diff --git a/capabilities/computation/measure/burst.ts b/capabilities/computation/measure/burst.ts index 6708c3c9d..6059ff6f0 100644 --- a/capabilities/computation/measure/burst.ts +++ b/capabilities/computation/measure/burst.ts @@ -2,7 +2,7 @@ import { checkedWorker } from '@on.auto/definitions'; import { programPool, type ProgramRequest } from '@on.auto/workflow-engine/dsl'; import { computationBounds } from '../src/run/run-bounds.ts'; -import { formatted, functionOf, inTurn, median, request } from './common.ts'; +import { formatted, functionOf, handed, inTurn, median, request } from './common.ts'; const jobs = 200; @@ -24,7 +24,11 @@ const twoModules: Naming = (index) => (index % 2 === 0 ? {} : withASchema); const oneModule: Naming = (index) => (index % 2 === 0 ? { worker: checkedWorker, context: null } : withASchema); async function burstOf(naming: Naming): Promise { - const pool = programPool({ workers: permits, heapMegabytes: computationBounds.heapMegabytes }); + const pool = programPool({ + handed, + workers: permits, + heapMegabytes: computationBounds.heapMegabytes, + }); const started = performance.now(); await Promise.all( Array.from({ length: jobs }, (_, index) => diff --git a/capabilities/computation/measure/common.ts b/capabilities/computation/measure/common.ts index c42f25553..dba480864 100644 --- a/capabilities/computation/measure/common.ts +++ b/capabilities/computation/measure/common.ts @@ -1,7 +1,11 @@ +import { handedWorkersWithChecks } from '@on.auto/definitions/check'; +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { programPool, type Json, type ProgramPool, type ProgramRequest } from '@on.auto/workflow-engine/dsl'; import { computationBounds } from '../src/run/run-bounds.ts'; +export const handed = await handedWorkersWithChecks(shippedFilesOnDisk); + export function millisecondsOf(work: () => void): number { const started = performance.now(); work(); @@ -46,5 +50,5 @@ export function request( } export function poolOfOne(): ProgramPool { - return programPool({ workers: 1, heapMegabytes: computationBounds.heapMegabytes }); + return programPool({ handed, workers: 1, heapMegabytes: computationBounds.heapMegabytes }); } diff --git a/capabilities/computation/measure/workers.ts b/capabilities/computation/measure/workers.ts index ef53a870a..61233157e 100644 --- a/capabilities/computation/measure/workers.ts +++ b/capabilities/computation/measure/workers.ts @@ -1,7 +1,7 @@ import { programPool, workerStackMegabytes, type ProgramPool } from '@on.auto/workflow-engine/dsl'; import { computationBounds, mebibytes } from '../src/run/run-bounds.ts'; -import { formatted, functionOf, inTurn, poolOfOne, request } from './common.ts'; +import { formatted, functionOf, handed, inTurn, poolOfOne, request } from './common.ts'; const blocking = new URL(`data:text/javascript,${encodeURIComponent('while (true) {}')}`); @@ -24,7 +24,12 @@ const bombs: readonly Bomb[] = [ ]; async function termination(): Promise { - const pool = programPool({ workers: 1, heapMegabytes: computationBounds.heapMegabytes, worker: blocking }); + const pool = programPool({ + handed, + workers: 1, + heapMegabytes: computationBounds.heapMegabytes, + worker: blocking, + }); const ended = await pool.run(request(functionOf('return input;'), 1, 500)); await pool.close(); return `a worker that never answers, under a deadline of 500 ms: ended ${ended.ran}, after ${formatted(ended.milliseconds, 1)} ms`; diff --git a/capabilities/computation/src/testing/computation-runs.ts b/capabilities/computation/src/testing/computation-runs.ts index 98b09945f..59f571984 100644 --- a/capabilities/computation/src/testing/computation-runs.ts +++ b/capabilities/computation/src/testing/computation-runs.ts @@ -1,5 +1,11 @@ import type { CapabilityAnswer, PreparedDefinition, Capability, RunContext } from '@on.auto/definitions'; -import { checkingWithin, noLongestRuns, nothingSaved, recordingJournal } from '@on.auto/definitions/testing'; +import { + checkingWithin, + measuredPoolSettings, + noLongestRuns, + nothingSaved, + recordingJournal, +} from '@on.auto/definitions/testing'; import { allPermissions, type Conflict, type InvalidInput, type Unavailable } from '@on.auto/operations'; import { programPool, type PoolSettings, type ProgramPool } from '@on.auto/workflow-engine/dsl'; import { Effect, type Exit, type Schema } from 'effect'; @@ -41,17 +47,11 @@ afterEach(async () => { await Promise.all(pools.splice(0).map((pool) => pool.close())); }); -const measured: Readonly> = Object.fromEntries( - Object.entries(process.env).flatMap(([key, value]: readonly [string, string | undefined]) => - key === 'NODE_V8_COVERAGE' && value !== undefined ? [[key, value]] : [], - ), -); - export function poolOf(settings: Partial = {}): ProgramPool { const pool = programPool({ workers: computationBounds.workers, heapMegabytes: computationBounds.heapMegabytes, - environment: measured, + ...measuredPoolSettings, ...settings, }); pools.push(pool); diff --git a/capabilities/coordination/src/capability/prediction-calls.test.ts b/capabilities/coordination/src/capability/prediction-calls.test.ts index 36e31b0f1..9a0eb108e 100644 --- a/capabilities/coordination/src/capability/prediction-calls.test.ts +++ b/capabilities/coordination/src/capability/prediction-calls.test.ts @@ -1,3 +1,4 @@ +import { measuredPoolSettings } from '@on.auto/definitions/testing'; import { memoryLedger } from '@on.auto/operations/testing'; import { makePredictionFunctionAdapter } from '@on.auto/prediction'; import { testPorts, weeklyOrders } from '@on.auto/prediction/testing'; @@ -11,7 +12,7 @@ import { recordingStarts, recordingWorkflows } from '../testing/recording-starts const runId = '0199a3c4-7d2e-7c1a-9b3f-555555555553'; -const pool = programPool({ workers: 1, heapMegabytes: 256 }); +const pool = programPool({ workers: 1, heapMegabytes: 256, ...measuredPoolSettings }); afterAll(() => pool.close()); diff --git a/capabilities/coordination/src/testing/expression-checks.ts b/capabilities/coordination/src/testing/expression-checks.ts index cd5c0fe8a..26b0d20ec 100644 --- a/capabilities/coordination/src/testing/expression-checks.ts +++ b/capabilities/coordination/src/testing/expression-checks.ts @@ -1,5 +1,5 @@ import { checkedAtSave } from '@on.auto/definitions/check'; -import { checksAwaitedAtMostMs } from '@on.auto/definitions/testing'; +import { checksAwaitedAtMostMs, measuredPoolSettings } from '@on.auto/definitions/testing'; import { programPool, type ProgramPool } from '@on.auto/workflow-engine/dsl'; import { afterAll } from 'vitest'; @@ -13,6 +13,6 @@ afterAll(async () => { }); export const testExpressionCheck: ExpressionCheck = (job) => { - pool ??= programPool({ workers: 1, heapMegabytes: 256 }); + pool ??= programPool({ workers: 1, heapMegabytes: 256, ...measuredPoolSettings }); return checkedAtSave(pool, job, checksAwaitedAtMostMs); }; diff --git a/capabilities/coordination/src/testing/test-host.ts b/capabilities/coordination/src/testing/test-host.ts index 9c96843b8..47ff63052 100644 --- a/capabilities/coordination/src/testing/test-host.ts +++ b/capabilities/coordination/src/testing/test-host.ts @@ -1,3 +1,4 @@ +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { machineSandboxOf } from '@on.auto/workflow-engine/dsl'; import { openWorkflowHost, type WorkflowHost } from '@on.auto/workflow-host'; import { recordedReactions, recordedWaiting } from '@on.auto/workflow-host/testing'; @@ -11,7 +12,7 @@ import { testPlaybookNames } from './test-sandbox.ts'; export function testHost(nested: Playbook): Promise { return openWorkflowHost({ database: { store: 'sqlite', file: ':memory:' }, - machine: workflowMachineOptionsOf(machineSandboxOf(), testPlaybookNames), + machine: workflowMachineOptionsOf(machineSandboxOf(shippedFilesOnDisk), testPlaybookNames), perform: definitionCalls(nested.executeNested), settle: nested.settle, reports: { diff --git a/capabilities/coordination/src/testing/test-sandbox.ts b/capabilities/coordination/src/testing/test-sandbox.ts index 558687ad0..b659c7cb1 100644 --- a/capabilities/coordination/src/testing/test-sandbox.ts +++ b/capabilities/coordination/src/testing/test-sandbox.ts @@ -1,5 +1,6 @@ import { playbookOwnNamesOf, type PlaybookOwnNames } from '@on.auto/definitions'; import { echo } from '@on.auto/definitions/testing'; +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { reusedSandbox, type MachineOptions, @@ -12,7 +13,10 @@ import { workflowFacts } from '../capability/workflow.ts'; import { workflowPolicyOf, type WorkflowPolicy } from '../document/workflow-functions.ts'; import { workflowMachineOptionsOf, workflowMachineSettingsOf } from '../runs/workflow-machine-options.ts'; -export const testSandbox: MachineSandbox = reusedSandbox(await freshInstance(unitMemoryBytes), () => 0); +export const testSandbox: MachineSandbox = reusedSandbox( + await freshInstance(shippedFilesOnDisk, unitMemoryBytes), + () => 0, +); export const testPlaybookNames: PlaybookOwnNames = playbookOwnNamesOf([echo, workflowFacts]); diff --git a/capabilities/prediction/measure.ts b/capabilities/prediction/measure.ts index c4f674a30..99430b4b5 100644 --- a/capabilities/prediction/measure.ts +++ b/capabilities/prediction/measure.ts @@ -1,4 +1,5 @@ -import { programPool } from '@on.auto/workflow-engine/dsl'; +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; +import { handedWorkersOf, programPool } from '@on.auto/workflow-engine/dsl'; import { rebuiltOn, type Folding, type History } from './measure/dataset-rebuild.ts'; import { dailyOrders, seriesMeasured, testedDailyOrders } from './measure/order-history.ts'; @@ -69,7 +70,7 @@ async function testedOn({ store, aDatabase }: MeasuredStore, history: History): } async function measuredOn(measured: MeasuredStore): Promise { - const pool = programPool({ workers: 1, heapMegabytes: 256 }); + const pool = programPool({ handed: await handedWorkersOf(shippedFilesOnDisk), workers: 1, heapMegabytes: 256 }); const history = { days, liveDays, filters: pool.evaluations.filters }; try { await untestedOn(measured, history); diff --git a/capabilities/prediction/src/dataset-testing/dataset-writes-suite.ts b/capabilities/prediction/src/dataset-testing/dataset-writes-suite.ts index d36333e79..9c08baf96 100644 --- a/capabilities/prediction/src/dataset-testing/dataset-writes-suite.ts +++ b/capabilities/prediction/src/dataset-testing/dataset-writes-suite.ts @@ -1,14 +1,12 @@ -import { filterSandboxOf } from '@on.auto/workflow-engine/dsl'; import { Effect } from 'effect'; import { describe, expect, it } from 'vitest'; -import { datasetFolder } from '../dataset/dataset-folder.ts'; import { datasetsPortOn } from '../dataset/datasets-port.ts'; import { uploadedWeeklyOrders, weeklyOrders, weeklyOrdersWith } from '../testing/weekly-orders.ts'; import { isLive, liveFolded } from './dataset-predicates.ts'; import type { SettingsOf } from './dataset-stores.ts'; import { north } from './orders.ts'; -import { alphaKey, projectedDatasets } from './projected-datasets.ts'; +import { alphaKey, projectedDatasets, testDatasetFolder } from './projected-datasets.ts'; const alpha = { org: 'acme', playbook: 'alpha' }; @@ -25,7 +23,7 @@ function staleWriteTest(settingsOf: SettingsOf): void { const row = await datasets.until(dataSha256, liveFolded(1)); const untouched = await datasets.until(idle.dataSha256, isLive); await datasets.stop(); - const folder = datasetFolder({ filters: filterSandboxOf(), triesBeforeStall: Number.POSITIVE_INFINITY }); + const folder = testDatasetFolder(); const unchanged = folder.unchanged(row); const stale = { row: Object.assign({}, row, { checkpointText: '["0"]' }), diff --git a/capabilities/prediction/src/dataset-testing/projected-datasets.ts b/capabilities/prediction/src/dataset-testing/projected-datasets.ts index 1547624a8..ee833ccfb 100644 --- a/capabilities/prediction/src/dataset-testing/projected-datasets.ts +++ b/capabilities/prediction/src/dataset-testing/projected-datasets.ts @@ -1,3 +1,4 @@ +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { filterSandboxOf } from '@on.auto/workflow-engine/dsl'; import { someFolder, type DatabaseSettings } from '@on.auto/workflow-host'; import { projectedFolders, type ProjectedFolders } from '@on.auto/workflow-host/testing'; @@ -81,11 +82,18 @@ function readingOf(folders: ProjectedFolders): Pick> = {}, -): Promise { - const folder = datasetFolder({ filters: filterSandboxOf(), triesBeforeStall: Number.POSITIVE_INFINITY, ...folding }); +type Folding = Partial>; + +export function testDatasetFolder(folding: Folding = {}): ReturnType { + return datasetFolder({ + filters: filterSandboxOf(shippedFilesOnDisk), + triesBeforeStall: Number.POSITIVE_INFINITY, + ...folding, + }); +} + +export async function projectedDatasets(settings: DatabaseSettings, folding: Folding = {}): Promise { + const folder = testDatasetFolder(folding); const folders = await projectedFolders(settings, [someFolder(folder)]); onTestFinished(folders.close); return { diff --git a/capabilities/prediction/src/dataset/dataset-matching.test.ts b/capabilities/prediction/src/dataset/dataset-matching.test.ts index 54ceec36f..433122b40 100644 --- a/capabilities/prediction/src/dataset/dataset-matching.test.ts +++ b/capabilities/prediction/src/dataset/dataset-matching.test.ts @@ -1,3 +1,4 @@ +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { filterSandboxOf } from '@on.auto/workflow-engine/dsl'; import type { PageEvent } from '@on.auto/workflow-host'; import { Effect } from 'effect'; @@ -36,7 +37,7 @@ describe('the matching of a dataset whose filters name their data before their t it('tests an event against the data tests of the filters of its own type alone, matching each type first', async () => { const matching = datasetMatchingOf( dataNamedFirst, - filterSandboxOf(() => 0), + filterSandboxOf(shippedFilesOnDisk, () => 0), ); const match = await Effect.runPromise(matching.matchOf(placed)); diff --git a/capabilities/prediction/src/testing/prediction-runs.ts b/capabilities/prediction/src/testing/prediction-runs.ts index f47ef2977..691a0d6d5 100644 --- a/capabilities/prediction/src/testing/prediction-runs.ts +++ b/capabilities/prediction/src/testing/prediction-runs.ts @@ -7,7 +7,13 @@ import type { SavedDefinition, SavedDefinitions, } from '@on.auto/definitions'; -import { checkingWithin, noLongestRuns, onTheStandingClock, recordingJournal } from '@on.auto/definitions/testing'; +import { + checkingWithin, + measuredPoolSettings, + noLongestRuns, + onTheStandingClock, + recordingJournal, +} from '@on.auto/definitions/testing'; import { allPermissions, contentOfText } from '@on.auto/operations'; import { programPool, type ProgramPool } from '@on.auto/workflow-engine/dsl'; import { Effect, type Exit, type Schema } from 'effect'; @@ -24,14 +30,8 @@ afterEach(async () => { await Promise.all(pools.splice(0).map((pool) => pool.close())); }); -const measured: Readonly> = Object.fromEntries( - Object.entries(process.env).flatMap(([key, value]: readonly [string, string | undefined]) => - key === 'NODE_V8_COVERAGE' && value !== undefined ? [[key, value]] : [], - ), -); - export function poolOf(): ProgramPool { - const pool = programPool({ workers: 1, heapMegabytes: 256, environment: measured }); + const pool = programPool({ workers: 1, heapMegabytes: 256, ...measuredPoolSettings }); pools.push(pool); return checkingWithin(pool); } diff --git a/capabilities/recall/measure/folds.ts b/capabilities/recall/measure/folds.ts index 08fa92558..d21edf00c 100644 --- a/capabilities/recall/measure/folds.ts +++ b/capabilities/recall/measure/folds.ts @@ -1,3 +1,5 @@ +import { handedWorkersWithChecks } from '@on.auto/definitions/check'; +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { programPool, type FoldRequest, type Json, type JsonObject } from '@on.auto/workflow-engine/dsl'; import { Effect, Result } from 'effect'; @@ -6,6 +8,8 @@ import { parseRecallDocument } from '../src/document/document-parsing.ts'; import { recallBounds, recallFolding } from '../src/run/recall-bounds.ts'; import { campaignReviews } from '../src/testing/campaign-reviews.ts'; +const handed = await handedWorkersWithChecks(shippedFilesOnDisk); + export interface Folded { readonly events: number; readonly pages: number; @@ -20,7 +24,7 @@ const document = Result.getOrThrow(parseRecallDocument(campaignReviews)); const { details } = document; async function strippedFold(): Promise { - const pool = programPool({ workers: 1, heapMegabytes: recallBounds.heapMegabytes }); + const pool = programPool({ handed, workers: 1, heapMegabytes: recallBounds.heapMegabytes }); const { module = details.fold } = await Effect.runPromise(documentCheck(pool)(document)); await pool.close(); return module; @@ -73,7 +77,7 @@ async function foldedInPages( events: number, size: number, ): Promise { - const pool = programPool({ workers: 1, heapMegabytes: recallBounds.heapMegabytes }); + const pool = programPool({ handed, workers: 1, heapMegabytes: recallBounds.heapMegabytes }); const starts = Array.from({ length: Math.ceil(events / size) }, (_, index) => index * size); const done = await starts.reduce>( async (before, start) => { diff --git a/capabilities/recall/src/testing/recall-runs.ts b/capabilities/recall/src/testing/recall-runs.ts index bf02169e7..00fed4871 100644 --- a/capabilities/recall/src/testing/recall-runs.ts +++ b/capabilities/recall/src/testing/recall-runs.ts @@ -2,6 +2,7 @@ import type { CapabilityAnswer, PreparedDefinition, Capability, RunContext } fro import { checkingWithin, checksAwaitedAtMostMs, + measuredPoolSettings, noLongestRuns, nothingSaved, recordingJournal, @@ -44,17 +45,11 @@ afterEach(async () => { await Promise.all(pools.splice(0).map((pool) => pool.close())); }); -const measured: Readonly> = Object.fromEntries( - Object.entries(process.env).flatMap(([key, value]: readonly [string, string | undefined]) => - key === 'NODE_V8_COVERAGE' && value !== undefined ? [[key, value]] : [], - ), -); - export function poolOf(settings: Partial = {}): ProgramPool { const pool = programPool({ workers: 2, heapMegabytes: recallBounds.heapMegabytes, - environment: measured, + ...measuredPoolSettings, ...settings, }); pools.push(pool); diff --git a/packages/api/README.md b/packages/api/README.md index 201298dd9..4fec4c499 100644 --- a/packages/api/README.md +++ b/packages/api/README.md @@ -13,7 +13,7 @@ Each request passes through the same chain before it reaches an operation, in th **Signed-in persons.** With sign-in on, the authenticator carries the protected resource, `PUBLIC_URL` and the Auto identity service's issuer, and the API serves `GET /.well-known/oauth-protected-resource` and the same path followed by `/mcp`, `/orgs/{org}/mcp` and `/orgs/{org}/playbooks/{playbook}/mcp` before the chain checks a credential, as public paths matching that prefix, answering [RFC 9728](https://datatracker.ietf.org/doc/html/rfc9728) metadata: `resource` is `PUBLIC_URL` followed by the path after the prefix, `authorization_servers` the identity service's issuer, `bearer_methods_supported` `["header"]`, `scopes_supported` the scopes a sign-in asks for, `signInScopes` of `@on.auto/identity`: `user:org:read`, which asks the identity service for a token bound to the organization the person signs in to, then the four permissions, which the identity service grants as scopes of its own, and `resource_documentation` the public MCP reference; a path under the prefix that names no endpoint, or an org or playbook id that is not one, is `404`. A `401` then carries `Bearer resource_metadata="/.well-known/oauth-protected-resource", scope=""`, the five separated by spaces, after `error="invalid_token"` when a credential was refused, and its detail says a key or a token. With sign-in off none of this is served, the metadata's path needs a key, and the challenges are `Bearer` and `Bearer error="invalid_token"`. The challenges are built in `src/authentication/challenges.ts` and the metadata in `src/authentication/protected-resource.ts`. -Before that chain, a browser opening `/` (a `GET` that accepts `text/html`) gets a page saying the server is running, with a button that opens Studio on this server: the Studio a person can reach, `studioBaseOf` of `@on.auto/identity`, followed by `?server=`, which is `/studio/?server=…` where the runtime serves Studio and `https://studio.on.auto/?server=…` for the default Studio host otherwise. The page needs no key and loads nothing from any server: its styles, its icon and its typefaces, DM Mono and DM Sans, are inside it. The two font files and their SIL Open Font Licenses are in `src/landing/fonts`. Any other request to `/` has no route. +Before that chain, a browser opening `/` (a `GET` that accepts `text/html`) gets a page saying the server is running, with a button that opens Studio on this server: the Studio a person can reach, `studioBaseOf` of `@on.auto/identity`, followed by `?server=`, which is `/studio/?server=…` where the runtime serves Studio and `https://studio.on.auto/?server=…` for the default Studio host otherwise. The page needs no key and loads nothing from any server: its styles, its icon and its typefaces, DM Mono and DM Sans, are inside it. The two font files and their SIL Open Font Licenses are in `src/landing/fonts`; `readPageStyle(shippedFiles)` reads them through the `ShippedFiles` of `@on.auto/operations` the server composes, once, and answers the page's styles, which its content security policy hashes, as a `PageStyle`, which the server reads at its start and gives the API as `ApiOptions.pageStyle`, so a typeface that cannot be read stops the start (`src/landing/page-style.ts`, `src/landing/fonts.ts`). Any other request to `/` has no route. **The served Studio.** A page on a public origin may not reach an address on the person's machine, so a runtime may serve Studio itself, at `/studio/` on its own origin. `ApiOptions.studio` says whether it does: a `HostedStudio`, `served: false`, serves none, and `/studio/` has no route; a `ServedStudio`, `served: true`, carries `host`, a `StudioHost`, which answers a path asked of the Studio host with a `StudioAnswer`, its status, content type, body, the seconds a browser may keep it, the host's validators, `ETag` and `Last-Modified`, and, for a redirect, the path on the host it sends the browser to, or with nothing when the host cannot be reached and nothing was kept. `GET` and `HEAD` under `/studio/` answer what the host answers for the same path under its root, without the query, with its status, its content type, its body, its `ETag` and `Last-Modified`, a redirect's `Location` under `/studio/`, and `Cache-Control` `public, max-age=, immutable` for what the host keeps fresh or `no-cache` for the rest; every other method is `405` `method_not_allowed` with `Allow: GET, HEAD`, `/studio` is sent on to `/studio/` with its query by `308`, since the page resolves its files against the folder it was served from, and a path that holds `..`, written or with a dot percent-encoded, is `404` `not_found` before the host is asked (`src/studio/served-studio.ts`). When the host answers nothing, the runtime answers `503` with a page of its own, `src/studio/unreachable-studio.ts`, which runs no script and says that Studio cannot be fetched now and from where. A request at `/studio` or under `/studio/` needs no key, since a browser opening a page sends none, and passes the `Origin` check and, in local mode, the `Host` check as every other request does (`src/middleware/public-paths.ts`). Its answers carry a content security policy of their own, `servedStudioPolicyOf` of `src/studio/studio-policy.ts`, written once from what Studio's build loads: `default-src 'none'; script-src 'self'; worker-src 'self'; style-src 'self' 'unsafe-inline'; font-src 'self'; img-src 'self' data:; connect-src 'self'; base-uri 'self'; form-action 'self'; frame-ancestors 'none'`, for its module scripts and the worker of its editor, its style sheets and the style elements it writes as it runs, its fonts and its images, the `data:` images its style sheets hold, and its calls to the runtime, every one from the runtime's origin. The page the host answers holds an inline script, so a `text/html` answer's `script-src` also allows each inline script of that page by its SHA-256, read from the page as the host answered it; every other answer under `/studio/` carries the policy with no hash, set by `src/middleware/response-headers.ts`. Every other path keeps `default-src 'none'; frame-ancestors 'none'`, and the other security headers are the same under `/studio/` as elsewhere. diff --git a/packages/api/src/api-options.ts b/packages/api/src/api-options.ts index a7165c491..32031ce1a 100644 --- a/packages/api/src/api-options.ts +++ b/packages/api/src/api-options.ts @@ -2,6 +2,7 @@ import type { Authenticator } from '@on.auto/identity'; import type { MiddlewareHandler } from 'hono'; import type { ApiEnv } from './api-env.ts'; +import type { PageStyle } from './landing/page-style.ts'; import type { RuntimeControl } from './landing/runtime-control.ts'; import type { ReportIncident } from './problem/error-boundary.ts'; import type { RegisterRoutes } from './routes.ts'; @@ -13,6 +14,7 @@ export interface ApiOptions { readonly authenticator: Authenticator; readonly routes: readonly RegisterRoutes[]; readonly reportIncident: ReportIncident; + readonly pageStyle: PageStyle; readonly requests?: MiddlewareHandler; readonly runtimeControl?: RuntimeControl; readonly installId?: () => string | undefined; diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts index 4a4fcf375..e1c9db364 100644 --- a/packages/api/src/index.ts +++ b/packages/api/src/index.ts @@ -1,6 +1,7 @@ export { createApiHandler } from './api-handler.ts'; export type { ApiHandler } from './api-handler.ts'; export type { ApiOptions } from './api-options.ts'; +export { readPageStyle, type PageStyle } from './landing/page-style.ts'; export type { ApiStudio, HostedStudio, ServedStudio, StudioAnswer, StudioHost } from './studio/served-studio.ts'; export { runtimeActionPaths, diff --git a/packages/api/src/landing/fonts.ts b/packages/api/src/landing/fonts.ts index f7d9d6d17..d8f846645 100644 --- a/packages/api/src/landing/fonts.ts +++ b/packages/api/src/landing/fonts.ts @@ -1,23 +1,29 @@ -import { readFileSync } from 'node:fs'; +import type { ShippedFiles } from '@on.auto/operations'; -function embedded(file: string): string { - const font = readFileSync(new URL(`./fonts/${file}`, import.meta.url)).toString('base64'); - return `url(data:font/woff2;base64,${font}) format('woff2')`; +const monoFile = new URL('./fonts/dm-mono-latin-500.woff2', import.meta.url); + +const sansFile = new URL('./fonts/dm-sans-latin-variable.woff2', import.meta.url); + +function embedded(font: Readonly): string { + return `url(data:font/woff2;base64,${Buffer.from(font).toString('base64')}) format('woff2')`; } -export const fontFaces = ` +export async function fontFacesOf(shippedFiles: ShippedFiles): Promise { + const [mono, sans] = await Promise.all([shippedFiles.bytes(monoFile), shippedFiles.bytes(sansFile)]); + return ` @font-face { font-family: 'DM Mono'; font-style: normal; font-weight: 500; font-display: swap; - src: ${embedded('dm-mono-latin-500.woff2')}; + src: ${embedded(mono)}; } @font-face { font-family: 'DM Sans'; font-style: normal; font-weight: 100 1000; font-display: swap; - src: ${embedded('dm-sans-latin-variable.woff2')}; + src: ${embedded(sans)}; } `; +} diff --git a/packages/api/src/landing/landing-page.test.ts b/packages/api/src/landing/landing-page.test.ts index 6bab1516a..75df0aab5 100644 --- a/packages/api/src/landing/landing-page.test.ts +++ b/packages/api/src/landing/landing-page.test.ts @@ -1,8 +1,11 @@ import { createHash } from 'node:crypto'; +import type { ShippedFiles } from '@on.auto/operations'; +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; +import { shippedFilesInMemory } from '@on.auto/operations/testing'; import { describe, expect, it } from 'vitest'; -import type { ApiHandler } from '../index.ts'; +import { readPageStyle, type ApiHandler } from '../index.ts'; import { call, createTestHandler } from '../testing/api-calls.ts'; import { operationServer } from '../testing/operation-server.ts'; import { servedStudio } from '../testing/studios.ts'; @@ -156,6 +159,29 @@ describe('what the page at the root of the server loads', () => { expect(page).toContain("font-family: 'DM Sans';"); }); + it('has its typefaces read through the files the server ships once, before the server answers, and none read again for a page', async () => { + const read: string[] = []; + const shippedFiles: ShippedFiles = { + ...shippedFilesOnDisk, + bytes: (file) => { + read.push(file.pathname.slice(file.pathname.lastIndexOf('/') + 1)); + return shippedFilesOnDisk.bytes(file); + }, + }; + const { handler } = createTestHandler({ pageStyle: await readPageStyle(shippedFiles) }); + const before = [...read]; + + await open(handler); + await open(handler); + + expect(before).toEqual(['dm-mono-latin-500.woff2', 'dm-sans-latin-variable.woff2']); + expect(read).toEqual(before); + }); + + it('cannot be styled when a typeface is not shipped, so the read refuses, naming the file', async () => { + await expect(readPageStyle(shippedFilesInMemory({}))).rejects.toThrow(/No file is shipped at .*\.woff2$/u); + }); + it('names no other server than the studio it opens', async () => { const page = await (await open(createTestHandler().handler)).text(); diff --git a/packages/api/src/landing/landing-page.ts b/packages/api/src/landing/landing-page.ts index cc135f503..274c2503b 100644 --- a/packages/api/src/landing/landing-page.ts +++ b/packages/api/src/landing/landing-page.ts @@ -4,7 +4,7 @@ import { html, raw } from 'hono/html'; import type { ApiEnv } from '../api-env.ts'; import { brand } from './auto-logo.ts'; -import { controlPageHeaders, headOf, readOnlyPageHeaders } from './page-style.ts'; +import type { PageStyle } from './page-style.ts'; import { act, refusalOf, @@ -21,10 +21,14 @@ const arrow = `; +interface Dressing { + readonly style: PageStyle; + readonly head: string; + readonly actionPage: (action: RuntimeAction) => Page; +} + function factsOf(status: RuntimeStatus): Page { const log = status.logFile === undefined @@ -48,7 +52,7 @@ function factsOf(status: RuntimeStatus): Page { `; } -function pageOf(studioLink: string, server: Readonly, status: RuntimeStatus | undefined): Page { +function pageOf(head: string, studioLink: string, server: Readonly, status: RuntimeStatus | undefined): Page { return html` @@ -67,35 +71,46 @@ function pageOf(studioLink: string, server: Readonly, status: RuntimeStatus `; } -const restartingPage = html` - - - ${raw(headOf(''))} - - -
- ${raw(brand)} -

Auto Runtime is restarting

-

This page opens again once it is back.

-
- - `; - -const stoppedPage = html` - - - ${raw(head)} - - -
- ${raw(brand)} -

Auto Runtime has stopped

-

Your agent starts it again when it next needs it. To start it yourself, run npx @on.auto/runtime.

-
- - `; - -const actionPages: Readonly> = { stop: stoppedPage, restart: restartingPage }; +function restartingPageOf(style: PageStyle): Page { + return html` + + + ${raw(style.head(''))} + + +
+ ${raw(brand)} +

Auto Runtime is restarting

+

This page opens again once it is back.

+
+ + `; +} + +function stoppedPageOf(head: string): Page { + return html` + + + ${raw(head)} + + +
+ ${raw(brand)} +

Auto Runtime has stopped

+

Your agent starts it again when it next needs it. To start it yourself, run npx @on.auto/runtime.

+
+ + `; +} + +function dressingOf(style: PageStyle): Dressing { + const head = style.head(''); + const actionPages: Readonly> = { + stop: stoppedPageOf(head), + restart: restartingPageOf(style), + }; + return { style, head, actionPage: (action) => actionPages[action] }; +} const actionStates: Readonly> = { stop: 'stopping', restart: 'restarting' }; @@ -114,7 +129,7 @@ function guarded(answer: Answer): Answer { }) ?? answer(c); } -function landingAnswerOf({ control, studio, installId }: Landing): Answer { +function landingAnswerOf({ control, studio, installId }: Landing, { head, style }: Dressing): Answer { return (c) => { const server = new URL(c.req.url); const studioLink = studioLinkOf(studio, { @@ -123,27 +138,27 @@ function landingAnswerOf({ control, studio, installId }: Landing): Answer { installId: installId?.(), }); return c.html( - pageOf(studioLink, server, control?.status()), + pageOf(head, studioLink, server, control?.status()), 200, - control === undefined ? readOnlyPageHeaders : controlPageHeaders, + control === undefined ? style.readOnlyHeaders : style.controlHeaders, ); }; } -function actionAnswerOf(control: RuntimeControl, action: RuntimeAction): Answer { +function actionAnswerOf(control: RuntimeControl, action: RuntimeAction, { actionPage, style }: Dressing): Answer { return guarded((c) => { act(control, action); return asksForAPage(c) - ? c.html(actionPages[action], 200, readOnlyPageHeaders) + ? c.html(actionPage(action), 200, style.readOnlyHeaders) : c.json({ status: actionStates[action] }, 202, statusHeaders); }); } -function controlAnswersOf(control: RuntimeControl): ReadonlyMap { +function controlAnswersOf(control: RuntimeControl, dressing: Dressing): ReadonlyMap { return new Map([ [`GET ${runtimePath}`, guarded((c) => c.json(statusDocumentOf(control.status()), 200, statusHeaders))], - [`POST ${runtimeActionPaths.stop}`, actionAnswerOf(control, 'stop')], - [`POST ${runtimeActionPaths.restart}`, actionAnswerOf(control, 'restart')], + [`POST ${runtimeActionPaths.stop}`, actionAnswerOf(control, 'stop', dressing)], + [`POST ${runtimeActionPaths.restart}`, actionAnswerOf(control, 'restart', dressing)], ]); } @@ -151,19 +166,23 @@ interface Landing { readonly control: RuntimeControl | undefined; readonly studio: StudioSettings; readonly installId: (() => string | undefined) | undefined; + readonly style: PageStyle; } -function answerFor(c: Context, landing: Landing, controls: ReadonlyMap): Answer | undefined { +function answerFor(c: Context, page: Answer, controls: ReadonlyMap): Answer | undefined { if (c.req.method === 'GET' && c.req.path === '/' && asksForAPage(c)) { - return landingAnswerOf(landing); + return page; } return controls.get(`${c.req.method} ${c.req.path}`); } export function landingPageFor(landing: Landing): MiddlewareHandler { - const controls = landing.control === undefined ? new Map() : controlAnswersOf(landing.control); + const dressing = dressingOf(landing.style); + const page = landingAnswerOf(landing, dressing); + const controls = + landing.control === undefined ? new Map() : controlAnswersOf(landing.control, dressing); return (c, next) => { - const answer = answerFor(c, landing, controls); + const answer = answerFor(c, page, controls); return answer === undefined ? next() : Promise.resolve(answer(c)); }; } diff --git a/packages/api/src/landing/page-style.ts b/packages/api/src/landing/page-style.ts index 0dae84b5c..ed0949bc3 100644 --- a/packages/api/src/landing/page-style.ts +++ b/packages/api/src/landing/page-style.ts @@ -1,9 +1,17 @@ import { createHash } from 'node:crypto'; +import type { ShippedFiles } from '@on.auto/operations'; + import { iconOnDark, iconOnLight } from './auto-logo.ts'; -import { fontFaces } from './fonts.ts'; +import { fontFacesOf } from './fonts.ts'; + +export interface PageStyle { + readonly head: (extra: string) => string; + readonly readOnlyHeaders: Readonly>; + readonly controlHeaders: Readonly>; +} -const styles = `${fontFaces} +const pageStyles = ` :root { color-scheme: light dark; --paper: #ffffff; @@ -137,7 +145,11 @@ p { .controls button:focus-visible { outline: 2px solid var(--focus); outline-offset: 2px; } `; -function policyOf(formAction: string): string { +function stylesOf(fontFaces: string): string { + return `${fontFaces}${pageStyles}`; +} + +function policyOf(styles: string, formAction: string): string { return [ "default-src 'none'", `style-src 'sha256-${createHash('sha256').update(styles).digest('base64')}'`, @@ -149,28 +161,34 @@ function policyOf(formAction: string): string { ].join('; '); } -function pageHeadersOf(formAction: string, referrerPolicy: string): Readonly> { +function pageHeadersOf(styles: string, formAction: string, referrerPolicy: string): Readonly> { return { 'cache-control': 'no-store', - 'content-security-policy': policyOf(formAction), + 'content-security-policy': policyOf(styles, formAction), 'referrer-policy': referrerPolicy, 'x-content-type-options': 'nosniff', 'x-frame-options': 'DENY', }; } -export const readOnlyPageHeaders = pageHeadersOf("'none'", 'no-referrer'); - -export const controlPageHeaders = pageHeadersOf("'self'", 'same-origin'); +function pageStyleOf(fontFaces: string): PageStyle { + const styles = stylesOf(fontFaces); + return { + head: (extra) => + [ + '', + '', + extra, + 'Auto Runtime', + ``, + ``, + ``, + ].join(''), + readOnlyHeaders: pageHeadersOf(styles, "'none'", 'no-referrer'), + controlHeaders: pageHeadersOf(styles, "'self'", 'same-origin'), + }; +} -export function headOf(extra: string): string { - return [ - '', - '', - extra, - 'Auto Runtime', - ``, - ``, - ``, - ].join(''); +export async function readPageStyle(shippedFiles: ShippedFiles): Promise { + return pageStyleOf(await fontFacesOf(shippedFiles)); } diff --git a/packages/api/src/middleware/middleware-chain.ts b/packages/api/src/middleware/middleware-chain.ts index 312ebdc7c..91fde8c25 100644 --- a/packages/api/src/middleware/middleware-chain.ts +++ b/packages/api/src/middleware/middleware-chain.ts @@ -11,7 +11,7 @@ import { validateHost } from './validate-host.ts'; import { validateOrigin } from './validate-origin.ts'; export function middlewareFor( - { allowedOrigins, studio, authenticator, requests, runtimeControl, installId }: ApiOptions, + { allowedOrigins, studio, authenticator, requests, runtimeControl, installId, pageStyle }: ApiOptions, publicPaths: ReadonlySet, ): readonly MiddlewareHandler[] { const isPublic = publicPathsOf(authenticator, publicPaths); @@ -19,7 +19,7 @@ export function middlewareFor( const pages = { studio, allowedOrigins }; return [ ...(requests === undefined ? [] : [requests]), - landingPageFor({ control: runtimeControl, studio, installId }), + landingPageFor({ control: runtimeControl, studio, installId, style: pageStyle }), ...requestIdAndSecurityHeadersFor(studio), ...[validateOrigin(pages, authenticator.resource), ...hostValidation].map((middleware) => exceptOn(isPublic, middleware), diff --git a/packages/api/src/testing/api-calls.ts b/packages/api/src/testing/api-calls.ts index e767a2e89..91f479acc 100644 --- a/packages/api/src/testing/api-calls.ts +++ b/packages/api/src/testing/api-calls.ts @@ -1,7 +1,9 @@ import type { Authenticator } from '@on.auto/identity'; import { allPermissions, type CallSummary } from '@on.auto/operations'; +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { createApiHandler, type ApiHandler, type ApiOptions, type RegisterRoutes } from '../index.ts'; +import { readPageStyle } from '../landing/page-style.ts'; import { hostedStudio } from './studios.ts'; export interface Call { @@ -35,6 +37,8 @@ const admitEveryone: Authenticator = { }), }; +const pageStyle = await readPageStyle(shippedFilesOnDisk); + export function createTestHandler(options: Partial = {}): { handler: ApiHandler; reported: ReportedThrown[]; @@ -45,6 +49,7 @@ export function createTestHandler(options: Partial = {}): { studio: hostedStudio, authenticator: admitEveryone, routes: [], + pageStyle, reportIncident: (incident, error, requestId, summary) => { reported.push({ incident, requestId, message: error.message, cause: error.cause, call: summary }); }, diff --git a/packages/connections/README.md b/packages/connections/README.md index abd5b5061..bdc8654a3 100644 --- a/packages/connections/README.md +++ b/packages/connections/README.md @@ -97,7 +97,7 @@ A client the consent client registers dynamically at an authorization server is ### The file's providers -`makeProviderResolver(providers)` (`src/providers/provider-resolver.ts`) is the `ProviderResolver` of `@on.auto/tool-servers` over a list of providers, by which the reading of the settings file resolves an entry of `tool_servers` that names a provider in place of its server. It answers what `readEntry` asks, the provider's name, its placeholders and the credential, its references read as the names of their variables, through the steps an addition takes, each the one there is: `providerNamed` of `@on.auto/providers` refuses a name the list does not have at `/provider`, `There is no provider named crm here`; `additionOfProvider`, the one resolution `add_connection` makes, places the placeholders and chooses the way, the one `kind` names or else the first the settings file can give, any but one of consent, refusing a placeholder or a kind at the key the operator wrote; a way of the kind `authorization` it chose, which a person gives by consent on Studio's page, is refused there, before the credential is placed, at `/credential/kind` where the entry names it and at `/credential` where the provider has no other way, in the form of the refusal of a kind, `The provider github takes no authorization credential in the settings file, since a person gives that consent on Studio's page; it takes secret`, and, for a provider with no other way, `The provider slack takes authorization alone, a consent a person gives on Studio's page, which an entry of the settings file cannot give, so a person connects it there`; `credentialIssuesOf`, the check of where a credential goes that an addition runs, refuses a way into a header the provider's entry sends already, in any case, or one the client sets itself, at `/provider`, as `add_connection` places it; the credential's fields are checked against the way by `givenFieldsOf` of `src/entries/given-values.ts`, the rule of the values a credential of each kind takes that `give_credential` holds a value given on the page to, in words of their own for the settings file, `wordsOfTheSettings`, beside the page's, `wordsOfThePage`: the credential as a reference alone for a `secret`, `client_id` with `client_secret`, or with `private_key` and `algorithm`, for `client_credentials`, each at its key under `/credential`, and none for `none`, `Expected no credential: the way none takes none`; and `placedReference` places the named reference where the way says, the client id where `{client_id}` stands, as the serving of a connection made from that provider places its credential, and answers where it placed each field it was given, `placedAt`. The resolver answers the entry as an entry of the file writes it, the provider's references kept, the folder of the providers' documents and `writtenAt`, which places a problem the reading then finds at a place in `placedAt` at the field that filled it under `/credential`, and any other at `/provider`. +`makeProviderResolver(providers)` (`src/providers/provider-resolver.ts`) is the `ProviderResolver` of `@on.auto/tool-servers` over a list of providers, by which the reading of the settings file resolves an entry of `tool_servers` that names a provider in place of its server. It answers what `readEntry` asks, the provider's name, its placeholders and the credential, its references read as the names of their variables, through the steps an addition takes, each the one there is: `providerNamed` of `@on.auto/providers` refuses a name the list does not have at `/provider`, `There is no provider named crm here`; `additionOfProvider`, the one resolution `add_connection` makes, places the placeholders and chooses the way, the one `kind` names or else the first the settings file can give, any but one of consent, refusing a placeholder or a kind at the key the operator wrote; a way of the kind `authorization` it chose, which a person gives by consent on Studio's page, is refused there, before the credential is placed, at `/credential/kind` where the entry names it and at `/credential` where the provider has no other way, in the form of the refusal of a kind, `The provider github takes no authorization credential in the settings file, since a person gives that consent on Studio's page; it takes secret`, and, for a provider with no other way, `The provider slack takes authorization alone, a consent a person gives on Studio's page, which an entry of the settings file cannot give, so a person connects it there`; `credentialIssuesOf`, the check of where a credential goes that an addition runs, refuses a way into a header the provider's entry sends already, in any case, or one the client sets itself, at `/provider`, as `add_connection` places it; the credential's fields are checked against the way by `givenFieldsOf` of `src/entries/given-values.ts`, the rule of the values a credential of each kind takes that `give_credential` holds a value given on the page to, in words of their own for the settings file, `wordsOfTheSettings`, beside the page's, `wordsOfThePage`: the credential as a reference alone for a `secret`, `client_id` with `client_secret`, or with `private_key` and `algorithm`, for `client_credentials`, each at its key under `/credential`, and none for `none`, `Expected no credential: the way none takes none`; and `placedReference` places the named reference where the way says, the client id where `{client_id}` stands, as the serving of a connection made from that provider places its credential, and answers where it placed each field it was given, `placedAt`. The resolver answers the entry as an entry of the file writes it, the provider's references kept, the texts of the providers' documents and `writtenAt`, which places a problem the reading then finds at a place in `placedAt` at the field that filled it under `/credential`, and any other at `/provider`. ## The calls diff --git a/packages/connections/src/providers/provider-additions.test.ts b/packages/connections/src/providers/provider-additions.test.ts index 85fc82c51..4c3a0c89b 100644 --- a/packages/connections/src/providers/provider-additions.test.ts +++ b/packages/connections/src/providers/provider-additions.test.ts @@ -1,4 +1,6 @@ import { providersOf, regions, tracker } from '@on.auto/providers/testing'; +import { shippedDocumentOf } from '@on.auto/tool-servers'; +import { fakeApiDocument } from '@on.auto/tool-servers/testing'; import { describe, expect, it } from 'vitest'; import { connectionPolicyOf } from '../policy/connection-policy.ts'; @@ -29,6 +31,10 @@ const notesApi = { credentials: [{ kind: 'none' }], }; +const notesDocument = new TextEncoder().encode(JSON.stringify(fakeApiDocument('3.1', notesApi.base_url))); + +const notesShipped = new Map([[notesApi.document, shippedDocumentOf(notesDocument)]]); + const openai = { ...tracker, name: 'openai', url: 'https://openai-tools.example.com/mcp', placeholders: {} }; describe('a connection added by a provider', () => { @@ -221,7 +227,7 @@ describe('the rules a connection added by a provider passes', () => { it('take the document beside the entries of a provider, which is no host, and judge the host its calls go to', async () => { const h = harness({ policy: connectionPolicyOf({ allowed_hosts: ['*.example.org'] }), - providers: providersOf(notesApi), + providers: { ...providersOf(notesApi), documents: notesShipped }, }); expect(await adding(h, { provider: 'notes-api' })).toMatchObject({ diff --git a/packages/connections/src/providers/provider-resolver.ts b/packages/connections/src/providers/provider-resolver.ts index aa61f7cdf..9326dd0f8 100644 --- a/packages/connections/src/providers/provider-resolver.ts +++ b/packages/connections/src/providers/provider-resolver.ts @@ -6,7 +6,13 @@ import { type ProviderEntry, type ProviderList, } from '@on.auto/providers'; -import type { CredentialGiven, ProviderNamed, ProviderResolved, ProviderResolver } from '@on.auto/tool-servers'; +import type { + CredentialGiven, + ProviderNamed, + ProviderResolved, + ProviderResolver, + ShippedDocument, +} from '@on.auto/tool-servers'; import { Result } from 'effect'; import { credentialIssuesOf } from '../entries/credential-checks.ts'; @@ -87,12 +93,16 @@ function writtenAtOf({ placedAt }: Placing): (pointer: string) => string { }; } -function resolvedOf(addition: ProviderAddition, { credential, reference }: Credentialled, documents: string) { +function resolvedOf( + addition: ProviderAddition, + { credential, reference }: Credentialled, + documents: ReadonlyMap, +) { const placing = placedReference({ provider: addition.provider, entry: addition.entry, credential }, reference); const resolved: ProviderResolved = { written: { ...placing.written, type: addition.type }, placedValues: placing.placedValues, - documentsDirectory: documents, + documents: { kind: 'shipped', byName: documents }, writtenAt: writtenAtOf(placing), }; return resolved; diff --git a/packages/connections/src/providers/provider-serving.test.ts b/packages/connections/src/providers/provider-serving.test.ts index 6435870ac..3f9dcbede 100644 --- a/packages/connections/src/providers/provider-serving.test.ts +++ b/packages/connections/src/providers/provider-serving.test.ts @@ -1,6 +1,12 @@ import type { ProviderList } from '@on.auto/providers'; import { fixtureProviders, providerListOf, providerOf, providersOf } from '@on.auto/providers/testing'; -import type { ConnectionServerSettings, ConnectionUnready, OrgConnection } from '@on.auto/tool-servers'; +import { + shippedDocumentOf, + type ConnectionServerSettings, + type ConnectionUnready, + type OrgConnection, +} from '@on.auto/tool-servers'; +import { fakeApiDocument } from '@on.auto/tool-servers/testing'; import { Effect, Option, Redacted } from 'effect'; import { describe, expect, it } from 'vitest'; @@ -205,7 +211,16 @@ const { base_url: _baseUrl, ...unbasedNotes } = { describe('a provider whose document beside the entries names its server', () => { it('has the host of that server judged where it gives no base_url, at add and at use', async () => { - const providers = providerListOf(providerOf(unbasedNotes, 'https://api.notes.example.com/v1')); + const server = 'https://api.notes.example.com/v1'; + const providers = { + ...providerListOf(providerOf(unbasedNotes, server)), + documents: new Map([ + [ + unbasedNotes.document, + shippedDocumentOf(new TextEncoder().encode(JSON.stringify(fakeApiDocument('3.1', server)))), + ], + ]), + }; const h = harness({ providers }); const { id, token } = await added(h, { provider: 'notes-api' }); await h.call(operationNamed(h.operations, 'approve'), ada, { id, token }); diff --git a/packages/connections/src/providers/shipped-consents.test.ts b/packages/connections/src/providers/shipped-consents.test.ts index e830aedc3..5fe822e16 100644 --- a/packages/connections/src/providers/shipped-consents.test.ts +++ b/packages/connections/src/providers/shipped-consents.test.ts @@ -1,3 +1,4 @@ +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { readProviders, type Placeholder, type ProviderEntry } from '@on.auto/providers'; import { Option, Result, Schema } from 'effect'; import { afterEach, describe, expect, it } from 'vitest'; @@ -5,7 +6,7 @@ import { afterEach, describe, expect, it } from 'vitest'; import { vendorsOfTheShippedWays, type VendorOfTheWay } from '../testing/shipped-vendors.ts'; import { consentedAtTheVendor, harnessAtTheVendor, servedAsTheVendor } from '../testing/vendor-fakes.ts'; -const shipped = Result.getOrThrow(readProviders()); +const shipped = Result.getOrThrow(await readProviders(shippedFilesOnDisk)); const closing: (() => Promise)[] = []; diff --git a/packages/connections/src/providers/shipped-providers.test.ts b/packages/connections/src/providers/shipped-providers.test.ts index c573f15f7..71b4d27e9 100644 --- a/packages/connections/src/providers/shipped-providers.test.ts +++ b/packages/connections/src/providers/shipped-providers.test.ts @@ -1,6 +1,4 @@ -import { readdirSync, readFileSync } from 'node:fs'; -import { join } from 'node:path'; - +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { additionOfProvider, holdsAClientId, @@ -22,7 +20,11 @@ import { describe, expect, it } from 'vitest'; import { additionIssuesOf, readAddition, type AdditionInput } from '../entries/entry-checks.ts'; import { credentialFieldIssuesOf, placesNoValue } from '../registry/credential.ts'; -const shipped = Result.getOrThrow(readProviders()); +const shipped = Result.getOrThrow(await readProviders(shippedFilesOnDisk)); + +const filesBesideTheEntries = await shippedFilesOnDisk.list( + new URL('entries/documents/', import.meta.resolve('@on.auto/providers')), +); const checking = { org: 'acme', @@ -92,13 +94,13 @@ function settingsOf(addition: ProviderAddition): ServerSettings { function operationsIn(document = ''): number { const { paths } = Schema.decodeUnknownSync(Schema.fromJsonString(DocumentSchema))( - readFileSync(join(shipped.documents, document), 'utf8'), + shipped.documents.get(document)?.text ?? '', ); return Object.values(paths).flatMap((operations) => Object.keys(operations)).length; } function documentsBesideTheEntries(): readonly string[] { - return readdirSync(join(shipped.documents, 'documents')) + return filesBesideTheEntries .filter((file) => file.endsWith('.openapi.json')) .map((file) => `documents/${file}`) .toSorted(); diff --git a/packages/connections/src/serving/placed-credentials.ts b/packages/connections/src/serving/placed-credentials.ts index 24ada732f..bbb8ae1d8 100644 --- a/packages/connections/src/serving/placed-credentials.ts +++ b/packages/connections/src/serving/placed-credentials.ts @@ -1,5 +1,11 @@ import { clientIdPlaceholder } from '@on.auto/providers'; -import { readEntry, type Environment, type ServerEntryFields } from '@on.auto/tool-servers'; +import { + readEntry, + type Environment, + type ServerEntryFields, + type ShippedDocument, + type ToolServerSettingsContext, +} from '@on.auto/tool-servers'; import { JsonPointer, Option, Redacted, Result, Schema } from 'effect'; import { ConnectionEntrySchema, type ConnectionEntry, type ConnectionType } from '../registry/connection-entry.ts'; @@ -176,7 +182,7 @@ interface Named { } export interface Providing { - readonly documents: string; + readonly documents: ReadonlyMap; readonly values: ReadonlyMap; readonly documentServer: string | undefined; } @@ -196,6 +202,10 @@ function operatorEnvironmentOf(providing: Providing | undefined): Environment { ); } +function documentsOf(providing: Providing | undefined): NonNullable { + return providing === undefined ? { kind: 'directory', path: '.' } : { kind: 'shipped', byName: providing.documents }; +} + export function readPlaced( { name, type, provider }: Named, { written, environment, placedValues }: Placed, @@ -206,7 +216,7 @@ export function readPlaced( name, { ...written, type, org }, { ...operatorEnvironmentOf(providing), ...environment }, - { modelProviders, documentsDirectory: providing?.documents ?? '.', placedValues }, + { modelProviders, documents: documentsOf(providing), placedValues }, ); return Result.map(read, (settings) => ({ ...settings, provider })); } diff --git a/packages/definitions/README.md b/packages/definitions/README.md index bc375f86f..f9df39ea1 100644 --- a/packages/definitions/README.md +++ b/packages/definitions/README.md @@ -180,7 +180,7 @@ A run rejected with `conflict` records the kind the capability gave, so `get_run `checkedAtSave(pool, job, deadlineMs)`, the one export of `@on.auto/definitions/check`, a subpath of its own because it names the engine's pool, which nothing else of the main entry needs, checks the programs and the expressions of a document as a job of the engine's pool, `check`, on the permit the pool keeps for checks beside the workers of runs, so a save never waits behind the runs (`src/program-check/save-check.ts`). The computation, recall, prediction and workflow capabilities call it from their `check`, with the document's module, its place, `computation` or `recall`, the schemas of its types, and every expression it holds with the names each may read; a workflow's expressions go as one job, whatever their number, and so do a prediction function's, the `data` expressions of its filters, which read `$data`, with no module. It answers the compiler's issues, each with the expression it belongs to, or `module`, and its line there, which the capability places in the document; or, when there are none, the document's `module` and the expressions stripping changed, by their source, which the capability answers from its `check` and the save keeps (`src/program-check/type-stripping.ts`), so no run strips anything and no runtime holds a compiler; or `InvalidInput` when the job does not answer within `deadlineMs`, 2,000 ms of the server's wall-clock time, counted from when its worker is ready, since the compiler took longer than a save allows over this document: its types should be simpler or its program split, and, since a busy server may also have slowed the check, the words say a later save may succeed; or `Unavailable` when the job stops otherwise: `busy` when no checker came free within that time or was ready within the start the pool allows it, the worker's memory, a cancelled save or the server stopping. The assignment bound stays the fast refusal of the one shape measured to grow the compiler's work with its square; others, such as hundreds of narrowings, meet the deadline. `warmedChecks(pool)` checks one trivial expression, which the server awaits before it accepts a request, so the compiler is loaded before the first save and no save waits behind the warming for the one permit; the pool keeps the worker of the checks however long it idles. A worker that crashes or cannot read the job is a defect. -The job runs in its own worker module, `src/program-check/check-worker.ts`, which loads the compiler once and keeps it, with the sandbox's lib, between checks, so a warm check pays only for the document. The compiler is TypeScript 6, `typescript` 6.0.3 installed under the alias `typescript6`, since TypeScript 7 has no compiler API to call and the repository's own toolchain keeps the name `typescript`; it is a dependency of this package alone. Its options are `strict`, `exactOptionalPropertyTypes`, `isolatedModules`, `erasableSyntaxOnly`, `noEmit` and `noLib`, with the sandbox's lib as the only lib and `types: []` (`src/program-check/checking.ts`). One program holds, each as a file of its own, so a line of the document's program is a line of its file and no offset is subtracted: +The job runs in its own worker module, `src/program-check/check-worker.ts`, which loads the compiler once and keeps it, with the sandbox's lib, between checks, so a warm check pays only for the document. A thread cannot be handed a reader, so the files of the kept lib are read once, before the pool is made, through the `ShippedFiles` the server composes: `handedWorkersWithChecks(shippedFiles)` of the `check` entry answers the files the pool hands its workers, the module of the sandbox, and for the check worker the kept lib in its place, which the server gives the one pool it makes (`src/program-check/save-check.ts`). The worker makes its checker of what it was handed with `serveJobsWith` of the engine's job loop, through `sandboxLibFrom(handed)` (`handedChecks`, `src/program-check/kept-lib.ts`), and a test makes its checker of the same lib read on disk (`keptLibOnDisk`, `src/testing/read-on-disk.ts`). The compiler is TypeScript 6, `typescript` 6.0.3 installed under the alias `typescript6`, since TypeScript 7 has no compiler API to call and the repository's own toolchain keeps the name `typescript`; it is a dependency of this package alone. Its options are `strict`, `exactOptionalPropertyTypes`, `isolatedModules`, `erasableSyntaxOnly`, `noEmit` and `noLib`, with the sandbox's lib as the only lib and `types: []` (`src/program-check/checking.ts`). One program holds, each as a file of its own, so a line of the document's program is a line of its file and no offset is subtracted: - the sandbox's lib and its iterator lib; - the declarations of the document, `Json`, the types its schemas give, `Input`, `Output`, each definition of a schema's `$defs` or `definitions` as a type of its own named after the schema and the definition, `Input_address`, a character a name cannot hold written as its code point between two `$`, `Input_street$45$address`, so that a `$ref` may recur, and, for a recall function, `View`, each `Json` without its schema, and, for a recall function, `Event`, the playbook's event with its attributes and no extension, so that a misspelt attribute is refused rather than read as an extension (`src/program-check/schema-types.ts`); @@ -192,7 +192,7 @@ Beside the compiler's diagnostics, a walk of the module (`src/program-check/modu ### The sandbox's lib -The lib is the sandbox's own, so a name the sandbox lacks is a name the check lacks. `sandboxLibOf(probe)` (`src/program-check/sandbox-lib.ts`) makes it from the compiler's ES2025 libs without `intl` and `weakref`, less the globals, `Math` members and `Date` methods of the engine's `sandboxRemovals`, the list the prelude removes them by, less the call signature of `DateConstructor` and its constructors of several numbers, which read the server's time zone, less the values of `Intl`, and less every global and member the sandbox answers it does not have, each asked of a fresh instance through the engine's `sandboxAnswers`, the members of an interface asked of a sample of it, which `src/program-check/instance-samples.ts` lists, a strict function among them, so the members that throw in a module, `arguments` and `caller`, go too. The comments go, and the licence notice of the compiler's libs is kept once at the top. The iterator lib stays a module of its own, as the compiler ships it, since its abstract class `Iterator` would collide with the global interface `Iterator` in a script (`lib.es2025.iterator.d.ts`, lines 19 to 22). +The lib is the sandbox's own, so a name the sandbox lacks is a name the check lacks. `sandboxLibOf(shippedFiles, probe)` (`src/program-check/sandbox-lib.ts`) makes it from the compiler's ES2025 libs, read through the `ShippedFiles` it is given, without `intl` and `weakref`, less the globals, `Math` members and `Date` methods of the engine's `sandboxRemovals`, the list the prelude removes them by, less the call signature of `DateConstructor` and its constructors of several numbers, which read the server's time zone, less the values of `Intl`, and less every global and member the sandbox answers it does not have, each asked of a fresh instance through the engine's `sandboxAnswers`, the members of an interface asked of a sample of it, which `src/program-check/instance-samples.ts` lists, a strict function among them, so the members that throw in a module, `arguments` and `caller`, go too. The comments go, and the licence notice of the compiler's libs is kept once at the top. The iterator lib stays a module of its own, as the compiler ships it, since its abstract class `Iterator` would collide with the global interface `Iterator` in a script (`lib.es2025.iterator.d.ts`, lines 19 to 22). The repository has no build step, so the lib is generated by a command, `pnpm --filter @on.auto/definitions sandbox-lib` (`write-sandbox-lib.ts`), and kept under `lib/`, beside `src`, as `sandbox.d.ts.txt` and `sandbox-iterator.d.ts.txt`: text files, which the repository's lint and format do not read as its own declarations. `src/program-check/sandbox-lib.test.ts` generates it again and fails when the kept lib differs, so a new build of the engine, a change to the prelude's list or a new compiler changes the kept lib in the same change. The server's image copies `lib/`. @@ -435,7 +435,7 @@ A run that finishes later records facts after the call that started it returned, ## Testing -`@on.auto/definitions/testing` exports `echo`, a small real capability for the tests of this and other packages. Its definition document is a JSON object with a string `greeting`, an optional string `description`, optional `warnings`, a list of strings, and optional `triggers`, which its summary gives back; a run takes a JSON object and answers `{ greeting, input }`. `nothingSaved` is the `SavedDefinitions` of a playbook with nothing active and no content kept, for a check a test runs outside a save. `steppingClock(stepMs, wall)` is the Effect `Clock` of a test that renders: its monotonic time, which a render counts on, starts at nothing and moves by `stepMs` each time it is read, so it stands still with a step of nothing, while it tells the time and sleeps as `wall` does, the real clock unless given. `standingClock` is it with a step of nothing, `standingRenderClock` the render clock that clock reads as, and `onTheStandingClock(effect)` runs an effect on it, so the tests of every capability render on one clock that stands still. `checkingWithin(pool, deadlineMs)` is the pool whose checks at save may take `deadlineMs`, `checksAwaitedAtMostMs`, 30 s, unless given, for the tests that check a document with the compiler in a real worker, which a busy machine can slow past the 2 s a save allows; the refusal at that deadline is held by a pool that answers it, in `src/program-check/save-check.test.ts`. Those tests keep the 10 s a checker's worker may take to start, `mostCheckStartMs` of `@on.auto/workflow-engine`, after which a check ends `busy`: the first check of a pool starts its worker within it on purpose, since the pool's own tests prove that bound with a worker slow to start. +`@on.auto/definitions/testing` exports `echo`, a small real capability for the tests of this and other packages. Its definition document is a JSON object with a string `greeting`, an optional string `description`, optional `warnings`, a list of strings, and optional `triggers`, which its summary gives back; a run takes a JSON object and answers `{ greeting, input }`. `nothingSaved` is the `SavedDefinitions` of a playbook with nothing active and no content kept, for a check a test runs outside a save. `steppingClock(stepMs, wall)` is the Effect `Clock` of a test that renders: its monotonic time, which a render counts on, starts at nothing and moves by `stepMs` each time it is read, so it stands still with a step of nothing, while it tells the time and sleeps as `wall` does, the real clock unless given. `standingClock` is it with a step of nothing, `standingRenderClock` the render clock that clock reads as, and `onTheStandingClock(effect)` runs an effect on it, so the tests of every capability render on one clock that stands still. `checkingWithin(pool, deadlineMs)` is the pool whose checks at save may take `deadlineMs`, `checksAwaitedAtMostMs`, 30 s, unless given, for the tests that check a document with the compiler in a real worker, which a busy machine can slow past the 2 s a save allows; the refusal at that deadline is held by a pool that answers it, in `src/program-check/save-check.test.ts`. `measuredPoolSettings` are the settings a pool of a test takes beside its bounds: the files its workers are handed, read on disk with `handedWorkersWithChecks`, and the `measuredEnvironment` of `@on.auto/operations/testing`, `NODE_V8_COVERAGE` alone, so the coverage of its workers is measured. Those tests keep the 10 s a checker's worker may take to start, `mostCheckStartMs` of `@on.auto/workflow-engine`, after which a check ends `busy`: the first check of a pool starts its worker within it on purpose, since the pool's own tests prove that bound with a worker slow to start. ## Source diff --git a/packages/definitions/measure/measure-checks.ts b/packages/definitions/measure/measure-checks.ts index dc26d751c..831f3ae29 100644 --- a/packages/definitions/measure/measure-checks.ts +++ b/packages/definitions/measure/measure-checks.ts @@ -1,11 +1,13 @@ import { cpus } from 'node:os'; +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { programPool, sandboxAnswers, type CheckJob } from '@on.auto/workflow-engine/dsl'; import { checkerOf } from '../src/program-check/checking.ts'; -import { keptSandboxLib } from '../src/program-check/kept-lib.ts'; +import { sandboxLibFrom } from '../src/program-check/kept-lib.ts'; import { sandboxLibOf } from '../src/program-check/sandbox-lib.ts'; -import { checkWorker } from '../src/program-check/save-check.ts'; +import { checkWorker, handedWorkersWithChecks } from '../src/program-check/save-check.ts'; +import { keptLibOnDisk } from '../src/testing/read-on-disk.ts'; const schemas = { input: { @@ -110,13 +112,13 @@ const [cpu] = cpus(); write(`Node ${process.version} on ${cpu?.model ?? 'an unknown processor'}, ${cpus().length} cores`); const generating = performance.now(); -const generated = await sandboxLibOf(sandboxAnswers); +const generated = await sandboxLibOf(shippedFilesOnDisk, (probed) => sandboxAnswers(shippedFilesOnDisk, probed)); write( `the lib generated in ${(performance.now() - generating).toFixed(0)} ms: ${generated.script.length} and ${generated.iterator.length} characters`, ); const making = performance.now(); -const check = checkerOf(keptSandboxLib()); +const check = checkerOf(sandboxLibFrom(keptLibOnDisk)); write(`the checker made from the kept lib in ${(performance.now() - making).toFixed(1)} ms`); write(`the example, its first check in the thread: ${timed(() => check(computation(pace))).toFixed(1)} ms`); @@ -138,9 +140,11 @@ for (const [name, job] of cases) { ); } -const pool = programPool({ workers: 1, heapMegabytes: 256 }); -const cold = await pool.check({ ...computation(pace), deadlineMs: 60_000, worker: checkWorker }); -const warm = await pool.check({ ...computation(pace), deadlineMs: 60_000, worker: checkWorker }); +const handed = await handedWorkersWithChecks(shippedFilesOnDisk); +const pool = programPool({ workers: 1, heapMegabytes: 256, handed }); +const checkedBy = { worker: checkWorker }; +const cold = await pool.check({ ...computation(pace), deadlineMs: 60_000, ...checkedBy }); +const warm = await pool.check({ ...computation(pace), deadlineMs: 60_000, ...checkedBy }); write( `the example in the pool's check worker: ${cold.milliseconds.toFixed(1)} ms the first, the worker's start included, then ${warm.milliseconds.toFixed(1)} ms`, ); diff --git a/packages/definitions/src/check.ts b/packages/definitions/src/check.ts index 991453b5a..76edda13e 100644 --- a/packages/definitions/src/check.ts +++ b/packages/definitions/src/check.ts @@ -1,2 +1,7 @@ export { mostAssignmentsInAFunction } from './program-check/assignment-bound.ts'; -export { checkedAtSave, warmedChecks, type CheckedDocument } from './program-check/save-check.ts'; +export { + checkedAtSave, + handedWorkersWithChecks, + warmedChecks, + type CheckedDocument, +} from './program-check/save-check.ts'; diff --git a/packages/definitions/src/program-check/assignment-bound.test.ts b/packages/definitions/src/program-check/assignment-bound.test.ts index 8064f6939..d73c91be1 100644 --- a/packages/definitions/src/program-check/assignment-bound.test.ts +++ b/packages/definitions/src/program-check/assignment-bound.test.ts @@ -2,11 +2,12 @@ import type { CheckJob } from '@on.auto/workflow-engine/worker'; import type { Schema } from 'effect'; import { describe, expect, it } from 'vitest'; +import { keptLibOnDisk } from '../testing/read-on-disk.ts'; import { fewerAssignments, mostAssignmentsInAFunction } from './assignment-bound.ts'; import { checkerOf } from './checking.ts'; -import { keptSandboxLib } from './kept-lib.ts'; +import { sandboxLibFrom } from './kept-lib.ts'; -const check = checkerOf(keptSandboxLib()); +const check = checkerOf(sandboxLibFrom(keptLibOnDisk)); const output: Schema.JsonObject = { type: 'object', diff --git a/packages/definitions/src/program-check/check-worker.ts b/packages/definitions/src/program-check/check-worker.ts index 84830efea..c03b92b2d 100644 --- a/packages/definitions/src/program-check/check-worker.ts +++ b/packages/definitions/src/program-check/check-worker.ts @@ -1,6 +1,5 @@ -import { serveJobs } from '@on.auto/workflow-engine/job-loop'; +import { serveJobsWith } from '@on.auto/workflow-engine/job-loop'; -import { checkerOf } from './checking.ts'; -import { keptSandboxLib } from './kept-lib.ts'; +import { handedChecks } from './kept-lib.ts'; -serveJobs({ check: checkerOf(keptSandboxLib()) }); +serveJobsWith(handedChecks); diff --git a/packages/definitions/src/program-check/kept-lib.ts b/packages/definitions/src/program-check/kept-lib.ts index 77411da4c..fa618488e 100644 --- a/packages/definitions/src/program-check/kept-lib.ts +++ b/packages/definitions/src/program-check/kept-lib.ts @@ -1,5 +1,6 @@ -import { readFileSync } from 'node:fs'; +import type { CheckHandler, HandedFile } from '@on.auto/workflow-engine/job-loop'; +import { checkerOf } from './checking.ts'; import type { SandboxLib } from './sandbox-lib.ts'; export const keptLibFiles: Readonly> = { @@ -7,9 +8,14 @@ export const keptLibFiles: Readonly> = { iterator: new URL('../../lib/sandbox-iterator.d.ts.txt', import.meta.url), }; -export function keptSandboxLib(): SandboxLib { +export function sandboxLibFrom(handed: HandedFile): SandboxLib { + const decoder = new TextDecoder(); return { - script: readFileSync(keptLibFiles.script, 'utf8'), - iterator: readFileSync(keptLibFiles.iterator, 'utf8'), + script: decoder.decode(handed(keptLibFiles.script)), + iterator: decoder.decode(handed(keptLibFiles.iterator)), }; } + +export function handedChecks(handed: HandedFile): { readonly check: CheckHandler } { + return { check: checkerOf(sandboxLibFrom(handed)) }; +} diff --git a/packages/definitions/src/program-check/module-checks.test.ts b/packages/definitions/src/program-check/module-checks.test.ts index 180bf4173..86f9b66ac 100644 --- a/packages/definitions/src/program-check/module-checks.test.ts +++ b/packages/definitions/src/program-check/module-checks.test.ts @@ -2,11 +2,12 @@ import type { CheckJob } from '@on.auto/workflow-engine/worker'; import type { Schema } from 'effect'; import { describe, expect, it } from 'vitest'; +import { keptLibOnDisk } from '../testing/read-on-disk.ts'; import { checkerOf } from './checking.ts'; -import { keptSandboxLib } from './kept-lib.ts'; +import { sandboxLibFrom } from './kept-lib.ts'; import { answersAtOnce, holdsNothing, readsNothing, signatures } from './module-rules.ts'; -const check = checkerOf(keptSandboxLib()); +const check = checkerOf(sandboxLibFrom(keptLibOnDisk)); const input: Schema.JsonObject = { type: 'object', diff --git a/packages/definitions/src/program-check/place-checks.test.ts b/packages/definitions/src/program-check/place-checks.test.ts index 83b947b48..6b51db7d8 100644 --- a/packages/definitions/src/program-check/place-checks.test.ts +++ b/packages/definitions/src/program-check/place-checks.test.ts @@ -2,12 +2,13 @@ import type { CheckJob } from '@on.auto/workflow-engine/worker'; import type { Schema } from 'effect'; import { describe, expect, it } from 'vitest'; +import { keptLibOnDisk } from '../testing/read-on-disk.ts'; import { checkerOf } from './checking.ts'; import { oneExpression } from './expression-issues.ts'; -import { keptSandboxLib } from './kept-lib.ts'; +import { sandboxLibFrom } from './kept-lib.ts'; import { holdsNothing, readsNothing, signatures } from './module-rules.ts'; -const check = checkerOf(keptSandboxLib()); +const check = checkerOf(sandboxLibFrom(keptLibOnDisk)); const view: Schema.JsonObject = { type: 'object', diff --git a/packages/definitions/src/program-check/sandbox-lib.test.ts b/packages/definitions/src/program-check/sandbox-lib.test.ts index 85faeddd3..5b2402b81 100644 --- a/packages/definitions/src/program-check/sandbox-lib.test.ts +++ b/packages/definitions/src/program-check/sandbox-lib.test.ts @@ -1,10 +1,27 @@ +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { sandboxAnswers } from '@on.auto/workflow-engine/dsl'; +import { handedFileIn } from '@on.auto/workflow-engine/job-loop'; +import type { CheckJob } from '@on.auto/workflow-engine/worker'; import { describe, expect, it } from 'vitest'; -import { keptLibFiles, keptSandboxLib } from './kept-lib.ts'; +import { keptLibOnDisk } from '../testing/read-on-disk.ts'; +import { handedChecks, keptLibFiles, sandboxLibFrom } from './kept-lib.ts'; import { sandboxLibOf } from './sandbox-lib.ts'; -const kept = keptSandboxLib(); +const kept = sandboxLibFrom(keptLibOnDisk); + +const probe = (expressions: readonly string[]) => sandboxAnswers(shippedFilesOnDisk, expressions); + +function computationReturning(output: string): CheckJob { + return { + module: { + place: 'computation', + source: `export default function (input: Input): Output {\n return ${output};\n}`, + }, + schemas: { input: { type: 'object' }, output: { type: 'array', items: { type: 'number' } } }, + expressions: [], + }; +} function declares(text: string): (fragment: string) => boolean { return (fragment) => text.includes(fragment); @@ -12,7 +29,7 @@ function declares(text: string): (fragment: string) => boolean { describe('the lib of the sandbox', () => { it('is what the generator makes from the compiler’s libs and the sandbox’s own answers, kept beside the package', async () => { - expect(await sandboxLibOf(sandboxAnswers)).toEqual(kept); + expect(await sandboxLibOf(shippedFilesOnDisk, probe)).toEqual(kept); expect(keptLibFiles.script.pathname).toMatch(/\/definitions\/lib\/sandbox\.d\.ts\.txt$/u); expect(keptLibFiles.iterator.pathname).toMatch(/\/definitions\/lib\/sandbox-iterator\.d\.ts\.txt$/u); }); @@ -56,3 +73,21 @@ describe('the lib of the sandbox', () => { expect(kept.script.startsWith('/*!')).toBe(true); }); }); + +describe('the checks of the worker the pool hands the kept lib', () => { + it('answer the compiler’s issues against the sandbox’s lib, so a method the sandbox has checks and one it lacks does not', () => { + const { check } = handedChecks(keptLibOnDisk); + + expect(check(computationReturning('[3, 1].toSorted()'))).toMatchObject({ ran: 'checked', issues: [] }); + expect(check(computationReturning('[Math.random()]'))).toMatchObject({ + ran: 'checked', + issues: [{ at: 'module', line: 2, detail: "Property 'random' does not exist on type 'Math'." }], + }); + }); + + it('cannot be made when the pool handed the worker no kept lib', () => { + expect(() => handedChecks(handedFileIn({}))).toThrow( + `The pool handed this worker no file at ${keptLibFiles.script.href}`, + ); + }); +}); diff --git a/packages/definitions/src/program-check/sandbox-lib.ts b/packages/definitions/src/program-check/sandbox-lib.ts index 2137b0d04..6dd8393af 100644 --- a/packages/definitions/src/program-check/sandbox-lib.ts +++ b/packages/definitions/src/program-check/sandbox-lib.ts @@ -1,5 +1,4 @@ -import { readFileSync } from 'node:fs'; - +import type { ShippedFiles } from '@on.auto/operations'; import { sandboxRemovals } from '@on.auto/workflow-engine/worker'; import ts from 'typescript6'; @@ -48,26 +47,29 @@ const libReferences = /^\/\/\/ \n/gmu; const iteratorLib = 'lib.es2025.iterator.d.ts'; -function libSource(name: string): ts.SourceFile { +async function libSource(shippedFiles: ShippedFiles, name: string): Promise { const fileName = `lib.${name}.d.ts`; - return ts.createSourceFile( - fileName, - readFileSync(new URL(fileName, libFolder), 'utf8'), - ts.ScriptTarget.ES2025, - true, - ); + const text = await shippedFiles.text(new URL(fileName, libFolder)); + return ts.createSourceFile(fileName, text, ts.ScriptTarget.ES2025, true); } -function libsFrom(name: string, seen: Set): readonly ts.SourceFile[] { +async function libsFrom( + shippedFiles: ShippedFiles, + name: string, + seen: Set, +): Promise { if (seen.has(name) || skipped.test(name)) { return []; } seen.add(name); - const source = libSource(name); - const referenced: ts.SourceFile[] = []; - for (const { fileName } of source.libReferenceDirectives) { - referenced.push(...libsFrom(fileName, seen)); - } + const source = await libSource(shippedFiles, name); + const referenced = await source.libReferenceDirectives.reduce>( + async (before, { fileName }: Readonly) => [ + ...(await before), + ...(await libsFrom(shippedFiles, fileName, seen)), + ], + Promise.resolve([]), + ); return [...referenced, source]; } @@ -218,8 +220,8 @@ function libFilesOf(sources: () => readonly ts.SourceFile[]): readonly LibFile[] return files; } -export async function sandboxLibOf(probe: Probe): Promise { - const sources = libsFrom('es2025', new Set()); +export async function sandboxLibOf(shippedFiles: ShippedFiles, probe: Probe): Promise { + const sources = await libsFrom(shippedFiles, 'es2025', new Set()); const files = libFilesOf(() => sources); const answers = await probe(files.flatMap(({ candidates }) => candidates.probed.map(({ expression }) => expression))); let answered = 0; diff --git a/packages/definitions/src/program-check/save-check.test.ts b/packages/definitions/src/program-check/save-check.test.ts index 9826971db..4a23c764e 100644 --- a/packages/definitions/src/program-check/save-check.test.ts +++ b/packages/definitions/src/program-check/save-check.test.ts @@ -3,11 +3,12 @@ import { programPool, type CheckOutcome, type CheckJob, type ProgramPool } from import { Effect, Exit } from 'effect'; import { afterAll, describe, expect, it } from 'vitest'; +import { measuredPoolSettings } from '../testing/read-on-disk.ts'; import { checkingWithin } from '../testing/time/patient-checks.ts'; import * as checkWorkerModule from './check-worker.ts'; import { checkDeadlineMs, checkedAtSave, checkWorker, warmedChecks } from './save-check.ts'; -const pool = programPool({ workers: 1, heapMegabytes: 256 }); +const pool = programPool({ workers: 1, heapMegabytes: 256, ...measuredPoolSettings }); const patient = checkingWithin(pool); diff --git a/packages/definitions/src/program-check/save-check.ts b/packages/definitions/src/program-check/save-check.ts index 76b11a225..87131c467 100644 --- a/packages/definitions/src/program-check/save-check.ts +++ b/packages/definitions/src/program-check/save-check.ts @@ -1,9 +1,13 @@ -import { InvalidInput, Unavailable } from '@on.auto/operations'; +import { InvalidInput, Unavailable, type ShippedFiles } from '@on.auto/operations'; import { + handedFilesOf, + handedWorkersOf, pooledJob, type CheckIssue, type CheckJob, type CheckOutcome, + type HandedFiles, + type HandedWorkers, type ProgramPool, type Stopped, type StrippedSources, @@ -11,9 +15,18 @@ import { import { Effect } from 'effect'; import { changedExpressions, type StrippedForms } from '../capability/stripped-forms.ts'; +import { keptLibFiles } from './kept-lib.ts'; export const checkWorker = new URL('./check-worker.ts', import.meta.url); +export function keptLibOf(shippedFiles: ShippedFiles): Promise { + return handedFilesOf(shippedFiles, Object.values(keptLibFiles)); +} + +export async function handedWorkersWithChecks(shippedFiles: ShippedFiles): Promise { + return handedWorkersOf(shippedFiles, { [checkWorker.href]: await keptLibOf(shippedFiles) }); +} + export const checkDeadlineMs = 2000; export type CheckedDocument = { readonly issues: readonly CheckIssue[] } | { readonly stripped: StrippedForms }; diff --git a/packages/definitions/src/testing/index.ts b/packages/definitions/src/testing/index.ts index dd9d2509e..56bc622a6 100644 --- a/packages/definitions/src/testing/index.ts +++ b/packages/definitions/src/testing/index.ts @@ -4,5 +4,6 @@ export { recordingJournal, type RecordingJournal } from './recording-journal.ts' export { noLongestRuns } from './longest-runs.ts'; export { nothingSaved } from './nothing-saved.ts'; export { checkingWithin, checksAwaitedAtMostMs } from './time/patient-checks.ts'; +export { measuredPoolSettings } from './read-on-disk.ts'; export { onTheStandingClock, standingClock, standingRenderClock, steppingClock } from './time/stepping-clock.ts'; export { shapeNamed } from './shapes.ts'; diff --git a/packages/definitions/src/testing/read-on-disk.ts b/packages/definitions/src/testing/read-on-disk.ts new file mode 100644 index 000000000..10e428bcf --- /dev/null +++ b/packages/definitions/src/testing/read-on-disk.ts @@ -0,0 +1,13 @@ +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; +import { measuredEnvironment } from '@on.auto/operations/testing'; +import type { PoolSettings } from '@on.auto/workflow-engine/dsl'; +import { handedFileIn, type HandedFile } from '@on.auto/workflow-engine/job-loop'; + +import { handedWorkersWithChecks, keptLibOf } from '../program-check/save-check.ts'; + +export const measuredPoolSettings: Pick = { + handed: await handedWorkersWithChecks(shippedFilesOnDisk), + environment: measuredEnvironment, +}; + +export const keptLibOnDisk: HandedFile = handedFileIn(await keptLibOf(shippedFilesOnDisk)); diff --git a/packages/definitions/write-sandbox-lib.ts b/packages/definitions/write-sandbox-lib.ts index 69a1b5688..4ed8ef662 100644 --- a/packages/definitions/write-sandbox-lib.ts +++ b/packages/definitions/write-sandbox-lib.ts @@ -1,11 +1,12 @@ import { writeFileSync } from 'node:fs'; +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { sandboxAnswers } from '@on.auto/workflow-engine/dsl'; import { keptLibFiles } from './src/program-check/kept-lib.ts'; import { sandboxLibOf } from './src/program-check/sandbox-lib.ts'; -const lib = await sandboxLibOf(sandboxAnswers); +const lib = await sandboxLibOf(shippedFilesOnDisk, (expressions) => sandboxAnswers(shippedFilesOnDisk, expressions)); writeFileSync(keptLibFiles.script, lib.script); writeFileSync(keptLibFiles.iterator, lib.iterator); diff --git a/packages/ledger/src/portable-entry.test.ts b/packages/ledger/src/portable-entry.test.ts index 5f9948929..41d66ea5d 100644 --- a/packages/ledger/src/portable-entry.test.ts +++ b/packages/ledger/src/portable-entry.test.ts @@ -1,39 +1,13 @@ -import { readFileSync } from 'node:fs'; -import { dirname, join } from 'node:path'; - +import { modulesReachedFrom } from '@on.auto/operations/testing'; import { describe, expect, it } from 'vitest'; -const runtimeImport = /^\s*(?:import|export)\s+(?!type\b)[^'"]*?\bfrom\s+'([^']+)'/gmu; - -function importsOf(file: string): readonly string[] { - const specifiers: string[] = []; - for (const match of readFileSync(file, 'utf8').matchAll(runtimeImport)) { - specifiers.push(String(match[1])); - } - return specifiers; -} - -function packagesReachableFrom(entry: string): ReadonlySet { - const visited = new Set(); - const packages = new Set(); - const pending = [join(import.meta.dirname, entry)]; - while (pending.length > 0) { - const file = String(pending.pop()); - const specifiers = visited.has(file) ? [] : importsOf(file); - visited.add(file); - pending.push( - ...specifiers.filter((specifier) => specifier.startsWith('.')).map((relative) => join(dirname(file), relative)), - ); - for (const name of specifiers.filter((specifier) => !specifier.startsWith('.'))) { - packages.add(name); - } - } - return packages; +function packagesReachableFrom(entry: string): readonly string[] { + return modulesReachedFrom(import.meta.dirname, entry).packages; } describe('the ledger entry points', () => { it('keep the drivers out of the main entry', () => { - const packages = [...packagesReachableFrom('index.ts')]; + const packages = packagesReachableFrom('index.ts'); expect(packages.filter((name) => name.includes('sqlite3'))).toEqual([]); expect(packages).not.toContain('pg'); @@ -42,7 +16,7 @@ describe('the ledger entry points', () => { }); it('keep the PostgreSQL event store and its driver behind the postgresql entry', () => { - const packages = [...packagesReachableFrom('postgresql/postgresql-ledger.ts')]; + const packages = packagesReachableFrom('postgresql/postgresql-ledger.ts'); expect(packages).toContain('@event-driven-io/emmett-postgresql'); expect(packages).toContain('@event-driven-io/emmett-postgresql/pg'); diff --git a/packages/operations/README.md b/packages/operations/README.md index 7d9bb0192..5986dcca0 100644 --- a/packages/operations/README.md +++ b/packages/operations/README.md @@ -208,12 +208,22 @@ The digest is Node's, behind the one module of the repository for it: the digest `cacheFolderProblem(directory, owner?)` (`src/folders/cache-folder.ts`) is the one rule of a folder the server keeps files of its own in, which it later trusts: it makes the folder when it is not there, with `cacheFolderMode`, `0700`, and answers what is wrong with it, that it could not be made, is not a folder, belongs to another user than the server's, or may be written by other users, or nothing. The model cache of prediction functions and the Studio the server serves read it, so no other user can make or change the files either serves from. +## The files a package ships + +`ShippedFiles` (`src/shipped-files/shipped-files.ts`) is the port through which a package reads the files it ships beside its code: `text(file)`, `bytes(file)` and `list(folder)`, the names of the files directly in a folder, sorted, each of a `URL` the package computes from its own `import.meta.url`, as `new URL('../entries/', import.meta.url)`. A package computes where its files are and never how they are read: it takes the reader as a dependency of the function or factory that reads, never from an import of the reader, and the reader never knows what a file is for, so how they are read is one decision, taken once by the server that composes the packages. What a package read it hands on as content, never as a path for another package to read again: `@on.auto/providers` hands the tool servers the texts of the documents beside its entries, and a pool of `@on.auto/workflow-engine` is handed the bytes its workers start with. `shippedFilesOnDisk` (`src/shipped-files/shipped-files-on-disk.ts`) is the one reader, and the one module of the port that imports `node:fs`; it is an entry of its own, `@on.auto/operations/shipped-files-on-disk`, as the drivers of `@on.auto/ledger` are, so that the main entry, which every package imports, and the testing entry reach no file system for it (`src/shipped-files/shipped-files-entries.test.ts`). + ## Testing `@on.auto/operations/testing` exports an in-memory `Ledger`, `PlaybookRegistry` and `IncidentReporter` for the tests of packages that define or serve operations. The in-memory ledger names and links its messages as the ledger does. `memoryPlaybookRegistry(active, retired)` takes the active playbooks and, optionally, the retired ones; any other playbook is unknown. `memoryLedger(runOutcomes)` keeps the outcomes of runs with the mapping it is given, and none without one; an append whose mapping throws keeps nothing. `runTallies` is a small `RunOutcomeMapping` over `runFacts`, a decider that appends the `run_began` and `run_ended` facts it is given, for the tests of a ledger. `memoryLedger(runOutcomes, projections)` keeps the projections it is given too, and `runTallyRows`, `tallyRowsOf(version)` and `readTallyRows` are a small projection over the same facts and a query of it, and `topicRows` over `topicFacts` a projection keyed by its mapping over two stream kinds with columns its reader advances, for the tests of a ledger's projections. The in-memory ledger reads what a playbook recorded as the ledger does, with the context of each message, and the behaviour suite of `@on.auto/ledger` runs on it; it keeps recorded content whole in memory, `memoryRecordedContent()`, by the rules of the port the ledger keeps, its refusals, its first media type and its end of staging among them, which the tests of other packages take alone too, and `keptTextOf(content, playbook, sha256)` reads a kept text back as text. `internalTermsIn(text, measuredIn?)` (`src/plain-language/internal-terms.ts`), beside the plain words it guards, finds the terms a person's words must not hold, which the tests of every package that words a result run: the internal words, a name in capitals with underscores, an id, a path, and a number from 100 to 599 that reads as a status code, unless a unit follows it, one of the units the runtime measures in or one the text names in `measuredIn`, such as the unit of a prediction function, alone or after a range written `N to M`, as in `100 to 160 orders`; a number before `and` is not in a range, so the `404` of `404 and 3 rows` is still found. +`shippedFilesInMemory(files)` (`src/shipped-files/shipped-files-in-memory.ts`) is a `ShippedFiles` over a record of contents by the `href` of their files, each a text or bytes, for the unit tests of a package that reads what it ships: a folder lists the files directly under it, and a file it does not hold is refused, as is a folder that holds none, since it holds no empty folder; the same tests run on it and on the reader on disk (`src/shipped-files/shipped-files.test.ts`). + `recordedContentBehaviour(entry)` is the behaviour suite of the content port, in `src/content-testing`: a content of no bytes to 64 MiB kept and streamed back, a text kept once with its first media type in its playbook alone, a content put again whole or with its digest, the refusals of a wrong digest, a wrong size and a stream that fails, wrong bytes put with the digest of a content held, a wrong put racing a right one, and staging, read through a second client, ended, refused and raced. An adapter runs it with a `ContentEntry` whose `aStore()` answers `{ content, elsewhere, unnamedLeft }`: the port, the port of a second client of the same store, and how many pieces of puts no kept content names the store holds, which is 0 after every test. This package runs it on `memoryRecordedContent`, the ledger on SQLite and PostgreSQL, and `@on.auto/object-storage` on its fake bucket and on a real S3-compatible server. The samples it is made of are exported with it: `generated(bytes, seed)` and `generatedStream`, bytes made the same for a seed, `answerOfAbout(bytes)`, a JSON answer of about that size, `readBack` and `readThrough`, which digest what a read streams, `deliveredBeforeFailing(stream)`, how many bytes a failing stream handed over, `brokenAfter(bytes)` and `unread()`, streams that fail or are never read, and `resultOf`, `refusalOf` and `present`, which run an effect for its value, its failure or its value present. `spawnForTheTest(command, args, options)` (`src/process-testing/test-processes.ts`) is how a test of the repository starts a process it runs itself. It starts it as `spawn` of `node:child_process` does and registers its end with `onTestFinished` of Vitest, the hook that runs once the test finished, whether it passed, failed or timed out: a process still there is sent SIGTERM, on which the server stops, and SIGKILL if it is still there `stoppedWithinMs`, 10 s, later, a bound the server's test of its longest stop holds that stop under, and well within the 60 s a hook may take. `endWithTheTest(pid)` ends in the same way, by its pid, a process the test did not start itself, such as a server started in the background, and signals no pid that `kill(pid, 0)` finds gone or another user's; it cannot tell a pid the system gave to another process of the same user since. `endWithTheTest(reader)` takes a `PidReader` in place of the pid, registered before the process is started: once the test finished, it reads the pid with `read()` again and again while `stillToCome()` holds, for at most `readWithinMs`, and ends the pid it read, if any. A worker of the test runner that dies runs no hook, so it ends nothing. +`measuredEnvironment` (`src/process-testing/measured-environment.ts`) is the environment a test gives a worker or a process in place of its own: `NODE_V8_COVERAGE` alone, when the run measures coverage, so the coverage of what it starts is measured and no other variable of the test reaches it. + +`modulesReachedFrom(folder, entry)` (`src/module-testing/modules-reached.ts`) answers the files an entry of a package reaches through the imports that load when it runs, a type import left out, each relative to the folder of its sources, and the packages those files import, by which the tests of the entries of this package, of `@on.auto/ledger` and of `@on.auto/workflow-engine` keep what an entry must not reach behind another. + `aFolderOfTheTestRun(runningIn)` (`src/process-testing/test-run-folder.ts`) is the global setup of the tests of every package, which `vitest.shared.ts` names with its default, `aFolderOfTheTestRun()`. Each run makes a folder of its own in the temporary folder, `auto-runtime-test-run--`, and sets `TMPDIR` to it before any worker starts, so `tmpdir()` of `node:os` answers it in every worker and in every process a test starts with the environment it inherits, and two runs at once have two folders. Its teardown, which a worker that dies, a hook that times out or a child that is killed does not keep from running, lists what the tests left there, ends each `RunningProcess` that `runningIn(entry)` finds still running in an entry, with SIGTERM and with SIGKILL if `isStillRunning()` still holds `stoppedWithinMs` later, sending a signal only while it holds, fails the run naming each entry and each process it ended, and removes the folder whatever happened. A package whose own processes may outlive their test gives it a `runningIn` of its own, as the server does for the runtimes its tests start in the background, through `@on.auto/operations/test-run-folder`. That is an entry of its own because a global setup runs in Vitest's own process and not in a worker, so neither it nor anything it imports takes Vitest, with whose hooks `@on.auto/operations/testing` registers what it ends; `endedWhileItRuns(running)` of `@on.auto/operations/testing` is the same end for the hook of a test, over the same `RunningProcess`. diff --git a/packages/operations/package.json b/packages/operations/package.json index 41a1650e9..7929c6ed0 100644 --- a/packages/operations/package.json +++ b/packages/operations/package.json @@ -6,6 +6,7 @@ "type": "module", "exports": { ".": "./src/index.ts", + "./shipped-files-on-disk": "./src/shipped-files/shipped-files-on-disk.ts", "./test-run-folder": "./src/process-testing/test-run-folder.ts", "./testing": "./src/testing/index.ts" }, diff --git a/packages/operations/src/index.ts b/packages/operations/src/index.ts index d49ef7a24..c0ef0b3c4 100644 --- a/packages/operations/src/index.ts +++ b/packages/operations/src/index.ts @@ -284,3 +284,4 @@ export { uuidV5 } from './uuid/uuid-v5.ts'; export { randomUUIDv7, uuidV7Before, uuidV7TimeOf } from './uuid/uuid-v7.ts'; export { loopbackNames, namesTheLoopback } from './addresses/loopback-names.ts'; export { cacheFolderMode, cacheFolderProblem } from './folders/cache-folder.ts'; +export type { ShippedFiles } from './shipped-files/shipped-files.ts'; diff --git a/packages/operations/src/module-testing/modules-reached.ts b/packages/operations/src/module-testing/modules-reached.ts new file mode 100644 index 000000000..df641f5d8 --- /dev/null +++ b/packages/operations/src/module-testing/modules-reached.ts @@ -0,0 +1,36 @@ +import { readFileSync } from 'node:fs'; +import { dirname, join, normalize } from 'node:path'; + +export interface ModulesReached { + readonly files: readonly string[]; + readonly packages: readonly string[]; +} + +const runtimeImport = /^\s*(?:import|export)\s+(?!type\b)[^'"]*?\bfrom\s+'([^']+)'/gmu; + +function importsOf(file: string): readonly string[] { + return [...readFileSync(file, 'utf8').matchAll(runtimeImport)].map( + ([, specifier = '']: readonly string[]) => specifier, + ); +} + +export function modulesReachedFrom(folder: string, entry: string): ModulesReached { + const files = new Set([entry]); + const packages = new Set(); + const pending = [entry]; + for (let file = pending.pop(); file !== undefined; file = pending.pop()) { + const specifiers = importsOf(join(folder, file)); + const unseen = specifiers + .filter((specifier) => specifier.startsWith('.')) + .map((relative) => normalize(join(dirname(file), relative))) + .filter((imported) => !files.has(imported)); + for (const imported of unseen) { + files.add(imported); + pending.push(imported); + } + for (const name of specifiers.filter((specifier) => !specifier.startsWith('.'))) { + packages.add(name); + } + } + return { files: [...files].toSorted(), packages: [...packages].toSorted() }; +} diff --git a/packages/operations/src/process-testing/measured-environment.test.ts b/packages/operations/src/process-testing/measured-environment.test.ts new file mode 100644 index 000000000..2d14a4f27 --- /dev/null +++ b/packages/operations/src/process-testing/measured-environment.test.ts @@ -0,0 +1,10 @@ +import { describe, expect, it } from 'vitest'; + +import { measuredEnvironment } from './measured-environment.ts'; + +describe('the environment a worker or a process of a test is given', () => { + it('holds the folder the coverage of the run is written to, as the run gives it, and nothing else', () => { + expect(Object.keys(measuredEnvironment).filter((key) => key !== 'NODE_V8_COVERAGE')).toEqual([]); + expect(measuredEnvironment['NODE_V8_COVERAGE']).toBe(process.env['NODE_V8_COVERAGE']); + }); +}); diff --git a/packages/operations/src/process-testing/measured-environment.ts b/packages/operations/src/process-testing/measured-environment.ts new file mode 100644 index 000000000..6923b75fb --- /dev/null +++ b/packages/operations/src/process-testing/measured-environment.ts @@ -0,0 +1,5 @@ +export const measuredEnvironment: Readonly> = Object.fromEntries( + Object.entries(process.env).flatMap(([key, value]: readonly [string, string | undefined]) => + key === 'NODE_V8_COVERAGE' && value !== undefined ? [[key, value]] : [], + ), +); diff --git a/packages/operations/src/shipped-files/shipped-files-entries.test.ts b/packages/operations/src/shipped-files/shipped-files-entries.test.ts new file mode 100644 index 000000000..1b64e9136 --- /dev/null +++ b/packages/operations/src/shipped-files/shipped-files-entries.test.ts @@ -0,0 +1,30 @@ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +import { Schema } from 'effect'; +import { describe, expect, it } from 'vitest'; + +import { modulesReachedFrom } from '../module-testing/modules-reached.ts'; + +const source = join(import.meta.dirname, '..'); + +const diskReader = 'shipped-files/shipped-files-on-disk.ts'; + +const decodeExports = Schema.decodeUnknownSync( + Schema.fromJsonString(Schema.Struct({ exports: Schema.Record(Schema.String, Schema.String) })), +); + +describe('the entries of the package', () => { + it('keep the reader of the shipped files on disk behind an entry of its own, so the main and testing entries reach no file system for it', () => { + const { exports } = decodeExports(readFileSync(join(source, '../package.json'), 'utf8')); + const main = modulesReachedFrom(source, 'index.ts'); + const testing = modulesReachedFrom(source, 'testing/index.ts'); + + expect(exports['./shipped-files-on-disk']).toBe(`./src/${diskReader}`); + expect(modulesReachedFrom(source, diskReader).packages).toEqual(['node:fs/promises']); + expect(main.files).toContain('dispatch/dispatcher.ts'); + expect(main.files).not.toContain(diskReader); + expect(testing.files).toContain('shipped-files/shipped-files-in-memory.ts'); + expect(testing.files).not.toContain(diskReader); + }); +}); diff --git a/packages/operations/src/shipped-files/shipped-files-in-memory.ts b/packages/operations/src/shipped-files/shipped-files-in-memory.ts new file mode 100644 index 000000000..79e95ccb9 --- /dev/null +++ b/packages/operations/src/shipped-files/shipped-files-in-memory.ts @@ -0,0 +1,42 @@ +import type { ShippedFiles } from './shipped-files.ts'; + +type Content = string | Readonly; + +type Held = ReadonlyMap; + +function contentOf(held: Held, file: Readonly): Promise { + const content = held.get(file.href); + return content === undefined + ? Promise.reject(new Error(`No file is shipped at ${file.href}`)) + : Promise.resolve(content); +} + +function textOf(content: Content): string { + return typeof content === 'string' ? content : new TextDecoder().decode(content); +} + +function bytesOf(content: Content): Uint8Array { + return typeof content === 'string' ? new TextEncoder().encode(content) : new Uint8Array(content); +} + +function namesIn(held: Held, folder: Readonly): Promise { + const under = [...held.keys()].filter((href) => href.startsWith(folder.href)); + if (under.length === 0) { + return Promise.reject(new Error(`No folder is shipped at ${folder.href}`)); + } + return Promise.resolve( + under + .map((href) => href.slice(folder.href.length)) + .filter((name) => !name.includes('/')) + .toSorted(), + ); +} + +export function shippedFilesInMemory(files: Readonly>): ShippedFiles { + const held: Held = new Map(Object.entries(files)); + return { + text: (file) => contentOf(held, file).then(textOf), + bytes: (file) => contentOf(held, file).then(bytesOf), + list: (folder) => namesIn(held, folder), + }; +} diff --git a/packages/operations/src/shipped-files/shipped-files-on-disk.ts b/packages/operations/src/shipped-files/shipped-files-on-disk.ts new file mode 100644 index 000000000..e6e2e5aaa --- /dev/null +++ b/packages/operations/src/shipped-files/shipped-files-on-disk.ts @@ -0,0 +1,16 @@ +import type { Dirent } from 'node:fs'; +import { readdir, readFile } from 'node:fs/promises'; + +import type { ShippedFiles } from './shipped-files.ts'; + +export const shippedFilesOnDisk: ShippedFiles = { + text: (file) => readFile(file, 'utf8'), + bytes: (file) => readFile(file), + list: async (folder) => { + const entries = await readdir(folder, { withFileTypes: true }); + return entries + .filter((entry: Readonly) => entry.isFile()) + .map(({ name }: Readonly) => name) + .toSorted(); + }, +}; diff --git a/packages/operations/src/shipped-files/shipped-files.test.ts b/packages/operations/src/shipped-files/shipped-files.test.ts new file mode 100644 index 000000000..8dc783ce0 --- /dev/null +++ b/packages/operations/src/shipped-files/shipped-files.test.ts @@ -0,0 +1,61 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { afterAll, describe, expect, it } from 'vitest'; + +import { shippedFilesInMemory } from './shipped-files-in-memory.ts'; +import { shippedFilesOnDisk } from './shipped-files-on-disk.ts'; +import type { ShippedFiles } from './shipped-files.ts'; + +const notes = 'Notes, in UTF-8: café\n'; + +const mark = Uint8Array.of(0, 255, 1); + +const inner = 'Inner notes\n'; + +const onDisk = mkdtempSync(join(tmpdir(), 'auto-runtime-shipped-files-')); +mkdirSync(join(onDisk, 'nested')); +writeFileSync(join(onDisk, 'notes.txt'), notes); +writeFileSync(join(onDisk, 'mark.bin'), mark); +writeFileSync(join(onDisk, 'nested', 'inner.txt'), inner); + +afterAll(() => { + rmSync(onDisk, { recursive: true, force: true }); +}); + +const inMemory = new URL('file:///shipped/'); + +const readers: readonly (readonly [string, ShippedFiles, Readonly])[] = [ + ['on disk', shippedFilesOnDisk, pathToFileURL(`${onDisk}/`)], + [ + 'in memory', + shippedFilesInMemory({ + [new URL('notes.txt', inMemory).href]: notes, + [new URL('mark.bin', inMemory).href]: mark, + [new URL('nested/inner.txt', inMemory).href]: inner, + }), + inMemory, + ], +]; + +describe.each(readers)('the files shipped %s', (_, reader, folder) => { + it('are read as text in UTF-8 and as bytes', async () => { + expect(await reader.text(new URL('notes.txt', folder))).toBe(notes); + expect([...(await reader.bytes(new URL('notes.txt', folder)))]).toEqual([...new TextEncoder().encode(notes)]); + expect([...(await reader.bytes(new URL('mark.bin', folder)))]).toEqual([0, 255, 1]); + expect(await reader.text(new URL('mark.bin', folder))).toBe('\u0000�\u0001'); + }); + + it('are listed by the names of the files directly in a folder, sorted', async () => { + expect(await reader.list(folder)).toEqual(['mark.bin', 'notes.txt']); + expect(await reader.list(new URL('nested/', folder))).toEqual(['inner.txt']); + }); + + it('refuse a file or a folder that is not shipped', async () => { + await expect(reader.text(new URL('missing.txt', folder))).rejects.toThrow('missing.txt'); + await expect(reader.bytes(new URL('missing.txt', folder))).rejects.toThrow('missing.txt'); + await expect(reader.list(new URL('missing/', folder))).rejects.toThrow('missing'); + }); +}); diff --git a/packages/operations/src/shipped-files/shipped-files.ts b/packages/operations/src/shipped-files/shipped-files.ts new file mode 100644 index 000000000..e28e8a539 --- /dev/null +++ b/packages/operations/src/shipped-files/shipped-files.ts @@ -0,0 +1,5 @@ +export interface ShippedFiles { + readonly text: (file: Readonly) => Promise; + readonly bytes: (file: Readonly) => Promise; + readonly list: (folder: Readonly) => Promise; +} diff --git a/packages/operations/src/testing/index.ts b/packages/operations/src/testing/index.ts index 4fdf3817c..cbd910cec 100644 --- a/packages/operations/src/testing/index.ts +++ b/packages/operations/src/testing/index.ts @@ -1,6 +1,7 @@ export { memoryPlaybookRegistry } from './memory-playbook-registry.ts'; export { memoryLedger, type MemoryLedger } from './memory-ledger.ts'; export { memoryRecordedContent } from '../content/memory-content.ts'; +export { shippedFilesInMemory } from '../shipped-files/shipped-files-in-memory.ts'; export { keptTextOf } from './kept-text.ts'; export { nothingKept } from '../reading/kept-content.ts'; export { recordingReporter, type RecordingReporter, type ReportedIncident } from './recording-reporter.ts'; @@ -35,3 +36,5 @@ export { export { internalTerms, internalTermsIn } from '../plain-language/internal-terms.ts'; export { endedWhileItRuns, stoppedWithinMs, type RunningProcess } from '../process-testing/process-endings.ts'; export { endWithTheTest, spawnForTheTest, type PidReader } from '../process-testing/test-processes.ts'; +export { measuredEnvironment } from '../process-testing/measured-environment.ts'; +export { modulesReachedFrom, type ModulesReached } from '../module-testing/modules-reached.ts'; diff --git a/packages/providers/README.md b/packages/providers/README.md index c403a9327..845a97cbe 100644 --- a/packages/providers/README.md +++ b/packages/providers/README.md @@ -38,7 +38,7 @@ The fields of a way, `wayFields`, are the one place of the placement, `in`, `nam ## The shipped entries -`readProviders()` (`src/reading/shipped-providers.ts`) reads the entries once, at start, from `new URL('../entries/', import.meta.url)` with a listing of the folder, through `readYaml` of `@on.auto/config` and the schema, and answers `ProviderList`: the providers sorted by their names, not by the names of their files, and the folder their documents are read from. A provider is its entry and its `address`, the address its calls go to as the entry writes it: `url`, `base_url`, the first server a document beside the entries names, read at start, or the URL of a document that is not. An entry the schema refuses, a file not named for its entry, a document beside the entries that is no JSON, and one whose first server, where the entry gives no `base_url`, is missing or is no https URL, or http URL on a loopback address, without a variable, are `ProvidersInvalid`, naming each problem at the file, its line and its key, as a settings file is named, and the server stops at start. The entries ship because they sit in `src`, in the image and in the npm package alike. +`readProviders(shippedFiles)` (`src/reading/shipped-providers.ts`) reads the entries once, at start, from `new URL('../entries/', import.meta.url)` with a listing of the folder, through the `ShippedFiles` of `@on.auto/operations` it is given, which the server composes, then `readYaml` of `@on.auto/config` and the schema, and answers `ProviderList`: the providers sorted by their names, not by the names of their files, and `documents`, each document beside the entries as its text and the bytes it takes, both of the bytes read through the same reader while the entry naming it is checked (`shippedDocumentOf` of `@on.auto/tool-servers`), keyed by the name the entry writes, which the tool servers describe the provider's API from, so no package reads a shipped document again. A provider is its entry and its `address`, the address its calls go to as the entry writes it: `url`, `base_url`, the first server a document beside the entries names, read at start, or the URL of a document that is not. An entry the schema refuses, a file not named for its entry, a document beside the entries that is no JSON, and one whose first server, where the entry gives no `base_url`, is missing or is no https URL, or http URL on a loopback address, without a variable, are `ProvidersInvalid`, naming each problem at the file, its line and its key, as a settings file is named, and the server stops at start. The entries ship because they sit in `src`, in the image and in the npm package alike. The entries this runtime ships, each verified against its vendor's documentation when it was written: @@ -79,7 +79,7 @@ A way of consent names as its issuer the authorization server its server's prote `AddedProviderEntrySchema` is the schema of a provider whoever runs the runtime adds beside those it ships: `ProviderEntrySchema`, but for a `document` beside the entries, refused at `/document`, since those documents are the runtime's own and an added entry names its document by its address. The fields of an entry carry descriptions, so a JSON Schema made from it describes each. `entryProse` names the keys of an entry that hold prose, `title` and `description`, a way's among them, for a check of credentials written out that leaves a sentence alone; the schema judges the server part and the placeholders of an added entry as it judges those of a shipped one. -`joinedProviders(list, added)` answers one list of the shipped providers and those added, each at its address as the shipped entries' reading takes it (`addressOf` of `src/reading/provider-list.ts`), sorted by name, the folder of the shipped documents kept, and an added entry winning the name of a shipped one, which is then not listed. +`joinedProviders(list, added)` answers one list of the shipped providers and those added, each at its address as the shipped entries' reading takes it (`addressOf` of `src/reading/provider-list.ts`), sorted by name, the texts of the shipped documents kept, and an added entry winning the name of a shipped one, which is then not listed. ## The resolution @@ -157,4 +157,4 @@ The conversion, `src/translating/swagger-conversion.ts`, writes an OpenAPI 3.1 d ## Source -`src/index.ts` is the entry point, and `translate-documents.ts` the command `documents:translate` runs, which writes what `src/translating/document-manifests.ts` translates. `src/entry` holds the schema of an entry, its ways, its placeholders and the reading of its templates; `src/reading` the reading of the shipped entries, the list and what a provider reaches; `src/resolution` the resolution and the checks of a placeholder's value; `src/translating` the translations of a Discovery document and of OpenAPI documents, the conversion of Swagger documents, the `oauth2` scheme they write, and the reading of the manifests of each kind; `src/entries` the shipped entries, their documents and the manifests of those translated; `src/testing` what the tests share, and `src/testing/index.ts`, the entry `@on.auto/providers/testing`, the fixture providers `tracker`, `ledgers`, `regions` and `metrics` and their list, `fixtureProviders`, `drive` and `adverts`, whose way is consent, the one discovering its server's metadata and the other giving its endpoints with a placeholder in them, and `audiences`, whose header holds `{client_id}` beside its way of consent, `providerOf(entry, documentServer?)`, `providerListOf(...providers)` and `providersOf(...entries)`, which the tests of the resolution and of the connections use in place of the shipped ones. +`src/index.ts` is the entry point, and `translate-documents.ts` the command `documents:translate` runs, which writes what `src/translating/document-manifests.ts` translates: the command reads the manifests through `shippedFilesOnDisk` of `@on.auto/operations/shipped-files-on-disk`, and the file a `=` names, which is no file the package ships, itself. `src/entry` holds the schema of an entry, its ways, its placeholders and the reading of its templates; `src/reading` the reading of the shipped entries, the list and what a provider reaches; `src/resolution` the resolution and the checks of a placeholder's value; `src/translating` the translations of a Discovery document and of OpenAPI documents, the conversion of Swagger documents, the `oauth2` scheme they write, and the reading of the manifests of each kind; `src/entries` the shipped entries, their documents and the manifests of those translated; `src/testing` what the tests share, and `src/testing/index.ts`, the entry `@on.auto/providers/testing`, the fixture providers `tracker`, `ledgers`, `regions` and `metrics` and their list, `fixtureProviders`, `drive` and `adverts`, whose way is consent, the one discovering its server's metadata and the other giving its endpoints with a placeholder in them, and `audiences`, whose header holds `{client_id}` beside its way of consent, `providerOf(entry, documentServer?)`, `providerListOf(...providers)` and `providersOf(...entries)`, which the tests of the resolution and of the connections use in place of the shipped ones. diff --git a/packages/providers/src/reading/entry-folders.test.ts b/packages/providers/src/reading/entry-folders.test.ts index 41b33fb1c..cc0b39c02 100644 --- a/packages/providers/src/reading/entry-folders.test.ts +++ b/packages/providers/src/reading/entry-folders.test.ts @@ -37,31 +37,31 @@ function servedAt(url: string): string { } describe('a folder of entries', () => { - it('is read with the server of a document beside it, or the address of a document an entry names by URL', () => { + it('is read with the server of a document beside it, or the address of a document an entry names by URL', async () => { const remote = notes.replace('documents/notes.openapi.json', 'https://notes.example.com/openapi.json'); const beside = { 'notes.yaml': notes, 'tracker.yaml': tracker, 'documents/notes.openapi.json': served }; - expect(entryFolderWith(beside).read()).toBe( + expect(await entryFolderWith(beside).read()).toBe( 'notes https://api.notes.example.com/v1; tracker https://tracker.example.com/mcp', ); - expect(entryFolderWith({ 'notes.yaml': remote, 'README.md': '# Entries' }).read()).toBe( + expect(await entryFolderWith({ 'notes.yaml': remote, 'README.md': '# Entries' }).read()).toBe( 'notes https://notes.example.com/openapi.json', ); }); - it('is sorted by the names of the providers, not of their files', () => { + it('is sorted by the names of the providers, not of their files', async () => { const slack = tracker.replace('name: tracker', 'name: slack'); const slackApi = tracker.replace('name: tracker', 'name: slack-api'); - expect(entryFolderWith({ 'slack.yaml': slack, 'slack-api.yaml': slackApi }).read()).toBe( + expect(await entryFolderWith({ 'slack.yaml': slack, 'slack-api.yaml': slackApi }).read()).toBe( 'slack https://tracker.example.com/mcp; slack-api https://tracker.example.com/mcp', ); }); }); describe('a folder of entries the runtime refuses', () => { - it('is refused with the file, the line and the key of each problem, as a settings file is', () => { - const refused = entryFolderWith({ + it('is refused with the file, the line and the key of each problem, as a settings file is', async () => { + const refused = await entryFolderWith({ 'tracker.yaml': tracker.replace(' - kind: none', ' - kind: reference'), 'notes.yaml': notes.replace('name: notes', 'name: notebook').replace('/notes.', '/notebook.'), 'broken.yaml': 'name: [broken\n', @@ -76,15 +76,17 @@ describe('a folder of entries the runtime refuses', () => { ]); }); - it('refuses a document that is not the one beside the entries named for the provider', () => { + it('refuses a document that is not the one beside the entries named for the provider', async () => { const elsewhere = notes.replace('documents/notes.openapi.json', 'documents/other.openapi.json'); const at = `${invalid}/notes.yaml:6:11 document`; - expect([ - entryFolderWith({ 'notes.yaml': elsewhere }).read(), - entryFolderWith({ 'notes.yaml': notes }).read(), - entryFolderWith({ 'notes.yaml': notes, 'documents/notes.openapi.json': '{ not json' }).read(), - ]).toEqual([ + expect( + await Promise.all([ + entryFolderWith({ 'notes.yaml': elsewhere }).read(), + entryFolderWith({ 'notes.yaml': notes }).read(), + entryFolderWith({ 'notes.yaml': notes, 'documents/notes.openapi.json': '{ not json' }).read(), + ]), + ).toEqual([ `${at}: Expected an https URL, or an http URL on a loopback address, written out up to its host, with each placeholder in its place, or documents/notes.openapi.json, the document beside the entries named for the provider`, `${at}: Expected documents/notes.openapi.json beside the entries, an OpenAPI document in JSON`, `${at}: Expected documents/notes.openapi.json beside the entries, an OpenAPI document in JSON`, @@ -93,31 +95,35 @@ describe('a folder of entries the runtime refuses', () => { }); describe('the server of a document beside the entries', () => { - it('is refused where the document names none, unless the entry gives base_url', () => { + it('is refused where the document names none, unless the entry gives base_url', async () => { const based = `${notes}base_url: https://api.notes.example.com\n`; - expect([ - entryFolderWith({ 'notes.yaml': notes, 'documents/notes.openapi.json': unserved }).read(), - entryFolderWith({ 'notes.yaml': based, 'documents/notes.openapi.json': unserved }).read(), - ]).toEqual([ + expect( + await Promise.all([ + entryFolderWith({ 'notes.yaml': notes, 'documents/notes.openapi.json': unserved }).read(), + entryFolderWith({ 'notes.yaml': based, 'documents/notes.openapi.json': unserved }).read(), + ]), + ).toEqual([ `${invalid}/notes.yaml:6:11 document: The document names no server, so the entry gives base_url`, 'notes https://api.notes.example.com', ]); }); - it('is refused where the first is no pinned URL written out, unless the entry gives base_url', () => { + it('is refused where the first is no pinned URL written out, unless the entry gives base_url', async () => { const refusal = `${invalid}/notes.yaml:6:11 document: The first server the document names is no https URL, or http URL on a loopback address, without a variable, so the entry gives base_url`; - expect([ - entryFolderWith({ 'notes.yaml': notes, 'documents/notes.openapi.json': servedAt('/v1') }).read(), - entryFolderWith({ - 'notes.yaml': notes, - 'documents/notes.openapi.json': servedAt('http://notes.example.com'), - }).read(), - entryFolderWith({ - 'notes.yaml': notes, - 'documents/notes.openapi.json': servedAt('https://{region}.notes.example.com'), - }).read(), - ]).toEqual([refusal, refusal, refusal]); + expect( + await Promise.all([ + entryFolderWith({ 'notes.yaml': notes, 'documents/notes.openapi.json': servedAt('/v1') }).read(), + entryFolderWith({ + 'notes.yaml': notes, + 'documents/notes.openapi.json': servedAt('http://notes.example.com'), + }).read(), + entryFolderWith({ + 'notes.yaml': notes, + 'documents/notes.openapi.json': servedAt('https://{region}.notes.example.com'), + }).read(), + ]), + ).toEqual([refusal, refusal, refusal]); }); }); diff --git a/packages/providers/src/reading/provider-list.ts b/packages/providers/src/reading/provider-list.ts index 6f744074b..444d4a875 100644 --- a/packages/providers/src/reading/provider-list.ts +++ b/packages/providers/src/reading/provider-list.ts @@ -1,4 +1,5 @@ import type { Issue } from '@on.auto/operations'; +import type { ShippedDocument } from '@on.auto/tool-servers'; import { Option, Order, Result } from 'effect'; import type { ProviderEntry } from '../entry/provider-entry.ts'; @@ -11,7 +12,7 @@ export interface Provider { export interface ProviderList { readonly providers: readonly Provider[]; - readonly documents: string; + readonly documents: ReadonlyMap; } function required(value: A | undefined): A { diff --git a/packages/providers/src/reading/search-console-properties.test.ts b/packages/providers/src/reading/search-console-properties.test.ts index 4d1f2be54..1a50d3a6f 100644 --- a/packages/providers/src/reading/search-console-properties.test.ts +++ b/packages/providers/src/reading/search-console-properties.test.ts @@ -1,6 +1,8 @@ import { readFileSync } from 'node:fs'; -import { join, relative } from 'node:path'; +import { relative } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { describedAccess, patientTiming, serveFakeApi } from '@on.auto/tool-servers/testing'; import { Effect, Result } from 'effect'; import { afterEach, describe, expect, it } from 'vitest'; @@ -17,7 +19,9 @@ const [, domainProperty = '', urlPrefixProperty = '', sentPath = ''] = [...sente ([, quoted = '']: readonly string[]) => quoted, ); -const shipped = Result.getOrThrow(readProviders()); +const shipped = Result.getOrThrow(await readProviders(shippedFilesOnDisk)); + +const entriesFolder = new URL('../entries/', import.meta.url); const { entry } = Result.getOrThrow(providerNamed(shipped, 'google-search-console')); @@ -36,7 +40,7 @@ async function pathSentFor(siteUrl: string): Promise { { 'search-console': { org: 'acme', - document: relative(process.cwd(), join(shipped.documents, entry.document ?? '')), + document: relative(process.cwd(), fileURLToPath(new URL(entry.document ?? '', entriesFolder))), base_url: fake.baseUrl, auth: { issuer: String(fake.issuing?.issuer), client_id: 'playbook', client_secret: '${CLIENT_SECRET}' }, }, diff --git a/packages/providers/src/reading/shipped-documents.test.ts b/packages/providers/src/reading/shipped-documents.test.ts index 7c0e76f75..e95fa7af5 100644 --- a/packages/providers/src/reading/shipped-documents.test.ts +++ b/packages/providers/src/reading/shipped-documents.test.ts @@ -1,6 +1,6 @@ import { readFileSync } from 'node:fs'; -import { join } from 'node:path'; +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { Schema } from 'effect'; import { describe, expect, it } from 'vitest'; @@ -32,7 +32,7 @@ type Held = ReturnType; function heldBy(name: string) { const { security, paths, components } = decodeOperations( - readFileSync(join(documentsBesideTheEntries, `${name}.openapi.json`), 'utf8'), + readFileSync(new URL(`${name}.openapi.json`, documentsBesideTheEntries), 'utf8'), ); const operations = Object.values(paths).flatMap((methods) => Object.values(methods)); const requirements = [...security, ...operations.flatMap((operation) => operation.security ?? [])]; @@ -63,8 +63,20 @@ function namesOf(manifest: DocumentManifest): Named { }; } -function namedBy(name: string): Held { - const { kept, scheme } = namesOf(manifestIn(documentsBesideTheEntries, name)); +interface Kept { + readonly name: string; + readonly manifest: DocumentManifest; +} + +const manifests: readonly Kept[] = await Promise.all( + (await manifestNamesIn(shippedFilesOnDisk, documentsBesideTheEntries)).map(async (name) => ({ + name, + manifest: await manifestIn(shippedFilesOnDisk, documentsBesideTheEntries, name), + })), +); + +function namedBy(manifest: DocumentManifest): Held { + const { kept, scheme } = namesOf(manifest); return { operations: kept.toSorted(), schemes: [scheme], required: [scheme] }; } @@ -72,20 +84,16 @@ function addressesOf(manifest: DocumentManifest): readonly string[] { return [...namesOf(manifest).sources, manifest.authorization_server]; } -const names = manifestNamesIn(documentsBesideTheEntries); - describe('the manifests beside the entries', () => { it('are each one the command reads, of one of its kinds, keeping each operation once, at https addresses', () => { - const kept: readonly (readonly string[])[] = names.map((name) => namedBy(name).operations); + const kept: readonly (readonly string[])[] = manifests.map(({ manifest }) => namedBy(manifest).operations); expect(kept.map(({ length }) => length)).toEqual(kept.map((operations) => new Set(operations).size)); expect( - names - .flatMap((name) => addressesOf(manifestIn(documentsBesideTheEntries, name))) - .filter((address) => !address.startsWith('https://')), + manifests.flatMap(({ manifest }) => addressesOf(manifest)).filter((address) => !address.startsWith('https://')), ).toEqual([]); - expect(names.map((name) => manifestIn(documentsBesideTheEntries, name).type).toSorted()).toEqual([ - ...Array.from({ length: names.length - 3 }, () => 'discovery'), + expect(manifests.map(({ manifest }) => manifest.type).toSorted()).toEqual([ + ...Array.from({ length: manifests.length - 3 }, () => 'discovery'), 'openapi', 'openapi', 'swagger', @@ -93,6 +101,6 @@ describe('the manifests beside the entries', () => { }); it('each have their document beside them, whose operations are those they keep, all requiring the one scheme they name', () => { - expect(names.map((name) => heldBy(name))).toEqual(names.map((name) => namedBy(name))); + expect(manifests.map(({ name }) => heldBy(name))).toEqual(manifests.map(({ manifest }) => namedBy(manifest))); }); }); diff --git a/packages/providers/src/reading/shipped-providers.test.ts b/packages/providers/src/reading/shipped-providers.test.ts index dde37d1cf..10b83b880 100644 --- a/packages/providers/src/reading/shipped-providers.test.ts +++ b/packages/providers/src/reading/shipped-providers.test.ts @@ -1,14 +1,21 @@ import { readdirSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { Result } from 'effect'; import { describe, expect, it } from 'vitest'; import { documentServerOf, operatorVariablesOf, readProviders, type Provider } from '../index.ts'; -const shipped = Result.getOrThrow(readProviders()); +const shipped = Result.getOrThrow(await readProviders(shippedFilesOnDisk)); -const entriesFolder = fileURLToPath(new URL('../entries/', import.meta.url)); +const entriesUrl = new URL('../entries/', import.meta.url); + +const entriesFolder = fileURLToPath(entriesUrl); + +const documentsBesideTheEntries = shipped.providers + .flatMap(({ entry }) => (entry.document === undefined || URL.canParse(entry.document) ? [] : [entry.document])) + .toSorted(); function addressesOf({ entry, address }: Provider): readonly string[] { return [entry.website, address, ...entry.credentials.flatMap(({ issuer }) => (issuer === undefined ? [] : [issuer]))]; @@ -163,7 +170,19 @@ describe('the servers of the shipped providers', () => { expect(shipped.providers.map((provider) => documentServerOf(provider))).toEqual( shipped.providers.map((provider) => documentServerExpectedOf(provider)), ); - expect(shipped.documents).toBe(entriesFolder); + }); + + it('hand on the documents beside the entries as the texts they read with the bytes each takes, by the name each entry writes', async () => { + const onDisk = await Promise.all( + documentsBesideTheEntries.map(async (written) => { + const file = new URL(written, entriesUrl); + return { text: await shippedFilesOnDisk.text(file), bytes: (await shippedFilesOnDisk.bytes(file)).byteLength }; + }), + ); + + expect([...shipped.documents.keys()].toSorted()).toEqual(documentsBesideTheEntries); + expect(documentsBesideTheEntries.map((written) => shipped.documents.get(written))).toEqual(onDisk); + expect(documentsBesideTheEntries.length).toBeGreaterThan(0); }); }); diff --git a/packages/providers/src/reading/shipped-providers.ts b/packages/providers/src/reading/shipped-providers.ts index ec1aee2d0..6c3a1b1ec 100644 --- a/packages/providers/src/reading/shipped-providers.ts +++ b/packages/providers/src/reading/shipped-providers.ts @@ -1,5 +1,3 @@ -import { readdirSync, readFileSync } from 'node:fs'; -import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { @@ -11,7 +9,8 @@ import { type Position, type YamlKind, } from '@on.auto/config'; -import { isPinnableUrl } from '@on.auto/tool-servers'; +import type { ShippedFiles } from '@on.auto/operations'; +import { isPinnableUrl, shippedDocumentOf, type ShippedDocument } from '@on.auto/tool-servers'; import { Data, Option, Result, Schema, SchemaIssue, type StandardSchema } from 'effect'; import { ProviderEntrySchema, type ProviderEntry } from '../entry/provider-entry.ts'; @@ -32,6 +31,11 @@ interface Located { type Read = Result.Result; +interface Entries { + readonly shippedFiles: ShippedFiles; + readonly folder: Readonly; +} + type PathSegment = PropertyKey | StandardSchema.StandardSchemaV1.PathSegment; const shippedEntries = new URL('../entries/', import.meta.url); @@ -51,6 +55,8 @@ const DocumentServersSchema = Schema.Struct({ servers: Schema.optionalKey(Schema.Array(Schema.Struct({ url: Schema.String }))), }); +type DocumentServers = typeof DocumentServersSchema.Type; + const decodeServers = Schema.decodeUnknownResult(DocumentServersSchema); function isPropertyKey(segment: PathSegment): segment is PropertyKey { @@ -68,66 +74,89 @@ function documentProblem(detail: string): Read { return Result.fail([{ pointer: '/document', detail }]); } -function parsedDocument(path: string): unknown { +interface DocumentBesideTheEntries { + readonly shipped: ShippedDocument; + readonly servers: DocumentServers; +} + +async function documentBesideTheEntries( + { shippedFiles, folder }: Entries, + written: string, +): Promise { try { - const parsed: unknown = JSON.parse(readFileSync(path, 'utf8')); - return parsed; + const shipped = shippedDocumentOf(await shippedFiles.bytes(new URL(written, folder))); + const parsed: unknown = JSON.parse(shipped.text); + return Result.getOrUndefined(Result.map(decodeServers(parsed), (servers) => ({ shipped, servers }))); } catch { return undefined; } } -interface DocumentServer { +type Shipped = readonly [written: string, document: ShippedDocument]; + +interface DocumentRead { readonly server?: string; + readonly shipped?: Shipped; +} + +interface ProviderRead { + readonly provider: Provider; + readonly shipped?: Shipped; } function isServerWrittenOut(url: string): boolean { return isPinnableUrl(url) && placeholdersIn(url).length === 0; } -function serverOfTheDocument(entry: ProviderEntry, folder: string): Read { +async function documentOfTheEntry(entry: ProviderEntry, entries: Entries): Promise> { const written = entry.document; if (written === undefined || isAddress(written)) { return Result.succeed({}); } - const servers = Result.getOrUndefined(decodeServers(parsedDocument(join(folder, written)))); - if (servers === undefined) { + const document = await documentBesideTheEntries(entries, written); + if (document === undefined) { return documentProblem(`Expected ${written} beside the entries, an OpenAPI document in JSON`); } + const shipped: Shipped = [written, document.shipped]; if (entry.base_url !== undefined) { - return Result.succeed({}); + return Result.succeed({ shipped }); } - const [first] = servers.servers ?? []; + const [first] = document.servers.servers ?? []; if (first === undefined) { return documentProblem('The document names no server, so the entry gives base_url'); } return isServerWrittenOut(first.url) - ? Result.succeed({ server: withoutTrailingSlashes(first.url) }) + ? Result.succeed({ server: withoutTrailingSlashes(first.url), shipped }) : documentProblem( 'The first server the document names is no https URL, or http URL on a loopback address, without a variable, so the entry gives base_url', ); } -function providerOf(entry: ProviderEntry, file: string, folder: string): Read { +async function providerOf(entry: ProviderEntry, file: string, entries: Entries): Promise> { const named = `${entry.name}.yaml`; if (file !== named) { return Result.fail([{ pointer: '/name', detail: `Expected the name the file is named for, as in ${named}` }]); } - return Result.map(serverOfTheDocument(entry, folder), ({ server }) => ({ entry, address: addressOf(entry, server) })); + return Result.map(await documentOfTheEntry(entry, entries), ({ server, shipped }): ProviderRead => ({ + provider: { entry, address: addressOf(entry, server) }, + ...(shipped === undefined ? {} : { shipped }), + })); } -function readProvider(folder: string, file: string): Result.Result { - const path = join(folder, file); - const reading = readYaml(readFileSync(path, 'utf8'), entryYaml); +async function readProvider(entries: Entries, file: string): Promise> { + const at = new URL(file, entries.folder); + const path = fileURLToPath(at); + const reading = readYaml(await entries.shippedFiles.text(at), entryYaml); if ('problems' in reading) { return Result.fail( reading.problems.map(({ position, detail }) => ({ place: placeIn(path, position, ''), detail })), ); } const { document } = reading; - const read = Result.flatMap(Result.mapError(decodeEntry(document.value), problemsOf), (entry) => - providerOf(entry, file, folder), - ); + const read = await Result.match(Result.mapError(decodeEntry(document.value), problemsOf), { + onFailure: (problems): Promise> => Promise.resolve(Result.fail(problems)), + onSuccess: (entry) => providerOf(entry, file, entries), + }); const located = ({ pointer, detail }: Problem): Located => { const position: Position = document.locate(pointer); return { place: placeIn(path, position, pointer), detail }; @@ -140,19 +169,23 @@ function invalid(located: readonly Located[]): ProvidersInvalid { return new ProvidersInvalid({ message: `The entries of the providers are invalid: ${listed}` }); } -export function readProviders(entries: Readonly = shippedEntries): Result.Result { - const folder = fileURLToPath(entries); - const files = readdirSync(folder) - .filter((file) => file.endsWith('.yaml')) - .toSorted(); - const read = files.map((file) => readProvider(folder, file)); +export async function readProviders( + shippedFiles: ShippedFiles, + folder: Readonly = shippedEntries, +): Promise> { + const entries: Entries = { shippedFiles, folder }; + const files = (await shippedFiles.list(folder)).filter((file) => file.endsWith('.yaml')); + const read = await Promise.all(files.map((file) => readProvider(entries, file))); const located: Located[] = []; for (const each of read) { located.push(...Result.getFailure(each).pipe(Option.getOrElse((): readonly Located[] => []))); } return Result.match(Result.all(read), { onFailure: () => Result.fail(invalid(located)), - onSuccess: (providers: readonly Provider[]) => - Result.succeed({ providers: providers.toSorted(byName), documents: folder }), + onSuccess: (providers: readonly ProviderRead[]) => + Result.succeed({ + providers: providers.map(({ provider }) => provider).toSorted(byName), + documents: new Map(providers.flatMap(({ shipped }) => (shipped === undefined ? [] : [shipped]))), + }), }); } diff --git a/packages/providers/src/reading/shipped-ways.test.ts b/packages/providers/src/reading/shipped-ways.test.ts index aab28290d..d6d948050 100644 --- a/packages/providers/src/reading/shipped-ways.test.ts +++ b/packages/providers/src/reading/shipped-ways.test.ts @@ -1,9 +1,10 @@ +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { Result } from 'effect'; import { describe, expect, it } from 'vitest'; import { hostAlternativesOf, readProviders, type CredentialWay, type ProviderEntry } from '../index.ts'; -const shipped = Result.getOrThrow(readProviders()); +const shipped = Result.getOrThrow(await readProviders(shippedFilesOnDisk)); const google = 'accounts.google.com'; diff --git a/packages/providers/src/testing/entry-problems.ts b/packages/providers/src/testing/entry-problems.ts index 59594b114..84d08efd0 100644 --- a/packages/providers/src/testing/entry-problems.ts +++ b/packages/providers/src/testing/entry-problems.ts @@ -1,10 +1,7 @@ -import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; import { pathToFileURL } from 'node:url'; +import { shippedFilesInMemory } from '@on.auto/operations/testing'; import { Result, Schema, SchemaIssue } from 'effect'; -import { onTestFinished } from 'vitest'; import { ProviderEntrySchema } from '../entry/provider-entry.ts'; import { readProviders } from '../reading/shipped-providers.ts'; @@ -26,23 +23,22 @@ export function decodedEntry(entry: unknown) { } export interface EntryFolder { - readonly folder: string; - readonly read: () => string; + readonly read: () => Promise; } +const folder = '/shipped/entries'; + +const entries = pathToFileURL(`${folder}/`); + export function entryFolderWith(files: Readonly>): EntryFolder { - const folder = mkdtempSync(join(tmpdir(), 'auto-runtime-providers-')); - onTestFinished(() => { - rmSync(folder, { recursive: true, force: true }); - }); - mkdirSync(join(folder, 'documents')); - for (const [name, text] of Object.entries(files)) { - writeFileSync(join(folder, name), text); - } + const shipped = shippedFilesInMemory( + Object.fromEntries( + Object.entries(files).map(([name, text]: readonly [string, string]) => [new URL(name, entries).href, text]), + ), + ); return { - folder, - read: () => - Result.match(readProviders(pathToFileURL(`${folder}/`)), { + read: async () => + Result.match(await readProviders(shipped, entries), { onSuccess: ({ providers }) => providers.map(({ entry, address }) => `${entry.name} ${address}`).join('; '), onFailure: ({ message }: { readonly message: string }) => message.replaceAll(folder, ''), }), diff --git a/packages/providers/src/testing/fixture-providers.ts b/packages/providers/src/testing/fixture-providers.ts index 0faca85dd..cb1552d75 100644 --- a/packages/providers/src/testing/fixture-providers.ts +++ b/packages/providers/src/testing/fixture-providers.ts @@ -204,7 +204,7 @@ export function providerOf(entry: unknown, documentServer?: string): Provider { } export function providerListOf(...providers: readonly Provider[]): ProviderList { - return { providers, documents: import.meta.dirname }; + return { providers, documents: new Map() }; } export function providersOf(...entries: readonly unknown[]): ProviderList { diff --git a/packages/providers/src/translating/document-manifests.test.ts b/packages/providers/src/translating/document-manifests.test.ts index d013adeda..582f4eec0 100644 --- a/packages/providers/src/translating/document-manifests.test.ts +++ b/packages/providers/src/translating/document-manifests.test.ts @@ -1,7 +1,7 @@ -import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import type { ShippedFiles } from '@on.auto/operations'; +import { shippedFilesInMemory } from '@on.auto/operations/testing'; import { describe, expect, it, onTestFinished } from 'vitest'; import { trackerTranslated } from '../testing/curated-documents.ts'; @@ -47,16 +47,24 @@ function fetchingFrom(answered: ReadonlyMap): Fetching { }; } -function folderWith(files: Readonly>): string { - const folder = mkdtempSync(join(tmpdir(), 'translated-documents-')); +const folder = pathToFileURL('/shipped/documents/'); + +function pathOf(file: string): string { + return fileURLToPath(new URL(file, folder)); +} + +function shippedWith(files: Readonly>): ShippedFiles { onTestFinished(() => { - rmSync(folder, { recursive: true, force: true }); fetched.splice(0); }); - for (const [name, content] of Object.entries(files)) { - writeFileSync(join(folder, name), JSON.stringify(content)); - } - return folder; + return shippedFilesInMemory( + Object.fromEntries( + Object.entries(files).map(([name, content]: readonly [string, unknown]) => [ + new URL(name, folder).href, + JSON.stringify(content), + ]), + ), + ); } function parsed(text: string): unknown { @@ -68,8 +76,25 @@ function textOf(value: unknown): string { return `${JSON.stringify(value, null, 2)}\n`; } -function translatedIn(folder: string, commandArguments: readonly string[] = [], answered = answers) { - return translatedDocuments(commandArguments, { folder, fetching: fetchingFrom(answered) }); +function readingFrom(given: ReadonlyMap): (path: string) => Promise { + return (path) => { + const text = given.get(path); + return text === undefined ? Promise.reject(new Error(`No file is given at ${path}`)) : Promise.resolve(text); + }; +} + +function translatedIn( + shipped: ShippedFiles, + commandArguments: readonly string[] = [], + answered = answers, + given: ReadonlyMap = new Map(), +) { + return translatedDocuments(commandArguments, { + folder, + shippedFiles: shipped, + fetching: fetchingFrom(answered), + readGiven: readingFrom(given), + }); } const { cuts, ...uncut } = trackerManifest; @@ -78,38 +103,32 @@ const remade = { ...trackerManifest, revision: trackerDiscovery.revision }; describe('the documents translated from their manifests', () => { it('are each the document of the methods its manifest keeps, cut as it says, with the manifest at the revision read', async () => { - const folder = folderWith({ 'tracker.manifest.json': trackerManifest }); + const shipped = shippedWith({ 'tracker.manifest.json': trackerManifest }); - const written = await translatedIn(folder); + const written = await translatedIn(shipped); const read = written.map(({ text }) => parsed(text)); - expect(written.map(({ path }) => path)).toEqual([ - join(folder, 'tracker.openapi.json'), - join(folder, 'tracker.manifest.json'), - ]); + expect(written.map(({ path }) => path)).toEqual([pathOf('tracker.openapi.json'), pathOf('tracker.manifest.json')]); expect(read).toEqual([trackerTranslated, remade]); expect(written.map(({ text }) => text)).toEqual(read.map((value) => textOf(value))); expect(fetched).toEqual([trackerManifest.discovery, trackerManifest.authorization_server]); }); it('are made for the manifests named alone, from the Discovery document given beside a name instead of fetching it', async () => { - const given = folderWith({ 'tracker-v1.json': trackerDiscovery }); - const folder = folderWith({ 'tracker.manifest.json': trackerManifest, 'bare.manifest.json': uncut }); + const shipped = shippedWith({ 'tracker.manifest.json': trackerManifest, 'bare.manifest.json': uncut }); + const given = new Map([['/given/tracker-v1.json', JSON.stringify(trackerDiscovery)]]); - const written = await translatedIn(folder, [`tracker=${join(given, 'tracker-v1.json')}`]); + const written = await translatedIn(shipped, ['tracker=/given/tracker-v1.json'], answers, given); - expect(written.map(({ path }) => path)).toEqual([ - join(folder, 'tracker.openapi.json'), - join(folder, 'tracker.manifest.json'), - ]); + expect(written.map(({ path }) => path)).toEqual([pathOf('tracker.openapi.json'), pathOf('tracker.manifest.json')]); expect(fetched).toEqual([trackerManifest.authorization_server]); - expect(manifestNamesIn(folder)).toEqual(['bare', 'tracker']); + expect(await manifestNamesIn(shipped, folder)).toEqual(['bare', 'tracker']); }); it('write a manifest without cuts as it is, and leave the schemas whole', async () => { - const folder = folderWith({ 'bare.manifest.json': { ...uncut, methods: ['tracker.projects.issues.mutate'] } }); + const shipped = shippedWith({ 'bare.manifest.json': { ...uncut, methods: ['tracker.projects.issues.mutate'] } }); - const written = await translatedIn(folder, ['bare']); + const written = await translatedIn(shipped, ['bare']); expect(written.map(({ text }) => parsed(text))).toMatchObject([ { @@ -127,31 +146,31 @@ describe('the documents translated from their manifests', () => { describe('the documents not translated', () => { it('are refused for a name no manifest has, saying how the command is used', async () => { - const folder = folderWith({ 'tracker.manifest.json': trackerManifest }); + const shipped = shippedWith({ 'tracker.manifest.json': trackerManifest }); - await expect(translatedIn(folder, ['crm'])).rejects.toThrow( + await expect(translatedIn(shipped, ['crm'])).rejects.toThrow( 'There is no manifest named crm beside the documents, which hold tracker. Usage: pnpm --filter @on.auto/providers documents:translate [[=]]..., each name a manifest beside the documents, every manifest when none is named', ); }); it('are refused where an address answers with a failure', async () => { - const folder = folderWith({ 'tracker.manifest.json': trackerManifest }); + const shipped = shippedWith({ 'tracker.manifest.json': trackerManifest }); - await expect(translatedIn(folder, [], new Map())).rejects.toThrow( + await expect(translatedIn(shipped, [], new Map())).rejects.toThrow( `${trackerManifest.discovery} answered 404, so nothing is translated from it`, ); }); it('are refused for a manifest that is not one', async () => { - const folder = folderWith({ 'tracker.manifest.json': { ...trackerManifest, methods: [] } }); + const shipped = shippedWith({ 'tracker.manifest.json': { ...trackerManifest, methods: [] } }); - await expect(translatedIn(folder)).rejects.toThrow('Expected a value with a length of at least 1'); + await expect(translatedIn(shipped)).rejects.toThrow('Expected a value with a length of at least 1'); }); }); function cutWith(cut: unknown, discovery: unknown = trackerDiscovery) { - const folder = folderWith({ 'tracker.manifest.json': { ...trackerManifest, cuts: [cut] } }); - return translatedIn(folder, [], new Map([...answers, [trackerManifest.discovery, discovery]])); + const shipped = shippedWith({ 'tracker.manifest.json': { ...trackerManifest, cuts: [cut] } }); + return translatedIn(shipped, [], new Map([...answers, [trackerManifest.discovery, discovery]])); } describe('a cut not made', () => { @@ -191,18 +210,18 @@ describe('a cut a manifest does not take', () => { describe('the documents translated from the manifests of OpenAPI and Swagger documents', () => { it('are each the document of the operations its manifest keeps, from the documents it names, fetched, with no manifest written', async () => { - const folder = folderWith({ + const shipped = shippedWith({ 'tracker.manifest.json': trackerOpenApiManifest, 'about.manifest.json': aboutManifest, 'notes.manifest.json': notesManifest, }); - const written = await translatedIn(folder); + const written = await translatedIn(shipped); const read = written.map(({ text }) => parsed(text)); const converted = { info: notesManifest.info, security: [{ oauth2: [] }] }; expect(written.map(({ path }) => path)).toEqual( - ['about', 'notes', 'tracker'].map((name) => join(folder, `${name}.openapi.json`)), + ['about', 'notes', 'tracker'].map((name) => pathOf(`${name}.openapi.json`)), ); expect(read).toEqual([aboutTranslated, expect.objectContaining(converted), trackerOpenApiTranslated]); expect(written.map(({ text }) => text)).toEqual(read.map((value) => textOf(value))); @@ -220,12 +239,12 @@ describe('the documents translated from the manifests of OpenAPI and Swagger doc it('are refused a file given in place of their documents, and a document that does not answer, in the words of the command', async () => { const gone = 'https://developers.tracker.example.com/api/gone/'; - const folder = folderWith({ 'tracker.manifest.json': trackerOpenApiManifest }); - const unanswered = folderWith({ + const shipped = shippedWith({ 'tracker.manifest.json': trackerOpenApiManifest }); + const unanswered = shippedWith({ 'tracker.manifest.json': { ...trackerOpenApiManifest, sources: [peopleAddress, gone] }, }); - await expect(translatedIn(folder, ['tracker=tracker-people.json'])).rejects.toThrow( + await expect(translatedIn(shipped, ['tracker=tracker-people.json'])).rejects.toThrow( 'The manifest tracker is of OpenAPI documents, which the command fetches at the addresses it names, so no file is given in place of one. Usage: pnpm --filter @on.auto/providers documents:translate', ); await expect(translatedIn(unanswered)).rejects.toThrow(`${gone} answered 404, so nothing is translated from it`); diff --git a/packages/providers/src/translating/document-manifests.ts b/packages/providers/src/translating/document-manifests.ts index 9364012a5..970791ac1 100644 --- a/packages/providers/src/translating/document-manifests.ts +++ b/packages/providers/src/translating/document-manifests.ts @@ -1,8 +1,6 @@ -import { readdirSync, readFileSync } from 'node:fs'; -import { readFile } from 'node:fs/promises'; -import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; -import { listed } from '@on.auto/operations'; +import { listed, type ShippedFiles } from '@on.auto/operations'; import { Schema } from 'effect'; import { CutSchema } from './discovery-cuts.ts'; @@ -11,7 +9,7 @@ import { convertedDocumentOf, SwaggerConversionSchema } from './swagger-conversi import { translatedDiscoveryOf } from './translate-discovery.ts'; import { curatedOpenApiOf } from './translate-openapi.ts'; -export const documentsBesideTheEntries = join(import.meta.dirname, '../entries/documents/'); +export const documentsBesideTheEntries = new URL('../entries/documents/', import.meta.url); const AuthorizationServerAddressSchema = Schema.String.annotate({ description: 'The address of the metadata of the authorization server whose endpoints the oauth2 scheme names', @@ -68,8 +66,10 @@ const documentSuffix = '.openapi.json'; export type Fetching = (address: string) => Promise; export interface Translating { - readonly folder: string; + readonly folder: Readonly; + readonly shippedFiles: ShippedFiles; readonly fetching: Fetching; + readonly readGiven: (path: string) => Promise; } export interface Written { @@ -85,15 +85,19 @@ interface Asked { const usage = 'Usage: pnpm --filter @on.auto/providers documents:translate [[=]]..., each name a manifest beside the documents, every manifest when none is named'; -export function manifestNamesIn(folder: string): readonly string[] { - return readdirSync(folder) +export async function manifestNamesIn(shippedFiles: ShippedFiles, folder: Readonly): Promise { + return (await shippedFiles.list(folder)) .filter((file) => file.endsWith(manifestSuffix)) .map((file) => file.slice(0, -manifestSuffix.length)) .toSorted(); } -export function manifestIn(folder: string, name: string): DocumentManifest { - return decodeManifest(readFileSync(join(folder, `${name}${manifestSuffix}`), 'utf8')); +export async function manifestIn( + shippedFiles: ShippedFiles, + folder: Readonly, + name: string, +): Promise { + return decodeManifest(await shippedFiles.text(new URL(`${name}${manifestSuffix}`, folder))); } function askedOf(commandArgument: string, names: readonly string[]): Asked { @@ -115,8 +119,8 @@ async function fetchedJson(address: string, fetching: Fetching): Promise { - const read: unknown = JSON.parse(await readFile(path, 'utf8')); +async function givenJson(path: string, { readGiven }: Translating): Promise { + const read: unknown = JSON.parse(await readGiven(path)); return read; } @@ -124,24 +128,25 @@ function jsonText(value: unknown): string { return `${JSON.stringify(value, null, 2)}\n`; } -function documentPathOf(folder: string, name: string): string { - return join(folder, `${name}${documentSuffix}`); +function pathIn(folder: Readonly, file: string): string { + return fileURLToPath(new URL(file, folder)); } async function discoveryTranslated( { name, given }: Asked, manifest: DiscoveryManifest, - { folder, fetching }: Translating, + translating: Translating, ): Promise { - const read = await (given === undefined ? fetchedJson(manifest.discovery, fetching) : givenJson(given)); + const { folder, fetching } = translating; + const read = await (given === undefined ? fetchedJson(manifest.discovery, fetching) : givenJson(given, translating)); const metadata = await fetchedJson(manifest.authorization_server, fetching); const { document, revision } = translatedDiscoveryOf(read, metadata, manifest); const { type, discovery: address, authorization_server: authorizationServer, methods, cuts } = manifest; const remade = { type, discovery: address, revision, authorization_server: authorizationServer, methods }; return [ - { path: documentPathOf(folder, name), text: jsonText(document) }, + { path: pathIn(folder, `${name}${documentSuffix}`), text: jsonText(document) }, { - path: join(folder, `${name}${manifestSuffix}`), + path: pathIn(folder, `${name}${manifestSuffix}`), text: jsonText(cuts === undefined ? remade : { ...remade, cuts }), }, ]; @@ -168,7 +173,7 @@ async function openApiTranslated( const documents = await fetchedAt(asked, manifest.sources, translating); const metadata = await fetchedJson(manifest.authorization_server, translating.fetching); const document = curatedOpenApiOf(documents, manifest, metadata); - return [{ path: documentPathOf(translating.folder, asked.name), text: jsonText(document) }]; + return [{ path: pathIn(translating.folder, `${asked.name}${documentSuffix}`), text: jsonText(document) }]; } async function swaggerConverted( @@ -183,11 +188,11 @@ async function swaggerConverted( ); const metadata = await fetchedJson(manifest.authorization_server, translating.fetching); const document = convertedDocumentOf(documents, manifest, metadata); - return [{ path: documentPathOf(translating.folder, asked.name), text: jsonText(document) }]; + return [{ path: pathIn(translating.folder, `${asked.name}${documentSuffix}`), text: jsonText(document) }]; } -function translatedOne(asked: Asked, translating: Translating): Promise { - const manifest = manifestIn(translating.folder, asked.name); +async function translatedOne(asked: Asked, translating: Translating): Promise { + const manifest = await manifestIn(translating.shippedFiles, translating.folder, asked.name); if (manifest.type === 'discovery') { return discoveryTranslated(asked, manifest, translating); } @@ -200,7 +205,7 @@ export async function translatedDocuments( commandArguments: readonly string[], translating: Translating, ): Promise { - const names = manifestNamesIn(translating.folder); + const names = await manifestNamesIn(translating.shippedFiles, translating.folder); const asked = commandArguments.length === 0 ? names.map((name): Asked => ({ name, given: undefined })) diff --git a/packages/providers/translate-documents.ts b/packages/providers/translate-documents.ts index 6566eda8c..d7c22f320 100644 --- a/packages/providers/translate-documents.ts +++ b/packages/providers/translate-documents.ts @@ -1,7 +1,10 @@ import { execFileSync } from 'node:child_process'; import { writeFileSync } from 'node:fs'; +import { readFile } from 'node:fs/promises'; import { fileURLToPath } from 'node:url'; +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; + import { documentsBesideTheEntries, translatedDocuments } from './src/translating/document-manifests.ts'; const repository = fileURLToPath(new URL('../../', import.meta.url)); @@ -10,7 +13,9 @@ const formatter = fileURLToPath(new URL('../../node_modules/.bin/oxfmt', import. const written = await translatedDocuments(process.argv.slice(2), { folder: documentsBesideTheEntries, + shippedFiles: shippedFilesOnDisk, fetching: fetch, + readGiven: (path) => readFile(path, 'utf8'), }); const paths = written.map(({ path }) => path); for (const { path, text } of written) { diff --git a/packages/server/README.md b/packages/server/README.md index f647af354..336dc29ca 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -1,6 +1,6 @@ # @on.auto/server -The Node.js server of Auto Runtime: `src/main.ts` starts it on the composition root of `src/composition/composition-root.ts`, `dev.ts` restarts it on every save, and `auto-runtime.ts` is the command of the npm package `@on.auto/runtime`, which `npm/pack.ts` packs with the `README.md` in `npm/`; the `Dockerfile` builds its container image. +The Node.js server of Auto Runtime: `src/main.ts` starts it on the composition root of `src/composition/composition-root.ts`, `dev.ts` restarts it on every save, and `auto-runtime.ts` is the command of the npm package `@on.auto/runtime`, which `npm/pack.ts` packs with the `README.md` in `npm/`; the `Dockerfile` builds its container image. How the files the packages ship are read is composed once, `shippedFilesOnDisk` of `@on.auto/operations/shipped-files-on-disk` in the options of `src/server-options/composed-options.ts`, and handed on as `ServerOptions.shippedFiles`: at the start, to the providers' entries the settings are read against (`readShippedSettings`, `src/settings/served-settings.ts`) and to the style of the API's pages, `readPageStyle` of `@on.auto/api`, each read once and handed on as content, so a file that cannot be read stops the start (`src/lifecycle/lifecycle.ts`, `src/lifecycle/page-style-startup.test.ts`); then to `composed`, and from there to the files the pool of workers starts with, read where the one pool is made (`src/composition/served-computation.ts`), and to the guides and recipes the server serves (`src/guides`). `src/server-options/server-options.ts` holds what a server is started with and answers, `ServerOptions`, `Composed`, `Served` and `RunningServer` among them, which the start in `src/lifecycle` and every part the composition serves import, so nothing imports the start for them. ## The secrets key diff --git a/packages/server/measure/measured-server.ts b/packages/server/measure/measured-server.ts index 78be72242..439564dd7 100644 --- a/packages/server/measure/measured-server.ts +++ b/packages/server/measure/measured-server.ts @@ -1,7 +1,8 @@ import { Schema } from 'effect'; import { compositionRoot } from '../src/composition/composition-root.ts'; -import { startServer, type RunningServer } from '../src/lifecycle/lifecycle.ts'; +import { startServer } from '../src/lifecycle/lifecycle.ts'; +import type { RunningServer } from '../src/server-options/server-options.ts'; export const playbook = '/v1/orgs/local/playbooks/measure'; diff --git a/packages/server/src/command-line/local-banner.ts b/packages/server/src/command-line/local-banner.ts index a5bab9e98..eb062b751 100644 --- a/packages/server/src/command-line/local-banner.ts +++ b/packages/server/src/command-line/local-banner.ts @@ -1,8 +1,8 @@ import { studioBaseOf, studioLinkOf, type AccessMode, type StudioSettings } from '@on.auto/identity'; -import type { Install } from '../lifecycle/lifecycle.ts'; import { defaultRuntimePort } from '../local-runtime/local-control.ts'; import { linkerFor, painterFor, type ColorDepth } from '../logging/terminal.ts'; +import type { Install } from '../server-options/server-options.ts'; import { firstStartNotice } from '../usage-ping/usage-ping.ts'; import { logoHeight, logoLines, type PaintingDepth } from './logo.ts'; diff --git a/packages/server/src/command-line/local-server.test.ts b/packages/server/src/command-line/local-server.test.ts index f67a4a224..bb8317234 100644 --- a/packages/server/src/command-line/local-server.test.ts +++ b/packages/server/src/command-line/local-server.test.ts @@ -4,8 +4,8 @@ import type { SignInSettings } from '@on.auto/identity'; import { describe, expect, it } from 'vitest'; import { release } from '../composition/release.ts'; -import type { Install } from '../lifecycle/lifecycle.ts'; import type { ColorDepth } from '../logging/terminal.ts'; +import type { Install } from '../server-options/server-options.ts'; import { servedLocally, type LocalServerHost } from './local-server.ts'; interface Served { diff --git a/packages/server/src/command-line/local-server.ts b/packages/server/src/command-line/local-server.ts index cf6b0d94e..b950ba972 100644 --- a/packages/server/src/command-line/local-server.ts +++ b/packages/server/src/command-line/local-server.ts @@ -1,8 +1,8 @@ import type { AccessMode, SignInSettings, StudioSettings } from '@on.auto/identity'; import { release } from '../composition/release.ts'; -import type { Install } from '../lifecycle/lifecycle.ts'; import type { ColorDepth } from '../logging/terminal.ts'; +import type { Install } from '../server-options/server-options.ts'; import { clearedLine, headerOf, readyNoticeOf, startingLine, stoppedNotice, stoppingNotice } from './local-banner.ts'; interface LocalServer { diff --git a/packages/server/src/composition/composition-root.ts b/packages/server/src/composition/composition-root.ts index 7d13bd6ab..13384d658 100644 --- a/packages/server/src/composition/composition-root.ts +++ b/packages/server/src/composition/composition-root.ts @@ -1,8 +1,6 @@ -import type { DispatcherServices } from '@on.auto/operations'; - import { storesAndConnectionsOf } from '../connections/served-connections.ts'; import { ledgerDatabaseOf, withARefusedLayoutExplained } from '../database/ledger-database.ts'; -import { defaultServerOptions, type ServerOptions } from '../lifecycle/lifecycle.ts'; +import { serverOptionsComposing, type ComposedServerOptions } from '../server-options/composed-options.ts'; import { serveWorkflows, type WorkflowParts } from '../workflows/workflows.ts'; import { applicationLayer } from './application-layer.ts'; import { functionWiringOf, type FunctionWiring } from './function-wiring.ts'; @@ -15,43 +13,42 @@ export function compositionRootWith( programPoolOf?: FunctionWiring['programPoolOf'], clock?: WorkflowParts['clock'], toolTiming?: FunctionWiring['toolTiming'], -): ServerOptions { +): ComposedServerOptions { const wiring = functionWiringOf(modelAccessOf, programPoolOf, toolTiming); - return { - ...defaultServerOptions, - composed: (settings, telemetry) => { - const database = ledgerDatabaseOf(settings, telemetry.appended); - return { - services: applicationLayer( - withARefusedLayoutExplained(ledgerLayerFor(settings, database, wiring.projections.appends), settings), - ), - serve: async (runtime, ownOrigin) => { - const { ledger, workflows, identityIssuer } = settings; - const { stores, retryStateKeys, consent, connections, source, offering, clientMetadata } = - await storesAndConnectionsOf(runtime, settings, database, ownOrigin); - const functions = await functionsServedBy(runtime, settings, database, { - ...wiring, - ...(clock === undefined ? {} : { now: clock.now }), - telemetry, - consent, - connections: source, - offering, - }); - const parts = { - ledger, - workflows, - identityIssuer, - retryStateKeys, - ...functions.parts, - orgOperations: [...functions.parts.orgOperations, ...connections], - ...(clock === undefined ? {} : { clock }), - }; - const served = stores.withSecretsClosed(functions.closing(await serveWorkflows(runtime, parts))); - return { ...served, routes: [...served.routes, clientMetadata] }; - }, - }; - }, - }; + return serverOptionsComposing((settings, telemetry, shippedFiles) => { + const database = ledgerDatabaseOf(settings, telemetry.appended); + return { + services: applicationLayer( + withARefusedLayoutExplained(ledgerLayerFor(settings, database, wiring.projections.appends), settings), + ), + serve: async (runtime, ownOrigin) => { + const { ledger, workflows, identityIssuer } = settings; + const { stores, retryStateKeys, consent, connections, source, offering, clientMetadata } = + await storesAndConnectionsOf(runtime, settings, database, ownOrigin); + const functions = await functionsServedBy(runtime, settings, database, { + ...wiring, + ...(clock === undefined ? {} : { now: clock.now }), + telemetry, + consent, + connections: source, + offering, + shippedFiles, + }); + const parts = { + ledger, + workflows, + identityIssuer, + retryStateKeys, + shippedFiles, + ...functions.parts, + orgOperations: [...functions.parts.orgOperations, ...connections], + ...(clock === undefined ? {} : { clock }), + }; + const served = stores.withSecretsClosed(functions.closing(await serveWorkflows(runtime, parts))); + return { ...served, routes: [...served.routes, clientMetadata] }; + }, + }; + }); } export const compositionRoot = compositionRootWith(loggedModelAccess); diff --git a/packages/server/src/composition/content-store.test.ts b/packages/server/src/composition/content-store.test.ts index 9be559181..7b9f146af 100644 --- a/packages/server/src/composition/content-store.test.ts +++ b/packages/server/src/composition/content-store.test.ts @@ -7,7 +7,7 @@ import { Effect, ManagedRuntime, Schema } from 'effect'; import { describe, expect, it, onTestFinished, vi } from 'vitest'; import { ledgerDatabaseOf } from '../database/ledger-database.ts'; -import { readSettings } from '../settings/settings.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; import { contentStoreOf } from './content-store.ts'; import { ledgerLayerFor, ledgerLayerOf } from './ledger-store.ts'; @@ -39,7 +39,7 @@ function ledgerOn(host: string): Promise { onTestFinished(() => { vi.unstubAllEnvs(); }); - const settings = readSettings({ + const settings = readSettingsOf({ LEDGER_FILE: ':memory:', CONTENT_STORE_URL: 's3://playbook-content/production', AWS_ENDPOINT_URL: `http://${host}`, @@ -52,7 +52,7 @@ function ledgerOn(host: string): Promise { describe('the store the content of a playbook is kept in', () => { it("is the ledger's own when no object store is set", () => { - expect(contentStoreOf(readSettings({}))).toBeUndefined(); + expect(contentStoreOf(readSettingsOf({}))).toBeUndefined(); }); it('is an object store when CONTENT_STORE_URL is set, whose bucket is checked before the ledger opens', async () => { diff --git a/packages/server/src/composition/served-computation.ts b/packages/server/src/composition/served-computation.ts index 30467e16b..69f683451 100644 --- a/packages/server/src/composition/served-computation.ts +++ b/packages/server/src/composition/served-computation.ts @@ -1,12 +1,13 @@ import { computationBounds, makeComputationFunctionAdapter } from '@on.auto/computation'; import type { Capability } from '@on.auto/definitions'; -import { checkedAtSave, warmedChecks } from '@on.auto/definitions/check'; -import { programPool, type CheckJob, type ProgramPool } from '@on.auto/workflow-engine/dsl'; +import { checkedAtSave, handedWorkersWithChecks, warmedChecks } from '@on.auto/definitions/check'; +import type { ShippedFiles } from '@on.auto/operations'; +import { programPool, type CheckJob, type HandedWorkers, type ProgramPool } from '@on.auto/workflow-engine/dsl'; import type { ComputationSettings } from '../function-settings/computation-settings.ts'; -import type { Served } from '../lifecycle/lifecycle.ts'; +import type { Served } from '../server-options/server-options.ts'; -export type ProgramPoolOf = (settings: ComputationSettings) => ProgramPool; +export type ProgramPoolOf = (settings: ComputationSettings, handed: HandedWorkers) => ProgramPool; export interface ServedComputation { readonly capability: Capability; @@ -15,14 +16,15 @@ export interface ServedComputation { readonly withPoolClosed: (served: Served) => Served; } -export const workerPool: ProgramPoolOf = ({ workers }) => - programPool({ workers, heapMegabytes: computationBounds.heapMegabytes }); +export const workerPool: ProgramPoolOf = ({ workers }, handed) => + programPool({ workers, heapMegabytes: computationBounds.heapMegabytes, handed }); export async function computationServedBy( settings: ComputationSettings, poolOf: ProgramPoolOf, + shippedFiles: ShippedFiles, ): Promise { - const pool = poolOf(settings); + const pool = poolOf(settings, await handedWorkersWithChecks(shippedFiles)); await warmedChecks(pool); return { capability: makeComputationFunctionAdapter({ pool }), diff --git a/packages/server/src/composition/served-functions.ts b/packages/server/src/composition/served-functions.ts index a41fc6fe1..b8d8e836f 100644 --- a/packages/server/src/composition/served-functions.ts +++ b/packages/server/src/composition/served-functions.ts @@ -1,5 +1,5 @@ import type { AppRuntime } from '@on.auto/api'; -import type { DispatcherServices } from '@on.auto/operations'; +import type { DispatcherServices, ShippedFiles } from '@on.auto/operations'; import { playbookOperations } from '@on.auto/playbooks'; import type { Settings } from '../settings/settings.ts'; @@ -17,7 +17,7 @@ import { toolUsersServedBy, type ToolUsersWiring } from './served-tools.ts'; type LedgerDatabase = Parameters[1]['database']; export interface ServedFunctions { - readonly parts: Omit; + readonly parts: Omit; readonly closing: ServedPredictionFunctions['withProcessesClosed']; } @@ -28,11 +28,13 @@ export async function functionsServedBy( { programPoolOf, projections: wiring, + shippedFiles, ...tools - }: FunctionWiring & Pick, + }: FunctionWiring & + Pick & { readonly shippedFiles: ShippedFiles }, ): Promise { const { reasoning, interaction, toolOperations, withToolsClosed } = await toolUsersServedBy(runtime, settings, tools); - const computation = await computationServedBy(settings.computation, programPoolOf); + const computation = await computationServedBy(settings.computation, programPoolOf, shippedFiles); const projections = await wiring.served(runtime, { recall: settings.recall, database }, computation.pool, [ datasetsFolderOf(computation.pool), ]); diff --git a/packages/server/src/composition/served-prediction.ts b/packages/server/src/composition/served-prediction.ts index a1f304a60..75f631ca3 100644 --- a/packages/server/src/composition/served-prediction.ts +++ b/packages/server/src/composition/served-prediction.ts @@ -4,8 +4,8 @@ import type { DispatcherServices } from '@on.auto/operations'; import type { PredictionPorts } from '@on.auto/prediction'; import type { WorkflowStore } from '@on.auto/workflow-host'; -import type { ServedClosing } from '../lifecycle/lifecycle.ts'; import { logPredictionFunctions } from '../logging/prediction-logging.ts'; +import type { ServedClosing } from '../server-options/server-options.ts'; import type { Settings } from '../settings/settings.ts'; import { runtimeLedger } from './runtime-ledger.ts'; import { servedModels } from './served-models.ts'; diff --git a/packages/server/src/composition/served-routes.ts b/packages/server/src/composition/served-routes.ts index adacf2dbf..a74c39e0b 100644 --- a/packages/server/src/composition/served-routes.ts +++ b/packages/server/src/composition/served-routes.ts @@ -16,6 +16,7 @@ import { type Catalog, type Dispatcher, type DispatcherServices, + type ShippedFiles, } from '@on.auto/operations'; import { servedGuidesOf } from '../guides/served-guides.ts'; @@ -30,13 +31,14 @@ export interface ServedCatalog { readonly capabilities: readonly Capability[]; readonly identityIssuer: string; readonly retryStateKeys: RetryStateKeys; + readonly shippedFiles: ShippedFiles; } -export function routesFor( +export async function routesFor( runtime: AppRuntime, - { catalog, dispatcher, capabilities, identityIssuer, retryStateKeys }: ServedCatalog, -): readonly RegisterRoutes[] { - const { guides, recipes, definitionTypes } = servedGuidesOf(capabilities); + { catalog, dispatcher, capabilities, identityIssuer, retryStateKeys, shippedFiles }: ServedCatalog, +): Promise { + const { guides, recipes, definitionTypes } = await servedGuidesOf(shippedFiles, capabilities); const shelf = guideShelfOf(guides, recipes, definitionTypes); return [ operationRoutes({ catalog, dispatcher, runCall: runtime.run }), @@ -59,9 +61,20 @@ export function routesFor( export function routesServing( operations: Operations, - { identityIssuer, retryStateKeys }: Pick, -): (runtime: AppRuntime) => readonly RegisterRoutes[] { + { + identityIssuer, + retryStateKeys, + shippedFiles, + }: Pick, +): (runtime: AppRuntime) => Promise { const catalog = makeCatalog(operations); - const served = { catalog, dispatcher: makeDispatcher([]), capabilities: [], identityIssuer, retryStateKeys }; + const served = { + catalog, + dispatcher: makeDispatcher([]), + capabilities: [], + identityIssuer, + retryStateKeys, + shippedFiles, + }; return (runtime) => routesFor(runtime, served); } diff --git a/packages/server/src/composition/served-tools.ts b/packages/server/src/composition/served-tools.ts index 8bdb4a1e4..4882998e3 100644 --- a/packages/server/src/composition/served-tools.ts +++ b/packages/server/src/composition/served-tools.ts @@ -2,7 +2,7 @@ import type { AppRuntime } from '@on.auto/api'; import type { DispatcherServices } from '@on.auto/operations'; import type { ToolAccessOptions } from '@on.auto/tool-servers'; -import type { Served } from '../lifecycle/lifecycle.ts'; +import type { Served } from '../server-options/server-options.ts'; import type { Settings } from '../settings/settings.ts'; import { interactionServedBy, type ServedInteraction } from './served-interaction.ts'; import { reasoningServedBy, type ModelAccessOf, type ServedReasoning } from './served-reasoning.ts'; diff --git a/packages/server/src/computation/checks-over-http.test.ts b/packages/server/src/computation/checks-over-http.test.ts index 90cca64e5..dafb32824 100644 --- a/packages/server/src/computation/checks-over-http.test.ts +++ b/packages/server/src/computation/checks-over-http.test.ts @@ -43,7 +43,7 @@ function countingChecks(): { readonly checks: Checks; readonly poolOf: ProgramPo stalled = { ran: 'stopped', because, milliseconds: 2000 }; }, }, - poolOf: (settings) => checking(patientWorkerPool(settings)), + poolOf: (settings, handed) => checking(patientWorkerPool(settings, handed)), }; } diff --git a/packages/server/src/computation/checks-over-mcp.test.ts b/packages/server/src/computation/checks-over-mcp.test.ts index fa6ba3e1c..811dbc204 100644 --- a/packages/server/src/computation/checks-over-mcp.test.ts +++ b/packages/server/src/computation/checks-over-mcp.test.ts @@ -36,8 +36,8 @@ function stallingChecks(): { stall: (because) => { state.because = because; }, - poolOf: (settings) => { - const pool = patientWorkerPool(settings); + poolOf: (settings, handed) => { + const pool = patientWorkerPool(settings, handed); return { ...pool, check: (request, signal) => diff --git a/packages/server/src/computation/computation-deadline.test.ts b/packages/server/src/computation/computation-deadline.test.ts index d70f9de72..685f216e2 100644 --- a/packages/server/src/computation/computation-deadline.test.ts +++ b/packages/server/src/computation/computation-deadline.test.ts @@ -45,9 +45,14 @@ async function ranOn(rows: number) { describe('a run of a computation function that does not end by itself, over HTTP', () => { it('is unavailable once its warm worker is terminated at the deadline, while another worker answers and the server answers other requests', async () => { server = await servingReasoning([], {}, undefined, { - programPoolOf: ({ workers }) => + programPoolOf: ({ workers }, handed) => checkingWithin( - programPool({ workers, heapMegabytes: computationBounds.heapMegabytes, worker: blockingOnTwoRows }), + programPool({ + workers, + heapMegabytes: computationBounds.heapMegabytes, + worker: blockingOnTwoRows, + handed, + }), ), }); await server.call('POST', '/v1/orgs/acme/playbooks', { body: { playbook: 'alpha', name: 'Alpha' } }); diff --git a/packages/server/src/config-file/connections-section.test.ts b/packages/server/src/config-file/connections-section.test.ts index e2b793b2f..a014a7ff7 100644 --- a/packages/server/src/config-file/connections-section.test.ts +++ b/packages/server/src/config-file/connections-section.test.ts @@ -6,7 +6,7 @@ import { configurationOf } from '@on.auto/config'; import { Redacted, Result } from 'effect'; import { afterEach, describe, expect, it } from 'vitest'; -import { readSettings } from '../settings/settings.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; import { fileSettings } from './file-settings.ts'; const directories: string[] = []; @@ -27,7 +27,7 @@ function fileOf(text: string): string { function refusalOf(path: string, environment: Readonly> = {}): string { try { - readSettings({ CONFIG_FILE: path, ...environment }); + readSettingsOf({ CONFIG_FILE: path, ...environment }); } catch (error) { return String(error).replaceAll(path, 'auto-runtime.yaml'); } @@ -45,7 +45,7 @@ const ownVariable = 'is a variable the runtime reads for itself, which no connec describe('the connections section of the configuration file', () => { it('is read as the policy of what people may add, its kinds of credential not taken for credentials', () => { - expect(readSettings({ CONFIG_FILE: fileOf(policy) }).connections).toEqual({ + expect(readSettingsOf({ CONFIG_FILE: fileOf(policy) }).connections).toEqual({ peopleMayAdd: true, allowedHosts: ['*.example.com', 'crm.example.com'], credentialKinds: ['none', 'reference', 'secret', 'client_credentials'], @@ -56,7 +56,7 @@ describe('the connections section of the configuration file', () => { describe('the variables of the connections section', () => { it('hold the value of each variable it lists that the environment sets, and nothing of one it leaves unset or empty', () => { - const settings = readSettings({ + const settings = readSettingsOf({ CONFIG_FILE: fileOf(policy.replace('[CRM_API_KEY]', '[CRM_API_KEY, WIKI_TOKEN, NOTES_KEY]')), CRM_API_KEY: 'crm-key-0f3c9a71', NOTES_KEY: '', diff --git a/packages/server/src/config-file/example-file.test.ts b/packages/server/src/config-file/example-file.test.ts index 5c39ef0a2..ed608182f 100644 --- a/packages/server/src/config-file/example-file.test.ts +++ b/packages/server/src/config-file/example-file.test.ts @@ -4,7 +4,7 @@ import { join } from 'node:path'; import { describe, expect, it, onTestFinished } from 'vitest'; -import { readSettings } from '../settings/settings.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; const example = readFileSync(new URL('../../../../auto-runtime.example.yaml', import.meta.url), 'utf8'); @@ -37,7 +37,7 @@ describe('the tool servers and the connections auto-runtime.example.yaml shows', it('read as written, an entry that names a provider the connections add among them', () => { const path = configFile(`${settingAsWritten('tool_servers')}\n${settingAsWritten('connections')}\n`); - const { toolServers, providers } = readSettings({ CONFIG_FILE: path, ...references }); + const { toolServers, providers } = readSettingsOf({ CONFIG_FILE: path, ...references }); expect(toolServers.servers.map(({ name, provider }) => [name, provider])).toEqual([ ['graph', null], diff --git a/packages/server/src/connections/operator-values.test.ts b/packages/server/src/connections/operator-values.test.ts index 1a591ea44..7f2ba665f 100644 --- a/packages/server/src/connections/operator-values.test.ts +++ b/packages/server/src/connections/operator-values.test.ts @@ -2,10 +2,10 @@ import { fixtureProviders } from '@on.auto/providers/testing'; import { Redacted } from 'effect'; import { describe, expect, it } from 'vitest'; -import { readSettings } from '../settings/settings.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; import { readConnections, readConnectionSettings } from './connection-settings.ts'; -const functions = readSettings({}); +const functions = readSettingsOf({}); describe('the values the operator sets for the providers', () => { it('are read from the environment for each variable a provider refers to, and for no other', () => { diff --git a/packages/server/src/connections/providers-added.test.ts b/packages/server/src/connections/providers-added.test.ts index 9ae361de9..a1388a94c 100644 --- a/packages/server/src/connections/providers-added.test.ts +++ b/packages/server/src/connections/providers-added.test.ts @@ -6,9 +6,9 @@ import { serveFakeMcp, type FakeMcpServer } from '@on.auto/tool-servers/testing' import { Schema } from 'effect'; import { afterEach, describe, expect, it, onTestFinished } from 'vitest'; -import { readSettings } from '../settings/settings.ts'; import { offeredByTheServer, shippedProviders } from '../testing/servers/connected-servers.ts'; import { servingReasoning, type ReasoningServer } from '../testing/servers/reasoning-server.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; const workspace = 'notes-workspace-6b2e0d'; @@ -169,7 +169,7 @@ describe('a provider the settings file adds, whose choices set a host named for function refusalOf(environment: Readonly>): string { try { - readSettings(environment); + readSettingsOf(environment); } catch (error) { return String(error).replaceAll(environment['CONFIG_FILE'] ?? 'no file', 'auto-runtime.yaml'); } diff --git a/packages/server/src/connections/providers-served.test.ts b/packages/server/src/connections/providers-served.test.ts index 1c57270a8..1a264e1f7 100644 --- a/packages/server/src/connections/providers-served.test.ts +++ b/packages/server/src/connections/providers-served.test.ts @@ -3,9 +3,9 @@ import { hostAlternativesOf, operatorVariablesIn, type Provider } from '@on.auto import { Schema } from 'effect'; import { afterAll, beforeAll, describe, expect, it, onTestFinished } from 'vitest'; -import { readSettings } from '../settings/settings.ts'; import { required, shippedList, shippedProviders } from '../testing/servers/connected-servers.ts'; import { servingReasoning, type ReasoningServer } from '../testing/servers/reasoning-server.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; import { variablesTheRuntimeReads } from './variables-the-runtime-reads.ts'; const names = shippedProviders.map(({ entry }) => entry.name); @@ -94,7 +94,7 @@ describe('the providers a server lists over HTTP', () => { }); it('take no name a built-in model provider has, which every runtime would refuse a connection', () => { - const { modelProviders } = readSettings({}); + const { modelProviders } = readSettingsOf({}); expect(names.filter((name) => modelProviders.includes(name))).toEqual([]); expect(modelProviders.length).toBeGreaterThan(5); diff --git a/packages/server/src/connections/registered-clients.test.ts b/packages/server/src/connections/registered-clients.test.ts index ce0f64f83..2748ccb2e 100644 --- a/packages/server/src/connections/registered-clients.test.ts +++ b/packages/server/src/connections/registered-clients.test.ts @@ -5,8 +5,8 @@ import { serveFakeMcp } from '@on.auto/tool-servers/testing'; import { Redacted, Schema } from 'effect'; import { afterEach, describe, expect, it } from 'vitest'; -import { readSettings } from '../settings/settings.ts'; import { alpha, servingReasoning, type ReasoningServer } from '../testing/servers/reasoning-server.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; const apiKey = 'graph-api-key-4f1d9a7c2b'; @@ -50,14 +50,14 @@ afterEach(async () => { describe('the registered clients of a server', () => { it('are among its settings, each secret redacted', () => { - const { registeredClients } = readSettings({ CONNECTIONS: connections, EXAMPLE_CLIENT_SECRET: clientSecret }); + const { registeredClients } = readSettingsOf({ CONNECTIONS: connections, EXAMPLE_CLIENT_SECRET: clientSecret }); expect(registeredClients).toMatchObject([ { clientId: '1234.apps.example.com', credential: { kind: 'client_secret' } }, ]); expect(JSON.stringify(registeredClients)).not.toContain(clientSecret); expect(registeredClientSecrets(registeredClients).map((secret) => Redacted.value(secret))).toEqual([clientSecret]); - expect(readSettings({}).registeredClients).toEqual([]); + expect(readSettingsOf({}).registeredClients).toEqual([]); }); it('have their secrets scrubbed from what a call answers and from what it records, as the secrets of its tool servers are', async () => { diff --git a/packages/server/src/consent/consent-settings.test.ts b/packages/server/src/consent/consent-settings.test.ts index 703545471..73bb80eb0 100644 --- a/packages/server/src/consent/consent-settings.test.ts +++ b/packages/server/src/consent/consent-settings.test.ts @@ -6,7 +6,7 @@ import { createApiKey } from '@on.auto/identity'; import { describe, expect, it, onTestFinished } from 'vitest'; import { variablesTheRuntimeReads } from '../connections/variables-the-runtime-reads.ts'; -import { readSettings } from '../settings/settings.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; import { autoClientsOf, returnKindOf } from './consent-settings.ts'; const { entry } = createApiKey({ id: 'ci-1', org: 'acme', permissions: ['org:read'], playbooks: '*' }); @@ -17,7 +17,7 @@ const signingIn = { PUBLIC_URL: 'https://runtime.example.com' }; function errorFrom(environment: Readonly>): string { try { - readSettings(environment); + readSettingsOf(environment); } catch (error) { return String(error); } @@ -25,7 +25,7 @@ function errorFrom(environment: Readonly>): string { } function switchOf(environment: Readonly>): boolean { - return readSettings(environment).autoClients.enabled; + return readSettingsOf(environment).autoClients.enabled; } function connectionsOf(setting: Readonly>): string { @@ -75,9 +75,9 @@ describe('the switch of Auto’s clients', () => { describe('the address of Auto’s consent service', () => { it('is https://consent.on.auto by default, and any https origin, or an http one on the loopback for a test', () => { expect([ - readSettings(loopback).autoClients.serviceUrl, - readSettings({ ...loopback, CONSENT_SERVICE_URL: 'https://consent.example.com' }).autoClients.serviceUrl, - readSettings({ ...loopback, CONSENT_SERVICE_URL: 'http://127.0.0.1:19650' }).autoClients.serviceUrl, + readSettingsOf(loopback).autoClients.serviceUrl, + readSettingsOf({ ...loopback, CONSENT_SERVICE_URL: 'https://consent.example.com' }).autoClients.serviceUrl, + readSettingsOf({ ...loopback, CONSENT_SERVICE_URL: 'http://127.0.0.1:19650' }).autoClients.serviceUrl, ]).toEqual(['https://consent.on.auto', 'https://consent.example.com', 'http://127.0.0.1:19650']); }); diff --git a/packages/server/src/database/database-pools.test.ts b/packages/server/src/database/database-pools.test.ts index 911fb173c..432c680d4 100644 --- a/packages/server/src/database/database-pools.test.ts +++ b/packages/server/src/database/database-pools.test.ts @@ -4,10 +4,10 @@ import type { DatabaseTokenRequest } from '@on.auto/outbound/aws'; import { describe, expect, it, onTestFinished } from 'vitest'; import type { PostgreSQLLedgerSettings } from '../settings/ledger-settings.ts'; -import { readSettings } from '../settings/settings.ts'; import { aFakePostgreSQL } from '../testing/databases/fake-postgresql.ts'; import { aCertificateOfTheLoopback, trustedForTheTest } from '../testing/databases/loopback-certificate.ts'; import { secretsKey } from '../testing/databases/secrets-key.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; import { databasePoolsOf, type DatabaseTokenSignerOf } from './database-pools.ts'; interface Asked { @@ -40,7 +40,7 @@ function aRecordingSigner(): RecordingSigner { const proxied = { NODE_USE_ENV_PROXY: '1', HTTPS_PROXY: 'http://proxy.internal:3128' }; function postgresqlLedgerOf(environment: Environment): PostgreSQLLedgerSettings { - const { ledger } = readSettings({ SECRETS_KEY: secretsKey, ...environment }); + const { ledger } = readSettingsOf({ SECRETS_KEY: secretsKey, ...environment }); if (ledger.store === 'sqlite') { throw new Error('Expected a ledger in PostgreSQL'); } @@ -62,7 +62,7 @@ describe('the pools of a ledger whose password is in DATABASE_URL', () => { const pool = databasePoolsOf( postgresqlLedgerOf({ DATABASE_URL: url }), - readSettings({}).proxy, + readSettingsOf({}).proxy, signer.signerOf, )({ max: 4, @@ -80,7 +80,7 @@ describe('the pools of a ledger that signs a token for each connection', () => { trustedForTheTest(certificate); const database = await aFakePostgreSQL(certificate); const signer = aRecordingSigner(); - const { proxy } = readSettings(proxied); + const { proxy } = readSettingsOf(proxied); const pool = databasePoolsOf(signingLedgerOn(database.port), proxy, signer.signerOf)({ max: 2 }); onTestFinished(() => pool.end()); @@ -99,7 +99,7 @@ describe('the pools of a ledger that signs a token for each connection', () => { it('reach only a database whose certificate they verify, and ask no token of one they cannot', async () => { const database = await aFakePostgreSQL(aCertificateOfTheLoopback()); const signer = aRecordingSigner(); - const pool = databasePoolsOf(signingLedgerOn(database.port), readSettings({}).proxy, signer.signerOf)({ max: 1 }); + const pool = databasePoolsOf(signingLedgerOn(database.port), readSettingsOf({}).proxy, signer.signerOf)({ max: 1 }); onTestFinished(() => pool.end()); await expect(pool.connect()).rejects.toMatchObject({ code: 'DEPTH_ZERO_SELF_SIGNED_CERT' }); @@ -110,7 +110,7 @@ describe('the pools of a ledger that signs a token for each connection', () => { it('refuse a database that offers no TLS, before any token is asked for', async () => { const database = await aFakePostgreSQL(); const signer = aRecordingSigner(); - const pool = databasePoolsOf(signingLedgerOn(database.port), readSettings({}).proxy, signer.signerOf)({ max: 1 }); + const pool = databasePoolsOf(signingLedgerOn(database.port), readSettingsOf({}).proxy, signer.signerOf)({ max: 1 }); onTestFinished(() => pool.end()); await expect(pool.connect()).rejects.toThrow('The server does not support SSL connections'); @@ -125,7 +125,7 @@ describe('the host name of a database that signs in with tokens', () => { trustedForTheTest(certificate); const database = await aFakePostgreSQL(certificate); const signer = aRecordingSigner(); - const pool = databasePoolsOf(signingLedgerOn(database.port), readSettings({}).proxy, signer.signerOf)({ max: 1 }); + const pool = databasePoolsOf(signingLedgerOn(database.port), readSettingsOf({}).proxy, signer.signerOf)({ max: 1 }); onTestFinished(() => pool.end()); await expect(pool.connect()).rejects.toMatchObject({ code: 'ERR_TLS_CERT_ALTNAME_INVALID' }); @@ -142,7 +142,7 @@ describe('a token the pools of a ledger cannot sign', () => { const unsigned = new Error('Could not load credentials from any providers'); const pool = databasePoolsOf( signingLedgerOn(database.port), - readSettings({}).proxy, + readSettingsOf({}).proxy, () => () => Promise.reject(unsigned), )({ max: 1 }); onTestFinished(() => pool.end()); diff --git a/packages/server/src/database/kms-page.test.ts b/packages/server/src/database/kms-page.test.ts index c11584b6a..609ce7456 100644 --- a/packages/server/src/database/kms-page.test.ts +++ b/packages/server/src/database/kms-page.test.ts @@ -5,8 +5,8 @@ import { serveFakeKms } from '@on.auto/secrets/testing'; import { Effect, Schema } from 'effect'; import { afterAll, describe, expect, it, onTestFinished } from 'vitest'; -import { readSettings } from '../settings/settings.ts'; import { temporaryLedger } from '../testing/records/temporary-ledger.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; import { ledgerStoresOpenedBy } from './ledger-stores.ts'; const page = readFileSync(new URL('../../../../docs/self-host/production.md', import.meta.url), 'utf8'); @@ -57,7 +57,7 @@ const quotedRefusal = String(/`(The secrets settings are invalid\. SECRETS_KEY_K describe('the page that holds the secrets key in a KMS', () => { it('sets SECRETS_KEY_KMS to the ARN of a key and of an alias, and rotates from a key in a file in one restart, as the settings read them', () => { - expect(shellBlocks.map((environment) => readSettings(environment).secrets)).toEqual([ + expect(shellBlocks.map((environment) => readSettingsOf(environment).secrets)).toEqual([ { current: { from: 'kms', setting: 'SECRETS_KEY_KMS', arn: keyArn, region: 'eu-west-1' }, previous: undefined }, { current: { @@ -110,7 +110,7 @@ async function startRefusedFor(environment: Readonly>): P const context = await Effect.runPromise(Effect.context()); return ledgerStoresOpenedBy( context, - readSettings({ LEDGER_FILE: fileName, ...environment }), + readSettingsOf({ LEDGER_FILE: fileName, ...environment }), { store: 'sqlite', file: fileName }, () => Promise.resolve(onTheFake), ).then(() => 'started', String); diff --git a/packages/server/src/database/ledger-database.test.ts b/packages/server/src/database/ledger-database.test.ts index b7b1ba90a..5a151bc64 100644 --- a/packages/server/src/database/ledger-database.test.ts +++ b/packages/server/src/database/ledger-database.test.ts @@ -3,8 +3,8 @@ import { Ledger } from '@on.auto/operations'; import { Cause, Effect, Exit, Layer } from 'effect'; import { describe, expect, it } from 'vitest'; -import { readSettings } from '../settings/settings.ts'; import { secretsKey } from '../testing/databases/secrets-key.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; import { ledgerDatabaseOf, withARefusedLayoutExplained } from './ledger-database.ts'; import { LedgerOfAnotherVersion } from './ledger-of-another-version.ts'; @@ -12,7 +12,7 @@ const somePools: unknown = expect.any(Function); describe('the database of the ledger and the workflow host', () => { it('is the file of the ledger when the ledger is kept in SQLite', () => { - expect(ledgerDatabaseOf(readSettings({ LEDGER_FILE: '/data/ledger.db' }))).toEqual({ + expect(ledgerDatabaseOf(readSettingsOf({ LEDGER_FILE: '/data/ledger.db' }))).toEqual({ store: 'sqlite', file: '/data/ledger.db', }); @@ -21,7 +21,7 @@ describe('the database of the ledger and the workflow host', () => { it('is reached through the pools the server composes when the ledger is kept in PostgreSQL', () => { expect( ledgerDatabaseOf( - readSettings({ + readSettingsOf({ DATABASE_URL: 'postgresql://playbooks:secret@db.example.com/playbooks', SECRETS_KEY: secretsKey, }), @@ -46,7 +46,7 @@ describe('a ledger that refuses the layout of its database as it opens', () => { const refused = await builtFrom( withARefusedLayoutExplained( Layer.effect(Ledger, Effect.die(refusedLayout)), - readSettings({ LEDGER_FILE: '/data/ledger.db' }), + readSettingsOf({ LEDGER_FILE: '/data/ledger.db' }), ), ); @@ -62,7 +62,7 @@ describe('a ledger that refuses the layout of its database as it opens', () => { const refused = await builtFrom( withARefusedLayoutExplained( Layer.effect(Ledger, Effect.die(refusedLayout)), - readSettings({ LEDGER_FILE: '/app/data/ledger.db', RUNTIME_DISTRIBUTION: 'docker', DO_NOT_TRACK: '1' }), + readSettingsOf({ LEDGER_FILE: '/app/data/ledger.db', RUNTIME_DISTRIBUTION: 'docker', DO_NOT_TRACK: '1' }), ), ); @@ -81,7 +81,7 @@ describe('a ledger that refuses the layout of its database as it opens', () => { await builtFrom( withARefusedLayoutExplained( Layer.effect(Ledger, Effect.die(unreachable)), - readSettings({ LEDGER_FILE: '/data/ledger.db' }), + readSettingsOf({ LEDGER_FILE: '/data/ledger.db' }), ), ), ).toBe(unreachable); diff --git a/packages/server/src/database/ledger-stores-keepers.test.ts b/packages/server/src/database/ledger-stores-keepers.test.ts index 1c83c4ef3..086e981d7 100644 --- a/packages/server/src/database/ledger-stores-keepers.test.ts +++ b/packages/server/src/database/ledger-stores-keepers.test.ts @@ -11,9 +11,9 @@ import { serveFakeParameterStore, type FakeParameterOperation } from '@on.auto/s import { Effect, Logger, Redacted, type Context } from 'effect'; import { afterAll, describe, expect, it, onTestFinished } from 'vitest'; -import { readSettings } from '../settings/settings.ts'; import { secretStoreInFile } from '../testing/records/kept-credentials.ts'; import { temporaryLedger, type TemporaryLedger } from '../testing/records/temporary-ledger.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; import { ledgerStoresOpenedBy, type LedgerStores } from './ledger-stores.ts'; const account = '111122223333'; @@ -73,7 +73,7 @@ async function openedWith( const { context, lines } = await aContextLogging(); const stores = await ledgerStoresOpenedBy( context, - readSettings({ LEDGER_FILE: fileName, ...environment }), + readSettingsOf({ LEDGER_FILE: fileName, ...environment }), { store: 'sqlite', file: fileName }, () => Promise.resolve(adaptersOn(parameterStoreUrl, waits)), ); diff --git a/packages/server/src/database/ledger-stores-kms.test.ts b/packages/server/src/database/ledger-stores-kms.test.ts index a733d1368..d462f80ca 100644 --- a/packages/server/src/database/ledger-stores-kms.test.ts +++ b/packages/server/src/database/ledger-stores-kms.test.ts @@ -5,9 +5,9 @@ import { serveFakeKms, type FakeKeyState } from '@on.auto/secrets/testing'; import { Effect, Logger, Redacted, type Context } from 'effect'; import { afterAll, describe, expect, it, onTestFinished } from 'vitest'; -import { readSettings } from '../settings/settings.ts'; import { secretStoreInFile } from '../testing/records/kept-credentials.ts'; import { temporaryLedger } from '../testing/records/temporary-ledger.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; import { ledgerStoresOpenedBy } from './ledger-stores.ts'; const account = '111122223333'; @@ -66,7 +66,7 @@ async function openedWith(fileName: string, environment: Readonly Promise.resolve(adaptersWithin(waits)), ); @@ -78,7 +78,7 @@ async function storesOf(fileName: string, environment: Readonly Promise.resolve(onTheFake), ); diff --git a/packages/server/src/database/ledger-stores-rotation.test.ts b/packages/server/src/database/ledger-stores-rotation.test.ts index f4b4f7db5..ebd2eb173 100644 --- a/packages/server/src/database/ledger-stores-rotation.test.ts +++ b/packages/server/src/database/ledger-stores-rotation.test.ts @@ -3,8 +3,8 @@ import { randomUUIDv7, sha256Of } from '@on.auto/operations'; import { Effect } from 'effect'; import { describe, expect, it, onTestFinished } from 'vitest'; -import { readSettings } from '../settings/settings.ts'; import { temporaryLedger } from '../testing/records/temporary-ledger.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; import { ledgerStoresOpenedBy } from './ledger-stores.ts'; const [previousKey, currentKey] = [Buffer.alloc(32, 1), Buffer.alloc(32, 2)]; @@ -31,7 +31,7 @@ describe('the stores of a ledger whose secrets the previous key cannot unwrap', it('stop the start, naming the setting of the previous key before the words of its custodian', async () => { const fileName = aLedgerFile(); await keptUnderThePreviousKey(fileName); - const settings = readSettings({ + const settings = readSettingsOf({ LEDGER_FILE: fileName, SECRETS_KEY: currentKey.toString('base64'), SECRETS_KEY_PREVIOUS: previousKey.toString('base64'), diff --git a/packages/server/src/database/ledger-stores.test.ts b/packages/server/src/database/ledger-stores.test.ts index b185f0d35..dba363591 100644 --- a/packages/server/src/database/ledger-stores.test.ts +++ b/packages/server/src/database/ledger-stores.test.ts @@ -5,9 +5,9 @@ import { Effect, Logger, type Context } from 'effect'; import type { Pool } from 'pg'; import { describe, expect, it } from 'vitest'; -import { readSettings } from '../settings/settings.ts'; import { secretsKey } from '../testing/databases/secrets-key.ts'; import { messagesOf } from '../testing/processes/stderr-lines.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; import { ledgerStoresOpenedBy } from './ledger-stores.ts'; async function aContextLogging(): Promise<{ readonly context: Context.Context; readonly lines: string[] }> { @@ -33,7 +33,7 @@ describe('the stores of a ledger in PostgreSQL, as the server opens them, withou return pool; }; - const stores = await ledgerStoresOpenedBy(context, readSettings({ DATABASE_URL: url, SECRETS_KEY: secretsKey }), { + const stores = await ledgerStoresOpenedBy(context, readSettingsOf({ DATABASE_URL: url, SECRETS_KEY: secretsKey }), { store: 'postgresql', pools: recordedPools, }); diff --git a/packages/server/src/database/ledger-stores.ts b/packages/server/src/database/ledger-stores.ts index 6ad88d4dd..a355ec968 100644 --- a/packages/server/src/database/ledger-stores.ts +++ b/packages/server/src/database/ledger-stores.ts @@ -2,12 +2,12 @@ import type { SecretStore } from '@on.auto/secrets'; import type { DatabaseSettings } from '@on.auto/workflow-host'; import { Effect, type Context } from 'effect'; -import type { Served } from '../lifecycle/lifecycle.ts'; import { logContentStore, logLedger } from '../logging/logging.ts'; import { loadAwsAdapters, type AwsAdaptersLoader } from '../secrets/aws-adapters.ts'; import { keepersOf } from '../secrets/secret-keepers.ts'; import { secretStoreStarted } from '../secrets/secret-store-start.ts'; import { custodiansOf } from '../secrets/secret-store.ts'; +import type { Served } from '../server-options/server-options.ts'; import type { Settings } from '../settings/settings.ts'; export interface LedgerStores { diff --git a/packages/server/src/database/secret-manager-page.test.ts b/packages/server/src/database/secret-manager-page.test.ts index 69ffc9924..2e60a3753 100644 --- a/packages/server/src/database/secret-manager-page.test.ts +++ b/packages/server/src/database/secret-manager-page.test.ts @@ -6,8 +6,8 @@ import { Effect, Logger, Schema } from 'effect'; import { afterAll, describe, expect, it, onTestFinished } from 'vitest'; import type { AwsAdapters } from '../secrets/aws-adapters.ts'; -import { readSettings } from '../settings/settings.ts'; import { temporaryLedger } from '../testing/records/temporary-ledger.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; import { ledgerStoresOpenedBy } from './ledger-stores.ts'; const page = readFileSync(new URL('../../../../docs/self-host/secret-manager.md', import.meta.url), 'utf8'); @@ -68,7 +68,7 @@ describe('the page that keeps secrets in a secret manager', () => { it('names the manager alone, then the move from the ledger and back, as the settings read them', () => { expect( environments.map((environment) => { - const { keepers, secrets } = readSettings(environment); + const { keepers, secrets } = readSettingsOf(environment); return { keepers, custodian: secrets?.current.from }; }), ).toEqual([ @@ -127,7 +127,7 @@ async function startedUnder(environment: Readonly>) { const context = await Effect.runPromise(Effect.context().pipe(Effect.provide(Logger.layer([capture])))); const started = await ledgerStoresOpenedBy( context, - readSettings({ LEDGER_FILE: fileName, ...environment }), + readSettingsOf({ LEDGER_FILE: fileName, ...environment }), { store: 'sqlite', file: fileName }, () => Promise.resolve(onTheFake), ).then( diff --git a/packages/server/src/development/env-files.test.ts b/packages/server/src/development/env-files.test.ts index 40363f370..6f1012480 100644 --- a/packages/server/src/development/env-files.test.ts +++ b/packages/server/src/development/env-files.test.ts @@ -6,7 +6,6 @@ import { parseEnv } from 'node:util'; import { providerStatus } from '@on.auto/reasoning'; import { describe, expect, it } from 'vitest'; -import { readSettings } from '../settings/settings.ts'; import { stoppedWith } from '../testing/processes/development-endings.ts'; import { developmentFiles, @@ -16,6 +15,7 @@ import { untilWritten, type DevelopmentOptions, } from '../testing/processes/development-process.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; const example = new URL('../../../../.env.example', import.meta.url); @@ -66,7 +66,7 @@ describe('local Studio under pnpm dev', () => { describe('the settings files pnpm dev reads', () => { it('run the server in local mode with dev.env alone, on a loopback address with neither PUBLIC_URL nor API_KEYS', () => { - expect(readSettings(parseEnv(readFileSync(devEnv, 'utf8')))).toMatchObject({ + expect(readSettingsOf(parseEnv(readFileSync(devEnv, 'utf8')))).toMatchObject({ host: '127.0.0.1', apiKeys: undefined, signIn: undefined, @@ -84,7 +84,7 @@ describe('the settings files pnpm dev reads', () => { it('documents the model settings in .env.example and auto-runtime.example.yaml in a form that reads as written', () => { const environment = parseEnv(uncommented(readFileSync(example, 'utf8'))); - const { models, allowedOrigins, configFile } = readSettings({ ...environment, CONFIG_FILE: exampleConfigFile }); + const { models, allowedOrigins, configFile } = readSettingsOf({ ...environment, CONFIG_FILE: exampleConfigFile }); expect(providerStatus(models, { entraId: false }).configured).toEqual(['anthropic', 'openai', 'google', 'gateway']); expect(Object.fromEntries(models.aliases)).toEqual({ 'anthropic/*': 'gateway/anthropic/*' }); @@ -95,7 +95,7 @@ describe('the settings files pnpm dev reads', () => { it('documents the ledger in PostgreSQL in .env.example in a form that reads as written over dev.env', () => { const environment = parseEnv(uncommented(readFileSync(example, 'utf8'))); - const { ledger } = readSettings({ LEDGER_FILE: '.data/ledger.db', ...environment }); + const { ledger } = readSettingsOf({ LEDGER_FILE: '.data/ledger.db', ...environment }); expect(ledger).toMatchObject({ store: 'postgresql', host: '127.0.0.1:5432', database: 'postgres' }); }); diff --git a/packages/server/src/function-settings/computation-settings.test.ts b/packages/server/src/function-settings/computation-settings.test.ts index 01d5aecca..c8d4ac60c 100644 --- a/packages/server/src/function-settings/computation-settings.test.ts +++ b/packages/server/src/function-settings/computation-settings.test.ts @@ -1,11 +1,11 @@ import type { Environment } from '@on.auto/config'; import { describe, expect, it } from 'vitest'; -import { readSettings } from '../settings/settings.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; function errorFrom(environment: Environment): unknown { try { - readSettings(environment); + readSettingsOf(environment); } catch (error) { return error; } @@ -14,11 +14,11 @@ function errorFrom(environment: Environment): unknown { describe('the computation settings', () => { it('run four computation functions at once when nothing is set', () => { - expect(readSettings({}).computation).toEqual({ workers: 4 }); + expect(readSettingsOf({}).computation).toEqual({ workers: 4 }); }); it('read how many computation functions run at once', () => { - expect(readSettings({ COMPUTATION_WORKERS: ' 12 ' }).computation).toEqual({ workers: 12 }); + expect(readSettingsOf({ COMPUTATION_WORKERS: ' 12 ' }).computation).toEqual({ workers: 12 }); }); it.each(['0', '65', 'many', '1.5'])( diff --git a/packages/server/src/function-settings/interaction-settings.test.ts b/packages/server/src/function-settings/interaction-settings.test.ts index c42849ae1..68bbb384d 100644 --- a/packages/server/src/function-settings/interaction-settings.test.ts +++ b/packages/server/src/function-settings/interaction-settings.test.ts @@ -1,13 +1,13 @@ import { describe, expect, it } from 'vitest'; -import { readSettings } from '../settings/settings.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; const openRequestsRefused = 'InvalidSettingsError: The interaction settings are invalid. INTERACTION_OPEN_REQUESTS: Expected a whole number from 1 to 1000000, such as 10000'; function errorFrom(environment: Readonly>): string { try { - readSettings(environment); + readSettingsOf(environment); } catch (error) { return String(error); } @@ -18,7 +18,7 @@ describe('the interaction function settings', () => { it('read the most open requests a playbook may hold, 10,000 unless set', () => { expect( [{}, { INTERACTION_OPEN_REQUESTS: '500' }, { INTERACTION_OPEN_REQUESTS: '1000000' }].map( - (environment: Readonly>) => readSettings(environment).interaction.mostOpenRequests, + (environment: Readonly>) => readSettingsOf(environment).interaction.mostOpenRequests, ), ).toEqual([10_000, 500, 1_000_000]); }); @@ -28,8 +28,8 @@ describe('the interaction function settings', () => { }); it('start as if neither were set when the environment still names CHANNELS or PUBLIC_ORIGIN', () => { - expect(readSettings({ CHANNELS: '{ not even JSON', PUBLIC_ORIGIN: 'nowhere' }).interaction).toEqual( - readSettings({}).interaction, + expect(readSettingsOf({ CHANNELS: '{ not even JSON', PUBLIC_ORIGIN: 'nowhere' }).interaction).toEqual( + readSettingsOf({}).interaction, ); }); }); diff --git a/packages/server/src/function-settings/prediction-settings.test.ts b/packages/server/src/function-settings/prediction-settings.test.ts index 57bc89020..49db5cedd 100644 --- a/packages/server/src/function-settings/prediction-settings.test.ts +++ b/packages/server/src/function-settings/prediction-settings.test.ts @@ -3,7 +3,7 @@ import { existsSync, readFileSync } from 'node:fs'; import type { Environment } from '@on.auto/config'; import { describe, expect, it } from 'vitest'; -import { readSettings } from '../settings/settings.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; const mebibyte = 1_048_576; @@ -45,7 +45,7 @@ function proofIn(file: string): Proof { function errorFrom(environment: Environment): unknown { try { - readSettings(environment); + readSettingsOf(environment); } catch (error) { return error; } @@ -54,7 +54,7 @@ function errorFrom(environment: Environment): unknown { describe('the prediction function settings', () => { it('keep 4 model processes of 1,280 MiB each, 2 of them for the builds being scored, 4 builds a function a day, 32 active functions a playbook, and the files of the models under /var/tmp/auto-runtime, when nothing is set', () => { - expect(readSettings({}).prediction).toEqual({ + expect(readSettingsOf({}).prediction).toEqual({ processes: 4, processMemoryMebibytes: 1280, buildsAtOnce: 2, @@ -66,8 +66,8 @@ describe('the prediction function settings', () => { it('keep the files of the models in the absolute folder PREDICTION_CACHE_DIR names, and refuse a relative one', () => { expect([ - readSettings({ PREDICTION_CACHE_DIR: ' /srv/models ' }).prediction.cacheDirectory, - readSettings({ PREDICTION_CACHE_DIR: '' }).prediction.cacheDirectory, + readSettingsOf({ PREDICTION_CACHE_DIR: ' /srv/models ' }).prediction.cacheDirectory, + readSettingsOf({ PREDICTION_CACHE_DIR: '' }).prediction.cacheDirectory, String(errorFrom({ PREDICTION_CACHE_DIR: 'models' })), ]).toEqual([ '/srv/models', @@ -78,8 +78,8 @@ describe('the prediction function settings', () => { it('look for prlimit in the PATH the server was started with, and nowhere without one', () => { expect([ - readSettings({ PATH: '/usr/local/bin:/usr/bin' }).prediction.path, - readSettings({}).prediction.path, + readSettingsOf({ PATH: '/usr/local/bin:/usr/bin' }).prediction.path, + readSettingsOf({}).prediction.path, ]).toEqual(['/usr/local/bin:/usr/bin', undefined]); }); }); @@ -87,7 +87,7 @@ describe('the prediction function settings', () => { describe('the prediction function settings given values', () => { it('read each of the five', () => { expect( - readSettings({ + readSettingsOf({ PREDICTION_PROCESSES: ' 8 ', PREDICTION_PROCESS_MEMORY: '1024', PREDICTION_BUILDS_AT_ONCE: '3', @@ -144,7 +144,7 @@ describe('the least memory of a model process', () => { true, true, ]); - expect(readSettings({ PREDICTION_PROCESS_MEMORY: String(floor) }).prediction.processMemoryMebibytes).toBe(floor); + expect(readSettingsOf({ PREDICTION_PROCESS_MEMORY: String(floor) }).prediction.processMemoryMebibytes).toBe(floor); expect(String(errorFrom({ PREDICTION_PROCESS_MEMORY: String(floor - 1) }))).toBe( `InvalidSettingsError: The prediction function settings are invalid. PREDICTION_PROCESS_MEMORY: Expected a whole number from ${floor} to 16384, such as 1280`, ); diff --git a/packages/server/src/function-settings/providers-named.test.ts b/packages/server/src/function-settings/providers-named.test.ts index 8b6f421d5..c376818fa 100644 --- a/packages/server/src/function-settings/providers-named.test.ts +++ b/packages/server/src/function-settings/providers-named.test.ts @@ -7,8 +7,8 @@ import type { ServerSettings } from '@on.auto/tool-servers'; import { Redacted } from 'effect'; import { describe, expect, it, onTestFinished } from 'vitest'; -import { readSettings } from '../settings/settings.ts'; import { required, shippedProviders } from '../testing/servers/connected-servers.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; const githubToken = 'github-token-0f3c9a71e2'; @@ -83,13 +83,13 @@ function configFile(text: string): string { } function settingsOf(servers: string, environment: Readonly> = secrets) { - return readSettings({ CONFIG_FILE: configFile(`${providersOfTheFile}${servers}`), ...environment }); + return readSettingsOf({ CONFIG_FILE: configFile(`${providersOfTheFile}${servers}`), ...environment }); } function errorOf(servers: string, environment: Readonly> = secrets): string { const path = configFile(`${providersOfTheFile}${servers}`); try { - readSettings({ CONFIG_FILE: path, ...environment }); + readSettingsOf({ CONFIG_FILE: path, ...environment }); } catch (error) { return String(error).replaceAll(path, 'auto-runtime.yaml'); } @@ -198,7 +198,7 @@ tool_servers: `; const path = configFile(keyed); - expect(readSettings({ CONFIG_FILE: path }).toolServers.servers).toMatchObject([ + expect(readSettingsOf({ CONFIG_FILE: path }).toolServers.servers).toMatchObject([ { name: 'store', url: 'https://acme.keyed.example.com/mcp', provider: 'keyed' }, ]); }); diff --git a/packages/server/src/function-settings/reasoning-settings.test.ts b/packages/server/src/function-settings/reasoning-settings.test.ts index 05fac07db..6f12bf3bf 100644 --- a/packages/server/src/function-settings/reasoning-settings.test.ts +++ b/packages/server/src/function-settings/reasoning-settings.test.ts @@ -1,11 +1,11 @@ import type { Environment } from '@on.auto/config'; import { describe, expect, it } from 'vitest'; -import { readSettings } from '../settings/settings.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; function errorFrom(environment: Environment): unknown { try { - readSettings(environment); + readSettingsOf(environment); } catch (error) { return error; } @@ -14,7 +14,7 @@ function errorFrom(environment: Environment): unknown { describe('the reasoning function settings', () => { it('let the model calls of a run add up to 2,000,000 input tokens when nothing is set, or the setting is empty', () => { - expect([readSettings({}).reasoning, readSettings({ REASONING_MAX_INPUT_TOKENS: '' }).reasoning]).toEqual([ + expect([readSettingsOf({}).reasoning, readSettingsOf({ REASONING_MAX_INPUT_TOKENS: '' }).reasoning]).toEqual([ { mostInputTokens: 2_000_000 }, { mostInputTokens: 2_000_000 }, ]); @@ -23,7 +23,7 @@ describe('the reasoning function settings', () => { it('read the most input tokens of a run, from the least to the most a whole number of seven digits allows', () => { expect( ['10000', ' 500000 ', '9999999'].map( - (tokens) => readSettings({ REASONING_MAX_INPUT_TOKENS: tokens }).reasoning.mostInputTokens, + (tokens) => readSettingsOf({ REASONING_MAX_INPUT_TOKENS: tokens }).reasoning.mostInputTokens, ), ).toEqual([10_000, 500_000, 9_999_999]); }); diff --git a/packages/server/src/function-settings/reasoning-settings.ts b/packages/server/src/function-settings/reasoning-settings.ts index b828c006c..860048de5 100644 --- a/packages/server/src/function-settings/reasoning-settings.ts +++ b/packages/server/src/function-settings/reasoning-settings.ts @@ -86,7 +86,7 @@ export function readReasoningSettings( readToolServerSettings(environment, { modelProviders, providers, - ...(file === undefined ? {} : { documentsDirectory: dirname(file.path) }), + ...(file === undefined ? {} : { documents: { kind: 'directory', path: dirname(file.path) } }), }).pipe(Effect.mapError((invalid: Problems) => serversPlaced(invalid, file))), ); const reasoning = Effect.runSync(countsIn(environment, { settings: 'reasoning function settings', bounds })); diff --git a/packages/server/src/function-settings/recall-settings.test.ts b/packages/server/src/function-settings/recall-settings.test.ts index 66774a534..dd401a31b 100644 --- a/packages/server/src/function-settings/recall-settings.test.ts +++ b/packages/server/src/function-settings/recall-settings.test.ts @@ -1,11 +1,11 @@ import type { Environment } from '@on.auto/config'; import { describe, expect, it } from 'vitest'; -import { readSettings } from '../settings/settings.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; function errorFrom(environment: Environment): unknown { try { - readSettings(environment); + readSettingsOf(environment); } catch (error) { return error; } @@ -14,12 +14,16 @@ function errorFrom(environment: Environment): unknown { describe('the recall function settings', () => { it('keep at most 32 recall functions a playbook, build 4 views of a playbook at once and follow 4 playbooks at once when nothing is set', () => { - expect(readSettings({}).recall).toEqual({ mostFunctions: 32, rebuildsAtOnce: 4, playbooksAtOnce: 4 }); + expect(readSettingsOf({}).recall).toEqual({ + mostFunctions: 32, + rebuildsAtOnce: 4, + playbooksAtOnce: 4, + }); }); it('read each of the three', () => { expect( - readSettings({ + readSettingsOf({ RECALL_MAX_FUNCTIONS: ' 100 ', RECALL_MAX_REBUILDS: '2', RECALL_PLAYBOOKS_AT_ONCE: '8', diff --git a/packages/server/src/function-settings/tool-server-settings.test.ts b/packages/server/src/function-settings/tool-server-settings.test.ts index ec931a78c..29edf5e48 100644 --- a/packages/server/src/function-settings/tool-server-settings.test.ts +++ b/packages/server/src/function-settings/tool-server-settings.test.ts @@ -5,7 +5,7 @@ import { dirname, join } from 'node:path'; import { Redacted } from 'effect'; import { describe, expect, it, onTestFinished } from 'vitest'; -import { readSettings } from '../settings/settings.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; const apiKey = 'graph-api-key-4f1d9a7c2b'; @@ -17,7 +17,7 @@ const graph = { function errorFrom(environment: Readonly>): string { try { - readSettings(environment); + readSettingsOf(environment); } catch (error) { return String(error); } @@ -34,7 +34,7 @@ function configFile(text: string): string { return path; } -function authorizationOf(settings: ReturnType): string | undefined { +function authorizationOf(settings: ReturnType): string | undefined { const [server] = settings.toolServers.servers; const header = server?.type === 'mcp' ? server.headers.get('Authorization') : undefined; return header === undefined ? undefined : Redacted.value(header); @@ -42,7 +42,7 @@ function authorizationOf(settings: ReturnType): string | un describe('the tool servers of the server', () => { it('reads the servers, each with the tools it allows, from the environment', () => { - const settings = readSettings({ + const settings = readSettingsOf({ TOOL_SERVERS: JSON.stringify({ graph: { ...graph, allowed: ['search', 'execute'] } }), GRAPH_API_KEY: apiKey, }); @@ -59,7 +59,7 @@ describe('the tool servers of the server', () => { 'tool_servers:\n graph:\n url: https://graph.example.com/mcp\n org: acme\n headers:\n Authorization: Bearer ${GRAPH_API_KEY}\n', ); - expect(authorizationOf(readSettings({ CONFIG_FILE: path, GRAPH_API_KEY: apiKey }))).toBe(`Bearer ${apiKey}`); + expect(authorizationOf(readSettingsOf({ CONFIG_FILE: path, GRAPH_API_KEY: apiKey }))).toBe(`Bearer ${apiKey}`); }); }); @@ -109,7 +109,10 @@ describe('the tools of a tool server in the configuration file', () => { ); expect( - readSettings({ CONFIG_FILE: path }).toolServers.servers.map(({ name, allowed }) => ({ name, allowed })), + readSettingsOf({ CONFIG_FILE: path }).toolServers.servers.map(({ name, allowed }) => ({ + name, + allowed, + })), ).toEqual([ { name: 'graph', allowed: ['search', 'execute'] }, { name: 'notes', allowed: null }, @@ -143,7 +146,7 @@ describe('the document of an openapi server named by a path in the configuration it('is read from the directory of the configuration file', () => { const path = configFile('tool_servers:\n billing:\n document: apis/billing.json\n org: acme\n'); - expect(readSettings({ CONFIG_FILE: path }).toolServers.servers).toMatchObject([ + expect(readSettingsOf({ CONFIG_FILE: path }).toolServers.servers).toMatchObject([ { name: 'billing', type: 'openapi', document: { kind: 'file', path: join(dirname(path), 'apis/billing.json') } }, ]); }); diff --git a/packages/server/src/guides/recipes.ts b/packages/server/src/guides/recipes.ts index 3832ba178..a93e80785 100644 --- a/packages/server/src/guides/recipes.ts +++ b/packages/server/src/guides/recipes.ts @@ -1,6 +1,5 @@ -import { readFileSync } from 'node:fs'; - import type { Guide, Recipe } from '@on.auto/api'; +import type { ShippedFiles } from '@on.auto/operations'; type RecipeOutline = Omit; @@ -85,15 +84,21 @@ const whereInteractionIsServed: Readonly> = { }, }; -function textOf({ name }: RecipeOutline, carried: ReadonlySet): string { - const text = readFileSync(new URL(`recipes/${name}.md`, import.meta.url), 'utf8'); +async function textOf( + shippedFiles: ShippedFiles, + { name }: RecipeOutline, + carried: ReadonlySet, +): Promise { + const text = await shippedFiles.text(new URL(`recipes/${name}.md`, import.meta.url)); const insertion = carried.has(interactionGuide) ? whereInteractionIsServed[name] : undefined; return insertion === undefined ? text : text.replace(insertion.after, `${insertion.after}${insertion.words}`); } -export function recipesFor(guides: readonly Guide[]): readonly Recipe[] { +export function recipesFor(shippedFiles: ShippedFiles, guides: readonly Guide[]): Promise { const carried = new Set(guides.map(({ name }) => name)); - return outlines - .filter(({ formatGuide }) => carried.has(formatGuide)) - .map((outline) => Object.assign({ text: textOf(outline, carried) }, outline)); + return Promise.all( + outlines + .filter(({ formatGuide }) => carried.has(formatGuide)) + .map(async (outline) => Object.assign({ text: await textOf(shippedFiles, outline, carried) }, outline)), + ); } diff --git a/packages/server/src/guides/served-guides.test.ts b/packages/server/src/guides/served-guides.test.ts index 34e7d8989..f599d5b55 100644 --- a/packages/server/src/guides/served-guides.test.ts +++ b/packages/server/src/guides/served-guides.test.ts @@ -3,11 +3,12 @@ import { readFileSync } from 'node:fs'; import { mostGuideBytes, mostRecipeBytes } from '@on.auto/api'; import { defineCapability, type Capability, type CapabilityGuide } from '@on.auto/definitions'; +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { Effect } from 'effect'; import { describe, expect, it } from 'vitest'; import { withLinksResolved } from './page-links.ts'; -import { servedGuidesOf } from './served-guides.ts'; +import { servedGuidesOf, type ServedGuides } from './served-guides.ts'; import { withoutSiteMarkup } from './site-markup.ts'; function capabilityOf(name: string, noun: string, guide: CapabilityGuide): Capability { @@ -40,8 +41,14 @@ const everyType = [reasoning, computation, recall, prediction, workflow]; const interaction = capabilityOf('interaction', 'interaction function', { name: 'interaction-function' }); -function recipeTextsOf(capabilities: readonly Capability[]): Readonly> { - return Object.fromEntries(servedGuidesOf(capabilities).recipes.map(({ name, text }) => [name, text])); +const served = await servedGuidesOf(shippedFilesOnDisk, everyType); + +const servedWithInteraction = await servedGuidesOf(shippedFilesOnDisk, [...everyType, interaction]); + +const servedForReasoningAndWorkflows = await servedGuidesOf(shippedFilesOnDisk, [reasoning, workflow]); + +function recipeTextsOf({ recipes }: ServedGuides): Readonly> { + return Object.fromEntries(recipes.map(({ name, text }) => [name, text])); } function bytesOf(texts: Readonly>, ...names: readonly string[]): readonly number[] { @@ -68,7 +75,7 @@ const pages: Readonly> = { }; describe('the guides of a server that runs every type of definition', () => { - const { definitionTypes, guides, recipes } = servedGuidesOf(everyType); + const { definitionTypes, guides, recipes } = served; it('are the terminology, a guide per type and the four recipes, ten in all', () => { expect([...guides, ...recipes].map(({ name }) => name)).toEqual([ @@ -106,7 +113,7 @@ describe('the guides of a server that runs every type of definition', () => { }); describe('the titles and links of the guides of a server that runs every type of definition', () => { - const { guides } = servedGuidesOf(everyType); + const { guides } = served; it('title each type guide as its page is titled', () => { expect(guides.slice(1).map(({ title }) => title)).toEqual([ @@ -125,7 +132,7 @@ describe('the titles and links of the guides of a server that runs every type of describe('the text of the guides of a server', () => { it('holds none of the markup the documentation site needs', () => { - const { guides } = servedGuidesOf(everyType); + const { guides } = served; expect( guides.filter(({ text }) => /^
|^<\/div>$|^$/mu.test(outsideCode(text))), @@ -134,10 +141,10 @@ describe('the text of the guides of a server', () => { }); describe('the size and the words of the guides of a server', () => { - const { guides, recipes } = servedGuidesOf(everyType); + const { guides, recipes } = served; it('stay under 64 KiB each, the guide of interaction functions among them, and each recipe under 4 KiB', () => { - const everyGuide = servedGuidesOf([...everyType, interaction]).guides; + const everyGuide = servedWithInteraction.guides; expect(everyGuide.map(({ name }) => name)).toContain('interaction-function'); expect(everyGuide.filter(({ text }) => Buffer.byteLength(text, 'utf8') > mostGuideBytes)).toEqual([]); @@ -157,16 +164,19 @@ describe('the size and the words of the guides of a server', () => { describe('the recipes of a server', () => { it('leave out a recipe whose format guide the server does not carry', () => { - expect(servedGuidesOf([reasoning, workflow]).recipes.map(({ name }) => name)).toEqual([ + expect(servedForReasoningAndWorkflows.recipes.map(({ name }) => name)).toEqual([ 'first-playbook', 'give-tools', 'schedule', ]); - expect(servedGuidesOf([]).recipes).toEqual([]); + }); + + it('are none for a server that runs no type of definition', async () => { + expect((await servedGuidesOf(shippedFilesOnDisk, [])).recipes).toEqual([]); }); it('ask the person in their own words, filled into what the prompt asks for', () => { - const asked = Object.fromEntries(servedGuidesOf(everyType).recipes.map(({ name, request }) => [name, request])); + const asked = Object.fromEntries(served.recipes.map(({ name, request }) => [name, request])); expect([ asked['first-playbook']?.({}), @@ -186,7 +196,7 @@ describe('the recipes of a server', () => { describe('the recipes that give a playbook tools', () => { it('call a tool to learn what it answers, and never make a function to look, within the bound of a recipe', () => { - const texts = Object.fromEntries(servedGuidesOf(everyType).recipes.map(({ name, text }) => [name, text])); + const texts = recipeTextsOf(served); expect(String(texts['give-tools'])).toContain( "4. To learn what a tool answers, call it with execute, with the arguments its input_schema takes, and read its `text`: that is what the function's model will see. Never make a function to look. A tool whose `read_only` is false may change something, so call it only once the person agrees. When a prompt needs an id, such as a channel's, call the tool that lists them and take the id from its answer, confirming the choice with the person.", @@ -206,7 +216,7 @@ describe('the recipes that give a playbook tools', () => { describe('the recipes of a server that serves interaction functions', () => { it('send a message a person answers to an interaction function, words a server without them does not serve', () => { - const texts = recipeTextsOf([...everyType, interaction]); + const texts = recipeTextsOf(servedWithInteraction); expect(texts['give-tools']).toContain( '5. Ask the person what the function should do with the tools, and which of them it needs. When the playbook only needs what a tool answers, with nothing to reason about, write an interaction function that calls the tool instead: it answers at once with what the tool answered and spends no model call, and the interaction-function guide says how to write it from the answer execute gave. When the playbook should send a person a message through a tool and take their answer, write an interaction function instead: the interaction-function guide says how it names the tool it sends through and the tool it reads replies with, each called the same way.\n', @@ -216,7 +226,7 @@ describe('the recipes of a server that serves interaction functions', () => { ); expect([ bytesOf(texts, 'first-playbook', 'remember', 'give-tools', 'schedule'), - bytesOf(recipeTextsOf(everyType), 'first-playbook', 'remember', 'give-tools', 'schedule'), + bytesOf(recipeTextsOf(served), 'first-playbook', 'remember', 'give-tools', 'schedule'), ]).toEqual([ [1988, 2327, 3277, 1294], [1898, 2327, 2716, 1294], @@ -224,7 +234,7 @@ describe('the recipes of a server that serves interaction functions', () => { }); it('name the same tools as on a server without them', () => { - const { recipes } = servedGuidesOf([...everyType, interaction]); + const { recipes } = servedWithInteraction; expect(recipes.map(({ name, text }) => [name, toolsNamedIn(text).toSorted()])).toEqual( recipes.map(({ name, calls }) => [name, calls.toSorted()]), @@ -246,7 +256,7 @@ function toolsNamedIn(text: string): readonly string[] { describe('the tools a recipe calls', () => { it('are every tool its steps name, but the guide tool, listed on every connection, list_models and add_connection, called only where they are listed, and the fields of a tool they name', () => { - const { recipes } = servedGuidesOf(everyType); + const { recipes } = served; expect(recipes.map(({ name, text }) => [name, toolsNamedIn(text).toSorted()])).toEqual( recipes.map(({ name, calls }) => [name, calls.toSorted()]), @@ -260,7 +270,7 @@ describe('the tools a recipe calls', () => { describe('the terminology guide of a server', () => { it('names only the types of definition the server runs', () => { - const [terminology] = servedGuidesOf([reasoning, workflow]).guides; + const [terminology] = servedForReasoningAndWorkflows.guides; expect(terminology?.text).toContain('- Reasoning: Reasoning function.'); expect(terminology?.text).toContain('- Coordination: Workflow.'); @@ -273,9 +283,9 @@ describe('the terminology guide of a server', () => { }); describe('a server without the page of a type it runs', () => { - it('fails to start', () => { - expect(() => servedGuidesOf([capabilityOf('drafting', 'draft', { name: 'drafting-function' })])).toThrow( - /no such file or directory.*drafting-format\.md/u, - ); + it('fails to start', async () => { + await expect( + servedGuidesOf(shippedFilesOnDisk, [capabilityOf('drafting', 'draft', { name: 'drafting-function' })]), + ).rejects.toThrow(/no such file or directory.*drafting-format\.md/u); }); }); diff --git a/packages/server/src/guides/served-guides.ts b/packages/server/src/guides/served-guides.ts index 7245a23b2..33870653a 100644 --- a/packages/server/src/guides/served-guides.ts +++ b/packages/server/src/guides/served-guides.ts @@ -1,8 +1,6 @@ -import { readFileSync } from 'node:fs'; - import type { DefinitionType, Guide, Recipe } from '@on.auto/api'; import type { Capability } from '@on.auto/definitions'; -import { capitalized } from '@on.auto/operations'; +import { capitalized, type ShippedFiles } from '@on.auto/operations'; import { withLinksResolved } from './page-links.ts'; import { recipesFor } from './recipes.ts'; @@ -23,13 +21,16 @@ function pageOfGuide(guide: string): string { return `reference/${guide.replace(/-function$/u, '')}-format.md`; } -function pageText(page: string): string { - return readFileSync(new URL(page, documentation), 'utf8'); +function pageText(shippedFiles: ShippedFiles, page: string): Promise { + return shippedFiles.text(new URL(page, documentation)); } -function typeGuideOf({ noun, guide }: Pick): Guide { +async function typeGuideOf( + shippedFiles: ShippedFiles, + { noun, guide }: Pick, +): Promise { const page = pageOfGuide(guide.name); - const text = withLinksResolved(withoutSiteMarkup(pageText(page)), page); + const text = withLinksResolved(withoutSiteMarkup(await pageText(shippedFiles, page)), page); return { name: guide.name, title: `${capitalized(noun.one)} format`, @@ -38,7 +39,10 @@ function typeGuideOf({ noun, guide }: Pick): Guide }; } -export function servedGuidesOf(capabilities: readonly Capability[]): ServedGuides { +export async function servedGuidesOf( + shippedFiles: ShippedFiles, + capabilities: readonly Capability[], +): Promise { const definitionTypes: readonly DefinitionType[] = capabilities.map(({ type, noun, guide }) => ({ type, noun: noun.one, @@ -46,10 +50,10 @@ export function servedGuidesOf(capabilities: readonly Capability[]): ServedGuide })); const guides = [ terminologyGuideOf( - pageText(terminologyPage), + await pageText(shippedFiles, terminologyPage), definitionTypes.map(({ noun }) => noun), ), - ...capabilities.map(({ noun, guide }) => typeGuideOf({ noun, guide })), + ...(await Promise.all(capabilities.map(({ noun, guide }) => typeGuideOf(shippedFiles, { noun, guide })))), ]; - return { definitionTypes, guides, recipes: recipesFor(guides) }; + return { definitionTypes, guides, recipes: await recipesFor(shippedFiles, guides) }; } diff --git a/packages/server/src/lifecycle/browser-requests.test.ts b/packages/server/src/lifecycle/browser-requests.test.ts new file mode 100644 index 000000000..1ed36876e --- /dev/null +++ b/packages/server/src/lifecycle/browser-requests.test.ts @@ -0,0 +1,54 @@ +import { get } from 'node:http'; + +import { createApiKey } from '@on.auto/identity'; +import { describe, expect, it } from 'vitest'; + +import { defaultServerOptions } from '../testing/entries/serving-options.ts'; +import { startServer } from './lifecycle.ts'; + +const loopback = { HOST: '127.0.0.1', PORT: '0' }; + +function statusOf(port: number, path: string, headers: Readonly>): Promise { + return new Promise((resolve, reject) => { + get({ host: '127.0.0.1', port, path, headers }, (response) => { + response.resume(); + resolve(response.statusCode); + }).on('error', reject); + }); +} + +describe('startServer validates requests from browsers', () => { + it('is in local mode on loopback without API keys, so it rejects a Host header that is not localhost', async () => { + const server = await startServer(loopback, defaultServerOptions); + + const foreign = await statusOf(server.port, '/nowhere', { host: 'evil.example' }); + const local = await statusOf(server.port, '/nowhere', { host: `localhost:${server.port}` }); + await server.stop(); + + expect({ foreign, local }).toEqual({ foreign: 403, local: 404 }); + }); + + it('is not in local mode once API keys are configured, so it checks the key and not the Host header', async () => { + const { key, entry } = createApiKey({ id: 'ci-1', org: 'acme', permissions: ['org:read'], playbooks: '*' }); + const server = await startServer({ ...loopback, API_KEYS: JSON.stringify([entry]) }, defaultServerOptions); + + const withoutKey = await statusOf(server.port, '/v1/orgs/acme/playbooks', { host: 'evil.example' }); + const withKey = await statusOf(server.port, '/v1/orgs/acme/playbooks', { + host: 'evil.example', + authorization: `Bearer ${key}`, + }); + await server.stop(); + + expect({ withoutKey, withKey }).toEqual({ withoutKey: 401, withKey: 404 }); + }); + + it('rejects a request from an origin ALLOWED_ORIGINS does not list', async () => { + const server = await startServer({ ...loopback, ALLOWED_ORIGINS: 'https://app.example.com' }, defaultServerOptions); + + const allowed = await statusOf(server.port, '/nowhere', { origin: 'https://app.example.com' }); + const foreign = await statusOf(server.port, '/nowhere', { origin: 'https://evil.example' }); + await server.stop(); + + expect({ allowed, foreign }).toEqual({ allowed: 404, foreign: 403 }); + }); +}); diff --git a/packages/server/src/lifecycle/composed-stop.test.ts b/packages/server/src/lifecycle/composed-stop.test.ts index e2b9c462b..eaabcbbeb 100644 --- a/packages/server/src/lifecycle/composed-stop.test.ts +++ b/packages/server/src/lifecycle/composed-stop.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, onTestFinished } from 'vitest'; import { compositionRoot } from '../composition/composition-root.ts'; +import { type RunningServer } from '../server-options/server-options.ts'; import { isAcceptingConnections } from '../testing/processes/accepting-connections.ts'; import { gatewayThatHangsFirst, @@ -12,7 +13,7 @@ import { temporaryLedger } from '../testing/records/temporary-ledger.ts'; import { runIdIn, workflowSource } from '../testing/servers/workflow-server.ts'; import { noConsentThroughAuto } from '../testing/settings/no-consent-through-auto.ts'; import { longerThanTheTestMs } from '../testing/waiting/until.ts'; -import { startServer, type RunningServer } from './lifecycle.ts'; +import { startServer } from './lifecycle.ts'; const greeting = workflowSource('greeting', "do:\n - greet: { set: { greeting: '${ `Hello, ${$data.name}` }' } }\n"); diff --git a/packages/server/src/lifecycle/http-server.test.ts b/packages/server/src/lifecycle/http-server.test.ts index 0f6b7486f..8168e0ff6 100644 --- a/packages/server/src/lifecycle/http-server.test.ts +++ b/packages/server/src/lifecycle/http-server.test.ts @@ -1,7 +1,9 @@ import { afterEach, describe, expect, it } from 'vitest'; +import { type RunningServer } from '../server-options/server-options.ts'; +import { defaultServerOptions } from '../testing/entries/serving-options.ts'; import { ownOriginOf, tcpPort } from './http-server.ts'; -import { startServer, defaultServerOptions, type RunningServer } from './lifecycle.ts'; +import { startServer } from './lifecycle.ts'; describe('HTTP server', () => { let server: RunningServer | undefined; diff --git a/packages/server/src/lifecycle/lifecycle.test.ts b/packages/server/src/lifecycle/lifecycle.test.ts index fc79750eb..b2c8b8599 100644 --- a/packages/server/src/lifecycle/lifecycle.test.ts +++ b/packages/server/src/lifecycle/lifecycle.test.ts @@ -1,15 +1,13 @@ -import { get } from 'node:http'; - import type { RegisterRoutes } from '@on.auto/api'; -import { createApiKey } from '@on.auto/identity'; import { Effect, Layer } from 'effect'; import { describe, expect, it } from 'vitest'; -import { optionsComposing, optionsServing } from '../testing/entries/serving-options.ts'; +import { servedBy, type Served, type ServerOptions } from '../server-options/server-options.ts'; +import { optionsComposing, optionsServing, defaultServerOptions } from '../testing/entries/serving-options.ts'; import { heldRequests } from '../testing/entries/test-routes.ts'; import { isAcceptingConnections } from '../testing/processes/accepting-connections.ts'; import { longerThanTheTestMs } from '../testing/waiting/until.ts'; -import { startServer, defaultServerOptions, servedBy, type Served, type ServerOptions } from './lifecycle.ts'; +import { startServer } from './lifecycle.ts'; const loopback = { HOST: '127.0.0.1', PORT: '0' }; @@ -50,15 +48,6 @@ function recordingWork(record: (event: string) => void): Served { }; } -function statusOf(port: number, path: string, headers: Readonly>): Promise { - return new Promise((resolve, reject) => { - get({ host: '127.0.0.1', port, path, headers }, (response) => { - response.resume(); - resolve(response.statusCode); - }).on('error', reject); - }); -} - describe('startServer', () => { it('listens on the port the operating system assigns for port 0', async () => { const server = await startServer({ HOST: '127.0.0.1', PORT: '0' }, defaultServerOptions); @@ -165,42 +154,6 @@ describe('the work a started server does besides answering requests', () => { }); }); -describe('startServer validates requests from browsers', () => { - it('is in local mode on loopback without API keys, so it rejects a Host header that is not localhost', async () => { - const server = await startServer(loopback, defaultServerOptions); - - const foreign = await statusOf(server.port, '/nowhere', { host: 'evil.example' }); - const local = await statusOf(server.port, '/nowhere', { host: `localhost:${server.port}` }); - await server.stop(); - - expect({ foreign, local }).toEqual({ foreign: 403, local: 404 }); - }); - - it('is not in local mode once API keys are configured, so it checks the key and not the Host header', async () => { - const { key, entry } = createApiKey({ id: 'ci-1', org: 'acme', permissions: ['org:read'], playbooks: '*' }); - const server = await startServer({ ...loopback, API_KEYS: JSON.stringify([entry]) }, defaultServerOptions); - - const withoutKey = await statusOf(server.port, '/v1/orgs/acme/playbooks', { host: 'evil.example' }); - const withKey = await statusOf(server.port, '/v1/orgs/acme/playbooks', { - host: 'evil.example', - authorization: `Bearer ${key}`, - }); - await server.stop(); - - expect({ withoutKey, withKey }).toEqual({ withoutKey: 401, withKey: 404 }); - }); - - it('rejects a request from an origin ALLOWED_ORIGINS does not list', async () => { - const server = await startServer({ ...loopback, ALLOWED_ORIGINS: 'https://app.example.com' }, defaultServerOptions); - - const allowed = await statusOf(server.port, '/nowhere', { origin: 'https://app.example.com' }); - const foreign = await statusOf(server.port, '/nowhere', { origin: 'https://evil.example' }); - await server.stop(); - - expect({ allowed, foreign }).toEqual({ allowed: 404, foreign: 403 }); - }); -}); - describe('startServer with routes', () => { it('serves the routes it is given', async () => { const held = heldRequests(); diff --git a/packages/server/src/lifecycle/lifecycle.ts b/packages/server/src/lifecycle/lifecycle.ts index c8d3ad78d..e6cd2f502 100644 --- a/packages/server/src/lifecycle/lifecycle.ts +++ b/packages/server/src/lifecycle/lifecycle.ts @@ -1,238 +1,19 @@ -import { - createApiHandler, - makeAppRuntime, - type ApiHandler, - type RegisterRoutes, - type AppRuntime, - type RuntimeControl, -} from '@on.auto/api'; +import { readPageStyle } from '@on.auto/api'; import type { Environment } from '@on.auto/config'; -import { - authenticatorFor, - identityServiceOf, - type AccessMode, - type Authenticator, - type SignInSettings, - type StudioSettings, -} from '@on.auto/identity'; -import { Data, Effect, Function, Layer } from 'effect'; +import { authenticatorFor, identityServiceOf } from '@on.auto/identity'; -import { - logAccessMode, - logConfigFile, - logIncident, - logServedStudio, - logStudioKeptInMemory, - logTelemetry, - serverLogs, - type LogSettings, -} from '../logging/logging.ts'; -import { readSettings, type Settings } from '../settings/settings.ts'; -import { servedStudioOf } from '../studio/served-studio.ts'; -import { startTelemetry, type Telemetry } from '../telemetry/start-telemetry.ts'; -import { createHttpServer, listen, ownOriginOf, tcpPort } from './http-server.ts'; -import { exitDeadlineMs, shutDown, shutdownTimeoutMs } from './shutdown.ts'; - -class StartupError extends Data.TaggedError('StartupError')<{ - readonly message: string; - readonly cause: unknown; -}> {} - -export interface Install { - readonly id: string; - readonly isNew: boolean; -} - -export interface StartPing { - readonly installId: () => string | undefined; - readonly sent: () => Install | undefined; - readonly stop: () => void; -} - -export type UsagePing = (settings: Pick) => StartPing | undefined; - -export interface RunningServer { - readonly port: number; - readonly mode: AccessMode; - readonly signIn: SignInSettings | undefined; - readonly studio: StudioSettings; - readonly install: Install | undefined; - stop(): Promise; -} - -export interface Served { - readonly routes: readonly RegisterRoutes[]; - readonly stopWork: () => Promise; -} - -export type ServedClosing = (served: Served) => Served; - -export interface Composed { - readonly services: Layer.Layer; - readonly serve: (runtime: AppRuntime, ownOrigin: () => string) => Served | Promise; -} - -export interface LocalServing { - readonly runtimeControl?: RuntimeControl; - readonly providerSetup: string; - readonly appended?: NonNullable; -} - -export interface ServerOptions { - readonly composed: (settings: Settings, telemetry: Telemetry) => Composed; - readonly shutdownTimeoutMs: number; - readonly exitDeadlineMs: number; - readonly localServingIn?: (mode: AccessMode) => LocalServing; - readonly usagePing?: UsagePing; -} - -export function servedBy(routes: readonly RegisterRoutes[]): Served { - return { routes, stopWork: () => Promise.resolve() }; -} - -export const defaultServerOptions: ServerOptions = { - composed: () => ({ services: Layer.empty, serve: () => servedBy([]) }), - shutdownTimeoutMs, - exitDeadlineMs, -}; - -interface Starting { - readonly settings: Settings; - readonly authenticator: Authenticator; - readonly telemetry: Telemetry; - readonly options: ServerOptions; -} - -async function servedOf(composed: Composed, runtime: AppRuntime, ownOrigin: () => string): Promise { - try { - return await composed.serve(runtime, ownOrigin); - } catch (failure) { - await runtime.dispose(); - throw failure; - } -} - -async function startRuntime( - services: Layer.Layer, - logs: LogSettings, - telemetry: Layer.Layer, -): Promise> { - try { - return await makeAppRuntime(services.pipe(Layer.provideMerge(telemetry), Layer.provideMerge(serverLogs(logs)))); - } catch (failure) { - throw new StartupError({ message: `The server's services could not start: ${String(failure)}`, cause: failure }); - } -} - -function settingsServing(settings: Settings, local: LocalServing | undefined): Settings { - return local === undefined ? settings : { ...settings, providerSetup: local.providerSetup }; -} - -function telemetryServing(telemetry: Telemetry, local: LocalServing | undefined): Telemetry { - const counting = local?.appended; - if (counting === undefined) { - return telemetry; - } - const traced = telemetry.appended; - return { - ...telemetry, - appended: { - appended: (stream, events, place) => { - traced?.appended(stream, events, place); - counting.appended(stream, events, place); - }, - }, - }; -} - -interface Answering { - readonly settings: Settings; - readonly authenticator: Authenticator; - readonly routes: readonly RegisterRoutes[]; - readonly runtime: AppRuntime; - readonly telemetry: Telemetry; - readonly local: LocalServing | undefined; - readonly startPing: StartPing; -} - -function apiOf({ settings, authenticator, routes, runtime, telemetry, local, startPing }: Answering): ApiHandler { - return createApiHandler({ - allowedOrigins: settings.allowedOrigins, - studio: servedStudioOf(settings, { - report: (problem) => { - void runtime.run(logStudioKeptInMemory(problem)); - }, - }), - authenticator, - routes, - reportIncident: (id, error, requestId, call) => { - void runtime.run(logIncident({ id, original: error, call }).pipe(Effect.annotateLogs({ requestId }))); - }, - ...(telemetry.requests === undefined ? {} : { requests: telemetry.requests }), - ...(local?.runtimeControl === undefined ? {} : { runtimeControl: local.runtimeControl }), - installId: startPing.installId, - }); -} - -const noStartPing: StartPing = { - installId: Function.constUndefined, - sent: Function.constUndefined, - stop: Function.constVoid, -}; - -async function started({ settings, authenticator, telemetry, options }: Starting): Promise { - const local = options.localServingIn?.(authenticator.mode); - const composed = options.composed(settingsServing(settings, local), telemetryServing(telemetry, local)); - const runtime = await startRuntime(composed.services, settings, telemetry.layer); - await runtime.run(logAccessMode({ ...settings, mode: authenticator.mode })); - await runtime.run(logServedStudio({ ...settings, mode: authenticator.mode })); - await runtime.run(logConfigFile(settings.configFile)); - await runtime.run(logTelemetry(settings.telemetry)); - const listening = { port: settings.port }; - const served = await servedOf( - composed, - runtime, - () => settings.publicUrl ?? ownOriginOf(settings.host, listening.port), - ); - const startPing = options.usagePing?.(settings) ?? noStartPing; - const api = apiOf({ settings, authenticator, routes: served.routes, runtime, telemetry, local, startPing }); - const server = createHttpServer(api.listener); - try { - await listen(server, settings.port, settings.host); - } catch (error) { - await served.stopWork(); - await runtime.dispose(); - await api.close(); - throw error; - } - listening.port = tcpPort(server.address()); - const install = startPing.sent(); - let stopping: Promise | undefined; - return { - port: listening.port, - mode: authenticator.mode, - signIn: settings.signIn, - studio: settings.studio, - install, - stop: async () => { - startPing.stop(); - stopping ??= shutDown( - server, - { - stopWork: served.stopWork, - disposeRuntime: runtime.dispose, - stopTelemetry: telemetry.shutdown, - closeApi: api.close, - }, - options.shutdownTimeoutMs, - ); - await stopping; - }, - }; -} +import type { RunningServer, ServerOptions } from '../server-options/server-options.ts'; +import { readShippedSettings } from '../settings/served-settings.ts'; +import { readSettings } from '../settings/settings.ts'; +import { startTelemetry } from '../telemetry/start-telemetry.ts'; +import { started } from './server-start.ts'; export async function startServer(environment: Environment, options: ServerOptions): Promise { - const settings = readSettings(environment); + const [shipped, pageStyle] = await Promise.all([ + readShippedSettings(options.shippedFiles), + readPageStyle(options.shippedFiles), + ]); + const settings = readSettings(environment, shipped); const identityService = settings.signIn === undefined ? undefined @@ -244,7 +25,7 @@ export async function startServer(environment: Environment, options: ServerOp const authenticator = authenticatorFor({ ...settings, identityService }); const telemetry = startTelemetry(settings.telemetry); try { - return await started({ settings, authenticator, telemetry, options }); + return await started({ settings, authenticator, telemetry, options, pageStyle }); } catch (failure) { await telemetry.shutdown(); throw failure; diff --git a/packages/server/src/lifecycle/page-style-startup.test.ts b/packages/server/src/lifecycle/page-style-startup.test.ts new file mode 100644 index 000000000..46cfec3e5 --- /dev/null +++ b/packages/server/src/lifecycle/page-style-startup.test.ts @@ -0,0 +1,24 @@ +import type { ShippedFiles } from '@on.auto/operations'; +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; +import { describe, expect, it } from 'vitest'; + +import { defaultServerOptions } from '../testing/entries/serving-options.ts'; +import { startServer } from './lifecycle.ts'; + +const withoutTypefaces: ShippedFiles = { + ...shippedFilesOnDisk, + bytes: (file) => + file.pathname.endsWith('.woff2') + ? Promise.reject(new Error(`No file is shipped at ${file.href}`)) + : shippedFilesOnDisk.bytes(file), +}; + +describe('a server whose pages cannot be styled', () => { + it('does not start when a typeface its pages carry cannot be read, and names the file', async () => { + const options = { ...defaultServerOptions, shippedFiles: withoutTypefaces }; + + await expect(startServer({ HOST: '127.0.0.1', PORT: '0' }, options)).rejects.toThrow( + /^No file is shipped at file:.*\.woff2$/u, + ); + }); +}); diff --git a/packages/server/src/lifecycle/run-server.test.ts b/packages/server/src/lifecycle/run-server.test.ts index 8044f725b..8a65ce4cc 100644 --- a/packages/server/src/lifecycle/run-server.test.ts +++ b/packages/server/src/lifecycle/run-server.test.ts @@ -3,9 +3,9 @@ import { Effect, Layer } from 'effect'; import { describe, expect, it } from 'vitest'; import { ledgerOfAnotherVersion } from '../database/ledger-of-another-version.ts'; -import { optionsComposing, optionsServing } from '../testing/entries/serving-options.ts'; +import { servedBy } from '../server-options/server-options.ts'; +import { optionsComposing, optionsServing, defaultServerOptions } from '../testing/entries/serving-options.ts'; import { isAcceptingConnections } from '../testing/processes/accepting-connections.ts'; -import { defaultServerOptions, servedBy } from './lifecycle.ts'; import { exitOnStartupFailure, runServer, type ServerProcess } from './run-server.ts'; import { stopInsistedBy, stopRequestedBy, type StopSignal } from './stop-request.ts'; diff --git a/packages/server/src/lifecycle/run-server.ts b/packages/server/src/lifecycle/run-server.ts index 795b16e69..49bc65823 100644 --- a/packages/server/src/lifecycle/run-server.ts +++ b/packages/server/src/lifecycle/run-server.ts @@ -3,8 +3,9 @@ import { setTimeout } from 'node:timers/promises'; import type { Environment } from '@on.auto/config'; import { ledgerOfAnotherVersionIn } from '../database/ledger-of-another-version.ts'; +import { type RunningServer, type ServerOptions } from '../server-options/server-options.ts'; import { firstStartNotice } from '../usage-ping/usage-ping.ts'; -import { startServer, type RunningServer, type ServerOptions } from './lifecycle.ts'; +import { startServer } from './lifecycle.ts'; export interface ServerProcess { readonly env: Environment; diff --git a/packages/server/src/lifecycle/server-start.ts b/packages/server/src/lifecycle/server-start.ts new file mode 100644 index 000000000..66aa2a821 --- /dev/null +++ b/packages/server/src/lifecycle/server-start.ts @@ -0,0 +1,178 @@ +import { + createApiHandler, + makeAppRuntime, + type ApiHandler, + type AppRuntime, + type PageStyle, + type RegisterRoutes, +} from '@on.auto/api'; +import type { Authenticator } from '@on.auto/identity'; +import { Data, Effect, Function, Layer } from 'effect'; + +import { + logAccessMode, + logConfigFile, + logIncident, + logServedStudio, + logStudioKeptInMemory, + logTelemetry, + serverLogs, + type LogSettings, +} from '../logging/logging.ts'; +import type { + Composed, + LocalServing, + RunningServer, + Served, + ServerOptions, + StartPing, +} from '../server-options/server-options.ts'; +import type { Settings } from '../settings/settings.ts'; +import { servedStudioOf } from '../studio/served-studio.ts'; +import type { Telemetry } from '../telemetry/start-telemetry.ts'; +import { createHttpServer, listen, ownOriginOf, tcpPort } from './http-server.ts'; +import { shutDown } from './shutdown.ts'; + +class StartupError extends Data.TaggedError('StartupError')<{ + readonly message: string; + readonly cause: unknown; +}> {} + +interface Starting { + readonly settings: Settings; + readonly authenticator: Authenticator; + readonly telemetry: Telemetry; + readonly options: ServerOptions; + readonly pageStyle: PageStyle; +} + +async function servedOf(composed: Composed, runtime: AppRuntime, ownOrigin: () => string): Promise { + try { + return await composed.serve(runtime, ownOrigin); + } catch (failure) { + await runtime.dispose(); + throw failure; + } +} + +async function startRuntime( + services: Layer.Layer, + logs: LogSettings, + telemetry: Layer.Layer, +): Promise> { + try { + return await makeAppRuntime(services.pipe(Layer.provideMerge(telemetry), Layer.provideMerge(serverLogs(logs)))); + } catch (failure) { + throw new StartupError({ message: `The server's services could not start: ${String(failure)}`, cause: failure }); + } +} + +function settingsServing(settings: Settings, local: LocalServing | undefined): Settings { + return local === undefined ? settings : { ...settings, providerSetup: local.providerSetup }; +} + +function telemetryServing(telemetry: Telemetry, local: LocalServing | undefined): Telemetry { + const counting = local?.appended; + if (counting === undefined) { + return telemetry; + } + const traced = telemetry.appended; + return { + ...telemetry, + appended: { + appended: (stream, events, place) => { + traced?.appended(stream, events, place); + counting.appended(stream, events, place); + }, + }, + }; +} + +interface Answering { + readonly settings: Settings; + readonly authenticator: Authenticator; + readonly routes: readonly RegisterRoutes[]; + readonly runtime: AppRuntime; + readonly telemetry: Telemetry; + readonly local: LocalServing | undefined; + readonly startPing: StartPing; + readonly pageStyle: PageStyle; +} + +function apiOf(answering: Answering): ApiHandler { + const { settings, authenticator, routes, runtime, telemetry, local, startPing, pageStyle } = answering; + return createApiHandler({ + allowedOrigins: settings.allowedOrigins, + studio: servedStudioOf(settings, { + report: (problem) => { + void runtime.run(logStudioKeptInMemory(problem)); + }, + }), + authenticator, + routes, + pageStyle, + reportIncident: (id, error, requestId, call) => { + void runtime.run(logIncident({ id, original: error, call }).pipe(Effect.annotateLogs({ requestId }))); + }, + ...(telemetry.requests === undefined ? {} : { requests: telemetry.requests }), + ...(local?.runtimeControl === undefined ? {} : { runtimeControl: local.runtimeControl }), + installId: startPing.installId, + }); +} + +const noStartPing: StartPing = { + installId: Function.constUndefined, + sent: Function.constUndefined, + stop: Function.constVoid, +}; + +export async function started(starting: Starting): Promise { + const { settings, authenticator, telemetry, options, pageStyle } = starting; + const local = options.localServingIn?.(authenticator.mode); + const { shippedFiles } = options; + const composed = options.composed(settingsServing(settings, local), telemetryServing(telemetry, local), shippedFiles); + const runtime = await startRuntime(composed.services, settings, telemetry.layer); + await runtime.run(logAccessMode({ ...settings, mode: authenticator.mode })); + await runtime.run(logServedStudio({ ...settings, mode: authenticator.mode })); + await runtime.run(logConfigFile(settings.configFile)); + await runtime.run(logTelemetry(settings.telemetry)); + const listening = { port: settings.port }; + const ownOrigin = (): string => settings.publicUrl ?? ownOriginOf(settings.host, listening.port); + const served = await servedOf(composed, runtime, ownOrigin); + const startPing = options.usagePing?.(settings) ?? noStartPing; + const routes = served.routes; + const api = apiOf({ settings, authenticator, routes, runtime, telemetry, local, startPing, pageStyle }); + const server = createHttpServer(api.listener); + try { + await listen(server, settings.port, settings.host); + } catch (error) { + await served.stopWork(); + await runtime.dispose(); + await api.close(); + throw error; + } + listening.port = tcpPort(server.address()); + const install = startPing.sent(); + let stopping: Promise | undefined; + return { + port: listening.port, + mode: authenticator.mode, + signIn: settings.signIn, + studio: settings.studio, + install, + stop: async () => { + startPing.stop(); + stopping ??= shutDown( + server, + { + stopWork: served.stopWork, + disposeRuntime: runtime.dispose, + stopTelemetry: telemetry.shutdown, + closeApi: api.close, + }, + options.shutdownTimeoutMs, + ); + await stopping; + }, + }; +} diff --git a/packages/server/src/lifecycle/shutdown.test.ts b/packages/server/src/lifecycle/shutdown.test.ts index 36e8f4fd0..4672d1e4b 100644 --- a/packages/server/src/lifecycle/shutdown.test.ts +++ b/packages/server/src/lifecycle/shutdown.test.ts @@ -3,11 +3,11 @@ import { fileURLToPath } from 'node:url'; import { stoppedWithinMs } from '@on.auto/operations/testing'; import { describe, expect, it } from 'vitest'; +import { defaultServerOptions } from '../testing/entries/serving-options.ts'; import { untilRefusingConnections } from '../testing/processes/accepting-connections.ts'; import { shortShutdownTimeoutMs, shutdownTimeoutVariable } from '../testing/processes/shutdown-timeouts.ts'; import { serverHoldingARequest, spawnServer } from '../testing/processes/spawned-server.ts'; import { longerThanTheTestMs } from '../testing/waiting/until.ts'; -import { defaultServerOptions } from './lifecycle.ts'; import { longestStopMs } from './shutdown.ts'; const serveWithTestRoutes = fileURLToPath(new URL('../testing/entries/serve-with-test-routes.ts', import.meta.url)); diff --git a/packages/server/src/lifecycle/telemetry-lifecycle.test.ts b/packages/server/src/lifecycle/telemetry-lifecycle.test.ts index 72b7d513e..16779d130 100644 --- a/packages/server/src/lifecycle/telemetry-lifecycle.test.ts +++ b/packages/server/src/lifecycle/telemetry-lifecycle.test.ts @@ -1,10 +1,11 @@ import { Effect, Layer } from 'effect'; import { describe, expect, it, onTestFinished } from 'vitest'; -import { optionsComposing } from '../testing/entries/serving-options.ts'; +import { servedBy } from '../server-options/server-options.ts'; +import { optionsComposing, defaultServerOptions } from '../testing/entries/serving-options.ts'; import { logsIn, spansIn } from '../testing/telemetry/otlp-exports.ts'; import { receivingOtlp, type OtlpReceiver } from '../testing/telemetry/otlp-receiver.ts'; -import { defaultServerOptions, servedBy, startServer } from './lifecycle.ts'; +import { startServer } from './lifecycle.ts'; async function collector(): Promise { const receiver = await receivingOtlp(); diff --git a/packages/server/src/local-runtime/controlled-server.test.ts b/packages/server/src/local-runtime/controlled-server.test.ts index 642f3c52b..c0b0f96e0 100644 --- a/packages/server/src/local-runtime/controlled-server.test.ts +++ b/packages/server/src/local-runtime/controlled-server.test.ts @@ -2,9 +2,10 @@ import { createApiKey, signInScopes } from '@on.auto/identity'; import { Layer } from 'effect'; import { describe, expect, it, onTestFinished } from 'vitest'; -import { defaultServerOptions, startServer, servedBy, type ServerOptions } from '../lifecycle/lifecycle.ts'; +import { startServer } from '../lifecycle/lifecycle.ts'; +import { servedBy, type ServerOptions } from '../server-options/server-options.ts'; import { accessTokenOf, issuerOverHttps } from '../sign-in/issuer-over-https.ts'; -import { optionsServing } from '../testing/entries/serving-options.ts'; +import { optionsServing, defaultServerOptions } from '../testing/entries/serving-options.ts'; import { localControlOf, type LocalControl } from './local-control.ts'; import { runsInFlight } from './runs-in-flight.ts'; diff --git a/packages/server/src/local-runtime/local-control.ts b/packages/server/src/local-runtime/local-control.ts index 140a5349d..e8c816534 100644 --- a/packages/server/src/local-runtime/local-control.ts +++ b/packages/server/src/local-runtime/local-control.ts @@ -6,7 +6,7 @@ import type { Environment } from '@on.auto/config'; import type { AccessMode } from '@on.auto/identity'; import { Option, Schema } from 'effect'; -import type { LocalServing } from '../lifecycle/lifecycle.ts'; +import type { LocalServing } from '../server-options/server-options.ts'; import type { RunsInFlight } from './runs-in-flight.ts'; import { restartWaitMs } from './runtime-probe.ts'; diff --git a/packages/server/src/local-runtime/provider-setup.test.ts b/packages/server/src/local-runtime/provider-setup.test.ts index 50635f358..34d784e1a 100644 --- a/packages/server/src/local-runtime/provider-setup.test.ts +++ b/packages/server/src/local-runtime/provider-setup.test.ts @@ -2,7 +2,8 @@ import { plainTextIn, withMcpSession } from '@on.auto/api/testing'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { compositionRoot } from '../composition/composition-root.ts'; -import { startServer, type RunningServer } from '../lifecycle/lifecycle.ts'; +import { startServer } from '../lifecycle/lifecycle.ts'; +import { type RunningServer } from '../server-options/server-options.ts'; import { temporaryLedger, type TemporaryLedger } from '../testing/records/temporary-ledger.ts'; import { request } from '../testing/servers/http-client.ts'; import { alpha } from '../testing/servers/reasoning-server.ts'; diff --git a/packages/server/src/local-runtime/runtime-probe.test.ts b/packages/server/src/local-runtime/runtime-probe.test.ts index 5e8028351..0e2b19fac 100644 --- a/packages/server/src/local-runtime/runtime-probe.test.ts +++ b/packages/server/src/local-runtime/runtime-probe.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest'; -import { defaultServerOptions } from '../lifecycle/lifecycle.ts'; import { longestStopMs } from '../lifecycle/shutdown.ts'; +import { defaultServerOptions } from '../testing/entries/serving-options.ts'; import { restartWaitMs, startWaitMs, throughARestart } from './runtime-probe.ts'; describe('the patience of an agent for a runtime that restarts', () => { diff --git a/packages/server/src/logging/ledger-logging.test.ts b/packages/server/src/logging/ledger-logging.test.ts index a7bb6967e..89fd7b8fc 100644 --- a/packages/server/src/logging/ledger-logging.test.ts +++ b/packages/server/src/logging/ledger-logging.test.ts @@ -3,9 +3,9 @@ import { resolve } from 'node:path'; import { Redacted } from 'effect'; import { describe, expect, it } from 'vitest'; -import { readSettings } from '../settings/settings.ts'; import { secretsKey } from '../testing/databases/secrets-key.ts'; import { linesLoggedBy } from '../testing/records/logged-lines.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; import { logLedger } from './logging.ts'; describe('logLedger on SQLite', () => { @@ -51,7 +51,7 @@ describe('logLedger on PostgreSQL', () => { }); it('names the user and the region a ledger in PostgreSQL signs its tokens for, and no token', async () => { - const { ledger } = readSettings({ + const { ledger } = readSettingsOf({ DATABASE_URL: 'postgresql://auto_runtime@ledger.c9akciq32.eu-west-1.rds.amazonaws.com/playbooks', DATABASE_CREDENTIALS: 'aws-iam', SECRETS_KEY: secretsKey, diff --git a/packages/server/src/logging/telemetry-logging.test.ts b/packages/server/src/logging/telemetry-logging.test.ts index 15e8b8522..f503dc80a 100644 --- a/packages/server/src/logging/telemetry-logging.test.ts +++ b/packages/server/src/logging/telemetry-logging.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest'; -import { readSettings } from '../settings/settings.ts'; import { linesLoggedBy } from '../testing/records/logged-lines.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; import { logTelemetry } from './logging.ts'; describe('logTelemetry', () => { @@ -10,7 +10,7 @@ describe('logTelemetry', () => { }); it('names where each signal goes by its origin alone, and whether content is recorded', async () => { - const telemetry = readSettings({ + const telemetry = readSettingsOf({ OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: 'https://collector.example.com:4318/v1/traces?token=secret', TELEMETRY_CONTENT: 'true', }).telemetry; @@ -34,7 +34,7 @@ describe('logTelemetry', () => { it('says content is left out unless TELEMETRY_CONTENT asks for it', async () => { const [line] = await linesLoggedBy( - logTelemetry(readSettings({ OTEL_EXPORTER_OTLP_ENDPOINT: 'http://collector:4318' }).telemetry), + logTelemetry(readSettingsOf({ OTEL_EXPORTER_OTLP_ENDPOINT: 'http://collector:4318' }).telemetry), ); expect(line).toContain('"content":"left out"'); diff --git a/packages/server/src/mcp/mcp-with-keys.test.ts b/packages/server/src/mcp/mcp-with-keys.test.ts index 958c02661..a7942b64d 100644 --- a/packages/server/src/mcp/mcp-with-keys.test.ts +++ b/packages/server/src/mcp/mcp-with-keys.test.ts @@ -5,7 +5,8 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { compositionRoot } from '../composition/composition-root.ts'; import { release } from '../composition/release.ts'; -import { startServer, type RunningServer } from '../lifecycle/lifecycle.ts'; +import { startServer } from '../lifecycle/lifecycle.ts'; +import { type RunningServer } from '../server-options/server-options.ts'; import { temporaryLedger, type TemporaryLedger } from '../testing/records/temporary-ledger.ts'; const acmeAdmin = createApiKey({ id: 'acme-admin', org: 'acme', permissions: allPermissions, playbooks: '*' }); diff --git a/packages/server/src/playbooks/playbooks-over-http.test.ts b/packages/server/src/playbooks/playbooks-over-http.test.ts index ef6157acf..b32df8a8c 100644 --- a/packages/server/src/playbooks/playbooks-over-http.test.ts +++ b/packages/server/src/playbooks/playbooks-over-http.test.ts @@ -1,7 +1,8 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { compositionRoot } from '../composition/composition-root.ts'; -import { startServer, type RunningServer } from '../lifecycle/lifecycle.ts'; +import { startServer } from '../lifecycle/lifecycle.ts'; +import { type RunningServer } from '../server-options/server-options.ts'; import { temporaryLedger, type TemporaryLedger } from '../testing/records/temporary-ledger.ts'; import { request, type TestResponse } from '../testing/servers/http-client.ts'; import { noConsentThroughAuto } from '../testing/settings/no-consent-through-auto.ts'; diff --git a/packages/server/src/playbooks/playbooks-with-keys.test.ts b/packages/server/src/playbooks/playbooks-with-keys.test.ts index f18366079..46d9ebdfc 100644 --- a/packages/server/src/playbooks/playbooks-with-keys.test.ts +++ b/packages/server/src/playbooks/playbooks-with-keys.test.ts @@ -3,7 +3,8 @@ import { allPermissions } from '@on.auto/operations'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { compositionRoot } from '../composition/composition-root.ts'; -import { startServer, type RunningServer } from '../lifecycle/lifecycle.ts'; +import { startServer } from '../lifecycle/lifecycle.ts'; +import { type RunningServer } from '../server-options/server-options.ts'; import { temporaryLedger, type TemporaryLedger } from '../testing/records/temporary-ledger.ts'; import { request } from '../testing/servers/http-client.ts'; diff --git a/packages/server/src/prediction/build-keys.test.ts b/packages/server/src/prediction/build-keys.test.ts index 6895563ae..f8cedc7e7 100644 --- a/packages/server/src/prediction/build-keys.test.ts +++ b/packages/server/src/prediction/build-keys.test.ts @@ -5,7 +5,8 @@ import { Schema } from 'effect'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { compositionRoot } from '../composition/composition-root.ts'; -import { startServer, type RunningServer } from '../lifecycle/lifecycle.ts'; +import { startServer } from '../lifecycle/lifecycle.ts'; +import { type RunningServer } from '../server-options/server-options.ts'; import { temporaryLedger, type TemporaryLedger } from '../testing/records/temporary-ledger.ts'; import { buildRoute, candidateFile, visits } from './build-calls.ts'; diff --git a/packages/server/src/prediction/dataset-uploads.ts b/packages/server/src/prediction/dataset-uploads.ts index 0f89d2beb..6ad96b5eb 100644 --- a/packages/server/src/prediction/dataset-uploads.ts +++ b/packages/server/src/prediction/dataset-uploads.ts @@ -6,7 +6,7 @@ import { Effect, ManagedRuntime } from 'effect'; import { ledgerLayerFor } from '../composition/ledger-store.ts'; import { ledgerDatabaseOf } from '../database/ledger-database.ts'; -import { readSettings } from '../settings/settings.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; const datasets = '/v1/orgs/acme/playbooks/alpha/datasets'; @@ -49,7 +49,7 @@ export async function keptText( environment: Readonly>, sha256: string, ): Promise { - const settings = readSettings(environment); + const settings = readSettingsOf(environment); const runtime = ManagedRuntime.make(ledgerLayerFor(settings, ledgerDatabaseOf(settings))); try { const kept = await runtime.runPromise( diff --git a/packages/server/src/reasoning/provider-messages-over-http.test.ts b/packages/server/src/reasoning/provider-messages-over-http.test.ts index 93c660fff..5e575eabe 100644 --- a/packages/server/src/reasoning/provider-messages-over-http.test.ts +++ b/packages/server/src/reasoning/provider-messages-over-http.test.ts @@ -3,7 +3,8 @@ import { gatewayError, gatewayErrorText, gatewayInternals, recordingReporter } f import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { compositionRootWith } from '../composition/composition-root.ts'; -import { startServer, type RunningServer } from '../lifecycle/lifecycle.ts'; +import { startServer } from '../lifecycle/lifecycle.ts'; +import { type RunningServer } from '../server-options/server-options.ts'; import { temporaryLedger, type TemporaryLedger } from '../testing/records/temporary-ledger.ts'; import { request, type TestResponse } from '../testing/servers/http-client.ts'; import { noConsentThroughAuto } from '../testing/settings/no-consent-through-auto.ts'; diff --git a/packages/server/src/reasoning/unconfigured-provider-over-http.test.ts b/packages/server/src/reasoning/unconfigured-provider-over-http.test.ts index f5a3a6f40..e0935376d 100644 --- a/packages/server/src/reasoning/unconfigured-provider-over-http.test.ts +++ b/packages/server/src/reasoning/unconfigured-provider-over-http.test.ts @@ -1,7 +1,8 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { compositionRoot } from '../composition/composition-root.ts'; -import { startServer, type RunningServer } from '../lifecycle/lifecycle.ts'; +import { startServer } from '../lifecycle/lifecycle.ts'; +import { type RunningServer } from '../server-options/server-options.ts'; import { temporaryLedger, type TemporaryLedger } from '../testing/records/temporary-ledger.ts'; import { request, type TestResponse } from '../testing/servers/http-client.ts'; import { alpha } from '../testing/servers/reasoning-server.ts'; diff --git a/packages/server/src/recall/checked-values-over-http.test.ts b/packages/server/src/recall/checked-values-over-http.test.ts index a48aa129f..df89bc171 100644 --- a/packages/server/src/recall/checked-values-over-http.test.ts +++ b/packages/server/src/recall/checked-values-over-http.test.ts @@ -57,8 +57,8 @@ function recordingModules(pool: ProgramPool, modules: Set): describe('values their schemas refuse, through the one checked worker, over HTTP', () => { it('refuse a computation output and a recall answer, and stall a view, in the one wording of the issues', async () => { const modules = new Set(); - const server = await servingRecall(verdicts({ campaign: 'spring', verdict: 'approve' }), {}, (settings) => - recordingModules(patientWorkerPool(settings), modules), + const server = await servingRecall(verdicts({ campaign: 'spring', verdict: 'approve' }), {}, (settings, handed) => + recordingModules(patientWorkerPool(settings, handed), modules), ); closing.push(server.stop); await playbookWithReviews(server, 1); diff --git a/packages/server/src/secrets/aws-loading.test.ts b/packages/server/src/secrets/aws-loading.test.ts index 099857c09..78b021185 100644 --- a/packages/server/src/secrets/aws-loading.test.ts +++ b/packages/server/src/secrets/aws-loading.test.ts @@ -5,8 +5,8 @@ import { Effect, Logger } from 'effect'; import { afterAll, describe, expect, it, onTestFinished, vi } from 'vitest'; import { ledgerStoresOpenedBy } from '../database/ledger-stores.ts'; -import { readSettings } from '../settings/settings.ts'; import { temporaryLedger } from '../testing/records/temporary-ledger.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; const modulesLoaded = createRequire(import.meta.url).cache; @@ -27,7 +27,7 @@ async function openedAndClosed(environment: Readonly>): P const { fileName, remove } = temporaryLedger(); onTestFinished(remove); const context = await Effect.runPromise(Effect.context().pipe(Effect.provide(Logger.layer([])))); - const stores = await ledgerStoresOpenedBy(context, readSettings({ LEDGER_FILE: fileName, ...environment }), { + const stores = await ledgerStoresOpenedBy(context, readSettingsOf({ LEDGER_FILE: fileName, ...environment }), { store: 'sqlite', file: fileName, }); diff --git a/packages/server/src/secrets/own-keys.test.ts b/packages/server/src/secrets/own-keys.test.ts index 543d6b085..9e6185a69 100644 --- a/packages/server/src/secrets/own-keys.test.ts +++ b/packages/server/src/secrets/own-keys.test.ts @@ -1,5 +1,3 @@ -import { makeAppRuntime } from '@on.auto/api'; -import { ledgerLayer } from '@on.auto/ledger/sqlite3'; import { Conflict, type StreamReader, type StreamWriter } from '@on.auto/operations'; import { memoryLedger, type MemoryLedger } from '@on.auto/operations/testing'; import { sealedSecretStoreOn, secretStoreOn, type SecretRows, type SecretStore } from '@on.auto/secrets'; @@ -14,7 +12,7 @@ import { import { DateTime, Effect, Redacted, Result } from 'effect'; import { describe, expect, it, onTestFinished } from 'vitest'; -import { applicationLayer } from '../composition/application-layer.ts'; +import { memoryRuntime } from '../testing/records/memory-runtime.ts'; import { ownKeysDecider, ownKeysStream, @@ -253,7 +251,7 @@ describe('a start under a secret manager that cannot have the key that signs the }); async function aRuntime() { - const runtime = await makeAppRuntime(applicationLayer(ledgerLayer({ fileName: ':memory:' }))); + const runtime = await memoryRuntime(); onTestFinished(() => runtime.dispose()); return runtime; } diff --git a/packages/server/src/secrets/secret-store.test.ts b/packages/server/src/secrets/secret-store.test.ts index a31ff5737..d6646c86c 100644 --- a/packages/server/src/secrets/secret-store.test.ts +++ b/packages/server/src/secrets/secret-store.test.ts @@ -3,7 +3,8 @@ import type { Keeper } from '@on.auto/secrets'; import { Effect, Option } from 'effect'; import { describe, expect, it } from 'vitest'; -import { readSettings, type Settings } from '../settings/settings.ts'; +import { type Settings } from '../settings/settings.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; import { loadAwsAdapters, type AwsAdapters } from './aws-adapters.ts'; import { keepersOf } from './secret-keepers.ts'; import { custodiansOf } from './secret-store.ts'; @@ -60,13 +61,13 @@ function custodiansFor(settings: Pick, loadAws: ( describe('the custodians of the secrets settings', () => { it('are custodians of keys in hand, and load nothing of AWS, where no setting names a key in KMS', async () => { const loader = recordingLoader(); - const settings = readSettings({ + const settings = readSettingsOf({ SECRETS_KEY: current.toString('base64'), SECRETS_KEY_PREVIOUS: previous.toString('base64'), }); const custodians = await custodiansFor(settings, loader.loadAws); - const alone = await custodiansFor(readSettings({ LEDGER_FILE: ':memory:' }), loader.loadAws); + const alone = await custodiansFor(readSettingsOf({ LEDGER_FILE: ':memory:' }), loader.loadAws); expect([custodians.current.kid, custodians.previous?.kid, alone.previous]).toEqual([ kidOf(current), @@ -81,12 +82,12 @@ describe('the custodians of the secrets settings', () => { describe('the custodians of the secrets settings that name a key in KMS', () => { it('load the custodian of KMS where a setting names a key there, for the key the ARN names in its region, with the default credential chain and the proxy settings', async () => { const loader = recordingLoader(); - const inKms = readSettings({ + const inKms = readSettingsOf({ SECRETS_KEY_KMS: keyArn, SECRETS_KEY_PREVIOUS: previous.toString('base64'), NODE_USE_ENV_PROXY: '1', }); - const outOfKms = readSettings({ SECRETS_KEY: current.toString('base64'), SECRETS_KEY_KMS_PREVIOUS: keyArn }); + const outOfKms = readSettingsOf({ SECRETS_KEY: current.toString('base64'), SECRETS_KEY_KMS_PREVIOUS: keyArn }); const intoKms = await custodiansFor(inKms, loader.loadAws); const back = await custodiansFor(outOfKms, loader.loadAws); @@ -124,7 +125,7 @@ describe('the custodians of the secrets settings that name a key in KMS', () => describe('the keepers of the secrets settings', () => { it('are the keeper that seals into the ledger under the custodians, and load nothing of AWS, where no setting names a secret manager', async () => { const loader = recordingLoader(); - const settings = readSettings({ SECRETS_KEY: current.toString('base64') }); + const settings = readSettingsOf({ SECRETS_KEY: current.toString('base64') }); const custodians = await custodiansFor(settings, loader.loadAws); const keepers = await keepersOf(settings.keepers, custodians, { proxy: settings.proxy, loadAws: loader.loadAws }); @@ -136,11 +137,11 @@ describe('the keepers of the secrets settings', () => { it('load the keeper of Parameter Store where a setting names it, current or previous, for its prefix in the region of the settings, with the default credential chain and the proxy settings', async () => { const loader = recordingLoader(); const making = { loadAws: loader.loadAws }; - const alone = readSettings({ + const alone = readSettingsOf({ SECRETS_KEEPER: 'aws-parameter-store:/auto-runtime/secrets', AWS_REGION: 'eu-west-1', }); - const back = readSettings({ + const back = readSettingsOf({ SECRETS_KEEPER: 'ledger', SECRETS_KEEPER_PREVIOUS: 'aws-parameter-store:/auto-runtime/secrets', SECRETS_KEY: current.toString('base64'), diff --git a/packages/server/src/server-options/composed-options.ts b/packages/server/src/server-options/composed-options.ts new file mode 100644 index 000000000..44c890fbd --- /dev/null +++ b/packages/server/src/server-options/composed-options.ts @@ -0,0 +1,11 @@ +import type { DispatcherServices } from '@on.auto/operations'; +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; + +import { exitDeadlineMs, shutdownTimeoutMs } from '../lifecycle/shutdown.ts'; +import type { ServerOptions } from './server-options.ts'; + +export type ComposedServerOptions = ServerOptions; + +export function serverOptionsComposing(composed: ServerOptions['composed']): ServerOptions { + return { composed, shutdownTimeoutMs, exitDeadlineMs, shippedFiles: shippedFilesOnDisk }; +} diff --git a/packages/server/src/server-options/server-options.ts b/packages/server/src/server-options/server-options.ts new file mode 100644 index 000000000..24709dd4e --- /dev/null +++ b/packages/server/src/server-options/server-options.ts @@ -0,0 +1,60 @@ +import type { AppRuntime, RegisterRoutes, RuntimeControl } from '@on.auto/api'; +import type { AccessMode, SignInSettings, StudioSettings } from '@on.auto/identity'; +import type { ShippedFiles } from '@on.auto/operations'; +import type { Layer } from 'effect'; + +import type { Settings } from '../settings/settings.ts'; +import type { Telemetry } from '../telemetry/start-telemetry.ts'; + +export interface Install { + readonly id: string; + readonly isNew: boolean; +} + +export interface StartPing { + readonly installId: () => string | undefined; + readonly sent: () => Install | undefined; + readonly stop: () => void; +} + +export type UsagePing = (settings: Pick) => StartPing | undefined; + +export interface RunningServer { + readonly port: number; + readonly mode: AccessMode; + readonly signIn: SignInSettings | undefined; + readonly studio: StudioSettings; + readonly install: Install | undefined; + stop(): Promise; +} + +export interface Served { + readonly routes: readonly RegisterRoutes[]; + readonly stopWork: () => Promise; +} + +export type ServedClosing = (served: Served) => Served; + +export interface Composed { + readonly services: Layer.Layer; + readonly serve: (runtime: AppRuntime, ownOrigin: () => string) => Served | Promise; +} + +export interface LocalServing { + readonly runtimeControl?: RuntimeControl; + readonly providerSetup: string; + readonly appended?: NonNullable; +} + +export interface ServerOptions { + readonly composed: (settings: Settings, telemetry: Telemetry, shippedFiles: ShippedFiles) => Composed; + readonly shutdownTimeoutMs: number; + readonly exitDeadlineMs: number; + readonly shippedFiles: ShippedFiles; + readonly localServingIn?: (mode: AccessMode) => LocalServing; + readonly usagePing?: UsagePing; +} + +export function servedBy(routes: readonly RegisterRoutes[]): Served { + return { routes, stopWork: () => Promise.resolve() }; +} diff --git a/packages/server/src/settings/access-settings.test.ts b/packages/server/src/settings/access-settings.test.ts index 15807cadd..8f38b66b7 100644 --- a/packages/server/src/settings/access-settings.test.ts +++ b/packages/server/src/settings/access-settings.test.ts @@ -1,11 +1,11 @@ import type { Environment } from '@on.auto/config'; import { describe, expect, it } from 'vitest'; -import { readSettings } from './settings.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; function errorFrom(environment: Environment): unknown { try { - readSettings(environment); + readSettingsOf(environment); } catch (error) { return error; } @@ -18,14 +18,14 @@ const httpOffLoopback = describe('readSettings signing persons in', () => { it('signs persons in through the Auto identity service once PUBLIC_URL is set, and on every interface through no one without it', () => { expect([ - readSettings({ PUBLIC_URL: 'https://runtime.example.com' }).signIn, - readSettings({ IDENTITY_ISSUER: 'https://identity.example' }).signIn, + readSettingsOf({ PUBLIC_URL: 'https://runtime.example.com' }).signIn, + readSettingsOf({ IDENTITY_ISSUER: 'https://identity.example' }).signIn, ]).toEqual([{ issuer: 'https://clerk.on.auto', publicUrl: 'https://runtime.example.com' }, undefined]); }); it('signs persons in through the identity service IDENTITY_ISSUER names, such as a development instance', () => { expect( - readSettings({ IDENTITY_ISSUER: 'https://identity.example', PUBLIC_URL: 'https://runtime.example.com' }).signIn, + readSettingsOf({ IDENTITY_ISSUER: 'https://identity.example', PUBLIC_URL: 'https://runtime.example.com' }).signIn, ).toEqual({ issuer: 'https://identity.example', publicUrl: 'https://runtime.example.com' }); }); @@ -35,7 +35,7 @@ describe('readSettings signing persons in', () => { ['127.0.0.1', 'http://[::1]:5555'], ['0.0.0.0', 'http://localhost:5555'], ])('takes on HOST=%s the http PUBLIC_URL=%s, whose host is a loopback name', (host, publicUrl) => { - expect(readSettings({ HOST: host, PUBLIC_URL: publicUrl }).signIn).toEqual({ + expect(readSettingsOf({ HOST: host, PUBLIC_URL: publicUrl }).signIn).toEqual({ issuer: 'https://clerk.on.auto', publicUrl, }); @@ -59,11 +59,11 @@ describe('readSettings without PUBLIC_URL', () => { [{ HOST: '0.0.0.0' }], [{ HOST: '127.0.0.1', IDENTITY_ISSUER: 'https://identity.example' }], ] as const)('signs no one in with %j, since PUBLIC_URL has no default', (server) => { - expect(readSettings(server).signIn).toBeUndefined(); + expect(readSettingsOf(server).signIn).toBeUndefined(); }); it('signs persons in at PUBLIC_URL on a loopback address when PUBLIC_URL is set', () => { - expect(readSettings({ HOST: '127.0.0.1', PUBLIC_URL: 'https://runtime.example.com' }).signIn).toEqual({ + expect(readSettingsOf({ HOST: '127.0.0.1', PUBLIC_URL: 'https://runtime.example.com' }).signIn).toEqual({ issuer: 'https://clerk.on.auto', publicUrl: 'https://runtime.example.com', }); diff --git a/packages/server/src/settings/content-settings.test.ts b/packages/server/src/settings/content-settings.test.ts index e2481dedb..4005cd7ea 100644 --- a/packages/server/src/settings/content-settings.test.ts +++ b/packages/server/src/settings/content-settings.test.ts @@ -1,11 +1,11 @@ import type { Environment } from '@on.auto/config'; import { describe, expect, it } from 'vitest'; -import { readSettings } from './settings.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; function errorFrom(environment: Environment): unknown { try { - readSettings(environment); + readSettingsOf(environment); } catch (error) { return error; } @@ -13,16 +13,18 @@ function errorFrom(environment: Environment): unknown { } function contentOf(environment: Environment) { - return readSettings({ CONTENT_STORE_URL: 's3://playbook-content/a/b', ...environment }).content; + return readSettingsOf({ CONTENT_STORE_URL: 's3://playbook-content/a/b', ...environment }).content; } describe('the content store settings', () => { it('keep content in the ledger when CONTENT_STORE_URL is not set, whatever the endpoint', () => { - expect(readSettings({ AWS_ENDPOINT_URL: 'http://127.0.0.1:9000' }).content).toEqual({ store: 'ledger' }); + expect(readSettingsOf({ AWS_ENDPOINT_URL: 'http://127.0.0.1:9000' }).content).toEqual({ + store: 'ledger', + }); }); it('keep content on object storage in the bucket and under the prefix of CONTENT_STORE_URL', () => { - expect(readSettings({ CONTENT_STORE_URL: 's3://playbook-content', AWS_REGION: 'eu-west-1' }).content).toEqual({ + expect(readSettingsOf({ CONTENT_STORE_URL: 's3://playbook-content', AWS_REGION: 'eu-west-1' }).content).toEqual({ store: 'object-storage', location: { bucket: 'playbook-content', prefix: '' }, endpoint: undefined, diff --git a/packages/server/src/settings/database-credentials.test.ts b/packages/server/src/settings/database-credentials.test.ts index f8ecfe120..31f16cf8b 100644 --- a/packages/server/src/settings/database-credentials.test.ts +++ b/packages/server/src/settings/database-credentials.test.ts @@ -2,10 +2,11 @@ import type { Environment } from '@on.auto/config'; import { describe, expect, it } from 'vitest'; import { secretsKey } from '../testing/databases/secrets-key.ts'; -import { readSettings, type Settings } from './settings.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; +import { type Settings } from './settings.ts'; function settingsOf(environment: Environment): Settings { - return readSettings({ SECRETS_KEY: secretsKey, ...environment }); + return readSettingsOf({ SECRETS_KEY: secretsKey, ...environment }); } function errorFrom(environment: Environment): string { diff --git a/packages/server/src/settings/declared-local-mode.test.ts b/packages/server/src/settings/declared-local-mode.test.ts index 4b43fdd08..733e8fd8d 100644 --- a/packages/server/src/settings/declared-local-mode.test.ts +++ b/packages/server/src/settings/declared-local-mode.test.ts @@ -6,9 +6,10 @@ import { principalIn } from '@on.auto/identity/testing'; import { Function } from 'effect'; import { describe, expect, it, onTestFinished } from 'vitest'; -import { defaultServerOptions, startServer, type RunningServer } from '../lifecycle/lifecycle.ts'; +import { startServer } from '../lifecycle/lifecycle.ts'; +import { defaultServerOptions } from '../testing/entries/serving-options.ts'; import { temporaryLedger, type TemporaryLedger } from '../testing/records/temporary-ledger.ts'; -import { readSettings } from './settings.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; const { entry } = createApiKey({ id: 'ci-1', org: 'acme', permissions: ['org:read'], playbooks: '*' }); @@ -33,7 +34,7 @@ function aLedger(): TemporaryLedger { return ledger; } -function startOf(environment: Readonly>): Promise { +function startOf(environment: Readonly>) { const starting = startServer({ PORT: '0', LEDGER_FILE: aLedger().fileName, ...environment }, defaultServerOptions); onTestFinished(async () => { await (await starting.catch(Function.constUndefined))?.stop(); @@ -43,7 +44,7 @@ function startOf(environment: Readonly>): Promise { it('is in local mode on every address, admitting a request with no credential as the local developer of the org local, with the served Studio on and no one signed in', async () => { - const settings = readSettings(inAContainer); + const settings = readSettingsOf(inAContainer); const authenticator = authenticatorFor({ ...settings, identityService: undefined }); const developer = principalIn(await authenticator.authenticate(undefined, '/v1/orgs/acme/playbooks')); @@ -53,13 +54,13 @@ describe('a runtime whose operator declares local mode with ACCESS_MODE=local', true, undefined, ]); - expect(() => authenticatorFor({ ...readSettings({ HOST: '0.0.0.0' }), identityService: undefined })).toThrow( + expect(() => authenticatorFor({ ...readSettingsOf({ HOST: '0.0.0.0' }), identityService: undefined })).toThrow( 'A server reachable beyond this computer needs PUBLIC_URL', ); }); it('takes its PUBLIC_URL on the loopback as the address of the runtime, and signs no one in with it', () => { - expect(readSettings(inAContainer)).toMatchObject({ + expect(readSettingsOf(inAContainer)).toMatchObject({ accessMode: 'local', publicUrl: 'http://localhost:5555', signIn: undefined, @@ -67,7 +68,7 @@ describe('a runtime whose operator declares local mode with ACCESS_MODE=local', }); it('needs no PUBLIC_URL on a loopback address, whose links point at that address', () => { - expect(readSettings({ HOST: '127.0.0.1', ACCESS_MODE: 'local' })).toMatchObject({ + expect(readSettingsOf({ HOST: '127.0.0.1', ACCESS_MODE: 'local' })).toMatchObject({ accessMode: 'local', publicUrl: undefined, studio: { served: true }, @@ -77,7 +78,7 @@ describe('a runtime whose operator declares local mode with ACCESS_MODE=local', it('is declared in the configuration file as access_mode', () => { const file = join(aLedger().directory, 'auto-runtime.yaml'); writeFileSync(file, 'access_mode: local\n'); - const settings = readSettings({ HOST: '0.0.0.0', PUBLIC_URL: 'http://localhost:5555', CONFIG_FILE: file }); + const settings = readSettingsOf({ HOST: '0.0.0.0', PUBLIC_URL: 'http://localhost:5555', CONFIG_FILE: file }); expect([settings.accessMode, authenticatorFor({ ...settings, identityService: undefined }).mode]).toEqual([ 'local', diff --git a/packages/server/src/settings/keeper-settings.test.ts b/packages/server/src/settings/keeper-settings.test.ts index 92bedc8b1..fd6fcf09e 100644 --- a/packages/server/src/settings/keeper-settings.test.ts +++ b/packages/server/src/settings/keeper-settings.test.ts @@ -2,11 +2,11 @@ import type { Environment } from '@on.auto/config'; import { describe, expect, it } from 'vitest'; import { variablesTheRuntimeReads } from '../connections/variables-the-runtime-reads.ts'; -import { readSettings } from './settings.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; function errorFrom(environment: Environment): string { try { - readSettings(environment); + readSettingsOf(environment); } catch (error) { return String(error); } @@ -14,7 +14,7 @@ function errorFrom(environment: Environment): string { } function keepingOf(environment: Environment) { - const { keepers, secrets } = readSettings(environment); + const { keepers, secrets } = readSettingsOf(environment); return { keepers, custodian: secrets?.current.from }; } @@ -117,7 +117,7 @@ describe('the keepers of the secrets settings at their bounds', () => { const prefix = '/level'.repeat(14); expect( - readSettings({ SECRETS_KEEPER: `aws-parameter-store:${prefix}`, AWS_REGION: 'eu-west-1' }).keepers.current, + readSettingsOf({ SECRETS_KEEPER: `aws-parameter-store:${prefix}`, AWS_REGION: 'eu-west-1' }).keepers.current, ).toEqual({ kind: 'aws-parameter-store', setting: 'SECRETS_KEEPER', diff --git a/packages/server/src/settings/ledger-settings.test.ts b/packages/server/src/settings/ledger-settings.test.ts index d1fdee726..0688fc8a1 100644 --- a/packages/server/src/settings/ledger-settings.test.ts +++ b/packages/server/src/settings/ledger-settings.test.ts @@ -5,11 +5,12 @@ import { Redacted } from 'effect'; import { describe, expect, it } from 'vitest'; import { secretsKey } from '../testing/databases/secrets-key.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; import type { LedgerSettings } from './ledger-settings.ts'; -import { readSettings, type Settings } from './settings.ts'; +import { type Settings } from './settings.ts'; function settingsOf(environment: Environment): Settings { - return readSettings({ SECRETS_KEY: secretsKey, ...environment }); + return readSettingsOf({ SECRETS_KEY: secretsKey, ...environment }); } function errorFrom(environment: Environment): unknown { diff --git a/packages/server/src/settings/secrets-settings.test.ts b/packages/server/src/settings/secrets-settings.test.ts index f32ae5fa0..b3458d47b 100644 --- a/packages/server/src/settings/secrets-settings.test.ts +++ b/packages/server/src/settings/secrets-settings.test.ts @@ -2,12 +2,12 @@ import type { Environment } from '@on.auto/config'; import { Option, Redacted } from 'effect'; import { describe, expect, it } from 'vitest'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; import { secretsVariables, type SecretsKeySource } from './secrets-settings.ts'; -import { readSettings } from './settings.ts'; function errorFrom(environment: Environment): string { try { - readSettings(environment); + readSettingsOf(environment); } catch (error) { return String(error); } @@ -19,7 +19,7 @@ function revealed(source: SecretsKeySource | undefined): unknown { } function custodiansIn(environment: Environment) { - return Option.getOrThrow(Option.fromNullishOr(readSettings(environment).secrets)); + return Option.getOrThrow(Option.fromNullishOr(readSettingsOf(environment).secrets)); } function secretsOf(environment: Environment) { diff --git a/packages/server/src/settings/served-settings.ts b/packages/server/src/settings/served-settings.ts index 7d4b4ebc5..2133c471d 100644 --- a/packages/server/src/settings/served-settings.ts +++ b/packages/server/src/settings/served-settings.ts @@ -1,6 +1,7 @@ import type { Environment, FileUse } from '@on.auto/config'; import { makeProviderResolver } from '@on.auto/connections'; -import { readProviders } from '@on.auto/providers'; +import type { ShippedFiles } from '@on.auto/operations'; +import { readProviders, type ProviderList } from '@on.auto/providers'; import { Effect, Result } from 'effect'; import { @@ -21,12 +22,21 @@ export interface ServedSettings extends FunctionSettings, ConnectionSettings { readonly autoClients: AutoClientsSettings; } +export interface ShippedSettings { + readonly providers: ProviderList; +} + +export async function readShippedSettings(shippedFiles: ShippedFiles): Promise { + return { providers: Result.getOrThrow(await readProviders(shippedFiles)) }; +} + export function readServedSettings( environment: Environment, file: FileUse | undefined, access: Omit, + shipped: ShippedSettings, ): ServedSettings { - const connections = readConnections(environment, file, Result.getOrThrow(readProviders())); + const connections = readConnections(environment, file, shipped.providers); const functions = readFunctionSettings(environment, file, makeProviderResolver(connections.providers)); const connectionSettings = readConnectionSettings(environment, file, functions, connections); return { diff --git a/packages/server/src/settings/settings.test.ts b/packages/server/src/settings/settings.test.ts index 86d7a63c3..7532d35aa 100644 --- a/packages/server/src/settings/settings.test.ts +++ b/packages/server/src/settings/settings.test.ts @@ -3,11 +3,11 @@ import { createApiKey } from '@on.auto/identity'; import { describe, expect, it } from 'vitest'; import { shippedList } from '../testing/servers/connected-servers.ts'; -import { readSettings } from './settings.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; function errorFrom(environment: Environment): unknown { try { - readSettings(environment); + readSettingsOf(environment); } catch (error) { return error; } @@ -42,7 +42,7 @@ const modelProvidersByDefault = [ describe('readSettings', () => { it('listens on every interface at port 5555, allows no origin, has no API keys and keeps the ledger in data/ by default', () => { - const { models, toolServers, interaction, ...server } = readSettings({}); + const { models, toolServers, interaction, ...server } = readSettingsOf({}); expect(models.openai).toEqual({ configured: false, missing: ['OPENAI_API_KEY'] }); expect(toolServers).toEqual({ servers: [], referencedVariables: [] }); @@ -160,7 +160,7 @@ const everySettingRead = { describe('readSettings given the distribution', () => { it('reads where the runtime came from even when the usage ping is turned off', () => { - expect(readSettings({ RUNTIME_DISTRIBUTION: 'docker', DO_NOT_TRACK: '1' })).toMatchObject({ + expect(readSettingsOf({ RUNTIME_DISTRIBUTION: 'docker', DO_NOT_TRACK: '1' })).toMatchObject({ usagePing: undefined, distribution: 'docker', }); @@ -169,7 +169,7 @@ describe('readSettings given the distribution', () => { describe('readSettings given every setting', () => { it('reads every setting from the environment it is given', () => { - const { models, toolServers, interaction, studio, accessMode, publicUrl, ...server } = readSettings(everySetting); + const { models, toolServers, interaction, studio, accessMode, publicUrl, ...server } = readSettingsOf(everySetting); expect([interaction, studio.served, accessMode]).toEqual([{ mostOpenRequests: 250 }, false, undefined]); expect([models.openai.configured, publicUrl]).toEqual([true, 'https://runtime.example.com']); @@ -181,7 +181,7 @@ describe('readSettings given every setting', () => { describe('readSettings with empty values', () => { it('treats empty variables as not set', () => { expect( - readSettings({ + readSettingsOf({ ALLOWED_ORIGINS: '', API_KEYS: '', LEDGER_FILE: '', @@ -207,7 +207,7 @@ describe('readSettings with empty values', () => { }); it.each([' ', '\t', ' \n '])('treats API_KEYS=%j, only whitespace, as no API keys', (keys) => { - expect(readSettings({ API_KEYS: keys })).toMatchObject({ apiKeys: undefined }); + expect(readSettingsOf({ API_KEYS: keys })).toMatchObject({ apiKeys: undefined }); }); }); diff --git a/packages/server/src/settings/settings.ts b/packages/server/src/settings/settings.ts index 3aa32925d..c502931ce 100644 --- a/packages/server/src/settings/settings.ts +++ b/packages/server/src/settings/settings.ts @@ -18,7 +18,7 @@ import { } from './access-settings.ts'; import { readContentSettings, type ContentSettings } from './content-settings.ts'; import { readLedgerAndSecrets, type LedgerAndSecrets } from './secrets-settings.ts'; -import { readServedSettings, type ServedSettings } from './served-settings.ts'; +import { readServedSettings, type ServedSettings, type ShippedSettings } from './served-settings.ts'; interface ConfigFileSources { readonly path: string; @@ -42,14 +42,14 @@ function sourcesOf(file: FileUse | undefined): ConfigFileSources | undefined { return file === undefined ? undefined : { path: file.path, fromFile: file.fromFile, overridden: file.overridden }; } -export function readSettings(given: Environment): Settings { +export function readSettings(given: Environment, shipped: ShippedSettings): Settings { const { environment, file } = Result.getOrThrow(configurationOf(given, fileSettings)); const { host, port } = readServerConfig(environment); const { allowedOrigins, logFormat, logLevel } = Effect.runSync(readAccessSettings(environment)); const credentials = readCredentialSettings(environment, host); const { ledger, secrets, keepers } = Effect.runSync(readLedgerAndSecrets(environment)); const content = Effect.runSync(readContentSettings(environment)); - const served = readServedSettings(environment, file, { host, ...credentials }); + const served = readServedSettings(environment, file, { host, ...credentials }, shipped); const telemetry = Effect.runSync(readTelemetrySettings(environment)); const usagePing = Effect.runSync(readUsagePingSettings(environment)); return { diff --git a/packages/server/src/settings/workflow-settings.test.ts b/packages/server/src/settings/workflow-settings.test.ts index bb2c0a7d6..53bbe667e 100644 --- a/packages/server/src/settings/workflow-settings.test.ts +++ b/packages/server/src/settings/workflow-settings.test.ts @@ -1,11 +1,11 @@ import type { Environment } from '@on.auto/config'; import { describe, expect, it } from 'vitest'; -import { readSettings } from './settings.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; function errorFrom(environment: Environment): unknown { try { - readSettings(environment); + readSettingsOf(environment); } catch (error) { return error; } @@ -14,7 +14,7 @@ function errorFrom(environment: Environment): unknown { describe('the workflow settings', () => { it('let a run last 30 days, run 32 calls at once, keep 1000 calls open under a run and sweep every second when nothing is set', () => { - expect(readSettings({}).workflows).toEqual({ + expect(readSettingsOf({}).workflows).toEqual({ mostDurationMs: 2_592_000_000, mostCallsAtOnce: 32, mostOpenCalls: 1000, @@ -23,7 +23,7 @@ describe('the workflow settings', () => { }); it('read how long a run may last, how many calls run at once, how many stay open and how often the runs are swept', () => { - const workflows = readSettings({ + const workflows = readSettingsOf({ WORKFLOW_MAX_DURATION: 'P7D', WORKFLOW_NESTED_RUNS: '8', WORKFLOW_MAX_OPEN_CALLS: '50', @@ -55,8 +55,8 @@ describe('the workflow settings', () => { describe('the most calls open under one run', () => { it('take from 1 to 9999, and refuse 10000', () => { expect([ - readSettings({ WORKFLOW_MAX_OPEN_CALLS: '1' }).workflows.mostOpenCalls, - readSettings({ WORKFLOW_MAX_OPEN_CALLS: '9999' }).workflows.mostOpenCalls, + readSettingsOf({ WORKFLOW_MAX_OPEN_CALLS: '1' }).workflows.mostOpenCalls, + readSettingsOf({ WORKFLOW_MAX_OPEN_CALLS: '9999' }).workflows.mostOpenCalls, ]).toEqual([1, 9999]); expect(String(errorFrom({ WORKFLOW_MAX_OPEN_CALLS: '10000' }))).toContain( 'WORKFLOW_MAX_OPEN_CALLS: Expected a whole number from 1 to 9999', diff --git a/packages/server/src/sign-in/sign-in-with-public-url.test.ts b/packages/server/src/sign-in/sign-in-with-public-url.test.ts index df0e5d398..84dabb422 100644 --- a/packages/server/src/sign-in/sign-in-with-public-url.test.ts +++ b/packages/server/src/sign-in/sign-in-with-public-url.test.ts @@ -6,7 +6,8 @@ import { describe, expect, it, onTestFinished } from 'vitest'; import { compositionRoot } from '../composition/composition-root.ts'; import { tcpPort } from '../lifecycle/http-server.ts'; -import { startServer, type RunningServer } from '../lifecycle/lifecycle.ts'; +import { startServer } from '../lifecycle/lifecycle.ts'; +import { type RunningServer } from '../server-options/server-options.ts'; import { temporaryLedger } from '../testing/records/temporary-ledger.ts'; import { accessTokenOf, issuerOverHttps } from './issuer-over-https.ts'; diff --git a/packages/server/src/studio/studio-over-http.test.ts b/packages/server/src/studio/studio-over-http.test.ts index ca3faa29b..ab119751e 100644 --- a/packages/server/src/studio/studio-over-http.test.ts +++ b/packages/server/src/studio/studio-over-http.test.ts @@ -2,7 +2,9 @@ import { createApiKey } from '@on.auto/identity'; import { describe, expect, it, onTestFinished } from 'vitest'; import { release } from '../composition/release.ts'; -import { defaultServerOptions, startServer, type RunningServer } from '../lifecycle/lifecycle.ts'; +import { startServer } from '../lifecycle/lifecycle.ts'; +import { type RunningServer } from '../server-options/server-options.ts'; +import { defaultServerOptions } from '../testing/entries/serving-options.ts'; import { temporaryLedger } from '../testing/records/temporary-ledger.ts'; import { fakeStudioHost, diff --git a/packages/server/src/studio/studio-settings.test.ts b/packages/server/src/studio/studio-settings.test.ts index 2c90bae75..6eba3bb19 100644 --- a/packages/server/src/studio/studio-settings.test.ts +++ b/packages/server/src/studio/studio-settings.test.ts @@ -7,7 +7,7 @@ import { Effect } from 'effect'; import { describe, expect, it, onTestFinished } from 'vitest'; import { besideTheLedgerFile, readLedgerSettings } from '../settings/ledger-settings.ts'; -import { readSettings } from '../settings/settings.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; import { studioFolderName } from './served-studio.ts'; const { entry } = createApiKey({ id: 'ci-1', org: 'acme', permissions: ['org:read'], playbooks: '*' }); @@ -16,7 +16,7 @@ const loopback = { HOST: '127.0.0.1' }; function errorFrom(environment: Readonly>): string { try { - readSettings(environment); + readSettingsOf(environment); } catch (error) { return String(error); } @@ -25,15 +25,15 @@ function errorFrom(environment: Readonly>): string { describe('the Studio settings of a runtime in local mode', () => { it('serve Studio, fetched from the Studio host of Auto, by default', () => { - expect(readSettings(loopback).studio).toEqual({ origin: 'https://studio.on.auto', served: true }); + expect(readSettingsOf(loopback).studio).toEqual({ origin: 'https://studio.on.auto', served: true }); }); it('serve none on an address beyond the loopback with neither PUBLIC_URL nor API_KEYS, which is no local mode', () => { - expect(readSettings({ HOST: '0.0.0.0' }).studio.served).toBe(false); + expect(readSettingsOf({ HOST: '0.0.0.0' }).studio.served).toBe(false); }); it('serve none where SERVE_STUDIO is false', () => { - expect(readSettings({ ...loopback, SERVE_STUDIO: 'false' }).studio.served).toBe(false); + expect(readSettingsOf({ ...loopback, SERVE_STUDIO: 'false' }).studio.served).toBe(false); }); }); @@ -44,11 +44,16 @@ const withCredentials: ReadonlyArray { it.each(withCredentials)('serve no Studio by default where %s is set', (_case, environment) => { - expect(readSettings(environment).studio).toEqual({ origin: 'https://studio.on.auto', served: false }); + expect(readSettingsOf(environment).studio).toEqual({ + origin: 'https://studio.on.auto', + served: false, + }); }); it('serve Studio where the operator sets SERVE_STUDIO to true', () => { - expect(readSettings({ PUBLIC_URL: 'https://runtime.example.com', SERVE_STUDIO: 'true' }).studio.served).toBe(true); + expect(readSettingsOf({ PUBLIC_URL: 'https://runtime.example.com', SERVE_STUDIO: 'true' }).studio.served).toBe( + true, + ); }); }); @@ -60,7 +65,7 @@ describe('the Studio host a runtime names', () => { 'http://127.0.0.1:3200', 'http://[::1]:3200', ])('is taken from STUDIO_ORIGIN=%s', (origin) => { - expect(readSettings({ STUDIO_ORIGIN: origin }).studio.origin).toBe(origin); + expect(readSettingsOf({ STUDIO_ORIGIN: origin }).studio.origin).toBe(origin); }); it.each([ @@ -90,7 +95,7 @@ describe('the Studio settings in the configuration file', () => { const configFile = join(folder, 'auto-runtime.yaml'); writeFileSync(configFile, 'studio_origin: http://127.0.0.1:3200\nserve_studio: false\n'); - expect(readSettings({ ...loopback, CONFIG_FILE: configFile }).studio).toEqual({ + expect(readSettingsOf({ ...loopback, CONFIG_FILE: configFile }).studio).toEqual({ origin: 'http://127.0.0.1:3200', served: false, }); diff --git a/packages/server/src/telemetry/object-commands.test.ts b/packages/server/src/telemetry/object-commands.test.ts index dc50a4f87..8825d71b1 100644 --- a/packages/server/src/telemetry/object-commands.test.ts +++ b/packages/server/src/telemetry/object-commands.test.ts @@ -6,7 +6,7 @@ import { Effect, Schema } from 'effect'; import { describe, expect, it, onTestFinished, vi } from 'vitest'; import { contentStoreOf, type ComposedContentStore } from '../composition/content-store.ts'; -import { readSettings } from '../settings/settings.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; import { spansIn, type ExportedSpan } from '../testing/telemetry/otlp-exports.ts'; import { receivingOtlp } from '../testing/telemetry/otlp-receiver.ts'; import { startTelemetry } from './start-telemetry.ts'; @@ -45,7 +45,7 @@ function composedOn(environment: Readonly>): ComposedCont onTestFinished(() => { vi.unstubAllEnvs(); }); - const composed = contentStoreOf(readSettings(environment)); + const composed = contentStoreOf(readSettingsOf(environment)); if (composed === undefined) { throw new TypeError('CONTENT_STORE_URL is set, so the content is kept in an object store'); } @@ -62,7 +62,7 @@ async function spansOfAReadOfNothing(): Promise { OTEL_EXPORTER_OTLP_ENDPOINT: receiver.endpoint, OTEL_EXPORTER_OTLP_PROTOCOL: 'http/json', }; - const telemetry = startTelemetry(readSettings(environment).telemetry); + const telemetry = startTelemetry(readSettingsOf(environment).telemetry); const { reached, store } = composedOn(environment); await Effect.runPromise(reached); await Effect.runPromise(store.content.get(alpha, sha256Of('{"kept":false}'))); diff --git a/packages/server/src/telemetry/recorded-events.test.ts b/packages/server/src/telemetry/recorded-events.test.ts index 4ece5e97a..463dfb0ef 100644 --- a/packages/server/src/telemetry/recorded-events.test.ts +++ b/packages/server/src/telemetry/recorded-events.test.ts @@ -2,7 +2,7 @@ import type { Environment } from '@on.auto/config'; import type { AppendObserver } from '@on.auto/ledger'; import { describe, expect, it, onTestFinished } from 'vitest'; -import { readSettings } from '../settings/settings.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; import { logsIn, metricNamesIn } from '../testing/telemetry/otlp-exports.ts'; import { receivingOtlp, type OtlpReceiver } from '../testing/telemetry/otlp-receiver.ts'; import { startTelemetry, type Telemetry } from './start-telemetry.ts'; @@ -30,7 +30,7 @@ async function observed(environment: Environment = {}): Promise { const receiver = await receivingOtlp(); onTestFinished(receiver.close); const telemetry = startTelemetry( - readSettings({ + readSettingsOf({ OTEL_EXPORTER_OTLP_ENDPOINT: receiver.endpoint, OTEL_EXPORTER_OTLP_PROTOCOL: 'http/json', ...environment, diff --git a/packages/server/src/telemetry/request-tracing.test.ts b/packages/server/src/telemetry/request-tracing.test.ts index 97d3339c0..0db97fc9a 100644 --- a/packages/server/src/telemetry/request-tracing.test.ts +++ b/packages/server/src/telemetry/request-tracing.test.ts @@ -2,19 +2,15 @@ import { INVALID_SPAN_CONTEXT, trace } from '@opentelemetry/api'; import { describe, expect, it, onTestFinished } from 'vitest'; import { compositionRoot } from '../composition/composition-root.ts'; -import { defaultServerOptions, startServer, type RunningServer } from '../lifecycle/lifecycle.ts'; +import { startServer } from '../lifecycle/lifecycle.ts'; +import { defaultServerOptions } from '../testing/entries/serving-options.ts'; import { temporaryLedger } from '../testing/records/temporary-ledger.ts'; import { noConsentThroughAuto } from '../testing/settings/no-consent-through-auto.ts'; import { metricNamesIn, spansIn } from '../testing/telemetry/otlp-exports.ts'; -import { receivingOtlp, type OtlpReceiver } from '../testing/telemetry/otlp-receiver.ts'; +import { receivingOtlp } from '../testing/telemetry/otlp-receiver.ts'; import { tracedRequestOf } from './request-tracing.ts'; -interface Traced { - readonly receiver: OtlpReceiver; - readonly server: RunningServer; -} - -async function tracedServer(endpoint = 'OTEL_EXPORTER_OTLP_ENDPOINT', path = ''): Promise { +async function tracedServer(endpoint = 'OTEL_EXPORTER_OTLP_ENDPOINT', path = '') { const receiver = await receivingOtlp(); onTestFinished(receiver.close); const server = await startServer( diff --git a/packages/server/src/telemetry/start-telemetry.test.ts b/packages/server/src/telemetry/start-telemetry.test.ts index 89bacf22d..32bab9722 100644 --- a/packages/server/src/telemetry/start-telemetry.test.ts +++ b/packages/server/src/telemetry/start-telemetry.test.ts @@ -5,7 +5,7 @@ import { Effect, ManagedRuntime, Metric } from 'effect'; import { describe, expect, it, onTestFinished } from 'vitest'; import { release } from '../composition/release.ts'; -import { readSettings } from '../settings/settings.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; import { logsIn, metricNamesIn, resourceIn, spansIn } from '../testing/telemetry/otlp-exports.ts'; import { receivingOtlp, type OtlpReceiver, type ReceivedExport } from '../testing/telemetry/otlp-receiver.ts'; import { noTelemetry, startTelemetry } from './start-telemetry.ts'; @@ -19,7 +19,7 @@ async function collector(): Promise { const effectRuns = Metric.counter('auto.on.test.effect_runs'); async function exportedBy(environment: Environment): Promise { - const telemetry = startTelemetry(readSettings(environment).telemetry); + const telemetry = startTelemetry(readSettingsOf(environment).telemetry); const runtime = ManagedRuntime.make(telemetry.layer); await runtime.runPromise( Effect.logInfo('working').pipe(Effect.andThen(Metric.update(effectRuns, 1)), Effect.withSpan('effect work')), @@ -35,7 +35,7 @@ describe('telemetry whose collector cannot be reached', () => { it('stops within a second and a half, leaving behind what it could not send', async () => { const gone = await receivingOtlp(); await gone.close(); - const telemetry = startTelemetry(readSettings({ OTEL_EXPORTER_OTLP_ENDPOINT: gone.endpoint }).telemetry); + const telemetry = startTelemetry(readSettingsOf({ OTEL_EXPORTER_OTLP_ENDPOINT: gone.endpoint }).telemetry); trace.getTracer('test').startSpan('lost work').end(); const stopping = performance.now(); await telemetry.shutdown(); @@ -46,7 +46,7 @@ describe('telemetry whose collector cannot be reached', () => { describe('telemetry with no OTLP endpoint', () => { it('is no telemetry at all', () => { - expect(startTelemetry(readSettings({}).telemetry)).toBe(noTelemetry); + expect(startTelemetry(readSettingsOf({}).telemetry)).toBe(noTelemetry); }); }); diff --git a/packages/server/src/telemetry/telemetry-settings.test.ts b/packages/server/src/telemetry/telemetry-settings.test.ts index 13008ef59..ede97cd67 100644 --- a/packages/server/src/telemetry/telemetry-settings.test.ts +++ b/packages/server/src/telemetry/telemetry-settings.test.ts @@ -2,13 +2,13 @@ import type { Environment } from '@on.auto/config'; import { Redacted } from 'effect'; import { describe, expect, it } from 'vitest'; -import { readSettings } from '../settings/settings.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; import type { NameValues } from './name-value-lists.ts'; import type { SignalExport } from './telemetry-settings.ts'; function errorFrom(environment: Environment): string { try { - readSettings(environment); + readSettingsOf(environment); } catch (error) { return String(error); } @@ -16,7 +16,7 @@ function errorFrom(environment: Environment): string { } function telemetryOf(environment: Environment) { - return readSettings(environment).telemetry; + return readSettingsOf(environment).telemetry; } function headersOf(target: SignalExport | undefined): NameValues { diff --git a/packages/server/src/testing/entries/serve-with-self-caused-error.ts b/packages/server/src/testing/entries/serve-with-self-caused-error.ts index 3ffcf87d4..c8ebf1452 100644 --- a/packages/server/src/testing/entries/serve-with-self-caused-error.ts +++ b/packages/server/src/testing/entries/serve-with-self-caused-error.ts @@ -1,8 +1,8 @@ import type { RegisterRoutes } from '@on.auto/api'; -import { servedBy } from '../../lifecycle/lifecycle.ts'; import { exitOnStartupFailure, runServer } from '../../lifecycle/run-server.ts'; import { stopInsistedBy, stopRequestedBy } from '../../lifecycle/stop-request.ts'; +import { servedBy } from '../../server-options/server-options.ts'; import { optionsServing } from './serving-options.ts'; const selfCaused = new Error('an error whose cause is itself'); diff --git a/packages/server/src/testing/entries/serve-with-test-routes.ts b/packages/server/src/testing/entries/serve-with-test-routes.ts index a5d4d5a93..b43cd48a2 100644 --- a/packages/server/src/testing/entries/serve-with-test-routes.ts +++ b/packages/server/src/testing/entries/serve-with-test-routes.ts @@ -2,9 +2,9 @@ import { once } from 'node:events'; import { Schema } from 'effect'; -import { servedBy } from '../../lifecycle/lifecycle.ts'; import { exitOnStartupFailure, runServer } from '../../lifecycle/run-server.ts'; import { stopInsistedBy, stopRequestedBy } from '../../lifecycle/stop-request.ts'; +import { servedBy } from '../../server-options/server-options.ts'; import { shortShutdownTimeoutMs, shutdownTimeoutVariable } from '../processes/shutdown-timeouts.ts'; import { callInFlightAnnouncedIn } from './calls-in-flight.ts'; import { optionsServing } from './serving-options.ts'; diff --git a/packages/server/src/testing/entries/serve-with-waiting-tool.ts b/packages/server/src/testing/entries/serve-with-waiting-tool.ts index 656757a95..99fe80cf3 100644 --- a/packages/server/src/testing/entries/serve-with-waiting-tool.ts +++ b/packages/server/src/testing/entries/serve-with-waiting-tool.ts @@ -3,10 +3,10 @@ import { playbookOperations } from '@on.auto/playbooks'; import { compositionRoot } from '../../composition/composition-root.ts'; import { routesServing } from '../../composition/served-routes.ts'; -import { servedBy } from '../../lifecycle/lifecycle.ts'; import { exitOnStartupFailure, runServer } from '../../lifecycle/run-server.ts'; import { stopInsistedBy, stopRequestedBy } from '../../lifecycle/stop-request.ts'; import { keyInHandOf } from '../../secrets/secrets-keys.ts'; +import { servedBy } from '../../server-options/server-options.ts'; import { shortShutdownTimeoutMs } from '../processes/shutdown-timeouts.ts'; import { waitForeverAnnouncedIn } from './calls-in-flight.ts'; @@ -14,13 +14,14 @@ await runServer( process, { ...compositionRoot, - composed: (settings, telemetry) => ({ - ...compositionRoot.composed(settings, telemetry), - serve: (runtime) => + composed: (settings, telemetry, shippedFiles) => ({ + ...compositionRoot.composed(settings, telemetry, shippedFiles), + serve: async (runtime) => servedBy( - routesServing([...playbookOperations, waitForeverAnnouncedIn(process.env)], { + await routesServing([...playbookOperations, waitForeverAnnouncedIn(process.env)], { identityIssuer: settings.identityIssuer, retryStateKeys: { current: keyInHandOf({ from: 'start' }) }, + shippedFiles, })(runtime), ), }), diff --git a/packages/server/src/testing/entries/serving-options.ts b/packages/server/src/testing/entries/serving-options.ts index 28a5d1213..ec6c8769e 100644 --- a/packages/server/src/testing/entries/serving-options.ts +++ b/packages/server/src/testing/entries/serving-options.ts @@ -1,11 +1,14 @@ import { Layer } from 'effect'; -import { defaultServerOptions, type Composed, type ServerOptions } from '../../lifecycle/lifecycle.ts'; +import { serverOptionsComposing } from '../../server-options/composed-options.ts'; +import { servedBy, type Composed, type ServerOptions } from '../../server-options/server-options.ts'; export function optionsComposing(services: Layer.Layer, serve: Composed['serve']): ServerOptions { - return { ...defaultServerOptions, composed: () => ({ services, serve }) }; + return serverOptionsComposing(() => ({ services, serve })); } export function optionsServing(serve: Composed['serve']): ServerOptions { return optionsComposing(Layer.empty, serve); } + +export const defaultServerOptions: ServerOptions = optionsServing(() => servedBy([])); diff --git a/packages/server/src/testing/records/memory-runtime.ts b/packages/server/src/testing/records/memory-runtime.ts new file mode 100644 index 000000000..72a6c5340 --- /dev/null +++ b/packages/server/src/testing/records/memory-runtime.ts @@ -0,0 +1,9 @@ +import { makeAppRuntime, type AppRuntime } from '@on.auto/api'; +import { ledgerLayer } from '@on.auto/ledger/sqlite3'; +import type { DispatcherServices } from '@on.auto/operations'; + +import { applicationLayer } from '../../composition/application-layer.ts'; + +export function memoryRuntime(): Promise> { + return makeAppRuntime(applicationLayer(ledgerLayer({ fileName: ':memory:' }))); +} diff --git a/packages/server/src/testing/records/server-pings.ts b/packages/server/src/testing/records/server-pings.ts new file mode 100644 index 000000000..7f59c78f5 --- /dev/null +++ b/packages/server/src/testing/records/server-pings.ts @@ -0,0 +1,63 @@ +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import type { Environment } from '@on.auto/config'; +import { onTestFinished } from 'vitest'; + +import { usagePingOf, type PingFetch } from '../../usage-ping/usage-ping.ts'; +import { defaultServerOptions } from '../entries/serving-options.ts'; + +export interface Pings { + readonly fetch: PingFetch; + readonly urls: readonly string[]; + readonly signals: readonly AbortSignal[]; +} + +export function folderOfTheTest(): string { + const folder = mkdtempSync(join(tmpdir(), 'auto-runtime-usage-ping-server-')); + onTestFinished(() => { + rmSync(folder, { recursive: true, force: true }); + }); + return folder; +} + +function recorded(answer: (signal: AbortSignal) => Promise): Pings { + const urls: string[] = []; + const signals: AbortSignal[] = []; + return { + urls, + signals, + fetch: (url, { signal }) => { + urls.push(url); + signals.push(signal); + return answer(signal); + }, + }; +} + +export function answered(): Pings { + return recorded(() => Promise.resolve(new Response(''))); +} + +export function unanswered(): Pings { + return recorded( + (signal) => + new Promise((_answer, refuse) => { + signal.addEventListener('abort', () => { + refuse(new Error('the ping was given up', { cause: signal.reason })); + }); + }), + ); +} + +export function optionsPingingThrough(pings: Pings, runtimeHome: string) { + return { + ...defaultServerOptions, + usagePing: usagePingOf({ fetch: pings.fetch, runtimeHome, running: process }), + }; +} + +export function settingsIn(folder: string, environment: Environment = {}): Environment { + return { HOST: '127.0.0.1', PORT: '0', LEDGER_FILE: join(folder, 'ledger.db'), ...environment }; +} diff --git a/packages/server/src/testing/servers/connected-servers.ts b/packages/server/src/testing/servers/connected-servers.ts index e99fdd931..03c722024 100644 --- a/packages/server/src/testing/servers/connected-servers.ts +++ b/packages/server/src/testing/servers/connected-servers.ts @@ -1,12 +1,13 @@ -import { readProviders, type CredentialWay } from '@on.auto/providers'; +import type { CredentialWay } from '@on.auto/providers'; import type { OfferedProvider } from '@on.auto/tool-servers'; -import { Option, Result, Schema } from 'effect'; +import { Option, Schema } from 'effect'; +import { shippedSettings } from '../settings/shipped-settings.ts'; import type { ReasoningServer } from './reasoning-server.ts'; export const connectionsOfAcme = '/v1/orgs/acme/connections'; -export const shippedList = Result.getOrThrow(readProviders()); +export const shippedList = shippedSettings.providers; export const shippedProviders = shippedList.providers; diff --git a/packages/server/src/testing/servers/recall-server.ts b/packages/server/src/testing/servers/recall-server.ts index 286dd8810..0829cfabc 100644 --- a/packages/server/src/testing/servers/recall-server.ts +++ b/packages/server/src/testing/servers/recall-server.ts @@ -61,7 +61,7 @@ export function foldsHeldUntilOpened(): Gate { }, }); return { - poolOf: (settings) => held(patientWorkerPool(settings)), + poolOf: (settings, handed) => held(patientWorkerPool(settings, handed)), letThrough: (folds) => { allowed.folds += folds; released(); diff --git a/packages/server/src/testing/servers/waiting-server.ts b/packages/server/src/testing/servers/waiting-server.ts index 56c0021da..484c5333f 100644 --- a/packages/server/src/testing/servers/waiting-server.ts +++ b/packages/server/src/testing/servers/waiting-server.ts @@ -4,8 +4,9 @@ import { playbookOperations } from '@on.auto/playbooks'; import { compositionRoot } from '../../composition/composition-root.ts'; import { routesServing } from '../../composition/served-routes.ts'; -import { servedBy, startServer } from '../../lifecycle/lifecycle.ts'; +import { startServer } from '../../lifecycle/lifecycle.ts'; import { keyInHandOf } from '../../secrets/secrets-keys.ts'; +import { servedBy } from '../../server-options/server-options.ts'; import { temporaryLedger } from '../records/temporary-ledger.ts'; import { loopbackSettingsOver } from './reasoning-server.ts'; @@ -22,13 +23,14 @@ export async function servingAToolThatWaits(): Promise { const calling = Promise.withResolvers(); const server = await startServer(loopbackSettingsOver(ledger), { ...compositionRoot, - composed: (settings, telemetry) => ({ - ...compositionRoot.composed(settings, telemetry), - serve: (runtime) => + composed: (settings, telemetry, shippedFiles) => ({ + ...compositionRoot.composed(settings, telemetry, shippedFiles), + serve: async (runtime) => servedBy( - routesServing([...playbookOperations, waitForeverAnnouncing(calling.resolve)], { + await routesServing([...playbookOperations, waitForeverAnnouncing(calling.resolve)], { identityIssuer: settings.identityIssuer, retryStateKeys: { current: keyInHandOf({ from: 'start' }) }, + shippedFiles, })(runtime), ), }), diff --git a/packages/server/src/testing/servers/worker-pools.ts b/packages/server/src/testing/servers/worker-pools.ts index da1bc5179..f05bbf4e7 100644 --- a/packages/server/src/testing/servers/worker-pools.ts +++ b/packages/server/src/testing/servers/worker-pools.ts @@ -11,8 +11,9 @@ function decidingPatiently(pool: ProgramPool): ProgramPool { return { ...pool, evaluations: { ...pool.evaluations, machine: decidingWithin(machine, inputsAwaitedAtMostMs) } }; } -export const patientWorkerPool: ProgramPoolOf = (settings) => decidingPatiently(checkingWithin(workerPool(settings))); +export const patientWorkerPool: ProgramPoolOf = (settings, handed) => + decidingPatiently(checkingWithin(workerPool(settings, handed))); export function scriptedWorkerPool(...outcomes: readonly PoolOutcome[]): ProgramPoolOf { - return (settings) => scriptedPool(outcomes, patientWorkerPool(settings)); + return (settings, handed) => scriptedPool(outcomes, patientWorkerPool(settings, handed)); } diff --git a/packages/server/src/testing/settings/shipped-settings.ts b/packages/server/src/testing/settings/shipped-settings.ts new file mode 100644 index 000000000..2d405db89 --- /dev/null +++ b/packages/server/src/testing/settings/shipped-settings.ts @@ -0,0 +1,11 @@ +import type { Environment } from '@on.auto/config'; +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; + +import { readShippedSettings, type ShippedSettings } from '../../settings/served-settings.ts'; +import { readSettings, type Settings } from '../../settings/settings.ts'; + +export const shippedSettings: ShippedSettings = await readShippedSettings(shippedFilesOnDisk); + +export function readSettingsOf(environment: Environment): Settings { + return readSettings(environment, shippedSettings); +} diff --git a/packages/server/src/usage-ping/install-id.ts b/packages/server/src/usage-ping/install-id.ts index 87fcbb1c9..804427c78 100644 --- a/packages/server/src/usage-ping/install-id.ts +++ b/packages/server/src/usage-ping/install-id.ts @@ -2,7 +2,7 @@ import { randomUUID } from 'node:crypto'; import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; -import type { Install } from '../lifecycle/lifecycle.ts'; +import type { Install } from '../server-options/server-options.ts'; import { besideTheLedgerFile, type LedgerSettings } from '../settings/ledger-settings.ts'; const installIdFile = 'install-id'; diff --git a/packages/server/src/usage-ping/usage-ping-announcement.test.ts b/packages/server/src/usage-ping/usage-ping-announcement.test.ts new file mode 100644 index 000000000..5266b324d --- /dev/null +++ b/packages/server/src/usage-ping/usage-ping-announcement.test.ts @@ -0,0 +1,41 @@ +import { Function } from 'effect'; +import { describe, expect, it } from 'vitest'; + +import { runServer } from '../lifecycle/run-server.ts'; +import { answered, folderOfTheTest, optionsPingingThrough, settingsIn } from '../testing/records/server-pings.ts'; +import { firstStartNotice } from './usage-ping.ts'; + +async function announced(folder: string): Promise { + const written: string[] = []; + const never = Promise.withResolvers().promise; + const serverProcess = { + env: settingsIn(folder), + stdout: { + write: (message: string) => { + written.push(message); + }, + }, + stderr: { + write: (_message: string, flushed: () => void) => { + flushed(); + }, + }, + exit: Function.constVoid, + }; + const server = await runServer(serverProcess, optionsPingingThrough(answered(), folder), never, never); + await server.stop(); + return written; +} + +describe('the announcement of a server whose usage ping is on', () => { + it('says after the port, at the first start alone, that it sends an anonymous usage ping', async () => { + const folder = folderOfTheTest(); + + const first = await announced(folder); + const second = await announced(folder); + + expect(first.slice(1)).toEqual([`${firstStartNotice}\n`]); + expect(second).toHaveLength(1); + expect(first[0]).toMatch(/^Auto Runtime listening on port \d+\n$/u); + }); +}); diff --git a/packages/server/src/usage-ping/usage-ping-in-the-server.test.ts b/packages/server/src/usage-ping/usage-ping-in-the-server.test.ts index 33d1b2884..85f803411 100644 --- a/packages/server/src/usage-ping/usage-ping-in-the-server.test.ts +++ b/packages/server/src/usage-ping/usage-ping-in-the-server.test.ts @@ -1,69 +1,20 @@ import { once } from 'node:events'; -import { existsSync, mkdtempSync, rmSync } from 'node:fs'; -import { tmpdir } from 'node:os'; +import { existsSync } from 'node:fs'; import { join } from 'node:path'; import type { Environment } from '@on.auto/config'; -import { Function } from 'effect'; import { describe, expect, it, onTestFinished } from 'vitest'; -import { defaultServerOptions, startServer, type ServerOptions } from '../lifecycle/lifecycle.ts'; -import { runServer } from '../lifecycle/run-server.ts'; -import { firstStartNotice, usagePingOf, type PingFetch } from './usage-ping.ts'; - -interface Pings { - readonly fetch: PingFetch; - readonly urls: readonly string[]; - readonly signals: readonly AbortSignal[]; -} - -function folderOfTheTest(): string { - const folder = mkdtempSync(join(tmpdir(), 'auto-runtime-usage-ping-server-')); - onTestFinished(() => { - rmSync(folder, { recursive: true, force: true }); - }); - return folder; -} - -function recorded(answer: (signal: AbortSignal) => Promise): Pings { - const urls: string[] = []; - const signals: AbortSignal[] = []; - return { - urls, - signals, - fetch: (url, { signal }) => { - urls.push(url); - signals.push(signal); - return answer(signal); - }, - }; -} - -function answered(): Pings { - return recorded(() => Promise.resolve(new Response(''))); -} - -function unanswered(): Pings { - return recorded( - (signal) => - new Promise((_answer, refuse) => { - signal.addEventListener('abort', () => { - refuse(new Error('the ping was given up', { cause: signal.reason })); - }); - }), - ); -} - -function optionsPingingThrough(pings: Pings, runtimeHome: string): ServerOptions { - return { - ...defaultServerOptions, - usagePing: usagePingOf({ fetch: pings.fetch, runtimeHome, running: process }), - }; -} - -function settingsIn(folder: string, environment: Environment = {}): Environment { - return { HOST: '127.0.0.1', PORT: '0', LEDGER_FILE: join(folder, 'ledger.db'), ...environment }; -} +import { startServer } from '../lifecycle/lifecycle.ts'; +import { defaultServerOptions } from '../testing/entries/serving-options.ts'; +import { + answered, + folderOfTheTest, + optionsPingingThrough, + settingsIn, + unanswered, + type Pings, +} from '../testing/records/server-pings.ts'; async function landingPageOf(port: number): Promise { const response = await fetch(`http://127.0.0.1:${port}/`, { headers: { accept: 'text/html' } }); @@ -75,28 +26,6 @@ function abortOfTheFirst({ signals }: Pings): Promise { return first === undefined ? Promise.reject(new Error('no usage ping was sent')) : once(first, 'abort'); } -async function announced(folder: string): Promise { - const written: string[] = []; - const never = Promise.withResolvers().promise; - const serverProcess = { - env: settingsIn(folder), - stdout: { - write: (message: string) => { - written.push(message); - }, - }, - stderr: { - write: (_message: string, flushed: () => void) => { - flushed(); - }, - }, - exit: Function.constVoid, - }; - const server = await runServer(serverProcess, optionsPingingThrough(answered(), folder), never, never); - await server.stop(); - return written; -} - describe('a server whose usage ping is on', () => { it('sends it once it listens, and names its install in the link of its page to Studio', async () => { const folder = folderOfTheTest(); @@ -156,16 +85,3 @@ describe('a server whose usage ping is off', () => { }, ); }); - -describe('the announcement of a server whose usage ping is on', () => { - it('says after the port, at the first start alone, that it sends an anonymous usage ping', async () => { - const folder = folderOfTheTest(); - - const first = await announced(folder); - const second = await announced(folder); - - expect(first.slice(1)).toEqual([`${firstStartNotice}\n`]); - expect(second).toHaveLength(1); - expect(first[0]).toMatch(/^Auto Runtime listening on port \d+\n$/u); - }); -}); diff --git a/packages/server/src/usage-ping/usage-ping-settings.test.ts b/packages/server/src/usage-ping/usage-ping-settings.test.ts index 3d779bedf..e3e47e372 100644 --- a/packages/server/src/usage-ping/usage-ping-settings.test.ts +++ b/packages/server/src/usage-ping/usage-ping-settings.test.ts @@ -5,15 +5,15 @@ import { join } from 'node:path'; import type { Environment } from '@on.auto/config'; import { describe, expect, it, onTestFinished } from 'vitest'; -import { readSettings } from '../settings/settings.ts'; +import { readSettingsOf } from '../testing/settings/shipped-settings.ts'; function usagePingIn(environment: Environment): unknown { - return readSettings(environment).usagePing; + return readSettingsOf(environment).usagePing; } function errorFrom(environment: Environment): string { try { - readSettings(environment); + readSettingsOf(environment); } catch (error) { return String(error); } diff --git a/packages/server/src/usage-ping/usage-ping.test.ts b/packages/server/src/usage-ping/usage-ping.test.ts index 832630613..9fe7a3e71 100644 --- a/packages/server/src/usage-ping/usage-ping.test.ts +++ b/packages/server/src/usage-ping/usage-ping.test.ts @@ -6,7 +6,7 @@ import { join } from 'node:path'; import { describe, expect, it, onTestFinished } from 'vitest'; import { release } from '../composition/release.ts'; -import type { StartPing } from '../lifecycle/lifecycle.ts'; +import type { StartPing } from '../server-options/server-options.ts'; import type { LedgerSettings } from '../settings/ledger-settings.ts'; import type { UsagePingSettings } from './usage-ping-settings.ts'; import { firstStartNotice, usagePingOf, type PingFetch, type UsagePingHost } from './usage-ping.ts'; diff --git a/packages/server/src/usage-ping/usage-ping.ts b/packages/server/src/usage-ping/usage-ping.ts index c461c5e32..15cd5a3d0 100644 --- a/packages/server/src/usage-ping/usage-ping.ts +++ b/packages/server/src/usage-ping/usage-ping.ts @@ -1,7 +1,7 @@ import { Function } from 'effect'; import { release } from '../composition/release.ts'; -import type { Install, StartPing, UsagePing } from '../lifecycle/lifecycle.ts'; +import type { Install, StartPing, UsagePing } from '../server-options/server-options.ts'; import type { LedgerSettings } from '../settings/ledger-settings.ts'; import { installIdFileOf, installIn } from './install-id.ts'; import type { UsagePingSettings } from './usage-ping-settings.ts'; diff --git a/packages/server/src/workflows/host-work.test.ts b/packages/server/src/workflows/host-work.test.ts index da9adae50..340916fb8 100644 --- a/packages/server/src/workflows/host-work.test.ts +++ b/packages/server/src/workflows/host-work.test.ts @@ -1,13 +1,12 @@ -import { makeAppRuntime } from '@on.auto/api'; import { playbookOwnNamesOf, defineStartVersion } from '@on.auto/definitions'; import { echo } from '@on.auto/definitions/testing'; -import { ledgerLayer } from '@on.auto/ledger/sqlite3'; import { makeDispatcher } from '@on.auto/operations'; -import { filterSandboxOf, machineSandboxOf, programPool } from '@on.auto/workflow-engine/dsl'; +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; +import { filterSandboxOf, handedWorkersOf, machineSandboxOf, programPool } from '@on.auto/workflow-engine/dsl'; import { Effect } from 'effect'; import { describe, expect, it } from 'vitest'; -import { applicationLayer } from '../composition/application-layer.ts'; +import { memoryRuntime } from '../testing/records/memory-runtime.ts'; import { hostWorkOf } from './host-work.ts'; const unknownRun = { org: 'acme', playbook: 'alpha', id: '0199a3c4-7d2e-7c1a-9b3f-2f1e0d9c8b7a' }; @@ -16,8 +15,8 @@ const lineage = { causationId: 'cancel-1', correlationId: unknownRun.id }; describe('the machine the workflow host decides runs with', () => { it("is the coordination's functions, runtime and context of each record on the pool's sandbox, so no setting of the server reaches the machine's bound on tasks without waiting", async () => { - const runtime = await makeAppRuntime(applicationLayer(ledgerLayer({ fileName: ':memory:' }))); - const pool = programPool({ workers: 1, heapMegabytes: 64 }); + const runtime = await memoryRuntime(); + const pool = programPool({ handed: await handedWorkersOf(shippedFilesOnDisk), workers: 1, heapMegabytes: 64 }); const { machine } = hostWorkOf(runtime, makeDispatcher([]), { capabilities: [echo], startVersion: defineStartVersion([echo]), @@ -34,12 +33,12 @@ describe('the machine the workflow host decides runs with', () => { describe('the cancels the workflow host hands to the runtime of the server', () => { it('settle nothing for a run the ledger does not have, and record the cancel of a run that has not started yet', async () => { - const runtime = await makeAppRuntime(applicationLayer(ledgerLayer({ fileName: ':memory:' }))); + const runtime = await memoryRuntime(); const { waiting } = hostWorkOf(runtime, makeDispatcher([]), { capabilities: [echo], startVersion: defineStartVersion([echo]), names: playbookOwnNamesOf([echo]), - evaluations: { machine: machineSandboxOf(), filters: filterSandboxOf() }, + evaluations: { machine: machineSandboxOf(shippedFilesOnDisk), filters: filterSandboxOf(shippedFilesOnDisk) }, }); const settled = await Effect.runPromise( diff --git a/packages/server/src/workflows/reaction-dependencies.test.ts b/packages/server/src/workflows/reaction-dependencies.test.ts index 7a6fcf019..67049bee6 100644 --- a/packages/server/src/workflows/reaction-dependencies.test.ts +++ b/packages/server/src/workflows/reaction-dependencies.test.ts @@ -1,13 +1,11 @@ -import { makeAppRuntime } from '@on.auto/api'; import { playbookOwnNamesOf, defineStartVersion } from '@on.auto/definitions'; import { echo } from '@on.auto/definitions/testing'; -import { ledgerLayer } from '@on.auto/ledger/sqlite3'; import type { PlaybookRequest, Dispatcher, Outcome } from '@on.auto/operations'; import { StartRefused, StartRejected } from '@on.auto/workflow-host'; import { Effect } from 'effect'; import { describe, expect, it, onTestFinished } from 'vitest'; -import { applicationLayer } from '../composition/application-layer.ts'; +import { memoryRuntime } from '../testing/records/memory-runtime.ts'; import { reactionsOf } from './reaction-dependencies.ts'; const start = { @@ -23,7 +21,7 @@ const start = { } as const; async function reactingWith(answers: readonly Outcome[]) { - const runtime = await makeAppRuntime(applicationLayer(ledgerLayer({ fileName: ':memory:' }))); + const runtime = await memoryRuntime(); onTestFinished(() => runtime.dispose()); const requests: PlaybookRequest[] = []; const dispatcher: Dispatcher = { diff --git a/packages/server/src/workflows/reaction-dependencies.ts b/packages/server/src/workflows/reaction-dependencies.ts index aae5c2f91..804e41462 100644 --- a/packages/server/src/workflows/reaction-dependencies.ts +++ b/packages/server/src/workflows/reaction-dependencies.ts @@ -59,7 +59,7 @@ export function reactionsOf( runtime: AppRuntime, dispatcher: Dispatcher, { startVersion, names }: Reacting, -): ReactionOptions { +): Omit { return { definitionType: workflows, start: (start) => diff --git a/packages/server/src/workflows/workflow-checks.test.ts b/packages/server/src/workflows/workflow-checks.test.ts index 243904995..850225af2 100644 --- a/packages/server/src/workflows/workflow-checks.test.ts +++ b/packages/server/src/workflows/workflow-checks.test.ts @@ -35,7 +35,7 @@ function recordingChecks(): { readonly checked: readonly CheckRequest[]; readonl return pool.check(request, signal); }, }); - return { checked, poolOf: (settings) => recording(patientWorkerPool(settings)) }; + return { checked, poolOf: (settings, handed) => recording(patientWorkerPool(settings, handed)) }; } function saving(name: string, source: string) { diff --git a/packages/server/src/workflows/workflow-deadlines.test.ts b/packages/server/src/workflows/workflow-deadlines.test.ts index d8f88ce20..a276c2be1 100644 --- a/packages/server/src/workflows/workflow-deadlines.test.ts +++ b/packages/server/src/workflows/workflow-deadlines.test.ts @@ -31,7 +31,7 @@ function decidingPastTheDeadline(pool: ProgramPool): ProgramPool { return { ...pool, evaluations: { ...pool.evaluations, machine: { ...machine, unit: unitPastItsDeadline } } }; } -const poolOf: ProgramPoolOf = (settings) => decidingPastTheDeadline(patientWorkerPool(settings)); +const poolOf: ProgramPoolOf = (settings, handed) => decidingPastTheDeadline(patientWorkerPool(settings, handed)); describe('the start of a workflow whose expressions run past the deadline of an input, over HTTP', () => { it('is unavailable, naming the expression, the deadline of an input and the step', async () => { diff --git a/packages/server/src/workflows/workflows.ts b/packages/server/src/workflows/workflows.ts index 5fd02abef..d8a7ecf48 100644 --- a/packages/server/src/workflows/workflows.ts +++ b/packages/server/src/workflows/workflows.ts @@ -1,13 +1,19 @@ import type { AppRuntime } from '@on.auto/api'; import { callMarginMs, defineSendRunEvent, makeWorkflowAdapter, type ExpressionCheck } from '@on.auto/coordination'; import { defineStartVersion, type PlaybookOwnNames, type Capability, type CapabilityFacts } from '@on.auto/definitions'; -import { makeCatalog, makeDispatcher, type DispatcherServices, type RouteRegistration } from '@on.auto/operations'; +import { + makeCatalog, + makeDispatcher, + type DispatcherServices, + type RouteRegistration, + type ShippedFiles, +} from '@on.auto/operations'; import type { WorkflowHost } from '@on.auto/workflow-host'; import { Effect } from 'effect'; import { playbookOperationsServing } from '../composition/playbook-operations.ts'; import { routesFor, type ServedCatalog } from '../composition/served-routes.ts'; -import type { Served } from '../lifecycle/lifecycle.ts'; +import type { Served } from '../server-options/server-options.ts'; import { openedHost, type HostParts } from './host-dependencies.ts'; interface OrgOperation { @@ -26,6 +32,7 @@ export interface WorkflowParts extends HostParts { readonly names: PlaybookOwnNames; readonly identityIssuer: string; readonly retryStateKeys: ServedCatalog['retryStateKeys']; + readonly shippedFiles: ShippedFiles; } export function longestCallOf(capabilities: readonly Pick[]): number { @@ -67,7 +74,7 @@ export async function serveWorkflows(runtime: AppRuntime, pa ...parts.playbookOperations, defineSendRunEvent(host, names), ]); - const { identityIssuer, retryStateKeys } = parts; - const served = { catalog, dispatcher, capabilities, identityIssuer, retryStateKeys }; - return { routes: routesFor(runtime, served), stopWork: host.stop }; + const { identityIssuer, retryStateKeys, shippedFiles } = parts; + const served = { catalog, dispatcher, capabilities, identityIssuer, retryStateKeys, shippedFiles }; + return { routes: await routesFor(runtime, served), stopWork: host.stop }; } diff --git a/packages/tool-servers/README.md b/packages/tool-servers/README.md index 0c9050a2f..fbc44a643 100644 --- a/packages/tool-servers/README.md +++ b/packages/tool-servers/README.md @@ -10,7 +10,7 @@ How a playbook reaches the outside world: the tool servers of its org, MCP serve ## Settings -`readToolServerSettings(environment, { modelProviders, documentsDirectory?, providers? })` reads `TOOL_SERVERS`, JSON, which the server also writes from the configuration file's `tool_servers`: an object with one entry per server, keyed by the name a function writes in `server/tool`, which holds no two hyphens together and none at the end (`endsAtTheFirstSeparator` of `src/names/model-facing-names.ts`), since the model sees a hyphen as an underscore and a tool as `server__tool`. Each entry is one of three types, `mcp`, a remote MCP server spoken to over Streamable HTTP, `stdio`, an MCP server the runtime runs as a process, and `openapi`, an HTTP API described by an OpenAPI document, taken from `url`, `command` or `document` when `type` is left out; nothing in a function says which type it names. Beside `servers`, the settings keep `referencedVariables`, the name of each variable the entries refer to, one with a fallback among them, from which the server learns the variables it reads for itself without decoding `TOOL_SERVERS` again. `ServerSettings` carry `connection`, `null` for an entry of the file and a connection's id for the settings of a connection's link, and `provider`, the name of the provider the server is made from, the one an entry of the file names or a connection was made from, `null` for a server given by hand, as `search` lists it. Each entry carries the tools of its server, in `ServerSettings` as `allowed`, the names or `null`: the tools of the server a function may name and `execute` may call, each named as the server lists it, without the server's name and without `*`. Every tool of the server when it is left out, so an entry always allows at least one tool. It is the operator's one list, for runs and for calls made by hand alike. +`readToolServerSettings(environment, { modelProviders, documents?, providers? })` reads `TOOL_SERVERS`, JSON, which the server also writes from the configuration file's `tool_servers`: an object with one entry per server, keyed by the name a function writes in `server/tool`, which holds no two hyphens together and none at the end (`endsAtTheFirstSeparator` of `src/names/model-facing-names.ts`), since the model sees a hyphen as an underscore and a tool as `server__tool`. Each entry is one of three types, `mcp`, a remote MCP server spoken to over Streamable HTTP, `stdio`, an MCP server the runtime runs as a process, and `openapi`, an HTTP API described by an OpenAPI document, taken from `url`, `command` or `document` when `type` is left out; nothing in a function says which type it names. Beside `servers`, the settings keep `referencedVariables`, the name of each variable the entries refer to, one with a fallback among them, from which the server learns the variables it reads for itself without decoding `TOOL_SERVERS` again. `ServerSettings` carry `connection`, `null` for an entry of the file and a connection's id for the settings of a connection's link, and `provider`, the name of the provider the server is made from, the one an entry of the file names or a connection was made from, `null` for a server given by hand, as `search` lists it. Each entry carries the tools of its server, in `ServerSettings` as `allowed`, the names or `null`: the tools of the server a function may name and `execute` may call, each named as the server lists it, without the server's name and without `*`. Every tool of the server when it is left out, so an entry always allows at least one tool. It is the operator's one list, for runs and for calls made by hand alike. The list is checked at start (`src/settings/tool-lists.ts`), at the pointer of the name and never with a value: a name that is not a tool name, such as `graph/search`; `*`, which is what leaving it out means; a name listed twice; and an empty list. Whether a server has a tool is learned from the server, or from its document, never from the settings. @@ -23,7 +23,7 @@ The list is checked at start (`src/settings/tool-lists.ts`), at the pointer of t | `command` | stdio | An installed, pinned command, never a package downloaded at start such as `npx -y` | | `args` | stdio | The arguments of the command | | `env` | stdio | The whole environment of the process, the value of a reference a secret: it inherits nothing else, so give `PATH` or an absolute `command` | -| `document` | openapi | The OpenAPI document, 3.0 or 3.1, in JSON or YAML: an https URL, an http URL on a loopback address, or a path relative to `documentsDirectory`, the configuration file's, that stays in it | +| `document` | openapi | The OpenAPI document, 3.0 or 3.1, in JSON or YAML: an https URL, an http URL on a loopback address, or a path relative to the directory `documents` names, the configuration file's, that stays in it | | `base_url` | openapi | Where the calls go, in place of the first server the document names; required when it names none or one relative to a file. An https URL, or an http URL on a loopback address | | `query_params` | openapi | Query parameters sent with every call that the document's security schemes do not place; the value of a reference is a secret, a value written out is not | | `security` | openapi | The credential of each security scheme of the document the entry gives, by the scheme's name (see [The credentials of an HTTP API](#the-credentials-of-an-http-api)) | @@ -126,7 +126,7 @@ The document's `components.securitySchemes` and the `security` of the document a An `openapi` entry is a `ServerLink` like the others (`src/clients/server-links.ts`), whose client is the bridge (`src/openapi/api-client.ts`): it answers `listTools` and `call` of the client contract (`src/clients/server-client.ts`), so the rest of `ToolAccess`, the bounds of a run, the scrubbing, the recording, a call's facts and the failures counted against a run are the same code for an MCP server and for an HTTP API, which the shared behaviour suite of `src/access-testing` proves by running the same tests over the fake of each. -At start `describedToolServers` reads the document, from its URL through the server's outbound fetch, without the entry's headers and following no redirect, which is refused naming the entry and its status, since the document chooses where the calls go, within 10 s and 4 MiB, or from its file, refuses one that is neither JSON nor YAML or nests deeper than 128 levels, and decodes it with a schema of the parts the bridge reads alone (`src/openapi/api-document.ts`): the version, the servers, the paths with their operations, parameters and request bodies, and the `components` they refer to. Swagger 2.0 and any version but 3.0 and 3.1 are refused at `/swagger` or `/openapi`, and a part of another shape at its place. A path that does not begin with `/` is refused at `/paths/`, since joined to `base_url` it would extend its host or its port, and so is one with a `.` or `..` segment, written plainly, percent-encoded or split by a tab or a line break, which an address drops, or with a `?` or a `#`. Each operation becomes one tool (`src/openapi/api-tools.ts`): +At start `describedToolServers` reads the document, from its URL through the server's outbound fetch, without the entry's headers and following no redirect, which is refused naming the entry and its status, since the document chooses where the calls go, within 10 s and 4 MiB, or from its file, or from the text the providers ship, whose bytes they measured as they read it, each within 4 MiB, refuses one that is neither JSON nor YAML or nests deeper than 128 levels, and decodes it with a schema of the parts the bridge reads alone (`src/openapi/api-document.ts`): the version, the servers, the paths with their operations, parameters and request bodies, and the `components` they refer to. Swagger 2.0 and any version but 3.0 and 3.1 are refused at `/swagger` or `/openapi`, and a part of another shape at its place. A path that does not begin with `/` is refused at `/paths/`, since joined to `base_url` it would extend its host or its port, and so is one with a `.` or `..` segment, written plainly, percent-encoded or split by a tab or a line break, which an address drops, or with a `?` or a `#`. Each operation becomes one tool (`src/openapi/api-tools.ts`): - its name is the `operationId`, each character outside the grammar of a tool name an underscore and cut at 128 characters, or, without one, the method and the path with the leading `/` dropped, `{x}` as `x` and `/` as `_`, in lower case, read in one pass however many braces the path holds, as the variables of a server's address are, as `get_invoices_id` for `GET /invoices/{id}`; two operations of one name refuse the document, naming both places, as do more than 500 operations; - its description is the operation's `summary` and `description`, cut at 4 KiB as descriptions are; @@ -152,7 +152,7 @@ The bridge's client adds about 0.01 ms to a `fetch` on loopback, building the re ## The connections of an org -A connection is a tool server a person added to an org, a provider of `@on.auto/providers` by its name or a server given by hand, which `@on.auto/connections` keeps; a connection made from a provider whose document lies beside the providers' entries is read with `readEntry` and the providers' folder as its `documentsDirectory`, as a file's entry is read with the configuration's, so its document is a file the link reads, and every other connection's document is a URL. The access takes them from a source, `connections` of its options, a `ConnectionSource` (`src/org-servers/connection-source.ts`), and knows nothing of the ledger: `connectionsOf(org)` answers each connection of the org that is not removed, with its id, `serving`, its name, `type` `mcp` or `openapi`, `provider`, the name of the provider it was added from or `null`, `orgs`, `playbooks`, `allowed`, `record_content` and `request_id`, and its state: `connected`, with `server`, an effect that reads its credential and checks the operator's rules and answers its `ServerSettings` and `reaches`, whose `auth`, for a connection connected by consent, is the third kind of credential, `consent`, which the source composes after reading the entry with no credential placed and the file never writes: the connection's way, its server, the client it recorded, the return address, `reaches`, the row of its tokens as the source read it and the store of that row, `ConnectionTokens`, or `unready`, with the `because` and words of a connection that is pending or broken. `takesConnections` says whether people may add connections. Without a source, `noConnections`, the access has the file's servers alone. +A connection is a tool server a person added to an org, a provider of `@on.auto/providers` by its name or a server given by hand, which `@on.auto/connections` keeps; a connection made from a provider whose document lies beside the providers' entries is read with `readEntry` and `documents` of the kind `shipped`, the documents `readProviders` of `@on.auto/providers` read, each a `ShippedDocument`, its text and the bytes it takes (`shippedDocumentOf`), where a file's entry is read with the directory of the configuration, so its document is the text the providers handed on, refused at its `document` where they ship none of that name, and every other connection's document is a URL. The access takes them from a source, `connections` of its options, a `ConnectionSource` (`src/org-servers/connection-source.ts`), and knows nothing of the ledger: `connectionsOf(org)` answers each connection of the org that is not removed, with its id, `serving`, its name, `type` `mcp` or `openapi`, `provider`, the name of the provider it was added from or `null`, `orgs`, `playbooks`, `allowed`, `record_content` and `request_id`, and its state: `connected`, with `server`, an effect that reads its credential and checks the operator's rules and answers its `ServerSettings` and `reaches`, whose `auth`, for a connection connected by consent, is the third kind of credential, `consent`, which the source composes after reading the entry with no credential placed and the file never writes: the connection's way, its server, the client it recorded, the return address, `reaches`, the row of its tokens as the source read it and the store of that row, `ConnectionTokens`, or `unready`, with the `because` and words of a connection that is pending or broken. `takesConnections` says whether people may add connections. Without a source, `noConnections`, the access has the file's servers alone. Every way in reads the servers of an org the same way (`src/org-servers/org-access.ts`): the file's entries, then the org's connections, each an `OrgServer` with the settings it serves by, the connection's id or `null` as `connection`, and its link. Within an org a name resolves to the file's entry first: a connection whose name an entry of the file holds, for this org or another, which a later edit of the file can give it, is unavailable, `connection_not_allowed`, in the sentence `add_connection` refuses that name with, `takenByTheConfiguration`, followed by what to do, `remove it and add it again under another name`, and is never called (`withTheFileFirst` and `namedIn` in `src/org-servers/org-servers.ts`, the naming in `src/org-servers/org-naming.ts`). diff --git a/packages/tool-servers/src/api-testing/described-documents.ts b/packages/tool-servers/src/api-testing/described-documents.ts index 9730f00cd..d18ba6b86 100644 --- a/packages/tool-servers/src/api-testing/described-documents.ts +++ b/packages/tool-servers/src/api-testing/described-documents.ts @@ -7,14 +7,14 @@ import { onTestFinished } from 'vitest'; import type { DescribedApiSettings, Describing } from '../openapi/described-api.ts'; import { describedToolServers } from '../openapi/described-servers.ts'; -import { readToolServerSettings } from '../settings/settings-reading.ts'; +import { readToolServerSettings, type ToolServerSettingsContext } from '../settings/settings-reading.ts'; import { serveOnLoopback } from '../testing/loopback-server.ts'; import { fakeApiDocument } from './fake-api-document.ts'; type Entries = Readonly>; export interface DescribingOptions extends Partial { - readonly directory?: string; + readonly documents?: ToolServerSettingsContext['documents']; } const refused = 'The documents of the tool servers could not be read. TOOL_SERVERS: /graph'; @@ -65,11 +65,15 @@ export function countedFetch( }; } -export function describing(entries: Entries, { directory, ...more }: DescribingOptions = {}) { +export function documentsIn(directory: string): DescribingOptions { + return { documents: { kind: 'directory', path: directory } }; +} + +export function describing(entries: Entries, { documents, ...more }: DescribingOptions = {}) { const leftOut: (Parameters[1] & { readonly server: string })[] = []; const effect = readToolServerSettings( { TOOL_SERVERS: JSON.stringify(entries), GRAPH_API_KEY: 'graph-api-key-4f1d9a7c2b' }, - { modelProviders: [], ...(directory === undefined ? {} : { documentsDirectory: directory }) }, + { modelProviders: [], ...(documents === undefined ? {} : { documents }) }, ).pipe( Effect.flatMap((settings) => describedToolServers(settings, { diff --git a/packages/tool-servers/src/index.ts b/packages/tool-servers/src/index.ts index d51015202..2cfdb0dc1 100644 --- a/packages/tool-servers/src/index.ts +++ b/packages/tool-servers/src/index.ts @@ -138,8 +138,10 @@ export type { ConsentAuth, ConsentCredential, ServerSettings, + ShippedDocument, ToolServerSettings, } from './settings/server-settings.ts'; +export { shippedDocumentOf } from './settings/server-settings.ts'; export type { Describing, ToolServers } from './openapi/described-api.ts'; export type { LeftOutOperation } from './openapi/api-tools.ts'; export { ToolServerDocumentRefused, describedToolServers } from './openapi/described-servers.ts'; diff --git a/packages/tool-servers/src/openapi/described-servers.test.ts b/packages/tool-servers/src/openapi/described-servers.test.ts index 2c5b1105b..55aae56b1 100644 --- a/packages/tool-servers/src/openapi/described-servers.test.ts +++ b/packages/tool-servers/src/openapi/described-servers.test.ts @@ -7,7 +7,10 @@ import { describing, directoryWith, documentWithServer, + documentsIn, + refusalOf, } from '../api-testing/described-documents.ts'; +import { shippedDocumentOf } from '../settings/server-settings.ts'; import { serveFakeApi, type FakeApiServer } from '../testing/index.ts'; const notes = @@ -78,6 +81,32 @@ describe('the operations of an openapi server that need a header', () => { }); }); +describe('the document of an openapi server a provider ships', () => { + it('is the text the providers read, described as a file is, and nothing is read from the directory of the server', async () => { + const graph = new TextEncoder().encode(documentWithServer('https://graph.example.com/v1')); + const byName = new Map([['documents/graph.json', shippedDocumentOf(graph)]]); + + const described = await describedApiOf( + { graph: { document: 'documents/graph.json', org: 'acme', security: keyed } }, + { documents: { kind: 'shipped', byName } }, + ); + + expect(described?.baseUrl).toBe('https://graph.example.com/v1'); + expect(described?.operations).toHaveLength(9); + }); + + it('is refused when the providers ship no document of that name', async () => { + const refused = await refusalOf( + { graph: { document: 'documents/notes.json', org: 'acme' } }, + { documents: { kind: 'shipped', byName: new Map() } }, + ); + + expect(refused).toBe( + 'The tool server settings are invalid. TOOL_SERVERS: /graph/document: Expected an https URL, or an http URL on a loopback address, or a document the providers ship', + ); + }); +}); + describe('the document of an openapi server, read from a file at start', () => { it('is JSON or YAML in the directory of the configuration file', async () => { const directory = directoryWith({ @@ -87,9 +116,9 @@ describe('the document of an openapi server, read from a file at start', () => { const graph = await describedApiOf( { graph: { document: 'graph.json', org: 'acme', security: keyed } }, - { directory }, + documentsIn(directory), ); - const listed = await describedApiOf({ notes: { document: 'notes.yaml', org: 'acme' } }, { directory }); + const listed = await describedApiOf({ notes: { document: 'notes.yaml', org: 'acme' } }, documentsIn(directory)); expect(graph?.baseUrl).toBe('https://graph.example.com/v1'); expect(graph?.operations).toHaveLength(9); @@ -104,9 +133,9 @@ describe('the document of an openapi server, read from a file at start', () => { 'graph.json': documentWithServer('https://{region}.graph.example.com/{stage}', { region: { default: 'eu' } }), }); - expect((await describedApiOf({ graph: { document: 'graph.json', org: 'acme' } }, { directory }))?.baseUrl).toBe( - 'https://eu.graph.example.com/%7Bstage%7D', - ); + expect( + (await describedApiOf({ graph: { document: 'graph.json', org: 'acme' } }, documentsIn(directory)))?.baseUrl, + ).toBe('https://eu.graph.example.com/%7Bstage%7D'); }); it('sends the calls to the first server it names without the slashes it ends in, as base_url is read', async () => { @@ -118,7 +147,8 @@ describe('the document of an openapi server, read from a file at start', () => { expect( await Promise.all( ['graph.json', 'root.json'].map( - async (document) => (await describedApiOf({ graph: { document, org: 'acme' } }, { directory }))?.baseUrl, + async (document) => + (await describedApiOf({ graph: { document, org: 'acme' } }, documentsIn(directory)))?.baseUrl, ), ), ).toEqual(['https://graph.example.com/v1', 'https://graph.example.com']); @@ -130,8 +160,8 @@ describe('the address of the first server a document names', () => { const braces = '{'.repeat(200_000); const directory = directoryWith({ 'graph.json': documentWithServer(`https://graph.example.com/${braces}`) }); - expect((await describedApiOf({ graph: { document: 'graph.json', org: 'acme' } }, { directory }))?.baseUrl).toBe( - `https://graph.example.com/${'%7B'.repeat(200_000)}`, - ); + expect( + (await describedApiOf({ graph: { document: 'graph.json', org: 'acme' } }, documentsIn(directory)))?.baseUrl, + ).toBe(`https://graph.example.com/${'%7B'.repeat(200_000)}`); }); }); diff --git a/packages/tool-servers/src/openapi/document-reading.ts b/packages/tool-servers/src/openapi/document-reading.ts index 346f7f85b..b28758cad 100644 --- a/packages/tool-servers/src/openapi/document-reading.ts +++ b/packages/tool-servers/src/openapi/document-reading.ts @@ -8,7 +8,7 @@ import { Data, Result } from 'effect'; import { at } from '../settings/checked.ts'; import { toolServersSetting } from '../settings/server-entries.ts'; -import type { ApiDocumentSource } from '../settings/server-settings.ts'; +import type { ApiDocumentSource, ShippedDocument } from '../settings/server-settings.ts'; import { DocumentRefused } from './described-api.ts'; export const documentBounds = { bytes: 4_194_304, operations: 500, readMs: 10_000, nesting: 128 } as const; @@ -124,7 +124,20 @@ async function textIn(path: string, entry: string): Promise { } } +function shippedText({ text, bytes }: ShippedDocument, entry: string): Text { + return bytes > documentBounds.bytes ? Result.fail(refused(entry, tooLarge)) : Result.succeed(text); +} + +function textOf(source: ApiDocumentSource, reading: DocumentReading): Promise { + if (source.kind === 'url') { + return textAt(source.url, reading); + } + return source.kind === 'file' + ? textIn(source.path, reading.entry) + : Promise.resolve(shippedText(source.document, reading.entry)); +} + export async function documentOf(source: ApiDocumentSource, reading: DocumentReading): Promise { - const text = source.kind === 'url' ? await textAt(source.url, reading) : await textIn(source.path, reading.entry); + const text = await textOf(source, reading); return Result.flatMap(text, (read) => boundedParse(read, reading.entry)); } diff --git a/packages/tool-servers/src/openapi/document-refusals.test.ts b/packages/tool-servers/src/openapi/document-refusals.test.ts index 9671ee5f3..102a6e8e7 100644 --- a/packages/tool-servers/src/openapi/document-refusals.test.ts +++ b/packages/tool-servers/src/openapi/document-refusals.test.ts @@ -9,9 +9,11 @@ import { directoryWith, documentProblem, documentWithServer, + documentsIn, refusalOf, servedDocumentAt, } from '../api-testing/described-documents.ts'; +import { shippedDocumentOf } from '../settings/server-settings.ts'; import { fakeApiDocument, serveFakeApi } from '../testing/index.ts'; const brokenFetch: typeof globalThis.fetch = () => @@ -26,7 +28,7 @@ const brokenFetch: typeof globalThis.fetch = () => ); function refusedIn(directory: string) { - return (document: string) => refusalOf({ graph: { document, org: 'acme' } }, { directory }); + return (document: string) => refusalOf({ graph: { document, org: 'acme' } }, documentsIn(directory)); } describe('a document refused at start because its URL', () => { @@ -72,16 +74,20 @@ describe('a document refused at start because its URL', () => { }); describe('a document refused at start because of its bytes', () => { - it('when it takes more than 4 MiB at its URL or in its file', async () => { + it('when it takes more than 4 MiB at its URL, in its file or in the bytes the providers measured as they read it', async () => { const large = 'x'.repeat(4_194_305); const served = await servedDocumentAt(() => Promise.resolve(new Response(large))); const directory = directoryWith({ 'graph.json': large }); + const byName = new Map([['graph.json', shippedDocumentOf(new TextEncoder().encode('é'.repeat(2_097_153)))]]); const tooLarge = documentProblem( 'The document takes more than the 4194304 bytes a document of a tool server may take', ); expect(await refusalOf({ graph: { document: served, org: 'acme' } })).toBe(tooLarge); expect(await refusedIn(directory)('graph.json')).toBe(tooLarge); + expect( + await refusalOf({ graph: { document: 'graph.json', org: 'acme' } }, { documents: { kind: 'shipped', byName } }), + ).toBe(tooLarge); }); it('when its file does not exist or cannot be read', async () => { @@ -156,7 +162,7 @@ describe('a document refused at start because a schema refers outside it', () => Object.fromEntries(elsewhere.map((address, index) => [`graph-${index}.json`, referringTo(address)])), ); const refused = (document: string) => - refusalOf({ graph: { document, org: 'acme' } }, { directory, fetch: counted.fetch }); + refusalOf({ graph: { document, org: 'acme' } }, { ...documentsIn(directory), fetch: counted.fetch }); expect(await Promise.all(elsewhere.map((_, index) => refused(`graph-${index}.json`)))).toEqual( elsewhere.map(() => @@ -182,7 +188,7 @@ describe('a document refused at start because a path would take its calls elsewh const refusal = await refusalOf( { graph: { document: 'graph.json', base_url: `http://127.0.0.1:${port.slice(0, -1)}`, org: 'acme' } }, - { directory }, + documentsIn(directory), ); expect(refusal).toContain( diff --git a/packages/tool-servers/src/settings/api-entry-checks.test.ts b/packages/tool-servers/src/settings/api-entry-checks.test.ts index 6149691a0..588c29000 100644 --- a/packages/tool-servers/src/settings/api-entry-checks.test.ts +++ b/packages/tool-servers/src/settings/api-entry-checks.test.ts @@ -18,10 +18,10 @@ const documentShape = `${pinned}, or a path relative to the directory of the con const misplaced = 'Holds a reference to an environment variable, which only headers, query_params, security, env and auth may hold: an argument shows in the listing of the processes of the machine, and the URL of a server or of a document is no place for a credential'; -function read(servers: unknown, documentsDirectory?: string) { +function read(servers: unknown, path?: string) { return readToolServerSettings( { TOOL_SERVERS: JSON.stringify(servers), BILLING_API_KEY: apiKey }, - { modelProviders: [], ...(documentsDirectory === undefined ? {} : { documentsDirectory }) }, + { modelProviders: [], ...(path === undefined ? {} : { documents: { kind: 'directory', path } }) }, ); } diff --git a/packages/tool-servers/src/settings/api-entry-checks.ts b/packages/tool-servers/src/settings/api-entry-checks.ts index 8ff318471..0fcaa6df8 100644 --- a/packages/tool-servers/src/settings/api-entry-checks.ts +++ b/packages/tool-servers/src/settings/api-entry-checks.ts @@ -14,7 +14,7 @@ import { } from './checked.ts'; import { checkedAuth, checkedHeaders } from './remote-checks.ts'; import type { ServerEntryFields } from './server-entries.ts'; -import type { ApiDocumentSource, OpenApiServerSettings } from './server-settings.ts'; +import type { ApiDocumentSource, DocumentsPlace, OpenApiServerSettings, ShippedDocument } from './server-settings.ts'; export type ApiTransport = Pick< OpenApiServerSettings, @@ -28,15 +28,32 @@ function isWithin(path: string): boolean { return !isAbsolute(path) && normal !== '..' && !normal.startsWith(`..${sep}`); } -function checkedDocument(name: string, document: string, directory: string): Checked { +function fileDocument(name: string, document: string, directory: string): Checked { + return isWithin(document) && document.trim() !== '' + ? accepted({ kind: 'file', path: resolve(directory, document) }) + : refused(name, ['document'], documentShape); +} + +function shippedDocument( + name: string, + document: string, + byName: ReadonlyMap, +): Checked { + const shipped = byName.get(document); + return shipped === undefined + ? refused(name, ['document'], `${pinnedUrlShape}, or a document the providers ship`) + : accepted({ kind: 'shipped', document: shipped }); +} + +function checkedDocument(name: string, document: string, documents: DocumentsPlace): Checked { if (URL.canParse(document)) { return isPinnableUrl(document) ? accepted({ kind: 'url', url: document }) : refused(name, ['document'], documentShape); } - return isWithin(document) && document.trim() !== '' - ? accepted({ kind: 'file', path: resolve(directory, document) }) - : refused(name, ['document'], documentShape); + return documents.kind === 'shipped' + ? shippedDocument(name, document, documents.byName) + : fileDocument(name, document, documents.path); } function checkedBaseUrl(name: string, baseUrl: string | undefined): Checked { @@ -83,9 +100,9 @@ export function apiTransport( name: string, fields: ServerEntryFields, document: string, - directory: string, + documents: DocumentsPlace, ): Checked { - const source = checkedDocument(name, document, directory); + const source = checkedDocument(name, document, documents); const baseUrl = checkedBaseUrl(name, fields.base_url); const headers = checkedHeaders(name, fields.headers ?? {}, 'openapi'); const query = checkedQuery(name, fields.query_params ?? {}); diff --git a/packages/tool-servers/src/settings/entry-checks.ts b/packages/tool-servers/src/settings/entry-checks.ts index 5036c426d..8aa1fa80d 100644 --- a/packages/tool-servers/src/settings/entry-checks.ts +++ b/packages/tool-servers/src/settings/entry-checks.ts @@ -7,7 +7,13 @@ import { apiTransport } from './api-entry-checks.ts'; import { accepted, at, problemsOf, refused, rejected, redactedValuesOf, type Checked } from './checked.ts'; import { checkedAuth, checkedHeaders } from './remote-checks.ts'; import { toolServersSetting, type ServerEntryFields } from './server-entries.ts'; -import type { McpServerSettings, ServerSettings, ServerType, StdioServerSettings } from './server-settings.ts'; +import type { + DocumentsPlace, + McpServerSettings, + ServerSettings, + ServerType, + StdioServerSettings, +} from './server-settings.ts'; import { toolListProblems } from './tool-lists.ts'; type Transport = S extends ServerSettings @@ -133,7 +139,7 @@ function transportOf( name: string, fields: ServerEntryFields, type: ServerType | undefined, - directory: string, + documents: DocumentsPlace, ): Checked> { const { url, command, document } = fields; if (type === 'mcp' && url !== undefined) { @@ -143,17 +149,17 @@ function transportOf( return stdioTransport(name, fields, command); } return type === 'openapi' && document !== undefined - ? apiTransport(name, fields, document, directory) + ? apiTransport(name, fields, document, documents) : missingTransport(name, type); } function checkedTransport( name: string, fields: ServerEntryFields, - directory: string, + documents: DocumentsPlace, ): Checked> { const type = typeOf(fields); - const transport = transportOf(name, fields, type, directory); + const transport = transportOf(name, fields, type, documents); const foreign = type === undefined ? [] : foreignFieldProblems(name, fields, type); return transport.ok && foreign.length === 0 ? transport : rejected([...foreign, ...problemsOf(transport)]); } @@ -173,7 +179,7 @@ export function servedFieldsOf({ playbooks, allowed, record_content, request_id interface EntryContext { readonly modelProviders: readonly string[]; - readonly directory: string; + readonly documents: DocumentsPlace; readonly secrets: readonly Redacted.Redacted[]; readonly provider: string | null; } @@ -181,11 +187,11 @@ interface EntryContext { export function checkedEntry( name: string, fields: ServerEntryFields, - { modelProviders, directory, secrets, provider }: EntryContext, + { modelProviders, documents, secrets, provider }: EntryContext, ): Result.Result { const named = checkedName(name, modelProviders); const scope = servedScopeOf(toolServersSetting, name, fields, 'server'); - const transport = checkedTransport(name, fields, directory); + const transport = checkedTransport(name, fields, documents); const toolLists = toolListProblems(name, fields); if (!named.ok || Result.isFailure(scope) || !transport.ok || toolLists.length > 0) { return Result.fail([ diff --git a/packages/tool-servers/src/settings/provider-entries.test.ts b/packages/tool-servers/src/settings/provider-entries.test.ts index 3eccecba8..9afff7aaf 100644 --- a/packages/tool-servers/src/settings/provider-entries.test.ts +++ b/packages/tool-servers/src/settings/provider-entries.test.ts @@ -55,7 +55,7 @@ function providers() { ['ledgers', ledgers], ['refusing', refusing], ]), - documents, + { kind: 'directory', path: documents }, ); } diff --git a/packages/tool-servers/src/settings/provider-entries.ts b/packages/tool-servers/src/settings/provider-entries.ts index b14d0e52b..2e859fea0 100644 --- a/packages/tool-servers/src/settings/provider-entries.ts +++ b/packages/tool-servers/src/settings/provider-entries.ts @@ -10,6 +10,7 @@ import { Function, Result } from 'effect'; import { at } from './checked.ts'; import { toolServersSetting, type ServerEntryFields } from './server-entries.ts'; +import type { DocumentsPlace } from './server-settings.ts'; export interface CredentialGiven { readonly kind?: string | undefined; @@ -29,7 +30,7 @@ export interface ProviderNamed { export interface ProviderResolved { readonly written: ServerEntryFields; readonly placedValues: ReadonlyMap; - readonly documentsDirectory: string; + readonly documents: DocumentsPlace; readonly writtenAt: (pointer: string) => string; } @@ -40,14 +41,14 @@ export const noProviders: ProviderResolver = ({ provider }) => export interface EntryReading { readonly placedValues: ReadonlyMap | undefined; - readonly documentsDirectory: string | undefined; + readonly documents: DocumentsPlace | undefined; readonly providers: ProviderResolver | undefined; } export interface EntryToRead { readonly written: ServerEntryFields; readonly placedValues: ReadonlyMap; - readonly documentsDirectory: string; + readonly documents: DocumentsPlace; readonly provider: string | null; readonly placed: (problems: readonly SettingProblem[]) => readonly SettingProblem[]; } @@ -185,7 +186,7 @@ function plainEntry(written: ServerEntryFields, reading: EntryReading): EntryToR return { written, placedValues: reading.placedValues ?? new Map(), - documentsDirectory: reading.documentsDirectory ?? process.cwd(), + documents: reading.documents ?? { kind: 'directory', path: process.cwd() }, provider: null, placed: Function.identity, }; @@ -209,7 +210,7 @@ function resolvedEntry( onSuccess: (resolved): EntryToRead => ({ written: { ...resolved.written, ...perConnectionOf(written) }, placedValues: resolved.placedValues, - documentsDirectory: resolved.documentsDirectory, + documents: resolved.documents, provider, placed: placedIn(name, (pointer) => writtenPointer(resolved.writtenAt(pointer))), }), diff --git a/packages/tool-servers/src/settings/server-settings.ts b/packages/tool-servers/src/settings/server-settings.ts index 55187d2aa..c6e5f85a3 100644 --- a/packages/tool-servers/src/settings/server-settings.ts +++ b/packages/tool-servers/src/settings/server-settings.ts @@ -50,9 +50,19 @@ export interface StdioServerSettings extends ServedBy { readonly env: ReadonlyMap; } +export interface ShippedDocument { + readonly text: string; + readonly bytes: number; +} + export type ApiDocumentSource = | { readonly kind: 'url'; readonly url: string } - | { readonly kind: 'file'; readonly path: string }; + | { readonly kind: 'file'; readonly path: string } + | { readonly kind: 'shipped'; readonly document: ShippedDocument }; + +export type DocumentsPlace = + | { readonly kind: 'directory'; readonly path: string } + | { readonly kind: 'shipped'; readonly byName: ReadonlyMap }; export interface OpenApiServerSettings extends ServedBy { readonly type: 'openapi'; @@ -90,6 +100,10 @@ export function isListedFor(server: Scoped, scope: ServersScope, named: string | return servesScope(server, scope) && (named === undefined || server.name === named); } +export function shippedDocumentOf(bytes: Readonly): ShippedDocument { + return { text: new TextDecoder().decode(bytes), bytes: bytes.byteLength }; +} + export function authOf(server: ServerSettings): AuthSettings | null { return server.type === 'stdio' ? null : server.auth; } diff --git a/packages/tool-servers/src/settings/settings-reading.ts b/packages/tool-servers/src/settings/settings-reading.ts index 10aa8e03b..e2e4ddf38 100644 --- a/packages/tool-servers/src/settings/settings-reading.ts +++ b/packages/tool-servers/src/settings/settings-reading.ts @@ -15,13 +15,13 @@ import { checkedEntry } from './entry-checks.ts'; import { misplacedReferencesOfEntry, secretsOfEntry, unreferencedSecurity } from './entry-references.ts'; import { entryToRead, type EntryToRead, type ProviderResolver } from './provider-entries.ts'; import { ServerEntrySchema, ToolServersSchema, toolServersSetting, type ServerEntryFields } from './server-entries.ts'; -import type { ServerSettings, ToolServerSettings } from './server-settings.ts'; +import type { DocumentsPlace, ServerSettings, ToolServerSettings } from './server-settings.ts'; export type Environment = Readonly>; export interface ToolServerSettingsContext { readonly modelProviders: readonly string[]; - readonly documentsDirectory?: string; + readonly documents?: DocumentsPlace; readonly placedValues?: ReadonlyMap; readonly providers?: ProviderResolver; } @@ -58,7 +58,7 @@ function withValuesPlaced(fields: ServerEntryFields, placed: PlacedValues): Serv function readWritten( name: string, - { written, placedValues, documentsDirectory, provider }: EntryToRead, + { written, placedValues, documents, provider }: EntryToRead, environment: Environment, { modelProviders }: ToolServerSettingsContext, ): Result.Result { @@ -77,7 +77,7 @@ function readWritten( const fields = withValuesPlaced(decodeEntry(resolved.value), placedValues); return checkedEntry(name, fields, { modelProviders, - directory: documentsDirectory, + documents, secrets: secretsOfEntry(name, fields, resolved.references), provider, }); @@ -91,7 +91,7 @@ export function readEntry( ): Result.Result { const reading = { placedValues: context.placedValues, - documentsDirectory: context.documentsDirectory, + documents: context.documents, providers: context.providers, }; return Result.flatMap(entryToRead(name, written, reading), (entry) => diff --git a/packages/tool-servers/src/testing/fake-providers.ts b/packages/tool-servers/src/testing/fake-providers.ts index 6eecc6b1b..af74ed209 100644 --- a/packages/tool-servers/src/testing/fake-providers.ts +++ b/packages/tool-servers/src/testing/fake-providers.ts @@ -3,6 +3,7 @@ import { Result } from 'effect'; import type { ProviderNamed, ProviderResolver } from '../settings/provider-entries.ts'; import type { ServerEntryFields } from '../settings/server-entries.ts'; +import type { DocumentsPlace } from '../settings/server-settings.ts'; export type FakeProvider = (named: ProviderNamed) => ServerEntryFields | { readonly refused: readonly Issue[] }; @@ -17,9 +18,11 @@ function placedValuesOf({ credential }: ProviderNamed): ReadonlyMap, - documentsDirectory: string = process.cwd(), + documents: DocumentsPlace = inTheWorkingDirectory, ): FakeProviders { const named: ProviderNamed[] = []; return { @@ -35,7 +38,7 @@ export function fakeProviders( : Result.succeed({ written: resolved, placedValues: placedValuesOf(asked), - documentsDirectory, + documents, writtenAt: (pointer) => (pointer === credentialHeader ? '/credential/value' : '/provider'), }); }, diff --git a/packages/workflow-engine/README.md b/packages/workflow-engine/README.md index 1ab1af21a..a75976177 100644 --- a/packages/workflow-engine/README.md +++ b/packages/workflow-engine/README.md @@ -9,9 +9,9 @@ The machine runs in Node, in the server: [`@on.auto/workflow-host`](../workflow- - `@on.auto/workflow-engine`: the contract, the machine, `workflowMachine(options)`, the engine an adapter builds over its ports, `workflowEngineOf(ports, options, cache)`, and what the workflow adapter reads a document with: JSON helpers, the task kinds, `taskKinds` and `kindOf`, and the walkers of a document's task lists, `taskEntries` for one list and `allTaskEntries` for every list within it, the policy, its checks and the limits of a document, from `src/index.ts`. - `@on.auto/workflow-engine/testing`: the memory adapter of `src/memory` and its virtual clock, a driver over it, the probes of the ports' contract that every adapter runs, `decidingWithin(sandbox, milliseconds)`, the same machine with a clock that reads ahead, so that the expressions of an input it decides may take `milliseconds` rather than `mostInputMs`, for tests that decide on a real worker a busy machine slows, and `scriptedPool(script, otherwise)`, a `ProgramPool` that answers runs with the outcomes of its script in turn and then hands them, its folds, its checks and its closing to `otherwise`, for tests that need an outcome a real worker gives only rarely (`src/testing/index.ts`). Neither the entry nor anything it imports takes a Node module or the YAML parser (`src/engine/portability.test.ts`). - `@on.auto/workflow-engine/worker`: the answerers a worker module hands its jobs to, `answerOf(request, host)`, which runs a program's request in the fresh instance of the sandbox its `host` gives, with the host's output `check`, and `foldAnswerOf(request, host)`, which folds a page in the instances, with the clock, the place it marks and the view checks of its `host` (see [A page of folds](#a-page-of-folds)), with `progressOf(shared)`, the place of a page's folds in memory shared with the pool, `mostValueDepth`, and `sandboxRemovals`, the one list of what the sandbox takes away, which the prelude is written from and the definitions package generates the lib of its check from (`src/worker.ts`). Neither they nor anything they import reaches the pool, the instances or a Node module: the fresh instance a request runs in comes from the `host` the worker module gives them (`src/engine/portability.test.ts`). -- `@on.auto/workflow-engine/job-loop`: `serveJobs({ program, fold, check, checks })`, the loop a worker module of the pool runs (`src/job-loop.ts`, `src/program-pool/job-loop.ts`; see [The workers of the pool](#the-workers-of-the-pool)). It is an entry of its own because it listens on `node:worker_threads`' parent port and takes fresh instances of the sandbox for each job, which the main, testing and worker entries never reach; the pool's own two worker modules run it, as do the checked worker of `@on.auto/definitions` and its check worker. +- `@on.auto/workflow-engine/job-loop`: `serveJobs({ program, fold, check, checks })`, the loop a worker module of the pool runs, `serveJobsWith(handlersOf)`, the same loop over the handlers a module makes of the files the pool handed its worker, and `handedFileIn(handed)`, which answers a file of those handed and fails for any other (`src/job-loop.ts`, `src/program-pool/job-loop.ts`; see [The workers of the pool](#the-workers-of-the-pool)). It is an entry of its own because it listens on `node:worker_threads`' parent port and takes fresh instances of the sandbox for each job, which the main, testing and worker entries never reach; the pool's own two worker modules run it, as do the checked worker of `@on.auto/definitions` and its check worker. - `@on.auto/workflow-engine/process-loop`: `serveProcessJobs(handlers, process)`, the loop a module the pool of processes runs serves its jobs with (`src/process-loop.ts`, `src/process-pool/process-loop.ts`; see [The processes of the pool](#the-processes-of-the-pool)). It is an entry of its own because a process loads it alone: it reaches the loop, the messages and the values of the process pool and the ready signal both pools send, `src/jobs/ready-signal.ts`, and no module of the worker pool or the instances of the sandbox (`src/engine/portability.test.ts`). -- `@on.auto/workflow-engine/dsl`: the sandbox's Node host, `freshInstance(memoryBytes)`, which instantiates the compiled QuickJS module with a memory of that maximum, `machineSandboxOf(clock)`, the instances a host prepares for the inputs of its workflows on its own thread, `filterSandboxOf(clock)`, a context on a fresh instance for each group of filters, on its own thread too, `hostClock` and `sandboxAnswers(expressions)`, what the sandbox answers to probes; the bounds of the sandbox; the format's duration reader, `readDuration`, which an interaction function reads its `expires` with; and a pool of worker threads, kept warm between jobs, that runs a program, a page of folds or a check in a worker of the module the job names, and whose `evaluations` are the sandboxes of `decide` and of a host's filters in the pool's evaluation worker, which the server's workflow host uses (`src/dsl.ts`; see [The evaluation worker](#the-evaluation-worker); see [The sandbox](#the-sandbox), [A page of folds](#a-page-of-folds) and [The workers of the pool](#the-workers-of-the-pool)). The computation capability runs computation functions with it, the recall capability the `answer` of a recall function, the workflow host the folds of the views recall functions keep, and the definitions package its checks at save. It is an entry of its own because the pool imports `node:worker_threads` and arms host timers, and the instances read the WebAssembly file: the main, testing and worker entries reach neither the pool, the instances nor anything that imports them, and only this entry and the job loop's do (`src/engine/portability.test.ts`). The entry also exports `processPool(settings)`, a pool of child processes for jobs a caller's module serves (see [The processes of the pool](#the-processes-of-the-pool)), which the server composes for the model processes of prediction functions. +- `@on.auto/workflow-engine/dsl`: the sandbox's Node host, `freshInstance(shippedFiles, memoryBytes)`, which instantiates the QuickJS module, compiled once from its file as the `ShippedFiles` of `@on.auto/operations` it is given reads it, with a memory of that maximum, `machineSandboxOf(shippedFiles, clock)`, the instances a host prepares for the inputs of its workflows on its own thread, `filterSandboxOf(shippedFiles, clock)`, a context on a fresh instance for each group of filters, on its own thread too, `hostClock` and `sandboxAnswers(shippedFiles, expressions)`, what the sandbox answers to probes; `handedWorkersOf(shippedFiles, modules)` and `handedFilesOf(shippedFiles, files)`, which read the files a pool hands its workers; the bounds of the sandbox; the format's duration reader, `readDuration`, which an interaction function reads its `expires` with; and a pool of worker threads, kept warm between jobs, that runs a program, a page of folds or a check in a worker of the module the job names, and whose `evaluations` are the sandboxes of `decide` and of a host's filters in the pool's evaluation worker, which the server's workflow host uses (`src/dsl.ts`; see [The evaluation worker](#the-evaluation-worker); see [The sandbox](#the-sandbox), [A page of folds](#a-page-of-folds) and [The workers of the pool](#the-workers-of-the-pool)). The computation capability runs computation functions with it, the recall capability the `answer` of a recall function, the workflow host the folds of the views recall functions keep, and the definitions package its checks at save. It is an entry of its own because the pool imports `node:worker_threads` and arms host timers, and the instances compile the WebAssembly module: the main, testing and worker entries reach neither the pool, the instances nor anything that imports them, and only this entry and the job loop's do (`src/engine/portability.test.ts`). The entry also exports `processPool(settings)`, a pool of child processes for jobs a caller's module serves (see [The processes of the pool](#the-processes-of-the-pool)), which the server composes for the model processes of prediction functions. ## How a run moves @@ -25,34 +25,34 @@ The machine runs in Node, in the server: [`@on.auto/workflow-host`](../workflow- ## Layers and ports -| Layer | Folder | What it holds | Port | -| ------------- | ------------------- | ----------------------------------------------------------------------------------------------------------------------- | ----------------------------------------- | -| DSL | `src/dsl` | JSON, durations, expressions and their budget, tasks, policy | none: pure | -| programs | `src/programs` | the sandbox: its prelude, its counter, its contexts | none: pure, given an instance and a clock | -| instances | `src/instances` | the compiled module, fresh instances, the hosts' sandboxes | none: Node only, reached from `dsl.ts` | -| jobs | `src/jobs` | the pool's contract, jobs and messages, an answer, the job loop's caches, the sandboxes another thread evaluates in | none: pure | -| program pool | `src/program-pool` | warm worker threads, their permits, the job loop, the evaluation worker, a deadline and a heap | none: Node only, reached from `dsl.ts` | -| workers | `src/workers` | the pool's own program and fold workers, on the job loop, and its evaluation worker | none: Node only, loaded by the pool | -| folds | `src/folds` | a page of events folded into views, and how a page crosses to a worker | none: pure, given its clock | -| machine | `src/machine` | inputs, state, held values, admission, the clock, UTC time, draws | none: pure | -| runner | `src/runner` | the session of one input, the list and task runners | none: pure | -| tasks | `src/tasks` | each task body: start, resume and cancel over its frame | none: pure | -| decider | `src/decider` | the run's lifecycle, its bounds, the patch and outputs of an event | none: pure | -| run log | `src/run-log` | events, state patches, state formats, the fold, snapshots | `RunLogStore` (one Emmett stream per run) | -| steps | `src/steps` | the step entries of an event, what caused each, the ids of step events | none: pure | -| timers | `src/timers` | timer ids and what each timer is for | `Timers` | -| inbox | `src/inbox` | the external events a run receives | none: events arrive as `event_received` | -| filters | `src/filters` | an event filter read, and matched over an event alone in one context | none: pure, given an instance | -| reactions | `src/reactions` | the ports a run reacts through, its listeners and its emissions | `Listeners`, `Emitter` | -| executor | `src/executor` | call keys | `Executor` | -| dispatch | `src/dispatch` | outputs, the watermark, the order of a dispatch, a run's next due time | `DispatchWatermark` | -| serialisation | `src/serialisation` | one input at a time for each run | `RunSerialiser` | -| settlement | `src/settlement` | settle receipts, due times, troubling receipts | `RecordStore`, `RunReporter` | -| cache | `src/cache` | the runs the engine keeps loaded between inputs, bounded | none | -| engine | `src/engine` | the loop on the ledger, the ports together, the engine's interface | `WorkflowEngine` | -| memory | `src/memory` | a run store, timers, executor, record store and watermark in memory | every port, in memory | -| testing | `src/testing` | a driver over the memory adapter, the ports' probes, run readers | none | -| pool testing | `src/pool-testing` | the scripted pool, the counting and evaluation modules, the waits for threads and jobs, the test sandbox and fold pages | none | +| Layer | Folder | What it holds | Port | +| ------------- | ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------- | +| DSL | `src/dsl` | JSON, durations, expressions and their budget, tasks, policy | none: pure | +| programs | `src/programs` | the sandbox: its prelude, its counter, its contexts | none: pure, given an instance and a clock | +| instances | `src/instances` | the compiled module, fresh instances, the hosts' sandboxes | none: Node only, reached from `dsl.ts` | +| jobs | `src/jobs` | the pool's contract, jobs and messages, an answer, the job loop's caches, the sandboxes another thread evaluates in | none: pure | +| program pool | `src/program-pool` | warm worker threads, their permits, the job loop, the evaluation worker, a deadline and a heap | none: Node only, reached from `dsl.ts` | +| workers | `src/workers` | the pool's own program and fold workers, on the job loop, and its evaluation worker | none: Node only, loaded by the pool | +| folds | `src/folds` | a page of events folded into views, and how a page crosses to a worker | none: pure, given its clock | +| machine | `src/machine` | inputs, state, held values, admission, the clock, UTC time, draws | none: pure | +| runner | `src/runner` | the session of one input, the list and task runners | none: pure | +| tasks | `src/tasks` | each task body: start, resume and cancel over its frame | none: pure | +| decider | `src/decider` | the run's lifecycle, its bounds, the patch and outputs of an event | none: pure | +| run log | `src/run-log` | events, state patches, state formats, the fold, snapshots | `RunLogStore` (one Emmett stream per run) | +| steps | `src/steps` | the step entries of an event, what caused each, the ids of step events | none: pure | +| timers | `src/timers` | timer ids and what each timer is for | `Timers` | +| inbox | `src/inbox` | the external events a run receives | none: events arrive as `event_received` | +| filters | `src/filters` | an event filter read, and matched over an event alone in one context | none: pure, given an instance | +| reactions | `src/reactions` | the ports a run reacts through, its listeners and its emissions | `Listeners`, `Emitter` | +| executor | `src/executor` | call keys | `Executor` | +| dispatch | `src/dispatch` | outputs, the watermark, the order of a dispatch, a run's next due time | `DispatchWatermark` | +| serialisation | `src/serialisation` | one input at a time for each run | `RunSerialiser` | +| settlement | `src/settlement` | settle receipts, due times, troubling receipts | `RecordStore`, `RunReporter` | +| cache | `src/cache` | the runs the engine keeps loaded between inputs, bounded | none | +| engine | `src/engine` | the loop on the ledger, the ports together, the engine's interface | `WorkflowEngine` | +| memory | `src/memory` | a run store, timers, executor, record store and watermark in memory | every port, in memory | +| testing | `src/testing` | a driver over the memory adapter, the ports' probes, run readers | none | +| pool testing | `src/pool-testing` | the scripted pool, the counting and evaluation modules, the waits for threads and jobs, the test sandbox and fold pages, and the settings of a test's pool, with the files on disk and the sandbox the pool hands | none | Every port answers with an Effect. None of them is a clock: time comes in with the inputs, and the sweep is given the time before which a run is overdue. @@ -68,7 +68,7 @@ An expression's budget is checkpoints of the sandbox (below): 250 for one expres ## The sandbox -Every program a person writes inside the playbook, a computation function, a recall function's fold and answer, a workflow's expressions and the `data` of a filter, is TypeScript in the erasable subset, run by QuickJS compiled to WebAssembly: `quickjs-emscripten-core` with the synchronous release variant, `@jitl/quickjs-wasmfile-release-sync`, both pinned at 0.32.0, since the work a program costs is counted by that build. `src/programs` is the one place that knows the sandbox, and it is portable: it imports the library by type alone and is given an instance and a clock. Each host compiles the module once, asynchronously, and makes instances of it (`src/instances`): a fresh instance, with a `WebAssembly.Memory` whose `maximum` is the unit's memory bound and whose `grow` is wrapped so the host sees a refusal even when the program catches the engine's `out of memory` (`freshInstance`). +Every program a person writes inside the playbook, a computation function, a recall function's fold and answer, a workflow's expressions and the `data` of a filter, is TypeScript in the erasable subset, run by QuickJS compiled to WebAssembly: `quickjs-emscripten-core` with the synchronous release variant, `@jitl/quickjs-wasmfile-release-sync`, both pinned at 0.32.0, since the work a program costs is counted by that build. `src/programs` is the one place that knows the sandbox, and it is portable: it imports the library by type alone and is given an instance and a clock. Each host compiles the module once, asynchronously, from the bytes of its file, `sandboxModuleFile`, which a host's own sandboxes read through the `ShippedFiles` they are given and a worker of the pool is handed (see [The workers of the pool](#the-workers-of-the-pool)), and makes instances of it (`src/instances`): a fresh instance, with a `WebAssembly.Memory` whose `maximum` is the unit's memory bound and whose `grow` is wrapped so the host sees a refusal even when the program catches the engine's `out of memory` (`freshInstance`). - **A fresh instance for every unit.** Every program job, every page of folds, every input of `decide` and every group of filters takes an instance of its own, discarded after the unit, so nothing of one unit reaches the next and a memory stop never leaves a wounded module behind. The memory's maximum is the one memory wall, in bytes: 256 MiB for a run, 64 MiB for a page, a group of filters and an input. A unit holds what the module's own memory of that maximum holds: when it needs more, the glue asks to grow by a fifth more than it needs, then a tenth, then a twentieth, and gives up after the three, so the host marks a refusal only when three attempts in a row fail and not at the first, and a program keeps 58 strings of a MiB under a maximum of 64 MiB, as the module alone does (`src/instances/fresh-instances.test.ts`). The engine's own `setMemoryLimit` counts allocations, not bytes, in this build, and is not relied on. - **The prelude** (`src/programs/sandbox-prelude.ts`) makes a context the playbook's: `Date` answers the run's moment with no argument, takes one number, one `Date`, a date alone or a date and time that names its offset, and refuses whatever reads the server's time zone, its original constructor out of reach through `getPrototypeOf`, `Reflect.construct`, `bind` and `class extends`; `Math.random`, `eval`, the `Function` constructors, `WeakRef`, `FinalizationRegistry` and `Promise`, whose callbacks the sandbox never runs, go, with every local-time method of `Date`, from the one list `sandboxRemovals` (`src/programs/sandbox-names.ts`), which also says that `Date` is not called and takes at most one argument, the rule the prelude keeps and the definitions package writes the lib of its check from. An expression runs as strict code, `'use strict'` the first line of the function it is wrapped in, as the check compiles it. Nothing is added: no `console`, no timers, no `fetch`, no `Intl`, no module loader. @@ -80,7 +80,7 @@ Every program a person writes inside the playbook, a computation function, a rec A unit answers a `ProgramRun`: `answered` with the text of its JSON, `raised` with the issue and its line, `exhausted` with its `limit` (`work`, `memory`, `stack` or `deadline`), `unfit` with the member JSON cannot carry, or `oversized`, each with the checkpoints it spent (`src/programs/program-run.ts`). `moduleRun(instance, settings, request)` runs a module's entry over its arguments, as JSON text, in one context; `expressionUnitOf(instances, settings)` evaluates the expressions of one input, each in a fresh context of one runtime on the instance it takes when it first needs one; `filterContextOf` and `foldingUnitOf` keep a frozen context over a group of filters and over a page's folds. -`programPool({ workers, heapMegabytes })` runs a `ProgramRequest` (the source, the entry, the arguments, the moment of `Date`, the budget in checkpoints, the instance's `memoryBytes`, the `stackBytes`, `deadlineMs`, the most bytes its output may take as JSON, and optionally the `worker` module that runs it and a JSON `context` that module reads from the request, as the checked worker of `@on.auto/definitions` checks an output against the schema it names; a pool given its own `worker` in its settings, as tests do, runs every program with that one) in a worker thread of that module, kept warm between jobs, at most `workers` at once (`src/program-pool`; see [The workers of the pool](#the-workers-of-the-pool)). A run waits for a permit until its deadline, and the deadline also stops it there: the worker's handler reads the deadline at every checkpoint, and the pool terminates the worker when the deadline passes, so a run whose worker never answers still ends. The program and the arguments cross to the worker as copies, the arguments as JSON text, and the answer comes back as JSON text with its size, so the thread that holds the ledger never waits on a program. Every worker has a heap of `heapMegabytes`, its own limit, which bounds no WebAssembly memory, a stack of `workerStackMegabytes`, 64 MiB, none of the server's flags, and an empty environment unless the pool's settings give it one, `environment`, which the server never does, so no setting of the server, its keys among them, reaches a worker. A worker given no `execArgv` takes the flags of the process that starts it, and Node would then read an `--env-file` of the server, which the development runner gives, into every worker over the environment the pool gives, and run a preload of the server, `--import`, in every worker; the pool therefore gives each an empty `execArgv` (`src/program-pool/worker-flags.test.ts`), and Node strips the types of a worker module without a flag. The tests give only `NODE_V8_COVERAGE`, so the coverage of a worker is measured. An outcome is the worker's answer, `oversized` for an output over its bytes, which the worker measures as JSON before it crosses, `refused` for a program the worker would not load, `mismatched` with the issues a request's worker found in the output, each a `pointer` and a `detail` cut at 1,024 bytes apart, and the detail of every issue in an answer cut at `mostIssueBytes`, 1,024 bytes of UTF-8, marked with `…` (`textWithin`), `stopped` because of the `deadline`, `memory` (the worker's own heap, `ERR_WORKER_OUT_OF_MEMORY`), `busy` when no worker came free, `cancelled` by the caller's signal or `closing`, or `crashed` for any other ending, and each says how many `milliseconds` it took. `close()` turns away the runs still waiting and terminates every worker, idle or busy, a busy job ending `closing`. +`programPool({ workers, heapMegabytes, handed })` runs a `ProgramRequest` (the source, the entry, the arguments, the moment of `Date`, the budget in checkpoints, the instance's `memoryBytes`, the `stackBytes`, `deadlineMs`, the most bytes its output may take as JSON, and optionally the `worker` module that runs it and a JSON `context` that module reads from the request, as the checked worker of `@on.auto/definitions` checks an output against the schema it names; a pool given its own `worker` in its settings, as tests do, runs every program with that one) in a worker thread of that module, kept warm between jobs, at most `workers` at once (`src/program-pool`; see [The workers of the pool](#the-workers-of-the-pool)). A run waits for a permit until its deadline, and the deadline also stops it there: the worker's handler reads the deadline at every checkpoint, and the pool terminates the worker when the deadline passes, so a run whose worker never answers still ends. The program and the arguments cross to the worker as copies, the arguments as JSON text, and the answer comes back as JSON text with its size, so the thread that holds the ledger never waits on a program. Every worker has a heap of `heapMegabytes`, its own limit, which bounds no WebAssembly memory, a stack of `workerStackMegabytes`, 64 MiB, none of the server's flags, and an empty environment unless the pool's settings give it one, `environment`, which the server never does, so no setting of the server, its keys among them, reaches a worker. A worker given no `execArgv` takes the flags of the process that starts it, and Node would then read an `--env-file` of the server, which the development runner gives, into every worker over the environment the pool gives, and run a preload of the server, `--import`, in every worker; the pool therefore gives each an empty `execArgv` (`src/program-pool/worker-flags.test.ts`), and Node strips the types of a worker module without a flag. The tests give only `NODE_V8_COVERAGE`, so the coverage of a worker is measured. An outcome is the worker's answer, `oversized` for an output over its bytes, which the worker measures as JSON before it crosses, `refused` for a program the worker would not load, `mismatched` with the issues a request's worker found in the output, each a `pointer` and a `detail` cut at 1,024 bytes apart, and the detail of every issue in an answer cut at `mostIssueBytes`, 1,024 bytes of UTF-8, marked with `…` (`textWithin`), `stopped` because of the `deadline`, `memory` (the worker's own heap, `ERR_WORKER_OUT_OF_MEMORY`), `busy` when no worker came free, `cancelled` by the caller's signal or `closing`, or `crashed` for any other ending, and each says how many `milliseconds` it took. `close()` turns away the runs still waiting and terminates every worker, idle or busy, a busy job ending `closing`. `programPool` also runs a page of folds in a worker, `fold(request, signal)` (see [A page of folds](#a-page-of-folds)), and a check, `check(request)`, on a permit of its own beside the `workers` of runs, `checkPermits`, 1, so a save never waits behind the runs that take every worker: the request names the module that checks, gives the expressions and the programs of a document with the schemas of its types, and waits for its permit at most its deadline, after which the check ends `busy`. Its deadline counts from when its worker is ready, which the job loop says once it listens, `{ ready: true }`, so starting the worker and loading its modules, a compiler among them, never spend it: the start has a bound of its own, `mostCheckStartMs`, 10 seconds, after which the check ends `busy` and the worker, still starting, is kept for the next check rather than ended, and a check that runs past its deadline ends `deadline`. The worker of the checks never idles out, since a save would pay its start again; it is still let go of after its 1,000th check (`src/program-pool/pool-checks.test.ts`). @@ -108,6 +108,8 @@ The pool keeps the workers of each module URL between jobs, every module's worke A worker module runs `serveJobs({ program, fold, check, checks })` of the `job-loop` entry, which listens on the parent port, decodes each envelope with the schema the pool encoded it with, and hands the whole request to the handler of its kind with a host: the clock, a fresh instance of the sandbox for a program and one for each view of a page, prepared while the job before ran, and the output check of the request's `context` or the view checks of each view's schema, compiled once for each schema from the `checks` the module gives and kept, by schema, in a cache of at most 262,144 characters (`src/jobs/job-kit.ts`); with no `checks` a program goes unchecked and a view that keeps a schema stalls. A request the loop cannot read is answered `unreadable`, which a page's job reads as such and a program's as a crash, and the worker is not kept. The pool's own `src/workers/program-worker.ts` and `fold-worker.ts` serve `answerOf` and `foldAnswerOf` so; the pool names them for a request that names no worker, as the engine's own tests do, while the capabilities name the checked worker of `@on.auto/definitions` for every run. +The pool hands every worker it starts, in its `workerData`, the files its module reads, from the `handed` of its settings: the files `handed.modules` names for the worker's module, or `handed.sandbox`, the module of the sandbox, for a module it does not name, so a worker reads no file itself. A thread cannot be handed a reader, so the files are read before the pool is made, once, through the `ShippedFiles` of `@on.auto/operations`: `handedWorkersOf(shippedFiles, modules)` reads the module of the sandbox and `handedFilesOf(shippedFiles, files)` the files of a module, and the server reads them where it composes its one pool, with the files of the check worker of `@on.auto/definitions` (`handedWorkersWithChecks`). `programPool` therefore stays synchronous, and every worker starts with its files, the evaluation worker of a unit asked before any `reserve` among them, which answers once it is ready (`src/program-pool/evaluation-thread.test.ts`), as does a worker the pool starts again while the thread that decides blocks on it. A worker compiles the sandbox from what it was handed, and a module that makes its handlers of handed files, as the check worker makes its checker of the lib it compiles against, serves them with `serveJobsWith(handlersOf)`, whose `handed(file)` answers the bytes of a file handed to the worker and fails for any other (`src/program-pool/handed-files.ts`, `src/program-pool/pool-checks.test.ts`). + The answer says whether the worker may be kept, by the rule of its kind: a program keeps it unless its evaluator ended it at the deadline, since the thread was busy past the job's deadline; a page keeps it unless a view ran past its fold's deadline, `overtime`, or the page was `unreadable`. The pool lets a worker go, terminating it before the job's permit is released: - after any job the pool stopped, by its deadline, the memory, a cancellation or the closing, after any job that crashed, and after any answer that says not to keep it; @@ -341,7 +343,7 @@ Each sentence is something a reviewer can check against the code or a test. **[e 17. **[engine]** The state of a run is plain JSON, and the data it holds stays at or under 4 MiB (`src/decider/run-bounds.test.ts`). 18. **[engine]** No event is larger than 1.5 MiB as JSON: the machine measures each event before the append and ends the run instead (`src/run-log/run-event.test.ts` for the measure, `src/decider/run-bounds.test.ts`); and no input runs more than 100 tasks (`src/runner/list-runner.test.ts`). 19. **[adapter]** Inputs of one run are applied one at a time; the machine relies only on the expected version of each append. -20. **[engine]** Nothing in this package but the program pool, the process pool and the instances of the sandbox uses a Node-only API, a dynamic import, code generation or a host timer; the pool, `src/program-pool`, uses `node:worker_threads` and host timers, the process pool, `src/process-pool`, uses `node:child_process`, the file system to find `prlimit` and host timers, the instances, `src/instances`, read and compile the WebAssembly file, and only the `dsl`, `job-loop` and `process-loop` entries reach them; `src/programs` imports QuickJS by type alone, no module of the package reaches `typescript`, and the prelude reads no clock, no random source and no locale (`src/engine/portability.test.ts`, `src/programs/no-compiler.test.ts`); outside this package, a lint rule refuses an import of `node:worker_threads` but in the pool and the measure scripts of the computation capability (`.oxlintrc.json`). +20. **[engine]** Nothing in this package but the program pool, the process pool and the instances of the sandbox uses a Node-only API, a dynamic import, code generation or a host timer; the pool, `src/program-pool`, uses `node:worker_threads` and host timers, the process pool, `src/process-pool`, uses `node:child_process`, the file system to find `prlimit` and host timers, the instances, `src/instances`, compile the WebAssembly module, whose bytes a host reads through the `ShippedFiles` it is given and the pool hands its workers, and only the `dsl`, `job-loop` and `process-loop` entries reach them; `src/programs` imports QuickJS by type alone, no module of the package reaches `typescript`, and the prelude reads no clock, no random source and no locale (`src/engine/portability.test.ts`, `src/programs/no-compiler.test.ts`); outside this package, a lint rule refuses an import of `node:worker_threads` but in the pool and the measure scripts of the computation capability (`.oxlintrc.json`). 21. **[engine]** No module of the engine keeps a cache that grows with the history of a run; each engine's cache of loaded runs is bounded, by runs and by bytes, and so are each job loop's caches of checks, which live in its closure (`src/engine/portability.test.ts`, `src/dsl/bounded-cache.test.ts`, `src/cache/run-cache.test.ts`, `src/program-pool/job-loop.test.ts`). 22. **[engine]** No two events of a stream have the same receipt kind and key (`src/decider/deduplication.test.ts`). 23. **[engine]** `lastInputAt` never decreases, and a fired timer's input is never earlier than the time it was due (`src/machine/input-receipt.test.ts`). diff --git a/packages/workflow-engine/measure/evaluations.ts b/packages/workflow-engine/measure/evaluations.ts index e02bbe3bd..16bb7f280 100644 --- a/packages/workflow-engine/measure/evaluations.ts +++ b/packages/workflow-engine/measure/evaluations.ts @@ -1,6 +1,7 @@ +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { Effect } from 'effect'; -import { machineSandboxOf, programPool } from '../src/dsl.ts'; +import { handedWorkersOf, machineSandboxOf, programPool } from '../src/dsl.ts'; import { mostFilterMs } from '../src/dsl/evaluation.ts'; import { filterVerdictsOf } from '../src/filters/filter-verdicts.ts'; import { decidedOnce } from '../src/pool-testing/decided-once.ts'; @@ -62,11 +63,11 @@ function evaluationMs(sandbox: MachineSandbox): number { } export async function evaluationsMeasured(): Promise { - const pool = programPool({ workers: 1, heapMegabytes: 256 }); + const pool = programPool({ handed: await handedWorkersOf(shippedFilesOnDisk), workers: 1, heapMegabytes: 256 }); const { machine, filters } = pool.evaluations; const coldMs = await timed(() => Effect.runPromise(machine.reserve)); const inWorker = evaluationMs(machine); - const here = machineSandboxOf(); + const here = machineSandboxOf(shippedFilesOnDisk); await Effect.runPromise(here.reserve); const inThread = evaluationMs(here); const decideMs = await timed(() => decidedOnce(machine, churning(4_194_304), startedAt)); diff --git a/packages/workflow-engine/measure/pages.ts b/packages/workflow-engine/measure/pages.ts index 06e353569..057461a34 100644 --- a/packages/workflow-engine/measure/pages.ts +++ b/packages/workflow-engine/measure/pages.ts @@ -1,7 +1,9 @@ +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { Function } from 'effect'; import { freshInstance, + handedWorkersOf, programPool, unitMemoryBytes, workerStackBytes, @@ -100,7 +102,7 @@ function inTurn(items: readonly A[], step: (item: A) => Promise): Promi } async function onAFreshPool(page: FoldRequest): Promise { - const pool = programPool({ workers: 1, heapMegabytes: 256 }); + const pool = programPool({ handed: await handedWorkersOf(shippedFilesOnDisk), workers: 1, heapMegabytes: 256 }); const { milliseconds } = await pool.fold(page); await pool.close(); return milliseconds; @@ -110,7 +112,7 @@ async function foldedOnThisThread( page: FoldRequest, ): Promise<{ readonly milliseconds: number; readonly views: string }> { const folded = { views: '' }; - const instances = await Promise.all(page.views.map(() => freshInstance(unitMemoryBytes))); + const instances = await Promise.all(page.views.map(() => freshInstance(shippedFilesOnDisk, unitMemoryBytes))); const host = { now: () => performance.now(), folding: Function.constVoid, @@ -153,7 +155,7 @@ function medianOf(all: readonly Turn[], of: (turn: Turn) => number): string { export async function pagesMeasured(): Promise { const page = pageOf(events); const start = await inTurn(Array.from({ length: turns }), () => onAFreshPool(pageOf(0))); - const pool = programPool({ workers: 1, heapMegabytes: 256 }); + const pool = programPool({ handed: await handedWorkersOf(shippedFilesOnDisk), workers: 1, heapMegabytes: 256 }); await pool.fold(page); const all = await turnsOf(page, pool); await pool.close(); diff --git a/packages/workflow-engine/measure/sandbox.ts b/packages/workflow-engine/measure/sandbox.ts index d1e1413fe..025699a09 100644 --- a/packages/workflow-engine/measure/sandbox.ts +++ b/packages/workflow-engine/measure/sandbox.ts @@ -1,13 +1,15 @@ -import { stat } from 'node:fs/promises'; +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { freshInstance, + handedWorkersOf, programPool, runMemoryBytes, threadStackBytes, unitMemoryBytes, type ProgramRun, } from '../src/dsl.ts'; +import { sandboxModuleFile } from '../src/instances/fresh-instances.ts'; import { expressionUnitOf } from '../src/programs/expression-units.ts'; import { filterContextOf } from '../src/programs/kept-contexts.ts'; import { moduleRun } from '../src/programs/module-runs.ts'; @@ -50,7 +52,7 @@ async function ranModule( args: readonly string[], evaluation: Evaluation = unbounded, ): Promise { - const run = moduleRun(await freshInstance(unitMemoryBytes), settings, { + const run = moduleRun(await freshInstance(shippedFilesOnDisk, unitMemoryBytes), settings, { source, entry: 'default', arguments: args, @@ -63,23 +65,23 @@ async function ranModule( } async function instanceLines(): Promise { - const wasm = await stat(new URL(import.meta.resolve('@jitl/quickjs-wasmfile-release-sync/wasm'))); - await freshInstance(unitMemoryBytes); + const wasm = await shippedFilesOnDisk.bytes(sandboxModuleFile); + await freshInstance(shippedFilesOnDisk, unitMemoryBytes); const fresh = await medianAsync(50, async () => { const started = performance.now(); - await freshInstance(unitMemoryBytes); + await freshInstance(shippedFilesOnDisk, unitMemoryBytes); return performance.now() - started; }); const withPrelude = await medianAsync(50, async () => { const started = performance.now(); - const runtime = sandboxRuntimeOf(await freshInstance(runMemoryBytes), settings); + const runtime = sandboxRuntimeOf(await freshInstance(shippedFilesOnDisk, runMemoryBytes), settings); runtime.context().close(); const milliseconds = performance.now() - started; runtime.close(); return milliseconds; }); return [ - `the WebAssembly file: ${formatted(wasm.size, 0)} bytes`, + `the WebAssembly file: ${formatted(wasm.byteLength, 0)} bytes`, `a fresh instance of the compiled module, memory maximum 64 MiB: ${formatted(fresh, 2)} ms at the median of 50; 256 MiB with a runtime, a context and the prelude: ${formatted(withPrelude, 2)} ms`, ]; } @@ -113,13 +115,13 @@ async function checkpointLines(): Promise { } async function expressionLines(): Promise { - const instance = await freshInstance(unitMemoryBytes); + const instance = await freshInstance(shippedFilesOnDisk, unitMemoryBytes); const unit = expressionUnitOf(() => instance, settings); const fresh = medianMillisecondsOf(201, () => { unit.evaluate('$context.decision.decision == "approve"', { context: decision }, unbounded); }); unit.close(); - const filters = filterContextOf(await freshInstance(unitMemoryBytes), settings, unbounded); + const filters = filterContextOf(await freshInstance(shippedFilesOnDisk, unitMemoryBytes), settings, unbounded); const test = filters.define(' $data.decision.decision == "approve" '); const freezing = millisecondsOf(() => { filters.freeze(); @@ -167,7 +169,7 @@ async function valueLines(): Promise { } async function stackLines(): Promise { - const pool = programPool({ workers: 1, heapMegabytes: 256 }); + const pool = programPool({ handed: await handedWorkersOf(shippedFilesOnDisk), workers: 1, heapMegabytes: 256 }); const deepest = 'export default function () {\n let deepest = 0;\n const down = (depth) => {\n deepest = depth;\n return down(depth + 1);\n };\n try {\n down(0);\n } catch {\n return deepest;\n }\n return -1;\n}'; const depthAt = async (stackBytes: number): Promise => { diff --git a/packages/workflow-engine/src/dsl.ts b/packages/workflow-engine/src/dsl.ts index e752eb35f..4e319b59f 100644 --- a/packages/workflow-engine/src/dsl.ts +++ b/packages/workflow-engine/src/dsl.ts @@ -20,6 +20,8 @@ export type { CheckRequest, FoldOutcome, FoldRequest, + HandedFiles, + HandedWorkers, PoolOutcome, PoolSettings, ProgramPool, @@ -27,6 +29,7 @@ export type { } from './jobs/pool-contract.ts'; export type { CheckAnswer, CheckIssue, CheckJob, StrippedSources } from './jobs/check-messages.ts'; export { checkPermits, programPool } from './program-pool/program-pool.ts'; +export { handedFilesOf, handedWorkersOf } from './program-pool/handed-files.ts'; export { pooledJob } from './jobs/pooled-jobs.ts'; export { idleWorkerMs, jobsBeforeRecycling } from './program-pool/pool-workers.ts'; export { workerStackMegabytes } from './program-pool/pool-threads.ts'; diff --git a/packages/workflow-engine/src/dsl/expressions.test.ts b/packages/workflow-engine/src/dsl/expressions.test.ts index 8d933912d..9057374e9 100644 --- a/packages/workflow-engine/src/dsl/expressions.test.ts +++ b/packages/workflow-engine/src/dsl/expressions.test.ts @@ -1,3 +1,4 @@ +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { describe, expect, it } from 'vitest'; import { freshInstance } from '../instances/fresh-instances.ts'; @@ -12,7 +13,7 @@ const now = Date.parse('2026-10-01T09:00:00.000Z'); const budget = { now, mostWork: 250, deadlineAt: Number.POSITIVE_INFINITY }; async function unitOf(clock: () => number = () => 0): Promise { - const instance = await freshInstance(unitMemoryBytes); + const instance = await freshInstance(shippedFilesOnDisk, unitMemoryBytes); return expressionUnitOf(() => instance, { stackBytes: threadStackBytes, mostAnswerBytes: mostEventBytes, clock }); } diff --git a/packages/workflow-engine/src/engine/portability.test.ts b/packages/workflow-engine/src/engine/portability.test.ts index ed161e0c5..b019aeba6 100644 --- a/packages/workflow-engine/src/engine/portability.test.ts +++ b/packages/workflow-engine/src/engine/portability.test.ts @@ -2,6 +2,7 @@ import { readdirSync, readFileSync } from 'node:fs'; import { dirname, join, normalize } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { modulesReachedFrom } from '@on.auto/operations/testing'; import { describe, expect, it } from 'vitest'; import { freezingPreludeSource, preludeSource } from '../programs/sandbox-prelude.ts'; @@ -19,7 +20,7 @@ const importedInAFolder: ReadonlyMap = new Map([ ['programs/', ['quickjs-emscripten-core']], ['program-pool/', ['node:worker_threads']], ['process-pool/', ['node:buffer', 'node:child_process', 'node:fs', 'node:path', 'node:url']], - ['instances/', ['node:fs/promises', 'quickjs-emscripten-core', '@jitl/quickjs-wasmfile-release-sync']], + ['instances/', ['quickjs-emscripten-core', '@jitl/quickjs-wasmfile-release-sync']], ]); const quickJs = 'quickjs-emscripten-core'; @@ -132,26 +133,8 @@ const machineAndRunLog = ['machine', 'runner', 'tasks', 'decider', 'run-log', 'd sourcesUnder(folder), ); -const relativeImport = /from '(\.{1,2}\/[^']+)'/gu; - -function importedBy(file: string): readonly string[] { - const text = readFileSync(join(source, file), 'utf8'); - return [...text.matchAll(relativeImport)].map(([, path = '']: readonly string[]) => - normalize(join(dirname(file), path)), - ); -} - function reachableFrom(entry: string): readonly string[] { - const reached = new Set([entry]); - const pending = [entry]; - for (let file = pending.pop(); file !== undefined; file = pending.pop()) { - const unseen = importedBy(file).filter((imported) => !reached.has(imported)); - for (const imported of unseen) { - reached.add(imported); - pending.push(imported); - } - } - return [...reached].toSorted(); + return modulesReachedFrom(source, entry).files; } const nodeOrYaml: readonly Forbidden[] = [ diff --git a/packages/workflow-engine/src/filters/event-filter.test.ts b/packages/workflow-engine/src/filters/event-filter.test.ts index e23a177bb..b55320a09 100644 --- a/packages/workflow-engine/src/filters/event-filter.test.ts +++ b/packages/workflow-engine/src/filters/event-filter.test.ts @@ -1,3 +1,4 @@ +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { describe, expect, it } from 'vitest'; import type { Json, JsonObject } from '../dsl/json.ts'; @@ -16,7 +17,7 @@ const at = '/schedule/on/one'; const now = Date.parse('2026-10-01T09:00:00.000Z'); -const sandbox: FilterSandbox = filterSandboxOf(() => 0); +const sandbox: FilterSandbox = filterSandboxOf(shippedFilesOnDisk, () => 0); const monthClosed: JsonObject = { specversion: '1.0', @@ -147,7 +148,7 @@ describe('matching an event whose data expression fails', () => { it('answers an error for a filter whose context did not freeze in time', async () => { const ticking = { at: 0 }; - const late = filterSandboxOf(() => { + const late = filterSandboxOf(shippedFilesOnDisk, () => { ticking.at += 5000; return ticking.at; }); diff --git a/packages/workflow-engine/src/filters/filter-isolation.test.ts b/packages/workflow-engine/src/filters/filter-isolation.test.ts index 412fec105..2eb25c830 100644 --- a/packages/workflow-engine/src/filters/filter-isolation.test.ts +++ b/packages/workflow-engine/src/filters/filter-isolation.test.ts @@ -1,3 +1,4 @@ +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { describe, expect, it } from 'vitest'; import type { JsonObject } from '../dsl/json.ts'; @@ -22,7 +23,7 @@ function filterAt(reference: string, data = writingTheIteratorPrototype) { describe('the filters of a sandbox that breaks', () => { it('pass on a failure that is not a raise of the filter, unchanged', async () => { const failure = new TypeError('the sandbox broke'); - const sandbox = filterSandboxOf(() => 0); + const sandbox = filterSandboxOf(shippedFilesOnDisk, () => 0); const breaking: FilterSandbox = { ...sandbox, session: async (opening) => ({ @@ -40,7 +41,7 @@ describe('the filters of a sandbox that breaks', () => { describe('the filters of one batch', () => { it('share no state through an intrinsic no global names, so each answers as it would alone', async () => { const [first, second] = [filterAt('/first'), filterAt('/second')]; - const sandbox = filterSandboxOf(() => 0); + const sandbox = filterSandboxOf(shippedFilesOnDisk, () => 0); const together = await filterVerdictsOf([first, second], closed, sandbox, now); const alone = [ @@ -53,7 +54,7 @@ describe('the filters of one batch', () => { }); it('take a fresh context after one its sandbox refused, so one filter that fills its memory fails no other', async () => { - const sandbox = filterSandboxOf(() => 0); + const sandbox = filterSandboxOf(shippedFilesOnDisk, () => 0); const verdicts = await filterVerdictsOf( [ @@ -71,7 +72,7 @@ describe('the filters of one batch', () => { }); it('say a filter was stopped when its memory or its deadline ended it, which no event causes, and not when its work did', async () => { - const sandbox = filterSandboxOf(() => 0); + const sandbox = filterSandboxOf(shippedFilesOnDisk, () => 0); const verdicts = await filterVerdictsOf( [filterAt('/working', working), filterAt('/filling', filling), filterAt('/eu', '${ $data.region == "eu" }')], diff --git a/packages/workflow-engine/src/filters/filter-testing.test.ts b/packages/workflow-engine/src/filters/filter-testing.test.ts index 505e7efbc..c428ed748 100644 --- a/packages/workflow-engine/src/filters/filter-testing.test.ts +++ b/packages/workflow-engine/src/filters/filter-testing.test.ts @@ -1,3 +1,4 @@ +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { describe, expect, it } from 'vitest'; import type { JsonObject } from '../dsl/json.ts'; @@ -50,7 +51,7 @@ const american = filterAt('/us', '${ $data.region == "us" }'); describe('the filters tested over a page of events', () => { it('prepare their tests in one session and test every event in it, answering as a group of each event alone does', async () => { - const counting = counted(filterSandboxOf(() => 0)); + const counting = counted(filterSandboxOf(shippedFilesOnDisk, () => 0)); const testing = filterTestingOf([european, american], counting.sandbox); const verdicts = [ @@ -68,11 +69,13 @@ describe('the filters tested over a page of events', () => { [false, false], ]); expect([sessionsOfThePage, counting.closed()]).toEqual([1, 1]); - expect(verdicts[0]).toEqual(await filterVerdictsOf([european, american], closedIn('eu'), filterSandboxOf(), now)); + expect(verdicts[0]).toEqual( + await filterVerdictsOf([european, american], closedIn('eu'), filterSandboxOf(shippedFilesOnDisk), now), + ); }); it('let a session the sandbox stopped go, finish the event in sessions of its own, and test the next event in a fresh one', async () => { - const counting = counted(filterSandboxOf(() => 0)); + const counting = counted(filterSandboxOf(shippedFilesOnDisk, () => 0)); const testing = filterTestingOf([filterAt('/filling', filling), european], counting.sandbox); const stopped = await testing.verdictsOf(closedIn('eu'), now); @@ -88,7 +91,7 @@ describe('the filters tested over a page of events', () => { describe('the filters tested over a page of events while a session opens', () => { it('close a session still being prepared once it opens, and answer that event and any after the close in sessions of their own', async () => { - const counting = counted(filterSandboxOf(() => 0)); + const counting = counted(filterSandboxOf(shippedFilesOnDisk, () => 0)); const testing = filterTestingOf([european], counting.sandbox); const opening = testing.verdictsOf(closedIn('eu'), now); @@ -100,7 +103,7 @@ describe('the filters tested over a page of events while a session opens', () => }); it('share one session between two events asked about at once, and close it once', async () => { - const counting = counted(filterSandboxOf(() => 0)); + const counting = counted(filterSandboxOf(shippedFilesOnDisk, () => 0)); const testing = filterTestingOf([european, american], counting.sandbox); const verdicts = await Promise.all([ @@ -127,7 +130,7 @@ describe('the filters of several types tested over a page of events', () => { const refunded = { reference: '/refunded', attributes: { type: 'com.acme.refunded', data: refundTest } }; const testing = filterTestingOf( [placed, refunded], - filterSandboxOf(() => 0), + filterSandboxOf(shippedFilesOnDisk, () => 0), ); const verdicts = await testing.verdictsOf(placedOrder, now); @@ -140,7 +143,7 @@ describe('the filters of several types tested over a page of events', () => { const refunded = { reference: '/refunded', attributes: { data: refundTest, type: 'com.acme.refunded' } }; const testing = filterTestingOf( [refunded], - filterSandboxOf(() => 0), + filterSandboxOf(shippedFilesOnDisk, () => 0), ); const verdicts = await testing.verdictsOf(placedOrder, now); @@ -153,7 +156,7 @@ describe('the filters of several types tested over a page of events', () => { describe('the filters tested over a page of events in a sandbox that breaks', () => { it('close a session whose tests could not be prepared, and pass on the failure unchanged', async () => { const failure = new TypeError('the sandbox broke'); - const counting = counted(filterSandboxOf(() => 0)); + const counting = counted(filterSandboxOf(shippedFilesOnDisk, () => 0)); const breaking: FilterSandbox = { ...counting.sandbox, session: async (opening) => ({ diff --git a/packages/workflow-engine/src/filters/filter-workers.test.ts b/packages/workflow-engine/src/filters/filter-workers.test.ts index d79fb1257..e9eb6dba0 100644 --- a/packages/workflow-engine/src/filters/filter-workers.test.ts +++ b/packages/workflow-engine/src/filters/filter-workers.test.ts @@ -12,6 +12,7 @@ import { plainFilter, replacementOf, } from '../pool-testing/evaluation-workers.ts'; +import { testPoolSettings } from '../pool-testing/test-pools.ts'; import { threadsAlive } from '../pool-testing/threads-alive.ts'; import { programPool } from '../program-pool/program-pool.ts'; import { filterVerdictsOf } from './filter-verdicts.ts'; @@ -20,10 +21,6 @@ const timerMs = 20; const pools: ProgramPool[] = []; -const coverage = process.env['NODE_V8_COVERAGE']; - -const measured = coverage === undefined ? {} : { NODE_V8_COVERAGE: coverage }; - function european(reference: string): typeof plainFilter { return { ...plainFilter, reference }; } @@ -33,7 +30,7 @@ afterEach(async () => { }); function poolOf(settings: Partial = {}): ProgramPool { - const pool = programPool({ workers: 1, heapMegabytes: 64, environment: measured, ...settings }); + const pool = programPool({ ...testPoolSettings(1), ...settings }); pools.push(pool); return pool; } diff --git a/packages/workflow-engine/src/folds/fold-answer.test.ts b/packages/workflow-engine/src/folds/fold-answer.test.ts index 1f87f7b7d..c20add268 100644 --- a/packages/workflow-engine/src/folds/fold-answer.test.ts +++ b/packages/workflow-engine/src/folds/fold-answer.test.ts @@ -1,3 +1,4 @@ +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { Function } from 'effect'; import { describe, expect, it } from 'vitest'; @@ -8,7 +9,9 @@ import type { FoldHost } from './fold-page.ts'; import { foldProgress, progressOf } from './fold-progress.ts'; async function hostOf(views: number, more: Partial = {}): Promise { - const instances = await Promise.all(Array.from({ length: views }, () => freshInstance(unitMemoryBytes))); + const instances = await Promise.all( + Array.from({ length: views }, () => freshInstance(shippedFilesOnDisk, unitMemoryBytes)), + ); return { now: () => 0, folding: Function.constVoid, diff --git a/packages/workflow-engine/src/folds/fold-filters.test.ts b/packages/workflow-engine/src/folds/fold-filters.test.ts index 954c53bff..ed1064af7 100644 --- a/packages/workflow-engine/src/folds/fold-filters.test.ts +++ b/packages/workflow-engine/src/folds/fold-filters.test.ts @@ -1,3 +1,4 @@ +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { describe, expect, it } from 'vitest'; import type { Json, JsonObject } from '../dsl/json.ts'; @@ -12,7 +13,7 @@ const reviewed = { type: 'run_succeeded', subject: 'reasoning/review-brief' }; const evaluation: Evaluation = { budget: 500, deadlineAt: Number.POSITIVE_INFINITY, moment: 0 }; const context = filterContextOf( - await freshInstance(unitMemoryBytes), + await freshInstance(shippedFilesOnDisk, unitMemoryBytes), { stackBytes: threadStackBytes, mostAnswerBytes: 1000, clock: () => 0 }, evaluation, ); diff --git a/packages/workflow-engine/src/folds/fold-pool.test.ts b/packages/workflow-engine/src/folds/fold-pool.test.ts index 863d4e31c..ea882679c 100644 --- a/packages/workflow-engine/src/folds/fold-pool.test.ts +++ b/packages/workflow-engine/src/folds/fold-pool.test.ts @@ -2,6 +2,7 @@ import { afterEach, describe, expect, it } from 'vitest'; import type { JsonObject } from '../dsl/json.ts'; import type { FoldRequest, PoolSettings, ProgramPool } from '../jobs/pool-contract.ts'; +import { testPoolSettings } from '../pool-testing/test-pools.ts'; import { longerThanTheTestMs } from '../pool-testing/waiting.ts'; import { programPool } from '../program-pool/program-pool.ts'; import { unitMemoryBytes, workerStackBytes } from '../programs/sandbox-bounds.ts'; @@ -42,12 +43,8 @@ const foldingAtOnce = workerOf( 'import { parentPort } from "node:worker_threads"; parentPort.on("message", ({ job }) => { parentPort.postMessage({ job, answer: { ran: "folded", early: false, views: [{ view: "3", folded: 2, lastFolded: 1, through: 1, work: 0 }] }, keep: true }); });', ); -const coverage = process.env['NODE_V8_COVERAGE']; - -const measured = coverage === undefined ? {} : { NODE_V8_COVERAGE: coverage }; - function poolOf(settings: Partial = {}): ProgramPool { - const pool = programPool({ workers: 4, heapMegabytes: 64, environment: measured, ...settings }); + const pool = programPool({ ...testPoolSettings(4), ...settings }); pools.push(pool); return pool; } diff --git a/packages/workflow-engine/src/instances/fresh-instances.test.ts b/packages/workflow-engine/src/instances/fresh-instances.test.ts index b90f641e2..6382c7417 100644 --- a/packages/workflow-engine/src/instances/fresh-instances.test.ts +++ b/packages/workflow-engine/src/instances/fresh-instances.test.ts @@ -1,4 +1,5 @@ import * as releaseSync from '@jitl/quickjs-wasmfile-release-sync'; +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { Schema } from 'effect'; import { newQuickJSWASMModuleFromVariant, newVariant, type QuickJSSyncVariant } from 'quickjs-emscripten-core'; import { describe, expect, it } from 'vitest'; @@ -11,7 +12,7 @@ import { instanceStock } from './instance-stock.ts'; const evaluation = { budget: 1_000_000, deadlineAt: Number.POSITIVE_INFINITY, moment: 0 }; function allocated(memoryBytes: number, megabytes: number): Promise { - return freshInstance(memoryBytes).then((instance) => { + return freshInstance(shippedFilesOnDisk, memoryBytes).then((instance) => { const unit = expressionUnitOf(() => instance, { stackBytes: threadStackBytes, mostAnswerBytes: 1024, @@ -78,7 +79,10 @@ describe('a fresh instance', () => { }); it('shares nothing with another instance made from the same compiled module', async () => { - const [first, second] = await Promise.all([freshInstance(pageBytes * 512), freshInstance(pageBytes * 512)]); + const [first, second] = await Promise.all([ + freshInstance(shippedFilesOnDisk, pageBytes * 512), + freshInstance(shippedFilesOnDisk, pageBytes * 512), + ]); expect(first.module).not.toBe(second.module); expect([first.memoryBytes, second.memoryBytes]).toEqual([pageBytes * 512, pageBytes * 512]); @@ -87,7 +91,7 @@ describe('a fresh instance', () => { describe('a stock of instances', () => { it('hands out a fresh instance of the memory asked for every time, preparing the next while the last is in use', async () => { - const instances = instanceStock(); + const instances = instanceStock((memoryBytes) => freshInstance(shippedFilesOnDisk, memoryBytes)); const [first, second, other] = [ await instances(pageBytes * 512), diff --git a/packages/workflow-engine/src/instances/fresh-instances.ts b/packages/workflow-engine/src/instances/fresh-instances.ts index 72da0fb7b..2168ab529 100644 --- a/packages/workflow-engine/src/instances/fresh-instances.ts +++ b/packages/workflow-engine/src/instances/fresh-instances.ts @@ -1,10 +1,10 @@ -import { readFile } from 'node:fs/promises'; - import * as releaseSync from '@jitl/quickjs-wasmfile-release-sync'; +import type { ShippedFiles } from '@on.auto/operations'; import { Schema } from 'effect'; import { newQuickJSWASMModuleFromVariant, newVariant, type QuickJSSyncVariant } from 'quickjs-emscripten-core'; import type { SandboxInstance } from '../programs/sandbox-session.ts'; +import type { Instances } from './instance-stock.ts'; interface GrowingMemory { readonly grow: (pages: number) => number; @@ -26,6 +26,8 @@ interface Bounded { export const pageBytes = 65_536; +export const sandboxModuleFile = new URL(import.meta.resolve('@jitl/quickjs-wasmfile-release-sync/wasm')); + const initialPages = 256; const attemptsOfOneGrowth = 3; @@ -41,14 +43,7 @@ const variant = Schema.decodeUnknownSync( Schema.declare((value: unknown): value is QuickJSSyncVariant => Reflect.get(new Object(value), 'type') === 'sync'), )(releaseSync.default); -const compiledModule: { pending?: Promise } = {}; - -function compiled(): Promise { - compiledModule.pending ??= readFile(new URL(import.meta.resolve('@jitl/quickjs-wasmfile-release-sync/wasm'))).then( - webAssembly.compile, - ); - return compiledModule.pending; -} +const instancesThrough = new WeakMap(); function boundedMemory(memoryBytes: number): Bounded { const growth = { failedInARow: 0, refused: false }; @@ -70,9 +65,19 @@ function boundedMemory(memoryBytes: number): Bounded { return { memory, refusedGrowth: () => growth.refused }; } -export async function freshInstance(memoryBytes: number): Promise { - const { memory, refusedGrowth } = boundedMemory(memoryBytes); - const wasmModule = await compiled(); - const module = await newQuickJSWASMModuleFromVariant(newVariant(variant, { wasmModule, wasmMemory: memory })); - return { module, memoryBytes, refusedGrowth }; +export function freshInstancesOf(sandboxModule: () => Promise>): Instances { + const compiled: { pending?: Promise } = {}; + return async (memoryBytes) => { + const { memory, refusedGrowth } = boundedMemory(memoryBytes); + compiled.pending ??= sandboxModule().then(webAssembly.compile); + const wasmModule = await compiled.pending; + const module = await newQuickJSWASMModuleFromVariant(newVariant(variant, { wasmModule, wasmMemory: memory })); + return { module, memoryBytes, refusedGrowth }; + }; +} + +export function freshInstance(shippedFiles: ShippedFiles, memoryBytes: number): Promise { + const instances = instancesThrough.get(shippedFiles) ?? freshInstancesOf(() => shippedFiles.bytes(sandboxModuleFile)); + instancesThrough.set(shippedFiles, instances); + return instances(memoryBytes); } diff --git a/packages/workflow-engine/src/instances/host-sandboxes.test.ts b/packages/workflow-engine/src/instances/host-sandboxes.test.ts index f108c87c7..ba46056d0 100644 --- a/packages/workflow-engine/src/instances/host-sandboxes.test.ts +++ b/packages/workflow-engine/src/instances/host-sandboxes.test.ts @@ -1,3 +1,4 @@ +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { Effect } from 'effect'; import { describe, expect, it } from 'vitest'; @@ -12,8 +13,8 @@ const overflows = '"x".repeat(96 * 1024 * 1024).length'; describe('the sandboxes of a host', () => { it('reserves an instance of 64 MiB for each input of a workflow, read against the clock of the host by default', async () => { - const sandbox = machineSandboxOf(); - const counted = machineSandboxOf(() => 12); + const sandbox = machineSandboxOf(shippedFilesOnDisk); + const counted = machineSandboxOf(shippedFilesOnDisk, () => 12); await Effect.runPromise(sandbox.reserve); const unit = sandbox.unit(); @@ -30,7 +31,7 @@ describe('the sandboxes of a host', () => { }); it('opens a context on an instance of 64 MiB for each group of filters, read against the clock of the host by default', async () => { - const sandbox = filterSandboxOf(); + const sandbox = filterSandboxOf(shippedFilesOnDisk); const session = await sandbox.session(() => evaluation); const [fitting, overflowing] = [session.define(fits), session.define(overflows)]; diff --git a/packages/workflow-engine/src/instances/host-sandboxes.ts b/packages/workflow-engine/src/instances/host-sandboxes.ts index 1bf853fe8..9ad65c9e4 100644 --- a/packages/workflow-engine/src/instances/host-sandboxes.ts +++ b/packages/workflow-engine/src/instances/host-sandboxes.ts @@ -1,3 +1,5 @@ +import type { ShippedFiles } from '@on.auto/operations'; + import type { FilterSandbox } from '../filters/filter-verdicts.ts'; import { filterContextOf } from '../programs/kept-contexts.ts'; import { reservedSandbox, type MachineSandbox } from '../programs/reserved-instances.ts'; @@ -9,12 +11,12 @@ export function hostClock(): number { return performance.now(); } -export function machineSandboxOf(clock: () => number = hostClock): MachineSandbox { - return reservedSandbox(() => freshInstance(unitMemoryBytes), clock); +export function machineSandboxOf(shippedFiles: ShippedFiles, clock: () => number = hostClock): MachineSandbox { + return reservedSandbox(() => freshInstance(shippedFiles, unitMemoryBytes), clock); } -export function filterSandboxOf(clock: () => number = hostClock): FilterSandbox { - const instances = instanceStock(); +export function filterSandboxOf(shippedFiles: ShippedFiles, clock: () => number = hostClock): FilterSandbox { + const instances = instanceStock((memoryBytes) => freshInstance(shippedFiles, memoryBytes)); return { session: async (opening) => { const settings = { stackBytes: threadStackBytes, mostAnswerBytes: unitMemoryBytes, clock }; diff --git a/packages/workflow-engine/src/instances/instance-stock.ts b/packages/workflow-engine/src/instances/instance-stock.ts index f5e7d7c6b..fcf46a2ec 100644 --- a/packages/workflow-engine/src/instances/instance-stock.ts +++ b/packages/workflow-engine/src/instances/instance-stock.ts @@ -1,13 +1,12 @@ import type { SandboxInstance } from '../programs/sandbox-session.ts'; -import { freshInstance } from './fresh-instances.ts'; export type Instances = (memoryBytes: number) => Promise; -export function instanceStock(): Instances { +export function instanceStock(fresh: Instances): Instances { const ready = new Map>(); return (memoryBytes) => { - const next = ready.get(memoryBytes) ?? freshInstance(memoryBytes); - ready.set(memoryBytes, freshInstance(memoryBytes)); + const next = ready.get(memoryBytes) ?? fresh(memoryBytes); + ready.set(memoryBytes, fresh(memoryBytes)); return next; }; } diff --git a/packages/workflow-engine/src/instances/sandbox-probes.test.ts b/packages/workflow-engine/src/instances/sandbox-probes.test.ts index a18ce1c63..b29b0350a 100644 --- a/packages/workflow-engine/src/instances/sandbox-probes.test.ts +++ b/packages/workflow-engine/src/instances/sandbox-probes.test.ts @@ -1,10 +1,11 @@ +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { describe, expect, it } from 'vitest'; import { sandboxAnswers } from './sandbox-probes.ts'; describe('the answers of the sandbox to probes', () => { it('answers each expression as JSON, or names how it ended', async () => { - expect(await sandboxAnswers(['typeof eval', 'Date.now()', 'null.x', '0 / 0'])).toEqual([ + expect(await sandboxAnswers(shippedFilesOnDisk, ['typeof eval', 'Date.now()', 'null.x', '0 / 0'])).toEqual([ '"undefined"', '0', "raised: TypeError: cannot read property 'x' of null", diff --git a/packages/workflow-engine/src/instances/sandbox-probes.ts b/packages/workflow-engine/src/instances/sandbox-probes.ts index e5e4aa90a..188390bf6 100644 --- a/packages/workflow-engine/src/instances/sandbox-probes.ts +++ b/packages/workflow-engine/src/instances/sandbox-probes.ts @@ -1,11 +1,16 @@ +import type { ShippedFiles } from '@on.auto/operations'; + import { expressionUnitOf } from '../programs/expression-units.ts'; import { threadStackBytes, unitMemoryBytes } from '../programs/sandbox-bounds.ts'; import { freshInstance } from './fresh-instances.ts'; const probing = { budget: Number.POSITIVE_INFINITY, deadlineAt: Number.POSITIVE_INFINITY, moment: 0 }; -export async function sandboxAnswers(expressions: readonly string[]): Promise { - const instance = await freshInstance(unitMemoryBytes); +export async function sandboxAnswers( + shippedFiles: ShippedFiles, + expressions: readonly string[], +): Promise { + const instance = await freshInstance(shippedFiles, unitMemoryBytes); const unit = expressionUnitOf(() => instance, { stackBytes: threadStackBytes, mostAnswerBytes: unitMemoryBytes, diff --git a/packages/workflow-engine/src/job-loop.ts b/packages/workflow-engine/src/job-loop.ts index 22abc0b6b..d106fff3f 100644 --- a/packages/workflow-engine/src/job-loop.ts +++ b/packages/workflow-engine/src/job-loop.ts @@ -1,2 +1,3 @@ export type { CheckHandler, FoldHandler, JobHandlers, ProgramHandler, ValueChecks } from './jobs/job-kit.ts'; -export { serveJobs } from './program-pool/job-loop.ts'; +export { handedFileIn, type HandedFile } from './program-pool/handed-files.ts'; +export { serveJobs, serveJobsWith } from './program-pool/job-loop.ts'; diff --git a/packages/workflow-engine/src/jobs/pool-contract.ts b/packages/workflow-engine/src/jobs/pool-contract.ts index 25cee4d05..32d1cbda0 100644 --- a/packages/workflow-engine/src/jobs/pool-contract.ts +++ b/packages/workflow-engine/src/jobs/pool-contract.ts @@ -7,9 +7,17 @@ import type { Ending, Interrupted } from './job-endings.ts'; import type { ProgramAnswer } from './program-messages.ts'; import type { Evaluations } from './remote-evaluations.ts'; +export type HandedFiles = Readonly>>; + +export interface HandedWorkers { + readonly sandbox: HandedFiles; + readonly modules?: Readonly>; +} + export interface PoolSettings { readonly workers: number; readonly heapMegabytes: number; + readonly handed: HandedWorkers; readonly worker?: Readonly; readonly foldWorker?: Readonly; readonly evaluationWorker?: Readonly; diff --git a/packages/workflow-engine/src/jobs/program-answer.test.ts b/packages/workflow-engine/src/jobs/program-answer.test.ts index 09763c319..fdc6baf04 100644 --- a/packages/workflow-engine/src/jobs/program-answer.test.ts +++ b/packages/workflow-engine/src/jobs/program-answer.test.ts @@ -1,3 +1,4 @@ +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { describe, expect, it } from 'vitest'; import { freshInstance } from '../instances/fresh-instances.ts'; @@ -8,7 +9,7 @@ import type { ProgramJob } from './program-messages.ts'; async function hostOf(more: Partial = {}): Promise { return { now: () => 0, - instance: await freshInstance(unitMemoryBytes), + instance: await freshInstance(shippedFilesOnDisk, unitMemoryBytes), check: unchecked, ...more, }; diff --git a/packages/workflow-engine/src/pool-testing/checking-workers.ts b/packages/workflow-engine/src/pool-testing/checking-workers.ts index 7314c2191..9d04a469c 100644 --- a/packages/workflow-engine/src/pool-testing/checking-workers.ts +++ b/packages/workflow-engine/src/pool-testing/checking-workers.ts @@ -28,3 +28,15 @@ export const checkingForever = workerOf([ ]); export const brokenAtStart = workerOf(["throw new Error('broken on purpose');"]); + +export function checkingWithTheFilesHanded(tag: string): URL { + return workerOf([ + "import { parentPort, workerData } from 'node:worker_threads';", + `const tag = ${JSON.stringify(tag)};`, + 'const issues = Object.keys(workerData.handed).map((detail) => ({ at: 0, line: tag.length, detail }));', + "parentPort.on('message', ({ job }) => {", + " parentPort.postMessage({ job, answer: { ran: 'checked', issues }, keep: true });", + '});', + 'parentPort.postMessage({ ready: true });', + ]); +} diff --git a/packages/workflow-engine/src/pool-testing/fold-pages.ts b/packages/workflow-engine/src/pool-testing/fold-pages.ts index 6dd345225..dc6e12709 100644 --- a/packages/workflow-engine/src/pool-testing/fold-pages.ts +++ b/packages/workflow-engine/src/pool-testing/fold-pages.ts @@ -1,3 +1,4 @@ +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { Function } from 'effect'; import type { Json, JsonObject } from '../dsl/json.ts'; @@ -98,6 +99,6 @@ export function slowFirstFold(stepMs: number): Timing { } export async function folded(page: FoldPage, timing: Timing = stillTiming()): Promise { - const instances = await Promise.all(page.views.map(() => freshInstance(page.memoryBytes))); + const instances = await Promise.all(page.views.map(() => freshInstance(shippedFilesOnDisk, page.memoryBytes))); return foldPage(page, { ...timing, checkOf: entriesAtMost, instances }); } diff --git a/packages/workflow-engine/src/pool-testing/test-pools.ts b/packages/workflow-engine/src/pool-testing/test-pools.ts new file mode 100644 index 000000000..d27cb5bcd --- /dev/null +++ b/packages/workflow-engine/src/pool-testing/test-pools.ts @@ -0,0 +1,19 @@ +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; +import { measuredEnvironment } from '@on.auto/operations/testing'; + +import { machineSandboxOf } from '../instances/host-sandboxes.ts'; +import type { HandedFiles, HandedWorkers, PoolSettings } from '../jobs/pool-contract.ts'; +import { handedWorkersOf } from '../program-pool/handed-files.ts'; +import type { MachineSandbox } from '../programs/reserved-instances.ts'; + +export const handedWorkers: HandedWorkers = await handedWorkersOf(shippedFilesOnDisk); + +export const handedSandbox: HandedFiles = handedWorkers.sandbox; + +export function testPoolSettings(workers: number): PoolSettings { + return { workers, heapMegabytes: 64, environment: measuredEnvironment, handed: handedWorkers }; +} + +export function machineOnTheThread(): MachineSandbox { + return machineSandboxOf(shippedFilesOnDisk); +} diff --git a/packages/workflow-engine/src/pool-testing/test-sandbox.ts b/packages/workflow-engine/src/pool-testing/test-sandbox.ts index 61d94e25f..9e7757202 100644 --- a/packages/workflow-engine/src/pool-testing/test-sandbox.ts +++ b/packages/workflow-engine/src/pool-testing/test-sandbox.ts @@ -1,3 +1,5 @@ +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; + import { workflowMachine } from '../decider/workflow-machine.ts'; import { freshInstance } from '../instances/fresh-instances.ts'; import type { RunDecider } from '../machine/run-decider.ts'; @@ -7,7 +9,10 @@ import type { MachineOptions } from '../runner/run-descriptors.ts'; import { testMachineOf } from '../testing/driver-inputs.ts'; import { memoryDriver, type DriverOptions, type MemoryDriver } from '../testing/memory-driver.ts'; -export const testSandbox: MachineSandbox = reusedSandbox(await freshInstance(unitMemoryBytes), () => 0); +export const testSandbox: MachineSandbox = reusedSandbox( + await freshInstance(shippedFilesOnDisk, unitMemoryBytes), + () => 0, +); export const testMachine: MachineOptions = testMachineOf(testSandbox); diff --git a/packages/workflow-engine/src/program-pool/evaluation-loop.test.ts b/packages/workflow-engine/src/program-pool/evaluation-loop.test.ts index 99123928c..d9d8c3266 100644 --- a/packages/workflow-engine/src/program-pool/evaluation-loop.test.ts +++ b/packages/workflow-engine/src/program-pool/evaluation-loop.test.ts @@ -1,5 +1,6 @@ import { afterEach, describe, expect, it } from 'vitest'; +import { handedSandbox } from '../pool-testing/test-pools.ts'; import type { Evaluation } from '../programs/program-run.ts'; import { serveEvaluations } from './evaluation-loop.ts'; @@ -24,7 +25,11 @@ afterEach(() => { function servedHere(): Served { const channel = new MessageChannel(); channels.push(channel); - serveEvaluations({ port: channel.port1, flags: new SharedArrayBuffer(2 * Int32Array.BYTES_PER_ELEMENT) }); + serveEvaluations({ + port: channel.port1, + flags: new SharedArrayBuffer(2 * Int32Array.BYTES_PER_ELEMENT), + handed: handedSandbox, + }); return { ask: (request) => { const { promise, resolve } = Promise.withResolvers(); diff --git a/packages/workflow-engine/src/program-pool/evaluation-loop.ts b/packages/workflow-engine/src/program-pool/evaluation-loop.ts index 44fbd12d9..70f6cd573 100644 --- a/packages/workflow-engine/src/program-pool/evaluation-loop.ts +++ b/packages/workflow-engine/src/program-pool/evaluation-loop.ts @@ -2,7 +2,6 @@ import { MessagePort, workerData } from 'node:worker_threads'; import { Option, Schema } from 'effect'; -import { freshInstance } from '../instances/fresh-instances.ts'; import { instanceStock, type Instances } from '../instances/instance-stock.ts'; import { answeredFlag, @@ -16,6 +15,7 @@ import { filterContextOf, type FilterContext, type FilterTest } from '../program import type { ProgramRun } from '../programs/program-run.ts'; import { threadStackBytes, unitMemoryBytes } from '../programs/sandbox-bounds.ts'; import type { SandboxSettings } from '../programs/sandbox-session.ts'; +import { handedSandboxOf } from './handed-files.ts'; interface PreparedFilters { readonly context: FilterContext; @@ -111,8 +111,9 @@ export function serveEvaluations(data: unknown = workerData): void { Atomics.notify(flags, answeredFlag); return run; }; - served(port, answer, { instances: instanceStock(), units: new Map(), filters: new Map() }); - void freshInstance(unitMemoryBytes).then((instance) => { + const sandbox = handedSandboxOf(data); + served(port, answer, { instances: instanceStock(sandbox), units: new Map(), filters: new Map() }); + void sandbox(unitMemoryBytes).then((instance) => { Atomics.store(flags, readyFlag, 1); Atomics.notify(flags, readyFlag); return instance; diff --git a/packages/workflow-engine/src/program-pool/evaluation-thread.test.ts b/packages/workflow-engine/src/program-pool/evaluation-thread.test.ts index 016b0b716..88c477f8e 100644 --- a/packages/workflow-engine/src/program-pool/evaluation-thread.test.ts +++ b/packages/workflow-engine/src/program-pool/evaluation-thread.test.ts @@ -1,9 +1,9 @@ import { Effect } from 'effect'; import { afterEach, describe, expect, it } from 'vitest'; -import { machineSandboxOf } from '../instances/host-sandboxes.ts'; import { decidedOnce } from '../pool-testing/decided-once.ts'; import { churn, evaluatedAt } from '../pool-testing/evaluation-workers.ts'; +import { machineOnTheThread, testPoolSettings } from '../pool-testing/test-pools.ts'; import { threadsAlive } from '../pool-testing/threads-alive.ts'; import { longerThanTheTestMs } from '../pool-testing/waiting.ts'; import type { Evaluation, ProgramRun } from '../programs/program-run.ts'; @@ -20,10 +20,6 @@ const severalTimesSlowerMs = 3000; const pools: ProgramPool[] = []; -const coverage = process.env['NODE_V8_COVERAGE']; - -const measured = coverage === undefined ? {} : { NODE_V8_COVERAGE: coverage }; - const filling = '(() => { const kept = []; for (;;) { kept.push("x".repeat(1048576) + kept.length); } })()'; const caughtStackBomb = @@ -36,7 +32,7 @@ afterEach(async () => { }); function poolOf(settings: Partial = {}): ProgramPool { - const pool = programPool({ workers: 1, heapMegabytes: 64, environment: measured, ...settings }); + const pool = programPool({ ...testPoolSettings(1), ...settings }); pools.push(pool); return pool; } @@ -64,6 +60,14 @@ async function timed(work: () => Answer | Promise): Promise<{ an return { answer, ms: performance.now() - started }; } +describe('the evaluation worker of a pool asked before any reserve', () => { + it('starts with the module of the sandbox the settings of the pool hand it, and answers once it is ready', () => { + const { machine } = poolOf().evaluations; + + expect(evaluatedIn(machine, '1 + 1', longerThanTheTestMs)).toMatchObject({ ran: 'answered', text: '2' }); + }); +}); + describe('the evaluation worker of a pool, given native work that no checkpoint interrupts', () => { it('ends the expression of a decision at the input deadline, within about 2.1 s, and decides the next input on a fresh worker', async () => { const { machine } = poolOf().evaluations; @@ -87,7 +91,7 @@ describe('the evaluation worker of a pool, given native work that no checkpoint describe('the evaluation worker of a pool, given a stack or a memory to exhaust', () => { it("gives a caught stack bomb one answer, from the shallow stack of the host, from 5,000 frames deep and on a host's own thread", async () => { const { machine } = poolOf().evaluations; - const here = machineSandboxOf(); + const here = machineOnTheThread(); await Promise.all([Effect.runPromise(machine.reserve), Effect.runPromise(here.reserve)]); const shallow = evaluatedIn(machine, caughtStackBomb); diff --git a/packages/workflow-engine/src/program-pool/evaluation-thread.ts b/packages/workflow-engine/src/program-pool/evaluation-thread.ts index c2f8197ee..c3d93dac0 100644 --- a/packages/workflow-engine/src/program-pool/evaluation-thread.ts +++ b/packages/workflow-engine/src/program-pool/evaluation-thread.ts @@ -12,6 +12,7 @@ import { type Evaluations, } from '../jobs/remote-evaluations.ts'; import { exhaustedBy, type ProgramRun } from '../programs/program-run.ts'; +import { handedTo } from './handed-files.ts'; import { workerOptions } from './pool-threads.ts'; export const answerGraceMs = 50; @@ -56,7 +57,7 @@ function started(module: Readonly, settings: PoolSettings): Live { const flags = new Int32Array(new SharedArrayBuffer(2 * Int32Array.BYTES_PER_ELEMENT)); const worker = new Worker(module, { ...workerOptions(settings), - workerData: { port: port2, flags: flags.buffer }, + workerData: { port: port2, flags: flags.buffer, handed: handedTo(settings.handed, module) }, transferList: [port2], }); const state = { gone: false }; diff --git a/packages/workflow-engine/src/program-pool/handed-files.test.ts b/packages/workflow-engine/src/program-pool/handed-files.test.ts new file mode 100644 index 000000000..a90c5c6b7 --- /dev/null +++ b/packages/workflow-engine/src/program-pool/handed-files.test.ts @@ -0,0 +1,61 @@ +import { shippedFilesInMemory } from '@on.auto/operations/testing'; +import { describe, expect, it } from 'vitest'; + +import { sandboxModuleFile } from '../instances/fresh-instances.ts'; +import { handedFile, handedFileIn, handedFilesOf, handedTo, handedWorkersOf } from './handed-files.ts'; + +const shipped = new URL('file:///shipped/'); + +const lib = new URL('lib.txt', shipped); + +const checker = new URL('file:///workers/checker.ts'); + +const runner = new URL('file:///workers/runner.ts'); + +const memory = shippedFilesInMemory({ + [sandboxModuleFile.href]: Uint8Array.of(1, 2), + [lib.href]: Uint8Array.of(3), +}); + +function bytesOf(handed: Readonly>>): Readonly> { + return Object.fromEntries( + Object.entries(handed).map(([href, bytes]: readonly [string, Readonly]) => [href, [...bytes]]), + ); +} + +describe('the files a composition hands the workers of a pool', () => { + it('are the bytes of the files it names, read through the reader, by the address of each', async () => { + expect(bytesOf(await handedFilesOf(memory, [lib, sandboxModuleFile]))).toEqual({ + [lib.href]: [3], + [sandboxModuleFile.href]: [1, 2], + }); + }); + + it('are, for every worker, the module of the sandbox, and for the workers of a module named, its own files instead', async () => { + const handed = await handedWorkersOf(memory, { [checker.href]: await handedFilesOf(memory, [lib]) }); + + expect(bytesOf(handedTo(handed, runner))).toEqual({ [sandboxModuleFile.href]: [1, 2] }); + expect(bytesOf(handedTo(handed, checker))).toEqual({ [lib.href]: [3] }); + expect(bytesOf(handedTo({ sandbox: handed.sandbox }, checker))).toEqual({ [sandboxModuleFile.href]: [1, 2] }); + }); + + it('fail to be read when the reader cannot read one of them', async () => { + await expect(handedFilesOf(memory, [new URL('missing.bin', shipped)])).rejects.toThrow('missing.bin'); + }); +}); + +describe('a file a worker reads of those it was handed', () => { + it('is the bytes the pool handed it', () => { + expect([...handedFile(lib, { handed: { [lib.href]: Uint8Array.of(7) } })]).toEqual([7]); + expect([...handedFileIn({ [lib.href]: Uint8Array.of(8) })(lib)]).toEqual([8]); + }); + + it('is refused when the pool handed the worker no such file, or nothing it can read', () => { + const refused = `The pool handed this worker no file at ${lib.href}`; + + expect(() => handedFile(lib, { handed: {} })).toThrow(refused); + expect(() => handedFile(lib, { handed: { [lib.href]: 'not bytes' } })).toThrow(refused); + expect(() => handedFile(lib)).toThrow(refused); + expect(() => handedFileIn({})(lib)).toThrow(refused); + }); +}); diff --git a/packages/workflow-engine/src/program-pool/handed-files.ts b/packages/workflow-engine/src/program-pool/handed-files.ts new file mode 100644 index 000000000..cf5d79800 --- /dev/null +++ b/packages/workflow-engine/src/program-pool/handed-files.ts @@ -0,0 +1,52 @@ +import { workerData } from 'node:worker_threads'; + +import type { ShippedFiles } from '@on.auto/operations'; +import { Option, Schema } from 'effect'; + +import { freshInstancesOf, sandboxModuleFile } from '../instances/fresh-instances.ts'; +import type { Instances } from '../instances/instance-stock.ts'; +import type { HandedFiles, HandedWorkers } from '../jobs/pool-contract.ts'; + +export type HandedFile = (file: Readonly) => Readonly; + +interface Handed { + readonly handed: HandedFiles; +} + +const decodeHanded = Schema.decodeUnknownOption( + Schema.Struct({ handed: Schema.Record(Schema.String, Schema.instanceOf(Uint8Array)) }), +); + +export async function handedFilesOf(shippedFiles: ShippedFiles, files: readonly Readonly[]): Promise { + const contents = await Promise.all(files.map(async (file) => [file.href, await shippedFiles.bytes(file)] as const)); + return Object.fromEntries(contents); +} + +export async function handedWorkersOf( + shippedFiles: ShippedFiles, + modules: Readonly> = {}, +): Promise { + return { sandbox: await handedFilesOf(shippedFiles, [sandboxModuleFile]), modules }; +} + +export function handedTo({ sandbox, modules = {} }: HandedWorkers, module: Readonly): HandedFiles { + return modules[module.href] ?? sandbox; +} + +export function handedFileIn(handed: HandedFiles): HandedFile { + return (file) => + Option.getOrThrowWith( + Option.fromNullishOr(handed[file.href]), + () => new Error(`The pool handed this worker no file at ${file.href}`), + ); +} + +export function handedFile(file: Readonly, data: unknown = workerData): Readonly { + return handedFileIn(Option.match(decodeHanded(data), { onNone: () => ({}), onSome: ({ handed }: Handed) => handed }))( + file, + ); +} + +export function handedSandboxOf(data: unknown): Instances { + return freshInstancesOf(() => Promise.try(() => handedFile(sandboxModuleFile, data))); +} diff --git a/packages/workflow-engine/src/program-pool/job-loop.test.ts b/packages/workflow-engine/src/program-pool/job-loop.test.ts index c397f253d..6394807c8 100644 --- a/packages/workflow-engine/src/program-pool/job-loop.test.ts +++ b/packages/workflow-engine/src/program-pool/job-loop.test.ts @@ -5,10 +5,12 @@ import { foldAnswerOf, foldPageData } from '../folds/fold-answer.ts'; import type { FoldPage } from '../folds/fold-page.ts'; import { foldProgress, type FoldProgress } from '../folds/fold-progress.ts'; import type { CheckJob } from '../jobs/check-messages.ts'; -import type { JobHandlers } from '../jobs/job-kit.ts'; import { answerOf, type OutputCheck } from '../jobs/program-answer.ts'; +import { handedSandbox } from '../pool-testing/test-pools.ts'; import { threadStackBytes, unitMemoryBytes } from '../programs/sandbox-bounds.ts'; -import { serveJobs } from './job-loop.ts'; +import { serveJobs, serveJobsWith } from './job-loop.ts'; + +type HandlersOf = Parameters[0]; type Ask = (message: unknown) => Promise; @@ -29,12 +31,12 @@ afterEach(() => { } }); -function served(handlers: JobHandlers): Ask { +function servedWith(handlersOf: HandlersOf, handed: Readonly>> = {}): Ask { const { port1, port2 } = new MessageChannel(); channels.push({ port1, port2 }); const ready = Promise.withResolvers(); port2.once('message', ready.resolve); - serveJobs(handlers, port1); + serveJobsWith(handlersOf, port1, { handed: { ...handedSandbox, ...handed } }); return async (message) => { await ready.promise; const { promise, resolve } = Promise.withResolvers(); @@ -46,6 +48,10 @@ function served(handlers: JobHandlers): Ask { }; } +function served(handlers: ReturnType): Ask { + return servedWith(() => handlers); +} + function inTurn(ask: Ask, messages: readonly unknown[]): Promise { return messages.reduce>( async (before, message) => [...(await before), await ask(message)], @@ -122,6 +128,21 @@ describe('a loop that starts serving jobs', () => { expect(await first.promise).toEqual({ ready: true }); }); + + it('makes its handlers of the files the pool handed its worker', async () => { + const note = new URL('file:///shipped/note.txt'); + const ask = servedWith( + (handed) => ({ + check: () => ({ ran: 'checked', issues: [{ at: 0, line: 1, detail: new TextDecoder().decode(handed(note)) }] }), + }), + { [note.href]: new TextEncoder().encode('Handed at start') }, + ); + + expect(await ask({ job: 1, kind: 'check', request: checkRequest })).toMatchObject({ + job: 1, + answer: { ran: 'checked', issues: [{ detail: 'Handed at start' }] }, + }); + }); }); describe('a loop that serves jobs', () => { diff --git a/packages/workflow-engine/src/program-pool/job-loop.ts b/packages/workflow-engine/src/program-pool/job-loop.ts index 3fd17de5a..fd8f568ab 100644 --- a/packages/workflow-engine/src/program-pool/job-loop.ts +++ b/packages/workflow-engine/src/program-pool/job-loop.ts @@ -1,4 +1,4 @@ -import { parentPort, type MessagePort } from 'node:worker_threads'; +import { parentPort, workerData, type MessagePort } from 'node:worker_threads'; import { Option, Schema } from 'effect'; @@ -9,6 +9,7 @@ import { JobSchema, type JobAnswer, type JobEnvelope } from '../jobs/job-envelop import { keptFor, type JobHandlers, type Kept } from '../jobs/job-kit.ts'; import type { ProgramAnswerData } from '../jobs/program-answer.ts'; import { readySignal } from '../jobs/ready-signal.ts'; +import { handedFile, handedSandboxOf, type HandedFile } from './handed-files.ts'; type ProgramEnvelope = Extract; @@ -95,11 +96,20 @@ function answerTo(message: unknown, serving: Serving): Promise { return value.kind === 'fold' ? foldAnswer(value, serving) : Promise.resolve(checkAnswer(value, serving)); } -export function serveJobs(handlers: JobHandlers, port: MessagePort | null = parentPort): void { +export function serveJobsWith( + handlersOf: (handed: HandedFile) => JobHandlers, + port: MessagePort | null = parentPort, + data: unknown = workerData, +): void { if (port === null) { return; } - const serving: Serving = { handlers, kept: keptFor(handlers.checks), instances: instanceStock() }; + const handlers = handlersOf((file) => handedFile(file, data)); + const serving: Serving = { + handlers, + kept: keptFor(handlers.checks), + instances: instanceStock(handedSandboxOf(data)), + }; port.on('message', (message: unknown) => { void answerTo(message, serving).then((answer) => { port.postMessage(answer, []); @@ -108,3 +118,11 @@ export function serveJobs(handlers: JobHandlers, port: MessagePort | null = pare }); port.postMessage(readySignal, []); } + +export function serveJobs( + handlers: JobHandlers, + port: MessagePort | null = parentPort, + data: unknown = workerData, +): void { + serveJobsWith(() => handlers, port, data); +} diff --git a/packages/workflow-engine/src/program-pool/pool-checks.test.ts b/packages/workflow-engine/src/program-pool/pool-checks.test.ts index b2095531e..d296a7689 100644 --- a/packages/workflow-engine/src/program-pool/pool-checks.test.ts +++ b/packages/workflow-engine/src/program-pool/pool-checks.test.ts @@ -2,8 +2,15 @@ import { Schema } from 'effect'; import { afterEach, describe, expect, it } from 'vitest'; import type { CheckOutcome, CheckRequest, PoolOutcome, PoolSettings, ProgramPool } from '../jobs/pool-contract.ts'; -import { brokenAtStart, checking, checkingForever, checkingReadyAfter } from '../pool-testing/checking-workers.ts'; +import { + brokenAtStart, + checking, + checkingForever, + checkingReadyAfter, + checkingWithTheFilesHanded, +} from '../pool-testing/checking-workers.ts'; import { counting } from '../pool-testing/counting-workers.ts'; +import { handedSandbox, handedWorkers, testPoolSettings } from '../pool-testing/test-pools.ts'; import { nextThreadStarted, threadEnded } from '../pool-testing/threads-alive.ts'; import { isWatched, longerThanTheTestMs, until } from '../pool-testing/waiting.ts'; import { unitMemoryBytes, workerStackBytes } from '../programs/sandbox-bounds.ts'; @@ -13,10 +20,6 @@ const slowStartMs = 2000; const pools: ProgramPool[] = []; -const coverage = process.env['NODE_V8_COVERAGE']; - -const measured = coverage === undefined ? {} : { NODE_V8_COVERAGE: coverage }; - const decodeRan = Schema.decodeUnknownSync(Schema.Struct({ thread: Schema.Number })); afterEach(async () => { @@ -24,7 +27,7 @@ afterEach(async () => { }); function poolOf(settings: Partial = {}): ProgramPool { - const pool = programPool({ workers: 4, heapMegabytes: 64, environment: measured, ...settings }); + const pool = programPool({ ...testPoolSettings(4), ...settings }); pools.push(pool); return pool; } @@ -39,6 +42,10 @@ function checkOf(more: Partial = {}): CheckRequest { }; } +function detailsOf(outcome: CheckOutcome): readonly string[] { + return outcome.ran === 'checked' ? outcome.issues.map(({ detail }) => detail) : []; +} + function threadOf(outcome: CheckOutcome): number | undefined { return outcome.ran === 'checked' ? outcome.issues[0]?.line : undefined; } @@ -101,6 +108,21 @@ describe('the permit of the checks of a pool', () => { }); }); +describe('the files the worker of a check starts with', () => { + it('are those the settings of the pool hand its module, and no other, while a module they name nothing for starts with the module of the sandbox', async () => { + const lib = new URL('file:///shipped/lib.txt'); + const named = checkingWithTheFilesHanded('named'); + const other = checkingWithTheFilesHanded('other'); + const pool = poolOf({ handed: { ...handedWorkers, modules: { [named.href]: { [lib.href]: Uint8Array.of(1) } } } }); + + const ofTheNamed = await pool.check(checkOf({ worker: named })); + const ofTheOther = await pool.check(checkOf({ worker: other })); + + expect(detailsOf(ofTheNamed)).toEqual([lib.href]); + expect(detailsOf(ofTheOther)).toEqual(Object.keys(handedSandbox)); + }); +}); + describe('the worker of the checks of a pool, while it starts', () => { it('counts a check’s deadline from when its worker is ready, so a worker slower to start than the deadline answers', async () => { const pool = poolOf(); diff --git a/packages/workflow-engine/src/program-pool/pool-permits.test.ts b/packages/workflow-engine/src/program-pool/pool-permits.test.ts index 8e293aecd..50192bec3 100644 --- a/packages/workflow-engine/src/program-pool/pool-permits.test.ts +++ b/packages/workflow-engine/src/program-pool/pool-permits.test.ts @@ -6,6 +6,7 @@ import { afterEach, describe, expect, it } from 'vitest'; import type { PoolOutcome, PoolSettings, ProgramPool } from '../jobs/pool-contract.ts'; import { begun, counting, countingElsewhere } from '../pool-testing/counting-workers.ts'; +import { testPoolSettings } from '../pool-testing/test-pools.ts'; import { threadsAlive } from '../pool-testing/threads-alive.ts'; import { unitMemoryBytes, workerStackBytes } from '../programs/sandbox-bounds.ts'; import { programPool } from './program-pool.ts'; @@ -17,10 +18,6 @@ interface Ran { const pools: ProgramPool[] = []; -const coverage = process.env['NODE_V8_COVERAGE']; - -const measured = coverage === undefined ? {} : { NODE_V8_COVERAGE: coverage }; - const decodeRan = Schema.decodeUnknownSync(Schema.Struct({ jobs: Schema.Number, thread: Schema.Number })); async function executed(script: string): Promise { @@ -42,7 +39,7 @@ afterEach(async () => { }); function poolOf(settings: Partial = {}): ProgramPool { - const pool = programPool({ workers: 4, heapMegabytes: 64, environment: measured, ...settings }); + const pool = programPool({ ...testPoolSettings(4), ...settings }); pools.push(pool); return pool; } @@ -183,7 +180,9 @@ describe('a pool that closes', () => { ])('holds no process open %s', async (_when, closing) => { const script = [ `const { programPool } = await import('${new URL('program-pool.ts', import.meta.url).href}');`, - 'const pool = programPool({ workers: 2, heapMegabytes: 64 });', + `const { handedWorkersOf } = await import('${new URL('handed-files.ts', import.meta.url).href}');`, + `const { shippedFilesOnDisk } = await import('${import.meta.resolve('@on.auto/operations/shipped-files-on-disk')}');`, + 'const pool = programPool({ handed: await handedWorkersOf(shippedFilesOnDisk), workers: 2, heapMegabytes: 64 });', "const source = 'export default function (input) { return input + 1; }';", "const outcome = await pool.run({ source, arguments: [1], entry: 'default', moment: 0, budget: 500, memoryBytes: 67108864, stackBytes: 1048576, deadlineMs: 10000, mostOutputBytes: 100 });", closing, diff --git a/packages/workflow-engine/src/program-pool/pool-threads.ts b/packages/workflow-engine/src/program-pool/pool-threads.ts index 6a18ada43..cf01e6996 100644 --- a/packages/workflow-engine/src/program-pool/pool-threads.ts +++ b/packages/workflow-engine/src/program-pool/pool-threads.ts @@ -7,6 +7,7 @@ import { JobAnswerSchema, type JobSchema } from '../jobs/job-envelopes.ts'; import type { PoolSettings } from '../jobs/pool-contract.ts'; import type { Job, Running } from '../jobs/pool-job.ts'; import { ReadySchema } from '../jobs/ready-signal.ts'; +import { handedTo } from './handed-files.ts'; export const workerStackMegabytes = 64; @@ -202,8 +203,12 @@ export function workerOptions(settings: PoolSettings): WorkerOptions { }; } +function workerStarting(module: Readonly, settings: PoolSettings): Worker { + return new Worker(module, { ...workerOptions(settings), workerData: { handed: handedTo(settings.handed, module) } }); +} + function threadOf(start: Job, settings: PoolSettings, hooks: ThreadHooks): Thread { - const worker = new Worker(start.module, workerOptions(settings)); + const worker = workerStarting(start.module, settings); const state: ThreadState = { jobs: 0, resting: undefined }; const loading = Promise.withResolvers(); const served = async (job: number, work: Job, running: Running): Promise> => { diff --git a/packages/workflow-engine/src/program-pool/program-pool.test.ts b/packages/workflow-engine/src/program-pool/program-pool.test.ts index bc8de530d..d9837427a 100644 --- a/packages/workflow-engine/src/program-pool/program-pool.test.ts +++ b/packages/workflow-engine/src/program-pool/program-pool.test.ts @@ -2,6 +2,7 @@ import { afterEach, describe, expect, it } from 'vitest'; import type { Json } from '../dsl/json.ts'; import type { PoolSettings, ProgramPool, ProgramRequest } from '../jobs/pool-contract.ts'; +import { testPoolSettings } from '../pool-testing/test-pools.ts'; import { nextThreadStarted, threadEnded } from '../pool-testing/threads-alive.ts'; import { isWatched, longerThanTheTestMs, until } from '../pool-testing/waiting.ts'; import { runMemoryBytes, unitMemoryBytes, workerStackBytes } from '../programs/sandbox-bounds.ts'; @@ -29,12 +30,8 @@ function answeringWith(output: string): URL { ); } -const coverage = process.env['NODE_V8_COVERAGE']; - -const measured = coverage === undefined ? {} : { NODE_V8_COVERAGE: coverage }; - function poolOf(settings: Partial = {}): ProgramPool { - const pool = programPool({ workers: 4, heapMegabytes: 64, environment: measured, ...settings }); + const pool = programPool({ ...testPoolSettings(4), ...settings }); pools.push(pool); return pool; } diff --git a/packages/workflow-engine/src/program-pool/warm-workers.test.ts b/packages/workflow-engine/src/program-pool/warm-workers.test.ts index 855fba830..8f98c7c0e 100644 --- a/packages/workflow-engine/src/program-pool/warm-workers.test.ts +++ b/packages/workflow-engine/src/program-pool/warm-workers.test.ts @@ -4,6 +4,7 @@ import { afterEach, describe, expect, it } from 'vitest'; import type { FoldingView } from '../folds/fold-page.ts'; import type { FoldOutcome, FoldRequest, PoolOutcome, PoolSettings, ProgramPool } from '../jobs/pool-contract.ts'; import { begun, counting, countingElsewhere, countingOnTheLoop } from '../pool-testing/counting-workers.ts'; +import { testPoolSettings } from '../pool-testing/test-pools.ts'; import { threadEnded, threadsAlive } from '../pool-testing/threads-alive.ts'; import { longerThanTheTestMs, until } from '../pool-testing/waiting.ts'; import { unitMemoryBytes, workerStackBytes } from '../programs/sandbox-bounds.ts'; @@ -22,10 +23,6 @@ interface Running { const pools: ProgramPool[] = []; -const coverage = process.env['NODE_V8_COVERAGE']; - -const measured = coverage === undefined ? {} : { NODE_V8_COVERAGE: coverage }; - const decodeRan = Schema.decodeUnknownSync(Schema.Struct({ jobs: Schema.Number, thread: Schema.Number })); const noted = { type: 'noted' }; @@ -39,7 +36,7 @@ afterEach(async () => { }); function poolOf(settings: Partial = {}): ProgramPool { - const pool = programPool({ workers: 4, heapMegabytes: 64, environment: measured, ...settings }); + const pool = programPool({ ...testPoolSettings(4), ...settings }); pools.push(pool); return pool; } diff --git a/packages/workflow-engine/src/program-pool/worker-flags.test.ts b/packages/workflow-engine/src/program-pool/worker-flags.test.ts index b0e7cc7ca..7aa7c99b1 100644 --- a/packages/workflow-engine/src/program-pool/worker-flags.test.ts +++ b/packages/workflow-engine/src/program-pool/worker-flags.test.ts @@ -36,6 +36,7 @@ const request: ProgramRequest = { function poolReading(environment?: Readonly>): ProgramPool { const pool = programPool({ + handed: { sandbox: {} }, workers: 1, heapMegabytes: 64, worker: reading, @@ -47,7 +48,7 @@ function poolReading(environment?: Readonly>): ProgramPoo const server = [ `const { programPool } = await import(${JSON.stringify(new URL('program-pool.ts', import.meta.url).href)});`, - `const pool = programPool({ workers: 1, heapMegabytes: 64, worker: new URL(${JSON.stringify(reading.href)}) });`, + `const pool = programPool({ handed: { sandbox: {} }, workers: 1, heapMegabytes: 64, worker: new URL(${JSON.stringify(reading.href)}) });`, "const outcome = await pool.run({ source: '.', arguments: [null], entry: 'default', moment: 0, budget: 500, memoryBytes: 67108864, stackBytes: 1048576, deadlineMs: 10000, mostOutputBytes: 1000 });", 'await pool.close();', 'process.stdout.write(JSON.stringify({ secret: process.env.WORKER_SECRET, preloaded: globalThis.preloaded === true, outcome }));', diff --git a/packages/workflow-engine/src/programs/expression-units.test.ts b/packages/workflow-engine/src/programs/expression-units.test.ts index 4e41866d5..2a3fe7d69 100644 --- a/packages/workflow-engine/src/programs/expression-units.test.ts +++ b/packages/workflow-engine/src/programs/expression-units.test.ts @@ -1,3 +1,4 @@ +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { describe, expect, it } from 'vitest'; import { freshInstance } from '../instances/fresh-instances.ts'; @@ -26,7 +27,7 @@ function countedInstances(instance: SandboxInstance): { describe('a unit of expressions', () => { it('takes an instance only when it first evaluates, and keeps it for every expression after', async () => { - const counted = countedInstances(await freshInstance(unitMemoryBytes)); + const counted = countedInstances(await freshInstance(shippedFilesOnDisk, unitMemoryBytes)); const unit = expressionUnitOf(counted.instances, settings); const untouched = [unit.took(), counted.taken()]; @@ -45,7 +46,7 @@ describe('a unit of expressions', () => { }); it('gives each expression a context of its own, so nothing one keeps reaches the next', async () => { - const instance = await freshInstance(unitMemoryBytes); + const instance = await freshInstance(shippedFilesOnDisk, unitMemoryBytes); const unit = expressionUnitOf(() => instance, settings); const kept = unit.evaluate('Reflect.set(globalThis, "seen", true)', {}, evaluation); @@ -56,7 +57,7 @@ describe('a unit of expressions', () => { }); it('passes only the names an expression uses', async () => { - const instance = await freshInstance(unitMemoryBytes); + const instance = await freshInstance(shippedFilesOnDisk, unitMemoryBytes); const unit = expressionUnitOf(() => instance, settings); const run = unit.evaluate( @@ -72,7 +73,7 @@ describe('a unit of expressions', () => { describe('an expression a unit refuses', () => { it('raises what strict code raises, as the check compiles an expression: a write to a name never declared or to a frozen object', async () => { - const instance = await freshInstance(unitMemoryBytes); + const instance = await freshInstance(shippedFilesOnDisk, unitMemoryBytes); const unit = expressionUnitOf(() => instance, settings); const runs = [ @@ -91,7 +92,7 @@ describe('an expression a unit refuses', () => { }); it('raises an expression that does not parse, with its line', async () => { - const instance = await freshInstance(unitMemoryBytes); + const instance = await freshInstance(shippedFilesOnDisk, unitMemoryBytes); const unit = expressionUnitOf(() => instance, settings); const run = unit.evaluate('$data +\n * 2', { data: 1 }, evaluation); @@ -107,9 +108,9 @@ describe('an expression a unit refuses', () => { describe('a unit whose instance broke', () => { it('answers every expression after its instance refused memory as out of memory, and after its stack broke as out of stack', async () => { - const refusing = await freshInstance(unitMemoryBytes); + const refusing = await freshInstance(shippedFilesOnDisk, unitMemoryBytes); const growing = expressionUnitOf(() => refusing, settings); - const breaking = await freshInstance(unitMemoryBytes); + const breaking = await freshInstance(shippedFilesOnDisk, unitMemoryBytes); const deep = expressionUnitOf(() => breaking, { ...settings, stackBytes: workerStackBytes }); const memory = [ diff --git a/packages/workflow-engine/src/programs/module-runs.test.ts b/packages/workflow-engine/src/programs/module-runs.test.ts index fc8a766e7..c03043ade 100644 --- a/packages/workflow-engine/src/programs/module-runs.test.ts +++ b/packages/workflow-engine/src/programs/module-runs.test.ts @@ -1,3 +1,4 @@ +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { describe, expect, it } from 'vitest'; import { freshInstance } from '../instances/fresh-instances.ts'; @@ -19,7 +20,7 @@ function requestOf(source: string, more: Partial = {}): ModuleReq } async function ran(source: string, more: Partial = {}): Promise { - return moduleRun(await freshInstance(unitMemoryBytes), settings, requestOf(source, more)); + return moduleRun(await freshInstance(shippedFilesOnDisk, unitMemoryBytes), settings, requestOf(source, more)); } describe('a run of a module', () => { diff --git a/packages/workflow-engine/src/programs/reserved-instances.test.ts b/packages/workflow-engine/src/programs/reserved-instances.test.ts index 35c03ffdd..c2e93d068 100644 --- a/packages/workflow-engine/src/programs/reserved-instances.test.ts +++ b/packages/workflow-engine/src/programs/reserved-instances.test.ts @@ -1,3 +1,4 @@ +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { Effect } from 'effect'; import { describe, expect, it } from 'vitest'; @@ -7,7 +8,7 @@ import { reservedSandbox, reusedSandbox, type MachineSandbox } from './reserved- import { unitMemoryBytes } from './sandbox-bounds.ts'; import type { SandboxInstance } from './sandbox-session.ts'; -const instance = await freshInstance(unitMemoryBytes); +const instance = await freshInstance(shippedFilesOnDisk, unitMemoryBytes); const evaluation: Evaluation = { budget: 250, deadlineAt: Number.POSITIVE_INFINITY, moment: 0 }; diff --git a/packages/workflow-engine/src/programs/sandbox-prelude.test.ts b/packages/workflow-engine/src/programs/sandbox-prelude.test.ts index 076f9e4b4..9a72d8102 100644 --- a/packages/workflow-engine/src/programs/sandbox-prelude.test.ts +++ b/packages/workflow-engine/src/programs/sandbox-prelude.test.ts @@ -1,3 +1,4 @@ +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { describe, expect, it } from 'vitest'; import { freshInstance } from '../instances/fresh-instances.ts'; @@ -14,7 +15,7 @@ const evaluation: Evaluation = { budget: 500, deadlineAt: Number.POSITIVE_INFINI const settings = { stackBytes: threadStackBytes, mostAnswerBytes: 1_048_576, clock: () => 0 }; -const instance = await freshInstance(unitMemoryBytes); +const instance = await freshInstance(shippedFilesOnDisk, unitMemoryBytes); const unit = expressionUnitOf(() => instance, settings); @@ -45,7 +46,7 @@ function foldOf(folding: FoldingUnit): FilterContext & { readonly fold: Fold } { } async function returned(body: string): Promise { - return moduleRun(await freshInstance(unitMemoryBytes), settings, { + return moduleRun(await freshInstance(shippedFilesOnDisk, unitMemoryBytes), settings, { source: `export default function () {\n ${body}\n}`, entry: 'default', arguments: [], @@ -253,7 +254,7 @@ describe('a context the sandbox freezes', () => { it('lets an Error of a program name itself and a value take a key its prototype has', async () => { const folding = foldOf( foldingUnitOf( - await freshInstance(unitMemoryBytes), + await freshInstance(shippedFilesOnDisk, unitMemoryBytes), settings, { fold: [ diff --git a/packages/workflow-engine/src/programs/sandbox-session.test.ts b/packages/workflow-engine/src/programs/sandbox-session.test.ts index 5da340a93..d4a7b00f8 100644 --- a/packages/workflow-engine/src/programs/sandbox-session.test.ts +++ b/packages/workflow-engine/src/programs/sandbox-session.test.ts @@ -1,3 +1,4 @@ +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { describe, expect, it } from 'vitest'; import { freshInstance } from '../instances/fresh-instances.ts'; @@ -15,7 +16,7 @@ async function ran( more: Partial = {}, runSettings: SandboxSettings = settings, ): Promise { - const run = moduleRun(await freshInstance(unitMemoryBytes), runSettings, { + const run = moduleRun(await freshInstance(shippedFilesOnDisk, unitMemoryBytes), runSettings, { source: `export default function (input) {\n ${body}\n}`, entry: 'default', arguments: ['null'], @@ -123,7 +124,7 @@ describe('the stack of a program', () => { }); it('ends a program whose bound is deeper than the stack of the thread it runs on, which then answers no more', async () => { - const instance = await freshInstance(unitMemoryBytes); + const instance = await freshInstance(shippedFilesOnDisk, unitMemoryBytes); const runtime = sandboxRuntimeOf(instance, { ...settings, stackBytes: workerStackBytes }); const context = runtime.context(); const fn = keptOf( @@ -177,7 +178,7 @@ describe('what a program throws', () => { describe('a context of the sandbox', () => { it('calls a function over the values it parsed, and lets go of what it is told to forget', async () => { - const instance = await freshInstance(unitMemoryBytes); + const instance = await freshInstance(shippedFilesOnDisk, unitMemoryBytes); const runtime = sandboxRuntimeOf(instance, settings); const context = runtime.context(); const fn = keptOf(context.expression('((value) => (\n[value]\n))', evaluation)); @@ -196,7 +197,7 @@ describe('a context of the sandbox', () => { }); it('answers no function for an export that is not one, and raises text that is not JSON as an argument', async () => { - const runtime = sandboxRuntimeOf(await freshInstance(unitMemoryBytes), settings); + const runtime = sandboxRuntimeOf(await freshInstance(shippedFilesOnDisk, unitMemoryBytes), settings); const context = runtime.context(); const fn = keptOf(context.expression('((value) => (\n[value]\n))', evaluation)); const unreadable = { @@ -215,7 +216,7 @@ describe('a context of the sandbox', () => { describe('a context whose instance breaks', () => { it('answers the failure of a program that breaks the instance it runs on, rather than throwing', async () => { - const instance = await freshInstance(16_777_216); + const instance = await freshInstance(shippedFilesOnDisk, 16_777_216); const runtime = sandboxRuntimeOf(instance, settings); const context = runtime.context(); @@ -230,7 +231,7 @@ describe('a context whose instance breaks', () => { }); it('throws what breaks it for another reason, as a defect of the host', async () => { - const instance = await freshInstance(unitMemoryBytes); + const instance = await freshInstance(shippedFilesOnDisk, unitMemoryBytes); const runtime = sandboxRuntimeOf(instance, settings); const context = runtime.context(); context.close(); diff --git a/packages/workflow-host/README.md b/packages/workflow-host/README.md index 49aa9616e..c617ed696 100644 --- a/packages/workflow-host/README.md +++ b/packages/workflow-host/README.md @@ -13,23 +13,23 @@ The workflow engine of [`@on.auto/workflow-engine`](../workflow-engine) on Node: Both fail with the engine's `Conflict` when the run's log kept changing while an input was decided, with `HostElsewhere` while another host holds the database's workflows, and with `HostStopped` once the host is stopping. The caller gives the host: -| Option | What it is | -| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `database` | `{ store: 'sqlite', file }` or `{ store: 'postgresql', pools }`, the database the ledger is kept in, or a `WorkflowStore` opened on it | -| `projector` | optional `ProjectorSettings`: the projector, with [its folders](#the-folders-of-the-projector), that keeps their rows, the views of recall functions among them, while the host holds the database's workflows | -| `machine` | the machine's options: the functions a workflow may call and the runtime its expressions see | -| `perform` | `(call, run) => Effect`: what a call does, a `CallResult` or `{ status: 'waiting', child }` for a call whose run finishes later; it never fails, a function that cannot answer answers `unreachable` | -| `waiting` | what [waiting calls](#waiting-calls) need: `resultOf`, the call's answer for the ending of the run it waits for, `cancel`, which records a cancel on a run, and `cancelDeferred`, which settles a cancelled run of another capability | -| `mostOpenCalls` | how many calls may be open under one run at the top of a tree, `mostOpenCallsOfATree`, 1,000, unless given | -| `settle` | `SettleRun` of `@on.auto/definitions`: how a workflow's outcome is recorded on its run | -| `reports` | where the host tells the operator of a run it could not settle (`unsettled`), of a failure it retries (`trouble`), of a lost connection, and its notes (`note`) | -| `sweepEveryMs` | how often the loop sweeps, 1,000 in the server | -| `mostCallsAtOnce` | how many calls run at once | -| `clock`, `cacheBounds` | the clock, `Date.now` unless given, and the bounds of the engine's cache of loaded runs, `runCacheBounds` unless given | -| `holder` | the id the host claims the database's workflows with, a random UUID unless given | -| `reactions` | what the host needs to react to the records of the playbooks, below: the capability whose definitions are workflows, how a trigger starts a run, how an emitted event is recorded, and the signal of an append | -| `consumers` | more consumers of the records of the playbooks, each naming the event types it wants, after the host's own, none unless given (see [The follower](#the-follower)) | -| `dueWork` | the due rows of projections the loop hands out before its timers, none unless given (see [Due rows](#due-rows)) | +| Option | What it is | +| ---------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `database` | `{ store: 'sqlite', file }` or `{ store: 'postgresql', pools }`, the database the ledger is kept in, or a `WorkflowStore` opened on it | +| `projector` | optional `ProjectorSettings`: the projector, with [its folders](#the-folders-of-the-projector), that keeps their rows, the views of recall functions among them, while the host holds the database's workflows | +| `machine` | the machine's options: the functions a workflow may call and the runtime its expressions see | +| `perform` | `(call, run) => Effect`: what a call does, a `CallResult` or `{ status: 'waiting', child }` for a call whose run finishes later; it never fails, a function that cannot answer answers `unreachable` | +| `waiting` | what [waiting calls](#waiting-calls) need: `resultOf`, the call's answer for the ending of the run it waits for, `cancel`, which records a cancel on a run, and `cancelDeferred`, which settles a cancelled run of another capability | +| `mostOpenCalls` | how many calls may be open under one run at the top of a tree, `mostOpenCallsOfATree`, 1,000, unless given | +| `settle` | `SettleRun` of `@on.auto/definitions`: how a workflow's outcome is recorded on its run | +| `reports` | where the host tells the operator of a run it could not settle (`unsettled`), of a failure it retries (`trouble`), of a lost connection, and its notes (`note`) | +| `sweepEveryMs` | how often the loop sweeps, 1,000 in the server | +| `mostCallsAtOnce` | how many calls run at once | +| `clock`, `cacheBounds` | the clock, `Date.now` unless given, and the bounds of the engine's cache of loaded runs, `runCacheBounds` unless given | +| `holder` | the id the host claims the database's workflows with, a random UUID unless given | +| `reactions` | what the host needs to react to the records of the playbooks, below: the capability whose definitions are workflows, how a trigger starts a run, how an emitted event is recorded, the signal of an append, and the sandbox its filters are matched in, the evaluation worker's of the server's pool | +| `consumers` | more consumers of the records of the playbooks, each naming the event types it wants, after the host's own, none unless given (see [The follower](#the-follower)) | +| `dueWork` | the due rows of projections the loop hands out before its timers, none unless given (see [Due rows](#due-rows)) | ## Where a run is kept @@ -433,7 +433,7 @@ The tests that kill a host and start another on the same database wait for the d The test databases on PostgreSQL commit without waiting for the disk (`synchronous_commit = off`), which loses nothing a test reads. -The suites of `src/views-testing` run on SQLite in `src/projector/views-on-sqlite.test.ts` and on PostgreSQL in `src/projector/views-on-postgresql.test.ts`: the example folded over runs whatever their output, the checkpoint at the end of the history, more than a page, a page ended by its time, a function's own runs, published events matched by data, the stalls, the retries of a deadline, a page whose worker broke keeping what its view folded before the event, even when folding up to that event again ends early on its budget, a slow fold charged to its own view alone and never to the views folding the same event, the versions, a playbook found without a restart, one shared read, rebuilds in their slots, and the conditional writes; PostgreSQL also folds an event committed late, in the order the ledger recorded it. Their pages fold in the modules of `src/views-testing/test-fold-workers.ts`, which serve their jobs on the engine's job loop: one checks a view's schema in the one wording of its issues and sleeps before folding one event, so a slow fold is told from its neighbours, and one breaks on an event, so a lost page is told from a stall. +The suites of `src/views-testing` run on SQLite in `src/projector/views-on-sqlite.test.ts` and on PostgreSQL in `src/projector/views-on-postgresql.test.ts`: the example folded over runs whatever their output, the checkpoint at the end of the history, more than a page, a page ended by its time, a function's own runs, published events matched by data, the stalls, the retries of a deadline, a page whose worker broke keeping what its view folded before the event, even when folding up to that event again ends early on its budget, a slow fold charged to its own view alone and never to the views folding the same event, the versions, a playbook found without a restart, one shared read, rebuilds in their slots, and the conditional writes; PostgreSQL also folds an event committed late, in the order the ledger recorded it. Their pages fold on the pool `testFoldPoolOf` makes with the `measuredPoolSettings` of `@on.auto/definitions/testing`, in the modules of `src/views-testing/test-fold-workers.ts`, which serve their jobs on the engine's job loop: one checks a view's schema in the one wording of its issues and sleeps before folding one event, so a slow fold is told from its neighbours, and one breaks on an event, so a lost page is told from a stall. The suites of `src/folder-testing` run on SQLite in `src/projector/folders-on-sqlite.test.ts` and on PostgreSQL in `src/projector/folders-on-postgresql.test.ts`: a second folder, which counts the events of a type into tables of its own, in the host's own process, keyed by the part of its definition it counts, as the views of recall functions are folded beside it: `folder_tallies`, a row a tally with its count, and `folder_tally_outputs`, the count of each output, which a page adds to with one upsert an output before it moves its row's checkpoint, all in one transaction conditional on the checkpoint it read, as a dataset of prediction functions writes its rows. It shares one read a wake with the views once both are caught up; a row waits for its slot, rebuilds in it, goes live, and stalls at an event it cannot fold, counting no output after it; over a restart its row never leaves `live`, so a new projector folds only what came after its checkpoint, each event once; it rebuilds from the start when the part of a definition it is keyed by changes, while an edit that leaves that part alone and the row another definition still names stay live, never rebuilt; a row is dropped, its outputs first, in one transaction, once no definition names it; it takes no write, neither its outputs nor its row, from a pass that read its row before another wrote it, and keeps nothing of a write whose last statement fails after its outputs were written; and its tables are made on every open of the store, which finds them made and keeps their rows. With the views beside it at another checkpoint, it folds on alone while a views fold that overran rests, the views read alone from their own checkpoint at the next sweep, and the two share one read again once both are caught up. diff --git a/packages/workflow-host/measure/idle-views.ts b/packages/workflow-host/measure/idle-views.ts index 79195d57d..c9a245bd8 100644 --- a/packages/workflow-host/measure/idle-views.ts +++ b/packages/workflow-host/measure/idle-views.ts @@ -1,6 +1,5 @@ import { setTimeout } from 'node:timers/promises'; -import { programPool } from '@on.auto/workflow-engine/dsl'; import { Effect, Function } from 'effect'; import type { DatabaseSettings } from '../src/database/host-databases.ts'; @@ -8,7 +7,7 @@ import { statement } from '../src/database/statement.ts'; import { openWorkflowStore, type WorkflowStore } from '../src/host/workflow-store.ts'; import { systemClock } from '../src/loop/host-clock.ts'; import { startProjector } from '../src/projector/projector.ts'; -import { measuredViewOf, measuredViews, recallContentOf } from './measured-views.ts'; +import { measuredViewOf, measuredViews, measuringPool, recallContentOf } from './measured-views.ts'; export interface IdleViews { readonly views: number; @@ -62,7 +61,7 @@ async function allLive(store: WorkflowStore, views: number): Promise { } export async function idleViewsOn(settings: DatabaseSettings, views: number, seconds: number): Promise { - const pool = programPool({ workers: 4, heapMegabytes: 256 }); + const pool = measuringPool(); const measured = measuredViews(pool); const store = await openWorkflowStore(settings, Function.constVoid, measured.folders); await savedInTurn(store, views); diff --git a/packages/workflow-host/measure/measured-views.ts b/packages/workflow-host/measure/measured-views.ts index f18ba32d5..6555368b8 100644 --- a/packages/workflow-host/measure/measured-views.ts +++ b/packages/workflow-host/measure/measured-views.ts @@ -1,4 +1,11 @@ -import { unitMemoryBytes, workerStackBytes, type ProgramPool } from '@on.auto/workflow-engine/dsl'; +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; +import { + handedWorkersOf, + programPool, + unitMemoryBytes, + workerStackBytes, + type ProgramPool, +} from '@on.auto/workflow-engine/dsl'; import { Effect, type Schema } from 'effect'; import type { HostDatabase } from '../src/database/host-database.ts'; @@ -8,6 +15,12 @@ import type { KeptView } from '../src/views/view-rows.ts'; import { viewsFolder } from '../src/views/views-folder.ts'; import { viewsPortOn } from '../src/views/views-port.ts'; +const handed = await handedWorkersOf(shippedFilesOnDisk); + +export function measuringPool(): ProgramPool { + return programPool({ handed, workers: 4, heapMegabytes: 256 }); +} + export function recallContentOf(name: string, details: Schema.JsonObject): Schema.JsonObject { return { source: name, details }; } diff --git a/packages/workflow-host/measure/rebuild.ts b/packages/workflow-host/measure/rebuild.ts index a5ceb4ce4..1c752a6dc 100644 --- a/packages/workflow-host/measure/rebuild.ts +++ b/packages/workflow-host/measure/rebuild.ts @@ -1,6 +1,5 @@ import { setTimeout } from 'node:timers/promises'; -import { programPool } from '@on.auto/workflow-engine/dsl'; import { Effect, Function } from 'effect'; import type { HostDatabase } from '../src/database/host-database.ts'; @@ -9,7 +8,7 @@ import { openWorkflowStore } from '../src/host/workflow-store.ts'; import { systemClock } from '../src/loop/host-clock.ts'; import { startProjector } from '../src/projector/projector.ts'; import { reviewsFold } from '../src/views-testing/campaign-reviews.ts'; -import { measuredViewOf, measuredViews, recallContentOf } from './measured-views.ts'; +import { measuredViewOf, measuredViews, measuringPool, recallContentOf } from './measured-views.ts'; export interface Rebuild { readonly events: number; @@ -107,7 +106,7 @@ function counted(database: HostDatabase): Counted { } export async function rebuildOn(settings: DatabaseSettings, events: number): Promise { - const pool = programPool({ workers: 4, heapMegabytes: 256 }); + const pool = measuringPool(); const measured = measuredViews(pool); const store = await openWorkflowStore(settings, Function.constVoid, measured.folders); await recorded(store.database, events); diff --git a/packages/workflow-host/src/filtering/filter-matching.ts b/packages/workflow-host/src/filtering/filter-matching.ts index 05db26c48..5d91391ad 100644 --- a/packages/workflow-host/src/filtering/filter-matching.ts +++ b/packages/workflow-host/src/filtering/filter-matching.ts @@ -6,7 +6,6 @@ import { type JsonObject, type MatchedFilter, } from '@on.auto/workflow-engine'; -import { filterSandboxOf } from '@on.auto/workflow-engine/dsl'; import { Array, Effect } from 'effect'; export const stopsInARowBeforeTheVersion = 3; @@ -130,7 +129,7 @@ export function filterStops(): FilterStops { }; } -export function filterMatchingOf(sandbox: FilterSandbox = filterSandboxOf()): MatchFilters { +export function filterMatchingOf(sandbox: FilterSandbox): MatchFilters { return (filters, event, now) => Effect.promise(() => filterVerdictsOf(filters, event, sandbox, now)); } diff --git a/packages/workflow-host/src/reaction-testing/recorded-reactions.ts b/packages/workflow-host/src/reaction-testing/recorded-reactions.ts index f31bd5cb4..484b1188c 100644 --- a/packages/workflow-host/src/reaction-testing/recorded-reactions.ts +++ b/packages/workflow-host/src/reaction-testing/recorded-reactions.ts @@ -1,4 +1,6 @@ import type { Emission } from '@on.auto/definitions'; +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; +import { filterSandboxOf } from '@on.auto/workflow-engine/dsl'; import { Effect } from 'effect'; import { StartRefused, type ReactionOptions, type ReactionStart } from '../reactions/reaction-options.ts'; @@ -32,6 +34,7 @@ export function recordedReactions(behaviour: ReactionBehaviour = {}): RecordedRe return { options: { definitionType: 'workflow', + filters: filterSandboxOf(shippedFilesOnDisk), start: (start) => Effect.suspend(() => { const failure = behaviour.failure?.(start) ?? null; diff --git a/packages/workflow-host/src/reaction-testing/trigger-starting.ts b/packages/workflow-host/src/reaction-testing/trigger-starting.ts index 3f71b4e92..ae6e1790f 100644 --- a/packages/workflow-host/src/reaction-testing/trigger-starting.ts +++ b/packages/workflow-host/src/reaction-testing/trigger-starting.ts @@ -1,4 +1,6 @@ +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import type { FilterVerdict, MatchedFilter } from '@on.auto/workflow-engine'; +import { filterSandboxOf } from '@on.auto/workflow-engine/dsl'; import { Array, Effect, Function } from 'effect'; import { filterMatchingOf, filterStops, type FilterStops, type MatchFilters } from '../filtering/filter-matching.ts'; @@ -63,7 +65,7 @@ interface CountingMatches { export function countingMatches(): CountingMatches { const evaluated: string[] = []; - const matching = filterMatchingOf(); + const matching = filterMatchingOf(filterSandboxOf(shippedFilesOnDisk)); return { match: (filters, event, now) => { evaluated.push(...filters.map(({ reference }) => reference)); diff --git a/packages/workflow-host/src/reactions/reaction-options.ts b/packages/workflow-host/src/reactions/reaction-options.ts index 90441519f..20c91437d 100644 --- a/packages/workflow-host/src/reactions/reaction-options.ts +++ b/packages/workflow-host/src/reactions/reaction-options.ts @@ -29,5 +29,5 @@ export interface ReactionOptions { readonly start: StartReaction; readonly emit: EmitEvent; readonly appended?: StreamSignal; - readonly filters?: FilterSandbox; + readonly filters: FilterSandbox; } diff --git a/packages/workflow-host/src/testing/host-documents.ts b/packages/workflow-host/src/testing/host-documents.ts index 8604e24ec..137a70f1c 100644 --- a/packages/workflow-host/src/testing/host-documents.ts +++ b/packages/workflow-host/src/testing/host-documents.ts @@ -1,3 +1,4 @@ +import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import type { MachineOptions } from '@on.auto/workflow-engine'; import { machineSandboxOf } from '@on.auto/workflow-engine/dsl'; import { defaultLimits, defaultSeed, testMachineOf } from '@on.auto/workflow-engine/testing'; @@ -13,7 +14,7 @@ const decodeObject = Schema.decodeUnknownSync(Schema.JsonObject); const header = { dsl: '1.0.3', namespace: 'acme', name: 'test', version: '1.0.0' }; -export const testMachine: MachineOptions = testMachineOf(machineSandboxOf()); +export const testMachine: MachineOptions = testMachineOf(machineSandboxOf(shippedFilesOnDisk)); export function workflow(source: string): JsonObject { return { document: header, ...decodeObject(parse(source)) }; diff --git a/packages/workflow-host/src/views-testing/test-fold-workers.ts b/packages/workflow-host/src/views-testing/test-fold-workers.ts index a6d8f4aad..10e6fe233 100644 --- a/packages/workflow-host/src/views-testing/test-fold-workers.ts +++ b/packages/workflow-host/src/views-testing/test-fold-workers.ts @@ -1,4 +1,5 @@ -import { mostValueDepth } from '@on.auto/workflow-engine/dsl'; +import { measuredPoolSettings } from '@on.auto/definitions/testing'; +import { mostValueDepth, programPool, type PoolSettings, type ProgramPool } from '@on.auto/workflow-engine/dsl'; export const breaksTheWorker = 'breaks the worker'; @@ -53,3 +54,7 @@ export const breakingFoldWorker = foldingWorker({ whenFolding: `if (events[event]?.data?.output === '${breaksTheWorker}') throw new Error('broken on purpose');`, serving: 'serveJobs({ fold });', }); + +export function testFoldPoolOf(settings: Partial): ProgramPool { + return programPool({ workers: 4, heapMegabytes: 64, ...measuredPoolSettings, ...settings }); +} diff --git a/packages/workflow-host/src/views-testing/view-documents.ts b/packages/workflow-host/src/views-testing/view-documents.ts index ce8a5bfa3..dbe715048 100644 --- a/packages/workflow-host/src/views-testing/view-documents.ts +++ b/packages/workflow-host/src/views-testing/view-documents.ts @@ -37,12 +37,6 @@ export const collecting = detailsOf(foldOf('return [...view, event.data.output]; initial: [], }); -export const measuredEnvironment: Readonly> = Object.fromEntries( - Object.entries(process.env).flatMap(([key, value]: readonly [string, string | undefined]) => - key === 'NODE_V8_COVERAGE' && value !== undefined ? [[key, value]] : [], - ), -); - export function foldingOf(): FoldingSettings { return { budget: 500, diff --git a/packages/workflow-host/src/views-testing/view-harness.ts b/packages/workflow-host/src/views-testing/view-harness.ts index 19bb27a9a..250265bc7 100644 --- a/packages/workflow-host/src/views-testing/view-harness.ts +++ b/packages/workflow-host/src/views-testing/view-harness.ts @@ -1,5 +1,4 @@ import { appendSignal, type AppendSignal } from '@on.auto/ledger'; -import { programPool, type PoolSettings, type ProgramPool } from '@on.auto/workflow-engine/dsl'; import { onTestFinished } from 'vitest'; import type { DatabaseSettings } from '../database/host-databases.ts'; @@ -7,24 +6,25 @@ import { openWorkflowStore, type WorkflowStore } from '../host/workflow-store.ts import { recordingReports, type RecordingReports } from '../testing/recording-reports.ts'; import { harnessProjectors, type HarnessProjectors } from './harness-projectors.ts'; import { playbookAppends, type PlaybookAppends } from './playbook-appends.ts'; -import { measuredEnvironment, settingsOver, type ViewSettingsOf, type ViewsProjecting } from './view-documents.ts'; +import { testFoldPoolOf } from './test-fold-workers.ts'; +import { settingsOver, type ViewSettingsOf, type ViewsProjecting } from './view-documents.ts'; import { viewReadingOf, type ViewReading } from './view-reading.ts'; export interface ViewHarness extends PlaybookAppends, ViewReading, HarnessProjectors { readonly store: WorkflowStore; readonly reports: RecordingReports; - readonly pool: ProgramPool; + readonly pool: ReturnType; readonly appends: AppendSignal; readonly settingsOf: ViewSettingsOf; } export async function viewHarness( settings: DatabaseSettings, - poolSettings: Partial = {}, + poolSettings: Parameters[0] = {}, besides: ViewsProjecting['besides'] = [], ): Promise { const reports = recordingReports(); - const pool = programPool({ workers: 4, heapMegabytes: 64, environment: measuredEnvironment, ...poolSettings }); + const pool = testFoldPoolOf(poolSettings); const appends = appendSignal(); const settingsOf = settingsOver(pool, appends); const store = await openWorkflowStore(settings, reports.reports.lostConnection, settingsOf({ besides }).folders);