From 80d7d5787614bc433182bdfb78e8d44c228b5fa8 Mon Sep 17 00:00:00 2001 From: nvidia Date: Fri, 11 Sep 2026 08:32:12 +0000 Subject: [PATCH 1/9] p2p: a publisher can push its body, so a node nobody can reach can still sell MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every blob transfer here is a pull — the fetcher goes to the holder. That is right for a CONSUMER behind a firewall and wrong for a PUBLISHER behind one: the verifier has to reach in, cannot, and the anchor sits at ANNOUNCED for ever. Nothing errors anywhere. The anchor gossips, the catalogue lists it, and the body is simply unobtainable — which is how ainize.ai came to show a knowledge whose live test cannot run, because the seller node that held it no longer runs. Three parts: POST /p2p/blob/:sha accepts a body on behalf of its author. Off unless `p2p.relayBlobs`, bounded by `p2p.maxRelayBytes`. P2P.offerBlob() pushes to peers, best-effort, never throws. Market.offerBody() called after announce; logs who took it, and warns plainly when nobody did. Accepting is not a matter of trusting the caller. The sha must name an anchor this node already knows from the gossiped ledger, so it is not open storage; the uploader must sign as that anchor's author, so only the publisher can place its own bytes; and importFile rehashes and refuses a mismatch — the signed anchor already fixes the hash, this checks the bytes against it. Discovery and fetch needed no change: holders() already works off the blobs a peer advertises in PeerInfo, so a relay shows up as a holder and fetchBlob finds it unmodified. Needs @ainize/core 0.1.2 for p2p.relayBlobs / p2p.maxRelayBytes. --- src/api.ts | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++- src/market.ts | 23 ++++++++++++++++++++++ src/p2p.ts | 33 ++++++++++++++++++++++++++++++- 3 files changed, 108 insertions(+), 2 deletions(-) diff --git a/src/api.ts b/src/api.ts index c983de3..7528a3d 100644 --- a/src/api.ts +++ b/src/api.ts @@ -5,7 +5,7 @@ * /p2p/* peer protocol (hello, peers, records, blobs) */ import { randomBytes } from 'node:crypto'; -import { createReadStream, existsSync, mkdirSync, readFileSync, rmSync, statSync } from 'node:fs'; +import { createReadStream, existsSync, mkdirSync, readFileSync, rmSync, statSync, unlinkSync } from 'node:fs'; import { join } from 'node:path'; import express, { type Request, type Response, type NextFunction, type Router } from 'express'; import multer from 'multer'; @@ -2289,6 +2289,58 @@ export function buildApi(deps: ApiDeps): Router { createReadStream(blob.path).pipe(res); })); + /** + * A publisher OFFERS its patch body to this node, so a node nobody can reach can still be a seller. + * + * Every other blob transfer here is a pull — the fetcher goes to the holder. That suits a consumer behind a + * firewall and fails a PUBLISHER behind one: the verifier has to reach in, cannot, and the anchor sits at + * ANNOUNCED for ever with no error raised anywhere. This is the one direction that has to be a push. + * + * WHY ACCEPTING IS SAFE, and it is not a matter of trusting the caller: + * + * - the sha in the path must name an anchor this node already knows, from the gossiped ledger. An + * offer for a body nobody has announced is refused, so this is not open storage. + * - the uploader must sign as that anchor's AUTHOR. Only the publisher can place their own bytes. + * - `importFile` rehashes the file and refuses a mismatch, and rejects anything that is not a patch + * (`addrs` missing). So a relay cannot be made to serve content other than what the author published + * — the signed anchor already fixes the hash, and the bytes are checked against it. + * + * Off unless `p2p.relayBlobs`, and bounded by `p2p.maxRelayBytes`: holding bytes for other people is a cost + * and a node should say yes to it deliberately. + */ + router.post('/p2p/blob/:sha', upload.single('blob'), wrap(async (req) => { + const sha = String(req.params.sha ?? '').toLowerCase(); + if (!/^[0-9a-f]{64}$/.test(sha)) throw bad('sha must be a 64-character hex sha256'); + const cfgP2p = market.cfg.p2p ?? {}; + if (!cfgP2p.relayBlobs) throw new HttpError(403, 'relay_disabled: this node does not hold blobs for other nodes (p2p.relayBlobs)'); + + const file = (req as Request & { file?: { path: string; size: number } }).file; + if (!file) throw bad('attach the patch body as multipart field `blob`'); + const cleanup = () => { try { unlinkSync(file.path); } catch { /* already gone */ } }; + + try { + const max = cfgP2p.maxRelayBytes ?? 0; + if (max > 0 && file.size > max) throw new HttpError(413, `blob is ${file.size} bytes; this node relays at most ${max} (p2p.maxRelayBytes)`); + + // The anchor must already be known, and the offer must be signed by ITS author. Both together are what + // make this a relay rather than free storage for anyone who can reach the port. + const entry = (await market.catalogAll()).find((e) => e.anchor.patch_sha256 === sha); + if (!entry) throw notFound(`no anchor known to this node names ${sha.slice(0, 12)} — announce it first, so the offer can be checked against a signed record`); + const offerer = verifyAuthHeader(req.header('x-ainize-auth'), `blob:${sha}`); + if (!offerer || !sameAddr(offerer, entry.anchor.author)) { + throw new HttpError(403, `only the author of ${entry.anchor.id} may place its body here`); + } + + if (market.blobs.get(sha)) return { ok: true, sha256: sha, already_held: true }; + // importFile rehashes and refuses a mismatch — the signed anchor fixes the hash, this checks the bytes. + const { blob } = await market.blobs.importFile(file.path, { copy: true, expectSha: sha }); + market.log('info', 'blob', `relaying ${sha.slice(0, 12)} for ${entry.anchor.id} (${blob.size_bytes} bytes) on behalf of ${entry.anchor.author.slice(0, 10)}`, entry.anchor.id); + return { ok: true, sha256: sha, size_bytes: blob.size_bytes, already_held: false }; + } finally { + cleanup(); + } + })); + // published training sets between nodes (lineage design §6.6): same gate as /p2p/blob, plus the access level const datasetGateP2p = async (req: Request, sha: string) => { if (!market.datasets.has(sha)) throw notFound('dataset not held by this node'); diff --git a/src/market.ts b/src/market.ts index cb1a5e2..675be54 100644 --- a/src/market.ts +++ b/src/market.ts @@ -1570,9 +1570,32 @@ export class Market { : opts.autoSupersede === false ? ' — nothing is retired (--keep-others)' : ''), id, { conflicts, retires }); await this.p2p?.broadcast(rec).catch(() => undefined); + /** + * Offer the BODY to peers, not just the anchor. + * + * Broadcast sends the record. The record names a sha, and every route that moves a sha is a pull — the + * verifier comes to the author. A publisher behind NAT or a firewall passes this line with a perfectly good + * announce and no way for anyone to fetch what it announced: the catalogue lists it, the status never leaves + * ANNOUNCED, and nothing anywhere reports an error. Pushing the body to whoever will hold it is what closes + * that, and it is best-effort — peers that decline are normal, so this cannot fail a publish. + */ + void this.offerBody(anchor.patch_sha256, blob.path, id); return rec; } + /** Push a just-announced body to relaying peers, and say plainly when nobody took it (see `announce`). */ + private async offerBody(sha: string, path: string, id: string): Promise { + if (!this.p2p) return; + try { + const took = await this.p2p.offerBlob(sha, path); + if (took.length) this.log('info', 'publish', `${took.length} peer(s) now hold the body of ${id}: ${took.join(', ')}`, id, { relays: took }); + else this.log('warn', 'publish', `no peer accepted the body of ${id} — verifiers must reach ${this.publicUrl} themselves to fetch it. ` + + `If this node is not reachable from outside, ${id} will stay ANNOUNCED: ask a peer to set \`p2p.relayBlobs true\`.`, id); + } catch (e) { + this.log('warn', 'publish', `could not offer the body of ${id} to peers: ${(e as Error).message}`, id); + } + } + /** What this announce will retire once verifiers pass it — read back by the API and the CLI (item 248). */ pendingSupersedes(id: string): ConflictInfo[] { try { return JSON.parse(this.store.get(`pending_supersede:${id}`) ?? '[]') as ConflictInfo[]; } catch { return []; } diff --git a/src/p2p.ts b/src/p2p.ts index 3ce94ea..79a3940 100644 --- a/src/p2p.ts +++ b/src/p2p.ts @@ -3,7 +3,7 @@ * (local-ledger mode: set reconciliation by `received_at` cursor + push on new record), * blob availability and authenticated blob fetch. */ -import { createWriteStream, mkdirSync, renameSync } from 'node:fs'; +import { createWriteStream, mkdirSync, renameSync, readFileSync } from 'node:fs'; import { dirname } from 'node:path'; import { pipeline } from 'node:stream/promises'; import { Readable } from 'node:stream'; @@ -377,6 +377,37 @@ export class P2P { throw lastErr ?? new Error(`no peer holds the training set ${sha.slice(0, 12)}`); } + /** + * PUSH this node's own blob to peers, so a publisher nobody can reach can still be a seller. + * + * Everything else here is a pull, which is right for a consumer behind a firewall and wrong for a PUBLISHER + * behind one: the verifier has to reach in, cannot, and the anchor sits at ANNOUNCED for ever. Nothing errors — + * the anchor gossips, the catalogue lists it, and the body is simply unobtainable. This is the one direction + * that has to be a push. + * + * Best-effort by design: a peer that refuses (relay off, too large, does not know the anchor yet) is not a + * publish failure, so this NEVER throws. It returns the peers that accepted, and the caller logs the count — + * a publisher who ends up with zero relays should be told, not left to find out at verification time. + */ + async offerBlob(sha: string, path: string, endpoints = this.peers().map((p) => p.endpoint)): Promise { + const accepted: string[] = []; + const body = readFileSync(path); + for (const ep of endpoints) { + if (this.normalize(ep) === this.normalize(this.selfEndpoint)) continue; + try { + const form = new FormData(); + form.append('blob', new Blob([body]), `${sha}.npz`); + const r = await fetch(`${ep}/p2p/blob/${sha}`, { + method: 'POST', body: form, + headers: { 'x-ainize-auth': authHeader(this.deps.identity, `blob:${sha}`) }, + signal: AbortSignal.timeout(10 * 60_000), + }); + if (r.ok) accepted.push(ep); + } catch { /* a peer that will not hold it is not a publish failure */ } + } + return accepted; + } + /** Fetch a blob from a peer with identity auth (verifier/author/purchaser rights are checked by the peer). */ async fetchBlob(sha: string, dest: string, endpoints = this.holders(sha), token?: string): Promise { let lastErr: Error | null = null; From d156d36253e7ec877ff21680ba0f44e31fde3771 Mon Sep 17 00:00:00 2001 From: Haechan Date: Fri, 11 Sep 2026 09:01:39 +0000 Subject: [PATCH 2/9] Harden outbound blob relay and retry existing publications --- README.md | 3 + docs/blob-relay.md | 94 ++++++++++++ scripts/test-blob-relay-docker.sh | 51 +++++++ src/api.ts | 65 ++------ src/blob-relay.ts | 87 +++++++++++ src/p2p.ts | 33 ++-- src/replica-npz.ts | 84 +++++++++++ test/blob-relay.test.ts | 241 ++++++++++++++++++++++++++++++ 8 files changed, 597 insertions(+), 61 deletions(-) create mode 100644 docs/blob-relay.md create mode 100644 scripts/test-blob-relay-docker.sh create mode 100644 src/blob-relay.ts create mode 100644 src/replica-npz.ts create mode 100644 test/blob-relay.test.ts diff --git a/README.md b/README.md index 4c07b59..23c0b84 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,9 @@ node's identity. The explorer UI is a separate build ([ainize-web](https://github.com/ainblockchain/ainize-web)); point `webDist` at it to have this process serve it too. +Publishers behind NAT can offer bodies to opt-in peers using the +[outbound P2P relay](docs/blob-relay.md), including retrying already-announced knowledge. + ## The rules this node will not bend - **Publishing is not selling.** Two independent nodes must load the patch into the real model and diff --git a/docs/blob-relay.md b/docs/blob-relay.md new file mode 100644 index 0000000..286227b --- /dev/null +++ b/docs/blob-relay.md @@ -0,0 +1,94 @@ +# Outbound P2P knowledge relay + +This extends the existing `POST /p2p/blob/:sha` protocol from PR #5. It does not +require a public seller URL, port forwarding, Tailscale Funnel, a new dataset +publication, or retraining. A publisher behind NAT makes an outbound connection +to an operator-configured peer that opts into holding knowledge bodies. + +## Receiver configuration + +The receiver needs both the node implementation and the relay configuration +fields in [ainize-core PR #3](https://github.com/ainblockchain/ainize-core/pull/3). +Do not identify that implementation by a package version alone: on 2026-09-11, +core main and this feature branch both called themselves 0.1.2, but only the +feature branch contained `relayBlobs` and `maxRelayBytes`. + +```bash +ainize config set p2p.relayBlobs true +ainize config set p2p.maxRelayBytes 10737418240 +``` + +Apply the config using the deployment's normal safe restart procedure. Do not +restart a shared trainer or model while an experiment holds its runtime lock. +Forward `POST /p2p/blob/:sha` to this node, in addition to the existing P2P routes. +Neither `/api/me/*` nor operator/teaching APIs need to become public for relaying. + +The size setting is a conservative aggregate storage budget: **all locally held +knowledge blobs**, plus reservations for concurrent incoming bodies, count +toward it. Existing local/imported bodies therefore reduce relay headroom. This +avoids losing accounting across restarts without a new database migration. It +is not a per-file limit and is not a whole-filesystem quota; temporary upload +and copy space require additional disk headroom. Zero or unset disables relay. +This is a single-node-process budget; do not share its data directory between +multiple independently running receivers. + +Additional fixed bounds: 256 MiB encoded, 512 MiB expanded NPZ, eight concurrent +offers, one file per request, and a 60-second upload deadline. Oversized files +remain transferable by the existing authenticated pull path; this new public +ingress deliberately accepts a smaller, bounded subset. Required knowledge +arrays are little-endian int64 addresses and C-order float32 before/after rows. +ZIP64 central directories and archives with more than 64 arrays are refused. + +## Protocol and recovery + +1. Gossip the signed anchor using the existing ledger protocol. +2. POST multipart field `blob` to `/p2p/blob/`, with the existing + `x-ainize-auth: authHeader(authorIdentity, "blob:")` signature. +3. Check the JSON receipt (`ok`, exact `sha256`, `size_bytes`, `already_held`). + An HTML 200 is not a successful relay. Repeated valid offers are idempotent. +4. Check `/p2p/blobs`, the public knowledge detail's `has_body`, and finally run + the real Live test. A successful file transfer does not prove answer quality. + +For knowledge announced before the receiver was deployed, log into the **author +node** as its operator and call `POST /api/patches//relay` (for example, from +that node's browser console with `fetch('/api/patches//relay', {method:'POST'})`). +It retries the existing body without adding another anchor, retraining, or +publishing a dataset. `relayed: false` and an empty `accepted` list are failures +to place the body, not a successful publication. Only explicitly configured +peers receive automatic offers; peer exchange cannot silently add recipients. + +Authentication, known published-public anchor, author, storage budget, and +in-flight checks happen **before** multipart parsing. The receiver validates +exact anchored size, hash and dimensions, bounded ZIP inflation, and existing +destination integrity before registration. It cleans temporary files on +rejection and disconnection. Drafts, test anchors, retired/rejected knowledge, +unrelated authors, and corrupt stored copies are not acknowledged as valid. + +Relaying does not grant a purchase, change verification quorum, mark knowledge +verified, or waive dataset access/PII rules. Paid-body relays must be peers the +publisher trusts to store those bytes; API download gates are not encryption +against the relay operator. The legacy signature purpose is retained for wire +compatibility and is not bound to an HTTP method or recipient; relay deployment +does not resolve that pre-existing protocol limitation. + +## Reproducible validation + +```bash +bash scripts/test-blob-relay-docker.sh /path/to/ainize-core /path/to/new-evidence +``` + +The default prebuilt dependency image is +`ain-cert-ainize-cli:hf-import-20260911-r5`; provide `AINIZE_TEST_IMAGE` for an +equivalent local image with Node 24, Python/NumPy, and core/node dependencies +under `/opt/ainize/ainize-{core,node}`. Source directories and the root filesystem +are read-only. Tests build both exact source trees in an executable tmpfs, with +network isolation, CPU quota 2, CPU set 0–7, RAM/swap ceiling 4 GiB, and no GPU. +The wrapper preserves build/test failures and container state as well as success. +The fixtures are synthetic, not the DART100 performance or public Live evidence. + +The public route probes at 2026-09-11 08:48 and 08:54 UTC returned HTTP 404 +(`Cannot POST /p2p/blob/...`) on both apex and www at the first check and www at +the second. Thus the visible endpoint had not yet demonstrated this receiver, +even though feature code was available. A relay-disabled **403** or signed-offer +**403** proves route matching; a **404** HTML `Cannot POST` does not. Record the +actual deployment commit and repeat the body transfer and Live test separately. diff --git a/scripts/test-blob-relay-docker.sh b/scripts/test-blob-relay-docker.sh new file mode 100644 index 0000000..ba57422 --- /dev/null +++ b/scripts/test-blob-relay-docker.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euo pipefail +root=$(cd "$(dirname "$0")/.." && pwd) +core=${1:?Pass the ainize-core source directory containing the relay config fields} +evidence=${2:?Pass a new evidence directory} +image=${AINIZE_TEST_IMAGE:-ain-cert-ainize-cli:hf-import-20260911-r5} +mkdir -p "$evidence" +evidence=$(realpath "$evidence") +core=$(realpath "$core") +name="ain-cert-blob-relay-tests-$(date -u +%Y%m%dT%H%M%S)" +git -C "$root" rev-parse HEAD > "$evidence/node-base-commit.txt" +git -C "$core" rev-parse HEAD > "$evidence/core-commit.txt" +docker image inspect "$image" --format '{{.Id}}' > "$evidence/image-id.txt" +for package in core node; do + source=$root + if [ "$package" = core ]; then source=$core; fi + target=$evidence/source/$package + mkdir -p "$target" + cp -a "$source/src" "$source/package.json" "$source"/tsconfig*.json "$target/" + for optional in test fixtures trainer; do + if [ -d "$source/$optional" ]; then cp -a "$source/$optional" "$target/"; fi + done +done +(cd "$evidence/source" && find . -type f -print0 | sort -z | xargs -0 sha256sum) > "$evidence/source-sha256.txt" +docker create --name "$name" --network none --cpus 2 --cpuset-cpus 0-7 --memory 4g --memory-swap 4g \ + --read-only --tmpfs /tmp:rw,exec,size=2g --mount "type=bind,src=$evidence/source/node,dst=/source/node,readonly" \ + --mount "type=bind,src=$evidence/source/core,dst=/source/core,readonly" --entrypoint bash "$image" -ceu ' +for package in core node; do + target=/tmp/ainize-$package + mkdir -p "$target" + cp -a /source/$package/src /source/$package/package.json /source/$package/tsconfig*.json "$target/" + cp -a /opt/ainize/ainize-$package/node_modules "$target/" + for optional in test fixtures trainer; do + if [ -d /source/$package/$optional ]; then cp -a /source/$package/$optional "$target/"; fi + done +done +cd /tmp/ainize-core +npm run build +cd /tmp/ainize-node +npm run build +node --test --import tsx test/blob-relay.test.ts +node --test --import tsx test/guard-api.test.ts test/cluster.test.ts +' > "$evidence/container-id.txt" +docker inspect "$name" --format '{{json .HostConfig}}' > "$evidence/host-config.json" +set +e +docker start -a "$name" > "$evidence/tests.log" 2>&1 +result=$? +set -e +docker inspect "$name" --format '{{json .State}}' > "$evidence/container-state.json" +cat "$evidence/tests.log" +exit "$result" diff --git a/src/api.ts b/src/api.ts index 7528a3d..8dbc37d 100644 --- a/src/api.ts +++ b/src/api.ts @@ -5,7 +5,7 @@ * /p2p/* peer protocol (hello, peers, records, blobs) */ import { randomBytes } from 'node:crypto'; -import { createReadStream, existsSync, mkdirSync, readFileSync, rmSync, statSync, unlinkSync } from 'node:fs'; +import { createReadStream, existsSync, mkdirSync, readFileSync, rmSync, statSync } from 'node:fs'; import { join } from 'node:path'; import express, { type Request, type Response, type NextFunction, type Router } from 'express'; import multer from 'multer'; @@ -17,6 +17,7 @@ import { type CatalogEntry, type LedgerRecord, type PatchAnchor, } from '@ainize/core'; import { verifyAuthHeader } from './p2p.js'; +import { blobRelay } from './blob-relay.js'; import { TeachAuth } from './teach-auth.js'; import { challengedMessage, ConflictError, MarketError, MAX_CHAT_PATCHES, NotFoundError, TREE_MAX_DEPTH, type Market, type MarketEntry } from './market.js'; import { publishedRows } from './dataset-blobs.js'; @@ -1163,6 +1164,16 @@ export function buildApi(deps: ApiDeps): Router { const record = await market.announce(id, { replaces: body.replaces, autoSupersede: body.auto_supersede }); return { record, retires: market.pendingSupersedes(id), verifiers: await market.verifierReach(), visibility: record.body.visibility ?? 'public' }; })); + router.post('/api/patches/:id/relay', requireOperator, wrap(async (req) => { + const entry = await market.entry(req.params.id as string); + if (!entry) throw notFound('knowledge not found'); + if (!sameAddr(entry.anchor.author, market.address)) throw new HttpError(403, 'only this knowledge author may retry its offer'); + if (entry.status === 'DRAFT' || entry.status === 'RETIRED' || entry.status === 'REJECTED' || entry.anchor.visibility === 'test') throw new HttpError(409, 'a published public knowledge is required'); + const blob = market.blobs.get(entry.anchor.patch_sha256); + if (!blob) throw new HttpError(409, 'knowledge body is not held by this author node'); + const accepted = await market.p2p.offerBlob(blob.sha256, blob.path); + return { id: entry.anchor.id, sha256: blob.sha256, accepted, relayed: accepted.length > 0 }; + })); /** * The exit (item 148): an author-signed `retire` record takes their own knowledge off sale for good. The anchor * stays on the permanent record, the catalogue drops it, /x402/patch/:id answers 410, and everyone who already @@ -2289,57 +2300,7 @@ export function buildApi(deps: ApiDeps): Router { createReadStream(blob.path).pipe(res); })); - /** - * A publisher OFFERS its patch body to this node, so a node nobody can reach can still be a seller. - * - * Every other blob transfer here is a pull — the fetcher goes to the holder. That suits a consumer behind a - * firewall and fails a PUBLISHER behind one: the verifier has to reach in, cannot, and the anchor sits at - * ANNOUNCED for ever with no error raised anywhere. This is the one direction that has to be a push. - * - * WHY ACCEPTING IS SAFE, and it is not a matter of trusting the caller: - * - * - the sha in the path must name an anchor this node already knows, from the gossiped ledger. An - * offer for a body nobody has announced is refused, so this is not open storage. - * - the uploader must sign as that anchor's AUTHOR. Only the publisher can place their own bytes. - * - `importFile` rehashes the file and refuses a mismatch, and rejects anything that is not a patch - * (`addrs` missing). So a relay cannot be made to serve content other than what the author published - * — the signed anchor already fixes the hash, and the bytes are checked against it. - * - * Off unless `p2p.relayBlobs`, and bounded by `p2p.maxRelayBytes`: holding bytes for other people is a cost - * and a node should say yes to it deliberately. - */ - router.post('/p2p/blob/:sha', upload.single('blob'), wrap(async (req) => { - const sha = String(req.params.sha ?? '').toLowerCase(); - if (!/^[0-9a-f]{64}$/.test(sha)) throw bad('sha must be a 64-character hex sha256'); - const cfgP2p = market.cfg.p2p ?? {}; - if (!cfgP2p.relayBlobs) throw new HttpError(403, 'relay_disabled: this node does not hold blobs for other nodes (p2p.relayBlobs)'); - - const file = (req as Request & { file?: { path: string; size: number } }).file; - if (!file) throw bad('attach the patch body as multipart field `blob`'); - const cleanup = () => { try { unlinkSync(file.path); } catch { /* already gone */ } }; - - try { - const max = cfgP2p.maxRelayBytes ?? 0; - if (max > 0 && file.size > max) throw new HttpError(413, `blob is ${file.size} bytes; this node relays at most ${max} (p2p.maxRelayBytes)`); - - // The anchor must already be known, and the offer must be signed by ITS author. Both together are what - // make this a relay rather than free storage for anyone who can reach the port. - const entry = (await market.catalogAll()).find((e) => e.anchor.patch_sha256 === sha); - if (!entry) throw notFound(`no anchor known to this node names ${sha.slice(0, 12)} — announce it first, so the offer can be checked against a signed record`); - const offerer = verifyAuthHeader(req.header('x-ainize-auth'), `blob:${sha}`); - if (!offerer || !sameAddr(offerer, entry.anchor.author)) { - throw new HttpError(403, `only the author of ${entry.anchor.id} may place its body here`); - } - - if (market.blobs.get(sha)) return { ok: true, sha256: sha, already_held: true }; - // importFile rehashes and refuses a mismatch — the signed anchor fixes the hash, this checks the bytes. - const { blob } = await market.blobs.importFile(file.path, { copy: true, expectSha: sha }); - market.log('info', 'blob', `relaying ${sha.slice(0, 12)} for ${entry.anchor.id} (${blob.size_bytes} bytes) on behalf of ${entry.anchor.author.slice(0, 10)}`, entry.anchor.id); - return { ok: true, sha256: sha, size_bytes: blob.size_bytes, already_held: false }; - } finally { - cleanup(); - } - })); + router.post('/p2p/blob/:sha', blobRelay(market)); // published training sets between nodes (lineage design §6.6): same gate as /p2p/blob, plus the access level const datasetGateP2p = async (req: Request, sha: string) => { diff --git a/src/blob-relay.ts b/src/blob-relay.ts new file mode 100644 index 0000000..4cc5bc1 --- /dev/null +++ b/src/blob-relay.ts @@ -0,0 +1,87 @@ +import { randomUUID } from 'node:crypto'; +import { existsSync, rmSync, statSync } from 'node:fs'; +import { join } from 'node:path'; +import type { Request, Response, RequestHandler } from 'express'; +import multer from 'multer'; +import { sameAddr } from '@ainize/core'; +import { MarketError, type Market } from './market.js'; +import { sha256File } from './blobs.js'; +import { verifyAuthHeader } from './p2p.js'; +import { inspectReplicaNpz } from './replica-npz.js'; + +export const RELAY_FILE_MAX_BYTES = 256 * 1024 ** 2; +export const RELAY_EXPANDED_MAX_BYTES = 512 * 1024 ** 2; + +export function blobRelay(market: Market): RequestHandler { + const reservations = new Map(); + const receive = async (req: Request, res: Response) => { + const sha = String(req.params.sha ?? '').toLowerCase(); + if (!/^[0-9a-f]{64}$/.test(sha)) throw new MarketError(400, 'sha must be a 64-character hex sha256'); + const budget = market.cfg.p2p?.maxRelayBytes ?? 0; + if (!market.cfg.p2p?.relayBlobs || !Number.isSafeInteger(budget) || budget <= 0) { + throw new MarketError(403, 'relay_disabled: enable p2p.relayBlobs and a positive p2p.maxRelayBytes'); + } + const offerer = verifyAuthHeader(req.header('x-ainize-auth'), `blob:${sha}`); + if (!offerer) throw new MarketError(403, 'a signed author offer is required'); + const entries = (await market.catalogAll()).filter(entry => entry.anchor.patch_sha256 === sha && entry.status !== 'DRAFT' && entry.status !== 'RETIRED' && entry.status !== 'REJECTED' && entry.anchor.visibility !== 'test'); + if (!entries.length) throw new MarketError(404, 'no published public anchor names this blob; announce it first'); + const entry = entries.find(candidate => sameAddr(candidate.anchor.author, offerer)); + if (!entry) throw new MarketError(403, 'only the author may offer this blob'); + const expected = entry.anchor.size_bytes; + if (!Number.isSafeInteger(expected) || expected <= 0 || expected > RELAY_FILE_MAX_BYTES) throw new MarketError(413, `anchor size must be 1–${RELAY_FILE_MAX_BYTES} bytes`); + if (reservations.has(sha)) throw new MarketError(409, 'this blob already has an upload in progress'); + if (reservations.size >= 8) throw new MarketError(503, 'relay upload concurrency limit reached; retry later'); + const held = market.blobs.get(sha); + const used = market.blobs.list().reduce((total, blob) => total + statSync(blob.path).size, 0); + const reserved = [...reservations.values()].reduce((total, bytes) => total + bytes, 0); + const needed = held ? 0 : expected; + if (!held && used + reserved + needed > budget) throw new MarketError(413, 'relay storage budget exhausted (held blobs plus in-flight reservations)'); + reservations.set(sha, needed); + let temporary: string | undefined; + try { + if (held) { + if (statSync(held.path).size !== expected || await sha256File(held.path) !== sha) throw new MarketError(409, 'held blob failed integrity check; operator repair required'); + return { ok: true, sha256: sha, size_bytes: expected, already_held: true }; + } + const storage = multer.diskStorage({ + destination: join(market.cfg.dataDir, 'uploads'), + filename: (_request, _file, done) => { + const filename = `relay-${randomUUID()}`; + temporary = join(market.cfg.dataDir, 'uploads', filename); + done(null, filename); + }, + }); + const upload = multer({ storage, limits: { fileSize: expected + 1, files: 1, fields: 0, parts: 2, fieldNameSize: 16 } }).single('blob'); + await new Promise((resolve, reject) => { + const aborted = () => reject(new MarketError(499, 'relay upload aborted')); + if (req.aborted) { aborted(); return; } + req.once('aborted', aborted); + const deadline = setTimeout(() => req.destroy(), 60_000); + req.once('close', () => clearTimeout(deadline)); + upload(req, res, error => { + req.off('aborted', aborted); + clearTimeout(deadline); + if (error) reject(new MarketError(error instanceof multer.MulterError && error.code === 'LIMIT_FILE_SIZE' ? 413 : 400, 'invalid or oversized relay multipart upload')); + else resolve(); + }); + }); + if (!req.file || req.file.size !== expected) throw new MarketError(400, 'attach multipart field blob with exactly the anchored size'); + if (await sha256File(req.file.path) !== sha) throw new MarketError(400, 'blob sha256 does not match the signed anchor'); + try { + const info = inspectReplicaNpz(req.file.path, RELAY_FILE_MAX_BYTES, RELAY_EXPANDED_MAX_BYTES); + if (info.rows !== entry.anchor.rows || (entry.anchor.model.row_dim !== undefined && info.rowDim !== entry.anchor.model.row_dim)) throw new Error('knowledge dimensions disagree with anchor'); + } catch (error) { + throw new MarketError(400, (error as Error).message); + } + const destination = market.blobs.pathFor(sha); + if (existsSync(destination) && await sha256File(destination) !== sha) throw new MarketError(409, 'stored file failed integrity check; operator repair required'); + const { blob } = await market.blobs.importFile(req.file.path, { copy: true, expectSha: sha }); + market.log('info', 'blob', `relaying ${sha.slice(0, 12)} for ${entry.anchor.id} (${blob.size_bytes} bytes)`, entry.anchor.id); + return { ok: true, sha256: sha, size_bytes: blob.size_bytes, already_held: false }; + } finally { + try { if (temporary) rmSync(temporary, { force: true }); } + finally { reservations.delete(sha); } + } + }; + return (req, res, next) => { receive(req, res).then(result => res.json(result)).catch(next); }; +} diff --git a/src/p2p.ts b/src/p2p.ts index 79a3940..68e1aba 100644 --- a/src/p2p.ts +++ b/src/p2p.ts @@ -3,12 +3,13 @@ * (local-ledger mode: set reconciliation by `received_at` cursor + push on new record), * blob availability and authenticated blob fetch. */ -import { createWriteStream, mkdirSync, renameSync, readFileSync } from 'node:fs'; +import { createWriteStream, mkdirSync, renameSync, openAsBlob } from 'node:fs'; import { dirname } from 'node:path'; import { pipeline } from 'node:stream/promises'; import { Readable } from 'node:stream'; import { signMessage, verifyMessage, type LedgerRecord, type PeerInfo, type Identity, type Ledger, isRecordRefusal } from '@ainize/core'; import type { Store } from './store.js'; +import { sha256File } from './blobs.js'; export interface P2PDeps { identity: Identity; @@ -389,20 +390,34 @@ export class P2P { * publish failure, so this NEVER throws. It returns the peers that accepted, and the caller logs the count — * a publisher who ends up with zero relays should be told, not left to find out at verification time. */ - async offerBlob(sha: string, path: string, endpoints = this.peers().map((p) => p.endpoint)): Promise { + async offerBlob(sha: string, path: string, endpoints = this.peers().filter(peer => peer.source === 'configured').map(peer => peer.endpoint)): Promise { const accepted: string[] = []; - const body = readFileSync(path); - for (const ep of endpoints) { - if (this.normalize(ep) === this.normalize(this.selfEndpoint)) continue; + let body: Blob; + try { + if (!/^[0-9a-f]{64}$/.test(sha) || await sha256File(path) !== sha) return accepted; + body = await openAsBlob(path); + } catch { return accepted; } + for (const endpoint of new Set(endpoints.map(value => this.normalize(value)))) { + if (endpoint === this.normalize(this.selfEndpoint)) continue; try { const form = new FormData(); - form.append('blob', new Blob([body]), `${sha}.npz`); - const r = await fetch(`${ep}/p2p/blob/${sha}`, { + form.append('blob', body, `${sha}.npz`); + const response = await fetch(`${endpoint}/p2p/blob/${sha}`, { method: 'POST', body: form, headers: { 'x-ainize-auth': authHeader(this.deps.identity, `blob:${sha}`) }, - signal: AbortSignal.timeout(10 * 60_000), + signal: AbortSignal.timeout(60_000), redirect: 'error', }); - if (r.ok) accepted.push(ep); + if (!response.ok || !response.headers.get('content-type')?.includes('application/json')) { await response.body?.cancel(); continue; } + const chunks: Uint8Array[] = []; + let bytes = 0; + if (!response.body) continue; + for await (const chunk of response.body) { + bytes += chunk.length; + if (bytes > 4096) throw new Error('relay acknowledgment exceeds the limit'); + chunks.push(chunk); + } + const receipt = JSON.parse(Buffer.concat(chunks).toString('utf8')) as Record; + if (receipt.ok === true && receipt.sha256 === sha && (receipt.size_bytes === body.size || (receipt.already_held === true && receipt.size_bytes === undefined))) accepted.push(endpoint); } catch { /* a peer that will not hold it is not a publish failure */ } } return accepted; diff --git a/src/replica-npz.ts b/src/replica-npz.ts new file mode 100644 index 0000000..b4a5be8 --- /dev/null +++ b/src/replica-npz.ts @@ -0,0 +1,84 @@ +import { readFileSync, statSync } from 'node:fs'; +import { inflateRawSync } from 'node:zlib'; +import { inspectNpz, parseNpyHeader, type NpzInfo } from '@ainize/core'; + +export function inspectReplicaNpz(path: string, maxBytes: number, maxExpandedBytes: number): NpzInfo { + const requireValid = (condition: boolean, message: string) => { if (!condition) throw new Error(`replica_npz: ${message}`); }; + requireValid(statSync(path).size <= maxBytes, 'archive exceeds the limit'); + const bytes = readFileSync(path); + requireValid(bytes.length >= 22 && bytes.length <= maxBytes, 'invalid archive size'); + const range = (offset: number, length: number, ceiling = bytes.length) => { + requireValid(Number.isSafeInteger(offset) && Number.isSafeInteger(length) && offset >= 0 && length >= 0 && offset + length <= ceiling, 'archive range exceeds its bounds'); + }; + let end = -1; + for (let offset = bytes.length - 22; offset >= Math.max(0, bytes.length - 65557); offset--) { + if (bytes.readUInt32LE(offset) === 0x06054b50) { end = offset; break; } + } + requireValid(end >= 0, 'missing central directory'); + requireValid(end + 22 + bytes.readUInt16LE(end + 20) === bytes.length, 'invalid archive ending'); + const count = bytes.readUInt16LE(end + 10); + requireValid(bytes.readUInt16LE(end + 4) === 0 && bytes.readUInt16LE(end + 6) === 0 && bytes.readUInt16LE(end + 8) === count, 'split archives are not supported'); + requireValid(count > 0 && count <= 64, 'archive must contain 1–64 arrays'); + const directorySize = bytes.readUInt32LE(end + 12); + const directoryStart = bytes.readUInt32LE(end + 16); + range(directoryStart, directorySize, end); + requireValid(directoryStart + directorySize === end, 'ZIP64 directories or trailing directory records are not supported'); + const names = new Set(); + const occupied: { start: number; end: number }[] = []; + let cursor = directoryStart; + let expanded = 0; + for (let index = 0; index < count; index++) { + range(cursor, 46, end); + requireValid(bytes.readUInt32LE(cursor) === 0x02014b50, 'invalid directory entry'); + const flags = bytes.readUInt16LE(cursor + 8); + const method = bytes.readUInt16LE(cursor + 10); + const compressedSize = bytes.readUInt32LE(cursor + 20); + const uncompressedSize = bytes.readUInt32LE(cursor + 24); + const nameLength = bytes.readUInt16LE(cursor + 28); + const extraLength = bytes.readUInt16LE(cursor + 30); + const commentLength = bytes.readUInt16LE(cursor + 32); + const local = bytes.readUInt32LE(cursor + 42); + range(cursor + 46, nameLength + extraLength + commentLength, end); + const nameBytes = bytes.subarray(cursor + 46, cursor + 46 + nameLength); + const name = nameBytes.toString('utf8'); + requireValid(/^[A-Za-z0-9_.-]+\.npy$/.test(name) && !names.has(name), 'invalid or duplicate array name'); + names.add(name); + requireValid((flags & ~0x0808) === 0 && (method === 0 || method === 8), 'unsupported encryption or compression'); + requireValid(bytes.readUInt16LE(cursor + 34) === 0, 'array is on another disk'); + expanded += uncompressedSize; + requireValid(uncompressedSize >= 10 && expanded <= maxExpandedBytes, 'expanded arrays exceed the limit'); + range(local, 30, directoryStart); + requireValid(bytes.readUInt32LE(local) === 0x04034b50 && bytes.readUInt16LE(local + 6) === flags && bytes.readUInt16LE(local + 8) === method, 'local header disagrees with directory'); + const localNameLength = bytes.readUInt16LE(local + 26); + const localExtraLength = bytes.readUInt16LE(local + 28); + range(local + 30, localNameLength + localExtraLength, directoryStart); + requireValid(bytes.subarray(local + 30, local + 30 + localNameLength).equals(nameBytes), 'local array name disagrees with directory'); + const start = local + 30 + localNameLength + localExtraLength; + range(start, compressedSize, directoryStart); + requireValid(!occupied.some(other => local < other.end && start + compressedSize > other.start), 'overlapping arrays'); + occupied.push({ start: local, end: start + compressedSize }); + const compressed = bytes.subarray(start, start + compressedSize); + const data = method === 0 ? compressed : inflateRawSync(compressed, { maxOutputLength: uncompressedSize }); + requireValid(data.length === uncompressedSize, 'expanded size disagrees with directory'); + requireValid(data.subarray(0, 6).toString('latin1') === '\x93NUMPY' && [1, 2, 3].includes(data[6]), 'invalid array header'); + const headerOffset = data[6] === 1 ? 10 : 12; + range(0, headerOffset, data.length); + const headerSize = data[6] === 1 ? data.readUInt16LE(8) : data.readUInt32LE(8); + requireValid(headerOffset + headerSize <= Math.min(4096, data.length), 'array header exceeds inspection bounds'); + if (['addrs.npy', 'before.npy', 'after.npy'].includes(name)) { + const header = parseNpyHeader(data); + const width = name === 'addrs.npy' ? 8 : 4; + requireValid(!header.fortranOrder && header.shape.every(dimension => Number.isSafeInteger(dimension) && dimension > 0), 'invalid knowledge layout'); + requireValid(header.shape.reduce((total, dimension) => total * dimension, width) === data.length - header.dataOffset, 'array payload size disagrees with its shape'); + } + cursor += 46 + nameLength + extraLength + commentLength; + } + requireValid(cursor === end && ['addrs.npy', 'before.npy', 'after.npy'].every(name => names.has(name)), 'missing knowledge arrays or incomplete directory'); + const info = inspectNpz(path); + const addrs = info.members.find(member => member.name === 'addrs')!; + const before = info.members.find(member => member.name === 'before')!; + const after = info.members.find(member => member.name === 'after')!; + requireValid(addrs.descr === ' 0, 'invalid address array'); + requireValid(Number.isSafeInteger(info.rowDim) && info.rowDim > 0 && [before, after].every(member => member.descr === '(resolve => node.server.listen(0, '127.0.0.1', resolve)); + const url = `http://127.0.0.1:${(node.server.address() as { port: number }).port}`; + const author = createIdentity(); + const publisher = new LocalLedger(join(home, 'publisher.sqlite'), author); + await publisher.init(); + context.after(async () => { await node.stop(); await publisher.close(); rmSync(home, { recursive: true, force: true }); }); + const file = join(home, 'lesson.npz'); + const addresses = Buffer.alloc(8); addresses.writeBigInt64LE(123n); + writeNpz(file, [ + { name: 'addrs', descr: ' = {}) => { + const record = await publisher.append('anchor', { ...anchor, ...overrides }); + assert.ok(LocalLedger.validate(record)); + await node.ledger.ingest(record); node.market.invalidate(); + }; + const offer = async (body = bytes, signature = authHeader(author, `blob:${sha}`), hash = sha) => { + const form = new FormData(); form.append('blob', new Blob([body]), 'lesson.npz'); + return fetch(`${url}/p2p/blob/${hash}`, { method: 'POST', headers: { 'x-ainize-auth': signature }, body: form }); + }; + const clean = () => assert.deepEqual(readdirSync(join(cfg.dataDir, 'uploads')), []); + const sender = new P2P({ identity: author, ledger: publisher, store: node.store, selfInfo: () => node.market.selfInfo(), log: () => {} }, [], 60_000, 'http://127.0.0.1:1'); + return { node, cfg, url, author, publisher, file, bytes, sha, anchor, publish, offer, clean, sender }; +} + +test('signed outbound push imports the exact body, is idempotent, and advertises its hash', async context => { + const setup = await fixture(context); await setup.publish(); + const first = await setup.offer(); + assert.equal(first.status, 200, await first.text()); + assert.deepEqual(await setup.sender.offerBlob(setup.sha, setup.file, [setup.url]), [setup.url]); + const held = setup.node.market.blobs.get(setup.sha)!; + assert.deepEqual(readFileSync(held.path), setup.bytes); + assert.ok((await setup.node.market.selfInfo()).blobs.includes(setup.sha)); + const duplicate = await setup.offer(); + assert.equal(duplicate.status, 200); + assert.deepEqual(await duplicate.json(), { ok: true, sha256: setup.sha, size_bytes: setup.bytes.length, already_held: true }); + assert.equal((await setup.node.market.entry(setup.anchor.id))?.status, 'ANNOUNCED'); + setup.clean(); +}); + +test('disabled, zero/unset budget, malformed SHA and unauthenticated offers never leave uploads', async context => { + const setup = await fixture(context); await setup.publish(); + setup.cfg.p2p!.relayBlobs = false; + assert.equal((await setup.offer()).status, 403); setup.clean(); + setup.cfg.p2p!.relayBlobs = true; + for (const budget of [0, undefined]) { + setup.cfg.p2p!.maxRelayBytes = budget; + assert.equal((await setup.offer()).status, 403); setup.clean(); + } + setup.cfg.p2p!.maxRelayBytes = 1024 ** 2; + assert.equal((await setup.offer(setup.bytes, '', 'invalid')).status, 400); setup.clean(); + assert.equal((await setup.offer(setup.bytes, '')).status, 403); setup.clean(); +}); + +test('unknown, draft, test and foreign-author bodies are refused before multipart parsing', async context => { + const setup = await fixture(context); + assert.equal((await setup.offer()).status, 404); + setup.node.store.putDraft(setup.anchor, setup.file); + setup.node.market.invalidate(); + assert.equal((await setup.offer()).status, 404); + await setup.publish({ visibility: 'test' }); + assert.equal((await setup.offer()).status, 404); + await setup.publish({ id: 'public-lesson' }); + assert.equal((await setup.offer(setup.bytes, authHeader(createIdentity(), `blob:${setup.sha}`))).status, 403); + setup.clean(); +}); + +test('shared content chooses the matching signed author, not the first anchor', async context => { + const setup = await fixture(context); + const stranger = createIdentity(); + const foreign = new LocalLedger(':memory:', stranger); + await foreign.init(); context.after(() => foreign.close()); + const record = await foreign.append('anchor', { ...setup.anchor, id: 'foreign-first', author: stranger.address }); + await setup.node.ledger.ingest(record); + await setup.publish(); + assert.equal((await setup.offer()).status, 200); +}); + +test('wrong hash, wrong size and oversized multipart are rejected and cleaned up', async context => { + const setup = await fixture(context); await setup.publish(); + assert.equal((await setup.offer(Buffer.alloc(setup.bytes.length))).status, 400); + assert.equal((await setup.offer(setup.bytes.subarray(1))).status, 400); + assert.equal((await setup.offer(Buffer.alloc(setup.bytes.length + 2))).status, 413); + assert.equal(setup.node.market.blobs.has(setup.sha), false); + setup.clean(); + assert.equal((await setup.offer()).status, 200); +}); + +test('storage cap is cumulative, and a held body is still idempotent at the cap', async context => { + const setup = await fixture(context); await setup.publish(); + setup.cfg.p2p!.maxRelayBytes = setup.bytes.length; + assert.equal((await setup.offer()).status, 200); + assert.equal((await setup.offer()).status, 200); + const other = Buffer.from(setup.bytes); other[50] ^= 1; + const otherSha = sha256Hex(other); + await setup.publish({ id: 'second-lesson', patch_sha256: otherSha }); + assert.equal((await setup.offer(other, authHeader(setup.author, `blob:${otherSha}`), otherSha)).status, 413); + setup.clean(); +}); + +test('in-flight reservations prevent concurrent budget oversubscription and release on abort', async context => { + const setup = await fixture(context); await setup.publish(); + setup.cfg.p2p!.maxRelayBytes = setup.bytes.length; + const first = request(`${setup.url}/p2p/blob/${setup.sha}`, { method: 'POST', headers: { + 'content-type': 'multipart/form-data; boundary=relay-test', 'x-ainize-auth': authHeader(setup.author, `blob:${setup.sha}`), + } }); + first.on('error', () => {}); + first.write('--relay-test\r\nContent-Disposition: form-data; name="blob"; filename="lesson.npz"\r\nContent-Type: application/octet-stream\r\n\r\n'); + first.write(setup.bytes.subarray(0, 100)); + for (let attempt = 0; attempt < 100 && readdirSync(join(setup.cfg.dataDir, 'uploads')).length === 0; attempt++) await new Promise(resolve => setTimeout(resolve, 10)); + assert.equal((await setup.offer()).status, 409); + const otherSha = 'b'.repeat(64); + await setup.publish({ id: 'concurrent-lesson', patch_sha256: otherSha }); + assert.equal((await setup.offer(setup.bytes, authHeader(setup.author, `blob:${otherSha}`), otherSha)).status, 413); + first.destroy(); + for (let attempt = 0; attempt < 100 && readdirSync(join(setup.cfg.dataDir, 'uploads')).length > 0; attempt++) await new Promise(resolve => setTimeout(resolve, 10)); + setup.clean(); + assert.equal((await setup.offer()).status, 200); +}); + +test('corrupt held or orphan destination files are never acknowledged as valid', async context => { + const setup = await fixture(context); await setup.publish(); + const destination = setup.node.market.blobs.pathFor(setup.sha); + writeFileSync(destination, Buffer.alloc(setup.bytes.length)); + assert.equal((await setup.offer()).status, 409); setup.clean(); + rmSync(destination); + assert.equal((await setup.offer()).status, 200); + writeFileSync(destination, Buffer.alloc(setup.bytes.length)); + assert.equal((await setup.offer()).status, 409); setup.clean(); +}); + +test('malformed signed NPZ and mismatched anchored dimensions never enter storage', async context => { + const setup = await fixture(context); + const invalid = Buffer.alloc(setup.bytes.length); + const invalidSha = sha256Hex(invalid); + await setup.publish({ patch_sha256: invalidSha }); + assert.equal((await setup.offer(invalid, authHeader(setup.author, `blob:${invalidSha}`), invalidSha)).status, 400); + await setup.publish({ id: 'wrong-dimensions', rows: 999 }); + assert.equal((await setup.offer()).status, 400); + setup.clean(); +}); + +test('NPZ preflight bounds encoded and expanded bytes and rejects truncated arrays', async context => { + const setup = await fixture(context); + assert.equal(inspectReplicaNpz(setup.file, setup.bytes.length, 1024 ** 2).rows, 1); + assert.throws(() => inspectReplicaNpz(setup.file, 1, 1024 ** 2), /archive exceeds/); + assert.throws(() => inspectReplicaNpz(setup.file, setup.bytes.length, 1), /expanded arrays/); + writeFileSync(setup.file, setup.bytes.subarray(0, -1)); + assert.throws(() => inspectReplicaNpz(setup.file, setup.bytes.length, 1024 ** 2)); +}); + +test('compressed NumPy archives work and a forged expansion length is bounded', async context => { + const setup = await fixture(context); + execFileSync('python3', ['-c', 'import numpy as np, sys; archive = np.load(sys.argv[1]); arrays = {name: archive[name] for name in archive.files}; archive.close(); np.savez_compressed(sys.argv[1], **arrays)', setup.file]); + const compressed = readFileSync(setup.file); + assert.equal(inspectReplicaNpz(setup.file, 1024 ** 2, 1024 ** 2).rowDim, 2); + const directory = compressed.indexOf(Buffer.from([0x50, 0x4b, 0x01, 0x02])); + assert.ok(directory > 0); + compressed.writeUInt32LE(10, directory + 24); + writeFileSync(setup.file, compressed); + assert.throws(() => inspectReplicaNpz(setup.file, 1024 ** 2, 1024 ** 2)); +}); + +test('sender contains file errors and rejects HTML, wrong hash, oversized acknowledgments and redirects', async context => { + const setup = await fixture(context); + assert.deepEqual(await setup.sender.offerBlob(setup.sha, `${setup.file}.missing`, [setup.url]), []); + assert.deepEqual(await setup.sender.offerBlob('a'.repeat(64), setup.file, [setup.url]), []); + let mode = 'html'; + let redirected = 0; + const peer = createServer((req, res) => { + req.resume(); + if (req.url === '/unexpected') { redirected++; res.end('{}'); return; } + if (mode === 'redirect') { res.writeHead(307, { location: '/unexpected' }); res.end(); return; } + if (mode === 'html') { res.setHeader('content-type', 'text/html'); res.end('ok'); return; } + res.setHeader('content-type', 'application/json'); + res.end(mode === 'large' ? ' '.repeat(5000) : JSON.stringify({ ok: true, sha256: 'a'.repeat(64), size_bytes: setup.bytes.length })); + }); + await new Promise(resolve => peer.listen(0, '127.0.0.1', resolve)); + context.after(() => new Promise(resolve => peer.close(() => resolve()))); + const endpoint = `http://127.0.0.1:${(peer.address() as { port: number }).port}`; + for (mode of ['html', 'wrong-hash', 'large', 'redirect']) assert.deepEqual(await setup.sender.offerBlob(setup.sha, setup.file, [endpoint]), []); + assert.equal(redirected, 0); +}); + +test('relay retry requires operator login and refuses another author', async context => { + const setup = await fixture(context); await setup.publish(); + const route = `${setup.url}/api/patches/${setup.anchor.id}/relay`; + assert.equal((await fetch(route, { method: 'POST' })).status, 401); + setup.node.store.putSession('relay-test-session', 60_000); + assert.equal((await fetch(route, { method: 'POST', headers: { authorization: 'Bearer relay-test-session' } })).status, 403); +}); + +test('operator retries an existing signed knowledge without another publish record or training', async context => { + const author = await fixture(context); + const receiver = await fixture(context); + const record = await author.node.ledger.append('anchor', { ...author.anchor, author: author.cfg.identity.address }); + await receiver.node.ledger.ingest(record); + author.node.market.invalidate(); receiver.node.market.invalidate(); + await author.node.market.blobs.importFile(author.file, { copy: true }); + author.node.store.upsertPeer(receiver.url, { source: 'configured' }); + author.node.store.putSession('retry-session', 60_000); + const before = (await author.node.ledger.anchors()).length; + const response = await fetch(`${author.url}/api/patches/${author.anchor.id}/relay`, { + method: 'POST', headers: { authorization: 'Bearer retry-session' }, + }); + assert.equal(response.status, 200); + const receipt = await response.json() as { relayed: boolean; accepted: string[] }; + assert.equal(receipt.relayed, true); + assert.deepEqual(receipt.accepted, [receiver.url]); + assert.deepEqual(readFileSync(receiver.node.market.blobs.get(author.sha)!.path), author.bytes); + assert.equal((await author.node.ledger.anchors()).length, before); +}); From a4753e0142f27d5c2ff0412d6a3d81ea3ef3dccd Mon Sep 17 00:00:00 2001 From: Haechan Date: Fri, 11 Sep 2026 09:20:57 +0000 Subject: [PATCH 3/9] Retain relayed bodies and recover published knowledge without restarting trainers --- docs/blob-relay-evidence-ko.md | 47 +++++++++++++++++++++ docs/blob-relay.md | 25 +++++++++++ scripts/retry-public-blob.mjs | 70 +++++++++++++++++++++++++++++++ scripts/test-blob-relay-docker.sh | 6 +-- src/blob-relay.ts | 2 + src/blobs.ts | 2 + src/gc.ts | 6 ++- src/store.ts | 4 +- src/verifier.ts | 3 +- test/blob-relay.test.ts | 58 +++++++++++++++++++++++++ test/retry-public-blob.test.ts | 62 +++++++++++++++++++++++++++ 11 files changed, 278 insertions(+), 7 deletions(-) create mode 100644 docs/blob-relay-evidence-ko.md create mode 100644 scripts/retry-public-blob.mjs create mode 100644 test/retry-public-blob.test.ts diff --git a/docs/blob-relay-evidence-ko.md b/docs/blob-relay-evidence-ko.md new file mode 100644 index 0000000..bacc6a3 --- /dev/null +++ b/docs/blob-relay-evidence-ko.md @@ -0,0 +1,47 @@ +# Ainize P2P 본문 전송 — 발행 노드와 공개 수신 노드 구분 + +## 확인한 사실 (2026-09-11 09:10–09:13 UTC) + +- 관리자 측 네 노드의 blob 0건 보고는 관리자 측 저장소 관측이다. 발행 원본까지 소실되었다는 증거는 아니다. +- 이 실험 머신의 `ain-cert-ainize-node-1`은 07:39:04 UTC부터 실행 중이며 healthy, PID 3616107이었다. 로컬 `http://127.0.0.1:3410/p2p/blobs`는 6건이며 아래 공개 본문 2건을 포함한다. +- 발행자 주소는 `0x20A4e266da261F187613efcb90b1eB131BC381a1`이다. 공개 원장의 두 anchor author와 일치하고, 실제 core의 LocalLedger.validate로 서명을 검증했다. +- 원본 파일은 `kpi/ainize/home-docker/data/drive/patches//.npz`에 있다. 파일 전체 SHA256·바이트 수·NPZ 구조를 확인했다. 재학습이나 새 anchor 발행은 필요하지 않다. +- 원격 노드에서 `localhost:3410`을 접속하면 원격 머신 자체를 가리킨다. 다른 머신의 발행 노드를 이 주소로 접속하지 못한 것을 원본 프로세스 종료·디스크 소실로 판정하면 안 된다. + +| 지식 ID | SHA256 | 실제 바이트 | 메모리 행/차원 | +|---|---|---:|---| +| taught-ainize-teach-first-20260-855df1 | f9f665f6fa1a6b37963a4845107c0c0a5d3b970bcd2af6e8f40938a0fbdf7acc | 3,679,278 | 2,856 × 160 | +| taught-ainize-lifecycle100-2026-cf9a6f | fb1cd41e2f6a26f785d72460a2eac4a62688ee4c70e5bee43187d734eeca2e64 | 3,719,206 | 2,887 × 160 | + +## 실제 전송 결과 + +본문 없는 route probe에 그치지 않고, 위 anchor author의 유효한 서명과 multipart `blob` 본문을 사용했다. Docker CPU 1개·RAM/전체 memory+swap 상한 1GiB·CPU set 0–7·read-only·GPU 없음이며 config와 해당 파일만 읽기 전용으로 마운트했다. 비밀키·인증 헤더 값은 출력하지 않았다. + +- 대표자명: 09:11:52 UTC, `POST https://www.ainize.ai/p2p/blob/f9f665...` → **HTML 404 Cannot POST**, accepted=false. +- 소재지: 09:12:53 UTC, `POST https://ainize.ai/p2p/blob/fb1cd4...` → **HTML 404 Cannot POST**, accepted=false. +- 09:05:51 UTC 실제 공개 Live test는 **409 body not held**, 공개 `/p2p/blobs`는 빈 배열이었다. +- `GET /p2p/blob/:sha`에서 없는 본문을 요청해 받는 404와 위 **수신용 POST의 Cannot POST** 응답은 구분한다. 이것만으로 프록시와 백엔드 중 어느 배포 단계가 원인인지 확정하지는 않는다. 다만 공개 주소가 제시된 수신 프로토콜을 처리하지 못했다는 직접 증거다. + +관리자 머신에서 **본문·인증 없이** 아래 요청으로 경계를 확인할 수 있다. 비밀키는 필요 없다. + +```bash +curl -i -X POST http://127.0.0.1:3400/p2p/blob/f9f665f6fa1a6b37963a4845107c0c0a5d3b970bcd2af6e8f40938a0fbdf7acc +``` + +수신 코드가 있으면 설정/인증/본문 검사에서 JSON 403 또는 400이 나온다. HTML Cannot POST 404면 해당 백엔드에 route가 매칭되지 않은 것이다. 내부에서는 JSON이고 공개에서는 HTML 404이면 전달 계층을 확인한다. 실행 커밋도 함께 확인한다. 공개 node PR #5와 core PR #3는 별도이며, npm core 0.1.2라는 버전 문자열만으로 relay 설정 필드 포함 여부를 판단하지 않는다. + +## 보완 내용과 완료 경계 + +- 기존 P2P 프로토콜 그대로 인증 전 업로드 차단, 총용량·동시 업로드 예약, NPZ 압축 해제 제한, hash/size/dimension 검사, 임시 파일 정리를 구현했다. +- 운영자 retry API와 `scripts/retry-public-blob.mjs`를 제공한다. 기존 노드의 학습을 중단하지 않고 이미 공개된 무료 본문만 전송하며, 성공 응답 뒤 실제 GET 재다운로드의 해시까지 검사한다. 새 dataset/anchor를 만들지 않는다. +- 자동 verifier 정리와 gc가 수신한 relay 본문을 바로 삭제할 수 있는 경로도 확인했다. 수신된 본문에 영속 retention 표시를 적용하고 이 두 정리 경로에서 보존한다. 재시작 후에도 유지하며 운영자의 명시적 forget은 가능하다. 이것은 저장 의무이지 유료 지식 사용권이나 검증 통과가 아니다. +- source commit/push/GitHub prerelease와 **실제 공개 서버 배포·본문 복제·Live 성공**은 각각 별도 확인한다. 현재 전송 실패를 성공으로 보고하지 않는다. Funnel이나 대체 HTTPS 파일 서버는 사용하지 않는다. + +## 증빙 + +- `kpi/evidence/blob_relay_public_probe_20260911/signed-offer-status.json`: 로컬 실행 상태, 두 원본 SHA/크기·anchor 서명 확인, 두 signed POST 원문 응답. +- `kpi/evidence/signed_p2p_offer_first_r2_20260911/`, `signed_p2p_offer_second_20260911/`: 실제 Docker 전송 실패와 자원·exit 상태. 첫 시도의 구 core export 불일치는 별도 원문으로 보존했다. +- `kpi/evidence/blob_relay_dart_npz_check_r2_20260911/`: 두 실제 NPZ의 bounded parser 통과. 본문 구조 확인이지 공개 추론 성공은 아니다. +- 기존 보안 보완: [d156d36 소스·39개 시험 릴리스](https://github.com/ainblockchain/ainize-node/releases/tag/p2p-blob-relay-hardening-20260911). +- 후속 재전송·본문 보존 코드는 같은 [node PR #5](https://github.com/ainblockchain/ainize-node/pull/5)에 반영한다. 원문 계획의 100개 학습/평가·70개 실제 샤드 파이프라인·나머지 성능 목표는 이 진단으로 완료되지 않는다. + diff --git a/docs/blob-relay.md b/docs/blob-relay.md index 286227b..0e43efc 100644 --- a/docs/blob-relay.md +++ b/docs/blob-relay.md @@ -32,6 +32,14 @@ and copy space require additional disk headroom. Zero or unset disables relay. This is a single-node-process budget; do not share its data directory between multiple independently running receivers. +Accepted relay bodies have a persistent retention flag in the blob database. +Verification cleanup and `gc` do not discard them: an offered copy must not +disappear immediately after attestation. Ordinary verification downloads retain +their existing cleanup policy. The operator can explicitly release a relay copy +with `ainize patch forget `. This flag is a storage obligation, not a purchase +or permission to apply paid knowledge. Older databases acquire the new column +with a zero default; re-offering an already-held body establishes retention. + Additional fixed bounds: 256 MiB encoded, 512 MiB expanded NPZ, eight concurrent offers, one file per request, and a 60-second upload deadline. Oversized files remain transferable by the existing authenticated pull path; this new public @@ -57,6 +65,23 @@ publishing a dataset. `relayed: false` and an empty `accepted` list are failures to place the body, not a successful publication. Only explicitly configured peers receive automatic offers; peer exchange cannot silently add recipients. +If the author node cannot safely restart because a trainer is active, the +standalone recovery client can offer an already-published **free public** body +using the same signed protocol and the existing identity file: + +```bash +node scripts/retry-public-blob.mjs /private/config.json /path/to/body.npz published-id https://ainize.ai +``` + +Run it with installed node/core dependencies (Node 24), preferably in a Docker +container with only the config and selected body mounted read-only. It checks +the source hash, anchored size, author and ledger signature before transmitting; +only the first 500 peer records are searched. It refuses redirects and bogus +acknowledgments, and requires an authenticated GET read-back with the same hash +after acceptance. It never prints the identity secret or request auth header. +The client does not retrain, create an anchor, publish a dataset, or modify the +running author node. A missing receiving route remains a failure, not success. + Authentication, known published-public anchor, author, storage budget, and in-flight checks happen **before** multipart parsing. The receiver validates exact anchored size, hash and dimensions, bounded ZIP inflation, and existing diff --git a/scripts/retry-public-blob.mjs b/scripts/retry-public-blob.mjs new file mode 100644 index 0000000..558745f --- /dev/null +++ b/scripts/retry-public-blob.mjs @@ -0,0 +1,70 @@ +import { createHash } from 'node:crypto'; +import { createReadStream, openAsBlob, readFileSync, statSync } from 'node:fs'; +import { LocalLedger, signMessage } from '@ainize/core'; + +const [configPath, filePath, patchId, peer = 'https://www.ainize.ai'] = process.argv.slice(2); +const emit = result => process.stdout.write(`${JSON.stringify({ at: new Date().toISOString(), ...result })}\n`); + +async function boundedBody(response, maximum) { + const chunks = []; + let size = 0; + if (!response.body) return Buffer.alloc(0); + for await (const chunk of response.body) { + size += chunk.length; + if (size > maximum) throw new Error('response exceeds its byte limit'); + chunks.push(chunk); + } + return Buffer.concat(chunks); +} + +async function main() { + if (!configPath || !filePath || !patchId) throw new Error('usage: node scripts/retry-public-blob.mjs [https-peer]'); + const origin = new URL(peer); + if (origin.protocol !== 'https:' || origin.username || origin.password || origin.pathname !== '/' || origin.search || origin.hash) throw new Error('peer must be an HTTPS origin without credentials'); + const cfg = JSON.parse(readFileSync(configPath, 'utf8')); + const identity = cfg.identity; + if (!identity?.address || !/^[0-9a-fA-F]{64}$/.test(identity.privateKey ?? '')) throw new Error('configured node identity is unavailable'); + const identityMatches = address => typeof address === 'string' && address.toLowerCase() === identity.address.toLowerCase(); + const size = statSync(filePath).size; + if (size < 1 || size > 256 * 1024 ** 2) throw new Error('body is outside the relay size bounds'); + const digest = createHash('sha256'); + for await (const chunk of createReadStream(filePath)) digest.update(chunk); + const sha = digest.digest('hex'); + const headers = () => { + const timestamp = Date.now(); + const signature = signMessage(`blob:${sha}:${timestamp}`, identity.privateKey); + return { 'x-ainize-auth': `${identity.address}:${timestamp}:${signature}` }; + }; + const recordsResponse = await fetch(`${origin.origin}/p2p/records?since=0&limit=500`, { redirect: 'error', signal: AbortSignal.timeout(20_000) }); + if (!recordsResponse.ok) throw new Error(`peer ledger request returned ${recordsResponse.status}`); + const records = JSON.parse((await boundedBody(recordsResponse, 8 * 1024 ** 2)).toString('utf8')).records; + const record = records?.find(candidate => candidate.kind === 'anchor' && candidate.body?.id === patchId && candidate.body.patch_sha256 === sha); + if (!record || !LocalLedger.validate(record) || !identityMatches(record.author) || !identityMatches(record.body.author)) throw new Error('matching author-signed public anchor not found in the first 500 peer records'); + if (record.body.visibility === 'test' || record.body.price !== '0' || record.body.size_bytes !== size) throw new Error('this recovery tool only offers exact-size, already-published free public knowledge'); + emit({ stage: 'local-body-verified', patchId, author: identity.address, sha256: sha, sizeBytes: size, anchorHash: record.hash, anchorSignatureValid: true }); + const form = new FormData(); + form.append('blob', await openAsBlob(filePath), `${sha}.npz`); + const endpoint = `${origin.origin}/p2p/blob/${sha}`; + const response = await fetch(endpoint, { method: 'POST', body: form, headers: headers(), redirect: 'error', signal: AbortSignal.timeout(60_000) }); + const raw = (await boundedBody(response, 4096)).toString('utf8'); + let receipt; + try { receipt = JSON.parse(raw); } catch { receipt = raw; } + const accepted = response.ok && receipt?.ok === true && receipt.sha256 === sha && (receipt.size_bytes === size || (receipt.already_held === true && receipt.size_bytes === undefined)); + emit({ stage: 'signed-p2p-offer', endpoint, method: 'POST', status: response.status, bytesOffered: size, receipt, accepted }); + if (!accepted) { process.exitCode = 1; return; } + const download = await fetch(endpoint, { headers: headers(), redirect: 'error', signal: AbortSignal.timeout(60_000) }); + if (!download.ok || !download.body) throw new Error(`relay read-back returned ${download.status}`); + const readBack = createHash('sha256'); + let downloaded = 0; + for await (const chunk of download.body) { + downloaded += chunk.length; + if (downloaded > size) throw new Error('read-back is larger than the signed body'); + readBack.update(chunk); + } + const readBackSha = readBack.digest('hex'); + const verified = readBackSha === sha && downloaded === size; + emit({ stage: 'relay-read-back', patchId, sha256: readBackSha, sizeBytes: downloaded, verified }); + if (!verified) process.exitCode = 1; +} + +main().catch(error => { emit({ stage: 'error', error: error.message }); process.exitCode = 1; }); diff --git a/scripts/test-blob-relay-docker.sh b/scripts/test-blob-relay-docker.sh index ba57422..676a34c 100644 --- a/scripts/test-blob-relay-docker.sh +++ b/scripts/test-blob-relay-docker.sh @@ -17,7 +17,7 @@ for package in core node; do target=$evidence/source/$package mkdir -p "$target" cp -a "$source/src" "$source/package.json" "$source"/tsconfig*.json "$target/" - for optional in test fixtures trainer; do + for optional in test fixtures trainer scripts; do if [ -d "$source/$optional" ]; then cp -a "$source/$optional" "$target/"; fi done done @@ -30,7 +30,7 @@ for package in core node; do mkdir -p "$target" cp -a /source/$package/src /source/$package/package.json /source/$package/tsconfig*.json "$target/" cp -a /opt/ainize/ainize-$package/node_modules "$target/" - for optional in test fixtures trainer; do + for optional in test fixtures trainer scripts; do if [ -d /source/$package/$optional ]; then cp -a /source/$package/$optional "$target/"; fi done done @@ -38,7 +38,7 @@ cd /tmp/ainize-core npm run build cd /tmp/ainize-node npm run build -node --test --import tsx test/blob-relay.test.ts +node --test --import tsx test/blob-relay.test.ts test/retry-public-blob.test.ts node --test --import tsx test/guard-api.test.ts test/cluster.test.ts ' > "$evidence/container-id.txt" docker inspect "$name" --format '{{json .HostConfig}}' > "$evidence/host-config.json" diff --git a/src/blob-relay.ts b/src/blob-relay.ts index 4cc5bc1..f366578 100644 --- a/src/blob-relay.ts +++ b/src/blob-relay.ts @@ -41,6 +41,7 @@ export function blobRelay(market: Market): RequestHandler { try { if (held) { if (statSync(held.path).size !== expected || await sha256File(held.path) !== sha) throw new MarketError(409, 'held blob failed integrity check; operator repair required'); + market.blobs.markRelayed(sha); return { ok: true, sha256: sha, size_bytes: expected, already_held: true }; } const storage = multer.diskStorage({ @@ -76,6 +77,7 @@ export function blobRelay(market: Market): RequestHandler { const destination = market.blobs.pathFor(sha); if (existsSync(destination) && await sha256File(destination) !== sha) throw new MarketError(409, 'stored file failed integrity check; operator repair required'); const { blob } = await market.blobs.importFile(req.file.path, { copy: true, expectSha: sha }); + market.blobs.markRelayed(sha); market.log('info', 'blob', `relaying ${sha.slice(0, 12)} for ${entry.anchor.id} (${blob.size_bytes} bytes)`, entry.anchor.id); return { ok: true, sha256: sha, size_bytes: blob.size_bytes, already_held: false }; } finally { diff --git a/src/blobs.ts b/src/blobs.ts index d42ec10..8f06b88 100644 --- a/src/blobs.ts +++ b/src/blobs.ts @@ -50,6 +50,8 @@ export class BlobStore { } has(sha: string): boolean { return !!this.get(sha); } + markRelayed(sha: string): void { this.store.markBlobRelayed(sha); } + isRelayed(sha: string): boolean { return this.get(sha)?.relayed === 1; } list(): BlobRow[] { return this.store.listBlobs().filter((b) => existsSync(b.path)); } addrSet(sha: string): BigInt64Array | null { diff --git a/src/gc.ts b/src/gc.ts index a3e0f5c..36b074b 100644 --- a/src/gc.ts +++ b/src/gc.ts @@ -19,7 +19,7 @@ export interface GcPlan { candidates: GcCandidate[]; bytes: number; /** Bodies looked at and kept, by the reason they were kept. */ - kept: { authored: number; purchased: number; applied: number; draft: number; unlisted: number; too_new: number; sole_copy: number }; + kept: { authored: number; purchased: number; applied: number; draft: number; unlisted: number; too_new: number; sole_copy: number; relayed: number }; } export interface GcOptions { @@ -35,7 +35,7 @@ export interface GcOptions { export async function gcPlan(market: Market, opts: GcOptions = {}): Promise { const keepPurchased = opts.keepPurchased !== false; const now = Date.now(); - const kept: GcPlan['kept'] = { authored: 0, purchased: 0, applied: 0, draft: 0, unlisted: 0, too_new: 0, sole_copy: 0 }; + const kept: GcPlan['kept'] = { authored: 0, purchased: 0, applied: 0, draft: 0, unlisted: 0, too_new: 0, sole_copy: 0, relayed: 0 }; const catalog = await market.catalogAll(); const bySha = new Map(); for (const e of catalog) { @@ -51,6 +51,7 @@ export async function gcPlan(market: Market, opts: GcOptions = {}): Promise { const m = this.market; const v = verifierConfig(m.cfg); - if (v.retainBodies) return; + if (v.retainBodies || m.blobs.isRelayed(sha)) return; const blob = m.blobs.get(sha); if (!blob) return; // Bodies are content-addressed: every id built from the same training output shares this file. @@ -521,6 +521,7 @@ export class Verifier { if (lic && lic.source !== 'verification') return; } if (blob.path.startsWith(m.datasets.dir)) return; // a training set, not a knowledge body + if (m.blobs.isRelayed(sha)) return; m.blobs.remove(sha); for (const e of sharing) m.store.clearLicense(e.anchor.id); this.released.files++; diff --git a/test/blob-relay.test.ts b/test/blob-relay.test.ts index da502b4..8d5517f 100644 --- a/test/blob-relay.test.ts +++ b/test/blob-relay.test.ts @@ -9,6 +9,10 @@ import { createIdentity, defaultConfig, LocalLedger, sha256Hex, writeNpz, type P import { startNode } from '../src/server.js'; import { authHeader, P2P } from '../src/p2p.js'; import { inspectReplicaNpz } from '../src/replica-npz.js'; +import { Verifier } from '../src/verifier.js'; +import { Store } from '../src/store.js'; +import { BlobStore } from '../src/blobs.js'; +import { gcPlan } from '../src/gc.js'; async function fixture(context: TestContext) { const home = mkdtempSync(join(tmpdir(), 'ainize-relay-')); @@ -239,3 +243,57 @@ test('operator retries an existing signed knowledge without another publish reco assert.deepEqual(readFileSync(receiver.node.market.blobs.get(author.sha)!.path), author.bytes); assert.equal((await author.node.ledger.anchors()).length, before); }); + +test('relayed bodies survive verification cleanup and GC without granting a paid license', async context => { + const setup = await fixture(context); await setup.publish({ price: '1' }); + assert.equal((await setup.offer()).status, 200); + const verifier = new Verifier(setup.node.market, 60_000); + const cleanup = verifier as unknown as { releaseBody: (anchor: PatchAnchor, sha: string) => Promise }; + await cleanup.releaseBody(setup.anchor, setup.sha); + assert.equal(setup.node.market.blobs.has(setup.sha), true); + assert.equal(setup.node.market.licenseOf((await setup.node.market.entry(setup.anchor.id))!), null); + const plan = await gcPlan(setup.node.market, { allowSoleCopy: true, keepPurchased: false }); + assert.equal(plan.kept.relayed, 1); + assert.deepEqual(plan.candidates, []); +}); + +test('relay retention persists across database reopening and explicit removal clears it', async context => { + const setup = await fixture(context); await setup.publish(); + assert.equal((await setup.offer()).status, 200); + const reopened = new Store(join(setup.cfg.dataDir, 'node.sqlite')); + try { + const blobs = new BlobStore(reopened, setup.cfg.dataDir); + assert.equal(blobs.isRelayed(setup.sha), true); + blobs.remove(setup.sha); + assert.equal(blobs.isRelayed(setup.sha), false); + await blobs.importFile(setup.file, { copy: true }); + assert.equal(blobs.isRelayed(setup.sha), false); + } finally { reopened.close(); } +}); + +test('ordinary verification copies still follow the existing release policy', async context => { + const setup = await fixture(context); await setup.publish({ price: '1' }); + await setup.node.market.blobs.importFile(setup.file, { copy: true }); + const verifier = new Verifier(setup.node.market, 60_000); + const cleanup = verifier as unknown as { releaseBody: (anchor: PatchAnchor, sha: string) => Promise }; + await cleanup.releaseBody(setup.anchor, setup.sha); + assert.equal(setup.node.market.blobs.has(setup.sha), false); +}); + +test('verification cleanup rechecks retention after awaiting the catalog', async context => { + const setup = await fixture(context); await setup.publish({ price: '1' }); + await setup.node.market.blobs.importFile(setup.file, { copy: true }); + const original = setup.node.market.catalogAll.bind(setup.node.market); + let releaseCatalog!: () => void; + const gate = new Promise(resolve => { releaseCatalog = resolve; }); + setup.node.market.catalogAll = async force => { await gate; return original(force); }; + const verifier = new Verifier(setup.node.market, 60_000); + const cleanup = verifier as unknown as { releaseBody: (anchor: PatchAnchor, sha: string) => Promise }; + try { + const pending = cleanup.releaseBody(setup.anchor, setup.sha); + setup.node.market.blobs.markRelayed(setup.sha); + releaseCatalog(); + await pending; + assert.equal(setup.node.market.blobs.has(setup.sha), true); + } finally { releaseCatalog(); setup.node.market.catalogAll = original; } +}); diff --git a/test/retry-public-blob.test.ts b/test/retry-public-blob.test.ts new file mode 100644 index 0000000..1585231 --- /dev/null +++ b/test/retry-public-blob.test.ts @@ -0,0 +1,62 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { pathToFileURL } from 'node:url'; +import { createIdentity, LocalLedger, sha256Hex } from '@ainize/core'; + +for (const mode of ['success', 'html-404', 'html-200', 'wrong-download', 'tampered-anchor', 'paid-anchor']) { + test(`standalone recovery client: ${mode}`, async context => { + const directory = mkdtempSync(join(tmpdir(), 'ainize-recovery-')); + const identity = createIdentity(); + const ledger = new LocalLedger(':memory:', identity); + await ledger.init(); + context.after(async () => { await ledger.close(); rmSync(directory, { recursive: true, force: true }); }); + const bytes = Buffer.from('synthetic recovery transport fixture'); + const sha = sha256Hex(bytes); + const configPath = join(directory, 'config.json'); + const filePath = join(directory, 'body.npz'); + writeFileSync(configPath, JSON.stringify({ identity }), { mode: 0o600 }); + writeFileSync(filePath, bytes); + const record = await ledger.append('anchor', { + id: 'recovery-test', author: identity.address, patch_sha256: sha, + size_bytes: bytes.length, visibility: 'public', price: mode === 'paid-anchor' ? '1' : '0', + }); + if (mode === 'tampered-anchor') (record.body as { size_bytes: number }).size_bytes++; + const bootstrap = ` +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import {createHash} from 'node:crypto'; +import {verifyMessage} from ${JSON.stringify(pathToFileURL(resolve('node_modules/@ainize/core/dist/index.js')).href)}; +const bytes = readFileSync(${JSON.stringify(filePath)}); +const mode = ${JSON.stringify(mode)}; +globalThis.fetch = async (url, options) => { + assert.equal(options.redirect, 'error'); + if (String(url).includes('/p2p/records?')) return Response.json({records: [${JSON.stringify(record)}]}); + assert.ok(!['paid-anchor','tampered-anchor'].includes(mode), 'invalid anchor must not send a body'); + const [address, timestamp, signature] = options.headers['x-ainize-auth'].split(':'); + assert.equal(address, ${JSON.stringify(identity.address)}); + assert.ok(verifyMessage('blob:${sha}:' + timestamp, signature, address)); + if (options.method === 'POST') { + const uploaded = Buffer.from(await options.body.get('blob').arrayBuffer()); + assert.deepEqual(uploaded, bytes); + if (mode.startsWith('html-')) return new Response('Cannot POST /p2p/blob', {status: mode === 'html-404' ? 404 : 200}); + return Response.json({ok:true,sha256:${JSON.stringify(sha)},size_bytes:bytes.length,already_held:false}); + } + return new Response(mode === 'wrong-download' ? Buffer.alloc(bytes.length) : bytes); +}; +process.argv = [process.execPath, 'retry-public-blob.mjs', ${JSON.stringify(configPath)}, ${JSON.stringify(filePath)}, 'recovery-test', 'https://relay.invalid']; +await import(${JSON.stringify(pathToFileURL(resolve('scripts/retry-public-blob.mjs')).href)}); +`; + const result = spawnSync(process.execPath, ['--input-type=module', '-e', bootstrap], { encoding: 'utf8', timeout: 20_000 }); + assert.equal(result.status, mode === 'success' ? 0 : 1, result.stdout + result.stderr); + assert.ok(!`${result.stdout}${result.stderr}`.includes(identity.privateKey)); + const output = result.stdout.split('\n').filter(line => line.startsWith('{')).map(line => JSON.parse(line)); + if (mode === 'success') assert.equal(output.at(-1)?.verified, true); + if (mode.startsWith('html-')) assert.equal(output.at(-1)?.accepted, false); + if (mode === 'wrong-download') assert.equal(output.at(-1)?.verified, false); + if (mode === 'tampered-anchor' || mode === 'paid-anchor') assert.deepEqual(output.map(entry => entry.stage), ['error']); + }); +} From 0e19caa0df5af9108d77e859bf9c82d019990c20 Mon Sep 17 00:00:00 2001 From: Haechan Date: Fri, 11 Sep 2026 10:57:35 +0000 Subject: [PATCH 4/9] Add reproducible API-only runtime refresh with linked-body preservation checks --- deploy/build-source-refresh.sh | 28 ++++++++ deploy/cert-docker/compose.ainize.json | 35 ++++++++++ deploy/cert-docker/upgrade-ainize.sh | 58 ++++++++++++++++ deploy/runtime-snapshot.mjs | 96 ++++++++++++++++++++++++++ deploy/source-refresh.Dockerfile | 15 ++++ deploy/source-refresh.md | 30 ++++++++ docs/blob-relay-evidence-ko.md | 7 ++ scripts/test-blob-relay-docker.sh | 6 +- test/runtime-snapshot.test.ts | 63 +++++++++++++++++ 9 files changed, 335 insertions(+), 3 deletions(-) create mode 100644 deploy/build-source-refresh.sh create mode 100644 deploy/cert-docker/compose.ainize.json create mode 100644 deploy/cert-docker/upgrade-ainize.sh create mode 100644 deploy/runtime-snapshot.mjs create mode 100644 deploy/source-refresh.Dockerfile create mode 100644 deploy/source-refresh.md create mode 100644 test/runtime-snapshot.test.ts diff --git a/deploy/build-source-refresh.sh b/deploy/build-source-refresh.sh new file mode 100644 index 0000000..3f41826 --- /dev/null +++ b/deploy/build-source-refresh.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail +NODE=$(cd "$(dirname "$0")/.." && pwd) +CORE=$(realpath "${1:?pass the compatible ainize-core worktree}") +CLI=$(realpath "${2:?pass the compatible ainize-cli worktree}") +OUT=${3:?pass a new build evidence directory} +IMAGE=${4:?pass a new local image tag} +BASE=${AINIZE_BASE_IMAGE:?pass the existing dependency/runtime image ID or tag} +mkdir "$OUT" +OUT=$(realpath "$OUT") +mkdir "$OUT/context" +BASE_ID=$(docker image inspect "$BASE" --format '{{.Id}}') +printf '%s\n' "$BASE_ID" > "$OUT/base-image.txt" +for name in core node cli; do + source=$NODE + if [ "$name" = core ]; then source=$CORE; fi + if [ "$name" = cli ]; then source=$CLI; fi + mkdir "$OUT/context/$name" + cp -a "$source/src" "$source/package.json" "$source"/tsconfig*.json "$OUT/context/$name/" + git -C "$source" rev-parse HEAD > "$OUT/$name-commit.txt" +done +cp -a "$NODE/trainer" "$OUT/context/node/" +cp "$NODE/deploy/source-refresh.Dockerfile" "$OUT/context/Dockerfile" +(cd "$OUT/context" && find . -type f -print0 | sort -z | xargs -0 sha256sum) > "$OUT/source-sha256.txt" +DOCKER_BUILDKIT=0 docker build --cpu-period 100000 --cpu-quota 200000 --cpuset-cpus "${AINIZE_CPUSET:-0-7}" \ + --memory 4g --memory-swap 4g --build-arg "BASE_IMAGE=$BASE_ID" -t "$IMAGE" "$OUT/context" > "$OUT/build.log" 2>&1 +docker image inspect "$IMAGE" > "$OUT/image.json" +echo "Built $IMAGE. This command does not replace a running node or publish an image." diff --git a/deploy/cert-docker/compose.ainize.json b/deploy/cert-docker/compose.ainize.json new file mode 100644 index 0000000..a284e38 --- /dev/null +++ b/deploy/cert-docker/compose.ainize.json @@ -0,0 +1,35 @@ +{ + "name": "ain-cert-ainize", + "services": { + "node": { + "image": "${AINIZE_IMAGE:-ain-cert-ainize:relay-runtime-20260911}", + "network_mode": "host", + "cpus": 2, + "cpuset": "0-7", + "mem_limit": "8g", + "memswap_limit": "8g", + "user": "${AINIZE_UID:?set AINIZE_UID}:${AINIZE_GID:?set AINIZE_GID}", + "group_add": ["${DOCKER_GID:?set DOCKER_GID}"], + "environment": { + "AINIZE_HOME": "/mnt/newdata/gov/kpi/ainize/home-docker" + }, + "volumes": [ + "/mnt/newdata/gov/kpi/ainize/home-docker:/mnt/newdata/gov/kpi/ainize/home-docker", + "/mnt/newdata/qwen3.8:/mnt/newdata/qwen3.8", + "/mnt/newdata/gov/kpi/harness/dart-datasets:/datasets:ro", + "/mnt/newdata/gov/kpi/evidence:/mnt/newdata/gov/kpi/evidence", + "/var/run/docker.sock:/var/run/docker.sock:ro" + ], + "healthcheck": { + "test": ["CMD", "node", "-e", "fetch('http://localhost:3410/healthz').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))"], + "interval": "15s", + "timeout": "5s", + "retries": 4, + "start_period": "30s" + }, + "restart": "no", + "labels": { "org.ain.cert.role": "ainize-node" }, + "logging": { "driver": "json-file", "options": { "max-size": "20m", "max-file": "4" } } + } + } +} diff --git a/deploy/cert-docker/upgrade-ainize.sh b/deploy/cert-docker/upgrade-ainize.sh new file mode 100644 index 0000000..9274956 --- /dev/null +++ b/deploy/cert-docker/upgrade-ainize.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +set -euo pipefail +KPI=$(cd "$(dirname "$0")/.." && pwd) +RUN_ID=${RUN_ID:-ainize_upgrade_$(date -u +%Y%m%dT%H%M%SZ)} +[[ "$RUN_ID" =~ ^[A-Za-z0-9_-]+$ ]] || exit 1 +OUT="$KPI/evidence/$RUN_ID" +BACKUP="$KPI/secrets/$RUN_ID" +SNAPSHOT="$KPI/pr/an-relay/deploy/runtime-snapshot.mjs" +TRAINER_ROOT=${AINIZE_TRAINER_ROOT:-/mnt/newdata/qwen3.8/.teach} +mkdir "$OUT" +mkdir -m 700 "$BACKUP" +docker top ain-cert-ainize-node-1 -eo pid,stat,args > "$OUT/processes-before.txt" +if awk '$0 ~ /node .*ainize-lifecycle[.]js/ && $2 !~ /T/ {found=1} END {exit !found}' "$OUT/processes-before.txt"; then + echo 'lifecycle observer is active; arrange a terminal/idle maintenance window first' >&2 + exit 1 +fi +for container in flashnext flashtrain; do + docker inspect "$container" --format '{{.Id}} {{.State.StartedAt}} {{.State.Pid}}' > "$OUT/$container-before.txt" +done +bash "$KPI/docker/ainize-cli.sh" teach jobs --json > "$OUT/jobs-before.json" +curl --fail --silent --show-error --max-time 30 http://localhost:3410/api/info > "$OUT/info-before.json" +cp "$SNAPSHOT" "$OUT/runtime-snapshot.mjs" +sha256sum "$OUT/runtime-snapshot.mjs" > "$OUT/source.sha256" +node "$OUT/runtime-snapshot.mjs" capture "$KPI/ainize/home-docker" "$OUT/jobs-before.json" "$OUT/info-before.json" "$OUT/inventory-before.json" "$TRAINER_ROOT" +docker inspect ain-cert-ainize-node-1 --format '{{json .State}}' > "$OUT/state-before.json" +docker inspect ain-cert-ainize-node-1 --format '{{json .Image}}' > "$OUT/image-before.json" +export AINIZE_UID="$(id -u)" AINIZE_GID="$(id -g)" DOCKER_GID="$(stat -c %g /var/run/docker.sock)" +docker compose -f "$KPI/docker/compose.ainize.json" config --format json > "$OUT/compose.json" +TARGET_IMAGE=$(node -e 'console.log(JSON.parse(require("fs").readFileSync(process.argv[1])).services.node.image)' "$OUT/compose.json") +docker image inspect "$TARGET_IMAGE" --format '{{json .Id}}' > "$OUT/target-image.json" +docker compose -f "$KPI/docker/compose.ainize.json" stop -t 60 node > "$OUT/stop.log" 2>&1 +umask 077 +tar -C "$KPI/ainize" -cf "$BACKUP/home-docker.tar" home-docker +tar -C "$TRAINER_ROOT" -cf "$BACKUP/trained-bodies.tar" . +umask 022 +sha256sum "$BACKUP/home-docker.tar" "$BACKUP/trained-bodies.tar" > "$OUT/backup.sha256" +docker compose -f "$KPI/docker/compose.ainize.json" up -d --no-deps node > "$OUT/start.log" 2>&1 +docker inspect ain-cert-ainize-node-1 --format '{{json .Image}}' > "$OUT/image-after.json" +cmp "$OUT/target-image.json" "$OUT/image-after.json" +docker inspect ain-cert-ainize-node-1 --format '{{json .HostConfig}}' > "$OUT/limits-after.json" +for attempt in {1..60}; do + if curl --fail --silent --max-time 5 http://localhost:3410/readyz > "$OUT/ready.json"; then + node -e 'const value=JSON.parse(require("fs").readFileSync(process.argv[1])); if (value.ready !== true && value.ok !== true) throw Error("not backend readiness JSON")' "$OUT/ready.json" + bash "$KPI/docker/ainize-cli.sh" teach jobs --json > "$OUT/jobs-after.json" + curl --fail --silent --show-error --max-time 30 http://localhost:3410/api/info > "$OUT/info-after.json" + node "$OUT/runtime-snapshot.mjs" capture "$KPI/ainize/home-docker" "$OUT/jobs-after.json" "$OUT/info-after.json" "$OUT/inventory-after.json" "$TRAINER_ROOT" + node "$OUT/runtime-snapshot.mjs" verify "$OUT/inventory-before.json" "$OUT/inventory-after.json" > "$OUT/preservation.json" + for container in flashnext flashtrain; do + docker inspect "$container" --format '{{.Id}} {{.State.StartedAt}} {{.State.Pid}}' > "$OUT/$container-after.txt" + cmp "$OUT/$container-before.txt" "$OUT/$container-after.txt" + done + echo "Ainize ready; private backup preserved at $BACKUP" + exit 0 + fi + sleep 2 +done +echo 'readiness observation expired; inspect the same container, do not restart blindly' +exit 1 diff --git a/deploy/runtime-snapshot.mjs b/deploy/runtime-snapshot.mjs new file mode 100644 index 0000000..889b741 --- /dev/null +++ b/deploy/runtime-snapshot.mjs @@ -0,0 +1,96 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { createReadStream } from 'node:fs'; +import { lstat, readdir, readFile, realpath, stat as fileStat, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +export function requireIdle(info, jobs) { + const terminal = new Set(['READY', 'FAILED', 'CANCELLED', 'NEEDS_MORE', 'REJECTED', 'ANNOUNCED', 'PENDING_REVIEW']); + assert.ok(Array.isArray(jobs.items) && jobs.items.length < 500, 'job list is missing or may be truncated'); + assert.ok(jobs.items.every(job => terminal.has(job.status)), 'unfinished teach job; do not restart'); + const runtime = info.runtime; + assert.equal(runtime?.available, true, 'runtime is unavailable'); + assert.deepEqual(runtime.applied, [], 'runtime has an applied patch'); + assert.equal(runtime.queue?.running, null); + assert.equal(runtime.queue.waiting, 0); + assert.equal(runtime.queue.lock, null); + assert.deepEqual(runtime.queue.queued, []); +} + +export function jobBindings(jobs) { + return jobs.items.map(job => ({ id: job.id, status: job.status, dataset: job.dataset, mode: job.mode, + context_patch_ids: job.context_patch_ids, draft_id: job.draft_id, result: job.result, checks: job.checks, + })).sort((left, right) => left.id.localeCompare(right.id)); +} + +export async function inventory(home, trainerRoot) { + const files = {}; + const allowed = trainerRoot ? await realpath(trainerRoot) : null; + async function visit(relative, base = home, prefix = '') { + let filename = path.join(base, relative); + let stat = await lstat(filename); + let target; + if (stat.isSymbolicLink()) { + assert.ok(allowed && base === home && /^[a-f0-9]{64}\.npz$/.test(path.basename(relative)), `refusing symlink in evidence inventory: ${relative}`); + filename = await realpath(filename); + assert.ok(filename.startsWith(allowed + path.sep), `symlink escapes the explicit trainer root: ${relative}`); + target = path.relative(allowed, filename); + stat = await fileStat(filename); + assert.ok(stat.isFile(), 'linked body must be a regular file'); + } + if (stat.isDirectory()) { + for (const name of (await readdir(filename)).sort()) await visit(path.join(relative, name), base, prefix); + } else { + assert.ok(stat.isFile(), `unexpected file type: ${relative}`); + const hash = createHash('sha256'); + for await (const chunk of createReadStream(filename)) hash.update(chunk); + const sha256 = hash.digest('hex'); + if (target) assert.equal(path.basename(relative, '.npz'), sha256, 'linked body does not match its content-addressed name'); + files[prefix + relative] = { bytes: stat.size, sha256, ...(target ? { trainerTarget: target } : {}) }; + } + } + for (const directory of ['data/teach/datasets', 'data/drive/patches']) await visit(directory); + if (allowed) await visit('', allowed, 'trainer/'); + assert.ok(Object.keys(files).length > 0, 'empty data inventory'); + return files; +} + +export function compareSnapshots(before, after) { + assert.deepEqual(after.jobs, before.jobs, 'job IDs, datasets, trained bodies or checks changed'); + for (const [filename, metadata] of Object.entries(before.files)) { + assert.deepEqual(after.files[filename], metadata, `existing dataset or trained body changed: ${filename}`); + } + assert.deepEqual(after.runtime, before.runtime, 'model runtime identity changed'); + assert.equal(after.nodeAddress, before.nodeAddress, 'publisher identity changed'); +} + +async function main() { + const [command, ...args] = process.argv.slice(2); + if (command === 'capture') { + const [home, jobsPath, infoPath, destination, trainerRoot] = args; + assert.ok(destination, 'capture [trainer-root]'); + const jobs = JSON.parse(await readFile(jobsPath, 'utf8')); + const info = JSON.parse(await readFile(infoPath, 'utf8')); + requireIdle(info, jobs); + const runtime = Object.fromEntries(['api', 'model', 'hook', 'repo', 'patch_dir'].map(key => [key, info.runtime[key]])); + const files = await inventory(home, trainerRoot); + if (trainerRoot) { + for (const job of jobs.items.filter(item => item.result?.sha256 && item.checks?.executed)) { + const body = files[`trainer/${job.id}/lesson.npz`]; + assert.equal(body?.sha256, job.result.sha256, `checked job body changed: ${job.id}`); + assert.equal(body.bytes, job.result.size_bytes, `checked job size changed: ${job.id}`); + } + } + const snapshot = { at: new Date().toISOString(), nodeAddress: info.node.address, runtime, jobs: jobBindings(jobs), files }; + await writeFile(destination, JSON.stringify(snapshot, null, 2) + '\n', { flag: 'wx' }); + } else if (command === 'verify') { + const [beforePath, afterPath] = args; + const before = JSON.parse(await readFile(beforePath, 'utf8')); + const after = JSON.parse(await readFile(afterPath, 'utf8')); + compareSnapshots(before, after); + console.log(JSON.stringify({ verified: true, jobs: before.jobs.length, preservedFiles: Object.keys(before.files).length })); + } else throw new Error('expected capture or verify'); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) main().catch(error => { console.error(error.message); process.exitCode = 1; }); diff --git a/deploy/source-refresh.Dockerfile b/deploy/source-refresh.Dockerfile new file mode 100644 index 0000000..641607d --- /dev/null +++ b/deploy/source-refresh.Dockerfile @@ -0,0 +1,15 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} +USER root +RUN rm -rf /opt/ainize/ainize-core/src /opt/ainize/ainize-core/dist /opt/ainize/ainize-node/src /opt/ainize/ainize-node/dist /opt/ainize/ainize-cli/src /opt/ainize/ainize-cli/dist +COPY core/src /opt/ainize/ainize-core/src +COPY core/package.json core/tsconfig*.json /opt/ainize/ainize-core/ +COPY node/src /opt/ainize/ainize-node/src +COPY node/package.json node/tsconfig*.json /opt/ainize/ainize-node/ +COPY node/trainer /opt/ainize/ainize-node/trainer +COPY cli/src /opt/ainize/ainize-cli/src +COPY cli/package.json cli/tsconfig*.json /opt/ainize/ainize-cli/ +RUN cd /opt/ainize/ainize-core && npm run build && cd /opt/ainize/ainize-node && npm run build && cd /opt/ainize/ainize-cli && npm run build +WORKDIR /opt/ainize/ainize-node +ENTRYPOINT ["node"] +CMD ["/opt/ainize/ainize-node/dist/bin.js"] diff --git a/deploy/source-refresh.md b/deploy/source-refresh.md new file mode 100644 index 0000000..d4aec10 --- /dev/null +++ b/deploy/source-refresh.md @@ -0,0 +1,30 @@ +# Certification runtime refresh + +These helpers are for the existing `/mnt/newdata/gov/kpi` deployment, not a clean-machine installer or a public-server deployment. They do not restart model containers or publish container images. + +`build-source-refresh.sh` freezes compatible core/node/CLI source and rebuilds all three against the immutable ID of an existing dependency/Python runtime image. It saves source hashes, Git provenance, build output and image ID. CPU quota2, cpuset0–7 and RAM/swap4GiB bound the build. Changing dependencies requires rebuilding the dependency image first; this helper does not run an unlocked `npm install`. + +```bash +AINIZE_BASE_IMAGE=sha256:563e96f6725136939bd6bac5cf6a6480fcc535f71fb51c0ab02ab19d7d8891d7 \ + bash deploy/build-source-refresh.sh /path/to/ainize-core /path/to/ainize-cli \ + /path/to/new-evidence ain-cert-ainize:relay-runtime-20260911 +``` + +The verified build combines core `695a8ad6` (0.1.3), node runtime `257bfe85` (0.1.2 plus hardening/watchdog fix), and CLI `9acc9de` (0.1.1 including HF URL import and VERIFIED compatibility). Keeping the older CLI source caused a real `LISTED`/`VERIFIED` TypeScript error; that failed build is preserved. Rebuilding with this helper returned the exact same final image ID `sha256:da23af1a5c75cfee61bdad8403f39cad31b6bd2860e7af590a6ffbf03f31c4b0`. Dependency checks and all three package builds pass. + +## Safe API-only replacement + +Install `cert-docker/upgrade-ainize.sh` and `cert-docker/compose.ainize.json` verbatim in the deployment's `kpi/docker/`. The existing clone at `kpi/pr/an-relay` supplies `deploy/runtime-snapshot.mjs`. The compose default is the verified local image; `AINIZE_IMAGE` can select an explicit available image. This is not permission to replace another deployment's configuration. + +Coordinate the lifecycle observer first. Pause it intentionally while it is only observing a known TRAINING job; do not interrupt an inference audit or cancel the actual job. Wait for the server's real terminal job state and an idle, empty model stack. The upgrade script refuses active observers and nonterminal jobs. A preflight check is not a distributed lock against another operator, so the runtime must be reserved for this maintenance. + +```bash +cd /mnt/newdata/gov +RUN_ID=ainize_runtime_upgrade_20260911 bash kpi/docker/upgrade-ainize.sh +``` + +The script snapshots the job IDs, dataset bindings, patch hashes/checks, publisher identity, data files and trainer files. Published drive entries can be symlinks into `.teach//lesson.npz`: only SHA-named links under the explicit trainer root are followed and their bytes must match the name. Other links are refused. All checked job bodies must match their recorded SHA and size. Neither a symlink nor a public node's empty blob list proves the original body is gone. + +Only the Ainize API container stops. Its private home and the linked trainer tree are backed up separately under `kpi/secrets/` (0700 directory/0600 archives). Backups are never release assets. After startup, actual JSON readiness, exact target image, every existing data/body hash, unchanged job binding and publisher/model identity are checked. `flashnext` and `flashtrain` container IDs/start times/PIDs must remain identical. An expired readiness observation is a failure to inspect, not a reason for blind restart. Resume the original lifecycle RUN_ID only after preservation checks pass; do not replace its frozen source or submit duplicate jobs. + +Offline validation: `ainize_runtime_maintenance_final_tests_20260911` passed55 tests (30 relay/client/watchdog/snapshot +25 cluster/chat guards). The new snapshot tests include linked-body handling and refusal of foreign/mutated state. Test fixtures and a successful local rebuild do not establish public-server deployment, P2P delivery, Live answer quality or all100 dataset evaluations. diff --git a/docs/blob-relay-evidence-ko.md b/docs/blob-relay-evidence-ko.md index f40d00c..8a3174e 100644 --- a/docs/blob-relay-evidence-ko.md +++ b/docs/blob-relay-evidence-ko.md @@ -52,3 +52,10 @@ curl -i -X POST http://127.0.0.1:3400/p2p/blob/f9f665f6fa1a6b37963a4845107c0c0a5 - DART7번째는 teach READY 및16개 비교추론 응답을 저장했지만 패치 제거 후 스택이 남아 감사가 실패했다. 유휴·소유 patch ID/SHA를 확인한 후 **그 패치만** 다시 제거하고,16개 원문을 재사용해 동일job/RUN_ID로 재개했다. 모델을 다시 올리거나 학습을 중복 제출하지 않았다.7개 감사완료·8번째 학습중이며 정답률은 기본44/56·대체9/56이다. - 별도 코드 검증에서 watchdog이 잠금 전에 읽은 옛 스택을 잠금 후 재적용하는 경쟁을 재현했다. 잠금 안에서 스택/최상위/복구 계획을 읽도록 수정했으며 회귀3건은 수정 전 모두 실패,수정 후 모두 통과했다. 이 경쟁이 실제7번째 실패의 유일한 원인이었다고 단정하지 않는다. - 추가 원문: `blob_relay_main_compat_20260911/`, `blob_relay_watchdog_red_20260911/`, `blob_relay_watchdog_green_20260911/`, `ainize_lifecycle100_20260911/recovery-7/`. 공개 본문 수신/Live 성공 및 현재 운영 노드에 이 수정이 배포됐는지는 여전히 별도 확인 대상이다. + +## 10:43 UTC 실제 본문 재전송과 유지보수 검증 + +- 두 원본은 로컬 publisher의 drive에서 `/mnt/newdata/qwen3.8/.teach/<기존job>/lesson.npz`를 가리키는 링크다. 링크 대상 바이트를 읽어 공개 anchor의 서명·작성자·SHA·크기를 다시 검증했다. 각각3,679,278/3,719,206bytes로 동일하다. 관리자가 확인한 공개 네 노드의 blobs0과 이 로컬 원본 보존은 서로 모순되지 않는다. +- 최신 호환 이미지(core0.1.3/node0.1.2+보강/CLI0.1.1)에서 **실제 signed multipart 본문**을 다시 보냈다. 10:43:29 www 첫 번째,10:43:31 apex 두 번째 모두 HTML404 `Cannot POST /p2p/blob/...`, accepted=false. 새 앵커/데이터셋이나 학습을 만들지 않았다. 원문은 `kpi/evidence/signed_p2p_offer_r3_20260911/`. +- blob0이면 없는 본문의 GET 실패는 설명된다. 그러나 빈 수신 노드도 처리해야 하는 POST 업로드의 HTML `Cannot POST`를 GET의 파일 부재와 같은 증거로 해석하지 않는다. 프록시인지 실행 바이너리인지의 확정은 관리자 내부3400 응답·실행 커밋 확인이 필요하다. +- 이미지 전체 빌드/의존성 검사와 회귀55개가 통과했다. API-only 교체 전후 원본·학습 파일/ID 검증 및 링크 대상 별도 비공개 백업을 추가했다. 실행·검증 절차는 `deploy/source-refresh.md`이며 공개 서버에 배포되었다고 보고하지 않는다. diff --git a/scripts/test-blob-relay-docker.sh b/scripts/test-blob-relay-docker.sh index ae38ceb..5b6fa63 100644 --- a/scripts/test-blob-relay-docker.sh +++ b/scripts/test-blob-relay-docker.sh @@ -17,7 +17,7 @@ for package in core node; do target=$evidence/source/$package mkdir -p "$target" cp -a "$source/src" "$source/package.json" "$source"/tsconfig*.json "$target/" - for optional in test fixtures trainer scripts; do + for optional in test fixtures trainer scripts deploy; do if [ -d "$source/$optional" ]; then cp -a "$source/$optional" "$target/"; fi done done @@ -30,7 +30,7 @@ for package in core node; do mkdir -p "$target" cp -a /source/$package/src /source/$package/package.json /source/$package/tsconfig*.json "$target/" cp -a /opt/ainize/ainize-$package/node_modules "$target/" - for optional in test fixtures trainer scripts; do + for optional in test fixtures trainer scripts deploy; do if [ -d /source/$package/$optional ]; then cp -a /source/$package/$optional "$target/"; fi done done @@ -38,7 +38,7 @@ cd /tmp/ainize-core npm run build cd /tmp/ainize-node npm run build -node --test --import tsx test/blob-relay.test.ts test/retry-public-blob.test.ts test/watchdog-snapshot.test.ts +node --test --import tsx test/blob-relay.test.ts test/retry-public-blob.test.ts test/watchdog-snapshot.test.ts test/runtime-snapshot.test.ts node --test --import tsx test/guard-api.test.ts test/cluster.test.ts ' > "$evidence/container-id.txt" docker inspect "$name" --format '{{json .HostConfig}}' > "$evidence/host-config.json" diff --git a/test/runtime-snapshot.test.ts b/test/runtime-snapshot.test.ts new file mode 100644 index 0000000..1fe9662 --- /dev/null +++ b/test/runtime-snapshot.test.ts @@ -0,0 +1,63 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { mkdtemp, mkdir, writeFile, symlink, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { test } from 'node:test'; +import { compareSnapshots, inventory, jobBindings, requireIdle } from '../deploy/runtime-snapshot.mjs'; + +const info = { runtime: { available: true, applied: [], queue: { running: null, waiting: 0, queued: [], lock: null } } }; +const jobs = { items: [{ id: 'job-one', status: 'READY', dataset: { id: 'dataset-one', sha256: 'body-hash' }, result: { sha256: 'patch-hash' }, checks: { executed: true } }] }; + +test('maintenance requires terminal jobs and an idle, empty runtime', () => { + requireIdle(info, jobs); + assert.throws(() => requireIdle(info, { items: [{ ...jobs.items[0], status: 'TRAINING' }] })); + for (const queue of [{ running: 'chat' }, { waiting: 1 }, { lock: { pid: 1 } }, { queued: ['chat'] }]) { + assert.throws(() => requireIdle({ runtime: { ...info.runtime, queue: { ...info.runtime.queue, ...queue } } }, jobs)); + } + assert.throws(() => requireIdle({ runtime: { ...info.runtime, applied: [{ id: 'foreign' }] } }, jobs)); +}); + +test('explicit trainer-root links are hashed as bodies, never treated as missing or followed outside the root', async () => { + const home = await mkdtemp(path.join(tmpdir(), 'ainize-linked-body-')); + try { + const trainer = path.join(home, 'trainer'); + await mkdir(trainer); + await mkdir(path.join(home, 'data/teach/datasets'), { recursive: true }); + await mkdir(path.join(home, 'data/drive/patches'), { recursive: true }); + const body = Buffer.from('trained-fixture'); + const sha = createHash('sha256').update(body).digest('hex'); + await writeFile(path.join(trainer, 'lesson.npz'), body); + await symlink(path.join(trainer, 'lesson.npz'), path.join(home, `data/drive/patches/${sha}.npz`)); + await assert.rejects(inventory(home), /symlink/); + const files = await inventory(home, trainer); + assert.equal(files[`data/drive/patches/${sha}.npz`].sha256, sha); + assert.equal(files['trainer/lesson.npz'].sha256, sha); + await writeFile(path.join(trainer, 'lesson.npz'), 'different'); + await assert.rejects(inventory(home, trainer), /content-addressed/); + } finally { await rm(home, { recursive: true, force: true }); } +}); + +test('maintenance verifies original job bindings, model and bytes without exposing credentials', async () => { + const home = await mkdtemp(path.join(tmpdir(), 'ainize-snapshot-')); + try { + await mkdir(path.join(home, 'data/teach/datasets'), { recursive: true }); + await mkdir(path.join(home, 'data/drive/patches'), { recursive: true }); + await writeFile(path.join(home, 'data/teach/datasets/questions.jsonl'), 'questions'); + await writeFile(path.join(home, 'data/drive/patches/body.npz'), 'body'); + await writeFile(path.join(home, 'config.json'), 'PRIVATE-NOT-FOR-INVENTORY'); + const before = { jobs: jobBindings(jobs), files: await inventory(home), runtime: { model: 'model-one' }, nodeAddress: 'publisher' }; + assert.equal(Object.keys(before.files).length, 2); + compareSnapshots(before, structuredClone(before)); + const after = structuredClone(before); + after.jobs[0].result.sha256 = 'different'; + assert.throws(() => compareSnapshots(before, after)); + await writeFile(path.join(home, 'data/drive/patches/body.npz'), 'tampered'); + assert.throws(() => compareSnapshots(before, { ...before, files: {} })); + const changed = await inventory(home); + assert.throws(() => compareSnapshots(before, { ...before, files: changed })); + assert.throws(() => compareSnapshots(before, { ...before, runtime: { model: 'different' } })); + await symlink(path.join(home, 'config.json'), path.join(home, 'data/drive/patches/secret')); + await assert.rejects(inventory(home), /symlink/); + } finally { await rm(home, { recursive: true, force: true }); } +}); From c6575d1818c5f69d169648a5b7a3151d9992f633 Mon Sep 17 00:00:00 2001 From: Haechan Date: Fri, 11 Sep 2026 11:03:13 +0000 Subject: [PATCH 5/9] Check all operator-visible jobs before shared-runtime maintenance --- deploy/cert-docker/upgrade-ainize.sh | 4 ++-- deploy/runtime-snapshot.mjs | 28 +++++++++++++++++++++++++++- deploy/source-refresh.md | 2 +- docs/blob-relay.md | 6 +++++- test/runtime-snapshot.test.ts | 20 +++++++++++++++++++- 5 files changed, 54 insertions(+), 6 deletions(-) diff --git a/deploy/cert-docker/upgrade-ainize.sh b/deploy/cert-docker/upgrade-ainize.sh index 9274956..9c7f901 100644 --- a/deploy/cert-docker/upgrade-ainize.sh +++ b/deploy/cert-docker/upgrade-ainize.sh @@ -17,7 +17,7 @@ fi for container in flashnext flashtrain; do docker inspect "$container" --format '{{.Id}} {{.State.StartedAt}} {{.State.Pid}}' > "$OUT/$container-before.txt" done -bash "$KPI/docker/ainize-cli.sh" teach jobs --json > "$OUT/jobs-before.json" +node "$SNAPSHOT" jobs "$KPI/ainize/home-docker" "$OUT/jobs-before.json" curl --fail --silent --show-error --max-time 30 http://localhost:3410/api/info > "$OUT/info-before.json" cp "$SNAPSHOT" "$OUT/runtime-snapshot.mjs" sha256sum "$OUT/runtime-snapshot.mjs" > "$OUT/source.sha256" @@ -41,7 +41,7 @@ docker inspect ain-cert-ainize-node-1 --format '{{json .HostConfig}}' > "$OUT/li for attempt in {1..60}; do if curl --fail --silent --max-time 5 http://localhost:3410/readyz > "$OUT/ready.json"; then node -e 'const value=JSON.parse(require("fs").readFileSync(process.argv[1])); if (value.ready !== true && value.ok !== true) throw Error("not backend readiness JSON")' "$OUT/ready.json" - bash "$KPI/docker/ainize-cli.sh" teach jobs --json > "$OUT/jobs-after.json" + node "$OUT/runtime-snapshot.mjs" jobs "$KPI/ainize/home-docker" "$OUT/jobs-after.json" curl --fail --silent --show-error --max-time 30 http://localhost:3410/api/info > "$OUT/info-after.json" node "$OUT/runtime-snapshot.mjs" capture "$KPI/ainize/home-docker" "$OUT/jobs-after.json" "$OUT/info-after.json" "$OUT/inventory-after.json" "$TRAINER_ROOT" node "$OUT/runtime-snapshot.mjs" verify "$OUT/inventory-before.json" "$OUT/inventory-after.json" > "$OUT/preservation.json" diff --git a/deploy/runtime-snapshot.mjs b/deploy/runtime-snapshot.mjs index 889b741..9c838f5 100644 --- a/deploy/runtime-snapshot.mjs +++ b/deploy/runtime-snapshot.mjs @@ -24,6 +24,28 @@ export function jobBindings(jobs) { })).sort((left, right) => left.id.localeCompare(right.id)); } +export async function operatorJobs(home, request = fetch) { + const credentials = path.join(home, 'cli.json'); + assert.equal((await lstat(credentials)).mode & 0o077, 0, 'operator credential file must be private'); + const state = JSON.parse(await readFile(credentials, 'utf8')); + const origin = new URL(state.nodeUrl); + assert.ok(origin.protocol === 'https:' || origin.protocol === 'http:' && ['localhost', '127.0.0.1', '[::1]'].includes(origin.hostname)); + assert.ok(!origin.username && !origin.password && origin.pathname === '/' && !origin.search && !origin.hash, 'origin-only operator URL required'); + assert.ok(typeof state.token === 'string' && state.token, 'operator login required'); + const response = await request(new URL('/api/me/teach/jobs', origin), { headers: { authorization: `Bearer ${state.token}` }, redirect: 'error', signal: AbortSignal.timeout(30_000) }); + assert.ok(response.ok && response.headers.get('content-type')?.includes('application/json'), `operator jobs returned HTTP ${response.status}`); + const chunks = []; + let size = 0; + for await (const chunk of response.body) { + size += chunk.length; + assert.ok(size <= 8 * 1024 ** 2, 'operator job list is too large'); + chunks.push(chunk); + } + const jobs = JSON.parse(Buffer.concat(chunks).toString('utf8')); + assert.ok(Array.isArray(jobs.items) && jobs.items.length < 500, 'operator job list may be truncated'); + return jobs; +} + export async function inventory(home, trainerRoot) { const files = {}; const allowed = trainerRoot ? await realpath(trainerRoot) : null; @@ -67,7 +89,11 @@ export function compareSnapshots(before, after) { async function main() { const [command, ...args] = process.argv.slice(2); - if (command === 'capture') { + if (command === 'jobs') { + const [home, destination] = args; + assert.ok(destination, 'jobs '); + await writeFile(destination, JSON.stringify(await operatorJobs(home), null, 2) + '\n', { flag: 'wx', mode: 0o600 }); + } else if (command === 'capture') { const [home, jobsPath, infoPath, destination, trainerRoot] = args; assert.ok(destination, 'capture [trainer-root]'); const jobs = JSON.parse(await readFile(jobsPath, 'utf8')); diff --git a/deploy/source-refresh.md b/deploy/source-refresh.md index d4aec10..98e199e 100644 --- a/deploy/source-refresh.md +++ b/deploy/source-refresh.md @@ -27,4 +27,4 @@ The script snapshots the job IDs, dataset bindings, patch hashes/checks, publish Only the Ainize API container stops. Its private home and the linked trainer tree are backed up separately under `kpi/secrets/` (0700 directory/0600 archives). Backups are never release assets. After startup, actual JSON readiness, exact target image, every existing data/body hash, unchanged job binding and publisher/model identity are checked. `flashnext` and `flashtrain` container IDs/start times/PIDs must remain identical. An expired readiness observation is a failure to inspect, not a reason for blind restart. Resume the original lifecycle RUN_ID only after preservation checks pass; do not replace its frozen source or submit duplicate jobs. -Offline validation: `ainize_runtime_maintenance_final_tests_20260911` passed55 tests (30 relay/client/watchdog/snapshot +25 cluster/chat guards). The new snapshot tests include linked-body handling and refusal of foreign/mutated state. Test fixtures and a successful local rebuild do not establish public-server deployment, P2P delivery, Live answer quality or all100 dataset evaluations. +Offline validation: `ainize_runtime_maintenance_final_tests_20260911` passed55 tests (30 relay/client/watchdog/snapshot +25 cluster/chat guards). Additional operator enumeration tests are recorded separately. The new snapshot tests include linked-body handling and refusal of foreign/mutated state. Enumeration uses the operator-only `/api/me/teach/jobs`, not the signature-only visitor endpoint or a teaching key's subset. An actual CHECKING job caused the guard to refuse the restart, leaving the same API instance running (`ainize_runtime_preflight_operator_reject_20260911`). An earlier `docker top` call missing its mandatory PID field is retained as a wrapper failure, not a successful guard test. Test fixtures and a successful local rebuild do not establish public-server deployment, P2P delivery, Live answer quality or all100 dataset evaluations. diff --git a/docs/blob-relay.md b/docs/blob-relay.md index 9925b3a..b368975 100644 --- a/docs/blob-relay.md +++ b/docs/blob-relay.md @@ -11,7 +11,11 @@ The receiver needs both the node implementation and the relay configuration fields in [ainize-core PR #3](https://github.com/ainblockchain/ainize-core/pull/3). Do not identify that implementation by a package version alone: on 2026-09-11, core main and this feature branch both called themselves 0.1.2, but only the -feature branch contained `relayBlobs` and `maxRelayBytes`. +feature branch contained `relayBlobs` and `maxRelayBytes`. Later that day main +advanced to core0.1.3 (`695a8ad6`) and node0.1.2 (`20e599a6`); this hardening branch +incorporates those changes. The compatible certification CLI is0.1.1 (`9acc9de`). +The currently observed public `/api/info` still reports0.1.0, build +`2026-09-10T14:15:13.517Z`; source availability is not proof of the running binary. ```bash ainize config set p2p.relayBlobs true diff --git a/test/runtime-snapshot.test.ts b/test/runtime-snapshot.test.ts index 1fe9662..1acb4e7 100644 --- a/test/runtime-snapshot.test.ts +++ b/test/runtime-snapshot.test.ts @@ -4,11 +4,29 @@ import { mkdtemp, mkdir, writeFile, symlink, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import path from 'node:path'; import { test } from 'node:test'; -import { compareSnapshots, inventory, jobBindings, requireIdle } from '../deploy/runtime-snapshot.mjs'; +import { compareSnapshots, inventory, jobBindings, operatorJobs, requireIdle } from '../deploy/runtime-snapshot.mjs'; const info = { runtime: { available: true, applied: [], queue: { running: null, waiting: 0, queued: [], lock: null } } }; const jobs = { items: [{ id: 'job-one', status: 'READY', dataset: { id: 'dataset-one', sha256: 'body-hash' }, result: { sha256: 'patch-hash' }, checks: { executed: true } }] }; +test('maintenance enumerates every operator-visible job, not only one teaching key owner', async () => { + const home = await mkdtemp(path.join(tmpdir(), 'ainize-operator-jobs-')); + try { + await writeFile(path.join(home, 'cli.json'), JSON.stringify({ nodeUrl: 'http://localhost:3410', token: 'fixture-token' }), { mode: 0o600 }); + const allJobs = { items: [...jobs.items, { id: 'another-owner', status: 'TRAINING' }] }; + const result = await operatorJobs(home, async (url, options) => { + assert.equal(url.pathname, '/api/me/teach/jobs'); + assert.equal(options.headers.authorization, 'Bearer fixture-token'); + assert.equal(options.redirect, 'error'); + return new Response(JSON.stringify(allJobs), { headers: { 'content-type': 'application/json' } }); + }); + assert.equal(result.items.length, 2); + assert.throws(() => requireIdle(info, result), /unfinished/); + await assert.rejects(operatorJobs(home, async () => new Response('frontend HTML')), /HTTP/); + await assert.rejects(operatorJobs(home, async () => new Response('{}', { status: 401 })), /401/); + } finally { await rm(home, { recursive: true, force: true }); } +}); + test('maintenance requires terminal jobs and an idle, empty runtime', () => { requireIdle(info, jobs); assert.throws(() => requireIdle(info, { items: [{ ...jobs.items[0], status: 'TRAINING' }] })); From 181d8fb065e3dc4aa01822d94d6ac0d674a0c5f4 Mon Sep 17 00:00:00 2001 From: Haechan Date: Fri, 11 Sep 2026 11:04:45 +0000 Subject: [PATCH 6/9] Record verified local runtime replacement and distinguish protocol version from package version --- docs/blob-relay-evidence-ko.md | 7 +++++++ docs/blob-relay.md | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/docs/blob-relay-evidence-ko.md b/docs/blob-relay-evidence-ko.md index 8a3174e..a2e35ae 100644 --- a/docs/blob-relay-evidence-ko.md +++ b/docs/blob-relay-evidence-ko.md @@ -59,3 +59,10 @@ curl -i -X POST http://127.0.0.1:3400/p2p/blob/f9f665f6fa1a6b37963a4845107c0c0a5 - 최신 호환 이미지(core0.1.3/node0.1.2+보강/CLI0.1.1)에서 **실제 signed multipart 본문**을 다시 보냈다. 10:43:29 www 첫 번째,10:43:31 apex 두 번째 모두 HTML404 `Cannot POST /p2p/blob/...`, accepted=false. 새 앵커/데이터셋이나 학습을 만들지 않았다. 원문은 `kpi/evidence/signed_p2p_offer_r3_20260911/`. - blob0이면 없는 본문의 GET 실패는 설명된다. 그러나 빈 수신 노드도 처리해야 하는 POST 업로드의 HTML `Cannot POST`를 GET의 파일 부재와 같은 증거로 해석하지 않는다. 프록시인지 실행 바이너리인지의 확정은 관리자 내부3400 응답·실행 커밋 확인이 필요하다. - 이미지 전체 빌드/의존성 검사와 회귀55개가 통과했다. API-only 교체 전후 원본·학습 파일/ID 검증 및 링크 대상 별도 비공개 백업을 추가했다. 실행·검증 절차는 `deploy/source-refresh.md`이며 공개 서버에 배포되었다고 보고하지 않는다. + +## 11:03 UTC 로컬 API 교체 완료 + +- 9번째 동일job의 실제 READY를 기다린 뒤 API 컨테이너만 교체했다. 새 인스턴스 시작11:02:46UTC, 이미지 `sha256:da23af1a5c75cfee61bdad8403f39cad31b6bd2860e7af590a6ffbf03f31c4b0`. 학습9개의 ID·dataset·result/checks와 기존 파일846개의 바이트/해시가 일치했다. `flashnext`/`flashtrain`의 컨테이너 ID·시작 시각·PID가 모두 그대로다. 링크 대상 학습 파일도 별도0600 비공개 백업했다. +- 로컬의 본문·인증 없는 POST는 **JSON403 relay_disabled**다. 빈 수신/비활성 수신에도 route 자체는 응답한다. 공개 signed POST의 HTML404와 구분한다. 로컬은 발행자이며 추가 relay 저장소로 임의 개방하지 않았다. +- `/api/info.version`은 npm 버전이 아니라 core의 `VERSION='0.1.0'` 상수다. 새 로컬 바이너리도0.1.0을 표시한다. 따라서 공개0.1.0 표시만으로 미배포를 단정하지 않는다. 공개 build9/10과 로컬 build9/11은 참고하되 실제 POST·실행 이미지/커밋으로 확인한다. +- 후속 회귀 **56/56**(31+25) 및 실제 미종료job 재시작 거부 시험이 통과했다. 증빙 `ainize_runtime_operator_guard_tests_20260911/`, `ainize_runtime_upgrade_20260911/`. 이는 공개 P2P 복제나 Live 성공이 아니다. diff --git a/docs/blob-relay.md b/docs/blob-relay.md index b368975..ecf4b9d 100644 --- a/docs/blob-relay.md +++ b/docs/blob-relay.md @@ -16,6 +16,10 @@ advanced to core0.1.3 (`695a8ad6`) and node0.1.2 (`20e599a6`); this hardening br incorporates those changes. The compatible certification CLI is0.1.1 (`9acc9de`). The currently observed public `/api/info` still reports0.1.0, build `2026-09-10T14:15:13.517Z`; source availability is not proof of the running binary. +The API version is the core `VERSION` constant, still0.1.0 in core0.1.3, not the +npm package version. The refreshed local node also reports0.1.0 with a newer +build stamp and handles the POST. Do not diagnose deployment from that version +string alone; compare the actual POST response, build/image and source identity. ```bash ainize config set p2p.relayBlobs true From b0160c7a96a4e7f3be8eea505f86cd77f5e7988c Mon Sep 17 00:00:00 2001 From: Haechan Date: Fri, 11 Sep 2026 11:58:20 +0000 Subject: [PATCH 7/9] Fix early-response blob upload crashes and replay preserved public originals over isolated P2P --- docs/blob-relay-evidence-ko.md | 11 ++ docs/blob-relay.md | 65 +++++++++++ scripts/replay-public-blobs.mjs | 181 ++++++++++++++++++++++++++++++ scripts/retry-public-blob.mjs | 11 +- scripts/run-public-blob-replay.sh | 93 +++++++++++++++ scripts/test-blob-relay-docker.sh | 2 +- src/blob-upload.ts | 52 +++++++++ src/p2p.ts | 29 ++--- test/blob-upload.test.ts | 93 +++++++++++++++ test/replay-public-blobs.test.ts | 34 ++++++ test/retry-public-blob.test.ts | 23 +++- 11 files changed, 560 insertions(+), 34 deletions(-) create mode 100644 scripts/replay-public-blobs.mjs create mode 100644 scripts/run-public-blob-replay.sh create mode 100644 src/blob-upload.ts create mode 100644 test/blob-upload.test.ts create mode 100644 test/replay-public-blobs.test.ts diff --git a/docs/blob-relay-evidence-ko.md b/docs/blob-relay-evidence-ko.md index a2e35ae..51fc57c 100644 --- a/docs/blob-relay-evidence-ko.md +++ b/docs/blob-relay-evidence-ko.md @@ -66,3 +66,14 @@ curl -i -X POST http://127.0.0.1:3400/p2p/blob/f9f665f6fa1a6b37963a4845107c0c0a5 - 로컬의 본문·인증 없는 POST는 **JSON403 relay_disabled**다. 빈 수신/비활성 수신에도 route 자체는 응답한다. 공개 signed POST의 HTML404와 구분한다. 로컬은 발행자이며 추가 relay 저장소로 임의 개방하지 않았다. - `/api/info.version`은 npm 버전이 아니라 core의 `VERSION='0.1.0'` 상수다. 새 로컬 바이너리도0.1.0을 표시한다. 따라서 공개0.1.0 표시만으로 미배포를 단정하지 않는다. 공개 build9/10과 로컬 build9/11은 참고하되 실제 POST·실행 이미지/커밋으로 확인한다. - 후속 회귀 **56/56**(31+25) 및 실제 미종료job 재시작 거부 시험이 통과했다. 증빙 `ainize_runtime_operator_guard_tests_20260911/`, `ainize_runtime_upgrade_20260911/`. 이는 공개 P2P 복제나 Live 성공이 아니다. + + +## 11:54 UTC 실제 원본의 빈 수신 노드 재현 + +- 공개 네 노드의 blobs0 보고를 부정하지 않는다. 그러나 이 머신에 남은 두 원본의 공개 anchor 서명·작성자·전체 SHA·크기를 다시 확인하고, **원본 파일을 전혀 마운트하지 않은 빈 별도 Ainize 노드**로 실제 P2P 전송했다. 새 학습이나 새 anchor를 만들지 않았다. +- 첫 실행은 두 본문 수신과 해시 재다운로드까지 성공했지만, 첫 중복 전송의 조기200 응답 뒤 Node24.21.0의 file-backed Blob 송신이 `ERR_INVALID_STATE: ReadableStream is already closed`로 종료됐다. 이 실패는 `ainize_original_replay_20260911/`에 그대로 보존했다. +- `src/blob-upload.ts`에 async-generator/Readable multipart 전송을 추가해 P2P와 standalone 복구 클라이언트가 함께 사용하게 했다. 256MiB 파일·4KiB 응답·60초 전체 기한·redirect 미추적·스트림 정리를 유지한다. 중간 native-http 대안의 EPIPE 실패2회도 보존했고, 최종 구현은 회귀62개와 별도40개 반복 시험(큰 파일 조기 응답400회)을 통과했다. +- 수정 이미지 `sha256:f45a08b206bc56a4c97a004e1229e7c64aceac7a17c2b300bf5b14470cae53ea`의 두 번째 실제 실행: 빈 수신0→2건, 본문 POST4회(초회2+중복2)·GET해시4회, 같은 수신 노드 재시작 후 GET해시2회 통과. 앵커2개의 원래 해시 유지, GC 실제 실행에서도2개 보존·삭제0이다. 원본없는 GET404·무인증 POST403·앵커없는 서명 POST JSON404·앵커전파 후 POST200을 각각 구분했다. +- 각 컨테이너는 Docker internal network·공개포트 없음·runc·GPU 미할당·CPU1·RAM/전체 memory+swap1GiB·CPU set0–7·PID128·read-only·tmpfs64MiB다. 수신 노드에 publisher 비밀키/원본은 없고, 재시작 후 확인 클라이언트에도 원본 마운트가 없다. 기존 flashnext/flashtrain/실제 Ainize API의 ID·시작시각·PID는 그대로다. 별도 수신 노드의 실험이지 공개 서버나 운영 API의 교체가 아니다. +- **공개 재전송은 여전히 실패**: 수정 클라이언트로11:54:44 www와11:54:47 apex에 실제 원본을 offer했으나 모두 HTML404 `Cannot POST`, accepted=false다. 관리자 내부3400의 POST 응답/실행 커밋 확인이 필요하며, 프록시와 실행 바이너리 중 원인을 단정하지 않는다. 공개 복제·VERIFIED·Live 성공으로 보고하지 않는다. +- 성공 원문: `kpi/evidence/ainize_original_replay_r2_20260911/summary.json` 및 전체 송수신/재시작 로그. 회귀 `ainize_stream_upload_tests_r3_20260911/`, 반복 `ainize_stream_upload_repeat_r3_20260911/`, 공개 실패 `signed_p2p_offer_stream_20260911/`. 실행법은 `scripts/run-public-blob-replay.sh`와 `docs/blob-relay.md`다. 비밀 홈·키·NPZ는 진단 릴리스에서 제외한다. diff --git a/docs/blob-relay.md b/docs/blob-relay.md index ecf4b9d..d6592ba 100644 --- a/docs/blob-relay.md +++ b/docs/blob-relay.md @@ -90,6 +90,71 @@ after acceptance. It never prints the identity secret or request auth header. The client does not retrain, create an anchor, publish a dataset, or modify the running author node. A missing receiving route remains a failure, not success. +Both `P2P.offerBlob` and the recovery client use the same bounded HTTP(S) +multipart file-stream transport in `src/blob-upload.ts`. A receiver can reply +before consuming the upload (disabled, unknown anchor, or already held). On the +tested Node24.21.0, the former `fetch`/file-backed Blob path terminated the sender +with an uncaught `ERR_INVALID_STATE` during a repeated3.68MB original offer. +The new transport supplies multipart bytes through an async-generator-backed +Node Readable instead of a file-backed Blob, bounds the response to4KiB and the +entire operation to60s, does not follow redirects, and closes file/request streams +on completion or failure. The response still has to finish within the bounds and +pass the caller's receipt checks. It does not suppress arbitrary uncaught +exceptions. An intermediate native-http implementation produced EPIPE failures +in the large early-response regression; those failed attempts are retained, not +counted as a validated implementation. See the [Node stream API](https://nodejs.org/docs/latest-v24.x/api/stream.html#streamreadablefromiterable-options). + +### Isolated replay of existing public originals + +This is a storage/transport experiment, not public-node deployment, independent +attestation, a Live inference test, or evidence that all100 datasets completed. +The fixture manifest contains `{"version":1,"records":[originalSignedAnchor,...]}` +copied from the public ledger, not newly signed replacements. Only already-public +free anchors are accepted; files must have exact signed hashes and sizes. + +```bash +AINIZE_REPLAY_IMAGE=your-locally-built-compatible-image \ + bash scripts/run-public-blob-replay.sh /input/manifest.json /private/publisher-config.json \ + /private/sha-named-originals /evidence/new-run /private/new-receiver-state +``` + +Build the compatible image with `deploy/build-source-refresh.sh`; the image must +contain the new `dist/blob-upload.js` as well as matching core/node code. The +wrapper requires Docker, Bash and host Node, freezes the replay client/manifest, +and records image identity, runtime hashes, network/limits, logs and exit states. +The evidence and private-state directories must not already exist. The receiver +has its own new identity/storage and no mount of publisher keys or source bodies. +Only the sender receives the original config and selected read-only file mounts. +The check client after restart has no source-body mounts. Private state, publisher +config and NPZ bodies must not be included in a diagnostic release archive. + +Containers use an internal Docker network with no published ports, `runc`, no +GPU/devices, one CPU quota each, CPU set0–7 (override `AINIZE_REPLAY_CPUSET`), RAM +and total RAM+swap ceiling1GiB,128PID cap, read-only rootfs,64MiB tmpfs and dropped +capabilities. The receiver cannot reach the active model/trainer: its runtime +endpoints are its own loopback port1, teaching is disabled, and it has no peers +or verifier role. Only this experiment's containers/network are stopped/removed; +the private receiver state is retained outside public evidence. + +The run first demonstrates these distinct states on an empty receiver: + +| Request | Expected response | +|---|---| +| GET missing body | JSON404, blob not held | +| POST without authentication | JSON403, signed offer required | +| Signed POST before the anchor | JSON404, no published public anchor | +| Original anchor gossip then signed original body POST | JSON200, exact hash/size | +| Repeated body POST at the full storage budget | JSON200, already_held=true | + +Then it checks authenticated GET byte hashes, stops/starts the same receiver, +checks both hashes again from a body-less check container, and executes GC with +`allowSoleCopy=true, keepPurchased=false` without losing relayed copies. Original +anchor hashes/counts must not change. Thus a JSON404 for an unknown anchor is a +valid receiver response; HTML `Cannot POST` is a different observed result. An +empty public blob list alone cannot distinguish these conditions or prove that +the original publisher's files are gone. Inspect the private backend response +and actual running commit before deciding whether a proxy or binary is at fault. + Authentication, known published-public anchor, author, storage budget, and in-flight checks happen **before** multipart parsing. The receiver validates exact anchored size, hash and dimensions, bounded ZIP inflation, and existing diff --git a/scripts/replay-public-blobs.mjs b/scripts/replay-public-blobs.mjs new file mode 100644 index 0000000..38bdb95 --- /dev/null +++ b/scripts/replay-public-blobs.mjs @@ -0,0 +1,181 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { existsSync, readFileSync, statSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { defaultConfig, loadConfig, LocalLedger, saveConfig } from '@ainize/core'; +import { authHeader, gcRun, sha256File, startNode } from '../dist/index.js'; +import { uploadBlob } from '../dist/blob-upload.js'; + +const emit = result => process.stdout.write(`${JSON.stringify({ at: new Date().toISOString(), ...result })}\n`); +const origin = 'http://relay:3400'; +const maximumBody = 256 * 1024 ** 2; + +export function validateReplayManifest(manifest) { + assert.equal(manifest?.version, 1, 'unsupported manifest'); + assert.ok(Array.isArray(manifest.records) && manifest.records.length > 0 && manifest.records.length <= 16, 'select 1–16 original anchors'); + const hashes = new Set(); + const ids = new Set(); + for (const record of manifest.records) { + assert.equal(record?.kind, 'anchor'); + assert.ok(LocalLedger.validate(record), 'invalid original anchor signature'); + const anchor = record.body; + assert.equal(anchor.author.toLowerCase(), record.author.toLowerCase(), 'anchor author mismatch'); + assert.ok(typeof anchor.id === 'string' && anchor.id.length > 0 && !ids.has(anchor.id), 'duplicate or empty knowledge ID'); + assert.match(anchor.patch_sha256, /^[0-9a-f]{64}$/); + assert.ok(!hashes.has(anchor.patch_sha256), 'duplicate blob'); + assert.equal(anchor.price, '0', 'only already-public free bodies may be replayed'); + assert.ok(anchor.visibility === undefined || anchor.visibility === 'public', 'nonpublic anchor'); + assert.ok(Number.isSafeInteger(anchor.size_bytes) && anchor.size_bytes > 0 && anchor.size_bytes <= maximumBody, 'invalid body size'); + ids.add(anchor.id); + hashes.add(anchor.patch_sha256); + } + return manifest.records.map(record => record.body); +} + +async function jsonRequest(path, options = {}) { + const response = await fetch(`${origin}${path}`, { ...options, redirect: 'error', signal: AbortSignal.timeout(60_000) }); + const chunks = []; + let size = 0; + for await (const chunk of response.body) { + size += chunk.length; + assert.ok(size <= 1024 ** 2, 'response too large'); + chunks.push(chunk); + } + const raw = Buffer.concat(chunks).toString('utf8'); + let body; + try { body = JSON.parse(raw); } catch { body = raw; } + emit({ stage: 'http', method: options.method ?? 'GET', path, status: response.status, body }); + return { status: response.status, body }; +} + +async function readBack(anchor, identity) { + const sha = anchor.patch_sha256; + const response = await fetch(`${origin}/p2p/blob/${sha}`, { + headers: { 'x-ainize-auth': authHeader(identity, `blob:${sha}`) }, + redirect: 'error', signal: AbortSignal.timeout(60_000), + }); + assert.equal(response.status, 200); + assert.equal(response.headers.get('x-content-sha256'), sha); + const digest = createHash('sha256'); + let size = 0; + for await (const chunk of response.body) { + size += chunk.length; + assert.ok(size <= anchor.size_bytes, 'read-back exceeds signed size'); + digest.update(chunk); + } + const actualSha = digest.digest('hex'); + assert.equal(size, anchor.size_bytes); + assert.equal(actualSha, sha); + emit({ stage: 'read-back', patchId: anchor.id, sha256: actualSha, sizeBytes: size, verified: true }); +} + +async function receiver(anchors) { + const home = '/private/receiver'; + let cfg = loadConfig(home); + if (!cfg) { + cfg = defaultConfig({ home, name: 'isolated-original-body-relay', host: '0.0.0.0', port: 3400, peers: [], roles: ['seller'], ledger: 'local' }); + cfg.runtime = { api: 'http://127.0.0.1:1', hookApi: 'http://127.0.0.1:1' }; + cfg.verifier = { ...cfg.verifier, auto: false }; + cfg.p2p = { relayBlobs: true, maxRelayBytes: anchors.reduce((total, anchor) => total + anchor.size_bytes, 0) }; + saveConfig(cfg, home); + } + assert.equal(cfg.ledger.kind, 'local'); + assert.deepEqual(cfg.peers, []); + assert.deepEqual(cfg.roles, ['seller']); + assert.equal(cfg.runtime.api, 'http://127.0.0.1:1'); + assert.equal(cfg.runtime.hookApi, 'http://127.0.0.1:1'); + const node = await startNode(cfg, { home, quiet: true, serveWeb: false, teachWorker: false }); + const snapshot = async stage => { + const blobs = []; + for (const blob of node.market.blobs.list()) { + const actualSha = await sha256File(blob.path); + assert.equal(actualSha, blob.sha256); + assert.ok(node.market.blobs.isRelayed(blob.sha256), 'missing durable relay retention'); + blobs.push({ sha256: actualSha, sizeBytes: statSync(blob.path).size, relayed: true }); + } + const collected = await gcRun(node.market, { allowSoleCopy: true, keepPurchased: false }); + assert.equal(collected.removed.length, 0); + emit({ stage, receiver: cfg.identity.address, blobs, gcRemoved: collected.removed.length, gcKeptRelayed: collected.kept.relayed, anchors: (await node.ledger.anchors()).map(record => ({ hash: record.hash, id: record.body.id })) }); + }; + await snapshot('receiver-start'); + let stopping = false; + const stop = async () => { + if (stopping) return; + stopping = true; + try { await snapshot('receiver-stop'); await node.stop(); process.exit(0); } + catch (error) { emit({ stage: 'error', error: error.message }); process.exit(1); } + }; + process.on('SIGINT', stop); + process.on('SIGTERM', stop); +} + +async function client(mode, manifest, anchors) { + const cfg = JSON.parse(readFileSync('/private/publisher.json', 'utf8')); + const identity = cfg.identity; + assert.match(identity?.privateKey ?? '', /^[0-9a-fA-F]{64}$/); + for (const anchor of anchors) { + assert.equal(identity.address.toLowerCase(), anchor.author.toLowerCase(), 'only the original author can replay'); + if (mode === 'send') { + const file = `/bodies/${anchor.patch_sha256}.npz`; + assert.ok(existsSync(file), 'original body missing'); + assert.equal(statSync(file).size, anchor.size_bytes); + assert.equal(await sha256File(file), anchor.patch_sha256); + } + } + const before = await jsonRequest('/p2p/blobs'); + assert.equal(before.status, 200); + assert.equal(before.body.blobs.length, mode === 'send' ? 0 : anchors.length); + if (mode === 'send') { + const first = anchors[0]; + const path = `/p2p/blob/${first.patch_sha256}`; + const missing = await jsonRequest(path); + assert.equal(missing.status, 404); + assert.equal(typeof missing.body, 'object'); + const unauthenticated = await jsonRequest(path, { method: 'POST' }); + assert.equal(unauthenticated.status, 403); + assert.equal(typeof unauthenticated.body, 'object'); + const unknown = await jsonRequest(path, { method: 'POST', headers: { 'x-ainize-auth': authHeader(identity, `blob:${first.patch_sha256}`) } }); + assert.equal(unknown.status, 404); + assert.match(unknown.body.error, /anchor/); + const announced = await jsonRequest('/p2p/records', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ records: manifest.records }) }); + assert.equal(announced.status, 200); + assert.equal(announced.body.added, anchors.length); + assert.deepEqual(announced.body.rejected, []); + for (const duplicate of [false, true]) { + for (const anchor of anchors) { + const sha = anchor.patch_sha256; + const path = `/p2p/blob/${sha}`; + const response = await uploadBlob(`${origin}${path}`, sha, `/bodies/${sha}.npz`, authHeader(identity, `blob:${sha}`)); + const receipt = { status: response.status, body: JSON.parse(response.body) }; + emit({ stage: 'http', method: 'POST', path, ...receipt }); + assert.equal(receipt.status, 200); + assert.deepEqual(receipt.body, { ok: true, sha256: sha, size_bytes: anchor.size_bytes, already_held: duplicate }); + await readBack(anchor, identity); + } + } + } else { + for (const anchor of anchors) await readBack(anchor, identity); + } + const after = await jsonRequest('/p2p/blobs'); + assert.equal(after.status, 200); + assert.deepEqual(after.body.blobs.map(blob => blob.sha256).sort(), anchors.map(anchor => anchor.patch_sha256).sort()); + const records = await jsonRequest('/p2p/records?since=0&limit=100'); + assert.equal(records.status, 200); + const finalAnchors = records.body.records.filter(record => record.kind === 'anchor'); + assert.deepEqual(finalAnchors.map(record => record.hash).sort(), manifest.records.map(record => record.hash).sort()); + emit({ stage: 'complete', mode, verifiedBodies: anchors.length, anchorHashesUnchanged: true, publicDelivery: false, liveInference: false }); +} + +export async function main() { + const mode = process.argv[2]; + assert.ok(['receiver', 'send', 'check'].includes(mode), 'usage: replay-public-blobs.mjs receiver|send|check'); + const manifest = JSON.parse(readFileSync('/input/manifest.json', 'utf8')); + const anchors = validateReplayManifest(manifest); + if (mode === 'receiver') await receiver(anchors); + else await client(mode, manifest, anchors); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + main().catch(error => { emit({ stage: 'error', error: error.message }); process.exitCode = 1; }); +} diff --git a/scripts/retry-public-blob.mjs b/scripts/retry-public-blob.mjs index 558745f..3f26411 100644 --- a/scripts/retry-public-blob.mjs +++ b/scripts/retry-public-blob.mjs @@ -1,6 +1,7 @@ import { createHash } from 'node:crypto'; -import { createReadStream, openAsBlob, readFileSync, statSync } from 'node:fs'; +import { createReadStream, readFileSync, statSync } from 'node:fs'; import { LocalLedger, signMessage } from '@ainize/core'; +import { uploadBlob } from '../dist/blob-upload.js'; const [configPath, filePath, patchId, peer = 'https://www.ainize.ai'] = process.argv.slice(2); const emit = result => process.stdout.write(`${JSON.stringify({ at: new Date().toISOString(), ...result })}\n`); @@ -42,14 +43,12 @@ async function main() { if (!record || !LocalLedger.validate(record) || !identityMatches(record.author) || !identityMatches(record.body.author)) throw new Error('matching author-signed public anchor not found in the first 500 peer records'); if (record.body.visibility === 'test' || record.body.price !== '0' || record.body.size_bytes !== size) throw new Error('this recovery tool only offers exact-size, already-published free public knowledge'); emit({ stage: 'local-body-verified', patchId, author: identity.address, sha256: sha, sizeBytes: size, anchorHash: record.hash, anchorSignatureValid: true }); - const form = new FormData(); - form.append('blob', await openAsBlob(filePath), `${sha}.npz`); const endpoint = `${origin.origin}/p2p/blob/${sha}`; - const response = await fetch(endpoint, { method: 'POST', body: form, headers: headers(), redirect: 'error', signal: AbortSignal.timeout(60_000) }); - const raw = (await boundedBody(response, 4096)).toString('utf8'); + const response = await uploadBlob(endpoint, sha, filePath, headers()['x-ainize-auth']); + const raw = response.body; let receipt; try { receipt = JSON.parse(raw); } catch { receipt = raw; } - const accepted = response.ok && receipt?.ok === true && receipt.sha256 === sha && (receipt.size_bytes === size || (receipt.already_held === true && receipt.size_bytes === undefined)); + const accepted = response.status >= 200 && response.status < 300 && response.contentType.includes('application/json') && receipt?.ok === true && receipt.sha256 === sha && (receipt.size_bytes === size || (receipt.already_held === true && receipt.size_bytes === undefined)); emit({ stage: 'signed-p2p-offer', endpoint, method: 'POST', status: response.status, bytesOffered: size, receipt, accepted }); if (!accepted) { process.exitCode = 1; return; } const download = await fetch(endpoint, { headers: headers(), redirect: 'error', signal: AbortSignal.timeout(60_000) }); diff --git a/scripts/run-public-blob-replay.sh b/scripts/run-public-blob-replay.sh new file mode 100644 index 0000000..04190fe --- /dev/null +++ b/scripts/run-public-blob-replay.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 +root=$(cd "$(dirname "$0")/.." && pwd) +manifest=$(realpath "${1:?Pass a manifest containing only the original signed public anchors}") +config=$(realpath "${2:?Pass the original publisher config (private, read-only)}") +bodies=$(realpath "${3:?Pass a directory containing SHA-named original NPZ files}") +evidence=${4:?Pass a new evidence directory} +private=${5:?Pass a new private receiver-state directory outside evidence} +image=${AINIZE_REPLAY_IMAGE:?Set a source-built image containing dist/blob-upload.js} +mkdir "$evidence" +mkdir "$private" +evidence=$(realpath "$evidence") +private=$(realpath "$private") +case "$private/" in "$evidence/"*) echo 'Private state must be outside evidence' >&2; exit 1;; esac +case "$evidence/" in "$private/"*) echo 'Evidence must be outside private state' >&2; exit 1;; esac +name="ain-cert-original-relay-$(date -u +%Y%m%dT%H%M%S)-$$" +receiver="$name-receiver" +network="$name-network" +containers=() +cleanup() { + result=$? + trap - EXIT + for container in "${containers[@]}"; do + docker stop -t 30 "$container" >/dev/null 2>&1 || true + docker logs "$container" > "$evidence/$container.log" 2>&1 || true + docker inspect "$container" --format '{{json .State}}' > "$evidence/$container-state.json" 2>/dev/null || true + docker rm "$container" >/dev/null 2>&1 || true + done + docker network rm "$network" >/dev/null 2>&1 || true + printf '%s\n' "$result" > "$evidence/exit-code.txt" + exit "$result" +} +trap cleanup EXIT +cp "$root/scripts/replay-public-blobs.mjs" "$evidence/" +cp "$root/scripts/run-public-blob-replay.sh" "$evidence/" +cp "$manifest" "$evidence/manifest.json" +git -C "$root" rev-parse HEAD > "$evidence/node-base-commit.txt" +docker image inspect "$image" --format '{{.Id}}' > "$evidence/image-id.txt" +(cd "$evidence" && sha256sum replay-public-blobs.mjs run-public-blob-replay.sh manifest.json) > "$evidence/source.sha256" +docker network create --internal "$network" > "$evidence/network-id.txt" +docker network inspect "$network" > "$evidence/network.json" +common=(--runtime runc --network "$network" --cpus 1 --cpuset-cpus "${AINIZE_REPLAY_CPUSET:-0-7}" --memory 1g --memory-swap 1g + --pids-limit 128 --read-only --cap-drop ALL --security-opt no-new-privileges --user "$(id -u):$(id -g)" + --tmpfs /tmp:rw,size=64m --env NVIDIA_VISIBLE_DEVICES=void --entrypoint node + --mount "type=bind,src=$evidence/replay-public-blobs.mjs,dst=/opt/ainize/ainize-node/scripts/replay-public-blobs.mjs,readonly" + --mount "type=bind,src=$evidence/manifest.json,dst=/input/manifest.json,readonly") +script=/opt/ainize/ainize-node/scripts/replay-public-blobs.mjs +docker create --name "$receiver" --network-alias relay "${common[@]}" \ + --mount "type=bind,src=$private,dst=/private/receiver" "$image" "$script" receiver > "$evidence/receiver-id.txt" +containers+=("$receiver") +docker inspect "$receiver" --format '{{json .HostConfig}}' > "$evidence/receiver-limits.json" +docker start "$receiver" >/dev/null +wait_ready() { + for attempt in $(seq 1 60); do + if docker exec "$receiver" node -e 'fetch("http://127.0.0.1:3400/healthz").then(response=>process.exit(response.ok?0:1)).catch(()=>process.exit(1))' >/dev/null 2>&1; then return; fi + sleep 1 + done + echo 'Isolated receiver failed readiness' >&2 + return 1 +} +wait_ready +docker exec "$receiver" sh -c 'cd /opt/ainize; find ainize-core/dist ainize-node/dist -type f -print0 | sort -z | xargs -0 sha256sum' > "$evidence/runtime.sha256" +mounts=() +while IFS= read -r sha; do + [[ "$sha" =~ ^[0-9a-f]{64}$ ]] || { echo 'Invalid manifest SHA' >&2; exit 1; } + body=$(realpath "$bodies/$sha.npz") + mounts+=(--mount "type=bind,src=$body,dst=/bodies/$sha.npz,readonly") +done < <(node -e 'for(const record of JSON.parse(require("fs").readFileSync(process.argv[1],"utf8")).records) console.log(record.body.patch_sha256)' "$manifest") +run_client() { + mode=$1 + container="$name-$mode" + body_mounts=() + if [ "$mode" = send ]; then body_mounts=("${mounts[@]}"); fi + docker create --name "$container" "${common[@]}" \ + --mount "type=bind,src=$config,dst=/private/publisher.json,readonly" "${body_mounts[@]}" \ + "$image" "$script" "$mode" > "$evidence/$mode-id.txt" + containers+=("$container") + docker inspect "$container" --format '{{json .HostConfig}}' > "$evidence/$mode-limits.json" + timeout 180 docker start -a "$container" > "$evidence/$mode.log" 2>&1 + docker inspect "$container" --format '{{json .State}}' > "$evidence/$mode-state.json" +} +run_client send +docker stop -t 30 "$receiver" >/dev/null +docker inspect "$receiver" --format '{{json .State}}' > "$evidence/receiver-before-restart-state.json" +test "$(docker inspect "$receiver" --format '{{.State.ExitCode}}')" = 0 +docker start "$receiver" >/dev/null +wait_ready +docker inspect "$receiver" --format '{{json .State}}' > "$evidence/receiver-after-restart-state.json" +run_client check +docker stop -t 30 "$receiver" >/dev/null +test "$(docker inspect "$receiver" --format '{{.State.ExitCode}}')" = 0 +printf 'Original-body replay passed; this is isolated P2P transfer, not public delivery or Live inference.\n' diff --git a/scripts/test-blob-relay-docker.sh b/scripts/test-blob-relay-docker.sh index 5b6fa63..bfe682d 100644 --- a/scripts/test-blob-relay-docker.sh +++ b/scripts/test-blob-relay-docker.sh @@ -38,7 +38,7 @@ cd /tmp/ainize-core npm run build cd /tmp/ainize-node npm run build -node --test --import tsx test/blob-relay.test.ts test/retry-public-blob.test.ts test/watchdog-snapshot.test.ts test/runtime-snapshot.test.ts +node --test --import tsx test/blob-upload.test.ts test/blob-relay.test.ts test/retry-public-blob.test.ts test/replay-public-blobs.test.ts test/watchdog-snapshot.test.ts test/runtime-snapshot.test.ts node --test --import tsx test/guard-api.test.ts test/cluster.test.ts ' > "$evidence/container-id.txt" docker inspect "$name" --format '{{json .HostConfig}}' > "$evidence/host-config.json" diff --git a/src/blob-upload.ts b/src/blob-upload.ts new file mode 100644 index 0000000..c699a1c --- /dev/null +++ b/src/blob-upload.ts @@ -0,0 +1,52 @@ +import { randomUUID } from 'node:crypto'; +import { createReadStream, statSync } from 'node:fs'; +import { Readable } from 'node:stream'; + +export interface BlobUploadResponse { + status: number; + contentType: string; + body: string; +} + +export async function uploadBlob(endpoint: string, sha: string, path: string, authorization: string, timeoutMs = 60_000): Promise { + const url = new URL(endpoint); + if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password || url.hash) throw new Error('invalid blob upload URL'); + if (!/^[0-9a-f]{64}$/.test(sha)) throw new Error('invalid blob SHA'); + const file = statSync(path); + if (!file.isFile() || !Number.isSafeInteger(file.size) || file.size <= 0 || file.size > 256 * 1024 ** 2) throw new Error('blob exceeds upload size bounds'); + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 60_000) throw new Error('invalid blob upload deadline'); + const boundary = `ainize-${randomUUID()}`; + const prefix = Buffer.from(`--${boundary}\r\nContent-Disposition: form-data; name="blob"; filename="${sha}.npz"\r\nContent-Type: application/octet-stream\r\n\r\n`); + const suffix = Buffer.from(`\r\n--${boundary}--\r\n`); + const source = createReadStream(path, { end: file.size - 1 }); + const body = Readable.from((async function* () { + yield prefix; + for await (const chunk of source) yield chunk; + yield suffix; + })()); + const controller = new AbortController(); + const deadline = setTimeout(() => controller.abort(new Error('blob upload deadline exceeded')), timeoutMs); + try { + const response = await fetch(url, { + method: 'POST', body: body as unknown as NonNullable, duplex: 'half', redirect: 'manual', signal: controller.signal, + headers: { 'content-type': `multipart/form-data; boundary=${boundary}`, 'content-length': String(prefix.length + file.size + suffix.length), 'x-ainize-auth': authorization }, + } as RequestInit & { duplex: 'half' }); + const chunks: Uint8Array[] = []; + let size = 0; + if (!response.body) throw new Error('relay acknowledgment is empty'); + for await (const chunk of response.body) { + size += chunk.length; + if (size > 4096) throw new Error('relay acknowledgment exceeds the limit'); + chunks.push(chunk); + } + return { status: response.status, contentType: response.headers.get('content-type') ?? '', body: Buffer.concat(chunks).toString('utf8') }; + } catch (error) { + if (controller.signal.aborted) throw controller.signal.reason; + throw error; + } finally { + clearTimeout(deadline); + controller.abort(); + body.destroy(); + source.destroy(); + } +} diff --git a/src/p2p.ts b/src/p2p.ts index 678893a..21edc7b 100644 --- a/src/p2p.ts +++ b/src/p2p.ts @@ -3,13 +3,14 @@ * (local-ledger mode: set reconciliation by `received_at` cursor + push on new record), * blob availability and authenticated blob fetch. */ -import { createWriteStream, mkdirSync, renameSync, openAsBlob } from 'node:fs'; +import { createWriteStream, mkdirSync, renameSync, statSync } from 'node:fs'; import { dirname } from 'node:path'; import { pipeline } from 'node:stream/promises'; import { Readable } from 'node:stream'; import { signMessage, verifyMessage, type LedgerRecord, type PeerInfo, type Identity, type Ledger, isRecordRefusal } from '@ainize/core'; import type { Store } from './store.js'; import { sha256File } from './blobs.js'; +import { uploadBlob } from './blob-upload.js'; export interface P2PDeps { identity: Identity; @@ -392,32 +393,18 @@ export class P2P { */ async offerBlob(sha: string, path: string, endpoints = this.peers().filter(peer => peer.source === 'configured').map(peer => peer.endpoint)): Promise { const accepted: string[] = []; - let body: Blob; + let size: number; try { if (!/^[0-9a-f]{64}$/.test(sha) || await sha256File(path) !== sha) return accepted; - body = await openAsBlob(path); + size = statSync(path).size; } catch { return accepted; } for (const endpoint of new Set(endpoints.map(value => this.normalize(value)))) { if (endpoint === this.normalize(this.selfEndpoint)) continue; try { - const form = new FormData(); - form.append('blob', body, `${sha}.npz`); - const response = await fetch(`${endpoint}/p2p/blob/${sha}`, { - method: 'POST', body: form, - headers: { 'x-ainize-auth': authHeader(this.deps.identity, `blob:${sha}`) }, - signal: AbortSignal.timeout(60_000), redirect: 'error', - }); - if (!response.ok || !response.headers.get('content-type')?.includes('application/json')) { await response.body?.cancel(); continue; } - const chunks: Uint8Array[] = []; - let bytes = 0; - if (!response.body) continue; - for await (const chunk of response.body) { - bytes += chunk.length; - if (bytes > 4096) throw new Error('relay acknowledgment exceeds the limit'); - chunks.push(chunk); - } - const receipt = JSON.parse(Buffer.concat(chunks).toString('utf8')) as Record; - if (receipt.ok === true && receipt.sha256 === sha && (receipt.size_bytes === body.size || (receipt.already_held === true && receipt.size_bytes === undefined))) accepted.push(endpoint); + const response = await uploadBlob(`${endpoint}/p2p/blob/${sha}`, sha, path, authHeader(this.deps.identity, `blob:${sha}`)); + if (response.status < 200 || response.status >= 300 || !response.contentType.includes('application/json')) continue; + const receipt = JSON.parse(response.body) as Record; + if (receipt.ok === true && receipt.sha256 === sha && (receipt.size_bytes === size || (receipt.already_held === true && receipt.size_bytes === undefined))) accepted.push(endpoint); } catch { /* a peer that will not hold it is not a publish failure */ } } return accepted; diff --git a/test/blob-upload.test.ts b/test/blob-upload.test.ts new file mode 100644 index 0000000..e668e21 --- /dev/null +++ b/test/blob-upload.test.ts @@ -0,0 +1,93 @@ +import { test, type TestContext } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync, truncateSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { createServer, type RequestListener } from 'node:http'; +import { uploadBlob } from '../src/blob-upload.js'; + +async function fixture(context: TestContext, handler: RequestListener) { + const directory = mkdtempSync(join(tmpdir(), 'ainize-upload-')); + const file = join(directory, 'body.npz'); + writeFileSync(file, ''); + truncateSync(file, 32 * 1024 ** 2); + const server = createServer(handler); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + context.after(async () => { + server.closeAllConnections(); + await new Promise(resolve => server.close(() => resolve())); + rmSync(directory, { recursive: true, force: true }); + }); + return { file, url: `http://127.0.0.1:${(server.address() as { port: number }).port}/p2p/blob/${'a'.repeat(64)}` }; +} + +test('streamed upload encodes exactly one file with bounded chunks and a correct content length', async context => { + const bytes = Buffer.alloc(2 * 1024 ** 2, 7); + let received = 0; + const setup = await fixture(context, async (request, response) => { + assert.equal(request.headers['x-ainize-auth'], 'test-authorization'); + const chunks: Buffer[] = []; + for await (const chunk of request) { received += chunk.length; chunks.push(chunk); } + const body = Buffer.concat(chunks); + assert.equal(received, Number(request.headers['content-length'])); + const boundary = request.headers['content-type']!.split('boundary=')[1]; + const prefix = Buffer.from(`--${boundary}\r\nContent-Disposition: form-data; name="blob"; filename="${'a'.repeat(64)}.npz"\r\nContent-Type: application/octet-stream\r\n\r\n`); + const suffix = Buffer.from(`\r\n--${boundary}--\r\n`); + assert.deepEqual(body, Buffer.concat([prefix, bytes, suffix])); + response.setHeader('content-type', 'application/json'); + response.end('{"ok":true}'); + }); + writeFileSync(setup.file, bytes); + assert.deepEqual(await uploadBlob(setup.url, 'a'.repeat(64), setup.file, 'test-authorization'), { status: 200, contentType: 'application/json', body: '{"ok":true}' }); + assert.ok(received > bytes.length); +}); + +test('large files survive repeated early acknowledgments, refusals and redirects without following them', async context => { + let mode = 200; + let requests = 0; + const setup = await fixture(context, (_request, response) => { + requests++; + response.writeHead(mode, { 'content-type': mode === 404 ? 'text/html' : 'application/json', location: '/unexpected' }); + response.end(mode === 404 ? 'Cannot POST' : '{"already_held":true}'); + }); + for (mode of [200, 403, 404, 307]) { + for (let attempt = 0; attempt < 10; attempt++) { + const response = await uploadBlob(setup.url, 'a'.repeat(64), setup.file, 'test-authorization'); + assert.equal(response.status, mode); + } + } + await new Promise(resolve => setTimeout(resolve, 100)); + assert.equal(requests, 40); +}); + +test('oversized acknowledgments, broken sockets and stalled responses fail within bounds', async context => { + let mode = 'oversized'; + const setup = await fixture(context, (request, response) => { + if (mode === 'oversized') { response.end('x'.repeat(5000)); return; } + if (mode === 'broken') { request.socket.destroy(); return; } + response.writeHead(200); + const timer = setInterval(() => response.write(' '), 10); + response.once('close', () => clearInterval(timer)); + }); + await assert.rejects(uploadBlob(setup.url, 'a'.repeat(64), setup.file, 'test', 2000), /acknowledgment exceeds/); + mode = 'broken'; + await assert.rejects(uploadBlob(setup.url, 'a'.repeat(64), setup.file, 'test', 2000)); + mode = 'stalled'; + await assert.rejects(uploadBlob(setup.url, 'a'.repeat(64), setup.file, 'test', 100), /deadline/); +}); + +test('invalid destinations, file sizes, hashes and deadlines are refused before HTTP', async context => { + let requests = 0; + const setup = await fixture(context, (_request, response) => { requests++; response.end('{}'); }); + for (const destination of ['file:///private/config.json', setup.url.replace('http://', 'http://user:password@'), `${setup.url}#fragment`]) { + await assert.rejects(uploadBlob(destination, 'a'.repeat(64), setup.file, 'test')); + } + await assert.rejects(uploadBlob(setup.url, '../bad', setup.file, 'test')); + await assert.rejects(uploadBlob(setup.url, 'a'.repeat(64), `${setup.file}.missing`, 'test')); + for (const deadline of [0, 60_001, NaN]) await assert.rejects(uploadBlob(setup.url, 'a'.repeat(64), setup.file, 'test', deadline)); + for (const size of [0, 256 * 1024 ** 2 + 1]) { + truncateSync(setup.file, size); + await assert.rejects(uploadBlob(setup.url, 'a'.repeat(64), setup.file, 'test')); + } + assert.equal(requests, 0); +}); diff --git a/test/replay-public-blobs.test.ts b/test/replay-public-blobs.test.ts new file mode 100644 index 0000000..073b6ff --- /dev/null +++ b/test/replay-public-blobs.test.ts @@ -0,0 +1,34 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { createIdentity, LocalLedger } from '@ainize/core'; + +const { validateReplayManifest } = await import('../scripts/replay-public-blobs.mjs'); + +test('original-body replay accepts original signatures without modifying their records', async context => { + const identity = createIdentity(); + const ledger = new LocalLedger(':memory:', identity); + await ledger.init(); + context.after(() => ledger.close()); + const record = await ledger.append('anchor', { id: 'original', author: identity.address, price: '0', patch_sha256: 'a'.repeat(64), size_bytes: 64 }); + const manifest = { version: 1, records: [record] }; + const before = JSON.stringify(manifest); + assert.deepEqual(validateReplayManifest(manifest), [record.body]); + assert.equal(JSON.stringify(manifest), before); +}); + +test('original-body replay rejects nonpublic, paid, duplicate, forged and oversized anchors', async context => { + const identity = createIdentity(); + const ledger = new LocalLedger(':memory:', identity); + await ledger.init(); + context.after(() => ledger.close()); + const anchor = { id: 'original', author: identity.address, price: '0', patch_sha256: 'a'.repeat(64), size_bytes: 64 }; + for (const changes of [{ visibility: 'test' }, { visibility: 'private' }, { price: '1' }, { author: createIdentity().address }, { size_bytes: 0 }, { size_bytes: 256 * 1024 ** 2 + 1 }, { patch_sha256: '../escape' }]) { + const record = await ledger.append('anchor', { ...anchor, ...changes }); + assert.throws(() => validateReplayManifest({ version: 1, records: [record] })); + } + const record = await ledger.append('anchor', anchor); + assert.throws(() => validateReplayManifest({ version: 1, records: [record, record] })); + assert.throws(() => validateReplayManifest({ version: 1, records: [{ ...record, hash: '0'.repeat(64) }] })); + assert.throws(() => validateReplayManifest({ version: 1, records: [] })); + assert.throws(() => validateReplayManifest({ version: 2, records: [record] })); +}); diff --git a/test/retry-public-blob.test.ts b/test/retry-public-blob.test.ts index 1585231..48e549f 100644 --- a/test/retry-public-blob.test.ts +++ b/test/retry-public-blob.test.ts @@ -29,9 +29,25 @@ for (const mode of ['success', 'html-404', 'html-200', 'wrong-download', 'tamper import assert from 'node:assert/strict'; import {readFileSync} from 'node:fs'; import {createHash} from 'node:crypto'; +import {registerHooks} from 'node:module'; import {verifyMessage} from ${JSON.stringify(pathToFileURL(resolve('node_modules/@ainize/core/dist/index.js')).href)}; const bytes = readFileSync(${JSON.stringify(filePath)}); const mode = ${JSON.stringify(mode)}; +globalThis.testUpload = async (url, sha, file, authorization) => { + assert.ok(!['paid-anchor','tampered-anchor'].includes(mode), 'invalid anchor must not send a body'); + assert.equal(url, 'https://relay.invalid/p2p/blob/${sha}'); + assert.equal(sha, '${sha}'); + assert.deepEqual(readFileSync(file), bytes); + const [address, timestamp, signature] = authorization.split(':'); + assert.equal(address, ${JSON.stringify(identity.address)}); + assert.ok(verifyMessage('blob:${sha}:' + timestamp, signature, address)); + if (mode.startsWith('html-')) return {status:mode === 'html-404' ? 404 : 200,contentType:'text/html',body:'Cannot POST /p2p/blob'}; + return {status:200,contentType:'application/json',body:JSON.stringify({ok:true,sha256:sha,size_bytes:bytes.length,already_held:false})}; +}; +registerHooks({load(url, context, nextLoad) { + if (url === ${JSON.stringify(pathToFileURL(resolve('dist/blob-upload.js')).href)}) return {format:'module',shortCircuit:true,source:'export const uploadBlob = (...args) => globalThis.testUpload(...args);'}; + return nextLoad(url, context); +}}); globalThis.fetch = async (url, options) => { assert.equal(options.redirect, 'error'); if (String(url).includes('/p2p/records?')) return Response.json({records: [${JSON.stringify(record)}]}); @@ -39,12 +55,7 @@ globalThis.fetch = async (url, options) => { const [address, timestamp, signature] = options.headers['x-ainize-auth'].split(':'); assert.equal(address, ${JSON.stringify(identity.address)}); assert.ok(verifyMessage('blob:${sha}:' + timestamp, signature, address)); - if (options.method === 'POST') { - const uploaded = Buffer.from(await options.body.get('blob').arrayBuffer()); - assert.deepEqual(uploaded, bytes); - if (mode.startsWith('html-')) return new Response('Cannot POST /p2p/blob', {status: mode === 'html-404' ? 404 : 200}); - return Response.json({ok:true,sha256:${JSON.stringify(sha)},size_bytes:bytes.length,already_held:false}); - } + assert.notEqual(options.method, 'POST'); return new Response(mode === 'wrong-download' ? Buffer.alloc(bytes.length) : bytes); }; process.argv = [process.execPath, 'retry-public-blob.mjs', ${JSON.stringify(configPath)}, ${JSON.stringify(filePath)}, 'recovery-test', 'https://relay.invalid']; From ad51002a6b0358ec751904fbe35f266644dd7a4c Mon Sep 17 00:00:00 2001 From: Haechan Date: Sat, 12 Sep 2026 11:47:50 +0000 Subject: [PATCH 8/9] Document verified public blob relay and add repeatable Live observations --- docs/public-relay-live-20260912.md | 71 +++++++++++++++++++++++++ scripts/observe-public-relay.mjs | 59 ++++++++++++++++++++ scripts/retry-preserved-public-blobs.sh | 39 ++++++++++++++ 3 files changed, 169 insertions(+) create mode 100644 docs/public-relay-live-20260912.md create mode 100644 scripts/observe-public-relay.mjs create mode 100644 scripts/retry-preserved-public-blobs.sh diff --git a/docs/public-relay-live-20260912.md b/docs/public-relay-live-20260912.md new file mode 100644 index 0000000..9f0a9f4 --- /dev/null +++ b/docs/public-relay-live-20260912.md @@ -0,0 +1,71 @@ +# 공개 P2P 본문 및 Live test 재현 — 2026-09-12 + +원래 공개한 DART 지식2개를 보존된 파일에서 다시 전송했다. 공개 peer가 +HTTP200으로 수신했고, 작성자 서명으로 다시 내려받은 전체SHA/바이트가 일치했다. +공개 Live compare도2건 HTTP200과 적용 전/후 답변을 반환했다. +원문 성능지표 전체 통과 수는0/6이며 이 결과는 공개 본문/추론 경로의 기능 실증이다. + +| 항목 | 대표자명 | 소재지 | +|---|---|---| +| knowledge ID | taught-ainize-teach-first-20260-855df1 | taught-ainize-lifecycle100-2026-cf9a6f | +| SHA256 | f9f665f6fa1a6b37963a4845107c0c0a5d3b970bcd2af6e8f40938a0fbdf7acc | fb1cd41e2f6a26f785d72460a2eac4a62688ee4c70e5bee43187d734eeca2e64 | +| 바이트 | 3,679,278 | 3,719,206 | +| 수신 POST / 인증 다운로드 | 200 / SHA 일치 | 200 / SHA 일치 | +| 익명 GET | 402: 다운로드 권한 요구 | 402: 다운로드 권한 요구 | +| Live compare | 200, 적용 답변 조원국 | 200, 적용 답변 경기도 남양주시 별내3로 391 | +| 공개 검증 | 6/8, REJECTED | 0/8, REJECTED | + +## 이 실험 폴더에서 재실행 + +`/mnt/newdata/gov`의 원래 publisher identity, 공개 anchor와 일치하는 NPZ, +아래에 고정된 로컬 Docker 이미지가 필요하다. 전송 도구는 공개 anchor의 작성자 +서명·가격0·크기·SHA를 먼저 검사한다. 이미 공개된 동일 본문만 재전송한다. + +```bash +cd /mnt/newdata/gov +run_id="relay-$(date -u +%Y%m%dT%H%M%SZ | tr '[:upper:]' '[:lower:]')" +output="$PWD/kpi/evidence/$run_id" +bash kpi/pr/an-relay/scripts/retry-preserved-public-blobs.sh "$output" "$run_id" +``` + +Docker CPU1/cpuset0–7/RAM·swap합계512MiB/read-only/no GPU, 현재 파일 소유자의 +UID/GID를 쓴다. 키와 원본은 읽기 전용이다. 각 POST의 JSON ACK와 실제 파일 +재다운로드 해시를 기록하며, 하나라도 실패하면 exit1이다. 중복 수신은 +`already_held:true`일 수 있으며 동일SHA 재다운로드까지 확인한다. + +공개 카탈로그·익명 GET·Live를 관측하는 명령이다. 두 번의 공개 Live quota를 쓴다. + +```bash +docker run --name "ain-cert-observe-$run_id" --user "$(id -u):$(id -g)" \ + --runtime runc --cpus 1 --cpuset-cpus 0-7 --memory 512m --memory-swap 512m \ + --read-only --pids-limit 128 --cap-drop ALL --security-opt no-new-privileges \ + -e NVIDIA_VISIBLE_DEVICES=void \ + --mount "type=bind,src=$PWD/kpi/pr/an-relay/scripts/observe-public-relay.mjs,dst=/observe.mjs,readonly" \ + --mount "type=bind,src=$output,dst=/evidence" \ + sha256:f45a08b206bc56a4c97a004e1229e7c64aceac7a17c2b300bf5b14470cae53ea \ + /observe.mjs https://www.ainize.ai /evidence/live \ + taught-ainize-lifecycle100-2026-cf9a6f taught-ainize-teach-first-20260-855df1 +``` + +관측 도구의 exit0은 응답 기록 완료다. `summary.json`의 HTTP 상태 및 +`*-chat-response.json`의 `base`/`patched` 실제 답변, 카탈로그의 검증 판정을 함께 본다. +모델이 반환한 기업 정보는 시험 응답이며 사실 확인 자료로 사용하지 않는다. + +## 설치·검증 상태 + +npm 조회에서 CLI `ainize@0.1.3`와 `@ainize/node@0.1.4` 게시를 확인했다. +이번 본문 복구는 이전에 시험한 서명 전송 도구로 실행했다. 실행 중인 로컬 +API는 node0.1.2 수정본, CLI0.1.1이며 버전 업그레이드 완료로 보고하지 않는다. +새 노드를 구성할 때는 관리자 안내의 CLI 설치·relay 설정·기동 후 publish 흐름을 +사용하되 실제 설치 버전과 ACK를 기록해야 한다. + +공개 검증은 파일 수신 직후 실제로 실행됐다. 공개 attestation의 잘린 답변과 +`src/runtime.ts`의 verify 생성한도8토큰을 확인했다. 소재지의 첫 정답은 공개 +Live의128토큰 한도에서14토큰으로 완성됐으므로 생성한도 차이가 실패에 기여할 +수 있다. 전체8문항 재평가나 검증정책 변경을 실행한 결과는 아니다. 공개 +`benchmark_hit:false` 등 원래 판정 필드도 보존하며 성공 값으로 덮지 않는다. + +최초 실측: `kpi/evidence/ainize_public_relay_20260912T1140/`. +11:38:46/51 UTC signed POST와 read-back, `live/`의 공개 compare/카탈로그가 원문이다. +첫 관측용 컨테이너는 잘못 고정한UID1000 때문에 실행 전 EACCES로 종료했고, +파일 소유자의UID/GID로 수정해 재실행했다. 기존 학습·모델·체인을 재시작하지 않았다. diff --git a/scripts/observe-public-relay.mjs b/scripts/observe-public-relay.mjs new file mode 100644 index 0000000..91b55fa --- /dev/null +++ b/scripts/observe-public-relay.mjs @@ -0,0 +1,59 @@ +import { createHash } from 'node:crypto'; +import { mkdir, writeFile } from 'node:fs/promises'; +import path from 'node:path'; + +const [peer, destination, ...patchIds] = process.argv.slice(2); + +async function observe(url, options = {}) { + const started = Date.now(); + const response = await fetch(url, { ...options, redirect: 'error', signal: AbortSignal.timeout(60_000) }); + const digest = createHash('sha256'); + const chunks = []; + let bytes = 0; + const json = response.headers.get('content-type')?.includes('application/json'); + for await (const chunk of response.body ?? []) { + bytes += chunk.length; + if (bytes > (json ? 8 * 1024 ** 2 : 256 * 1024 ** 2)) throw new Error('response exceeds observation limit'); + digest.update(chunk); + if (json) chunks.push(chunk); + } + return { + at: new Date().toISOString(), url: String(url), status: response.status, + durationMs: Date.now() - started, contentType: response.headers.get('content-type'), + bytes, sha256: digest.digest('hex'), + ...(json ? { body: JSON.parse(Buffer.concat(chunks).toString('utf8')) } : {}), + }; +} + +async function main() { + if (!peer || !destination || !patchIds.length) throw new Error('usage: node observe-public-relay.mjs [...]'); + const origin = new URL(peer); + if (origin.protocol !== 'https:' || origin.username || origin.password || origin.pathname !== '/' || origin.search || origin.hash) throw new Error('HTTPS origin required'); + await mkdir(destination, { mode: 0o700 }); + const save = async (name, value) => writeFile(path.join(destination, name), JSON.stringify(value, null, 2) + '\n', { flag: 'wx', mode: 0o600 }); + const catalog = await observe(new URL('/api/catalog', origin)); + await save('catalog-before.json', catalog); + if (catalog.status !== 200 || !Array.isArray(catalog.body?.items)) throw new Error('public catalog unavailable'); + const results = []; + for (const patchId of patchIds) { + const entry = catalog.body.items.find(item => item.anchor?.id === patchId); + if (!entry || !/^[0-9a-f]{64}$/.test(entry.anchor.patch_sha256)) throw new Error(`missing catalog entry: ${patchId}`); + const blob = await observe(new URL(`/p2p/blob/${entry.anchor.patch_sha256}`, origin)); + await save(`${results.length}-anonymous-blob.json`, blob); + const sample = entry.anchor.benchmark?.samples?.[0]; + const prompt = sample?.prompt; + if (typeof prompt !== 'string' || !prompt) throw new Error(`benchmark prompt unavailable: ${patchId}`); + const request = { patch_id: patchId, mode: 'compare', messages: [{ role: 'user', content: prompt }], max_tokens: 128 }; + await save(`${results.length}-chat-request.json`, request); + const chat = await observe(new URL('/api/chat', origin), { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(request) }); + await save(`${results.length}-chat-response.json`, chat); + results.push({ patchId, sha256: entry.anchor.patch_sha256, anonymousGet: blob.status, chatStatus: chat.status, catalogStatus: entry.status, passed: entry.passed, quorum: entry.quorum }); + console.log(JSON.stringify(results[results.length - 1])); + } + await save('catalog-after.json', await observe(new URL('/api/catalog', origin))); + await save('chat-patches.json', await observe(new URL('/api/chat/patches', origin))); + await save('blobs.json', await observe(new URL('/p2p/blobs', origin))); + await save('summary.json', { at: new Date().toISOString(), peer: origin.origin, results }); +} + +main().catch(error => { console.error(error.message); process.exitCode = 1; }); diff --git a/scripts/retry-preserved-public-blobs.sh b/scripts/retry-preserved-public-blobs.sh new file mode 100644 index 0000000..8075084 --- /dev/null +++ b/scripts/retry-preserved-public-blobs.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail +root=${GOV_ROOT:-/mnt/newdata/gov} +scripts=$(cd "$(dirname "$0")" && pwd) +output=${1:?Pass a NEW evidence directory} +run_id=${2:?Pass a unique lowercase RUN_ID} +[[ "$run_id" =~ ^[a-z0-9][a-z0-9_-]{0,40}$ ]] || exit 2 +mkdir -m 700 "$output" +output=$(realpath "$output") +image=sha256:f45a08b206bc56a4c97a004e1229e7c64aceac7a17c2b300bf5b14470cae53ea +printf '%s\n' "$image" > "$output/image-id.txt" +date -u +%FT%TZ > "$output/at.txt" +sha256sum "$scripts/retry-public-blob.mjs" "$scripts/retry-preserved-public-blobs.sh" > "$output/source.sha256" +result=0 +for patch in taught-ainize-teach-first-20260-855df1 taught-ainize-lifecycle100-2026-cf9a6f; do + if [[ "$patch" == taught-ainize-teach-first-20260-855df1 ]]; then + sha=f9f665f6fa1a6b37963a4845107c0c0a5d3b970bcd2af6e8f40938a0fbdf7acc + else + sha=fb1cd41e2f6a26f785d72460a2eac4a62688ee4c70e5bee43187d734eeca2e64 + fi + file=$(realpath "$root/kpi/ainize/home-docker/data/drive/patches/$patch/$sha.npz") + name="ain-cert-reoffer-$run_id-${sha:0:8}" + docker create --name "$name" --user "$(id -u):$(id -g)" \ + --runtime runc --cpus 1 --cpuset-cpus 0-7 --memory 512m --memory-swap 512m \ + --read-only --pids-limit 128 --cap-drop ALL --security-opt no-new-privileges \ + -e NVIDIA_VISIBLE_DEVICES=void \ + --mount "type=bind,src=$root/kpi/ainize/home-docker/config.json,dst=/private-config.json,readonly" \ + --mount "type=bind,src=$file,dst=/body.npz,readonly" \ + --mount "type=bind,src=$scripts/retry-public-blob.mjs,dst=/opt/ainize/ainize-node/scripts/retry-public-blob.mjs,readonly" \ + "$image" scripts/retry-public-blob.mjs /private-config.json /body.npz "$patch" https://www.ainize.ai \ + > "$output/$sha-container-id.txt" + docker inspect "$name" --format '{{json .HostConfig}}' > "$output/$sha-limits.json" + exit_code=0 + docker start -a "$name" > "$output/$sha-signed.jsonl" 2>&1 || exit_code=$? + printf '%s\n' "$exit_code" > "$output/$sha-exit-code.txt" + cat "$output/$sha-signed.jsonl" + if (( exit_code != 0 )); then result=1; fi +done +exit "$result" From 954d42ad90dc6b49b7241f3502033e0d36fea812 Mon Sep 17 00:00:00 2001 From: Haechan Date: Sat, 12 Sep 2026 12:10:28 +0000 Subject: [PATCH 9/9] Enforce configurable parallel teach admission with a 70-job regression test --- src/teach.ts | 6 ++-- test/parallel-teach-admission.test.ts | 48 +++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 test/parallel-teach-admission.test.ts diff --git a/src/teach.ts b/src/teach.ts index 0b1694f..8c91517 100644 --- a/src/teach.ts +++ b/src/teach.ts @@ -188,7 +188,7 @@ export interface TeachPolicyView { backend: 'gradient' | 'stub'; queue: { depth: number; max: number; position_eta_s?: number | null; queued_rows: number; queued_rows_max: number }; limits: { - facts_per_job: number; jobs_per_key_per_day: number; jobs_per_ip_per_day: number; prompt_max: number; answer_max: number; + facts_per_job: number; jobs_per_key_per_day: number; jobs_per_ip_per_day: number; active_jobs_per_key: number; prompt_max: number; answer_max: number; dataset_max_bytes: number; dataset_max_rows: number; dataset_max_source_lines: number; rows_per_job: number; rows_per_job_source: 'default' | 'measured' | 'operator'; rows_per_key_per_day: number; rows_per_ip_per_day: number; datasets_per_key_per_day: number; dataset_ttl_days: number; @@ -579,6 +579,7 @@ export class TeachWorker { }, limits: { facts_per_job: c.factsPerJob, jobs_per_key_per_day: c.jobsPerKeyPerDay, jobs_per_ip_per_day: c.jobsPerIpPerDay, prompt_max: PROMPT_MAX, answer_max: ANSWER_MAX, + active_jobs_per_key: c.activeJobsPerKey ?? ACTIVE_JOBS_PER_KEY, dataset_max_bytes: c.dataset.maxBytes, dataset_max_rows: c.dataset.maxRows, dataset_max_source_lines: c.dataset.maxSourceLines, rows_per_job: rows.rows, rows_per_job_source: rows.source, rows_per_key_per_day: c.dataset.rowsPerKeyPerDay, rows_per_ip_per_day: c.dataset.rowsPerIpPerDay, @@ -1298,7 +1299,8 @@ export class TeachWorker { const rowsWaiting = active.reduce((n, j) => n + (j.dataset_rows ?? j.facts.length), 0); if (rowsWaiting >= c.queuedRowsMax) throw new TeachError(503, `trainer_paused: ${rowsWaiting} questions are already waiting on this node — try again later`); const mineActive = active.filter((j) => j.contributor.toLowerCase() === address.toLowerCase()).length; - if (mineActive >= ACTIVE_JOBS_PER_KEY) throw new TeachError(429, `quota_key: you already have ${mineActive} lesson(s) in progress on this node — wait for them to finish`); + const activeLimit = c.activeJobsPerKey ?? ACTIVE_JOBS_PER_KEY; + if (mineActive >= activeLimit) throw new TeachError(429, `quota_key: you already have ${mineActive} lesson(s) in progress on this node — the configured limit is ${activeLimit}`); } // ------------------------------------------------------------ merge (design §9, §12.2 POST /api/teach/merge/preview) diff --git a/test/parallel-teach-admission.test.ts b/test/parallel-teach-admission.test.ts new file mode 100644 index 0000000..41e7fb5 --- /dev/null +++ b/test/parallel-teach-admission.test.ts @@ -0,0 +1,48 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { createIdentity, defaultConfig } from '@ainize/core'; +import { startNode } from '../src/server.js'; +import { teachAuthHeaderFor } from '../src/teach-auth.js'; + +test('one teacher can admit 70 concurrent jobs; the 71st and the global queue cap remain enforced', async context => { + const home = mkdtempSync(join(tmpdir(), 'ainize-parallel-admission-')); + const teacher = createIdentity(); + const config = defaultConfig({ home, port: 34048, peers: [], roles: ['seller'], ledger: 'local' }); + config.host = '127.0.0.1'; + config.teach = { + ...config.teach!, enabled: true, backend: 'stub', stubOffline: true, + queueMax: 80, activeJobsPerKey: 70, trustedKeys: [teacher.address], + jobsPerIpPerDay: 1000, jobsPerKeyPerDay: 1000, + }; + const node = await startNode(config, { quiet: true, serveWeb: false, teachHooks: { intervalMs: 600_000 } }); + context.after(async () => { await node.stop(); rmSync(home, { recursive: true, force: true }); }); + await node.teach!.stop(); + const endpoint = 'http://127.0.0.1:34048'; + const submit = async (index: number) => { + const route = '/api/teach/jobs'; + const body = JSON.stringify({ patch_ids: [], facts: [{ prompt: `Parallel lesson ${index} code?`, answer: `value-${index}` }], name: `parallel-${index}` }); + const response = await fetch(endpoint + route, { method: 'POST', headers: { + 'content-type': 'application/json', + 'x-ainize-auth': teachAuthHeaderFor(teacher, { node: node.market.address, method: 'POST', path: route, body }), + }, body }); + return { status: response.status, body: await response.json() as { job?: { id: string; status: string }; error?: string } }; + }; + const responses = await Promise.all(Array.from({ length: 70 }, (_, index) => submit(index))); + for (const response of responses) assert.equal(response.status, 202, JSON.stringify(response.body)); + assert.equal(new Set(responses.map(response => response.body.job?.id)).size, 70); + assert.ok(responses.every(response => response.body.job?.status === 'QUEUED')); + const rejected = await submit(70); + assert.equal(rejected.status, 429); + assert.match(rejected.body.error ?? '', /configured limit is 70/); + config.teach!.activeJobsPerKey = 80; + config.teach!.queueMax = 70; + const full = await submit(71); + assert.equal(full.status, 503); + assert.match(full.body.error ?? '', /training queue is full/); + node.teach!.invalidatePolicy(); + const policy = await fetch(endpoint + '/api/teach/policy').then(response => response.json()) as { limits: { active_jobs_per_key: number } }; + assert.equal(policy.limits.active_jobs_per_key, 80); +});