From c03be79c9ea9ef6777bf69a921e5fd883a8ce2e0 Mon Sep 17 00:00:00 2001 From: Svilen Stefanov Date: Thu, 24 Sep 2026 00:18:07 +0200 Subject: [PATCH 1/2] feat: check every run with CodeBoarding before the engine install and report its outcome A preflight step calls the proxy's /run/start with the job's OIDC token, in every credential mode, before the engine is installed. Its answer decides whether the run goes ahead and the depth it runs at: the workflow's depth_cap is only a request, and state identity, sync and review all read the preflight's depth_cap. When the proxy says the author may go deeper than the committed baseline, the review rebuilds the base in full at that cap. The run id rides last in every hosted call's bearer (~codeboarding-run~, after any licence), and a final always() step reports produced, failed or cancelled to /run/finish. Success is the map, not the exit code: a map written before a crash on shutdown still counts as produced. Everything fails open: an unreachable or broken proxy runs the analysis at up to 3 levels with a warning; a job without id-token: write (own-key only) skips the check. license_key is deprecated with a warning and keeps working. The proxy URL moves to scripts/action/hosted-proxy-url, overridable with CODEBOARDING_PROXY_URL for the dev stack. tests/contracts vendors licensing-aws contracts/ at cf8ae92; CI installs jsonschema for them. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/test.yml | 5 +- README.md | 50 ++- action.yml | 59 ++- scripts/action/analyze.sh | 12 +- scripts/action/configure-auth.sh | 7 +- scripts/action/hosted-proxy-url | 1 + scripts/action/run_meter.py | 204 ++++++++++ scripts/action/verify-credentials.sh | 5 + scripts/oidc_relay.py | 16 +- tests/contracts/README.md | 17 + .../contracts/examples/me.free-exhausted.json | 85 +++++ .../contracts/examples/me.free-one-left.json | 83 ++++ tests/contracts/examples/me.past-due.json | 83 ++++ tests/contracts/examples/me.pro.json | 83 ++++ tests/contracts/examples/me.team.json | 86 +++++ tests/contracts/examples/me.trial.json | 83 ++++ .../examples/meter-consume.allowed.json | 14 + .../examples/meter-consume.locked.json | 28 ++ .../examples/meter-consume.request.json | 11 + .../examples/run-finish.charged.json | 12 + .../examples/run-finish.released.json | 12 + .../examples/run-finish.request.json | 5 + .../contracts/examples/run-start.allowed.json | 23 ++ .../contracts/examples/run-start.counted.json | 23 ++ .../examples/run-start.fail-open.json | 23 ++ .../contracts/examples/run-start.legacy.json | 23 ++ .../examples/run-start.request.action.json | 10 + .../examples/run-start.request.vscode.json | 14 + .../examples/run-start.wall-bot.json | 37 ++ .../examples/run-start.wall-runs.json | 37 ++ .../contracts/examples/session.extension.json | 15 + tests/contracts/examples/session.web.json | 15 + .../examples/wall.token-ceiling.json | 15 + tests/contracts/me.schema.json | 359 ++++++++++++++++++ .../meter-consume.request.schema.json | 53 +++ .../meter-consume.response.schema.json | 51 +++ tests/contracts/meter.schema.json | 38 ++ .../contracts/run-finish.request.schema.json | 30 ++ .../contracts/run-finish.response.schema.json | 53 +++ tests/contracts/run-start.request.schema.json | 79 ++++ .../contracts/run-start.response.schema.json | 152 ++++++++ tests/contracts/session.schema.json | 78 ++++ tests/contracts/wall.schema.json | 103 +++++ tests/test_action_auth.py | 2 + tests/test_action_inputs.py | 49 ++- tests/test_action_state.py | 15 + tests/test_oidc_relay.py | 37 ++ tests/test_run_meter.py | 282 ++++++++++++++ 48 files changed, 2549 insertions(+), 28 deletions(-) create mode 100644 scripts/action/hosted-proxy-url create mode 100755 scripts/action/run_meter.py create mode 100644 tests/contracts/README.md create mode 100644 tests/contracts/examples/me.free-exhausted.json create mode 100644 tests/contracts/examples/me.free-one-left.json create mode 100644 tests/contracts/examples/me.past-due.json create mode 100644 tests/contracts/examples/me.pro.json create mode 100644 tests/contracts/examples/me.team.json create mode 100644 tests/contracts/examples/me.trial.json create mode 100644 tests/contracts/examples/meter-consume.allowed.json create mode 100644 tests/contracts/examples/meter-consume.locked.json create mode 100644 tests/contracts/examples/meter-consume.request.json create mode 100644 tests/contracts/examples/run-finish.charged.json create mode 100644 tests/contracts/examples/run-finish.released.json create mode 100644 tests/contracts/examples/run-finish.request.json create mode 100644 tests/contracts/examples/run-start.allowed.json create mode 100644 tests/contracts/examples/run-start.counted.json create mode 100644 tests/contracts/examples/run-start.fail-open.json create mode 100644 tests/contracts/examples/run-start.legacy.json create mode 100644 tests/contracts/examples/run-start.request.action.json create mode 100644 tests/contracts/examples/run-start.request.vscode.json create mode 100644 tests/contracts/examples/run-start.wall-bot.json create mode 100644 tests/contracts/examples/run-start.wall-runs.json create mode 100644 tests/contracts/examples/session.extension.json create mode 100644 tests/contracts/examples/session.web.json create mode 100644 tests/contracts/examples/wall.token-ceiling.json create mode 100644 tests/contracts/me.schema.json create mode 100644 tests/contracts/meter-consume.request.schema.json create mode 100644 tests/contracts/meter-consume.response.schema.json create mode 100644 tests/contracts/meter.schema.json create mode 100644 tests/contracts/run-finish.request.schema.json create mode 100644 tests/contracts/run-finish.response.schema.json create mode 100644 tests/contracts/run-start.request.schema.json create mode 100644 tests/contracts/run-start.response.schema.json create mode 100644 tests/contracts/session.schema.json create mode 100644 tests/contracts/wall.schema.json create mode 100644 tests/test_run_meter.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e910cb9..de014c5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,7 +18,10 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.12' - - name: Run unit tests (stdlib only) + # The action itself stays stdlib only; the tests validate against the paywall contracts. + - name: Install the contract validator + run: python -m pip install --disable-pip-version-check jsonschema + - name: Run unit tests run: python -m unittest discover -s tests -v core-compatibility: diff --git a/README.md b/README.md index 8f13b94..955a0a6 100644 --- a/README.md +++ b/README.md @@ -150,10 +150,49 @@ The same rule makes the combinations explicit rather than order-dependent: A licence alongside your own key is deliberately allowed: it says "my CodeBoarding plan, my own tokens". **Your key always wins.** A direct provider call never reaches -CodeBoarding, so the licence is recorded and reported but not spent, and nothing meters -that combination today. The job summary says so on every run, rather than leaving you to +CodeBoarding, so the licence is recorded and reported but not spent; the run is still +checked and counted like any own-key run (see [Plans and allowances](#plans-and-allowances)). The job summary says so on every run, rather than leaving you to infer it from the tier name. +### Plans and allowances + +Every run asks CodeBoarding first. Before the engine is installed, the action calls the +proxy's `/run/start` with the job's OIDC token, which names the repository, the pull +request and the GitHub user who triggered the run; the answer says whether the run goes +ahead and how deep it may go. A final step reports whether a map was produced. + +| | Free | Pro | +|---|---|---| +| Runs a week, across the Action and VS Code | 5 | 40 | +| Private pull request reviews a week, in the web app | 3 | 60 | +| Map depth | up to 3 levels | no cap | + +- **A run is one pull request a week.** The first map that week counts; every push after + it that week is included, even once the allowance is used up. Allowances reset Monday + 00:00 UTC. +- **Failed runs are never charged.** Only a run that writes its map counts. A failed or + cancelled run releases its place, and so does a runner that disappears (after six hours). +- **Who is charged.** The GitHub user who triggered the run: whoever opened the pull + request or pushed the commit, on their own CodeBoarding account (created the first time + they are seen, and theirs to claim by signing in with GitHub). Bot pull requests + (`dependabot[bot]`, `renovate[bot]`) are charged to the organisation's bot allowance. + Baseline syncs are not counted. A Team plan covers everyone in the organisation. +- **Depth.** `depth_cap` chooses how deep the map goes and the plan of whoever is charged + caps it: 3 on Free, no cap on Pro, the trial, Team and Enterprise. With the default of 2 + nothing differs between plans. When an author may go deeper than the committed baseline + was drawn, the review is analysed in full at their cap instead of incrementally. +- **Your own key.** A provider key changes where the model calls go, not whether the run is + checked: own-key runs still call `/run/start` and count the same, with no token ceiling, + since the model bill is yours. A job without `id-token: write` skips the check with a + warning. +- **CodeBoarding down is never your problem.** If the proxy cannot be reached, the run goes + ahead at up to 3 levels with a warning. + +`license_key` is deprecated: plans now follow your GitHub account. It keeps working until +the license key cutoff and is ignored after it; each run that sets it says so. Link the key +to your account on the [plan page](https://app.codeboarding.org/dashboard/plan), then +remove it (and use `llm: hosted` instead of `llm: license`). + ### Providers Each provider has its own inputs, so which key a workflow uses is readable from the file @@ -306,11 +345,11 @@ With the default `github.token`, the repository or organization must allow GitHu | `_api_key` | both | empty | That provider's key, e.g. `anthropic_api_key`. See [Providers](#providers). | | `_base_url` | both | empty | That provider's endpoint, where it has one. | | `aws_bedrock_region` | both | empty | Bedrock region. Core defaults to `us-east-1`. | -| `license_key` | both | empty | CodeBoarding license. Required by `llm: license`. | +| `license_key` | both | empty | Deprecated. CodeBoarding license, required by `llm: license`; ignored after the license key cutoff. | | `model` | both | empty | Default model for both analysis and parsing. | | `agent_model` | both | empty | Analysis-only override for `model`. | | `parsing_model` | both | empty | Parsing-only override for `model`. | -| `depth_cap` | both | `2` | Positive integer maximum analysis depth, including full-analysis fallbacks. Changing it rebuilds incompatible state. | +| `depth_cap` | both | `2` | Positive integer maximum analysis depth, including full-analysis fallbacks, capped by the plan of whoever the run is charged to (3 on Free). Changing it rebuilds incompatible state. | | `github_token` | both | `${{ github.token }}` | Token for comments and sync delivery. | | `sync_strategy` | sync | `push` | `push` or `pull_request`. | | `target_branch` | sync | event branch | Branch receiving the baseline or rolling PR. | @@ -326,7 +365,8 @@ runs prefer compatible prior PR state for incremental updates, while the review still compares the merge base with the current head. Set `depth_cap` in the action's `with:` block (for example, `depth_cap: 4`). -This configuration is authoritative: stored `metadata.depth_cap` is checked for +The run uses it as given unless the plan caps it lower (see +[Plans and allowances](#plans-and-allowances)). Stored `metadata.depth_cap` is checked for compatibility, not inherited, and legacy `metadata.depth_level` is not used as a fallback. Missing or incompatible baseline depth triggers a rebuild. diff --git a/action.yml b/action.yml index a371995..395575f 100644 --- a/action.yml +++ b/action.yml @@ -13,7 +13,7 @@ inputs: description: 'Required. Where analysis credentials come from: hosted, license, or a provider name (anthropic, aws_bedrock, cerebras, deepseek, glm, google, kimi, litellm, ollama, openai, openrouter, orcarouter, vercel).' required: true license_key: - description: 'CodeBoarding license key. Required by llm: license; optional alongside a provider key.' + description: 'Deprecated: plans now follow your GitHub account. A CodeBoarding license key, required by llm: license and optional alongside a provider key; it keeps working until the license key cutoff and is ignored after it.' required: false default: '' # Anthropic - selected by llm: anthropic @@ -348,15 +348,28 @@ runs: persist-credentials: false path: .codeboarding-target - - name: Setup Java for CodeBoarding + # Before the engine install, in every credential mode: the plan of whoever this run is + # charged to decides whether it runs and caps how deep, so nothing below reads + # inputs.depth_cap directly. Own-key runs ask too, although their model calls never reach + # CodeBoarding. After the checkout only because it reads the committed baseline's depth. + - name: Start the run with CodeBoarding + id: preflight if: steps.guard.outputs.skip != 'true' + shell: bash + env: + DEPTH_CAP: ${{ inputs.depth_cap }} + CHECKOUT_DIR: ${{ github.workspace }}/.codeboarding-target + run: python3 "$GITHUB_ACTION_PATH/scripts/action/run_meter.py" start + + - name: Setup Java for CodeBoarding + if: steps.guard.outputs.skip != 'true' && steps.preflight.outputs.allowed != 'false' uses: actions/setup-java@v5 with: distribution: temurin java-version: '21' - name: Install CodeBoarding - if: steps.guard.outputs.skip != 'true' + if: steps.guard.outputs.skip != 'true' && steps.preflight.outputs.allowed != 'false' shell: bash run: | python -m pip install --disable-pip-version-check 'codeboarding==0.14.4' @@ -370,7 +383,7 @@ runs: } - name: Configure analysis authentication - if: steps.guard.outputs.skip != 'true' + if: steps.guard.outputs.skip != 'true' && steps.preflight.outputs.allowed != 'false' shell: bash env: ACTION_PATH: ${{ github.action_path }} @@ -378,7 +391,7 @@ runs: - name: Resolve analysis identity id: state - if: steps.guard.outputs.skip != 'true' + if: steps.guard.outputs.skip != 'true' && steps.preflight.outputs.allowed != 'false' continue-on-error: true shell: bash env: @@ -388,7 +401,7 @@ runs: IS_FORK: ${{ steps.guard.outputs.is_fork }} LLM_PROVIDER: ${{ steps.llm.outputs.provider }} BACKEND_ID: ${{ steps.llm.outputs.backend_id }} - DEPTH_CAP: ${{ inputs.depth_cap }} + DEPTH_CAP: ${{ steps.preflight.outputs.depth_cap }} MODEL: ${{ inputs.model }} AGENT_MODEL_INPUT: ${{ inputs.agent_model }} PARSING_MODEL_INPUT: ${{ inputs.parsing_model }} @@ -429,7 +442,7 @@ runs: - name: Analyze baseline id: sync_analyze - if: steps.guard.outputs.skip != 'true' && steps.guard.outputs.mode == 'sync' + if: steps.guard.outputs.skip != 'true' && steps.preflight.outputs.allowed != 'false' && steps.guard.outputs.mode == 'sync' shell: bash env: ACTION_PATH: ${{ github.action_path }} @@ -437,7 +450,7 @@ runs: CHECKOUT_DIR: ${{ github.workspace }}/.codeboarding-target STAGE_DIR: ${{ runner.temp }}/cb-state/${{ github.action }}/out FORCE_FULL: ${{ inputs.force_full }} - DEPTH_CAP: ${{ inputs.depth_cap }} + DEPTH_CAP: ${{ steps.preflight.outputs.depth_cap }} MODEL: ${{ inputs.model }} AGENT_MODEL_INPUT: ${{ inputs.agent_model }} PARSING_MODEL_INPUT: ${{ inputs.parsing_model }} @@ -445,7 +458,7 @@ runs: - name: Deliver baseline id: sync_commit - if: steps.guard.outputs.skip != 'true' && steps.guard.outputs.mode == 'sync' + if: steps.guard.outputs.skip != 'true' && steps.preflight.outputs.allowed != 'false' && steps.guard.outputs.mode == 'sync' shell: bash env: ACTION_PATH: ${{ github.action_path }} @@ -511,7 +524,7 @@ runs: - name: Analyze pull request id: review_analyze - if: steps.guard.outputs.skip != 'true' && steps.guard.outputs.mode == 'review' + if: steps.guard.outputs.skip != 'true' && steps.preflight.outputs.allowed != 'false' && steps.guard.outputs.mode == 'review' shell: bash env: ACTION_PATH: ${{ github.action_path }} @@ -529,7 +542,8 @@ runs: CFG_HASH: ${{ steps.state.outputs.cfg_hash }} GIT_TOKEN: ${{ inputs.github_token }} GITHUB_SERVER_URL: ${{ github.server_url }} - DEPTH_CAP: ${{ inputs.depth_cap }} + DEPTH_CAP: ${{ steps.preflight.outputs.depth_cap }} + FULL_ANALYSIS: ${{ steps.preflight.outputs.full_analysis }} MODEL: ${{ inputs.model }} AGENT_MODEL_INPUT: ${{ inputs.agent_model }} PARSING_MODEL_INPUT: ${{ inputs.parsing_model }} @@ -570,7 +584,7 @@ runs: - name: Render review diagram id: review_render - if: steps.guard.outputs.skip != 'true' && steps.guard.outputs.mode == 'review' + if: steps.guard.outputs.skip != 'true' && steps.preflight.outputs.allowed != 'false' && steps.guard.outputs.mode == 'review' shell: bash env: ACTION_PATH: ${{ github.action_path }} @@ -580,7 +594,7 @@ runs: - name: Build review artifact id: review_artifact - if: steps.guard.outputs.skip != 'true' && steps.guard.outputs.mode == 'review' + if: steps.guard.outputs.skip != 'true' && steps.preflight.outputs.allowed != 'false' && steps.guard.outputs.mode == 'review' shell: bash env: ANALYSIS_PATH: ${{ steps.review_analyze.outputs.analysis_path }} @@ -601,7 +615,7 @@ runs: - name: Upload review artifact id: upload_review_artifact_dotcom - if: steps.guard.outputs.skip != 'true' && steps.guard.outputs.mode == 'review' && github.server_url == 'https://github.com' + if: steps.guard.outputs.skip != 'true' && steps.preflight.outputs.allowed != 'false' && steps.guard.outputs.mode == 'review' && github.server_url == 'https://github.com' uses: actions/upload-artifact@v4 with: name: codeboarding-review-${{ github.run_id }}-${{ github.run_attempt }} @@ -615,7 +629,7 @@ runs: - name: Build review comment id: review_body - if: steps.guard.outputs.skip != 'true' && steps.guard.outputs.mode == 'review' + if: steps.guard.outputs.skip != 'true' && steps.preflight.outputs.allowed != 'false' && steps.guard.outputs.mode == 'review' shell: bash env: DIAGRAM: ${{ steps.review_render.outputs.diagram_md }} @@ -632,7 +646,7 @@ runs: - name: Post review comment id: review_comment - if: steps.guard.outputs.skip != 'true' && steps.guard.outputs.mode == 'review' + if: steps.guard.outputs.skip != 'true' && steps.preflight.outputs.allowed != 'false' && steps.guard.outputs.mode == 'review' uses: marocchino/sticky-pull-request-comment@v2 with: header: ${{ steps.guard.outputs.comment_id }} @@ -669,3 +683,16 @@ runs: ### CodeBoarding review · failed See the [workflow logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}). + + # Last, and on every outcome, so the hold is charged or released now rather than lapsing + # after six hours. Its own OIDC token, because the auth directory went with the analysis. + - name: Finish the run with CodeBoarding + if: always() && steps.preflight.outputs.run_id != '' + continue-on-error: true + shell: bash + env: + RUN_ID: ${{ steps.preflight.outputs.run_id }} + JOB_STATUS: ${{ job.status }} + ANALYSIS_PATH: ${{ steps.review_analyze.outputs.analysis_path || steps.sync_analyze.outputs.analysis_path }} + MAP_MARKER: ${{ runner.temp }}/codeboarding-map + run: python3 "$GITHUB_ACTION_PATH/scripts/action/run_meter.py" finish diff --git a/scripts/action/analyze.sh b/scripts/action/analyze.sh index 5f246a0..b9c7fb7 100755 --- a/scripts/action/analyze.sh +++ b/scripts/action/analyze.sh @@ -42,6 +42,13 @@ metadata = json.load(open(sys.argv[1])).get("metadata", {}) print(metadata.get("depth_cap", ""))' "$analysis" 2>/dev/null || true } +# Names the analysis this run is about to write, and when it started writing it, for the +# finish step: an engine that crashes on shutdown after writing the map loses this step's +# outputs, but the run still produced a map and is charged for it. +mark_map() { + printf '%s' "$1" > "$RUNNER_TEMP/codeboarding-map" +} + seed_state() { local checkout="$1" state="$2" mkdir -p "$state" @@ -123,6 +130,7 @@ analyze_sync() { local work="$RUNNER_TEMP/codeboarding-sync" state="$RUNNER_TEMP/codeboarding-sync/analysis" rm -rf "$work" seed_state "$CHECKOUT_DIR" "$state" + mark_map "$state/analysis.json" if [ "${FORCE_FULL,,}" = true ] || [ "$(depth_cap_from "$state/analysis.json")" != "$DEPTH_CAP" ]; then full "$CHECKOUT_DIR" "$state" "$DEPTH_CAP" @@ -185,7 +193,8 @@ analyze_review() { git -C "$CHECKOUT_DIR" worktree add --detach "$base_checkout" "$REVIEW_BASE_SHA" >/dev/null seed_state "$base_checkout" "$base_state" REQUIRES_FULL=true - if [ "$(depth_cap_from "$base_state/analysis.json")" = "$DEPTH_CAP" ]; then + # FULL_ANALYSIS is the proxy saying this run may go deeper than the baseline was drawn. + if [ "$(depth_cap_from "$base_state/analysis.json")" = "$DEPTH_CAP" ] && [ "${FULL_ANALYSIS:-false}" != true ]; then incremental "$base_checkout" "$base_state" fi if [ "$REQUIRES_FULL" = true ]; then @@ -213,6 +222,7 @@ analyze_review() { fi rm -f "$head_state/origin.json" + mark_map "$head_state/analysis.json" incremental "$CHECKOUT_DIR" "$head_state" if [ "$REQUIRES_FULL" = true ]; then full "$CHECKOUT_DIR" "$head_state" "$DEPTH_CAP" diff --git a/scripts/action/configure-auth.sh b/scripts/action/configure-auth.sh index 6abcc9d..649eb63 100755 --- a/scripts/action/configure-auth.sh +++ b/scripts/action/configure-auth.sh @@ -6,7 +6,10 @@ # nothing to do here: its key never leaves the runner. set -euo pipefail AUTH_DIR="${RUNNER_TEMP}/codeboarding-auth" -HOSTED_PROXY_URL="https://auduihjmm4b735zci7vyabuikq0hppqn.lambda-url.us-east-1.on.aws" +# One URL for the model calls here and the run's start and finish in run_meter.py. +# CODEBOARDING_PROXY_URL points a workflow at the dev stack; it only redirects that +# workflow's own OIDC tokens, so it grants nothing. +HOSTED_PROXY_URL="${CODEBOARDING_PROXY_URL:-$(cat "$ACTION_PATH/scripts/action/hosted-proxy-url")}" umask 077 if [ ! -s "$AUTH_DIR/tier" ]; then @@ -28,6 +31,8 @@ RELAY_ARGS=(--upstream-base-url "$HOSTED_PROXY_URL" --ready-file "$READY") if [ -s "$AUTH_DIR/license.txt" ]; then RELAY_ARGS+=(--license-file "$AUTH_DIR/license.txt") fi +# Written by the preflight when the proxy gave this run an id; read per request. +RELAY_ARGS+=(--run-id-file "$AUTH_DIR/run-id") python3 "$ACTION_PATH/scripts/oidc_relay.py" "${RELAY_ARGS[@]}" > "$LOG" 2>&1 & echo $! > "$PID" diff --git a/scripts/action/hosted-proxy-url b/scripts/action/hosted-proxy-url new file mode 100644 index 0000000..7081ca1 --- /dev/null +++ b/scripts/action/hosted-proxy-url @@ -0,0 +1 @@ +https://auduihjmm4b735zci7vyabuikq0hppqn.lambda-url.us-east-1.on.aws diff --git a/scripts/action/run_meter.py b/scripts/action/run_meter.py new file mode 100755 index 0000000..d6e361d --- /dev/null +++ b/scripts/action/run_meter.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""A run's start and finish on CodeBoarding's proxy, in every credential mode. + +``start`` runs before the engine install and asks ``POST /run/start`` whether this run may +go ahead, and how deep: the plan of whoever the run is charged to caps ``depth_cap``, so +no workflow can raise it. Own-key runs ask too, although their model calls never reach +CodeBoarding; that is the honour-system half of the paywall. The proxy's answer becomes +this step's outputs, and its run id is written where the relay packs it into every hosted +call. + +``finish`` runs last, on every outcome, and reports ``POST /run/finish``: ``produced`` +charges the run, anything else releases its hold. Success is the map, not the exit code. + +Both fail open. The proxy being down or wrong never stops or fails a run: ``start`` then +allows it at up to three levels, and ``finish`` only warns. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +from pathlib import Path +from urllib.request import Request, urlopen + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from oidc_relay import RelayConfig, authorization # noqa: E402 + +ACTION_ROOT = Path(__file__).resolve().parent.parent.parent +#: The cap a run gets when the proxy cannot say: Free's, so an outage lifts nobody past it. +FAIL_OPEN_DEPTH = 3 + + +def proxy_url(environ: dict[str, str]) -> str: + return environ.get("CODEBOARDING_PROXY_URL") or (Path(__file__).parent / "hosted-proxy-url").read_text().strip() + + +def post(environ: dict[str, str], path: str, body: dict, license_file: Path | None = None) -> dict: + config = RelayConfig( + proxy_url(environ), + environ["ACTIONS_ID_TOKEN_REQUEST_URL"], + environ["ACTIONS_ID_TOKEN_REQUEST_TOKEN"], + license_file, + ) + request = Request( + proxy_url(environ).rstrip("/") + path, + data=json.dumps(body).encode(), + headers={"Authorization": authorization(config), "Content-Type": "application/json"}, + method="POST", + ) + with urlopen(request, timeout=30) as response: # nosec B310 - CodeBoarding's proxy + return json.loads(response.read()) + + +def baseline_depth(checkout: Path) -> int | None: + """The committed baseline's cap, so the proxy can tell a PR that goes deeper to run in full.""" + try: + cap = json.loads((checkout / ".codeboarding" / "analysis.json").read_text(encoding="utf-8"))["metadata"][ + "depth_cap" + ] + except (OSError, ValueError, KeyError, TypeError): + return None + return cap if isinstance(cap, int) and not isinstance(cap, bool) and cap >= 1 else None + + +def start_request(environ: dict[str, str], depth: int, tier: str) -> dict: + manifest = json.loads((ACTION_ROOT / ".release-please-manifest.json").read_text(encoding="utf-8")) + return { + "depth": depth, + "credential": "hosted" if tier in ("hosted", "license") else "own_key", + "baseline_depth": baseline_depth(Path(environ.get("CHECKOUT_DIR", ""))), + "client": {"surface": "action", "version": manifest["."]}, + } + + +def start(environ: dict[str, str]) -> tuple[dict[str, str], int]: + """The step outputs, and the exit code: non-zero only for a depth_cap that is not a number.""" + raw = environ.get("DEPTH_CAP", "2") + if not re.fullmatch(r"[1-9][0-9]*", raw): + print("::error::depth_cap must be a positive integer.") + return {}, 1 + depth = int(raw) + auth_dir = Path(environ["RUNNER_TEMP"]) / "codeboarding-auth" + for stale in (Path(environ["RUNNER_TEMP"]) / "codeboarding-map", auth_dir / "run-id"): + stale.unlink(missing_ok=True) + outputs = { + "allowed": "true", + "depth_cap": str(depth), + "run_id": "", + "full_analysis": "false", + "wall_message": "", + "mode": "", + } + + # Only own-key workflows get here without one: the hosted tiers refuse to start without it. + if not (environ.get("ACTIONS_ID_TOKEN_REQUEST_URL") and environ.get("ACTIONS_ID_TOKEN_REQUEST_TOKEN")): + print( + "::warning title=CodeBoarding run check::This job cannot mint a GitHub OIDC token, so the run " + "was not checked against your CodeBoarding plan. Add `id-token: write` to the job's permissions." + ) + return outputs, 0 + + tier = (auth_dir / "tier").read_text(encoding="utf-8").strip() + license_file = auth_dir / "license.txt" + try: + answer = post( + environ, + "/run/start", + start_request(environ, depth, tier), + license_file if tier == "license" and license_file.is_file() else None, + ) + allowed, cap = answer["allowed"], answer["depth_cap"] + if not isinstance(allowed, bool) or not isinstance(cap, int) or cap < 1: + raise ValueError("unexpected /run/start answer") + except Exception as exc: # noqa: BLE001 - our outage must never block a run + capped = min(depth, FAIL_OPEN_DEPTH) + print( + f"::warning title=CodeBoarding run check::CodeBoarding could not check this run ({type(exc).__name__}), " + f"so it runs at up to {capped} levels." + ) + outputs["depth_cap"] = str(capped) + return outputs, 0 + + run_id = answer.get("run_id") or "" + wall = answer.get("wall") or {} + outputs.update( + { + "allowed": str(allowed).lower(), + "depth_cap": str(min(depth, cap)), + "run_id": run_id, + "full_analysis": str(answer.get("full_analysis") is True).lower(), + "wall_message": " ".join(str(wall.get("message", "")).split()), + "mode": str(answer.get("mode") or ""), + } + ) + if run_id: + (auth_dir / "run-id").write_text(run_id, encoding="utf-8") + print( + f"CodeBoarding run check: allowed={outputs['allowed']} depth_cap={outputs['depth_cap']} " + f"plan={answer.get('plan')} mode={outputs['mode']} charged_to={(answer.get('charged_to') or {}).get('login')}." + ) + if answer.get("depth_reason"): + print(f"::notice title=CodeBoarding depth::{answer['depth_reason']}") + if not allowed: + print(f"::notice title=CodeBoarding::{outputs['wall_message'] or 'This run is over the plan allowance.'}") + return outputs, 0 + + +def map_written(environ: dict[str, str]) -> bool: + """Whether the analysis this run set out to write exists. + + The step output is the normal answer. It is lost when the engine crashes on shutdown + after writing, so the marker analyze.sh leaves just before the final engine call is the + fallback: the file it names, rewritten since, is this run's map. A file older than the + marker is the seed the run started from, not something it produced. + """ + path = environ.get("ANALYSIS_PATH", "") + if path and Path(path).is_file(): + return True + marker = Path(environ.get("MAP_MARKER") or "/nonexistent") + try: + expected = Path(marker.read_text(encoding="utf-8").strip()) + return expected.is_file() and expected.stat().st_mtime_ns >= marker.stat().st_mtime_ns + except OSError: + return False + + +def outcome(environ: dict[str, str]) -> str: + if map_written(environ): + return "produced" + if environ.get("JOB_STATUS") == "cancelled": + return "cancelled" + return "failed" + + +def finish(environ: dict[str, str]) -> int: + """Always 0: reporting the outcome must never be what fails the job.""" + body = {"run_id": environ["RUN_ID"], "outcome": outcome(environ), "error": None} + try: + answer = post(environ, "/run/finish", body) + except Exception as exc: # noqa: BLE001 - an unreported run is released after the stale-hold timeout + print(f"::warning title=CodeBoarding run check::Could not report this run's outcome ({type(exc).__name__}).") + return 0 + print(f"Reported {body['outcome']} to CodeBoarding: charged={answer.get('charged')} ({answer.get('cause')}).") + return 0 + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=["start", "finish"]) + args = parser.parse_args(argv) + environ = dict(os.environ) + if args.command == "finish": + return finish(environ) + outputs, code = start(environ) + with open(environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as handle: + handle.writelines(f"{key}={value}\n" for key, value in outputs.items()) + return code + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/action/verify-credentials.sh b/scripts/action/verify-credentials.sh index 6963142..9779edf 100755 --- a/scripts/action/verify-credentials.sh +++ b/scripts/action/verify-credentials.sh @@ -18,6 +18,11 @@ while IFS= read -r var; do done < <(compgen -v | grep -E '^CB_IN_.*_API_KEY$' || true) [ -z "${CB_IN_LICENSE_KEY:-}" ] || echo "::add-mask::${CB_IN_LICENSE_KEY}" +# Still honoured: the proxy decides what a key is worth, and after the cutoff it is ignored there. +if [ -n "${CB_IN_LICENSE_KEY:-}" ]; then + echo "::warning title=CodeBoarding license_key is deprecated::License keys stop working after the key cutoff. Plans now follow your GitHub account: link this key at https://app.codeboarding.org/dashboard/plan, then remove license_key (and replace llm: license with llm: hosted)." +fi + set +e plan="$(python3 "${ACTION_PATH}/scripts/action/credential_check.py" --auth-dir "$AUTH_DIR")" resolved=$? diff --git a/scripts/oidc_relay.py b/scripts/oidc_relay.py index cd5f36a..5d128dd 100644 --- a/scripts/oidc_relay.py +++ b/scripts/oidc_relay.py @@ -48,6 +48,7 @@ class RelayConfig: id_token_request_url: str id_token_request_token: str license_file: Path | None = None + run_id_file: Path | None = None def _with_audience(url: str) -> str: @@ -76,7 +77,7 @@ def _mint_oidc_token(config: RelayConfig) -> str: return token.strip() -def _authorization(config: RelayConfig) -> str: +def authorization(config: RelayConfig) -> str: token = _mint_oidc_token(config) if config.license_file is not None: try: @@ -86,6 +87,12 @@ def _authorization(config: RelayConfig) -> str: if not license_key: raise RuntimeError("CodeBoarding license is empty") token = f"{token}~codeboarding-license~{license_key}" + # Last, because the proxy splits it off from the right before it looks for a licence. + # Absent when the preflight failed open: the proxy then decides what an unheld call gets. + if config.run_id_file is not None and config.run_id_file.is_file(): + run_id = config.run_id_file.read_text(encoding="utf-8").strip() + if run_id: + token = f"{token}~codeboarding-run~{run_id}" return f"Bearer {token}" @@ -128,7 +135,7 @@ def _handle(self) -> None: for key, value in self.headers.items() if key.lower() not in _HOP_BY_HOP and key.lower() != "authorization" } - headers["Authorization"] = _authorization(self.config) + headers["Authorization"] = authorization(self.config) request = Request( _upstream_url(self.config.upstream_base_url, self.path), data=self._request_body(), @@ -166,6 +173,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--upstream-base-url", required=True) parser.add_argument("--ready-file", required=True, type=Path) parser.add_argument("--license-file", type=Path) + parser.add_argument("--run-id-file", type=Path) args = parser.parse_args(argv) request_url = os.environ.get("ACTIONS_ID_TOKEN_REQUEST_URL", "") @@ -174,7 +182,9 @@ def main(argv: list[str] | None = None) -> int: print("A GitHub OIDC request URL/token is unavailable.", file=sys.stderr) return 2 - server = RelayServer(RelayConfig(args.upstream_base_url, request_url, request_token, args.license_file)) + server = RelayServer( + RelayConfig(args.upstream_base_url, request_url, request_token, args.license_file, args.run_id_file) + ) args.ready_file.write_text(str(server.server_port), encoding="utf-8") try: server.serve_forever() diff --git a/tests/contracts/README.md b/tests/contracts/README.md new file mode 100644 index 0000000..5fc4122 --- /dev/null +++ b/tests/contracts/README.md @@ -0,0 +1,17 @@ +# Paywall contracts + +JSON Schemas (draft 2020-12) for the wire shapes licensing-aws answers and accepts. The webview, the VS Code extension and the Action each keep a copy of this directory and validate their own fixtures against it in their test suites; `tests/unit/test_contracts/test_schemas.py` validates the examples here. + +| Schema | Route | +| --- | --- | +| `session.schema.json` | answer of `POST /session/github` and `POST /session/extension` | +| `me.schema.json` | answer of `GET /me` | +| `run-start.request.schema.json`, `run-start.response.schema.json` | `POST /run/start` on gha_proxy (OIDC) and license_proxy (extension token) | +| `run-finish.request.schema.json`, `run-finish.response.schema.json` | `POST /run/finish` on both | +| `meter-consume.request.schema.json`, `meter-consume.response.schema.json` | `POST /meter/consume` | +| `wall.schema.json` | the refusal inside the answers above, and the body of a 402 from either proxy (`{"error": {"message", "type"}, "wall": …}`) | +| `meter.schema.json` | one weekly allowance, used by the others | + +Schemas reference each other by `$id` (`https://codeboarding.org/schemas/licensing/v1/`), so load the whole directory into one registry. Examples are named `..json`, and `.request.` examples validate against the request schema. + +A change here is a change to four repositories: bump the copies in the same series of pull requests. diff --git a/tests/contracts/examples/me.free-exhausted.json b/tests/contracts/examples/me.free-exhausted.json new file mode 100644 index 0000000..92a55a4 --- /dev/null +++ b/tests/contracts/examples/me.free-exhausted.json @@ -0,0 +1,85 @@ +{ + "account": { + "id": "acc_3qk2m7x9p4t8w1v6r5ya", + "login": "svilen", + "provider": "github", + "provider_id": 1296269, + "claimed": true, + "email": "svilen@example.com" + }, + "plan": { + "name": "free", + "source": "default", + "status": "active", + "expires_at": null, + "grace_ends_at": null, + "interval": null, + "cancel_at_period_end": false, + "covered_by": null + }, + "trial": { + "state": "ended", + "started_at": "2026-09-04T09:00:00+00:00", + "ends_at": "2026-09-18T09:00:00+00:00", + "days_left": 0 + }, + "depth_cap": 3, + "meters": [ + { + "meter": "runs", + "used": 5, + "limit": 5, + "resets_at": "2026-09-28T00:00:00+00:00" + }, + { + "meter": "reading", + "used": 3, + "limit": 3, + "resets_at": "2026-09-28T00:00:00+00:00" + } + ], + "orgs": [ + { + "workspace": "workspace:github:9919", + "login": "acme-corp", + "plan": "free", + "covered": false, + "members_permission": true, + "bot_runs": { + "meter": "bot_runs", + "used": 2, + "limit": 5, + "resets_at": "2026-09-28T00:00:00+00:00" + } + } + ], + "sessions": [ + { + "id": "ses_web_1", + "kind": "web", + "editor": null, + "created_at": "2026-09-18T09:00:00+00:00", + "last_used_at": "2026-09-23T14:05:00+00:00", + "current": true + }, + { + "id": "ses_ext_1", + "kind": "extension", + "editor": "vscode", + "created_at": "2026-09-18T09:02:00+00:00", + "last_used_at": "2026-09-22T17:40:00+00:00", + "current": false + } + ], + "offers": { + "cancel_personal_pro": false, + "legacy_link": { + "license": "Pro license from May 2026" + } + }, + "billing": { + "customer": false + }, + "mode": "shadow", + "now": "2026-09-23T14:05:00+00:00" +} diff --git a/tests/contracts/examples/me.free-one-left.json b/tests/contracts/examples/me.free-one-left.json new file mode 100644 index 0000000..3067de9 --- /dev/null +++ b/tests/contracts/examples/me.free-one-left.json @@ -0,0 +1,83 @@ +{ + "account": { + "id": "acc_3qk2m7x9p4t8w1v6r5ya", + "login": "svilen", + "provider": "github", + "provider_id": 1296269, + "claimed": true, + "email": "svilen@example.com" + }, + "plan": { + "name": "free", + "source": "default", + "status": "active", + "expires_at": null, + "grace_ends_at": null, + "interval": null, + "cancel_at_period_end": false, + "covered_by": null + }, + "trial": { + "state": "ended", + "started_at": "2026-09-04T09:00:00+00:00", + "ends_at": "2026-09-18T09:00:00+00:00", + "days_left": 0 + }, + "depth_cap": 3, + "meters": [ + { + "meter": "runs", + "used": 4, + "limit": 5, + "resets_at": "2026-09-28T00:00:00+00:00" + }, + { + "meter": "reading", + "used": 1, + "limit": 3, + "resets_at": "2026-09-28T00:00:00+00:00" + } + ], + "orgs": [ + { + "workspace": "workspace:github:9919", + "login": "acme-corp", + "plan": "free", + "covered": false, + "members_permission": true, + "bot_runs": { + "meter": "bot_runs", + "used": 2, + "limit": 5, + "resets_at": "2026-09-28T00:00:00+00:00" + } + } + ], + "sessions": [ + { + "id": "ses_web_1", + "kind": "web", + "editor": null, + "created_at": "2026-09-18T09:00:00+00:00", + "last_used_at": "2026-09-23T14:05:00+00:00", + "current": true + }, + { + "id": "ses_ext_1", + "kind": "extension", + "editor": "vscode", + "created_at": "2026-09-18T09:02:00+00:00", + "last_used_at": "2026-09-22T17:40:00+00:00", + "current": false + } + ], + "offers": { + "cancel_personal_pro": false, + "legacy_link": null + }, + "billing": { + "customer": false + }, + "mode": "shadow", + "now": "2026-09-23T14:05:00+00:00" +} diff --git a/tests/contracts/examples/me.past-due.json b/tests/contracts/examples/me.past-due.json new file mode 100644 index 0000000..4d5a2e8 --- /dev/null +++ b/tests/contracts/examples/me.past-due.json @@ -0,0 +1,83 @@ +{ + "account": { + "id": "acc_3qk2m7x9p4t8w1v6r5ya", + "login": "svilen", + "provider": "github", + "provider_id": 1296269, + "claimed": true, + "email": "svilen@example.com" + }, + "plan": { + "name": "pro", + "source": "subscription", + "status": "past_due", + "expires_at": "2026-10-23T14:00:00+00:00", + "grace_ends_at": "2026-09-30T14:00:00+00:00", + "interval": "month", + "cancel_at_period_end": false, + "covered_by": null + }, + "trial": { + "state": "ended", + "started_at": "2026-09-04T09:00:00+00:00", + "ends_at": "2026-09-18T09:00:00+00:00", + "days_left": 0 + }, + "depth_cap": null, + "meters": [ + { + "meter": "runs", + "used": 12, + "limit": 40, + "resets_at": "2026-09-28T00:00:00+00:00" + }, + { + "meter": "reading", + "used": 7, + "limit": 60, + "resets_at": "2026-09-28T00:00:00+00:00" + } + ], + "orgs": [ + { + "workspace": "workspace:github:9919", + "login": "acme-corp", + "plan": "free", + "covered": false, + "members_permission": true, + "bot_runs": { + "meter": "bot_runs", + "used": 2, + "limit": 5, + "resets_at": "2026-09-28T00:00:00+00:00" + } + } + ], + "sessions": [ + { + "id": "ses_web_1", + "kind": "web", + "editor": null, + "created_at": "2026-09-18T09:00:00+00:00", + "last_used_at": "2026-09-23T14:05:00+00:00", + "current": true + }, + { + "id": "ses_ext_1", + "kind": "extension", + "editor": "vscode", + "created_at": "2026-09-18T09:02:00+00:00", + "last_used_at": "2026-09-22T17:40:00+00:00", + "current": false + } + ], + "offers": { + "cancel_personal_pro": false, + "legacy_link": null + }, + "billing": { + "customer": true + }, + "mode": "shadow", + "now": "2026-09-23T14:05:00+00:00" +} diff --git a/tests/contracts/examples/me.pro.json b/tests/contracts/examples/me.pro.json new file mode 100644 index 0000000..c480e44 --- /dev/null +++ b/tests/contracts/examples/me.pro.json @@ -0,0 +1,83 @@ +{ + "account": { + "id": "acc_3qk2m7x9p4t8w1v6r5ya", + "login": "svilen", + "provider": "github", + "provider_id": 1296269, + "claimed": true, + "email": "svilen@example.com" + }, + "plan": { + "name": "pro", + "source": "subscription", + "status": "active", + "expires_at": "2026-10-23T14:00:00+00:00", + "grace_ends_at": null, + "interval": "month", + "cancel_at_period_end": false, + "covered_by": null + }, + "trial": { + "state": "ended", + "started_at": "2026-09-04T09:00:00+00:00", + "ends_at": "2026-09-18T09:00:00+00:00", + "days_left": 0 + }, + "depth_cap": null, + "meters": [ + { + "meter": "runs", + "used": 12, + "limit": 40, + "resets_at": "2026-09-28T00:00:00+00:00" + }, + { + "meter": "reading", + "used": 7, + "limit": 60, + "resets_at": "2026-09-28T00:00:00+00:00" + } + ], + "orgs": [ + { + "workspace": "workspace:github:9919", + "login": "acme-corp", + "plan": "free", + "covered": false, + "members_permission": true, + "bot_runs": { + "meter": "bot_runs", + "used": 2, + "limit": 5, + "resets_at": "2026-09-28T00:00:00+00:00" + } + } + ], + "sessions": [ + { + "id": "ses_web_1", + "kind": "web", + "editor": null, + "created_at": "2026-09-18T09:00:00+00:00", + "last_used_at": "2026-09-23T14:05:00+00:00", + "current": true + }, + { + "id": "ses_ext_1", + "kind": "extension", + "editor": "vscode", + "created_at": "2026-09-18T09:02:00+00:00", + "last_used_at": "2026-09-22T17:40:00+00:00", + "current": false + } + ], + "offers": { + "cancel_personal_pro": false, + "legacy_link": null + }, + "billing": { + "customer": true + }, + "mode": "shadow", + "now": "2026-09-23T14:05:00+00:00" +} diff --git a/tests/contracts/examples/me.team.json b/tests/contracts/examples/me.team.json new file mode 100644 index 0000000..6f2a209 --- /dev/null +++ b/tests/contracts/examples/me.team.json @@ -0,0 +1,86 @@ +{ + "account": { + "id": "acc_3qk2m7x9p4t8w1v6r5ya", + "login": "svilen", + "provider": "github", + "provider_id": 1296269, + "claimed": true, + "email": "svilen@example.com" + }, + "plan": { + "name": "team", + "source": "admin", + "status": "active", + "expires_at": "2027-01-01T00:00:00+00:00", + "grace_ends_at": null, + "interval": "month", + "cancel_at_period_end": false, + "covered_by": { + "kind": "workspace", + "name": "acme-corp" + } + }, + "trial": { + "state": "none", + "started_at": null, + "ends_at": null, + "days_left": null + }, + "depth_cap": null, + "meters": [ + { + "meter": "runs", + "used": 2, + "limit": 40, + "resets_at": "2026-09-28T00:00:00+00:00" + }, + { + "meter": "reading", + "used": 0, + "limit": 60, + "resets_at": "2026-09-28T00:00:00+00:00" + } + ], + "orgs": [ + { + "workspace": "workspace:github:9919", + "login": "acme-corp", + "plan": "team", + "covered": true, + "members_permission": true, + "bot_runs": { + "meter": "bot_runs", + "used": 4, + "limit": null, + "resets_at": "2026-09-28T00:00:00+00:00" + } + } + ], + "sessions": [ + { + "id": "ses_web_1", + "kind": "web", + "editor": null, + "created_at": "2026-09-18T09:00:00+00:00", + "last_used_at": "2026-09-23T14:05:00+00:00", + "current": true + }, + { + "id": "ses_ext_1", + "kind": "extension", + "editor": "vscode", + "created_at": "2026-09-18T09:02:00+00:00", + "last_used_at": "2026-09-22T17:40:00+00:00", + "current": false + } + ], + "offers": { + "cancel_personal_pro": true, + "legacy_link": null + }, + "billing": { + "customer": false + }, + "mode": "shadow", + "now": "2026-09-23T14:05:00+00:00" +} diff --git a/tests/contracts/examples/me.trial.json b/tests/contracts/examples/me.trial.json new file mode 100644 index 0000000..68b9c7a --- /dev/null +++ b/tests/contracts/examples/me.trial.json @@ -0,0 +1,83 @@ +{ + "account": { + "id": "acc_3qk2m7x9p4t8w1v6r5ya", + "login": "svilen", + "provider": "github", + "provider_id": 1296269, + "claimed": true, + "email": "svilen@example.com" + }, + "plan": { + "name": "pro", + "source": "trial", + "status": "active", + "expires_at": "2026-10-02T09:00:00+00:00", + "grace_ends_at": null, + "interval": null, + "cancel_at_period_end": false, + "covered_by": null + }, + "trial": { + "state": "active", + "started_at": "2026-09-18T09:00:00+00:00", + "ends_at": "2026-10-02T09:00:00+00:00", + "days_left": 9 + }, + "depth_cap": null, + "meters": [ + { + "meter": "runs", + "used": 3, + "limit": 40, + "resets_at": "2026-09-28T00:00:00+00:00" + }, + { + "meter": "reading", + "used": 1, + "limit": 60, + "resets_at": "2026-09-28T00:00:00+00:00" + } + ], + "orgs": [ + { + "workspace": "workspace:github:9919", + "login": "acme-corp", + "plan": "free", + "covered": false, + "members_permission": true, + "bot_runs": { + "meter": "bot_runs", + "used": 2, + "limit": 5, + "resets_at": "2026-09-28T00:00:00+00:00" + } + } + ], + "sessions": [ + { + "id": "ses_web_1", + "kind": "web", + "editor": null, + "created_at": "2026-09-18T09:00:00+00:00", + "last_used_at": "2026-09-23T14:05:00+00:00", + "current": true + }, + { + "id": "ses_ext_1", + "kind": "extension", + "editor": "vscode", + "created_at": "2026-09-18T09:02:00+00:00", + "last_used_at": "2026-09-22T17:40:00+00:00", + "current": false + } + ], + "offers": { + "cancel_personal_pro": false, + "legacy_link": null + }, + "billing": { + "customer": false + }, + "mode": "shadow", + "now": "2026-09-23T14:05:00+00:00" +} diff --git a/tests/contracts/examples/meter-consume.allowed.json b/tests/contracts/examples/meter-consume.allowed.json new file mode 100644 index 0000000..fdc65e8 --- /dev/null +++ b/tests/contracts/examples/meter-consume.allowed.json @@ -0,0 +1,14 @@ +{ + "allowed": true, + "consumed": true, + "already_opened": false, + "meter": { + "meter": "reading", + "used": 2, + "limit": 3, + "resets_at": "2026-09-28T00:00:00+00:00" + }, + "mode": "enforce", + "fail_open": false, + "wall": null +} diff --git a/tests/contracts/examples/meter-consume.locked.json b/tests/contracts/examples/meter-consume.locked.json new file mode 100644 index 0000000..31a5c26 --- /dev/null +++ b/tests/contracts/examples/meter-consume.locked.json @@ -0,0 +1,28 @@ +{ + "allowed": false, + "consumed": false, + "already_opened": false, + "meter": { + "meter": "reading", + "used": 3, + "limit": 3, + "resets_at": "2026-09-28T00:00:00+00:00" + }, + "mode": "enforce", + "fail_open": false, + "wall": { + "reason": "reviews_exhausted", + "meter": "reading", + "used": 3, + "limit": 3, + "resets_at": "2026-09-28T00:00:00+00:00", + "plan": "free", + "subject": { + "kind": "person", + "login": "svilen" + }, + "message": "This is your 4th private review this week. Free includes 3 a week; the next one unlocks Monday 00:00 UTC.", + "upgrade_url": "https://app.codeboarding.org/dashboard/plan", + "plans_url": "https://codeboarding.org/pricing" + } +} diff --git a/tests/contracts/examples/meter-consume.request.json b/tests/contracts/examples/meter-consume.request.json new file mode 100644 index 0000000..14031a1 --- /dev/null +++ b/tests/contracts/examples/meter-consume.request.json @@ -0,0 +1,11 @@ +{ + "meter": "reading", + "repository": { + "provider": "github", + "id": 1296269, + "owner": "acme-corp", + "name": "billing-service", + "private": true + }, + "pr": 482 +} diff --git a/tests/contracts/examples/run-finish.charged.json b/tests/contracts/examples/run-finish.charged.json new file mode 100644 index 0000000..4357503 --- /dev/null +++ b/tests/contracts/examples/run-finish.charged.json @@ -0,0 +1,12 @@ +{ + "run_id": "github:acme-corp/billing-service#11809532110", + "charged": true, + "cause": "produced", + "meter": { + "meter": "runs", + "used": 5, + "limit": 5, + "resets_at": "2026-09-28T00:00:00+00:00" + }, + "mode": "enforce" +} diff --git a/tests/contracts/examples/run-finish.released.json b/tests/contracts/examples/run-finish.released.json new file mode 100644 index 0000000..1511ac9 --- /dev/null +++ b/tests/contracts/examples/run-finish.released.json @@ -0,0 +1,12 @@ +{ + "run_id": "vscode:acc_3qk2m7x9p4t8w1v6r5ya#5b0c1b1e-7f7e-4f3a-9d1a-0d6f2a1c9e11", + "charged": false, + "cause": "failed", + "meter": { + "meter": "runs", + "used": 4, + "limit": 5, + "resets_at": "2026-09-28T00:00:00+00:00" + }, + "mode": "enforce" +} diff --git a/tests/contracts/examples/run-finish.request.json b/tests/contracts/examples/run-finish.request.json new file mode 100644 index 0000000..879a491 --- /dev/null +++ b/tests/contracts/examples/run-finish.request.json @@ -0,0 +1,5 @@ +{ + "run_id": "github:acme-corp/billing-service#11809532110", + "outcome": "produced", + "error": null +} diff --git a/tests/contracts/examples/run-start.allowed.json b/tests/contracts/examples/run-start.allowed.json new file mode 100644 index 0000000..f875bb8 --- /dev/null +++ b/tests/contracts/examples/run-start.allowed.json @@ -0,0 +1,23 @@ +{ + "depth_reason": "3 levels: m.koch is on Free", + "plan": "free", + "trial": false, + "charged_to": { + "kind": "person", + "subject": "account:acc_3qk2m7x9p4t8w1v6r5ya", + "login": "m.koch" + }, + "meter": "runs", + "resets_at": "2026-09-28T00:00:00+00:00", + "full_analysis": true, + "mode": "enforce", + "fail_open": false, + "wall": null, + "allowed": true, + "run_id": "github:acme-corp/billing-service#11809532110", + "depth_cap": 3, + "counted": false, + "used": 4, + "limit": 5, + "remaining": 1 +} diff --git a/tests/contracts/examples/run-start.counted.json b/tests/contracts/examples/run-start.counted.json new file mode 100644 index 0000000..03d776a --- /dev/null +++ b/tests/contracts/examples/run-start.counted.json @@ -0,0 +1,23 @@ +{ + "depth_reason": null, + "plan": "free", + "trial": false, + "charged_to": { + "kind": "person", + "subject": "account:acc_3qk2m7x9p4t8w1v6r5ya", + "login": "m.koch" + }, + "meter": "runs", + "resets_at": "2026-09-28T00:00:00+00:00", + "full_analysis": false, + "mode": "enforce", + "fail_open": false, + "wall": null, + "allowed": true, + "run_id": "vscode:acc_3qk2m7x9p4t8w1v6r5ya#5b0c1b1e-7f7e-4f3a-9d1a-0d6f2a1c9e11", + "depth_cap": 3, + "counted": true, + "used": 5, + "limit": 5, + "remaining": 0 +} diff --git a/tests/contracts/examples/run-start.fail-open.json b/tests/contracts/examples/run-start.fail-open.json new file mode 100644 index 0000000..1d63fd3 --- /dev/null +++ b/tests/contracts/examples/run-start.fail-open.json @@ -0,0 +1,23 @@ +{ + "depth_reason": null, + "plan": "free", + "trial": false, + "charged_to": { + "kind": "person", + "subject": "account:acc_3qk2m7x9p4t8w1v6r5ya", + "login": "m.koch" + }, + "meter": "runs", + "resets_at": "2026-09-28T00:00:00+00:00", + "full_analysis": false, + "mode": "enforce", + "fail_open": true, + "wall": null, + "allowed": true, + "run_id": "prov.1790000000.q8Zr1c.3f1d2a", + "depth_cap": 3, + "counted": false, + "used": null, + "limit": null, + "remaining": null +} diff --git a/tests/contracts/examples/run-start.legacy.json b/tests/contracts/examples/run-start.legacy.json new file mode 100644 index 0000000..09969f3 --- /dev/null +++ b/tests/contracts/examples/run-start.legacy.json @@ -0,0 +1,23 @@ +{ + "depth_reason": null, + "plan": "free", + "trial": false, + "charged_to": { + "kind": "none", + "subject": null, + "login": null + }, + "meter": null, + "resets_at": "2026-09-28T00:00:00+00:00", + "full_analysis": false, + "mode": "legacy", + "fail_open": false, + "wall": null, + "allowed": true, + "run_id": "github:acme-corp/billing-service#11809532110", + "depth_cap": 2, + "counted": false, + "used": null, + "limit": null, + "remaining": null +} diff --git a/tests/contracts/examples/run-start.request.action.json b/tests/contracts/examples/run-start.request.action.json new file mode 100644 index 0000000..82681a5 --- /dev/null +++ b/tests/contracts/examples/run-start.request.action.json @@ -0,0 +1,10 @@ +{ + "depth": 5, + "credential": "hosted", + "baseline_depth": 2, + "client": { + "surface": "action", + "version": "1.16.0", + "editor": null + } +} diff --git a/tests/contracts/examples/run-start.request.vscode.json b/tests/contracts/examples/run-start.request.vscode.json new file mode 100644 index 0000000..88b5254 --- /dev/null +++ b/tests/contracts/examples/run-start.request.vscode.json @@ -0,0 +1,14 @@ +{ + "run_id": "5b0c1b1e-7f7e-4f3a-9d1a-0d6f2a1c9e11", + "repository": { + "key": "github.com/codeboarding/codeboarding", + "name": "CodeBoarding" + }, + "depth": 3, + "credential": "hosted", + "client": { + "surface": "vscode", + "version": "0.15.0", + "editor": "cursor" + } +} diff --git a/tests/contracts/examples/run-start.wall-bot.json b/tests/contracts/examples/run-start.wall-bot.json new file mode 100644 index 0000000..d05dbb1 --- /dev/null +++ b/tests/contracts/examples/run-start.wall-bot.json @@ -0,0 +1,37 @@ +{ + "depth_reason": null, + "plan": "free", + "trial": false, + "charged_to": { + "kind": "workspace", + "subject": "workspace:github:9919", + "login": "acme-corp" + }, + "meter": "bot_runs", + "resets_at": "2026-09-28T00:00:00+00:00", + "full_analysis": false, + "mode": "enforce", + "fail_open": false, + "wall": { + "reason": "bot_runs_exhausted", + "meter": "bot_runs", + "used": 5, + "limit": 5, + "resets_at": "2026-09-28T00:00:00+00:00", + "plan": "free", + "subject": { + "kind": "workspace", + "login": "acme-corp" + }, + "message": "CodeBoarding map not drawn: acme-corp has used 5 of 5 free bot pull requests this week (resets Monday 00:00 UTC). A Team plan includes bot pull requests.", + "upgrade_url": "https://app.codeboarding.org/dashboard/plan", + "plans_url": "https://codeboarding.org/pricing" + }, + "allowed": false, + "run_id": null, + "depth_cap": 3, + "counted": false, + "used": 5, + "limit": 5, + "remaining": 0 +} diff --git a/tests/contracts/examples/run-start.wall-runs.json b/tests/contracts/examples/run-start.wall-runs.json new file mode 100644 index 0000000..0fc2828 --- /dev/null +++ b/tests/contracts/examples/run-start.wall-runs.json @@ -0,0 +1,37 @@ +{ + "depth_reason": null, + "plan": "free", + "trial": false, + "charged_to": { + "kind": "person", + "subject": "account:acc_3qk2m7x9p4t8w1v6r5ya", + "login": "m.koch" + }, + "meter": "runs", + "resets_at": "2026-09-28T00:00:00+00:00", + "full_analysis": false, + "mode": "enforce", + "fail_open": false, + "wall": { + "reason": "runs_exhausted", + "meter": "runs", + "used": 5, + "limit": 5, + "resets_at": "2026-09-28T00:00:00+00:00", + "plan": "free", + "subject": { + "kind": "person", + "login": "m.koch" + }, + "message": "CodeBoarding map not drawn: m.koch has used 5 of 5 free runs this week (resets Monday 00:00 UTC). Pro gives 40 a week; a Team plan covers everyone in acme-corp.", + "upgrade_url": "https://app.codeboarding.org/dashboard/plan", + "plans_url": "https://codeboarding.org/pricing" + }, + "allowed": false, + "run_id": null, + "depth_cap": 3, + "counted": false, + "used": 5, + "limit": 5, + "remaining": 0 +} diff --git a/tests/contracts/examples/session.extension.json b/tests/contracts/examples/session.extension.json new file mode 100644 index 0000000..b654cf1 --- /dev/null +++ b/tests/contracts/examples/session.extension.json @@ -0,0 +1,15 @@ +{ + "token": "cbx_EXAMPLEEXAMPLEEXAMPLEEXAMPLEEXAMPLEEXAMPLEE", + "session_id": "ses_ext_1", + "kind": "extension", + "expires_at": null, + "account": { + "id": "acc_3qk2m7x9p4t8w1v6r5ya", + "login": "svilen", + "provider": "github", + "provider_id": 1296269, + "claimed": true, + "email": "svilen@example.com" + }, + "trial_started": false +} diff --git a/tests/contracts/examples/session.web.json b/tests/contracts/examples/session.web.json new file mode 100644 index 0000000..6e33a78 --- /dev/null +++ b/tests/contracts/examples/session.web.json @@ -0,0 +1,15 @@ +{ + "token": "cbs_EXAMPLEEXAMPLEEXAMPLEEXAMPLEEXAMPLEEXAMPLEE", + "session_id": "ses_web_1", + "kind": "web", + "expires_at": "2026-09-24T14:05:00+00:00", + "account": { + "id": "acc_3qk2m7x9p4t8w1v6r5ya", + "login": "svilen", + "provider": "github", + "provider_id": 1296269, + "claimed": true, + "email": "svilen@example.com" + }, + "trial_started": true +} diff --git a/tests/contracts/examples/wall.token-ceiling.json b/tests/contracts/examples/wall.token-ceiling.json new file mode 100644 index 0000000..49e9cf6 --- /dev/null +++ b/tests/contracts/examples/wall.token-ceiling.json @@ -0,0 +1,15 @@ +{ + "reason": "token_ceiling", + "meter": "tokens", + "used": 1500000, + "limit": 1500000, + "resets_at": "2026-09-28T00:00:00+00:00", + "plan": "free", + "subject": { + "kind": "person", + "login": "svilen" + }, + "message": "This repository is large: free runs cover about 1.5M tokens a week and this one needs more. It resets Monday 00:00 UTC; Pro covers it.", + "upgrade_url": "https://app.codeboarding.org/dashboard/plan", + "plans_url": "https://codeboarding.org/pricing" +} diff --git a/tests/contracts/me.schema.json b/tests/contracts/me.schema.json new file mode 100644 index 0000000..b1d6a1d --- /dev/null +++ b/tests/contracts/me.schema.json @@ -0,0 +1,359 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://codeboarding.org/schemas/licensing/v1/me.schema.json", + "title": "Answer of GET /me: the plan, the counts and the reset time every surface shows", + "type": "object", + "properties": { + "account": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^acc_" + }, + "login": { + "type": "string" + }, + "provider": { + "enum": [ + "github" + ] + }, + "provider_id": { + "type": "integer" + }, + "claimed": { + "type": "boolean" + }, + "email": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "id", + "login", + "provider", + "provider_id", + "claimed", + "email" + ], + "additionalProperties": false + }, + "plan": { + "type": "object", + "properties": { + "name": { + "enum": [ + "free", + "pro", + "max", + "team", + "enterprise" + ] + }, + "source": { + "enum": [ + "default", + "trial", + "subscription", + "admin", + "legacy" + ] + }, + "status": { + "enum": [ + "active", + "past_due", + "canceled" + ] + }, + "expires_at": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "grace_ends_at": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "interval": { + "enum": [ + "month", + "year", + null + ] + }, + "cancel_at_period_end": { + "type": "boolean" + }, + "covered_by": { + "anyOf": [ + { + "type": "object", + "properties": { + "kind": { + "enum": [ + "person", + "workspace", + "company" + ] + }, + "name": { + "type": "string" + } + }, + "required": [ + "kind", + "name" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "name", + "source", + "status", + "expires_at", + "grace_ends_at", + "interval", + "cancel_at_period_end", + "covered_by" + ], + "additionalProperties": false + }, + "trial": { + "type": "object", + "properties": { + "state": { + "enum": [ + "active", + "ended", + "none", + "ineligible" + ] + }, + "started_at": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "ends_at": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "days_left": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + } + }, + "required": [ + "state", + "started_at", + "ends_at", + "days_left" + ], + "additionalProperties": false + }, + "depth_cap": { + "type": [ + "integer", + "null" + ], + "minimum": 1, + "description": "null means no cap" + }, + "meters": { + "type": "array", + "items": { + "$ref": "https://codeboarding.org/schemas/licensing/v1/meter.schema.json" + } + }, + "orgs": { + "type": "array", + "items": { + "type": "object", + "properties": { + "workspace": { + "type": "string", + "pattern": "^workspace:github:" + }, + "login": { + "type": "string" + }, + "plan": { + "enum": [ + "free", + "pro", + "max", + "team", + "enterprise" + ] + }, + "covered": { + "type": "boolean", + "description": "true when this organisation's plan covers the signed-in person" + }, + "members_permission": { + "type": "boolean" + }, + "bot_runs": { + "anyOf": [ + { + "$ref": "https://codeboarding.org/schemas/licensing/v1/meter.schema.json" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "workspace", + "login", + "plan", + "covered", + "members_permission", + "bot_runs" + ], + "additionalProperties": false + } + }, + "sessions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "kind": { + "enum": [ + "web", + "extension" + ] + }, + "editor": { + "type": [ + "string", + "null" + ] + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "last_used_at": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "current": { + "type": "boolean" + } + }, + "required": [ + "id", + "kind", + "editor", + "created_at", + "last_used_at", + "current" + ], + "additionalProperties": false + } + }, + "offers": { + "type": "object", + "properties": { + "cancel_personal_pro": { + "type": "boolean" + }, + "legacy_link": { + "anyOf": [ + { + "type": "object", + "properties": { + "license": { + "type": "string", + "description": "e.g. 'Pro license from May 2026'" + } + }, + "required": [ + "license" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "cancel_personal_pro", + "legacy_link" + ], + "additionalProperties": false + }, + "billing": { + "type": "object", + "properties": { + "customer": { + "type": "boolean" + } + }, + "required": [ + "customer" + ], + "additionalProperties": false + }, + "mode": { + "enum": [ + "legacy", + "shadow", + "enforce" + ] + }, + "now": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "account", + "plan", + "trial", + "depth_cap", + "meters", + "orgs", + "sessions", + "offers", + "billing", + "mode", + "now" + ], + "additionalProperties": false +} diff --git a/tests/contracts/meter-consume.request.schema.json b/tests/contracts/meter-consume.request.schema.json new file mode 100644 index 0000000..46d63f1 --- /dev/null +++ b/tests/contracts/meter-consume.request.schema.json @@ -0,0 +1,53 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://codeboarding.org/schemas/licensing/v1/meter-consume.request.schema.json", + "title": "Body of POST /meter/consume (the webview server, only when the answer carries a private map)", + "type": "object", + "properties": { + "meter": { + "enum": [ + "reading" + ] + }, + "repository": { + "type": "object", + "properties": { + "provider": { + "enum": [ + "github" + ] + }, + "id": { + "type": "integer" + }, + "owner": { + "type": "string" + }, + "name": { + "type": "string" + }, + "private": { + "type": "boolean" + } + }, + "required": [ + "provider", + "id", + "owner", + "name", + "private" + ], + "additionalProperties": false + }, + "pr": { + "type": "integer", + "minimum": 1 + } + }, + "required": [ + "meter", + "repository", + "pr" + ], + "additionalProperties": false +} diff --git a/tests/contracts/meter-consume.response.schema.json b/tests/contracts/meter-consume.response.schema.json new file mode 100644 index 0000000..35b1343 --- /dev/null +++ b/tests/contracts/meter-consume.response.schema.json @@ -0,0 +1,51 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://codeboarding.org/schemas/licensing/v1/meter-consume.response.schema.json", + "title": "Answer of POST /meter/consume", + "type": "object", + "properties": { + "allowed": { + "type": "boolean" + }, + "consumed": { + "type": "boolean", + "description": "this call used one review" + }, + "already_opened": { + "type": "boolean" + }, + "meter": { + "$ref": "https://codeboarding.org/schemas/licensing/v1/meter.schema.json" + }, + "mode": { + "enum": [ + "legacy", + "shadow", + "enforce" + ] + }, + "fail_open": { + "type": "boolean" + }, + "wall": { + "anyOf": [ + { + "$ref": "https://codeboarding.org/schemas/licensing/v1/wall.schema.json" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "allowed", + "consumed", + "already_opened", + "meter", + "mode", + "fail_open", + "wall" + ], + "additionalProperties": false +} diff --git a/tests/contracts/meter.schema.json b/tests/contracts/meter.schema.json new file mode 100644 index 0000000..9cd8c66 --- /dev/null +++ b/tests/contracts/meter.schema.json @@ -0,0 +1,38 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://codeboarding.org/schemas/licensing/v1/meter.schema.json", + "title": "A weekly allowance and how much of it is used", + "type": "object", + "properties": { + "meter": { + "enum": [ + "runs", + "reading", + "bot_runs" + ] + }, + "used": { + "type": "integer", + "minimum": 0 + }, + "limit": { + "type": [ + "integer", + "null" + ], + "minimum": 0, + "description": "null when the plan includes the meter without a count" + }, + "resets_at": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "meter", + "used", + "limit", + "resets_at" + ], + "additionalProperties": false +} diff --git a/tests/contracts/run-finish.request.schema.json b/tests/contracts/run-finish.request.schema.json new file mode 100644 index 0000000..b154d75 --- /dev/null +++ b/tests/contracts/run-finish.request.schema.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://codeboarding.org/schemas/licensing/v1/run-finish.request.schema.json", + "title": "Body of POST /run/finish", + "type": "object", + "properties": { + "run_id": { + "type": "string" + }, + "outcome": { + "enum": [ + "produced", + "failed", + "cancelled" + ] + }, + "error": { + "type": [ + "string", + "null" + ], + "maxLength": 200 + } + }, + "required": [ + "run_id", + "outcome" + ], + "additionalProperties": false +} diff --git a/tests/contracts/run-finish.response.schema.json b/tests/contracts/run-finish.response.schema.json new file mode 100644 index 0000000..aaa8bea --- /dev/null +++ b/tests/contracts/run-finish.response.schema.json @@ -0,0 +1,53 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://codeboarding.org/schemas/licensing/v1/run-finish.response.schema.json", + "title": "Answer of POST /run/finish", + "type": "object", + "properties": { + "run_id": { + "type": "string" + }, + "charged": { + "type": "boolean" + }, + "cause": { + "enum": [ + "produced", + "already_counted", + "failed", + "cancelled", + "lapsed", + "over_allowance", + "uncounted", + "legacy", + "not_found", + "fail_open" + ] + }, + "meter": { + "anyOf": [ + { + "$ref": "https://codeboarding.org/schemas/licensing/v1/meter.schema.json" + }, + { + "type": "null" + } + ] + }, + "mode": { + "enum": [ + "legacy", + "shadow", + "enforce" + ] + } + }, + "required": [ + "run_id", + "charged", + "cause", + "meter", + "mode" + ], + "additionalProperties": false +} diff --git a/tests/contracts/run-start.request.schema.json b/tests/contracts/run-start.request.schema.json new file mode 100644 index 0000000..cc2d1ad --- /dev/null +++ b/tests/contracts/run-start.request.schema.json @@ -0,0 +1,79 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://codeboarding.org/schemas/licensing/v1/run-start.request.schema.json", + "title": "Body of POST /run/start (both proxies)", + "type": "object", + "properties": { + "run_id": { + "type": "string", + "minLength": 8, + "description": "VS Code: a client uuid for this run. The Action omits it; gha_proxy takes run_id from the OIDC token" + }, + "repository": { + "type": "object", + "properties": { + "key": { + "type": "string", + "minLength": 1, + "description": "VS Code: a stable key for the local repository (normalised origin URL, else absolute path)" + }, + "name": { + "type": "string" + } + }, + "required": [ + "key" + ], + "additionalProperties": false + }, + "depth": { + "type": "integer", + "minimum": 1, + "description": "The depth the setting asks for" + }, + "credential": { + "enum": [ + "hosted", + "own_key" + ] + }, + "baseline_depth": { + "type": [ + "integer", + "null" + ], + "minimum": 1, + "description": "Action: the depth of the default-branch baseline, when one exists" + }, + "client": { + "type": "object", + "properties": { + "surface": { + "enum": [ + "vscode", + "action" + ] + }, + "version": { + "type": "string" + }, + "editor": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "surface" + ], + "additionalProperties": false + } + }, + "required": [ + "depth", + "credential", + "client" + ], + "additionalProperties": false +} diff --git a/tests/contracts/run-start.response.schema.json b/tests/contracts/run-start.response.schema.json new file mode 100644 index 0000000..98e95f2 --- /dev/null +++ b/tests/contracts/run-start.response.schema.json @@ -0,0 +1,152 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://codeboarding.org/schemas/licensing/v1/run-start.response.schema.json", + "title": "Answer of POST /run/start", + "type": "object", + "properties": { + "allowed": { + "type": "boolean" + }, + "run_id": { + "type": [ + "string", + "null" + ], + "description": "Carry on every hosted model call and on /run/finish; null only when refused" + }, + "depth_cap": { + "type": "integer", + "minimum": 1, + "description": "The requested depth clamped to the payer's plan; use it as the run's depth" + }, + "depth_reason": { + "type": [ + "string", + "null" + ], + "description": "e.g. '3 levels: m.koch is on Free'; null when nothing was clamped" + }, + "plan": { + "enum": [ + "free", + "pro", + "max", + "team", + "enterprise" + ] + }, + "trial": { + "type": "boolean" + }, + "charged_to": { + "type": "object", + "properties": { + "kind": { + "enum": [ + "person", + "workspace", + "none" + ] + }, + "subject": { + "type": [ + "string", + "null" + ] + }, + "login": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "kind", + "subject", + "login" + ], + "additionalProperties": false + }, + "counted": { + "type": "boolean", + "description": "true when this pull request or repository already counted this week, so the run is free" + }, + "meter": { + "enum": [ + "runs", + "bot_runs", + null + ] + }, + "used": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "limit": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "remaining": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "resets_at": { + "type": "string", + "format": "date-time" + }, + "full_analysis": { + "type": "boolean", + "description": "Action: analyse in full because the cap is deeper than the baseline" + }, + "mode": { + "enum": [ + "legacy", + "shadow", + "enforce" + ] + }, + "fail_open": { + "type": "boolean" + }, + "wall": { + "anyOf": [ + { + "$ref": "https://codeboarding.org/schemas/licensing/v1/wall.schema.json" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "allowed", + "run_id", + "depth_cap", + "depth_reason", + "plan", + "trial", + "charged_to", + "counted", + "meter", + "used", + "limit", + "remaining", + "resets_at", + "full_analysis", + "mode", + "fail_open", + "wall" + ], + "additionalProperties": false +} diff --git a/tests/contracts/session.schema.json b/tests/contracts/session.schema.json new file mode 100644 index 0000000..cdcb8e9 --- /dev/null +++ b/tests/contracts/session.schema.json @@ -0,0 +1,78 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://codeboarding.org/schemas/licensing/v1/session.schema.json", + "title": "Answer of POST /session/github and POST /session/extension", + "type": "object", + "properties": { + "token": { + "type": "string", + "pattern": "^cb[sx]_" + }, + "session_id": { + "type": "string" + }, + "kind": { + "enum": [ + "web", + "extension" + ] + }, + "expires_at": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "account": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^acc_" + }, + "login": { + "type": "string" + }, + "provider": { + "enum": [ + "github" + ] + }, + "provider_id": { + "type": "integer" + }, + "claimed": { + "type": "boolean" + }, + "email": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "id", + "login", + "provider", + "provider_id", + "claimed", + "email" + ], + "additionalProperties": false + }, + "trial_started": { + "type": "boolean" + } + }, + "required": [ + "token", + "session_id", + "kind", + "expires_at", + "account", + "trial_started" + ], + "additionalProperties": false +} diff --git a/tests/contracts/wall.schema.json b/tests/contracts/wall.schema.json new file mode 100644 index 0000000..356fdc8 --- /dev/null +++ b/tests/contracts/wall.schema.json @@ -0,0 +1,103 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://codeboarding.org/schemas/licensing/v1/wall.schema.json", + "title": "Why a run or a map was refused, in the person's unit", + "type": "object", + "properties": { + "reason": { + "enum": [ + "runs_exhausted", + "reviews_exhausted", + "bot_runs_exhausted", + "token_ceiling", + "no_live_run", + "sign_in_required", + "key_retired" + ] + }, + "meter": { + "enum": [ + "runs", + "reading", + "bot_runs", + "tokens", + null + ] + }, + "used": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "limit": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "resets_at": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "plan": { + "enum": [ + "free", + "pro", + "max", + "team", + "enterprise" + ] + }, + "subject": { + "type": "object", + "properties": { + "kind": { + "enum": [ + "person", + "workspace" + ] + }, + "login": { + "type": "string" + } + }, + "required": [ + "kind", + "login" + ], + "additionalProperties": false + }, + "message": { + "type": "string", + "minLength": 1, + "description": "A complete sentence naming the unit, the count and the reset time" + }, + "upgrade_url": { + "type": "string", + "format": "uri" + }, + "plans_url": { + "type": "string", + "format": "uri" + } + }, + "required": [ + "reason", + "meter", + "used", + "limit", + "resets_at", + "plan", + "subject", + "message", + "upgrade_url", + "plans_url" + ], + "additionalProperties": false +} diff --git a/tests/test_action_auth.py b/tests/test_action_auth.py index 1cf2568..53829dd 100644 --- a/tests/test_action_auth.py +++ b/tests/test_action_auth.py @@ -285,6 +285,7 @@ def test_hosted_auth_relays_to_the_codeboarding_proxy(self) -> None: self.assertEqual(result.returncode, 0, result.stderr or result.stdout) self.assertEqual(outputs["tier"], "license") self.assertIn("::add-mask::a-license", result.stdout) + self.assertIn("license_key is deprecated", result.stdout, "still honoured, but said to be going") configured = subprocess.run( [str(CONFIGURE_AUTH)], @@ -305,6 +306,7 @@ def test_hosted_auth_relays_to_the_codeboarding_proxy(self) -> None: upstream = args[args.index("--upstream-base-url") + 1] self.assertEqual(upstream, "https://auduihjmm4b735zci7vyabuikq0hppqn.lambda-url.us-east-1.on.aws") self.assertIn("--license-file", args) + self.assertEqual(args[args.index("--run-id-file") + 1], str(auth_dir / "run-id")) self.assertEqual((auth_dir / "env" / "OPENROUTER_API_KEY").read_text(), "github-actions-oidc-relay") self.assertEqual((auth_dir / "env" / "OPENROUTER_BASE_URL").read_text(), "http://127.0.0.1:12345") diff --git a/tests/test_action_inputs.py b/tests/test_action_inputs.py index 4757bc7..139864a 100644 --- a/tests/test_action_inputs.py +++ b/tests/test_action_inputs.py @@ -64,12 +64,59 @@ def test_llm_is_required_and_has_no_default(self) -> None: self.assertNotIn("default:", block) def test_depth_is_wired_to_state_identity_and_both_analysis_modes(self) -> None: + """The workflow's depth_cap is only a request: the preflight's answer, clamped to the + payer's plan, is what the identity and both analyses run at (R4, R5).""" self.assertIn("default: '2'", self.inputs["depth_cap"]) self.assertNotIn("depth_level", self.inputs) for identifier in ("id: state", "id: sync_analyze", "id: review_analyze"): start = ACTION.index(identifier) block = ACTION[start : ACTION.index("\n run:", start)] - self.assertIn("DEPTH_CAP: ${{ inputs.depth_cap }}", block) + self.assertIn("DEPTH_CAP: ${{ steps.preflight.outputs.depth_cap }}", block) + preflight = ACTION[ACTION.index("id: preflight") :] + self.assertIn("DEPTH_CAP: ${{ inputs.depth_cap }}", preflight[: preflight.index("\n run:")]) + self.assertEqual(ACTION.count("${{ inputs.depth_cap }}"), 1, "only the preflight reads the input") + review = ACTION[ACTION.index("id: review_analyze") :] + self.assertIn( + "FULL_ANALYSIS: ${{ steps.preflight.outputs.full_analysis }}", review[: review.index("\n run:")] + ) + + def test_the_preflight_runs_before_the_engine_install_in_every_mode(self) -> None: + start = ACTION.index("- name: Start the run with CodeBoarding") + self.assertLess(ACTION.index("- name: Stop on LLM configuration failure"), start, "it reads the resolved tier") + self.assertLess(ACTION.index("- name: Checkout analysis target"), start, "it reads the baseline's depth") + self.assertLess(start, ACTION.index("- name: Install CodeBoarding")) + condition = ACTION[start : ACTION.index("shell:", start)] + self.assertIn("if: steps.guard.outputs.skip != 'true'\n", condition, "no credential mode is exempt") + + def test_a_refused_run_does_no_analysis_work(self) -> None: + for step in ( + "Setup Java for CodeBoarding", + "Install CodeBoarding", + "Configure analysis authentication", + "Resolve analysis identity", + "Analyze baseline", + "Deliver baseline", + "Analyze pull request", + "Render review diagram", + "Post review comment", + ): + with self.subTest(step=step): + start = ACTION.index(f"- name: {step}\n") + condition = ACTION[start : ACTION.index("\n", ACTION.index("if:", start))] + self.assertIn("steps.preflight.outputs.allowed != 'false'", condition) + + def test_the_finish_runs_last_on_every_outcome_and_never_fails_the_job(self) -> None: + start = ACTION.index("- name: Finish the run with CodeBoarding") + self.assertEqual(ACTION[start:].count("- name:"), 1, "the finish is the last step") + block = ACTION[start:] + self.assertIn("if: always() && steps.preflight.outputs.run_id != ''", block) + self.assertIn("continue-on-error: true", block) + self.assertIn("steps.review_analyze.outputs.analysis_path || steps.sync_analyze.outputs.analysis_path", block) + self.assertIn("JOB_STATUS: ${{ job.status }}", block) + + def test_license_key_is_deprecated_but_still_wired(self) -> None: + self.assertIn("Deprecated", self.inputs["license_key"]) + self.assertIn("CB_IN_LICENSE_KEY: ${{ inputs.license_key }}", ACTION) def test_default_workflow_reviews_drafts_on_open_and_new_commits(self) -> None: self.assertNotIn("github.event.pull_request.draft", DOGFOOD) diff --git a/tests/test_action_state.py b/tests/test_action_state.py index 7b5a8aa..13a8717 100644 --- a/tests/test_action_state.py +++ b/tests/test_action_state.py @@ -286,6 +286,21 @@ def test_compatible_committed_baseline_runs_incrementally(self) -> None: self._analyze(REVIEW_BASE_SHA=sha, DEPTH_CAP="4") self.assertEqual([c["mode"] for c in self._engine_calls()], ["incremental", "incremental"]) + def test_the_proxy_asking_for_a_full_analysis_rebuilds_the_committed_baseline(self) -> None: + """A Pro author may go deeper than the default branch was drawn: the review then runs + in full at the author's cap, never incrementally from the shallower baseline.""" + sha = self._commit_base(cap=4) + self._analyze(REVIEW_BASE_SHA=sha, DEPTH_CAP="4", FULL_ANALYSIS="true") + self.assertEqual([c["mode"] for c in self._engine_calls()], ["full", "incremental"]) + self.assertEqual(self._engine_calls()[0]["depth"], "4") + + def test_the_head_analysis_is_named_for_the_finish_step_before_it_runs(self) -> None: + _state(self.base_dir, cap=4) + values = self._analyze(DEPTH_CAP="4") + marker = self.runner_temp / "codeboarding-map" + self.assertEqual(marker.read_text(), values["analysis_path"]) + self.assertGreaterEqual(Path(values["analysis_path"]).stat().st_mtime_ns, marker.stat().st_mtime_ns) + def test_legacy_committed_depth_is_not_inherited(self) -> None: sha = self._commit_base(legacy=True) self._analyze(REVIEW_BASE_SHA=sha, DEPTH_CAP="4") diff --git a/tests/test_oidc_relay.py b/tests/test_oidc_relay.py index 5430681..95fd87d 100644 --- a/tests/test_oidc_relay.py +++ b/tests/test_oidc_relay.py @@ -5,6 +5,7 @@ import importlib.util import json import sys +import tempfile import threading import unittest from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -101,6 +102,42 @@ def log_message(self, *_args): self.assertEqual([auth for _, auth, _ in received], ["Bearer jwt-1", "Bearer jwt-2"]) self.assertEqual([path for path, _, _ in received], ["/api/v1/chat/completions?model=test"] * 2) + def test_the_run_id_rides_last_in_the_bearer_with_or_without_a_licence(self): + """The proxy splits `~codeboarding-run~` off from the right, then the licence.""" + + class OidcIssuer(BaseHTTPRequestHandler): + def do_GET(self): + payload = b'{"value": "jwt"}' + self.send_response(200) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *_args): + pass + + issuer = _Server(OidcIssuer) + issuer.start() + self.addCleanup(issuer.close) + with tempfile.TemporaryDirectory() as temp: + license_file, run_id_file = Path(temp) / "license.txt", Path(temp) / "run-id" + license_file.write_text("LIC\n") + + def bearer(**files): + config = oidc_relay.RelayConfig( + "https://proxy.example", f"{issuer.url}/token", "request-token", **files + ) + return oidc_relay.authorization(config) + + self.assertEqual(bearer(run_id_file=run_id_file), "Bearer jwt", "no run id until the preflight wrote one") + run_id_file.write_text("github:o/r#42\n") + self.assertEqual(bearer(run_id_file=run_id_file), "Bearer jwt~codeboarding-run~github:o/r#42") + self.assertEqual( + bearer(license_file=license_file, run_id_file=run_id_file), + "Bearer jwt~codeboarding-license~LIC~codeboarding-run~github:o/r#42", + ) + self.assertEqual(bearer(license_file=license_file), "Bearer jwt~codeboarding-license~LIC") + def test_audience_replaces_an_existing_value(self): self.assertEqual( oidc_relay._with_audience("https://issuer.example/token?audience=old&x=1"), diff --git a/tests/test_run_meter.py b/tests/test_run_meter.py new file mode 100644 index 0000000..3cf869c --- /dev/null +++ b/tests/test_run_meter.py @@ -0,0 +1,282 @@ +"""The run's start and finish against CodeBoarding's proxy, checked against the shared contract. + +tests/contracts is a copy of licensing-aws's contracts/ directory (CodeBoarding/licensing-aws#17, +cf8ae92). The proxy answers in those shapes, so the preflight's request must validate against +them and every example answer must turn into sensible step outputs. +""" + +from __future__ import annotations + +import contextlib +import importlib.util +import io +import json +import os +import subprocess +import sys +import tempfile +import threading +import time +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +try: + from jsonschema import Draft202012Validator + from referencing import Registry, Resource +except ImportError: # CI installs it; a bare local checkout skips the contract checks + Draft202012Validator = None + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "action" / "run_meter.py" +CONTRACTS = Path(__file__).resolve().parent / "contracts" +NEEDS_JSONSCHEMA = unittest.skipIf( + Draft202012Validator is None, "jsonschema is not installed: `pip install jsonschema` to check the paywall contracts" +) + +_SPEC = importlib.util.spec_from_file_location("run_meter", SCRIPT) +assert _SPEC and _SPEC.loader +run_meter = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(run_meter) + + +def example(name: str) -> dict: + return json.loads((CONTRACTS / "examples" / name).read_text(encoding="utf-8")) + + +def validate(document: dict, schema_name: str) -> list[str]: + schemas = [json.loads(p.read_text(encoding="utf-8")) for p in CONTRACTS.glob("*.schema.json")] + registry = Registry().with_resources((s["$id"], Resource.from_contents(s)) for s in schemas) + schema = json.loads((CONTRACTS / schema_name).read_text(encoding="utf-8")) + return [error.message for error in Draft202012Validator(schema, registry=registry).iter_errors(document)] + + +class FakeProxy: + """GitHub's OIDC issuer and CodeBoarding's proxy on one loopback server.""" + + def __init__(self, answer: dict | None = None, status: int = 200) -> None: + self.requests: list[tuple[str, str, dict]] = [] + proxy = self + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + self._reply(200, {"value": "oidc-jwt"}) + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + proxy.requests.append((self.path, self.headers["Authorization"], body)) + self._reply(status, answer or {}) + + def _reply(self, code, payload): + data = json.dumps(payload).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, *_args): + pass + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.thread = threading.Thread(target=lambda: self.server.serve_forever(poll_interval=0.05), daemon=True) + self.thread.start() + + @property + def url(self) -> str: + return f"http://127.0.0.1:{self.server.server_port}" + + def close(self) -> None: + self.server.shutdown() + self.server.server_close() + self.thread.join(timeout=2) + + +class RunMeterTests(unittest.TestCase): + def setUp(self) -> None: + temp_dir = tempfile.TemporaryDirectory() + self.addCleanup(temp_dir.cleanup) + self.root = Path(temp_dir.name) + self.runner_temp = self.root / "runner" + self.auth_dir = self.runner_temp / "codeboarding-auth" + self.auth_dir.mkdir(parents=True) + self.checkout = self.root / "checkout" + (self.checkout / ".codeboarding").mkdir(parents=True) + + def _environ(self, proxy: FakeProxy | None, tier: str = "hosted", **extra: str) -> dict[str, str]: + (self.auth_dir / "tier").write_text(tier, encoding="utf-8") + environ = {"RUNNER_TEMP": str(self.runner_temp), "CHECKOUT_DIR": str(self.checkout), "DEPTH_CAP": "5"} + if proxy is not None: + environ |= { + "CODEBOARDING_PROXY_URL": proxy.url, + "ACTIONS_ID_TOKEN_REQUEST_URL": f"{proxy.url}/token", + "ACTIONS_ID_TOKEN_REQUEST_TOKEN": "request-token", + } + return environ | extra + + def _start(self, answer: dict | None, tier: str = "hosted", **extra: str): + proxy = FakeProxy(answer) + self.addCleanup(proxy.close) + log = io.StringIO() + with contextlib.redirect_stdout(log): + outputs, code = run_meter.start(self._environ(proxy, tier, **extra)) + self.assertEqual(code, 0, log.getvalue()) + return outputs, proxy.requests, log.getvalue() + + # -- the request ------------------------------------------------------- + + @NEEDS_JSONSCHEMA + def test_the_preflight_request_matches_the_contract_in_every_credential_mode(self) -> None: + (self.checkout / ".codeboarding" / "analysis.json").write_text('{"metadata": {"depth_cap": 2}}') + for tier, credential in (("hosted", "hosted"), ("license", "hosted"), ("byok", "own_key")): + with self.subTest(tier=tier): + _, requests, _ = self._start(example("run-start.allowed.json"), tier) + path, _, body = requests[0] + self.assertEqual(path, "/run/start") + self.assertEqual(validate(body, "run-start.request.schema.json"), []) + self.assertEqual(body["credential"], credential) + self.assertEqual((body["depth"], body["baseline_depth"]), (5, 2)) + self.assertEqual(body["client"]["surface"], "action") + manifest = json.loads((ROOT / ".release-please-manifest.json").read_text()) + self.assertEqual(body["client"]["version"], manifest["."]) + + def test_an_unknown_baseline_depth_is_sent_as_null(self) -> None: + _, requests, _ = self._start(example("run-start.allowed.json")) + self.assertIsNone(requests[0][2]["baseline_depth"]) + + def test_a_licence_rides_in_the_bearer_only_on_the_license_tier(self) -> None: + (self.auth_dir / "license.txt").write_text("LIC-123", encoding="utf-8") + _, licensed, _ = self._start(example("run-start.allowed.json"), "license") + _, own_key, _ = self._start(example("run-start.allowed.json"), "byok+license") + self.assertEqual(licensed[0][1], "Bearer oidc-jwt~codeboarding-license~LIC-123") + self.assertEqual(own_key[0][1], "Bearer oidc-jwt") + + # -- the answer -------------------------------------------------------- + + @NEEDS_JSONSCHEMA + def test_every_contract_example_validates(self) -> None: + for path in sorted((CONTRACTS / "examples").glob("run-*.json")): + stem, _, _ = path.name.partition(".") + kind = "request" if ".request." in path.name else "response" + with self.subTest(example=path.name): + self.assertEqual(validate(example(path.name), f"{stem}.{kind}.schema.json"), []) + + def test_every_run_start_answer_becomes_step_outputs(self) -> None: + for path in sorted((CONTRACTS / "examples").glob("run-start.*.json")): + if ".request." in path.name: + continue + answer = example(path.name) + with self.subTest(example=path.name): + outputs, _, _ = self._start(answer) + self.assertEqual(outputs["allowed"], str(answer["allowed"]).lower()) + self.assertEqual(outputs["depth_cap"], str(min(5, answer["depth_cap"]))) + self.assertEqual(outputs["run_id"], answer["run_id"] or "") + self.assertEqual(outputs["full_analysis"], str(answer["full_analysis"]).lower()) + self.assertEqual(outputs["mode"], answer["mode"]) + self.assertEqual(outputs["wall_message"], (answer["wall"] or {}).get("message", "")) + run_id_file = self.auth_dir / "run-id" + self.assertEqual(run_id_file.read_text() if run_id_file.exists() else None, answer["run_id"]) + + def test_the_proxy_can_only_lower_the_depth(self) -> None: + outputs, _, _ = self._start(example("run-start.legacy.json") | {"depth_cap": 10}, DEPTH_CAP="4") + self.assertEqual(outputs["depth_cap"], "4") + + def test_an_unreachable_proxy_fails_open_at_three_levels(self) -> None: + environ = self._environ(None) | { + "CODEBOARDING_PROXY_URL": "http://127.0.0.1:9", + "ACTIONS_ID_TOKEN_REQUEST_URL": "http://127.0.0.1:9/token", + "ACTIONS_ID_TOKEN_REQUEST_TOKEN": "request-token", + "GITHUB_OUTPUT": str(self.root / "github-output"), + "PATH": os.environ["PATH"], + } + (self.auth_dir / "run-id").write_text("from-an-earlier-invocation") + result = subprocess.run( + [sys.executable, str(SCRIPT), "start"], env=environ, capture_output=True, text=True, check=False + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("::warning", result.stdout) + outputs = dict(line.split("=", 1) for line in (self.root / "github-output").read_text().splitlines()) + self.assertEqual((outputs["allowed"], outputs["depth_cap"], outputs["run_id"]), ("true", "3", "")) + self.assertFalse((self.auth_dir / "run-id").exists(), "no hosted call may carry a stale run id") + + def test_an_answer_that_breaks_the_contract_fails_open(self) -> None: + outputs, _, log = self._start({"error": {"message": "Internal"}}) + self.assertEqual((outputs["allowed"], outputs["depth_cap"]), ("true", "3")) + self.assertIn("::warning", log) + + def test_a_job_without_id_token_permission_skips_the_check_and_keeps_its_depth(self) -> None: + log = io.StringIO() + with contextlib.redirect_stdout(log): + outputs, code = run_meter.start(self._environ(None, "byok")) + self.assertEqual(code, 0) + self.assertIn("id-token: write", log.getvalue()) + self.assertEqual((outputs["allowed"], outputs["depth_cap"], outputs["run_id"]), ("true", "5", "")) + + def test_a_depth_that_is_not_a_positive_integer_stops_the_run(self) -> None: + with contextlib.redirect_stdout(io.StringIO()): + _, code = run_meter.start(self._environ(None, DEPTH_CAP="0")) + self.assertEqual(code, 1) + + # -- the finish -------------------------------------------------------- + + def _finish(self, **environ: str) -> tuple[dict, str]: + proxy = FakeProxy(example("run-finish.charged.json")) + self.addCleanup(proxy.close) + log = io.StringIO() + with contextlib.redirect_stdout(log): + code = run_meter.finish(self._environ(proxy, RUN_ID="github:o/r#1", **environ)) + self.assertEqual(code, 0) + return proxy.requests[0][2], log.getvalue() + + def _marked(self, *, rewritten: bool) -> dict[str, str]: + """A map seeded before analyze.sh's marker, then rewritten by the engine or not.""" + analysis = self.root / "head-state" / "analysis.json" + analysis.parent.mkdir() + analysis.write_text("{}") + time.sleep(0.01) + marker = self.runner_temp / "codeboarding-map" + marker.write_text(str(analysis)) + if rewritten: + time.sleep(0.01) + analysis.write_text('{"components": []}') + return {"MAP_MARKER": str(marker)} + + @NEEDS_JSONSCHEMA + def test_the_finish_request_matches_the_contract(self) -> None: + body, _ = self._finish(JOB_STATUS="failure") + self.assertEqual(validate(body, "run-finish.request.schema.json"), []) + + def test_a_written_map_is_produced(self) -> None: + analysis = self.root / "analysis.json" + analysis.write_text("{}") + body, log = self._finish(ANALYSIS_PATH=str(analysis), JOB_STATUS="success") + self.assertEqual(body["outcome"], "produced") + self.assertIn("charged=True", log) + + def test_a_crash_after_the_map_was_written_is_still_produced(self) -> None: + body, _ = self._finish(JOB_STATUS="failure", **self._marked(rewritten=True)) + self.assertEqual(body["outcome"], "produced") + + def test_the_seed_a_run_started_from_is_not_its_map(self) -> None: + body, _ = self._finish(JOB_STATUS="failure", **self._marked(rewritten=False)) + self.assertEqual(body["outcome"], "failed") + + def test_a_cancelled_job_without_a_map_is_cancelled(self) -> None: + body, _ = self._finish(JOB_STATUS="cancelled", ANALYSIS_PATH="") + self.assertEqual(body["outcome"], "cancelled") + + def test_an_unreachable_proxy_never_fails_the_finish(self) -> None: + environ = self._environ(None) | { + "RUN_ID": "github:o/r#1", + "CODEBOARDING_PROXY_URL": "http://127.0.0.1:9", + "ACTIONS_ID_TOKEN_REQUEST_URL": "http://127.0.0.1:9/token", + "ACTIONS_ID_TOKEN_REQUEST_TOKEN": "request-token", + } + log = io.StringIO() + with contextlib.redirect_stdout(log): + self.assertEqual(run_meter.finish(environ), 0) + self.assertIn("::warning", log.getvalue()) + + +if __name__ == "__main__": + unittest.main() From 6f552db09be57c758505152a9b205f14ec7087ae Mon Sep 17 00:00:00 2001 From: Svilen Stefanov Date: Thu, 24 Sep 2026 01:24:13 +0200 Subject: [PATCH 2/2] fix: pack a staged licence on own-key runs and re-arm the map marker before a full fallback Review round on the preflight: - A byok+license run now stages its licence for the preflight, which packs it into the /run/start bearer whenever license.txt exists; the credential stays own_key and the relay that would spend it is still never started. Without it those holders read as Free at the proxy. - analyze.sh writes the codeboarding-map marker again before a full fallback, so a map an incremental pass wrote never counts as produced once the pass replacing it fails or is walled. - FULL_ANALYSIS is no longer wired into analyze.sh: it was only consulted where the baseline's cap already equalled the run's, so it could never change anything. A depth above the baseline's already forces the full rebuild (the cap is part of the state identity and is checked against the stored metadata). - README: pinned pre-release Action versions fail hosted calls once plans are enforced. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 8 ++++-- action.yml | 1 - scripts/action/analyze.sh | 9 ++++-- scripts/action/credential_check.py | 14 +++++----- scripts/action/run_meter.py | 2 +- tests/test_action_auth.py | 4 ++- tests/test_action_inputs.py | 4 --- tests/test_action_state.py | 44 ++++++++++++++++++++++++------ tests/test_run_meter.py | 14 ++++++---- 9 files changed, 67 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 955a0a6..3fb6290 100644 --- a/README.md +++ b/README.md @@ -150,8 +150,9 @@ The same rule makes the combinations explicit rather than order-dependent: A licence alongside your own key is deliberately allowed: it says "my CodeBoarding plan, my own tokens". **Your key always wins.** A direct provider call never reaches -CodeBoarding, so the licence is recorded and reported but not spent; the run is still -checked and counted like any own-key run (see [Plans and allowances](#plans-and-allowances)). The job summary says so on every run, rather than leaving you to +CodeBoarding, so the licence is never spent on a model call. It only tells the run's +check whose plan the run is on, and the run is counted like any own-key run (see +[Plans and allowances](#plans-and-allowances)). The job summary says so on every run, rather than leaving you to infer it from the tier name. ### Plans and allowances @@ -187,6 +188,9 @@ ahead and how deep it may go. A final step reports whether a map was produced. warning. - **CodeBoarding down is never your problem.** If the proxy cannot be reached, the run goes ahead at up to 3 levels with a warning. +- **Update pinned versions.** Action versions from before this release do not start runs + with the proxy, so their hosted calls fail once it enforces plans. `@v1` gets this + release automatically; a workflow pinned to an older tag or SHA must update. `license_key` is deprecated: plans now follow your GitHub account. It keeps working until the license key cutoff and is ignored after it; each run that sets it says so. Link the key diff --git a/action.yml b/action.yml index 395575f..0ec752b 100644 --- a/action.yml +++ b/action.yml @@ -543,7 +543,6 @@ runs: GIT_TOKEN: ${{ inputs.github_token }} GITHUB_SERVER_URL: ${{ github.server_url }} DEPTH_CAP: ${{ steps.preflight.outputs.depth_cap }} - FULL_ANALYSIS: ${{ steps.preflight.outputs.full_analysis }} MODEL: ${{ inputs.model }} AGENT_MODEL_INPUT: ${{ inputs.agent_model }} PARSING_MODEL_INPUT: ${{ inputs.parsing_model }} diff --git a/scripts/action/analyze.sh b/scripts/action/analyze.sh index b9c7fb7..4b3c85f 100755 --- a/scripts/action/analyze.sh +++ b/scripts/action/analyze.sh @@ -44,7 +44,9 @@ print(metadata.get("depth_cap", ""))' "$analysis" 2>/dev/null || true # Names the analysis this run is about to write, and when it started writing it, for the # finish step: an engine that crashes on shutdown after writing the map loses this step's -# outputs, but the run still produced a map and is charged for it. +# outputs, but the run still produced a map and is charged for it. Called again before a +# full fallback, so a map the incremental pass wrote does not count once the pass that +# replaces it fails or is walled. mark_map() { printf '%s' "$1" > "$RUNNER_TEMP/codeboarding-map" } @@ -137,6 +139,7 @@ analyze_sync() { else incremental "$CHECKOUT_DIR" "$state" if [ "$REQUIRES_FULL" = true ]; then + mark_map "$state/analysis.json" full "$CHECKOUT_DIR" "$state" "$DEPTH_CAP" fi fi @@ -193,8 +196,7 @@ analyze_review() { git -C "$CHECKOUT_DIR" worktree add --detach "$base_checkout" "$REVIEW_BASE_SHA" >/dev/null seed_state "$base_checkout" "$base_state" REQUIRES_FULL=true - # FULL_ANALYSIS is the proxy saying this run may go deeper than the baseline was drawn. - if [ "$(depth_cap_from "$base_state/analysis.json")" = "$DEPTH_CAP" ] && [ "${FULL_ANALYSIS:-false}" != true ]; then + if [ "$(depth_cap_from "$base_state/analysis.json")" = "$DEPTH_CAP" ]; then incremental "$base_checkout" "$base_state" fi if [ "$REQUIRES_FULL" = true ]; then @@ -225,6 +227,7 @@ analyze_review() { mark_map "$head_state/analysis.json" incremental "$CHECKOUT_DIR" "$head_state" if [ "$REQUIRES_FULL" = true ]; then + mark_map "$head_state/analysis.json" full "$CHECKOUT_DIR" "$head_state" "$DEPTH_CAP" fi diff --git a/scripts/action/credential_check.py b/scripts/action/credential_check.py index 8bd1949..92372b3 100755 --- a/scripts/action/credential_check.py +++ b/scripts/action/credential_check.py @@ -302,13 +302,13 @@ def resolve(table: dict, environ: dict[str, str]) -> dict: env = _resolve_byok(table, name, given, environ) # A licence alongside a provider key is deliberately allowed, not an error: it says - # "my CodeBoarding plan, my own tokens". Nothing enforces it yet -- direct provider - # calls never reach our proxy -- so it is recorded for the surfaces that read it. - return { - "tier": "byok+license" if license_key else "byok", - "provider": name, - "env": env, - } + # "my CodeBoarding plan, my own tokens". Direct provider calls never reach our proxy, so + # it is never spent on a model call; it is staged only for the run's preflight, which + # tells the proxy whose plan this run is on. + plan = {"tier": "byok+license" if license_key else "byok", "provider": name, "env": env} + if license_key: + plan["license"] = license_key + return plan def _is_endpoint(var: str) -> bool: diff --git a/scripts/action/run_meter.py b/scripts/action/run_meter.py index d6e361d..917c499 100755 --- a/scripts/action/run_meter.py +++ b/scripts/action/run_meter.py @@ -109,7 +109,7 @@ def start(environ: dict[str, str]) -> tuple[dict[str, str], int]: environ, "/run/start", start_request(environ, depth, tier), - license_file if tier == "license" and license_file.is_file() else None, + license_file if license_file.is_file() else None, ) allowed, cap = answer["allowed"], answer["depth_cap"] if not isinstance(allowed, bool) or not isinstance(cap, int) or cap < 1: diff --git a/tests/test_action_auth.py b/tests/test_action_auth.py index 53829dd..5b74d1b 100644 --- a/tests/test_action_auth.py +++ b/tests/test_action_auth.py @@ -335,7 +335,9 @@ def test_your_own_key_wins_outright_when_a_licence_is_wired_beside_it(self) -> N self.assertEqual(result.returncode, 0, result.stderr or result.stdout) self.assertEqual(outputs["tier"], "byok+license") self.assertEqual((auth_dir / "env" / "ANTHROPIC_API_KEY").read_text(), "my-own-key") - self.assertFalse((auth_dir / "license.txt").exists(), "no licence is staged for the relay") + # Staged for the preflight, which tells the proxy whose plan the run is on; the relay + # that would spend it on a model call is never started (asserted below). + self.assertEqual((auth_dir / "license.txt").read_text(), "a-licence") self.assertNotIn("a-licence", (auth_dir / "env" / "ANTHROPIC_API_KEY").read_text()) configured = subprocess.run( diff --git a/tests/test_action_inputs.py b/tests/test_action_inputs.py index 139864a..4e2490c 100644 --- a/tests/test_action_inputs.py +++ b/tests/test_action_inputs.py @@ -75,10 +75,6 @@ def test_depth_is_wired_to_state_identity_and_both_analysis_modes(self) -> None: preflight = ACTION[ACTION.index("id: preflight") :] self.assertIn("DEPTH_CAP: ${{ inputs.depth_cap }}", preflight[: preflight.index("\n run:")]) self.assertEqual(ACTION.count("${{ inputs.depth_cap }}"), 1, "only the preflight reads the input") - review = ACTION[ACTION.index("id: review_analyze") :] - self.assertIn( - "FULL_ANALYSIS: ${{ steps.preflight.outputs.full_analysis }}", review[: review.index("\n run:")] - ) def test_the_preflight_runs_before_the_engine_install_in_every_mode(self) -> None: start = ACTION.index("- name: Start the run with CodeBoarding") diff --git a/tests/test_action_state.py b/tests/test_action_state.py index 13a8717..e6bf574 100644 --- a/tests/test_action_state.py +++ b/tests/test_action_state.py @@ -2,6 +2,7 @@ from __future__ import annotations +import importlib.util import json import os import subprocess @@ -11,6 +12,10 @@ ROOT = Path(__file__).resolve().parent.parent +_SPEC = importlib.util.spec_from_file_location("run_meter", ROOT / "scripts" / "action" / "run_meter.py") +assert _SPEC and _SPEC.loader +run_meter = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(run_meter) STATE_NAMES = ROOT / "scripts" / "action" / "state-names.sh" ANALYZE = ROOT / "scripts" / "action" / "analyze.sh" @@ -30,12 +35,18 @@ analysis = os.path.join(output, "analysis.json") metadata = json.load(open(analysis))["metadata"] if os.path.isfile(analysis) else {} if argv[0] == "incremental" and os.environ.get("CB_REQUIRE_FULL") == "true": + if os.environ.get("CB_INCREMENTAL_WRITES") == "true": + json.dump({"metadata": metadata, "components": []}, open(analysis, "w")) print(json.dumps({"requiresFullAnalysis": True})) sys.exit(0) if argv[0] == "full": + if os.environ.get("CB_FULL_CRASHES") == "before_write": + sys.exit(1) metadata = {"depth_cap": int(argv[argv.index("--depth-cap") + 1])} with open(analysis, "w") as handle: json.dump({"metadata": metadata, "components": [], "components_relations": []}, handle) +if argv[0] == "full" and os.environ.get("CB_FULL_CRASHES") == "after_write": + sys.exit(139) print(json.dumps({"requiresFullAnalysis": False, "analysis_path": analysis})) ''' @@ -175,7 +186,7 @@ def setUp(self) -> None: def tearDown(self) -> None: self.temp_dir.cleanup() - def _analyze(self, **extra: str) -> dict[str, str]: + def _analyze(self, *, check: bool = True, **extra: str) -> dict[str, str]: self.output.write_text("", encoding="utf-8") result = subprocess.run( [str(ANALYZE)], @@ -203,7 +214,8 @@ def _analyze(self, **extra: str) -> dict[str, str]: text=True, check=False, ) - self.assertEqual(result.returncode, 0, result.stderr or result.stdout) + if check: + self.assertEqual(result.returncode, 0, result.stderr or result.stdout) values: dict[str, str] = {} for line in self.output.read_text(encoding="utf-8").splitlines(): key, _, value = line.partition("=") @@ -286,13 +298,27 @@ def test_compatible_committed_baseline_runs_incrementally(self) -> None: self._analyze(REVIEW_BASE_SHA=sha, DEPTH_CAP="4") self.assertEqual([c["mode"] for c in self._engine_calls()], ["incremental", "incremental"]) - def test_the_proxy_asking_for_a_full_analysis_rebuilds_the_committed_baseline(self) -> None: - """A Pro author may go deeper than the default branch was drawn: the review then runs - in full at the author's cap, never incrementally from the shallower baseline.""" - sha = self._commit_base(cap=4) - self._analyze(REVIEW_BASE_SHA=sha, DEPTH_CAP="4", FULL_ANALYSIS="true") - self.assertEqual([c["mode"] for c in self._engine_calls()], ["full", "incremental"]) - self.assertEqual(self._engine_calls()[0]["depth"], "4") + def _map_written(self) -> bool: + """The finish step's verdict on what this analysis left behind.""" + return run_meter.map_written({"ANALYSIS_PATH": "", "MAP_MARKER": str(self.runner_temp / "codeboarding-map")}) + + def test_a_full_fallback_that_crashes_after_writing_the_map_still_produced_it(self) -> None: + _state(self.base_dir, cap=4) + self._analyze(check=False, DEPTH_CAP="4", CB_REQUIRE_FULL="true", CB_FULL_CRASHES="after_write") + self.assertTrue(self._map_written()) + + def test_an_incremental_map_does_not_count_when_the_full_fallback_fails(self) -> None: + """A 402 mid-run fails the pass the same way, so this also covers a walled fallback.""" + _state(self.base_dir, cap=4) + self._analyze( + check=False, + DEPTH_CAP="4", + CB_REQUIRE_FULL="true", + CB_INCREMENTAL_WRITES="true", + CB_FULL_CRASHES="before_write", + ) + self.assertEqual([c["mode"] for c in self._engine_calls()], ["incremental", "full"]) + self.assertFalse(self._map_written()) def test_the_head_analysis_is_named_for_the_finish_step_before_it_runs(self) -> None: _state(self.base_dir, cap=4) diff --git a/tests/test_run_meter.py b/tests/test_run_meter.py index 3cf869c..0ce657c 100644 --- a/tests/test_run_meter.py +++ b/tests/test_run_meter.py @@ -144,12 +144,16 @@ def test_an_unknown_baseline_depth_is_sent_as_null(self) -> None: _, requests, _ = self._start(example("run-start.allowed.json")) self.assertIsNone(requests[0][2]["baseline_depth"]) - def test_a_licence_rides_in_the_bearer_only_on_the_license_tier(self) -> None: + def test_a_staged_licence_rides_in_the_bearer_on_the_license_and_own_key_tiers(self) -> None: + """Otherwise a licence holder on their own key reads as Free at the proxy.""" + _, bare, _ = self._start(example("run-start.allowed.json"), "byok") (self.auth_dir / "license.txt").write_text("LIC-123", encoding="utf-8") - _, licensed, _ = self._start(example("run-start.allowed.json"), "license") - _, own_key, _ = self._start(example("run-start.allowed.json"), "byok+license") - self.assertEqual(licensed[0][1], "Bearer oidc-jwt~codeboarding-license~LIC-123") - self.assertEqual(own_key[0][1], "Bearer oidc-jwt") + for tier, credential in (("license", "hosted"), ("byok+license", "own_key")): + with self.subTest(tier=tier): + _, requests, _ = self._start(example("run-start.allowed.json"), tier) + self.assertEqual(requests[0][1], "Bearer oidc-jwt~codeboarding-license~LIC-123") + self.assertEqual(requests[0][2]["credential"], credential) + self.assertEqual(bare[0][1], "Bearer oidc-jwt") # -- the answer --------------------------------------------------------