diff --git a/.github/workflows/build-boards.yml b/.github/workflows/build-boards.yml index bf1734a19c2..016a2f2f32e 100644 --- a/.github/workflows/build-boards.yml +++ b/.github/workflows/build-boards.yml @@ -94,6 +94,17 @@ jobs: name: ${{ matrix.board }} path: bin/${{ matrix.board }} + - name: Upload firmware sizes + # "zz-" sorts the size records after every board in the run's artifact list. + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: zz-sizes-${{ matrix.board }} + path: sizes/${{ matrix.board }}.json + if-no-files-found: ignore + # build-ci merges these into the 0-sizes reports; delete them after one day. + retention-days: 1 + - name: Upload to S3 uses: ./.github/actions/upload_aws with: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e250c8debd7..f0f57a74243 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -241,10 +241,90 @@ jobs: build-ci: # Final gate for ruleset checks: always runs so the required check is # reported, and fails if any job (or matrix leg) failed or was cancelled. - needs: [tests, zephyr-tests, mpy-cross, mpy-cross-mac, docs, ports] + needs: [scheduler, tests, zephyr-tests, mpy-cross, mpy-cross-mac, docs, ports] if: ${{ always() }} runs-on: ubuntu-24.04 + env: + CP_VERSION: ${{ needs.scheduler.outputs.cp-version }} steps: + # Firmware size report. Each board job uploads its sizes/.json as zz-sizes-; + # merge them into one table and a job summary. This is reporting only, so none of it + # may fail the gate: every step continues on error. + - name: Set up repository + continue-on-error: true + uses: actions/checkout@v6 + with: + submodules: false + show-progress: false + fetch-depth: 1 + persist-credentials: false + + # actions/download-artifact matches a pattern against a listing of the run's + # artifacts that stops at 1000, and a full build has more. The script fetches each + # board's record by exact name instead, from the board list the scheduler produced. + # It runs from github-script because only action steps get the artifact API token. + - name: Install the artifact client + continue-on-error: true + run: npm install --no-save --no-package-lock --no-audit --no-fund @actions/artifact@6 + + - name: Download firmware sizes + continue-on-error: true + uses: actions/github-script@v9 + env: + BOARDS: ${{ needs.scheduler.outputs.ports }} + with: + script: | + const { pathToFileURL } = require("node:url"); + const sizes = await import(pathToFileURL(`${process.env.GITHUB_WORKSPACE}/tools/ci_download_sizes.mjs`)); + await sizes.downloadSizes({ + boards: sizes.boardsFromSchedule(JSON.parse(process.env.BOARDS || "{}")), + outDir: "sizes", + info: core.info, + warning: core.warning, + }); + + - name: Summarize firmware sizes + continue-on-error: true + run: python3 tools/ci_firmware_sizes.py sizes sizes-report + + # Uploaded unzipped, one file per artifact, so each opens directly; with + # `archive: false` the file name is the artifact name. + - name: Upload firmware size report (HTML) + continue-on-error: true + uses: actions/upload-artifact@v7 + with: + path: sizes-report/0-sizes.html + archive: false + if-no-files-found: ignore + + - name: Upload firmware size report (JSON) + continue-on-error: true + uses: actions/upload-artifact@v7 + with: + path: sizes-report/0-sizes.json + archive: false + if-no-files-found: ignore + + # The action itself only uploads on pushes to main and release branches and on + # published releases. The S3 names carry the version, so the directory keeps a history. + - name: Upload firmware size report to S3 (HTML) + continue-on-error: true + uses: ./.github/actions/upload_aws + with: + source: sizes-report/0-sizes.html + destination: 0-sizes/adafruit-circuitpython-sizes-${{ env.CP_VERSION }}.html + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + - name: Upload firmware size report to S3 (JSON) + continue-on-error: true + uses: ./.github/actions/upload_aws + with: + source: sizes-report/0-sizes.json + destination: 0-sizes/adafruit-circuitpython-sizes-${{ env.CP_VERSION }}.json + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + - name: Check build results run: | echo "Job results: ${{ toJSON(needs.*.result) }}" diff --git a/.gitignore b/.gitignore index 659801277f3..ec4811a6e30 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,8 @@ dist/ ###################### build/ bin/ +sizes/ +node_modules/ circuitpython-stubs/ test-stubs/ build-*/ diff --git a/tools/build_release_files.py b/tools/build_release_files.py index b65b2aeb723..ac6580063de 100755 --- a/tools/build_release_files.py +++ b/tools/build_release_files.py @@ -70,6 +70,29 @@ def flash_usage(port, build_dir): return None, None +def record_size(port, board, language, used, region, status, source): + """Record one language build's flash usage in sizes/.json. + + CI uploads the file as its own small artifact so a later job can gather every + board's sizes without scraping logs. `source` is "measured" for a linked build and + "predicted" for a language skipped by the size prediction; `used` is None, written as + null, when the build failed or the port does not report its flash region. + """ + os.makedirs("../sizes", exist_ok=True) + path = f"../sizes/{board}.json" + try: + with open(path, "r") as f: + record = json.load(f) + except FileNotFoundError: + record = {"port": port, "board": board, "region": None, "languages": {}} + if region is not None: + record["region"] = region + record["languages"][language] = {"used": used, "status": status, "source": source} + with open(path, "w") as f: + json.dump(record, f, indent=1) + f.write("\n") + + def c_array_bytes(path): """Sum the bytes of the const arrays initialised in a generated C file.""" if not path.exists(): @@ -218,6 +241,15 @@ def generate_translation(port, board, build_dir, language): flush=True, ) if skip: + record_size( + board_info["port"], + board, + language, + predicted_flash, + flash_region, + "skipped", + "predicted", + ) continue make_result = subprocess.run( @@ -275,19 +307,32 @@ def generate_translation(port, board, build_dir, language): print(make_result.stdout.decode("utf-8")) print(other_output) - if predicted_flash is not None and make_result.returncode == 0: - actual_flash, _ = flash_usage(board_info["port"], build_dir) - if actual_flash is not None: - print( - "Flash size check {board} {language}: predicted {predicted}," - " actual {actual}, error {error:+d}".format( - board=board, - language=language, - predicted=predicted_flash, - actual=actual_flash, - error=predicted_flash - actual_flash, - ) + # Languages share a build directory, so a failed link leaves the previous + # language's firmware.size.json behind; only trust it after a successful build. + actual_flash, actual_region = None, None + if make_result.returncode == 0: + actual_flash, actual_region = flash_usage(board_info["port"], build_dir) + record_size( + board_info["port"], + board, + language, + actual_flash, + actual_region, + "succeeded" if make_result.returncode == 0 else "failed", + "measured", + ) + + if predicted_flash is not None and actual_flash is not None: + print( + "Flash size check {board} {language}: predicted {predicted}," + " actual {actual}, error {error:+d}".format( + board=board, + language=language, + predicted=predicted_flash, + actual=actual_flash, + error=predicted_flash - actual_flash, ) + ) # Flush so we will see something before 10 minutes has passed. print(flush=True) diff --git a/tools/ci_download_sizes.mjs b/tools/ci_download_sizes.mjs new file mode 100755 index 00000000000..4e199ab368c --- /dev/null +++ b/tools/ci_download_sizes.mjs @@ -0,0 +1,57 @@ +// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) +// +// SPDX-License-Identifier: MIT + +// Download every board's firmware size record from this run's artifacts, by exact name. +// +// build_release_files.py records a board's sizes in sizes/.json and the board job +// uploads it as the artifact zz-sizes-. actions/download-artifact matches a name +// pattern against a listing of the run's artifacts, and that listing stops at 1000, which +// a full build exceeds. Looking each record up by name has no such limit and uses the same +// internal artifact API, so it costs nothing against the REST rate limit. +// +// That API needs the runner's ACTIONS_RUNTIME_TOKEN, which only `uses:` action steps get, +// so build.yml calls downloadSizes() from an actions/github-script step rather than from +// `node` in a `run:` step. @actions/artifact must be installed where this file can resolve +// it: `npm install` at the repository root. + +import { DefaultArtifactClient } from "@actions/artifact"; + +// The boards in the scheduler job's "ports" output: {: [boards...], ports: [...]}. +export function boardsFromSchedule(schedule) { + return (schedule.ports ?? []).flatMap((port) => schedule[port]); +} + +export async function downloadSizes({ boards, outDir, info = console.log, warning = console.warn }) { + const client = new DefaultArtifactClient(); + const missing = []; + let downloaded = 0; + + async function fetchRecord(board) { + try { + const { artifact } = await client.getArtifact(`zz-sizes-${board}`); + await client.downloadArtifact(artifact.id, { path: outDir }); + downloaded++; + } catch (error) { + missing.push(board); + info(`No size record for ${board}: ${error.message}`); + } + } + + // A few downloads at a time: enough to get through a full build in a minute or two, + // few enough not to trip the artifact service's throttling. + const queue = [...boards]; + await Promise.all( + Array.from({ length: 8 }, async () => { + while (queue.length) { + await fetchRecord(queue.shift()); + } + }), + ); + + info(`Downloaded ${downloaded} of ${boards.length} size records to ${outDir}`); + if (missing.length) { + warning(`No firmware size record for ${missing.length} board(s): ${missing.join(" ")}`); + } + return { downloaded, missing }; +} diff --git a/tools/ci_firmware_sizes.py b/tools/ci_firmware_sizes.py new file mode 100755 index 00000000000..ad8be014c21 --- /dev/null +++ b/tools/ci_firmware_sizes.py @@ -0,0 +1,380 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) +# +# SPDX-License-Identifier: MIT + +"""Merge the per-board firmware size records of a CI run into one report. + +build_release_files.py writes sizes/.json for every board it builds, and the board +job uploads it as the artifact zz-sizes-. The build-ci job downloads those into one +directory and runs: + + ci_firmware_sizes.py + +which writes + + /0-sizes.json every board with its per-language builds and derived free flash + /0-sizes.html a sortable, filterable table of the same boards, data embedded + +The names match the artifacts build-ci uploads them as, one file each, unzipped. The +CircuitPython version, from --version or $CP_VERSION, goes into the reports. The script +also appends a line to $GITHUB_STEP_SUMMARY when that is set, or to --summary FILE, +saying how many boards were built and where the reports are. The run description in the +reports comes from the GITHUB_* environment variables; outside CI the report says so. +""" + +import argparse +import collections +import datetime +import html +import json +import os +import pathlib +import sys + +# Both reports share this name; build-ci uploads each as an artifact of the same name. +REPORT_NAME = "0-sizes" + + +def load_records(records_dir): + """Read every sizes/.json under records_dir, keyed by board.""" + records = {} + for path in sorted(pathlib.Path(records_dir).rglob("*.json")): + with path.open("r") as f: + record = json.load(f) + if not isinstance(record, dict) or "languages" not in record: + print(f"Skipping {path}: not a size record", file=sys.stderr) + continue + records[record["board"]] = record + return records + + +def summarize(record): + """Add the derived fields the table and the summary sort and colour by.""" + region = record.get("region") + languages = record["languages"] + board = { + "port": record["port"], + "board": record["board"], + "region": region, + "en_US": (languages.get("en_US") or {}).get("used"), + "largest_language": None, + "largest_used": None, + "largest_source": None, + "min_free": None, + "pct": None, + "measured": sum(1 for b in languages.values() if b["source"] == "measured"), + "predicted": sum(1 for b in languages.values() if b["source"] == "predicted"), + "failed": sorted(lang for lang, b in languages.items() if b["status"] == "failed"), + "languages": languages, + } + sized = [(b["used"], lang) for lang, b in languages.items() if b["used"] is not None] + if sized: + used, lang = max(sized) + board["largest_language"] = lang + board["largest_used"] = used + board["largest_source"] = languages[lang]["source"] + if region: + board["min_free"] = region - used + board["pct"] = round(100.0 * used / region, 2) + return board + + +def run_info(version): + """Where the numbers came from, from the variables GitHub Actions sets.""" + env = os.environ + info = { + "version": version, + "repository": env.get("GITHUB_REPOSITORY"), + "run_id": env.get("GITHUB_RUN_ID"), + "event": env.get("GITHUB_EVENT_NAME"), + "ref": env.get("GITHUB_REF_NAME"), + "sha": env.get("GITHUB_SHA"), + "url": None, + } + if info["repository"] and info["run_id"]: + server = env.get("GITHUB_SERVER_URL", "https://github.com") + info["url"] = f"{server}/{info['repository']}/actions/runs/{info['run_id']}" + return info + + +def describe_run(info): + """One sentence for the report header.""" + if not info["url"]: + return "Built outside GitHub Actions." + parts = [f"{info['event']} build" if info["event"] else "build"] + if info["version"]: + parts.append(f"of CircuitPython {info['version']}") + elif info["ref"]: + parts.append(f"of {info['ref']}") + if info["sha"]: + parts.append(f"at {info['sha'][:10]}") + return " ".join(parts) + + +def sort_key(board): + # Fullest first; boards without a measurable region go last, alphabetically. + return (board["pct"] is None, -(board["pct"] or 0), board["board"]) + + +def write_json(boards, info, out_dir): + document = { + "generated": datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds"), + "run": info, + "boards": boards, + } + with (out_dir / f"{REPORT_NAME}.json").open("w") as f: + json.dump(document, f, indent=1) + f.write("\n") + + +HTML_TEMPLATE = r""" + + + + +Firmware Sizes + + + +

Firmware sizes

+

__RUN__ __COUNTS__ +"Largest" is the language build that uses the most flash, which is what has to fit. +A size in grey was predicted for a language that was not built. +Click a column header to sort.

+
+ + +
+
Free flash: + + + + + +
+
+
+ + + + + + + + + + + + +
PortBoardRegionen_US usedLargest langLargest usedMin free% fullLangs
+ + + +""" + + +def write_html(boards, info, out_dir): + if info["url"]: + run = 'From GitHub Actions run {run_id} ({desc}).'.format( + url=html.escape(info["url"], quote=True), + run_id=html.escape(info["run_id"]), + desc=html.escape(describe_run(info)), + ) + else: + run = html.escape(describe_run(info)) + with_region = sum(1 for b in boards if b["region"]) + counts = f"{len(boards)} boards, {with_region} with a measurable flash region." + # The table rows carry only what the page shows; the per-language detail is in + # the JSON report. "6}" + "".join(f" {label:>8}" for label in labels)) + for name in sorted(ports): + c = ports[name] + print(f"{name:16} {c['boards']:6}" + "".join(f" {c[label]:8}" for label in labels)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("records_dir", help="directory holding the per-board size records") + parser.add_argument("out_dir", help="where to write the JSON and HTML reports") + parser.add_argument( + "--version", + default=os.environ.get("CP_VERSION"), + help="CircuitPython version named in the reports (default: $CP_VERSION)", + ) + parser.add_argument( + "--summary", + default=os.environ.get("GITHUB_STEP_SUMMARY"), + help="append the markdown summary to this file (default: $GITHUB_STEP_SUMMARY)", + ) + args = parser.parse_args() + + records = load_records(args.records_dir) + boards = sorted((summarize(r) for r in records.values()), key=sort_key) + info = run_info(args.version) + + out_dir = pathlib.Path(args.out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + write_json(boards, info, out_dir) + write_html(boards, info, out_dir) + + summary = markdown_summary(boards) + if args.summary: + with open(args.summary, "a") as f: + f.write(summary) + else: + print(summary) + print_ports(boards) + + +if __name__ == "__main__": + main()