From 4a1f5eaadab841306292809b243342c14b79a923 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 27 Sep 2026 22:44:32 +0200 Subject: [PATCH 1/9] An image release: main's disk image once its nightly is green `cargo run -- --release-boot --build-only` builds `target/bootable-release.img`: the shipped config less every program whose package the licence gate holds pending the owner (doom), and so less the assets only doom opens (DOOM1.WAD and the SoundFont). `cargo run -- --ci release`, a new nightly job that needs every other lane green and runs on main alone, installs the toolchain, builds that image, and publishes it as `image-x86_64-<12 hex of the commit>`: the image gzipped, its SHA256SUMS, the notes as README.md, and ToyOS's licences, NOTICE and the texts NOTICE names for what the image ships. It keeps the newest seven image releases and deletes the rest with their tags. The notes print one QEMU command line per host, declared once in `imagerelease::Host::command` with QEMU's own edk2 firmware; the harness boots exactly those lines: `release_command_boots` to the desktop, and `release_writes_no_other_disk` beside an NVMe disk laid out as another operating system's and a stick with no table, comparing both byte for byte. Measured on the dev host before this commit: the macOS line over a copy of `target/bootable.img` early-panics on Homebrew QEMU 11.1.1's edk2 firmware (memory allocation of 4096 bytes failed, right after pmm), and reaches `compositor: ready` with the repository's `ovmf/` firmware in its place. Filed as issues/kernel/the-kernel-dies-at-boot-on-the-edk2-firmware-qemu-ships.md. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly.yml | 22 +- Cargo.toml | 4 +- ...the-latest-badge-from-the-image-release.md | 20 + ...arries-no-notice-of-the-crates-it-links.md | 19 + ...at-boot-on-the-edk2-firmware-qemu-ships.md | 34 ++ src/build.rs | 101 +++- src/ci.rs | 15 +- src/flags.rs | 2 + src/imagerelease.rs | 468 ++++++++++++++++++ src/lib.rs | 1 + src/licence.rs | 38 +- src/main.rs | 22 + src/sourcegate.rs | 6 + tests/common/mod.rs | 2 + tests/common/release.rs | 293 +++++++++++ tests/toyos.rs | 11 +- 16 files changed, 1040 insertions(+), 18 deletions(-) create mode 100644 issues/build/a-toolchain-release-takes-the-latest-badge-from-the-image-release.md create mode 100644 issues/build/the-image-release-carries-no-notice-of-the-crates-it-links.md create mode 100644 issues/kernel/the-kernel-dies-at-boot-on-the-edk2-firmware-qemu-ships.md create mode 100644 src/imagerelease.rs create mode 100644 tests/common/release.rs diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index cba134ab63..fef5722edb 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -63,7 +63,8 @@ jobs: with: fetch-depth: 0 - - name: disk and QEMU + - &runner-deps + name: disk and QEMU run: | sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc \ /usr/local/share/boost /usr/local/.ghcup @@ -105,7 +106,7 @@ jobs: for attempt in 1 2 3; do apt-get -o Acquire::Check-Valid-Until=false update -qq > /tmp/apt.log 2>&1 \ && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq git curl ca-certificates \ - zstd xz-utils build-essential qemu-system-x86 >> /tmp/apt.log 2>&1 \ + zstd xz-utils build-essential qemu-system-x86 ovmf >> /tmp/apt.log 2>&1 \ && break [ "$attempt" = 3 ] && { cat /tmp/apt.log; exit 1; } sleep 20 @@ -250,6 +251,23 @@ jobs: - *checkout - run: cargo build -p toyos-build + # The disk image a person downloads, of the commit every lane above passed; + # `src/imagerelease.rs` says what it carries and how many stay. Bare + # `ubuntu-24.04` for `gh`, as `build` is. + release: + needs: [host, build, guest, tcg, audio, portability-linux, portability-macos] + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-24.04 + timeout-minutes: 120 + permissions: + contents: write + steps: + - *checkout + - *runner-deps + - env: + GH_TOKEN: ${{ github.token }} + run: cargo run -- --ci release + # One standing issue, found by title and commented on; a dispatch is somebody # watching the run, so only the schedule files. nightly-red: diff --git a/Cargo.toml b/Cargo.toml index e71be2c363..36a154b9c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -160,6 +160,9 @@ image = { version = "0.25", default-features = false, features = ["jpeg"] } # (`src/fingerprint.rs`). Already resolved here through a dev-dependency, so # nothing new is fetched — but `cargo run` did not compile it before this. sha2 = "0.10" +# The image release's compressed asset, which every stock macOS and Linux +# unpacks (`src/imagerelease.rs`). Already resolved here through `bcachefs`. +flate2 = { version = "1", default-features = false, features = ["rust_backend"] } [dev-dependencies] toyos-cc = { path = "toyos-cc" } @@ -195,7 +198,6 @@ toyos-xhci = { path = "toyos-xhci" } # the guest's volume is compared with a third party's decoding of the committed # archive, never with `userland/pkg`'s own. The archive itself is committed # under `tests/fixtures` and no test fetches anything. -flate2 = { version = "1", default-features = false, features = ["rust_backend"] } tar = "0.4" [profile.release] diff --git a/issues/build/a-toolchain-release-takes-the-latest-badge-from-the-image-release.md b/issues/build/a-toolchain-release-takes-the-latest-badge-from-the-image-release.md new file mode 100644 index 0000000000..08896fb4fb --- /dev/null +++ b/issues/build/a-toolchain-release-takes-the-latest-badge-from-the-image-release.md @@ -0,0 +1,20 @@ +--- +status: open +kind: tooling +opened: 2026-09-27 +--- + +# A toolchain release takes GitHub's Latest badge from the image release + +`gh release create` in `src/release.rs` names no `--latest`, so GitHub marks +each new toolchain release Latest: every release this repository has published +is a toolchain's, and the newest carries the badge. The image release +(`src/imagerelease.rs`) is created `--latest`, and the next toolchain release +takes the badge back, so `/releases/latest` names a toolchain rather than the +image a person downloads until the next image release. + +Not changed beside the image release because `src/release.rs` is one of the +trees the toolchain's tag hashes (`TREES`): editing it publishes a toolchain. + +**Exit condition.** A toolchain release is created `--latest=false`, and +`/releases/latest` names the newest image release whenever there is one. diff --git a/issues/build/the-image-release-carries-no-notice-of-the-crates-it-links.md b/issues/build/the-image-release-carries-no-notice-of-the-crates-it-links.md new file mode 100644 index 0000000000..d668acdb10 --- /dev/null +++ b/issues/build/the-image-release-carries-no-notice-of-the-crates-it-links.md @@ -0,0 +1,19 @@ +--- +status: open +kind: defect +opened: 2026-09-27 +--- + +# The image release carries no notice of the crates its programs link + +The image release carries ToyOS's own licences, `NOTICE`, and the texts +`NOTICE` names for the fonts and icons on the image (`LICENCE_ASSETS` in +`src/imagerelease.rs`). Every program on the image is also compiled from +third-party crates, the ones `src/licence.rs` walks, and MIT, BSD and +Apache-2.0 each ask that a copy of the software carry the licence text, and +MIT and BSD its copyright notice too. `NOTICE` says those crates keep their own +upstream licences, and nothing the release publishes carries one. + +**Exit condition.** The release carries the licence text and copyright notice +of every package the licence gate finds in the release image, generated from +the gate's own walk, and a test fails when a shipped package has none. diff --git a/issues/kernel/the-kernel-dies-at-boot-on-the-edk2-firmware-qemu-ships.md b/issues/kernel/the-kernel-dies-at-boot-on-the-edk2-firmware-qemu-ships.md new file mode 100644 index 0000000000..41d3a0a0ab --- /dev/null +++ b/issues/kernel/the-kernel-dies-at-boot-on-the-edk2-firmware-qemu-ships.md @@ -0,0 +1,34 @@ +--- +status: open +kind: defect +opened: 2026-09-27 +--- + +# The kernel dies at boot on the edk2 firmware QEMU ships + +Booted with `edk2-x86_64-code.fd` and `edk2-i386-vars.fd` from Homebrew's QEMU +11.1.1 (`/opt/homebrew/share/qemu/`), the kernel panics right after the +physical memory manager comes up: + + [kernel 0.000 cpu0 boot] pmm: the firmware map calls 2140844032 bytes usable in 110 entries; managed=2017460224 withheld=94371840 unaligned=29011968, and the three sum to it; frames=962 reserved_frames=45 base=0x200000 span=1022 + [kernel 0.000 cpu0 boot] EARLY PANIC: panicked at library/alloc/src/alloc.rs:659:9: + memory allocation of 4096 bytes failed + +The same image under the same command line with `ovmf/OVMF_CODE-pure-efi.fd` +and `ovmf/OVMF_VARS-pure-efi.fd` in their place reaches the desktop +(`compositor: ready`). Two things the firmwares hand over differ in the two +logs: the memory map, 110 entries against 95, and the GOP framebuffer, at +`0x80000000` against `0xc0000000`. On the firmware that boots, the line after +`pmm:` is the framebuffer's mapping, `mmio: 0xc0000000+0x400000 PAT +WriteCombining`. + +The command line is `imagerelease::Host::MacosAppleSilicon`'s +(`src/imagerelease.rs`) with `-display none`, over a copy of a +`target/bootable.img`. + +It blocks the image release on macOS: the notes' macOS command line boots this +firmware, and so do `release_command_boots` and `release_writes_no_other_disk` +on the dev host. + +**Exit condition.** `release_command_boots` is green on the dev host with the +firmware Homebrew's QEMU ships. diff --git a/src/build.rs b/src/build.rs index 34dba99c00..3957af0280 100644 --- a/src/build.rs +++ b/src/build.rs @@ -169,6 +169,20 @@ fn parse_config(path: &Path) -> SystemConfig { .unwrap_or_else(|e| panic!("Failed to parse {}: {e}", path.display())) } +/// `config` less every program the licence gate names as a package pending +/// the owner. A program's key is its package's name +/// (`the_release_withholds_every_pending_package`); one another row starts or +/// receives stays named there, and `build_and_assemble` refuses the image. +fn withhold_pending(config: &mut SystemConfig) { + for subject in crate::licence::pending_owner() { + if let crate::licence::Subject::Crate(name) = subject { + if config.programs.remove(name).is_some() { + eprintln!("release: leaving out {name}, whose licence the owner has yet to rule on"); + } + } + } +} + // --- Freshness checking --- /// Fingerprint all external build dependencies that cargo cannot track: the @@ -1008,8 +1022,13 @@ pub struct Boot { /// yet — `issues/build/two-sequences-build-one-image.md` — and until they /// are, this is what keeps each artifact to a single writer. case: bool, + /// Leave out every program [`crate::licence::pending_owner`] names. + public: bool, } +/// What [`Boot::release`] writes. +pub const RELEASE_IMAGE: &str = "target/bootable-release.img"; + impl Boot { /// **The one naming rule**: the artifact is named after the directory /// holding the config, and the shipped config sits at the root and keeps @@ -1032,7 +1051,16 @@ impl Boot { Some(name) => format!("target/bootable-{}.img", name.to_string_lossy()), None => "target/bootable.img".to_string(), }; - Ok(Self { config: dir.join(CONFIG), image: PathBuf::from(image), case }) + Ok(Self { config: dir.join(CONFIG), image: PathBuf::from(image), case, public: false }) + } + + /// The config this boot builds. + fn system(&self) -> SystemConfig { + let mut config = parse_config(&self.config); + if self.public { + withhold_pending(&mut config); + } + config } /// The image `arch`'s build of this boot writes. x86-64 keeps the name @@ -1055,6 +1083,14 @@ impl Boot { Self::mode(root, root) } + /// The public release: the shipped config less every program whose + /// package the licence gate holds pending the owner's ruling, and so less + /// the assets only those programs open ([`assets::collect`]). Its own + /// artifact, because it is not the shipped image. + pub fn release(root: &Path) -> Self { + Self { image: PathBuf::from(RELEASE_IMAGE), public: true, ..Self::shipped(root) } + } + /// The config declares no `devices`, so nothing started there claims the /// framebuffer and the kernel's last boot checkpoint stays on screen. /// `screen_diag_boot` boots this same config, so the tested image and the @@ -1841,7 +1877,7 @@ fn shipped_parts(root: &Path, boot: &Boot, rebuild_toolchain: bool, plan: &Plan) let sysroot = toolchain::ensure(root, rebuild_toolchain, &mut lock); let env = GuestEnv::new(&sysroot); - let config = parse_config(&boot.config); + let config = boot.system(); invalidate_stale(root, &mut lock, &env.toolchain, &config_targets(root, &config)); @@ -2361,6 +2397,67 @@ fn collect_hosted_rustc(root: &Path, toolchain: &Path, root_files: &mut Vec<(Str mod tests { use super::*; + fn pending(of: fn(crate::licence::Subject) -> Option<&'static str>) -> Vec<&'static str> { + crate::licence::pending_owner().filter_map(of).collect() + } + + /// The release leaves out every program whose package the licence gate + /// holds pending the owner and keeps every other, and each withheld key is + /// its package's name, which is what the rule reads it by. + #[test] + fn the_release_withholds_every_pending_package() { + use crate::licence::Subject; + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let shipped = Boot::shipped(root).system(); + let release = Boot::release(root).system(); + let withheld = pending(|s| match s { + Subject::Crate(name) => Some(name), + _ => None, + }); + assert!(!withheld.is_empty(), "the gate holds no package pending, so this reads nothing"); + for name in &withheld { + let program = shipped + .programs + .get(*name) + .unwrap_or_else(|| panic!("the shipped config builds no {name}")); + let manifest = fs::read_to_string(program.crate_dir(root, name).join("Cargo.toml")).unwrap(); + let manifest: toml::Value = toml::from_str(&manifest).unwrap(); + assert_eq!(manifest["package"]["name"].as_str(), Some(*name)); + } + let kept: Vec<&String> = + shipped.programs.keys().filter(|key| !withheld.contains(&key.as_str())).collect(); + assert_eq!(release.programs.keys().collect::>(), kept); + assert_eq!(Boot::release(root).image_for(Arch::X86_64), PathBuf::from(RELEASE_IMAGE)); + assert_eq!(Boot::release(root).config, Boot::shipped(root).config); + } + + /// No file the licence gate holds pending the owner is on the release's + /// ROOT, and every one is on the shipped image's. + #[test] + fn the_release_image_carries_no_pending_file() { + use crate::licence::Subject; + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let files = |config: &SystemConfig| -> Vec { + let programs: BTreeSet<&str> = config.programs.keys().map(String::as_str).collect(); + assets::collect(&config.assets, &programs).into_iter().map(|(name, _)| name).collect() + }; + let shipped = files(&Boot::shipped(root).system()); + let release = files(&Boot::release(root).system()); + let withheld = pending(|s| match s { + Subject::File(path) => Some(path), + _ => None, + }); + assert!(!withheld.is_empty(), "the gate holds no file pending, so this reads nothing"); + for path in withheld { + let name = Path::new(path).file_name().unwrap().to_string_lossy().to_lowercase(); + let on = |root_files: &[String]| { + root_files.iter().any(|f| f.rsplit('/').next() == Some(name.as_str())) + }; + assert!(on(&shipped), "the shipped image carries no {name}, so its absence below says nothing"); + assert!(!on(&release), "the release image carries {name}"); + } + } + /// `console` is reached by `console/system.toml` alone and `init` by no /// `[programs]` row, so a reader that drops a mode or init loses one. #[test] diff --git a/src/ci.rs b/src/ci.rs index 5518b76c6b..9c047071df 100644 --- a/src/ci.rs +++ b/src/ci.rs @@ -7,7 +7,8 @@ //! [`Job::GateStage`] run as `host`. Every test that boots no guest is in //! [`Job::Host`], so a merge is gated on all of them. `nightly.yml` runs //! everything that boots a guest, `host` again to write the cache the merge -//! queue restores, and portability. `publish.yml` puts a landing's crates on +//! queue restores, and portability, and publishes `main`'s image once all of +//! that is green ([`Job::Release`]). `publish.yml` puts a landing's crates on //! crates.io. //! //! A host job runs every step and reds if any failed; a guest job stops at the @@ -30,7 +31,7 @@ use std::path::Path; use std::process::Command; use crate::arch::Arch; -use crate::{flags, pr, release, sdkversion}; +use crate::{flags, imagerelease, pr, release, sdkversion}; /// The checks `main`'s ruleset must require, as `gate-stage` reads them back: /// a minimum, never an equality, so a name GitHub requires and this does not @@ -49,6 +50,7 @@ const USAGE: &str = "cargo run -- --ci , where is one of: guest / one shard of the whole guest suite, nightly tier included (nightly) tcg one test on an emulated CPU (nightly) audio / one shard of gate A (nightly) + release publish main's image once its nightly is green (nightly) nightly-red file or update the nightly-red issue from $NEEDS (nightly) publish put main's SDK crates on crates.io (publish.yml)"; @@ -60,6 +62,7 @@ enum Job { Guest(String), Tcg, Audio(String), + Release, NightlyRed, Publish, } @@ -77,6 +80,7 @@ fn parse(words: &[String]) -> Result { Some("guest") => Job::Guest(shard(words.get(1))?), Some("tcg") => Job::Tcg, Some("audio") => Job::Audio(shard(words.get(1))?), + Some("release") => Job::Release, Some("nightly-red") => Job::NightlyRed, Some("publish") => Job::Publish, Some(other) => return Err(format!("no CI job is called {other:?}")), @@ -103,6 +107,13 @@ pub fn dispatch(root: &Path, args: &[String]) { } Job::Tcg => guest(root, &suite_args(&["--jobs", "1", "process_stats"])), Job::Audio(shard) => guest(root, &suite_args(&["--audio-gate", "30", "--shard", shard])), + Job::Release => { + let mut steps = vec![step("the toolchain", || release::install(root))]; + if steps.iter().all(|s| s.verdict.is_ok()) { + steps.push(step("the image release", || imagerelease::publish(root))); + } + steps + } Job::NightlyRed => vec![step("the nightly-red issue", nightly_red)], Job::Publish => vec![step("the SDK crates on crates.io", || publish(root))], }; diff --git a/src/flags.rs b/src/flags.rs index eb8fb50d0c..3938e17872 100644 --- a/src/flags.rs +++ b/src/flags.rs @@ -74,6 +74,8 @@ declare_flags!(pub CARGO_RUN = { pub KERNEL_FEATURE = "--kernel-feature", Each; pub DIAG_BOOT = "--diag-boot", None; pub CONSOLE_BOOT = "--console-boot", None; + /// Build the image the release job publishes (`build::Boot::release`). + pub RELEASE_BOOT = "--release-boot", None; pub BOOT_CONFIG = "--boot-config", Next; pub ARCH = "--arch", Next; pub REGEN_FONT = "--regen-font", None; diff --git a/src/imagerelease.rs b/src/imagerelease.rs new file mode 100644 index 0000000000..4fb3cf31ab --- /dev/null +++ b/src/imagerelease.rs @@ -0,0 +1,468 @@ +//! The image release: the disk a person downloads and boots under QEMU or +//! writes to a stick, published by `cargo run -- --ci release` from a commit of +//! `main` whose whole nightly was green. +//! +//! **Named by that commit**, [`tag`]: every build draws its partition GUIDs and +//! a runner mints its own throwaway signing key (`src/signing.rs`), so the +//! bytes name nothing a second build reproduces and a content hash would name +//! one build. **Kept to [`KEEP`]**: each publish deletes every older image +//! release and its tag ([`stale`]). +//! +//! The image is `build::Boot::release`'s. The notes carry the two command lines +//! [`Host::command`] declares, which are the argv `release_command_boots` and +//! `release_writes_no_other_disk` boot. +//! +//! Not in `src/release.rs`, whose bytes are hashed into the toolchain's tag. + +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use crate::arch::{Accel, Arch}; + +/// What every image release's tag starts with; the rest is the commit's first +/// twelve hex digits. +pub const TAG_PREFIX: &str = "image-x86_64-"; + +/// How many image releases stay published. +pub const KEEP: usize = 7; + +/// The compressed disk, as a release asset. +pub const IMAGE_ASSET: &str = "toyos.img.gz"; + +/// The image as the notes' commands name it once unpacked. +pub const IMAGE: &str = "toyos.img"; + +/// The SHA-256 of [`IMAGE_ASSET`], in the form `sha256sum -c` reads. +pub const SUMS_ASSET: &str = "SHA256SUMS"; + +/// The notes, also carried as an asset. +pub const NOTES_ASSET: &str = "README.md"; + +/// Every licence text the release carries beside the image, as (the file in +/// this tree, its asset name): ToyOS's own two, the ledger, and each text +/// `NOTICE` names for the third-party files the release image ships +/// (`every_text_notice_names_for_what_the_release_ships_is_carried`). +pub const LICENCE_ASSETS: &[(&str, &str)] = &[ + ("LICENSE-MIT", "LICENSE-MIT"), + ("LICENSE-APACHE", "LICENSE-APACHE"), + ("NOTICE", "NOTICE"), + ("licenses/OFL-1.1-JetBrainsMono.txt", "OFL-1.1-JetBrainsMono.txt"), + ("licenses/MIT-PhosphorIcons.txt", "MIT-PhosphorIcons.txt"), + ("assets/fonts/OFL.txt", "OFL-1.1-OpenSans.txt"), +]; + +/// `image-x86_64-<12 hex>` of a full commit id. +pub fn tag(commit: &str) -> Result { + let full = commit.len() == 40 && commit.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')); + if !full { + return Err(format!("{commit:?} is not a full commit id")); + } + Ok(format!("{TAG_PREFIX}{}", &commit[..12])) +} + +/// The hosts the notes give a command line for. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Host { + /// Homebrew's QEMU, emulating the x86-64 guest. + MacosAppleSilicon, + /// Debian's `qemu-system-x86` and `ovmf`, on the host's own CPU. + LinuxKvm, +} + +impl Host { + pub const ALL: [Host; 2] = [Host::MacosAppleSilicon, Host::LinuxKvm]; + + /// The one of the two this machine is, or why it is neither. + pub fn this() -> Result { + if cfg!(all(target_os = "macos", target_arch = "aarch64")) { + return Ok(Host::MacosAppleSilicon); + } + if cfg!(all(target_os = "linux", target_arch = "x86_64")) && Arch::X86_64.accel() == Accel::Kvm { + return Ok(Host::LinuxKvm); + } + Err("this host is neither an Apple Silicon Mac nor an x86-64 Linux whose /dev/kvm opens, \ + so no command line the release notes print is this host's" + .into()) + } + + /// Where the notes say this host is. + pub fn named(self) -> &'static str { + match self { + Host::MacosAppleSilicon => "macOS on Apple Silicon, with Homebrew's `qemu`", + Host::LinuxKvm => { + "Linux on x86-64 with KVM, with Debian's `qemu-system-x86` and `ovmf` and a user \ + that can open `/dev/kvm`" + } + } + } + + fn accel(self) -> Accel { + match self { + Host::MacosAppleSilicon => Accel::Tcg, + Host::LinuxKvm => Accel::Kvm, + } + } + + /// The edk2 firmware the host's QEMU installation ships: its code, and the + /// variable-store template beside it, which the guest gets read-only. + pub fn firmware(self) -> [&'static str; 2] { + match self { + Host::MacosAppleSilicon => [ + "/opt/homebrew/share/qemu/edk2-x86_64-code.fd", + "/opt/homebrew/share/qemu/edk2-i386-vars.fd", + ], + Host::LinuxKvm => ["/usr/share/OVMF/OVMF_CODE_4M.fd", "/usr/share/OVMF/OVMF_VARS_4M.fd"], + } + } + + /// The whole command line, program first, that boots `image` as a USB + /// stick on the machine shape `cargo run` boots: q35 with a vIOMMU, a + /// firmware framebuffer, a USB keyboard and tablet, and virtio-net. The + /// kernel's log goes to the terminal. + pub fn command(self, image: &str) -> Vec { + let accel = self.accel(); + let [code, vars] = self.firmware(); + [ + "qemu-system-x86_64", + "-nodefaults", + "-accel", + accel.name(), + "-cpu", + Arch::X86_64.cpu(accel), + "-machine", + "q35,kernel-irqchip=split", + "-smp", + "4", + "-m", + "2G", + "-drive", + &format!("if=pflash,format=raw,unit=0,file={code},readonly=on"), + "-drive", + &format!("if=pflash,format=raw,unit=1,file={vars},readonly=on"), + "-device", + "intel-iommu,intremap=on,caching-mode=on,aw-bits=48", + "-device", + "nec-usb-xhci,id=xhci", + "-drive", + &format!("if=none,id=stick,format=raw,file={image}"), + "-device", + "usb-storage,bus=xhci.0,drive=stick,bootindex=0", + "-device", + "usb-kbd,bus=xhci.0", + "-device", + "usb-tablet,bus=xhci.0", + "-vga", + "std", + "-netdev", + "user,id=net0", + "-device", + "virtio-net-pci-non-transitional,netdev=net0,iommu_platform=on", + "-serial", + "stdio", + ] + .map(String::from) + .to_vec() + } +} + +/// [`Host::command`] as the notes print it: an option and its value to a line. +fn shell(argv: &[String]) -> String { + let mut out = format!(" {}", argv[0]); + for word in &argv[1..] { + if word.starts_with('-') { + out.push_str(" \\\n "); + } else { + out.push(' '); + } + out.push_str(word); + } + out +} + +/// The release notes, which are also [`NOTES_ASSET`]. +pub fn notes(root: &Path, tag: &str, commit: &str) -> Result { + let qemu = crate::ci::declared_qemu_version(root).ok_or(".github/qemu-version declares no version")?; + let data = toyos_gpt::Guid::TOYOS_DATA_TEXT; + let mut commands = String::new(); + for host in Host::ALL { + commands.push_str(&format!("{}:\n\n{}\n\n", host.named(), shell(&host.command(IMAGE)))); + } + let texts: Vec = LICENCE_ASSETS.iter().map(|(_, asset)| format!("`{asset}`")).collect(); + Ok(format!( + "# ToyOS {tag} + +The ToyOS disk image of commit {commit} on `main`, whose nightly was green. It boots under QEMU, or from a USB stick on a UEFI x86-64 machine. + +## Verify and unpack + +Download `{IMAGE_ASSET}` and `{SUMS_ASSET}` from this release into one directory, and in it: + + sha256sum -c {SUMS_ASSET} + gunzip {IMAGE_ASSET} + +## Under QEMU + +QEMU {qemu} is the version ToyOS is measured with. The firmware is the edk2 build the host's QEMU installation ships. + +{commands}The kernel's log is on the terminal and the desktop is in QEMU's window. The running system writes to `{IMAGE}` itself, as it would to a stick. `/apps`, `/config`, `/home` and `/state` are kept in memory and are gone at the next boot. + +## On a USB stick + +The machine has to be an x86-64 PC from 2020 or later, booting UEFI with Secure Boot off. The only hardware ToyOS is known to work on is a Lenovo ThinkPad T14; on anything else it is untried. + +Write `{IMAGE}` to the whole stick, not to a partition of it; what the stick held is lost: + + dd if={IMAGE} of=/dev/ bs=4194304 + sync + +What a boot writes on the machine: + +- the stick: its log partition and its boot volume; +- the firmware's variable store: the loader's anti-rollback floor, and `BootNext` where a boot entry names the stick; +- another disk only where it carries a partition of ToyOS's DATA type, `{data}`, which only a disk ToyOS was set up on has. That is where `/apps`, `/config`, `/home` and `/state` live. Every other disk is read for its partition table and never written: `release_writes_no_other_disk` boots this image beside a disk laid out as another operating system's and compares every byte of it. + +## Terms + +ToyOS is MIT OR Apache-2.0. `NOTICE` names every third-party file in the repository and its terms; this release carries it and the texts for what the image ships: {texts}. + +The image leaves out doom, `DOOM1.WAD` and the SoundFont: whether they may ship is the owner's to rule (`src/licence.rs`), and nothing is published while it is not. +", + texts = texts.join(", "), + )) +} + +/// The tags among `listed`, as (tag, creation time in RFC 3339 UTC), that are +/// image releases older than the [`KEEP`] newest. +pub fn stale(listed: &[(String, String)], keep: usize) -> Vec { + let mut images: Vec<&(String, String)> = + listed.iter().filter(|(tag, _)| tag.starts_with(TAG_PREFIX)).collect(); + images.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| b.0.cmp(&a.0))); + images.into_iter().skip(keep).map(|(tag, _)| tag.clone()).collect() +} + +/// Write every asset of `tag`'s release into `out`: the image at `image` +/// compressed, its sum, the notes and the licence texts. Answers the paths in +/// upload order. +pub fn write_assets(root: &Path, image: &Path, out: &Path, tag: &str, commit: &str) -> Result, String> { + use flate2::write::GzEncoder; + use sha2::{Digest, Sha256}; + + let compressed = out.join(IMAGE_ASSET); + let file = fs::File::create(&compressed).map_err(|e| format!("{}: {e}", compressed.display()))?; + let mut gz = GzEncoder::new(file, flate2::Compression::default()); + let mut raw = fs::File::open(image).map_err(|e| format!("{}: {e}", image.display()))?; + std::io::copy(&mut raw, &mut gz).map_err(|e| format!("compressing {}: {e}", image.display()))?; + gz.finish().map_err(|e| format!("compressing {}: {e}", image.display()))?.flush().map_err(|e| e.to_string())?; + + let bytes = fs::read(&compressed).map_err(|e| format!("{}: {e}", compressed.display()))?; + let sum: String = Sha256::digest(&bytes).iter().map(|b| format!("{b:02x}")).collect(); + let sums = out.join(SUMS_ASSET); + fs::write(&sums, format!("{sum} {IMAGE_ASSET}\n")).map_err(|e| e.to_string())?; + + let readme = out.join(NOTES_ASSET); + fs::write(&readme, notes(root, tag, commit)?).map_err(|e| e.to_string())?; + + let mut assets = vec![compressed, sums, readme]; + for (from, name) in LICENCE_ASSETS { + let to = out.join(name); + fs::copy(root.join(from), &to).map_err(|e| format!("{from}: {e}"))?; + assets.push(to); + } + Ok(assets) +} + +fn gh(root: &Path, args: &[&str]) -> Result { + let out = Command::new("gh").args(args).current_dir(root).output().map_err(|e| format!("gh: {e}"))?; + if out.status.success() { + Ok(String::from_utf8_lossy(&out.stdout).into_owned()) + } else { + Err(format!("gh {} exited {}: {}", args.join(" "), out.status, String::from_utf8_lossy(&out.stderr).trim())) + } +} + +/// Whether `tag` is a release carrying [`IMAGE_ASSET`]. +fn published(root: &Path, tag: &str) -> bool { + gh(root, &["release", "view", tag, "--json", "assets", "--jq", ".assets[].name"]) + .is_ok_and(|names| names.lines().any(|name| name == IMAGE_ASSET)) +} + +/// `cargo run -- --ci release`: publish this commit's image unless it is +/// published, then delete the image releases past [`KEEP`]. Only a nightly on +/// `main` publishes, and `nightly.yml` runs this only once every lane of that +/// nightly is green. +pub fn publish(root: &Path) -> Result { + let on_runner = std::env::var("GITHUB_ACTIONS").is_ok_and(|v| v == "true"); + if !on_runner || std::env::var("GITHUB_REF").ok().as_deref() != Some("refs/heads/main") { + return Err("only a nightly on main publishes an image".into()); + } + let commit = std::env::var("GITHUB_SHA").map_err(|_| "GITHUB_SHA is unset".to_string())?; + let head = crate::pr::git(root, &["rev-parse", "HEAD"])?; + if head != commit { + return Err(format!("the checkout is {head} and the nightly ran {commit}")); + } + let tag = tag(&commit)?; + + let mut said = if published(root, &tag) { + format!("{tag} is already published") + } else { + let built = Command::new("cargo") + .args(["run", "--", "--release-boot", "--build-only"]) + .current_dir(root) + .status() + .map_err(|e| format!("cargo: {e}"))?; + if !built.success() { + return Err(format!("cargo run -- --release-boot --build-only exited {built}")); + } + let out = toyos_tmpdir::TempDir::new("image-release"); + let assets = write_assets(root, &root.join(crate::build::RELEASE_IMAGE), &out, &tag, &commit)?; + let notes = out.join(NOTES_ASSET); + let mut args: Vec = ["release", "create", &tag, "--title", &tag, "--target", &commit, "--latest", "--notes-file"] + .map(String::from) + .to_vec(); + args.push(notes.display().to_string()); + args.extend(assets.iter().map(|a| a.display().to_string())); + gh(root, &args.iter().map(String::as_str).collect::>())?; + if !published(root, &tag) { + return Err(format!("{tag} was created and carries no {IMAGE_ASSET}")); + } + format!("{tag} published") + }; + + let listed = gh(root, &["release", "list", "--limit", "1000", "--json", "tagName,createdAt", "--jq", ".[] | .tagName + \" \" + .createdAt"])?; + let listed: Vec<(String, String)> = listed + .lines() + .filter_map(|l| l.split_once(' ')) + .map(|(t, c)| (t.to_string(), c.to_string())) + .collect(); + let old = stale(&listed, KEEP); + for old in &old { + gh(root, &["release", "delete", old, "--cleanup-tag", "--yes"])?; + } + said.push_str(&format!("; {} older image release(s) deleted, {KEEP} kept", old.len())); + Ok(said) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + } + + #[test] + fn a_tag_is_the_commit_and_a_short_or_foreign_id_is_refused() { + let commit = "c55189490123456789abcdef0123456789abcdef"; + assert_eq!(tag(commit).unwrap(), "image-x86_64-c55189490123"); + assert!(tag("c5518949").is_err()); + assert!(tag(&commit.to_uppercase()).is_err()); + } + + /// Only image releases are judged, the newest are kept by creation time, + /// and the toolchain's releases are never named. + #[test] + fn retention_deletes_only_image_releases_past_the_newest_kept() { + let at = |day: u32| format!("2026-09-{day:02}T03:00:00Z"); + let mut listed: Vec<(String, String)> = + (1..=10).map(|day| (format!("{TAG_PREFIX}{day:012}"), at(day))).collect(); + listed.push(("toolchain-linux-x86_64-0123456789abcdef".into(), at(1))); + let old = stale(&listed, 7); + assert_eq!(old, [3, 2, 1].map(|day| format!("{TAG_PREFIX}{day:012}"))); + assert!(stale(&listed[..7], 7).is_empty()); + assert_eq!(stale(&listed, 0).len(), 10); + } + + /// Each host's command boots the image it is given as the one stick, on + /// that host's own accelerator and firmware, and the notes print it whole. + #[test] + fn the_notes_print_each_hosts_command_line_whole() { + let notes = notes(&root(), "image-x86_64-c55189490123", &"c".repeat(40)).unwrap(); + for host in Host::ALL { + let argv = host.command(IMAGE); + assert!(argv.iter().any(|a| a == &format!("if=none,id=stick,format=raw,file={IMAGE}"))); + assert!(argv.iter().any(|a| a == host.accel().name())); + for firmware in host.firmware() { + assert!(argv.iter().any(|a| a.contains(firmware)), "{host:?}: {firmware}"); + } + let printed = shell(&argv); + assert!(notes.contains(&printed), "{host:?}'s command is not in the notes"); + let words: Vec<&str> = printed.split_whitespace().filter(|w| *w != "\\").collect(); + assert_eq!(words, argv.iter().map(String::as_str).collect::>()); + } + } + + /// The release carries the text `NOTICE` names for every third-party file + /// under an asset directory the release image ships, and none only + /// withheld material names. + #[test] + fn every_text_notice_names_for_what_the_release_ships_is_carried() { + use crate::licence::Subject; + let root = root(); + let notice = fs::read_to_string(root.join("NOTICE")).unwrap(); + let assets: Vec = crate::build::shipped(&root) + .unwrap() + .assets + .iter() + .map(|dir| format!("{}/", dir.strip_prefix(&root).unwrap().display())) + .collect(); + let withheld: Vec<&str> = crate::licence::pending_owner() + .map(|s| match s { + Subject::Crate(p) | Subject::Notice(p) | Subject::File(p) => p, + }) + .collect(); + let carried: Vec<&str> = LICENCE_ASSETS.iter().map(|(from, _)| *from).collect(); + let mut needed = vec!["LICENSE-MIT", "LICENSE-APACHE", "NOTICE"]; + let mut shipped_sections = 0; + let sections = crate::licence::sections(¬ice); + for section in §ions { + let ships = assets.iter().any(|dir| section.path.starts_with(dir.as_str())) + && !withheld.contains(§ion.path.as_str()); + if !ships { + continue; + } + shipped_sections += 1; + for text in §ion.texts { + assert!(carried.contains(&text.as_str()), "{} names {text}, which the release does not carry", section.path); + needed.push(text.as_str()); + } + } + assert!(shipped_sections >= 3, "{shipped_sections} NOTICE sections read as shipped"); + for from in &carried { + assert!(root.join(from).is_file(), "{from}"); + assert!(needed.contains(from), "{from} is carried and nothing the release ships names it"); + } + let names: Vec<&str> = LICENCE_ASSETS.iter().map(|(_, name)| *name).collect(); + let mut unique = names.clone(); + unique.sort_unstable(); + unique.dedup(); + assert_eq!(unique.len(), names.len(), "two texts under one asset name: {names:?}"); + } + + /// What `sha256sum -c` checks is the compressed asset's own digest, and the + /// asset decompresses to the image byte for byte. + #[test] + fn the_sum_is_the_compressed_images_and_it_decompresses_to_the_image() { + use sha2::{Digest, Sha256}; + use std::io::Read; + let dir = toyos_tmpdir::TempDir::new("image-assets"); + let image = dir.join("in.img"); + let bytes: Vec = (0..300_000u32).map(|i| (i % 251) as u8).chain(std::iter::repeat_n(0, 1 << 20)).collect(); + fs::write(&image, &bytes).unwrap(); + let out = dir.join("out"); + fs::create_dir(&out).unwrap(); + let assets = write_assets(&root(), &image, &out, "image-x86_64-c55189490123", &"c".repeat(40)).unwrap(); + let names: Vec = + assets.iter().map(|a| a.file_name().unwrap().to_string_lossy().into_owned()).collect(); + assert_eq!(names[..3], [IMAGE_ASSET, SUMS_ASSET, NOTES_ASSET]); + assert_eq!(names.len(), 3 + LICENCE_ASSETS.len()); + + let gz = fs::read(out.join(IMAGE_ASSET)).unwrap(); + let sum: String = Sha256::digest(&gz).iter().map(|b| format!("{b:02x}")).collect(); + assert_eq!(fs::read_to_string(out.join(SUMS_ASSET)).unwrap(), format!("{sum} {IMAGE_ASSET}\n")); + let mut back = Vec::new(); + flate2::read::GzDecoder::new(&gz[..]).read_to_end(&mut back).unwrap(); + assert!(back == bytes, "the asset does not decompress to the image"); + } +} diff --git a/src/lib.rs b/src/lib.rs index 2ef6e4d30d..524be5b4f5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,6 +18,7 @@ pub mod hostws; pub mod icmp; pub mod identity; pub mod image; +pub mod imagerelease; pub mod signing; /// Which kernel containers may be hashed, and by whose keys; read by nothing /// but its own tests. diff --git a/src/licence.rs b/src/licence.rs index 45753e3158..c2f3671262 100644 --- a/src/licence.rs +++ b/src/licence.rs @@ -194,6 +194,15 @@ pub const EXCEPTIONS: &[Exception] = &[ }, ]; +/// Everything that ships while the owner has yet to rule whether it may, which +/// is what a public release leaves out (`build::Boot::release`). +pub fn pending_owner() -> impl Iterator { + EXCEPTIONS + .iter() + .filter(|e| matches!(e.standing, Standing::PendingOwner(_))) + .map(|e| e.subject) +} + /// Every committed file whose terms somebody had to establish, with the digest /// of what is committed and where the terms are recorded. /// @@ -956,19 +965,24 @@ fn judge_files(ledger: &[Row], tracked: &[String], shipping: &Shipping, report: // --- NOTICE ------------------------------------------------------------------ -/// One `NOTICE` section: its heading, the path the heading starts with, and the -/// SPDX lines it carries. +/// One `NOTICE` section: its heading, the path the heading starts with, the +/// SPDX lines it carries, and the files its terms are written in. #[derive(Debug, PartialEq)] -struct Section { +pub(crate) struct Section { heading: String, - path: String, + pub(crate) path: String, spdx: Vec, + /// The path each `Licence text:` or `Terms:` line starts with. + pub(crate) texts: Vec, } const SPDX_TAG: &str = "SPDX-License-Identifier:"; +/// What a section names the file its terms are written in with. +const TEXT_TAGS: [&str; 2] = ["Licence text:", "Terms:"]; + /// The `-`-underlined sections of `text`, in order. -fn sections(text: &str) -> Vec
{ +pub(crate) fn sections(text: &str) -> Vec
{ let lines: Vec<&str> = text.lines().collect(); let mut out: Vec
= Vec::new(); for (i, line) in lines.iter().enumerate() { @@ -980,11 +994,17 @@ fn sections(text: &str) -> Vec
{ heading: line.trim().to_string(), path: line.split_whitespace().next().unwrap_or("").to_string(), spdx: Vec::new(), + texts: Vec::new(), }); - } else if let (Some(section), Some(expr)) = - (out.last_mut(), line.trim().strip_prefix(SPDX_TAG)) - { - section.spdx.push(expr.trim().to_string()); + } else if let Some(section) = out.last_mut() { + let line = line.trim(); + if let Some(expr) = line.strip_prefix(SPDX_TAG) { + section.spdx.push(expr.trim().to_string()); + } + let rest = TEXT_TAGS.iter().find_map(|tag| line.strip_prefix(tag)); + if let Some(path) = rest.and_then(|rest| rest.split_whitespace().next()) { + section.texts.push(path.trim_end_matches(',').to_string()); + } } } out diff --git a/src/main.rs b/src/main.rs index e0d675c47a..e5e3c1a7e1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -149,6 +149,27 @@ fn main() { } return; } + // The release job publishes what this builds, so nothing on the line may + // make it another image. + let release = asked(&flags::RELEASE_BOOT); + if release { + for other in [ + &flags::DIAG_BOOT, + &flags::CONSOLE_BOOT, + &flags::BOOT_CONFIG, + &flags::KERNEL_PARAM, + &flags::KERNEL_FEATURE, + &flags::DEBUG, + &flags::OWNER_KEY, + &flags::UPDATE_IMAGE, + &flags::ARCH, + ] { + if asked(other) { + eprintln!("Error: --release-boot builds the published image and takes no {}", other.name); + std::process::exit(2); + } + } + } // Before anything is built, so a missing key is refused before any lock // and no image this run writes is signed by two keys. let update_image = CARGO_RUN.value(&args, &flags::UPDATE_IMAGE).map(PathBuf::from); @@ -211,6 +232,7 @@ fn main() { } (None, true, _) => toyos_build::build::Boot::diag(&root), (None, _, true) => toyos_build::build::Boot::console(&root), + _ if release => toyos_build::build::Boot::release(&root), _ => toyos_build::build::Boot::shipped(&root), }; assert!( diff --git a/src/sourcegate.rs b/src/sourcegate.rs index ba7fdb0624..9df3d4e136 100644 --- a/src/sourcegate.rs +++ b/src/sourcegate.rs @@ -724,6 +724,12 @@ const CI_PACKAGES: &[Package] = &[ name: "git", why: "the version control this repository is, and `REQUIRED` in src/main.rs", }, + Package { + name: "ovmf", + why: "the edk2 firmware Debian's QEMU boots a UEFI guest with: the image release notes' \ + Linux command line names it, and the guest suite boots that line \ + (src/imagerelease.rs)", + }, Package { name: "python3", why: "the Python standing failure CLAUDE.md:56 declares, wearing a package name — \ diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 0c64deee56..b5f32a2135 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -43,6 +43,8 @@ pub mod metal; pub mod origin; #[allow(dead_code)] pub mod partclaim; +/// The image release's command lines, and the disks its image was not given. +pub mod release; #[allow(dead_code)] pub mod passcost; #[allow(dead_code)] diff --git a/tests/common/release.rs b/tests/common/release.rs new file mode 100644 index 0000000000..f9dd289bc1 --- /dev/null +++ b/tests/common/release.rs @@ -0,0 +1,293 @@ +//! The image release's command lines booted as its notes print them +//! (`toyos_build::imagerelease::Host::command`), and what the release image +//! does to a disk it was not given. +//! +//! Each guest boots a copy of the image `cargo run -- --release-boot +//! --build-only` writes, built once per run by that same sequence, and is +//! judged by its console alone: these boots are not the harness's profiles. + +use std::collections::BTreeMap; +use std::io::{BufRead, BufReader, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::sync::mpsc::{self, Receiver, RecvTimeoutError}; +use std::sync::OnceLock; +use std::time::{Duration, Instant}; + +use toyos_build::build::{self, Boot}; +use toyos_build::fingerprint::{first_difference, whole_device}; +use toyos_build::imagerelease::Host; + +/// One guest's ceiling from power-on to a painting desktop, before +/// `super::qemu::budget` scales it: a liveness guard, never a verdict. +const DESKTOP: Duration = Duration::from_secs(120); + +/// The compositor's report, one every two seconds of a painting desktop. +const FRAMES: &str = "compositor: frames="; + +/// The release image, built once per run. +fn image() -> &'static Path { + static BUILT: OnceLock = OnceLock::new(); + BUILT.get_or_init(|| { + let root = super::compile::repo_root(); + let boot = Boot::release(&root); + let plan = build::plan_for(&root, &boot, false, &[]); + build::build(&root, boot, false, &plan) + }) +} + +/// A copy of the release image under this lane, for one guest to write to. +fn stick(name: &str) -> Result { + let to = super::lane::dir().join(name); + std::fs::copy(image(), &to).map_err(|e| format!("copy the release image to {}: {e}", to.display()))?; + Ok(to) +} + +/// A QEMU started from a release command line, and its console so far. +struct Guest { + child: Child, + lines: Receiver, + log: String, + stderr: PathBuf, +} + +impl Guest { + /// `argv` with `extra` after it and no window: the notes' line, headless. + fn start(argv: &[String], extra: &[String], stderr: PathBuf) -> Result { + let err = std::fs::File::create(&stderr).map_err(|e| format!("{}: {e}", stderr.display()))?; + let mut child = Command::new(&argv[0]) + .args(&argv[1..]) + .args(extra) + .args(["-display", "none"]) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(err) + .spawn() + .map_err(|e| format!("{}: {e}", argv[0]))?; + let out = child.stdout.take().expect("piped"); + let (send, lines) = mpsc::channel(); + std::thread::spawn(move || { + for line in BufReader::new(out).split(b'\n') { + let Ok(line) = line else { return }; + if send.send(String::from_utf8_lossy(&line).into_owned()).is_err() { + return; + } + } + }); + Ok(Guest { child, lines, log: String::new(), stderr }) + } + + /// Read the console until `done` holds of it, refusing a panic the moment + /// one is printed. + fn until(&mut self, what: &str, done: impl Fn(&str) -> bool) -> Result<(), String> { + let budget = super::qemu::budget(DESKTOP); + let deadline = Instant::now() + budget; + loop { + if let Some(line) = self.log.lines().find(|l| l.contains("PANIC") || l.contains("panicked at")) { + return Err(format!("the guest panicked before {what}: {line}\n{}", self.log)); + } + if done(&self.log) { + return Ok(()); + } + let left = deadline.saturating_duration_since(Instant::now()); + match self.lines.recv_timeout(left) { + Ok(line) => { + self.log.push_str(&line); + self.log.push('\n'); + } + Err(RecvTimeoutError::Timeout) => { + return Err(format!("no {what} within {budget:?}:\n{}", self.log)); + } + Err(RecvTimeoutError::Disconnected) => { + let stderr = std::fs::read_to_string(&self.stderr).unwrap_or_default(); + return Err(format!( + "QEMU closed its console before {what}:\n{}\nQEMU's stderr:\n{stderr}", + self.log + )); + } + } + } + } + + /// The desktop is up: every program the shipped config starts said it + /// started, the three that announce themselves did, and the compositor + /// has painted. + fn desktop(&mut self) -> Result<(), String> { + let mut said: Vec = build::boot_start(&super::compile::repo_root().join("system.toml")) + .iter() + .map(|program| format!("init: started {program}")) + .collect(); + said.extend( + ["compositor: ready", "netd: ready", "logd: this boot's kernel log is", FRAMES].map(String::from), + ); + self.until("the desktop", |log| said.iter().all(|line| log.contains(line.as_str()))) + } +} + +impl Drop for Guest { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +/// The notes' command line for this host boots the release image to a +/// painting desktop. +pub fn release_command_boots() -> Result<(), String> { + let host = Host::this()?; + let stick = stick("release-command.img")?; + let argv = host.command(&stick.display().to_string()); + let mut guest = Guest::start(&argv, &[], super::lane::dir().join("release-command.stderr"))?; + guest.desktop()?; + eprintln!(" [release] {host:?}'s command line reached a painting desktop"); + Ok(()) +} + +/// Beside the release image, a disk laid out as another operating system's +/// and a stick with no partition table come back byte for byte after a boot +/// to the desktop, while the kernel says it read both and opened a volume on +/// neither. +pub fn release_writes_no_other_disk() -> Result<(), String> { + const OTHER_BYTES: u64 = 192 << 20; + const SPARE_BYTES: u64 = 64 << 20; + let host = Host::this()?; + let dir = super::lane::dir(); + let stick = stick("release-disks.img")?; + let other = dir.join("another-os.img"); + let parts = another_os_disk(&other, OTHER_BYTES)?; + let spare = dir.join("spare-stick.img"); + pattern_file(&spare, SPARE_BYTES, 0x5A)?; + let before = [whole_device(&stick), whole_device(&other), whole_device(&spare)]; + + let extra: Vec = [ + "-drive".to_string(), + format!("if=none,id=other,format=raw,file={}", other.display()), + "-device".to_string(), + "nvme,serial=other,drive=other".to_string(), + "-drive".to_string(), + format!("if=none,id=spare,format=raw,file={}", spare.display()), + "-device".to_string(), + "usb-storage,bus=xhci.0,drive=spare".to_string(), + ] + .to_vec(); + let argv = host.command(&stick.display().to_string()); + let mut guest = Guest::start(&argv, &extra, dir.join("release-disks.stderr"))?; + guest.desktop()?; + + // What the kernel opened, before anything about the bytes: a page cache is + // what every write to a DATA volume goes through. + if let Some(line) = guest.log.lines().find(|l| l.contains("page cache: device")) { + return Err(format!("the release opened a DATA volume on a disk it was not given: {line}\n{}", guest.log)); + } + // The case was reached: the kernel read both tables and took neither. + let read_other = format!("gpt: device 1 has {parts} partitions and none of them is ours"); + for said in [ + read_other.as_str(), + "has no partition table we can use", + "storage: this machine carries 0 TOYOS-DATA partitions", + ] { + if !guest.log.contains(said) { + return Err(format!("the kernel never said {said:?}\n{}", guest.log)); + } + } + // The machine runs on past the desktop, so what a boot writes late is in. + let settled = guest.log.matches(FRAMES).count() + 3; + guest.until("three more frame reports", |log| log.matches(FRAMES).count() >= settled)?; + drop(guest); + + let after = [whole_device(&stick), whole_device(&other), whole_device(&spare)]; + if first_difference(&before[0], &after[0]).is_none() { + return Err( + "the stick came back unchanged, so no write of this boot reached a backing file and \ + the comparison below proves nothing" + .into(), + ); + } + for (name, (b, a)) in ["the other operating system's disk", "the spare stick"] + .iter() + .zip(before[1..].iter().zip(&after[1..])) + { + if let Some(diff) = first_difference(b, a) { + return Err(format!("the release wrote to {name}: {diff}")); + } + } + eprintln!( + " [release] {host:?}: beside the stick, an NVMe disk of {parts} foreign partitions and a \ + stick with no table came back byte for byte" + ); + Ok(()) +} + +/// `len` bytes of `fill`, so a write of anything, zeros included, moves the +/// fingerprint. +fn pattern_file(path: &Path, len: u64, fill: u8) -> Result<(), String> { + let mut file = std::fs::File::create(path).map_err(|e| format!("{}: {e}", path.display()))?; + let chunk = vec![fill; 1 << 20]; + for _ in 0..len / chunk.len() as u64 { + file.write_all(&chunk).map_err(|e| format!("{}: {e}", path.display()))?; + } + Ok(()) +} + +/// A disk laid out as a PC's with another operating system on it: an EFI +/// system partition, Microsoft's reserved and basic-data partitions and a +/// Linux filesystem, each carrying its format's signature, over a filled +/// disk. Answers how many partitions it carries. +fn another_os_disk(path: &Path, len: u64) -> Result { + use gpt::partition_types::{BASIC, EFI, LINUX_FS, MICROSOFT_RESERVED}; + const MIB: u64 = 1 << 20; + pattern_file(path, len, 0xA5)?; + + let mut file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(path) + .map_err(|e| format!("{}: {e}", path.display()))?; + let mbr = gpt::mbr::ProtectiveMBR::with_lb_size(u32::try_from(len / 512 - 1).unwrap_or(u32::MAX)); + mbr.overwrite_lba0(&mut file).map_err(|e| format!("protective MBR: {e}"))?; + let mut disk = gpt::GptConfig::default() + .initialized(false) + .writable(true) + .logical_block_size(gpt::disk::LogicalBlockSize::Lb512) + .create_from_device(Box::new(file), None) + .map_err(|e| format!("a table on {}: {e}", path.display()))?; + disk.update_partitions(BTreeMap::::new()) + .map_err(|e| format!("an empty table: {e}"))?; + let layout = [ + ("EFI system partition", 32 * MIB, EFI), + ("Microsoft reserved partition", 16 * MIB, MICROSOFT_RESERVED), + ("Basic data partition", 64 * MIB, BASIC), + ("Linux filesystem", 64 * MIB, LINUX_FS), + ]; + let mut starts = Vec::new(); + for (name, bytes, kind) in layout { + let id = disk + .add_partition(name, bytes, kind, 0, Some(2048)) + .map_err(|e| format!("add {name}: {e}"))?; + let placed = &disk.partitions()[&id]; + starts.push(placed.bytes_start(gpt::disk::LogicalBlockSize::Lb512).map_err(|e| e.to_string())?); + } + let mut device = disk.write().map_err(|e| format!("write the table: {e}"))?; + + // FAT32's and NTFS's boot sectors, and ext4's superblock magic. + let mut fat = [0u8; 512]; + fat[..3].copy_from_slice(&[0xEB, 0x58, 0x90]); + fat[3..11].copy_from_slice(b"MSDOS5.0"); + fat[82..90].copy_from_slice(b"FAT32 "); + fat[510..].copy_from_slice(&[0x55, 0xAA]); + let mut ntfs = [0u8; 512]; + ntfs[..3].copy_from_slice(&[0xEB, 0x52, 0x90]); + ntfs[3..11].copy_from_slice(b"NTFS "); + ntfs[510..].copy_from_slice(&[0x55, 0xAA]); + for (at, bytes) in [(starts[0], &fat[..]), (starts[2], &ntfs[..]), (starts[3] + 1024 + 56, &[0x53, 0xEF][..])] { + device.seek(SeekFrom::Start(at)).map_err(|e| e.to_string())?; + device.write_all(bytes).map_err(|e| e.to_string())?; + } + device.flush().map_err(|e| e.to_string())?; + + // The premise, by the parser the kernel selects DATA with. + if toyos_build::image::data_partition_of(path).is_ok() { + return Err(format!("{} carries a TOYOS-DATA partition, so it is no other system's disk", path.display())); + } + Ok(starts.len()) +} diff --git a/tests/toyos.rs b/tests/toyos.rs index f2ed0e9097..d8844b8fec 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -12,8 +12,8 @@ use common::qemu::{ STALLED, }; use common::{ - audio, compile, devices, faults, hostload, lan, metal, partclaim, pkg, power, screen, serial, - stats, storage, usb, + audio, compile, devices, faults, hostload, lan, metal, partclaim, pkg, power, release, screen, + serial, stats, storage, usb, }; use toyos_build::bootlog::{self, boot_millis}; use toyos_build::heartbeat; @@ -983,6 +983,11 @@ const MACHINE_TESTS: &[(&str, Sched, Tier)] = &[ ("volume_from_another_disk", Sched::Parallel, Tier::Fast), ("broken_data_volume_is_absent", Sched::Parallel, Tier::Fast), ("data_candidate_with_bad_geometry_is_absent", Sched::Parallel, Tier::Fast), + // The image release's command line booted as its notes print it, and beside it + // two disks it was not given: console lines and image bytes, and the + // ceiling is a liveness guard. + ("release_command_boots", Sched::Parallel, Tier::Fast), + ("release_writes_no_other_disk", Sched::Parallel, Tier::Fast), // Four kernel lines and a file read off the image once the guest is gone; no clock in any of them. ("internal_disk_boot", Sched::Parallel, Tier::Fast), // One boot each, kernel lines and image bytes for verdicts, no clock in either. @@ -11469,6 +11474,8 @@ fn run_machine_test( // Body in `tests/common/storage.rs`, so the hunk in this shared file // stays one line. "foreign_disk_untouched" => storage::foreign_disk_untouched(test_config, c_bins, rust_bins), + "release_command_boots" => release::release_command_boots(), + "release_writes_no_other_disk" => release::release_writes_no_other_disk(), "internal_disk_boot" => storage::internal_disk_boot(test_config, c_bins, rust_bins), "partition_claim" => partclaim::partition_claim(test_config, c_bins, rust_bins), "partition_claim_gives_up" => { From dcee994a70c745d061e028c0b4fcee00420a1c4f Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 27 Sep 2026 22:49:47 +0200 Subject: [PATCH 2/9] imagerelease: the host is read through Arch, and QEMU is spawned as Arch names it Co-Authored-By: Claude Opus 5.5 --- src/imagerelease.rs | 14 +++++++------- src/sourcegate.rs | 11 ++++++++--- tests/common/release.rs | 9 +++++++-- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/imagerelease.rs b/src/imagerelease.rs index 4fb3cf31ab..d03c96df4f 100644 --- a/src/imagerelease.rs +++ b/src/imagerelease.rs @@ -76,10 +76,10 @@ impl Host { /// The one of the two this machine is, or why it is neither. pub fn this() -> Result { - if cfg!(all(target_os = "macos", target_arch = "aarch64")) { + if cfg!(target_os = "macos") && Arch::HOST == Some(Arch::Aarch64) { return Ok(Host::MacosAppleSilicon); } - if cfg!(all(target_os = "linux", target_arch = "x86_64")) && Arch::X86_64.accel() == Accel::Kvm { + if cfg!(target_os = "linux") && Arch::HOST == Some(Arch::X86_64) && Arch::X86_64.accel() == Accel::Kvm { return Ok(Host::LinuxKvm); } Err("this host is neither an Apple Silicon Mac nor an x86-64 Linux whose /dev/kvm opens, \ @@ -125,7 +125,7 @@ impl Host { let accel = self.accel(); let [code, vars] = self.firmware(); [ - "qemu-system-x86_64", + Arch::X86_64.qemu(), "-nodefaults", "-accel", accel.name(), @@ -204,7 +204,7 @@ Download `{IMAGE_ASSET}` and `{SUMS_ASSET}` from this release into one directory ## Under QEMU -QEMU {qemu} is the version ToyOS is measured with. The firmware is the edk2 build the host's QEMU installation ships. +QEMU {qemu} is the version ToyOS is measured with. The firmware is edk2, from Homebrew's QEMU on macOS and from Debian's `ovmf` on Linux. {commands}The kernel's log is on the terminal and the desktop is in QEMU's window. The running system writes to `{IMAGE}` itself, as it would to a stick. `/apps`, `/config`, `/home` and `/state` are kept in memory and are gone at the next boot. @@ -212,16 +212,16 @@ QEMU {qemu} is the version ToyOS is measured with. The firmware is the edk2 buil The machine has to be an x86-64 PC from 2020 or later, booting UEFI with Secure Boot off. The only hardware ToyOS is known to work on is a Lenovo ThinkPad T14; on anything else it is untried. -Write `{IMAGE}` to the whole stick, not to a partition of it; what the stick held is lost: +Write `{IMAGE}` to the whole stick, not to a partition of it, as root; what the stick held is lost: dd if={IMAGE} of=/dev/ bs=4194304 sync What a boot writes on the machine: -- the stick: its log partition and its boot volume; +- the stick it booted from; - the firmware's variable store: the loader's anti-rollback floor, and `BootNext` where a boot entry names the stick; -- another disk only where it carries a partition of ToyOS's DATA type, `{data}`, which only a disk ToyOS was set up on has. That is where `/apps`, `/config`, `/home` and `/state` live. Every other disk is read for its partition table and never written: `release_writes_no_other_disk` boots this image beside a disk laid out as another operating system's and compares every byte of it. +- another disk only where it carries a partition of ToyOS's DATA type, `{data}`, a type no other system uses. That is where `/apps`, `/config`, `/home` and `/state` live. Every other disk is read for its partition table and never written: `release_writes_no_other_disk` boots this image beside a disk laid out as another operating system's and compares every byte of it. ## Terms diff --git a/src/sourcegate.rs b/src/sourcegate.rs index 9df3d4e136..a9a4093041 100644 --- a/src/sourcegate.rs +++ b/src/sourcegate.rs @@ -555,15 +555,20 @@ const HOST_SPAWNS: &[Spawn] = &[ }, Spawn { arg: "arch.qemu()", - sites: &[("src/qemu.rs", 1), ("src/ci.rs", 2), ("tests/common/qemu.rs", 1)], + sites: &[ + ("src/qemu.rs", 1), + ("src/ci.rs", 2), + ("tests/common/qemu.rs", 1), + ("tests/common/release.rs", 1), + ], why: "QEMU, the other half of the bar: `Arch::qemu` names `qemu-system-x86_64` and \ `qemu-system-aarch64`, and `check_prerequisites` requires the one being booted", }, Spawn { arg: "\"gh\"", sites: &[], - why: "GitHub's CLI, outside the bar: CI's release, protection and nightly-red jobs \ - (src/ci.rs, src/release.rs) ask GitHub with it. Nothing that builds or boots \ + why: "GitHub's CLI, outside the bar: CI's releases, protection and nightly-red jobs \ + (src/ci.rs, src/release.rs, src/imagerelease.rs) ask GitHub with it. Nothing that builds or boots \ reaches it", }, Spawn { diff --git a/tests/common/release.rs b/tests/common/release.rs index f9dd289bc1..8449ab3fb1 100644 --- a/tests/common/release.rs +++ b/tests/common/release.rs @@ -14,6 +14,7 @@ use std::sync::mpsc::{self, Receiver, RecvTimeoutError}; use std::sync::OnceLock; use std::time::{Duration, Instant}; +use toyos_build::arch::Arch; use toyos_build::build::{self, Boot}; use toyos_build::fingerprint::{first_difference, whole_device}; use toyos_build::imagerelease::Host; @@ -55,7 +56,11 @@ impl Guest { /// `argv` with `extra` after it and no window: the notes' line, headless. fn start(argv: &[String], extra: &[String], stderr: PathBuf) -> Result { let err = std::fs::File::create(&stderr).map_err(|e| format!("{}: {e}", stderr.display()))?; - let mut child = Command::new(&argv[0]) + let arch = Arch::X86_64; + if argv[0] != arch.qemu() { + return Err(format!("a release command line starts {:?}, not {}", argv[0], arch.qemu())); + } + let mut child = Command::new(arch.qemu()) .args(&argv[1..]) .args(extra) .args(["-display", "none"]) @@ -63,7 +68,7 @@ impl Guest { .stdout(Stdio::piped()) .stderr(err) .spawn() - .map_err(|e| format!("{}: {e}", argv[0]))?; + .map_err(|e| format!("{}: {e}", arch.qemu()))?; let out = child.stdout.take().expect("piped"); let (send, lines) = mpsc::channel(); std::thread::spawn(move || { From 748890ec413ad341b6a3d8ba6d1568aedb858e94 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 27 Sep 2026 23:04:48 +0200 Subject: [PATCH 3/9] build: say which rows refuse a withheld program Co-Authored-By: Claude Opus 5.5 --- src/build.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/build.rs b/src/build.rs index 3957af0280..985e568910 100644 --- a/src/build.rs +++ b/src/build.rs @@ -171,8 +171,8 @@ fn parse_config(path: &Path) -> SystemConfig { /// `config` less every program the licence gate names as a package pending /// the owner. A program's key is its package's name -/// (`the_release_withholds_every_pending_package`); one another row starts or -/// receives stays named there, and `build_and_assemble` refuses the image. +/// (`the_release_withholds_every_pending_package`). One that `[boot] start` or +/// a symlink still names is refused by `build_and_assemble`. fn withhold_pending(config: &mut SystemConfig) { for subject in crate::licence::pending_owner() { if let crate::licence::Subject::Crate(name) = subject { From 9ba1b2af8a0d947e59ad82cc301c47155b8da002 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 27 Sep 2026 23:49:53 +0200 Subject: [PATCH 4/9] imagerelease: publish the bytes that booted, with the image's licence notice The release job needs only `build`, runs in the pinned Debian KVM container, builds `Boot::release`'s image once in-process, boots a copy of it under the Linux line the notes print (`imagerelease::boots`), and on main uploads those same bytes: created as a draft, checked to carry the image, then published. Off main it builds and boots and publishes nothing, so a branch dispatch measures the Debian OVMF paths and the KVM line. `--release-boot` and its refusal list are gone; `--ci release` shares the guest jobs' frame (instrument, toolchain, private $TMPDIR). `boots` fingerprints both firmware files before and after the guest, so a line that lets edk2 write the shared variable-store template is red. `licence::notices` is written from the licence gate's own walk: every package with its own licence files, or the standard texts of its licence from the fork's LICENSES/ where the package publishes none, with its source (the crates.io download, the git URL, or the fork at its pinned commit for std); and every NOTICE section over a shipped file with its texts, less what is pending the owner. It is on the release's ROOT at share/licences.txt and beside the release. A package with no text reds `the_release_notice_carries_every_package_and_file_it_ships`. Where the toolchain was installed, std's library is fetched sparse beside rust/, since a source tree at rust/ makes the toolchain the checkout's to build. This replaces LICENCE_ASSETS and its hand-mirror test. The notes name the floor variable (its name's head and the vendor GUID, now `toyos_update::floor::VENDOR`, which the loader parses), that only `dmpstore -d` removes it, and that BootNext boots the stick once, before the dd line; the dd line gains its unmount step; the withheld list comes from `pending_owner`. `gh release view` answering anything but `release not found` is an error, and a listing that fills its limit or carries an entry without a tag and a time is refused. `release_writes_no_other_disk` is deleted. `foreign_disk_untouched` gains a USB disk whose table names only other systems' partitions and a USB stick with no table, asserts the kernel read both, and waits for QEMU to exit after `run shutdown` rather than draining for a fixed time. `release_command_boots` is disabled while the kernel dies on Homebrew's edk2; the loader writing a second stick of one release is filed. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly.yml | 30 +- Cargo.toml | 2 +- bootloader/src/floor.rs | 6 +- ...its-log-guid-not-the-one-it-booted-from.md | 24 + ...arries-no-notice-of-the-crates-it-links.md | 19 - ...at-boot-on-the-edk2-firmware-qemu-ships.md | 9 +- src/build.rs | 66 ++- src/ci.rs | 45 +- src/flags.rs | 2 - src/imagerelease.rs | 439 ++++++++++++------ src/licence.rs | 419 ++++++++++++++++- src/main.rs | 22 - src/redlist.rs | 4 + src/sourcegate.rs | 15 +- src/sysroot.rs | 2 +- tests/common/mod.rs | 2 +- tests/common/release.rs | 304 +----------- tests/common/storage.rs | 94 +++- tests/toyos.rs | 7 +- toyos-update/src/floor.rs | 7 +- 20 files changed, 925 insertions(+), 593 deletions(-) create mode 100644 issues/boot-media/the-loader-writes-the-first-disk-carrying-its-log-guid-not-the-one-it-booted-from.md delete mode 100644 issues/build/the-image-release-carries-no-notice-of-the-crates-it-links.md diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index fef5722edb..784105e5bd 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -63,8 +63,7 @@ jobs: with: fetch-depth: 0 - - &runner-deps - name: disk and QEMU + - name: disk and QEMU run: | sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc \ /usr/local/share/boost /usr/local/.ghcup @@ -251,27 +250,32 @@ jobs: - *checkout - run: cargo build -p toyos-build - # The disk image a person downloads, of the commit every lane above passed; - # `src/imagerelease.rs` says what it carries and how many stay. Bare - # `ubuntu-24.04` for `gh`, as `build` is. + # The disk image a person downloads: built once, booted under the Linux line + # its notes print, and on main published as those same bytes + # (`src/imagerelease.rs`). release: - needs: [host, build, guest, tcg, audio, portability-linux, portability-macos] - if: github.ref == 'refs/heads/main' + needs: build runs-on: ubuntu-24.04 - timeout-minutes: 120 + # A wedge guard, not a budget. + timeout-minutes: 60 + container: *kvm permissions: contents: write + env: + GH_TOKEN: ${{ github.token }} steps: + - *deps + - name: gh + run: | + DEBIAN_FRONTEND=noninteractive apt-get install -y -qq gh - *checkout - - *runner-deps - - env: - GH_TOKEN: ${{ github.token }} - run: cargo run -- --ci release + - *guest-cache + - run: cargo run -- --ci release # One standing issue, found by title and commented on; a dispatch is somebody # watching the run, so only the schedule files. nightly-red: - needs: [host, build, guest, tcg, audio, portability-linux, portability-macos] + needs: [host, build, guest, tcg, audio, portability-linux, portability-macos, release] if: ${{ !cancelled() && github.event_name == 'schedule' }} runs-on: ubuntu-latest permissions: diff --git a/Cargo.toml b/Cargo.toml index e60b07b8ce..700f84c71e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -162,7 +162,7 @@ image = { version = "0.25", default-features = false, features = ["jpeg"] } # nothing new is fetched — but `cargo run` did not compile it before this. sha2 = "0.10" # The image release's compressed asset, which every stock macOS and Linux -# unpacks (`src/imagerelease.rs`). Already resolved here through `bcachefs`. +# unpacks (`src/imagerelease.rs`). flate2 = { version = "1", default-features = false, features = ["rust_backend"] } [dev-dependencies] diff --git a/bootloader/src/floor.rs b/bootloader/src/floor.rs index b0210e5615..d75b4181ca 100644 --- a/bootloader/src/floor.rs +++ b/bootloader/src/floor.rs @@ -17,14 +17,12 @@ use alloc::vec::Vec; use toyos_update::floor::{self, Read, Scope, Stored}; use uefi::prelude::*; use uefi::table::runtime::{VariableAttributes, VariableVendor}; -use uefi::{guid, CString16}; +use uefi::{CString16, Guid}; /// Whose images this loader's floor holds, as its build decided. const SCOPE: Scope = Scope::from_word(env!("TOYOS_IMAGE_FLOOR")); -/// The vendor every floor is under: `33BE3D4A-30E6-49F5-8050-F169D93A20FB`, -/// minted for this and used for nothing else. -const VENDOR: VariableVendor = VariableVendor(guid!("33be3d4a-30e6-49f5-8050-f169d93a20fb")); +const VENDOR: VariableVendor = VariableVendor(Guid::parse_or_panic(floor::VENDOR)); /// The only attributes the floor is written with. const ATTRIBUTES: VariableAttributes = diff --git a/issues/boot-media/the-loader-writes-the-first-disk-carrying-its-log-guid-not-the-one-it-booted-from.md b/issues/boot-media/the-loader-writes-the-first-disk-carrying-its-log-guid-not-the-one-it-booted-from.md new file mode 100644 index 0000000000..cdfe55b74f --- /dev/null +++ b/issues/boot-media/the-loader-writes-the-first-disk-carrying-its-log-guid-not-the-one-it-booted-from.md @@ -0,0 +1,24 @@ +--- +status: open +kind: defect +opened: 2026-09-27 +--- + +# The loader writes the first disk carrying its log GUID, not the one it booted from + +`loaderlog::volume_handle` (`bootloader/src/loaderlog.rs`) takes the first +filesystem on the machine whose GPT partition's unique GUID is the log +partition's, and `loader.log` and the attempt records are written there. Every +copy of one image carries the same partition unique GUIDs, and the image +release (`src/imagerelease.rs`) is written byte for byte to every stick made +from it. With two sticks of one release plugged in, the loader can write the +log partition of the stick it did not boot from: a disk it was not given, and +one that carries no TOYOS-DATA partition, which the release notes say a boot +never writes. + +Owner: the bootloader. + +**Exit condition.** The loader writes only the device it booted from: the log +volume is looked up on the device of the loaded image's own handle, and a boot +with two copies of one image attached writes the other copy's bytes not at +all. diff --git a/issues/build/the-image-release-carries-no-notice-of-the-crates-it-links.md b/issues/build/the-image-release-carries-no-notice-of-the-crates-it-links.md deleted file mode 100644 index d668acdb10..0000000000 --- a/issues/build/the-image-release-carries-no-notice-of-the-crates-it-links.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-09-27 ---- - -# The image release carries no notice of the crates its programs link - -The image release carries ToyOS's own licences, `NOTICE`, and the texts -`NOTICE` names for the fonts and icons on the image (`LICENCE_ASSETS` in -`src/imagerelease.rs`). Every program on the image is also compiled from -third-party crates, the ones `src/licence.rs` walks, and MIT, BSD and -Apache-2.0 each ask that a copy of the software carry the licence text, and -MIT and BSD its copyright notice too. `NOTICE` says those crates keep their own -upstream licences, and nothing the release publishes carries one. - -**Exit condition.** The release carries the licence text and copyright notice -of every package the licence gate finds in the release image, generated from -the gate's own walk, and a test fails when a shipped package has none. diff --git a/issues/kernel/the-kernel-dies-at-boot-on-the-edk2-firmware-qemu-ships.md b/issues/kernel/the-kernel-dies-at-boot-on-the-edk2-firmware-qemu-ships.md index 41d3a0a0ab..86db2c2079 100644 --- a/issues/kernel/the-kernel-dies-at-boot-on-the-edk2-firmware-qemu-ships.md +++ b/issues/kernel/the-kernel-dies-at-boot-on-the-edk2-firmware-qemu-ships.md @@ -1,5 +1,5 @@ --- -status: open +status: expected-red kind: defect opened: 2026-09-27 --- @@ -26,9 +26,8 @@ The command line is `imagerelease::Host::MacosAppleSilicon`'s (`src/imagerelease.rs`) with `-display none`, over a copy of a `target/bootable.img`. -It blocks the image release on macOS: the notes' macOS command line boots this -firmware, and so do `release_command_boots` and `release_writes_no_other_disk` -on the dev host. +`release_command_boots` boots that line on the dev host, and is disabled in +`src/redlist.rs` while this stands. **Exit condition.** `release_command_boots` is green on the dev host with the -firmware Homebrew's QEMU ships. +firmware Homebrew's QEMU ships, and its row leaves `src/redlist.rs`. diff --git a/src/build.rs b/src/build.rs index 4b39f1ec14..99b202475b 100644 --- a/src/build.rs +++ b/src/build.rs @@ -1026,8 +1026,10 @@ pub struct Boot { public: bool, } -/// What [`Boot::release`] writes. +/// What [`Boot::release`] writes: the image, and its licence notice +/// ([`crate::licence::notices`]), which is also on its ROOT. pub const RELEASE_IMAGE: &str = "target/bootable-release.img"; +pub const RELEASE_NOTICES: &str = "target/bootable-release-licences.txt"; impl Boot { /// **The one naming rule**: the artifact is named after the directory @@ -1091,6 +1093,26 @@ impl Boot { Self { image: PathBuf::from(RELEASE_IMAGE), public: true, ..Self::shipped(root) } } + /// [`Shipped`] for this boot's image, read out of its config the way + /// [`build`] reads it. + /// + /// **A config that ships the hosted compiler is refused**: its dependencies + /// are the rust fork's `compiler/` workspace, which no reader of this + /// answer walks. + pub fn parts(&self, root: &Path) -> Result { + let config = self.system(); + if config.hosted_rustc { + return Err(format!( + "{} sets hosted-rustc, and nothing reads the licences of the compiler it ships", + self.config.display() + )); + } + Ok(Shipped { + crates: config_crates(root, &config).into_iter().map(|c| (c.dir, c.features)).collect(), + assets: config.assets.iter().map(|dir| root.join(dir)).collect(), + }) + } + /// The config declares no `devices`, so nothing started there claims the /// framebuffer and the kernel's last boot checkpoint stays on screen. /// `screen_diag_boot` boots this same config, so the tested image and the @@ -1140,34 +1162,24 @@ impl Boot { } } -/// What the three modes' images are built from, besides std: every crate -/// [`config_crates`] names for one of them with the features the build gives -/// it, and the asset directories their configs copy onto ROOT. A case's config -/// is a test image and is not here. +/// What images are built from, besides std: every crate [`config_crates`] +/// names for them with the features the build gives it, and the asset +/// directories their configs copy onto ROOT. A case's config is a test image +/// and is not here. pub struct Shipped { pub crates: BTreeSet<(PathBuf, Features)>, pub assets: BTreeSet, } -/// [`Shipped`], read out of the modes' configs the way [`build`] reads them. -/// -/// **A config that ships the hosted compiler is refused**: its dependencies are -/// the rust fork's `compiler/` workspace, which no reader of this answer walks. +/// [`Boot::parts`] of the three modes together. pub fn shipped(root: &Path) -> Result { - let mut crates = BTreeSet::new(); - let mut assets = BTreeSet::new(); + let mut all = Shipped { crates: BTreeSet::new(), assets: BTreeSet::new() }; for boot in [Boot::shipped(root), Boot::diag(root), Boot::console(root)] { - let config = parse_config(&boot.config); - if config.hosted_rustc { - return Err(format!( - "{} sets hosted-rustc, and nothing reads the licences of the compiler it ships", - boot.config.display() - )); - } - crates.extend(config_crates(root, &config).into_iter().map(|c| (c.dir, c.features))); - assets.extend(config.assets.iter().map(|dir| root.join(dir))); + let parts = boot.parts(root)?; + all.crates.extend(parts.crates); + all.assets.extend(parts.assets); } - Ok(Shipped { crates, assets }) + Ok(all) } /// The parameters an image built for flashing may carry: the kernel's own boot @@ -1908,7 +1920,17 @@ fn shipped_parts(root: &Path, boot: &Boot, rebuild_toolchain: bool, plan: &Plan) ) }; - let root_bytes = build_and_assemble(root, &config, &env, &[], false, arch); + let mut extra = Vec::new(); + if boot.public { + let notices = boot + .parts(root) + .and_then(|parts| crate::licence::notices(root, parts)) + .unwrap_or_else(|why| panic!("the release's licence notice: {why}")); + let at = root.join(RELEASE_NOTICES); + fs::write(&at, ¬ices).unwrap_or_else(|e| panic!("write {}: {e}", at.display())); + extra.push((crate::licence::NOTICES_ON_ROOT.to_string(), notices.into_bytes())); + } + let root_bytes = build_and_assemble(root, &config, &env, &extra, false, arch); let bl_bytes = fs::read(&bl_art).expect("Failed to read staged bootloader"); (kernel_bytes, bl_bytes, root_bytes) diff --git a/src/ci.rs b/src/ci.rs index e79bf0e5fc..898a707d4c 100644 --- a/src/ci.rs +++ b/src/ci.rs @@ -7,8 +7,8 @@ //! [`Job::GateStage`] run as `host`. Every test that boots no guest is in //! [`Job::Host`], so a merge is gated on all of them. `nightly.yml` runs //! everything that boots a guest, `host` again to write the cache the merge -//! queue restores, and portability, and publishes `main`'s image once all of -//! that is green ([`Job::Release`]). `publish.yml` puts a landing's crates on +//! queue restores, portability, and the image release, which boots the image it +//! publishes ([`Job::Release`]). `publish.yml` puts a landing's crates on //! crates.io. //! //! A host job runs every step and reds if any failed; a guest job stops at the @@ -50,7 +50,7 @@ const USAGE: &str = "cargo run -- --ci , where is one of: guest / one shard of the whole guest suite, nightly tier included (nightly) tcg one test on an emulated CPU (nightly) audio / one shard of gate A (nightly) - release publish main's image once its nightly is green (nightly) + release build the release image, boot it, and on main publish it (nightly) nightly-red file or update the nightly-red issue from $NEEDS (nightly) publish put main's SDK crates on crates.io (publish.yml)"; @@ -103,17 +103,13 @@ pub fn dispatch(root: &Path, args: &[String]) { Job::GateStage => vec![step("what protects main", || gate_stage(root))], Job::Toolchain => vec![step("the toolchain release", || release::ensure_published(root))], Job::Guest(shard) => { - guest(root, &suite_args(&["--shard", shard, "--jobs", "1", "--nightly"])) + guest(root, "the suite", || suite(root, &suite_args(&["--shard", shard, "--jobs", "1", "--nightly"]))) } - Job::Tcg => guest(root, &suite_args(&["--jobs", "1", "process_stats"])), - Job::Audio(shard) => guest(root, &suite_args(&["--audio-gate", "30", "--shard", shard])), - Job::Release => { - let mut steps = vec![step("the toolchain", || release::install(root))]; - if steps.iter().all(|s| s.verdict.is_ok()) { - steps.push(step("the image release", || imagerelease::publish(root))); - } - steps + Job::Tcg => guest(root, "the suite", || suite(root, &suite_args(&["--jobs", "1", "process_stats"]))), + Job::Audio(shard) => { + guest(root, "the suite", || suite(root, &suite_args(&["--audio-gate", "30", "--shard", shard]))) } + Job::Release => guest(root, "the image release", || imagerelease::release(root)), Job::NightlyRed => vec![step("the nightly-red issue", nightly_red)], Job::Publish => vec![step("the SDK crates on crates.io", || publish(root))], }; @@ -627,7 +623,7 @@ fn suite_args(args: &[&str]) -> Vec { /// A guest job's own `$TMPDIR`, same rule as [`host`]: nothing the suite /// writes past a `toyos_tmpdir::TempDir` — the harness's own `Run`, its lanes, /// every boot image — survives past the last step, which reds on it. -fn guest(root: &Path, suite: &[String]) -> Vec { +fn guest(root: &Path, label: &str, boots: impl FnOnce() -> Result) -> Vec { let tmp = toyos_tmpdir::TempDir::new("ci-guest"); // Before any thread, same as `host`: every child this process spawns below // inherits this, and nothing here reads the environment concurrently with @@ -640,16 +636,7 @@ fn guest(root: &Path, suite: &[String]) -> Vec { steps.push(step("the toolchain", || release::install(root))); } if steps.iter().all(|s| s.verdict.is_ok()) { - steps.push(step("the suite", || { - let args: Vec<&str> = suite.iter().map(String::as_str).collect(); - let (green, log) = cargo_logged(root, &args)?; - let said = verdicts(&log); - if green { - Ok(said) - } else { - Err(said) - } - })); + steps.push(step(label, boots)); } // Unconditional: whatever stopped earlier, this $TMPDIR is still this // process's own to judge, and a leak past a failing suite is still a leak. @@ -657,6 +644,18 @@ fn guest(root: &Path, suite: &[String]) -> Vec { steps } +/// `cargo `, judged by its exit and summarised by [`verdicts`]. +fn suite(root: &Path, suite: &[String]) -> Result { + let args: Vec<&str> = suite.iter().map(String::as_str).collect(); + let (green, log) = cargo_logged(root, &args)?; + let said = verdicts(&log); + if green { + Ok(said) + } else { + Err(said) + } +} + /// The suite's own count line and every line naming a verdict worth reading /// without the log: a failure. fn verdicts(log: &str) -> String { diff --git a/src/flags.rs b/src/flags.rs index 3938e17872..eb8fb50d0c 100644 --- a/src/flags.rs +++ b/src/flags.rs @@ -74,8 +74,6 @@ declare_flags!(pub CARGO_RUN = { pub KERNEL_FEATURE = "--kernel-feature", Each; pub DIAG_BOOT = "--diag-boot", None; pub CONSOLE_BOOT = "--console-boot", None; - /// Build the image the release job publishes (`build::Boot::release`). - pub RELEASE_BOOT = "--release-boot", None; pub BOOT_CONFIG = "--boot-config", Next; pub ARCH = "--arch", Next; pub REGEN_FONT = "--regen-font", None; diff --git a/src/imagerelease.rs b/src/imagerelease.rs index d03c96df4f..268d67f91d 100644 --- a/src/imagerelease.rs +++ b/src/imagerelease.rs @@ -1,25 +1,28 @@ //! The image release: the disk a person downloads and boots under QEMU or -//! writes to a stick, published by `cargo run -- --ci release` from a commit of -//! `main` whose whole nightly was green. +//! writes to a stick, published by `cargo run -- --ci release`. //! -//! **Named by that commit**, [`tag`]: every build draws its partition GUIDs and -//! a runner mints its own throwaway signing key (`src/signing.rs`), so the -//! bytes name nothing a second build reproduces and a content hash would name -//! one build. **Kept to [`KEEP`]**: each publish deletes every older image -//! release and its tag ([`stale`]). -//! -//! The image is `build::Boot::release`'s. The notes carry the two command lines -//! [`Host::command`] declares, which are the argv `release_command_boots` and -//! `release_writes_no_other_disk` boot. +//! **What is published is what booted**: the job builds `build::Boot::release`'s +//! image once, boots a copy of it under the Linux command line its notes print +//! ([`boots`]), and on `main` uploads those bytes. **Named by the commit**, +//! [`tag`]: every build draws its partition GUIDs and a runner mints its own +//! throwaway signing key (`src/signing.rs`), so a second build reproduces no +//! byte of the first. **Kept to [`KEEP`]**: each publish deletes every older +//! image release and its tag ([`stale`]). //! //! Not in `src/release.rs`, whose bytes are hashed into the toolchain's tag. use std::fs; -use std::io::Write; +use std::io::{BufRead, BufReader, Write}; use std::path::{Path, PathBuf}; -use std::process::Command; +use std::process::{Child, Command, Stdio}; +use std::sync::mpsc::{self, Receiver, RecvTimeoutError}; +use std::time::{Duration, Instant}; + +use serde_json::Value; use crate::arch::{Accel, Arch}; +use crate::fingerprint::{first_difference, whole_device}; +use crate::licence::{pending_owner, Subject}; /// What every image release's tag starts with; the rest is the commit's first /// twelve hex digits. @@ -40,18 +43,16 @@ pub const SUMS_ASSET: &str = "SHA256SUMS"; /// The notes, also carried as an asset. pub const NOTES_ASSET: &str = "README.md"; -/// Every licence text the release carries beside the image, as (the file in -/// this tree, its asset name): ToyOS's own two, the ledger, and each text -/// `NOTICE` names for the third-party files the release image ships -/// (`every_text_notice_names_for_what_the_release_ships_is_carried`). -pub const LICENCE_ASSETS: &[(&str, &str)] = &[ - ("LICENSE-MIT", "LICENSE-MIT"), - ("LICENSE-APACHE", "LICENSE-APACHE"), - ("NOTICE", "NOTICE"), - ("licenses/OFL-1.1-JetBrainsMono.txt", "OFL-1.1-JetBrainsMono.txt"), - ("licenses/MIT-PhosphorIcons.txt", "MIT-PhosphorIcons.txt"), - ("assets/fonts/OFL.txt", "OFL-1.1-OpenSans.txt"), -]; +/// The image's licence notice (`build::RELEASE_NOTICES`), as an asset. +pub const LICENCES_ASSET: &str = "licences.txt"; + +/// One guest's ceiling from power-on to a painting desktop, unscaled: a +/// liveness guard, never a verdict. +pub const DESKTOP: Duration = Duration::from_secs(120); + +/// How many releases one listing asks for; a listing that fills it is refused, +/// since what it left out cannot be judged. +const LISTED: usize = 1000; /// `image-x86_64-<12 hex>` of a full commit id. pub fn tag(commit: &str) -> Result { @@ -74,7 +75,8 @@ pub enum Host { impl Host { pub const ALL: [Host; 2] = [Host::MacosAppleSilicon, Host::LinuxKvm]; - /// The one of the two this machine is, or why it is neither. + /// The one of the two this machine is, or, refused by name, that it is + /// neither: the notes print no line for it, so there is no line to boot. pub fn this() -> Result { if cfg!(target_os = "macos") && Arch::HOST == Some(Arch::Aarch64) { return Ok(Host::MacosAppleSilicon); @@ -82,8 +84,8 @@ impl Host { if cfg!(target_os = "linux") && Arch::HOST == Some(Arch::X86_64) && Arch::X86_64.accel() == Accel::Kvm { return Ok(Host::LinuxKvm); } - Err("this host is neither an Apple Silicon Mac nor an x86-64 Linux whose /dev/kvm opens, \ - so no command line the release notes print is this host's" + Err("the release notes print a command line for an Apple Silicon Mac and for an x86-64 \ + Linux whose /dev/kvm opens, and this host is neither" .into()) } @@ -167,6 +169,108 @@ impl Host { } } +/// A QEMU started from a release command line, and its console so far. +struct Guest { + child: Child, + lines: Receiver, + log: String, + stderr: PathBuf, +} + +impl Guest { + /// `argv` with no window: the notes' line, headless. + fn start(argv: &[String], stderr: &Path) -> Result { + let err = fs::File::create(stderr).map_err(|e| format!("{}: {e}", stderr.display()))?; + let arch = Arch::X86_64; + if argv[0] != arch.qemu() { + return Err(format!("a release command line starts {:?}, not {}", argv[0], arch.qemu())); + } + let mut child = Command::new(arch.qemu()) + .args(&argv[1..]) + .args(["-display", "none"]) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(err) + .spawn() + .map_err(|e| format!("{}: {e}", arch.qemu()))?; + let out = child.stdout.take().expect("piped"); + let (send, lines) = mpsc::channel(); + std::thread::spawn(move || { + for line in BufReader::new(out).split(b'\n') { + let Ok(line) = line else { return }; + if send.send(String::from_utf8_lossy(&line).into_owned()).is_err() { + return; + } + } + }); + Ok(Guest { child, lines, log: String::new(), stderr: stderr.to_path_buf() }) + } + + /// Read the console until every line of `want` is in it, within + /// `ceiling`, refusing a panic the moment one is printed. + fn until_said(&mut self, want: &[String], ceiling: Duration) -> Result<(), String> { + let deadline = Instant::now() + ceiling; + loop { + if let Some(line) = self.log.lines().find(|l| l.contains("PANIC") || l.contains("panicked at")) { + return Err(format!("the guest panicked before the desktop: {line}\n{}", self.log)); + } + if want.iter().all(|line| self.log.contains(line.as_str())) { + return Ok(()); + } + match self.lines.recv_timeout(deadline.saturating_duration_since(Instant::now())) { + Ok(line) => { + self.log.push_str(&line); + self.log.push('\n'); + } + Err(RecvTimeoutError::Timeout) => { + return Err(format!("no desktop within {ceiling:?}:\n{}", self.log)); + } + Err(RecvTimeoutError::Disconnected) => { + let stderr = fs::read_to_string(&self.stderr).unwrap_or_default(); + return Err(format!( + "QEMU closed its console before the desktop:\n{}\nQEMU's stderr:\n{stderr}", + self.log + )); + } + } + } + } +} + +impl Drop for Guest { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +/// Boot `stick` under `host`'s command line until the desktop paints: every +/// program `system.toml` starts said it started, the three that announce +/// themselves did, and the compositor reported a frame. And neither firmware +/// file changed, since the line gives the guest both read-only. +pub fn boots(root: &Path, host: Host, stick: &Path, ceiling: Duration, stderr: &Path) -> Result<(), String> { + let firmware = host.firmware(); + if let Some(missing) = firmware.iter().find(|f| !Path::new(f).is_file()) { + return Err(format!("{missing} is no file: this host lacks the firmware {host:?}'s line names")); + } + let before = firmware.map(|f| whole_device(Path::new(f))); + let mut want: Vec = crate::build::boot_start(&root.join("system.toml")) + .iter() + .map(|program| format!("init: started {program}")) + .collect(); + want.extend( + ["compositor: ready", "netd: ready", "logd: this boot's kernel log is", "compositor: frames="] + .map(String::from), + ); + let desktop = Guest::start(&host.command(&stick.display().to_string()), stderr)?.until_said(&want, ceiling); + for (file, before) in firmware.iter().zip(&before) { + if let Some(diff) = first_difference(before, &whole_device(Path::new(file))) { + return Err(format!("the boot wrote {file}, which the line gives the guest read-only: {diff}")); + } + } + desktop +} + /// [`Host::command`] as the notes print it: an option and its value to a line. fn shell(argv: &[String]) -> String { let mut out = format!(" {}", argv[0]); @@ -181,19 +285,35 @@ fn shell(argv: &[String]) -> String { out } +/// The floor variable a boot of the release leaves in the firmware, as the +/// notes name it: its name's fixed head, and how many hex digits follow. +fn floor_variable() -> (String, usize) { + use toyos_update::floor::{Scope, NAME_BYTES, PREFIX}; + let head = format!("{PREFIX}-{}", Scope::Image.tag() as char); + let hex = NAME_BYTES - head.len(); + (head, hex) +} + /// The release notes, which are also [`NOTES_ASSET`]. pub fn notes(root: &Path, tag: &str, commit: &str) -> Result { let qemu = crate::ci::declared_qemu_version(root).ok_or(".github/qemu-version declares no version")?; let data = toyos_gpt::Guid::TOYOS_DATA_TEXT; + let (floor, hex) = floor_variable(); + let vendor = toyos_update::floor::VENDOR; + let on_root = crate::licence::NOTICES_ON_ROOT; + let withheld: Vec = pending_owner() + .map(|s| match s { + Subject::Crate(path) | Subject::Notice(path) | Subject::File(path) => format!("`{path}`"), + }) + .collect(); let mut commands = String::new(); for host in Host::ALL { commands.push_str(&format!("{}:\n\n{}\n\n", host.named(), shell(&host.command(IMAGE)))); } - let texts: Vec = LICENCE_ASSETS.iter().map(|(_, asset)| format!("`{asset}`")).collect(); Ok(format!( "# ToyOS {tag} -The ToyOS disk image of commit {commit} on `main`, whose nightly was green. It boots under QEMU, or from a USB stick on a UEFI x86-64 machine. +The ToyOS disk image of commit {commit} on `main`. It booted to its desktop under the Linux command line below before it was published. It boots under QEMU, or from a USB stick on a UEFI x86-64 machine. ## Verify and unpack @@ -204,7 +324,7 @@ Download `{IMAGE_ASSET}` and `{SUMS_ASSET}` from this release into one directory ## Under QEMU -QEMU {qemu} is the version ToyOS is measured with. The firmware is edk2, from Homebrew's QEMU on macOS and from Debian's `ovmf` on Linux. +QEMU {qemu} is the version ToyOS is measured with. The firmware is edk2, from Homebrew's QEMU on macOS and from Debian's `ovmf` on Linux, and the guest gets both of its files read-only. {commands}The kernel's log is on the terminal and the desktop is in QEMU's window. The running system writes to `{IMAGE}` itself, as it would to a stick. `/apps`, `/config`, `/home` and `/state` are kept in memory and are gone at the next boot. @@ -212,24 +332,25 @@ QEMU {qemu} is the version ToyOS is measured with. The firmware is edk2, from Ho The machine has to be an x86-64 PC from 2020 or later, booting UEFI with Secure Boot off. The only hardware ToyOS is known to work on is a Lenovo ThinkPad T14; on anything else it is untried. -Write `{IMAGE}` to the whole stick, not to a partition of it, as root; what the stick held is lost: +Booting the stick writes two things into the machine's firmware: + +- a variable named `{floor}` and {hex} hex digits, under the vendor GUID `{vendor}`: the loader's anti-rollback floor, eight bytes. It stays after the stick is gone. It is readable only before an operating system starts, so no operating system can see or remove it; only a UEFI shell can, with `dmpstore -d -guid {vendor}`. Booting another ToyOS image's stick replaces it rather than adding one; +- `BootNext`, where one of the machine's boot entries names the stick: the next restart boots the stick once. + +Write `{IMAGE}` to the whole stick, not to a partition of it; what the stick held is lost. Unmount it first: on macOS `diskutil unmountDisk /dev/`, on Linux `umount` each of its partitions that is mounted (`lsblk /dev/` lists them). Then, as root: dd if={IMAGE} of=/dev/ bs=4194304 sync -What a boot writes on the machine: - -- the stick it booted from; -- the firmware's variable store: the loader's anti-rollback floor, and `BootNext` where a boot entry names the stick; -- another disk only where it carries a partition of ToyOS's DATA type, `{data}`, a type no other system uses. That is where `/apps`, `/config`, `/home` and `/state` live. Every other disk is read for its partition table and never written: `release_writes_no_other_disk` boots this image beside a disk laid out as another operating system's and compares every byte of it. +A boot writes to the stick it booted from, and to another disk only where that disk carries a partition of ToyOS's DATA type, `{data}`, a type no other system uses. That is where `/apps`, `/config`, `/home` and `/state` live. Every other disk is read for its partition table and never written. ## Terms -ToyOS is MIT OR Apache-2.0. `NOTICE` names every third-party file in the repository and its terms; this release carries it and the texts for what the image ships: {texts}. +ToyOS is MIT OR Apache-2.0. `{LICENCES_ASSET}`, beside the image and on it at `/system/{on_root}`, gives every package the image is built from with its licence, where its source is and its licence texts, and every third-party file on the image with its terms and texts. -The image leaves out doom, `DOOM1.WAD` and the SoundFont: whether they may ship is the owner's to rule (`src/licence.rs`), and nothing is published while it is not. +The image leaves out {withheld}, which the repository carries. ", - texts = texts.join(", "), + withheld = withheld.join(", "), )) } @@ -243,9 +364,16 @@ pub fn stale(listed: &[(String, String)], keep: usize) -> Vec { } /// Write every asset of `tag`'s release into `out`: the image at `image` -/// compressed, its sum, the notes and the licence texts. Answers the paths in -/// upload order. -pub fn write_assets(root: &Path, image: &Path, out: &Path, tag: &str, commit: &str) -> Result, String> { +/// compressed, its sum, the notes and the licence notice at `licences`. +/// Answers the paths in upload order. +pub fn write_assets( + root: &Path, + image: &Path, + licences: &Path, + out: &Path, + tag: &str, + commit: &str, +) -> Result, String> { use flate2::write::GzEncoder; use sha2::{Digest, Sha256}; @@ -264,15 +392,12 @@ pub fn write_assets(root: &Path, image: &Path, out: &Path, tag: &str, commit: &s let readme = out.join(NOTES_ASSET); fs::write(&readme, notes(root, tag, commit)?).map_err(|e| e.to_string())?; - let mut assets = vec![compressed, sums, readme]; - for (from, name) in LICENCE_ASSETS { - let to = out.join(name); - fs::copy(root.join(from), &to).map_err(|e| format!("{from}: {e}"))?; - assets.push(to); - } - Ok(assets) + let notice = out.join(LICENCES_ASSET); + fs::copy(licences, ¬ice).map_err(|e| format!("{}: {e}", licences.display()))?; + Ok(vec![compressed, sums, readme, notice]) } +/// `gh `: what it printed, or its exit and what it said. fn gh(root: &Path, args: &[&str]) -> Result { let out = Command::new("gh").args(args).current_dir(root).output().map_err(|e| format!("gh: {e}"))?; if out.status.success() { @@ -282,61 +407,108 @@ fn gh(root: &Path, args: &[&str]) -> Result { } } -/// Whether `tag` is a release carrying [`IMAGE_ASSET`]. -fn published(root: &Path, tag: &str) -> bool { - gh(root, &["release", "view", tag, "--json", "assets", "--jq", ".assets[].name"]) - .is_ok_and(|names| names.lines().any(|name| name == IMAGE_ASSET)) +/// A release GitHub holds under a tag: whether it is still a draft, and +/// whether it carries [`IMAGE_ASSET`]. +#[derive(Debug, PartialEq, Eq)] +struct Held { + draft: bool, + image: bool, +} + +/// What GitHub holds under `tag`: nothing, which `gh` says as exactly +/// `release not found`, or a [`Held`]. Any other failure of `gh` is one. +fn held(root: &Path, tag: &str) -> Result, String> { + let json = match gh(root, &["release", "view", tag, "--json", "isDraft,assets"]) { + Err(why) if why.ends_with(": release not found") => return Ok(None), + said => said?, + }; + let v: Value = serde_json::from_str(&json).map_err(|e| format!("gh release view {tag}: {e}"))?; + let draft = v["isDraft"].as_bool().ok_or_else(|| format!("gh release view {tag} gave no isDraft: {json}"))?; + let assets = v["assets"].as_array().ok_or_else(|| format!("gh release view {tag} gave no assets: {json}"))?; + let image = assets.iter().any(|a| a["name"].as_str() == Some(IMAGE_ASSET)); + Ok(Some(Held { draft, image })) } -/// `cargo run -- --ci release`: publish this commit's image unless it is -/// published, then delete the image releases past [`KEEP`]. Only a nightly on -/// `main` publishes, and `nightly.yml` runs this only once every lane of that -/// nightly is green. -pub fn publish(root: &Path) -> Result { - let on_runner = std::env::var("GITHUB_ACTIONS").is_ok_and(|v| v == "true"); - if !on_runner || std::env::var("GITHUB_REF").ok().as_deref() != Some("refs/heads/main") { - return Err("only a nightly on main publishes an image".into()); +/// Every release, as (tag, creation time), refusing a listing that filled +/// [`LISTED`] or an entry without both. +fn listed(root: &Path) -> Result, String> { + let limit = LISTED.to_string(); + let json = gh(root, &["release", "list", "--limit", &limit, "--json", "tagName,createdAt"])?; + let v: Value = serde_json::from_str(&json).map_err(|e| format!("gh release list: {e}"))?; + let all = v.as_array().ok_or_else(|| format!("gh release list gave no list: {json}"))?; + if all.len() >= LISTED { + return Err(format!("gh release list gave {} releases, its limit, so some are not in it", all.len())); + } + all.iter() + .map(|r| match (r["tagName"].as_str(), r["createdAt"].as_str()) { + (Some(tag), Some(at)) => Ok((tag.to_string(), at.to_string())), + _ => Err(format!("gh release list gave a release without a tag and a time: {r}")), + }) + .collect() +} + +/// Upload `assets` as a draft of `tag`, and publish it once GitHub holds the +/// image, so a failed upload leaves nothing public. +fn publish(root: &Path, tag: &str, commit: &str, notes: &Path, assets: &[PathBuf]) -> Result<(), String> { + let notes = notes.display().to_string(); + let mut args: Vec<&str> = + vec!["release", "create", tag, "--draft", "--title", tag, "--target", commit, "--notes-file", ¬es]; + let assets: Vec = assets.iter().map(|a| a.display().to_string()).collect(); + args.extend(assets.iter().map(String::as_str)); + gh(root, &args)?; + let drafted = held(root, tag)?; + if drafted != Some(Held { draft: true, image: true }) { + return Err(format!("{tag} was created as a draft carrying {IMAGE_ASSET}, and GitHub holds {drafted:?}")); + } + gh(root, &["release", "edit", tag, "--draft=false", "--latest"])?; + let published = held(root, tag)?; + if published != Some(Held { draft: false, image: true }) { + return Err(format!("{tag} was published, and GitHub holds {published:?}")); + } + Ok(()) +} + +/// `cargo run -- --ci release`: build the release image once, boot a copy of +/// it under this host's line, and on `main` publish those bytes unless this +/// commit's are, then delete the image releases past [`KEEP`]. +pub fn release(root: &Path) -> Result { + if !std::env::var("GITHUB_ACTIONS").is_ok_and(|v| v == "true") { + return Err("only a runner releases an image".into()); } let commit = std::env::var("GITHUB_SHA").map_err(|_| "GITHUB_SHA is unset".to_string())?; let head = crate::pr::git(root, &["rev-parse", "HEAD"])?; if head != commit { - return Err(format!("the checkout is {head} and the nightly ran {commit}")); + return Err(format!("the checkout is {head} and the run is of {commit}")); } let tag = tag(&commit)?; - - let mut said = if published(root, &tag) { - format!("{tag} is already published") - } else { - let built = Command::new("cargo") - .args(["run", "--", "--release-boot", "--build-only"]) - .current_dir(root) - .status() - .map_err(|e| format!("cargo: {e}"))?; - if !built.success() { - return Err(format!("cargo run -- --release-boot --build-only exited {built}")); - } - let out = toyos_tmpdir::TempDir::new("image-release"); - let assets = write_assets(root, &root.join(crate::build::RELEASE_IMAGE), &out, &tag, &commit)?; - let notes = out.join(NOTES_ASSET); - let mut args: Vec = ["release", "create", &tag, "--title", &tag, "--target", &commit, "--latest", "--notes-file"] - .map(String::from) - .to_vec(); - args.push(notes.display().to_string()); - args.extend(assets.iter().map(|a| a.display().to_string())); - gh(root, &args.iter().map(String::as_str).collect::>())?; - if !published(root, &tag) { - return Err(format!("{tag} was created and carries no {IMAGE_ASSET}")); + let on_main = std::env::var("GITHUB_REF").ok().as_deref() == Some("refs/heads/main"); + let host = Host::this()?; + + let mut said = match held(root, &tag)? { + Some(Held { draft: false, image: true }) => format!("{tag} is already published"), + Some(other) => return Err(format!("GitHub holds {tag} as {other:?}: a failed run's, to delete by hand")), + None => { + let boot = crate::build::Boot::release(root); + let plan = crate::build::plan_for(root, &boot, false, &[]); + let image = crate::build::build(root, boot, false, &plan); + let out = toyos_tmpdir::TempDir::new("image-release"); + let stick = out.join("stick.img"); + fs::copy(&image, &stick).map_err(|e| format!("copy {} to {}: {e}", image.display(), stick.display()))?; + boots(root, host, &stick, DESKTOP, &out.join("qemu.stderr"))?; + fs::remove_file(&stick).map_err(|e| format!("{}: {e}", stick.display()))?; + if !on_main { + return Ok(format!("{tag} booted to its desktop under {host:?}'s line; off main, so not published")); + } + let licences = root.join(crate::build::RELEASE_NOTICES); + let assets = write_assets(root, &image, &licences, &out, &tag, &commit)?; + publish(root, &tag, &commit, &out.join(NOTES_ASSET), &assets)?; + format!("{tag} booted to its desktop under {host:?}'s line and is published") } - format!("{tag} published") }; - - let listed = gh(root, &["release", "list", "--limit", "1000", "--json", "tagName,createdAt", "--jq", ".[] | .tagName + \" \" + .createdAt"])?; - let listed: Vec<(String, String)> = listed - .lines() - .filter_map(|l| l.split_once(' ')) - .map(|(t, c)| (t.to_string(), c.to_string())) - .collect(); - let old = stale(&listed, KEEP); + if !on_main { + return Ok(said); + } + let old = stale(&listed(root)?, KEEP); for old in &old { gh(root, &["release", "delete", old, "--cleanup-tag", "--yes"])?; } @@ -352,6 +524,10 @@ mod tests { PathBuf::from(env!("CARGO_MANIFEST_DIR")) } + fn notes_of_a_commit() -> String { + notes(&root(), "image-x86_64-c55189490123", &"c".repeat(40)).unwrap() + } + #[test] fn a_tag_is_the_commit_and_a_short_or_foreign_id_is_refused() { let commit = "c55189490123456789abcdef0123456789abcdef"; @@ -378,7 +554,7 @@ mod tests { /// that host's own accelerator and firmware, and the notes print it whole. #[test] fn the_notes_print_each_hosts_command_line_whole() { - let notes = notes(&root(), "image-x86_64-c55189490123", &"c".repeat(40)).unwrap(); + let notes = notes_of_a_commit(); for host in Host::ALL { let argv = host.command(IMAGE); assert!(argv.iter().any(|a| a == &format!("if=none,id=stick,format=raw,file={IMAGE}"))); @@ -393,55 +569,30 @@ mod tests { } } - /// The release carries the text `NOTICE` names for every third-party file - /// under an asset directory the release image ships, and none only - /// withheld material names. + /// Before the line that writes the stick, the notes name the firmware + /// variable a boot leaves behind by the name and vendor the loader writes + /// it under, how it is removed, and `BootNext`; and they name everything + /// the release leaves out. #[test] - fn every_text_notice_names_for_what_the_release_ships_is_carried() { - use crate::licence::Subject; - let root = root(); - let notice = fs::read_to_string(root.join("NOTICE")).unwrap(); - let assets: Vec = crate::build::shipped(&root) - .unwrap() - .assets - .iter() - .map(|dir| format!("{}/", dir.strip_prefix(&root).unwrap().display())) - .collect(); - let withheld: Vec<&str> = crate::licence::pending_owner() - .map(|s| match s { - Subject::Crate(p) | Subject::Notice(p) | Subject::File(p) => p, - }) - .collect(); - let carried: Vec<&str> = LICENCE_ASSETS.iter().map(|(from, _)| *from).collect(); - let mut needed = vec!["LICENSE-MIT", "LICENSE-APACHE", "NOTICE"]; - let mut shipped_sections = 0; - let sections = crate::licence::sections(¬ice); - for section in §ions { - let ships = assets.iter().any(|dir| section.path.starts_with(dir.as_str())) - && !withheld.contains(§ion.path.as_str()); - if !ships { - continue; - } - shipped_sections += 1; - for text in §ion.texts { - assert!(carried.contains(&text.as_str()), "{} names {text}, which the release does not carry", section.path); - needed.push(text.as_str()); - } + fn the_notes_name_what_a_boot_writes_to_the_firmware_before_the_stick_is_written() { + let notes = notes_of_a_commit(); + let (floor, hex) = floor_variable(); + let dd = notes.find(" dd if=").expect("no dd line"); + let named = format!("`{floor}` and {hex} hex digits"); + let vendor = format!("`{}`", toyos_update::floor::VENDOR); + for said in [named.as_str(), &vendor, "dmpstore -d", "`BootNext`", "the next restart boots the stick once", "unmountDisk"] { + let at = notes.find(said).unwrap_or_else(|| panic!("the notes never say {said:?}")); + assert!(at < dd, "{said:?} comes after the dd line"); } - assert!(shipped_sections >= 3, "{shipped_sections} NOTICE sections read as shipped"); - for from in &carried { - assert!(root.join(from).is_file(), "{from}"); - assert!(needed.contains(from), "{from} is carried and nothing the release ships names it"); + for withheld in pending_owner() { + let (Subject::Crate(path) | Subject::Notice(path) | Subject::File(path)) = withheld; + assert!(notes.contains(&format!("`{path}`")), "the notes do not say {path} is left out"); } - let names: Vec<&str> = LICENCE_ASSETS.iter().map(|(_, name)| *name).collect(); - let mut unique = names.clone(); - unique.sort_unstable(); - unique.dedup(); - assert_eq!(unique.len(), names.len(), "two texts under one asset name: {names:?}"); } - /// What `sha256sum -c` checks is the compressed asset's own digest, and the - /// asset decompresses to the image byte for byte. + /// What `sha256sum -c` checks is the compressed asset's own digest, the + /// asset decompresses to the image byte for byte, and the notice is + /// carried as it was written. #[test] fn the_sum_is_the_compressed_images_and_it_decompresses_to_the_image() { use sha2::{Digest, Sha256}; @@ -450,13 +601,15 @@ mod tests { let image = dir.join("in.img"); let bytes: Vec = (0..300_000u32).map(|i| (i % 251) as u8).chain(std::iter::repeat_n(0, 1 << 20)).collect(); fs::write(&image, &bytes).unwrap(); + let licences = dir.join("notice.txt"); + fs::write(&licences, "the notice").unwrap(); let out = dir.join("out"); fs::create_dir(&out).unwrap(); - let assets = write_assets(&root(), &image, &out, "image-x86_64-c55189490123", &"c".repeat(40)).unwrap(); + let assets = write_assets(&root(), &image, &licences, &out, "image-x86_64-c55189490123", &"c".repeat(40)).unwrap(); let names: Vec = assets.iter().map(|a| a.file_name().unwrap().to_string_lossy().into_owned()).collect(); - assert_eq!(names[..3], [IMAGE_ASSET, SUMS_ASSET, NOTES_ASSET]); - assert_eq!(names.len(), 3 + LICENCE_ASSETS.len()); + assert_eq!(names, [IMAGE_ASSET, SUMS_ASSET, NOTES_ASSET, LICENCES_ASSET]); + assert_eq!(fs::read_to_string(out.join(LICENCES_ASSET)).unwrap(), "the notice"); let gz = fs::read(out.join(IMAGE_ASSET)).unwrap(); let sum: String = Sha256::digest(&gz).iter().map(|b| format!("{b:02x}")).collect(); diff --git a/src/licence.rs b/src/licence.rs index c2f3671262..4b1b348ad3 100644 --- a/src/licence.rs +++ b/src/licence.rs @@ -40,6 +40,11 @@ //! section. And std's graph is re-locked when the gate runs, not read from a //! committed lock, so two runs at one head can resolve it differently. //! +//! **The same walk writes an image's licence notice** ([`notices`]): each +//! package it reaches with its own licence files, or with the standard texts +//! of its licence where it publishes none, and each `NOTICE` section over a +//! file that ships, with the texts that section names. +//! //! The expression grammar is SPDX's (`OR`, `AND`, `WITH`, parentheses) plus //! cargo's legacy `/` for `OR`. An identifier outside the allowlist is refused //! whether SPDX knows it or not, so no licence list is needed to refuse the @@ -51,6 +56,7 @@ use std::process::Command; use serde_json::Value; +use crate::arch::Arch; use crate::build::Features; /// What a shipped crate or file may be under. `OR` passes if any branch does, @@ -841,9 +847,32 @@ struct Local { scripts: BTreeSet, } -/// Judge every package `roots` reach in one metadata document, and return the -/// path packages among them. -fn judge_crates(metadata: &Value, roots: &[PathBuf], report: &mut Report) -> Result { +/// A package the walk reached, as its notice names it. +struct Reached { + name: String, + version: String, + licence: Option, + /// `cargo metadata`'s `source`: `None` for a path package. + source: Option, + dir: PathBuf, + licence_file: Option, + authors: Vec, + /// The `cfg` or target of every edge into it, `None` for an unconditional + /// one and for a root. + into: BTreeSet>, +} + +/// Every package the walk reached, by name, version and origin. +type ReachedBy = BTreeMap<(String, String, String), Reached>; + +/// Judge every package `roots` reach in one metadata document, record each in +/// `reached_by`, and return the path packages among them. +fn judge_crates( + metadata: &Value, + roots: &[PathBuf], + report: &mut Report, + reached_by: &mut ReachedBy, +) -> Result { let graph = Graph::new(metadata)?; let ids = roots .iter() @@ -869,10 +898,26 @@ fn judge_crates(metadata: &Value, roots: &[PathBuf], report: &mut Report) -> Res let version = str_of(package, "version").unwrap_or("?"); let manifest = str_of(package, "manifest_path").unwrap_or("?"); let source = str_of(package, "source"); + let dir = Path::new(manifest) + .parent() + .ok_or_else(|| format!("{name}'s manifest {manifest} is in no directory"))?; + let authors = package["authors"].as_array().map(Vec::as_slice).unwrap_or(&[]); + reached_by + .entry((name.to_string(), version.to_string(), source.unwrap_or(manifest).to_string())) + .or_insert_with(|| Reached { + name: name.to_string(), + version: version.to_string(), + licence: str_of(package, "license").map(String::from), + source: source.map(String::from), + dir: dir.to_path_buf(), + licence_file: str_of(package, "license_file").map(String::from), + authors: authors.iter().filter_map(Value::as_str).map(String::from).collect(), + into: BTreeSet::new(), + }) + .into + .extend(into[dep].iter().cloned()); if source.is_none() { - if let Some(dir) = Path::new(manifest).parent() { - local.dirs.insert(dir.to_path_buf()); - } + local.dirs.insert(dir.to_path_buf()); let targets = package["targets"].as_array().map(Vec::as_slice).unwrap_or(&[]); local.scripts.extend( targets @@ -968,12 +1013,12 @@ fn judge_files(ledger: &[Row], tracked: &[String], shipping: &Shipping, report: /// One `NOTICE` section: its heading, the path the heading starts with, the /// SPDX lines it carries, and the files its terms are written in. #[derive(Debug, PartialEq)] -pub(crate) struct Section { +struct Section { heading: String, - pub(crate) path: String, + path: String, spdx: Vec, /// The path each `Licence text:` or `Terms:` line starts with. - pub(crate) texts: Vec, + texts: Vec, } const SPDX_TAG: &str = "SPDX-License-Identifier:"; @@ -982,7 +1027,7 @@ const SPDX_TAG: &str = "SPDX-License-Identifier:"; const TEXT_TAGS: [&str; 2] = ["Licence text:", "Terms:"]; /// The `-`-underlined sections of `text`, in order. -pub(crate) fn sections(text: &str) -> Vec
{ +fn sections(text: &str) -> Vec
{ let lines: Vec<&str> = text.lines().collect(); let mut out: Vec
= Vec::new(); for (i, line) in lines.iter().enumerate() { @@ -1189,8 +1234,13 @@ fn ls_files(root: &Path, pathspecs: &[String]) -> Result, String> { /// The fork's `library/`, checked out at the commit this tree pins. A checkout /// whose `rust/` was never initialised — a CI runner's — fetches that commit -/// alone. +/// alone. One whose toolchain was installed fetches it beside `rust/` instead: +/// a source tree there makes the toolchain the checkout's own to build +/// (`toolchain::owner`). fn std_library(root: &Path) -> Result { + if matches!(crate::toolchain::owner(root), crate::toolchain::Owner::Installed) { + return fetched_library(root); + } let fork = crate::sysroot::fork_checkout(root); if !fork.join("library/Cargo.toml").exists() { run( @@ -1203,6 +1253,43 @@ fn std_library(root: &Path) -> Result { Ok(fork.join("library")) } +/// Where the fork is fetched to where `rust/` may hold no source. +const FETCHED_FORK: &str = "target/licence/fork"; + +/// The fork's `library/` and its own licence files at the pinned commit, and +/// nothing else of it. +fn fetched_library(root: &Path) -> Result { + let fork = root.join(FETCHED_FORK); + let commit = crate::sysroot::pinned_fork(root); + let git = |args: &[&str]| -> Result, String> { + run(Command::new("git").args(args).current_dir(&fork), &format!("git {}", args.join(" "))) + }; + if fork.join(".git").exists() && git(&["rev-parse", "HEAD"])? == format!("{commit}\n").into_bytes() { + return Ok(fork.join("library")); + } + let url = fork_url(root)?; + if fork.exists() { + std::fs::remove_dir_all(&fork).map_err(|e| format!("remove {}: {e}", fork.display()))?; + } + std::fs::create_dir_all(&fork).map_err(|e| format!("create {}: {e}", fork.display()))?; + git(&["init", "-q"])?; + git(&["fetch", "-q", "--depth", "1", "--filter=blob:none", &url, &commit])?; + git(&["sparse-checkout", "set", "--no-cone", "/library/", "/COPYRIGHT", "/LICENSE-*", "/LICENSES/"])?; + git(&["checkout", "-q", "FETCH_HEAD"])?; + Ok(fork.join("library")) +} + +/// The URL `.gitmodules` gives the fork. +fn fork_url(root: &Path) -> Result { + let out = run( + Command::new("git") + .args(["config", "-f", ".gitmodules", "submodule.rust.url"]) + .current_dir(root), + "git config -f .gitmodules submodule.rust.url", + )?; + Ok(String::from_utf8_lossy(&out).trim().to_string()) +} + /// One metadata document, and the shipped crates judged out of it. struct Doc { metadata: Value, @@ -1211,13 +1298,22 @@ struct Doc { roots: Vec, } -/// The gate `cargo run -- --ci host` runs. -pub fn judge(root: &Path) -> Result { - // Cargo names every manifest by its canonical path. - let root = &std::fs::canonicalize(root).map_err(|e| format!("{}: {e}", root.display()))?; - let shipped = crate::build::shipped(root)?; - let mut report = Report::default(); +/// What the licence walk read out of what ships. +struct Walk { + reached: ReachedBy, + /// The fork's `library/` std was resolved from. + library: PathBuf, + shipping: Shipping, + tracked: Vec, + sections: Vec
, + /// The tracked files each section's path names. + files: BTreeMap>, +} +/// Walk every crate `shipped` names, libc and std, judging each package into +/// `report`, and read where committed files ship from. `root` is canonical: +/// cargo names every manifest by its canonical path. +fn walk(root: &Path, shipped: crate::build::Shipped, report: &mut Report) -> Result { let mut roots: Vec<(PathBuf, Features)> = shipped.crates.into_iter().collect(); roots.push(( root.join(crate::libc::CRATE), @@ -1272,8 +1368,9 @@ pub fn judge(root: &Path) -> Result { }); let mut local = Local::default(); + let mut reached = ReachedBy::new(); for doc in &docs { - let found = judge_crates(&doc.metadata, &doc.roots, &mut report)?; + let found = judge_crates(&doc.metadata, &doc.roots, report, &mut reached)?; local.dirs.extend(found.dirs); local.scripts.extend(found.scripts); } @@ -1295,7 +1392,6 @@ pub fn judge(root: &Path) -> Result { .map_err(|e| format!("read {}: {e}", file.display()))?; shipping.named.extend(embeds(&text, script, &names).into_iter().map(String::from)); } - judge_files(COMMITTED_FILES, &tracked, &shipping, &mut report); let notice = std::fs::read_to_string(root.join("NOTICE")).map_err(|e| format!("read NOTICE: {e}"))?; @@ -1305,11 +1401,214 @@ pub fn judge(root: &Path) -> Result { let named = ls_files(root, &[format!(":(glob){}", section.path)])?; files.insert(section.path.clone(), named); } - judge_notice(§ions, &files, COMMITTED_FILES, &shipping, &mut report); + Ok(Walk { reached, library, shipping, tracked, sections, files }) +} + +fn canonical(root: &Path) -> Result { + std::fs::canonicalize(root).map_err(|e| format!("{}: {e}", root.display())) +} +/// The gate `cargo run -- --ci host` runs. +pub fn judge(root: &Path) -> Result { + let root = &canonical(root)?; + let mut report = Report::default(); + let walk = walk(root, crate::build::shipped(root)?, &mut report)?; + judge_files(COMMITTED_FILES, &walk.tracked, &walk.shipping, &mut report); + judge_notice(&walk.sections, &walk.files, COMMITTED_FILES, &walk.shipping, &mut report); verdict(report, EXCEPTIONS) } +// --- The notice -------------------------------------------------------------- + +/// Where [`notices`] is on an image's ROOT. +pub const NOTICES_ON_ROOT: &str = "share/licences.txt"; + +/// What a file carrying a package's licence is called, lower-cased, at its start. +const LICENCE_FILES: &[&str] = &["license", "licence", "unlicense", "copying", "copyright", "notice"]; + +/// The one registry a package's source can be named in. +const CRATES_IO: &str = "registry+https://github.com/rust-lang/crates.io-index"; + +/// The files carrying `p`'s licence: every regular file its directory holds +/// whose name [`LICENCE_FILES`] starts, and the one its `license_file` names. +/// A package outside a registry with none carries its repository's: those of +/// the nearest enclosing directory holding any, up to the one holding `.git`. +fn licence_files(p: &Reached) -> Result, String> { + let registry = p.source.as_deref().is_some_and(|s| s.starts_with("registry+")); + let mut dir = p.dir.clone(); + loop { + let mut found = Vec::new(); + for entry in std::fs::read_dir(&dir).map_err(|e| format!("{}: {e}", dir.display()))? { + let path = entry.map_err(|e| format!("{}: {e}", dir.display()))?.path(); + let name = file_name(&path.to_string_lossy()).to_lowercase(); + // The fork's `license-metadata.json` is REUSE's data about its texts, and no text. + if LICENCE_FILES.iter().any(|n| name.starts_with(n)) && !name.ends_with(".json") && path.is_file() { + found.push(path); + } + } + if let (true, Some(file)) = (dir == p.dir, &p.licence_file) { + let file = dir.join(file); + if !file.is_file() { + return Err(format!("{} {} names {} as its licence file, and it is none", p.name, p.version, file.display())); + } + found.push(file); + } + found.sort(); + found.dedup(); + if !found.is_empty() { + return Ok(found); + } + if registry || dir.join(".git").exists() || !dir.pop() { + return Err(format!("{} {} ({}) carries no licence file", p.name, p.version, p.dir.display())); + } + } +} + +/// The standard texts of the licences `p` declares, from the fork's +/// `LICENSES/`, which holds one `.txt` per licence: each of an `AND`, +/// an exception with its licence, and of an `OR` the first branch held whole. +/// `None` where no branch is. +fn standard_texts(p: &Reached, fork: &Path) -> Option> { + fn held(expr: &Expr, dir: &Path) -> Option> { + let text = |id: &str| Some(dir.join(format!("{id}.txt"))).filter(|f| f.is_file()); + match expr { + Expr::Id(id) => Some(vec![text(id)?]), + Expr::With(id, exception) => Some(vec![text(id)?, text(exception)?]), + Expr::And(parts) => Some(parts.iter().map(|p| held(p, dir)).collect::>>()?.concat()), + Expr::Or(parts) => parts.iter().find_map(|p| held(p, dir)), + } + } + held(&parse(p.licence.as_deref()?).ok()?, &fork.join("LICENSES")) +} + +/// Every licence text a notice carries, each once, by its first carrier. +#[derive(Default)] +struct Texts { + ids: BTreeMap, + listed: Vec<(String, String)>, +} + +impl Texts { + fn id(&mut self, file: &Path, carrier: &str) -> Result { + let bytes = std::fs::read(file).map_err(|e| format!("{}: {e}", file.display()))?; + let text = String::from_utf8_lossy(&bytes).into_owned(); + let next = self.listed.len() + 1; + let id = *self.ids.entry(text.clone()).or_insert(next); + if id == next { + self.listed.push((format!("{carrier}: {}", file_name(&file.to_string_lossy())), text)); + } + Ok(id) + } + + fn cite(&mut self, files: &[PathBuf], carrier: &str) -> Result { + let ids = files.iter().map(|f| self.id(f, carrier).map(|id| format!("[{id}]"))); + Ok(ids.collect::, _>>()?.join(" ")) + } +} + +/// The licence notice of an image built from `shipped`: every package the +/// walk reaches with its licence, where its source is and the texts it +/// carries; every `NOTICE` section over a file it ships, with its terms and +/// texts, less what [`pending_owner`] names; and each text once. Refused while +/// any package carries no licence text. +pub fn notices(root: &Path, shipped: crate::build::Shipped) -> Result { + let root = &canonical(root)?; + let walk = walk(root, shipped, &mut Report::default())?; + let fork = walk.library.parent().ok_or("std's library/ is in no directory")?; + let (url, commit) = (fork_url(root)?, crate::sysroot::pinned_fork(root)); + let mut texts = Texts::default(); + let mut refused = Vec::new(); + let mut out = String::from( + "Licence notices\n===============\n\n\ + Every package in the dependency graphs of this image's kernel, loader and programs,\n\ + some of them compiled only for other platforms, with its licence, where its source\n\ + is, and its licence texts; every third-party file on the image, with its terms; and\n\ + each of those texts once, at the end.\n\n\ + Packages\n--------\n", + ); + let guest: Vec<&str> = Arch::ALL.iter().flat_map(|a| [a.userland(), a.kernel(), a.loader()]).collect(); + for p in walk.reached.values() { + // A package every edge into which names another target by its triple + // is linked on none of ours. + let foreign = p.into.iter().all(|edge| { + edge.as_deref() + .is_some_and(|t| t.split(" | ").all(|t| !t.starts_with("cfg(") && !guest.contains(&t))) + }); + if foreign { + continue; + } + let (files, standard) = match licence_files(p) { + Ok(files) => (files, ""), + Err(why) => match standard_texts(p, fork) { + Some(files) => (files, ", the licences' standard texts, since the package carries none"), + None => { + refused.push(format!("{why}, and the fork's LICENSES/ holds no branch of its licence")); + continue; + } + }, + }; + let source = match (&p.source, p.dir.strip_prefix(fork), p.dir.strip_prefix(root)) { + (Some(s), ..) if s == CRATES_IO => { + format!("https://static.crates.io/crates/{0}/{0}-{1}.crate", p.name, p.version) + } + (Some(s), ..) => match s.strip_prefix("git+") { + Some(git) => git.to_string(), + None => return Err(format!("{} {} is from {s}, which no notice names a download in", p.name, p.version)), + }, + (None, Ok(at), _) => format!("{url} at {commit}, {}", at.display()), + (None, _, Ok(at)) => format!("the ToyOS source this image was built from, {}", at.display()), + (None, ..) => return Err(format!("{} {} is a path package outside {}", p.name, p.version, root.display())), + }; + let carrier = match standard { + "" => format!("{} {}", p.name, p.version), + _ => "the Rust fork's LICENSES".to_string(), + }; + let cited = texts.cite(&files, &carrier)?; + let licence = p.licence.as_deref().unwrap_or("none declared"); + let authors = match p.authors.as_slice() { + [] => String::new(), + all => format!("\n authors: {}", all.join(", ")), + }; + out.push_str(&format!( + "\n{} {}\n licence: {licence}\n source: {source}{authors}\n texts: {cited}{standard}\n", + p.name, p.version + )); + } + if !refused.is_empty() { + return Err(format!("{} shipped package(s) carry no licence text:\n {}", refused.len(), refused.join("\n "))); + } + + out.push_str("\nFiles\n-----\n"); + let withheld: Vec<&str> = pending_owner() + .filter_map(|s| match s { + Subject::Notice(path) | Subject::File(path) => Some(path), + Subject::Crate(_) => None, + }) + .collect(); + for section in &walk.sections { + let named = walk.files.get(§ion.path).map(Vec::as_slice).unwrap_or(&[]); + if withheld.contains(§ion.path.as_str()) || !named.iter().any(|f| walk.shipping.ships(f)) { + continue; + } + let files: Vec = section.texts.iter().map(|t| root.join(t)).collect(); + let cited = match texts.cite(&files, §ion.path)? { + cited if cited.is_empty() => cited, + cited => format!("\n texts: {cited}"), + }; + out.push_str(&format!( + "\n{}\n licence: {}{cited}\n", + section.heading, + section.spdx.join(" AND ") + )); + } + + out.push_str("\nTexts\n-----\n"); + for (at, (carrier, text)) in texts.listed.iter().enumerate() { + out.push_str(&format!("\n[{}] {carrier}\n\n{}\n", at + 1, text.trim_end())); + } + Ok(out) +} + #[cfg(test)] mod tests { use super::*; @@ -1449,7 +1748,7 @@ mod tests { fn crates(doc: &Value) -> Report { let mut report = Report::default(); - judge_crates(doc, &[PathBuf::from("/t/app")], &mut report).expect("judged"); + judge_crates(doc, &[PathBuf::from("/t/app")], &mut report, &mut ReachedBy::new()).expect("judged"); report } @@ -1688,7 +1987,7 @@ ub_checks"#; fn a_root_the_metadata_does_not_hold_is_refused() { let d = doc(&[("other", Some("MIT"), None, None)], &[]); let mut report = Report::default(); - let why = judge_crates(&d, &[PathBuf::from("/t/app")], &mut report).unwrap_err(); + let why = judge_crates(&d, &[PathBuf::from("/t/app")], &mut report, &mut ReachedBy::new()).unwrap_err(); assert!(why.contains("/t/app/Cargo.toml"), "{why}"); } @@ -1940,4 +2239,80 @@ prose. } } } + + fn reached(dir: &Path, source: Option<&str>, licence_file: Option<&str>) -> Reached { + Reached { + name: "x".into(), + version: "1.0.0".into(), + licence: Some("MIT".into()), + source: source.map(String::from), + dir: dir.to_path_buf(), + licence_file: licence_file.map(String::from), + authors: Vec::new(), + into: BTreeSet::new(), + } + } + + /// A package's own licence files are read and nothing else of its + /// directory; a registry package with none is refused by name, and a path + /// package with none carries its repository's, never past `.git`. + #[test] + fn a_package_carries_its_own_licence_files_or_its_repositorys() { + let tmp = toyos_tmpdir::TempDir::new("licence-files"); + let repo = tmp.join("repo"); + let package = repo.join("crates/x"); + std::fs::create_dir_all(package.join("src")).unwrap(); + std::fs::write(package.join("src/lib.rs"), "").unwrap(); + std::fs::write(package.join("README.md"), "").unwrap(); + let why = licence_files(&reached(&package, CRATES_IO, None)).unwrap_err(); + assert!(why.contains("x 1.0.0") && why.contains("carries no licence file"), "{why}"); + + std::fs::create_dir(repo.join(".git")).unwrap(); + std::fs::write(repo.join("LICENSE-MIT"), "mit").unwrap(); + std::fs::write(tmp.join("LICENSE"), "outside").unwrap(); + assert_eq!(licence_files(&reached(&package, None, None)).unwrap(), [repo.join("LICENSE-MIT")]); + assert!(licence_files(&reached(&package, CRATES_IO, None)).is_err(), "a registry package walked up"); + + for name in ["COPYING", "Licence.txt", "NOTICE", "UNLICENSE"] { + std::fs::write(package.join(name), name).unwrap(); + } + std::fs::write(package.join("terms.txt"), "").unwrap(); + let files = licence_files(&reached(&package, CRATES_IO, Some("terms.txt"))).unwrap(); + let names: Vec<&str> = files.iter().map(|f| file_name(f.to_str().unwrap())).collect(); + assert_eq!(names, ["COPYING", "Licence.txt", "NOTICE", "UNLICENSE", "terms.txt"]); + let why = licence_files(&reached(&package, CRATES_IO, Some("gone.txt"))).unwrap_err(); + assert!(why.contains("gone.txt"), "{why}"); + } + + /// The release's notice: every package its walk reaches carries a licence + /// text, the loader's MPL crates name where their source is, std carries + /// the fork's texts, and every third-party file the release ships is there + /// while nothing pending the owner is. + #[test] + fn the_release_notice_carries_every_package_and_file_it_ships() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let parts = crate::build::Boot::release(root).parts(root).unwrap(); + let text = notices(root, parts).unwrap_or_else(|why| panic!("{why}")); + let block = |head: &str| -> &str { + let at = text.find(&format!("\n{head}")).unwrap_or_else(|| panic!("no {head:?} in the notice")); + text[at + 1..].split("\n\n").next().unwrap() + }; + let uefi = block("uefi "); + assert!(uefi.contains("licence: MPL-2.0"), "{uefi}"); + assert!(uefi.contains("source: https://static.crates.io/crates/uefi/uefi-"), "{uefi}"); + let std = block("std "); + assert!(std.contains("source: https://github.com/ToyOSOrg/rust.git at "), "{std}"); + assert!(text.contains(": COPYRIGHT\n\nShort version for non-lawyers:"), "std carries no fork COPYRIGHT"); + for shipped in ["assets/JetBrainsMono-Regular.ttf", "assets/icons/*.svg", "assets/fonts/OpenSans-*.ttf"] { + assert!(block(&format!("{shipped} — ")).contains("texts: ["), "{shipped}"); + } + let listed = &text[..text.find("\nTexts\n-----\n").expect("no texts")]; + for withheld in pending_owner() { + let head = match withheld { + Subject::Crate(name) => format!("\n{name} "), + Subject::Notice(path) | Subject::File(path) => format!("\n{path} "), + }; + assert!(!listed.contains(&head), "the release's notice lists {withheld:?}"); + } + } } diff --git a/src/main.rs b/src/main.rs index e5e3c1a7e1..e0d675c47a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -149,27 +149,6 @@ fn main() { } return; } - // The release job publishes what this builds, so nothing on the line may - // make it another image. - let release = asked(&flags::RELEASE_BOOT); - if release { - for other in [ - &flags::DIAG_BOOT, - &flags::CONSOLE_BOOT, - &flags::BOOT_CONFIG, - &flags::KERNEL_PARAM, - &flags::KERNEL_FEATURE, - &flags::DEBUG, - &flags::OWNER_KEY, - &flags::UPDATE_IMAGE, - &flags::ARCH, - ] { - if asked(other) { - eprintln!("Error: --release-boot builds the published image and takes no {}", other.name); - std::process::exit(2); - } - } - } // Before anything is built, so a missing key is refused before any lock // and no image this run writes is signed by two keys. let update_image = CARGO_RUN.value(&args, &flags::UPDATE_IMAGE).map(PathBuf::from); @@ -232,7 +211,6 @@ fn main() { } (None, true, _) => toyos_build::build::Boot::diag(&root), (None, _, true) => toyos_build::build::Boot::console(&root), - _ if release => toyos_build::build::Boot::release(&root), _ => toyos_build::build::Boot::shipped(&root), }; assert!( diff --git a/src/redlist.rs b/src/redlist.rs index f941c3eacc..b73204e8de 100644 --- a/src/redlist.rs +++ b/src/redlist.rs @@ -51,6 +51,10 @@ pub const DISABLED: &[Disabled] = &[ test: "quiesce_wakes_on_the_last_exit", issue: "issues/build/quiesce-wakes-on-the-last-exit-lost-its-serial-ready-beside-other-guests.md", }, + Disabled { + test: "release_command_boots", + issue: "issues/kernel/the-kernel-dies-at-boot-on-the-edk2-firmware-qemu-ships.md", + }, Disabled { test: "sched_check_build", issue: "issues/build/the-pass-cost-gates-ci-sample-is-eight-days-stale-twice.md", diff --git a/src/sourcegate.rs b/src/sourcegate.rs index a9a4093041..2a9731080f 100644 --- a/src/sourcegate.rs +++ b/src/sourcegate.rs @@ -559,7 +559,7 @@ const HOST_SPAWNS: &[Spawn] = &[ ("src/qemu.rs", 1), ("src/ci.rs", 2), ("tests/common/qemu.rs", 1), - ("tests/common/release.rs", 1), + ("src/imagerelease.rs", 1), ], why: "QEMU, the other half of the bar: `Arch::qemu` names `qemu-system-x86_64` and \ `qemu-system-aarch64`, and `check_prerequisites` requires the one being booted", @@ -568,8 +568,8 @@ const HOST_SPAWNS: &[Spawn] = &[ arg: "\"gh\"", sites: &[], why: "GitHub's CLI, outside the bar: CI's releases, protection and nightly-red jobs \ - (src/ci.rs, src/release.rs, src/imagerelease.rs) ask GitHub with it. Nothing that builds or boots \ - reaches it", + (src/ci.rs, src/release.rs, src/imagerelease.rs) ask GitHub with it. No build and no \ + boot calls it", }, Spawn { arg: "\"curl\"", @@ -725,6 +725,11 @@ const CI_PACKAGES: &[Package] = &[ why: "outside the bar, and declared by nothing else: it fetches rustup-init.sh, and \ src/release.rs and src/ci.rs ask GitHub and the crates.io index with it", }, + Package { + name: "gh", + why: "GitHub's CLI, which the image release publishes with (src/imagerelease.rs), in the \ + one container job that publishes", + }, Package { name: "git", why: "the version control this repository is, and `REQUIRED` in src/main.rs", @@ -732,8 +737,8 @@ const CI_PACKAGES: &[Package] = &[ Package { name: "ovmf", why: "the edk2 firmware Debian's QEMU boots a UEFI guest with: the image release notes' \ - Linux command line names it, and the guest suite boots that line \ - (src/imagerelease.rs)", + Linux command line names it, and the release job and `release_command_boots` \ + boot that line (src/imagerelease.rs)", }, Package { name: "python3", diff --git a/src/sysroot.rs b/src/sysroot.rs index 5fbc008d8a..670112b517 100644 --- a/src/sysroot.rs +++ b/src/sysroot.rs @@ -201,7 +201,7 @@ pub fn key(root: &Path, compiler: &Compiler, fork: &Path) -> String { /// The commit this checkout's tree pins the std fork at: the index's, so a /// staged gitlink counts as the tree's. -fn pinned_fork(root: &Path) -> String { +pub(crate) fn pinned_fork(root: &Path) -> String { let entry = git_out(root, &["ls-files", "-s", "--", "rust"]); let mut words = entry.split_whitespace(); match (words.next(), words.next()) { diff --git a/tests/common/mod.rs b/tests/common/mod.rs index b5f32a2135..3304a1091d 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -43,7 +43,7 @@ pub mod metal; pub mod origin; #[allow(dead_code)] pub mod partclaim; -/// The image release's command lines, and the disks its image was not given. +/// The image release's command line for this host. pub mod release; #[allow(dead_code)] pub mod passcost; diff --git a/tests/common/release.rs b/tests/common/release.rs index 8449ab3fb1..73dfdb6da6 100644 --- a/tests/common/release.rs +++ b/tests/common/release.rs @@ -1,298 +1,22 @@ -//! The image release's command lines booted as its notes print them -//! (`toyos_build::imagerelease::Host::command`), and what the release image -//! does to a disk it was not given. -//! -//! Each guest boots a copy of the image `cargo run -- --release-boot -//! --build-only` writes, built once per run by that same sequence, and is -//! judged by its console alone: these boots are not the harness's profiles. +//! The image release's command line for this host, booted as its notes print +//! it (`toyos_build::imagerelease::boots`) over a copy of the release image. -use std::collections::BTreeMap; -use std::io::{BufRead, BufReader, Seek, SeekFrom, Write}; -use std::path::{Path, PathBuf}; -use std::process::{Child, Command, Stdio}; -use std::sync::mpsc::{self, Receiver, RecvTimeoutError}; -use std::sync::OnceLock; -use std::time::{Duration, Instant}; - -use toyos_build::arch::Arch; use toyos_build::build::{self, Boot}; -use toyos_build::fingerprint::{first_difference, whole_device}; -use toyos_build::imagerelease::Host; - -/// One guest's ceiling from power-on to a painting desktop, before -/// `super::qemu::budget` scales it: a liveness guard, never a verdict. -const DESKTOP: Duration = Duration::from_secs(120); - -/// The compositor's report, one every two seconds of a painting desktop. -const FRAMES: &str = "compositor: frames="; - -/// The release image, built once per run. -fn image() -> &'static Path { - static BUILT: OnceLock = OnceLock::new(); - BUILT.get_or_init(|| { - let root = super::compile::repo_root(); - let boot = Boot::release(&root); - let plan = build::plan_for(&root, &boot, false, &[]); - build::build(&root, boot, false, &plan) - }) -} - -/// A copy of the release image under this lane, for one guest to write to. -fn stick(name: &str) -> Result { - let to = super::lane::dir().join(name); - std::fs::copy(image(), &to).map_err(|e| format!("copy the release image to {}: {e}", to.display()))?; - Ok(to) -} - -/// A QEMU started from a release command line, and its console so far. -struct Guest { - child: Child, - lines: Receiver, - log: String, - stderr: PathBuf, -} - -impl Guest { - /// `argv` with `extra` after it and no window: the notes' line, headless. - fn start(argv: &[String], extra: &[String], stderr: PathBuf) -> Result { - let err = std::fs::File::create(&stderr).map_err(|e| format!("{}: {e}", stderr.display()))?; - let arch = Arch::X86_64; - if argv[0] != arch.qemu() { - return Err(format!("a release command line starts {:?}, not {}", argv[0], arch.qemu())); - } - let mut child = Command::new(arch.qemu()) - .args(&argv[1..]) - .args(extra) - .args(["-display", "none"]) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(err) - .spawn() - .map_err(|e| format!("{}: {e}", arch.qemu()))?; - let out = child.stdout.take().expect("piped"); - let (send, lines) = mpsc::channel(); - std::thread::spawn(move || { - for line in BufReader::new(out).split(b'\n') { - let Ok(line) = line else { return }; - if send.send(String::from_utf8_lossy(&line).into_owned()).is_err() { - return; - } - } - }); - Ok(Guest { child, lines, log: String::new(), stderr }) - } - - /// Read the console until `done` holds of it, refusing a panic the moment - /// one is printed. - fn until(&mut self, what: &str, done: impl Fn(&str) -> bool) -> Result<(), String> { - let budget = super::qemu::budget(DESKTOP); - let deadline = Instant::now() + budget; - loop { - if let Some(line) = self.log.lines().find(|l| l.contains("PANIC") || l.contains("panicked at")) { - return Err(format!("the guest panicked before {what}: {line}\n{}", self.log)); - } - if done(&self.log) { - return Ok(()); - } - let left = deadline.saturating_duration_since(Instant::now()); - match self.lines.recv_timeout(left) { - Ok(line) => { - self.log.push_str(&line); - self.log.push('\n'); - } - Err(RecvTimeoutError::Timeout) => { - return Err(format!("no {what} within {budget:?}:\n{}", self.log)); - } - Err(RecvTimeoutError::Disconnected) => { - let stderr = std::fs::read_to_string(&self.stderr).unwrap_or_default(); - return Err(format!( - "QEMU closed its console before {what}:\n{}\nQEMU's stderr:\n{stderr}", - self.log - )); - } - } - } - } - - /// The desktop is up: every program the shipped config starts said it - /// started, the three that announce themselves did, and the compositor - /// has painted. - fn desktop(&mut self) -> Result<(), String> { - let mut said: Vec = build::boot_start(&super::compile::repo_root().join("system.toml")) - .iter() - .map(|program| format!("init: started {program}")) - .collect(); - said.extend( - ["compositor: ready", "netd: ready", "logd: this boot's kernel log is", FRAMES].map(String::from), - ); - self.until("the desktop", |log| said.iter().all(|line| log.contains(line.as_str()))) - } -} - -impl Drop for Guest { - fn drop(&mut self) { - let _ = self.child.kill(); - let _ = self.child.wait(); - } -} +use toyos_build::imagerelease::{self, Host}; -/// The notes' command line for this host boots the release image to a -/// painting desktop. +/// The notes' line for this host boots the release image to a painting +/// desktop and leaves both firmware files as they were. pub fn release_command_boots() -> Result<(), String> { let host = Host::this()?; - let stick = stick("release-command.img")?; - let argv = host.command(&stick.display().to_string()); - let mut guest = Guest::start(&argv, &[], super::lane::dir().join("release-command.stderr"))?; - guest.desktop()?; - eprintln!(" [release] {host:?}'s command line reached a painting desktop"); - Ok(()) -} - -/// Beside the release image, a disk laid out as another operating system's -/// and a stick with no partition table come back byte for byte after a boot -/// to the desktop, while the kernel says it read both and opened a volume on -/// neither. -pub fn release_writes_no_other_disk() -> Result<(), String> { - const OTHER_BYTES: u64 = 192 << 20; - const SPARE_BYTES: u64 = 64 << 20; - let host = Host::this()?; + let root = super::compile::repo_root(); + let boot = Boot::release(&root); + let plan = build::plan_for(&root, &boot, false, &[]); + let image = build::build(&root, boot, false, &plan); let dir = super::lane::dir(); - let stick = stick("release-disks.img")?; - let other = dir.join("another-os.img"); - let parts = another_os_disk(&other, OTHER_BYTES)?; - let spare = dir.join("spare-stick.img"); - pattern_file(&spare, SPARE_BYTES, 0x5A)?; - let before = [whole_device(&stick), whole_device(&other), whole_device(&spare)]; - - let extra: Vec = [ - "-drive".to_string(), - format!("if=none,id=other,format=raw,file={}", other.display()), - "-device".to_string(), - "nvme,serial=other,drive=other".to_string(), - "-drive".to_string(), - format!("if=none,id=spare,format=raw,file={}", spare.display()), - "-device".to_string(), - "usb-storage,bus=xhci.0,drive=spare".to_string(), - ] - .to_vec(); - let argv = host.command(&stick.display().to_string()); - let mut guest = Guest::start(&argv, &extra, dir.join("release-disks.stderr"))?; - guest.desktop()?; - - // What the kernel opened, before anything about the bytes: a page cache is - // what every write to a DATA volume goes through. - if let Some(line) = guest.log.lines().find(|l| l.contains("page cache: device")) { - return Err(format!("the release opened a DATA volume on a disk it was not given: {line}\n{}", guest.log)); - } - // The case was reached: the kernel read both tables and took neither. - let read_other = format!("gpt: device 1 has {parts} partitions and none of them is ours"); - for said in [ - read_other.as_str(), - "has no partition table we can use", - "storage: this machine carries 0 TOYOS-DATA partitions", - ] { - if !guest.log.contains(said) { - return Err(format!("the kernel never said {said:?}\n{}", guest.log)); - } - } - // The machine runs on past the desktop, so what a boot writes late is in. - let settled = guest.log.matches(FRAMES).count() + 3; - guest.until("three more frame reports", |log| log.matches(FRAMES).count() >= settled)?; - drop(guest); - - let after = [whole_device(&stick), whole_device(&other), whole_device(&spare)]; - if first_difference(&before[0], &after[0]).is_none() { - return Err( - "the stick came back unchanged, so no write of this boot reached a backing file and \ - the comparison below proves nothing" - .into(), - ); - } - for (name, (b, a)) in ["the other operating system's disk", "the spare stick"] - .iter() - .zip(before[1..].iter().zip(&after[1..])) - { - if let Some(diff) = first_difference(b, a) { - return Err(format!("the release wrote to {name}: {diff}")); - } - } - eprintln!( - " [release] {host:?}: beside the stick, an NVMe disk of {parts} foreign partitions and a \ - stick with no table came back byte for byte" - ); - Ok(()) -} - -/// `len` bytes of `fill`, so a write of anything, zeros included, moves the -/// fingerprint. -fn pattern_file(path: &Path, len: u64, fill: u8) -> Result<(), String> { - let mut file = std::fs::File::create(path).map_err(|e| format!("{}: {e}", path.display()))?; - let chunk = vec![fill; 1 << 20]; - for _ in 0..len / chunk.len() as u64 { - file.write_all(&chunk).map_err(|e| format!("{}: {e}", path.display()))?; - } + let stick = dir.join("release-command.img"); + std::fs::copy(&image, &stick).map_err(|e| format!("copy the release image to {}: {e}", stick.display()))?; + let ceiling = super::qemu::budget(imagerelease::DESKTOP); + imagerelease::boots(&root, host, &stick, ceiling, &dir.join("release-command.stderr"))?; + eprintln!(" [release] {host:?}'s command line reached a painting desktop"); Ok(()) } - -/// A disk laid out as a PC's with another operating system on it: an EFI -/// system partition, Microsoft's reserved and basic-data partitions and a -/// Linux filesystem, each carrying its format's signature, over a filled -/// disk. Answers how many partitions it carries. -fn another_os_disk(path: &Path, len: u64) -> Result { - use gpt::partition_types::{BASIC, EFI, LINUX_FS, MICROSOFT_RESERVED}; - const MIB: u64 = 1 << 20; - pattern_file(path, len, 0xA5)?; - - let mut file = std::fs::OpenOptions::new() - .read(true) - .write(true) - .open(path) - .map_err(|e| format!("{}: {e}", path.display()))?; - let mbr = gpt::mbr::ProtectiveMBR::with_lb_size(u32::try_from(len / 512 - 1).unwrap_or(u32::MAX)); - mbr.overwrite_lba0(&mut file).map_err(|e| format!("protective MBR: {e}"))?; - let mut disk = gpt::GptConfig::default() - .initialized(false) - .writable(true) - .logical_block_size(gpt::disk::LogicalBlockSize::Lb512) - .create_from_device(Box::new(file), None) - .map_err(|e| format!("a table on {}: {e}", path.display()))?; - disk.update_partitions(BTreeMap::::new()) - .map_err(|e| format!("an empty table: {e}"))?; - let layout = [ - ("EFI system partition", 32 * MIB, EFI), - ("Microsoft reserved partition", 16 * MIB, MICROSOFT_RESERVED), - ("Basic data partition", 64 * MIB, BASIC), - ("Linux filesystem", 64 * MIB, LINUX_FS), - ]; - let mut starts = Vec::new(); - for (name, bytes, kind) in layout { - let id = disk - .add_partition(name, bytes, kind, 0, Some(2048)) - .map_err(|e| format!("add {name}: {e}"))?; - let placed = &disk.partitions()[&id]; - starts.push(placed.bytes_start(gpt::disk::LogicalBlockSize::Lb512).map_err(|e| e.to_string())?); - } - let mut device = disk.write().map_err(|e| format!("write the table: {e}"))?; - - // FAT32's and NTFS's boot sectors, and ext4's superblock magic. - let mut fat = [0u8; 512]; - fat[..3].copy_from_slice(&[0xEB, 0x58, 0x90]); - fat[3..11].copy_from_slice(b"MSDOS5.0"); - fat[82..90].copy_from_slice(b"FAT32 "); - fat[510..].copy_from_slice(&[0x55, 0xAA]); - let mut ntfs = [0u8; 512]; - ntfs[..3].copy_from_slice(&[0xEB, 0x52, 0x90]); - ntfs[3..11].copy_from_slice(b"NTFS "); - ntfs[510..].copy_from_slice(&[0x55, 0xAA]); - for (at, bytes) in [(starts[0], &fat[..]), (starts[2], &ntfs[..]), (starts[3] + 1024 + 56, &[0x53, 0xEF][..])] { - device.seek(SeekFrom::Start(at)).map_err(|e| e.to_string())?; - device.write_all(bytes).map_err(|e| e.to_string())?; - } - device.flush().map_err(|e| e.to_string())?; - - // The premise, by the parser the kernel selects DATA with. - if toyos_build::image::data_partition_of(path).is_ok() { - return Err(format!("{} carries a TOYOS-DATA partition, so it is no other system's disk", path.display())); - } - Ok(starts.len()) -} diff --git a/tests/common/storage.rs b/tests/common/storage.rs index fe93ca226d..94a67e2091 100644 --- a/tests/common/storage.rs +++ b/tests/common/storage.rs @@ -16,8 +16,12 @@ use toyos_build::fingerprint::{first_difference, whole_device}; use super::qemu::{self, BootOptions, QemuInstance}; -/// Boot the guest against a disk that belongs to somebody else, and prove it -/// comes back untouched. +/// Boot the guest against three disks that belong to somebody else, and prove +/// each comes back untouched: an NVMe disk whose TOYOS-DATA partition holds +/// another system's volume, a USB disk whose table names only other systems' +/// partitions, and a USB stick with no table. The two USB disks ride USB +/// because the kernel drives one NVMe controller, and that one carries the +/// first. /// /// Lives here so the registration hunk in `toyos.rs` stays one line: every /// agent edits that file. @@ -27,12 +31,18 @@ pub fn foreign_disk_untouched( rust_bins: &[(String, Vec)], ) -> Result<(), String> { const BYTES: u64 = 128 * 1024 * 1024; - // The same directory `boot_with_options` uses, named here because this - // image has to exist before the boot that must not touch it. + const USB_BYTES: u64 = 64 * 1024 * 1024; + // The same directory `boot_with_options` uses, named here because these + // images have to exist before the boot that must not touch them. let dir = super::lane::dir(); let image = dir.join("foreign-disk.img"); let (data_at, _) = foreign_disk_image(&image, BYTES); - let before = whole_device(&image); + let other = dir.join("other-systems-disk.img"); + let parts = other_systems_disk(&other, USB_BYTES)?; + let bare = dir.join("bare-stick.img"); + filled(&bare, USB_BYTES, 0x5A)?; + let disks = [image.clone(), other.clone(), bare.clone()]; + let before: Vec> = disks.iter().map(|disk| whole_device(disk)).collect(); // The premise, checked before the boot rather than assumed: if this volume // somehow already parsed as a ToyOS volume, the kernel would mount it and @@ -46,8 +56,9 @@ pub fn foreign_disk_untouched( c_bins, rust_bins, BootOptions { - profile: qemu::Profile::Metal, + profile: qemu::Profile::UsbDiskCrowd, nvme_image: Some(image.clone()), + usb_images: vec![other.clone(), bare.clone()], ..Default::default() }, ); @@ -79,27 +90,82 @@ pub fn foreign_disk_untouched( return Err(format!("the kernel decided to format a disk it was not given\n{log}")); } - // Shut down rather than kill: `PageCache::sync` at shutdown is the only - // thing that moves a format from the cache to the device, so a killed QEMU - // fingerprints an image a formatting kernel would also have left untouched. + // Shut down rather than kill, and wait for QEMU to exit: `PageCache::sync` + // at shutdown is the only thing that moves a format from the cache to the + // device, so a killed QEMU fingerprints an image a formatting kernel would + // also have left untouched. writeln!(qemu.stdin_mut(), "run shutdown").expect("write to QEMU stdin"); qemu.flush_stdin(); - let tail = qemu.drain_serial(Duration::from_secs(20)); + let tail = qemu.await_exit(Duration::from_secs(20))?; for bad in ["PANIC:", "panicked at"] { if tail.contains(bad) { return Err(format!("{bad:?} during shutdown\n{tail}")); } } drop(qemu); + // The USB disks were read, so the comparison below is about disks the + // kernel saw. + let said = format!("{log}{tail}"); + for read in [format!("has {parts} partitions and none of them is ours"), "has no partition table we can use".into()] { + if !said.contains(&read) { + return Err(format!("the kernel never said {read:?}, so it never read that disk\n{said}")); + } + } - let after = whole_device(&image); - if let Some(diff) = first_difference(&before, &after) { - return Err(format!("the kernel wrote to a disk it was not given: {diff}")); + for (disk, before) in disks.iter().zip(&before) { + if let Some(diff) = first_difference(before, &whole_device(disk)) { + return Err(format!("the kernel wrote to {}, a disk it was not given: {diff}", disk.display())); + } + } + for disk in &disks { + let _ = std::fs::remove_file(disk); } - let _ = std::fs::remove_file(&image); Ok(()) } +/// `len` bytes of `fill`, so a write of anything, zeros included, moves the +/// fingerprint. +fn filled(path: &Path, len: u64, fill: u8) -> Result<(), String> { + let len = usize::try_from(len).map_err(|e| format!("{len} bytes: {e}"))?; + std::fs::write(path, vec![fill; len]).map_err(|e| format!("{}: {e}", path.display())) +} + +/// A filled disk whose table names an EFI system partition, a Microsoft basic +/// data partition and a Linux filesystem, and none of ToyOS's types. Answers +/// how many partitions it carries. +fn other_systems_disk(path: &Path, len: u64) -> Result { + use gpt::partition_types::{BASIC, EFI, LINUX_FS}; + filled(path, len, 0xA5)?; + let mut file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(path) + .map_err(|e| format!("{}: {e}", path.display()))?; + let sectors = u32::try_from(len / 512 - 1).map_err(|e| format!("{len} bytes: {e}"))?; + gpt::mbr::ProtectiveMBR::with_lb_size(sectors) + .overwrite_lba0(&mut file) + .map_err(|e| format!("protective MBR: {e}"))?; + let mut disk = gpt::GptConfig::default() + .initialized(false) + .writable(true) + .logical_block_size(gpt::disk::LogicalBlockSize::Lb512) + .create_from_device(Box::new(file), None) + .map_err(|e| format!("a table on {}: {e}", path.display()))?; + disk.update_partitions(std::collections::BTreeMap::new()) + .map_err(|e| format!("an empty table: {e}"))?; + let layout = [("EFI system partition", EFI), ("Basic data partition", BASIC), ("Linux filesystem", LINUX_FS)]; + let parts = layout.len(); + for (name, kind) in layout { + disk.add_partition(name, 16 << 20, kind, 0, Some(2048)).map_err(|e| format!("add {name}: {e}"))?; + } + disk.write().map_err(|e| format!("write the table: {e}"))?; + // The premise, by the parser the kernel selects DATA with. + if toyos_build::image::data_partition_of(path).is_ok() { + return Err(format!("{} carries a TOYOS-DATA partition", path.display())); + } + Ok(parts) +} + /// The volume is genuine and the disk is not: block 0 here carries the magic, /// the version and the CRC this crate wrote, and every other stimulus in this /// file is refused before any of that is read. What it does not carry is this diff --git a/tests/toyos.rs b/tests/toyos.rs index d8844b8fec..57ca64c9f2 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -983,11 +983,9 @@ const MACHINE_TESTS: &[(&str, Sched, Tier)] = &[ ("volume_from_another_disk", Sched::Parallel, Tier::Fast), ("broken_data_volume_is_absent", Sched::Parallel, Tier::Fast), ("data_candidate_with_bad_geometry_is_absent", Sched::Parallel, Tier::Fast), - // The image release's command line booted as its notes print it, and beside it - // two disks it was not given: console lines and image bytes, and the - // ceiling is a liveness guard. + // The image release's command line booted as its notes print it: console + // lines and firmware bytes, and the ceiling is a liveness guard. ("release_command_boots", Sched::Parallel, Tier::Fast), - ("release_writes_no_other_disk", Sched::Parallel, Tier::Fast), // Four kernel lines and a file read off the image once the guest is gone; no clock in any of them. ("internal_disk_boot", Sched::Parallel, Tier::Fast), // One boot each, kernel lines and image bytes for verdicts, no clock in either. @@ -11475,7 +11473,6 @@ fn run_machine_test( // stays one line. "foreign_disk_untouched" => storage::foreign_disk_untouched(test_config, c_bins, rust_bins), "release_command_boots" => release::release_command_boots(), - "release_writes_no_other_disk" => release::release_writes_no_other_disk(), "internal_disk_boot" => storage::internal_disk_boot(test_config, c_bins, rust_bins), "partition_claim" => partclaim::partition_claim(test_config, c_bins, rust_bins), "partition_claim_gives_up" => { diff --git a/toyos-update/src/floor.rs b/toyos-update/src/floor.rs index 379b3fdc36..bfde62e326 100644 --- a/toyos-update/src/floor.rs +++ b/toyos-update/src/floor.rs @@ -76,7 +76,8 @@ impl Scope { } } - const fn tag(self) -> u8 { + /// The letter a floor's name carries after [`PREFIX`] and a dash. + pub const fn tag(self) -> u8 { match self { Self::Machine => b'K', Self::Image => b'I', @@ -98,6 +99,10 @@ const fn eq(a: &[u8], b: &[u8]) -> bool { true } +/// The vendor GUID every floor is under, minted for this and used for nothing +/// else. +pub const VENDOR: &str = "33be3d4a-30e6-49f5-8050-f169d93a20fb"; + /// What every floor variable's name begins with. pub const PREFIX: &str = "ToyOSImageFloor"; From 3ee66e9f47f8864640d3cdb576774645453ad49c Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 27 Sep 2026 23:55:00 +0200 Subject: [PATCH 5/9] imagerelease: what gh answers is read by two pure functions, each tested `held_of` reads `gh release view` and takes only its own `release not found` as nothing published; `releases` reads `gh release list` and refuses a listing that filled its limit or an entry without a tag and a time. foreign_disk_untouched hands its disks to the boot without clones. Co-Authored-By: Claude Opus 5.5 --- src/imagerelease.rs | 52 ++++++++++++++++++++++++++++++++++------- tests/common/storage.rs | 8 +++---- 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/src/imagerelease.rs b/src/imagerelease.rs index 268d67f91d..0e6cc388e8 100644 --- a/src/imagerelease.rs +++ b/src/imagerelease.rs @@ -418,23 +418,32 @@ struct Held { /// What GitHub holds under `tag`: nothing, which `gh` says as exactly /// `release not found`, or a [`Held`]. Any other failure of `gh` is one. fn held(root: &Path, tag: &str) -> Result, String> { - let json = match gh(root, &["release", "view", tag, "--json", "isDraft,assets"]) { + held_of(gh(root, &["release", "view", tag, "--json", "isDraft,assets"])) +} + +/// [`held`] of what `gh release view --json isDraft,assets` answered. +fn held_of(said: Result) -> Result, String> { + let json = match said { Err(why) if why.ends_with(": release not found") => return Ok(None), said => said?, }; - let v: Value = serde_json::from_str(&json).map_err(|e| format!("gh release view {tag}: {e}"))?; - let draft = v["isDraft"].as_bool().ok_or_else(|| format!("gh release view {tag} gave no isDraft: {json}"))?; - let assets = v["assets"].as_array().ok_or_else(|| format!("gh release view {tag} gave no assets: {json}"))?; + let v: Value = serde_json::from_str(&json).map_err(|e| format!("gh release view: {e}: {json}"))?; + let draft = v["isDraft"].as_bool().ok_or_else(|| format!("gh release view gave no isDraft: {json}"))?; + let assets = v["assets"].as_array().ok_or_else(|| format!("gh release view gave no assets: {json}"))?; let image = assets.iter().any(|a| a["name"].as_str() == Some(IMAGE_ASSET)); Ok(Some(Held { draft, image })) } -/// Every release, as (tag, creation time), refusing a listing that filled -/// [`LISTED`] or an entry without both. +/// Every release, as (tag, creation time). fn listed(root: &Path) -> Result, String> { let limit = LISTED.to_string(); - let json = gh(root, &["release", "list", "--limit", &limit, "--json", "tagName,createdAt"])?; - let v: Value = serde_json::from_str(&json).map_err(|e| format!("gh release list: {e}"))?; + releases(&gh(root, &["release", "list", "--limit", &limit, "--json", "tagName,createdAt"])?) +} + +/// [`listed`] of what `gh release list --json tagName,createdAt` answered, +/// refusing a listing that filled [`LISTED`] or an entry without both. +fn releases(json: &str) -> Result, String> { + let v: Value = serde_json::from_str(json).map_err(|e| format!("gh release list: {e}"))?; let all = v.as_array().ok_or_else(|| format!("gh release list gave no list: {json}"))?; if all.len() >= LISTED { return Err(format!("gh release list gave {} releases, its limit, so some are not in it", all.len())); @@ -528,6 +537,33 @@ mod tests { notes(&root(), "image-x86_64-c55189490123", &"c".repeat(40)).unwrap() } + /// Only `gh`'s own not-found answer reads as nothing published; a failure + /// of any other kind is one, and a draft is told from a release. + #[test] + fn a_gh_failure_is_not_read_as_nothing_published() { + let gh = |said: &str| Err(format!("gh release view image-x86_64-c55189490123 exited exit status: 1: {said}")); + assert_eq!(held_of(gh("release not found")), Ok(None)); + let why = held_of(gh("non-200 OK status code: 401 Unauthorized body: \"Bad credentials\"")).unwrap_err(); + assert!(why.contains("401"), "{why}"); + let json = |draft: bool| Ok(format!(r#"{{"isDraft":{draft},"assets":[{{"name":"{IMAGE_ASSET}"}}]}}"#)); + assert_eq!(held_of(json(true)), Ok(Some(Held { draft: true, image: true }))); + assert_eq!(held_of(json(false)), Ok(Some(Held { draft: false, image: true }))); + assert_eq!(held_of(Ok(r#"{"isDraft":false,"assets":[]}"#.into())), Ok(Some(Held { draft: false, image: false }))); + assert!(held_of(Ok(r#"{"assets":[]}"#.into())).is_err()); + } + + /// A listing that filled its limit, or an entry without a tag and a time, + /// is refused rather than read short. + #[test] + fn a_listing_that_may_have_left_releases_out_is_refused() { + let entry = |tag: &str| format!(r#"{{"tagName":"{tag}","createdAt":"2026-09-27T03:00:00Z"}}"#); + let list = |n: usize| format!("[{}]", (0..n).map(|i| entry(&format!("t{i}"))).collect::>().join(",")); + assert_eq!(releases(&list(LISTED - 1)).unwrap().len(), LISTED - 1); + assert!(releases(&list(LISTED)).unwrap_err().contains("its limit")); + let why = releases(r#"[{"tagName":"t"}]"#).unwrap_err(); + assert!(why.contains("without a tag and a time"), "{why}"); + } + #[test] fn a_tag_is_the_commit_and_a_short_or_foreign_id_is_refused() { let commit = "c55189490123456789abcdef0123456789abcdef"; diff --git a/tests/common/storage.rs b/tests/common/storage.rs index 94a67e2091..0a2faf08c8 100644 --- a/tests/common/storage.rs +++ b/tests/common/storage.rs @@ -41,13 +41,13 @@ pub fn foreign_disk_untouched( let parts = other_systems_disk(&other, USB_BYTES)?; let bare = dir.join("bare-stick.img"); filled(&bare, USB_BYTES, 0x5A)?; - let disks = [image.clone(), other.clone(), bare.clone()]; + let disks = [image, other, bare]; let before: Vec> = disks.iter().map(|disk| whole_device(disk)).collect(); // The premise, checked before the boot rather than assumed: if this volume // somehow already parsed as a ToyOS volume, the kernel would mount it and // the assertion below would pass for the wrong reason. - if front(&image, data_at, 4) == *b"BCFS" { + if front(&disks[0], data_at, 4) == *b"BCFS" { return Err("the foreign volume starts with a bcachefs superblock".to_string()); } @@ -57,8 +57,8 @@ pub fn foreign_disk_untouched( rust_bins, BootOptions { profile: qemu::Profile::UsbDiskCrowd, - nvme_image: Some(image.clone()), - usb_images: vec![other.clone(), bare.clone()], + nvme_image: Some(disks[0].clone()), + usb_images: disks[1..].to_vec(), ..Default::default() }, ); From 11b61c71570e95d42392bc45c11284db891b95b1 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 28 Sep 2026 00:43:55 +0200 Subject: [PATCH 6/9] imagerelease: the write token leaves the job that boots; std is fetched one way The release job builds and boots with a token that only reads, and hands the four assets on as an artifact and their digest as a job output. A new `release-publish` job holds `contents: write`, restores no cache, builds only toyos-build with no token in reach, rechecks the digest (`imagerelease::digest`) and publishes on main. `ci::a_token_that_writes_meets_no_cache_and_only_the_step_that_needs_it` holds every workflow to that. The post-publish re-read and the GITHUB_ACTIONS guard are gone; `gh` is no longer installed in the container. std's `library/` comes from a linked worktree's fork checkout, from `rust/library` where the checkout holds it, and from `fetched_library` everywhere else; the submodule-init path is deleted. The fetch sits at `.licence-fork/`, a direct child of the checkout, because std's manifest names `toyos` and `toyos-abi` three levels above its crates: under `target/` it never resolved. It is fetched into `target/licence/fork.partial` under its own lock and renamed into place, reused only at the pinned commit and sparse set, and the release build writes the notice before any build lock. The release notice test fetches the way the release job does. An OR takes Apache-2.0's standard text where it is a branch. The notice's opening sentence says what the walk leaves out. The notes name the loader defect as the one disk a boot may write though it was not given it. `foreign_disk_untouched` boots twice on `Profile::UsbDisk`, one foreign USB disk beside the NVMe disk each time: the USB driver serves two disks and the boot stick is one, so `UsbDiskCrowd`'s third was never read. `release_command_boots` is in a new Apple Silicon tier and boots that line; `Host::this` is gone, and the release job boots the Linux line. Filed: the toolchain install unpacks an asset no digest vouches for. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly.yml | 49 +++- .gitignore | 3 + ...its-log-guid-not-the-one-it-booted-from.md | 4 +- ...-unpacks-an-asset-no-digest-vouches-for.md | 25 ++ src/build.rs | 22 +- src/buildlock.rs | 7 + src/ci.rs | 69 ++++- src/imagerelease.rs | 246 +++++++++++------ src/licence.rs | 250 ++++++++++++++---- src/sourcegate.rs | 13 +- src/tiers.rs | 11 +- tests/common/release.rs | 11 +- tests/common/storage.rs | 55 ++-- tests/toyos.rs | 16 +- 14 files changed, 580 insertions(+), 201 deletions(-) create mode 100644 issues/build/the-toolchain-install-unpacks-an-asset-no-digest-vouches-for.md diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 784105e5bd..68292a9040 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -250,9 +250,10 @@ jobs: - *checkout - run: cargo build -p toyos-build - # The disk image a person downloads: built once, booted under the Linux line - # its notes print, and on main published as those same bytes - # (`src/imagerelease.rs`). + # The disk image a person downloads, built once and booted under the Linux + # line its notes print (`src/imagerelease.rs`). It compiles and runs + # third-party code out of a cache another such job wrote, so its token only + # reads: the assets go on as an artifact and their digest as an output. release: needs: build runs-on: ubuntu-24.04 @@ -260,22 +261,52 @@ jobs: timeout-minutes: 60 container: *kvm permissions: - contents: write + contents: read env: GH_TOKEN: ${{ github.token }} + outputs: + digest: ${{ steps.release.outputs.digest }} steps: - *deps - - name: gh - run: | - DEBIAN_FRONTEND=noninteractive apt-get install -y -qq gh - *checkout - *guest-cache - - run: cargo run -- --ci release + - id: release + run: cargo run -- --ci release + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: image-release + path: target/image-release/ + if-no-files-found: error + retention-days: 1 + + # The one job that writes releases, publishing on main what `release` booted + # once its digest is the one `release` answered. No cache and nothing built + # but the build system, and the token only in the publishing step: the + # checkout keeps no credential, and the build step has none. + release-publish: + needs: release + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: write + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: image-release + path: target/image-release + - run: cargo build -p toyos-build + - env: + GH_TOKEN: ${{ github.token }} + IMAGE_RELEASE_DIGEST: ${{ needs.release.outputs.digest }} + run: cargo run -- --ci release-publish # One standing issue, found by title and commented on; a dispatch is somebody # watching the run, so only the schedule files. nightly-red: - needs: [host, build, guest, tcg, audio, portability-linux, portability-macos, release] + needs: [host, build, guest, tcg, audio, portability-linux, portability-macos, release, release-publish] if: ${{ !cancelled() && github.event_name == 'schedule' }} runs-on: ubuntu-latest permissions: diff --git a/.gitignore b/.gitignore index 09df96c737..784c83812c 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,6 @@ assets/*.sf2 phosphor-icons .cargo/config.toml .build-locks/ +# The std fork's `library/` and licence files, fetched where `rust/` holds no source +# (`src/licence.rs`). +.licence-fork/ diff --git a/issues/boot-media/the-loader-writes-the-first-disk-carrying-its-log-guid-not-the-one-it-booted-from.md b/issues/boot-media/the-loader-writes-the-first-disk-carrying-its-log-guid-not-the-one-it-booted-from.md index cdfe55b74f..ecbb7f31c8 100644 --- a/issues/boot-media/the-loader-writes-the-first-disk-carrying-its-log-guid-not-the-one-it-booted-from.md +++ b/issues/boot-media/the-loader-writes-the-first-disk-carrying-its-log-guid-not-the-one-it-booted-from.md @@ -13,8 +13,8 @@ copy of one image carries the same partition unique GUIDs, and the image release (`src/imagerelease.rs`) is written byte for byte to every stick made from it. With two sticks of one release plugged in, the loader can write the log partition of the stick it did not boot from: a disk it was not given, and -one that carries no TOYOS-DATA partition, which the release notes say a boot -never writes. +one that carries no TOYOS-DATA partition. The release notes name this as the +one such disk a boot may write. Owner: the bootloader. diff --git a/issues/build/the-toolchain-install-unpacks-an-asset-no-digest-vouches-for.md b/issues/build/the-toolchain-install-unpacks-an-asset-no-digest-vouches-for.md new file mode 100644 index 0000000000..0ad6413014 --- /dev/null +++ b/issues/build/the-toolchain-install-unpacks-an-asset-no-digest-vouches-for.md @@ -0,0 +1,25 @@ +--- +status: open +kind: tooling +opened: 2026-09-28 +--- + +# The toolchain install unpacks an asset no digest vouches for + +`release::install` (`src/release.rs`) downloads the `toyos-toolchain.tar.zst` +asset of the release its tree's tag names, unpacks it into `rust/build` and +links it as rustup's `toyos`, and checks nothing about its bytes: the tag is +the hash of the tarball's inputs (`TREES`), not of the tarball. Every guest +job and the image release's boot job compile and boot with what it installs. + +A job holding this repository's `contents: write` token can replace a +release asset, so any code such a job runs can replace the toolchain every +later job installs, and through it the image the release publishes. The +nightly's `build` job holds that token while it bootstraps the toolchain, and +the image release's publish job while it publishes. + +Owner: the release module (`src/release.rs`). + +**Exit condition.** `install` unpacks only an asset whose SHA-256 is one that +no job holding a write token can rewrite — committed to the tree it installs +for — and refuses any other by name. diff --git a/src/build.rs b/src/build.rs index 99b202475b..a88fb8d775 100644 --- a/src/build.rs +++ b/src/build.rs @@ -1878,6 +1878,18 @@ fn shipped_parts(root: &Path, boot: &Boot, rebuild_toolchain: bool, plan: &Plan) let kernel_features = plan.features.join(","); let arch = plan.arch; + // The notice asks the network, so it is written before any lock is taken. + let mut extra = Vec::new(); + if boot.public { + let notices = boot + .parts(root) + .and_then(|parts| crate::licence::notices(root, parts, &crate::licence::std_library(root)?)) + .unwrap_or_else(|why| panic!("the release's licence notice: {why}")); + let at = root.join(RELEASE_NOTICES); + fs::write(&at, ¬ices).unwrap_or_else(|e| panic!("write {}: {e}", at.display())); + extra.push((crate::licence::NOTICES_ON_ROOT.to_string(), notices.into_bytes())); + } + // Before every build lock, which is the order `buildlock`'s header fixes. // What it bounds is the host: ten agents' builds spend the same fourteen // cores, and nothing was counting them. @@ -1920,16 +1932,6 @@ fn shipped_parts(root: &Path, boot: &Boot, rebuild_toolchain: bool, plan: &Plan) ) }; - let mut extra = Vec::new(); - if boot.public { - let notices = boot - .parts(root) - .and_then(|parts| crate::licence::notices(root, parts)) - .unwrap_or_else(|why| panic!("the release's licence notice: {why}")); - let at = root.join(RELEASE_NOTICES); - fs::write(&at, ¬ices).unwrap_or_else(|e| panic!("write {}: {e}", at.display())); - extra.push((crate::licence::NOTICES_ON_ROOT.to_string(), notices.into_bytes())); - } let root_bytes = build_and_assemble(root, &config, &env, &extra, false, arch); let bl_bytes = fs::read(&bl_art).expect("Failed to read staged bootloader"); diff --git a/src/buildlock.rs b/src/buildlock.rs index fa9fc5cf07..c2a53242cf 100644 --- a/src/buildlock.rs +++ b/src/buildlock.rs @@ -234,6 +234,13 @@ pub fn artifact(root: &Path) -> Guard { exclusive(&root.join(LOCK_DIR).join("artifact"), "artifact lock", "artifact staging") } +/// This worktree's one fetch of the std fork's licence sources +/// (`licence::fetched_library`). It waits on the network, so it is taken with +/// no other lock held. +pub fn fork_fetch(root: &Path) -> Guard { + exclusive(&root.join(LOCK_DIR).join("fork-fetch"), "fork fetch lock", "fetching std's licence sources") +} + /// The integration lock: one process at a time moves this host's `main`. /// /// It used to hold a whole landing — lock, merge, gate, fast-forward. diff --git a/src/ci.rs b/src/ci.rs index 898a707d4c..7a4d8b3c0d 100644 --- a/src/ci.rs +++ b/src/ci.rs @@ -7,9 +7,10 @@ //! [`Job::GateStage`] run as `host`. Every test that boots no guest is in //! [`Job::Host`], so a merge is gated on all of them. `nightly.yml` runs //! everything that boots a guest, `host` again to write the cache the merge -//! queue restores, portability, and the image release, which boots the image it -//! publishes ([`Job::Release`]). `publish.yml` puts a landing's crates on -//! crates.io. +//! queue restores, portability, and the image release: [`Job::Release`] boots the +//! image it stages with a token that only reads, and [`Job::ReleasePublish`] +//! publishes those bytes with one that writes. `publish.yml` puts a landing's +//! crates on crates.io. //! //! A host job runs every step and reds if any failed; a guest job stops at the //! first failure among the instrument, the toolchain and the suite, because @@ -50,7 +51,8 @@ const USAGE: &str = "cargo run -- --ci , where is one of: guest / one shard of the whole guest suite, nightly tier included (nightly) tcg one test on an emulated CPU (nightly) audio / one shard of gate A (nightly) - release build the release image, boot it, and on main publish it (nightly) + release build the release image, boot it, and stage its assets (nightly) + release-publish on main, publish the assets `release` staged (nightly) nightly-red file or update the nightly-red issue from $NEEDS (nightly) publish put main's SDK crates on crates.io (publish.yml)"; @@ -63,6 +65,7 @@ enum Job { Tcg, Audio(String), Release, + ReleasePublish, NightlyRed, Publish, } @@ -81,6 +84,7 @@ fn parse(words: &[String]) -> Result { Some("tcg") => Job::Tcg, Some("audio") => Job::Audio(shard(words.get(1))?), Some("release") => Job::Release, + Some("release-publish") => Job::ReleasePublish, Some("nightly-red") => Job::NightlyRed, Some("publish") => Job::Publish, Some(other) => return Err(format!("no CI job is called {other:?}")), @@ -110,6 +114,7 @@ pub fn dispatch(root: &Path, args: &[String]) { guest(root, "the suite", || suite(root, &suite_args(&["--audio-gate", "30", "--shard", shard]))) } Job::Release => guest(root, "the image release", || imagerelease::release(root)), + Job::ReleasePublish => vec![step("the image release's publication", || imagerelease::publish(root))], Job::NightlyRed => vec![step("the nightly-red issue", nightly_red)], Job::Publish => vec![step("the SDK crates on crates.io", || publish(root))], }; @@ -155,6 +160,14 @@ fn on_runner() -> bool { std::env::var("GITHUB_ACTIONS").is_ok_and(|v| v == "true") } +/// Set this step's output `name`, which a later job reads through its own +/// job's `outputs:`. Refused where no runner gave the step an output file. +pub fn output(name: &str, value: &str) -> Result<(), String> { + let path = std::env::var("GITHUB_OUTPUT").map_err(|_| "GITHUB_OUTPUT is unset".to_string())?; + let mut file = std::fs::OpenOptions::new().append(true).open(&path).map_err(|e| format!("{path}: {e}"))?; + writeln!(file, "{name}={value}").map_err(|e| format!("{path}: {e}")) +} + /// Append to the runner's job summary; nowhere off a runner. fn summary(text: &str) { let Ok(path) = std::env::var("GITHUB_STEP_SUMMARY") else { return }; @@ -915,6 +928,7 @@ mod tests { fn a_job_is_named_and_a_shard_is_a_shard() { assert_eq!(parse(&words("host")), Ok(Job::Host)); assert_eq!(parse(&words("guest 3/12")), Ok(Job::Guest("3/12".into()))); + assert_eq!(parse(&words("release-publish")), Ok(Job::ReleasePublish)); assert!(parse(&words("guest")).is_err()); assert!(parse(&words("guest 13/12")).is_err()); assert!(parse(&words("host extra")).is_err()); @@ -1065,6 +1079,53 @@ mod tests { assert!(writers.iter().all(|(f, _)| f == "nightly.yml"), "{writers:?}"); } + /// A job whose token writes restores no cache, since a cache is a tree a + /// job running third-party code wrote, and hands `GH_TOKEN` to a step + /// rather than to every step. + #[test] + fn a_token_that_writes_meets_no_cache_and_only_the_step_that_needs_it() { + let dir = repo_root().join(".github/workflows"); + let mut writing = 0; + for entry in std::fs::read_dir(&dir).expect(".github/workflows is readable").flatten() { + let text = std::fs::read_to_string(entry.path()).expect("a readable workflow"); + let file = entry.file_name().to_string_lossy().into_owned(); + let (head, jobs) = text.split_once("\njobs:\n").expect("a workflow has jobs"); + let lines: Vec<&str> = text.lines().collect(); + let restoring: Vec = lines + .windows(2) + .filter(|w| w[1].contains("actions/cache/restore@")) + .filter_map(|w| w[0].trim_start().strip_prefix("- &")) + .map(|anchor| format!("*{anchor}")) + .collect(); + let mut chunks: Vec<(String, String)> = Vec::new(); + for line in jobs.lines() { + let key = line.strip_prefix(" ").and_then(|l| l.strip_suffix(':')); + match key.filter(|k| !k.starts_with([' ', '#'])) { + Some(job) => chunks.push((job.to_string(), String::new())), + None => { + if let Some((_, body)) = chunks.last_mut() { + body.push_str(&format!("{line}\n")); + } + } + } + } + for (job, body) in &chunks { + let permissions = if body.contains("\n permissions:\n") { body.as_str() } else { head }; + if !permissions.contains(": write") { + continue; + } + writing += 1; + let restores = body.contains("actions/cache/restore@") || restoring.iter().any(|a| body.contains(a.as_str())); + assert!(!restores, "{file}: {job} restores a cache with a token that writes"); + let job_env = body.split("\n env:\n").nth(1).map(|env| { + env.lines().take_while(|l| l.starts_with(" ")).any(|l| l.contains("GH_TOKEN")) + }); + assert_ne!(job_env, Some(true), "{file}: {job} hands a token that writes to every step"); + } + } + assert!(writing > 0, "no job's token writes, so this checked nothing"); + } + #[test] fn the_declared_version_is_a_version() { let declared = diff --git a/src/imagerelease.rs b/src/imagerelease.rs index 0e6cc388e8..8405d9c77a 100644 --- a/src/imagerelease.rs +++ b/src/imagerelease.rs @@ -1,13 +1,19 @@ //! The image release: the disk a person downloads and boots under QEMU or -//! writes to a stick, published by `cargo run -- --ci release`. +//! writes to a stick, published by the nightly's `release` and +//! `release-publish` jobs. //! -//! **What is published is what booted**: the job builds `build::Boot::release`'s -//! image once, boots a copy of it under the Linux command line its notes print -//! ([`boots`]), and on `main` uploads those bytes. **Named by the commit**, -//! [`tag`]: every build draws its partition GUIDs and a runner mints its own -//! throwaway signing key (`src/signing.rs`), so a second build reproduces no -//! byte of the first. **Kept to [`KEEP`]**: each publish deletes every older -//! image release and its tag ([`stale`]). +//! **What is published is what booted**: [`release`] builds +//! `build::Boot::release`'s image once, boots a copy of it under the Linux +//! command line its notes print ([`boots`]), and stages those bytes' assets in +//! [`STAGED`]. **The token that writes releases never meets the code a build +//! runs**: that job's token reads, and it hands the assets on as an artifact +//! and their [`digest`] as a job output; [`publish`], in a job that restores no +//! cache and builds only this crate, publishes them on `main` once their digest +//! is the one it was handed. **Named by the commit**, [`tag`]: every build draws +//! its partition GUIDs and a runner mints its own throwaway signing key +//! (`src/signing.rs`), so a second build reproduces no byte of the first. +//! **Kept to [`KEEP`]**: each publish deletes every older image release and its +//! tag ([`stale`]). //! //! Not in `src/release.rs`, whose bytes are hashed into the toolchain's tag. @@ -23,6 +29,7 @@ use serde_json::Value; use crate::arch::{Accel, Arch}; use crate::fingerprint::{first_difference, whole_device}; use crate::licence::{pending_owner, Subject}; +use crate::release::sha256_hex; /// What every image release's tag starts with; the rest is the commit's first /// twelve hex digits. @@ -46,6 +53,21 @@ pub const NOTES_ASSET: &str = "README.md"; /// The image's licence notice (`build::RELEASE_NOTICES`), as an asset. pub const LICENCES_ASSET: &str = "licences.txt"; +/// Every asset, in upload order. +pub const ASSETS: [&str; 4] = [IMAGE_ASSET, SUMS_ASSET, NOTES_ASSET, LICENCES_ASSET]; + +/// Where [`release`] stages the assets and [`publish`] reads them: the +/// directory the nightly's artifact carries between the two jobs. +pub const STAGED: &str = "target/image-release"; + +/// What the publish job is handed the release job's [`digest`] in. +pub const DIGEST_VAR: &str = "IMAGE_RELEASE_DIGEST"; + +/// The defect behind the one disk the notes say a boot may write though it +/// was not given it. +const TWO_STICKS: &str = + "issues/boot-media/the-loader-writes-the-first-disk-carrying-its-log-guid-not-the-one-it-booted-from.md"; + /// One guest's ceiling from power-on to a painting desktop, unscaled: a /// liveness guard, never a verdict. pub const DESKTOP: Duration = Duration::from_secs(120); @@ -75,20 +97,6 @@ pub enum Host { impl Host { pub const ALL: [Host; 2] = [Host::MacosAppleSilicon, Host::LinuxKvm]; - /// The one of the two this machine is, or, refused by name, that it is - /// neither: the notes print no line for it, so there is no line to boot. - pub fn this() -> Result { - if cfg!(target_os = "macos") && Arch::HOST == Some(Arch::Aarch64) { - return Ok(Host::MacosAppleSilicon); - } - if cfg!(target_os = "linux") && Arch::HOST == Some(Arch::X86_64) && Arch::X86_64.accel() == Accel::Kvm { - return Ok(Host::LinuxKvm); - } - Err("the release notes print a command line for an Apple Silicon Mac and for an x86-64 \ - Linux whose /dev/kvm opens, and this host is neither" - .into()) - } - /// Where the notes say this host is. pub fn named(self) -> &'static str { match self { @@ -342,7 +350,7 @@ Write `{IMAGE}` to the whole stick, not to a partition of it; what the stick hel dd if={IMAGE} of=/dev/ bs=4194304 sync -A boot writes to the stick it booted from, and to another disk only where that disk carries a partition of ToyOS's DATA type, `{data}`, a type no other system uses. That is where `/apps`, `/config`, `/home` and `/state` live. Every other disk is read for its partition table and never written. +A boot writes to the stick it booted from, and to another disk only where that disk carries a partition of ToyOS's DATA type, `{data}`, a type no other system uses. That is where `/apps`, `/config`, `/home` and `/state` live. Every other disk is read for its partition table and not written, but for one known defect: with two sticks made from one image plugged in, the loader can write its log to the one it did not boot from (`{TWO_STICKS}` in the ToyOS source). ## Terms @@ -365,7 +373,6 @@ pub fn stale(listed: &[(String, String)], keep: usize) -> Vec { /// Write every asset of `tag`'s release into `out`: the image at `image` /// compressed, its sum, the notes and the licence notice at `licences`. -/// Answers the paths in upload order. pub fn write_assets( root: &Path, image: &Path, @@ -373,9 +380,8 @@ pub fn write_assets( out: &Path, tag: &str, commit: &str, -) -> Result, String> { +) -> Result<(), String> { use flate2::write::GzEncoder; - use sha2::{Digest, Sha256}; let compressed = out.join(IMAGE_ASSET); let file = fs::File::create(&compressed).map_err(|e| format!("{}: {e}", compressed.display()))?; @@ -385,16 +391,34 @@ pub fn write_assets( gz.finish().map_err(|e| format!("compressing {}: {e}", image.display()))?.flush().map_err(|e| e.to_string())?; let bytes = fs::read(&compressed).map_err(|e| format!("{}: {e}", compressed.display()))?; - let sum: String = Sha256::digest(&bytes).iter().map(|b| format!("{b:02x}")).collect(); - let sums = out.join(SUMS_ASSET); - fs::write(&sums, format!("{sum} {IMAGE_ASSET}\n")).map_err(|e| e.to_string())?; - - let readme = out.join(NOTES_ASSET); - fs::write(&readme, notes(root, tag, commit)?).map_err(|e| e.to_string())?; + let sums = format!("{} {IMAGE_ASSET}\n", sha256_hex(&bytes)); + fs::write(out.join(SUMS_ASSET), sums).map_err(|e| e.to_string())?; + fs::write(out.join(NOTES_ASSET), notes(root, tag, commit)?).map_err(|e| e.to_string())?; + fs::copy(licences, out.join(LICENCES_ASSET)).map_err(|e| format!("{}: {e}", licences.display()))?; + Ok(()) +} - let notice = out.join(LICENCES_ASSET); - fs::copy(licences, ¬ice).map_err(|e| format!("{}: {e}", licences.display()))?; - Ok(vec![compressed, sums, readme, notice]) +/// The SHA-256 of every asset's own, one `sha256sum` line each in upload +/// order: what the release job answers and the publish job recomputes. +/// Refused while `dir` holds anything but the four assets. +pub fn digest(dir: &Path) -> Result { + let entries = fs::read_dir(dir).map_err(|e| format!("{}: {e}", dir.display()))?; + let mut held = entries + .map(|e| e.map(|e| e.file_name().to_string_lossy().into_owned())) + .collect::, _>>() + .map_err(|e| format!("{}: {e}", dir.display()))?; + held.sort(); + let mut want = ASSETS.to_vec(); + want.sort(); + if held != want { + return Err(format!("{} holds {held:?}, and the assets are {ASSETS:?}", dir.display())); + } + let mut sums = String::new(); + for name in ASSETS { + let bytes = fs::read(dir.join(name)).map_err(|e| format!("{}: {e}", dir.join(name).display()))?; + sums.push_str(&format!("{} {name}\n", sha256_hex(&bytes))); + } + Ok(sha256_hex(sums.as_bytes())) } /// `gh `: what it printed, or its exit and what it said. @@ -456,13 +480,13 @@ fn releases(json: &str) -> Result, String> { .collect() } -/// Upload `assets` as a draft of `tag`, and publish it once GitHub holds the -/// image, so a failed upload leaves nothing public. -fn publish(root: &Path, tag: &str, commit: &str, notes: &Path, assets: &[PathBuf]) -> Result<(), String> { - let notes = notes.display().to_string(); +/// Upload the assets in `staged` as a draft of `tag`, and publish it once +/// GitHub holds the image, so a failed upload leaves nothing public. +fn create(root: &Path, tag: &str, commit: &str, staged: &Path) -> Result<(), String> { + let notes = staged.join(NOTES_ASSET).display().to_string(); let mut args: Vec<&str> = vec!["release", "create", tag, "--draft", "--title", tag, "--target", commit, "--notes-file", ¬es]; - let assets: Vec = assets.iter().map(|a| a.display().to_string()).collect(); + let assets: Vec = ASSETS.iter().map(|a| staged.join(a).display().to_string()).collect(); args.extend(assets.iter().map(String::as_str)); gh(root, &args)?; let drafted = held(root, tag)?; @@ -470,53 +494,68 @@ fn publish(root: &Path, tag: &str, commit: &str, notes: &Path, assets: &[PathBuf return Err(format!("{tag} was created as a draft carrying {IMAGE_ASSET}, and GitHub holds {drafted:?}")); } gh(root, &["release", "edit", tag, "--draft=false", "--latest"])?; - let published = held(root, tag)?; - if published != Some(Held { draft: false, image: true }) { - return Err(format!("{tag} was published, and GitHub holds {published:?}")); - } Ok(()) } -/// `cargo run -- --ci release`: build the release image once, boot a copy of -/// it under this host's line, and on `main` publish those bytes unless this -/// commit's are, then delete the image releases past [`KEEP`]. -pub fn release(root: &Path) -> Result { - if !std::env::var("GITHUB_ACTIONS").is_ok_and(|v| v == "true") { - return Err("only a runner releases an image".into()); - } +/// The commit this run is of, which the checkout has to be, as its [`tag`] +/// and itself. +fn this_run(root: &Path) -> Result<(String, String), String> { let commit = std::env::var("GITHUB_SHA").map_err(|_| "GITHUB_SHA is unset".to_string())?; let head = crate::pr::git(root, &["rev-parse", "HEAD"])?; if head != commit { return Err(format!("the checkout is {head} and the run is of {commit}")); } - let tag = tag(&commit)?; - let on_main = std::env::var("GITHUB_REF").ok().as_deref() == Some("refs/heads/main"); - let host = Host::this()?; + Ok((tag(&commit)?, commit)) +} + +/// `cargo run -- --ci release`: build the release image once, boot a copy of +/// it under the Linux line, stage its assets in [`STAGED`], and answer their +/// [`digest`] as the step's `digest` output. +pub fn release(root: &Path) -> Result { + let (tag, commit) = this_run(root)?; + let host = Host::LinuxKvm; + let boot = crate::build::Boot::release(root); + let plan = crate::build::plan_for(root, &boot, false, &[]); + let image = crate::build::build(root, boot, false, &plan); + let scratch = toyos_tmpdir::TempDir::new("image-release"); + let stick = scratch.join("stick.img"); + fs::copy(&image, &stick).map_err(|e| format!("copy {} to {}: {e}", image.display(), stick.display()))?; + boots(root, host, &stick, DESKTOP, &scratch.join("qemu.stderr"))?; + + let staged = root.join(STAGED); + if staged.exists() { + fs::remove_dir_all(&staged).map_err(|e| format!("remove {}: {e}", staged.display()))?; + } + fs::create_dir_all(&staged).map_err(|e| format!("create {}: {e}", staged.display()))?; + let licences = root.join(crate::build::RELEASE_NOTICES); + write_assets(root, &image, &licences, &staged, &tag, &commit)?; + let digest = digest(&staged)?; + crate::ci::output("digest", &digest)?; + Ok(format!("{tag} booted to its desktop under {host:?}'s line; its assets are staged, digest {digest}")) +} +/// `cargo run -- --ci release-publish`: the assets [`release`] staged, refused +/// unless their [`digest`] is the one it answered, published on `main` unless +/// this commit's are; then the image releases past [`KEEP`] deleted. +pub fn publish(root: &Path) -> Result { + let (tag, commit) = this_run(root)?; + let staged = root.join(STAGED); + let handed = std::env::var(DIGEST_VAR).map_err(|_| format!("{DIGEST_VAR} is unset"))?; + let digest = digest(&staged)?; + if digest != handed { + return Err(format!("the assets' digest is {digest}, and the release job answered {handed:?}")); + } + if std::env::var("GITHUB_REF").ok().as_deref() != Some("refs/heads/main") { + return Ok(format!("{tag}'s assets arrived as the release job staged them; off main, so not published")); + } let mut said = match held(root, &tag)? { Some(Held { draft: false, image: true }) => format!("{tag} is already published"), Some(other) => return Err(format!("GitHub holds {tag} as {other:?}: a failed run's, to delete by hand")), None => { - let boot = crate::build::Boot::release(root); - let plan = crate::build::plan_for(root, &boot, false, &[]); - let image = crate::build::build(root, boot, false, &plan); - let out = toyos_tmpdir::TempDir::new("image-release"); - let stick = out.join("stick.img"); - fs::copy(&image, &stick).map_err(|e| format!("copy {} to {}: {e}", image.display(), stick.display()))?; - boots(root, host, &stick, DESKTOP, &out.join("qemu.stderr"))?; - fs::remove_file(&stick).map_err(|e| format!("{}: {e}", stick.display()))?; - if !on_main { - return Ok(format!("{tag} booted to its desktop under {host:?}'s line; off main, so not published")); - } - let licences = root.join(crate::build::RELEASE_NOTICES); - let assets = write_assets(root, &image, &licences, &out, &tag, &commit)?; - publish(root, &tag, &commit, &out.join(NOTES_ASSET), &assets)?; - format!("{tag} booted to its desktop under {host:?}'s line and is published") + create(root, &tag, &commit, &staged)?; + format!("{tag} is published") } }; - if !on_main { - return Ok(said); - } let old = stale(&listed(root)?, KEEP); for old in &old { gh(root, &["release", "delete", old, "--cleanup-tag", "--yes"])?; @@ -626,14 +665,18 @@ mod tests { } } - /// What `sha256sum -c` checks is the compressed asset's own digest, the - /// asset decompresses to the image byte for byte, and the notice is - /// carried as it was written. + /// The one disk the notes say a boot may write though it was not given it + /// is cited by an issue that is open, so the sentence goes when the defect + /// does. #[test] - fn the_sum_is_the_compressed_images_and_it_decompresses_to_the_image() { - use sha2::{Digest, Sha256}; - use std::io::Read; - let dir = toyos_tmpdir::TempDir::new("image-assets"); + fn the_notes_cite_the_open_defect_behind_the_disk_a_boot_may_write() { + assert!(notes_of_a_commit().contains(&format!("`{TWO_STICKS}`"))); + let issue = fs::read_to_string(root().join(TWO_STICKS)).unwrap_or_else(|e| panic!("{TWO_STICKS}: {e}")); + assert!(issue.contains("\nstatus: open\n"), "{TWO_STICKS} is not open"); + } + + /// Four assets written into a directory of their own. + fn staged(dir: &Path) -> PathBuf { let image = dir.join("in.img"); let bytes: Vec = (0..300_000u32).map(|i| (i % 251) as u8).chain(std::iter::repeat_n(0, 1 << 20)).collect(); fs::write(&image, &bytes).unwrap(); @@ -641,17 +684,48 @@ mod tests { fs::write(&licences, "the notice").unwrap(); let out = dir.join("out"); fs::create_dir(&out).unwrap(); - let assets = write_assets(&root(), &image, &licences, &out, "image-x86_64-c55189490123", &"c".repeat(40)).unwrap(); - let names: Vec = - assets.iter().map(|a| a.file_name().unwrap().to_string_lossy().into_owned()).collect(); - assert_eq!(names, [IMAGE_ASSET, SUMS_ASSET, NOTES_ASSET, LICENCES_ASSET]); - assert_eq!(fs::read_to_string(out.join(LICENCES_ASSET)).unwrap(), "the notice"); + write_assets(&root(), &image, &licences, &out, "image-x86_64-c55189490123", &"c".repeat(40)).unwrap(); + out + } + /// What `sha256sum -c` checks is the compressed asset's own digest, the + /// asset decompresses to the image byte for byte, and the notice is + /// carried as it was written. + #[test] + fn the_sum_is_the_compressed_images_and_it_decompresses_to_the_image() { + use std::io::Read; + let dir = toyos_tmpdir::TempDir::new("image-assets"); + let out = staged(&dir); + assert_eq!(fs::read_to_string(out.join(LICENCES_ASSET)).unwrap(), "the notice"); let gz = fs::read(out.join(IMAGE_ASSET)).unwrap(); - let sum: String = Sha256::digest(&gz).iter().map(|b| format!("{b:02x}")).collect(); + let sum = sha256_hex(&gz); assert_eq!(fs::read_to_string(out.join(SUMS_ASSET)).unwrap(), format!("{sum} {IMAGE_ASSET}\n")); let mut back = Vec::new(); flate2::read::GzDecoder::new(&gz[..]).read_to_end(&mut back).unwrap(); - assert!(back == bytes, "the asset does not decompress to the image"); + assert!(back == fs::read(dir.join("in.img")).unwrap(), "the asset does not decompress to the image"); + } + + /// The digest the publish job checks moves with a byte of any asset, and a + /// directory holding more or fewer files than the assets is refused. + #[test] + fn the_digest_covers_every_asset_and_refuses_anything_else() { + let dir = toyos_tmpdir::TempDir::new("image-digest"); + let out = staged(&dir); + let whole = digest(&out).unwrap(); + for asset in ASSETS { + let at = out.join(asset); + let bytes = fs::read(&at).unwrap(); + let mut changed = bytes.clone(); + changed[0] ^= 1; + fs::write(&at, &changed).unwrap(); + assert_ne!(digest(&out).unwrap(), whole, "{asset} is outside the digest"); + fs::write(&at, &bytes).unwrap(); + } + assert_eq!(digest(&out).unwrap(), whole); + fs::write(out.join("extra"), "").unwrap(); + assert!(digest(&out).unwrap_err().contains("extra")); + fs::remove_file(out.join("extra")).unwrap(); + fs::remove_file(out.join(NOTES_ASSET)).unwrap(); + assert!(digest(&out).is_err(), "a directory without the notes was digested"); } } diff --git a/src/licence.rs b/src/licence.rs index 4b1b348ad3..63c5d84373 100644 --- a/src/licence.rs +++ b/src/licence.rs @@ -1232,50 +1232,80 @@ fn ls_files(root: &Path, pathspecs: &[String]) -> Result, String> { .collect()) } -/// The fork's `library/`, checked out at the commit this tree pins. A checkout -/// whose `rust/` was never initialised — a CI runner's — fetches that commit -/// alone. One whose toolchain was installed fetches it beside `rust/` instead: -/// a source tree there makes the toolchain the checkout's own to build -/// (`toolchain::owner`). -fn std_library(root: &Path) -> Result { - if matches!(crate::toolchain::owner(root), crate::toolchain::Owner::Installed) { - return fetched_library(root); - } - let fork = crate::sysroot::fork_checkout(root); - if !fork.join("library/Cargo.toml").exists() { - run( - Command::new("git") - .args(["submodule", "update", "--init", "--depth", "1", "rust"]) - .current_dir(root), - "git submodule update --init --depth 1 rust", - )?; - } - Ok(fork.join("library")) +/// The fork's `library/` at the commit this tree pins: a linked worktree's +/// fork checkout, `rust/library` where this checkout holds it, and everywhere +/// else — a runner, an installed toolchain — [`fetched_library`]. Asks the +/// network, so it is called with no build lock held. +pub fn std_library(root: &Path) -> Result { + if let crate::toolchain::Owner::Elsewhere(_) = crate::toolchain::owner(root) { + return Ok(crate::sysroot::fork_checkout(root).join("library")); + } + let library = root.join("rust/library"); + if library.join("Cargo.toml").is_file() { + return Ok(library); + } + fetched_library(root) } -/// Where the fork is fetched to where `rust/` may hold no source. -const FETCHED_FORK: &str = "target/licence/fork"; +/// Where the fork is fetched to where `rust/` holds no source. A directory of +/// the checkout's own, because std's manifest names `toyos` and `toyos-abi` +/// three levels above its crates; hidden and ignored, as `.build-locks/` is. +const FETCHED_FORK: &str = ".licence-fork"; + +/// Where a fetch is made before it is renamed into place, and where the fork +/// it replaces goes before it is removed: on the checkout's own filesystem. +const FETCHING: &str = "target/licence/fork.partial"; +const REPLACED: &str = "target/licence/fork.replaced"; + +/// What of the fork is checked out: its `library/` and its licence texts. +const SPARSE: [&str; 4] = ["/library/", "/COPYRIGHT", "/LICENSE-*", "/LICENSES/"]; /// The fork's `library/` and its own licence files at the pinned commit, and -/// nothing else of it. +/// nothing else of it. Fetched beside its place and renamed into it, so what +/// is there is always a whole fetch; a source tree at `rust/` would make the +/// toolchain the checkout's own to build (`toolchain::owner`). fn fetched_library(root: &Path) -> Result { - let fork = root.join(FETCHED_FORK); let commit = crate::sysroot::pinned_fork(root); - let git = |args: &[&str]| -> Result, String> { - run(Command::new("git").args(args).current_dir(&fork), &format!("git {}", args.join(" "))) + let fork = root.join(FETCHED_FORK); + let git = |dir: &Path, args: &[&str]| -> Result, String> { + run(Command::new("git").args(args).current_dir(dir), &format!("git {}", args.join(" "))) + }; + let sparse: String = SPARSE.iter().map(|p| format!("{p}\n")).collect(); + let pinned = || -> Result { + Ok(fork.is_dir() + && git(&fork, &["rev-parse", "HEAD"])? == format!("{commit}\n").into_bytes() + && git(&fork, &["sparse-checkout", "list"])? == sparse.as_bytes()) }; - if fork.join(".git").exists() && git(&["rev-parse", "HEAD"])? == format!("{commit}\n").into_bytes() { + if pinned()? { + return Ok(fork.join("library")); + } + let _fetching = crate::buildlock::fork_fetch(root); + if pinned()? { return Ok(fork.join("library")); } let url = fork_url(root)?; + let (partial, replaced) = (root.join(FETCHING), root.join(REPLACED)); + // Under the lock, either is only ever what a call cut short left. + for dir in [&partial, &replaced] { + if dir.exists() { + std::fs::remove_dir_all(dir).map_err(|e| format!("remove {}: {e}", dir.display()))?; + } + } + std::fs::create_dir_all(&partial).map_err(|e| format!("create {}: {e}", partial.display()))?; + git(&partial, &["init", "-q"])?; + git(&partial, &["fetch", "-q", "--depth", "1", "--filter=blob:none", &url, &commit])?; + git(&partial, &[&["sparse-checkout", "set", "--no-cone"][..], &SPARSE].concat())?; + git(&partial, &["checkout", "-q", "FETCH_HEAD"])?; + let rename = |from: &Path, to: &Path| { + std::fs::rename(from, to).map_err(|e| format!("rename {} to {}: {e}", from.display(), to.display())) + }; if fork.exists() { - std::fs::remove_dir_all(&fork).map_err(|e| format!("remove {}: {e}", fork.display()))?; + rename(&fork, &replaced)?; + } + rename(&partial, &fork)?; + if replaced.exists() { + std::fs::remove_dir_all(&replaced).map_err(|e| format!("remove {}: {e}", replaced.display()))?; } - std::fs::create_dir_all(&fork).map_err(|e| format!("create {}: {e}", fork.display()))?; - git(&["init", "-q"])?; - git(&["fetch", "-q", "--depth", "1", "--filter=blob:none", &url, &commit])?; - git(&["sparse-checkout", "set", "--no-cone", "/library/", "/COPYRIGHT", "/LICENSE-*", "/LICENSES/"])?; - git(&["checkout", "-q", "FETCH_HEAD"])?; Ok(fork.join("library")) } @@ -1310,10 +1340,15 @@ struct Walk { files: BTreeMap>, } -/// Walk every crate `shipped` names, libc and std, judging each package into -/// `report`, and read where committed files ship from. `root` is canonical: -/// cargo names every manifest by its canonical path. -fn walk(root: &Path, shipped: crate::build::Shipped, report: &mut Report) -> Result { +/// Walk every crate `shipped` names, libc, and std out of `library`, judging +/// each package into `report`, and read where committed files ship from. +/// `root` is canonical: cargo names every manifest by its canonical path. +fn walk( + root: &Path, + shipped: crate::build::Shipped, + library: &Path, + report: &mut Report, +) -> Result { let mut roots: Vec<(PathBuf, Features)> = shipped.crates.into_iter().collect(); roots.push(( root.join(crate::libc::CRATE), @@ -1345,7 +1380,7 @@ fn walk(root: &Path, shipped: crate::build::Shipped, report: &mut Report) -> Res // committed lock is stale by design: it is re-locked into a scratch copy, // and the fork's is never written. `RUSTC_BOOTSTRAP` because the fork's // manifests use cargo features a stable cargo otherwise refuses. - let library = std_library(root)?; + let library = canonical(library)?; let scratch = root.join("target/licence"); std::fs::create_dir_all(&scratch).map_err(|e| format!("create {}: {e}", scratch.display()))?; let lock = scratch.join("Cargo.lock"); @@ -1412,7 +1447,7 @@ fn canonical(root: &Path) -> Result { pub fn judge(root: &Path) -> Result { let root = &canonical(root)?; let mut report = Report::default(); - let walk = walk(root, crate::build::shipped(root)?, &mut report)?; + let walk = walk(root, crate::build::shipped(root)?, &std_library(root)?, &mut report)?; judge_files(COMMITTED_FILES, &walk.tracked, &walk.shipping, &mut report); judge_notice(&walk.sections, &walk.files, COMMITTED_FILES, &walk.shipping, &mut report); verdict(report, EXCEPTIONS) @@ -1466,8 +1501,8 @@ fn licence_files(p: &Reached) -> Result, String> { /// The standard texts of the licences `p` declares, from the fork's /// `LICENSES/`, which holds one `.txt` per licence: each of an `AND`, -/// an exception with its licence, and of an `OR` the first branch held whole. -/// `None` where no branch is. +/// an exception with its licence, and of an `OR` its Apache-2.0 branch where +/// it has one, else the first branch held whole. `None` where no branch is. fn standard_texts(p: &Reached, fork: &Path) -> Option> { fn held(expr: &Expr, dir: &Path) -> Option> { let text = |id: &str| Some(dir.join(format!("{id}.txt"))).filter(|f| f.is_file()); @@ -1475,7 +1510,12 @@ fn standard_texts(p: &Reached, fork: &Path) -> Option> { Expr::Id(id) => Some(vec![text(id)?]), Expr::With(id, exception) => Some(vec![text(id)?, text(exception)?]), Expr::And(parts) => Some(parts.iter().map(|p| held(p, dir)).collect::>>()?.concat()), - Expr::Or(parts) => parts.iter().find_map(|p| held(p, dir)), + // Apache-2.0's text is whole as it stands; MIT's asks for a + // copyright line that a package publishing no text never gave. + Expr::Or(parts) => { + let apache = parts.iter().filter(|p| matches!(p, Expr::Id(id) if id == "Apache-2.0")); + apache.chain(parts).find_map(|p| held(p, dir)) + } } } held(&parse(p.licence.as_deref()?).ok()?, &fork.join("LICENSES")) @@ -1506,14 +1546,15 @@ impl Texts { } } -/// The licence notice of an image built from `shipped`: every package the -/// walk reaches with its licence, where its source is and the texts it -/// carries; every `NOTICE` section over a file it ships, with its terms and -/// texts, less what [`pending_owner`] names; and each text once. Refused while -/// any package carries no licence text. -pub fn notices(root: &Path, shipped: crate::build::Shipped) -> Result { +/// The licence notice of an image built from `shipped` with std out of +/// `library` ([`std_library`]): every package the walk reaches with its +/// licence, where its source is and the texts it carries; every `NOTICE` +/// section over a file it ships, with its terms and texts, less what +/// [`pending_owner`] names; and each text once. Refused while any package +/// carries no licence text. +pub fn notices(root: &Path, shipped: crate::build::Shipped, library: &Path) -> Result { let root = &canonical(root)?; - let walk = walk(root, shipped, &mut Report::default())?; + let walk = walk(root, shipped, library, &mut Report::default())?; let fork = walk.library.parent().ok_or("std's library/ is in no directory")?; let (url, commit) = (fork_url(root)?, crate::sysroot::pinned_fork(root)); let mut texts = Texts::default(); @@ -1521,9 +1562,10 @@ pub fn notices(root: &Path, shipped: crate::build::Shipped) -> Result = Arch::ALL.iter().flat_map(|a| [a.userland(), a.kernel(), a.loader()]).collect(); @@ -2284,15 +2326,111 @@ prose. assert!(why.contains("gone.txt"), "{why}"); } - /// The release's notice: every package its walk reaches carries a licence - /// text, the loader's MPL crates name where their source is, std carries - /// the fork's texts, and every third-party file the release ships is there - /// while nothing pending the owner is. + /// `git ` in `dir`, which has to succeed: what it printed, trimmed. + fn git_in(dir: &Path, args: &[&str]) -> String { + let out = Command::new("git") + .args(["-c", "user.name=t", "-c", "user.email=t@t", "-c", "init.defaultBranch=main", "-c", "commit.gpgsign=false"]) + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!(out.status.success(), "git {args:?}: {}", String::from_utf8_lossy(&out.stderr)); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + + /// `root` pins the fork at `url` at `commit`, as `.gitmodules` and the + /// index say it. + fn pinning(root: &Path, url: &str, commit: &str) { + let modules = format!("[submodule \"rust\"]\n\tpath = rust\n\turl = {url}\n"); + std::fs::write(root.join(".gitmodules"), modules).unwrap(); + git_in(root, &["update-index", "--add", "--cacheinfo", &format!("160000,{commit},rust")]); + } + + /// The fetched fork is a whole fetch or nothing: one that failed leaves + /// nothing a later call refuses on, only `library/` and the licence texts + /// are checked out, a whole fetch is reused without the network, and a new + /// pin replaces it. + #[test] + fn a_fetch_cut_short_is_fetched_again_and_a_new_pin_replaces_the_fork() { + let tmp = toyos_tmpdir::TempDir::new("fetched-fork"); + let remote = tmp.join("fork"); + for dir in ["library", "LICENSES", "src"] { + std::fs::create_dir_all(remote.join(dir)).unwrap(); + } + let files = [ + ("library/Cargo.toml", "[workspace]\n"), + ("COPYRIGHT", "one"), + ("LICENSE-MIT", "mit"), + ("LICENSES/MIT.txt", "mit"), + ("x.py", ""), + ("src/lib.rs", ""), + ]; + for (file, text) in files { + std::fs::write(remote.join(file), text).unwrap(); + } + git_in(&remote, &["init", "-q"]); + git_in(&remote, &["config", "uploadpack.allowAnySHA1InWant", "true"]); + git_in(&remote, &["config", "uploadpack.allowFilter", "true"]); + git_in(&remote, &["add", "-A"]); + git_in(&remote, &["commit", "-q", "-m", "one"]); + let first = git_in(&remote, &["rev-parse", "HEAD"]); + let root = tmp.join("toyos"); + std::fs::create_dir(&root).unwrap(); + git_in(&root, &["init", "-q"]); + let url = format!("file://{}", remote.display()); + let gone = format!("file://{}", tmp.join("gone").display()); + + pinning(&root, &gone, &first); + assert!(fetched_library(&root).is_err(), "a fork nobody serves was fetched"); + pinning(&root, &url, &first); + let library = fetched_library(&root).unwrap_or_else(|why| panic!("{why}")); + let fork = library.parent().unwrap(); + for file in ["library/Cargo.toml", "COPYRIGHT", "LICENSE-MIT", "LICENSES/MIT.txt"] { + assert!(fork.join(file).is_file(), "{file} was not checked out"); + } + assert!(!fork.join("x.py").exists() && !fork.join("src").exists(), "more than the licence sources was checked out"); + + pinning(&root, &gone, &first); + assert_eq!(fetched_library(&root).unwrap_or_else(|why| panic!("{why}")), library, "a whole fetch was not reused"); + + std::fs::write(remote.join("COPYRIGHT"), "two").unwrap(); + git_in(&remote, &["commit", "-q", "-am", "two"]); + pinning(&root, &url, &git_in(&remote, &["rev-parse", "HEAD"])); + let library = fetched_library(&root).unwrap_or_else(|why| panic!("{why}")); + assert_eq!(std::fs::read_to_string(library.parent().unwrap().join("COPYRIGHT")).unwrap(), "two"); + } + + /// A package that publishes no text is given Apache-2.0's where its `OR` + /// offers it, wherever it stands, and otherwise the first branch the fork + /// holds. + #[test] + fn an_or_is_given_apache_2_0s_standard_text_where_it_is_a_branch() { + let fork = toyos_tmpdir::TempDir::new("standard-texts"); + std::fs::create_dir(fork.join("LICENSES")).unwrap(); + for id in ["MIT", "Apache-2.0", "BSD-3-Clause"] { + std::fs::write(fork.join(format!("LICENSES/{id}.txt")), id).unwrap(); + } + let given = |licence: &str| -> Vec { + let p = Reached { licence: Some(licence.into()), ..reached(&fork, None, None) }; + let texts = standard_texts(&p, &fork).unwrap_or_else(|| panic!("{licence}: no text")); + texts.iter().map(|t| file_name(t.to_str().unwrap()).to_string()).collect() + }; + assert_eq!(given("MIT OR Apache-2.0"), ["Apache-2.0.txt"]); + assert_eq!(given("MIT/Apache-2.0"), ["Apache-2.0.txt"]); + assert_eq!(given("Zlib OR MIT OR BSD-3-Clause"), ["MIT.txt"]); + } + + /// The release's notice, with std fetched the way the release job fetches + /// it: every package its walk reaches carries a licence text, the loader's + /// MPL crates name where their source is, std carries the fork's texts, and + /// every third-party file the release ships is there while nothing pending + /// the owner is. #[test] fn the_release_notice_carries_every_package_and_file_it_ships() { let root = Path::new(env!("CARGO_MANIFEST_DIR")); let parts = crate::build::Boot::release(root).parts(root).unwrap(); - let text = notices(root, parts).unwrap_or_else(|why| panic!("{why}")); + let library = fetched_library(root).unwrap_or_else(|why| panic!("{why}")); + let text = notices(root, parts, &library).unwrap_or_else(|why| panic!("{why}")); let block = |head: &str| -> &str { let at = text.find(&format!("\n{head}")).unwrap_or_else(|| panic!("no {head:?} in the notice")); text[at + 1..].split("\n\n").next().unwrap() diff --git a/src/sourcegate.rs b/src/sourcegate.rs index 2a9731080f..98586222f5 100644 --- a/src/sourcegate.rs +++ b/src/sourcegate.rs @@ -725,11 +725,6 @@ const CI_PACKAGES: &[Package] = &[ why: "outside the bar, and declared by nothing else: it fetches rustup-init.sh, and \ src/release.rs and src/ci.rs ask GitHub and the crates.io index with it", }, - Package { - name: "gh", - why: "GitHub's CLI, which the image release publishes with (src/imagerelease.rs), in the \ - one container job that publishes", - }, Package { name: "git", why: "the version control this repository is, and `REQUIRED` in src/main.rs", @@ -815,7 +810,13 @@ const CI_ACTIONS: &[Action] = &[ }, Action { name: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02", - why: "how a red guest job keeps its boots' serial logs (v4.6.2)", + why: "how a red guest job keeps its boots' serial logs, and how the image release's \ + boot job hands its assets to the job that publishes them (v4.6.2)", + }, + Action { + name: "actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093", + why: "the read half of the same store: the image release's publish job takes the \ + assets the boot job staged (v4.3.0)", }, Action { name: "rust-lang/crates-io-auth-action@c6f97d42243bad5fab37ca0427f495c86d5b1a18", diff --git a/src/tiers.rs b/src/tiers.rs index f0c57216b9..1231010b63 100644 --- a/src/tiers.rs +++ b/src/tiers.rs @@ -12,8 +12,10 @@ //! time, or because it shares a boot with one that is slow, stays where it is //! whatever it measures. //! -//! The local tier is the third, and the only one CI never runs: its guests are -//! of an architecture no hosted runner has been measured to boot. +//! The local tier is the third, and CI never runs it: its guests are of an +//! architecture no hosted runner has been measured to boot. The Apple Silicon +//! tier is the fourth, and CI never runs it either: its tests boot a command +//! line only an Apple Silicon Mac has, so no other host runs them. /// Which run a registered test belongs to. #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -27,6 +29,8 @@ pub enum Tier { /// (`issues/kernel/toyos-runs-on-arm64.md`) measures the runners and /// moves these rows to `Fast` or `Nightly`. Local, + /// Every unsharded `cargo test` on an Apple Silicon Mac. + AppleSilicon, } impl Tier { @@ -37,6 +41,9 @@ impl Tier { Self::Fast => true, Self::Nightly => nightly, Self::Local => !sharded, + Self::AppleSilicon => { + !sharded && cfg!(target_os = "macos") && crate::arch::Arch::HOST == Some(crate::arch::Arch::Aarch64) + } } } } diff --git a/tests/common/release.rs b/tests/common/release.rs index 73dfdb6da6..a4d6147c25 100644 --- a/tests/common/release.rs +++ b/tests/common/release.rs @@ -1,13 +1,14 @@ -//! The image release's command line for this host, booted as its notes print -//! it (`toyos_build::imagerelease::boots`) over a copy of the release image. +//! The image release's macOS command line, booted as its notes print it +//! (`toyos_build::imagerelease::boots`) over a copy of the release image. use toyos_build::build::{self, Boot}; use toyos_build::imagerelease::{self, Host}; -/// The notes' line for this host boots the release image to a painting -/// desktop and leaves both firmware files as they were. +/// The notes' Apple Silicon line boots the release image to a painting +/// desktop and leaves both firmware files as they were. Registered in the +/// Apple Silicon tier, so no other host runs it. pub fn release_command_boots() -> Result<(), String> { - let host = Host::this()?; + let host = Host::MacosAppleSilicon; let root = super::compile::repo_root(); let boot = Boot::release(&root); let plan = build::plan_for(&root, &boot, false, &[]); diff --git a/tests/common/storage.rs b/tests/common/storage.rs index 0a2faf08c8..4f097a94cf 100644 --- a/tests/common/storage.rs +++ b/tests/common/storage.rs @@ -19,9 +19,11 @@ use super::qemu::{self, BootOptions, QemuInstance}; /// Boot the guest against three disks that belong to somebody else, and prove /// each comes back untouched: an NVMe disk whose TOYOS-DATA partition holds /// another system's volume, a USB disk whose table names only other systems' -/// partitions, and a USB stick with no table. The two USB disks ride USB -/// because the kernel drives one NVMe controller, and that one carries the -/// first. +/// partitions, and a USB stick with no table. +/// +/// Two boots, one per USB disk, each beside the NVMe disk: the boot stick +/// takes one of the USB driver's two disks, so a machine carries one more +/// (`Profile::UsbDiskCrowd`'s third is never served). /// /// Lives here so the registration hunk in `toyos.rs` stays one line: every /// agent edits that file. @@ -41,24 +43,48 @@ pub fn foreign_disk_untouched( let parts = other_systems_disk(&other, USB_BYTES)?; let bare = dir.join("bare-stick.img"); filled(&bare, USB_BYTES, 0x5A)?; - let disks = [image, other, bare]; - let before: Vec> = disks.iter().map(|disk| whole_device(disk)).collect(); // The premise, checked before the boot rather than assumed: if this volume // somehow already parsed as a ToyOS volume, the kernel would mount it and // the assertion below would pass for the wrong reason. - if front(&disks[0], data_at, 4) == *b"BCFS" { + if front(&image, data_at, 4) == *b"BCFS" { return Err("the foreign volume starts with a bcachefs superblock".to_string()); } + let sticks = [ + (&other, format!("has {parts} partitions and none of them is ours")), + (&bare, "has no partition table we can use".to_string()), + ]; + for (stick, read) in sticks { + untouched_beside(test_config, c_bins, rust_bins, &image, stick, &read)?; + } + for disk in [&image, &other, &bare] { + let _ = std::fs::remove_file(disk); + } + Ok(()) +} + +/// One boot with `nvme` and `stick` beside the boot stick, each compared byte +/// for byte after QEMU has exited. `read` is what the kernel says reading +/// `stick`'s table, so the comparison is about a disk it saw. +fn untouched_beside( + test_config: &Path, + c_bins: &[(String, Vec)], + rust_bins: &[(String, Vec)], + nvme: &Path, + stick: &Path, + read: &str, +) -> Result<(), String> { + let disks = [nvme, stick]; + let before: Vec> = disks.iter().map(|disk| whole_device(disk)).collect(); let mut qemu = QemuInstance::boot_with_options( test_config, c_bins, rust_bins, BootOptions { - profile: qemu::Profile::UsbDiskCrowd, - nvme_image: Some(disks[0].clone()), - usb_images: disks[1..].to_vec(), + profile: qemu::Profile::UsbDisk, + nvme_image: Some(nvme.to_path_buf()), + usb_images: vec![stick.to_path_buf()], ..Default::default() }, ); @@ -103,13 +129,9 @@ pub fn foreign_disk_untouched( } } drop(qemu); - // The USB disks were read, so the comparison below is about disks the - // kernel saw. let said = format!("{log}{tail}"); - for read in [format!("has {parts} partitions and none of them is ours"), "has no partition table we can use".into()] { - if !said.contains(&read) { - return Err(format!("the kernel never said {read:?}, so it never read that disk\n{said}")); - } + if !said.contains(read) { + return Err(format!("the kernel never said {read:?}, so it never read {}\n{said}", stick.display())); } for (disk, before) in disks.iter().zip(&before) { @@ -117,9 +139,6 @@ pub fn foreign_disk_untouched( return Err(format!("the kernel wrote to {}, a disk it was not given: {diff}", disk.display())); } } - for disk in &disks { - let _ = std::fs::remove_file(disk); - } Ok(()) } diff --git a/tests/toyos.rs b/tests/toyos.rs index 57ca64c9f2..e0988c25b6 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -983,9 +983,10 @@ const MACHINE_TESTS: &[(&str, Sched, Tier)] = &[ ("volume_from_another_disk", Sched::Parallel, Tier::Fast), ("broken_data_volume_is_absent", Sched::Parallel, Tier::Fast), ("data_candidate_with_bad_geometry_is_absent", Sched::Parallel, Tier::Fast), - // The image release's command line booted as its notes print it: console - // lines and firmware bytes, and the ceiling is a liveness guard. - ("release_command_boots", Sched::Parallel, Tier::Fast), + // The image release's macOS command line booted as its notes print it: + // console lines and firmware bytes, and the ceiling is a liveness guard. + // The Linux line is the nightly `release` job's own boot. + ("release_command_boots", Sched::Parallel, Tier::AppleSilicon), // Four kernel lines and a file read off the image once the guest is gone; no clock in any of them. ("internal_disk_boot", Sched::Parallel, Tier::Fast), // One boot each, kernel lines and image bytes for verdicts, no clock in either. @@ -20613,6 +20614,15 @@ fn main() { }; let held_back = held(Tier::Nightly); let held_local = held(Tier::Local); + let held_apple = held(Tier::AppleSilicon); + if !held_apple.is_empty() { + eprintln!( + "[toyos] Apple Silicon tier: {} test(s) NOT run, because they boot a command line \ + only an Apple Silicon Mac has and this run is sharded or on another host.", + held_apple.len(), + ); + eprintln!("[toyos] {}", held_apple.join(", ")); + } if !held_local.is_empty() { eprintln!( "[toyos] local tier: {} test(s) NOT run, because a sharded run is CI's and no CI \ From 0845777f83af5733d0763499ef3a3dd277b59cd7 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 28 Sep 2026 00:47:29 +0200 Subject: [PATCH 7/9] licence: the release notice test asks that std cite the COPYRIGHT itself A copy of the Rust project's COPYRIGHT carried by another package satisfied the old check, so a fetch that dropped the fork's own left it green. Co-Authored-By: Claude Opus 5.5 --- src/licence.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/licence.rs b/src/licence.rs index 63c5d84373..bf27f91116 100644 --- a/src/licence.rs +++ b/src/licence.rs @@ -2440,7 +2440,15 @@ prose. assert!(uefi.contains("source: https://static.crates.io/crates/uefi/uefi-"), "{uefi}"); let std = block("std "); assert!(std.contains("source: https://github.com/ToyOSOrg/rust.git at "), "{std}"); - assert!(text.contains(": COPYRIGHT\n\nShort version for non-lawyers:"), "std carries no fork COPYRIGHT"); + // std cites the Rust project's COPYRIGHT itself, not merely a notice + // where some other package carries a copy of it. + let lines: Vec<&str> = text.lines().collect(); + let copyright: Vec<&str> = lines + .windows(3) + .filter(|w| w[0].ends_with(": COPYRIGHT") && w[2] == "Short version for non-lawyers:") + .filter_map(|w| w[0].split_once(']').map(|(id, _)| id)) + .collect(); + assert!(copyright.iter().any(|id| std.contains(&format!("{id}]"))), "std cites no COPYRIGHT: {std}"); for shipped in ["assets/JetBrainsMono-Regular.ttf", "assets/icons/*.svg", "assets/fonts/OpenSans-*.ttf"] { assert!(block(&format!("{shipped} — ")).contains("texts: ["), "{shipped}"); } From d3a26272b5f5899fbcc14dbc7cbb4870da0d23d1 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 28 Sep 2026 00:53:31 +0200 Subject: [PATCH 8/9] imagerelease: the staging names stay private; ovmf's reason names who boots it Co-Authored-By: Claude Opus 5.5 --- src/imagerelease.rs | 8 ++++---- src/sourcegate.rs | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/imagerelease.rs b/src/imagerelease.rs index 8405d9c77a..830e610a90 100644 --- a/src/imagerelease.rs +++ b/src/imagerelease.rs @@ -54,14 +54,14 @@ pub const NOTES_ASSET: &str = "README.md"; pub const LICENCES_ASSET: &str = "licences.txt"; /// Every asset, in upload order. -pub const ASSETS: [&str; 4] = [IMAGE_ASSET, SUMS_ASSET, NOTES_ASSET, LICENCES_ASSET]; +const ASSETS: [&str; 4] = [IMAGE_ASSET, SUMS_ASSET, NOTES_ASSET, LICENCES_ASSET]; /// Where [`release`] stages the assets and [`publish`] reads them: the /// directory the nightly's artifact carries between the two jobs. -pub const STAGED: &str = "target/image-release"; +const STAGED: &str = "target/image-release"; /// What the publish job is handed the release job's [`digest`] in. -pub const DIGEST_VAR: &str = "IMAGE_RELEASE_DIGEST"; +const DIGEST_VAR: &str = "IMAGE_RELEASE_DIGEST"; /// The defect behind the one disk the notes say a boot may write though it /// was not given it. @@ -401,7 +401,7 @@ pub fn write_assets( /// The SHA-256 of every asset's own, one `sha256sum` line each in upload /// order: what the release job answers and the publish job recomputes. /// Refused while `dir` holds anything but the four assets. -pub fn digest(dir: &Path) -> Result { +fn digest(dir: &Path) -> Result { let entries = fs::read_dir(dir).map_err(|e| format!("{}: {e}", dir.display()))?; let mut held = entries .map(|e| e.map(|e| e.file_name().to_string_lossy().into_owned())) diff --git a/src/sourcegate.rs b/src/sourcegate.rs index 98586222f5..d8e981b54e 100644 --- a/src/sourcegate.rs +++ b/src/sourcegate.rs @@ -732,8 +732,8 @@ const CI_PACKAGES: &[Package] = &[ Package { name: "ovmf", why: "the edk2 firmware Debian's QEMU boots a UEFI guest with: the image release notes' \ - Linux command line names it, and the release job and `release_command_boots` \ - boot that line (src/imagerelease.rs)", + Linux command line names it, and the release job boots that line \ + (src/imagerelease.rs)", }, Package { name: "python3", From d30dfff3b7ea797eafb3ba9ab7f61703554c6ced Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 28 Sep 2026 08:31:09 +0200 Subject: [PATCH 9/9] Image release: publish only on a green night; the Mac line in portability-macos; the writing-token gate reads write-all and persisted credentials - release-publish needs build, guest and tcg besides release. - Tier::AppleSilicon is gone. release_command_boots is in no tier: the suite's --release-command runs it and nothing else, and the nightly's portability-macos runs that on its Apple Silicon runner after the build, with the QEMU Homebrew already installed there. - The writing-token gate reads a job's permissions inline or nested, from any line of the job, and requires persist-credentials: false on every checkout of a job whose token writes. build, nightly-red and publish.yml's publish take it. - The notice test takes std from std_library, so a host test fetches nothing where the checkout holds the fork. - The fork's sparse set is derived from LICENCE_FILES, in any case. - The licence lock (was the fork-fetch lock) also covers the walk's scratch Cargo.lock and the release notice's write. - The re-read of the draft after `gh release create` is deleted. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6rME2DoqwjcYFStYHHY4j --- .github/workflows/nightly.yml | 17 ++-- .github/workflows/publish.yml | 2 + ...at-boot-on-the-edk2-firmware-qemu-ships.md | 5 +- src/build.rs | 3 +- src/buildlock.rs | 11 +-- src/ci.rs | 79 +++++++++++++++---- src/imagerelease.rs | 4 - src/licence.rs | 47 +++++++---- src/testargs.rs | 19 +++++ src/tiers.rs | 11 +-- tests/common/mod.rs | 2 +- tests/common/release.rs | 7 +- tests/common/storage.rs | 3 +- tests/toyos.rs | 32 ++++---- 14 files changed, 166 insertions(+), 76 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 68292a9040..a1928be9f3 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -62,6 +62,7 @@ jobs: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 0 + persist-credentials: false - name: disk and QEMU run: | @@ -238,6 +239,9 @@ jobs: sh "$RUNNER_TEMP/rustup-init.sh" -y --profile minimal --default-toolchain stable echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" - run: env -u GITHUB_ACTIONS -u CI cargo run -- --build-only + # The image release notes' macOS line (`src/imagerelease.rs`), booted as + # they print it: this runner is the Apple Silicon Mac it names. + - run: env -u GITHUB_ACTIONS -u CI cargo test --test toyos-build -- --release-command # The declared Windows frontier # (`issues/build/the-build-system-does-not-compile-on-windows.md`): red @@ -280,11 +284,12 @@ jobs: retention-days: 1 # The one job that writes releases, publishing on main what `release` booted - # once its digest is the one `release` answered. No cache and nothing built - # but the build system, and the token only in the publishing step: the - # checkout keeps no credential, and the build step has none. + # on a night the guest suite passed, once its digest is the one `release` + # answered. No cache and nothing built but the build system, and the token + # only in the publishing step: the checkout keeps no credential, and the + # build step has none. release-publish: - needs: release + needs: [build, guest, tcg, release] runs-on: ubuntu-24.04 timeout-minutes: 30 permissions: @@ -313,7 +318,9 @@ jobs: contents: read issues: write steps: - - *checkout + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false - env: GH_TOKEN: ${{ github.token }} NEEDS: ${{ toJSON(needs) }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 1c2e8f9e06..db95fd2574 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -29,6 +29,8 @@ jobs: # No submodules: `cargo publish` walks the repository's vcs state, and an # initialised but empty `rust/` breaks that walk. - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false - id: auth uses: rust-lang/crates-io-auth-action@c6f97d42243bad5fab37ca0427f495c86d5b1a18 # v1.0.5 diff --git a/issues/kernel/the-kernel-dies-at-boot-on-the-edk2-firmware-qemu-ships.md b/issues/kernel/the-kernel-dies-at-boot-on-the-edk2-firmware-qemu-ships.md index 86db2c2079..9486c58b58 100644 --- a/issues/kernel/the-kernel-dies-at-boot-on-the-edk2-firmware-qemu-ships.md +++ b/issues/kernel/the-kernel-dies-at-boot-on-the-edk2-firmware-qemu-ships.md @@ -26,8 +26,9 @@ The command line is `imagerelease::Host::MacosAppleSilicon`'s (`src/imagerelease.rs`) with `-display none`, over a copy of a `target/bootable.img`. -`release_command_boots` boots that line on the dev host, and is disabled in -`src/redlist.rs` while this stands. +`release_command_boots` (`cargo test --test toyos-build -- --release-command`, +which the nightly's `portability-macos` runs) boots that line, and is disabled +in `src/redlist.rs` while this stands. **Exit condition.** `release_command_boots` is green on the dev host with the firmware Homebrew's QEMU ships, and its row leaves `src/redlist.rs`. diff --git a/src/build.rs b/src/build.rs index 1d3bf7b171..6ae239999f 100644 --- a/src/build.rs +++ b/src/build.rs @@ -1878,7 +1878,6 @@ fn shipped_parts(root: &Path, boot: &Boot, rebuild_toolchain: bool, plan: &Plan) let kernel_features = plan.features.join(","); let arch = plan.arch; - // The notice asks the network, so it is written before any lock is taken. let mut extra = Vec::new(); if boot.public { let notices = boot @@ -1886,7 +1885,9 @@ fn shipped_parts(root: &Path, boot: &Boot, rebuild_toolchain: bool, plan: &Plan) .and_then(|parts| crate::licence::notices(root, parts, &crate::licence::std_library(root)?)) .unwrap_or_else(|why| panic!("the release's licence notice: {why}")); let at = root.join(RELEASE_NOTICES); + let held = buildlock::licence(root); fs::write(&at, ¬ices).unwrap_or_else(|e| panic!("write {}: {e}", at.display())); + drop(held); extra.push((crate::licence::NOTICES_ON_ROOT.to_string(), notices.into_bytes())); } diff --git a/src/buildlock.rs b/src/buildlock.rs index 95297a5045..c850618bf0 100644 --- a/src/buildlock.rs +++ b/src/buildlock.rs @@ -217,11 +217,12 @@ pub fn artifact(root: &Path) -> Guard { exclusive(&root.join(LOCK_DIR).join("artifact"), "artifact lock", "artifact staging") } -/// This worktree's one fetch of the std fork's licence sources -/// (`licence::fetched_library`). It waits on the network, so it is taken with -/// no other lock held. -pub fn fork_fetch(root: &Path) -> Guard { - exclusive(&root.join(LOCK_DIR).join("fork-fetch"), "fork fetch lock", "fetching std's licence sources") +/// This worktree's licence files: the std fork's fetch +/// (`licence::fetched_library`), the walk's scratch `Cargo.lock` and the +/// release's notice. It waits on the network, so it is taken with no other lock +/// held. +pub fn licence(root: &Path) -> Guard { + exclusive(&root.join(LOCK_DIR).join("licence"), "licence lock", "writing the licence files") } /// The integration lock: one process at a time moves this host's `main`. diff --git a/src/ci.rs b/src/ci.rs index 3ed426ac09..24e3478c8e 100644 --- a/src/ci.rs +++ b/src/ci.rs @@ -884,6 +884,7 @@ fn at_tip(ls_remote: &str, head: &str) -> Result<(), String> { #[cfg(test)] mod tests { use super::*; + use std::collections::BTreeMap; use std::path::PathBuf; /// A deterministic control on `host`'s `std::env::set_var("TMPDIR", ...)`: @@ -1071,9 +1072,43 @@ mod tests { assert!(writers.iter().all(|(f, _)| f == "nightly.yml"), "{writers:?}"); } + /// The value of `key` at `indent` in `block`: the rest of its line and the + /// lines nested under it, comments left out. + fn yaml_value(block: &str, indent: &str, key: &str) -> Option { + let head = format!("{indent}{key}:"); + let nested = format!("{indent} "); + let mut lines = block + .lines() + .skip_while(|l| !l.strip_prefix(head.as_str()).is_some_and(|rest| rest.is_empty() || rest.starts_with(' '))); + let first = lines.next()?; + let rest = lines.take_while(|l| l.starts_with(nested.as_str()) || l.trim().is_empty()); + let value = std::iter::once(&first[head.len()..]) + .chain(rest) + .filter(|l| !l.trim_start().starts_with('#')) + .map(|l| l.split(" #").next().unwrap_or(l)) + .collect::>() + .join("\n"); + Some(value) + } + + /// A job's steps, each its own lines. + fn job_steps(body: &str) -> Vec { + let mut steps: Vec = Vec::new(); + for line in yaml_value(body, " ", "steps").unwrap_or_default().lines() { + if line.starts_with(" - ") { + steps.push(String::new()); + } + if let Some(step) = steps.last_mut() { + step.push_str(&format!("{line}\n")); + } + } + steps + } + /// A job whose token writes restores no cache, since a cache is a tree a - /// job running third-party code wrote, and hands `GH_TOKEN` to a step - /// rather than to every step. + /// job running third-party code wrote; leaves no credential in the + /// checkout's `.git/config`, where every later step's code reads it; and + /// hands `GH_TOKEN` to a step rather than to every step. #[test] fn a_token_that_writes_meets_no_cache_and_only_the_step_that_needs_it() { let dir = repo_root().join(".github/workflows"); @@ -1082,13 +1117,6 @@ mod tests { let text = std::fs::read_to_string(entry.path()).expect("a readable workflow"); let file = entry.file_name().to_string_lossy().into_owned(); let (head, jobs) = text.split_once("\njobs:\n").expect("a workflow has jobs"); - let lines: Vec<&str> = text.lines().collect(); - let restoring: Vec = lines - .windows(2) - .filter(|w| w[1].contains("actions/cache/restore@")) - .filter_map(|w| w[0].trim_start().strip_prefix("- &")) - .map(|anchor| format!("*{anchor}")) - .collect(); let mut chunks: Vec<(String, String)> = Vec::new(); for line in jobs.lines() { let key = line.strip_prefix(" ").and_then(|l| l.strip_suffix(':')); @@ -1101,18 +1129,37 @@ mod tests { } } } + let mut anchors = BTreeMap::new(); + for (_, body) in &chunks { + for step in job_steps(body) { + let first = step.lines().next().unwrap_or_default().trim_start(); + if let Some(anchor) = first.strip_prefix("- &") { + anchors.insert(anchor.trim().to_string(), step.clone()); + } + } + } + let workflow = yaml_value(head, "", "permissions"); for (job, body) in &chunks { - let permissions = if body.contains("\n permissions:\n") { body.as_str() } else { head }; - if !permissions.contains(": write") { + let permissions = yaml_value(body, " ", "permissions").or_else(|| workflow.clone()); + if !permissions.is_some_and(|p| p.contains("write")) { continue; } writing += 1; - let restores = body.contains("actions/cache/restore@") || restoring.iter().any(|a| body.contains(a.as_str())); + // A `- *anchor` step is the step anchored `&anchor`. + let own = job_steps(body); + let steps: Vec<&String> = own + .iter() + .map(|step| match step.trim_start().strip_prefix("- *") { + Some(alias) => anchors.get(alias.trim()).unwrap_or_else(|| panic!("{file}: no step is anchored &{alias}")), + None => step, + }) + .collect(); + let restores = steps.iter().any(|s| s.contains("actions/cache/restore@")); assert!(!restores, "{file}: {job} restores a cache with a token that writes"); - let job_env = body.split("\n env:\n").nth(1).map(|env| { - env.lines().take_while(|l| l.starts_with(" ")).any(|l| l.contains("GH_TOKEN")) - }); - assert_ne!(job_env, Some(true), "{file}: {job} hands a token that writes to every step"); + let persists = steps.iter().any(|s| s.contains("actions/checkout@") && !s.contains("persist-credentials: false")); + assert!(!persists, "{file}: {job} keeps a token that writes in its checkout"); + let job_env = yaml_value(body, " ", "env").is_some_and(|env| env.contains("GH_TOKEN")); + assert!(!job_env, "{file}: {job} hands a token that writes to every step"); } } assert!(writing > 0, "no job's token writes, so this checked nothing"); diff --git a/src/imagerelease.rs b/src/imagerelease.rs index 830e610a90..cc2dbc4646 100644 --- a/src/imagerelease.rs +++ b/src/imagerelease.rs @@ -489,10 +489,6 @@ fn create(root: &Path, tag: &str, commit: &str, staged: &Path) -> Result<(), Str let assets: Vec = ASSETS.iter().map(|a| staged.join(a).display().to_string()).collect(); args.extend(assets.iter().map(String::as_str)); gh(root, &args)?; - let drafted = held(root, tag)?; - if drafted != Some(Held { draft: true, image: true }) { - return Err(format!("{tag} was created as a draft carrying {IMAGE_ASSET}, and GitHub holds {drafted:?}")); - } gh(root, &["release", "edit", tag, "--draft=false", "--latest"])?; Ok(()) } diff --git a/src/licence.rs b/src/licence.rs index e27ded65a9..49179f5cce 100644 --- a/src/licence.rs +++ b/src/licence.rs @@ -1241,8 +1241,14 @@ const FETCHED_FORK: &str = ".licence-fork"; const FETCHING: &str = "target/licence/fork.partial"; const REPLACED: &str = "target/licence/fork.replaced"; -/// What of the fork is checked out: its `library/` and its licence texts. -const SPARSE: [&str; 4] = ["/library/", "/COPYRIGHT", "/LICENSE-*", "/LICENSES/"]; +/// What of the fork is checked out: its `library/`, and every top-level entry +/// whose name [`LICENCE_FILES`] starts, in any case. +fn sparse() -> Vec { + let any_case = |name: &str| -> String { name.chars().map(|c| format!("[{c}{}]", c.to_ascii_uppercase())).collect() }; + std::iter::once("/library/".to_string()) + .chain(LICENCE_FILES.iter().map(|name| format!("/{}*", any_case(name)))) + .collect() +} /// The fork's `library/` and its own licence files at the pinned commit, and /// nothing else of it. Fetched beside its place and renamed into it, so what @@ -1254,7 +1260,8 @@ fn fetched_library(root: &Path) -> Result { let git = |dir: &Path, args: &[&str]| -> Result, String> { run(Command::new("git").args(args).current_dir(dir), &format!("git {}", args.join(" "))) }; - let sparse: String = SPARSE.iter().map(|p| format!("{p}\n")).collect(); + let patterns = sparse(); + let sparse: String = patterns.iter().map(|p| format!("{p}\n")).collect(); let pinned = || -> Result { Ok(fork.is_dir() && git(&fork, &["rev-parse", "HEAD"])? == format!("{commit}\n").into_bytes() @@ -1263,7 +1270,7 @@ fn fetched_library(root: &Path) -> Result { if pinned()? { return Ok(fork.join("library")); } - let _fetching = crate::buildlock::fork_fetch(root); + let _fetching = crate::buildlock::licence(root); if pinned()? { return Ok(fork.join("library")); } @@ -1278,7 +1285,9 @@ fn fetched_library(root: &Path) -> Result { std::fs::create_dir_all(&partial).map_err(|e| format!("create {}: {e}", partial.display()))?; git(&partial, &["init", "-q"])?; git(&partial, &["fetch", "-q", "--depth", "1", "--filter=blob:none", &url, &commit])?; - git(&partial, &[&["sparse-checkout", "set", "--no-cone"][..], &SPARSE].concat())?; + let mut set = vec!["sparse-checkout", "set", "--no-cone"]; + set.extend(patterns.iter().map(String::as_str)); + git(&partial, &set)?; git(&partial, &["checkout", "-q", "FETCH_HEAD"])?; let rename = |from: &Path, to: &Path| { std::fs::rename(from, to).map_err(|e| format!("rename {} to {}: {e}", from.display(), to.display())) @@ -1368,6 +1377,7 @@ fn walk( let scratch = root.join("target/licence"); std::fs::create_dir_all(&scratch).map_err(|e| format!("create {}: {e}", scratch.display()))?; let lock = scratch.join("Cargo.lock"); + let held = crate::buildlock::licence(root); std::fs::copy(library.join("Cargo.lock"), &lock) .map_err(|e| format!("copy std's Cargo.lock: {e}"))?; let lockfile = format!("resolver.lockfile-path={:?}", lock.display().to_string()); @@ -1379,6 +1389,7 @@ fn walk( &args, &[("RUSTC_BOOTSTRAP", "1")], )?; + drop(held); docs.push(Doc { members: members(&std_doc)?, metadata: std_doc, @@ -2331,9 +2342,10 @@ prose. } /// The fetched fork is a whole fetch or nothing: one that failed leaves - /// nothing a later call refuses on, only `library/` and the licence texts - /// are checked out, a whole fetch is reused without the network, and a new - /// pin replaces it. + /// nothing a later call refuses on, only `library/` and every top-level + /// file a name in [`LICENCE_FILES`] starts, in either case, are checked + /// out, a whole fetch is reused without the network, and a new pin + /// replaces it. #[test] fn a_fetch_cut_short_is_fetched_again_and_a_new_pin_replaces_the_fork() { let tmp = toyos_tmpdir::TempDir::new("fetched-fork"); @@ -2349,6 +2361,10 @@ prose. ("x.py", ""), ("src/lib.rs", ""), ]; + // Two per name, which a case-insensitive disk still holds apart. + let licences: Vec = + LICENCE_FILES.iter().flat_map(|name| [format!("{}-A", name.to_uppercase()), format!("{name}-b")]).collect(); + let files = files.into_iter().chain(licences.iter().map(|file| (file.as_str(), "text"))); for (file, text) in files { std::fs::write(remote.join(file), text).unwrap(); } @@ -2369,7 +2385,8 @@ prose. pinning(&root, &url, &first); let library = fetched_library(&root).unwrap_or_else(|why| panic!("{why}")); let fork = library.parent().unwrap(); - for file in ["library/Cargo.toml", "COPYRIGHT", "LICENSE-MIT", "LICENSES/MIT.txt"] { + let wanted = ["library/Cargo.toml", "COPYRIGHT", "LICENSE-MIT", "LICENSES/MIT.txt"]; + for file in wanted.into_iter().chain(licences.iter().map(String::as_str)) { assert!(fork.join(file).is_file(), "{file} was not checked out"); } assert!(!fork.join("x.py").exists() && !fork.join("src").exists(), "more than the licence sources was checked out"); @@ -2404,16 +2421,16 @@ prose. assert_eq!(given("Zlib OR MIT OR BSD-3-Clause"), ["MIT.txt"]); } - /// The release's notice, with std fetched the way the release job fetches - /// it: every package its walk reaches carries a licence text, the loader's - /// MPL crates name where their source is, std carries the fork's texts, and - /// every third-party file the release ships is there while nothing pending - /// the owner is. + /// The release's notice, with std where the build takes it from: every + /// package its walk reaches carries a licence text, the loader's MPL crates + /// name where their source is, std carries the fork's texts, and every + /// third-party file the release ships is there while nothing pending the + /// owner is. #[test] fn the_release_notice_carries_every_package_and_file_it_ships() { let root = Path::new(env!("CARGO_MANIFEST_DIR")); let parts = crate::build::Boot::release(root).parts(root).unwrap(); - let library = fetched_library(root).unwrap_or_else(|why| panic!("{why}")); + let library = std_library(root).unwrap_or_else(|why| panic!("{why}")); let text = notices(root, parts, &library).unwrap_or_else(|why| panic!("{why}")); let block = |head: &str| -> &str { let at = text.find(&format!("\n{head}")).unwrap_or_else(|| panic!("no {head:?} in the notice")); diff --git a/src/testargs.rs b/src/testargs.rs index 58d0fa8c51..bc69bddb43 100644 --- a/src/testargs.rs +++ b/src/testargs.rs @@ -162,6 +162,9 @@ declare_flags!(pub SUITE = { /// machine is not touched**: the run builds the images and writes down what /// to run on them, or judges readbacks a driver already left there. pub METAL_READBACK = "--metal-readback", Next; + /// The image release's macOS command line, booted as its notes print it: + /// one test outside every tier, and the only way to run it. + pub RELEASE_COMMAND = "--release-command", None; }); /// Validate the harness's argv and return the run's filter. @@ -197,6 +200,13 @@ pub fn parse(args: &[String]) -> Result, String> { } let has = |want| SUITE.present(args, want); + if has(&RELEASE_COMMAND) && args.len() > 1 { + return Err( + "--release-command runs one test in no tier and takes no other word, which it \ + would drop in silence" + .to_string(), + ); + } if has(&JOBS) && has(&JOBS_SHORT) { return Err( "--jobs and -j are two spellings of one width, and the run would read one of \ @@ -285,6 +295,15 @@ mod tests { ); } + #[test] + fn the_release_command_takes_no_other_word() { + assert_eq!(parse_owned(&["--release-command"]).unwrap(), None); + for other in ["release_command_boots", "--nightly", "--shard=1/2"] { + let refusal = parse_owned(&["--release-command", other]).unwrap_err(); + assert!(refusal.contains("takes no other word"), "{other}: {refusal}"); + } + } + #[test] fn two_filters_are_refused_because_only_one_would_run() { let refusal = parse_owned(&["futex", "dlopen"]).unwrap_err(); diff --git a/src/tiers.rs b/src/tiers.rs index 1231010b63..f0c57216b9 100644 --- a/src/tiers.rs +++ b/src/tiers.rs @@ -12,10 +12,8 @@ //! time, or because it shares a boot with one that is slow, stays where it is //! whatever it measures. //! -//! The local tier is the third, and CI never runs it: its guests are of an -//! architecture no hosted runner has been measured to boot. The Apple Silicon -//! tier is the fourth, and CI never runs it either: its tests boot a command -//! line only an Apple Silicon Mac has, so no other host runs them. +//! The local tier is the third, and the only one CI never runs: its guests are +//! of an architecture no hosted runner has been measured to boot. /// Which run a registered test belongs to. #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -29,8 +27,6 @@ pub enum Tier { /// (`issues/kernel/toyos-runs-on-arm64.md`) measures the runners and /// moves these rows to `Fast` or `Nightly`. Local, - /// Every unsharded `cargo test` on an Apple Silicon Mac. - AppleSilicon, } impl Tier { @@ -41,9 +37,6 @@ impl Tier { Self::Fast => true, Self::Nightly => nightly, Self::Local => !sharded, - Self::AppleSilicon => { - !sharded && cfg!(target_os = "macos") && crate::arch::Arch::HOST == Some(crate::arch::Arch::Aarch64) - } } } } diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 3304a1091d..c9dad9b097 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -43,7 +43,7 @@ pub mod metal; pub mod origin; #[allow(dead_code)] pub mod partclaim; -/// The image release's command line for this host. +/// The image release's macOS command line, which `--release-command` boots. pub mod release; #[allow(dead_code)] pub mod passcost; diff --git a/tests/common/release.rs b/tests/common/release.rs index a4d6147c25..0684161aa3 100644 --- a/tests/common/release.rs +++ b/tests/common/release.rs @@ -4,9 +4,12 @@ use toyos_build::build::{self, Boot}; use toyos_build::imagerelease::{self, Host}; +/// The name `src/redlist.rs` knows this test by. +pub const NAME: &str = "release_command_boots"; + /// The notes' Apple Silicon line boots the release image to a painting -/// desktop and leaves both firmware files as they were. Registered in the -/// Apple Silicon tier, so no other host runs it. +/// desktop and leaves both firmware files as they were. Run by the suite's +/// `--release-command` and by nothing else. pub fn release_command_boots() -> Result<(), String> { let host = Host::MacosAppleSilicon; let root = super::compile::repo_root(); diff --git a/tests/common/storage.rs b/tests/common/storage.rs index 4f097a94cf..99099e0943 100644 --- a/tests/common/storage.rs +++ b/tests/common/storage.rs @@ -22,8 +22,7 @@ use super::qemu::{self, BootOptions, QemuInstance}; /// partitions, and a USB stick with no table. /// /// Two boots, one per USB disk, each beside the NVMe disk: the boot stick -/// takes one of the USB driver's two disks, so a machine carries one more -/// (`Profile::UsbDiskCrowd`'s third is never served). +/// takes one of the USB driver's two disks, so a machine carries one more. /// /// Lives here so the registration hunk in `toyos.rs` stays one line: every /// agent edits that file. diff --git a/tests/toyos.rs b/tests/toyos.rs index 204428d171..fa7958e87d 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -942,10 +942,6 @@ const MACHINE_TESTS: &[(&str, Sched, Tier)] = &[ ("volume_from_another_disk", Sched::Parallel, Tier::Fast), ("broken_data_volume_is_absent", Sched::Parallel, Tier::Fast), ("data_candidate_with_bad_geometry_is_absent", Sched::Parallel, Tier::Fast), - // The image release's macOS command line booted as its notes print it: - // console lines and firmware bytes, and the ceiling is a liveness guard. - // The Linux line is the nightly `release` job's own boot. - ("release_command_boots", Sched::Parallel, Tier::AppleSilicon), // Four kernel lines and a file read off the image once the guest is gone; no clock in any of them. ("internal_disk_boot", Sched::Parallel, Tier::Fast), // One boot each, kernel lines and image bytes for verdicts, no clock in either. @@ -11293,7 +11289,6 @@ fn run_machine_test( // Body in `tests/common/storage.rs`, so the hunk in this shared file // stays one line. "foreign_disk_untouched" => storage::foreign_disk_untouched(test_config, c_bins, rust_bins), - "release_command_boots" => release::release_command_boots(), "internal_disk_boot" => storage::internal_disk_boot(test_config, c_bins, rust_bins), "partition_claim" => partclaim::partition_claim(test_config, c_bins, rust_bins), "partition_claim_gives_up" => { @@ -20021,6 +20016,7 @@ fn check_redlist(all_tests: &[TestDef]) -> Result<(), String> { .chain(AUDIO_TESTS.iter().map(|(name, _)| *name)) .chain(SCREEN_TESTS.iter().map(|(n, _, _)| *n)) .chain(MACHINE_TESTS.iter().map(|(n, _, _)| *n)) + .chain([release::NAME]) .collect(); redlist::check(redlist::DISABLED, |name| runnable.contains(name), &compile::repo_root()) } @@ -20103,6 +20099,23 @@ fn main() { common::qemu::VERBOSE.store(true, std::sync::atomic::Ordering::Relaxed); } + // **The image release's macOS line**, in no tier: it is an Apple Silicon + // Mac's, and the nightly's `portability-macos` is the run that has one. + if SUITE.present(&args, &testargs::RELEASE_COMMAND) { + let verdict = if keep(release::NAME) { + release::release_command_boots() + } else { + Err("disabled, so the macOS line was not booted".to_string()) + }; + match verdict { + Ok(()) => run.exit(0), + Err(why) => { + eprintln!("[toyos] {}: {why}", release::NAME); + run.exit(1) + } + } + } + // **The metal profile, before the C corpus.** It boots the T14 and not a // guest, so none of the C compile, the HTTPS judge's hosts or the tier // arithmetic below is any of its business; running it here is what keeps a @@ -20304,15 +20317,6 @@ fn main() { }; let held_back = held(Tier::Nightly); let held_local = held(Tier::Local); - let held_apple = held(Tier::AppleSilicon); - if !held_apple.is_empty() { - eprintln!( - "[toyos] Apple Silicon tier: {} test(s) NOT run, because they boot a command line \ - only an Apple Silicon Mac has and this run is sharded or on another host.", - held_apple.len(), - ); - eprintln!("[toyos] {}", held_apple.join(", ")); - } if !held_local.is_empty() { eprintln!( "[toyos] local tier: {} test(s) NOT run, because a sharded run is CI's and no CI \