diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index cba134ab63..a1928be9f3 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -62,6 +62,7 @@ jobs: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 0 + persist-credentials: false - name: disk and QEMU run: | @@ -105,7 +106,7 @@ jobs: for attempt in 1 2 3; do apt-get -o Acquire::Check-Valid-Until=false update -qq > /tmp/apt.log 2>&1 \ && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq git curl ca-certificates \ - zstd xz-utils build-essential qemu-system-x86 >> /tmp/apt.log 2>&1 \ + zstd xz-utils build-essential qemu-system-x86 ovmf >> /tmp/apt.log 2>&1 \ && break [ "$attempt" = 3 ] && { cat /tmp/apt.log; exit 1; } sleep 20 @@ -238,6 +239,9 @@ jobs: sh "$RUNNER_TEMP/rustup-init.sh" -y --profile minimal --default-toolchain stable echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" - run: env -u GITHUB_ACTIONS -u CI cargo run -- --build-only + # The image release notes' macOS line (`src/imagerelease.rs`), booted as + # they print it: this runner is the Apple Silicon Mac it names. + - run: env -u GITHUB_ACTIONS -u CI cargo test --test toyos-build -- --release-command # The declared Windows frontier # (`issues/build/the-build-system-does-not-compile-on-windows.md`): red @@ -250,17 +254,73 @@ jobs: - *checkout - run: cargo build -p toyos-build + # The disk image a person downloads, built once and booted under the Linux + # line its notes print (`src/imagerelease.rs`). It compiles and runs + # third-party code out of a cache another such job wrote, so its token only + # reads: the assets go on as an artifact and their digest as an output. + release: + needs: build + runs-on: ubuntu-24.04 + # A wedge guard, not a budget. + timeout-minutes: 60 + container: *kvm + permissions: + contents: read + env: + GH_TOKEN: ${{ github.token }} + outputs: + digest: ${{ steps.release.outputs.digest }} + steps: + - *deps + - *checkout + - *guest-cache + - id: release + run: cargo run -- --ci release + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: image-release + path: target/image-release/ + if-no-files-found: error + retention-days: 1 + + # The one job that writes releases, publishing on main what `release` booted + # on a night the guest suite passed, once its digest is the one `release` + # answered. No cache and nothing built but the build system, and the token + # only in the publishing step: the checkout keeps no credential, and the + # build step has none. + release-publish: + needs: [build, guest, tcg, release] + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: write + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: image-release + path: target/image-release + - run: cargo build -p toyos-build + - env: + GH_TOKEN: ${{ github.token }} + IMAGE_RELEASE_DIGEST: ${{ needs.release.outputs.digest }} + run: cargo run -- --ci release-publish + # One standing issue, found by title and commented on; a dispatch is somebody # watching the run, so only the schedule files. nightly-red: - needs: [host, build, guest, tcg, audio, portability-linux, portability-macos] + needs: [host, build, guest, tcg, audio, portability-linux, portability-macos, release, release-publish] if: ${{ !cancelled() && github.event_name == 'schedule' }} runs-on: ubuntu-latest permissions: contents: read issues: write steps: - - *checkout + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false - env: GH_TOKEN: ${{ github.token }} NEEDS: ${{ toJSON(needs) }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 1c2e8f9e06..db95fd2574 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -29,6 +29,8 @@ jobs: # No submodules: `cargo publish` walks the repository's vcs state, and an # initialised but empty `rust/` breaks that walk. - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false - id: auth uses: rust-lang/crates-io-auth-action@c6f97d42243bad5fab37ca0427f495c86d5b1a18 # v1.0.5 diff --git a/.gitignore b/.gitignore index 09df96c737..784c83812c 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,6 @@ assets/*.sf2 phosphor-icons .cargo/config.toml .build-locks/ +# The std fork's `library/` and licence files, fetched where `rust/` holds no source +# (`src/licence.rs`). +.licence-fork/ diff --git a/Cargo.toml b/Cargo.toml index ccc0d35460..5b751e7d1e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -163,6 +163,9 @@ image = { version = "0.25", default-features = false, features = ["jpeg"] } # (`src/fingerprint.rs`). Already resolved here through a dev-dependency, so # nothing new is fetched — but `cargo run` did not compile it before this. sha2 = "0.10" +# The image release's compressed asset, which every stock macOS and Linux +# unpacks (`src/imagerelease.rs`). +flate2 = { version = "1", default-features = false, features = ["rust_backend"] } [dev-dependencies] toyos-cc = { path = "toyos-cc" } @@ -198,7 +201,6 @@ toyos-xhci = { path = "toyos-xhci" } # the guest's volume is compared with a third party's decoding of the committed # archive, never with `userland/pkg`'s own. The archive itself is committed # under `tests/fixtures` and no test fetches anything. -flate2 = { version = "1", default-features = false, features = ["rust_backend"] } tar = "0.4" [profile.release] diff --git a/bootloader/src/floor.rs b/bootloader/src/floor.rs index b0210e5615..d75b4181ca 100644 --- a/bootloader/src/floor.rs +++ b/bootloader/src/floor.rs @@ -17,14 +17,12 @@ use alloc::vec::Vec; use toyos_update::floor::{self, Read, Scope, Stored}; use uefi::prelude::*; use uefi::table::runtime::{VariableAttributes, VariableVendor}; -use uefi::{guid, CString16}; +use uefi::{CString16, Guid}; /// Whose images this loader's floor holds, as its build decided. const SCOPE: Scope = Scope::from_word(env!("TOYOS_IMAGE_FLOOR")); -/// The vendor every floor is under: `33BE3D4A-30E6-49F5-8050-F169D93A20FB`, -/// minted for this and used for nothing else. -const VENDOR: VariableVendor = VariableVendor(guid!("33be3d4a-30e6-49f5-8050-f169d93a20fb")); +const VENDOR: VariableVendor = VariableVendor(Guid::parse_or_panic(floor::VENDOR)); /// The only attributes the floor is written with. const ATTRIBUTES: VariableAttributes = diff --git a/issues/boot-media/the-loader-writes-the-first-disk-carrying-its-log-guid-not-the-one-it-booted-from.md b/issues/boot-media/the-loader-writes-the-first-disk-carrying-its-log-guid-not-the-one-it-booted-from.md new file mode 100644 index 0000000000..ecbb7f31c8 --- /dev/null +++ b/issues/boot-media/the-loader-writes-the-first-disk-carrying-its-log-guid-not-the-one-it-booted-from.md @@ -0,0 +1,24 @@ +--- +status: open +kind: defect +opened: 2026-09-27 +--- + +# The loader writes the first disk carrying its log GUID, not the one it booted from + +`loaderlog::volume_handle` (`bootloader/src/loaderlog.rs`) takes the first +filesystem on the machine whose GPT partition's unique GUID is the log +partition's, and `loader.log` and the attempt records are written there. Every +copy of one image carries the same partition unique GUIDs, and the image +release (`src/imagerelease.rs`) is written byte for byte to every stick made +from it. With two sticks of one release plugged in, the loader can write the +log partition of the stick it did not boot from: a disk it was not given, and +one that carries no TOYOS-DATA partition. The release notes name this as the +one such disk a boot may write. + +Owner: the bootloader. + +**Exit condition.** The loader writes only the device it booted from: the log +volume is looked up on the device of the loaded image's own handle, and a boot +with two copies of one image attached writes the other copy's bytes not at +all. diff --git a/issues/build/a-toolchain-release-takes-the-latest-badge-from-the-image-release.md b/issues/build/a-toolchain-release-takes-the-latest-badge-from-the-image-release.md new file mode 100644 index 0000000000..08896fb4fb --- /dev/null +++ b/issues/build/a-toolchain-release-takes-the-latest-badge-from-the-image-release.md @@ -0,0 +1,20 @@ +--- +status: open +kind: tooling +opened: 2026-09-27 +--- + +# A toolchain release takes GitHub's Latest badge from the image release + +`gh release create` in `src/release.rs` names no `--latest`, so GitHub marks +each new toolchain release Latest: every release this repository has published +is a toolchain's, and the newest carries the badge. The image release +(`src/imagerelease.rs`) is created `--latest`, and the next toolchain release +takes the badge back, so `/releases/latest` names a toolchain rather than the +image a person downloads until the next image release. + +Not changed beside the image release because `src/release.rs` is one of the +trees the toolchain's tag hashes (`TREES`): editing it publishes a toolchain. + +**Exit condition.** A toolchain release is created `--latest=false`, and +`/releases/latest` names the newest image release whenever there is one. diff --git a/issues/build/the-toolchain-install-unpacks-an-asset-no-digest-vouches-for.md b/issues/build/the-toolchain-install-unpacks-an-asset-no-digest-vouches-for.md new file mode 100644 index 0000000000..0ad6413014 --- /dev/null +++ b/issues/build/the-toolchain-install-unpacks-an-asset-no-digest-vouches-for.md @@ -0,0 +1,25 @@ +--- +status: open +kind: tooling +opened: 2026-09-28 +--- + +# The toolchain install unpacks an asset no digest vouches for + +`release::install` (`src/release.rs`) downloads the `toyos-toolchain.tar.zst` +asset of the release its tree's tag names, unpacks it into `rust/build` and +links it as rustup's `toyos`, and checks nothing about its bytes: the tag is +the hash of the tarball's inputs (`TREES`), not of the tarball. Every guest +job and the image release's boot job compile and boot with what it installs. + +A job holding this repository's `contents: write` token can replace a +release asset, so any code such a job runs can replace the toolchain every +later job installs, and through it the image the release publishes. The +nightly's `build` job holds that token while it bootstraps the toolchain, and +the image release's publish job while it publishes. + +Owner: the release module (`src/release.rs`). + +**Exit condition.** `install` unpacks only an asset whose SHA-256 is one that +no job holding a write token can rewrite — committed to the tree it installs +for — and refuses any other by name. diff --git a/issues/kernel/the-kernel-dies-at-boot-on-the-edk2-firmware-qemu-ships.md b/issues/kernel/the-kernel-dies-at-boot-on-the-edk2-firmware-qemu-ships.md new file mode 100644 index 0000000000..9486c58b58 --- /dev/null +++ b/issues/kernel/the-kernel-dies-at-boot-on-the-edk2-firmware-qemu-ships.md @@ -0,0 +1,34 @@ +--- +status: expected-red +kind: defect +opened: 2026-09-27 +--- + +# The kernel dies at boot on the edk2 firmware QEMU ships + +Booted with `edk2-x86_64-code.fd` and `edk2-i386-vars.fd` from Homebrew's QEMU +11.1.1 (`/opt/homebrew/share/qemu/`), the kernel panics right after the +physical memory manager comes up: + + [kernel 0.000 cpu0 boot] pmm: the firmware map calls 2140844032 bytes usable in 110 entries; managed=2017460224 withheld=94371840 unaligned=29011968, and the three sum to it; frames=962 reserved_frames=45 base=0x200000 span=1022 + [kernel 0.000 cpu0 boot] EARLY PANIC: panicked at library/alloc/src/alloc.rs:659:9: + memory allocation of 4096 bytes failed + +The same image under the same command line with `ovmf/OVMF_CODE-pure-efi.fd` +and `ovmf/OVMF_VARS-pure-efi.fd` in their place reaches the desktop +(`compositor: ready`). Two things the firmwares hand over differ in the two +logs: the memory map, 110 entries against 95, and the GOP framebuffer, at +`0x80000000` against `0xc0000000`. On the firmware that boots, the line after +`pmm:` is the framebuffer's mapping, `mmio: 0xc0000000+0x400000 PAT +WriteCombining`. + +The command line is `imagerelease::Host::MacosAppleSilicon`'s +(`src/imagerelease.rs`) with `-display none`, over a copy of a +`target/bootable.img`. + +`release_command_boots` (`cargo test --test toyos-build -- --release-command`, +which the nightly's `portability-macos` runs) boots that line, and is disabled +in `src/redlist.rs` while this stands. + +**Exit condition.** `release_command_boots` is green on the dev host with the +firmware Homebrew's QEMU ships, and its row leaves `src/redlist.rs`. diff --git a/src/build.rs b/src/build.rs index 0e54936f36..6ae239999f 100644 --- a/src/build.rs +++ b/src/build.rs @@ -169,6 +169,20 @@ fn parse_config(path: &Path) -> SystemConfig { .unwrap_or_else(|e| panic!("Failed to parse {}: {e}", path.display())) } +/// `config` less every program the licence gate names as a package pending +/// the owner. A program's key is its package's name +/// (`the_release_withholds_every_pending_package`). One that `[boot] start` or +/// a symlink still names is refused by `build_and_assemble`. +fn withhold_pending(config: &mut SystemConfig) { + for subject in crate::licence::pending_owner() { + if let crate::licence::Subject::Crate(name) = subject { + if config.programs.remove(name).is_some() { + eprintln!("release: leaving out {name}, whose licence the owner has yet to rule on"); + } + } + } +} + // --- Freshness checking --- /// Fingerprint all external build dependencies that cargo cannot track: the @@ -1008,8 +1022,15 @@ pub struct Boot { /// yet — `issues/build/two-sequences-build-one-image.md` — and until they /// are, this is what keeps each artifact to a single writer. case: bool, + /// Leave out every program [`crate::licence::pending_owner`] names. + public: bool, } +/// What [`Boot::release`] writes: the image, and its licence notice +/// ([`crate::licence::notices`]), which is also on its ROOT. +pub const RELEASE_IMAGE: &str = "target/bootable-release.img"; +pub const RELEASE_NOTICES: &str = "target/bootable-release-licences.txt"; + impl Boot { /// **The one naming rule**: the artifact is named after the directory /// holding the config, and the shipped config sits at the root and keeps @@ -1032,7 +1053,16 @@ impl Boot { Some(name) => format!("target/bootable-{}.img", name.to_string_lossy()), None => "target/bootable.img".to_string(), }; - Ok(Self { config: dir.join(CONFIG), image: PathBuf::from(image), case }) + Ok(Self { config: dir.join(CONFIG), image: PathBuf::from(image), case, public: false }) + } + + /// The config this boot builds. + fn system(&self) -> SystemConfig { + let mut config = parse_config(&self.config); + if self.public { + withhold_pending(&mut config); + } + config } /// The image `arch`'s build of this boot writes. x86-64 keeps the name @@ -1055,6 +1085,34 @@ impl Boot { Self::mode(root, root) } + /// The public release: the shipped config less every program whose + /// package the licence gate holds pending the owner's ruling, and so less + /// the assets only those programs open ([`assets::collect`]). Its own + /// artifact, because it is not the shipped image. + pub fn release(root: &Path) -> Self { + Self { image: PathBuf::from(RELEASE_IMAGE), public: true, ..Self::shipped(root) } + } + + /// [`Shipped`] for this boot's image, read out of its config the way + /// [`build`] reads it. + /// + /// **A config that ships the hosted compiler is refused**: its dependencies + /// are the rust fork's `compiler/` workspace, which no reader of this + /// answer walks. + pub fn parts(&self, root: &Path) -> Result { + let config = self.system(); + if config.hosted_rustc { + return Err(format!( + "{} sets hosted-rustc, and nothing reads the licences of the compiler it ships", + self.config.display() + )); + } + Ok(Shipped { + crates: config_crates(root, &config).into_iter().map(|c| (c.dir, c.features)).collect(), + assets: config.assets.iter().map(|dir| root.join(dir)).collect(), + }) + } + /// The config declares no `devices`, so nothing started there claims the /// framebuffer and the kernel's last boot checkpoint stays on screen. /// `screen_diag_boot` boots this same config, so the tested image and the @@ -1104,34 +1162,24 @@ impl Boot { } } -/// What the three modes' images are built from, besides std: every crate -/// [`config_crates`] names for one of them with the features the build gives -/// it, and the asset directories their configs copy onto ROOT. A case's config -/// is a test image and is not here. +/// What images are built from, besides std: every crate [`config_crates`] +/// names for them with the features the build gives it, and the asset +/// directories their configs copy onto ROOT. A case's config is a test image +/// and is not here. pub struct Shipped { pub crates: BTreeSet<(PathBuf, Features)>, pub assets: BTreeSet, } -/// [`Shipped`], read out of the modes' configs the way [`build`] reads them. -/// -/// **A config that ships the hosted compiler is refused**: its dependencies are -/// the rust fork's `compiler/` workspace, which no reader of this answer walks. +/// [`Boot::parts`] of the three modes together. pub fn shipped(root: &Path) -> Result { - let mut crates = BTreeSet::new(); - let mut assets = BTreeSet::new(); + let mut all = Shipped { crates: BTreeSet::new(), assets: BTreeSet::new() }; for boot in [Boot::shipped(root), Boot::diag(root), Boot::console(root)] { - let config = parse_config(&boot.config); - if config.hosted_rustc { - return Err(format!( - "{} sets hosted-rustc, and nothing reads the licences of the compiler it ships", - boot.config.display() - )); - } - crates.extend(config_crates(root, &config).into_iter().map(|c| (c.dir, c.features))); - assets.extend(config.assets.iter().map(|dir| root.join(dir))); + let parts = boot.parts(root)?; + all.crates.extend(parts.crates); + all.assets.extend(parts.assets); } - Ok(Shipped { crates, assets }) + Ok(all) } /// The parameters an image built for flashing may carry: the kernel's own boot @@ -1830,13 +1878,26 @@ fn shipped_parts(root: &Path, boot: &Boot, rebuild_toolchain: bool, plan: &Plan) let kernel_features = plan.features.join(","); let arch = plan.arch; + let mut extra = Vec::new(); + if boot.public { + let notices = boot + .parts(root) + .and_then(|parts| crate::licence::notices(root, parts, &crate::licence::std_library(root)?)) + .unwrap_or_else(|why| panic!("the release's licence notice: {why}")); + let at = root.join(RELEASE_NOTICES); + let held = buildlock::licence(root); + fs::write(&at, ¬ices).unwrap_or_else(|e| panic!("write {}: {e}", at.display())); + drop(held); + extra.push((crate::licence::NOTICES_ON_ROOT.to_string(), notices.into_bytes())); + } + // Held until the last staged artifact has been read back, so no clean of // this worktree's crate targets can land inside this build. let mut lock = buildlock::shared(root, "build"); let sysroot = toolchain::ensure(root, rebuild_toolchain, &mut lock); let env = GuestEnv::new(&sysroot); - let config = parse_config(&boot.config); + let config = boot.system(); invalidate_stale(root, &mut lock, &env.toolchain, &config_targets(root, &config)); @@ -1867,7 +1928,7 @@ fn shipped_parts(root: &Path, boot: &Boot, rebuild_toolchain: bool, plan: &Plan) ) }; - let root_bytes = build_and_assemble(root, &config, &env, &[], false, arch); + let root_bytes = build_and_assemble(root, &config, &env, &extra, false, arch); let bl_bytes = fs::read(&bl_art).expect("Failed to read staged bootloader"); (kernel_bytes, bl_bytes, root_bytes) @@ -2346,6 +2407,67 @@ fn collect_hosted_rustc(root: &Path, toolchain: &Path, root_files: &mut Vec<(Str mod tests { use super::*; + fn pending(of: fn(crate::licence::Subject) -> Option<&'static str>) -> Vec<&'static str> { + crate::licence::pending_owner().filter_map(of).collect() + } + + /// The release leaves out every program whose package the licence gate + /// holds pending the owner and keeps every other, and each withheld key is + /// its package's name, which is what the rule reads it by. + #[test] + fn the_release_withholds_every_pending_package() { + use crate::licence::Subject; + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let shipped = Boot::shipped(root).system(); + let release = Boot::release(root).system(); + let withheld = pending(|s| match s { + Subject::Crate(name) => Some(name), + _ => None, + }); + assert!(!withheld.is_empty(), "the gate holds no package pending, so this reads nothing"); + for name in &withheld { + let program = shipped + .programs + .get(*name) + .unwrap_or_else(|| panic!("the shipped config builds no {name}")); + let manifest = fs::read_to_string(program.crate_dir(root, name).join("Cargo.toml")).unwrap(); + let manifest: toml::Value = toml::from_str(&manifest).unwrap(); + assert_eq!(manifest["package"]["name"].as_str(), Some(*name)); + } + let kept: Vec<&String> = + shipped.programs.keys().filter(|key| !withheld.contains(&key.as_str())).collect(); + assert_eq!(release.programs.keys().collect::>(), kept); + assert_eq!(Boot::release(root).image_for(Arch::X86_64), PathBuf::from(RELEASE_IMAGE)); + assert_eq!(Boot::release(root).config, Boot::shipped(root).config); + } + + /// No file the licence gate holds pending the owner is on the release's + /// ROOT, and every one is on the shipped image's. + #[test] + fn the_release_image_carries_no_pending_file() { + use crate::licence::Subject; + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let files = |config: &SystemConfig| -> Vec { + let programs: BTreeSet<&str> = config.programs.keys().map(String::as_str).collect(); + assets::collect(&config.assets, &programs).into_iter().map(|(name, _)| name).collect() + }; + let shipped = files(&Boot::shipped(root).system()); + let release = files(&Boot::release(root).system()); + let withheld = pending(|s| match s { + Subject::File(path) => Some(path), + _ => None, + }); + assert!(!withheld.is_empty(), "the gate holds no file pending, so this reads nothing"); + for path in withheld { + let name = Path::new(path).file_name().unwrap().to_string_lossy().to_lowercase(); + let on = |root_files: &[String]| { + root_files.iter().any(|f| f.rsplit('/').next() == Some(name.as_str())) + }; + assert!(on(&shipped), "the shipped image carries no {name}, so its absence below says nothing"); + assert!(!on(&release), "the release image carries {name}"); + } + } + /// `console` is reached by `console/system.toml` alone and `init` by no /// `[programs]` row, so a reader that drops a mode or init loses one. #[test] diff --git a/src/buildlock.rs b/src/buildlock.rs index d42efedf17..c850618bf0 100644 --- a/src/buildlock.rs +++ b/src/buildlock.rs @@ -217,6 +217,14 @@ pub fn artifact(root: &Path) -> Guard { exclusive(&root.join(LOCK_DIR).join("artifact"), "artifact lock", "artifact staging") } +/// This worktree's licence files: the std fork's fetch +/// (`licence::fetched_library`), the walk's scratch `Cargo.lock` and the +/// release's notice. It waits on the network, so it is taken with no other lock +/// held. +pub fn licence(root: &Path) -> Guard { + exclusive(&root.join(LOCK_DIR).join("licence"), "licence lock", "writing the licence files") +} + /// The integration lock: one process at a time moves this host's `main`. /// /// It used to hold a whole landing — lock, merge, gate, fast-forward. diff --git a/src/ci.rs b/src/ci.rs index eb7700dce9..24e3478c8e 100644 --- a/src/ci.rs +++ b/src/ci.rs @@ -7,8 +7,10 @@ //! [`Job::GateStage`] run as `host`. Every test that boots no guest is in //! [`Job::Host`], so a merge is gated on all of them. `nightly.yml` runs //! everything that boots a guest, `host` again to write the cache the merge -//! queue restores, and portability. `publish.yml` puts a landing's crates on -//! crates.io. +//! queue restores, portability, and the image release: [`Job::Release`] boots the +//! image it stages with a token that only reads, and [`Job::ReleasePublish`] +//! publishes those bytes with one that writes. `publish.yml` puts a landing's +//! crates on crates.io. //! //! A host job runs every step and reds if any failed; a guest job stops at the //! first failure among the instrument, the toolchain and the suite, because @@ -30,7 +32,7 @@ use std::path::Path; use std::process::Command; use crate::arch::Arch; -use crate::{flags, pr, release, sdkversion}; +use crate::{flags, imagerelease, pr, release, sdkversion}; /// The checks `main`'s ruleset must require, as `gate-stage` reads them back: /// a minimum, never an equality, so a name GitHub requires and this does not @@ -49,6 +51,8 @@ const USAGE: &str = "cargo run -- --ci , where is one of: guest / one shard of the whole guest suite, nightly tier included (nightly) tcg one test on an emulated CPU (nightly) audio / one shard of gate A (nightly) + release build the release image, boot it, and stage its assets (nightly) + release-publish on main, publish the assets `release` staged (nightly) nightly-red file or update the nightly-red issue from $NEEDS (nightly) publish put main's SDK crates on crates.io (publish.yml)"; @@ -60,6 +64,8 @@ enum Job { Guest(String), Tcg, Audio(String), + Release, + ReleasePublish, NightlyRed, Publish, } @@ -77,6 +83,8 @@ fn parse(words: &[String]) -> Result { Some("guest") => Job::Guest(shard(words.get(1))?), Some("tcg") => Job::Tcg, Some("audio") => Job::Audio(shard(words.get(1))?), + Some("release") => Job::Release, + Some("release-publish") => Job::ReleasePublish, Some("nightly-red") => Job::NightlyRed, Some("publish") => Job::Publish, Some(other) => return Err(format!("no CI job is called {other:?}")), @@ -99,10 +107,14 @@ pub fn dispatch(root: &Path, args: &[String]) { Job::GateStage => vec![step("what protects main", || gate_stage(root))], Job::Toolchain => vec![step("the toolchain release", || release::ensure_published(root))], Job::Guest(shard) => { - guest(root, &suite_args(&["--shard", shard, "--jobs", "1", "--nightly"])) + guest(root, "the suite", || suite(root, &suite_args(&["--shard", shard, "--jobs", "1", "--nightly"]))) } - Job::Tcg => guest(root, &suite_args(&["--jobs", "1", "process_stats"])), - Job::Audio(shard) => guest(root, &suite_args(&["--audio-gate", "30", "--shard", shard])), + Job::Tcg => guest(root, "the suite", || suite(root, &suite_args(&["--jobs", "1", "process_stats"]))), + Job::Audio(shard) => { + guest(root, "the suite", || suite(root, &suite_args(&["--audio-gate", "30", "--shard", shard]))) + } + Job::Release => guest(root, "the image release", || imagerelease::release(root)), + Job::ReleasePublish => vec![step("the image release's publication", || imagerelease::publish(root))], Job::NightlyRed => vec![step("the nightly-red issue", nightly_red)], Job::Publish => vec![step("the SDK crates on crates.io", || publish(root))], }; @@ -148,6 +160,14 @@ fn on_runner() -> bool { std::env::var("GITHUB_ACTIONS").is_ok_and(|v| v == "true") } +/// Set this step's output `name`, which a later job reads through its own +/// job's `outputs:`. Refused where no runner gave the step an output file. +pub fn output(name: &str, value: &str) -> Result<(), String> { + let path = std::env::var("GITHUB_OUTPUT").map_err(|_| "GITHUB_OUTPUT is unset".to_string())?; + let mut file = std::fs::OpenOptions::new().append(true).open(&path).map_err(|e| format!("{path}: {e}"))?; + writeln!(file, "{name}={value}").map_err(|e| format!("{path}: {e}")) +} + /// Append to the runner's job summary; nowhere off a runner. fn summary(text: &str) { let Ok(path) = std::env::var("GITHUB_STEP_SUMMARY") else { return }; @@ -608,7 +628,7 @@ fn suite_args(args: &[&str]) -> Vec { /// A guest job's own `$TMPDIR`, same rule as [`host`]: nothing the suite /// writes past a `toyos_tmpdir::TempDir` — the harness's own `Run`, its lanes, /// every boot image — survives past the last step, which reds on it. -fn guest(root: &Path, suite: &[String]) -> Vec { +fn guest(root: &Path, label: &str, boots: impl FnOnce() -> Result) -> Vec { let tmp = toyos_tmpdir::TempDir::new("ci-guest"); // Before any thread, same as `host`: every child this process spawns below // inherits this, and nothing here reads the environment concurrently with @@ -621,16 +641,7 @@ fn guest(root: &Path, suite: &[String]) -> Vec { steps.push(step("the toolchain", || release::install(root))); } if steps.iter().all(|s| s.verdict.is_ok()) { - steps.push(step("the suite", || { - let args: Vec<&str> = suite.iter().map(String::as_str).collect(); - let (green, log) = cargo_logged(root, &args)?; - let said = verdicts(&log); - if green { - Ok(said) - } else { - Err(said) - } - })); + steps.push(step(label, boots)); } // Unconditional: whatever stopped earlier, this $TMPDIR is still this // process's own to judge, and a leak past a failing suite is still a leak. @@ -638,6 +649,18 @@ fn guest(root: &Path, suite: &[String]) -> Vec { steps } +/// `cargo `, judged by its exit and summarised by [`verdicts`]. +fn suite(root: &Path, suite: &[String]) -> Result { + let args: Vec<&str> = suite.iter().map(String::as_str).collect(); + let (green, log) = cargo_logged(root, &args)?; + let said = verdicts(&log); + if green { + Ok(said) + } else { + Err(said) + } +} + /// The suite's own count line and every line naming a verdict worth reading /// without the log: a failure. fn verdicts(log: &str) -> String { @@ -861,6 +884,7 @@ fn at_tip(ls_remote: &str, head: &str) -> Result<(), String> { #[cfg(test)] mod tests { use super::*; + use std::collections::BTreeMap; use std::path::PathBuf; /// A deterministic control on `host`'s `std::env::set_var("TMPDIR", ...)`: @@ -897,6 +921,7 @@ mod tests { fn a_job_is_named_and_a_shard_is_a_shard() { assert_eq!(parse(&words("host")), Ok(Job::Host)); assert_eq!(parse(&words("guest 3/12")), Ok(Job::Guest("3/12".into()))); + assert_eq!(parse(&words("release-publish")), Ok(Job::ReleasePublish)); assert!(parse(&words("guest")).is_err()); assert!(parse(&words("guest 13/12")).is_err()); assert!(parse(&words("host extra")).is_err()); @@ -1047,6 +1072,99 @@ mod tests { assert!(writers.iter().all(|(f, _)| f == "nightly.yml"), "{writers:?}"); } + /// The value of `key` at `indent` in `block`: the rest of its line and the + /// lines nested under it, comments left out. + fn yaml_value(block: &str, indent: &str, key: &str) -> Option { + let head = format!("{indent}{key}:"); + let nested = format!("{indent} "); + let mut lines = block + .lines() + .skip_while(|l| !l.strip_prefix(head.as_str()).is_some_and(|rest| rest.is_empty() || rest.starts_with(' '))); + let first = lines.next()?; + let rest = lines.take_while(|l| l.starts_with(nested.as_str()) || l.trim().is_empty()); + let value = std::iter::once(&first[head.len()..]) + .chain(rest) + .filter(|l| !l.trim_start().starts_with('#')) + .map(|l| l.split(" #").next().unwrap_or(l)) + .collect::>() + .join("\n"); + Some(value) + } + + /// A job's steps, each its own lines. + fn job_steps(body: &str) -> Vec { + let mut steps: Vec = Vec::new(); + for line in yaml_value(body, " ", "steps").unwrap_or_default().lines() { + if line.starts_with(" - ") { + steps.push(String::new()); + } + if let Some(step) = steps.last_mut() { + step.push_str(&format!("{line}\n")); + } + } + steps + } + + /// A job whose token writes restores no cache, since a cache is a tree a + /// job running third-party code wrote; leaves no credential in the + /// checkout's `.git/config`, where every later step's code reads it; and + /// hands `GH_TOKEN` to a step rather than to every step. + #[test] + fn a_token_that_writes_meets_no_cache_and_only_the_step_that_needs_it() { + let dir = repo_root().join(".github/workflows"); + let mut writing = 0; + for entry in std::fs::read_dir(&dir).expect(".github/workflows is readable").flatten() { + let text = std::fs::read_to_string(entry.path()).expect("a readable workflow"); + let file = entry.file_name().to_string_lossy().into_owned(); + let (head, jobs) = text.split_once("\njobs:\n").expect("a workflow has jobs"); + let mut chunks: Vec<(String, String)> = Vec::new(); + for line in jobs.lines() { + let key = line.strip_prefix(" ").and_then(|l| l.strip_suffix(':')); + match key.filter(|k| !k.starts_with([' ', '#'])) { + Some(job) => chunks.push((job.to_string(), String::new())), + None => { + if let Some((_, body)) = chunks.last_mut() { + body.push_str(&format!("{line}\n")); + } + } + } + } + let mut anchors = BTreeMap::new(); + for (_, body) in &chunks { + for step in job_steps(body) { + let first = step.lines().next().unwrap_or_default().trim_start(); + if let Some(anchor) = first.strip_prefix("- &") { + anchors.insert(anchor.trim().to_string(), step.clone()); + } + } + } + let workflow = yaml_value(head, "", "permissions"); + for (job, body) in &chunks { + let permissions = yaml_value(body, " ", "permissions").or_else(|| workflow.clone()); + if !permissions.is_some_and(|p| p.contains("write")) { + continue; + } + writing += 1; + // A `- *anchor` step is the step anchored `&anchor`. + let own = job_steps(body); + let steps: Vec<&String> = own + .iter() + .map(|step| match step.trim_start().strip_prefix("- *") { + Some(alias) => anchors.get(alias.trim()).unwrap_or_else(|| panic!("{file}: no step is anchored &{alias}")), + None => step, + }) + .collect(); + let restores = steps.iter().any(|s| s.contains("actions/cache/restore@")); + assert!(!restores, "{file}: {job} restores a cache with a token that writes"); + let persists = steps.iter().any(|s| s.contains("actions/checkout@") && !s.contains("persist-credentials: false")); + assert!(!persists, "{file}: {job} keeps a token that writes in its checkout"); + let job_env = yaml_value(body, " ", "env").is_some_and(|env| env.contains("GH_TOKEN")); + assert!(!job_env, "{file}: {job} hands a token that writes to every step"); + } + } + assert!(writing > 0, "no job's token writes, so this checked nothing"); + } + #[test] fn the_declared_version_is_a_version() { let declared = diff --git a/src/imagerelease.rs b/src/imagerelease.rs new file mode 100644 index 0000000000..cc2dbc4646 --- /dev/null +++ b/src/imagerelease.rs @@ -0,0 +1,727 @@ +//! The image release: the disk a person downloads and boots under QEMU or +//! writes to a stick, published by the nightly's `release` and +//! `release-publish` jobs. +//! +//! **What is published is what booted**: [`release`] builds +//! `build::Boot::release`'s image once, boots a copy of it under the Linux +//! command line its notes print ([`boots`]), and stages those bytes' assets in +//! [`STAGED`]. **The token that writes releases never meets the code a build +//! runs**: that job's token reads, and it hands the assets on as an artifact +//! and their [`digest`] as a job output; [`publish`], in a job that restores no +//! cache and builds only this crate, publishes them on `main` once their digest +//! is the one it was handed. **Named by the commit**, [`tag`]: every build draws +//! its partition GUIDs and a runner mints its own throwaway signing key +//! (`src/signing.rs`), so a second build reproduces no byte of the first. +//! **Kept to [`KEEP`]**: each publish deletes every older image release and its +//! tag ([`stale`]). +//! +//! Not in `src/release.rs`, whose bytes are hashed into the toolchain's tag. + +use std::fs; +use std::io::{BufRead, BufReader, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::sync::mpsc::{self, Receiver, RecvTimeoutError}; +use std::time::{Duration, Instant}; + +use serde_json::Value; + +use crate::arch::{Accel, Arch}; +use crate::fingerprint::{first_difference, whole_device}; +use crate::licence::{pending_owner, Subject}; +use crate::release::sha256_hex; + +/// What every image release's tag starts with; the rest is the commit's first +/// twelve hex digits. +pub const TAG_PREFIX: &str = "image-x86_64-"; + +/// How many image releases stay published. +pub const KEEP: usize = 7; + +/// The compressed disk, as a release asset. +pub const IMAGE_ASSET: &str = "toyos.img.gz"; + +/// The image as the notes' commands name it once unpacked. +pub const IMAGE: &str = "toyos.img"; + +/// The SHA-256 of [`IMAGE_ASSET`], in the form `sha256sum -c` reads. +pub const SUMS_ASSET: &str = "SHA256SUMS"; + +/// The notes, also carried as an asset. +pub const NOTES_ASSET: &str = "README.md"; + +/// The image's licence notice (`build::RELEASE_NOTICES`), as an asset. +pub const LICENCES_ASSET: &str = "licences.txt"; + +/// Every asset, in upload order. +const ASSETS: [&str; 4] = [IMAGE_ASSET, SUMS_ASSET, NOTES_ASSET, LICENCES_ASSET]; + +/// Where [`release`] stages the assets and [`publish`] reads them: the +/// directory the nightly's artifact carries between the two jobs. +const STAGED: &str = "target/image-release"; + +/// What the publish job is handed the release job's [`digest`] in. +const DIGEST_VAR: &str = "IMAGE_RELEASE_DIGEST"; + +/// The defect behind the one disk the notes say a boot may write though it +/// was not given it. +const TWO_STICKS: &str = + "issues/boot-media/the-loader-writes-the-first-disk-carrying-its-log-guid-not-the-one-it-booted-from.md"; + +/// One guest's ceiling from power-on to a painting desktop, unscaled: a +/// liveness guard, never a verdict. +pub const DESKTOP: Duration = Duration::from_secs(120); + +/// How many releases one listing asks for; a listing that fills it is refused, +/// since what it left out cannot be judged. +const LISTED: usize = 1000; + +/// `image-x86_64-<12 hex>` of a full commit id. +pub fn tag(commit: &str) -> Result { + let full = commit.len() == 40 && commit.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')); + if !full { + return Err(format!("{commit:?} is not a full commit id")); + } + Ok(format!("{TAG_PREFIX}{}", &commit[..12])) +} + +/// The hosts the notes give a command line for. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Host { + /// Homebrew's QEMU, emulating the x86-64 guest. + MacosAppleSilicon, + /// Debian's `qemu-system-x86` and `ovmf`, on the host's own CPU. + LinuxKvm, +} + +impl Host { + pub const ALL: [Host; 2] = [Host::MacosAppleSilicon, Host::LinuxKvm]; + + /// Where the notes say this host is. + pub fn named(self) -> &'static str { + match self { + Host::MacosAppleSilicon => "macOS on Apple Silicon, with Homebrew's `qemu`", + Host::LinuxKvm => { + "Linux on x86-64 with KVM, with Debian's `qemu-system-x86` and `ovmf` and a user \ + that can open `/dev/kvm`" + } + } + } + + fn accel(self) -> Accel { + match self { + Host::MacosAppleSilicon => Accel::Tcg, + Host::LinuxKvm => Accel::Kvm, + } + } + + /// The edk2 firmware the host's QEMU installation ships: its code, and the + /// variable-store template beside it, which the guest gets read-only. + pub fn firmware(self) -> [&'static str; 2] { + match self { + Host::MacosAppleSilicon => [ + "/opt/homebrew/share/qemu/edk2-x86_64-code.fd", + "/opt/homebrew/share/qemu/edk2-i386-vars.fd", + ], + Host::LinuxKvm => ["/usr/share/OVMF/OVMF_CODE_4M.fd", "/usr/share/OVMF/OVMF_VARS_4M.fd"], + } + } + + /// The whole command line, program first, that boots `image` as a USB + /// stick on the machine shape `cargo run` boots: q35 with a vIOMMU, a + /// firmware framebuffer, a USB keyboard and tablet, and virtio-net. The + /// kernel's log goes to the terminal. + pub fn command(self, image: &str) -> Vec { + let accel = self.accel(); + let [code, vars] = self.firmware(); + [ + Arch::X86_64.qemu(), + "-nodefaults", + "-accel", + accel.name(), + "-cpu", + Arch::X86_64.cpu(accel), + "-machine", + "q35,kernel-irqchip=split", + "-smp", + "4", + "-m", + "2G", + "-drive", + &format!("if=pflash,format=raw,unit=0,file={code},readonly=on"), + "-drive", + &format!("if=pflash,format=raw,unit=1,file={vars},readonly=on"), + "-device", + "intel-iommu,intremap=on,caching-mode=on,aw-bits=48", + "-device", + "nec-usb-xhci,id=xhci", + "-drive", + &format!("if=none,id=stick,format=raw,file={image}"), + "-device", + "usb-storage,bus=xhci.0,drive=stick,bootindex=0", + "-device", + "usb-kbd,bus=xhci.0", + "-device", + "usb-tablet,bus=xhci.0", + "-vga", + "std", + "-netdev", + "user,id=net0", + "-device", + "virtio-net-pci-non-transitional,netdev=net0,iommu_platform=on", + "-serial", + "stdio", + ] + .map(String::from) + .to_vec() + } +} + +/// A QEMU started from a release command line, and its console so far. +struct Guest { + child: Child, + lines: Receiver, + log: String, + stderr: PathBuf, +} + +impl Guest { + /// `argv` with no window: the notes' line, headless. + fn start(argv: &[String], stderr: &Path) -> Result { + let err = fs::File::create(stderr).map_err(|e| format!("{}: {e}", stderr.display()))?; + let arch = Arch::X86_64; + if argv[0] != arch.qemu() { + return Err(format!("a release command line starts {:?}, not {}", argv[0], arch.qemu())); + } + let mut child = Command::new(arch.qemu()) + .args(&argv[1..]) + .args(["-display", "none"]) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(err) + .spawn() + .map_err(|e| format!("{}: {e}", arch.qemu()))?; + let out = child.stdout.take().expect("piped"); + let (send, lines) = mpsc::channel(); + std::thread::spawn(move || { + for line in BufReader::new(out).split(b'\n') { + let Ok(line) = line else { return }; + if send.send(String::from_utf8_lossy(&line).into_owned()).is_err() { + return; + } + } + }); + Ok(Guest { child, lines, log: String::new(), stderr: stderr.to_path_buf() }) + } + + /// Read the console until every line of `want` is in it, within + /// `ceiling`, refusing a panic the moment one is printed. + fn until_said(&mut self, want: &[String], ceiling: Duration) -> Result<(), String> { + let deadline = Instant::now() + ceiling; + loop { + if let Some(line) = self.log.lines().find(|l| l.contains("PANIC") || l.contains("panicked at")) { + return Err(format!("the guest panicked before the desktop: {line}\n{}", self.log)); + } + if want.iter().all(|line| self.log.contains(line.as_str())) { + return Ok(()); + } + match self.lines.recv_timeout(deadline.saturating_duration_since(Instant::now())) { + Ok(line) => { + self.log.push_str(&line); + self.log.push('\n'); + } + Err(RecvTimeoutError::Timeout) => { + return Err(format!("no desktop within {ceiling:?}:\n{}", self.log)); + } + Err(RecvTimeoutError::Disconnected) => { + let stderr = fs::read_to_string(&self.stderr).unwrap_or_default(); + return Err(format!( + "QEMU closed its console before the desktop:\n{}\nQEMU's stderr:\n{stderr}", + self.log + )); + } + } + } + } +} + +impl Drop for Guest { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +/// Boot `stick` under `host`'s command line until the desktop paints: every +/// program `system.toml` starts said it started, the three that announce +/// themselves did, and the compositor reported a frame. And neither firmware +/// file changed, since the line gives the guest both read-only. +pub fn boots(root: &Path, host: Host, stick: &Path, ceiling: Duration, stderr: &Path) -> Result<(), String> { + let firmware = host.firmware(); + if let Some(missing) = firmware.iter().find(|f| !Path::new(f).is_file()) { + return Err(format!("{missing} is no file: this host lacks the firmware {host:?}'s line names")); + } + let before = firmware.map(|f| whole_device(Path::new(f))); + let mut want: Vec = crate::build::boot_start(&root.join("system.toml")) + .iter() + .map(|program| format!("init: started {program}")) + .collect(); + want.extend( + ["compositor: ready", "netd: ready", "logd: this boot's kernel log is", "compositor: frames="] + .map(String::from), + ); + let desktop = Guest::start(&host.command(&stick.display().to_string()), stderr)?.until_said(&want, ceiling); + for (file, before) in firmware.iter().zip(&before) { + if let Some(diff) = first_difference(before, &whole_device(Path::new(file))) { + return Err(format!("the boot wrote {file}, which the line gives the guest read-only: {diff}")); + } + } + desktop +} + +/// [`Host::command`] as the notes print it: an option and its value to a line. +fn shell(argv: &[String]) -> String { + let mut out = format!(" {}", argv[0]); + for word in &argv[1..] { + if word.starts_with('-') { + out.push_str(" \\\n "); + } else { + out.push(' '); + } + out.push_str(word); + } + out +} + +/// The floor variable a boot of the release leaves in the firmware, as the +/// notes name it: its name's fixed head, and how many hex digits follow. +fn floor_variable() -> (String, usize) { + use toyos_update::floor::{Scope, NAME_BYTES, PREFIX}; + let head = format!("{PREFIX}-{}", Scope::Image.tag() as char); + let hex = NAME_BYTES - head.len(); + (head, hex) +} + +/// The release notes, which are also [`NOTES_ASSET`]. +pub fn notes(root: &Path, tag: &str, commit: &str) -> Result { + let qemu = crate::ci::declared_qemu_version(root).ok_or(".github/qemu-version declares no version")?; + let data = toyos_gpt::Guid::TOYOS_DATA_TEXT; + let (floor, hex) = floor_variable(); + let vendor = toyos_update::floor::VENDOR; + let on_root = crate::licence::NOTICES_ON_ROOT; + let withheld: Vec = pending_owner() + .map(|s| match s { + Subject::Crate(path) | Subject::Notice(path) | Subject::File(path) => format!("`{path}`"), + }) + .collect(); + let mut commands = String::new(); + for host in Host::ALL { + commands.push_str(&format!("{}:\n\n{}\n\n", host.named(), shell(&host.command(IMAGE)))); + } + Ok(format!( + "# ToyOS {tag} + +The ToyOS disk image of commit {commit} on `main`. It booted to its desktop under the Linux command line below before it was published. It boots under QEMU, or from a USB stick on a UEFI x86-64 machine. + +## Verify and unpack + +Download `{IMAGE_ASSET}` and `{SUMS_ASSET}` from this release into one directory, and in it: + + sha256sum -c {SUMS_ASSET} + gunzip {IMAGE_ASSET} + +## Under QEMU + +QEMU {qemu} is the version ToyOS is measured with. The firmware is edk2, from Homebrew's QEMU on macOS and from Debian's `ovmf` on Linux, and the guest gets both of its files read-only. + +{commands}The kernel's log is on the terminal and the desktop is in QEMU's window. The running system writes to `{IMAGE}` itself, as it would to a stick. `/apps`, `/config`, `/home` and `/state` are kept in memory and are gone at the next boot. + +## On a USB stick + +The machine has to be an x86-64 PC from 2020 or later, booting UEFI with Secure Boot off. The only hardware ToyOS is known to work on is a Lenovo ThinkPad T14; on anything else it is untried. + +Booting the stick writes two things into the machine's firmware: + +- a variable named `{floor}` and {hex} hex digits, under the vendor GUID `{vendor}`: the loader's anti-rollback floor, eight bytes. It stays after the stick is gone. It is readable only before an operating system starts, so no operating system can see or remove it; only a UEFI shell can, with `dmpstore -d -guid {vendor}`. Booting another ToyOS image's stick replaces it rather than adding one; +- `BootNext`, where one of the machine's boot entries names the stick: the next restart boots the stick once. + +Write `{IMAGE}` to the whole stick, not to a partition of it; what the stick held is lost. Unmount it first: on macOS `diskutil unmountDisk /dev/`, on Linux `umount` each of its partitions that is mounted (`lsblk /dev/` lists them). Then, as root: + + dd if={IMAGE} of=/dev/ bs=4194304 + sync + +A boot writes to the stick it booted from, and to another disk only where that disk carries a partition of ToyOS's DATA type, `{data}`, a type no other system uses. That is where `/apps`, `/config`, `/home` and `/state` live. Every other disk is read for its partition table and not written, but for one known defect: with two sticks made from one image plugged in, the loader can write its log to the one it did not boot from (`{TWO_STICKS}` in the ToyOS source). + +## Terms + +ToyOS is MIT OR Apache-2.0. `{LICENCES_ASSET}`, beside the image and on it at `/system/{on_root}`, gives every package the image is built from with its licence, where its source is and its licence texts, and every third-party file on the image with its terms and texts. + +The image leaves out {withheld}, which the repository carries. +", + withheld = withheld.join(", "), + )) +} + +/// The tags among `listed`, as (tag, creation time in RFC 3339 UTC), that are +/// image releases older than the [`KEEP`] newest. +pub fn stale(listed: &[(String, String)], keep: usize) -> Vec { + let mut images: Vec<&(String, String)> = + listed.iter().filter(|(tag, _)| tag.starts_with(TAG_PREFIX)).collect(); + images.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| b.0.cmp(&a.0))); + images.into_iter().skip(keep).map(|(tag, _)| tag.clone()).collect() +} + +/// Write every asset of `tag`'s release into `out`: the image at `image` +/// compressed, its sum, the notes and the licence notice at `licences`. +pub fn write_assets( + root: &Path, + image: &Path, + licences: &Path, + out: &Path, + tag: &str, + commit: &str, +) -> Result<(), String> { + use flate2::write::GzEncoder; + + let compressed = out.join(IMAGE_ASSET); + let file = fs::File::create(&compressed).map_err(|e| format!("{}: {e}", compressed.display()))?; + let mut gz = GzEncoder::new(file, flate2::Compression::default()); + let mut raw = fs::File::open(image).map_err(|e| format!("{}: {e}", image.display()))?; + std::io::copy(&mut raw, &mut gz).map_err(|e| format!("compressing {}: {e}", image.display()))?; + gz.finish().map_err(|e| format!("compressing {}: {e}", image.display()))?.flush().map_err(|e| e.to_string())?; + + let bytes = fs::read(&compressed).map_err(|e| format!("{}: {e}", compressed.display()))?; + let sums = format!("{} {IMAGE_ASSET}\n", sha256_hex(&bytes)); + fs::write(out.join(SUMS_ASSET), sums).map_err(|e| e.to_string())?; + fs::write(out.join(NOTES_ASSET), notes(root, tag, commit)?).map_err(|e| e.to_string())?; + fs::copy(licences, out.join(LICENCES_ASSET)).map_err(|e| format!("{}: {e}", licences.display()))?; + Ok(()) +} + +/// The SHA-256 of every asset's own, one `sha256sum` line each in upload +/// order: what the release job answers and the publish job recomputes. +/// Refused while `dir` holds anything but the four assets. +fn digest(dir: &Path) -> Result { + let entries = fs::read_dir(dir).map_err(|e| format!("{}: {e}", dir.display()))?; + let mut held = entries + .map(|e| e.map(|e| e.file_name().to_string_lossy().into_owned())) + .collect::, _>>() + .map_err(|e| format!("{}: {e}", dir.display()))?; + held.sort(); + let mut want = ASSETS.to_vec(); + want.sort(); + if held != want { + return Err(format!("{} holds {held:?}, and the assets are {ASSETS:?}", dir.display())); + } + let mut sums = String::new(); + for name in ASSETS { + let bytes = fs::read(dir.join(name)).map_err(|e| format!("{}: {e}", dir.join(name).display()))?; + sums.push_str(&format!("{} {name}\n", sha256_hex(&bytes))); + } + Ok(sha256_hex(sums.as_bytes())) +} + +/// `gh `: what it printed, or its exit and what it said. +fn gh(root: &Path, args: &[&str]) -> Result { + let out = Command::new("gh").args(args).current_dir(root).output().map_err(|e| format!("gh: {e}"))?; + if out.status.success() { + Ok(String::from_utf8_lossy(&out.stdout).into_owned()) + } else { + Err(format!("gh {} exited {}: {}", args.join(" "), out.status, String::from_utf8_lossy(&out.stderr).trim())) + } +} + +/// A release GitHub holds under a tag: whether it is still a draft, and +/// whether it carries [`IMAGE_ASSET`]. +#[derive(Debug, PartialEq, Eq)] +struct Held { + draft: bool, + image: bool, +} + +/// What GitHub holds under `tag`: nothing, which `gh` says as exactly +/// `release not found`, or a [`Held`]. Any other failure of `gh` is one. +fn held(root: &Path, tag: &str) -> Result, String> { + held_of(gh(root, &["release", "view", tag, "--json", "isDraft,assets"])) +} + +/// [`held`] of what `gh release view --json isDraft,assets` answered. +fn held_of(said: Result) -> Result, String> { + let json = match said { + Err(why) if why.ends_with(": release not found") => return Ok(None), + said => said?, + }; + let v: Value = serde_json::from_str(&json).map_err(|e| format!("gh release view: {e}: {json}"))?; + let draft = v["isDraft"].as_bool().ok_or_else(|| format!("gh release view gave no isDraft: {json}"))?; + let assets = v["assets"].as_array().ok_or_else(|| format!("gh release view gave no assets: {json}"))?; + let image = assets.iter().any(|a| a["name"].as_str() == Some(IMAGE_ASSET)); + Ok(Some(Held { draft, image })) +} + +/// Every release, as (tag, creation time). +fn listed(root: &Path) -> Result, String> { + let limit = LISTED.to_string(); + releases(&gh(root, &["release", "list", "--limit", &limit, "--json", "tagName,createdAt"])?) +} + +/// [`listed`] of what `gh release list --json tagName,createdAt` answered, +/// refusing a listing that filled [`LISTED`] or an entry without both. +fn releases(json: &str) -> Result, String> { + let v: Value = serde_json::from_str(json).map_err(|e| format!("gh release list: {e}"))?; + let all = v.as_array().ok_or_else(|| format!("gh release list gave no list: {json}"))?; + if all.len() >= LISTED { + return Err(format!("gh release list gave {} releases, its limit, so some are not in it", all.len())); + } + all.iter() + .map(|r| match (r["tagName"].as_str(), r["createdAt"].as_str()) { + (Some(tag), Some(at)) => Ok((tag.to_string(), at.to_string())), + _ => Err(format!("gh release list gave a release without a tag and a time: {r}")), + }) + .collect() +} + +/// Upload the assets in `staged` as a draft of `tag`, and publish it once +/// GitHub holds the image, so a failed upload leaves nothing public. +fn create(root: &Path, tag: &str, commit: &str, staged: &Path) -> Result<(), String> { + let notes = staged.join(NOTES_ASSET).display().to_string(); + let mut args: Vec<&str> = + vec!["release", "create", tag, "--draft", "--title", tag, "--target", commit, "--notes-file", ¬es]; + let assets: Vec = ASSETS.iter().map(|a| staged.join(a).display().to_string()).collect(); + args.extend(assets.iter().map(String::as_str)); + gh(root, &args)?; + gh(root, &["release", "edit", tag, "--draft=false", "--latest"])?; + Ok(()) +} + +/// The commit this run is of, which the checkout has to be, as its [`tag`] +/// and itself. +fn this_run(root: &Path) -> Result<(String, String), String> { + let commit = std::env::var("GITHUB_SHA").map_err(|_| "GITHUB_SHA is unset".to_string())?; + let head = crate::pr::git(root, &["rev-parse", "HEAD"])?; + if head != commit { + return Err(format!("the checkout is {head} and the run is of {commit}")); + } + Ok((tag(&commit)?, commit)) +} + +/// `cargo run -- --ci release`: build the release image once, boot a copy of +/// it under the Linux line, stage its assets in [`STAGED`], and answer their +/// [`digest`] as the step's `digest` output. +pub fn release(root: &Path) -> Result { + let (tag, commit) = this_run(root)?; + let host = Host::LinuxKvm; + let boot = crate::build::Boot::release(root); + let plan = crate::build::plan_for(root, &boot, false, &[]); + let image = crate::build::build(root, boot, false, &plan); + let scratch = toyos_tmpdir::TempDir::new("image-release"); + let stick = scratch.join("stick.img"); + fs::copy(&image, &stick).map_err(|e| format!("copy {} to {}: {e}", image.display(), stick.display()))?; + boots(root, host, &stick, DESKTOP, &scratch.join("qemu.stderr"))?; + + let staged = root.join(STAGED); + if staged.exists() { + fs::remove_dir_all(&staged).map_err(|e| format!("remove {}: {e}", staged.display()))?; + } + fs::create_dir_all(&staged).map_err(|e| format!("create {}: {e}", staged.display()))?; + let licences = root.join(crate::build::RELEASE_NOTICES); + write_assets(root, &image, &licences, &staged, &tag, &commit)?; + let digest = digest(&staged)?; + crate::ci::output("digest", &digest)?; + Ok(format!("{tag} booted to its desktop under {host:?}'s line; its assets are staged, digest {digest}")) +} + +/// `cargo run -- --ci release-publish`: the assets [`release`] staged, refused +/// unless their [`digest`] is the one it answered, published on `main` unless +/// this commit's are; then the image releases past [`KEEP`] deleted. +pub fn publish(root: &Path) -> Result { + let (tag, commit) = this_run(root)?; + let staged = root.join(STAGED); + let handed = std::env::var(DIGEST_VAR).map_err(|_| format!("{DIGEST_VAR} is unset"))?; + let digest = digest(&staged)?; + if digest != handed { + return Err(format!("the assets' digest is {digest}, and the release job answered {handed:?}")); + } + if std::env::var("GITHUB_REF").ok().as_deref() != Some("refs/heads/main") { + return Ok(format!("{tag}'s assets arrived as the release job staged them; off main, so not published")); + } + let mut said = match held(root, &tag)? { + Some(Held { draft: false, image: true }) => format!("{tag} is already published"), + Some(other) => return Err(format!("GitHub holds {tag} as {other:?}: a failed run's, to delete by hand")), + None => { + create(root, &tag, &commit, &staged)?; + format!("{tag} is published") + } + }; + let old = stale(&listed(root)?, KEEP); + for old in &old { + gh(root, &["release", "delete", old, "--cleanup-tag", "--yes"])?; + } + said.push_str(&format!("; {} older image release(s) deleted, {KEEP} kept", old.len())); + Ok(said) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + } + + fn notes_of_a_commit() -> String { + notes(&root(), "image-x86_64-c55189490123", &"c".repeat(40)).unwrap() + } + + /// Only `gh`'s own not-found answer reads as nothing published; a failure + /// of any other kind is one, and a draft is told from a release. + #[test] + fn a_gh_failure_is_not_read_as_nothing_published() { + let gh = |said: &str| Err(format!("gh release view image-x86_64-c55189490123 exited exit status: 1: {said}")); + assert_eq!(held_of(gh("release not found")), Ok(None)); + let why = held_of(gh("non-200 OK status code: 401 Unauthorized body: \"Bad credentials\"")).unwrap_err(); + assert!(why.contains("401"), "{why}"); + let json = |draft: bool| Ok(format!(r#"{{"isDraft":{draft},"assets":[{{"name":"{IMAGE_ASSET}"}}]}}"#)); + assert_eq!(held_of(json(true)), Ok(Some(Held { draft: true, image: true }))); + assert_eq!(held_of(json(false)), Ok(Some(Held { draft: false, image: true }))); + assert_eq!(held_of(Ok(r#"{"isDraft":false,"assets":[]}"#.into())), Ok(Some(Held { draft: false, image: false }))); + assert!(held_of(Ok(r#"{"assets":[]}"#.into())).is_err()); + } + + /// A listing that filled its limit, or an entry without a tag and a time, + /// is refused rather than read short. + #[test] + fn a_listing_that_may_have_left_releases_out_is_refused() { + let entry = |tag: &str| format!(r#"{{"tagName":"{tag}","createdAt":"2026-09-27T03:00:00Z"}}"#); + let list = |n: usize| format!("[{}]", (0..n).map(|i| entry(&format!("t{i}"))).collect::>().join(",")); + assert_eq!(releases(&list(LISTED - 1)).unwrap().len(), LISTED - 1); + assert!(releases(&list(LISTED)).unwrap_err().contains("its limit")); + let why = releases(r#"[{"tagName":"t"}]"#).unwrap_err(); + assert!(why.contains("without a tag and a time"), "{why}"); + } + + #[test] + fn a_tag_is_the_commit_and_a_short_or_foreign_id_is_refused() { + let commit = "c55189490123456789abcdef0123456789abcdef"; + assert_eq!(tag(commit).unwrap(), "image-x86_64-c55189490123"); + assert!(tag("c5518949").is_err()); + assert!(tag(&commit.to_uppercase()).is_err()); + } + + /// Only image releases are judged, the newest are kept by creation time, + /// and the toolchain's releases are never named. + #[test] + fn retention_deletes_only_image_releases_past_the_newest_kept() { + let at = |day: u32| format!("2026-09-{day:02}T03:00:00Z"); + let mut listed: Vec<(String, String)> = + (1..=10).map(|day| (format!("{TAG_PREFIX}{day:012}"), at(day))).collect(); + listed.push(("toolchain-linux-x86_64-0123456789abcdef".into(), at(1))); + let old = stale(&listed, 7); + assert_eq!(old, [3, 2, 1].map(|day| format!("{TAG_PREFIX}{day:012}"))); + assert!(stale(&listed[..7], 7).is_empty()); + assert_eq!(stale(&listed, 0).len(), 10); + } + + /// Each host's command boots the image it is given as the one stick, on + /// that host's own accelerator and firmware, and the notes print it whole. + #[test] + fn the_notes_print_each_hosts_command_line_whole() { + let notes = notes_of_a_commit(); + for host in Host::ALL { + let argv = host.command(IMAGE); + assert!(argv.iter().any(|a| a == &format!("if=none,id=stick,format=raw,file={IMAGE}"))); + assert!(argv.iter().any(|a| a == host.accel().name())); + for firmware in host.firmware() { + assert!(argv.iter().any(|a| a.contains(firmware)), "{host:?}: {firmware}"); + } + let printed = shell(&argv); + assert!(notes.contains(&printed), "{host:?}'s command is not in the notes"); + let words: Vec<&str> = printed.split_whitespace().filter(|w| *w != "\\").collect(); + assert_eq!(words, argv.iter().map(String::as_str).collect::>()); + } + } + + /// Before the line that writes the stick, the notes name the firmware + /// variable a boot leaves behind by the name and vendor the loader writes + /// it under, how it is removed, and `BootNext`; and they name everything + /// the release leaves out. + #[test] + fn the_notes_name_what_a_boot_writes_to_the_firmware_before_the_stick_is_written() { + let notes = notes_of_a_commit(); + let (floor, hex) = floor_variable(); + let dd = notes.find(" dd if=").expect("no dd line"); + let named = format!("`{floor}` and {hex} hex digits"); + let vendor = format!("`{}`", toyos_update::floor::VENDOR); + for said in [named.as_str(), &vendor, "dmpstore -d", "`BootNext`", "the next restart boots the stick once", "unmountDisk"] { + let at = notes.find(said).unwrap_or_else(|| panic!("the notes never say {said:?}")); + assert!(at < dd, "{said:?} comes after the dd line"); + } + for withheld in pending_owner() { + let (Subject::Crate(path) | Subject::Notice(path) | Subject::File(path)) = withheld; + assert!(notes.contains(&format!("`{path}`")), "the notes do not say {path} is left out"); + } + } + + /// The one disk the notes say a boot may write though it was not given it + /// is cited by an issue that is open, so the sentence goes when the defect + /// does. + #[test] + fn the_notes_cite_the_open_defect_behind_the_disk_a_boot_may_write() { + assert!(notes_of_a_commit().contains(&format!("`{TWO_STICKS}`"))); + let issue = fs::read_to_string(root().join(TWO_STICKS)).unwrap_or_else(|e| panic!("{TWO_STICKS}: {e}")); + assert!(issue.contains("\nstatus: open\n"), "{TWO_STICKS} is not open"); + } + + /// Four assets written into a directory of their own. + fn staged(dir: &Path) -> PathBuf { + let image = dir.join("in.img"); + let bytes: Vec = (0..300_000u32).map(|i| (i % 251) as u8).chain(std::iter::repeat_n(0, 1 << 20)).collect(); + fs::write(&image, &bytes).unwrap(); + let licences = dir.join("notice.txt"); + fs::write(&licences, "the notice").unwrap(); + let out = dir.join("out"); + fs::create_dir(&out).unwrap(); + write_assets(&root(), &image, &licences, &out, "image-x86_64-c55189490123", &"c".repeat(40)).unwrap(); + out + } + + /// What `sha256sum -c` checks is the compressed asset's own digest, the + /// asset decompresses to the image byte for byte, and the notice is + /// carried as it was written. + #[test] + fn the_sum_is_the_compressed_images_and_it_decompresses_to_the_image() { + use std::io::Read; + let dir = toyos_tmpdir::TempDir::new("image-assets"); + let out = staged(&dir); + assert_eq!(fs::read_to_string(out.join(LICENCES_ASSET)).unwrap(), "the notice"); + let gz = fs::read(out.join(IMAGE_ASSET)).unwrap(); + let sum = sha256_hex(&gz); + assert_eq!(fs::read_to_string(out.join(SUMS_ASSET)).unwrap(), format!("{sum} {IMAGE_ASSET}\n")); + let mut back = Vec::new(); + flate2::read::GzDecoder::new(&gz[..]).read_to_end(&mut back).unwrap(); + assert!(back == fs::read(dir.join("in.img")).unwrap(), "the asset does not decompress to the image"); + } + + /// The digest the publish job checks moves with a byte of any asset, and a + /// directory holding more or fewer files than the assets is refused. + #[test] + fn the_digest_covers_every_asset_and_refuses_anything_else() { + let dir = toyos_tmpdir::TempDir::new("image-digest"); + let out = staged(&dir); + let whole = digest(&out).unwrap(); + for asset in ASSETS { + let at = out.join(asset); + let bytes = fs::read(&at).unwrap(); + let mut changed = bytes.clone(); + changed[0] ^= 1; + fs::write(&at, &changed).unwrap(); + assert_ne!(digest(&out).unwrap(), whole, "{asset} is outside the digest"); + fs::write(&at, &bytes).unwrap(); + } + assert_eq!(digest(&out).unwrap(), whole); + fs::write(out.join("extra"), "").unwrap(); + assert!(digest(&out).unwrap_err().contains("extra")); + fs::remove_file(out.join("extra")).unwrap(); + fs::remove_file(out.join(NOTES_ASSET)).unwrap(); + assert!(digest(&out).is_err(), "a directory without the notes was digested"); + } +} diff --git a/src/lib.rs b/src/lib.rs index 2ef6e4d30d..524be5b4f5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,6 +18,7 @@ pub mod hostws; pub mod icmp; pub mod identity; pub mod image; +pub mod imagerelease; pub mod signing; /// Which kernel containers may be hashed, and by whose keys; read by nothing /// but its own tests. diff --git a/src/licence.rs b/src/licence.rs index a8dc1c73b9..49179f5cce 100644 --- a/src/licence.rs +++ b/src/licence.rs @@ -40,6 +40,11 @@ //! section. And std's graph is re-locked when the gate runs, not read from a //! committed lock, so two runs at one head can resolve it differently. //! +//! **The same walk writes an image's licence notice** ([`notices`]): each +//! package it reaches with its own licence files, or with the standard texts +//! of its licence where it publishes none, and each `NOTICE` section over a +//! file that ships, with the texts that section names. +//! //! The expression grammar is SPDX's (`OR`, `AND`, `WITH`, parentheses) plus //! cargo's legacy `/` for `OR`. An identifier outside the allowlist is refused //! whether SPDX knows it or not, so no licence list is needed to refuse the @@ -51,6 +56,7 @@ use std::process::Command; use serde_json::Value; +use crate::arch::Arch; use crate::build::Features; /// What a shipped crate or file may be under. `OR` passes if any branch does, @@ -194,6 +200,15 @@ pub const EXCEPTIONS: &[Exception] = &[ }, ]; +/// Everything that ships while the owner has yet to rule whether it may, which +/// is what a public release leaves out (`build::Boot::release`). +pub fn pending_owner() -> impl Iterator { + EXCEPTIONS + .iter() + .filter(|e| matches!(e.standing, Standing::PendingOwner(_))) + .map(|e| e.subject) +} + /// Every committed file whose terms somebody had to establish, with the digest /// of what is committed and where the terms are recorded. /// @@ -832,9 +847,32 @@ struct Local { scripts: BTreeSet, } -/// Judge every package `roots` reach in one metadata document, and return the -/// path packages among them. -fn judge_crates(metadata: &Value, roots: &[PathBuf], report: &mut Report) -> Result { +/// A package the walk reached, as its notice names it. +struct Reached { + name: String, + version: String, + licence: Option, + /// `cargo metadata`'s `source`: `None` for a path package. + source: Option, + dir: PathBuf, + licence_file: Option, + authors: Vec, + /// The `cfg` or target of every edge into it, `None` for an unconditional + /// one and for a root. + into: BTreeSet>, +} + +/// Every package the walk reached, by name, version and origin. +type ReachedBy = BTreeMap<(String, String, String), Reached>; + +/// Judge every package `roots` reach in one metadata document, record each in +/// `reached_by`, and return the path packages among them. +fn judge_crates( + metadata: &Value, + roots: &[PathBuf], + report: &mut Report, + reached_by: &mut ReachedBy, +) -> Result { let graph = Graph::new(metadata)?; let ids = roots .iter() @@ -860,10 +898,26 @@ fn judge_crates(metadata: &Value, roots: &[PathBuf], report: &mut Report) -> Res let version = str_of(package, "version").unwrap_or("?"); let manifest = str_of(package, "manifest_path").unwrap_or("?"); let source = str_of(package, "source"); + let dir = Path::new(manifest) + .parent() + .ok_or_else(|| format!("{name}'s manifest {manifest} is in no directory"))?; + let authors = package["authors"].as_array().map(Vec::as_slice).unwrap_or(&[]); + reached_by + .entry((name.to_string(), version.to_string(), source.unwrap_or(manifest).to_string())) + .or_insert_with(|| Reached { + name: name.to_string(), + version: version.to_string(), + licence: str_of(package, "license").map(String::from), + source: source.map(String::from), + dir: dir.to_path_buf(), + licence_file: str_of(package, "license_file").map(String::from), + authors: authors.iter().filter_map(Value::as_str).map(String::from).collect(), + into: BTreeSet::new(), + }) + .into + .extend(into[dep].iter().cloned()); if source.is_none() { - if let Some(dir) = Path::new(manifest).parent() { - local.dirs.insert(dir.to_path_buf()); - } + local.dirs.insert(dir.to_path_buf()); let targets = package["targets"].as_array().map(Vec::as_slice).unwrap_or(&[]); local.scripts.extend( targets @@ -956,17 +1010,22 @@ fn judge_files(ledger: &[Row], tracked: &[String], shipping: &Shipping, report: // --- NOTICE ------------------------------------------------------------------ -/// One `NOTICE` section: its heading, the path the heading starts with, and the -/// SPDX lines it carries. +/// One `NOTICE` section: its heading, the path the heading starts with, the +/// SPDX lines it carries, and the files its terms are written in. #[derive(Debug, PartialEq)] struct Section { heading: String, path: String, spdx: Vec, + /// The path each `Licence text:` or `Terms:` line starts with. + texts: Vec, } const SPDX_TAG: &str = "SPDX-License-Identifier:"; +/// What a section names the file its terms are written in with. +const TEXT_TAGS: [&str; 2] = ["Licence text:", "Terms:"]; + /// The `-`-underlined sections of `text`, in order. fn sections(text: &str) -> Vec
{ let lines: Vec<&str> = text.lines().collect(); @@ -980,11 +1039,17 @@ fn sections(text: &str) -> Vec
{ heading: line.trim().to_string(), path: line.split_whitespace().next().unwrap_or("").to_string(), spdx: Vec::new(), + texts: Vec::new(), }); - } else if let (Some(section), Some(expr)) = - (out.last_mut(), line.trim().strip_prefix(SPDX_TAG)) - { - section.spdx.push(expr.trim().to_string()); + } else if let Some(section) = out.last_mut() { + let line = line.trim(); + if let Some(expr) = line.strip_prefix(SPDX_TAG) { + section.spdx.push(expr.trim().to_string()); + } + let rest = TEXT_TAGS.iter().find_map(|tag| line.strip_prefix(tag)); + if let Some(path) = rest.and_then(|rest| rest.split_whitespace().next()) { + section.texts.push(path.trim_end_matches(',').to_string()); + } } } out @@ -1151,22 +1216,103 @@ fn metadata( serde_json::from_slice(&out).map_err(|e| format!("cargo metadata printed no JSON: {e}")) } -/// The fork's `library/`, checked out at the commit this tree pins. A checkout -/// whose `rust/` was never initialised — a CI runner's — fetches that commit -/// alone. -fn std_library(root: &Path) -> Result { - let fork = crate::sysroot::fork_checkout(root); - if !fork.join("library/Cargo.toml").exists() { - run( - Command::new("git") - .args(["submodule", "update", "--init", "--depth", "1", "rust"]) - .current_dir(root), - "git submodule update --init --depth 1 rust", - )?; +/// The fork's `library/` at the commit this tree pins: a linked worktree's +/// fork checkout, `rust/library` where this checkout holds it, and everywhere +/// else — a runner, an installed toolchain — [`fetched_library`]. Asks the +/// network, so it is called with no build lock held. +pub fn std_library(root: &Path) -> Result { + if let crate::toolchain::Owner::Elsewhere(_) = crate::toolchain::owner(root) { + return Ok(crate::sysroot::fork_checkout(root).join("library")); + } + let library = root.join("rust/library"); + if library.join("Cargo.toml").is_file() { + return Ok(library); + } + fetched_library(root) +} + +/// Where the fork is fetched to where `rust/` holds no source. A directory of +/// the checkout's own, because std's manifest names `toyos` and `toyos-abi` +/// three levels above its crates; hidden and ignored, as `.build-locks/` is. +const FETCHED_FORK: &str = ".licence-fork"; + +/// Where a fetch is made before it is renamed into place, and where the fork +/// it replaces goes before it is removed: on the checkout's own filesystem. +const FETCHING: &str = "target/licence/fork.partial"; +const REPLACED: &str = "target/licence/fork.replaced"; + +/// What of the fork is checked out: its `library/`, and every top-level entry +/// whose name [`LICENCE_FILES`] starts, in any case. +fn sparse() -> Vec { + let any_case = |name: &str| -> String { name.chars().map(|c| format!("[{c}{}]", c.to_ascii_uppercase())).collect() }; + std::iter::once("/library/".to_string()) + .chain(LICENCE_FILES.iter().map(|name| format!("/{}*", any_case(name)))) + .collect() +} + +/// The fork's `library/` and its own licence files at the pinned commit, and +/// nothing else of it. Fetched beside its place and renamed into it, so what +/// is there is always a whole fetch; a source tree at `rust/` would make the +/// toolchain the checkout's own to build (`toolchain::owner`). +fn fetched_library(root: &Path) -> Result { + let commit = crate::sysroot::pinned_fork(root); + let fork = root.join(FETCHED_FORK); + let git = |dir: &Path, args: &[&str]| -> Result, String> { + run(Command::new("git").args(args).current_dir(dir), &format!("git {}", args.join(" "))) + }; + let patterns = sparse(); + let sparse: String = patterns.iter().map(|p| format!("{p}\n")).collect(); + let pinned = || -> Result { + Ok(fork.is_dir() + && git(&fork, &["rev-parse", "HEAD"])? == format!("{commit}\n").into_bytes() + && git(&fork, &["sparse-checkout", "list"])? == sparse.as_bytes()) + }; + if pinned()? { + return Ok(fork.join("library")); + } + let _fetching = crate::buildlock::licence(root); + if pinned()? { + return Ok(fork.join("library")); + } + let url = fork_url(root)?; + let (partial, replaced) = (root.join(FETCHING), root.join(REPLACED)); + // Under the lock, either is only ever what a call cut short left. + for dir in [&partial, &replaced] { + if dir.exists() { + std::fs::remove_dir_all(dir).map_err(|e| format!("remove {}: {e}", dir.display()))?; + } + } + std::fs::create_dir_all(&partial).map_err(|e| format!("create {}: {e}", partial.display()))?; + git(&partial, &["init", "-q"])?; + git(&partial, &["fetch", "-q", "--depth", "1", "--filter=blob:none", &url, &commit])?; + let mut set = vec!["sparse-checkout", "set", "--no-cone"]; + set.extend(patterns.iter().map(String::as_str)); + git(&partial, &set)?; + git(&partial, &["checkout", "-q", "FETCH_HEAD"])?; + let rename = |from: &Path, to: &Path| { + std::fs::rename(from, to).map_err(|e| format!("rename {} to {}: {e}", from.display(), to.display())) + }; + if fork.exists() { + rename(&fork, &replaced)?; + } + rename(&partial, &fork)?; + if replaced.exists() { + std::fs::remove_dir_all(&replaced).map_err(|e| format!("remove {}: {e}", replaced.display()))?; } Ok(fork.join("library")) } +/// The URL `.gitmodules` gives the fork. +fn fork_url(root: &Path) -> Result { + let out = run( + Command::new("git") + .args(["config", "-f", ".gitmodules", "submodule.rust.url"]) + .current_dir(root), + "git config -f .gitmodules submodule.rust.url", + )?; + Ok(String::from_utf8_lossy(&out).trim().to_string()) +} + /// One metadata document, and the shipped crates judged out of it. struct Doc { metadata: Value, @@ -1175,13 +1321,27 @@ struct Doc { roots: Vec, } -/// The gate `cargo run -- --ci host` runs. -pub fn judge(root: &Path) -> Result { - // Cargo names every manifest by its canonical path. - let root = &std::fs::canonicalize(root).map_err(|e| format!("{}: {e}", root.display()))?; - let shipped = crate::build::shipped(root)?; - let mut report = Report::default(); +/// What the licence walk read out of what ships. +struct Walk { + reached: ReachedBy, + /// The fork's `library/` std was resolved from. + library: PathBuf, + shipping: Shipping, + tracked: Vec, + sections: Vec
, + /// The tracked files each section's path names. + files: BTreeMap>, +} +/// Walk every crate `shipped` names, libc, and std out of `library`, judging +/// each package into `report`, and read where committed files ship from. +/// `root` is canonical: cargo names every manifest by its canonical path. +fn walk( + root: &Path, + shipped: crate::build::Shipped, + library: &Path, + report: &mut Report, +) -> Result { let mut roots: Vec<(PathBuf, Features)> = shipped.crates.into_iter().collect(); roots.push(( root.join(crate::libc::CRATE), @@ -1213,10 +1373,11 @@ pub fn judge(root: &Path) -> Result { // committed lock is stale by design: it is re-locked into a scratch copy, // and the fork's is never written. `RUSTC_BOOTSTRAP` because the fork's // manifests use cargo features a stable cargo otherwise refuses. - let library = std_library(root)?; + let library = canonical(library)?; let scratch = root.join("target/licence"); std::fs::create_dir_all(&scratch).map_err(|e| format!("create {}: {e}", scratch.display()))?; let lock = scratch.join("Cargo.lock"); + let held = crate::buildlock::licence(root); std::fs::copy(library.join("Cargo.lock"), &lock) .map_err(|e| format!("copy std's Cargo.lock: {e}"))?; let lockfile = format!("resolver.lockfile-path={:?}", lock.display().to_string()); @@ -1228,6 +1389,7 @@ pub fn judge(root: &Path) -> Result { &args, &[("RUSTC_BOOTSTRAP", "1")], )?; + drop(held); docs.push(Doc { members: members(&std_doc)?, metadata: std_doc, @@ -1236,8 +1398,9 @@ pub fn judge(root: &Path) -> Result { }); let mut local = Local::default(); + let mut reached = ReachedBy::new(); for doc in &docs { - let found = judge_crates(&doc.metadata, &doc.roots, &mut report)?; + let found = judge_crates(&doc.metadata, &doc.roots, report, &mut reached)?; local.dirs.extend(found.dirs); local.scripts.extend(found.scripts); } @@ -1259,7 +1422,6 @@ pub fn judge(root: &Path) -> Result { .map_err(|e| format!("read {}: {e}", file.display()))?; shipping.named.extend(embeds(&text, script, &names).into_iter().map(String::from)); } - judge_files(COMMITTED_FILES, &tracked, &shipping, &mut report); let notice = std::fs::read_to_string(root.join("NOTICE")).map_err(|e| format!("read NOTICE: {e}"))?; @@ -1269,11 +1431,221 @@ pub fn judge(root: &Path) -> Result { let named = crate::sysroot::tracked_files(root, &[&format!(":(glob){}", section.path)])?; files.insert(section.path.clone(), named); } - judge_notice(§ions, &files, COMMITTED_FILES, &shipping, &mut report); + Ok(Walk { reached, library, shipping, tracked, sections, files }) +} + +fn canonical(root: &Path) -> Result { + std::fs::canonicalize(root).map_err(|e| format!("{}: {e}", root.display())) +} +/// The gate `cargo run -- --ci host` runs. +pub fn judge(root: &Path) -> Result { + let root = &canonical(root)?; + let mut report = Report::default(); + let walk = walk(root, crate::build::shipped(root)?, &std_library(root)?, &mut report)?; + judge_files(COMMITTED_FILES, &walk.tracked, &walk.shipping, &mut report); + judge_notice(&walk.sections, &walk.files, COMMITTED_FILES, &walk.shipping, &mut report); verdict(report, EXCEPTIONS) } +// --- The notice -------------------------------------------------------------- + +/// Where [`notices`] is on an image's ROOT. +pub const NOTICES_ON_ROOT: &str = "share/licences.txt"; + +/// What a file carrying a package's licence is called, lower-cased, at its start. +const LICENCE_FILES: &[&str] = &["license", "licence", "unlicense", "copying", "copyright", "notice"]; + +/// The one registry a package's source can be named in. +const CRATES_IO: &str = "registry+https://github.com/rust-lang/crates.io-index"; + +/// The files carrying `p`'s licence: every regular file its directory holds +/// whose name [`LICENCE_FILES`] starts, and the one its `license_file` names. +/// A package outside a registry with none carries its repository's: those of +/// the nearest enclosing directory holding any, up to the one holding `.git`. +fn licence_files(p: &Reached) -> Result, String> { + let registry = p.source.as_deref().is_some_and(|s| s.starts_with("registry+")); + let mut dir = p.dir.clone(); + loop { + let mut found = Vec::new(); + for entry in std::fs::read_dir(&dir).map_err(|e| format!("{}: {e}", dir.display()))? { + let path = entry.map_err(|e| format!("{}: {e}", dir.display()))?.path(); + let name = file_name(&path.to_string_lossy()).to_lowercase(); + // The fork's `license-metadata.json` is REUSE's data about its texts, and no text. + if LICENCE_FILES.iter().any(|n| name.starts_with(n)) && !name.ends_with(".json") && path.is_file() { + found.push(path); + } + } + if let (true, Some(file)) = (dir == p.dir, &p.licence_file) { + let file = dir.join(file); + if !file.is_file() { + return Err(format!("{} {} names {} as its licence file, and it is none", p.name, p.version, file.display())); + } + found.push(file); + } + found.sort(); + found.dedup(); + if !found.is_empty() { + return Ok(found); + } + if registry || dir.join(".git").exists() || !dir.pop() { + return Err(format!("{} {} ({}) carries no licence file", p.name, p.version, p.dir.display())); + } + } +} + +/// The standard texts of the licences `p` declares, from the fork's +/// `LICENSES/`, which holds one `.txt` per licence: each of an `AND`, +/// an exception with its licence, and of an `OR` its Apache-2.0 branch where +/// it has one, else the first branch held whole. `None` where no branch is. +fn standard_texts(p: &Reached, fork: &Path) -> Option> { + fn held(expr: &Expr, dir: &Path) -> Option> { + let text = |id: &str| Some(dir.join(format!("{id}.txt"))).filter(|f| f.is_file()); + match expr { + Expr::Id(id) => Some(vec![text(id)?]), + Expr::With(id, exception) => Some(vec![text(id)?, text(exception)?]), + Expr::And(parts) => Some(parts.iter().map(|p| held(p, dir)).collect::>>()?.concat()), + // Apache-2.0's text is whole as it stands; MIT's asks for a + // copyright line that a package publishing no text never gave. + Expr::Or(parts) => { + let apache = parts.iter().filter(|p| matches!(p, Expr::Id(id) if id == "Apache-2.0")); + apache.chain(parts).find_map(|p| held(p, dir)) + } + } + } + held(&parse(p.licence.as_deref()?).ok()?, &fork.join("LICENSES")) +} + +/// Every licence text a notice carries, each once, by its first carrier. +#[derive(Default)] +struct Texts { + ids: BTreeMap, + listed: Vec<(String, String)>, +} + +impl Texts { + fn id(&mut self, file: &Path, carrier: &str) -> Result { + let bytes = std::fs::read(file).map_err(|e| format!("{}: {e}", file.display()))?; + let text = String::from_utf8_lossy(&bytes).into_owned(); + let next = self.listed.len() + 1; + let id = *self.ids.entry(text.clone()).or_insert(next); + if id == next { + self.listed.push((format!("{carrier}: {}", file_name(&file.to_string_lossy())), text)); + } + Ok(id) + } + + fn cite(&mut self, files: &[PathBuf], carrier: &str) -> Result { + let ids = files.iter().map(|f| self.id(f, carrier).map(|id| format!("[{id}]"))); + Ok(ids.collect::, _>>()?.join(" ")) + } +} + +/// The licence notice of an image built from `shipped` with std out of +/// `library` ([`std_library`]): every package the walk reaches with its +/// licence, where its source is and the texts it carries; every `NOTICE` +/// section over a file it ships, with its terms and texts, less what +/// [`pending_owner`] names; and each text once. Refused while any package +/// carries no licence text. +pub fn notices(root: &Path, shipped: crate::build::Shipped, library: &Path) -> Result { + let root = &canonical(root)?; + let walk = walk(root, shipped, library, &mut Report::default())?; + let fork = walk.library.parent().ok_or("std's library/ is in no directory")?; + let (url, commit) = (fork_url(root)?, crate::sysroot::pinned_fork(root)); + let mut texts = Texts::default(); + let mut refused = Vec::new(); + let mut out = String::from( + "Licence notices\n===============\n\n\ + Every package in the dependency graphs of this image's kernel, loader and programs,\n\ + less those depended on only under another target's triple, with its licence, where\n\ + its source is, and its licence texts: some of them are compiled only for other\n\ + platforms. Then every third-party file on the image, with its terms; and each of\n\ + those texts once, at the end.\n\n\ + Packages\n--------\n", + ); + let guest: Vec<&str> = Arch::ALL.iter().flat_map(|a| [a.userland(), a.kernel(), a.loader()]).collect(); + for p in walk.reached.values() { + // A package every edge into which names another target by its triple + // is linked on none of ours. + let foreign = p.into.iter().all(|edge| { + edge.as_deref() + .is_some_and(|t| t.split(" | ").all(|t| !t.starts_with("cfg(") && !guest.contains(&t))) + }); + if foreign { + continue; + } + let (files, standard) = match licence_files(p) { + Ok(files) => (files, ""), + Err(why) => match standard_texts(p, fork) { + Some(files) => (files, ", the licences' standard texts, since the package carries none"), + None => { + refused.push(format!("{why}, and the fork's LICENSES/ holds no branch of its licence")); + continue; + } + }, + }; + let source = match (&p.source, p.dir.strip_prefix(fork), p.dir.strip_prefix(root)) { + (Some(s), ..) if s == CRATES_IO => { + format!("https://static.crates.io/crates/{0}/{0}-{1}.crate", p.name, p.version) + } + (Some(s), ..) => match s.strip_prefix("git+") { + Some(git) => git.to_string(), + None => return Err(format!("{} {} is from {s}, which no notice names a download in", p.name, p.version)), + }, + (None, Ok(at), _) => format!("{url} at {commit}, {}", at.display()), + (None, _, Ok(at)) => format!("the ToyOS source this image was built from, {}", at.display()), + (None, ..) => return Err(format!("{} {} is a path package outside {}", p.name, p.version, root.display())), + }; + let carrier = match standard { + "" => format!("{} {}", p.name, p.version), + _ => "the Rust fork's LICENSES".to_string(), + }; + let cited = texts.cite(&files, &carrier)?; + let licence = p.licence.as_deref().unwrap_or("none declared"); + let authors = match p.authors.as_slice() { + [] => String::new(), + all => format!("\n authors: {}", all.join(", ")), + }; + out.push_str(&format!( + "\n{} {}\n licence: {licence}\n source: {source}{authors}\n texts: {cited}{standard}\n", + p.name, p.version + )); + } + if !refused.is_empty() { + return Err(format!("{} shipped package(s) carry no licence text:\n {}", refused.len(), refused.join("\n "))); + } + + out.push_str("\nFiles\n-----\n"); + let withheld: Vec<&str> = pending_owner() + .filter_map(|s| match s { + Subject::Notice(path) | Subject::File(path) => Some(path), + Subject::Crate(_) => None, + }) + .collect(); + for section in &walk.sections { + let named = walk.files.get(§ion.path).map(Vec::as_slice).unwrap_or(&[]); + if withheld.contains(§ion.path.as_str()) || !named.iter().any(|f| walk.shipping.ships(f)) { + continue; + } + let files: Vec = section.texts.iter().map(|t| root.join(t)).collect(); + let cited = match texts.cite(&files, §ion.path)? { + cited if cited.is_empty() => cited, + cited => format!("\n texts: {cited}"), + }; + out.push_str(&format!( + "\n{}\n licence: {}{cited}\n", + section.heading, + section.spdx.join(" AND ") + )); + } + + out.push_str("\nTexts\n-----\n"); + for (at, (carrier, text)) in texts.listed.iter().enumerate() { + out.push_str(&format!("\n[{}] {carrier}\n\n{}\n", at + 1, text.trim_end())); + } + Ok(out) +} + #[cfg(test)] mod tests { use super::*; @@ -1413,7 +1785,7 @@ mod tests { fn crates(doc: &Value) -> Report { let mut report = Report::default(); - judge_crates(doc, &[PathBuf::from("/t/app")], &mut report).expect("judged"); + judge_crates(doc, &[PathBuf::from("/t/app")], &mut report, &mut ReachedBy::new()).expect("judged"); report } @@ -1652,7 +2024,7 @@ ub_checks"#; fn a_root_the_metadata_does_not_hold_is_refused() { let d = doc(&[("other", Some("MIT"), None, None)], &[]); let mut report = Report::default(); - let why = judge_crates(&d, &[PathBuf::from("/t/app")], &mut report).unwrap_err(); + let why = judge_crates(&d, &[PathBuf::from("/t/app")], &mut report, &mut ReachedBy::new()).unwrap_err(); assert!(why.contains("/t/app/Cargo.toml"), "{why}"); } @@ -1904,4 +2276,190 @@ prose. } } } + + fn reached(dir: &Path, source: Option<&str>, licence_file: Option<&str>) -> Reached { + Reached { + name: "x".into(), + version: "1.0.0".into(), + licence: Some("MIT".into()), + source: source.map(String::from), + dir: dir.to_path_buf(), + licence_file: licence_file.map(String::from), + authors: Vec::new(), + into: BTreeSet::new(), + } + } + + /// A package's own licence files are read and nothing else of its + /// directory; a registry package with none is refused by name, and a path + /// package with none carries its repository's, never past `.git`. + #[test] + fn a_package_carries_its_own_licence_files_or_its_repositorys() { + let tmp = toyos_tmpdir::TempDir::new("licence-files"); + let repo = tmp.join("repo"); + let package = repo.join("crates/x"); + std::fs::create_dir_all(package.join("src")).unwrap(); + std::fs::write(package.join("src/lib.rs"), "").unwrap(); + std::fs::write(package.join("README.md"), "").unwrap(); + let why = licence_files(&reached(&package, CRATES_IO, None)).unwrap_err(); + assert!(why.contains("x 1.0.0") && why.contains("carries no licence file"), "{why}"); + + std::fs::create_dir(repo.join(".git")).unwrap(); + std::fs::write(repo.join("LICENSE-MIT"), "mit").unwrap(); + std::fs::write(tmp.join("LICENSE"), "outside").unwrap(); + assert_eq!(licence_files(&reached(&package, None, None)).unwrap(), [repo.join("LICENSE-MIT")]); + assert!(licence_files(&reached(&package, CRATES_IO, None)).is_err(), "a registry package walked up"); + + for name in ["COPYING", "Licence.txt", "NOTICE", "UNLICENSE"] { + std::fs::write(package.join(name), name).unwrap(); + } + std::fs::write(package.join("terms.txt"), "").unwrap(); + let files = licence_files(&reached(&package, CRATES_IO, Some("terms.txt"))).unwrap(); + let names: Vec<&str> = files.iter().map(|f| file_name(f.to_str().unwrap())).collect(); + assert_eq!(names, ["COPYING", "Licence.txt", "NOTICE", "UNLICENSE", "terms.txt"]); + let why = licence_files(&reached(&package, CRATES_IO, Some("gone.txt"))).unwrap_err(); + assert!(why.contains("gone.txt"), "{why}"); + } + + /// `git ` in `dir`, which has to succeed: what it printed, trimmed. + fn git_in(dir: &Path, args: &[&str]) -> String { + let out = Command::new("git") + .args(["-c", "user.name=t", "-c", "user.email=t@t", "-c", "init.defaultBranch=main", "-c", "commit.gpgsign=false"]) + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!(out.status.success(), "git {args:?}: {}", String::from_utf8_lossy(&out.stderr)); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + + /// `root` pins the fork at `url` at `commit`, as `.gitmodules` and the + /// index say it. + fn pinning(root: &Path, url: &str, commit: &str) { + let modules = format!("[submodule \"rust\"]\n\tpath = rust\n\turl = {url}\n"); + std::fs::write(root.join(".gitmodules"), modules).unwrap(); + git_in(root, &["update-index", "--add", "--cacheinfo", &format!("160000,{commit},rust")]); + } + + /// The fetched fork is a whole fetch or nothing: one that failed leaves + /// nothing a later call refuses on, only `library/` and every top-level + /// file a name in [`LICENCE_FILES`] starts, in either case, are checked + /// out, a whole fetch is reused without the network, and a new pin + /// replaces it. + #[test] + fn a_fetch_cut_short_is_fetched_again_and_a_new_pin_replaces_the_fork() { + let tmp = toyos_tmpdir::TempDir::new("fetched-fork"); + let remote = tmp.join("fork"); + for dir in ["library", "LICENSES", "src"] { + std::fs::create_dir_all(remote.join(dir)).unwrap(); + } + let files = [ + ("library/Cargo.toml", "[workspace]\n"), + ("COPYRIGHT", "one"), + ("LICENSE-MIT", "mit"), + ("LICENSES/MIT.txt", "mit"), + ("x.py", ""), + ("src/lib.rs", ""), + ]; + // Two per name, which a case-insensitive disk still holds apart. + let licences: Vec = + LICENCE_FILES.iter().flat_map(|name| [format!("{}-A", name.to_uppercase()), format!("{name}-b")]).collect(); + let files = files.into_iter().chain(licences.iter().map(|file| (file.as_str(), "text"))); + for (file, text) in files { + std::fs::write(remote.join(file), text).unwrap(); + } + git_in(&remote, &["init", "-q"]); + git_in(&remote, &["config", "uploadpack.allowAnySHA1InWant", "true"]); + git_in(&remote, &["config", "uploadpack.allowFilter", "true"]); + git_in(&remote, &["add", "-A"]); + git_in(&remote, &["commit", "-q", "-m", "one"]); + let first = git_in(&remote, &["rev-parse", "HEAD"]); + let root = tmp.join("toyos"); + std::fs::create_dir(&root).unwrap(); + git_in(&root, &["init", "-q"]); + let url = format!("file://{}", remote.display()); + let gone = format!("file://{}", tmp.join("gone").display()); + + pinning(&root, &gone, &first); + assert!(fetched_library(&root).is_err(), "a fork nobody serves was fetched"); + pinning(&root, &url, &first); + let library = fetched_library(&root).unwrap_or_else(|why| panic!("{why}")); + let fork = library.parent().unwrap(); + let wanted = ["library/Cargo.toml", "COPYRIGHT", "LICENSE-MIT", "LICENSES/MIT.txt"]; + for file in wanted.into_iter().chain(licences.iter().map(String::as_str)) { + assert!(fork.join(file).is_file(), "{file} was not checked out"); + } + assert!(!fork.join("x.py").exists() && !fork.join("src").exists(), "more than the licence sources was checked out"); + + pinning(&root, &gone, &first); + assert_eq!(fetched_library(&root).unwrap_or_else(|why| panic!("{why}")), library, "a whole fetch was not reused"); + + std::fs::write(remote.join("COPYRIGHT"), "two").unwrap(); + git_in(&remote, &["commit", "-q", "-am", "two"]); + pinning(&root, &url, &git_in(&remote, &["rev-parse", "HEAD"])); + let library = fetched_library(&root).unwrap_or_else(|why| panic!("{why}")); + assert_eq!(std::fs::read_to_string(library.parent().unwrap().join("COPYRIGHT")).unwrap(), "two"); + } + + /// A package that publishes no text is given Apache-2.0's where its `OR` + /// offers it, wherever it stands, and otherwise the first branch the fork + /// holds. + #[test] + fn an_or_is_given_apache_2_0s_standard_text_where_it_is_a_branch() { + let fork = toyos_tmpdir::TempDir::new("standard-texts"); + std::fs::create_dir(fork.join("LICENSES")).unwrap(); + for id in ["MIT", "Apache-2.0", "BSD-3-Clause"] { + std::fs::write(fork.join(format!("LICENSES/{id}.txt")), id).unwrap(); + } + let given = |licence: &str| -> Vec { + let p = Reached { licence: Some(licence.into()), ..reached(&fork, None, None) }; + let texts = standard_texts(&p, &fork).unwrap_or_else(|| panic!("{licence}: no text")); + texts.iter().map(|t| file_name(t.to_str().unwrap()).to_string()).collect() + }; + assert_eq!(given("MIT OR Apache-2.0"), ["Apache-2.0.txt"]); + assert_eq!(given("MIT/Apache-2.0"), ["Apache-2.0.txt"]); + assert_eq!(given("Zlib OR MIT OR BSD-3-Clause"), ["MIT.txt"]); + } + + /// The release's notice, with std where the build takes it from: every + /// package its walk reaches carries a licence text, the loader's MPL crates + /// name where their source is, std carries the fork's texts, and every + /// third-party file the release ships is there while nothing pending the + /// owner is. + #[test] + fn the_release_notice_carries_every_package_and_file_it_ships() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let parts = crate::build::Boot::release(root).parts(root).unwrap(); + let library = std_library(root).unwrap_or_else(|why| panic!("{why}")); + let text = notices(root, parts, &library).unwrap_or_else(|why| panic!("{why}")); + let block = |head: &str| -> &str { + let at = text.find(&format!("\n{head}")).unwrap_or_else(|| panic!("no {head:?} in the notice")); + text[at + 1..].split("\n\n").next().unwrap() + }; + let uefi = block("uefi "); + assert!(uefi.contains("licence: MPL-2.0"), "{uefi}"); + assert!(uefi.contains("source: https://static.crates.io/crates/uefi/uefi-"), "{uefi}"); + let std = block("std "); + assert!(std.contains("source: https://github.com/ToyOSOrg/rust.git at "), "{std}"); + // std cites the Rust project's COPYRIGHT itself, not merely a notice + // where some other package carries a copy of it. + let lines: Vec<&str> = text.lines().collect(); + let copyright: Vec<&str> = lines + .windows(3) + .filter(|w| w[0].ends_with(": COPYRIGHT") && w[2] == "Short version for non-lawyers:") + .filter_map(|w| w[0].split_once(']').map(|(id, _)| id)) + .collect(); + assert!(copyright.iter().any(|id| std.contains(&format!("{id}]"))), "std cites no COPYRIGHT: {std}"); + for shipped in ["assets/JetBrainsMono-Regular.ttf", "assets/icons/*.svg", "assets/fonts/OpenSans-*.ttf"] { + assert!(block(&format!("{shipped} — ")).contains("texts: ["), "{shipped}"); + } + let listed = &text[..text.find("\nTexts\n-----\n").expect("no texts")]; + for withheld in pending_owner() { + let head = match withheld { + Subject::Crate(name) => format!("\n{name} "), + Subject::Notice(path) | Subject::File(path) => format!("\n{path} "), + }; + assert!(!listed.contains(&head), "the release's notice lists {withheld:?}"); + } + } } diff --git a/src/redlist.rs b/src/redlist.rs index f941c3eacc..b73204e8de 100644 --- a/src/redlist.rs +++ b/src/redlist.rs @@ -51,6 +51,10 @@ pub const DISABLED: &[Disabled] = &[ test: "quiesce_wakes_on_the_last_exit", issue: "issues/build/quiesce-wakes-on-the-last-exit-lost-its-serial-ready-beside-other-guests.md", }, + Disabled { + test: "release_command_boots", + issue: "issues/kernel/the-kernel-dies-at-boot-on-the-edk2-firmware-qemu-ships.md", + }, Disabled { test: "sched_check_build", issue: "issues/build/the-pass-cost-gates-ci-sample-is-eight-days-stale-twice.md", diff --git a/src/sourcegate.rs b/src/sourcegate.rs index f328ac9ed4..9c4c58f895 100644 --- a/src/sourcegate.rs +++ b/src/sourcegate.rs @@ -555,16 +555,21 @@ const HOST_SPAWNS: &[Spawn] = &[ }, Spawn { arg: "arch.qemu()", - sites: &[("src/qemu.rs", 1), ("src/ci.rs", 2), ("tests/common/qemu.rs", 1)], + sites: &[ + ("src/qemu.rs", 1), + ("src/ci.rs", 2), + ("tests/common/qemu.rs", 1), + ("src/imagerelease.rs", 1), + ], why: "QEMU, the other half of the bar: `Arch::qemu` names `qemu-system-x86_64` and \ `qemu-system-aarch64`, and `check_prerequisites` requires the one being booted", }, Spawn { arg: "\"gh\"", sites: &[], - why: "GitHub's CLI, outside the bar: CI's release, protection and nightly-red jobs \ - (src/ci.rs, src/release.rs) ask GitHub with it. Nothing that builds or boots \ - reaches it", + why: "GitHub's CLI, outside the bar: CI's releases, protection and nightly-red jobs \ + (src/ci.rs, src/release.rs, src/imagerelease.rs) ask GitHub with it. No build and no \ + boot calls it", }, Spawn { arg: "\"curl\"", @@ -724,6 +729,12 @@ const CI_PACKAGES: &[Package] = &[ name: "git", why: "the version control this repository is, and `REQUIRED` in src/main.rs", }, + Package { + name: "ovmf", + why: "the edk2 firmware Debian's QEMU boots a UEFI guest with: the image release notes' \ + Linux command line names it, and the release job boots that line \ + (src/imagerelease.rs)", + }, Package { name: "python3", why: "the Python standing failure CLAUDE.md:56 declares, wearing a package name — \ @@ -799,7 +810,13 @@ const CI_ACTIONS: &[Action] = &[ }, Action { name: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02", - why: "how a red guest job keeps its boots' serial logs (v4.6.2)", + why: "how a red guest job keeps its boots' serial logs, and how the image release's \ + boot job hands its assets to the job that publishes them (v4.6.2)", + }, + Action { + name: "actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093", + why: "the read half of the same store: the image release's publish job takes the \ + assets the boot job staged (v4.3.0)", }, Action { name: "rust-lang/crates-io-auth-action@c6f97d42243bad5fab37ca0427f495c86d5b1a18", diff --git a/src/sysroot.rs b/src/sysroot.rs index 348d300183..55bda1280d 100644 --- a/src/sysroot.rs +++ b/src/sysroot.rs @@ -201,7 +201,7 @@ pub fn key(root: &Path, compiler: &Compiler, fork: &Path) -> String { /// The commit this checkout's tree pins the std fork at: the index's, so a /// staged gitlink counts as the tree's. -fn pinned_fork(root: &Path) -> String { +pub(crate) fn pinned_fork(root: &Path) -> String { let entry = git_out(root, &["ls-files", "-s", "--", "rust"]); let mut words = entry.split_whitespace(); match (words.next(), words.next()) { diff --git a/src/testargs.rs b/src/testargs.rs index 58d0fa8c51..bc69bddb43 100644 --- a/src/testargs.rs +++ b/src/testargs.rs @@ -162,6 +162,9 @@ declare_flags!(pub SUITE = { /// machine is not touched**: the run builds the images and writes down what /// to run on them, or judges readbacks a driver already left there. pub METAL_READBACK = "--metal-readback", Next; + /// The image release's macOS command line, booted as its notes print it: + /// one test outside every tier, and the only way to run it. + pub RELEASE_COMMAND = "--release-command", None; }); /// Validate the harness's argv and return the run's filter. @@ -197,6 +200,13 @@ pub fn parse(args: &[String]) -> Result, String> { } let has = |want| SUITE.present(args, want); + if has(&RELEASE_COMMAND) && args.len() > 1 { + return Err( + "--release-command runs one test in no tier and takes no other word, which it \ + would drop in silence" + .to_string(), + ); + } if has(&JOBS) && has(&JOBS_SHORT) { return Err( "--jobs and -j are two spellings of one width, and the run would read one of \ @@ -285,6 +295,15 @@ mod tests { ); } + #[test] + fn the_release_command_takes_no_other_word() { + assert_eq!(parse_owned(&["--release-command"]).unwrap(), None); + for other in ["release_command_boots", "--nightly", "--shard=1/2"] { + let refusal = parse_owned(&["--release-command", other]).unwrap_err(); + assert!(refusal.contains("takes no other word"), "{other}: {refusal}"); + } + } + #[test] fn two_filters_are_refused_because_only_one_would_run() { let refusal = parse_owned(&["futex", "dlopen"]).unwrap_err(); diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 0c64deee56..c9dad9b097 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -43,6 +43,8 @@ pub mod metal; pub mod origin; #[allow(dead_code)] pub mod partclaim; +/// The image release's macOS command line, which `--release-command` boots. +pub mod release; #[allow(dead_code)] pub mod passcost; #[allow(dead_code)] diff --git a/tests/common/release.rs b/tests/common/release.rs new file mode 100644 index 0000000000..0684161aa3 --- /dev/null +++ b/tests/common/release.rs @@ -0,0 +1,26 @@ +//! The image release's macOS command line, booted as its notes print it +//! (`toyos_build::imagerelease::boots`) over a copy of the release image. + +use toyos_build::build::{self, Boot}; +use toyos_build::imagerelease::{self, Host}; + +/// The name `src/redlist.rs` knows this test by. +pub const NAME: &str = "release_command_boots"; + +/// The notes' Apple Silicon line boots the release image to a painting +/// desktop and leaves both firmware files as they were. Run by the suite's +/// `--release-command` and by nothing else. +pub fn release_command_boots() -> Result<(), String> { + let host = Host::MacosAppleSilicon; + let root = super::compile::repo_root(); + let boot = Boot::release(&root); + let plan = build::plan_for(&root, &boot, false, &[]); + let image = build::build(&root, boot, false, &plan); + let dir = super::lane::dir(); + let stick = dir.join("release-command.img"); + std::fs::copy(&image, &stick).map_err(|e| format!("copy the release image to {}: {e}", stick.display()))?; + let ceiling = super::qemu::budget(imagerelease::DESKTOP); + imagerelease::boots(&root, host, &stick, ceiling, &dir.join("release-command.stderr"))?; + eprintln!(" [release] {host:?}'s command line reached a painting desktop"); + Ok(()) +} diff --git a/tests/common/storage.rs b/tests/common/storage.rs index fe93ca226d..99099e0943 100644 --- a/tests/common/storage.rs +++ b/tests/common/storage.rs @@ -16,8 +16,13 @@ use toyos_build::fingerprint::{first_difference, whole_device}; use super::qemu::{self, BootOptions, QemuInstance}; -/// Boot the guest against a disk that belongs to somebody else, and prove it -/// comes back untouched. +/// Boot the guest against three disks that belong to somebody else, and prove +/// each comes back untouched: an NVMe disk whose TOYOS-DATA partition holds +/// another system's volume, a USB disk whose table names only other systems' +/// partitions, and a USB stick with no table. +/// +/// Two boots, one per USB disk, each beside the NVMe disk: the boot stick +/// takes one of the USB driver's two disks, so a machine carries one more. /// /// Lives here so the registration hunk in `toyos.rs` stays one line: every /// agent edits that file. @@ -27,12 +32,16 @@ pub fn foreign_disk_untouched( rust_bins: &[(String, Vec)], ) -> Result<(), String> { const BYTES: u64 = 128 * 1024 * 1024; - // The same directory `boot_with_options` uses, named here because this - // image has to exist before the boot that must not touch it. + const USB_BYTES: u64 = 64 * 1024 * 1024; + // The same directory `boot_with_options` uses, named here because these + // images have to exist before the boot that must not touch them. let dir = super::lane::dir(); let image = dir.join("foreign-disk.img"); let (data_at, _) = foreign_disk_image(&image, BYTES); - let before = whole_device(&image); + let other = dir.join("other-systems-disk.img"); + let parts = other_systems_disk(&other, USB_BYTES)?; + let bare = dir.join("bare-stick.img"); + filled(&bare, USB_BYTES, 0x5A)?; // The premise, checked before the boot rather than assumed: if this volume // somehow already parsed as a ToyOS volume, the kernel would mount it and @@ -41,13 +50,40 @@ pub fn foreign_disk_untouched( return Err("the foreign volume starts with a bcachefs superblock".to_string()); } + let sticks = [ + (&other, format!("has {parts} partitions and none of them is ours")), + (&bare, "has no partition table we can use".to_string()), + ]; + for (stick, read) in sticks { + untouched_beside(test_config, c_bins, rust_bins, &image, stick, &read)?; + } + for disk in [&image, &other, &bare] { + let _ = std::fs::remove_file(disk); + } + Ok(()) +} + +/// One boot with `nvme` and `stick` beside the boot stick, each compared byte +/// for byte after QEMU has exited. `read` is what the kernel says reading +/// `stick`'s table, so the comparison is about a disk it saw. +fn untouched_beside( + test_config: &Path, + c_bins: &[(String, Vec)], + rust_bins: &[(String, Vec)], + nvme: &Path, + stick: &Path, + read: &str, +) -> Result<(), String> { + let disks = [nvme, stick]; + let before: Vec> = disks.iter().map(|disk| whole_device(disk)).collect(); let mut qemu = QemuInstance::boot_with_options( test_config, c_bins, rust_bins, BootOptions { - profile: qemu::Profile::Metal, - nvme_image: Some(image.clone()), + profile: qemu::Profile::UsbDisk, + nvme_image: Some(nvme.to_path_buf()), + usb_images: vec![stick.to_path_buf()], ..Default::default() }, ); @@ -79,27 +115,75 @@ pub fn foreign_disk_untouched( return Err(format!("the kernel decided to format a disk it was not given\n{log}")); } - // Shut down rather than kill: `PageCache::sync` at shutdown is the only - // thing that moves a format from the cache to the device, so a killed QEMU - // fingerprints an image a formatting kernel would also have left untouched. + // Shut down rather than kill, and wait for QEMU to exit: `PageCache::sync` + // at shutdown is the only thing that moves a format from the cache to the + // device, so a killed QEMU fingerprints an image a formatting kernel would + // also have left untouched. writeln!(qemu.stdin_mut(), "run shutdown").expect("write to QEMU stdin"); qemu.flush_stdin(); - let tail = qemu.drain_serial(Duration::from_secs(20)); + let tail = qemu.await_exit(Duration::from_secs(20))?; for bad in ["PANIC:", "panicked at"] { if tail.contains(bad) { return Err(format!("{bad:?} during shutdown\n{tail}")); } } drop(qemu); + let said = format!("{log}{tail}"); + if !said.contains(read) { + return Err(format!("the kernel never said {read:?}, so it never read {}\n{said}", stick.display())); + } - let after = whole_device(&image); - if let Some(diff) = first_difference(&before, &after) { - return Err(format!("the kernel wrote to a disk it was not given: {diff}")); + for (disk, before) in disks.iter().zip(&before) { + if let Some(diff) = first_difference(before, &whole_device(disk)) { + return Err(format!("the kernel wrote to {}, a disk it was not given: {diff}", disk.display())); + } } - let _ = std::fs::remove_file(&image); Ok(()) } +/// `len` bytes of `fill`, so a write of anything, zeros included, moves the +/// fingerprint. +fn filled(path: &Path, len: u64, fill: u8) -> Result<(), String> { + let len = usize::try_from(len).map_err(|e| format!("{len} bytes: {e}"))?; + std::fs::write(path, vec![fill; len]).map_err(|e| format!("{}: {e}", path.display())) +} + +/// A filled disk whose table names an EFI system partition, a Microsoft basic +/// data partition and a Linux filesystem, and none of ToyOS's types. Answers +/// how many partitions it carries. +fn other_systems_disk(path: &Path, len: u64) -> Result { + use gpt::partition_types::{BASIC, EFI, LINUX_FS}; + filled(path, len, 0xA5)?; + let mut file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(path) + .map_err(|e| format!("{}: {e}", path.display()))?; + let sectors = u32::try_from(len / 512 - 1).map_err(|e| format!("{len} bytes: {e}"))?; + gpt::mbr::ProtectiveMBR::with_lb_size(sectors) + .overwrite_lba0(&mut file) + .map_err(|e| format!("protective MBR: {e}"))?; + let mut disk = gpt::GptConfig::default() + .initialized(false) + .writable(true) + .logical_block_size(gpt::disk::LogicalBlockSize::Lb512) + .create_from_device(Box::new(file), None) + .map_err(|e| format!("a table on {}: {e}", path.display()))?; + disk.update_partitions(std::collections::BTreeMap::new()) + .map_err(|e| format!("an empty table: {e}"))?; + let layout = [("EFI system partition", EFI), ("Basic data partition", BASIC), ("Linux filesystem", LINUX_FS)]; + let parts = layout.len(); + for (name, kind) in layout { + disk.add_partition(name, 16 << 20, kind, 0, Some(2048)).map_err(|e| format!("add {name}: {e}"))?; + } + disk.write().map_err(|e| format!("write the table: {e}"))?; + // The premise, by the parser the kernel selects DATA with. + if toyos_build::image::data_partition_of(path).is_ok() { + return Err(format!("{} carries a TOYOS-DATA partition", path.display())); + } + Ok(parts) +} + /// The volume is genuine and the disk is not: block 0 here carries the magic, /// the version and the CRC this crate wrote, and every other stimulus in this /// file is refused before any of that is read. What it does not carry is this diff --git a/tests/toyos.rs b/tests/toyos.rs index f705cf09f5..fa7958e87d 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -12,8 +12,8 @@ use common::qemu::{ STALLED, }; use common::{ - audio, compile, devices, faults, hostload, lan, metal, partclaim, pkg, power, screen, serial, - stats, storage, usb, + audio, compile, devices, faults, hostload, lan, metal, partclaim, pkg, power, release, screen, + serial, stats, storage, usb, }; use toyos_build::bootlog::{self, boot_millis}; use toyos_build::heartbeat; @@ -20016,6 +20016,7 @@ fn check_redlist(all_tests: &[TestDef]) -> Result<(), String> { .chain(AUDIO_TESTS.iter().map(|(name, _)| *name)) .chain(SCREEN_TESTS.iter().map(|(n, _, _)| *n)) .chain(MACHINE_TESTS.iter().map(|(n, _, _)| *n)) + .chain([release::NAME]) .collect(); redlist::check(redlist::DISABLED, |name| runnable.contains(name), &compile::repo_root()) } @@ -20098,6 +20099,23 @@ fn main() { common::qemu::VERBOSE.store(true, std::sync::atomic::Ordering::Relaxed); } + // **The image release's macOS line**, in no tier: it is an Apple Silicon + // Mac's, and the nightly's `portability-macos` is the run that has one. + if SUITE.present(&args, &testargs::RELEASE_COMMAND) { + let verdict = if keep(release::NAME) { + release::release_command_boots() + } else { + Err("disabled, so the macOS line was not booted".to_string()) + }; + match verdict { + Ok(()) => run.exit(0), + Err(why) => { + eprintln!("[toyos] {}: {why}", release::NAME); + run.exit(1) + } + } + } + // **The metal profile, before the C corpus.** It boots the T14 and not a // guest, so none of the C compile, the HTTPS judge's hosts or the tier // arithmetic below is any of its business; running it here is what keeps a diff --git a/toyos-update/src/floor.rs b/toyos-update/src/floor.rs index 379b3fdc36..bfde62e326 100644 --- a/toyos-update/src/floor.rs +++ b/toyos-update/src/floor.rs @@ -76,7 +76,8 @@ impl Scope { } } - const fn tag(self) -> u8 { + /// The letter a floor's name carries after [`PREFIX`] and a dash. + pub const fn tag(self) -> u8 { match self { Self::Machine => b'K', Self::Image => b'I', @@ -98,6 +99,10 @@ const fn eq(a: &[u8], b: &[u8]) -> bool { true } +/// The vendor GUID every floor is under, minted for this and used for nothing +/// else. +pub const VENDOR: &str = "33be3d4a-30e6-49f5-8050-f169d93a20fb"; + /// What every floor variable's name begins with. pub const PREFIX: &str = "ToyOSImageFloor";