diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 120c71de0..dece75b3d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,8 +67,10 @@ jobs: set -xeu # Build images to test; TODO investigate doing single container builds # via GHA and pushing to a temporary registry to share among workflows? - sudo just build - sudo just build-install-test-image + # Preserve rustup/cargo environment for sudo (rustup needs RUSTUP_HOME to find toolchains) + sudojust() { sudo env PATH="$PATH" CARGO_HOME="${CARGO_HOME:-$HOME/.cargo}" RUSTUP_HOME="${RUSTUP_HOME:-$HOME/.rustup}" just "$@"; } + sudojust build + sudojust build-install-test-image sudo podman build -t localhost/bootc-fsverity -f ci/Containerfile.install-fsverity # Grant permission diff --git a/Cargo.lock b/Cargo.lock index 45399dec0..5a20c0d59 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -440,6 +440,29 @@ dependencies = [ "uuid", ] +[[package]] +name = "cargo-platform" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e35af189006b9c0f00a064685c727031e3ed2d8020f7ba284d78cc2671bd36ea" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd5eb614ed4c27c5d706420e4320fbe3216ab31fa1c33cd8246ac36dae4479ba" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", + "thiserror 2.0.17", +] + [[package]] name = "cc" version = "1.2.51" @@ -2275,6 +2298,10 @@ name = "semver" version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +dependencies = [ + "serde", + "serde_core", +] [[package]] name = "serde" @@ -3338,6 +3365,7 @@ dependencies = [ "anstream", "anyhow", "camino", + "cargo_metadata", "chrono", "clap", "fn-error-context", diff --git a/Cargo.toml b/Cargo.toml index 8cc076c0d..91a56ac60 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,16 +43,15 @@ clap_mangen = { version = "0.2.20" } # Reviewers (including AI tools): The composefs-rs git revision is duplicated for each crate. # If adding/removing crates here, also update docs/Dockerfile.mdbook and docs/src/internals.md. # -# To develop against a local composefs-rs checkout: -# 1. Set BOOTC_extra_src to your composefs-rs path when building: -# BOOTC_extra_src=$HOME/src/composefs-rs just build -# 2. Comment out the git refs below and uncomment the path refs: +# To develop against a local composefs-rs checkout, add a [patch] section at the end of this file: +# [patch."https://github.com/containers/composefs-rs"] +# composefs = { path = "/home/user/src/composefs-rs/crates/composefs" } +# composefs-boot = { path = "/home/user/src/composefs-rs/crates/composefs-boot" } +# composefs-oci = { path = "/home/user/src/composefs-rs/crates/composefs-oci" } +# The Justfile will auto-detect these and bind-mount them into container builds. composefs = { git = "https://github.com/containers/composefs-rs", rev = "4a060161e0122bd2727e639437c61e05ecc7cab3", package = "composefs", features = ["rhel9"] } composefs-boot = { git = "https://github.com/containers/composefs-rs", rev = "4a060161e0122bd2727e639437c61e05ecc7cab3", package = "composefs-boot" } composefs-oci = { git = "https://github.com/containers/composefs-rs", rev = "4a060161e0122bd2727e639437c61e05ecc7cab3", package = "composefs-oci" } -# composefs = { path = "/run/extra-src/crates/composefs", package = "composefs", features = ["rhel9"] } -# composefs-boot = { path = "/run/extra-src/crates/composefs-boot", package = "composefs-boot" } -# composefs-oci = { path = "/run/extra-src/crates/composefs-oci", package = "composefs-oci" } fn-error-context = "0.2.1" hex = "0.4.3" indicatif = "0.18.0" diff --git a/Dockerfile b/Dockerfile index 42eb3e69b..27abdbd37 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,7 +33,7 @@ WORKDIR /src # See https://www.reddit.com/r/rust/comments/126xeyx/exploring_the_problem_of_faster_cargo_docker/ # We aren't using the full recommendations there, just the simple bits. # First we download all of our Rust dependencies -# Note: /run/extra-src is optionally bind-mounted via BOOTC_extra_src for local composefs-rs development +# Note: Local path dependencies (from [patch] sections) are auto-detected and bind-mounted by the Justfile RUN --mount=type=tmpfs,target=/run --mount=type=tmpfs,target=/tmp --mount=type=cache,target=/src/target --mount=type=cache,target=/var/roothome cargo fetch # We always do a "from scratch" build diff --git a/Justfile b/Justfile index 4c1f4af08..e3997ba90 100644 --- a/Justfile +++ b/Justfile @@ -25,7 +25,10 @@ base := env("BOOTC_base", "quay.io/centos-bootc/centos-bootc:stream10") # Buildroot base image buildroot_base := env("BOOTC_buildroot_base", "quay.io/centos/centos:stream10") # Optional: path to extra source (e.g. composefs-rs) for local development +# DEPRECATED: Use [patch] sections in Cargo.toml instead, which are auto-detected extra_src := env("BOOTC_extra_src", "") +# Set to "1" to disable auto-detection of local Rust dependencies +no_auto_local_deps := env("BOOTC_no_auto_local_deps", "") # Internal variables nocache := env("BOOTC_nocache", "") @@ -231,7 +234,12 @@ package: fi eval $(just _git-build-vars) echo "Building RPM with version: ${VERSION}" - podman build {{base_buildargs}} --build-arg=SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH} --build-arg=pkgversion=${VERSION} -t localhost/bootc-pkg --target=build . + # Auto-detect local Rust path dependencies (e.g., from [patch] sections) + local_deps_args="" + if [[ -z "{{no_auto_local_deps}}" ]]; then + local_deps_args=$(cargo xtask local-rust-deps) + fi + podman build {{base_buildargs}} --build-arg=SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH} --build-arg=pkgversion=${VERSION} -t localhost/bootc-pkg --target=build $local_deps_args . mkdir -p "${packages}" rm -vf "${packages}"/*.rpm podman run --rm localhost/bootc-pkg tar -C /out/ -cf - . | tar -C "${packages}"/ -xvf - diff --git a/crates/xtask/Cargo.toml b/crates/xtask/Cargo.toml index 81758e791..03ee341de 100644 --- a/crates/xtask/Cargo.toml +++ b/crates/xtask/Cargo.toml @@ -27,6 +27,7 @@ toml = { workspace = true } xshell = { workspace = true } # Crate-specific dependencies +cargo_metadata = "0.19" mandown = "1.1.0" rand = "0.9" serde_yaml = "0.9" diff --git a/crates/xtask/src/tmt.rs b/crates/xtask/src/tmt.rs index 0dd4634aa..5452aea83 100644 --- a/crates/xtask/src/tmt.rs +++ b/crates/xtask/src/tmt.rs @@ -834,7 +834,7 @@ struct TmtMetadata { tmt: serde_yaml::Value, } -#[derive(Debug)] +#[derive(Debug, Eq, PartialEq)] struct TestDef { number: u32, name: String, @@ -845,6 +845,20 @@ struct TestDef { tmt: serde_yaml::Value, } +impl Ord for TestDef { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.number + .cmp(&other.number) + .then_with(|| self.name.cmp(&other.name)) + } +} + +impl PartialOrd for TestDef { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + /// Generate tmt/plans/integration.fmf from test definitions #[context("Updating TMT integration.fmf")] pub(crate) fn update_integration() -> Result<()> { @@ -925,8 +939,7 @@ pub(crate) fn update_integration() -> Result<()> { }); } - // Sort tests by number - tests.sort_by_key(|t| t.number); + tests.sort(); // Generate single tests.fmf file using structured YAML let tests_dir = Utf8Path::new("tmt/tests"); diff --git a/crates/xtask/src/xtask.rs b/crates/xtask/src/xtask.rs index d01c9a6f4..eb8f61ddf 100644 --- a/crates/xtask/src/xtask.rs +++ b/crates/xtask/src/xtask.rs @@ -56,6 +56,8 @@ enum Commands { CheckBuildsys, /// Validate composefs digests match between build-time and install-time views ValidateComposefsDigest(ValidateComposefsDigestArgs), + /// Print podman bind mount arguments for local path dependencies + LocalRustDeps(LocalRustDepsArgs), } /// Arguments for validate-composefs-digest command @@ -65,6 +67,14 @@ pub(crate) struct ValidateComposefsDigestArgs { pub(crate) image: String, } +/// Arguments for local-rust-deps command +#[derive(Debug, Args)] +pub(crate) struct LocalRustDepsArgs { + /// Output format: "podman" for -v arguments, "json" for structured data + #[arg(long, default_value = "podman")] + pub(crate) format: String, +} + /// Arguments for run-tmt command #[derive(Debug, Args)] pub(crate) struct RunTmtArgs { @@ -149,6 +159,7 @@ fn try_main() -> Result<()> { Commands::TmtProvision(args) => tmt::tmt_provision(&sh, &args), Commands::CheckBuildsys => buildsys::check_buildsys(&sh, "Dockerfile".into()), Commands::ValidateComposefsDigest(args) => validate_composefs_digest(&sh, &args), + Commands::LocalRustDeps(args) => local_rust_deps(&sh, &args), } } @@ -417,6 +428,75 @@ fn update_generated(sh: &Shell) -> Result<()> { Ok(()) } +/// Find local path dependencies outside the workspace and output podman bind mount arguments. +/// +/// This uses `cargo metadata` to find all packages with no source (i.e., local path deps). +/// For packages outside the workspace root, it computes the minimal set of directories +/// to bind mount into the container. +#[context("Finding local Rust dependencies")] +fn local_rust_deps(_sh: &Shell, args: &LocalRustDepsArgs) -> Result<()> { + let metadata = cargo_metadata::MetadataCommand::new() + .exec() + .context("Running cargo metadata")?; + + let workspace_root = &metadata.workspace_root; + + let mut external_roots: std::collections::BTreeSet = + std::collections::BTreeSet::new(); + + for pkg in &metadata.packages { + // Packages with source are from registries/git, skip them + if pkg.source.is_some() { + continue; + } + + // Get the package directory (parent of Cargo.toml) + let pkg_dir = pkg + .manifest_path + .parent() + .ok_or_else(|| anyhow::anyhow!("No parent for manifest_path"))?; + + // Skip packages inside the workspace + if pkg_dir.starts_with(workspace_root) { + continue; + } + + // Find the workspace root for this external package by running cargo metadata + // in the package directory + let external_metadata = cargo_metadata::MetadataCommand::new() + .current_dir(pkg_dir) + .exec() + .with_context(|| format!("Running cargo metadata in {pkg_dir}"))?; + + external_roots.insert(external_metadata.workspace_root.clone()); + } + + match args.format.as_str() { + "podman" => { + // Output podman -v arguments + let mut args_out = Vec::new(); + for root in &external_roots { + // Mount read-only with SELinux disabled (for cross-context access) + args_out.push("-v".to_string()); + args_out.push(format!("{}:{}:ro", root, root)); + args_out.push("--security-opt=label=disable".to_string()); + } + if !args_out.is_empty() { + println!("{}", args_out.join(" ")); + } + } + "json" => { + let roots: Vec<&str> = external_roots.iter().map(|p| p.as_str()).collect(); + println!("{}", serde_json::to_string_pretty(&roots)?); + } + other => { + anyhow::bail!("Unknown format: {other}. Use 'podman' or 'json'."); + } + } + + Ok(()) +} + /// Validate that composefs digests match between build-time and install-time views. /// /// Compares dumpfiles generated from: diff --git a/tmt/plans/integration.fmf b/tmt/plans/integration.fmf index 40ca2135e..4dcb9c60a 100644 --- a/tmt/plans/integration.fmf +++ b/tmt/plans/integration.fmf @@ -71,19 +71,19 @@ execute: test: - /tmt/tests/tests/test-22-logically-bound-install -/plan-23-usroverlay: - summary: Execute tests for bootc usrover +/plan-23-install-outside-container: + summary: Execute tests for installing outside of a container discover: how: fmf test: - - /tmt/tests/tests/test-23-usroverlay + - /tmt/tests/tests/test-23-install-outside-container -/plan-23-install-outside-container: - summary: Execute tests for installing outside of a container +/plan-23-usroverlay: + summary: Execute tests for bootc usrover discover: how: fmf test: - - /tmt/tests/tests/test-23-install-outside-container + - /tmt/tests/tests/test-23-usroverlay /plan-24-image-upgrade-reboot: summary: Execute local upgrade tests diff --git a/tmt/tests/tests.fmf b/tmt/tests/tests.fmf index 319918962..eeb9228a6 100644 --- a/tmt/tests/tests.fmf +++ b/tmt/tests/tests.fmf @@ -21,16 +21,16 @@ duration: 30m test: nu booted/test-logically-bound-install.nu -/test-23-usroverlay: - summary: Execute tests for bootc usrover - duration: 30m - test: nu booted/test-usroverlay.nu - /test-23-install-outside-container: summary: Execute tests for installing outside of a container duration: 30m test: nu booted/test-install-outside-container.nu +/test-23-usroverlay: + summary: Execute tests for bootc usrover + duration: 30m + test: nu booted/test-usroverlay.nu + /test-24-image-upgrade-reboot: summary: Execute local upgrade tests duration: 30m