From 109d04f2e50b1d12268b1c3d0903b7d217dc667f Mon Sep 17 00:00:00 2001 From: Ryan Mulligan Date: Sun, 8 Nov 2026 23:02:33 +0000 Subject: [PATCH] Build direct dynamic output installables Assisted-by: Replit --- docs/reference/commands.md | 8 ++ examples/build.rs | 16 +-- src/api.rs | 29 ++--- src/api/build.rs | 5 +- src/api/installable.rs | 74 +++++++++--- src/api/run.rs | 6 +- src/build/backend.rs | 38 ++++--- src/build/mod.rs | 88 ++++++++++---- src/cli/build.rs | 40 ++++--- src/cli/classic/fmt.rs | 4 +- src/cli/classic/nix.rs | 4 +- src/cli/classic/shell.rs | 6 +- src/cli/devenv.rs | 2 +- src/cli/plumbing.rs | 6 +- src/remote/grpc/backend.rs | 18 ++- src/remote/worker/mod.rs | 4 +- src/remote/worker/wire.rs | 19 ++-- src/store/derived.rs | 38 +++++++ src/store/mod.rs | 1 + src/trace/run.rs | 5 +- src/trace/wire.rs | 32 +++++- tests/dynamic_derivations.rs | 214 +++++++++++++++++++++++++++++++++++ 22 files changed, 530 insertions(+), 127 deletions(-) create mode 100644 src/store/derived.rs diff --git a/docs/reference/commands.md b/docs/reference/commands.md index b29b4c93c..1e82a4dc3 100644 --- a/docs/reference/commands.md +++ b/docs/reference/commands.md @@ -162,6 +162,14 @@ every derivation in it gets built. So `iets build .#checks` builds all the checks. The walk is `nix-build`'s: names in order, a nested set only with `recurseForDerivations = true`, and a list always. +Strings returned by `builtins.outputOf` can be built directly, including +nested dynamic derivations. Each target keeps its own output selection. +The worker-protocol builder resolves the chain at build time; evaluation +does not build anything, and `--no-build` only uploads the concrete +producer's closure. `--dry-run` asks the daemon without realising the +chain. The same strings work with the modern `nix build` compatibility +command. The gRPC transport does not support dynamic installables. + In a set that has a member named after your system, iets goes into that member first. That is how a flake lays out `checks`, `packages` and `devShells`. To build for another system, name it in full: diff --git a/examples/build.rs b/examples/build.rs index f30d97844..5a4b1c154 100644 --- a/examples/build.rs +++ b/examples/build.rs @@ -9,11 +9,11 @@ use std::collections::HashMap; use std::env; use std::process::{self, ExitCode}; -use iets::api::installable::drv_roots; +use iets::api::installable::derived_roots; use iets::api::{self, EvalArgs, Input}; +use iets::build::Dispatcher; use iets::build::render::Style; use iets::build::upload::PlanUpload; -use iets::build::{Dispatcher, Target}; use iets::eval::Evaluator; use iets::remote::{plan, worker}; use iets::store::locate::locate_in_store; @@ -61,12 +61,13 @@ fn main() -> ExitCode { fn eval_and_build(ev: &mut Evaluator, a: EvalArgs, url: String) -> u8 { let mut roots = Vec::new(); - if let Err(e) = api::evaluate(ev, a).and_then(|v| drv_roots(ev, v, &mut roots)) { + if let Err(e) = api::evaluate(ev, a).and_then(|v| derived_roots(ev, v, &mut roots)) { api::report(ev, &e); return 1; } // `--out` would add wanted outputs; the roots' string context is enough here. - let (root_paths, outputs) = api::build_targets(&[], &roots); + let mut targets = api::build_targets(&[], &roots); + let root_paths: Vec<_> = targets.iter().map(|t| t.drv.root.clone()).collect(); let prov = iets::provenance::Provenance::snapshot(ev); // Every derivation and source this evaluation created, straight from @@ -105,10 +106,9 @@ fn eval_and_build(ev: &mut Evaluator, a: EvalArgs, url: String) -> u8 { return 1; } }; - let targets: Vec = root_paths - .iter() - .map(|r| Target::new(r.name(), r.clone(), outputs.clone())) - .collect(); + for t in &mut targets { + t.attr = t.drv.name().to_string(); + } let outcomes = match d.submit(&mut PlanUpload::new(&plan, &locate), targets) { Ok(o) => o, Err(e) => { diff --git a/src/api.rs b/src/api.rs index a4b72fd01..6529a9861 100644 --- a/src/api.rs +++ b/src/api.rs @@ -20,6 +20,8 @@ pub mod testkit; pub mod value; pub mod watch; +use crate::api::build::DerivedPath; +use crate::build::Target; use crate::compile::cache::IrCache; use crate::env::Env; use crate::eval::{self, Error, ErrorKind, EvalConfig, Evaluator, Pos, search_path}; @@ -29,7 +31,6 @@ use crate::provenance::{describe_pos, rewrite_msg}; use crate::runtime::Word; use crate::session::Session; use crate::source::nix_conf; -use crate::store::path::StorePath; use crate::sys::{self, OsStringExt}; use crate::trace::key::{Key, has_relative_path}; use crate::trace::run::Cache; @@ -438,19 +439,19 @@ pub fn full_includes(includes: &[(Vec, Vec)]) -> Vec<(Vec, Vec)> /// The derivations to build and the outputs to print: `wanted` plus the /// outputs the roots' string context named. -pub fn build_targets( - wanted: &[String], - roots: &[(StorePath, Option)], -) -> (Vec, Vec) { - let mut outputs = wanted.to_vec(); - for (_, out) in roots { - if let Some(o) = out - && !outputs.contains(o) - { - outputs.push(o.clone()); - } - } - (roots.iter().map(|(p, _)| p.clone()).collect(), outputs) +pub fn build_targets(wanted: &[String], roots: &[(DerivedPath, Option)]) -> Vec { + roots + .iter() + .map(|(drv, output)| { + let mut outputs = wanted.to_vec(); + if let Some(output) = output + && !outputs.contains(output) + { + outputs.push(output.clone()); + } + Target::new("", drv.clone(), outputs) + }) + .collect() } /// The tracing cache for this run, keyed on the eval arguments. `None` when the diff --git a/src/api/build.rs b/src/api/build.rs index 515625133..fe0ba65b6 100644 --- a/src/api/build.rs +++ b/src/api/build.rs @@ -19,6 +19,7 @@ use crate::provenance::Provenance; use crate::remote::plan; use crate::remote::worker::Missing; use crate::settings; +pub use crate::store::derived::DerivedPath; use crate::store::locate::locate_in_store; use crate::store::path::StorePath; use crate::store::source::SourceOrigin; @@ -34,13 +35,13 @@ use crate::store::drv::Derivation; /// One derivation and one of its outputs a command asked for. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Root { - pub drv: StorePath, + pub drv: DerivedPath, pub output: String, } impl Root { pub fn paths(roots: &[Root]) -> Vec { - roots.iter().map(|r| r.drv.clone()).collect() + roots.iter().map(|r| r.drv.root.clone()).collect() } } diff --git a/src/api/installable.rs b/src/api/installable.rs index 35a3e9756..fb0997978 100644 --- a/src/api/installable.rs +++ b/src/api/installable.rs @@ -5,7 +5,7 @@ use std::collections::HashSet; use std::str::from_utf8; -use crate::api::build::Root; +use crate::api::build::{DerivedPath, Root}; use crate::builtins::strings::parse_drv_name_bytes; use crate::eval::{Error, ErrorKind, EvalResult, Evaluator, Pos}; use crate::runtime::{self, AttrsObj, Cell, Ctx, Word}; @@ -379,6 +379,26 @@ pub fn drv_roots( ev: &mut Evaluator, v: Word, roots: &mut Vec<(StorePath, Option)>, +) -> EvalResult<()> { + let mut derived = Vec::new(); + derived_roots(ev, v, &mut derived)?; + for (drv, output) in derived { + if !drv.chain.is_empty() { + return Err(Error::boxed( + ErrorKind::Unsupported, + "this operation needs a concrete derivation; use build for dynamic outputs", + Pos::NONE, + )); + } + roots.push((drv.root, output)); + } + Ok(()) +} + +pub fn derived_roots( + ev: &mut Evaluator, + v: Word, + roots: &mut Vec<(DerivedPath, Option)>, ) -> EvalResult<()> { let bad = |msg: String| Error::boxed(ErrorKind::Eval, msg, Pos::NONE); let v = ev.force(v)?; @@ -392,7 +412,7 @@ pub fn drv_roots( if !p.is_derivation() { return Err(bad(format!("'{p}' is not a derivation path"))); } - roots.push((p, None)); + roots.push((p.into(), None)); return Ok(()); } if let Some(s) = v.as_string() { @@ -401,18 +421,20 @@ pub fn drv_roots( match c { Ctx::Built { drv, output } => { let (layers, root) = runtime::split_built(drv); - if !layers.is_empty() { - return Err(bad( - "iets: the output of a dynamic derivation is not an installable" - .to_string(), - )); - } + let drv = DerivedPath { + root: parse_drv(root)?, + chain: layers + .into_iter() + .rev() + .map(|n| String::from_utf8_lossy(n).into_owned()) + .collect(), + }; let out = String::from_utf8_lossy(output).into_owned(); - roots.push((parse_drv(root)?, Some(out))); + roots.push((drv, Some(out))); found = true; } Ctx::DrvDeep(drv) => { - roots.push((parse_drv(drv)?, None)); + roots.push((parse_drv(drv)?.into(), None)); found = true; } Ctx::Opaque(_) | Ctx::Impure(_) => {} @@ -428,7 +450,7 @@ pub fn drv_roots( } if let Some(l) = v.as_list() { for c in l.cells() { - drv_roots(ev, c.get(), roots)?; + derived_roots(ev, c.get(), roots)?; } return Ok(()); } @@ -446,11 +468,11 @@ pub fn roots_of( v: Word, args: Word, attr: &str, - roots: &mut Vec<(StorePath, Option)>, + roots: &mut Vec<(DerivedPath, Option)>, ) -> EvalResult<()> { let v = ev.auto_call(v, args)?; let Some(tree) = v.as_attrs().filter(|a| a.get(ev.s.drv_path).is_none()) else { - return drv_roots(ev, v, roots); + return derived_roots(ev, v, roots); }; let system = ev.current_system(); let sym = ev.sym(system.as_bytes()); @@ -570,7 +592,10 @@ fn one_derivation( } ev.drv.set_attr(&drv, attr); if seen.insert((drv.to_string(), output.clone())) { - out.push(Root { drv, output }); + out.push(Root { + drv: drv.into(), + output, + }); } Ok(true) } @@ -675,7 +700,7 @@ fn main_program(ev: &mut Evaluator, attrs: &'static AttrsObj) -> EvalResult, } @@ -707,6 +732,23 @@ pub fn to_derivation(ev: &mut Evaluator, v: Word, what: &str) -> EvalResult EvalResult { let v = ev.force(v)?; + if let Some(s) = v.as_string() + && s.ctx.iter().any(|c| matches!(c, Ctx::Built { .. })) + { + let mut roots = Vec::new(); + derived_roots(ev, v, &mut roots)?; + if let [(drv, Some(output))] = roots.as_slice() { + return Ok(Install { + drv: drv.clone(), + outputs: vec![output.clone()], + }); + } + return Err(Error::boxed( + ErrorKind::Eval, + "a string installable must name exactly one derivation output", + Pos::NONE, + )); + } let unsupported = |msg: String| Error::boxed(ErrorKind::Unsupported, msg, Pos::NONE); let not_a_drv = || unsupported(format!("the installable '{what}' is not a derivation")); let Some(attrs) = v.as_attrs() else { @@ -715,7 +757,7 @@ pub fn to_install(ev: &mut Evaluator, v: Word, what: &str, flake: bool) -> EvalR if !is_derivation(ev, attrs)? { return Err(not_a_drv()); } - let drv = drv_of(ev, attrs)?; + let drv: DerivedPath = drv_of(ev, attrs)?.into(); let specified_sym = ev.sym(b"outputSpecified"); let specified = match attrs.get(specified_sym) { diff --git a/src/api/run.rs b/src/api/run.rs index ebb3e060d..06638e60c 100644 --- a/src/api/run.rs +++ b/src/api/run.rs @@ -11,7 +11,7 @@ use crate::store::path::StorePath; use crate::trace::run::Cache; use crate::trace::store::Part; -use super::build::{self, Plan}; +use super::build::{self, DerivedPath, Plan}; use super::{EvalArgs, project_root, report, setup_cache, setup_cache_more}; use crate::error::{self, Error}; @@ -25,7 +25,7 @@ pub enum Outcome { } /// A build entry's result: each root derivation and the outputs wanted of it. -pub type Roots = Vec<(StorePath, Option)>; +pub type Roots = Vec<(DerivedPath, Option)>; pub struct Run { /// The evaluator is built from it once, in `go`. @@ -112,7 +112,7 @@ impl Run { pub fn hit_plan(&self) -> Option<(Roots, Plan)> { self.hit(|c| { let t = c.read_build()?; - let paths: Vec = t.roots.iter().map(|(p, _)| p.clone()).collect(); + let paths: Vec = t.roots.iter().map(|(p, _)| p.root.clone()).collect(); let inputs = build::Inputs { drvs: &t.drvs, sources: &t.sources, diff --git a/src/build/backend.rs b/src/build/backend.rs index 7fbbaa748..58551d8fd 100644 --- a/src/build/backend.rs +++ b/src/build/backend.rs @@ -26,6 +26,7 @@ use crate::remote::grpc::backend::Grpc; use crate::remote::plan::{self, Object, Plan}; use crate::remote::worker::wire::{self, write_object}; use crate::remote::worker::{Connection, Missing, Options, Stats, Transport, default_socket}; +use crate::store::derived::DerivedPath; use crate::store::nar; use crate::store::narinfo::Fetched; use crate::store::path::StorePath; @@ -33,12 +34,16 @@ use crate::store::path::StorePath; /// One derivation of a ready set, and the outputs the caller wants. #[derive(Debug, Clone)] pub struct Job { - pub drv: StorePath, + pub drv: DerivedPath, /// Empty means every output. pub outputs: Vec, } impl Job { + pub fn key(&self) -> String { + format!("{}!{}", self.drv, self.spec()) + } + /// `` or `*`, the output half of a derived path. pub fn spec(&self) -> String { if self.outputs.is_empty() { @@ -135,8 +140,8 @@ pub trait Backend { self.add_objects(&objects.iter().collect::>(), on_event) } - /// Builds a ready set. Every job gets a verdict, keyed by its derivation - /// path; a job the store never reached is absent from the map. + /// Builds a ready set. Every job gets a verdict keyed by `Job::key`, + /// including its output selection; an unreached job is absent. fn submit( &mut self, batch: &Batch<'_>, @@ -151,7 +156,7 @@ pub trait Backend { )) } - /// Output name to store path for a derivation that built. + /// Output name to store path for a job key, or a concrete derivation. fn outputs( &mut self, drv: &str, @@ -448,17 +453,13 @@ impl Backend for Worker { batch: &Batch<'_>, on_event: &mut dyn FnMut(Event), ) -> Result> { - let targets: Vec = batch - .jobs - .iter() - .map(|j| format!("{}!{}", j.drv, j.spec())) - .collect(); + let targets: Vec = batch.jobs.iter().map(Job::key).collect(); let t0 = Instant::now(); let results = self.conn.build_paths_with_results(&targets, on_event)?; self.t.build += t0.elapsed(); let mut out: HashMap = HashMap::new(); for r in &results { - let drv = r.path.split('!').next().unwrap_or(&r.path).to_string(); + let drv = r.path.clone(); if r.success() { self.built_outputs .entry(drv.clone()) @@ -475,10 +476,7 @@ impl Backend for Worker { } fn missing(&mut self, jobs: &[Job], on_event: &mut dyn FnMut(Event)) -> Result { - let paths: Vec = jobs - .iter() - .map(|j| format!("{}!{}", j.drv, j.spec())) - .collect(); + let paths: Vec = jobs.iter().map(Job::key).collect(); self.conn.query_missing(&paths, on_event) } @@ -491,7 +489,13 @@ impl Backend for Worker { return Ok(outputs.clone()); } let t0 = Instant::now(); - let map = self.conn.query_derivation_output_map(drv, on_event)?; + let path = drv.rsplit_once('!').map_or(drv, |(path, _)| path); + if path.contains('!') { + return Err(Error::Store(format!( + "the builder returned no output paths for dynamic target '{drv}'" + ))); + } + let map = self.conn.query_derivation_output_map(path, on_event)?; self.t.outputs += t0.elapsed(); Ok(map) } @@ -639,12 +643,12 @@ pub(crate) mod tests { #[test] fn output_specs() { let j = Job { - drv: drv("x"), + drv: drv("x").into(), outputs: vec![], }; assert_eq!(j.spec(), "*"); let j = Job { - drv: drv("x"), + drv: drv("x").into(), outputs: vec!["dev".into(), "out".into(), "out".into()], }; assert_eq!(j.spec(), "dev,out"); diff --git a/src/build/mod.rs b/src/build/mod.rs index 98c24628d..4eed61508 100644 --- a/src/build/mod.rs +++ b/src/build/mod.rs @@ -40,6 +40,7 @@ use crate::remote::event::Event; use crate::remote::plan::Plan; use crate::remote::worker::{Missing, Options, Transport}; use crate::session::{self, Session}; +use crate::store::derived::DerivedPath; use crate::store::path::{StorePath, drv_name}; use crate::store::drv::{Derivation, DerivationOutput}; @@ -54,7 +55,7 @@ use upload::Uploader; pub struct Target { /// Attribute path, or the derivation name when the caller has none. pub attr: String, - pub drv: StorePath, + pub drv: DerivedPath, /// Wanted output names; empty means every output (`!*`). pub outputs: Vec, } @@ -69,10 +70,14 @@ impl From<&Target> for backend::Job { } impl Target { - pub fn new(attr: &str, drv: StorePath, outputs: Vec) -> Target { + fn key(&self) -> String { + backend::Job::from(self).key() + } + + pub fn new(attr: &str, drv: impl Into, outputs: Vec) -> Target { Target { attr: attr.to_string(), - drv, + drv: drv.into(), outputs, } } @@ -82,7 +87,7 @@ impl Target { #[derive(Debug, Clone)] pub struct Outcome { pub attr: String, - pub drv: StorePath, + pub drv: DerivedPath, pub success: bool, pub skipped: bool, pub reason: Option, @@ -222,7 +227,7 @@ pub struct Dispatcher { backend: Box, pub reporter: Reporter, prov: Provenance, - seen: HashSet, + seen: HashSet<(DerivedPath, Vec)>, /// Nix's failingExitStatus of the failed builds, 0 while none failed. pub exit: i32, pub upload: upload::Stats, @@ -274,11 +279,16 @@ impl Dispatcher { } /// Uploads what the batch needs and builds it in one op. Targets whose - /// drv was already dispatched are dropped. + /// full path and output selection were already dispatched are dropped. pub fn submit(&mut self, up: &mut dyn Uploader, batch: Vec) -> Result> { let batch: Vec = batch .into_iter() - .filter(|t| self.seen.insert(t.drv.clone())) + .filter(|t| { + let mut outputs = t.outputs.clone(); + outputs.sort(); + outputs.dedup(); + self.seen.insert((t.drv.clone(), outputs)) + }) .collect(); if batch.is_empty() { return Ok(Vec::new()); @@ -292,7 +302,7 @@ impl Dispatcher { let by_drv: HashMap = (batch.iter()) .map(|t| (t.drv.to_string(), t.attr.clone())) .collect(); - let roots: Vec = batch.iter().map(|t| t.drv.clone()).collect(); + let roots: Vec = batch.iter().map(|t| t.drv.root.clone()).collect(); // Until this op reports its verdicts, the end of a build's activity // decides nothing: with a build hook the daemon runs two activities // for one derivation, and neither says whether it succeeded. @@ -309,8 +319,9 @@ impl Dispatcher { None => false, }; let status = match held { - true => (by_drv.keys()) - .map(|d| (d.clone(), backend::Status::ok())) + true => batch + .iter() + .map(|t| (t.key(), backend::Status::ok())) .collect(), false => { let stats = { @@ -339,13 +350,14 @@ impl Dispatcher { for t in batch { let drv = t.drv.to_string(); let elapsed = self.reporter.duration(&drv); - match status.get(&drv) { + let key = t.key(); + match status.get(&key) { Some(s) if s.ok => { - let map = match built.as_ref().and_then(|b| b.get(&drv)) { + let map = match built.as_ref().and_then(|b| b.get(&key)) { Some(m) => m.clone(), None => { let mut on_event = event_sink(&mut self.reporter, &self.prov, &by_drv); - self.backend.outputs(&drv, &mut on_event)? + self.backend.outputs(&key, &mut on_event)? } }; let outputs: BTreeMap = map @@ -467,7 +479,7 @@ impl Dispatcher { /// `--dry-run`: register the closure, then ask what building `targets` /// takes. pub fn missing(&mut self, up: &mut dyn Uploader, targets: &[Target]) -> Result { - let roots: Vec = targets.iter().map(|t| t.drv.clone()).collect(); + let roots: Vec = targets.iter().map(|t| t.drv.root.clone()).collect(); self.upload_only(up, &roots)?; let jobs: Vec = targets.iter().map(backend::Job::from).collect(); let by_drv = HashMap::new(); @@ -519,7 +531,10 @@ fn static_outputs( let drvs = up.drvs(roots, false).ok()?; let mut all = HashMap::new(); for t in batch { - let d = drvs.iter().find(|d| d.path == t.drv)?; + if !t.drv.chain.is_empty() { + return None; + } + let d = drvs.iter().find(|d| d.path == t.drv.root)?; let parsed = Derivation::parse(&d.aterm, Derivation::name_from_path(&t.drv).ok()?).ok()?; if t.outputs.iter().any(|n| !parsed.outputs.contains_key(n)) { return None; @@ -531,7 +546,7 @@ fn static_outputs( outs.insert(n.clone(), Some(p.to_string())); } } - all.insert(t.drv.to_string(), outs); + all.insert(t.key(), outs); } Some(all) } @@ -731,6 +746,41 @@ mod tests { assert_eq!(j.spec(), "dev,out"); } + #[test] + fn dispatch_keeps_distinct_chains_and_output_selections() { + let mut d = Dispatcher::with_backend( + Box::new(Fan { + limit: 1, + peak: Arc::new(AtomicBool::new(false)), + verdicts: true, + }), + Provenance::off(), + Reporter::with_output( + Style { + mode: render::Mode::Ci, + color: false, + fold: false, + quiet: true, + }, + Box::new(Sink::default()), + ), + ); + let path = |chain: &[&str]| DerivedPath { + root: drv("producer"), + chain: chain.iter().map(|s| s.to_string()).collect(), + }; + let targets = vec![ + Target::new("one", path(&["out"]), vec!["out".into()]), + Target::new("two", path(&["dev"]), vec!["out".into()]), + Target::new("three", path(&["out", "nested"]), vec!["out".into()]), + Target::new("four", path(&["out"]), vec!["dev".into()]), + ]; + let result = d.submit(&mut NoUpload, targets.clone()).unwrap(); + assert_eq!(result.len(), 4); + assert!(result.iter().all(|o| o.success)); + assert!(d.submit(&mut NoUpload, targets).unwrap().is_empty()); + } + /// A backend that builds every job of a batch concurrently, the way the /// `grpc://` farm does, and tags every line it streams with its drv. struct Fan { @@ -806,7 +856,7 @@ mod tests { } Ok(jobs .iter() - .map(|j| (j.drv.to_string(), backend::Status::ok())) + .map(|j| (j.key(), backend::Status::ok())) .collect()) } @@ -815,7 +865,7 @@ mod tests { drv: &str, _on_event: &mut dyn FnMut(Event), ) -> Result>> { - let p = StorePath::parse(drv)?; + let p = StorePath::parse(drv.split('!').next().unwrap())?; Ok(BTreeMap::from([( "out".to_string(), Some(format!("/nix/store/{}", p.base_name().replace(".drv", ""))), @@ -964,7 +1014,7 @@ mod tests { } let why = format!("1 dependencies of derivation '{}' failed", drv("pkg")); Ok((batch.jobs.iter()) - .map(|j| (j.drv.to_string(), backend::Status::failed(why.clone(), 1))) + .map(|j| (j.key(), backend::Status::failed(why.clone(), 1))) .collect()) } diff --git a/src/cli/build.rs b/src/cli/build.rs index 753332e67..9f5afd86e 100644 --- a/src/cli/build.rs +++ b/src/cli/build.rs @@ -245,14 +245,14 @@ fn build_once(argv: &[String]) -> i32 { t0.elapsed().as_secs_f64() ); } - let (root_paths, outputs) = build_targets(&b.outputs, &roots); + let targets = build_targets(&b.outputs, &roots); let prov = Provenance::from_map(SourceMap::from_sources(&sources)); let inputs = Inputs { drvs: &drvs, sources: &sources, fetched: &fetched, }; - Some(build_roots(&b, &root_paths, &outputs, inputs, &prov)) + Some(build_roots(&b, targets, inputs, &prov)) }) { return code; } @@ -260,7 +260,7 @@ fn build_once(argv: &[String]) -> i32 { cli::run_parallel(run, move |ev, cache| build_eval(ev, &b, all, t0, cache)) } -type Roots = Vec<(StorePath, Option)>; +type Roots = Vec<(apibuild::DerivedPath, Option)>; /// The roots of every installable, loading each flake once. fn resolve_all(ev: &mut Evaluator, all: Vec) -> Result> { @@ -323,7 +323,7 @@ fn build_eval( if let Some(c) = &cache { c.write(ev, &wire::encode_roots(&roots), true); } - let (root_paths, outputs) = build_targets(&b.outputs, &roots); + let targets = build_targets(&b.outputs, &roots); let prov = Provenance::snapshot(ev); let drvs = ev.drv.drvs.lock(); let sources = ev.known.origins(); @@ -333,7 +333,7 @@ fn build_eval( sources: &sources, fetched: &fetched, }; - build_roots(b, &root_paths, &outputs, inputs, &prov) + build_roots(b, targets, inputs, &prov) } /// Legacy mode: `DRV` is a `.drv` store path, or a copy of the file named @@ -366,8 +366,7 @@ fn build_drv_file(b: &BuildArgs, root_arg: &str) -> i32 { let prov = Provenance::from_map(SourceMap::from_sources(&sources)); build_roots( b, - &[root], - &b.outputs, + vec![Target::new("", root, b.outputs.clone())], Inputs { drvs: &drvs, sources: &sources, @@ -379,11 +378,12 @@ fn build_drv_file(b: &BuildArgs, root_arg: &str) -> i32 { fn build_roots( b: &BuildArgs, - roots: &[StorePath], - outputs: &[String], + mut targets: Vec, inputs: Inputs<'_>, prov: &Provenance, ) -> i32 { + let root_paths: Vec = targets.iter().map(|t| t.drv.root.clone()).collect(); + let roots = root_paths.as_slice(); if roots.is_empty() { return no_output("nothing to build: the expression has no derivation in it"); } @@ -423,16 +423,12 @@ fn build_roots( return 0; } - let targets: Vec = roots - .iter() - .map(|r| { - // The bare attribute: the dispatcher adds the position, once. - let name = (inputs.drvs.get(r)) - .and_then(|i| i.attr.clone()) - .unwrap_or_else(|| drv_name(&r.to_string()).to_string()); - Target::new(&name, r.clone(), outputs.to_vec()) - }) - .collect(); + for target in &mut targets { + let r = &target.drv.root; + target.attr = (inputs.drvs.get(r)) + .and_then(|i| i.attr.clone()) + .unwrap_or_else(|| drv_name(&r.to_string()).to_string()); + } if b.dry_run { return match d.missing(&targets) { Ok(m) => print_missing(&m, d.plan(), b.json), @@ -475,7 +471,7 @@ fn build_roots( text::built(&report); } render::emit(&report, b.json); - match silence(&outcomes, outputs) { + match silence(&outcomes, &b.outputs) { Some(why) => log::combine_exit(d.d.exit, no_output(&why)), None => d.d.exit, } @@ -584,7 +580,9 @@ mod tests { fn outcome(success: bool, skipped: bool, outputs: &[&str]) -> Outcome { Outcome { attr: "top".into(), - drv: StorePath::parse("/nix/store/00000000000000000000000000000000-top.drv").unwrap(), + drv: StorePath::parse("/nix/store/00000000000000000000000000000000-top.drv") + .unwrap() + .into(), success, skipped, reason: skipped.then(|| "the builder returned no build result".to_string()), diff --git a/src/cli/classic/fmt.rs b/src/cli/classic/fmt.rs index 4affd97b9..96019a294 100644 --- a/src/cli/classic/fmt.rs +++ b/src/cli/classic/fmt.rs @@ -191,7 +191,7 @@ fn resolve(ev: &mut Evaluator, fr: &FlakeRef, attr: &str) -> error::Result error::Result { let roots = vec![Root { - drv: app.drv.clone(), + drv: app.drv.clone().into(), output: app.output.clone(), }]; let style = Style { diff --git a/src/cli/classic/nix.rs b/src/cli/classic/nix.rs index 8d57e099e..a9456793f 100644 --- a/src/cli/classic/nix.rs +++ b/src/cli/classic/nix.rs @@ -520,13 +520,13 @@ pub(super) fn resolve(ev: &mut Evaluator, n: &Nix, what: &str, cwd: &[u8]) -> Ev /// are joined with a comma, which no output name can hold (store path names /// are `[A-Za-z0-9+._?=-]`), so the installables stay apart: the n-th one /// names its own out-link. -fn remember(targets: &[installable::Install]) -> Vec<(StorePath, Option)> { +fn remember(targets: &[installable::Install]) -> Vec<(build::DerivedPath, Option)> { (targets.iter()) .map(|t| (t.drv.clone(), Some(t.outputs.join(",")))) .collect() } -fn remembered(roots: Vec<(StorePath, Option)>) -> Vec { +fn remembered(roots: Vec<(build::DerivedPath, Option)>) -> Vec { (roots.into_iter()) .map(|(drv, outputs)| installable::Install { drv, diff --git a/src/cli/classic/shell.rs b/src/cli/classic/shell.rs index f0691670b..b934d37cd 100644 --- a/src/cli/classic/shell.rs +++ b/src/cli/classic/shell.rs @@ -287,14 +287,14 @@ fn prepare(ev: &mut Evaluator, s: &Shell) -> error::Result> { let mut targets = vec![roots[0].clone()]; if let Bash::Drv(p) = &bash { targets.push(Root { - drv: p.clone(), + drv: p.clone().into(), output: "out".into(), }); } key.extend(s.c.flags.attrs.iter().map(|a| a.as_bytes().to_vec())); let plan = apibuild::plan(ev, &Root::paths(&targets))?; Ok(Box::new(Ready { - drv_path: roots[0].drv.clone(), + drv_path: roots[0].drv.root.clone(), drv, bash, plan, @@ -315,7 +315,7 @@ fn bash(ev: &mut Evaluator, cwd: &[u8]) -> Bash { installable::get_drvs(ev, v, empty, &mut roots, &mut HashSet::new()) })(); match r { - Ok(()) if roots.len() == 1 => Bash::Drv(roots[0].drv.clone()), + Ok(()) if roots.len() == 1 => Bash::Drv(roots[0].drv.root.clone()), Ok(()) => { let msg = "the 'bashInteractive' attribute in did not evaluate to a derivation"; diff --git a/src/cli/devenv.rs b/src/cli/devenv.rs index e55dbaf42..6910c9484 100644 --- a/src/cli/devenv.rs +++ b/src/cli/devenv.rs @@ -623,7 +623,7 @@ fn dev_env_build(ev: &mut Evaluator, a: EvalArgs, d: &DevEnvArgs) -> Result Result<(), Box> { installable::get_drvs_at(ev, v, args, &mut rs, &mut seen, &start)?; for r in rs { // the walk records the path it reached each derivation by - let at = (ev.drv.drvs.get(&r.drv)) + let at = (ev.drv.drvs.get(&r.drv.root)) .and_then(|i| i.attr) .or_else(|| attr.clone()); roots.push(root_json(&r.drv, Some(r.output), at)); @@ -354,7 +354,7 @@ fn roots(args: &[String], stdin: bool, drvs: Option) -> Roots { } fn paths_of(targets: &[Target]) -> Vec { - targets.iter().map(|t| t.drv.clone()).collect() + targets.iter().map(|t| t.drv.root.clone()).collect() } /// The plan of the roots: from the derivations `--dump-drvs` wrote when there @@ -478,7 +478,7 @@ fn cache_status( lines.push(Line::Job(Job { attr: t.attr.clone(), attr_path: Vec::new(), - drv_path: t.drv.clone(), + drv_path: t.drv.root.clone(), outputs, constituents: Vec::new(), named_constituents: None, diff --git a/src/remote/grpc/backend.rs b/src/remote/grpc/backend.rs index 34f878c0d..526c15584 100644 --- a/src/remote/grpc/backend.rs +++ b/src/remote/grpc/backend.rs @@ -81,6 +81,9 @@ fn build_batch( batch: &Batch<'_>, on_event: &mut dyn FnMut(Event), ) -> Result> { + if batch.jobs.iter().any(|j| !j.drv.chain.is_empty()) { + return Err(Error::Usage("dynamic derivation installables need a worker-protocol builder; gRPC does not support them".into())); + } let mut out: HashMap = batch .jobs .iter() @@ -90,7 +93,7 @@ fn build_batch( .jobs .iter() .map(|j| { - drvs.get_key_value(&j.drv) + drvs.get_key_value(&j.drv.root) .map(|(k, _)| (k, j.outputs.as_slice())) .ok_or_else(|| Error::Plan(format!("'{}' is not in the batch's closure", j.drv))) }) @@ -338,7 +341,16 @@ impl Backend for Grpc { on_event, ); self.t.common.build += t0.elapsed(); - r + let statuses = r?; + Ok(batch + .jobs + .iter() + .filter_map(|job| { + statuses + .get(&job.drv.to_string()) + .map(|status| (job.key(), status.clone())) + }) + .collect()) } fn outputs( @@ -348,7 +360,7 @@ impl Backend for Grpc { ) -> Result>> { let _ = on_event; let t0 = Instant::now(); - let path = StorePath::parse(drv)?; + let path = StorePath::parse(drv.split('!').next().unwrap())?; let d = self .drvs .get(&path) diff --git a/src/remote/worker/mod.rs b/src/remote/worker/mod.rs index a3b516232..c7e241648 100644 --- a/src/remote/worker/mod.rs +++ b/src/remote/worker/mod.rs @@ -1090,7 +1090,9 @@ impl Connection { /// checks of Nix's client included. fn read_build_result(&mut self) -> Result { let r = self.r(); - let path = wire::derived_path(&read_str(r)?).map_err(conn)?; + // Nix prints nested SingleDerivedPath steps with `^`, even in the + // legacy derived-path field used by the worker protocol. + let path = wire::derived_path(&read_str(r)?.replace('^', "!")).map_err(conn)?; // common-protocol.cc:buildResultStatusTable let status = wire::read_num(r, u8::MAX.into()).map_err(conn)?; if status > 14 { diff --git a/src/remote/worker/wire.rs b/src/remote/worker/wire.rs index 09e304490..1c1332e35 100644 --- a/src/remote/worker/wire.rs +++ b/src/remote/worker/wire.rs @@ -184,7 +184,7 @@ pub fn read_num(r: &mut dyn Read, max: u64) -> io::Result { Ok(n) } -/// derived-path.cc:DerivedPath::parseLegacy without dynamic derivations: +/// derived-path.cc:DerivedPath::parseLegacy: /// `PATH`, `PATH!*` or `PATH!NAME,...`, held as to_string_legacy writes it, /// with the names sorted and each once. pub fn derived_path(s: &str) -> io::Result { @@ -192,12 +192,14 @@ pub fn derived_path(s: &str) -> io::Result { StorePath::parse(s).map_err(|e| bad(&e.to_string()))?; return Ok(s.to_string()); }; - if drv.contains('!') { - return Err(bad(&format!( - "'{s}' builds an output of a derivation output, which needs dynamic-derivations" - ))); + let mut parts = drv.split('!'); + StorePath::parse(parts.next().unwrap()).map_err(|e| bad(&e.to_string()))?; + for output in parts { + check_name(output).map_err(|e| bad(&e.to_string()))?; + if output.contains(',') || output == "*" { + return Err(bad("a dynamic derivation step needs one output name")); + } } - StorePath::parse(drv).map_err(|e| bad(&e.to_string()))?; if outputs == "*" { return Ok(s.to_string()); } @@ -546,7 +548,10 @@ mod tests { ); assert_eq!(derived_path(&format!("{d}!*")).unwrap(), format!("{d}!*")); assert!(derived_path(d).is_ok()); - assert!(derived_path(&format!("{d}!out!bin")).is_err()); + assert_eq!( + derived_path(&format!("{d}!out!bin")).unwrap(), + format!("{d}!out!bin") + ); assert!(derived_path(&format!("{d}!")).is_err()); assert!(derived_path("/etc/passwd!out").is_err()); let hex = "4ab29c33e1fd9f3e3e1ebf6e0da0e8f8c3a5e3a73c92c4e9d6e8c04b1a5f2b3d"; diff --git a/src/store/derived.rs b/src/store/derived.rs new file mode 100644 index 000000000..2c86b3669 --- /dev/null +++ b/src/store/derived.rs @@ -0,0 +1,38 @@ +//! A derivation known now, or reached through outputs at build time. +use std::fmt; +use std::ops::Deref; + +use super::path::StorePath; + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct DerivedPath { + pub root: StorePath, + /// Output names in producer-to-consumer order. + pub chain: Vec, +} + +impl From for DerivedPath { + fn from(root: StorePath) -> Self { + Self { + root, + chain: Vec::new(), + } + } +} + +impl Deref for DerivedPath { + type Target = StorePath; + fn deref(&self) -> &StorePath { + &self.root + } +} + +impl fmt::Display for DerivedPath { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.root)?; + for output in &self.chain { + write!(f, "!{output}")?; + } + Ok(()) + } +} diff --git a/src/store/mod.rs b/src/store/mod.rs index 6bbb8ae59..3153d515a 100644 --- a/src/store/mod.rs +++ b/src/store/mod.rs @@ -1,6 +1,7 @@ //! Pure store-layer functions: hashing, NAR, store paths and derivations. //! No dependency on the runtime. +pub mod derived; pub mod drv; pub mod drvdiff; pub mod hash; diff --git a/src/trace/run.rs b/src/trace/run.rs index 5d67cf4f4..ad944494a 100644 --- a/src/trace/run.rs +++ b/src/trace/run.rs @@ -13,6 +13,7 @@ use crate::eval::Evaluator; use crate::ignored; use crate::log; use crate::session::Session; +use crate::store::derived::DerivedPath; use crate::store::narinfo::Fetched; use crate::store::path::StorePath; use crate::store::source::SourceOrigin; @@ -23,7 +24,7 @@ use crate::trace::{validate, wire}; /// What a `build` entry holds besides its read set. pub struct BuildTables { - pub roots: Vec<(StorePath, Option)>, + pub roots: Vec<(DerivedPath, Option)>, pub drvs: FxHashMap, pub sources: HashMap, pub fetched: HashMap>, @@ -231,7 +232,7 @@ impl Cache { /// [`Cache::write`] for a command that builds: `roots` is the result, and /// the tables go with it, for [`Cache::read_build`]. - pub fn write_build(&self, ev: &Evaluator, roots: &[(StorePath, Option)]) { + pub fn write_build(&self, ev: &Evaluator, roots: &[(DerivedPath, Option)]) { self.write(ev, &wire::encode_roots(roots), true); } diff --git a/src/trace/wire.rs b/src/trace/wire.rs index 17aeb1de5..294fd2790 100644 --- a/src/trace/wire.rs +++ b/src/trace/wire.rs @@ -8,6 +8,7 @@ use std::sync::Arc; use super::{Digest32, Filtered, Kind, Obs, Query}; use crate::builtins::derivation::DrvInfo; +use crate::store::derived::DerivedPath; use crate::store::hash::{Hash, HashAlgo}; use crate::store::narinfo::{Fetched, NarInfo}; use crate::store::path::StorePath; @@ -583,22 +584,37 @@ pub fn decode_sources(b: &[u8]) -> Option> { } /// The roots `iets build` resolved: drv path and the output named by context. -pub fn encode_roots(roots: &[(StorePath, Option)]) -> Vec { +pub fn encode_roots(roots: &[(DerivedPath, Option)]) -> Vec { let mut e = Enc::new(); + // Separate from old root tuples without invalidating other cache parts. + e.u64(u64::MAX); e.u64(roots.len() as u64); for (p, out) in roots { e.bytes(p.base_name().as_bytes()); + e.u64(p.chain.len() as u64); + for output in &p.chain { + e.bytes(output.as_bytes()); + } e.opt_bytes(out.as_deref().map(str::as_bytes)); } e.0 } -pub fn decode_roots(b: &[u8]) -> Option)>> { +pub fn decode_roots(b: &[u8]) -> Option)>> { let mut d = Dec::new(b).ok()?; + if d.u64().ok()? != u64::MAX { + return None; + } let n = d.u64().ok()? as usize; let mut v = Vec::with_capacity(n.min(1 << 16)); for _ in 0..n { let p = store_path(d.bytes().ok()?).ok()?; + let count = usize::try_from(d.u64().ok()?).ok()?; + let mut chain = Vec::new(); + for _ in 0..count { + chain.push(String::from_utf8(d.bytes().ok()?.to_vec()).ok()?); + } + let p = DerivedPath { root: p, chain }; let out = match d.opt_vec().ok()? { Some(o) => Some(String::from_utf8(o).ok()?), None => None, @@ -748,7 +764,17 @@ mod tests { assert_eq!((o.at.rel.as_slice(), o.nar), (&b"lib"[..], Some((hash, 7)))); assert!(o.at.accepts(Path::new("/lib")) && !o.at.accepts(Path::new("/lib/x"))); - let roots = vec![(p.clone(), Some("out".to_string())), (p, None)]; + let roots = vec![ + (p.clone().into(), Some("out".to_string())), + ( + DerivedPath { + root: p.clone(), + chain: vec!["drv".into(), "nested".into()], + }, + Some("dev".into()), + ), + (p.into(), None), + ]; assert_eq!(decode_roots(&encode_roots(&roots)).unwrap(), roots); } } diff --git a/tests/dynamic_derivations.rs b/tests/dynamic_derivations.rs index e40561e77..c054e4bfe 100644 --- a/tests/dynamic_derivations.rs +++ b/tests/dynamic_derivations.rs @@ -4,8 +4,222 @@ mod e2e; mod hermetic; +use e2e::classic::Farm; use e2e::{Store, ok, stdout}; use std::fs; +use std::path::{Path, PathBuf}; + +fn store() -> Option { + Store::start_with(&[ + ( + "extra-experimental-features", + "ca-derivations dynamic-derivations", + ), + ("allow-import-from-derivation", "false"), + ]) +} + +/// A text-addressed output containing a derivation; instantiation only, +/// never building, prepares the fixture's bytes. +fn producer(s: &Store, leaf: &Path, name: &str) -> PathBuf { + let drv = stdout(&ok(s.nix("nix-instantiate").arg(leaf))); + let drv_name = Path::new(&drv).file_name().unwrap().to_str().unwrap(); + let drv_name = drv_name.split_once('-').unwrap().1; + s.write(&format!("{name}.aterm"), &fs::read_to_string(&drv).unwrap()); + s.write( + &format!("{name}.nix"), + &format!( + r#" + derivation {{ + name = "{drv_name}"; + system = "@SYSTEM@"; + builder = "@SHELL@"; + __contentAddressed = true; + outputHashMode = "text"; + outputHashAlgo = "sha256"; + args = [ "-c" ''IFS= read -r aterm < "$source"; printf '%s' "$aterm" > "$out"'' ]; + source = ./{name}.aterm; + references = (import {leaf}).source or ""; + }} + "#, + leaf = leaf.display() + ), + ) +} + +#[test] +fn direct_dynamic_output_installables_cache_link_and_match_nix() { + let Some(s) = store() else { return }; + let leaf = s.write( + "direct-leaf.nix", + r#" + derivation { + name = "direct-leaf-@NONCE@"; + system = "@SYSTEM@"; + builder = "@SHELL@"; + outputs = [ "out" "dev" ]; + args = [ "-c" ''echo direct > "$out"; echo headers > "$dev"'' ]; + } + "#, + ); + producer(&s, &leaf, "producer"); + let file = s.write( + "direct.nix", + r#" + builtins.outputOf (import ./producer.nix).outPath "dev" + "#, + ); + let result = stdout(&ok(s.iets().args(["build", "--impure"]).arg(&file))); + assert_eq!(fs::read_to_string(&result).unwrap(), "headers\n"); + let warm = ok(s.iets().args(["build", "--impure", "--timings"]).arg(&file)); + assert_eq!(stdout(&warm), result); + assert!(String::from_utf8_lossy(&warm.stderr).contains("eval: skipped")); + let reference = stdout(&ok(s + .nix("nix") + .args([ + "build", + "--impure", + "--no-link", + "--print-out-paths", + "--file", + ]) + .arg(&file))); + assert_eq!(result, reference); + let link = s.dir().join("direct-result"); + ok(s.iets() + .args(["build", "--impure", "--out-link"]) + .arg(&link) + .arg(&file)); + assert_eq!( + fs::read_link(s.dir().join("direct-result-dev")).unwrap(), + Path::new(&result) + ); + let farm = Farm::new(&s); + let modern = stdout(&ok(farm + .ours("nix") + .args([ + "build", + "--impure", + "--no-link", + "--print-out-paths", + "--file", + ]) + .arg(&file) + .env("IETS_NIX_FALLBACK", "never"))); + assert_eq!(modern, reference); + + let multiple = s.write( + "multiple.nix", + r#" + let p = import ./producer.nix; + in [ (builtins.outputOf p.outPath "out") (builtins.outputOf p.outPath "dev") ] + "#, + ); + let both = stdout(&ok(s.iets().args(["build", "--impure"]).arg(&multiple))); + let values: Vec<_> = both + .lines() + .map(|p| fs::read_to_string(p).unwrap()) + .collect(); + assert_eq!(values, ["direct\n", "headers\n"]); +} + +#[test] +fn nested_dynamic_installables_do_not_build_during_evaluation() { + let Some(s) = store() else { return }; + let leaf = s.write( + "nested-leaf.nix", + r#" + derivation { + name = "nested-leaf-@NONCE@"; + system = "@SYSTEM@"; + builder = "@SHELL@"; + args = [ "-c" ''echo nested > "$out"'' ]; + } + "#, + ); + let first = producer(&s, &leaf, "inner"); + producer(&s, &first, "outer"); + let file = s.write( + "nested.nix", + r#" + builtins.outputOf (builtins.outputOf (import ./outer.nix).outPath "out") "out" + "#, + ); + let leaf_drv = stdout(&ok(s.nix("nix-instantiate").arg(&leaf))); + let expected = stdout(&ok(s.nix("nix-store").args([ + "--query", + "--outputs", + &leaf_drv, + ]))); + ok(s.iets() + .args(["build", "--impure", "--no-build"]) + .arg(&file)); + assert!(!Path::new(&expected).exists()); + ok(s.iets().args(["build", "--impure", "--dry-run"]).arg(&file)); + assert!(!Path::new(&expected).exists()); + let result = stdout(&ok(s.iets().args(["build", "--impure"]).arg(&file))); + assert_eq!(fs::read_to_string(&result).unwrap(), "nested\n"); + assert_eq!(result, expected); + let warm = stdout(&ok(s.iets().args(["build", "--impure"]).arg(&file))); + assert_eq!(warm, result); + let farm = Farm::new(&s); + let modern = stdout(&ok(farm + .ours("nix") + .args([ + "build", + "--impure", + "--no-link", + "--print-out-paths", + "--file", + ]) + .arg(&file) + .env("IETS_NIX_FALLBACK", "never"))); + assert_eq!(modern, result); + // Determinate 3.24.2 cannot parse its own nested result (`^out!out!out`). + // Compare the resulting path to Nix building the concrete leaf instead. + let reference = stdout(&ok(s + .nix("nix") + .args([ + "build", + "--impure", + "--no-link", + "--print-out-paths", + "--file", + ]) + .arg(&leaf))); + assert_eq!(result, reference); +} + +#[test] +fn missing_dynamic_output_fails_without_reporting_a_producer_as_success() { + let Some(s) = store() else { return }; + let leaf = s.write( + "failure-leaf.nix", + r#" + derivation { + name = "failure-leaf-@NONCE@"; + system = "@SYSTEM@"; + builder = "@SHELL@"; + args = [ "-c" ''echo good > "$out"'' ]; + } + "#, + ); + producer(&s, &leaf, "failure-producer"); + let file = s.write( + "missing.nix", + r#" + builtins.outputOf (import ./failure-producer.nix).outPath "missing" + "#, + ); + let result = s + .iets() + .args(["build", "--impure"]) + .arg(&file) + .output() + .unwrap(); + assert!(!result.status.success()); + assert!(result.stdout.is_empty()); +} #[test] fn builds_generated_derivation_without_import_from_derivation() {