From f11504de9c7d2e428d89eb4dc9418155f771f3d9 Mon Sep 17 00:00:00 2001 From: Ryan Mulligan Date: Sun, 8 Nov 2026 22:43:47 +0000 Subject: [PATCH 1/3] Support dynamic derivation build results Assisted-by: Replit --- bench/dynamic-upm-compile.py | 62 +++++++++++++++++++++++ bench/dynamic-upm.py | 74 ++++++++++++++++++++++++++++ docs/how-to/dynamic-derivations.md | 55 +++++++++++++++++++++ src/api/drvshow.rs | 54 +++++++++++++++++--- src/build/backend.rs | 11 +++++ src/remote/grpc/drvwire.rs | 23 ++++++++- src/remote/worker/mod.rs | 19 +++++-- src/remote/worker/wire.rs | 7 +-- tests/dynamic_derivations.rs | 79 ++++++++++++++++++++++++++++++ 9 files changed, 370 insertions(+), 14 deletions(-) create mode 100644 bench/dynamic-upm-compile.py create mode 100644 bench/dynamic-upm.py create mode 100644 docs/how-to/dynamic-derivations.md create mode 100644 tests/dynamic_derivations.rs diff --git a/bench/dynamic-upm-compile.py b/bench/dynamic-upm-compile.py new file mode 100644 index 000000000..12ff51d09 --- /dev/null +++ b/bench/dynamic-upm-compile.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Measure UPM application recompilation with cached external dependencies. + +Each trial gets a fresh source snapshot with an equal-length Go comment added +to every local Go file. This invalidates UPM packages without deleting store +objects. It is not a cold toolchain/dependency benchmark. +""" +import argparse +import json +import os +import pathlib +import shutil +import subprocess +import tempfile +import time + +p = argparse.ArgumentParser(description=__doc__) +p.add_argument("--drvgo", type=pathlib.Path, required=True) +p.add_argument("--source", type=pathlib.Path, required=True) +p.add_argument("--iets", required=True) +p.add_argument("--runs", type=int, default=3) +p.add_argument("--output", type=pathlib.Path, required=True) +a = p.parse_args() +if a.runs < 1: + p.error("--runs must be positive") +env = dict(os.environ, NIX_REMOTE="daemon", IETS_PROGRESS="false") +results = [] +with tempfile.TemporaryDirectory(prefix="upm-compile-") as tmp: + for trial in range(a.runs): + clients = ["iets", "determinate"] + if trial % 2: + clients.reverse() + for client in clients: + src = pathlib.Path(tmp) / f"{trial}-{client}" + shutil.copytree(a.source, src) + for path in src.rglob("*.go"): + path.chmod(0o644) + with path.open("a") as f: + f.write(f"\n// benchmark {trial:03d} {client:11s}\n") + expr = f'(import ./examples/upm/dynamic.nix {{ source = {src}; }}).package' + cmd = ([a.iets, "build", "--impure", "--no-tracing-cache", "--workers", "1"] + if client == "iets" else ["nix-build", "--no-out-link"]) + cmd += ["-E", expr] + start = time.perf_counter() + r = subprocess.run(cmd, cwd=a.drvgo, env=env, text=True, capture_output=True) + elapsed = time.perf_counter() - start + log = a.output.with_suffix(f".{trial}-{client}.log") + log.write_text(r.stderr) + if r.returncode: + raise RuntimeError(f"{cmd} failed: see {log}") + output = r.stdout.strip() + smoke = subprocess.run([output + "/bin/upm", "--help"], + text=True, capture_output=True) + if smoke.returncode: + raise RuntimeError(f"UPM smoke test failed: {smoke.stderr}") + results.append(dict(client=client, trial=trial, seconds=elapsed, + output=output, command=cmd, help_exit=smoke.returncode)) + a.output.write_text(json.dumps({ + "note": "Local UPM package recompilation plus planning/evaluation/database/wrapping; external dependencies cached.", + "samples": results, + }, indent=2) + "\n") + print(client, trial, elapsed, flush=True) diff --git a/bench/dynamic-upm.py b/bench/dynamic-upm.py new file mode 100644 index 000000000..5b0c0cdf3 --- /dev/null +++ b/bench/dynamic-upm.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Alternate UPM dynamic-derivation evaluation and warm builds. + +Run after provisioning drvgo's pinned toolchain and building UPM once. +Compilation is deliberately not inferred from a cached realization. +""" +import argparse +import json +import os +import pathlib +import statistics +import subprocess +import time + +p = argparse.ArgumentParser(description=__doc__) +p.add_argument("--drvgo", required=True, type=pathlib.Path) +p.add_argument("--source", required=True, help="Pinned UPM source store path") +p.add_argument("--iets", required=True) +p.add_argument("--runs", type=int, default=5) +p.add_argument("--output", type=pathlib.Path, required=True) +a = p.parse_args() +if a.runs < 1: + p.error("--runs must be positive") +expr = f'(import ./examples/upm/dynamic.nix {{ source = {a.source}; }}).package' +commands = { + "eval": { + "iets": [a.iets, "eval", "--impure", "--raw", "--no-tracing-cache", + "--workers", "1", "-E", expr + ".drvPath"], + # Read-write mode is essential: outputOf reads its producer's .drv. + "determinate": ["nix-instantiate", "--eval", "--strict", "--read-write-mode", + "-E", expr + ".drvPath"], + }, + "warm-build": { + "iets": [a.iets, "build", "--impure", "--no-tracing-cache", + "--workers", "1", "-E", expr], + "determinate": ["nix-build", "--no-out-link", "-E", expr], + }, +} +env = dict(os.environ, NIX_REMOTE="daemon", IETS_PROGRESS="false") +records = [] +for phase, clients in commands.items(): + expected = None + for run in range(a.runs + 1): + for client in (["iets", "determinate"] if run % 2 == 0 + else ["determinate", "iets"]): + start = time.perf_counter() + r = subprocess.run(clients[client], cwd=a.drvgo, env=env, + text=True, capture_output=True) + elapsed = time.perf_counter() - start + if r.returncode: + raise RuntimeError(f"{clients[client]}:\n{r.stderr}") + result = r.stdout.strip().strip('"') + if expected is None: + expected = result + if result != expected: + raise RuntimeError(f"{phase}: results differ: {result} != {expected}") + if run: + records.append(dict(phase=phase, client=client, run=run, + seconds=elapsed, result=result)) +summary = { + phase: { + client: statistics.median(x["seconds"] for x in records + if x["phase"] == phase and x["client"] == client) + for client in clients + } for phase, clients in commands.items() +} +report = dict(commands=commands, samples=records, medians_seconds=summary, + source=a.source, + nix_version=subprocess.check_output(["nix", "--version"], text=True).strip(), + drvgo_revision=subprocess.check_output( + ["git", "-C", str(a.drvgo), "rev-parse", "HEAD"], text=True).strip(), + note="Warm page/store caches; tracing cache disabled; compilation excluded.") +a.output.write_text(json.dumps(report, indent=2) + "\n") +print(json.dumps(summary, indent=2)) diff --git a/docs/how-to/dynamic-derivations.md b/docs/how-to/dynamic-derivations.md new file mode 100644 index 000000000..678c0425b --- /dev/null +++ b/docs/how-to/dynamic-derivations.md @@ -0,0 +1,55 @@ +# Build generated derivations + +iets evaluates `builtins.outputOf`, text-addressed derivation producers and +their consumers without import-from-derivation. Builds use a Nix daemon with +dynamic derivations enabled: + +```nix +nix.settings.experimental-features = [ + "nix-command" "flakes" "ca-derivations" "dynamic-derivations" + "recursive-nix" +]; +nix.settings.system-features = [ "recursive-nix" "ca-derivations" ]; +``` + +`recursive-nix` is needed when the producer runs a nested Nix evaluator, +as drvgo does. It is not needed for a producer that simply writes a `.drv`. +Enable these features on the build daemon, not just on the client. + +A consumer can refer to a generated derivation's output: + +```nix +pkgs.runCommand "consumer" {} '' + ln -s ${builtins.outputOf planner.outPath "out"} "$out" +'' +``` + +Build the consumer with `iets build --impure consumer.nix`. The daemon builds +the planner, loads the generated derivation and resolves the output before +building the consumer. iets uses the realized output paths returned by the +build operation; a deferred derivation need not have an output path available +through `QueryDerivationOutputMap`. + +`nix derivation show` preserves the nested `dynamicOutputs` tree. The gRPC +build farm requires static input paths and rejects dynamic inputs explicitly; +use the local daemon or a worker-protocol remote builder instead. + +Directly selecting a dynamic `outputOf` string as the installable is not yet +supported. Select a consumer derivation such as the example above. Evaluation +still never builds: `getContext` on a dynamic output and reading its contents +during evaluation remain unsupported. + +## UPM with drvgo + +drvgo's `examples/upm/dynamic.nix` builds UPM, its Python database generator, +the database and the executable wrapper. Pass a pinned source store path via +the example's `source` argument: its default fetch lacks the `narHash` that +iets requires. Both clients must receive the same argument. + +`bench/dynamic-upm.py` alternates evaluation and warm realization runs, checks +that both clients return the same paths, and records every sample. +`bench/dynamic-upm-compile.py` measures application recompilation by adding a +fresh comment to local Go sources for each trial. External dependencies stay +cached. This includes planning, evaluation, local package compilation, database +generation and wrapping; it is not a cold dependency build or a measurement of +a different Go compiler. diff --git a/src/api/drvshow.rs b/src/api/drvshow.rs index df44973aa..a5cb4ea3a 100644 --- a/src/api/drvshow.rs +++ b/src/api/drvshow.rs @@ -15,7 +15,7 @@ use crate::error::{Error, Result}; use crate::eval::Evaluator; use crate::print; use crate::remote::plan::{load_drv, walk}; -use crate::store::drv::{Derivation, DerivationOutput}; +use crate::store::drv::{Derivation, DerivationOutput, InputNode}; use crate::store::path::{IngestionMethod, StorePath}; use crate::syntax::symbol::FxHashMap; @@ -187,7 +187,7 @@ fn inputs(drv: &Derivation) -> Value { "drvs".into(), Value::Object( (drv.input_drvs.iter()) - .map(|(p, node)| (p.base_name().to_string(), input_drv(&node.outputs))) + .map(|(p, node)| (p.base_name().to_string(), input_drv(node))) .collect(), ), ); @@ -195,14 +195,26 @@ fn inputs(drv: &Derivation) -> Value { Value::Object(o) } -/// One `inputDrvs` entry. `dynamicOutputs` is always empty: iets refuses a -/// dynamic-derivation input when it parses the ATerm. -fn input_drv(outputs: &BTreeSet) -> Value { +/// Keep each layer of generated derivations, not just the static outputs. +fn input_drv(node: &InputNode) -> Value { let mut o = Map::new(); - o.insert("dynamicOutputs".into(), Value::Object(Map::new())); + o.insert( + "dynamicOutputs".into(), + Value::Object( + node.children + .iter() + .map(|(name, child)| (name.clone(), input_drv(child))) + .collect(), + ), + ); o.insert( "outputs".into(), - Value::Array(outputs.iter().map(|s| Value::from(s.as_str())).collect()), + Value::Array( + node.outputs + .iter() + .map(|s| Value::from(s.as_str())) + .collect(), + ), ); Value::Object(o) } @@ -256,12 +268,40 @@ fn paths(v: &BTreeSet) -> Value { mod tests { use super::*; use rstest::rstest; + use std::collections::BTreeMap; fn json_of(aterm: &str, name: &str) -> String { let drv = Derivation::parse(aterm.as_bytes(), name).unwrap(); derivation(&drv).unwrap().to_string() } + #[test] + fn dynamic_inputs_preserve_every_layer_and_static_outputs() { + let leaf = InputNode::of(BTreeSet::from(["bin".into(), "dev".into()])); + let nested = InputNode { + outputs: BTreeSet::from(["log".into()]), + children: BTreeMap::from([("drv".into(), leaf)]), + }; + let root = InputNode { + outputs: BTreeSet::from(["out".into()]), + children: BTreeMap::from([("generated".into(), nested)]), + }; + assert_eq!( + input_drv(&root), + serde_json::json!({ + "outputs": ["out"], + "dynamicOutputs": { + "generated": { + "outputs": ["log"], + "dynamicOutputs": { + "drv": {"outputs": ["bin", "dev"], "dynamicOutputs": {}} + } + } + } + }) + ); + } + #[rstest] #[case::a_multi_output_derivation( r#"Derive([("dev","/nix/store/y7a2ldlsm4a9b89dy8gjq9w2jxrg9ni7-drvshow-multi-dev","",""),("out","/nix/store/glmm5g4k4ni9sqr84qzq47c7z839qy2v-drvshow-multi","","")],[],[],"x86_64-linux","/bin/sh",["-c","echo a > $out; echo b > $dev"],[("builder","/bin/sh"),("dev","/nix/store/y7a2ldlsm4a9b89dy8gjq9w2jxrg9ni7-drvshow-multi-dev"),("name","drvshow-multi"),("out","/nix/store/glmm5g4k4ni9sqr84qzq47c7z839qy2v-drvshow-multi"),("outputs","out dev"),("system","x86_64-linux")])"#, diff --git a/src/build/backend.rs b/src/build/backend.rs index 1e1197a20..7fbbaa748 100644 --- a/src/build/backend.rs +++ b/src/build/backend.rs @@ -325,6 +325,7 @@ pub struct Worker { transport: Transport, options: Options, t: Timings, + built_outputs: HashMap>>, } impl Worker { @@ -342,6 +343,7 @@ impl Worker { conn, transport: transport.clone(), options: options.clone(), + built_outputs: HashMap::new(), }) } } @@ -457,6 +459,12 @@ impl Backend for Worker { let mut out: HashMap = HashMap::new(); for r in &results { let drv = r.path.split('!').next().unwrap_or(&r.path).to_string(); + if r.success() { + self.built_outputs + .entry(drv.clone()) + .or_default() + .extend(r.outputs.clone()); + } let e = out.entry(drv).or_insert_with(Status::ok); if !r.success() { let exit = log::build_exit(r.status, &r.msg); @@ -479,6 +487,9 @@ impl Backend for Worker { drv: &str, on_event: &mut dyn FnMut(Event), ) -> Result>> { + if let Some(outputs) = self.built_outputs.get(drv).filter(|o| !o.is_empty()) { + return Ok(outputs.clone()); + } let t0 = Instant::now(); let map = self.conn.query_derivation_output_map(drv, on_event)?; self.t.outputs += t0.elapsed(); diff --git a/src/remote/grpc/drvwire.rs b/src/remote/grpc/drvwire.rs index 4e759a4ac..3e97df9cb 100644 --- a/src/remote/grpc/drvwire.rs +++ b/src/remote/grpc/drvwire.rs @@ -230,6 +230,12 @@ pub fn basic_input_srcs( ) -> Result> { let mut srcs = drv.input_srcs.clone(); for (dep, node) in &drv.input_drvs { + if !node.children.is_empty() { + return Err(Error::Plan(format!( + "'{drv_path}': input '{dep}' contains dynamic derivations; \ + use a worker-protocol builder to resolve generated inputs" + ))); + } for o in &node.outputs { let Some(p) = output_of(dep, o) else { return Err(Error::Plan(format!( @@ -246,7 +252,7 @@ pub fn basic_input_srcs( #[cfg(test)] mod tests { use super::*; - use crate::store::drv::Derivation; + use crate::store::drv::{Derivation, InputNode}; const ATERM: &str = r#"Derive([("out","/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo","","")],[("/nix/store/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-dep.drv",["out"])],["/nix/store/cccccccccccccccccccccccccccccccc-src"],"x86_64-linux","/bin/sh",["-c","true"],[("__json","{\"a\":1}"),("out","/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo"),("zz","last")])"#; @@ -254,6 +260,21 @@ mod tests { Derivation::parse(ATERM.as_bytes(), "foo").unwrap() } + #[test] + fn dynamic_inputs_are_not_silently_dropped() { + let mut drv = fixture(); + let node = drv.input_drvs.values_mut().next().unwrap(); + node.outputs.clear(); + node.children + .insert("out".into(), InputNode::of(BTreeSet::from(["out".into()]))); + let path = StorePath::parse("/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo.drv").unwrap(); + let err = basic_input_srcs(&path, &drv, &|_, _| { + panic!("a dynamic input must not be resolved as a static one") + }) + .unwrap_err(); + assert!(err.to_string().contains("worker-protocol builder")); + } + /// Reproduces `derivation::write` by hand: the field order and the empty /// hash-algo/hash strings for an input-addressed output. #[test] diff --git a/src/remote/worker/mod.rs b/src/remote/worker/mod.rs index 88fcacfea..a3b516232 100644 --- a/src/remote/worker/mod.rs +++ b/src/remote/worker/mod.rs @@ -484,6 +484,8 @@ pub struct BuildResult { pub path: String, pub status: u64, pub msg: String, + /// Resolved paths, including outputs of deferred derivations. + pub outputs: BTreeMap>, } impl BuildResult { @@ -1119,11 +1121,21 @@ impl Connection { } let r = self.r(); let outputs = read_u64(r)?; + let mut built_outputs = BTreeMap::new(); for _ in 0..outputs { - wire::drv_output(&read_str(r)?).map_err(conn)?; - wire::realisation(&wire::read_bytes(r, wire::MAX_STRING)?).map_err(conn)?; + let id = read_str(r)?; + wire::drv_output(&id).map_err(conn)?; + let name = id.split_once('!').expect("validated output id").1; + check_name(name).map_err(|e| Error::Protocol(e.to_string()))?; + let path = wire::realisation(&wire::read_bytes(r, wire::MAX_STRING)?).map_err(conn)?; + built_outputs.insert(name.to_string(), Some(path.to_string())); } - Ok(BuildResult { path, status, msg }) + Ok(BuildResult { + path, + status, + msg, + outputs: built_outputs, + }) } /// `QueryMissing`; `paths` are derived paths, as for `BuildPathsWithResults`. @@ -1314,6 +1326,7 @@ mod tests { path: String::new(), status, msg: String::new(), + outputs: BTreeMap::new(), }; assert!(r(0).success() && r(13).success()); assert!(!r(256).success() && !r(269).success() && !r(3).success()); diff --git a/src/remote/worker/wire.rs b/src/remote/worker/wire.rs index 8fa295252..09e304490 100644 --- a/src/remote/worker/wire.rs +++ b/src/remote/worker/wire.rs @@ -222,7 +222,7 @@ pub fn drv_output(s: &str) -> io::Result<()> { /// `common-protocol.cc:Serialise`: the JSON of realisation.cc, /// `outPath` a store path's base name, `signatures` if there, and `id`. -pub fn realisation(raw: &[u8]) -> io::Result<()> { +pub fn realisation(raw: &[u8]) -> io::Result { let bad_json = |why: &str| bad(&format!("while parsing a realisation object: {why}")); let v: serde_json::Value = serde_json::from_slice(raw).map_err(|e| bad_json(&e.to_string()))?; let o = v.as_object().ok_or_else(|| bad_json("not an object"))?; @@ -232,7 +232,7 @@ pub fn realisation(raw: &[u8]) -> io::Result<()> { .as_str() .ok_or_else(|| bad_json(&format!("'{key}' is not a string"))) }; - StorePath::from_base_name(string("outPath")?).map_err(|e| bad(&e.to_string()))?; + let path = StorePath::from_base_name(string("outPath")?).map_err(|e| bad(&e.to_string()))?; if let Some(sigs) = o.get("signatures") { for s in sigs .as_array() @@ -244,7 +244,8 @@ pub fn realisation(raw: &[u8]) -> io::Result<()> { )?; } } - drv_output(string("id")?) + drv_output(string("id")?)?; + Ok(path) } /// Mirrors serialise.hh:FramedSink: the payload of `AddMultipleToStore` goes out as diff --git a/tests/dynamic_derivations.rs b/tests/dynamic_derivations.rs new file mode 100644 index 000000000..e40561e77 --- /dev/null +++ b/tests/dynamic_derivations.rs @@ -0,0 +1,79 @@ +//! Exercise a generated derivation through the real daemon, without IFD. +#![cfg(unix)] + +mod e2e; +mod hermetic; + +use e2e::{Store, ok, stdout}; +use std::fs; + +#[test] +fn builds_generated_derivation_without_import_from_derivation() { + let Some(s) = Store::start_with(&[ + ( + "extra-experimental-features", + "ca-derivations dynamic-derivations", + ), + ("allow-import-from-derivation", "false"), + ]) else { + return; + }; + let leaf = s.write( + "leaf.nix", + r#" + derivation { + name = "dynamic-leaf-@NONCE@"; + system = "@SYSTEM@"; + builder = "@SHELL@"; + args = [ "-c" "echo dynamic-success > $out" ]; + } + "#, + ); + let drv = stdout(&ok(s.nix("nix-instantiate").arg(&leaf))); + // The producer copies the ATerm as text, not as a derivation dependency. + s.write("leaf.aterm", &fs::read_to_string(&drv).unwrap()); + let file = s.write( + "dynamic.nix", + r#" + let + producer = derivation { + name = "dynamic-leaf-@NONCE@.drv"; + system = "@SYSTEM@"; + builder = "@SHELL@"; + __contentAddressed = true; + outputHashMode = "text"; + outputHashAlgo = "sha256"; + args = [ "-c" ''printf '%s' "$aterm" > "$out"'' ]; + aterm = builtins.readFile ./leaf.aterm; + }; + in derivation { + name = "dynamic-consumer-@NONCE@"; + system = "@SYSTEM@"; + builder = "@SHELL@"; + args = [ "-c" ''read value < "$input"; printf '%s\n' "$value" > "$out"'' ]; + input = builtins.outputOf producer.outPath "out"; + } + "#, + ); + let expected = stdout(&ok(s.nix("nix-instantiate").arg(&file))); + // Leave no generated output for iets to accidentally reuse. + let built = ok(s + .iets() + .args(["build", "--impure", "--no-tracing-cache"]) + .arg(&file)); + let result = stdout(&built); + assert_eq!( + fs::read_to_string(result.trim()).unwrap(), + "dynamic-success\n" + ); + let reference = stdout(&ok(s.nix("nix-build").arg(&file).arg("--no-out-link"))); + assert_eq!(result, reference); + // AlreadyValid must carry the resolved path too, not only a fresh build. + let warm = stdout(&ok(s.iets().args(["build", "--impure"]).arg(&file))); + assert_eq!(warm, reference); + let actual = stdout(&ok(s + .iets() + .args(["eval", "--impure", "--raw", "-E"]) + .arg(format!("(import {}).drvPath", file.display())))); + assert_eq!(actual, expected); +} From 29d78b86c6cfbdd3c2c42666079423bfeac4d3d0 Mon Sep 17 00:00:00 2001 From: Ryan Mulligan Date: Sun, 8 Nov 2026 22:53:09 +0000 Subject: [PATCH 2/3] Clarify source-change benchmark scope Assisted-by: Replit --- bench/dynamic-upm-compile.py | 7 ++++--- docs/how-to/dynamic-derivations.md | 8 ++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/bench/dynamic-upm-compile.py b/bench/dynamic-upm-compile.py index 12ff51d09..0f9bc339d 100644 --- a/bench/dynamic-upm-compile.py +++ b/bench/dynamic-upm-compile.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 -"""Measure UPM application recompilation with cached external dependencies. +"""Measure UPM source-change builds in a prewarmed store. Each trial gets a fresh source snapshot with an equal-length Go comment added to every local Go file. This invalidates UPM packages without deleting store -objects. It is not a cold toolchain/dependency benchmark. +objects. Some external cgo packages may also rebuild; it is not a cold-store +benchmark or an isolated measurement of the compiler. """ import argparse import json @@ -56,7 +57,7 @@ results.append(dict(client=client, trial=trial, seconds=elapsed, output=output, command=cmd, help_exit=smoke.returncode)) a.output.write_text(json.dumps({ - "note": "Local UPM package recompilation plus planning/evaluation/database/wrapping; external dependencies cached.", + "note": "Fresh UPM source snapshots in a prewarmed store; includes planning, evaluation, compilation (including invalidated cgo dependencies), database generation and wrapping.", "samples": results, }, indent=2) + "\n") print(client, trial, elapsed, flush=True) diff --git a/docs/how-to/dynamic-derivations.md b/docs/how-to/dynamic-derivations.md index 678c0425b..43bd5e3f4 100644 --- a/docs/how-to/dynamic-derivations.md +++ b/docs/how-to/dynamic-derivations.md @@ -49,7 +49,7 @@ iets requires. Both clients must receive the same argument. `bench/dynamic-upm.py` alternates evaluation and warm realization runs, checks that both clients return the same paths, and records every sample. `bench/dynamic-upm-compile.py` measures application recompilation by adding a -fresh comment to local Go sources for each trial. External dependencies stay -cached. This includes planning, evaluation, local package compilation, database -generation and wrapping; it is not a cold dependency build or a measurement of -a different Go compiler. +fresh comment to local Go sources for each trial in a prewarmed store. +External cgo dependencies can also rebuild. This includes planning, evaluation, +compilation, database generation and wrapping; it is not a cold dependency +build or a measurement of a different Go compiler. From 4f0fe7e6fe76984c7eceefafa174e47862f9241e Mon Sep 17 00:00:00 2001 From: Ryan Mulligan Date: Sun, 8 Nov 2026 22:57:26 +0000 Subject: [PATCH 3/3] Record UPM measurements and compilation limitations Assisted-by: Replit --- bench/dynamic-upm-compile.py | 4 +- bench/results/dynamic-upm-compile.json | 95 ++++++++++++ bench/results/dynamic-upm-warm.json | 199 +++++++++++++++++++++++++ bench/results/dynamic-upm.md | 52 +++++++ 4 files changed, 348 insertions(+), 2 deletions(-) create mode 100644 bench/results/dynamic-upm-compile.json create mode 100644 bench/results/dynamic-upm-warm.json create mode 100644 bench/results/dynamic-upm.md diff --git a/bench/dynamic-upm-compile.py b/bench/dynamic-upm-compile.py index 0f9bc339d..6681a9d4c 100644 --- a/bench/dynamic-upm-compile.py +++ b/bench/dynamic-upm-compile.py @@ -37,10 +37,10 @@ for path in src.rglob("*.go"): path.chmod(0o644) with path.open("a") as f: - f.write(f"\n// benchmark {trial:03d} {client:11s}\n") + f.write(f"\n// benchmark {pathlib.Path(tmp).name} {trial:03d} {client:11s}\n") expr = f'(import ./examples/upm/dynamic.nix {{ source = {src}; }}).package' cmd = ([a.iets, "build", "--impure", "--no-tracing-cache", "--workers", "1"] - if client == "iets" else ["nix-build", "--no-out-link"]) + if client == "iets" else ["nix-build", "--no-out-link", "--max-jobs", "4", "--cores", "0"]) cmd += ["-E", expr] start = time.perf_counter() r = subprocess.run(cmd, cwd=a.drvgo, env=env, text=True, capture_output=True) diff --git a/bench/results/dynamic-upm-compile.json b/bench/results/dynamic-upm-compile.json new file mode 100644 index 000000000..cd7e084c8 --- /dev/null +++ b/bench/results/dynamic-upm-compile.json @@ -0,0 +1,95 @@ +{ + "note": "Fresh source builds in a prewarmed store; includes invalidated external cgo dependencies, planning, evaluation, database generation and wrapping.", + "samples": [ + { + "client": "iets", + "trial": 0, + "seconds": 89.08709269700012, + "output": "/nix/store/g1dxrz346vxd3iziwy7lbb6bimg3cmx0-upm-drvgo", + "command": [ + "/workspace/iets/target/release/iets", + "build", + "--impure", + "--no-tracing-cache", + "--workers", + "1", + "-E", + "(import ./examples/upm/dynamic.nix { source = /tmp/upm-compile-yr_ropw8/0-iets; }).package" + ], + "help_exit": 0 + }, + { + "client": "determinate", + "trial": 0, + "seconds": 88.91069620399958, + "output": "/nix/store/jrh0l9v6m9sqk4hn1pf9a4x74skh5i0h-upm-drvgo", + "command": [ + "nix-build", + "--no-out-link", + "-E", + "(import ./examples/upm/dynamic.nix { source = /tmp/upm-compile-yr_ropw8/0-determinate; }).package" + ], + "help_exit": 0 + }, + { + "client": "determinate", + "trial": 1, + "seconds": 89.80313975599984, + "output": "/nix/store/y3fbdpkkvyhlavd31kgbyq9zgzgz1phk-upm-drvgo", + "command": [ + "nix-build", + "--no-out-link", + "-E", + "(import ./examples/upm/dynamic.nix { source = /tmp/upm-compile-yr_ropw8/1-determinate; }).package" + ], + "help_exit": 0 + }, + { + "client": "iets", + "trial": 1, + "seconds": 84.0740111820005, + "output": "/nix/store/l4f61bkb8j9idgxp48fwbf0jydrn99aq-upm-drvgo", + "command": [ + "/workspace/iets/target/release/iets", + "build", + "--impure", + "--no-tracing-cache", + "--workers", + "1", + "-E", + "(import ./examples/upm/dynamic.nix { source = /tmp/upm-compile-yr_ropw8/1-iets; }).package" + ], + "help_exit": 0 + }, + { + "client": "iets", + "trial": 2, + "seconds": 81.07874024699959, + "output": "/nix/store/ydh2p8dbqa13sm8wbcv4d5cvcxbnn98z-upm-drvgo", + "command": [ + "/workspace/iets/target/release/iets", + "build", + "--impure", + "--no-tracing-cache", + "--workers", + "1", + "-E", + "(import ./examples/upm/dynamic.nix { source = /tmp/upm-compile-yr_ropw8/2-iets; }).package" + ], + "help_exit": 0 + }, + { + "client": "determinate", + "trial": 2, + "seconds": 89.69095421800012, + "output": "/nix/store/1f4k1l1w0qfs2fcb3rmw0r0kgx7zghcx-upm-drvgo", + "command": [ + "nix-build", + "--no-out-link", + "-E", + "(import ./examples/upm/dynamic.nix { source = /tmp/upm-compile-yr_ropw8/2-determinate; }).package" + ], + "help_exit": 0 + } + ] +} diff --git a/bench/results/dynamic-upm-warm.json b/bench/results/dynamic-upm-warm.json new file mode 100644 index 000000000..f52936482 --- /dev/null +++ b/bench/results/dynamic-upm-warm.json @@ -0,0 +1,199 @@ +{ + "commands": { + "eval": { + "iets": [ + "/workspace/iets/target/release/iets", + "eval", + "--impure", + "--raw", + "--no-tracing-cache", + "--workers", + "1", + "-E", + "(import ./examples/upm/dynamic.nix { source = /nix/store/r3azknj8cz74gnw86kdq2jcjvg4vc1wd-source; }).package.drvPath" + ], + "determinate": [ + "nix-instantiate", + "--eval", + "--strict", + "--read-write-mode", + "-E", + "(import ./examples/upm/dynamic.nix { source = /nix/store/r3azknj8cz74gnw86kdq2jcjvg4vc1wd-source; }).package.drvPath" + ] + }, + "warm-build": { + "iets": [ + "/workspace/iets/target/release/iets", + "build", + "--impure", + "--no-tracing-cache", + "--workers", + "1", + "-E", + "(import ./examples/upm/dynamic.nix { source = /nix/store/r3azknj8cz74gnw86kdq2jcjvg4vc1wd-source; }).package" + ], + "determinate": [ + "nix-build", + "--no-out-link", + "-E", + "(import ./examples/upm/dynamic.nix { source = /nix/store/r3azknj8cz74gnw86kdq2jcjvg4vc1wd-source; }).package" + ] + } + }, + "samples": [ + { + "phase": "eval", + "client": "determinate", + "run": 1, + "seconds": 1.3048401899995952, + "result": "/nix/store/r5q8qap0bs40lgwjgn41qwizpcdi5n26-upm-drvgo.drv" + }, + { + "phase": "eval", + "client": "iets", + "run": 1, + "seconds": 0.6081037739995736, + "result": "/nix/store/r5q8qap0bs40lgwjgn41qwizpcdi5n26-upm-drvgo.drv" + }, + { + "phase": "eval", + "client": "iets", + "run": 2, + "seconds": 0.5506039339998097, + "result": "/nix/store/r5q8qap0bs40lgwjgn41qwizpcdi5n26-upm-drvgo.drv" + }, + { + "phase": "eval", + "client": "determinate", + "run": 2, + "seconds": 1.4588833759999034, + "result": "/nix/store/r5q8qap0bs40lgwjgn41qwizpcdi5n26-upm-drvgo.drv" + }, + { + "phase": "eval", + "client": "determinate", + "run": 3, + "seconds": 1.326071896999565, + "result": "/nix/store/r5q8qap0bs40lgwjgn41qwizpcdi5n26-upm-drvgo.drv" + }, + { + "phase": "eval", + "client": "iets", + "run": 3, + "seconds": 0.6089354599998842, + "result": "/nix/store/r5q8qap0bs40lgwjgn41qwizpcdi5n26-upm-drvgo.drv" + }, + { + "phase": "eval", + "client": "iets", + "run": 4, + "seconds": 0.581556850999732, + "result": "/nix/store/r5q8qap0bs40lgwjgn41qwizpcdi5n26-upm-drvgo.drv" + }, + { + "phase": "eval", + "client": "determinate", + "run": 4, + "seconds": 1.3731455639999695, + "result": "/nix/store/r5q8qap0bs40lgwjgn41qwizpcdi5n26-upm-drvgo.drv" + }, + { + "phase": "eval", + "client": "determinate", + "run": 5, + "seconds": 1.593434630000047, + "result": "/nix/store/r5q8qap0bs40lgwjgn41qwizpcdi5n26-upm-drvgo.drv" + }, + { + "phase": "eval", + "client": "iets", + "run": 5, + "seconds": 0.5877071329996397, + "result": "/nix/store/r5q8qap0bs40lgwjgn41qwizpcdi5n26-upm-drvgo.drv" + }, + { + "phase": "warm-build", + "client": "determinate", + "run": 1, + "seconds": 1.5679668160000801, + "result": "/nix/store/b6j7yh4ys9594j6yqzsdph1lnk2m0rdn-upm-drvgo" + }, + { + "phase": "warm-build", + "client": "iets", + "run": 1, + "seconds": 0.5791242459999921, + "result": "/nix/store/b6j7yh4ys9594j6yqzsdph1lnk2m0rdn-upm-drvgo" + }, + { + "phase": "warm-build", + "client": "iets", + "run": 2, + "seconds": 0.6153836760004197, + "result": "/nix/store/b6j7yh4ys9594j6yqzsdph1lnk2m0rdn-upm-drvgo" + }, + { + "phase": "warm-build", + "client": "determinate", + "run": 2, + "seconds": 1.4069957619994966, + "result": "/nix/store/b6j7yh4ys9594j6yqzsdph1lnk2m0rdn-upm-drvgo" + }, + { + "phase": "warm-build", + "client": "determinate", + "run": 3, + "seconds": 1.414531684999929, + "result": "/nix/store/b6j7yh4ys9594j6yqzsdph1lnk2m0rdn-upm-drvgo" + }, + { + "phase": "warm-build", + "client": "iets", + "run": 3, + "seconds": 0.603366731999813, + "result": "/nix/store/b6j7yh4ys9594j6yqzsdph1lnk2m0rdn-upm-drvgo" + }, + { + "phase": "warm-build", + "client": "iets", + "run": 4, + "seconds": 0.603380556999582, + "result": "/nix/store/b6j7yh4ys9594j6yqzsdph1lnk2m0rdn-upm-drvgo" + }, + { + "phase": "warm-build", + "client": "determinate", + "run": 4, + "seconds": 1.3520378690000143, + "result": "/nix/store/b6j7yh4ys9594j6yqzsdph1lnk2m0rdn-upm-drvgo" + }, + { + "phase": "warm-build", + "client": "determinate", + "run": 5, + "seconds": 1.2411019099999976, + "result": "/nix/store/b6j7yh4ys9594j6yqzsdph1lnk2m0rdn-upm-drvgo" + }, + { + "phase": "warm-build", + "client": "iets", + "run": 5, + "seconds": 0.5695490429998245, + "result": "/nix/store/b6j7yh4ys9594j6yqzsdph1lnk2m0rdn-upm-drvgo" + } + ], + "medians_seconds": { + "eval": { + "iets": 0.5877071329996397, + "determinate": 1.3731455639999695 + }, + "warm-build": { + "iets": 0.603366731999813, + "determinate": 1.4069957619994966 + } + }, + "source": "/nix/store/r3azknj8cz74gnw86kdq2jcjvg4vc1wd-source", + "nix_version": "nix (Determinate Nix 3.24.2) 2.35.3", + "drvgo_revision": "957f2c8805c639848ba1ddcae3097586d50d2a25", + "note": "Warm page/store caches; tracing cache disabled; compilation excluded." +} diff --git a/bench/results/dynamic-upm.md b/bench/results/dynamic-upm.md new file mode 100644 index 000000000..f44ad254e --- /dev/null +++ b/bench/results/dynamic-upm.md @@ -0,0 +1,52 @@ +# UPM dynamic derivations + +Measured on 2026-11-08 with Determinate Nix 3.24.2 (Nix 2.35.3), +iets's worker-output fix, and drvgo's pinned Go toolchain. The machine was +x86_64 Linux on an Intel Xeon Platinum 8481C, limited to 4 CPUs and 12 GiB. +Both clients used the same daemon. The first compilation trials inadvertently +used different scheduling: iets's native build path used four jobs and all +cores per job, while the Nix client used two jobs and two cores per job. +Nix sandboxing was unavailable in this VM; the daemon used sandbox fallback. + +The UPM source was the revision pinned by drvgo's UPM example. Both clients +received the same source store path for evaluation and cached-build trials. +iets used one evaluator worker with its tracing cache disabled. Determinate +Nix used its configured evaluator defaults. Measurements ran after setup +builds and Rust compilation finished, with alternating client order. + +| Workload | iets median | Determinate median | Speedup | +|---|---:|---:|---:| +| Evaluation, five measured runs | 0.588 s | 1.373 s | 2.34× | +| Cached build, five measured runs | 0.603 s | 1.407 s | 2.33× | + +The evaluation and cached-build scripts checked identical output paths. +Each fresh-source trial added an equal-length comment to local Go files. +These trials used a prewarmed store but rebuilt some external cgo dependencies +(including sqlite3), as well as local packages, the database and wrapper. +Every resulting UPM executable passed `--help`. This is end-to-end +source-change build time, not an isolated compiler or cold-store measurement. +Both clients use the same Go compiler and Nix build daemon. + +Initial fresh-source samples in seconds (different budgets, not comparable): + +| Trial | iets | Determinate | +|---|---:|---:| +| 1 | 89.087 | 88.911 | +| 2 | 84.074 | 89.803 | +| 3 | 81.079 | 89.691 | + +No compilation speedup can be concluded from those samples. A new run with +both clients using four jobs and all cores completed the first iets build +in 76.186 seconds, but the Determinate build failed: its generated forwarding +derivation was killed with signal 9. The cgroup reported no OOM events or +OOM kills. The cause is unresolved. A similar forwarding-build failure +occurred during the initial iets setup; neither failure supports a claim +that one client is more reliable. + +The matched run was excluded rather than retrying a partly cached build as +if it were fresh. Evaluation and warm-build results remain valid; all fresh +builds share the same compiler. A sandbox-capable host is recommended for a +reliable compilation comparison. + +Reproduce with `bench/dynamic-upm.py` and `bench/dynamic-upm-compile.py`. +Raw samples and commands are in the adjacent JSON files.