From 959782cb19d136da4eef1e9e5af97e2549075c55 Mon Sep 17 00:00:00 2001 From: Frank McSherry Date: Fri, 20 Nov 2026 09:32:00 -0500 Subject: [PATCH] DDIR: sum integer values by key --- interactive/src/parse/pipe.rs | 14 ++++++++++ interactive/tests/sum.rs | 49 +++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 interactive/tests/sum.rs diff --git a/interactive/src/parse/pipe.rs b/interactive/src/parse/pipe.rs index 2534eee6f..267a728e3 100644 --- a/interactive/src/parse/pipe.rs +++ b/interactive/src/parse/pipe.rs @@ -36,6 +36,8 @@ //! - `| join(other, (k, v, v1) -> (k', v'))` — equijoin on the key. //! - `| min` / `| distinct` / `| count` / `| collect` — reduce; `distinct` keeps each `(key, val)` //! record once, and `collect` is NEST (gather a key's values into a `List`). +//! - `| sum` — sum integer values per key, weighted by their signed diffs. A zero total has no row. +//! For example, `(deposits - debits) | sum` maintains balances from `(account, amount)` rows. //! - `| reduce((k, vals) -> out)` — reduce by a lambda: `vals` is the key's values with their //! non-zero diffs, a `List` of `(val, diff)` in value order, and `out` the key's output, a //! `List` of `(val, diff)`. `| reduce(limit n, (k, vals) -> out)` hands the lambda only each @@ -679,6 +681,18 @@ impl Parser { Token::Distinct => { self.next(); Expr::Reduce(Box::new(lhs), Reducer::Distinct) }, Token::Count => { self.next(); Expr::Reduce(Box::new(lhs), Reducer::Count) }, Token::Collect => { self.next(); Expr::Reduce(Box::new(lhs), Reducer::Collect) }, + Token::Ident(name) if name == "sum" => { + self.next(); + // Move the integer value into the diff before arranging by key. Joining the + // update's time with itself preserves every coordinate, including inside loops. + let weight = Lambda::new( + "let (k, (v, t)) = input in ((k, ()), t, v) enlist".into(), + &corgi::Scope::default(), + ); + let timed = Expr::Lift(Box::new(lhs)); + let weighted = Expr::JoinFunction(Box::new(timed), weight); + Expr::Reduce(Box::new(weighted), Reducer::Count) + }, Token::Ident(name) if name == "reduce" => { self.next(); self.expect(&Token::LParen); // `reduce(limit k, (k, vals) -> …)`: the lambda is handed each key's first `k` values. diff --git a/interactive/tests/sum.rs b/interactive/tests/sum.rs new file mode 100644 index 000000000..b7beef0a2 --- /dev/null +++ b/interactive/tests/sum.rs @@ -0,0 +1,49 @@ +use interactive::server::{HostTime, Server}; +use interactive::{lower, parse}; +use timely::progress::Antichain; + +#[test] +fn signed_sums_at_product_times() { + timely::execute_directly(|worker| { + let mut server = Server::new(); + let src = r#" + let sum = input "amounts" : (int, int); + output "root" = sum | sum; + nested: { set total = sum | sum; } + output "nested" = nested::total; + "#; + let mut program = lower::lower_tree(parse::pipe::parse(src)); + interactive::passes::optimize(&mut program); + server.install_with_width(worker, "p", &program, 2).unwrap(); + let time = |a, b| HostTime::new(smallvec::smallvec![a, b]); + server.feed_at("p", 0, "1 val=5 diff=2\n2 val=-3\n3 val=0", &time(0, 1)).unwrap(); + server.feed_at("p", 0, "1 val=10 diff=-1\n2 val=-3 diff=-2", &time(1, 0)).unwrap(); + server.feed_at("p", 0, "1 val=4", &time(2, 1)).unwrap(); + server.seal("p", 0, &Antichain::new()).unwrap(); + + for output in ["p::root", "p::nested"] { + for (point, expected) in [ + (time(0, 1), vec![(1, 10), (2, -3)]), + (time(1, 0), vec![(1, -10), (2, 6)]), + (time(1, 1), vec![(2, 3)]), + (time(2, 1), vec![(1, 4), (2, 3)]), + ] { + let rows = server.snapshot_as_of(worker, output, &point).unwrap().rows(); + let expected: Vec<_> = expected.into_iter().map(|(k, v)| (k.to_string(), v.to_string(), 1)).collect(); + assert_eq!(rows, expected, "{output} at {point:?}"); + } + } + }); +} + +#[test] +fn sum_requires_integer_values() { + timely::execute_directly(|worker| { + for value in ["()", "float", "(int, int)", "List(int)"] { + let src = format!(r#"output "total" = input 0 : (int, {value}) | sum;"#); + let program = lower::lower_tree(parse::pipe::parse(&src)); + let error = Server::new().install(worker, "p", &program).unwrap_err(); + assert!(error.contains("join_function") && error.contains("Int"), "{value}: {error}"); + } + }); +}