Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions interactive/src/parse/pipe.rs
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@
//! - `| join(other, (k, v, v1) -> (k', v'))` — equijoin on the key.
//! - `| min` / `| distinct` / `| count` / `| collect` — reduce; `distinct` keeps each `(key, val)`
//! record once, and `collect` is NEST (gather a key's values into a `List`).
//! - `| sum` — sum integer values per key, weighted by their signed diffs. A zero total has no row.
//! For example, `(deposits - debits) | sum` maintains balances from `(account, amount)` rows.
//! - `| reduce((k, vals) -> out)` — reduce by a lambda: `vals` is the key's values with their
//! non-zero diffs, a `List` of `(val, diff)` in value order, and `out` the key's output, a
//! `List` of `(val, diff)`. `| reduce(limit n, (k, vals) -> out)` hands the lambda only each
Expand Down Expand Up @@ -679,6 +681,18 @@ impl Parser {
Token::Distinct => { self.next(); Expr::Reduce(Box::new(lhs), Reducer::Distinct) },
Token::Count => { self.next(); Expr::Reduce(Box::new(lhs), Reducer::Count) },
Token::Collect => { self.next(); Expr::Reduce(Box::new(lhs), Reducer::Collect) },
Token::Ident(name) if name == "sum" => {
self.next();
// Move the integer value into the diff before arranging by key. Joining the
// update's time with itself preserves every coordinate, including inside loops.
let weight = Lambda::new(
"let (k, (v, t)) = input in ((k, ()), t, v) enlist".into(),
&corgi::Scope::default(),
);
let timed = Expr::Lift(Box::new(lhs));
let weighted = Expr::JoinFunction(Box::new(timed), weight);
Expr::Reduce(Box::new(weighted), Reducer::Count)
},
Token::Ident(name) if name == "reduce" => {
self.next(); self.expect(&Token::LParen);
// `reduce(limit k, (k, vals) -> …)`: the lambda is handed each key's first `k` values.
Expand Down
49 changes: 49 additions & 0 deletions interactive/tests/sum.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
use interactive::server::{HostTime, Server};
use interactive::{lower, parse};
use timely::progress::Antichain;

#[test]
fn signed_sums_at_product_times() {
timely::execute_directly(|worker| {
let mut server = Server::new();
let src = r#"
let sum = input "amounts" : (int, int);
output "root" = sum | sum;
nested: { set total = sum | sum; }
output "nested" = nested::total;
"#;
let mut program = lower::lower_tree(parse::pipe::parse(src));
interactive::passes::optimize(&mut program);
server.install_with_width(worker, "p", &program, 2).unwrap();
let time = |a, b| HostTime::new(smallvec::smallvec![a, b]);
server.feed_at("p", 0, "1 val=5 diff=2\n2 val=-3\n3 val=0", &time(0, 1)).unwrap();
server.feed_at("p", 0, "1 val=10 diff=-1\n2 val=-3 diff=-2", &time(1, 0)).unwrap();
server.feed_at("p", 0, "1 val=4", &time(2, 1)).unwrap();
server.seal("p", 0, &Antichain::new()).unwrap();

for output in ["p::root", "p::nested"] {
for (point, expected) in [
(time(0, 1), vec![(1, 10), (2, -3)]),
(time(1, 0), vec![(1, -10), (2, 6)]),
(time(1, 1), vec![(2, 3)]),
(time(2, 1), vec![(1, 4), (2, 3)]),
] {
let rows = server.snapshot_as_of(worker, output, &point).unwrap().rows();
let expected: Vec<_> = expected.into_iter().map(|(k, v)| (k.to_string(), v.to_string(), 1)).collect();
assert_eq!(rows, expected, "{output} at {point:?}");
}
}
});
}

#[test]
fn sum_requires_integer_values() {
timely::execute_directly(|worker| {
for value in ["()", "float", "(int, int)", "List(int)"] {
let src = format!(r#"output "total" = input 0 : (int, {value}) | sum;"#);
let program = lower::lower_tree(parse::pipe::parse(&src));
let error = Server::new().install(worker, "p", &program).unwrap_err();
assert!(error.contains("join_function") && error.contains("Int"), "{value}: {error}");
}
});
}
Loading