From 51b979339d3f5bb2809482e649279eadcec3bce7 Mon Sep 17 00:00:00 2001 From: Frank McSherry Date: Fri, 20 Nov 2026 06:23:49 -0500 Subject: [PATCH 1/3] DDIR: count projects its values away; remove value and lift_iter count summed diffs over a key's whole value list, so its reduce arranged and read every distinct value of a key on each change. Lowering now writes map((k, _) -> (k, ())) before the reduce when the value is not unit, as distinct already rekeys by the whole record once shapes are known. With 200,000 values under one key, 200 one-row updates took 0.39 s before and 0.01 s after. value was sugar for map and is gone; its uses are now maps. lift_iter had no users outside one test, and lift gives the round as the time's last coordinate. Explain still writes LinearOp::LiftIter internally, so the IR op stays, with its discipline comment moved onto it. The two half_join projection tests now end in an explicit reduce. With count's new map in between, pushdown narrows nothing there: a map that drops a half join's whole value does not let it mask the half join, while a reduce does. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01H6wB4Gjwwivtq59LaFYbF5 --- interactive/bench/ldbc/snb/rel.py | 2 +- interactive/examples/aoc2023/day01/part1.ddp | 2 +- interactive/examples/aoc2023/day01/part2.ddp | 2 +- interactive/examples/aoc2023/day02/part1.ddp | 2 +- interactive/examples/aoc2023/day02/part2.ddp | 2 +- interactive/examples/aoc2023/day03/part1.ddp | 2 +- interactive/examples/aoc2023/day03/part2.ddp | 2 +- interactive/examples/aoc2023/day04/part1.ddp | 2 +- interactive/src/ir.rs | 10 ++++-- interactive/src/lower.rs | 31 ++++++++-------- interactive/src/parse/mod.rs | 8 ----- interactive/src/parse/pipe.rs | 13 ++----- interactive/src/passes/projection.rs | 9 +++-- interactive/src/plan/ddp.rs | 2 +- interactive/src/plan/ddp/parsed.rs | 7 ++-- interactive/src/reducers.rs | 35 +++++++++++++------ interactive/tests/bitemporal.rs | 4 +-- interactive/tests/difftest.rs | 11 ++---- interactive/tests/explain.rs | 4 +-- .../arrange_collapse.parsed--lowered.diff | 2 +- .../expected/arrange_collapse.parsed.ddp | 2 +- .../ldbc_ic03_oneoff.parsed--lowered.diff | 4 +-- .../expected/ldbc_ic03_oneoff.parsed.ddp | 4 +-- .../ldbc_ic04_oneoff.parsed--lowered.diff | 2 +- .../expected/ldbc_ic04_oneoff.parsed.ddp | 2 +- .../ldbc_ic05_oneoff.parsed--lowered.diff | 2 +- .../expected/ldbc_ic05_oneoff.parsed.ddp | 2 +- .../ldbc_ic05_standing.parsed--lowered.diff | 2 +- .../expected/ldbc_ic05_standing.parsed.ddp | 2 +- .../ldbc_ic06_oneoff.parsed--lowered.diff | 2 +- .../expected/ldbc_ic06_oneoff.parsed.ddp | 2 +- .../ldbc_ic10_oneoff.parsed--lowered.diff | 2 +- .../expected/ldbc_ic10_oneoff.parsed.ddp | 2 +- .../ldbc_ic12_oneoff.parsed--lowered.diff | 2 +- .../expected/ldbc_ic12_oneoff.parsed.ddp | 2 +- .../ldbc_ic14_standing.parsed--lowered.diff | 2 +- .../expected/ldbc_ic14_standing.parsed.ddp | 2 +- .../plan_pairs/expected/reduce_masks.ddp | 4 ++- .../reduce_masks.lowered--optimized.diff | 18 +++++----- .../reduce_masks.optimized--projected.diff | 12 ++++--- .../reduce_masks.parsed--lowered.diff | 17 ++++----- .../expected/reduce_masks.parsed.ddp | 4 +-- .../plan_pairs/expected/shared_demand.ddp | 5 +-- .../shared_demand.lowered--optimized.diff | 2 +- .../shared_demand.parsed--lowered.diff | 9 ++--- .../expected/shared_demand.parsed.ddp | 2 +- interactive/tests/programs/lambda_slots.ddp | 20 +++++------ 47 files changed, 146 insertions(+), 137 deletions(-) diff --git a/interactive/bench/ldbc/snb/rel.py b/interactive/bench/ldbc/snb/rel.py index b6c7abd0c..c02f8a895 100644 --- a/interactive/bench/ldbc/snb/rel.py +++ b/interactive/bench/ldbc/snb/rel.py @@ -385,7 +385,7 @@ def keyed(source, fields, keys, keep=True): # A dedicated value-only collect avoids lists of singleton # tuples and is useful for nested IC1/IC12 results. assert len(aggs)==1 - code = f'{source_name} | map((row, ()) -> ({group_key}, ({scalar(expr(e),refs)}))) | value((_, (item)) -> item) | collect | map((key, values) -> ({product([*outs,"values"])}, ()))' + code = f'{source_name} | map((row, ()) -> ({group_key}, ({scalar(expr(e),refs)}))) | map((k, (item)) -> (k, item)) | collect | map((key, values) -> ({product([*outs,"values"])}, ()))' self.lines.append(f'-- {name}: ({", ".join(r.fields)})\nlet {name} = {code};') return name else: raise ValueError(kind) diff --git a/interactive/examples/aoc2023/day01/part1.ddp b/interactive/examples/aoc2023/day01/part1.ddp index 642e08e63..67281209e 100644 --- a/interactive/examples/aoc2023/day01/part1.ddp +++ b/interactive/examples/aoc2023/day01/part1.ddp @@ -14,5 +14,5 @@ let calib = first | join(last, (line, (f, l)) -> (line, ((f, 10) mul, l) add)); output "result" = calib | map((_, v) -> ((), v)) | collect - | value((_, vs) -> vs fold_add) + | map((k, vs) -> (k, vs fold_add)) | arrange | inspect(part1); diff --git a/interactive/examples/aoc2023/day01/part2.ddp b/interactive/examples/aoc2023/day01/part2.ddp index 9b2e7f3aa..c639104f4 100644 --- a/interactive/examples/aoc2023/day01/part2.ddp +++ b/interactive/examples/aoc2023/day01/part2.ddp @@ -39,5 +39,5 @@ let calib = first | join(last, (line, (f, l)) -> (line, ((f, 10) mul, l) add)); output "result" = calib | map((_, v) -> ((), v)) | collect - | value((_, vs) -> vs fold_add) + | map((k, vs) -> (k, vs fold_add)) | arrange | inspect(part2); diff --git a/interactive/examples/aoc2023/day02/part1.ddp b/interactive/examples/aoc2023/day02/part1.ddp index 9c8c0ea25..a686d41db 100644 --- a/interactive/examples/aoc2023/day02/part1.ddp +++ b/interactive/examples/aoc2023/day02/part1.ddp @@ -10,5 +10,5 @@ let good = games - bad; output "result" = good | map((game, _) -> ((), game)) | collect - | value((_, gs) -> gs fold_add) + | map((k, gs) -> (k, gs fold_add)) | arrange | inspect(part1); diff --git a/interactive/examples/aoc2023/day02/part2.ddp b/interactive/examples/aoc2023/day02/part2.ddp index ed1938968..da1f16942 100644 --- a/interactive/examples/aoc2023/day02/part2.ddp +++ b/interactive/examples/aoc2023/day02/part2.ddp @@ -13,5 +13,5 @@ let power = maxg output "result" = power | map((_, p) -> ((), p)) | collect - | value((_, ps) -> ps fold_add) + | map((k, ps) -> (k, ps fold_add)) | arrange | inspect(part2); diff --git a/interactive/examples/aoc2023/day03/part1.ddp b/interactive/examples/aoc2023/day03/part1.ddp index 64a9e81c4..34462a2ca 100644 --- a/interactive/examples/aoc2023/day03/part1.ddp +++ b/interactive/examples/aoc2023/day03/part1.ddp @@ -40,5 +40,5 @@ let part_numbers = maximal | map(((r, s, _, v), _) -> ((r, s), v)) | join(active output "result" = part_numbers | map((_, v) -> ((), v)) | collect - | value((_, vs) -> vs fold_add) + | map((k, vs) -> (k, vs fold_add)) | arrange | inspect(part1); diff --git a/interactive/examples/aoc2023/day03/part2.ddp b/interactive/examples/aoc2023/day03/part2.ddp index f3dfcad62..35003de9b 100644 --- a/interactive/examples/aoc2023/day03/part2.ddp +++ b/interactive/examples/aoc2023/day03/part2.ddp @@ -45,5 +45,5 @@ output "result" = gearvals | filter((_, vs) -> (vs len, 2) eq) | map((_, vs) -> ((), ((0, vs) get, (1, vs) get) mul)) | map((_, p) -> ((), p)) | collect - | value((_, ps) -> ps fold_add) + | map((k, ps) -> (k, ps fold_add)) | arrange | inspect(part2); diff --git a/interactive/examples/aoc2023/day04/part1.ddp b/interactive/examples/aoc2023/day04/part1.ddp index 795ce7879..ac473e673 100644 --- a/interactive/examples/aoc2023/day04/part1.ddp +++ b/interactive/examples/aoc2023/day04/part1.ddp @@ -9,5 +9,5 @@ let points = dups output "result" = points | map((_, p) -> ((), p)) | collect - | value((_, ps) -> ps fold_add) + | map((k, ps) -> (k, ps fold_add)) | arrange | inspect(part1); diff --git a/interactive/src/ir.rs b/interactive/src/ir.rs index efa663562..0c383e993 100644 --- a/interactive/src/ir.rs +++ b/interactive/src/ir.rs @@ -13,7 +13,7 @@ pub use crate::lambda::Lambda; /// table (`parse::pipe`) must agree, which its tests check. pub(crate) const KEYWORDS: &[&str] = &[ "let", "var", "set", "output", "type", "enum", "input", "import", "map", "join", "min", "distinct", "count", - "collect", "flatmap", "arrange", "negate", "filter", "inspect", "lift_iter", + "collect", "flatmap", "arrange", "negate", "filter", "inspect", ]; #[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] @@ -36,8 +36,12 @@ pub enum LinearOp { /// Negate the diff. Negate, /// Append the current user-iter coord (at the row's scope depth) to - /// the value. Time itself is unchanged. See `Expr::LiftIter` for the - /// discipline restriction. + /// the value. Time itself is unchanged. Only explain writes it; the language has no spelling. + /// + /// Discipline, which nothing checks: the result of `LiftIter` should not be referenced inside the same + /// scope it appears in — only from an enclosing scope, after the + /// implicit leave. This preserves the "loop body is a time-invariant + /// function" property; in-scope use risks defeating the fixpoint. LiftIter, /// Each row becomes the lambda's list of complete `(key, val)` rows. FlatMap(Lambda), diff --git a/interactive/src/lower.rs b/interactive/src/lower.rs index f5dd87fbd..fbbb73f62 100644 --- a/interactive/src/lower.rs +++ b/interactive/src/lower.rs @@ -188,7 +188,7 @@ fn resolve_expr(expr: &mut Expr, here: &[String], item: Option<&str>, *expr = Expr::ResolvedPath { anchor: key.anchor, target: key.target, field: key.field }; } Expr::TypedSource(e, _, _) | Expr::Map(e, _) | Expr::Reduce(e, _) | Expr::Filter(e, _) - | Expr::Negate(e) | Expr::LiftIter(e) | Expr::FlatMap(e, _) + | Expr::Negate(e) | Expr::FlatMap(e, _) | Expr::Lift(e) | Expr::JoinFunction(e, _) | Expr::Inspect(e, _) | Expr::Arrange(e) => resolve_expr(e, here, item, scopes, plan), Expr::Join(l, r, _) | Expr::HalfJoin(l, r, _, _) => { @@ -247,7 +247,7 @@ fn expr_free_names(expr: &Expr, out: &mut BTreeSet) { Expr::ResolvedPath { .. } => {}, Expr::Path(..) => unreachable!("paths must be resolved before lowering"), Expr::TypedSource(e, _, _) | Expr::Map(e, _) | Expr::Reduce(e, _) | Expr::Filter(e, _) - | Expr::Negate(e) | Expr::LiftIter(e) + | Expr::Negate(e) | Expr::FlatMap(e, _) | Expr::Lift(e) | Expr::JoinFunction(e, _) | Expr::Inspect(e, _) | Expr::Arrange(e) => expr_free_names(e, out), Expr::Join(l, r, _) | Expr::HalfJoin(l, r, _, _) => { expr_free_names(l, out); expr_free_names(r, out); }, @@ -310,17 +310,17 @@ pub fn lower_tree(mut stmts: Vec) -> st::Program { let mut plan = PathPlan::default(); resolve_body(&mut stmts, &[], &scopes, &mut plan); let empty = HashMap::new(); - let mut distinct = HashMap::new(); - let mut program = st::Program { root: lower_scope_tree(&[], &stmts, &empty, &[], &plan, true, &mut distinct) }; + let mut by_shape = HashMap::new(); + let mut program = st::Program { root: lower_scope_tree(&[], &stmts, &empty, &[], &plan, true, &mut by_shape) }; let (shapes, _) = crate::shapes::infer(&program, &|_| None, 1); - crate::reducers::expand_distinct(&mut program.root, &shapes, &distinct, &mut Vec::new()); + crate::reducers::expand_by_shape(&mut program.root, &shapes, &by_shape, &mut Vec::new()); program } /// Expr-lowering state for one scope. Resolves names to scope-local `Ref`s and /// accumulates this scope's nodes and (input/trace) imports. struct ScopeLower { - distinct: Vec, + by_shape: Vec<(usize, crate::reducers::ByShape)>, is_root: bool, items: Vec, /// Each item's statement name (see `st::Scope::labels`). @@ -386,7 +386,6 @@ impl ScopeLower { Expr::Filter(e, c) => { let r = self.lower_expr(e); self.push(st::Node::Linear { input: r, ops: vec![LinearOp::Filter(c.clone())] }) }, Expr::Negate(e) => { let r = self.lower_expr(e); self.push(st::Node::Linear { input: r, ops: vec![LinearOp::Negate] }) }, Expr::FlatMap(e, t) => { let r = self.lower_expr(e); self.push(st::Node::Linear { input: r, ops: vec![LinearOp::FlatMap(t.clone())] }) }, - Expr::LiftIter(e) => { let r = self.lower_expr(e); self.push(st::Node::Linear { input: r, ops: vec![LinearOp::LiftIter] }) }, Expr::Lift(e) => { let r = self.lower_expr(e); self.push(st::Node::Linear { input: r, ops: vec![LinearOp::Lift] }) }, Expr::JoinFunction(e, t) => { let r = self.lower_expr(e); self.push(st::Node::Linear { input: r, ops: vec![LinearOp::JoinFunction(t.clone())] }) }, Expr::Arrange(e) => { let r = self.lower_expr(e); self.push(st::Node::Arrange(r)) }, @@ -404,11 +403,13 @@ impl ScopeLower { }, Expr::Reduce(e, red)=> { let r = self.lower_expr(e); - let a = if matches!(red, Reducer::Distinct) { + let a = match red { // Once shapes are known, lower distinct by key or by whole record. - self.distinct.push(self.items.len()); - r - } else { self.arrange(r) }; + Reducer::Distinct => { self.by_shape.push((self.items.len(), crate::reducers::ByShape::Distinct)); r } + // Once shapes are known, project count's values away if they are not unit. + Reducer::Count => { self.by_shape.push((self.items.len(), crate::reducers::ByShape::Count)); r } + _ => self.arrange(r), + }; self.push(st::Node::Reduce { input: a, reducer: crate::ir::Reducer::Expr(red.lambda()), limit: red.limit() }) }, Expr::Inspect(e, l) => { let r = self.lower_expr(e); self.push(st::Node::Inspect { input: r, label: l.clone() }) }, @@ -424,11 +425,11 @@ fn lower_scope_tree( exports_wanted: &[PathKey], plan: &PathPlan, is_root: bool, - distinct: &mut HashMap, Vec>, + by_shape: &mut HashMap, Vec<(usize, crate::reducers::ByShape)>>, ) -> st::Scope { let scope_name = scope_label(address); let mut s = ScopeLower { - distinct: Vec::new(), + by_shape: Vec::new(), is_root, items: Vec::new(), labels: Vec::new(), @@ -560,7 +561,7 @@ fn lower_scope_tree( let mut child_address = address.to_vec(); child_address.push(name.clone()); let wanted: Vec = plan.exports.get(&child_address).into_iter().flatten().cloned().collect(); - let child = lower_scope_tree(&child_address, b, &s.env, &wanted, plan, false, distinct); + let child = lower_scope_tree(&child_address, b, &s.env, &wanted, plan, false, by_shape); let slots = export_slots(&wanted, &child_address); let cidx = s.items.len(); s.items.push(st::Item::Sub(child)); @@ -611,7 +612,7 @@ fn lower_scope_tree( } } - distinct.insert(address.to_vec(), s.distinct); + by_shape.insert(address.to_vec(), s.by_shape); st::Scope { name: scope_name.to_string(), imports: s.imports, vars, items: s.items, labels: s.labels, names, binds, exports } } diff --git a/interactive/src/parse/mod.rs b/interactive/src/parse/mod.rs index e1ed9b063..5d0d2dc4e 100644 --- a/interactive/src/parse/mod.rs +++ b/interactive/src/parse/mod.rs @@ -60,14 +60,6 @@ pub enum Expr { Reduce(Box, Reducer), Filter(Box, Lambda), Negate(Box), - /// Append the current user-iter coord (at the operator's scope depth) - /// to each row's value. Time itself is unchanged. - /// - /// Discipline, which nothing checks: the result of `LiftIter` should not be referenced inside the same - /// scope it appears in — only from an enclosing scope, after the - /// implicit leave. This preserves the "loop body is a time-invariant - /// function" property; in-scope use risks defeating the fixpoint. - LiftIter(Box), /// Replace each row with the lambda's list of complete `(key, val)` rows. FlatMap(Box, Lambda), /// Pair each row's value with its whole time. See `LinearOp::Lift`. diff --git a/interactive/src/parse/pipe.rs b/interactive/src/parse/pipe.rs index 8798cc662..5a390694d 100644 --- a/interactive/src/parse/pipe.rs +++ b/interactive/src/parse/pipe.rs @@ -32,7 +32,6 @@ //! Operators chain with `|`; a slot is written `(pat -> body)`: //! //! - `| map((k, v) -> (k', v'))` — reshape each row to the `(key, val)` the lambda gives. -//! - `| value((k, v) -> v')` — replace the value, keeping the key. //! - `| join(other)` — equijoin on the key, yielding `(k, (v, v1))`. //! - `| join(other, f)` — project the joined row, as in `| join(other) | map(f)`. //! - `| min` / `| distinct` / `| count` / `| collect` — reduce; `distinct` keeps each `(key, val)` @@ -47,8 +46,7 @@ //! - `| filter((k, v) -> mask)` — keep rows where the lambda gives a nonzero `Int`. //! - `| flatmap((k, v) -> list)` — any number of complete `(key, val)` rows. //! For positions, write `((0, l len) range, l) zip`; keys are chosen by the lambda. -//! - `| negate` · `| arrange` · `| inspect(label)` · -//! `| lift_iter`. +//! - `| negate` · `| arrange` · `| inspect(label)`. //! - `| lift` — pair each value with its whole time, `(key, (val, time))`, the time a tuple //! of the epoch and every scope coordinate. //! - `| join_function((k, v) -> list)` — replace each row by the lambda's list of @@ -151,7 +149,7 @@ mod fragment; #[derive(Debug, Clone, PartialEq)] enum Token { Let, Var, Set, Output, Type, Enum, - Input, Import, Map, Join, Min, Distinct, Count, Collect, Arrange, Negate, Filter, LiftIter, FlatMap, Inspect, + Input, Import, Map, Join, Min, Distinct, Count, Collect, Arrange, Negate, Filter, FlatMap, Inspect, Ident(String), Int(i64), Str(String), /// A slot written as a corgi lambda, `pat -> body`, kept as corgi source. Lambda { pat: String, body: String }, @@ -345,7 +343,6 @@ fn keyword(ident: &str) -> Option { "flatmap" => Token::FlatMap, "arrange" => Token::Arrange, "negate" => Token::Negate, "filter" => Token::Filter, "inspect" => Token::Inspect, - "lift_iter" => Token::LiftIter, _ => return None, }) } @@ -672,11 +669,6 @@ impl Parser { fn parse_pipe_op(&mut self, lhs: Expr) -> Expr { match self.peek().clone() { Token::Map => { self.next(); Expr::Map(Box::new(lhs), self.slot()) }, - Token::Ident(name) if name == "value" => { - self.next(); - let val = self.slot(); - Expr::Map(Box::new(lhs), Lambda::compose("(k, v)", &[(&val, "(k, v)".into())], |r| format!("(k, {})", r[0]))) - }, Token::Join => { self.next(); self.expect(&Token::LParen); @@ -726,7 +718,6 @@ impl Parser { Token::Arrange => { self.next(); Expr::Arrange(Box::new(lhs)) }, Token::Negate => { self.next(); Expr::Negate(Box::new(lhs)) }, Token::Filter => { self.next(); Expr::Filter(Box::new(lhs), self.slot()) }, - Token::LiftIter => { self.next(); Expr::LiftIter(Box::new(lhs)) }, Token::Ident(name) if name == "lift" => { self.next(); Expr::Lift(Box::new(lhs)) }, Token::Ident(name) if name == "half_join" => { self.next(); self.expect(&Token::LParen); diff --git a/interactive/src/passes/projection.rs b/interactive/src/passes/projection.rs index a6e43b650..4d5a8fa31 100644 --- a/interactive/src/passes/projection.rs +++ b/interactive/src/passes/projection.rs @@ -1050,7 +1050,8 @@ mod tests { let distinct = format!("{source}output \"out\" = x | distinct;\n"); let mut count = lower::lower_tree(parse::pipe::parse(&count)); let mut distinct = lower::lower_tree(parse::pipe::parse(&distinct)); - assert_eq!(pushdown(&mut count, &|_| None).unwrap(), 0); + // Count projects its values away as it is lowered, so the map's value goes unread. + assert_eq!(pushdown(&mut count, &|_| None).unwrap(), 2); assert_eq!(pushdown(&mut distinct, &|_| None).unwrap(), 0); } @@ -1164,7 +1165,8 @@ mod tests { let b = input 1 : ((int, int, int), ()) | map(((k, y, z), ()) -> (k, (y, z))); output \"out\" = a | lift - | half_join(b, 0, (k, x, yz) -> (k, x)) | count;"; + | half_join(b, 0, (k, x, yz) -> (k, x)) + | reduce((k, vals) -> let n = vals map ((_, d) -> d) fold_add in (n, (n, 1)) enlist filter);"; let mut program = lower::lower_tree(parse::pipe::parse(text)); crate::passes::optimize(&mut program); assert_eq!(pushdown(&mut program, &|_| None).unwrap(), 2); @@ -1180,7 +1182,8 @@ mod tests { let b = input 1 : ((int, int, int), ()) | map(((k, y, z), ()) -> (k, (y, z))); output \"out\" = a | lift - | half_join(b, 0, (k, x, yz) -> (k, x)) | count;"; + | half_join(b, 0, (k, x, yz) -> (k, x)) + | reduce((k, vals) -> let n = vals map ((_, d) -> d) fold_add in (n, (n, 1)) enlist filter);"; let mut original = lower::lower_tree(parse::pipe::parse(text)); crate::passes::optimize(&mut original); let mut projected = original.clone(); diff --git a/interactive/src/plan/ddp.rs b/interactive/src/plan/ddp.rs index 3c3f31732..7f2d91f33 100644 --- a/interactive/src/plan/ddp.rs +++ b/interactive/src/plan/ddp.rs @@ -614,7 +614,7 @@ mod tests { nested: { let both = (a + b) + (a | negate); } let x = nested::both + (b | map((k, v) -> (v, k))); output "all" = x | map((k, v) -> (v, k)); - output "other" = a | value((_, v) -> (v, 1) add); + output "other" = a | map((k, v) -> (k, (v, 1) add)); "#; let before = plan(source); let parsed = before.emit_ddp_at(DdpStage::Parsed).unwrap(); diff --git a/interactive/src/plan/ddp/parsed.rs b/interactive/src/plan/ddp/parsed.rs index 8669a8c7c..87b8a7999 100644 --- a/interactive/src/plan/ddp/parsed.rs +++ b/interactive/src/plan/ddp/parsed.rs @@ -81,12 +81,15 @@ impl Printer<'_> { // Parsed syntax has no inferred shapes. Key by the whole record, // which also preserves unit-valued distinct's meaning. format!("{input} | map((k, v) -> ((k, v), ())) | reduce({lambda}) | map((kv, ()) -> kv)") + } else if matches!(reducer, crate::parse::Reducer::Count) { + // Parsed syntax has no inferred shapes. Count projects its values away. + format!("{input} | map((k, _) -> (k, ())) | reduce({lambda})") } else { format!("{input} | reduce({}{lambda})", super::limit_text(reducer.limit())) } } - Expr::Negate(input) | Expr::Lift(input) | Expr::LiftIter(input) | Expr::Arrange(input) => { - let word = match expr { Expr::Negate(_) => "negate", Expr::Lift(_) => "lift", Expr::LiftIter(_) => "lift_iter", _ => "arrange" }; + Expr::Negate(input) | Expr::Lift(input) | Expr::Arrange(input) => { + let word = match expr { Expr::Negate(_) => "negate", Expr::Lift(_) => "lift", _ => "arrange" }; format!("{} | {word}", self.expression(input)?) } Expr::Inspect(input, label) => format!("{} | inspect({label})", self.expression(input)?), diff --git a/interactive/src/reducers.rs b/interactive/src/reducers.rs index 0f3680b47..9813aba0a 100644 --- a/interactive/src/reducers.rs +++ b/interactive/src/reducers.rs @@ -9,13 +9,23 @@ pub(crate) fn unit_distinct() -> Lambda { Lambda::new("let (_, vals) = input in vals map (((), _) -> ((), 1))".into(), &corgi::Scope::default()) } -/// Finish lowering distinct once the tree's shapes are known. A unit-valued input -/// is already keyed by its whole record. Otherwise write the record key and its -/// inverse into the IR, where arrangements, projections and readers can see them. -pub(crate) fn expand_distinct( +/// A reduce sugar whose lowering waits for its input's shape. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ByShape { + /// Keyed by the whole record, so that a key's values are not read together. + Distinct, + /// Values projected away, so that a key's values are not read together. + Count, +} + +/// Finish lowering distinct and count once the tree's shapes are known. A unit-valued input +/// needs nothing. Otherwise distinct writes the record key and its inverse into the IR, and +/// count writes the projection to a unit value, where arrangements, projections and readers +/// can see them. +pub(crate) fn expand_by_shape( scope: &mut crate::scope_ir::Scope, shapes: &crate::shapes::ScopeShapes, - pending: &std::collections::HashMap, Vec>, + pending: &std::collections::HashMap, Vec<(usize, ByShape)>>, path: &mut Vec, ) { use crate::scope_ir::{Item, Node, Ref, Source}; @@ -29,9 +39,8 @@ pub(crate) fn expand_distinct( if let Ref::Local(i) | Ref::ChildExport(i, _) = r { *i = remap[*i]; } }; for (i, mut item) in std::mem::take(&mut scope.items).into_iter().enumerate() { - let distinct = sites.peek() == Some(&i); - if distinct { sites.next(); } - let by_record = distinct && match &item { + let site = if sites.peek().map(|s| s.0) == Some(i) { sites.next().map(|s| s.1) } else { None }; + let by_record = site.is_some() && match &item { Item::Op(Node::Reduce { input, .. }) => !shapes.of(input).is_some_and(|s| s.val == corgi::Shape::Unit), _ => false, }; @@ -40,7 +49,7 @@ pub(crate) fn expand_distinct( Item::Sub(child) => { let crate::shapes::ItemShapes::Sub(shapes) = &shapes.items[i] else { unreachable!() }; path.push(child.name.clone()); - expand_distinct(child, shapes, pending, path); + expand_by_shape(child, shapes, pending, path); path.pop(); for import in &mut child.imports { if let Source::Parent(r) = &mut import.from { remap_ref(r, &remap); } @@ -69,7 +78,13 @@ pub(crate) fn expand_distinct( labels.push(scope.labels[i].clone()); r }; - if distinct { + if site == Some(ByShape::Count) { + let Item::Op(Node::Reduce { input, reducer, limit }) = item else { unreachable!() }; + let input = if by_record { + push(Node::Linear { input, ops: vec![LinearOp::Project(Lambda::new("let (k, _) = input in (k, ())".into(), &corgi::Scope::default()))] }) + } else { input }; + push(Node::Reduce { input, reducer, limit }); + } else if site == Some(ByShape::Distinct) { let Item::Op(Node::Reduce { input, .. }) = item else { unreachable!() }; let input = if by_record { push(Node::Linear { input, ops: vec![LinearOp::Project(Lambda::new("let (k, v) = input in ((k, v), ())".into(), &corgi::Scope::default()))] }) diff --git a/interactive/tests/bitemporal.rs b/interactive/tests/bitemporal.rs index fc9260cb0..99cdcda25 100644 --- a/interactive/tests/bitemporal.rs +++ b/interactive/tests/bitemporal.rs @@ -456,8 +456,8 @@ fn independent_inputs_and_time_shapes() { let a = input 0 : (int, int); let b = input 1 : (int, int); s: { - let seen = a | delay((_, v) -> (0, 0, (256, v call bit_length) mul)) | lift_iter | lift - | map((k, ((v, iter), (s, e, i))) -> (k, (s, e, i, iter))); + let seen = a | delay((_, v) -> (0, 0, (256, v call bit_length) mul)) | lift + | map((k, (_, (s, e, i))) -> (k, (s, e, i, i))); } output "left" = a; output "right" = b; diff --git a/interactive/tests/difftest.rs b/interactive/tests/difftest.rs index 2ebc2d1dc..55816973a 100644 --- a/interactive/tests/difftest.rs +++ b/interactive/tests/difftest.rs @@ -3,7 +3,7 @@ //! //! **What a seed generates.** One case, the same every time for the same seed: a program over up //! to six declared inputs (int pairs, `(int, int)`, a list key, a declared sum, floats), with a -//! few `let`s of `map`, `value`, `filter`, `flatmap`, `join`, `negate`, concat and difference, +//! few `let`s of `map`, `filter`, `flatmap`, `join`, `negate`, concat and difference, //! `arrange`, the reductions (`distinct`, `count`, `min`, `collect`, `reduce` by a lambda), //! `delay` and `join_function`, and small iterative scopes (reachability or label propagation, //! sometimes nested); and its feeds, 3 to 30 epochs of inserts and retractions over small values, @@ -465,7 +465,7 @@ impl Gen { let same_shape = |text: String| Op::Plain { text, same_shape: true }; let roll = self.rng.below(100); let op = match roll { - 0..=29 => { + 0..=34 => { let (k2, v2) = (self.shape(&env, true, false), self.shape(&env, false, false)); let body = format!("({}, {})", self.expr(&k2, &env, false), self.expr(&v2, &env, false)); let op = Op::Plain { text: format!("map({input} -> {body})"), same_shape: k2 == key && v2 == val }; @@ -473,13 +473,6 @@ impl Gen { val = v2; op } - 30..=34 => { - let v2 = self.shape(&env, false, false); - let body = self.expr(&v2, &env, false); - let op = Op::Plain { text: format!("value({input} -> {body})"), same_shape: v2 == val }; - val = v2; - op - } 35..=49 => same_shape(format!("filter({input} -> {})", self.mask(&env, 2))), 50..=54 => { let elem = if self.rng.chance(70) { Sh::Int } else { self.shape(&env, false, false) }; diff --git a/interactive/tests/explain.rs b/interactive/tests/explain.rs index 570b9450a..13df4f31c 100644 --- a/interactive/tests/explain.rs +++ b/interactive/tests/explain.rs @@ -643,8 +643,8 @@ fn lambda_explanations_sufficient_small() { const SCALARS: &str = r#" let pairs = input 0 : (int, int); let counted = pairs | filter((_, v) -> (v, 3) gt) | count; - let lists = pairs | value((_, v) -> (v, 2) mul) | collect; - let flat = lists | flatmap((k, l) -> (k, l) cap_list) | value((_, x) -> x); + let lists = pairs | map((k, v) -> (k, (v, 2) mul)) | collect; + let flat = lists | flatmap((k, l) -> (k, l) cap_list) | map((k, x) -> (k, x)); inner: { set low = (flat | filter((_, x) -> (x, 15) lt)) + (low | filter((k, _) -> (k, k) eq)) | min; } diff --git a/interactive/tests/plan_pairs/expected/arrange_collapse.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/arrange_collapse.parsed--lowered.diff index 702f78b13..b49d56ce5 100644 --- a/interactive/tests/plan_pairs/expected/arrange_collapse.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/arrange_collapse.parsed--lowered.diff @@ -4,7 +4,7 @@ +-- Time lanes: epoch. let rows = input 0 : (int, ()); -let arranged = rows | arrange | arrange; --output "counts" = arranged | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter) | arrange; +-output "counts" = arranged | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter) | arrange; +let rows_idx = rows | arrange; +let arranged = rows_idx | arrange; +let counts_reduce = arranged | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); diff --git a/interactive/tests/plan_pairs/expected/arrange_collapse.parsed.ddp b/interactive/tests/plan_pairs/expected/arrange_collapse.parsed.ddp index 9d3592976..a9980d56b 100644 --- a/interactive/tests/plan_pairs/expected/arrange_collapse.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/arrange_collapse.parsed.ddp @@ -1,3 +1,3 @@ let rows = input 0 : (int, ()); let arranged = rows | arrange | arrange; -output "counts" = arranged | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter) | arrange; +output "counts" = arranged | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter) | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.parsed--lowered.diff index 88cf06531..caa911a45 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.parsed--lowered.diff @@ -17,8 +17,8 @@ -let hop2 = hop1 | join(friend, (_, (req, (g, _))) -> (g, req)); -let circle = hop1 + hop2 | filter((node, (_, who, _, _)) -> (node, who) ne) | join(person, (node, ((rid, _, x, y), (_, _, _, _, _, _, _, city, _, _, _))) -> (city, (rid, node, x, y))) | join(place, (_, ((rid, node, x, y), (_, _, home))) -> ((rid, node), (x, y, home))) | filter((_, (x, y, home)) -> ((home, x) ne, (home, y) ne) all) | map((key, _) -> (key, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); -let seen = visits | join(circle, (key, ((ix, iy), ())) -> (key, (ix, iy))); --let in_x = seen | filter((_, (ix, _)) -> (ix, 1) eq) | map((key, _) -> (key, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); --let in_y = seen | filter((_, (_, iy)) -> (iy, 1) eq) | map((key, _) -> (key, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +-let in_x = seen | filter((_, (ix, _)) -> (ix, 1) eq) | map((key, _) -> (key, ())) | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +-let in_y = seen | filter((_, (_, iy)) -> (iy, 1) eq) | map((key, _) -> (key, ())) | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +let asked_map = request | map(((rid, who, x, y, start, days), ()) -> (x, (rid, who, y, start, (start, (days, 86400000) mul) add))); +let asked_idx = asked_map | arrange; +let by_name_idx = by_name | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.parsed.ddp index 482619a57..31a8259b4 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.parsed.ddp @@ -12,8 +12,8 @@ let hop1 = asked | map((rid, (who, x, y, _, _)) -> (who, (rid, who, x, y))) | jo let hop2 = hop1 | join(friend, (_, (req, (g, _))) -> (g, req)); let circle = hop1 + hop2 | filter((node, (_, who, _, _)) -> (node, who) ne) | join(person, (node, ((rid, _, x, y), (_, _, _, _, _, _, _, city, _, _, _))) -> (city, (rid, node, x, y))) | join(place, (_, ((rid, node, x, y), (_, _, home))) -> ((rid, node), (x, y, home))) | filter((_, (x, y, home)) -> ((home, x) ne, (home, y) ne) all) | map((key, _) -> (key, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); let seen = visits | join(circle, (key, ((ix, iy), ())) -> (key, (ix, iy))); -let in_x = seen | filter((_, (ix, _)) -> (ix, 1) eq) | map((key, _) -> (key, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); -let in_y = seen | filter((_, (_, iy)) -> (iy, 1) eq) | map((key, _) -> (key, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +let in_x = seen | filter((_, (ix, _)) -> (ix, 1) eq) | map((key, _) -> (key, ())) | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +let in_y = seen | filter((_, (_, iy)) -> (iy, 1) eq) | map((key, _) -> (key, ())) | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); let counts = in_x | join(in_y); let ranked = counts | map(((rid, node), (nx, ny)) -> (node, (rid, nx, ny))) | join(person, (node, ((rid, nx, ny), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, (((nx, ny) add neg, node), (first, last, nx, ny)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((_, node), (first, last, nx, ny)))) -> ((rid, rank), (node, first, last, nx, ny, (nx, ny) add))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.parsed--lowered.diff index fe8689260..243353ccf 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.parsed--lowered.diff @@ -28,7 +28,7 @@ +let before = before_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); let during = seen | filter((_, (_, created, start, end)) -> ((created, start) ge, (created, end) lt) all); -let fresh = during + (during | join(before, (key, (row, ())) -> (key, row)) | negate); --let counts = fresh | map(((rid, _), (name, _, _, _)) -> ((rid, name), ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +-let counts = fresh | map(((rid, _), (name, _, _, _)) -> ((rid, name), ())) | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); -let ranked = counts | map(((rid, name), n) -> (rid, (n neg, name))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); +let during_idx = during | arrange; +let fresh_join = during_idx | join(before, (key, (row, ())) -> (key, row)); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.parsed.ddp index b99333028..4624efaba 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.parsed.ddp @@ -9,7 +9,7 @@ let seen = request | map(((rid, who, start, days), ()) -> (who, (rid, start, (st let before = seen | filter((_, (_, created, start, _)) -> (created, start) lt) | map((key, _) -> (key, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); let during = seen | filter((_, (_, created, start, end)) -> ((created, start) ge, (created, end) lt) all); let fresh = during + (during | join(before, (key, (row, ())) -> (key, row)) | negate); -let counts = fresh | map(((rid, _), (name, _, _, _)) -> ((rid, name), ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +let counts = fresh | map(((rid, _), (name, _, _, _)) -> ((rid, name), ())) | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); let ranked = counts | map(((rid, name), n) -> (rid, (n neg, name))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, (neg, name))) -> ((rid, rank), (name, neg neg))); output "answer" = answer | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic05_oneoff.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic05_oneoff.parsed--lowered.diff index 358873ba5..6d7342f2c 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic05_oneoff.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic05_oneoff.parsed--lowered.diff @@ -14,7 +14,7 @@ -let hop2 = hop1 | join(friend, (_, (req, (g, _))) -> (g, req)); -let circle = hop1 + hop2 | filter((node, (_, who, _)) -> (node, who) ne) | map((node, (rid, _, date)) -> ((node, rid, date), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((node, rid, date), ()) -> (node, (rid, date))); -let after = circle | join(member, (node, ((rid, date), (f, since))) -> ((node, f), (rid, date, since))) | filter((_, (_, date, since)) -> (since, date) gt) | map((key, (rid, _, _)) -> (key, rid)); --let busy = after | join(posted, ((_, f), (rid, n)) -> ((rid, f), n)) | join_function((key, n) -> ((key, ()), 0, n) enlist) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +-let busy = after | join(posted, ((_, f), (rid, n)) -> ((rid, f), n)) | join_function((key, n) -> ((key, ()), 0, n) enlist) | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); -let timed = circle | join(first, (node, ((rid, date), since)) -> (node, (rid, date, since))); -let fast = timed | filter((_, (_, date, since)) -> (since, date) gt) | join(lowest, (_, ((rid, _, _), f)) -> ((rid, f), ())); -let slow = timed | filter((_, (_, date, since)) -> (since, date) le) | map((node, (rid, _, _)) -> ((node, rid), ())) | join(after | map(((node, f), rid) -> ((node, rid), f)), ((_, rid), ((), f)) -> ((rid, f), ())); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic05_oneoff.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic05_oneoff.parsed.ddp index a2bf2c443..08b4afe85 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic05_oneoff.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic05_oneoff.parsed.ddp @@ -11,7 +11,7 @@ let hop1 = request | map(((rid, who, date), ()) -> (who, (rid, who, date))) | jo let hop2 = hop1 | join(friend, (_, (req, (g, _))) -> (g, req)); let circle = hop1 + hop2 | filter((node, (_, who, _)) -> (node, who) ne) | map((node, (rid, _, date)) -> ((node, rid, date), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((node, rid, date), ()) -> (node, (rid, date))); let after = circle | join(member, (node, ((rid, date), (f, since))) -> ((node, f), (rid, date, since))) | filter((_, (_, date, since)) -> (since, date) gt) | map((key, (rid, _, _)) -> (key, rid)); -let busy = after | join(posted, ((_, f), (rid, n)) -> ((rid, f), n)) | join_function((key, n) -> ((key, ()), 0, n) enlist) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +let busy = after | join(posted, ((_, f), (rid, n)) -> ((rid, f), n)) | join_function((key, n) -> ((key, ()), 0, n) enlist) | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); let timed = circle | join(first, (node, ((rid, date), since)) -> (node, (rid, date, since))); let fast = timed | filter((_, (_, date, since)) -> (since, date) gt) | join(lowest, (_, ((rid, _, _), f)) -> ((rid, f), ())); let slow = timed | filter((_, (_, date, since)) -> (since, date) le) | map((node, (rid, _, _)) -> ((node, rid), ())) | join(after | map(((node, f), rid) -> ((node, rid), f)), ((_, rid), ((), f)) -> ((rid, f), ())); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic05_standing.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic05_standing.parsed--lowered.diff index 0a945b30f..68827ca02 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic05_standing.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic05_standing.parsed--lowered.diff @@ -8,7 +8,7 @@ let friend = import "graph::friend" : (int, (int, int)); let forum = import "graph::forum" : (int, (List(int), int, int)); let member = import "graph::member.by_person" : (int, (int, int)); --let posted = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 0) eq) | map((_, (_, _, who, _, _, _, _, _, f, _, _, _, _)) -> ((who, f), ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +-let posted = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 0) eq) | map((_, (_, _, who, _, _, _, _, _, f, _, _, _, _)) -> ((who, f), ())) | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); -let first = member | map((who, (_, since)) -> (who, since)) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); -let lowest = member | map((who, (f, _)) -> (who, f)) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 40) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 40) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 40) min in let before = (end, count) sub in let keep = (((40, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten) | map((who, (_, f)) -> (who, f)); -output "posted" = posted | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic05_standing.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic05_standing.parsed.ddp index 2c3d84d4d..70a2eea08 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic05_standing.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic05_standing.parsed.ddp @@ -3,7 +3,7 @@ let message = import "graph::message" : (int, (int, int, int, int, List(int), Li let friend = import "graph::friend" : (int, (int, int)); let forum = import "graph::forum" : (int, (List(int), int, int)); let member = import "graph::member.by_person" : (int, (int, int)); -let posted = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 0) eq) | map((_, (_, _, who, _, _, _, _, _, f, _, _, _, _)) -> ((who, f), ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +let posted = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 0) eq) | map((_, (_, _, who, _, _, _, _, _, f, _, _, _, _)) -> ((who, f), ())) | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); let first = member | map((who, (_, since)) -> (who, since)) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); let lowest = member | map((who, (f, _)) -> (who, f)) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 40) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 40) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 40) min in let before = (end, count) sub in let keep = (((40, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten) | map((who, (_, f)) -> (who, f)); output "posted" = posted | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic06_oneoff.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic06_oneoff.parsed--lowered.diff index 498e978ca..565f18631 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic06_oneoff.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic06_oneoff.parsed--lowered.diff @@ -14,7 +14,7 @@ -let hop2 = hop1 | join(friend, (_, ((rid, who), (g, _))) -> (g, (rid, who))); -let circle = hop1 + hop2 | filter((node, (_, who)) -> (node, who) ne) | map((node, (rid, _)) -> ((rid, node), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); -let with_it = request | map(((rid, _, name), ()) -> (name, rid)) | join(by_name, (_, (rid, t)) -> (t, rid)) | join(tagged, (t, (rid, m)) -> (m, (rid, t))) | join(message, (m, ((rid, t), (kind, _, author, _, _, _, _, _, _, _, _, _, _))) -> ((rid, author), (kind, m, t))) | filter((_, (kind, _, _)) -> (kind, 0) eq) | join(circle, ((rid, _), ((_, m, t), ())) -> (m, (rid, t))); --let others = with_it | join(tags_of, (_, ((rid, t), other)) -> ((rid, t, other), ())) | filter(((_, t, other), ()) -> (t, other) ne) | map(((rid, _, other), ()) -> ((rid, other), ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +-let others = with_it | join(tags_of, (_, ((rid, t), other)) -> ((rid, t, other), ())) | filter(((_, t, other), ()) -> (t, other) ne) | map(((rid, _, other), ()) -> ((rid, other), ())) | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); -let ranked = others | map(((rid, other), n) -> (other, (rid, n))) | join(tag, (_, ((rid, n), (name, _))) -> (rid, (n neg, name))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); +let hop1_map = request | map(((rid, who, _), ()) -> (who, (rid, who))); +let hop1_idx = hop1_map | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic06_oneoff.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic06_oneoff.parsed.ddp index a792c7fd7..aa3092ceb 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic06_oneoff.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic06_oneoff.parsed.ddp @@ -10,7 +10,7 @@ let hop1 = request | map(((rid, who, _), ()) -> (who, (rid, who))) | join(person let hop2 = hop1 | join(friend, (_, ((rid, who), (g, _))) -> (g, (rid, who))); let circle = hop1 + hop2 | filter((node, (_, who)) -> (node, who) ne) | map((node, (rid, _)) -> ((rid, node), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); let with_it = request | map(((rid, _, name), ()) -> (name, rid)) | join(by_name, (_, (rid, t)) -> (t, rid)) | join(tagged, (t, (rid, m)) -> (m, (rid, t))) | join(message, (m, ((rid, t), (kind, _, author, _, _, _, _, _, _, _, _, _, _))) -> ((rid, author), (kind, m, t))) | filter((_, (kind, _, _)) -> (kind, 0) eq) | join(circle, ((rid, _), ((_, m, t), ())) -> (m, (rid, t))); -let others = with_it | join(tags_of, (_, ((rid, t), other)) -> ((rid, t, other), ())) | filter(((_, t, other), ()) -> (t, other) ne) | map(((rid, _, other), ()) -> ((rid, other), ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +let others = with_it | join(tags_of, (_, ((rid, t), other)) -> ((rid, t, other), ())) | filter(((_, t, other), ()) -> (t, other) ne) | map(((rid, _, other), ()) -> ((rid, other), ())) | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); let ranked = others | map(((rid, other), n) -> (other, (rid, n))) | join(tag, (_, ((rid, n), (name, _))) -> (rid, (n neg, name))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, (neg, other))) -> ((rid, rank), (other, neg neg))); output "answer" = answer | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic10_oneoff.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic10_oneoff.parsed--lowered.diff index d0ced800e..354836234 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic10_oneoff.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic10_oneoff.parsed--lowered.diff @@ -17,7 +17,7 @@ -let wrote = cands | map(((rid, who, node), _) -> (node, (rid, who))) | join(posts, (node, ((rid, who), m)) -> ((rid, who, node, m), ())); -let liked = wrote | map(((rid, who, node, m), ()) -> (m, (rid, who, node))) | join(tags_of, (m, ((rid, who, node), t)) -> ((who, t), (rid, node, m))) | join(likes, ((who, _), ((rid, node, m), ())) -> ((rid, who, node, m), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); -let points = (liked | map(((rid, who, node, _), ()) -> ((rid, who, node), 1))) + (wrote + (liked | negate) | map(((rid, who, node, _), ()) -> ((rid, who, node), -1))); --let summed = points | join_function((key, p) -> ((key, ()), 0, p) enlist) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +-let summed = points | join_function((key, p) -> ((key, ()), 0, p) enlist) | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +let hop1_map = request | map(((rid, who, month), ()) -> (who, (rid, who, month))); +let hop1_idx = hop1_map | arrange; +let friend_idx = friend | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic10_oneoff.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic10_oneoff.parsed.ddp index 1d731ea74..1e8984b9c 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic10_oneoff.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic10_oneoff.parsed.ddp @@ -15,7 +15,7 @@ let cands = twohop | map(((rid, who, month, node), _) -> (node, (rid, who, month let wrote = cands | map(((rid, who, node), _) -> (node, (rid, who))) | join(posts, (node, ((rid, who), m)) -> ((rid, who, node, m), ())); let liked = wrote | map(((rid, who, node, m), ()) -> (m, (rid, who, node))) | join(tags_of, (m, ((rid, who, node), t)) -> ((who, t), (rid, node, m))) | join(likes, ((who, _), ((rid, node, m), ())) -> ((rid, who, node, m), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); let points = (liked | map(((rid, who, node, _), ()) -> ((rid, who, node), 1))) + (wrote + (liked | negate) | map(((rid, who, node, _), ()) -> ((rid, who, node), -1))); -let summed = points | join_function((key, p) -> ((key, ()), 0, p) enlist) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +let summed = points | join_function((key, p) -> ((key, ()), 0, p) enlist) | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); let keys = cands | map((key, _) -> (key, ())); let score = summed + (keys + (keys | join(summed, (key, ((), _)) -> (key, ())) | negate) | map((key, ()) -> (key, 0))); let ranked = score | join(cands, ((rid, _, node), (s, (first, last, gender, town))) -> (rid, ((s neg, node), (first, last, s, gender, town)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic12_oneoff.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic12_oneoff.parsed--lowered.diff index ca16e38f2..b06bcf5f5 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic12_oneoff.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic12_oneoff.parsed--lowered.diff @@ -12,7 +12,7 @@ let reply = import "ic12s::reply" : (int, (int, int)); -let intag = request | map(((rid, _, name), ()) -> (name, rid)) | join(classes, (_, (rid, high)) -> (high, rid)) | join(below_hl, (_, (rid, low)) -> (low, rid)) | join(by_class, (_, (rid, t)) -> (t, rid)) | join(tag, (t, (rid, (name, _))) -> (t, (rid, name))) | join(mtag, (_, ((rid, name), m)) -> (m, (rid, name))) | join(message, (m, ((rid, name), (kind, _, _, _, _, _, _, _, _, _, _, _, _))) -> ((rid, m, name), kind)) | filter((_, kind) -> (kind, 0) eq) | map((key, _) -> (key, ())); -let hits = request | map(((rid, who, _), ()) -> (who, rid)) | join(friend, (_, (rid, (f, _))) -> (f, rid)) | join(reply, (f, (rid, (id, parent))) -> ((rid, parent), (f, id))) | join(intag | map(((rid, m, name), ()) -> ((rid, m), name)), ((rid, _), ((f, id), name)) -> ((rid, f), (id, name))); --let counts = hits | map(((rid, f), (id, _)) -> ((rid, f, id), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((rid, f, _), ()) -> ((rid, f), ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +-let counts = hits | map(((rid, f), (id, _)) -> ((rid, f, id), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((rid, f, _), ()) -> ((rid, f), ())) | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); -let tagset = hits | map(((rid, f), (_, name)) -> ((rid, f, name), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((rid, f, name), ()) -> ((rid, f), name)) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); -let ranked = counts | join(tagset, ((rid, f), (n, list)) -> (f, (rid, n, list))) | join(person, (f, ((rid, n, list), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((n neg, f), (first, last, list)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); +let intag_map = request | map(((rid, _, name), ()) -> (name, rid)); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic12_oneoff.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic12_oneoff.parsed.ddp index 6405f167e..9d653dc62 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic12_oneoff.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic12_oneoff.parsed.ddp @@ -11,7 +11,7 @@ let below_hl = import "ic12s::below_hl" : (int, int); let reply = import "ic12s::reply" : (int, (int, int)); let intag = request | map(((rid, _, name), ()) -> (name, rid)) | join(classes, (_, (rid, high)) -> (high, rid)) | join(below_hl, (_, (rid, low)) -> (low, rid)) | join(by_class, (_, (rid, t)) -> (t, rid)) | join(tag, (t, (rid, (name, _))) -> (t, (rid, name))) | join(mtag, (_, ((rid, name), m)) -> (m, (rid, name))) | join(message, (m, ((rid, name), (kind, _, _, _, _, _, _, _, _, _, _, _, _))) -> ((rid, m, name), kind)) | filter((_, kind) -> (kind, 0) eq) | map((key, _) -> (key, ())); let hits = request | map(((rid, who, _), ()) -> (who, rid)) | join(friend, (_, (rid, (f, _))) -> (f, rid)) | join(reply, (f, (rid, (id, parent))) -> ((rid, parent), (f, id))) | join(intag | map(((rid, m, name), ()) -> ((rid, m), name)), ((rid, _), ((f, id), name)) -> ((rid, f), (id, name))); -let counts = hits | map(((rid, f), (id, _)) -> ((rid, f, id), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((rid, f, _), ()) -> ((rid, f), ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +let counts = hits | map(((rid, f), (id, _)) -> ((rid, f, id), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((rid, f, _), ()) -> ((rid, f), ())) | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); let tagset = hits | map(((rid, f), (_, name)) -> ((rid, f, name), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((rid, f, name), ()) -> ((rid, f), name)) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); let ranked = counts | join(tagset, ((rid, f), (n, list)) -> (f, (rid, n, list))) | join(person, (f, ((rid, n, list), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((n neg, f), (first, last, list)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((neg, f), (first, last, list)))) -> ((rid, rank), (f, first, last, list, neg neg))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic14_standing.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic14_standing.parsed--lowered.diff index 49f7fda28..1d48c2abf 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic14_standing.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic14_standing.parsed--lowered.diff @@ -7,7 +7,7 @@ let pals = import "graph::friend.pair" : ((int, int), ()); -let answered = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 1) eq) | map((_, (_, _, who, _, _, _, _, _, _, parent, _, _, _)) -> (parent, who)) | join(message, (_, (who, (_, _, to, _, _, _, _, _, _, _, _, _, _))) -> (who, to)) | filter((who, to) -> (who, to) ne); -let talks = (answered | map((a, b) -> ((a, b), ()))) + (answered | map((a, b) -> ((b, a), ()))); --let weighted = talks | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter) | join(pals, (pair, (n, ())) -> (pair, n)) | map(((a, b), n) -> (a, (b, (((0, n), (1, 40) range) fold (((acc, n), k) -> +-let weighted = talks | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter) | join(pals, (pair, (n, ())) -> (pair, n)) | map(((a, b), n) -> (a, (b, (((0, n), (1, 40) range) fold (((acc, n), k) -> +let answered_filter = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 1) eq); +let answered_map = answered_filter | map((_, (_, _, who, _, _, _, _, _, _, parent, _, _, _)) -> (parent, who)); +let answered_idx = answered_map | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic14_standing.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic14_standing.parsed.ddp index f461b9ec9..55a639aea 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic14_standing.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic14_standing.parsed.ddp @@ -2,6 +2,6 @@ let message = import "graph::message" : (int, (int, int, int, int, List(int), Li let pals = import "graph::friend.pair" : ((int, int), ()); let answered = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 1) eq) | map((_, (_, _, who, _, _, _, _, _, _, parent, _, _, _)) -> (parent, who)) | join(message, (_, (who, (_, _, to, _, _, _, _, _, _, _, _, _, _))) -> (who, to)) | filter((who, to) -> (who, to) ne); let talks = (answered | map((a, b) -> ((a, b), ()))) + (answered | map((a, b) -> ((b, a), ()))); -let weighted = talks | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter) | join(pals, (pair, (n, ())) -> (pair, n)) | map(((a, b), n) -> (a, (b, (((0, n), (1, 40) range) fold (((acc, n), k) -> +let weighted = talks | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter) | join(pals, (pair, (n, ())) -> (pair, n)) | map(((a, b), n) -> (a, (b, (((0, n), (1, 40) range) fold (((acc, n), k) -> ((acc, ((((k, (k, 1) sub) mul, 1) add, n) gt, 1, 0) select) add, n)) .0, 1) add))); output "weighted" = weighted | arrange; diff --git a/interactive/tests/plan_pairs/expected/reduce_masks.ddp b/interactive/tests/plan_pairs/expected/reduce_masks.ddp index 70d551db2..39701c7cd 100644 --- a/interactive/tests/plan_pairs/expected/reduce_masks.ddp +++ b/interactive/tests/plan_pairs/expected/reduce_masks.ddp @@ -11,7 +11,9 @@ let unique_map = values | map((k, v) -> ((k, v), ())); let unique_idx = unique_map | arrange; let unique_reduce = unique_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); let unique_map2 = unique_reduce | map(((a, (_, b)), _) -> (a, ((), b))); -let counted_reduce = values_idx | reduce((_, a) -> let v0 = a map ((_, a) -> a) fold_add in (v0, (v0, 1)) enlist filter map ((_, a) -> ((), a))); +let counted_map = values | map((k, _) -> (k, ())); +let counted_idx = counted_map | arrange; +let counted_reduce = counted_idx | reduce((_, a) -> let v0 = a map ((_, a) -> a) fold_add in (v0, (v0, 1)) enlist filter map ((_, a) -> ((), a))); let collected_reduce = values_idx | reduce(_ -> ((), 1) enlist); let custom_map = custom | map((k, (_, v)) -> ((k, v), ())); -- Renderer: build an export arrangement. diff --git a/interactive/tests/plan_pairs/expected/reduce_masks.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/reduce_masks.lowered--optimized.diff index 5eda204b3..dcd596a4a 100644 --- a/interactive/tests/plan_pairs/expected/reduce_masks.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/reduce_masks.lowered--optimized.diff @@ -1,6 +1,6 @@ --- lowered +++ optimized -@@ -6,19 +6,15 @@ +@@ -6,20 +6,17 @@ let values_idx = values | arrange; let custom = values_idx | reduce((_, vals) -> vals map ((v, d) -> (v, (d, 2) mul))); let out_map = counted | map(((a, _), n) -> (a, n)); @@ -13,23 +13,23 @@ let unique_reduce = unique_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); -let unique_map2 = unique_reduce | map((kv, ()) -> kv); -let unique_map3 = unique_map2 | map((k, (_, v)) -> (k, ((), v))); --let values_idx3 = values | arrange; --let counted_reduce = values_idx3 | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +let unique_map2 = unique_reduce | map(((a, (_, b)), _) -> (a, ((), b))); -+let counted_reduce = values_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); - let counted_map = counted_reduce | map((k, _) -> (k, ())); --let values_idx4 = values | arrange; --let collected_reduce = values_idx4 | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); + let counted_map = values | map((k, _) -> (k, ())); + let counted_idx = counted_map | arrange; + let counted_reduce = counted_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); + let counted_map2 = counted_reduce | map((k, _) -> (k, ())); +-let values_idx3 = values | arrange; +-let collected_reduce = values_idx3 | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); +let collected_reduce = values_idx | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); let collected_map = collected_reduce | map((k, _) -> (k, ())); let custom_map = custom | map((k, (_, v)) -> ((k, v), ())); -- Renderer: build an export arrangement. -@@ -26,7 +22,7 @@ +@@ -27,7 +24,7 @@ -- Renderer: build an export arrangement. output "minimum" = minimum_map; -- Renderer: build an export arrangement. -output "unique" = unique_map3; +output "unique" = unique_map2; -- Renderer: build an export arrangement. - output "counted" = counted_map; + output "counted" = counted_map2; -- Renderer: build an export arrangement. diff --git a/interactive/tests/plan_pairs/expected/reduce_masks.optimized--projected.diff b/interactive/tests/plan_pairs/expected/reduce_masks.optimized--projected.diff index 436e8e04f..d160b1dae 100644 --- a/interactive/tests/plan_pairs/expected/reduce_masks.optimized--projected.diff +++ b/interactive/tests/plan_pairs/expected/reduce_masks.optimized--projected.diff @@ -1,6 +1,6 @@ --- optimized +++ projected -@@ -4,28 +4,25 @@ +@@ -4,30 +4,27 @@ let rows_idx = rows | arrange; let counted = rows_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); let values_idx = values | arrange; @@ -14,11 +14,13 @@ let unique_idx = unique_map | arrange; let unique_reduce = unique_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); let unique_map2 = unique_reduce | map(((a, (_, b)), _) -> (a, ((), b))); --let counted_reduce = values_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); --let counted_map = counted_reduce | map((k, _) -> (k, ())); + let counted_map = values | map((k, _) -> (k, ())); + let counted_idx = counted_map | arrange; +-let counted_reduce = counted_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +-let counted_map2 = counted_reduce | map((k, _) -> (k, ())); -let collected_reduce = values_idx | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); -let collected_map = collected_reduce | map((k, _) -> (k, ())); -+let counted_reduce = values_idx | reduce((_, a) -> let v0 = a map ((_, a) -> a) fold_add in (v0, (v0, 1)) enlist filter map ((_, a) -> ((), a))); ++let counted_reduce = counted_idx | reduce((_, a) -> let v0 = a map ((_, a) -> a) fold_add in (v0, (v0, 1)) enlist filter map ((_, a) -> ((), a))); +let collected_reduce = values_idx | reduce(_ -> ((), 1) enlist); let custom_map = custom | map((k, (_, v)) -> ((k, v), ())); -- Renderer: build an export arrangement. @@ -30,7 +32,7 @@ -- Renderer: build an export arrangement. output "unique" = unique_map2; --- Renderer: build an export arrangement. --output "counted" = counted_map; +-output "counted" = counted_map2; --- Renderer: build an export arrangement. -output "collected" = collected_map; +-- Renderer: publish the existing arrangement. diff --git a/interactive/tests/plan_pairs/expected/reduce_masks.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/reduce_masks.parsed--lowered.diff index 2fcc8023c..dab9e8726 100644 --- a/interactive/tests/plan_pairs/expected/reduce_masks.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/reduce_masks.parsed--lowered.diff @@ -1,14 +1,14 @@ --- parsed +++ lowered -@@ -1,10 +1,35 @@ +@@ -1,10 +1,36 @@ +-- Time lanes: epoch. let rows = input 0 : ((int, int), ()); --let counted = rows | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +-let counted = rows | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); -output "out" = counted | map(((a, _), n) -> (a, n)); let values = input 1 : (int, (int, int)); -output "minimum" = values | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist) | map((k, (_, v)) -> (k, ((), v))); -output "unique" = values | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map((k, (_, v)) -> (k, ((), v))); --output "counted" = values | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter) | map((k, _) -> (k, ())); +-output "counted" = values | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter) | map((k, _) -> (k, ())); -output "collected" = values | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist) | map((k, _) -> (k, ())); -let custom = values | reduce((_, vals) -> vals map ((v, d) -> (v, (d, 2) mul))); -output "custom" = custom | map((k, (_, v)) -> ((k, v), ())); @@ -25,11 +25,12 @@ +let unique_reduce = unique_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); +let unique_map2 = unique_reduce | map((kv, ()) -> kv); +let unique_map3 = unique_map2 | map((k, (_, v)) -> (k, ((), v))); ++let counted_map = values | map((k, _) -> (k, ())); ++let counted_idx = counted_map | arrange; ++let counted_reduce = counted_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); ++let counted_map2 = counted_reduce | map((k, _) -> (k, ())); +let values_idx3 = values | arrange; -+let counted_reduce = values_idx3 | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); -+let counted_map = counted_reduce | map((k, _) -> (k, ())); -+let values_idx4 = values | arrange; -+let collected_reduce = values_idx4 | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); ++let collected_reduce = values_idx3 | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); +let collected_map = collected_reduce | map((k, _) -> (k, ())); +let custom_map = custom | map((k, (_, v)) -> ((k, v), ())); +-- Renderer: build an export arrangement. @@ -39,7 +40,7 @@ +-- Renderer: build an export arrangement. +output "unique" = unique_map3; +-- Renderer: build an export arrangement. -+output "counted" = counted_map; ++output "counted" = counted_map2; +-- Renderer: build an export arrangement. +output "collected" = collected_map; +-- Renderer: build an export arrangement. diff --git a/interactive/tests/plan_pairs/expected/reduce_masks.parsed.ddp b/interactive/tests/plan_pairs/expected/reduce_masks.parsed.ddp index 13187a077..426a8b4a8 100644 --- a/interactive/tests/plan_pairs/expected/reduce_masks.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/reduce_masks.parsed.ddp @@ -1,10 +1,10 @@ let rows = input 0 : ((int, int), ()); -let counted = rows | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +let counted = rows | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); output "out" = counted | map(((a, _), n) -> (a, n)); let values = input 1 : (int, (int, int)); output "minimum" = values | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist) | map((k, (_, v)) -> (k, ((), v))); output "unique" = values | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map((k, (_, v)) -> (k, ((), v))); -output "counted" = values | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter) | map((k, _) -> (k, ())); +output "counted" = values | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter) | map((k, _) -> (k, ())); output "collected" = values | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist) | map((k, _) -> (k, ())); let custom = values | reduce((_, vals) -> vals map ((v, d) -> (v, (d, 2) mul))); output "custom" = custom | map((k, (_, v)) -> ((k, v), ())); diff --git a/interactive/tests/plan_pairs/expected/shared_demand.ddp b/interactive/tests/plan_pairs/expected/shared_demand.ddp index ffef77e6e..a42ea3e97 100644 --- a/interactive/tests/plan_pairs/expected/shared_demand.ddp +++ b/interactive/tests/plan_pairs/expected/shared_demand.ddp @@ -2,8 +2,9 @@ let rows = input 0 : (int, (int, int)); let first_map = rows | map((k, (a, _)) -> (k, a)); let second_map = rows | map((k, (_, b)) -> (k, b)); -let rows_idx = rows | arrange; -let counts_reduce = rows_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +let counts_map = rows | map((k, _) -> (k, ())); +let counts_idx = counts_map | arrange; +let counts_reduce = counts_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); -- Renderer: build an export arrangement. output "first" = first_map; -- Renderer: build an export arrangement. diff --git a/interactive/tests/plan_pairs/expected/shared_demand.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/shared_demand.lowered--optimized.diff index ea67221f5..f62522f82 100644 --- a/interactive/tests/plan_pairs/expected/shared_demand.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/shared_demand.lowered--optimized.diff @@ -7,4 +7,4 @@ +let rows = input 0 : (int, (int, int)); let first_map = rows | map((k, (a, _)) -> (k, a)); let second_map = rows | map((k, (_, b)) -> (k, b)); - let rows_idx = rows | arrange; + let counts_map = rows | map((k, _) -> (k, ())); diff --git a/interactive/tests/plan_pairs/expected/shared_demand.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/shared_demand.parsed--lowered.diff index 1b7e1ac64..b9fe0095f 100644 --- a/interactive/tests/plan_pairs/expected/shared_demand.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/shared_demand.parsed--lowered.diff @@ -1,17 +1,18 @@ --- parsed +++ lowered -@@ -1,4 +1,13 @@ +@@ -1,4 +1,14 @@ -let rows = input 0 : (int, (int, int)) | map((k, v) -> (k, v)); -output "first" = rows | map((k, (a, _)) -> (k, a)); -output "second" = rows | map((k, (_, b)) -> (k, b)); --output "counts" = rows | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +-output "counts" = rows | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +-- Time lanes: epoch. +let input0 = input 0 : (int, (int, int)); +let rows = input0 | map((k, v) -> (k, v)); +let first_map = rows | map((k, (a, _)) -> (k, a)); +let second_map = rows | map((k, (_, b)) -> (k, b)); -+let rows_idx = rows | arrange; -+let counts_reduce = rows_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); ++let counts_map = rows | map((k, _) -> (k, ())); ++let counts_idx = counts_map | arrange; ++let counts_reduce = counts_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +-- Renderer: build an export arrangement. +output "first" = first_map; +-- Renderer: build an export arrangement. diff --git a/interactive/tests/plan_pairs/expected/shared_demand.parsed.ddp b/interactive/tests/plan_pairs/expected/shared_demand.parsed.ddp index bdb858143..047dda5e0 100644 --- a/interactive/tests/plan_pairs/expected/shared_demand.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/shared_demand.parsed.ddp @@ -1,4 +1,4 @@ let rows = input 0 : (int, (int, int)) | map((k, v) -> (k, v)); output "first" = rows | map((k, (a, _)) -> (k, a)); output "second" = rows | map((k, (_, b)) -> (k, b)); -output "counts" = rows | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +output "counts" = rows | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); diff --git a/interactive/tests/programs/lambda_slots.ddp b/interactive/tests/programs/lambda_slots.ddp index b9619c197..915a67b2d 100644 --- a/interactive/tests/programs/lambda_slots.ddp +++ b/interactive/tests/programs/lambda_slots.ddp @@ -1,4 +1,4 @@ --- The slots that take one lambda: `value` (the value as written, with no tuple around it), +-- The slots that take one lambda: `map` (the value as written, with no tuple around it), -- `filter`, `flatmap` (complete rows, with positions explicitly zipped here) and `delay` (a row -- advances to the coordinate-wise maximum of its time and the tuple it names). A `--` comment -- may sit inside a lambda written over several lines. Declared types and the built-in `Option` @@ -10,7 +10,7 @@ enum Outer = Wrap Inner | Opt Option(int) | Nums List(int); let pairs = input 0 : ((int, int), ()) | map(((a, b), ()) -> (a, b)); -output "value" = pairs | value((a, b) -> ((b, 10) mul, a) add); +output "value" = pairs | map((a, b) -> (a, ((b, 10) mul, a) add)); output "filter" = pairs | filter((a, b) -> (a, b) lt); output "flatmap" = pairs | flatmap(row -> let (a, b) = row in let l = (a enlist, b enlist, (a, b) add enlist) append in (row.0, ((0, l len) range, l) zip) cap_list); stepped: { @@ -18,14 +18,14 @@ stepped: { set counted = delayed | count; } output "delay" = stepped::counted; -output "comment" = pairs | value((a, b) -> +output "comment" = pairs | map((a, b) -> (a, -- the key and the value, added (a, b) add -- one Int -); +)); -let sized = pairs | value((a, b) -> ((b, 3) gt, (b, a) inject Big, b inject Small) select); -output "sized" = sized | value((_, s) -> s match (Small (x -> x), Big ((x, y) -> (x, y) mul))); -let maybe = pairs | value((_, b) -> (b, (b, 3) gt) branch Option map_variant None (_ -> ())); -output "option" = maybe | value((_, m) -> m match (None (_ -> 0), Some (x -> x))); -let outer = pairs | value((_, b) -> b inject A inject Wrap); -output "nested" = outer | value((_, o) -> o match (Wrap (i -> i match (A (x -> x), B (_ -> 0))), Opt (_ -> -1), Nums (_ -> -1))); +let sized = pairs | map((a, b) -> (a, ((b, 3) gt, (b, a) inject Big, b inject Small) select)); +output "sized" = sized | map((k, s) -> (k, s match (Small (x -> x), Big ((x, y) -> (x, y) mul)))); +let maybe = pairs | map((k, b) -> (k, (b, (b, 3) gt) branch Option map_variant None (_ -> ()))); +output "option" = maybe | map((k, m) -> (k, m match (None (_ -> 0), Some (x -> x)))); +let outer = pairs | map((k, b) -> (k, b inject A inject Wrap)); +output "nested" = outer | map((k, o) -> (k, o match (Wrap (i -> i match (A (x -> x), B (_ -> 0))), Opt (_ -> -1), Nums (_ -> -1)))); From 270819df89050f92bdd45ba36d8a518c565360e7 Mon Sep 17 00:00:00 2001 From: Frank McSherry Date: Fri, 20 Nov 2026 06:36:38 -0500 Subject: [PATCH 2/3] DDIR projection: stop re-masking a half join's unit value A half join appends its time to the value its lambda gives, and keeps one demand fact for that value. Once the value was masked to unit, the fact stayed dead, so each round masked it again and counted a change. Pushdown hit its 16-round limit and reverted the whole program's projection. That happened whenever a half join's value went unread, as when count now drops its values after one. A unit value is now left alone, and the half_join tests end in count again. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01H6wB4Gjwwivtq59LaFYbF5 --- interactive/src/passes/projection.rs | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/interactive/src/passes/projection.rs b/interactive/src/passes/projection.rs index 4d5a8fa31..4efd99794 100644 --- a/interactive/src/passes/projection.rs +++ b/interactive/src/passes/projection.rs @@ -704,12 +704,19 @@ fn mask_half_join(original: &Lambda, output: &Site, live: &[bool]) -> Option u (Item::Op(Node::HalfJoin { projection, .. }), PlannedItem::Op { output, .. }) => { if let Some(new) = mask_half_join(projection, output, live) { masked += output.key.iter().filter(|&&id| !live[id]).count(); - masked += usize::from(!live[output.val[0]]); + let unit = matches!(&output.shape.1, Shape::Prod(parts) if parts[0] == Shape::Unit); + masked += usize::from(!unit && !live[output.val[0]]); *projection = new; } } @@ -1165,11 +1173,10 @@ mod tests { let b = input 1 : ((int, int, int), ()) | map(((k, y, z), ()) -> (k, (y, z))); output \"out\" = a | lift - | half_join(b, 0, (k, x, yz) -> (k, x)) - | reduce((k, vals) -> let n = vals map ((_, d) -> d) fold_add in (n, (n, 1)) enlist filter);"; + | half_join(b, 0, (k, x, yz) -> (k, x)) | count;"; let mut program = lower::lower_tree(parse::pipe::parse(text)); crate::passes::optimize(&mut program); - assert_eq!(pushdown(&mut program, &|_| None).unwrap(), 2); + assert_eq!(pushdown(&mut program, &|_| None).unwrap(), 4); let (_, check) = shapes::infer(&program, &|_| None, 1); assert!(check.conflicts.is_empty(), "{:?}", check.conflicts); } @@ -1182,12 +1189,11 @@ mod tests { let b = input 1 : ((int, int, int), ()) | map(((k, y, z), ()) -> (k, (y, z))); output \"out\" = a | lift - | half_join(b, 0, (k, x, yz) -> (k, x)) - | reduce((k, vals) -> let n = vals map ((_, d) -> d) fold_add in (n, (n, 1)) enlist filter);"; + | half_join(b, 0, (k, x, yz) -> (k, x)) | count;"; let mut original = lower::lower_tree(parse::pipe::parse(text)); crate::passes::optimize(&mut original); let mut projected = original.clone(); - assert_eq!(pushdown(&mut projected, &|_| None).unwrap(), 2); + assert_eq!(pushdown(&mut projected, &|_| None).unwrap(), 4); timely::execute_directly(move |worker| { let mut server = Server::new(); server.install(worker, "before", &original).unwrap(); From 02c1aecff31aee6513399462d7312636dadcec2b Mon Sep 17 00:00:00 2001 From: Frank McSherry Date: Fri, 20 Nov 2026 15:21:46 -0500 Subject: [PATCH 3/3] DDIR: remove LinearOp::LiftIter; explain writes lift and a map Explain's scope exits now fold the round into the chain with `lift` and a map that keeps the value's fields and the time's last coordinate. The op, its shape rule, its projection rule, its corgi kernel and its printer line go. Removing the `lift_iter` spelling had left the printer writing a word the parser rejects. Review nits: the lambda_slots header no longer describes `value`, the bitemporal test drops its repeated column, an identity map and a doubled map go, and `by_record` is now `valued`. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01H6wB4Gjwwivtq59LaFYbF5 --- interactive/bench/ldbc/snb/rel.py | 2 +- interactive/src/backend/corgi.rs | 26 +++------------------ interactive/src/explain/mod.rs | 20 +++++++++++----- interactive/src/ir.rs | 8 ------- interactive/src/level.rs | 1 - interactive/src/passes/as_of.rs | 4 ++-- interactive/src/passes/projection.rs | 8 ------- interactive/src/plan.rs | 3 +-- interactive/src/plan/compare.rs | 2 +- interactive/src/plan/ddp.rs | 2 +- interactive/src/reducers.rs | 9 +++---- interactive/src/scope_ir.rs | 2 +- interactive/src/shapes.rs | 6 ----- interactive/tests/bitemporal.rs | 4 ++-- interactive/tests/explain.rs | 7 +++--- interactive/tests/programs/lambda_slots.ddp | 2 +- 16 files changed, 35 insertions(+), 71 deletions(-) diff --git a/interactive/bench/ldbc/snb/rel.py b/interactive/bench/ldbc/snb/rel.py index c02f8a895..ba1e409a6 100644 --- a/interactive/bench/ldbc/snb/rel.py +++ b/interactive/bench/ldbc/snb/rel.py @@ -385,7 +385,7 @@ def keyed(source, fields, keys, keep=True): # A dedicated value-only collect avoids lists of singleton # tuples and is useful for nested IC1/IC12 results. assert len(aggs)==1 - code = f'{source_name} | map((row, ()) -> ({group_key}, ({scalar(expr(e),refs)}))) | map((k, (item)) -> (k, item)) | collect | map((key, values) -> ({product([*outs,"values"])}, ()))' + code = f'{source_name} | map((row, ()) -> ({group_key}, {scalar(expr(e),refs)})) | collect | map((key, values) -> ({product([*outs,"values"])}, ()))' self.lines.append(f'-- {name}: ({", ".join(r.fields)})\nlet {name} = {code};') return name else: raise ValueError(kind) diff --git a/interactive/src/backend/corgi.rs b/interactive/src/backend/corgi.rs index 3c2282574..2f192222a 100644 --- a/interactive/src/backend/corgi.rs +++ b/interactive/src/backend/corgi.rs @@ -67,7 +67,7 @@ impl Plan { LinearOp::Filter(cond) => ("filter", Some(compile_predicate(cond, k, v))), LinearOp::FlatMap(list) => ("flatmap", Some(compile_flatmap(list, k, v))), LinearOp::JoinFunction(lambda) => ("join_function", Some(compile_join_function(lambda, k, v, level.width()))), - LinearOp::Negate | LinearOp::LiftIter | LinearOp::Lift => ("", None), + LinearOp::Negate | LinearOp::Lift => ("", None), }; let graph = graph.map(|g| g.unwrap_or_else(|e| panic!("{what}: type error at shapes ({kshape}, {vshape}): {e}"))); Plan { shape: (kshape, vshape), graph } @@ -86,8 +86,7 @@ impl Plan { /// column + a structural explode; Negate = Rust. Every slot is columnar — there is no row-wise /// path inside the dataflow — and each op's graph was compiled when the operator was built /// (`plans`). An empty batch passes through untouched: it has no rows to compute. -/// The data<->time ops are columnar and total: LiftIter reads the iteration coordinate out of -/// `times` and appends it to `vals`; Lift reads every lane of `times` into `vals`; JoinFunction +/// The data<->time ops are columnar and total: Lift reads every lane of `times` into `vals`; JoinFunction /// explodes its list column as FlatMap does and joins each element's time lanes into `times`. /// `level` locates the iteration coordinate and fixes the number of lanes as data. fn apply_ops(mut c: CC, ops: &[LinearOp], level: Level, plans: &[Plan]) -> CC { @@ -127,25 +126,6 @@ fn apply_ops(mut c: CC, ops: &[LinearOp], level: Level, plans: &[Plan]) -> CC { } c } - // A value read OUT of each row's time. Vals gain one - // integer field; keys, times and diffs are untouched, and no slot is compiled. - // - // A tuple gains a field, and unit is the empty tuple: columnar it arrives as - // `CValue::Unit`, not an empty `Prod`, so `Unit` must become `Prod([iter])`. - // `Prod([Unit, iter])` would be a silent one-field-too-many divergence from the - // shape `shapes` infers. - LinearOp::LiftIter => { - // At the root there is no iteration - // coordinate and the value is zero. - let iters: Vec = level.own_lane().map_or_else(|| vec![0; c.times.len()], |lane| c.times.lane(lane)); - let lane = CValue::i64(iters.into_iter().map(|i| i as i64).collect()); - let vals = match c.vals { - CValue::Prod(mut fields) => { fields.push(lane); CValue::Prod(fields) } - CValue::Unit(_) => CValue::Prod(vec![lane]), - other => CValue::Prod(vec![other, lane]), - }; - CorgiContainer { keys: c.keys, vals, times: c.times, diffs: c.diffs } - } // A time as data is the root coordinates and one per scope: `level.width()` lanes, zero // where the column is narrower. Keys, times and diffs are untouched. LinearOp::Lift => { @@ -309,7 +289,7 @@ impl Backend for CorgiBackend { fn linear<'s>(c: Collection<'s, Time, CC>, ops: Vec, level: Level, (k, v): st::RowShape) -> Collection<'s, Time, CC> { // Container-level: fold the LinearOp chain over each corgi batch (no inter-op transcode). - // `level` is the scope depth (locates the iteration coordinate for LiftIter). + // `level` is the scope depth, which fixes the number of time lanes. // Every op compiles now, at the shape it will see. let steps = crate::shapes::linear_shapes(&ops, k, v, level).expect("install inferred these shapes"); let plans: Vec = ops.iter().zip(steps).map(|(op, (key, val))| Plan::new(op, key, val, level)).collect(); diff --git a/interactive/src/explain/mod.rs b/interactive/src/explain/mod.rs index 8621ff0d9..a0fc96a93 100644 --- a/interactive/src/explain/mod.rs +++ b/interactive/src/explain/mod.rs @@ -9,7 +9,7 @@ //! The clone-with-lifts is the foundation the rewrite's witness and //! forward copies are built from. `clone_into` clones an original //! program's scopes into an output scope under construction; every nested -//! scope additionally `lift_iter`s and exports each internal collection, so +//! scope additionally lifts its round into each internal collection and exports it, so //! every value-producing site in the subtree has a "host-visible" form — its //! value as `(V, chain…)`, the user's value whole and then its user-iter //! coordinates, innermost first — at the @@ -176,6 +176,11 @@ fn clone_rec(orig: &Scope, out: &mut Scope, import_map: &[Ref], path: &[usize], // and the export is the host-visible edge upward. A site of // this scope first has its value wrapped as `(V)`, so the // lifts extend it to `(V, chain…)` with `V` whole. + // + // The fold is `lift` and then a map that keeps the value's fields and the time's + // last coordinate, which is this scope's round. Times here have one root + // coordinate, the forward clone's explanation round if `forward`, and one per scope. + let round = format!("v.1.{}", child_path.len() + usize::from(forward)); let sub_idx = out.items.len(); for (a, vref) in child_visible { let vref = if a.path == child_path { @@ -185,8 +190,13 @@ fn clone_rec(orig: &Scope, out: &mut Scope, import_map: &[Ref], path: &[usize], } else { vref }; + // The value is `(V)` and one coordinate for each deeper exit. + let width = 1 + a.path.len() - child_path.len(); + let mut fields: Vec = (0..width).map(|j| format!("v.0.{j}")).collect(); + fields.push(round.clone()); + let fold = LinearOp::Project(projection(var(0), tup(fields), 2)); let lift_idx = child_out.items.len(); - child_out.items.push(Item::Op(Node::Linear { input: vref, ops: vec![LinearOp::LiftIter] })); + child_out.items.push(Item::Op(Node::Linear { input: vref, ops: vec![LinearOp::Lift, fold] })); let export_idx = child_out.exports.len(); child_out.exports.push(Export { name: format!("$host:{}", export_idx), @@ -1017,7 +1027,6 @@ impl<'a> Reverse<'a> { LinearOp::Negate => { self.push(ex, path, input, dep_this); } - LinearOp::LiftIter => panic!("explain: LiftIter in user program"), // Lift reads the whole time, and the forward clone runs one scope deeper. LinearOp::Lift => panic!("explain: lift is not supported"), LinearOp::JoinFunction(lambda) => { @@ -1127,7 +1136,7 @@ mod tests { let mut out = Scope { imports: p.root.imports.clone(), ..Scope::default() }; let import_map: Vec = (0..out.imports.len()).map(Ref::Import).collect(); clone_into(&p.root, &mut out, &import_map); - // outer's clone: every $host: export is a LiftIter Linear; and the + // outer's clone: every $host: export is a lift and a map; and the // ones re-exporting fwd/bwd internals chain TWO lifts (one per level): // the value behind the lift is itself a ChildExport of a lift. let Item::Sub(outer) = out.items.iter().find(|i| matches!(i, Item::Sub(_))).unwrap() else { unreachable!() }; @@ -1135,8 +1144,7 @@ mod tests { for e in outer.exports.iter().filter(|e| e.name.starts_with("$host:")) { let Ref::Local(li) = &e.value else { panic!("$host export should be a fresh lift") }; let Item::Op(Node::Linear { input, ops }) = &outer.items[*li] else { panic!("expected a lift") }; - assert_eq!(ops.as_slice().len(), 1); - assert!(matches!(ops[0], LinearOp::LiftIter)); + assert!(matches!(ops.as_slice(), [LinearOp::Lift, LinearOp::Project(_)])); if matches!(input, Ref::ChildExport(..)) { depth2_chains += 1; } } assert!(depth2_chains > 0, "fwd/bwd internals re-lift at outer's exit"); diff --git a/interactive/src/ir.rs b/interactive/src/ir.rs index 0c383e993..e6a16d30e 100644 --- a/interactive/src/ir.rs +++ b/interactive/src/ir.rs @@ -35,14 +35,6 @@ pub enum LinearOp { Filter(Lambda), /// Negate the diff. Negate, - /// Append the current user-iter coord (at the row's scope depth) to - /// the value. Time itself is unchanged. Only explain writes it; the language has no spelling. - /// - /// Discipline, which nothing checks: the result of `LiftIter` should not be referenced inside the same - /// scope it appears in — only from an enclosing scope, after the - /// implicit leave. This preserves the "loop body is a time-invariant - /// function" property; in-scope use risks defeating the fixpoint. - LiftIter, /// Each row becomes the lambda's list of complete `(key, val)` rows. FlatMap(Lambda), /// Pair each value with its whole time: `(key, val)` becomes `(key, (val, time))`. diff --git a/interactive/src/level.rs b/interactive/src/level.rs index 820f8f517..4ea413efa 100644 --- a/interactive/src/level.rs +++ b/interactive/src/level.rs @@ -9,7 +9,6 @@ pub(crate) struct Level { impl Level { pub(crate) fn root(root: usize) -> Self { Self { root, depth: 0 } } pub(crate) fn width(self) -> usize { self.root + self.depth } - pub(crate) fn own_lane(self) -> Option { (self.depth > 0).then(|| self.width() - 1) } pub(crate) fn child(self) -> Self { Self { depth: self.depth + 1, ..self } } pub(crate) fn parent(self) -> Self { Self { depth: self.depth.saturating_sub(1), ..self } } } diff --git a/interactive/src/passes/as_of.rs b/interactive/src/passes/as_of.rs index a1616e369..f933536e6 100644 --- a/interactive/src/passes/as_of.rs +++ b/interactive/src/passes/as_of.rs @@ -31,7 +31,7 @@ pub(super) fn drop_dead_as_of(prog: &mut Program, as_of: &Time, root: usize) -> /// half join that makes its updates at the as-of time dead. fn keeps_times(op: &LinearOp) -> bool { match op { - LinearOp::Project(_) | LinearOp::Filter(_) | LinearOp::Negate | LinearOp::LiftIter | LinearOp::FlatMap(_) | LinearOp::Lift => true, + LinearOp::Project(_) | LinearOp::Filter(_) | LinearOp::Negate | LinearOp::FlatMap(_) | LinearOp::Lift => true, LinearOp::JoinFunction(_) => false, } } @@ -74,7 +74,7 @@ fn drop_in_scope(s: &mut Scope, level: Level, as_of: &Time) -> usize { if !ops[at + 1..].iter().all(keeps_times) || uses[i].is_empty() { continue; } let dead: Vec = uses[i].iter().copied().filter(|u| dead_on(*u, &covered)).collect(); // The edge filter reads the time from the row value written by `lift`. Filter and Negate - // leave that value intact. Project, FlatMap, LiftIter and other tails may move or drop it; + // leave that value intact. Project, FlatMap and other tails may move or drop it; // handling them needs tracking time through lambdas or splitting the node and duplicating // its tail on dead edges, adding work every tick and losing the "never worse" property. if dead.len() == uses[i].len() { diff --git a/interactive/src/passes/projection.rs b/interactive/src/passes/projection.rs index 4efd99794..6046dea2c 100644 --- a/interactive/src/passes/projection.rs +++ b/interactive/src/passes/projection.rs @@ -206,13 +206,6 @@ impl Builder { } } - fn lift_iter(&mut self, from: &Site, input: &Site) { - self.key_identity(from, input); - for (&out, &src) in from.val.iter().zip(&input.val) { - self.edge(out, src); - } - } - fn lift(&mut self, from: &Site, input: &Site) { self.key_identity(from, input); if let Some(&value) = from.val.first() { @@ -502,7 +495,6 @@ fn build_scope( builder.scalar_lambda(&after, &[&before], lambda); } LinearOp::Negate => builder.identity(&after, &before), - LinearOp::LiftIter => builder.lift_iter(&after, &before), LinearOp::Lift => builder.lift(&after, &before), LinearOp::FlatMap(lambda) => { builder.presence(&after, &before); diff --git a/interactive/src/plan.rs b/interactive/src/plan.rs index d9de5be0f..84fd78d90 100644 --- a/interactive/src/plan.rs +++ b/interactive/src/plan.rs @@ -393,7 +393,6 @@ fn linear(op: &LinearOp) -> String { LinearOp::Project(l) => format!("map({})", source_lambda(l)), LinearOp::Filter(l) => format!("filter({})", source_lambda(l)), LinearOp::Negate => "negate".into(), - LinearOp::LiftIter => "lift_iter".into(), LinearOp::FlatMap(l) => format!("flatmap({})", source_lambda(l)), LinearOp::Lift => "lift".into(), LinearOp::JoinFunction(l) => format!("join_function({})", source_lambda(l)), @@ -431,7 +430,7 @@ fn printed_node(node: &Node, names: &Names, shapes: &ScopeShapes, depth: usize, let name = match op { LinearOp::Project(_) => "map", LinearOp::Filter(_) => "filter", LinearOp::FlatMap(_) => "flatmap", LinearOp::JoinFunction(_) => "join_function", - LinearOp::Negate => "negate", LinearOp::Lift => "lift", LinearOp::LiftIter => "lift_iter", + LinearOp::Negate => "negate", LinearOp::Lift => "lift", }; printed.push(match op { LinearOp::Project(l) | LinearOp::Filter(l) | LinearOp::FlatMap(l) | LinearOp::JoinFunction(l) => diff --git a/interactive/src/plan/compare.rs b/interactive/src/plan/compare.rs index fe7d3f57a..9d0da83c1 100644 --- a/interactive/src/plan/compare.rs +++ b/interactive/src/plan/compare.rs @@ -80,7 +80,7 @@ fn linear(a: &LinearOp, b: &LinearOp, input: Option<&corgi::Shape>) -> bool { (LinearOp::Project(a), LinearOp::Project(b)) | (LinearOp::Filter(a), LinearOp::Filter(b)) | (LinearOp::FlatMap(a), LinearOp::FlatMap(b)) | (LinearOp::JoinFunction(a), LinearOp::JoinFunction(b)) => lambda(a, b, input), - (LinearOp::Negate, LinearOp::Negate) | (LinearOp::Lift, LinearOp::Lift) | (LinearOp::LiftIter, LinearOp::LiftIter) => true, + (LinearOp::Negate, LinearOp::Negate) | (LinearOp::Lift, LinearOp::Lift) => true, _ => false, } } diff --git a/interactive/src/plan/ddp.rs b/interactive/src/plan/ddp.rs index 7f2d91f33..5d96e1637 100644 --- a/interactive/src/plan/ddp.rs +++ b/interactive/src/plan/ddp.rs @@ -399,7 +399,7 @@ impl Printer<'_> { LinearOp::Project(l) => ("map", Some(l)), LinearOp::Filter(l) => ("filter", Some(l)), LinearOp::FlatMap(l) => ("flatmap", Some(l)), LinearOp::JoinFunction(l) => ("join_function", Some(l)), LinearOp::Negate => ("negate", None), - LinearOp::Lift => ("lift", None), LinearOp::LiftIter => ("lift_iter", None), + LinearOp::Lift => ("lift", None), }; expr += &format!(" | {word}"); if let Some(l) = lambda { expr += &format!("({})", self.lambda(l, shape)?); } diff --git a/interactive/src/reducers.rs b/interactive/src/reducers.rs index 9813aba0a..babd6cbad 100644 --- a/interactive/src/reducers.rs +++ b/interactive/src/reducers.rs @@ -40,7 +40,8 @@ pub(crate) fn expand_by_shape( }; for (i, mut item) in std::mem::take(&mut scope.items).into_iter().enumerate() { let site = if sites.peek().map(|s| s.0) == Some(i) { sites.next().map(|s| s.1) } else { None }; - let by_record = site.is_some() && match &item { + // Whether the reduce's values are not unit: count then drops them, and distinct keys by the whole record. + let valued = site.is_some() && match &item { Item::Op(Node::Reduce { input, .. }) => !shapes.of(input).is_some_and(|s| s.val == corgi::Shape::Unit), _ => false, }; @@ -80,20 +81,20 @@ pub(crate) fn expand_by_shape( }; if site == Some(ByShape::Count) { let Item::Op(Node::Reduce { input, reducer, limit }) = item else { unreachable!() }; - let input = if by_record { + let input = if valued { push(Node::Linear { input, ops: vec![LinearOp::Project(Lambda::new("let (k, _) = input in (k, ())".into(), &corgi::Scope::default()))] }) } else { input }; push(Node::Reduce { input, reducer, limit }); } else if site == Some(ByShape::Distinct) { let Item::Op(Node::Reduce { input, .. }) = item else { unreachable!() }; - let input = if by_record { + let input = if valued { push(Node::Linear { input, ops: vec![LinearOp::Project(Lambda::new("let (k, v) = input in ((k, v), ())".into(), &corgi::Scope::default()))] }) } else { input }; // Both paths now have unit values: either they were known to be unit, // or the map above keyed the collection by its whole record. let reducer = Reducer::Expr(unit_distinct()); let output = push(Node::Reduce { input, reducer, limit: None }); - if by_record { + if valued { push(Node::Linear { input: output, ops: vec![LinearOp::Project(Lambda::new("let (kv, ()) = input in kv".into(), &corgi::Scope::default()))] }); } } else if let Item::Op(node) = item { diff --git a/interactive/src/scope_ir.rs b/interactive/src/scope_ir.rs index bd2cad888..1910e7752 100644 --- a/interactive/src/scope_ir.rs +++ b/interactive/src/scope_ir.rs @@ -316,7 +316,7 @@ fn dump_scope_body(s: &Scope, indent: usize) { let ops: Vec<&str> = ops.iter().map(|op| match op { LinearOp::Project(_) => "project", LinearOp::Filter(_) => "filter", LinearOp::Negate => "negate", - LinearOp::LiftIter => "lift_iter", LinearOp::FlatMap(_) => "flatmap", + LinearOp::FlatMap(_) => "flatmap", LinearOp::Lift => "lift", LinearOp::JoinFunction(_) => "join_function", }).collect(); format!("{} | {}", fmt_ref(input), ops.join(" | ")) diff --git a/interactive/src/shapes.rs b/interactive/src/shapes.rs index 312574432..dceb1e857 100644 --- a/interactive/src/shapes.rs +++ b/interactive/src/shapes.rs @@ -263,12 +263,6 @@ pub(crate) fn linear_shapes(ops: &[LinearOp], key: Shape, val: Shape, level: Lev Shape::List(e) => (k, v) = row_shape(*e)?, s => return Err(format!("flatmap of shape {s}, not a list")), }, - // Extend a tuple, or pair any other value. - LinearOp::LiftIter => v = match v { - Shape::Prod(mut fs) => { fs.push(Shape::Int); Shape::Prod(fs) } - Shape::Unit => Shape::Prod(vec![Shape::Int]), - other => Shape::Prod(vec![other, Shape::Int]), - }, LinearOp::Lift => v = Shape::Prod(vec![v, time_shape(level)]), LinearOp::JoinFunction(t) => (k, v) = join_function_shape(t, &env, level)?, } diff --git a/interactive/tests/bitemporal.rs b/interactive/tests/bitemporal.rs index 99cdcda25..b3fc98fad 100644 --- a/interactive/tests/bitemporal.rs +++ b/interactive/tests/bitemporal.rs @@ -457,7 +457,7 @@ fn independent_inputs_and_time_shapes() { let b = input 1 : (int, int); s: { let seen = a | delay((_, v) -> (0, 0, (256, v call bit_length) mul)) | lift - | map((k, (_, (s, e, i))) -> (k, (s, e, i, i))); + | map((k, (_, (s, e, i))) -> (k, (s, e, i))); } output "left" = a; output "right" = b; @@ -494,7 +494,7 @@ fn independent_inputs_and_time_shapes() { assert_eq!(left.len(), 2); assert_eq!(keyed.rows(), vec![("7".into(), "1".into(), 1)]); let rows: Rows = seen.rows().into_iter().map(|(k, v, d)| (numbers(&k), numbers(&v), d)).collect(); - assert_eq!(sorted(rows), vec![(vec![7], vec![0, 0, 256, 256], 1), (vec![8], vec![0, 5, 256, 256], 1)]); + assert_eq!(sorted(rows), vec![(vec![7], vec![0, 0, 256], 1), (vec![8], vec![0, 5, 256], 1)]); } let delta = server.snapshot_as_of(worker, "split::delta", &at((0, u64::MAX))).unwrap(); if worker.index() == 0 { diff --git a/interactive/tests/explain.rs b/interactive/tests/explain.rs index 13df4f31c..e8635fbdc 100644 --- a/interactive/tests/explain.rs +++ b/interactive/tests/explain.rs @@ -644,7 +644,7 @@ const SCALARS: &str = r#" let pairs = input 0 : (int, int); let counted = pairs | filter((_, v) -> (v, 3) gt) | count; let lists = pairs | map((k, v) -> (k, (v, 2) mul)) | collect; - let flat = lists | flatmap((k, l) -> (k, l) cap_list) | map((k, x) -> (k, x)); + let flat = lists | flatmap((k, l) -> (k, l) cap_list); inner: { set low = (flat | filter((_, x) -> (x, 15) lt)) + (low | filter((k, _) -> (k, k) eq)) | min; } @@ -678,9 +678,8 @@ fn collect_explanations_sufficient_small() { // --------------------------------------------------------------------------- // Explanations of loops: `delay`, a `min` in a loop, and SCC's shape at one -// scope, with its feedback negated and not. `LinearOp::LiftIter` is synthesized -// by the rewrite (every `$host:` export is one), so explained programs are what -// exercise it. +// scope, with its feedback negated and not. The rewrite writes a `lift` and a map +// at every `$host:` export, so explained programs are what exercise `lift` in loops. // --------------------------------------------------------------------------- /// `delay` through the rewrite, at depth 1. diff --git a/interactive/tests/programs/lambda_slots.ddp b/interactive/tests/programs/lambda_slots.ddp index 915a67b2d..3915a6afe 100644 --- a/interactive/tests/programs/lambda_slots.ddp +++ b/interactive/tests/programs/lambda_slots.ddp @@ -1,4 +1,4 @@ --- The slots that take one lambda: `map` (the value as written, with no tuple around it), +-- The slots that take one lambda: `map` (a row to a row), -- `filter`, `flatmap` (complete rows, with positions explicitly zipped here) and `delay` (a row -- advances to the coordinate-wise maximum of its time and the tuple it names). A `--` comment -- may sit inside a lambda written over several lines. Declared types and the built-in `Option`