diff --git a/interactive/bench/ldbc/snb/rel.py b/interactive/bench/ldbc/snb/rel.py index b6c7abd0c..ba1e409a6 100644 --- a/interactive/bench/ldbc/snb/rel.py +++ b/interactive/bench/ldbc/snb/rel.py @@ -385,7 +385,7 @@ def keyed(source, fields, keys, keep=True): # A dedicated value-only collect avoids lists of singleton # tuples and is useful for nested IC1/IC12 results. assert len(aggs)==1 - code = f'{source_name} | map((row, ()) -> ({group_key}, ({scalar(expr(e),refs)}))) | value((_, (item)) -> item) | collect | map((key, values) -> ({product([*outs,"values"])}, ()))' + code = f'{source_name} | map((row, ()) -> ({group_key}, {scalar(expr(e),refs)})) | collect | map((key, values) -> ({product([*outs,"values"])}, ()))' self.lines.append(f'-- {name}: ({", ".join(r.fields)})\nlet {name} = {code};') return name else: raise ValueError(kind) diff --git a/interactive/examples/aoc2023/day01/part1.ddp b/interactive/examples/aoc2023/day01/part1.ddp index 642e08e63..67281209e 100644 --- a/interactive/examples/aoc2023/day01/part1.ddp +++ b/interactive/examples/aoc2023/day01/part1.ddp @@ -14,5 +14,5 @@ let calib = first | join(last, (line, (f, l)) -> (line, ((f, 10) mul, l) add)); output "result" = calib | map((_, v) -> ((), v)) | collect - | value((_, vs) -> vs fold_add) + | map((k, vs) -> (k, vs fold_add)) | arrange | inspect(part1); diff --git a/interactive/examples/aoc2023/day01/part2.ddp b/interactive/examples/aoc2023/day01/part2.ddp index 9b2e7f3aa..c639104f4 100644 --- a/interactive/examples/aoc2023/day01/part2.ddp +++ b/interactive/examples/aoc2023/day01/part2.ddp @@ -39,5 +39,5 @@ let calib = first | join(last, (line, (f, l)) -> (line, ((f, 10) mul, l) add)); output "result" = calib | map((_, v) -> ((), v)) | collect - | value((_, vs) -> vs fold_add) + | map((k, vs) -> (k, vs fold_add)) | arrange | inspect(part2); diff --git a/interactive/examples/aoc2023/day02/part1.ddp b/interactive/examples/aoc2023/day02/part1.ddp index 9c8c0ea25..a686d41db 100644 --- a/interactive/examples/aoc2023/day02/part1.ddp +++ b/interactive/examples/aoc2023/day02/part1.ddp @@ -10,5 +10,5 @@ let good = games - bad; output "result" = good | map((game, _) -> ((), game)) | collect - | value((_, gs) -> gs fold_add) + | map((k, gs) -> (k, gs fold_add)) | arrange | inspect(part1); diff --git a/interactive/examples/aoc2023/day02/part2.ddp b/interactive/examples/aoc2023/day02/part2.ddp index ed1938968..da1f16942 100644 --- a/interactive/examples/aoc2023/day02/part2.ddp +++ b/interactive/examples/aoc2023/day02/part2.ddp @@ -13,5 +13,5 @@ let power = maxg output "result" = power | map((_, p) -> ((), p)) | collect - | value((_, ps) -> ps fold_add) + | map((k, ps) -> (k, ps fold_add)) | arrange | inspect(part2); diff --git a/interactive/examples/aoc2023/day03/part1.ddp b/interactive/examples/aoc2023/day03/part1.ddp index 64a9e81c4..34462a2ca 100644 --- a/interactive/examples/aoc2023/day03/part1.ddp +++ b/interactive/examples/aoc2023/day03/part1.ddp @@ -40,5 +40,5 @@ let part_numbers = maximal | map(((r, s, _, v), _) -> ((r, s), v)) | join(active output "result" = part_numbers | map((_, v) -> ((), v)) | collect - | value((_, vs) -> vs fold_add) + | map((k, vs) -> (k, vs fold_add)) | arrange | inspect(part1); diff --git a/interactive/examples/aoc2023/day03/part2.ddp b/interactive/examples/aoc2023/day03/part2.ddp index f3dfcad62..35003de9b 100644 --- a/interactive/examples/aoc2023/day03/part2.ddp +++ b/interactive/examples/aoc2023/day03/part2.ddp @@ -45,5 +45,5 @@ output "result" = gearvals | filter((_, vs) -> (vs len, 2) eq) | map((_, vs) -> ((), ((0, vs) get, (1, vs) get) mul)) | map((_, p) -> ((), p)) | collect - | value((_, ps) -> ps fold_add) + | map((k, ps) -> (k, ps fold_add)) | arrange | inspect(part2); diff --git a/interactive/examples/aoc2023/day04/part1.ddp b/interactive/examples/aoc2023/day04/part1.ddp index 795ce7879..ac473e673 100644 --- a/interactive/examples/aoc2023/day04/part1.ddp +++ b/interactive/examples/aoc2023/day04/part1.ddp @@ -9,5 +9,5 @@ let points = dups output "result" = points | map((_, p) -> ((), p)) | collect - | value((_, ps) -> ps fold_add) + | map((k, ps) -> (k, ps fold_add)) | arrange | inspect(part1); diff --git a/interactive/src/backend/corgi.rs b/interactive/src/backend/corgi.rs index 3c2282574..2f192222a 100644 --- a/interactive/src/backend/corgi.rs +++ b/interactive/src/backend/corgi.rs @@ -67,7 +67,7 @@ impl Plan { LinearOp::Filter(cond) => ("filter", Some(compile_predicate(cond, k, v))), LinearOp::FlatMap(list) => ("flatmap", Some(compile_flatmap(list, k, v))), LinearOp::JoinFunction(lambda) => ("join_function", Some(compile_join_function(lambda, k, v, level.width()))), - LinearOp::Negate | LinearOp::LiftIter | LinearOp::Lift => ("", None), + LinearOp::Negate | LinearOp::Lift => ("", None), }; let graph = graph.map(|g| g.unwrap_or_else(|e| panic!("{what}: type error at shapes ({kshape}, {vshape}): {e}"))); Plan { shape: (kshape, vshape), graph } @@ -86,8 +86,7 @@ impl Plan { /// column + a structural explode; Negate = Rust. Every slot is columnar — there is no row-wise /// path inside the dataflow — and each op's graph was compiled when the operator was built /// (`plans`). An empty batch passes through untouched: it has no rows to compute. -/// The data<->time ops are columnar and total: LiftIter reads the iteration coordinate out of -/// `times` and appends it to `vals`; Lift reads every lane of `times` into `vals`; JoinFunction +/// The data<->time ops are columnar and total: Lift reads every lane of `times` into `vals`; JoinFunction /// explodes its list column as FlatMap does and joins each element's time lanes into `times`. /// `level` locates the iteration coordinate and fixes the number of lanes as data. fn apply_ops(mut c: CC, ops: &[LinearOp], level: Level, plans: &[Plan]) -> CC { @@ -127,25 +126,6 @@ fn apply_ops(mut c: CC, ops: &[LinearOp], level: Level, plans: &[Plan]) -> CC { } c } - // A value read OUT of each row's time. Vals gain one - // integer field; keys, times and diffs are untouched, and no slot is compiled. - // - // A tuple gains a field, and unit is the empty tuple: columnar it arrives as - // `CValue::Unit`, not an empty `Prod`, so `Unit` must become `Prod([iter])`. - // `Prod([Unit, iter])` would be a silent one-field-too-many divergence from the - // shape `shapes` infers. - LinearOp::LiftIter => { - // At the root there is no iteration - // coordinate and the value is zero. - let iters: Vec = level.own_lane().map_or_else(|| vec![0; c.times.len()], |lane| c.times.lane(lane)); - let lane = CValue::i64(iters.into_iter().map(|i| i as i64).collect()); - let vals = match c.vals { - CValue::Prod(mut fields) => { fields.push(lane); CValue::Prod(fields) } - CValue::Unit(_) => CValue::Prod(vec![lane]), - other => CValue::Prod(vec![other, lane]), - }; - CorgiContainer { keys: c.keys, vals, times: c.times, diffs: c.diffs } - } // A time as data is the root coordinates and one per scope: `level.width()` lanes, zero // where the column is narrower. Keys, times and diffs are untouched. LinearOp::Lift => { @@ -309,7 +289,7 @@ impl Backend for CorgiBackend { fn linear<'s>(c: Collection<'s, Time, CC>, ops: Vec, level: Level, (k, v): st::RowShape) -> Collection<'s, Time, CC> { // Container-level: fold the LinearOp chain over each corgi batch (no inter-op transcode). - // `level` is the scope depth (locates the iteration coordinate for LiftIter). + // `level` is the scope depth, which fixes the number of time lanes. // Every op compiles now, at the shape it will see. let steps = crate::shapes::linear_shapes(&ops, k, v, level).expect("install inferred these shapes"); let plans: Vec = ops.iter().zip(steps).map(|(op, (key, val))| Plan::new(op, key, val, level)).collect(); diff --git a/interactive/src/explain/mod.rs b/interactive/src/explain/mod.rs index 8621ff0d9..a0fc96a93 100644 --- a/interactive/src/explain/mod.rs +++ b/interactive/src/explain/mod.rs @@ -9,7 +9,7 @@ //! The clone-with-lifts is the foundation the rewrite's witness and //! forward copies are built from. `clone_into` clones an original //! program's scopes into an output scope under construction; every nested -//! scope additionally `lift_iter`s and exports each internal collection, so +//! scope additionally lifts its round into each internal collection and exports it, so //! every value-producing site in the subtree has a "host-visible" form — its //! value as `(V, chain…)`, the user's value whole and then its user-iter //! coordinates, innermost first — at the @@ -176,6 +176,11 @@ fn clone_rec(orig: &Scope, out: &mut Scope, import_map: &[Ref], path: &[usize], // and the export is the host-visible edge upward. A site of // this scope first has its value wrapped as `(V)`, so the // lifts extend it to `(V, chain…)` with `V` whole. + // + // The fold is `lift` and then a map that keeps the value's fields and the time's + // last coordinate, which is this scope's round. Times here have one root + // coordinate, the forward clone's explanation round if `forward`, and one per scope. + let round = format!("v.1.{}", child_path.len() + usize::from(forward)); let sub_idx = out.items.len(); for (a, vref) in child_visible { let vref = if a.path == child_path { @@ -185,8 +190,13 @@ fn clone_rec(orig: &Scope, out: &mut Scope, import_map: &[Ref], path: &[usize], } else { vref }; + // The value is `(V)` and one coordinate for each deeper exit. + let width = 1 + a.path.len() - child_path.len(); + let mut fields: Vec = (0..width).map(|j| format!("v.0.{j}")).collect(); + fields.push(round.clone()); + let fold = LinearOp::Project(projection(var(0), tup(fields), 2)); let lift_idx = child_out.items.len(); - child_out.items.push(Item::Op(Node::Linear { input: vref, ops: vec![LinearOp::LiftIter] })); + child_out.items.push(Item::Op(Node::Linear { input: vref, ops: vec![LinearOp::Lift, fold] })); let export_idx = child_out.exports.len(); child_out.exports.push(Export { name: format!("$host:{}", export_idx), @@ -1017,7 +1027,6 @@ impl<'a> Reverse<'a> { LinearOp::Negate => { self.push(ex, path, input, dep_this); } - LinearOp::LiftIter => panic!("explain: LiftIter in user program"), // Lift reads the whole time, and the forward clone runs one scope deeper. LinearOp::Lift => panic!("explain: lift is not supported"), LinearOp::JoinFunction(lambda) => { @@ -1127,7 +1136,7 @@ mod tests { let mut out = Scope { imports: p.root.imports.clone(), ..Scope::default() }; let import_map: Vec = (0..out.imports.len()).map(Ref::Import).collect(); clone_into(&p.root, &mut out, &import_map); - // outer's clone: every $host: export is a LiftIter Linear; and the + // outer's clone: every $host: export is a lift and a map; and the // ones re-exporting fwd/bwd internals chain TWO lifts (one per level): // the value behind the lift is itself a ChildExport of a lift. let Item::Sub(outer) = out.items.iter().find(|i| matches!(i, Item::Sub(_))).unwrap() else { unreachable!() }; @@ -1135,8 +1144,7 @@ mod tests { for e in outer.exports.iter().filter(|e| e.name.starts_with("$host:")) { let Ref::Local(li) = &e.value else { panic!("$host export should be a fresh lift") }; let Item::Op(Node::Linear { input, ops }) = &outer.items[*li] else { panic!("expected a lift") }; - assert_eq!(ops.as_slice().len(), 1); - assert!(matches!(ops[0], LinearOp::LiftIter)); + assert!(matches!(ops.as_slice(), [LinearOp::Lift, LinearOp::Project(_)])); if matches!(input, Ref::ChildExport(..)) { depth2_chains += 1; } } assert!(depth2_chains > 0, "fwd/bwd internals re-lift at outer's exit"); diff --git a/interactive/src/ir.rs b/interactive/src/ir.rs index efa663562..e6a16d30e 100644 --- a/interactive/src/ir.rs +++ b/interactive/src/ir.rs @@ -13,7 +13,7 @@ pub use crate::lambda::Lambda; /// table (`parse::pipe`) must agree, which its tests check. pub(crate) const KEYWORDS: &[&str] = &[ "let", "var", "set", "output", "type", "enum", "input", "import", "map", "join", "min", "distinct", "count", - "collect", "flatmap", "arrange", "negate", "filter", "inspect", "lift_iter", + "collect", "flatmap", "arrange", "negate", "filter", "inspect", ]; #[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] @@ -35,10 +35,6 @@ pub enum LinearOp { Filter(Lambda), /// Negate the diff. Negate, - /// Append the current user-iter coord (at the row's scope depth) to - /// the value. Time itself is unchanged. See `Expr::LiftIter` for the - /// discipline restriction. - LiftIter, /// Each row becomes the lambda's list of complete `(key, val)` rows. FlatMap(Lambda), /// Pair each value with its whole time: `(key, val)` becomes `(key, (val, time))`. diff --git a/interactive/src/level.rs b/interactive/src/level.rs index 820f8f517..4ea413efa 100644 --- a/interactive/src/level.rs +++ b/interactive/src/level.rs @@ -9,7 +9,6 @@ pub(crate) struct Level { impl Level { pub(crate) fn root(root: usize) -> Self { Self { root, depth: 0 } } pub(crate) fn width(self) -> usize { self.root + self.depth } - pub(crate) fn own_lane(self) -> Option { (self.depth > 0).then(|| self.width() - 1) } pub(crate) fn child(self) -> Self { Self { depth: self.depth + 1, ..self } } pub(crate) fn parent(self) -> Self { Self { depth: self.depth.saturating_sub(1), ..self } } } diff --git a/interactive/src/lower.rs b/interactive/src/lower.rs index f5dd87fbd..fbbb73f62 100644 --- a/interactive/src/lower.rs +++ b/interactive/src/lower.rs @@ -188,7 +188,7 @@ fn resolve_expr(expr: &mut Expr, here: &[String], item: Option<&str>, *expr = Expr::ResolvedPath { anchor: key.anchor, target: key.target, field: key.field }; } Expr::TypedSource(e, _, _) | Expr::Map(e, _) | Expr::Reduce(e, _) | Expr::Filter(e, _) - | Expr::Negate(e) | Expr::LiftIter(e) | Expr::FlatMap(e, _) + | Expr::Negate(e) | Expr::FlatMap(e, _) | Expr::Lift(e) | Expr::JoinFunction(e, _) | Expr::Inspect(e, _) | Expr::Arrange(e) => resolve_expr(e, here, item, scopes, plan), Expr::Join(l, r, _) | Expr::HalfJoin(l, r, _, _) => { @@ -247,7 +247,7 @@ fn expr_free_names(expr: &Expr, out: &mut BTreeSet) { Expr::ResolvedPath { .. } => {}, Expr::Path(..) => unreachable!("paths must be resolved before lowering"), Expr::TypedSource(e, _, _) | Expr::Map(e, _) | Expr::Reduce(e, _) | Expr::Filter(e, _) - | Expr::Negate(e) | Expr::LiftIter(e) + | Expr::Negate(e) | Expr::FlatMap(e, _) | Expr::Lift(e) | Expr::JoinFunction(e, _) | Expr::Inspect(e, _) | Expr::Arrange(e) => expr_free_names(e, out), Expr::Join(l, r, _) | Expr::HalfJoin(l, r, _, _) => { expr_free_names(l, out); expr_free_names(r, out); }, @@ -310,17 +310,17 @@ pub fn lower_tree(mut stmts: Vec) -> st::Program { let mut plan = PathPlan::default(); resolve_body(&mut stmts, &[], &scopes, &mut plan); let empty = HashMap::new(); - let mut distinct = HashMap::new(); - let mut program = st::Program { root: lower_scope_tree(&[], &stmts, &empty, &[], &plan, true, &mut distinct) }; + let mut by_shape = HashMap::new(); + let mut program = st::Program { root: lower_scope_tree(&[], &stmts, &empty, &[], &plan, true, &mut by_shape) }; let (shapes, _) = crate::shapes::infer(&program, &|_| None, 1); - crate::reducers::expand_distinct(&mut program.root, &shapes, &distinct, &mut Vec::new()); + crate::reducers::expand_by_shape(&mut program.root, &shapes, &by_shape, &mut Vec::new()); program } /// Expr-lowering state for one scope. Resolves names to scope-local `Ref`s and /// accumulates this scope's nodes and (input/trace) imports. struct ScopeLower { - distinct: Vec, + by_shape: Vec<(usize, crate::reducers::ByShape)>, is_root: bool, items: Vec, /// Each item's statement name (see `st::Scope::labels`). @@ -386,7 +386,6 @@ impl ScopeLower { Expr::Filter(e, c) => { let r = self.lower_expr(e); self.push(st::Node::Linear { input: r, ops: vec![LinearOp::Filter(c.clone())] }) }, Expr::Negate(e) => { let r = self.lower_expr(e); self.push(st::Node::Linear { input: r, ops: vec![LinearOp::Negate] }) }, Expr::FlatMap(e, t) => { let r = self.lower_expr(e); self.push(st::Node::Linear { input: r, ops: vec![LinearOp::FlatMap(t.clone())] }) }, - Expr::LiftIter(e) => { let r = self.lower_expr(e); self.push(st::Node::Linear { input: r, ops: vec![LinearOp::LiftIter] }) }, Expr::Lift(e) => { let r = self.lower_expr(e); self.push(st::Node::Linear { input: r, ops: vec![LinearOp::Lift] }) }, Expr::JoinFunction(e, t) => { let r = self.lower_expr(e); self.push(st::Node::Linear { input: r, ops: vec![LinearOp::JoinFunction(t.clone())] }) }, Expr::Arrange(e) => { let r = self.lower_expr(e); self.push(st::Node::Arrange(r)) }, @@ -404,11 +403,13 @@ impl ScopeLower { }, Expr::Reduce(e, red)=> { let r = self.lower_expr(e); - let a = if matches!(red, Reducer::Distinct) { + let a = match red { // Once shapes are known, lower distinct by key or by whole record. - self.distinct.push(self.items.len()); - r - } else { self.arrange(r) }; + Reducer::Distinct => { self.by_shape.push((self.items.len(), crate::reducers::ByShape::Distinct)); r } + // Once shapes are known, project count's values away if they are not unit. + Reducer::Count => { self.by_shape.push((self.items.len(), crate::reducers::ByShape::Count)); r } + _ => self.arrange(r), + }; self.push(st::Node::Reduce { input: a, reducer: crate::ir::Reducer::Expr(red.lambda()), limit: red.limit() }) }, Expr::Inspect(e, l) => { let r = self.lower_expr(e); self.push(st::Node::Inspect { input: r, label: l.clone() }) }, @@ -424,11 +425,11 @@ fn lower_scope_tree( exports_wanted: &[PathKey], plan: &PathPlan, is_root: bool, - distinct: &mut HashMap, Vec>, + by_shape: &mut HashMap, Vec<(usize, crate::reducers::ByShape)>>, ) -> st::Scope { let scope_name = scope_label(address); let mut s = ScopeLower { - distinct: Vec::new(), + by_shape: Vec::new(), is_root, items: Vec::new(), labels: Vec::new(), @@ -560,7 +561,7 @@ fn lower_scope_tree( let mut child_address = address.to_vec(); child_address.push(name.clone()); let wanted: Vec = plan.exports.get(&child_address).into_iter().flatten().cloned().collect(); - let child = lower_scope_tree(&child_address, b, &s.env, &wanted, plan, false, distinct); + let child = lower_scope_tree(&child_address, b, &s.env, &wanted, plan, false, by_shape); let slots = export_slots(&wanted, &child_address); let cidx = s.items.len(); s.items.push(st::Item::Sub(child)); @@ -611,7 +612,7 @@ fn lower_scope_tree( } } - distinct.insert(address.to_vec(), s.distinct); + by_shape.insert(address.to_vec(), s.by_shape); st::Scope { name: scope_name.to_string(), imports: s.imports, vars, items: s.items, labels: s.labels, names, binds, exports } } diff --git a/interactive/src/parse/mod.rs b/interactive/src/parse/mod.rs index e1ed9b063..5d0d2dc4e 100644 --- a/interactive/src/parse/mod.rs +++ b/interactive/src/parse/mod.rs @@ -60,14 +60,6 @@ pub enum Expr { Reduce(Box, Reducer), Filter(Box, Lambda), Negate(Box), - /// Append the current user-iter coord (at the operator's scope depth) - /// to each row's value. Time itself is unchanged. - /// - /// Discipline, which nothing checks: the result of `LiftIter` should not be referenced inside the same - /// scope it appears in — only from an enclosing scope, after the - /// implicit leave. This preserves the "loop body is a time-invariant - /// function" property; in-scope use risks defeating the fixpoint. - LiftIter(Box), /// Replace each row with the lambda's list of complete `(key, val)` rows. FlatMap(Box, Lambda), /// Pair each row's value with its whole time. See `LinearOp::Lift`. diff --git a/interactive/src/parse/pipe.rs b/interactive/src/parse/pipe.rs index 8798cc662..5a390694d 100644 --- a/interactive/src/parse/pipe.rs +++ b/interactive/src/parse/pipe.rs @@ -32,7 +32,6 @@ //! Operators chain with `|`; a slot is written `(pat -> body)`: //! //! - `| map((k, v) -> (k', v'))` — reshape each row to the `(key, val)` the lambda gives. -//! - `| value((k, v) -> v')` — replace the value, keeping the key. //! - `| join(other)` — equijoin on the key, yielding `(k, (v, v1))`. //! - `| join(other, f)` — project the joined row, as in `| join(other) | map(f)`. //! - `| min` / `| distinct` / `| count` / `| collect` — reduce; `distinct` keeps each `(key, val)` @@ -47,8 +46,7 @@ //! - `| filter((k, v) -> mask)` — keep rows where the lambda gives a nonzero `Int`. //! - `| flatmap((k, v) -> list)` — any number of complete `(key, val)` rows. //! For positions, write `((0, l len) range, l) zip`; keys are chosen by the lambda. -//! - `| negate` · `| arrange` · `| inspect(label)` · -//! `| lift_iter`. +//! - `| negate` · `| arrange` · `| inspect(label)`. //! - `| lift` — pair each value with its whole time, `(key, (val, time))`, the time a tuple //! of the epoch and every scope coordinate. //! - `| join_function((k, v) -> list)` — replace each row by the lambda's list of @@ -151,7 +149,7 @@ mod fragment; #[derive(Debug, Clone, PartialEq)] enum Token { Let, Var, Set, Output, Type, Enum, - Input, Import, Map, Join, Min, Distinct, Count, Collect, Arrange, Negate, Filter, LiftIter, FlatMap, Inspect, + Input, Import, Map, Join, Min, Distinct, Count, Collect, Arrange, Negate, Filter, FlatMap, Inspect, Ident(String), Int(i64), Str(String), /// A slot written as a corgi lambda, `pat -> body`, kept as corgi source. Lambda { pat: String, body: String }, @@ -345,7 +343,6 @@ fn keyword(ident: &str) -> Option { "flatmap" => Token::FlatMap, "arrange" => Token::Arrange, "negate" => Token::Negate, "filter" => Token::Filter, "inspect" => Token::Inspect, - "lift_iter" => Token::LiftIter, _ => return None, }) } @@ -672,11 +669,6 @@ impl Parser { fn parse_pipe_op(&mut self, lhs: Expr) -> Expr { match self.peek().clone() { Token::Map => { self.next(); Expr::Map(Box::new(lhs), self.slot()) }, - Token::Ident(name) if name == "value" => { - self.next(); - let val = self.slot(); - Expr::Map(Box::new(lhs), Lambda::compose("(k, v)", &[(&val, "(k, v)".into())], |r| format!("(k, {})", r[0]))) - }, Token::Join => { self.next(); self.expect(&Token::LParen); @@ -726,7 +718,6 @@ impl Parser { Token::Arrange => { self.next(); Expr::Arrange(Box::new(lhs)) }, Token::Negate => { self.next(); Expr::Negate(Box::new(lhs)) }, Token::Filter => { self.next(); Expr::Filter(Box::new(lhs), self.slot()) }, - Token::LiftIter => { self.next(); Expr::LiftIter(Box::new(lhs)) }, Token::Ident(name) if name == "lift" => { self.next(); Expr::Lift(Box::new(lhs)) }, Token::Ident(name) if name == "half_join" => { self.next(); self.expect(&Token::LParen); diff --git a/interactive/src/passes/as_of.rs b/interactive/src/passes/as_of.rs index a1616e369..f933536e6 100644 --- a/interactive/src/passes/as_of.rs +++ b/interactive/src/passes/as_of.rs @@ -31,7 +31,7 @@ pub(super) fn drop_dead_as_of(prog: &mut Program, as_of: &Time, root: usize) -> /// half join that makes its updates at the as-of time dead. fn keeps_times(op: &LinearOp) -> bool { match op { - LinearOp::Project(_) | LinearOp::Filter(_) | LinearOp::Negate | LinearOp::LiftIter | LinearOp::FlatMap(_) | LinearOp::Lift => true, + LinearOp::Project(_) | LinearOp::Filter(_) | LinearOp::Negate | LinearOp::FlatMap(_) | LinearOp::Lift => true, LinearOp::JoinFunction(_) => false, } } @@ -74,7 +74,7 @@ fn drop_in_scope(s: &mut Scope, level: Level, as_of: &Time) -> usize { if !ops[at + 1..].iter().all(keeps_times) || uses[i].is_empty() { continue; } let dead: Vec = uses[i].iter().copied().filter(|u| dead_on(*u, &covered)).collect(); // The edge filter reads the time from the row value written by `lift`. Filter and Negate - // leave that value intact. Project, FlatMap, LiftIter and other tails may move or drop it; + // leave that value intact. Project, FlatMap and other tails may move or drop it; // handling them needs tracking time through lambdas or splitting the node and duplicating // its tail on dead edges, adding work every tick and losing the "never worse" property. if dead.len() == uses[i].len() { diff --git a/interactive/src/passes/projection.rs b/interactive/src/passes/projection.rs index a6e43b650..6046dea2c 100644 --- a/interactive/src/passes/projection.rs +++ b/interactive/src/passes/projection.rs @@ -206,13 +206,6 @@ impl Builder { } } - fn lift_iter(&mut self, from: &Site, input: &Site) { - self.key_identity(from, input); - for (&out, &src) in from.val.iter().zip(&input.val) { - self.edge(out, src); - } - } - fn lift(&mut self, from: &Site, input: &Site) { self.key_identity(from, input); if let Some(&value) = from.val.first() { @@ -502,7 +495,6 @@ fn build_scope( builder.scalar_lambda(&after, &[&before], lambda); } LinearOp::Negate => builder.identity(&after, &before), - LinearOp::LiftIter => builder.lift_iter(&after, &before), LinearOp::Lift => builder.lift(&after, &before), LinearOp::FlatMap(lambda) => { builder.presence(&after, &before); @@ -704,12 +696,19 @@ fn mask_half_join(original: &Lambda, output: &Site, live: &[bool]) -> Option u (Item::Op(Node::HalfJoin { projection, .. }), PlannedItem::Op { output, .. }) => { if let Some(new) = mask_half_join(projection, output, live) { masked += output.key.iter().filter(|&&id| !live[id]).count(); - masked += usize::from(!live[output.val[0]]); + let unit = matches!(&output.shape.1, Shape::Prod(parts) if parts[0] == Shape::Unit); + masked += usize::from(!unit && !live[output.val[0]]); *projection = new; } } @@ -1050,7 +1050,8 @@ mod tests { let distinct = format!("{source}output \"out\" = x | distinct;\n"); let mut count = lower::lower_tree(parse::pipe::parse(&count)); let mut distinct = lower::lower_tree(parse::pipe::parse(&distinct)); - assert_eq!(pushdown(&mut count, &|_| None).unwrap(), 0); + // Count projects its values away as it is lowered, so the map's value goes unread. + assert_eq!(pushdown(&mut count, &|_| None).unwrap(), 2); assert_eq!(pushdown(&mut distinct, &|_| None).unwrap(), 0); } @@ -1167,7 +1168,7 @@ mod tests { | half_join(b, 0, (k, x, yz) -> (k, x)) | count;"; let mut program = lower::lower_tree(parse::pipe::parse(text)); crate::passes::optimize(&mut program); - assert_eq!(pushdown(&mut program, &|_| None).unwrap(), 2); + assert_eq!(pushdown(&mut program, &|_| None).unwrap(), 4); let (_, check) = shapes::infer(&program, &|_| None, 1); assert!(check.conflicts.is_empty(), "{:?}", check.conflicts); } @@ -1184,7 +1185,7 @@ mod tests { let mut original = lower::lower_tree(parse::pipe::parse(text)); crate::passes::optimize(&mut original); let mut projected = original.clone(); - assert_eq!(pushdown(&mut projected, &|_| None).unwrap(), 2); + assert_eq!(pushdown(&mut projected, &|_| None).unwrap(), 4); timely::execute_directly(move |worker| { let mut server = Server::new(); server.install(worker, "before", &original).unwrap(); diff --git a/interactive/src/plan.rs b/interactive/src/plan.rs index d9de5be0f..84fd78d90 100644 --- a/interactive/src/plan.rs +++ b/interactive/src/plan.rs @@ -393,7 +393,6 @@ fn linear(op: &LinearOp) -> String { LinearOp::Project(l) => format!("map({})", source_lambda(l)), LinearOp::Filter(l) => format!("filter({})", source_lambda(l)), LinearOp::Negate => "negate".into(), - LinearOp::LiftIter => "lift_iter".into(), LinearOp::FlatMap(l) => format!("flatmap({})", source_lambda(l)), LinearOp::Lift => "lift".into(), LinearOp::JoinFunction(l) => format!("join_function({})", source_lambda(l)), @@ -431,7 +430,7 @@ fn printed_node(node: &Node, names: &Names, shapes: &ScopeShapes, depth: usize, let name = match op { LinearOp::Project(_) => "map", LinearOp::Filter(_) => "filter", LinearOp::FlatMap(_) => "flatmap", LinearOp::JoinFunction(_) => "join_function", - LinearOp::Negate => "negate", LinearOp::Lift => "lift", LinearOp::LiftIter => "lift_iter", + LinearOp::Negate => "negate", LinearOp::Lift => "lift", }; printed.push(match op { LinearOp::Project(l) | LinearOp::Filter(l) | LinearOp::FlatMap(l) | LinearOp::JoinFunction(l) => diff --git a/interactive/src/plan/compare.rs b/interactive/src/plan/compare.rs index fe7d3f57a..9d0da83c1 100644 --- a/interactive/src/plan/compare.rs +++ b/interactive/src/plan/compare.rs @@ -80,7 +80,7 @@ fn linear(a: &LinearOp, b: &LinearOp, input: Option<&corgi::Shape>) -> bool { (LinearOp::Project(a), LinearOp::Project(b)) | (LinearOp::Filter(a), LinearOp::Filter(b)) | (LinearOp::FlatMap(a), LinearOp::FlatMap(b)) | (LinearOp::JoinFunction(a), LinearOp::JoinFunction(b)) => lambda(a, b, input), - (LinearOp::Negate, LinearOp::Negate) | (LinearOp::Lift, LinearOp::Lift) | (LinearOp::LiftIter, LinearOp::LiftIter) => true, + (LinearOp::Negate, LinearOp::Negate) | (LinearOp::Lift, LinearOp::Lift) => true, _ => false, } } diff --git a/interactive/src/plan/ddp.rs b/interactive/src/plan/ddp.rs index 3c3f31732..5d96e1637 100644 --- a/interactive/src/plan/ddp.rs +++ b/interactive/src/plan/ddp.rs @@ -399,7 +399,7 @@ impl Printer<'_> { LinearOp::Project(l) => ("map", Some(l)), LinearOp::Filter(l) => ("filter", Some(l)), LinearOp::FlatMap(l) => ("flatmap", Some(l)), LinearOp::JoinFunction(l) => ("join_function", Some(l)), LinearOp::Negate => ("negate", None), - LinearOp::Lift => ("lift", None), LinearOp::LiftIter => ("lift_iter", None), + LinearOp::Lift => ("lift", None), }; expr += &format!(" | {word}"); if let Some(l) = lambda { expr += &format!("({})", self.lambda(l, shape)?); } @@ -614,7 +614,7 @@ mod tests { nested: { let both = (a + b) + (a | negate); } let x = nested::both + (b | map((k, v) -> (v, k))); output "all" = x | map((k, v) -> (v, k)); - output "other" = a | value((_, v) -> (v, 1) add); + output "other" = a | map((k, v) -> (k, (v, 1) add)); "#; let before = plan(source); let parsed = before.emit_ddp_at(DdpStage::Parsed).unwrap(); diff --git a/interactive/src/plan/ddp/parsed.rs b/interactive/src/plan/ddp/parsed.rs index 8669a8c7c..87b8a7999 100644 --- a/interactive/src/plan/ddp/parsed.rs +++ b/interactive/src/plan/ddp/parsed.rs @@ -81,12 +81,15 @@ impl Printer<'_> { // Parsed syntax has no inferred shapes. Key by the whole record, // which also preserves unit-valued distinct's meaning. format!("{input} | map((k, v) -> ((k, v), ())) | reduce({lambda}) | map((kv, ()) -> kv)") + } else if matches!(reducer, crate::parse::Reducer::Count) { + // Parsed syntax has no inferred shapes. Count projects its values away. + format!("{input} | map((k, _) -> (k, ())) | reduce({lambda})") } else { format!("{input} | reduce({}{lambda})", super::limit_text(reducer.limit())) } } - Expr::Negate(input) | Expr::Lift(input) | Expr::LiftIter(input) | Expr::Arrange(input) => { - let word = match expr { Expr::Negate(_) => "negate", Expr::Lift(_) => "lift", Expr::LiftIter(_) => "lift_iter", _ => "arrange" }; + Expr::Negate(input) | Expr::Lift(input) | Expr::Arrange(input) => { + let word = match expr { Expr::Negate(_) => "negate", Expr::Lift(_) => "lift", _ => "arrange" }; format!("{} | {word}", self.expression(input)?) } Expr::Inspect(input, label) => format!("{} | inspect({label})", self.expression(input)?), diff --git a/interactive/src/reducers.rs b/interactive/src/reducers.rs index 0f3680b47..babd6cbad 100644 --- a/interactive/src/reducers.rs +++ b/interactive/src/reducers.rs @@ -9,13 +9,23 @@ pub(crate) fn unit_distinct() -> Lambda { Lambda::new("let (_, vals) = input in vals map (((), _) -> ((), 1))".into(), &corgi::Scope::default()) } -/// Finish lowering distinct once the tree's shapes are known. A unit-valued input -/// is already keyed by its whole record. Otherwise write the record key and its -/// inverse into the IR, where arrangements, projections and readers can see them. -pub(crate) fn expand_distinct( +/// A reduce sugar whose lowering waits for its input's shape. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ByShape { + /// Keyed by the whole record, so that a key's values are not read together. + Distinct, + /// Values projected away, so that a key's values are not read together. + Count, +} + +/// Finish lowering distinct and count once the tree's shapes are known. A unit-valued input +/// needs nothing. Otherwise distinct writes the record key and its inverse into the IR, and +/// count writes the projection to a unit value, where arrangements, projections and readers +/// can see them. +pub(crate) fn expand_by_shape( scope: &mut crate::scope_ir::Scope, shapes: &crate::shapes::ScopeShapes, - pending: &std::collections::HashMap, Vec>, + pending: &std::collections::HashMap, Vec<(usize, ByShape)>>, path: &mut Vec, ) { use crate::scope_ir::{Item, Node, Ref, Source}; @@ -29,9 +39,9 @@ pub(crate) fn expand_distinct( if let Ref::Local(i) | Ref::ChildExport(i, _) = r { *i = remap[*i]; } }; for (i, mut item) in std::mem::take(&mut scope.items).into_iter().enumerate() { - let distinct = sites.peek() == Some(&i); - if distinct { sites.next(); } - let by_record = distinct && match &item { + let site = if sites.peek().map(|s| s.0) == Some(i) { sites.next().map(|s| s.1) } else { None }; + // Whether the reduce's values are not unit: count then drops them, and distinct keys by the whole record. + let valued = site.is_some() && match &item { Item::Op(Node::Reduce { input, .. }) => !shapes.of(input).is_some_and(|s| s.val == corgi::Shape::Unit), _ => false, }; @@ -40,7 +50,7 @@ pub(crate) fn expand_distinct( Item::Sub(child) => { let crate::shapes::ItemShapes::Sub(shapes) = &shapes.items[i] else { unreachable!() }; path.push(child.name.clone()); - expand_distinct(child, shapes, pending, path); + expand_by_shape(child, shapes, pending, path); path.pop(); for import in &mut child.imports { if let Source::Parent(r) = &mut import.from { remap_ref(r, &remap); } @@ -69,16 +79,22 @@ pub(crate) fn expand_distinct( labels.push(scope.labels[i].clone()); r }; - if distinct { + if site == Some(ByShape::Count) { + let Item::Op(Node::Reduce { input, reducer, limit }) = item else { unreachable!() }; + let input = if valued { + push(Node::Linear { input, ops: vec![LinearOp::Project(Lambda::new("let (k, _) = input in (k, ())".into(), &corgi::Scope::default()))] }) + } else { input }; + push(Node::Reduce { input, reducer, limit }); + } else if site == Some(ByShape::Distinct) { let Item::Op(Node::Reduce { input, .. }) = item else { unreachable!() }; - let input = if by_record { + let input = if valued { push(Node::Linear { input, ops: vec![LinearOp::Project(Lambda::new("let (k, v) = input in ((k, v), ())".into(), &corgi::Scope::default()))] }) } else { input }; // Both paths now have unit values: either they were known to be unit, // or the map above keyed the collection by its whole record. let reducer = Reducer::Expr(unit_distinct()); let output = push(Node::Reduce { input, reducer, limit: None }); - if by_record { + if valued { push(Node::Linear { input: output, ops: vec![LinearOp::Project(Lambda::new("let (kv, ()) = input in kv".into(), &corgi::Scope::default()))] }); } } else if let Item::Op(node) = item { diff --git a/interactive/src/scope_ir.rs b/interactive/src/scope_ir.rs index bd2cad888..1910e7752 100644 --- a/interactive/src/scope_ir.rs +++ b/interactive/src/scope_ir.rs @@ -316,7 +316,7 @@ fn dump_scope_body(s: &Scope, indent: usize) { let ops: Vec<&str> = ops.iter().map(|op| match op { LinearOp::Project(_) => "project", LinearOp::Filter(_) => "filter", LinearOp::Negate => "negate", - LinearOp::LiftIter => "lift_iter", LinearOp::FlatMap(_) => "flatmap", + LinearOp::FlatMap(_) => "flatmap", LinearOp::Lift => "lift", LinearOp::JoinFunction(_) => "join_function", }).collect(); format!("{} | {}", fmt_ref(input), ops.join(" | ")) diff --git a/interactive/src/shapes.rs b/interactive/src/shapes.rs index 312574432..dceb1e857 100644 --- a/interactive/src/shapes.rs +++ b/interactive/src/shapes.rs @@ -263,12 +263,6 @@ pub(crate) fn linear_shapes(ops: &[LinearOp], key: Shape, val: Shape, level: Lev Shape::List(e) => (k, v) = row_shape(*e)?, s => return Err(format!("flatmap of shape {s}, not a list")), }, - // Extend a tuple, or pair any other value. - LinearOp::LiftIter => v = match v { - Shape::Prod(mut fs) => { fs.push(Shape::Int); Shape::Prod(fs) } - Shape::Unit => Shape::Prod(vec![Shape::Int]), - other => Shape::Prod(vec![other, Shape::Int]), - }, LinearOp::Lift => v = Shape::Prod(vec![v, time_shape(level)]), LinearOp::JoinFunction(t) => (k, v) = join_function_shape(t, &env, level)?, } diff --git a/interactive/tests/bitemporal.rs b/interactive/tests/bitemporal.rs index fc9260cb0..b3fc98fad 100644 --- a/interactive/tests/bitemporal.rs +++ b/interactive/tests/bitemporal.rs @@ -456,8 +456,8 @@ fn independent_inputs_and_time_shapes() { let a = input 0 : (int, int); let b = input 1 : (int, int); s: { - let seen = a | delay((_, v) -> (0, 0, (256, v call bit_length) mul)) | lift_iter | lift - | map((k, ((v, iter), (s, e, i))) -> (k, (s, e, i, iter))); + let seen = a | delay((_, v) -> (0, 0, (256, v call bit_length) mul)) | lift + | map((k, (_, (s, e, i))) -> (k, (s, e, i))); } output "left" = a; output "right" = b; @@ -494,7 +494,7 @@ fn independent_inputs_and_time_shapes() { assert_eq!(left.len(), 2); assert_eq!(keyed.rows(), vec![("7".into(), "1".into(), 1)]); let rows: Rows = seen.rows().into_iter().map(|(k, v, d)| (numbers(&k), numbers(&v), d)).collect(); - assert_eq!(sorted(rows), vec![(vec![7], vec![0, 0, 256, 256], 1), (vec![8], vec![0, 5, 256, 256], 1)]); + assert_eq!(sorted(rows), vec![(vec![7], vec![0, 0, 256], 1), (vec![8], vec![0, 5, 256], 1)]); } let delta = server.snapshot_as_of(worker, "split::delta", &at((0, u64::MAX))).unwrap(); if worker.index() == 0 { diff --git a/interactive/tests/difftest.rs b/interactive/tests/difftest.rs index 2ebc2d1dc..55816973a 100644 --- a/interactive/tests/difftest.rs +++ b/interactive/tests/difftest.rs @@ -3,7 +3,7 @@ //! //! **What a seed generates.** One case, the same every time for the same seed: a program over up //! to six declared inputs (int pairs, `(int, int)`, a list key, a declared sum, floats), with a -//! few `let`s of `map`, `value`, `filter`, `flatmap`, `join`, `negate`, concat and difference, +//! few `let`s of `map`, `filter`, `flatmap`, `join`, `negate`, concat and difference, //! `arrange`, the reductions (`distinct`, `count`, `min`, `collect`, `reduce` by a lambda), //! `delay` and `join_function`, and small iterative scopes (reachability or label propagation, //! sometimes nested); and its feeds, 3 to 30 epochs of inserts and retractions over small values, @@ -465,7 +465,7 @@ impl Gen { let same_shape = |text: String| Op::Plain { text, same_shape: true }; let roll = self.rng.below(100); let op = match roll { - 0..=29 => { + 0..=34 => { let (k2, v2) = (self.shape(&env, true, false), self.shape(&env, false, false)); let body = format!("({}, {})", self.expr(&k2, &env, false), self.expr(&v2, &env, false)); let op = Op::Plain { text: format!("map({input} -> {body})"), same_shape: k2 == key && v2 == val }; @@ -473,13 +473,6 @@ impl Gen { val = v2; op } - 30..=34 => { - let v2 = self.shape(&env, false, false); - let body = self.expr(&v2, &env, false); - let op = Op::Plain { text: format!("value({input} -> {body})"), same_shape: v2 == val }; - val = v2; - op - } 35..=49 => same_shape(format!("filter({input} -> {})", self.mask(&env, 2))), 50..=54 => { let elem = if self.rng.chance(70) { Sh::Int } else { self.shape(&env, false, false) }; diff --git a/interactive/tests/explain.rs b/interactive/tests/explain.rs index 570b9450a..e8635fbdc 100644 --- a/interactive/tests/explain.rs +++ b/interactive/tests/explain.rs @@ -643,8 +643,8 @@ fn lambda_explanations_sufficient_small() { const SCALARS: &str = r#" let pairs = input 0 : (int, int); let counted = pairs | filter((_, v) -> (v, 3) gt) | count; - let lists = pairs | value((_, v) -> (v, 2) mul) | collect; - let flat = lists | flatmap((k, l) -> (k, l) cap_list) | value((_, x) -> x); + let lists = pairs | map((k, v) -> (k, (v, 2) mul)) | collect; + let flat = lists | flatmap((k, l) -> (k, l) cap_list); inner: { set low = (flat | filter((_, x) -> (x, 15) lt)) + (low | filter((k, _) -> (k, k) eq)) | min; } @@ -678,9 +678,8 @@ fn collect_explanations_sufficient_small() { // --------------------------------------------------------------------------- // Explanations of loops: `delay`, a `min` in a loop, and SCC's shape at one -// scope, with its feedback negated and not. `LinearOp::LiftIter` is synthesized -// by the rewrite (every `$host:` export is one), so explained programs are what -// exercise it. +// scope, with its feedback negated and not. The rewrite writes a `lift` and a map +// at every `$host:` export, so explained programs are what exercise `lift` in loops. // --------------------------------------------------------------------------- /// `delay` through the rewrite, at depth 1. diff --git a/interactive/tests/plan_pairs/expected/arrange_collapse.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/arrange_collapse.parsed--lowered.diff index 702f78b13..b49d56ce5 100644 --- a/interactive/tests/plan_pairs/expected/arrange_collapse.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/arrange_collapse.parsed--lowered.diff @@ -4,7 +4,7 @@ +-- Time lanes: epoch. let rows = input 0 : (int, ()); -let arranged = rows | arrange | arrange; --output "counts" = arranged | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter) | arrange; +-output "counts" = arranged | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter) | arrange; +let rows_idx = rows | arrange; +let arranged = rows_idx | arrange; +let counts_reduce = arranged | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); diff --git a/interactive/tests/plan_pairs/expected/arrange_collapse.parsed.ddp b/interactive/tests/plan_pairs/expected/arrange_collapse.parsed.ddp index 9d3592976..a9980d56b 100644 --- a/interactive/tests/plan_pairs/expected/arrange_collapse.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/arrange_collapse.parsed.ddp @@ -1,3 +1,3 @@ let rows = input 0 : (int, ()); let arranged = rows | arrange | arrange; -output "counts" = arranged | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter) | arrange; +output "counts" = arranged | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter) | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.parsed--lowered.diff index 88cf06531..caa911a45 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.parsed--lowered.diff @@ -17,8 +17,8 @@ -let hop2 = hop1 | join(friend, (_, (req, (g, _))) -> (g, req)); -let circle = hop1 + hop2 | filter((node, (_, who, _, _)) -> (node, who) ne) | join(person, (node, ((rid, _, x, y), (_, _, _, _, _, _, _, city, _, _, _))) -> (city, (rid, node, x, y))) | join(place, (_, ((rid, node, x, y), (_, _, home))) -> ((rid, node), (x, y, home))) | filter((_, (x, y, home)) -> ((home, x) ne, (home, y) ne) all) | map((key, _) -> (key, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); -let seen = visits | join(circle, (key, ((ix, iy), ())) -> (key, (ix, iy))); --let in_x = seen | filter((_, (ix, _)) -> (ix, 1) eq) | map((key, _) -> (key, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); --let in_y = seen | filter((_, (_, iy)) -> (iy, 1) eq) | map((key, _) -> (key, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +-let in_x = seen | filter((_, (ix, _)) -> (ix, 1) eq) | map((key, _) -> (key, ())) | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +-let in_y = seen | filter((_, (_, iy)) -> (iy, 1) eq) | map((key, _) -> (key, ())) | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +let asked_map = request | map(((rid, who, x, y, start, days), ()) -> (x, (rid, who, y, start, (start, (days, 86400000) mul) add))); +let asked_idx = asked_map | arrange; +let by_name_idx = by_name | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.parsed.ddp index 482619a57..31a8259b4 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.parsed.ddp @@ -12,8 +12,8 @@ let hop1 = asked | map((rid, (who, x, y, _, _)) -> (who, (rid, who, x, y))) | jo let hop2 = hop1 | join(friend, (_, (req, (g, _))) -> (g, req)); let circle = hop1 + hop2 | filter((node, (_, who, _, _)) -> (node, who) ne) | join(person, (node, ((rid, _, x, y), (_, _, _, _, _, _, _, city, _, _, _))) -> (city, (rid, node, x, y))) | join(place, (_, ((rid, node, x, y), (_, _, home))) -> ((rid, node), (x, y, home))) | filter((_, (x, y, home)) -> ((home, x) ne, (home, y) ne) all) | map((key, _) -> (key, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); let seen = visits | join(circle, (key, ((ix, iy), ())) -> (key, (ix, iy))); -let in_x = seen | filter((_, (ix, _)) -> (ix, 1) eq) | map((key, _) -> (key, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); -let in_y = seen | filter((_, (_, iy)) -> (iy, 1) eq) | map((key, _) -> (key, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +let in_x = seen | filter((_, (ix, _)) -> (ix, 1) eq) | map((key, _) -> (key, ())) | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +let in_y = seen | filter((_, (_, iy)) -> (iy, 1) eq) | map((key, _) -> (key, ())) | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); let counts = in_x | join(in_y); let ranked = counts | map(((rid, node), (nx, ny)) -> (node, (rid, nx, ny))) | join(person, (node, ((rid, nx, ny), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, (((nx, ny) add neg, node), (first, last, nx, ny)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((_, node), (first, last, nx, ny)))) -> ((rid, rank), (node, first, last, nx, ny, (nx, ny) add))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.parsed--lowered.diff index fe8689260..243353ccf 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.parsed--lowered.diff @@ -28,7 +28,7 @@ +let before = before_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); let during = seen | filter((_, (_, created, start, end)) -> ((created, start) ge, (created, end) lt) all); -let fresh = during + (during | join(before, (key, (row, ())) -> (key, row)) | negate); --let counts = fresh | map(((rid, _), (name, _, _, _)) -> ((rid, name), ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +-let counts = fresh | map(((rid, _), (name, _, _, _)) -> ((rid, name), ())) | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); -let ranked = counts | map(((rid, name), n) -> (rid, (n neg, name))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); +let during_idx = during | arrange; +let fresh_join = during_idx | join(before, (key, (row, ())) -> (key, row)); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.parsed.ddp index b99333028..4624efaba 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.parsed.ddp @@ -9,7 +9,7 @@ let seen = request | map(((rid, who, start, days), ()) -> (who, (rid, start, (st let before = seen | filter((_, (_, created, start, _)) -> (created, start) lt) | map((key, _) -> (key, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); let during = seen | filter((_, (_, created, start, end)) -> ((created, start) ge, (created, end) lt) all); let fresh = during + (during | join(before, (key, (row, ())) -> (key, row)) | negate); -let counts = fresh | map(((rid, _), (name, _, _, _)) -> ((rid, name), ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +let counts = fresh | map(((rid, _), (name, _, _, _)) -> ((rid, name), ())) | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); let ranked = counts | map(((rid, name), n) -> (rid, (n neg, name))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, (neg, name))) -> ((rid, rank), (name, neg neg))); output "answer" = answer | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic05_oneoff.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic05_oneoff.parsed--lowered.diff index 358873ba5..6d7342f2c 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic05_oneoff.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic05_oneoff.parsed--lowered.diff @@ -14,7 +14,7 @@ -let hop2 = hop1 | join(friend, (_, (req, (g, _))) -> (g, req)); -let circle = hop1 + hop2 | filter((node, (_, who, _)) -> (node, who) ne) | map((node, (rid, _, date)) -> ((node, rid, date), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((node, rid, date), ()) -> (node, (rid, date))); -let after = circle | join(member, (node, ((rid, date), (f, since))) -> ((node, f), (rid, date, since))) | filter((_, (_, date, since)) -> (since, date) gt) | map((key, (rid, _, _)) -> (key, rid)); --let busy = after | join(posted, ((_, f), (rid, n)) -> ((rid, f), n)) | join_function((key, n) -> ((key, ()), 0, n) enlist) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +-let busy = after | join(posted, ((_, f), (rid, n)) -> ((rid, f), n)) | join_function((key, n) -> ((key, ()), 0, n) enlist) | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); -let timed = circle | join(first, (node, ((rid, date), since)) -> (node, (rid, date, since))); -let fast = timed | filter((_, (_, date, since)) -> (since, date) gt) | join(lowest, (_, ((rid, _, _), f)) -> ((rid, f), ())); -let slow = timed | filter((_, (_, date, since)) -> (since, date) le) | map((node, (rid, _, _)) -> ((node, rid), ())) | join(after | map(((node, f), rid) -> ((node, rid), f)), ((_, rid), ((), f)) -> ((rid, f), ())); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic05_oneoff.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic05_oneoff.parsed.ddp index a2bf2c443..08b4afe85 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic05_oneoff.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic05_oneoff.parsed.ddp @@ -11,7 +11,7 @@ let hop1 = request | map(((rid, who, date), ()) -> (who, (rid, who, date))) | jo let hop2 = hop1 | join(friend, (_, (req, (g, _))) -> (g, req)); let circle = hop1 + hop2 | filter((node, (_, who, _)) -> (node, who) ne) | map((node, (rid, _, date)) -> ((node, rid, date), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((node, rid, date), ()) -> (node, (rid, date))); let after = circle | join(member, (node, ((rid, date), (f, since))) -> ((node, f), (rid, date, since))) | filter((_, (_, date, since)) -> (since, date) gt) | map((key, (rid, _, _)) -> (key, rid)); -let busy = after | join(posted, ((_, f), (rid, n)) -> ((rid, f), n)) | join_function((key, n) -> ((key, ()), 0, n) enlist) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +let busy = after | join(posted, ((_, f), (rid, n)) -> ((rid, f), n)) | join_function((key, n) -> ((key, ()), 0, n) enlist) | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); let timed = circle | join(first, (node, ((rid, date), since)) -> (node, (rid, date, since))); let fast = timed | filter((_, (_, date, since)) -> (since, date) gt) | join(lowest, (_, ((rid, _, _), f)) -> ((rid, f), ())); let slow = timed | filter((_, (_, date, since)) -> (since, date) le) | map((node, (rid, _, _)) -> ((node, rid), ())) | join(after | map(((node, f), rid) -> ((node, rid), f)), ((_, rid), ((), f)) -> ((rid, f), ())); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic05_standing.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic05_standing.parsed--lowered.diff index 0a945b30f..68827ca02 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic05_standing.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic05_standing.parsed--lowered.diff @@ -8,7 +8,7 @@ let friend = import "graph::friend" : (int, (int, int)); let forum = import "graph::forum" : (int, (List(int), int, int)); let member = import "graph::member.by_person" : (int, (int, int)); --let posted = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 0) eq) | map((_, (_, _, who, _, _, _, _, _, f, _, _, _, _)) -> ((who, f), ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +-let posted = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 0) eq) | map((_, (_, _, who, _, _, _, _, _, f, _, _, _, _)) -> ((who, f), ())) | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); -let first = member | map((who, (_, since)) -> (who, since)) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); -let lowest = member | map((who, (f, _)) -> (who, f)) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 40) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 40) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 40) min in let before = (end, count) sub in let keep = (((40, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten) | map((who, (_, f)) -> (who, f)); -output "posted" = posted | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic05_standing.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic05_standing.parsed.ddp index 2c3d84d4d..70a2eea08 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic05_standing.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic05_standing.parsed.ddp @@ -3,7 +3,7 @@ let message = import "graph::message" : (int, (int, int, int, int, List(int), Li let friend = import "graph::friend" : (int, (int, int)); let forum = import "graph::forum" : (int, (List(int), int, int)); let member = import "graph::member.by_person" : (int, (int, int)); -let posted = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 0) eq) | map((_, (_, _, who, _, _, _, _, _, f, _, _, _, _)) -> ((who, f), ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +let posted = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 0) eq) | map((_, (_, _, who, _, _, _, _, _, f, _, _, _, _)) -> ((who, f), ())) | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); let first = member | map((who, (_, since)) -> (who, since)) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); let lowest = member | map((who, (f, _)) -> (who, f)) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 40) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 40) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 40) min in let before = (end, count) sub in let keep = (((40, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten) | map((who, (_, f)) -> (who, f)); output "posted" = posted | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic06_oneoff.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic06_oneoff.parsed--lowered.diff index 498e978ca..565f18631 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic06_oneoff.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic06_oneoff.parsed--lowered.diff @@ -14,7 +14,7 @@ -let hop2 = hop1 | join(friend, (_, ((rid, who), (g, _))) -> (g, (rid, who))); -let circle = hop1 + hop2 | filter((node, (_, who)) -> (node, who) ne) | map((node, (rid, _)) -> ((rid, node), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); -let with_it = request | map(((rid, _, name), ()) -> (name, rid)) | join(by_name, (_, (rid, t)) -> (t, rid)) | join(tagged, (t, (rid, m)) -> (m, (rid, t))) | join(message, (m, ((rid, t), (kind, _, author, _, _, _, _, _, _, _, _, _, _))) -> ((rid, author), (kind, m, t))) | filter((_, (kind, _, _)) -> (kind, 0) eq) | join(circle, ((rid, _), ((_, m, t), ())) -> (m, (rid, t))); --let others = with_it | join(tags_of, (_, ((rid, t), other)) -> ((rid, t, other), ())) | filter(((_, t, other), ()) -> (t, other) ne) | map(((rid, _, other), ()) -> ((rid, other), ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +-let others = with_it | join(tags_of, (_, ((rid, t), other)) -> ((rid, t, other), ())) | filter(((_, t, other), ()) -> (t, other) ne) | map(((rid, _, other), ()) -> ((rid, other), ())) | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); -let ranked = others | map(((rid, other), n) -> (other, (rid, n))) | join(tag, (_, ((rid, n), (name, _))) -> (rid, (n neg, name))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); +let hop1_map = request | map(((rid, who, _), ()) -> (who, (rid, who))); +let hop1_idx = hop1_map | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic06_oneoff.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic06_oneoff.parsed.ddp index a792c7fd7..aa3092ceb 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic06_oneoff.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic06_oneoff.parsed.ddp @@ -10,7 +10,7 @@ let hop1 = request | map(((rid, who, _), ()) -> (who, (rid, who))) | join(person let hop2 = hop1 | join(friend, (_, ((rid, who), (g, _))) -> (g, (rid, who))); let circle = hop1 + hop2 | filter((node, (_, who)) -> (node, who) ne) | map((node, (rid, _)) -> ((rid, node), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); let with_it = request | map(((rid, _, name), ()) -> (name, rid)) | join(by_name, (_, (rid, t)) -> (t, rid)) | join(tagged, (t, (rid, m)) -> (m, (rid, t))) | join(message, (m, ((rid, t), (kind, _, author, _, _, _, _, _, _, _, _, _, _))) -> ((rid, author), (kind, m, t))) | filter((_, (kind, _, _)) -> (kind, 0) eq) | join(circle, ((rid, _), ((_, m, t), ())) -> (m, (rid, t))); -let others = with_it | join(tags_of, (_, ((rid, t), other)) -> ((rid, t, other), ())) | filter(((_, t, other), ()) -> (t, other) ne) | map(((rid, _, other), ()) -> ((rid, other), ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +let others = with_it | join(tags_of, (_, ((rid, t), other)) -> ((rid, t, other), ())) | filter(((_, t, other), ()) -> (t, other) ne) | map(((rid, _, other), ()) -> ((rid, other), ())) | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); let ranked = others | map(((rid, other), n) -> (other, (rid, n))) | join(tag, (_, ((rid, n), (name, _))) -> (rid, (n neg, name))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, (neg, other))) -> ((rid, rank), (other, neg neg))); output "answer" = answer | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic10_oneoff.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic10_oneoff.parsed--lowered.diff index d0ced800e..354836234 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic10_oneoff.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic10_oneoff.parsed--lowered.diff @@ -17,7 +17,7 @@ -let wrote = cands | map(((rid, who, node), _) -> (node, (rid, who))) | join(posts, (node, ((rid, who), m)) -> ((rid, who, node, m), ())); -let liked = wrote | map(((rid, who, node, m), ()) -> (m, (rid, who, node))) | join(tags_of, (m, ((rid, who, node), t)) -> ((who, t), (rid, node, m))) | join(likes, ((who, _), ((rid, node, m), ())) -> ((rid, who, node, m), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); -let points = (liked | map(((rid, who, node, _), ()) -> ((rid, who, node), 1))) + (wrote + (liked | negate) | map(((rid, who, node, _), ()) -> ((rid, who, node), -1))); --let summed = points | join_function((key, p) -> ((key, ()), 0, p) enlist) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +-let summed = points | join_function((key, p) -> ((key, ()), 0, p) enlist) | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +let hop1_map = request | map(((rid, who, month), ()) -> (who, (rid, who, month))); +let hop1_idx = hop1_map | arrange; +let friend_idx = friend | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic10_oneoff.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic10_oneoff.parsed.ddp index 1d731ea74..1e8984b9c 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic10_oneoff.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic10_oneoff.parsed.ddp @@ -15,7 +15,7 @@ let cands = twohop | map(((rid, who, month, node), _) -> (node, (rid, who, month let wrote = cands | map(((rid, who, node), _) -> (node, (rid, who))) | join(posts, (node, ((rid, who), m)) -> ((rid, who, node, m), ())); let liked = wrote | map(((rid, who, node, m), ()) -> (m, (rid, who, node))) | join(tags_of, (m, ((rid, who, node), t)) -> ((who, t), (rid, node, m))) | join(likes, ((who, _), ((rid, node, m), ())) -> ((rid, who, node, m), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); let points = (liked | map(((rid, who, node, _), ()) -> ((rid, who, node), 1))) + (wrote + (liked | negate) | map(((rid, who, node, _), ()) -> ((rid, who, node), -1))); -let summed = points | join_function((key, p) -> ((key, ()), 0, p) enlist) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +let summed = points | join_function((key, p) -> ((key, ()), 0, p) enlist) | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); let keys = cands | map((key, _) -> (key, ())); let score = summed + (keys + (keys | join(summed, (key, ((), _)) -> (key, ())) | negate) | map((key, ()) -> (key, 0))); let ranked = score | join(cands, ((rid, _, node), (s, (first, last, gender, town))) -> (rid, ((s neg, node), (first, last, s, gender, town)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic12_oneoff.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic12_oneoff.parsed--lowered.diff index ca16e38f2..b06bcf5f5 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic12_oneoff.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic12_oneoff.parsed--lowered.diff @@ -12,7 +12,7 @@ let reply = import "ic12s::reply" : (int, (int, int)); -let intag = request | map(((rid, _, name), ()) -> (name, rid)) | join(classes, (_, (rid, high)) -> (high, rid)) | join(below_hl, (_, (rid, low)) -> (low, rid)) | join(by_class, (_, (rid, t)) -> (t, rid)) | join(tag, (t, (rid, (name, _))) -> (t, (rid, name))) | join(mtag, (_, ((rid, name), m)) -> (m, (rid, name))) | join(message, (m, ((rid, name), (kind, _, _, _, _, _, _, _, _, _, _, _, _))) -> ((rid, m, name), kind)) | filter((_, kind) -> (kind, 0) eq) | map((key, _) -> (key, ())); -let hits = request | map(((rid, who, _), ()) -> (who, rid)) | join(friend, (_, (rid, (f, _))) -> (f, rid)) | join(reply, (f, (rid, (id, parent))) -> ((rid, parent), (f, id))) | join(intag | map(((rid, m, name), ()) -> ((rid, m), name)), ((rid, _), ((f, id), name)) -> ((rid, f), (id, name))); --let counts = hits | map(((rid, f), (id, _)) -> ((rid, f, id), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((rid, f, _), ()) -> ((rid, f), ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +-let counts = hits | map(((rid, f), (id, _)) -> ((rid, f, id), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((rid, f, _), ()) -> ((rid, f), ())) | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); -let tagset = hits | map(((rid, f), (_, name)) -> ((rid, f, name), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((rid, f, name), ()) -> ((rid, f), name)) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); -let ranked = counts | join(tagset, ((rid, f), (n, list)) -> (f, (rid, n, list))) | join(person, (f, ((rid, n, list), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((n neg, f), (first, last, list)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); +let intag_map = request | map(((rid, _, name), ()) -> (name, rid)); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic12_oneoff.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic12_oneoff.parsed.ddp index 6405f167e..9d653dc62 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic12_oneoff.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic12_oneoff.parsed.ddp @@ -11,7 +11,7 @@ let below_hl = import "ic12s::below_hl" : (int, int); let reply = import "ic12s::reply" : (int, (int, int)); let intag = request | map(((rid, _, name), ()) -> (name, rid)) | join(classes, (_, (rid, high)) -> (high, rid)) | join(below_hl, (_, (rid, low)) -> (low, rid)) | join(by_class, (_, (rid, t)) -> (t, rid)) | join(tag, (t, (rid, (name, _))) -> (t, (rid, name))) | join(mtag, (_, ((rid, name), m)) -> (m, (rid, name))) | join(message, (m, ((rid, name), (kind, _, _, _, _, _, _, _, _, _, _, _, _))) -> ((rid, m, name), kind)) | filter((_, kind) -> (kind, 0) eq) | map((key, _) -> (key, ())); let hits = request | map(((rid, who, _), ()) -> (who, rid)) | join(friend, (_, (rid, (f, _))) -> (f, rid)) | join(reply, (f, (rid, (id, parent))) -> ((rid, parent), (f, id))) | join(intag | map(((rid, m, name), ()) -> ((rid, m), name)), ((rid, _), ((f, id), name)) -> ((rid, f), (id, name))); -let counts = hits | map(((rid, f), (id, _)) -> ((rid, f, id), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((rid, f, _), ()) -> ((rid, f), ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +let counts = hits | map(((rid, f), (id, _)) -> ((rid, f, id), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((rid, f, _), ()) -> ((rid, f), ())) | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); let tagset = hits | map(((rid, f), (_, name)) -> ((rid, f, name), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((rid, f, name), ()) -> ((rid, f), name)) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); let ranked = counts | join(tagset, ((rid, f), (n, list)) -> (f, (rid, n, list))) | join(person, (f, ((rid, n, list), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((n neg, f), (first, last, list)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((neg, f), (first, last, list)))) -> ((rid, rank), (f, first, last, list, neg neg))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic14_standing.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic14_standing.parsed--lowered.diff index 49f7fda28..1d48c2abf 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic14_standing.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic14_standing.parsed--lowered.diff @@ -7,7 +7,7 @@ let pals = import "graph::friend.pair" : ((int, int), ()); -let answered = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 1) eq) | map((_, (_, _, who, _, _, _, _, _, _, parent, _, _, _)) -> (parent, who)) | join(message, (_, (who, (_, _, to, _, _, _, _, _, _, _, _, _, _))) -> (who, to)) | filter((who, to) -> (who, to) ne); -let talks = (answered | map((a, b) -> ((a, b), ()))) + (answered | map((a, b) -> ((b, a), ()))); --let weighted = talks | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter) | join(pals, (pair, (n, ())) -> (pair, n)) | map(((a, b), n) -> (a, (b, (((0, n), (1, 40) range) fold (((acc, n), k) -> +-let weighted = talks | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter) | join(pals, (pair, (n, ())) -> (pair, n)) | map(((a, b), n) -> (a, (b, (((0, n), (1, 40) range) fold (((acc, n), k) -> +let answered_filter = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 1) eq); +let answered_map = answered_filter | map((_, (_, _, who, _, _, _, _, _, _, parent, _, _, _)) -> (parent, who)); +let answered_idx = answered_map | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic14_standing.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic14_standing.parsed.ddp index f461b9ec9..55a639aea 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic14_standing.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic14_standing.parsed.ddp @@ -2,6 +2,6 @@ let message = import "graph::message" : (int, (int, int, int, int, List(int), Li let pals = import "graph::friend.pair" : ((int, int), ()); let answered = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 1) eq) | map((_, (_, _, who, _, _, _, _, _, _, parent, _, _, _)) -> (parent, who)) | join(message, (_, (who, (_, _, to, _, _, _, _, _, _, _, _, _, _))) -> (who, to)) | filter((who, to) -> (who, to) ne); let talks = (answered | map((a, b) -> ((a, b), ()))) + (answered | map((a, b) -> ((b, a), ()))); -let weighted = talks | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter) | join(pals, (pair, (n, ())) -> (pair, n)) | map(((a, b), n) -> (a, (b, (((0, n), (1, 40) range) fold (((acc, n), k) -> +let weighted = talks | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter) | join(pals, (pair, (n, ())) -> (pair, n)) | map(((a, b), n) -> (a, (b, (((0, n), (1, 40) range) fold (((acc, n), k) -> ((acc, ((((k, (k, 1) sub) mul, 1) add, n) gt, 1, 0) select) add, n)) .0, 1) add))); output "weighted" = weighted | arrange; diff --git a/interactive/tests/plan_pairs/expected/reduce_masks.ddp b/interactive/tests/plan_pairs/expected/reduce_masks.ddp index 70d551db2..39701c7cd 100644 --- a/interactive/tests/plan_pairs/expected/reduce_masks.ddp +++ b/interactive/tests/plan_pairs/expected/reduce_masks.ddp @@ -11,7 +11,9 @@ let unique_map = values | map((k, v) -> ((k, v), ())); let unique_idx = unique_map | arrange; let unique_reduce = unique_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); let unique_map2 = unique_reduce | map(((a, (_, b)), _) -> (a, ((), b))); -let counted_reduce = values_idx | reduce((_, a) -> let v0 = a map ((_, a) -> a) fold_add in (v0, (v0, 1)) enlist filter map ((_, a) -> ((), a))); +let counted_map = values | map((k, _) -> (k, ())); +let counted_idx = counted_map | arrange; +let counted_reduce = counted_idx | reduce((_, a) -> let v0 = a map ((_, a) -> a) fold_add in (v0, (v0, 1)) enlist filter map ((_, a) -> ((), a))); let collected_reduce = values_idx | reduce(_ -> ((), 1) enlist); let custom_map = custom | map((k, (_, v)) -> ((k, v), ())); -- Renderer: build an export arrangement. diff --git a/interactive/tests/plan_pairs/expected/reduce_masks.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/reduce_masks.lowered--optimized.diff index 5eda204b3..dcd596a4a 100644 --- a/interactive/tests/plan_pairs/expected/reduce_masks.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/reduce_masks.lowered--optimized.diff @@ -1,6 +1,6 @@ --- lowered +++ optimized -@@ -6,19 +6,15 @@ +@@ -6,20 +6,17 @@ let values_idx = values | arrange; let custom = values_idx | reduce((_, vals) -> vals map ((v, d) -> (v, (d, 2) mul))); let out_map = counted | map(((a, _), n) -> (a, n)); @@ -13,23 +13,23 @@ let unique_reduce = unique_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); -let unique_map2 = unique_reduce | map((kv, ()) -> kv); -let unique_map3 = unique_map2 | map((k, (_, v)) -> (k, ((), v))); --let values_idx3 = values | arrange; --let counted_reduce = values_idx3 | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +let unique_map2 = unique_reduce | map(((a, (_, b)), _) -> (a, ((), b))); -+let counted_reduce = values_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); - let counted_map = counted_reduce | map((k, _) -> (k, ())); --let values_idx4 = values | arrange; --let collected_reduce = values_idx4 | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); + let counted_map = values | map((k, _) -> (k, ())); + let counted_idx = counted_map | arrange; + let counted_reduce = counted_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); + let counted_map2 = counted_reduce | map((k, _) -> (k, ())); +-let values_idx3 = values | arrange; +-let collected_reduce = values_idx3 | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); +let collected_reduce = values_idx | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); let collected_map = collected_reduce | map((k, _) -> (k, ())); let custom_map = custom | map((k, (_, v)) -> ((k, v), ())); -- Renderer: build an export arrangement. -@@ -26,7 +22,7 @@ +@@ -27,7 +24,7 @@ -- Renderer: build an export arrangement. output "minimum" = minimum_map; -- Renderer: build an export arrangement. -output "unique" = unique_map3; +output "unique" = unique_map2; -- Renderer: build an export arrangement. - output "counted" = counted_map; + output "counted" = counted_map2; -- Renderer: build an export arrangement. diff --git a/interactive/tests/plan_pairs/expected/reduce_masks.optimized--projected.diff b/interactive/tests/plan_pairs/expected/reduce_masks.optimized--projected.diff index 436e8e04f..d160b1dae 100644 --- a/interactive/tests/plan_pairs/expected/reduce_masks.optimized--projected.diff +++ b/interactive/tests/plan_pairs/expected/reduce_masks.optimized--projected.diff @@ -1,6 +1,6 @@ --- optimized +++ projected -@@ -4,28 +4,25 @@ +@@ -4,30 +4,27 @@ let rows_idx = rows | arrange; let counted = rows_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); let values_idx = values | arrange; @@ -14,11 +14,13 @@ let unique_idx = unique_map | arrange; let unique_reduce = unique_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); let unique_map2 = unique_reduce | map(((a, (_, b)), _) -> (a, ((), b))); --let counted_reduce = values_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); --let counted_map = counted_reduce | map((k, _) -> (k, ())); + let counted_map = values | map((k, _) -> (k, ())); + let counted_idx = counted_map | arrange; +-let counted_reduce = counted_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +-let counted_map2 = counted_reduce | map((k, _) -> (k, ())); -let collected_reduce = values_idx | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); -let collected_map = collected_reduce | map((k, _) -> (k, ())); -+let counted_reduce = values_idx | reduce((_, a) -> let v0 = a map ((_, a) -> a) fold_add in (v0, (v0, 1)) enlist filter map ((_, a) -> ((), a))); ++let counted_reduce = counted_idx | reduce((_, a) -> let v0 = a map ((_, a) -> a) fold_add in (v0, (v0, 1)) enlist filter map ((_, a) -> ((), a))); +let collected_reduce = values_idx | reduce(_ -> ((), 1) enlist); let custom_map = custom | map((k, (_, v)) -> ((k, v), ())); -- Renderer: build an export arrangement. @@ -30,7 +32,7 @@ -- Renderer: build an export arrangement. output "unique" = unique_map2; --- Renderer: build an export arrangement. --output "counted" = counted_map; +-output "counted" = counted_map2; --- Renderer: build an export arrangement. -output "collected" = collected_map; +-- Renderer: publish the existing arrangement. diff --git a/interactive/tests/plan_pairs/expected/reduce_masks.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/reduce_masks.parsed--lowered.diff index 2fcc8023c..dab9e8726 100644 --- a/interactive/tests/plan_pairs/expected/reduce_masks.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/reduce_masks.parsed--lowered.diff @@ -1,14 +1,14 @@ --- parsed +++ lowered -@@ -1,10 +1,35 @@ +@@ -1,10 +1,36 @@ +-- Time lanes: epoch. let rows = input 0 : ((int, int), ()); --let counted = rows | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +-let counted = rows | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); -output "out" = counted | map(((a, _), n) -> (a, n)); let values = input 1 : (int, (int, int)); -output "minimum" = values | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist) | map((k, (_, v)) -> (k, ((), v))); -output "unique" = values | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map((k, (_, v)) -> (k, ((), v))); --output "counted" = values | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter) | map((k, _) -> (k, ())); +-output "counted" = values | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter) | map((k, _) -> (k, ())); -output "collected" = values | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist) | map((k, _) -> (k, ())); -let custom = values | reduce((_, vals) -> vals map ((v, d) -> (v, (d, 2) mul))); -output "custom" = custom | map((k, (_, v)) -> ((k, v), ())); @@ -25,11 +25,12 @@ +let unique_reduce = unique_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); +let unique_map2 = unique_reduce | map((kv, ()) -> kv); +let unique_map3 = unique_map2 | map((k, (_, v)) -> (k, ((), v))); ++let counted_map = values | map((k, _) -> (k, ())); ++let counted_idx = counted_map | arrange; ++let counted_reduce = counted_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); ++let counted_map2 = counted_reduce | map((k, _) -> (k, ())); +let values_idx3 = values | arrange; -+let counted_reduce = values_idx3 | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); -+let counted_map = counted_reduce | map((k, _) -> (k, ())); -+let values_idx4 = values | arrange; -+let collected_reduce = values_idx4 | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); ++let collected_reduce = values_idx3 | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); +let collected_map = collected_reduce | map((k, _) -> (k, ())); +let custom_map = custom | map((k, (_, v)) -> ((k, v), ())); +-- Renderer: build an export arrangement. @@ -39,7 +40,7 @@ +-- Renderer: build an export arrangement. +output "unique" = unique_map3; +-- Renderer: build an export arrangement. -+output "counted" = counted_map; ++output "counted" = counted_map2; +-- Renderer: build an export arrangement. +output "collected" = collected_map; +-- Renderer: build an export arrangement. diff --git a/interactive/tests/plan_pairs/expected/reduce_masks.parsed.ddp b/interactive/tests/plan_pairs/expected/reduce_masks.parsed.ddp index 13187a077..426a8b4a8 100644 --- a/interactive/tests/plan_pairs/expected/reduce_masks.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/reduce_masks.parsed.ddp @@ -1,10 +1,10 @@ let rows = input 0 : ((int, int), ()); -let counted = rows | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +let counted = rows | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); output "out" = counted | map(((a, _), n) -> (a, n)); let values = input 1 : (int, (int, int)); output "minimum" = values | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist) | map((k, (_, v)) -> (k, ((), v))); output "unique" = values | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map((k, (_, v)) -> (k, ((), v))); -output "counted" = values | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter) | map((k, _) -> (k, ())); +output "counted" = values | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter) | map((k, _) -> (k, ())); output "collected" = values | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist) | map((k, _) -> (k, ())); let custom = values | reduce((_, vals) -> vals map ((v, d) -> (v, (d, 2) mul))); output "custom" = custom | map((k, (_, v)) -> ((k, v), ())); diff --git a/interactive/tests/plan_pairs/expected/shared_demand.ddp b/interactive/tests/plan_pairs/expected/shared_demand.ddp index ffef77e6e..a42ea3e97 100644 --- a/interactive/tests/plan_pairs/expected/shared_demand.ddp +++ b/interactive/tests/plan_pairs/expected/shared_demand.ddp @@ -2,8 +2,9 @@ let rows = input 0 : (int, (int, int)); let first_map = rows | map((k, (a, _)) -> (k, a)); let second_map = rows | map((k, (_, b)) -> (k, b)); -let rows_idx = rows | arrange; -let counts_reduce = rows_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +let counts_map = rows | map((k, _) -> (k, ())); +let counts_idx = counts_map | arrange; +let counts_reduce = counts_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); -- Renderer: build an export arrangement. output "first" = first_map; -- Renderer: build an export arrangement. diff --git a/interactive/tests/plan_pairs/expected/shared_demand.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/shared_demand.lowered--optimized.diff index ea67221f5..f62522f82 100644 --- a/interactive/tests/plan_pairs/expected/shared_demand.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/shared_demand.lowered--optimized.diff @@ -7,4 +7,4 @@ +let rows = input 0 : (int, (int, int)); let first_map = rows | map((k, (a, _)) -> (k, a)); let second_map = rows | map((k, (_, b)) -> (k, b)); - let rows_idx = rows | arrange; + let counts_map = rows | map((k, _) -> (k, ())); diff --git a/interactive/tests/plan_pairs/expected/shared_demand.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/shared_demand.parsed--lowered.diff index 1b7e1ac64..b9fe0095f 100644 --- a/interactive/tests/plan_pairs/expected/shared_demand.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/shared_demand.parsed--lowered.diff @@ -1,17 +1,18 @@ --- parsed +++ lowered -@@ -1,4 +1,13 @@ +@@ -1,4 +1,14 @@ -let rows = input 0 : (int, (int, int)) | map((k, v) -> (k, v)); -output "first" = rows | map((k, (a, _)) -> (k, a)); -output "second" = rows | map((k, (_, b)) -> (k, b)); --output "counts" = rows | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +-output "counts" = rows | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +-- Time lanes: epoch. +let input0 = input 0 : (int, (int, int)); +let rows = input0 | map((k, v) -> (k, v)); +let first_map = rows | map((k, (a, _)) -> (k, a)); +let second_map = rows | map((k, (_, b)) -> (k, b)); -+let rows_idx = rows | arrange; -+let counts_reduce = rows_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); ++let counts_map = rows | map((k, _) -> (k, ())); ++let counts_idx = counts_map | arrange; ++let counts_reduce = counts_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +-- Renderer: build an export arrangement. +output "first" = first_map; +-- Renderer: build an export arrangement. diff --git a/interactive/tests/plan_pairs/expected/shared_demand.parsed.ddp b/interactive/tests/plan_pairs/expected/shared_demand.parsed.ddp index bdb858143..047dda5e0 100644 --- a/interactive/tests/plan_pairs/expected/shared_demand.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/shared_demand.parsed.ddp @@ -1,4 +1,4 @@ let rows = input 0 : (int, (int, int)) | map((k, v) -> (k, v)); output "first" = rows | map((k, (a, _)) -> (k, a)); output "second" = rows | map((k, (_, b)) -> (k, b)); -output "counts" = rows | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +output "counts" = rows | map((k, _) -> (k, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); diff --git a/interactive/tests/programs/lambda_slots.ddp b/interactive/tests/programs/lambda_slots.ddp index b9619c197..3915a6afe 100644 --- a/interactive/tests/programs/lambda_slots.ddp +++ b/interactive/tests/programs/lambda_slots.ddp @@ -1,4 +1,4 @@ --- The slots that take one lambda: `value` (the value as written, with no tuple around it), +-- The slots that take one lambda: `map` (a row to a row), -- `filter`, `flatmap` (complete rows, with positions explicitly zipped here) and `delay` (a row -- advances to the coordinate-wise maximum of its time and the tuple it names). A `--` comment -- may sit inside a lambda written over several lines. Declared types and the built-in `Option` @@ -10,7 +10,7 @@ enum Outer = Wrap Inner | Opt Option(int) | Nums List(int); let pairs = input 0 : ((int, int), ()) | map(((a, b), ()) -> (a, b)); -output "value" = pairs | value((a, b) -> ((b, 10) mul, a) add); +output "value" = pairs | map((a, b) -> (a, ((b, 10) mul, a) add)); output "filter" = pairs | filter((a, b) -> (a, b) lt); output "flatmap" = pairs | flatmap(row -> let (a, b) = row in let l = (a enlist, b enlist, (a, b) add enlist) append in (row.0, ((0, l len) range, l) zip) cap_list); stepped: { @@ -18,14 +18,14 @@ stepped: { set counted = delayed | count; } output "delay" = stepped::counted; -output "comment" = pairs | value((a, b) -> +output "comment" = pairs | map((a, b) -> (a, -- the key and the value, added (a, b) add -- one Int -); +)); -let sized = pairs | value((a, b) -> ((b, 3) gt, (b, a) inject Big, b inject Small) select); -output "sized" = sized | value((_, s) -> s match (Small (x -> x), Big ((x, y) -> (x, y) mul))); -let maybe = pairs | value((_, b) -> (b, (b, 3) gt) branch Option map_variant None (_ -> ())); -output "option" = maybe | value((_, m) -> m match (None (_ -> 0), Some (x -> x))); -let outer = pairs | value((_, b) -> b inject A inject Wrap); -output "nested" = outer | value((_, o) -> o match (Wrap (i -> i match (A (x -> x), B (_ -> 0))), Opt (_ -> -1), Nums (_ -> -1))); +let sized = pairs | map((a, b) -> (a, ((b, 3) gt, (b, a) inject Big, b inject Small) select)); +output "sized" = sized | map((k, s) -> (k, s match (Small (x -> x), Big ((x, y) -> (x, y) mul)))); +let maybe = pairs | map((k, b) -> (k, (b, (b, 3) gt) branch Option map_variant None (_ -> ()))); +output "option" = maybe | map((k, m) -> (k, m match (None (_ -> 0), Some (x -> x)))); +let outer = pairs | map((k, b) -> (k, b inject A inject Wrap)); +output "nested" = outer | map((k, o) -> (k, o match (Wrap (i -> i match (A (x -> x), B (_ -> 0))), Opt (_ -> -1), Nums (_ -> -1))));