Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion interactive/bench/ldbc/snb/rel.py
Original file line number Diff line number Diff line change
Expand Up @@ -385,7 +385,7 @@ def keyed(source, fields, keys, keep=True):
# A dedicated value-only collect avoids lists of singleton
# tuples and is useful for nested IC1/IC12 results.
assert len(aggs)==1
code = f'{source_name} | map((row, ()) -> ({group_key}, ({scalar(expr(e),refs)}))) | value((_, (item)) -> item) | collect | map((key, values) -> ({product([*outs,"values"])}, ()))'
code = f'{source_name} | map((row, ()) -> ({group_key}, {scalar(expr(e),refs)})) | collect | map((key, values) -> ({product([*outs,"values"])}, ()))'
self.lines.append(f'-- {name}: ({", ".join(r.fields)})\nlet {name} = {code};')
return name
else: raise ValueError(kind)
Expand Down
2 changes: 1 addition & 1 deletion interactive/examples/aoc2023/day01/part1.ddp
Original file line number Diff line number Diff line change
Expand Up @@ -14,5 +14,5 @@ let calib = first | join(last, (line, (f, l)) -> (line, ((f, 10) mul, l) add));

output "result" = calib
| map((_, v) -> ((), v)) | collect
| value((_, vs) -> vs fold_add)
| map((k, vs) -> (k, vs fold_add))
| arrange | inspect(part1);
2 changes: 1 addition & 1 deletion interactive/examples/aoc2023/day01/part2.ddp
Original file line number Diff line number Diff line change
Expand Up @@ -39,5 +39,5 @@ let calib = first | join(last, (line, (f, l)) -> (line, ((f, 10) mul, l) add));

output "result" = calib
| map((_, v) -> ((), v)) | collect
| value((_, vs) -> vs fold_add)
| map((k, vs) -> (k, vs fold_add))
| arrange | inspect(part2);
2 changes: 1 addition & 1 deletion interactive/examples/aoc2023/day02/part1.ddp
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,5 @@ let good = games - bad;

output "result" = good
| map((game, _) -> ((), game)) | collect
| value((_, gs) -> gs fold_add)
| map((k, gs) -> (k, gs fold_add))
| arrange | inspect(part1);
2 changes: 1 addition & 1 deletion interactive/examples/aoc2023/day02/part2.ddp
Original file line number Diff line number Diff line change
Expand Up @@ -13,5 +13,5 @@ let power = maxg

output "result" = power
| map((_, p) -> ((), p)) | collect
| value((_, ps) -> ps fold_add)
| map((k, ps) -> (k, ps fold_add))
| arrange | inspect(part2);
2 changes: 1 addition & 1 deletion interactive/examples/aoc2023/day03/part1.ddp
Original file line number Diff line number Diff line change
Expand Up @@ -40,5 +40,5 @@ let part_numbers = maximal | map(((r, s, _, v), _) -> ((r, s), v)) | join(active

output "result" = part_numbers
| map((_, v) -> ((), v)) | collect
| value((_, vs) -> vs fold_add)
| map((k, vs) -> (k, vs fold_add))
| arrange | inspect(part1);
2 changes: 1 addition & 1 deletion interactive/examples/aoc2023/day03/part2.ddp
Original file line number Diff line number Diff line change
Expand Up @@ -45,5 +45,5 @@ output "result" = gearvals
| filter((_, vs) -> (vs len, 2) eq)
| map((_, vs) -> ((), ((0, vs) get, (1, vs) get) mul))
| map((_, p) -> ((), p)) | collect
| value((_, ps) -> ps fold_add)
| map((k, ps) -> (k, ps fold_add))
| arrange | inspect(part2);
2 changes: 1 addition & 1 deletion interactive/examples/aoc2023/day04/part1.ddp
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,5 @@ let points = dups

output "result" = points
| map((_, p) -> ((), p)) | collect
| value((_, ps) -> ps fold_add)
| map((k, ps) -> (k, ps fold_add))
| arrange | inspect(part1);
26 changes: 3 additions & 23 deletions interactive/src/backend/corgi.rs
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ impl Plan {
LinearOp::Filter(cond) => ("filter", Some(compile_predicate(cond, k, v))),
LinearOp::FlatMap(list) => ("flatmap", Some(compile_flatmap(list, k, v))),
LinearOp::JoinFunction(lambda) => ("join_function", Some(compile_join_function(lambda, k, v, level.width()))),
LinearOp::Negate | LinearOp::LiftIter | LinearOp::Lift => ("", None),
LinearOp::Negate | LinearOp::Lift => ("", None),
};
let graph = graph.map(|g| g.unwrap_or_else(|e| panic!("{what}: type error at shapes ({kshape}, {vshape}): {e}")));
Plan { shape: (kshape, vshape), graph }
Expand All @@ -86,8 +86,7 @@ impl Plan {
/// column + a structural explode; Negate = Rust. Every slot is columnar — there is no row-wise
/// path inside the dataflow — and each op's graph was compiled when the operator was built
/// (`plans`). An empty batch passes through untouched: it has no rows to compute.
/// The data<->time ops are columnar and total: LiftIter reads the iteration coordinate out of
/// `times` and appends it to `vals`; Lift reads every lane of `times` into `vals`; JoinFunction
/// The data<->time ops are columnar and total: Lift reads every lane of `times` into `vals`; JoinFunction
/// explodes its list column as FlatMap does and joins each element's time lanes into `times`.
/// `level` locates the iteration coordinate and fixes the number of lanes as data.
fn apply_ops(mut c: CC, ops: &[LinearOp], level: Level, plans: &[Plan]) -> CC {
Expand Down Expand Up @@ -127,25 +126,6 @@ fn apply_ops(mut c: CC, ops: &[LinearOp], level: Level, plans: &[Plan]) -> CC {
}
c
}
// A value read OUT of each row's time. Vals gain one
// integer field; keys, times and diffs are untouched, and no slot is compiled.
//
// A tuple gains a field, and unit is the empty tuple: columnar it arrives as
// `CValue::Unit`, not an empty `Prod`, so `Unit` must become `Prod([iter])`.
// `Prod([Unit, iter])` would be a silent one-field-too-many divergence from the
// shape `shapes` infers.
LinearOp::LiftIter => {
// At the root there is no iteration
// coordinate and the value is zero.
let iters: Vec<u64> = level.own_lane().map_or_else(|| vec![0; c.times.len()], |lane| c.times.lane(lane));
let lane = CValue::i64(iters.into_iter().map(|i| i as i64).collect());
let vals = match c.vals {
CValue::Prod(mut fields) => { fields.push(lane); CValue::Prod(fields) }
CValue::Unit(_) => CValue::Prod(vec![lane]),
other => CValue::Prod(vec![other, lane]),
};
CorgiContainer { keys: c.keys, vals, times: c.times, diffs: c.diffs }
}
// A time as data is the root coordinates and one per scope: `level.width()` lanes, zero
// where the column is narrower. Keys, times and diffs are untouched.
LinearOp::Lift => {
Expand Down Expand Up @@ -309,7 +289,7 @@ impl Backend for CorgiBackend {

fn linear<'s>(c: Collection<'s, Time, CC>, ops: Vec<LinearOp>, level: Level, (k, v): st::RowShape) -> Collection<'s, Time, CC> {
// Container-level: fold the LinearOp chain over each corgi batch (no inter-op transcode).
// `level` is the scope depth (locates the iteration coordinate for LiftIter).
// `level` is the scope depth, which fixes the number of time lanes.
// Every op compiles now, at the shape it will see.
let steps = crate::shapes::linear_shapes(&ops, k, v, level).expect("install inferred these shapes");
let plans: Vec<Plan> = ops.iter().zip(steps).map(|(op, (key, val))| Plan::new(op, key, val, level)).collect();
Expand Down
20 changes: 14 additions & 6 deletions interactive/src/explain/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
//! The clone-with-lifts is the foundation the rewrite's witness and
//! forward copies are built from. `clone_into` clones an original
//! program's scopes into an output scope under construction; every nested
//! scope additionally `lift_iter`s and exports each internal collection, so
//! scope additionally lifts its round into each internal collection and exports it, so
//! every value-producing site in the subtree has a "host-visible" form — its
//! value as `(V, chain…)`, the user's value whole and then its user-iter
//! coordinates, innermost first — at the
Expand Down Expand Up @@ -176,6 +176,11 @@ fn clone_rec(orig: &Scope, out: &mut Scope, import_map: &[Ref], path: &[usize],
// and the export is the host-visible edge upward. A site of
// this scope first has its value wrapped as `(V)`, so the
// lifts extend it to `(V, chain…)` with `V` whole.
//
// The fold is `lift` and then a map that keeps the value's fields and the time's
// last coordinate, which is this scope's round. Times here have one root
// coordinate, the forward clone's explanation round if `forward`, and one per scope.
let round = format!("v.1.{}", child_path.len() + usize::from(forward));
let sub_idx = out.items.len();
for (a, vref) in child_visible {
let vref = if a.path == child_path {
Expand All @@ -185,8 +190,13 @@ fn clone_rec(orig: &Scope, out: &mut Scope, import_map: &[Ref], path: &[usize],
} else {
vref
};
// The value is `(V)` and one coordinate for each deeper exit.
let width = 1 + a.path.len() - child_path.len();
let mut fields: Vec<String> = (0..width).map(|j| format!("v.0.{j}")).collect();
fields.push(round.clone());
let fold = LinearOp::Project(projection(var(0), tup(fields), 2));
let lift_idx = child_out.items.len();
child_out.items.push(Item::Op(Node::Linear { input: vref, ops: vec![LinearOp::LiftIter] }));
child_out.items.push(Item::Op(Node::Linear { input: vref, ops: vec![LinearOp::Lift, fold] }));
let export_idx = child_out.exports.len();
child_out.exports.push(Export {
name: format!("$host:{}", export_idx),
Expand Down Expand Up @@ -1017,7 +1027,6 @@ impl<'a> Reverse<'a> {
LinearOp::Negate => {
self.push(ex, path, input, dep_this);
}
LinearOp::LiftIter => panic!("explain: LiftIter in user program"),
// Lift reads the whole time, and the forward clone runs one scope deeper.
LinearOp::Lift => panic!("explain: lift is not supported"),
LinearOp::JoinFunction(lambda) => {
Expand Down Expand Up @@ -1127,16 +1136,15 @@ mod tests {
let mut out = Scope { imports: p.root.imports.clone(), ..Scope::default() };
let import_map: Vec<Ref> = (0..out.imports.len()).map(Ref::Import).collect();
clone_into(&p.root, &mut out, &import_map);
// outer's clone: every $host: export is a LiftIter Linear; and the
// outer's clone: every $host: export is a lift and a map; and the
// ones re-exporting fwd/bwd internals chain TWO lifts (one per level):
// the value behind the lift is itself a ChildExport of a lift.
let Item::Sub(outer) = out.items.iter().find(|i| matches!(i, Item::Sub(_))).unwrap() else { unreachable!() };
let mut depth2_chains = 0;
for e in outer.exports.iter().filter(|e| e.name.starts_with("$host:")) {
let Ref::Local(li) = &e.value else { panic!("$host export should be a fresh lift") };
let Item::Op(Node::Linear { input, ops }) = &outer.items[*li] else { panic!("expected a lift") };
assert_eq!(ops.as_slice().len(), 1);
assert!(matches!(ops[0], LinearOp::LiftIter));
assert!(matches!(ops.as_slice(), [LinearOp::Lift, LinearOp::Project(_)]));
if matches!(input, Ref::ChildExport(..)) { depth2_chains += 1; }
}
assert!(depth2_chains > 0, "fwd/bwd internals re-lift at outer's exit");
Expand Down
6 changes: 1 addition & 5 deletions interactive/src/ir.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ pub use crate::lambda::Lambda;
/// table (`parse::pipe`) must agree, which its tests check.
pub(crate) const KEYWORDS: &[&str] = &[
"let", "var", "set", "output", "type", "enum", "input", "import", "map", "join", "min", "distinct", "count",
"collect", "flatmap", "arrange", "negate", "filter", "inspect", "lift_iter",
"collect", "flatmap", "arrange", "negate", "filter", "inspect",
];

#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
Expand All @@ -35,10 +35,6 @@ pub enum LinearOp {
Filter(Lambda),
/// Negate the diff.
Negate,
/// Append the current user-iter coord (at the row's scope depth) to
/// the value. Time itself is unchanged. See `Expr::LiftIter` for the
/// discipline restriction.
LiftIter,
/// Each row becomes the lambda's list of complete `(key, val)` rows.
FlatMap(Lambda),
/// Pair each value with its whole time: `(key, val)` becomes `(key, (val, time))`.
Expand Down
1 change: 0 additions & 1 deletion interactive/src/level.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ pub(crate) struct Level {
impl Level {
pub(crate) fn root(root: usize) -> Self { Self { root, depth: 0 } }
pub(crate) fn width(self) -> usize { self.root + self.depth }
pub(crate) fn own_lane(self) -> Option<usize> { (self.depth > 0).then(|| self.width() - 1) }
pub(crate) fn child(self) -> Self { Self { depth: self.depth + 1, ..self } }
pub(crate) fn parent(self) -> Self { Self { depth: self.depth.saturating_sub(1), ..self } }
}
31 changes: 16 additions & 15 deletions interactive/src/lower.rs
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@ fn resolve_expr(expr: &mut Expr, here: &[String], item: Option<&str>,
*expr = Expr::ResolvedPath { anchor: key.anchor, target: key.target, field: key.field };
}
Expr::TypedSource(e, _, _) | Expr::Map(e, _) | Expr::Reduce(e, _) | Expr::Filter(e, _)
| Expr::Negate(e) | Expr::LiftIter(e) | Expr::FlatMap(e, _)
| Expr::Negate(e) | Expr::FlatMap(e, _)
| Expr::Lift(e) | Expr::JoinFunction(e, _) | Expr::Inspect(e, _) | Expr::Arrange(e) =>
resolve_expr(e, here, item, scopes, plan),
Expr::Join(l, r, _) | Expr::HalfJoin(l, r, _, _) => {
Expand Down Expand Up @@ -247,7 +247,7 @@ fn expr_free_names(expr: &Expr, out: &mut BTreeSet<RefName>) {
Expr::ResolvedPath { .. } => {},
Expr::Path(..) => unreachable!("paths must be resolved before lowering"),
Expr::TypedSource(e, _, _) | Expr::Map(e, _) | Expr::Reduce(e, _) | Expr::Filter(e, _)
| Expr::Negate(e) | Expr::LiftIter(e)
| Expr::Negate(e)
| Expr::FlatMap(e, _) | Expr::Lift(e) | Expr::JoinFunction(e, _)
| Expr::Inspect(e, _) | Expr::Arrange(e) => expr_free_names(e, out),
Expr::Join(l, r, _) | Expr::HalfJoin(l, r, _, _) => { expr_free_names(l, out); expr_free_names(r, out); },
Expand Down Expand Up @@ -310,17 +310,17 @@ pub fn lower_tree(mut stmts: Vec<Stmt>) -> st::Program {
let mut plan = PathPlan::default();
resolve_body(&mut stmts, &[], &scopes, &mut plan);
let empty = HashMap::new();
let mut distinct = HashMap::new();
let mut program = st::Program { root: lower_scope_tree(&[], &stmts, &empty, &[], &plan, true, &mut distinct) };
let mut by_shape = HashMap::new();
let mut program = st::Program { root: lower_scope_tree(&[], &stmts, &empty, &[], &plan, true, &mut by_shape) };
let (shapes, _) = crate::shapes::infer(&program, &|_| None, 1);
crate::reducers::expand_distinct(&mut program.root, &shapes, &distinct, &mut Vec::new());
crate::reducers::expand_by_shape(&mut program.root, &shapes, &by_shape, &mut Vec::new());
program
}

/// Expr-lowering state for one scope. Resolves names to scope-local `Ref`s and
/// accumulates this scope's nodes and (input/trace) imports.
struct ScopeLower {
distinct: Vec<usize>,
by_shape: Vec<(usize, crate::reducers::ByShape)>,
is_root: bool,
items: Vec<st::Item>,
/// Each item's statement name (see `st::Scope::labels`).
Expand Down Expand Up @@ -386,7 +386,6 @@ impl ScopeLower {
Expr::Filter(e, c) => { let r = self.lower_expr(e); self.push(st::Node::Linear { input: r, ops: vec![LinearOp::Filter(c.clone())] }) },
Expr::Negate(e) => { let r = self.lower_expr(e); self.push(st::Node::Linear { input: r, ops: vec![LinearOp::Negate] }) },
Expr::FlatMap(e, t) => { let r = self.lower_expr(e); self.push(st::Node::Linear { input: r, ops: vec![LinearOp::FlatMap(t.clone())] }) },
Expr::LiftIter(e) => { let r = self.lower_expr(e); self.push(st::Node::Linear { input: r, ops: vec![LinearOp::LiftIter] }) },
Expr::Lift(e) => { let r = self.lower_expr(e); self.push(st::Node::Linear { input: r, ops: vec![LinearOp::Lift] }) },
Expr::JoinFunction(e, t) => { let r = self.lower_expr(e); self.push(st::Node::Linear { input: r, ops: vec![LinearOp::JoinFunction(t.clone())] }) },
Expr::Arrange(e) => { let r = self.lower_expr(e); self.push(st::Node::Arrange(r)) },
Expand All @@ -404,11 +403,13 @@ impl ScopeLower {
},
Expr::Reduce(e, red)=> {
let r = self.lower_expr(e);
let a = if matches!(red, Reducer::Distinct) {
let a = match red {
// Once shapes are known, lower distinct by key or by whole record.
self.distinct.push(self.items.len());
r
} else { self.arrange(r) };
Reducer::Distinct => { self.by_shape.push((self.items.len(), crate::reducers::ByShape::Distinct)); r }
// Once shapes are known, project count's values away if they are not unit.
Reducer::Count => { self.by_shape.push((self.items.len(), crate::reducers::ByShape::Count)); r }
_ => self.arrange(r),
};
self.push(st::Node::Reduce { input: a, reducer: crate::ir::Reducer::Expr(red.lambda()), limit: red.limit() })
},
Expr::Inspect(e, l) => { let r = self.lower_expr(e); self.push(st::Node::Inspect { input: r, label: l.clone() }) },
Expand All @@ -424,11 +425,11 @@ fn lower_scope_tree(
exports_wanted: &[PathKey],
plan: &PathPlan,
is_root: bool,
distinct: &mut HashMap<Vec<String>, Vec<usize>>,
by_shape: &mut HashMap<Vec<String>, Vec<(usize, crate::reducers::ByShape)>>,
) -> st::Scope {
let scope_name = scope_label(address);
let mut s = ScopeLower {
distinct: Vec::new(),
by_shape: Vec::new(),
is_root,
items: Vec::new(),
labels: Vec::new(),
Expand Down Expand Up @@ -560,7 +561,7 @@ fn lower_scope_tree(
let mut child_address = address.to_vec();
child_address.push(name.clone());
let wanted: Vec<PathKey> = plan.exports.get(&child_address).into_iter().flatten().cloned().collect();
let child = lower_scope_tree(&child_address, b, &s.env, &wanted, plan, false, distinct);
let child = lower_scope_tree(&child_address, b, &s.env, &wanted, plan, false, by_shape);
let slots = export_slots(&wanted, &child_address);
let cidx = s.items.len();
s.items.push(st::Item::Sub(child));
Expand Down Expand Up @@ -611,7 +612,7 @@ fn lower_scope_tree(
}
}

distinct.insert(address.to_vec(), s.distinct);
by_shape.insert(address.to_vec(), s.by_shape);
st::Scope { name: scope_name.to_string(), imports: s.imports, vars, items: s.items, labels: s.labels, names, binds, exports }
}

Expand Down
8 changes: 0 additions & 8 deletions interactive/src/parse/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -60,14 +60,6 @@ pub enum Expr {
Reduce(Box<Expr>, Reducer),
Filter(Box<Expr>, Lambda),
Negate(Box<Expr>),
/// Append the current user-iter coord (at the operator's scope depth)
/// to each row's value. Time itself is unchanged.
///
/// Discipline, which nothing checks: the result of `LiftIter` should not be referenced inside the same
/// scope it appears in — only from an enclosing scope, after the
/// implicit leave. This preserves the "loop body is a time-invariant
/// function" property; in-scope use risks defeating the fixpoint.
LiftIter(Box<Expr>),
/// Replace each row with the lambda's list of complete `(key, val)` rows.
FlatMap(Box<Expr>, Lambda),
/// Pair each row's value with its whole time. See `LinearOp::Lift`.
Expand Down
Loading
Loading