diff --git a/interactive/README.md b/interactive/README.md index ea09a1bb4..d543ccf41 100644 --- a/interactive/README.md +++ b/interactive/README.md @@ -70,6 +70,12 @@ For each pair of matches `(k, l)` and `(k, r)`, `left | join(right)` produces `( Passing `f` to the join lets it emit the projected fields directly. A separate `map` can require the full joined row as an intermediate result. Existing three-field join patterns migrate from `(k, l, r) -> …` to `(k, (l, r)) -> …`. +For updates from `left`, `left | lift | half_join(right, 0)` produces `(k, ((l, r), t))`. +Here `t` is the coordinate-wise maximum of the lifted data time and the matching right update's timestamp. +Use `0` to include equal timestamps and `1` for strictly earlier right updates in the total order. +An optional projection receives the entire row. For example, `half_join(right, 0, (k, ((l, r), _)) -> (k, r))` drops the data time. +The output update still has the left update's timestamp. To migrate an old projection `(k, l, r) -> (k2, v2)`, write `(k, ((l, r), t)) -> (k2, (v2, t))` to retain its previous output. + A currently unfortunate amount of symbolic load is in shuffling around fields of collection elements. The `map` operator applies to each record, and the `->` fragment is an expression converting the input to the output. `flatmap` gives a `List((key, val))` of complete rows, choosing both keys and values. diff --git a/interactive/src/lambda.rs b/interactive/src/lambda.rs index 34c00b493..7f502e52c 100644 --- a/interactive/src/lambda.rs +++ b/interactive/src/lambda.rs @@ -93,6 +93,10 @@ impl Lambda { pub(crate) fn on_join_columns(&self) -> Self { Self::compose("(k, l, r)", &[(self, "(k, (l, r))".into())], |out| out[0].clone()) } + /// Apply a half-join projection to the gathered columns and their joined data time. + pub(crate) fn on_half_join_columns(&self) -> Self { + Self::compose("(k, l, r, t)", &[(self, "(k, ((l, r), t))".into())], |out| out[0].clone()) + } /// The graph, or the parse error naming the program. pub(crate) fn graph(&self) -> Result<&std::sync::Arc>, String> { self.graph.as_ref().map_err(|e| format!("corgi program `{}`: {e}", self.src)) diff --git a/interactive/src/lower.rs b/interactive/src/lower.rs index fbbb73f62..a9c1d789e 100644 --- a/interactive/src/lower.rs +++ b/interactive/src/lower.rs @@ -638,7 +638,7 @@ mod tree_tests { let text = format!(r#" let rows = input 0 : (int, ()) | {producer}; output "join" = rows | join(rows, (k, (_, _)) -> (k, ())); - output "half" = rows | lift | half_join(rows, 0, (k, _, _) -> (k, ())); + output "half" = rows | lift | half_join(rows, 0, (k, ((_, _), t)) -> (k, ((), t))); output "reduce" = rows | min; output "distinct" = rows | distinct; "#); @@ -657,7 +657,7 @@ mod tree_tests { let program = lower_tree(parse(r#" let rows = input 0 : (int, int) | distinct; output "join" = rows | join(rows, (k, (a, _)) -> (k, a)); - output "half" = rows | lift | half_join(rows, 0, (k, a, _) -> (k, a)); + output "half" = rows | lift | half_join(rows, 0, (k, ((a, _), t)) -> (k, (a, t))); output "reduce" = rows | min; output "distinct" = rows | distinct; "#)); diff --git a/interactive/src/parse/pipe.rs b/interactive/src/parse/pipe.rs index 5a390694d..62ee607ff 100644 --- a/interactive/src/parse/pipe.rs +++ b/interactive/src/parse/pipe.rs @@ -51,10 +51,11 @@ //! of the epoch and every scope coordinate. //! - `| join_function((k, v) -> list)` — replace each row by the lambda's list of //! `((key, val), time, diff)`, at the row's time joined with `time`, diffs multiplied. -//! - `| half_join(other, strict, (k, v, v1) -> (k', v'))` — each update of a lifted stream, +//! - `| half_join(other, strict)` — each update of a lifted stream, //! `(k, (v, time))`, meets `other`'s rows at its key whose times precede the update's own in the //! total order on times (strictly when `strict` is 1, or also equal when 0), giving -//! `(k', (v', time ∨ t2))`. The lambda does not see the times. `other`'s changes produce nothing. +//! `(k, ((v, v1), time ∨ t2))`. `half_join(other, strict, f)` projects this whole row. +//! The output retains the stream update's timestamp. `other`'s changes produce nothing. //! - `| delay((k, v) -> time)` — `join_function` with the list `[((key, val), time, 1)]`: advance //! each row to at least the time the lambda gives. The tuple names the epoch, then each //! enclosing loop from outermost to innermost; `0` leaves a coordinate unchanged. Inside one @@ -666,6 +667,15 @@ impl Parser { Some((key, val)) } + fn join_projection(&mut self) -> Lambda { + if *self.peek() == Token::Comma { + self.next(); + self.lambda() + } else { + Lambda::new("let row = input in row".into(), &corgi::Scope::default()) + } + } + fn parse_pipe_op(&mut self, lhs: Expr) -> Expr { match self.peek().clone() { Token::Map => { self.next(); Expr::Map(Box::new(lhs), self.slot()) }, @@ -673,12 +683,7 @@ impl Parser { self.next(); self.expect(&Token::LParen); let rhs = self.parse_pipe_expr(); - let projection = if *self.peek() == Token::Comma { - self.next(); - self.lambda() - } else { - Lambda::new("let row = input in row".into(), &corgi::Scope::default()) - }; + let projection = self.join_projection(); self.expect(&Token::RParen); Expr::Join(Box::new(lhs), Box::new(rhs), projection) }, @@ -727,8 +732,7 @@ impl Parser { Token::Int(1) => true, other => panic!("half_join: strict is 0 or 1, not {other:?}"), }; - self.expect(&Token::Comma); - let p = self.lambda(); + let p = self.join_projection(); self.expect(&Token::RParen); Expr::HalfJoin(Box::new(lhs), Box::new(r), strict, p) }, diff --git a/interactive/src/passes/arrangements_tests.rs b/interactive/src/passes/arrangements_tests.rs index 1171f0edb..38a8a8d3f 100644 --- a/interactive/src/passes/arrangements_tests.rs +++ b/interactive/src/passes/arrangements_tests.rs @@ -210,7 +210,7 @@ fn variant_inputs_and_half_joins_stop_motion_before_fusion() { let p = rewrite(r#" let x = input 0 : (int, int); let timed = x | map((k, v) -> (k, (v, (0, 0)))); - child: { let out = ::timed | half_join(::x, 0, (k, l, r) -> (k, l)); } + child: { let out = ::timed | half_join(::x, 0, (k, ((l, r), t)) -> (k, (l, t))); } output "out" = child::out; "#, hoist(), 0); let Item::Sub(child) = p.root.items.last().unwrap() else { panic!() }; diff --git a/interactive/src/passes/as_of.rs b/interactive/src/passes/as_of.rs index f933536e6..e16305c8f 100644 --- a/interactive/src/passes/as_of.rs +++ b/interactive/src/passes/as_of.rs @@ -221,8 +221,8 @@ mod tests { let (prog, inserted) = rewrite(r#" let a = import "a"; let b = import "b"; - output "r" = ((a | lift | half_join(b, 0, (k, x, y) -> (k, (x, y)))) - + (b | lift | half_join(a, 1, (k, y, x) -> (k, (x, y))))); + output "r" = ((a | lift | half_join(b, 0)) + + (b | lift | half_join(a, 1, (k, ((y, x), t)) -> (k, ((x, y), t))))); "#); assert_eq!(inserted, 1); let found = filters(&prog.root); @@ -238,9 +238,9 @@ mod tests { let a = import "a"; let b = import "b"; let c = import "c"; - let ab = a | lift | half_join(b, 0, (k, x, y) -> (k, x)) | half_join(c, 0, (k, x, z) -> (k, x)); - let ba = b | lift | half_join(a, 1, (k, y, x) -> (k, x)) | half_join(c, 0, (k, x, z) -> (k, x)); - let ca = c | lift | half_join(a, 1, (k, z, x) -> (k, x)) | half_join(b, 1, (k, x, y) -> (k, x)); + let ab = a | lift | half_join(b, 0, (k, ((x, y), t)) -> (k, (x, t))) | half_join(c, 0, (k, ((x, z), t)) -> (k, (x, t))); + let ba = b | lift | half_join(a, 1, (k, ((y, x), t)) -> (k, (x, t))) | half_join(c, 0, (k, ((x, z), t)) -> (k, (x, t))); + let ca = c | lift | half_join(a, 1, (k, ((z, x), t)) -> (k, (x, t))) | half_join(b, 1, (k, ((x, y), t)) -> (k, (x, t))); output "r" = ab + ba + ca; "#); assert_eq!(inserted, 2, "every path but `a`'s, which is non-strict throughout"); @@ -250,8 +250,8 @@ mod tests { fn a_shared_lift_filters_only_the_edge_to_its_strict_reader() { let (prog, inserted) = rewrite(r#" let p = import "p"; - output "r" = ((p | lift | half_join(p, 0, (k, x, y) -> (k, (x, y)))) - + (p | lift | half_join(p, 1, (k, x, y) -> (k, (x, y))))); + output "r" = ((p | lift | half_join(p, 0)) + + (p | lift | half_join(p, 1))); "#); assert_eq!(inserted, 1); let found = filters(&prog.root); @@ -268,7 +268,7 @@ mod tests { let (prog, inserted) = rewrite(r#" let p = import "p"; let lifted = p | lift | filter((k, _) -> (k, k) eq) | negate; - output "strict" = lifted | half_join(p, 1, (k, x, y) -> (k, (x, y))); + output "strict" = lifted | half_join(p, 1); output "live" = lifted; "#); assert_eq!(inserted, 1); @@ -288,7 +288,7 @@ mod tests { let a = import "a"; let b = import "b"; let la = a | lift; - output "r" = la | half_join(b, 1, (k, x, y) -> (k, (x, y))); + output "r" = la | half_join(b, 1); output "lifted" = la; "#); assert_eq!(inserted, 1, "the edge to the strict half join is filtered"); @@ -299,13 +299,13 @@ mod tests { let (_, live) = rewrite(r#" let a = import "a"; let b = import "b"; - output "r" = a | lift | half_join(b, 0, (k, x, y) -> (k, (x, y))); + output "r" = a | lift | half_join(b, 0); "#); assert_eq!(live, 0, "a non-strict path is live at the as-of time"); let (_, moved) = rewrite(r#" let a = import "a"; let b = import "b"; - output "r" = a | lift | delay((_, (_, t)) -> t) | half_join(b, 1, (k, x, y) -> (k, (x, y))); + output "r" = a | lift | delay((_, (_, t)) -> t) | half_join(b, 1); "#); assert_eq!(moved, 0, "a delay between the lift and the strict half join can move the time"); } @@ -316,7 +316,7 @@ mod tests { let a = import "a"; let b = import "b"; s: { - let r = a | lift | half_join(b, 1, (k, x, y) -> (k, x)) | map((k, (x, _)) -> (k, x)); + let r = a | lift | half_join(b, 1, (k, ((x, y), t)) -> (k, (x, t))) | map((k, (x, _)) -> (k, x)); set x = (r + x) | distinct; } output "r" = s::x; diff --git a/interactive/src/passes/projection.rs b/interactive/src/passes/projection.rs index 6046dea2c..1f7d867a9 100644 --- a/interactive/src/passes/projection.rs +++ b/interactive/src/passes/projection.rs @@ -252,8 +252,8 @@ impl Builder { ) -> Result<(), String> { self.presence(from, stream); self.presence(from, arrangement); - // Keys decide matching, and the lifted stream time decides whether a - // match is admissible and supplies the output's data time. + // Keys decide matching. The operator always consumes a lifted stream, + // including its data time, even when the projection drops the time. for id in from.all() { for &key in stream.key.iter().chain(&arrangement.key) { self.edge(id, key); @@ -269,30 +269,18 @@ impl Builder { let Shape::Prod(stream_parts) = &stream.shape.1 else { return Err("projection: half_join stream value is not (value, time)".into()); }; - let stream_value = stream_parts - .first() - .ok_or("projection: half_join has no value field")?; - let shapes = [ - stream.shape.0.clone(), - stream_value.clone(), - arrangement.shape.1.clone(), - ]; + let [stream_value, time] = stream_parts.as_slice() else { + return Err("projection: half_join stream value is not (value, time)".into()); + }; + let shapes = [stream.shape.0.clone(), stream_value.clone(), arrangement.shape.1.clone(), time.clone()]; + let columns = lambda.on_half_join_columns(); for (output_id, path) in from.field_paths() { - if path.as_slice() == [1, 1] { - continue; - } - // The half join appends time after the projected value. Its value - // is one atomic fact at this top-level field precision. - let projected = if path.as_slice() == [1, 0] { - vec![1] - } else { - path - }; - for (slot, field) in graph_fields(lambda, &shapes, &projected) { + for (slot, field) in graph_fields(&columns, &shapes, &path) { let source = match slot { 0 => stream.key.get(field), 1 => stream.val.first(), 2 => arrangement.val.get(field), + 3 => stream.val.get(1), _ => None, }; if let Some(&source) = source { @@ -623,46 +611,16 @@ fn mask_part(name: &str, shape: &Shape, facts: &[usize], live: &[bool]) -> Strin } } -fn mask_lambda( - original: &Lambda, - key_shape: &Shape, - key_facts: &[usize], - value_shape: Option<&Shape>, - value_facts: &[usize], - live: &[bool], -) -> Option { - // A half join appends time after its projection. Its projected value is - // one demand fact even when the value itself is a tuple. - if width(key_shape) != key_facts.len() - || match value_shape { - Some(shape) => width(shape) != value_facts.len(), - None => value_facts.len() != 1, - } - { - return None; - } - if !needs_mask(key_shape, key_facts, live) && match value_shape { - Some(shape) => !needs_mask(shape, value_facts, live), - None => live[value_facts[0]], - } { +fn mask_site_lambda(original: &Lambda, output: &Site, live: &[bool]) -> Option { + if !needs_mask(&output.shape.0, &output.key, live) && !needs_mask(&output.shape.1, &output.val, live) { return None; } - // Select live fields directly from the call result. Corgi can fold those - // field selections through the tuple and eliminate dead work. The parsed - // source and the running graph then take the same optimization path. - Some(Lambda::compose( - "arg", - &[(original, "arg".into())], - |result| { - let key = mask_part(&format!("{}.0", result[0]), key_shape, key_facts, live); - let val = match value_shape { - Some(shape) => mask_part(&format!("{}.1", result[0]), shape, value_facts, live), - None if live[value_facts[0]] => format!("{}.1", result[0]), - None => "()".into(), - }; - format!("({key}, {val})") - }, - )) + // Select live fields directly from the call result so corgi can eliminate dead work. + Some(Lambda::compose("arg", &[(original, "arg".into())], |result| { + let key = mask_part(&format!("{}.0", result[0]), &output.shape.0, &output.key, live); + let val = mask_part(&format!("{}.1", result[0]), &output.shape.1, &output.val, live); + format!("({key}, {val})") + })) } fn needs_mask(shape: &Shape, facts: &[usize], live: &[bool]) -> bool { @@ -673,46 +631,6 @@ fn needs_mask(shape: &Shape, facts: &[usize], live: &[bool]) -> bool { } } -fn mask_site_lambda(original: &Lambda, output: &Site, live: &[bool]) -> Option { - mask_lambda( - original, - &output.shape.0, - &output.key, - Some(&output.shape.1), - &output.val, - live, - ) -} - -fn mask_half_join(original: &Lambda, output: &Site, live: &[bool]) -> Option { - // The half-join operator appends time after this lambda has produced - // (key, value), so only the first output value fact belongs to the lambda. - let Shape::Prod(parts) = &output.shape.1 else { - return None; - }; - if parts.len() != 2 { - return None; - } - let Some(&value_fact) = output.val.first() else { - return None; - }; - // A unit value has nothing to mask, though the half join keeps a fact for it. - // Masking it again would count a change each round, and never converge. - let (value_shape, value_facts) = if parts[0] == Shape::Unit { - (Some(&parts[0]), &[][..]) - } else { - (None, std::slice::from_ref(&value_fact)) - }; - mask_lambda( - original, - &output.shape.0, - &output.key, - value_shape, - value_facts, - live, - ) -} - fn rewrite_scope(scope: &mut st::Scope, plan: &PlannedScope, live: &[bool]) -> usize { let mut masked = 0; let labeled = scope.labels.len() == scope.items.len(); @@ -741,7 +659,7 @@ fn rewrite_scope(scope: &mut st::Scope, plan: &PlannedScope, live: &[bool]) -> u } } } - (Item::Op(Node::Join { projection, .. }), PlannedItem::Op { output, .. }) => { + (Item::Op(Node::Join { projection, .. } | Node::HalfJoin { projection, .. }), PlannedItem::Op { output, .. }) => { if let Some(new) = mask_site_lambda(projection, output, live) { masked += output .key @@ -752,14 +670,6 @@ fn rewrite_scope(scope: &mut st::Scope, plan: &PlannedScope, live: &[bool]) -> u *projection = new; } } - (Item::Op(Node::HalfJoin { projection, .. }), PlannedItem::Op { output, .. }) => { - if let Some(new) = mask_half_join(projection, output, live) { - masked += output.key.iter().filter(|&&id| !live[id]).count(); - let unit = matches!(&output.shape.1, Shape::Prod(parts) if parts[0] == Shape::Unit); - masked += usize::from(!unit && !live[output.val[0]]); - *projection = new; - } - } (Item::Op(Node::Reduce { input, reducer, .. }), PlannedItem::Op { output, .. }) => { if needs_mask(&output.shape.1, &output.val, live) { let value = mask_part("input", &output.shape.1, &output.val, live); @@ -1165,14 +1075,31 @@ mod tests { let b = input 1 : ((int, int, int), ()) | map(((k, y, z), ()) -> (k, (y, z))); output \"out\" = a | lift - | half_join(b, 0, (k, x, yz) -> (k, x)) | count;"; + | half_join(b, 0, (k, ((x, yz), t)) -> (k, (x, t))) | count;"; let mut program = lower::lower_tree(parse::pipe::parse(text)); crate::passes::optimize(&mut program); - assert_eq!(pushdown(&mut program, &|_| None).unwrap(), 4); + // Count needs neither the payload nor the joined data time in the output. + assert_eq!(pushdown(&mut program, &|_| None).unwrap(), 5); + assert_eq!(pushdown(&mut program, &|_| None).unwrap(), 0); let (_, check) = shapes::infer(&program, &|_| None, 1); assert!(check.conflicts.is_empty(), "{:?}", check.conflicts); } + #[test] + fn half_join_with_unit_payload_reaches_a_fixed_point() { + let text = r#"let rows = input 0 : (int, ()); + output "out" = rows | lift + | half_join(rows, 0, (k, (_, t)) -> (k, ((), t))) | count;"#; + let mut program = lower::lower_tree(parse::pipe::parse(text)); + crate::passes::optimize(&mut program); + // The unread time can be masked on the first pass. The unit payload must + // not keep reporting changes until pushdown gives up and reverts the plan. + assert!(pushdown(&mut program, &|_| None).unwrap() > 0); + let once = format!("{program:?}"); + assert_eq!(pushdown(&mut program, &|_| None).unwrap(), 0); + assert_eq!(format!("{program:?}"), once); + } + #[test] fn masked_half_join_matches_the_original_rows() { use crate::server::Server; @@ -1181,11 +1108,11 @@ mod tests { let b = input 1 : ((int, int, int), ()) | map(((k, y, z), ()) -> (k, (y, z))); output \"out\" = a | lift - | half_join(b, 0, (k, x, yz) -> (k, x)) | count;"; + | half_join(b, 0, (k, ((x, yz), t)) -> (k, (x, t))) | count;"; let mut original = lower::lower_tree(parse::pipe::parse(text)); crate::passes::optimize(&mut original); let mut projected = original.clone(); - assert_eq!(pushdown(&mut projected, &|_| None).unwrap(), 4); + assert_eq!(pushdown(&mut projected, &|_| None).unwrap(), 5); timely::execute_directly(move |worker| { let mut server = Server::new(); server.install(worker, "before", &original).unwrap(); diff --git a/interactive/src/plan.rs b/interactive/src/plan.rs index 84fd78d90..21c33b7ba 100644 --- a/interactive/src/plan.rs +++ b/interactive/src/plan.rs @@ -412,14 +412,14 @@ fn node(node: &Node, names: &Names) -> String { Node::Arrange(input) => format!("{} | arrange", names.of(input)), Node::Join { left, right, projection } if projection.returns_input() => format!("{} | join({})", names.of(left), names.of(right)), Node::Join { left, right, projection } => format!("{} | join({}, {})", names.of(left), names.of(right), source_lambda(projection)), + Node::HalfJoin { stream, arrangement, strict, projection } if projection.returns_input() => format!("{} | half_join({}, strict={strict})", names.of(stream), names.of(arrangement)), Node::HalfJoin { stream, arrangement, strict, projection } => format!("{} | half_join({}, strict={strict}, {})", names.of(stream), names.of(arrangement), source_lambda(projection)), Node::Reduce { input, reducer: r, limit } => format!("{} | {}", names.of(input), reducer(r, *limit)), Node::Inspect { input, label } => format!("{} | inspect({})", names.of(input), quote(label)), } } -// Slot inputs mirror shapes::linear_shapes and the join/reduce typing rules. In particular a -// half join's lambda sees neither the lifted time nor the arrangement's time. +// Slot inputs mirror shapes::linear_shapes and the join/reduce typing rules. fn printed_node(node: &Node, names: &Names, shapes: &ScopeShapes, depth: usize, at: &str, slots: &mut Slots) -> String { match node { Node::Linear { input, ops } => { @@ -446,11 +446,8 @@ fn printed_node(node: &Node, names: &Names, shapes: &ScopeShapes, depth: usize, let input = shapes.of(left).zip(shapes.of(right)).map(|(l, r)| Shape::Prod(vec![l.key, Shape::Prod(vec![l.val, r.val])])); format!("{} | join({}, {})", names.of(left), names.of(right), slots.lambda(projection, input, &format!("{at}/join"))) } - Node::HalfJoin { stream, arrangement, strict, projection } => { - let input = shapes.of(stream).zip(shapes.of(arrangement)).and_then(|(s, a)| { - let Shape::Prod(fs) = s.val else { return None }; - (fs.len() == 2).then(|| Shape::Prod(vec![s.key, fs[0].clone(), a.val])) - }); + Node::HalfJoin { stream, arrangement, strict, projection } if !projection.returns_input() => { + let input = shapes.of(stream).zip(shapes.of(arrangement)).and_then(|(s, a)| s.half_join_row(a.val).ok()); format!("{} | half_join({}, strict={strict}, {})", names.of(stream), names.of(arrangement), slots.lambda(projection, input, &format!("{at}/half_join"))) } Node::Reduce { input, reducer: Reducer::Expr(l), limit } => { @@ -646,7 +643,7 @@ mod tests { let provider = report(r#"output "rows" = input 0 : (int, int);"#, &|_| None, &Config::all()).unwrap(); let text = r#" let rows = import "rows" : (int, int); - output "out" = rows | lift | half_join(rows, 1, (k, v, w) -> (k, v)); + output "out" = rows | lift | half_join(rows, 1, (k, ((v, w), t)) -> (k, (v, t))); "#; let missing = report(text, &|_| None, &Config::all()).unwrap(); assert_eq!(missing.unresolved, ["rows"]); diff --git a/interactive/src/plan/compare.rs b/interactive/src/plan/compare.rs index 9d0da83c1..cbd3111a7 100644 --- a/interactive/src/plan/compare.rs +++ b/interactive/src/plan/compare.rs @@ -64,8 +64,7 @@ fn input_shapes(node: &Node, shapes: &ScopeShapes) -> Option> } Node::HalfJoin { stream, arrangement, .. } => { let (s, a) = (shapes.of(stream)?, shapes.of(arrangement)?); - let Shape::Prod(fields) = s.val else { return None }; - vec![Shape::Prod(vec![s.key, fields.first()?.clone(), a.val])] + vec![s.half_join_row(a.val).ok()?] } Node::Reduce { input, .. } => { let shape = shapes.of(input)?; diff --git a/interactive/src/plan/ddp.rs b/interactive/src/plan/ddp.rs index 5d96e1637..21823aa0d 100644 --- a/interactive/src/plan/ddp.rs +++ b/interactive/src/plan/ddp.rs @@ -414,11 +414,9 @@ impl Printer<'_> { let input = shapes.of(left).zip(shapes.of(right)).map(|(l, r)| Shape::Prod(vec![l.key, Shape::Prod(vec![l.val, r.val])])); format!("{} | join({}, {})", names.of(left), names.of(right), self.lambda(projection, input)?) } + Node::HalfJoin { stream, arrangement, strict, projection } if projection.returns_input() => format!("{} | half_join({}, {})", names.of(stream), names.of(arrangement), u8::from(*strict)), Node::HalfJoin { stream, arrangement, strict, projection } => { - let input = shapes.of(stream).zip(shapes.of(arrangement)).and_then(|(s, a)| { - let Shape::Prod(fs) = s.val else { return None }; - Some(Shape::Prod(vec![s.key, fs[0].clone(), a.val])) - }); + let input = shapes.of(stream).zip(shapes.of(arrangement)).and_then(|(s, a)| s.half_join_row(a.val).ok()); format!("{} | half_join({}, {}, {})", names.of(stream), names.of(arrangement), u8::from(*strict), self.lambda(projection, input)?) } Node::Reduce { input, reducer, limit } => { @@ -545,17 +543,21 @@ mod tests { #[test] fn identity_joins_print_without_a_projection() { - for (args, explicit) in [ - ("b", false), - ("b, x -> x", false), - ("b, (k, (l, r)) -> (k, (r, l))", true), + for (operator, args, explicit) in [ + ("join", "b", false), + ("join", "b, x -> x", false), + ("join", "b, (k, (l, r)) -> (k, (r, l))", true), + ("half_join", "b, 0", false), + ("half_join", "b, 0, x -> x", false), + ("half_join", "b, 0, (k, ((l, r), _)) -> (k, (r, l))", true), ] { + let left = if operator == "half_join" { "a | lift" } else { "a" }; let text = format!(r#"let a = input 0 : (int, int); let b = input 1 : (int, int); - output "out" = a | join({args});"#); + output "out" = {left} | {operator}({args});"#); let report = plan(&text); for stage in DdpStage::ALL { let printed = report.emit_ddp_at(stage).unwrap().text; - let join_args = printed.split_once("| join(").unwrap().1.split_once(';').unwrap().0; + let join_args = printed.split_once(&format!("| {operator}(")).unwrap().1.split_once(';').unwrap().0; assert_eq!(join_args.contains(" -> "), explicit, "{stage:?}: {printed}"); assert!(report.compare_final(&plan(&printed)).is_empty(), "{stage:?}: {printed}"); } @@ -646,7 +648,7 @@ mod tests { assert!(before.compare_final(&plan(&output.text)).is_empty()); assert_eq!(output.text, plan(&output.text).emit_ddp().unwrap().text); let before = plan(r#"let a = input 0 : (int, ()); let b = input 1 : (int, ()); - output "out" = a | lift | half_join(b, 1, (k, x, _) -> (k, x));"#); + output "out" = a | lift | half_join(b, 1, (k, ((x, _), t)) -> (k, (x, t)));"#); let output = before.emit_ddp().unwrap().text; assert!(output.contains("| lift | filter("), "{output}"); assert!(!output.contains("Added by")); diff --git a/interactive/src/plan/ddp/parsed.rs b/interactive/src/plan/ddp/parsed.rs index 87b8a7999..280c59d2a 100644 --- a/interactive/src/plan/ddp/parsed.rs +++ b/interactive/src/plan/ddp/parsed.rs @@ -71,6 +71,7 @@ impl Printer<'_> { } Expr::Join(left, right, l) if l.returns_input() => format!("{} | join({})", self.expression(left)?, self.expression(right)?), Expr::Join(left, right, l) => format!("{} | join({}, {})", self.expression(left)?, self.expression(right)?, self.parsed_lambda(l)?), + Expr::HalfJoin(stream, arrangement, strict, l) if l.returns_input() => format!("{} | half_join({}, {})", self.expression(stream)?, self.expression(arrangement)?, u8::from(*strict)), Expr::HalfJoin(stream, arrangement, strict, l) => format!("{} | half_join({}, {}, {})", self.expression(stream)?, self.expression(arrangement)?, u8::from(*strict), self.parsed_lambda(l)?), Expr::Reduce(input, reducer) => { let lambda = if matches!(reducer, crate::parse::Reducer::Distinct) { diff --git a/interactive/src/runtime/half_join/backend.rs b/interactive/src/runtime/half_join/backend.rs index 173d1d61a..2409d13a8 100644 --- a/interactive/src/runtime/half_join/backend.rs +++ b/interactive/src/runtime/half_join/backend.rs @@ -25,6 +25,7 @@ use crate::runtime::col_times::{ColTimes, Lanes}; use crate::runtime::container::CorgiContainer; use crate::runtime::join::{demanded, primitive_values, stage_collision, Probe, Reads, RunRef, SideProj, SideScratch, COORD_BITS}; use crate::lambda::compile_projection; +use crate::shapes::time_shape; use crate::ir::{Diff, Lambda, Time}; use super::{NativeBatch, Run, CC}; @@ -42,8 +43,8 @@ pub(crate) struct CorgiHalfJoinBackend { width: usize, /// The projection, narrowed to take its input projected by `reads.argument`. graph: Graph, - /// What the graph reads of the key, the update's value, and the arranged value. - reads: Reads, + /// What the graph reads of the key, stream value, arranged value, and joined data time. + reads: Reads<4>, /// Whether the stream value loses fields, and can coalesce previously distinct rows. project_stream: bool, /// Identifiers in the current block that cover more than one real key, ascending. `advance` @@ -57,9 +58,10 @@ impl CorgiHalfJoinBackend { /// type error is a panic with corgi's message: install type-checked the program, so it is not /// reached. pub(crate) fn new(projection: &Lambda, width: usize, [k, v0, v1]: [Shape; 3]) -> Self { - let compiled = compile_projection(projection, &[k.clone(), v0.clone(), v1.clone()]) - .unwrap_or_else(|e| panic!("half_join: type error at shapes ({k}, {v0}, {v1}): {e}")); - let input = Shape::Prod(vec![k, v0, v1]); + let shapes = [k, v0, v1, time_shape(width)]; + let compiled = compile_projection(&projection.on_half_join_columns(), &shapes) + .unwrap_or_else(|e| panic!("half_join: type error at {shapes:?}: {e}")); + let input = Shape::Prod(shapes.into()); let narrowed = corgi::narrow(compiled, &Demand::All) .unwrap_or_else(|e| panic!("half_join: narrowing a compiled projection at {input}: {e}")); let reads = Reads::new(narrowed.input, &input); @@ -256,8 +258,7 @@ impl ProxyHalfJoinBackend for CorgiHalfJoinBackend { /// The matched pairs' keys, values and arranged values gathered into columns and projected; /// pairs under a colliding identifier whose real keys differ are dropped first. The data times - /// are the tactic's rule, evaluated lane by lane: the presented times of the block's rows are - /// read into lanes once, and each match takes the lane-wise max. + /// follow the tactic's rule: each match takes the lane-wise maximum of the presented times. fn cross( &mut self, instance: &HalfJoinInstance>, @@ -317,11 +318,9 @@ impl ProxyHalfJoinBackend for CorgiHalfJoinBackend { } let pair = |i: usize| matches.pairs[kept.as_ref().map_or(i, |kept| kept[i])]; - // The projection over (key, val, arranged val), each projected to what it reads before - // gathering; the collision checks above compared the original keys. The time as data - // (the value's second field) is not the projection's to read: `advance` presented it. + // Gather only the fields needed by the projection, after checking the original keys. let rows = rs.len(); - let [dk, d0, d1] = &self.reads.columns; + let [dk, d0, d1, dt] = &self.reads.columns; let fields: Vec<&[CValue]> = runs.iter().map(run_fields).collect(); let gathered = |cols: &[Cow<'_, CValue>], rs: &[usize], us: &[usize]| { gather_lanes(&cols.iter().map(|c| Some(&**c)).collect::>(), rs, us) @@ -331,38 +330,37 @@ impl ProxyHalfJoinBackend for CorgiHalfJoinBackend { let arranged_vals = if *d1 == Demand::None { CValue::Unit(rows) } else if primitive { d1.project(rewrap(chunks[0].vals(), tokens)).expect("the arranged value's demand") } else { gathered(&demanded(d1, chunks.iter().map(|c| c.vals())), &cs, &js) }; - let argument = self.reads.argument.project(CValue::Prod(vec![out_keys, vals, arranged_vals])).expect("the half join's input has the shape its demand is on"); + // The joined data time is a fourth column. Build just the requested lanes of the + // coordinate-wise maximum; missing trailing coordinates in either time are zero. + assert!( + stream.iter().all(|(_, lifted, _)| lifted.data.len() <= self.width) + && arranged.iter().all(|(_, time, _)| time.len() <= self.width), + "half_join: a time wider than its scope", + ); + let time = if *dt == Demand::None { CValue::Unit(rows) } else { + let lanes = (0..self.width).map(|lane| { + if matches!(dt, Demand::Prod(fields) if fields[lane] == Demand::None) { return CValue::Unit(rows); } + let left: Vec<_> = stream.iter().map(|(_, lifted, _)| lifted.data.get(lane).copied().unwrap_or(0)).collect(); + let right: Vec<_> = arranged.iter().map(|(_, time, _)| time.get(lane).copied().unwrap_or(0)).collect(); + CValue::i64((0..rows).map(|i| { + let (a, b) = pair(i); + let coordinate = left[a].max(right[b]); + debug_assert_eq!( + coordinate, data_time(&stream[a].1, &arranged[b].1).get(lane).copied().unwrap_or(0), + "half_join: joined data-time lane {lane} disagrees with the tactic", + ); + coordinate as i64 + }).collect()) + }).collect(); + dt.project(CValue::Prod(lanes)).expect("the joined data time's demand") + }; + let argument = self.reads.argument.project(CValue::Prod(vec![out_keys, vals, arranged_vals, time])).expect("the half join's input has the shape its demand is on"); let shape = shape_of_value(&argument); assert!(shape == self.reads.shape, "half_join: a block read as {shape} reached a projection narrowed to {}", self.reads.shape); let mut out = corgi::eval_graph(&self.graph, argument).into_prod("half_join projection").unwrap(); let out_val = out.pop().unwrap(); let out_key = out.pop().unwrap(); - // The presented times as lanes, once per block: a time is as wide as its last nonzero - // coordinate, and the rest are zero. - let width = self.width; - let to_lanes = |times: &mut dyn Iterator, rows: usize| -> Vec> { - let mut lanes = vec![vec![0i64; rows]; width]; - for (i, t) in times.enumerate() { - for (l, c) in t.coordinates().enumerate() { - assert!(l < width, "half_join: a time wider than its scope"); - lanes[l][i] = c as i64; - } - } - lanes - }; - let data = to_lanes(&mut stream.iter().map(|row| &row.1.data), stream.len()); - let time2 = to_lanes(&mut arranged.iter().map(|row| &row.1), arranged.len()); - // The data time of each match: the lane-wise max is the join in a product of chains. - let joined: Vec = (0..width).map(|l| { - CValue::i64((0..rows).map(|i| { let (a, b) = pair(i); data[l][a].max(time2[l][b]) }).collect()) - }).collect(); - debug_assert!((0..rows).all(|i| { - let (a, b) = pair(i); - let t = data_time(&stream[a].1, &arranged[b].1); - let coords: Vec = t.coordinates().map(|c| c as i64).collect(); - coords.iter().enumerate().all(|(l, &c)| joined[l].as_i64("lane").unwrap()[i] == c) - }), "half_join: the lane-wise data time must agree with the tactic's rule"); // Each row keeps its update's own time, which the unit's capabilities cover: the driver // ships the block stamped with all of them. let mut times = ColTimes::new(); @@ -373,7 +371,7 @@ impl ProxyHalfJoinBackend for CorgiHalfJoinBackend { }; output.push(CorgiContainer { keys: out_key, - vals: CValue::Prod(vec![out_val, CValue::Prod(joined.into_iter().collect())]), + vals: out_val, times, diffs, }); @@ -393,14 +391,29 @@ mod tests { use super::super::test_util::*; use crate::runtime::chunk::{columns_to_batch, present_key}; - /// The projection `(k, (v0, v1))`. - fn pairing() -> Lambda { Lambda::new("let (k, v0, v1) = input in (k, (v0, v1))".into(), &corgi::Scope::default()) } + /// Keep the complete timed match row. + fn pairing() -> Lambda { Lambda::new("let row = input in row".into(), &corgi::Scope::default()) } /// A tactic over the backend at these shapes. fn tactic(strict: bool, shapes: [Shape; 3]) -> ProxyHalfJoinTactic { ProxyHalfJoinTactic::new(CorgiHalfJoinBackend::new(&pairing(), 2, shapes), strict) } + #[test] + #[should_panic(expected = "half_join: a time wider than its scope")] + fn rejects_an_extra_coordinate_even_when_time_is_unread() { + let batch = Rc::new(columns_to_batch( + present_key(ints(&[1])), ints(&[2]), + [Time::from_coordinates([0, 0, 1].into_iter())].into_iter().collect(), vec![1], + )); + let rows = container(&[1], &[3], &[(1, 0)], &[t(1, 0)], &[1]); + let run = Run::new(rows.keys, rows.vals, rows.times, rows.diffs); + let projection = Lambda::new("let (k, _) = input in (k, ())".into(), &corgi::Scope::default()); + let backend = CorgiHalfJoinBackend::new(&projection, 2, [Shape::Int, Shape::Int, Shape::Int]); + let mut tactic = ProxyHalfJoinTactic::new(backend, true); + tactic.prep(vec![run], vec![batch], Antichain::from_elem(t(1, 0))).for_each(drop); + } + /// Projection cancels unread churn, but neither of a lifted update's times may be merged. #[test] fn projected_churn_keeps_both_stream_times() { @@ -414,7 +427,7 @@ mod tests { lifted(&[(3, 0), (3, 0), (4, 0), if change_data { (5, 0) } else { (4, 0) }]), ]), [t(2, 0), t(2, 0), t(2, 0), if change_data { t(2, 0) } else { t(3, 0) }].into_iter().collect(), vec![1, -1, 1, -1]); let pair = Shape::Prod(vec![Shape::Int, Shape::Int]); - let lambda = Lambda::new("let (k, (a, _), (b, _)) = input in (k, (a, b))".into(), &corgi::Scope::default()); + let lambda = Lambda::new("let (k, (((a, _), (b, _)), _)) = input in (k, (a, b))".into(), &corgi::Scope::default()); let mut backend = CorgiHalfJoinBackend::new(&lambda, 2, [Shape::Int, pair.clone(), pair]); let instance = HalfJoinInstance { stream: vec![run], batches: vec![arranged] }; let (mut stream, mut arranged) = (Vec::new(), Vec::new()); @@ -595,7 +608,9 @@ mod tests { for i in 0..n { for (k2, v1, t2, d2) in arranged_rows.iter() { if *k2 != ks[i] || !(*t2 < ts[i] || (!strict && *t2 == ts[i])) { continue; } - let s = tup(vec![int(ss[i].0.max(t2.first().copied().unwrap_or(0) as i64)), int(ss[i].1.max(t2.get(1).copied().unwrap_or(0) as i64))]); + let lifted = Lifted { time: ts[i].clone(), data: t(ss[i].0 as u64, ss[i].1 as u64) }; + let joined = data_time(&lifted, t2); + let s = tup((0..2).map(|lane| int(joined.get(lane).copied().unwrap_or(0) as i64)).collect()); expect.push(((key_val(ks[i]), tup(vec![tup(vec![int(vs[i]), int(*v1)]), s])), ts[i].clone(), ds[i] * d2)); } } @@ -640,13 +655,14 @@ mod tests { let lower: Antichain