Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions interactive/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,12 @@ For each pair of matches `(k, l)` and `(k, r)`, `left | join(right)` produces `(
Passing `f` to the join lets it emit the projected fields directly. A separate `map` can require the full joined row as an intermediate result.
Existing three-field join patterns migrate from `(k, l, r) -> …` to `(k, (l, r)) -> …`.

For updates from `left`, `left | lift | half_join(right, 0)` produces `(k, ((l, r), t))`.
Here `t` is the coordinate-wise maximum of the lifted data time and the matching right update's timestamp.
Use `0` to include equal timestamps and `1` for strictly earlier right updates in the total order.
An optional projection receives the entire row. For example, `half_join(right, 0, (k, ((l, r), _)) -> (k, r))` drops the data time.
The output update still has the left update's timestamp. To migrate an old projection `(k, l, r) -> (k2, v2)`, write `(k, ((l, r), t)) -> (k2, (v2, t))` to retain its previous output.

A currently unfortunate amount of symbolic load is in shuffling around fields of collection elements.
The `map` operator applies to each record, and the `->` fragment is an expression converting the input to the output.
`flatmap` gives a `List((key, val))` of complete rows, choosing both keys and values.
Expand Down
4 changes: 4 additions & 0 deletions interactive/src/lambda.rs
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,10 @@ impl Lambda {
pub(crate) fn on_join_columns(&self) -> Self {
Self::compose("(k, l, r)", &[(self, "(k, (l, r))".into())], |out| out[0].clone())
}
/// Apply a half-join projection to the gathered columns and their joined data time.
pub(crate) fn on_half_join_columns(&self) -> Self {
Self::compose("(k, l, r, t)", &[(self, "(k, ((l, r), t))".into())], |out| out[0].clone())
}
/// The graph, or the parse error naming the program.
pub(crate) fn graph(&self) -> Result<&std::sync::Arc<corgi::Graph<corgi::NumOp>>, String> {
self.graph.as_ref().map_err(|e| format!("corgi program `{}`: {e}", self.src))
Expand Down
4 changes: 2 additions & 2 deletions interactive/src/lower.rs
Original file line number Diff line number Diff line change
Expand Up @@ -638,7 +638,7 @@ mod tree_tests {
let text = format!(r#"
let rows = input 0 : (int, ()) | {producer};
output "join" = rows | join(rows, (k, (_, _)) -> (k, ()));
output "half" = rows | lift | half_join(rows, 0, (k, _, _) -> (k, ()));
output "half" = rows | lift | half_join(rows, 0, (k, ((_, _), t)) -> (k, ((), t)));
output "reduce" = rows | min;
output "distinct" = rows | distinct;
"#);
Expand All @@ -657,7 +657,7 @@ mod tree_tests {
let program = lower_tree(parse(r#"
let rows = input 0 : (int, int) | distinct;
output "join" = rows | join(rows, (k, (a, _)) -> (k, a));
output "half" = rows | lift | half_join(rows, 0, (k, a, _) -> (k, a));
output "half" = rows | lift | half_join(rows, 0, (k, ((a, _), t)) -> (k, (a, t)));
output "reduce" = rows | min;
output "distinct" = rows | distinct;
"#));
Expand Down
24 changes: 14 additions & 10 deletions interactive/src/parse/pipe.rs
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,11 @@
//! of the epoch and every scope coordinate.
//! - `| join_function((k, v) -> list)` — replace each row by the lambda's list of
//! `((key, val), time, diff)`, at the row's time joined with `time`, diffs multiplied.
//! - `| half_join(other, strict, (k, v, v1) -> (k', v'))` — each update of a lifted stream,
//! - `| half_join(other, strict)` — each update of a lifted stream,
//! `(k, (v, time))`, meets `other`'s rows at its key whose times precede the update's own in the
//! total order on times (strictly when `strict` is 1, or also equal when 0), giving
//! `(k', (v', time ∨ t2))`. The lambda does not see the times. `other`'s changes produce nothing.
//! `(k, ((v, v1), time ∨ t2))`. `half_join(other, strict, f)` projects this whole row.
//! The output retains the stream update's timestamp. `other`'s changes produce nothing.
//! - `| delay((k, v) -> time)` — `join_function` with the list `[((key, val), time, 1)]`: advance
//! each row to at least the time the lambda gives. The tuple names the epoch, then each
//! enclosing loop from outermost to innermost; `0` leaves a coordinate unchanged. Inside one
Expand Down Expand Up @@ -666,19 +667,23 @@ impl Parser {
Some((key, val))
}

fn join_projection(&mut self) -> Lambda {
if *self.peek() == Token::Comma {
self.next();
self.lambda()
} else {
Lambda::new("let row = input in row".into(), &corgi::Scope::default())
}
}

fn parse_pipe_op(&mut self, lhs: Expr) -> Expr {
match self.peek().clone() {
Token::Map => { self.next(); Expr::Map(Box::new(lhs), self.slot()) },
Token::Join => {
self.next();
self.expect(&Token::LParen);
let rhs = self.parse_pipe_expr();
let projection = if *self.peek() == Token::Comma {
self.next();
self.lambda()
} else {
Lambda::new("let row = input in row".into(), &corgi::Scope::default())
};
let projection = self.join_projection();
self.expect(&Token::RParen);
Expr::Join(Box::new(lhs), Box::new(rhs), projection)
},
Expand Down Expand Up @@ -727,8 +732,7 @@ impl Parser {
Token::Int(1) => true,
other => panic!("half_join: strict is 0 or 1, not {other:?}"),
};
self.expect(&Token::Comma);
let p = self.lambda();
let p = self.join_projection();
self.expect(&Token::RParen);
Expr::HalfJoin(Box::new(lhs), Box::new(r), strict, p)
},
Expand Down
2 changes: 1 addition & 1 deletion interactive/src/passes/arrangements_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ fn variant_inputs_and_half_joins_stop_motion_before_fusion() {
let p = rewrite(r#"
let x = input 0 : (int, int);
let timed = x | map((k, v) -> (k, (v, (0, 0))));
child: { let out = ::timed | half_join(::x, 0, (k, l, r) -> (k, l)); }
child: { let out = ::timed | half_join(::x, 0, (k, ((l, r), t)) -> (k, (l, t))); }
output "out" = child::out;
"#, hoist(), 0);
let Item::Sub(child) = p.root.items.last().unwrap() else { panic!() };
Expand Down
24 changes: 12 additions & 12 deletions interactive/src/passes/as_of.rs
Original file line number Diff line number Diff line change
Expand Up @@ -221,8 +221,8 @@ mod tests {
let (prog, inserted) = rewrite(r#"
let a = import "a";
let b = import "b";
output "r" = ((a | lift | half_join(b, 0, (k, x, y) -> (k, (x, y))))
+ (b | lift | half_join(a, 1, (k, y, x) -> (k, (x, y)))));
output "r" = ((a | lift | half_join(b, 0))
+ (b | lift | half_join(a, 1, (k, ((y, x), t)) -> (k, ((x, y), t)))));
"#);
assert_eq!(inserted, 1);
let found = filters(&prog.root);
Expand All @@ -238,9 +238,9 @@ mod tests {
let a = import "a";
let b = import "b";
let c = import "c";
let ab = a | lift | half_join(b, 0, (k, x, y) -> (k, x)) | half_join(c, 0, (k, x, z) -> (k, x));
let ba = b | lift | half_join(a, 1, (k, y, x) -> (k, x)) | half_join(c, 0, (k, x, z) -> (k, x));
let ca = c | lift | half_join(a, 1, (k, z, x) -> (k, x)) | half_join(b, 1, (k, x, y) -> (k, x));
let ab = a | lift | half_join(b, 0, (k, ((x, y), t)) -> (k, (x, t))) | half_join(c, 0, (k, ((x, z), t)) -> (k, (x, t)));
let ba = b | lift | half_join(a, 1, (k, ((y, x), t)) -> (k, (x, t))) | half_join(c, 0, (k, ((x, z), t)) -> (k, (x, t)));
let ca = c | lift | half_join(a, 1, (k, ((z, x), t)) -> (k, (x, t))) | half_join(b, 1, (k, ((x, y), t)) -> (k, (x, t)));
output "r" = ab + ba + ca;
"#);
assert_eq!(inserted, 2, "every path but `a`'s, which is non-strict throughout");
Expand All @@ -250,8 +250,8 @@ mod tests {
fn a_shared_lift_filters_only_the_edge_to_its_strict_reader() {
let (prog, inserted) = rewrite(r#"
let p = import "p";
output "r" = ((p | lift | half_join(p, 0, (k, x, y) -> (k, (x, y))))
+ (p | lift | half_join(p, 1, (k, x, y) -> (k, (x, y)))));
output "r" = ((p | lift | half_join(p, 0))
+ (p | lift | half_join(p, 1)));
"#);
assert_eq!(inserted, 1);
let found = filters(&prog.root);
Expand All @@ -268,7 +268,7 @@ mod tests {
let (prog, inserted) = rewrite(r#"
let p = import "p";
let lifted = p | lift | filter((k, _) -> (k, k) eq) | negate;
output "strict" = lifted | half_join(p, 1, (k, x, y) -> (k, (x, y)));
output "strict" = lifted | half_join(p, 1);
output "live" = lifted;
"#);
assert_eq!(inserted, 1);
Expand All @@ -288,7 +288,7 @@ mod tests {
let a = import "a";
let b = import "b";
let la = a | lift;
output "r" = la | half_join(b, 1, (k, x, y) -> (k, (x, y)));
output "r" = la | half_join(b, 1);
output "lifted" = la;
"#);
assert_eq!(inserted, 1, "the edge to the strict half join is filtered");
Expand All @@ -299,13 +299,13 @@ mod tests {
let (_, live) = rewrite(r#"
let a = import "a";
let b = import "b";
output "r" = a | lift | half_join(b, 0, (k, x, y) -> (k, (x, y)));
output "r" = a | lift | half_join(b, 0);
"#);
assert_eq!(live, 0, "a non-strict path is live at the as-of time");
let (_, moved) = rewrite(r#"
let a = import "a";
let b = import "b";
output "r" = a | lift | delay((_, (_, t)) -> t) | half_join(b, 1, (k, x, y) -> (k, (x, y)));
output "r" = a | lift | delay((_, (_, t)) -> t) | half_join(b, 1);
"#);
assert_eq!(moved, 0, "a delay between the lift and the strict half join can move the time");
}
Expand All @@ -316,7 +316,7 @@ mod tests {
let a = import "a";
let b = import "b";
s: {
let r = a | lift | half_join(b, 1, (k, x, y) -> (k, x)) | map((k, (x, _)) -> (k, x));
let r = a | lift | half_join(b, 1, (k, ((x, y), t)) -> (k, (x, t))) | map((k, (x, _)) -> (k, x));
set x = (r + x) | distinct;
}
output "r" = s::x;
Expand Down
Loading
Loading