From 9e8d8efcd8a4b4d1321e1620d0ad5b87a7c352d0 Mon Sep 17 00:00:00 2001 From: Frank McSherry Date: Fri, 20 Nov 2026 11:50:21 -0500 Subject: [PATCH 1/2] DDIR: make the join projection optional and give it the joined row --- interactive/README.md | 9 +- interactive/bench/ldbc/authored/README.md | 2 +- .../bench/ldbc/authored/plans/bi01.ddp | 6 +- .../bench/ldbc/authored/plans/bi02.ddp | 10 +- .../bench/ldbc/authored/plans/bi03.ddp | 22 ++--- .../bench/ldbc/authored/plans/bi04.ddp | 22 ++--- .../bench/ldbc/authored/plans/bi05.ddp | 10 +- .../bench/ldbc/authored/plans/bi06.ddp | 14 +-- .../bench/ldbc/authored/plans/bi07.ddp | 12 +-- .../bench/ldbc/authored/plans/bi08.ddp | 18 ++-- .../bench/ldbc/authored/plans/bi09.ddp | 18 ++-- .../bench/ldbc/authored/plans/bi10.ddp | 24 ++--- .../bench/ldbc/authored/plans/bi11.ddp | 16 ++-- .../bench/ldbc/authored/plans/bi12.ddp | 10 +- .../bench/ldbc/authored/plans/bi13.ddp | 24 ++--- .../bench/ldbc/authored/plans/bi14.ddp | 32 +++---- .../bench/ldbc/authored/plans/bi15.ddp | 28 +++--- .../bench/ldbc/authored/plans/bi16.ddp | 16 ++-- .../bench/ldbc/authored/plans/bi17.ddp | 24 ++--- .../bench/ldbc/authored/plans/bi18.ddp | 10 +- .../bench/ldbc/authored/plans/bi19.ddp | 14 +-- .../bench/ldbc/authored/plans/bi20.ddp | 16 ++-- .../bench/ldbc/authored/plans/ic01.ddp | 42 ++++----- .../bench/ldbc/authored/plans/ic02.ddp | 12 +-- .../bench/ldbc/authored/plans/ic03.ddp | 24 ++--- .../bench/ldbc/authored/plans/ic04.ddp | 10 +- .../bench/ldbc/authored/plans/ic05.ddp | 20 ++-- .../bench/ldbc/authored/plans/ic06.ddp | 18 ++-- .../bench/ldbc/authored/plans/ic07.ddp | 10 +- .../bench/ldbc/authored/plans/ic08.ddp | 10 +- .../bench/ldbc/authored/plans/ic09.ddp | 16 ++-- .../bench/ldbc/authored/plans/ic10.ddp | 18 ++-- .../bench/ldbc/authored/plans/ic11.ddp | 14 +-- .../bench/ldbc/authored/plans/ic12.ddp | 24 ++--- .../bench/ldbc/authored/plans/ic13.ddp | 12 +-- .../bench/ldbc/authored/plans/ic14.ddp | 24 ++--- .../bench/ldbc/authored/plans/is01.ddp | 2 +- .../bench/ldbc/authored/plans/is02.ddp | 12 +-- .../bench/ldbc/authored/plans/is03.ddp | 4 +- .../bench/ldbc/authored/plans/is04.ddp | 2 +- .../bench/ldbc/authored/plans/is05.ddp | 4 +- .../bench/ldbc/authored/plans/is06.ddp | 8 +- .../bench/ldbc/authored/plans/is07.ddp | 10 +- interactive/bench/ldbc/bi11.ddp | 12 +-- interactive/bench/ldbc/bi5.ddp | 8 +- interactive/bench/ldbc/graph.ddp | 6 +- interactive/bench/ldbc/ic6.ddp | 10 +- .../bench/ldbc/interactive_v2/graph.py | 14 +-- .../interactive_v2/oneoff/ic01.oneoff.ddp | 42 ++++----- .../interactive_v2/oneoff/ic02.oneoff.ddp | 12 +-- .../interactive_v2/oneoff/ic03.oneoff.ddp | 24 ++--- .../interactive_v2/oneoff/ic04.oneoff.ddp | 8 +- .../interactive_v2/oneoff/ic04.standing.ddp | 2 +- .../interactive_v2/oneoff/ic05.oneoff.ddp | 20 ++-- .../interactive_v2/oneoff/ic06.oneoff.ddp | 18 ++-- .../interactive_v2/oneoff/ic07.oneoff.ddp | 10 +- .../interactive_v2/oneoff/ic08.oneoff.ddp | 10 +- .../interactive_v2/oneoff/ic09.oneoff.ddp | 16 ++-- .../interactive_v2/oneoff/ic10.oneoff.ddp | 16 ++-- .../interactive_v2/oneoff/ic10.standing.ddp | 2 +- .../interactive_v2/oneoff/ic11.oneoff.ddp | 14 +-- .../interactive_v2/oneoff/ic12.oneoff.ddp | 22 ++--- .../interactive_v2/oneoff/ic12.standing.ddp | 2 +- .../interactive_v2/oneoff/ic13.oneoff.ddp | 12 +-- .../interactive_v2/oneoff/ic14.oneoff.ddp | 20 ++-- .../interactive_v2/oneoff/ic14.standing.ddp | 4 +- interactive/bench/ldbc/is3.ddp | 4 +- interactive/bench/ldbc/snb/indexes.py | 6 +- interactive/bench/ldbc/snb/rel.py | 8 +- interactive/bench/txn/maintained.ddp | 8 +- interactive/bench/txn/resolve.ddp | 12 +-- interactive/bench/txn/resolve_by_key.ddp | 8 +- interactive/examples/aoc2023/day01/part1.ddp | 2 +- interactive/examples/aoc2023/day01/part2.ddp | 10 +- interactive/examples/aoc2023/day02/part2.ddp | 4 +- interactive/examples/aoc2023/day03/part1.ddp | 12 +-- interactive/examples/aoc2023/day03/part2.ddp | 12 +-- interactive/examples/aoc2023/day04/part2.ddp | 6 +- interactive/examples/aoc2023/day05/part1.ddp | 2 +- interactive/examples/aoc2023/day05/part2.ddp | 4 +- interactive/examples/aoc2023/day06/part1.ddp | 2 +- interactive/examples/aoc2023/day07/part1.ddp | 4 +- interactive/examples/aoc2023/day07/part2.ddp | 4 +- interactive/examples/aoc2023/day10/part1.ddp | 2 +- interactive/examples/aoc2023/day10/part2.ddp | 14 +-- interactive/examples/aoc2023/day11/part1.ddp | 12 +-- interactive/examples/aoc2023/day11/part2.ddp | 12 +-- interactive/examples/aoc2023/day13/part1.ddp | 8 +- interactive/examples/aoc2023/day13/part2.ddp | 8 +- interactive/examples/aoc2023/day14/part1.ddp | 6 +- interactive/examples/aoc2023/day14/part2.ddp | 22 ++--- interactive/examples/aoc2023/day15/part1.ddp | 6 +- interactive/examples/aoc2023/day15/part2.ddp | 12 +-- interactive/examples/aoc2023/day16/part1.ddp | 6 +- interactive/examples/aoc2023/day16/part2.ddp | 14 +-- interactive/examples/aoc2023/day17/part1.ddp | 10 +- interactive/examples/aoc2023/day17/part2.ddp | 10 +- interactive/examples/aoc2023/day18/part1.ddp | 8 +- interactive/examples/aoc2023/day18/part2.ddp | 8 +- interactive/examples/aoc2023/day19/part1.ddp | 2 +- interactive/examples/aoc2023/day19/part2.ddp | 4 +- interactive/examples/aoc2023/day22/part1.ddp | 12 +-- interactive/examples/aoc2023/day22/part2.ddp | 14 +-- interactive/examples/programs/kcore.ddp | 2 +- interactive/examples/programs/reach.ddp | 2 +- interactive/examples/programs/scc.ddp | 12 +-- interactive/examples/programs/tour.ddp | 2 +- .../examples/server/programs/access.ddp | 6 +- .../examples/server/programs/arrange_idem.ddp | 12 +-- .../examples/server/programs/reach_import.ddp | 2 +- interactive/server/demo/txn.txt | 8 +- interactive/server/diag.ddp | 4 +- interactive/server/src/cmd.rs | 4 +- interactive/server/tests/multiworker.rs | 4 +- interactive/src/backend/corgi.rs | 2 +- interactive/src/explain/decouple.rs | 10 +- interactive/src/explain/mod.rs | 8 +- interactive/src/lambda.rs | 8 +- interactive/src/lower.rs | 14 +-- interactive/src/parse/pipe.rs | 19 +++- interactive/src/parse/pipe/fragment.rs | 4 +- interactive/src/passes/arrangements_tests.rs | 6 +- interactive/src/passes/projection.rs | 2 +- interactive/src/plan.rs | 2 +- interactive/src/plan/compare.rs | 2 +- interactive/src/plan/ddp.rs | 6 +- interactive/src/scope_ir.rs | 2 +- interactive/src/server/read.rs | 2 +- interactive/src/shapes.rs | 4 +- interactive/tests/bitemporal.ddp | 6 +- interactive/tests/corgi_backend.rs | 36 ++++++++ interactive/tests/dedup.rs | 2 +- interactive/tests/diag.rs | 2 +- interactive/tests/difftest.rs | 12 +-- interactive/tests/explain.rs | 50 +++++----- interactive/tests/fragments.rs | 2 +- interactive/tests/half_join.rs | 4 +- interactive/tests/labels.rs | 4 +- interactive/tests/pass_roundtrip.rs | 6 +- .../tests/plan_pairs/cases/access_shared.ddp | 6 +- interactive/tests/plan_pairs/cases/nested.ddp | 2 +- .../plan_pairs/expected/access_shared.ddp | 6 +- .../access_shared.lowered--optimized.diff | 12 +-- .../access_shared.parsed--lowered.diff | 12 +-- .../expected/access_shared.parsed.ddp | 6 +- .../tests/plan_pairs/expected/explain.ddp | 8 +- .../expected/explain.lowered--optimized.diff | 16 ++-- .../plan_pairs/expected/explain.lowered.ddp | 8 +- .../explain.optimized--projected.diff | 6 +- .../plan_pairs/expected/ldbc_cascade.ddp | 54 +++++------ .../ldbc_cascade.lowered--optimized.diff | 24 ++--- .../ldbc_cascade.parsed--lowered.diff | 92 +++++++++---------- .../expected/ldbc_cascade.parsed.ddp | 38 ++++---- .../tests/plan_pairs/expected/ldbc_graph.ddp | 6 +- .../ldbc_graph.lowered--optimized.diff | 4 +- .../expected/ldbc_graph.parsed--lowered.diff | 10 +- .../plan_pairs/expected/ldbc_graph.parsed.ddp | 4 +- .../plan_pairs/expected/ldbc_ic01_oneoff.ddp | 42 ++++----- .../ldbc_ic01_oneoff.lowered--optimized.diff | 66 ++++++------- .../ldbc_ic01_oneoff.parsed--lowered.diff | 66 ++++++------- .../expected/ldbc_ic01_oneoff.parsed.ddp | 24 ++--- .../plan_pairs/expected/ldbc_ic02_oneoff.ddp | 12 +-- .../ldbc_ic02_oneoff.lowered--optimized.diff | 2 +- .../ldbc_ic02_oneoff.parsed--lowered.diff | 22 ++--- .../expected/ldbc_ic02_oneoff.parsed.ddp | 10 +- .../plan_pairs/expected/ldbc_ic03_oneoff.ddp | 24 ++--- .../ldbc_ic03_oneoff.lowered--optimized.diff | 32 +++---- .../ldbc_ic03_oneoff.parsed--lowered.diff | 38 ++++---- .../expected/ldbc_ic03_oneoff.parsed.ddp | 16 ++-- .../plan_pairs/expected/ldbc_ic04_oneoff.ddp | 8 +- .../ldbc_ic04_oneoff.lowered--optimized.diff | 2 +- ...ldbc_ic04_oneoff.optimized--projected.diff | 4 +- .../ldbc_ic04_oneoff.parsed--lowered.diff | 12 +-- .../expected/ldbc_ic04_oneoff.parsed.ddp | 4 +- .../expected/ldbc_ic04_standing.ddp | 2 +- ...ldbc_ic04_standing.lowered--optimized.diff | 2 +- .../ldbc_ic04_standing.parsed--lowered.diff | 4 +- .../expected/ldbc_ic04_standing.parsed.ddp | 2 +- .../plan_pairs/expected/ldbc_ic05_oneoff.ddp | 20 ++-- .../ldbc_ic05_oneoff.lowered--optimized.diff | 18 ++-- .../ldbc_ic05_oneoff.parsed--lowered.diff | 38 ++++---- .../expected/ldbc_ic05_oneoff.parsed.ddp | 18 ++-- .../plan_pairs/expected/ldbc_ic06_oneoff.ddp | 18 ++-- .../ldbc_ic06_oneoff.lowered--optimized.diff | 8 +- .../ldbc_ic06_oneoff.parsed--lowered.diff | 28 +++--- .../expected/ldbc_ic06_oneoff.parsed.ddp | 10 +- .../plan_pairs/expected/ldbc_ic07_oneoff.ddp | 10 +- ...ldbc_ic07_oneoff.optimized--projected.diff | 6 +- .../ldbc_ic07_oneoff.parsed--lowered.diff | 18 ++-- .../expected/ldbc_ic07_oneoff.parsed.ddp | 8 +- .../plan_pairs/expected/ldbc_ic08_oneoff.ddp | 10 +- .../ldbc_ic08_oneoff.lowered--optimized.diff | 8 +- .../ldbc_ic08_oneoff.parsed--lowered.diff | 14 +-- .../expected/ldbc_ic08_oneoff.parsed.ddp | 4 +- .../plan_pairs/expected/ldbc_ic09_oneoff.ddp | 16 ++-- .../ldbc_ic09_oneoff.lowered--optimized.diff | 14 +-- ...ldbc_ic09_oneoff.optimized--projected.diff | 2 +- .../ldbc_ic09_oneoff.parsed--lowered.diff | 28 +++--- .../expected/ldbc_ic09_oneoff.parsed.ddp | 12 +-- .../plan_pairs/expected/ldbc_ic10_oneoff.ddp | 16 ++-- .../ldbc_ic10_oneoff.lowered--optimized.diff | 8 +- .../ldbc_ic10_oneoff.parsed--lowered.diff | 30 +++--- .../expected/ldbc_ic10_oneoff.parsed.ddp | 14 +-- .../expected/ldbc_ic10_standing.ddp | 2 +- ...ldbc_ic10_standing.lowered--optimized.diff | 2 +- .../ldbc_ic10_standing.parsed--lowered.diff | 4 +- .../expected/ldbc_ic10_standing.parsed.ddp | 2 +- .../plan_pairs/expected/ldbc_ic11_oneoff.ddp | 14 +-- .../ldbc_ic11_oneoff.lowered--optimized.diff | 12 +-- .../ldbc_ic11_oneoff.parsed--lowered.diff | 22 ++--- .../expected/ldbc_ic11_oneoff.parsed.ddp | 8 +- .../plan_pairs/expected/ldbc_ic12_oneoff.ddp | 22 ++--- .../ldbc_ic12_oneoff.lowered--optimized.diff | 6 +- .../ldbc_ic12_oneoff.parsed--lowered.diff | 28 +++--- .../expected/ldbc_ic12_oneoff.parsed.ddp | 6 +- .../expected/ldbc_ic12_standing.ddp | 2 +- ...ldbc_ic12_standing.lowered--optimized.diff | 4 +- .../ldbc_ic12_standing.parsed--lowered.diff | 4 +- .../expected/ldbc_ic12_standing.parsed.ddp | 2 +- .../plan_pairs/expected/ldbc_ic13_oneoff.ddp | 12 +-- .../ldbc_ic13_oneoff.lowered--optimized.diff | 14 +-- .../ldbc_ic13_oneoff.parsed--lowered.diff | 24 ++--- .../expected/ldbc_ic13_oneoff.parsed.ddp | 12 +-- .../plan_pairs/expected/ldbc_ic14_oneoff.ddp | 20 ++-- .../ldbc_ic14_oneoff.lowered--optimized.diff | 30 +++--- .../ldbc_ic14_oneoff.parsed--lowered.diff | 36 ++++---- .../expected/ldbc_ic14_oneoff.parsed.ddp | 16 ++-- .../expected/ldbc_ic14_standing.ddp | 4 +- ...ldbc_ic14_standing.lowered--optimized.diff | 2 +- .../ldbc_ic14_standing.parsed--lowered.diff | 8 +- .../expected/ldbc_ic14_standing.parsed.ddp | 4 +- .../tests/plan_pairs/expected/ldbc_is01.ddp | 2 +- .../expected/ldbc_is01.parsed--lowered.diff | 4 +- .../plan_pairs/expected/ldbc_is01.parsed.ddp | 2 +- .../tests/plan_pairs/expected/ldbc_is02.ddp | 12 +-- .../ldbc_is02.lowered--optimized.diff | 10 +- .../expected/ldbc_is02.parsed--lowered.diff | 16 ++-- .../plan_pairs/expected/ldbc_is02.parsed.ddp | 4 +- .../tests/plan_pairs/expected/ldbc_is03.ddp | 4 +- .../ldbc_is03.lowered--optimized.diff | 2 +- .../expected/ldbc_is03.parsed--lowered.diff | 6 +- .../plan_pairs/expected/ldbc_is03.parsed.ddp | 2 +- .../tests/plan_pairs/expected/ldbc_is04.ddp | 2 +- .../expected/ldbc_is04.parsed--lowered.diff | 4 +- .../plan_pairs/expected/ldbc_is04.parsed.ddp | 2 +- .../tests/plan_pairs/expected/ldbc_is05.ddp | 4 +- .../expected/ldbc_is05.parsed--lowered.diff | 6 +- .../plan_pairs/expected/ldbc_is05.parsed.ddp | 2 +- .../tests/plan_pairs/expected/ldbc_is06.ddp | 8 +- .../expected/ldbc_is06.parsed--lowered.diff | 10 +- .../plan_pairs/expected/ldbc_is06.parsed.ddp | 2 +- .../tests/plan_pairs/expected/ldbc_is07.ddp | 10 +- .../ldbc_is07.lowered--optimized.diff | 10 +- .../ldbc_is07.optimized--projected.diff | 2 +- .../expected/ldbc_is07.parsed--lowered.diff | 16 ++-- .../plan_pairs/expected/ldbc_is07.parsed.ddp | 6 +- .../tests/plan_pairs/expected/nested.ddp | 2 +- .../expected/nested.lowered--optimized.diff | 4 +- .../expected/nested.parsed--lowered.diff | 4 +- .../plan_pairs/expected/nested.parsed.ddp | 2 +- .../plan_pairs/expected/server_access.ddp | 6 +- .../server_access.lowered--optimized.diff | 12 +-- .../server_access.parsed--lowered.diff | 12 +-- .../expected/server_access.parsed.ddp | 6 +- .../expected/server_arrange_idem.ddp | 12 +-- ...erver_arrange_idem.lowered--optimized.diff | 22 ++--- .../server_arrange_idem.parsed--lowered.diff | 24 ++--- .../expected/server_arrange_idem.parsed.ddp | 12 +-- .../expected/server_reach_import.ddp | 2 +- ...erver_reach_import.lowered--optimized.diff | 4 +- .../server_reach_import.parsed--lowered.diff | 4 +- .../expected/server_reach_import.parsed.ddp | 2 +- .../plan_pairs/expected/txn_maintained.ddp | 8 +- .../txn_maintained.lowered--optimized.diff | 14 +-- .../txn_maintained.optimized--projected.diff | 2 +- .../txn_maintained.parsed--lowered.diff | 16 ++-- .../expected/txn_maintained.parsed.ddp | 8 +- .../tests/plan_pairs/expected/txn_resolve.ddp | 12 +-- .../txn_resolve.lowered--optimized.diff | 18 ++-- .../expected/txn_resolve.parsed--lowered.diff | 24 ++--- .../expected/txn_resolve.parsed.ddp | 12 +-- .../expected/txn_resolve_by_key.ddp | 8 +- ...txn_resolve_by_key.lowered--optimized.diff | 12 +-- .../txn_resolve_by_key.parsed--lowered.diff | 16 ++-- .../expected/txn_resolve_by_key.parsed.ddp | 8 +- .../tests/plan_pairs/generated/cascade.ddp | 42 ++++----- .../tests/plan_pairs/generated/graph.ddp | 6 +- interactive/tests/programs/cc_records.ddp | 2 +- interactive/tests/programs/cc_stages.ddp | 2 +- interactive/tests/programs/f64_math.ddp | 2 +- interactive/tests/programs/join_fallback.ddp | 2 +- interactive/tests/programs/pair_keys.ddp | 2 +- interactive/tests/programs/registered.ddp | 2 +- interactive/tests/root_var.rs | 2 +- interactive/tests/server_arranged_imports.rs | 4 +- interactive/tests/server_late_import.rs | 10 +- interactive/tests/server_shapes.rs | 2 +- interactive/tests/shared_arrangements.rs | 2 +- interactive/tests/tritemporal.rs | 20 ++-- 299 files changed, 1763 insertions(+), 1707 deletions(-) diff --git a/interactive/README.md b/interactive/README.md index 3841fdf29..26476b02c 100644 --- a/interactive/README.md +++ b/interactive/README.md @@ -65,6 +65,9 @@ let blocks = input 2 : (int, Member); -- doc -> blocked member The shape of records is load bearing when we perform joins and other data-parallel operations. The pair structure `(key, val)` calls out first how the data are organized, and then any associated values. +For each pair of matches `(k, l)` and `(k, r)`, `left | join(right)` produces `(k, (l, r))`. +You can project that row inside the join: `left | join(right, f)` computes the same collection as `left | join(right) | map(f)`. +Existing three-field join patterns migrate from `(k, l, r) -> …` to `(k, (l, r)) -> …`. A currently unfortunate amount of symbolic load is in shuffling around fields of collection elements. The `map` operator applies to each record, and the `->` fragment is an expression converting the input to the output. @@ -115,7 +118,7 @@ To see what a variable is about to become, bind the `set` value with a `let` and let barred = barring::barred | map((who, doc) -> ((who, doc), ())); barring: { var barred : (Member, int); -- member -> doc: expanded blocked list. - let beyond = barred | join(members, (_grp, doc, who) -> (who, doc)); + let beyond = barred | join(members, (_grp, (doc, who)) -> (who, doc)); set barred = blocks | map((doc, who) -> (who, doc)) + beyond | distinct; } ``` @@ -145,7 +148,7 @@ output "access" = sharing::allowed; sharing: { var allowed : ((Member, int), Access); -- (member, doc) -> strongest set allowed = offered - - (offered | join(barred, (pair, acc, ()) -> (pair, acc))) + - (offered | join(barred, (pair, (acc, ())) -> (pair, acc))) | min; let offered = grants | map((doc, (who, acc)) -> ((who, doc), acc)) @@ -153,7 +156,7 @@ sharing: { let inherited = allowed | map(((who, doc), acc) -> (who, (doc, acc))) - | join(members, (_grp, (doc, acc), who) -> ((who, doc), acc)); + | join(members, (_grp, ((doc, acc), who)) -> ((who, doc), acc)); } ``` diff --git a/interactive/bench/ldbc/authored/README.md b/interactive/bench/ldbc/authored/README.md index a705de412..d7be8d644 100644 --- a/interactive/bench/ldbc/authored/README.md +++ b/interactive/bench/ldbc/authored/README.md @@ -152,7 +152,7 @@ Two conventions apply throughout: ## Idioms - **Keyed collections.** Keep a collection as `(key, value)` by the key it is - joined on: `friend` is `(person, friend)`, so `join(friend, (_, req, f) -> …)` + joined on: `friend` is `(person, friend)`, so `join(friend, (_, (req, f)) -> …)` reads as "for each friend". Map back to `((fields), ())` only where a step needs a whole-row key. - **Destructuring.** Lambdas name what they take apart: diff --git a/interactive/bench/ldbc/authored/plans/bi01.ddp b/interactive/bench/ldbc/authored/plans/bi01.ddp index 1568839bc..ab254ba3c 100644 --- a/interactive/bench/ldbc/authored/plans/bi01.ddp +++ b/interactive/bench/ldbc/authored/plans/bi01.ddp @@ -23,12 +23,12 @@ let per_day = classified | join_function(((day, year, kind, cat), (_, length)) - let cutoff = request | map(((rid, end), ()) -> (rid, (end, ((end, 86400000) div, 86400000) mul))); let whole = cutoff | map((rid, (_, day)) -> ((), (rid, day))) | join(per_day | map(((day, year, kind, cat, c), v) -> ((), (day, year, kind, cat, c, v))), - (_, (rid, cut), (day, year, kind, cat, c, v)) -> ((rid, cut, day), (year, kind, cat, c, v))) + (_, ((rid, cut), (day, year, kind, cat, c, v))) -> ((rid, cut, day), (year, kind, cat, c, v))) | filter(((_, cut, day), _) -> (day, cut) lt) | join_function(((rid, _, _), (year, kind, cat, c, v)) -> (((rid, year, kind, cat, c), ()), 0, v) enlist); let part = cutoff | map((rid, (end, day)) -> (day, (rid, end))) | join(classified | map(((day, year, kind, cat), (created, length)) -> (day, (year, kind, cat, created, length))), - (_, (rid, end), (year, kind, cat, created, length)) -> ((rid, year, kind, cat), (created, end, length))) + (_, ((rid, end), (year, kind, cat, created, length))) -> ((rid, year, kind, cat), (created, end, length))) | filter((_, (created, end, _)) -> (created, end) lt) | join_function(((rid, year, kind, cat), (_, _, length)) -> ((((rid, year, kind, cat, 0), ()), 0, 1) enlist, (((rid, year, kind, cat, 1), ()), 0, length) enlist) append); @@ -42,7 +42,7 @@ let groups = sums | map(((rid, year, kind, cat, c), v) -> ((rid, year, kind, cat let total = sums | filter(((_, _, _, _, c), _) -> (c, 0) eq) | join_function(((rid, _, _, _, _), n) -> ((rid, ()), 0, n) enlist) | count; let ranked = groups | map(((rid, year, kind, cat), (n, s)) -> (rid, (year, kind, cat, n, s))) - | join(total, (rid, (year, kind, cat, n, s), all) -> (rid, ((year neg, kind, cat), (n, s, all)))) + | join(total, (rid, ((year, kind, cat, n, s), all)) -> (rid, ((year neg, kind, cat), (n, s, all)))) | collect | flatmap((k, entries) -> (k, ((0, entries len) range, entries) zip) cap_list); let answer = ranked | map((rid, (rank, ((neg_year, kind, cat), (n, s, all)))) -> ((rid, rank), (neg_year neg, kind, cat, n, (s to_float, n to_float) div, s, ((n, 100) mul to_float, all to_float) div))); diff --git a/interactive/bench/ldbc/authored/plans/bi02.ddp b/interactive/bench/ldbc/authored/plans/bi02.ddp index 0f6cdfc76..e245f5230 100644 --- a/interactive/bench/ldbc/authored/plans/bi02.ddp +++ b/interactive/bench/ldbc/authored/plans/bi02.ddp @@ -12,18 +12,18 @@ let tagged = import "graph::mtag.by_tag" : (int, int); - let class_id = import "graph::tagclass.by_name" : (List(int), int); -- (name, id) -- ((class, day), (tag name, created)) for every tagged message, and each class's days. -let byday = tag | join(tagged, (_, (name, cls), m) -> (m, (cls, name))) - | join(message, (_, (cls, name), (_, created, _, _, _, _, _, _, _, _, _, _, day)) -> ((cls, day), (name, created))); +let byday = tag | join(tagged, (_, ((name, cls), m)) -> (m, (cls, name))) + | join(message, (_, ((cls, name), (_, created, _, _, _, _, _, _, _, _, _, _, day))) -> ((cls, day), (name, created))); let days = byday | map(((cls, day), _) -> ((cls, day), ())) | distinct | map(((cls, day), ()) -> (cls, day)); -- The request's class and its 200 window days (days with a message of the class). let wanted = request | map(((rid, date, name), ()) -> (name, (rid, date))) - | join(class_id, (_, (rid, date), cls) -> (cls, (rid, date))) - | join(days, (cls, (rid, date), day) -> ((cls, day), (rid, date))) + | join(class_id, (_, ((rid, date), cls)) -> (cls, (rid, date))) + | join(days, (cls, ((rid, date), day)) -> ((cls, day), (rid, date))) | filter(((_, day), (_, date)) -> ((day, ((date, 86400000) div, 86400000) mul) ge, (day, (date, 17280000000) add) lt) all); -- Count each tag's messages in the two halves. -let halves = wanted | join(byday, (_, (rid, date), (name, created)) -> ((rid, name), (created, date))) +let halves = wanted | join(byday, (_, ((rid, date), (name, created))) -> ((rid, name), (created, date))) | filter((_, (created, date)) -> ((created, date) ge, (created, (date, 17280000000) add) lt) all) | map((key, (created, date)) -> (key, ((created, (date, 8640000000) add) lt, (1, 0), (0, 1)) select)); let counts = halves | map(((rid, name), (a, _)) -> ((rid, name, a), ())) | count diff --git a/interactive/bench/ldbc/authored/plans/bi03.ddp b/interactive/bench/ldbc/authored/plans/bi03.ddp index 36a4b85b8..86371e277 100644 --- a/interactive/bench/ldbc/authored/plans/bi03.ddp +++ b/interactive/bench/ldbc/authored/plans/bi03.ddp @@ -18,24 +18,24 @@ let root_of = import "graph::message.root" : (int, int); -- Forums moderated by someone living in a city of the country. let local = request | map(((rid, _, country), ()) -> (country, rid)) - | join(place_id, (_, rid, country) -> (country, rid)) - | join(within, (_, rid, city) -> (city, rid)) - | join(residents, (_, rid, who) -> (who, rid)) - | join(moderates, (_, rid, f) -> ((rid, f), ())); + | join(place_id, (_, (rid, country)) -> (country, rid)) + | join(within, (_, (rid, city)) -> (city, rid)) + | join(residents, (_, (rid, who)) -> (who, rid)) + | join(moderates, (_, (rid, f)) -> ((rid, f), ())); -- Messages tagged with a tag of the class (once per message), and their threads' forums. let inclass = request | map(((rid, cls, _), ()) -> (cls, rid)) - | join(class_id, (_, rid, c) -> (c, rid)) - | join(in_class, (_, rid, t) -> (t, rid)) - | join(tagged, (_, rid, m) -> ((rid, m), ())) | distinct; + | join(class_id, (_, (rid, c)) -> (c, rid)) + | join(in_class, (_, (rid, t)) -> (t, rid)) + | join(tagged, (_, (rid, m)) -> ((rid, m), ())) | distinct; let counts = inclass | map(((rid, m), ()) -> (m, rid)) - | join(root_of, (_, rid, root) -> (root, rid)) - | join(message, (_, rid, (_, _, _, _, _, _, _, _, f, _, _, _, _)) -> ((rid, f), ())) - | join(local, (key, (), ()) -> (key, ())) | count; + | join(root_of, (_, (rid, root)) -> (root, rid)) + | join(message, (_, (rid, (_, _, _, _, _, _, _, _, f, _, _, _, _))) -> ((rid, f), ())) + | join(local, (key, ((), ())) -> (key, ())) | count; -- Top k: `reduce` sees a key's values in sort order and keeps the first k (copies capped -- at k), emitting (rank, value). Changes below the top k produce no output. let ranked = counts | map(((rid, f), n) -> (f, (rid, n))) - | join(forum, (f, (rid, n), (title, created, moderator)) -> (rid, ((n neg, f), (title, created, moderator)))) + | join(forum, (f, ((rid, n), (title, created, moderator))) -> (rid, ((n neg, f), (title, created, moderator)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((neg, f), (title, created, moderator)))) -> ((rid, rank), (f, title, created, moderator, neg neg))); diff --git a/interactive/bench/ldbc/authored/plans/bi04.ddp b/interactive/bench/ldbc/authored/plans/bi04.ddp index 5857a47bc..d7578b20b 100644 --- a/interactive/bench/ldbc/authored/plans/bi04.ddp +++ b/interactive/bench/ldbc/authored/plans/bi04.ddp @@ -18,16 +18,16 @@ let posts_in = import "graph::message.by_forum" : (int, int); let thread = import "graph::message.by_root" : (int, int); -- (root, message) -- Each person's country, and each forum's popularity (negated, so min is max). -let country_of = place | join(residents, (_, (_, _, country), who) -> (who, country)); -let popularity = country_of | join(joins, (_, country, (f, _)) -> ((f, country), ())) | count +let country_of = place | join(residents, (_, ((_, _, country), who)) -> (who, country)); +let popularity = country_of | join(joins, (_, (country, (f, _))) -> ((f, country), ())) | count | map(((f, _), n) -> (f, n neg)) | min; -- The request's top 100 forums created after its date. -- Top k: `reduce` sees a key's values in sort order and keeps the first k (copies capped -- at k), emitting (rank, value). Changes below the top k produce no output. let top = request | map(((rid, date), ()) -> ((), (rid, date))) - | join(popularity | join(forum, (f, neg, (_, when, _)) -> ((), (f, neg, when))), - (_, (rid, date), (f, neg, when)) -> (rid, ((neg, f), (date, when)))) + | join(popularity | join(forum, (f, (neg, (_, when, _))) -> ((), (f, neg, when))), + (_, ((rid, date), (f, neg, when))) -> (rid, ((neg, f), (date, when)))) | filter((_, (_, (date, when))) -> (when, date) gt) | map((rid, (order, _)) -> (rid, order)) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 100) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 100) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 100) min in let before = (end, count) sub in let keep = (((100, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten) @@ -35,16 +35,16 @@ let top = request | map(((rid, date), ()) -> ((), (rid, date))) -- Their members, and how many messages each person posted in them (every message in -- a thread rooted at one of the forum's posts). -let joined = top | join(members, (_, rid, (who, _)) -> ((rid, who), ())) | distinct; -let posted = top | join(posts_in, (_, rid, post) -> (post, rid)) - | join(thread, (_, rid, mid) -> (mid, rid)) - | join(message, (_, rid, (_, _, author, _, _, _, _, _, _, _, _, _, _)) -> ((rid, author), 1)); -let wrote = posted | join(joined, (key, _, ()) -> (key, ())) | count; -let counts = wrote + ((joined - (joined | join(wrote, (key, (), _) -> (key, ())))) | map((key, ()) -> (key, 0))); +let joined = top | join(members, (_, (rid, (who, _))) -> ((rid, who), ())) | distinct; +let posted = top | join(posts_in, (_, (rid, post)) -> (post, rid)) + | join(thread, (_, (rid, mid)) -> (mid, rid)) + | join(message, (_, (rid, (_, _, author, _, _, _, _, _, _, _, _, _, _))) -> ((rid, author), 1)); +let wrote = posted | join(joined, (key, (_, ())) -> (key, ())) | count; +let counts = wrote + ((joined - (joined | join(wrote, (key, ((), _)) -> (key, ())))) | map((key, ()) -> (key, 0))); -- Rank members by messages (most first), then id. let ranked = counts | map(((rid, who), n) -> (who, (rid, n))) - | join(person, (who, (rid, n), (first, last, _, _, created, _, _, _, _, _, _)) -> (rid, ((n neg, who), (first, last, created, n)))) + | join(person, (who, ((rid, n), (first, last, _, _, created, _, _, _, _, _, _))) -> (rid, ((n neg, who), (first, last, created, n)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 100) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 100) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 100) min in let before = (end, count) sub in let keep = (((100, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((_, who), (first, last, created, n)))) -> ((rid, rank), (who, first, last, created, n))); diff --git a/interactive/bench/ldbc/authored/plans/bi05.ddp b/interactive/bench/ldbc/authored/plans/bi05.ddp index 24e8f13bf..8a5863705 100644 --- a/interactive/bench/ldbc/authored/plans/bi05.ddp +++ b/interactive/bench/ldbc/authored/plans/bi05.ddp @@ -13,14 +13,14 @@ let liked_by = import "graph::likes.by_message" : (int, (int, int)); -- ( -- The request's tagged messages and their authors: (message, (rid, author)). -- Tag names are unique and (message, tag) pairs are a set, so each message appears once. let written = request | map(((rid, name), ()) -> (name, rid)) - | join(tag_id, (_, rid, tag) -> (tag, rid)) - | join(tagged, (_, rid, m) -> (m, rid)) - | join(message, (m, rid, (_, _, author, _, _, _, _, _, _, _, _, _, _)) -> (m, (rid, author))); + | join(tag_id, (_, (rid, tag)) -> (tag, rid)) + | join(tagged, (_, (rid, m)) -> (m, rid)) + | join(message, (m, (rid, (_, _, author, _, _, _, _, _, _, _, _, _, _))) -> (m, (rid, author))); -- Count each author's tagged messages (kind 0), replies to them (1) and likes on them (2). let as_message = written | map((_, (rid, author)) -> ((rid, author, 0), ())); -let as_like = written | join(liked_by, (_, (rid, author), _) -> ((rid, author, 2), ())); -let as_reply = written | join(replies, (_, (rid, author), _) -> ((rid, author, 1), ())); +let as_like = written | join(liked_by, (_, ((rid, author), _)) -> ((rid, author, 2), ())); +let as_reply = written | join(replies, (_, ((rid, author), _)) -> ((rid, author, 1), ())); let tally = (as_message + as_reply + as_like) | count; -- Pivot the (at most three) counts per author into (messages, replies, likes). diff --git a/interactive/bench/ldbc/authored/plans/bi06.ddp b/interactive/bench/ldbc/authored/plans/bi06.ddp index d8860057e..b1176ff6f 100644 --- a/interactive/bench/ldbc/authored/plans/bi06.ddp +++ b/interactive/bench/ldbc/authored/plans/bi06.ddp @@ -12,22 +12,22 @@ let liked_by = import "graph::likes.by_message" : (int, (int, int)); -- ( -- The request's tagged messages and their authors (one tag; (message, tag) is a set). let written = request | map(((rid, name), ()) -> (name, rid)) - | join(tag_id, (_, rid, t) -> (t, rid)) - | join(tagged, (_, rid, m) -> (m, rid)) - | join(message, (m, rid, (_, _, who, _, _, _, _, _, _, _, _, _, _)) -> ((rid, m, who), ())); + | join(tag_id, (_, (rid, t)) -> (t, rid)) + | join(tagged, (_, (rid, m)) -> (m, rid)) + | join(message, (m, (rid, (_, _, who, _, _, _, _, _, _, _, _, _, _))) -> ((rid, m, who), ())); -- Popularity: likes received on all of a person's messages. -let popular = liked_by | join(message, (_, _, (_, _, who, _, _, _, _, _, _, _, _, _, _)) -> (who, ())) | count; +let popular = liked_by | join(message, (_, (_, (_, _, who, _, _, _, _, _, _, _, _, _, _))) -> (who, ())) | count; -- Each (author, liker) pair once, scored by the liker's popularity; zero for the rest. let pairs = written | map(((rid, m, who), ()) -> (m, (rid, who))) - | join(liked_by, (_, (rid, who), (liker, _)) -> ((rid, who, liker), ())) | distinct; + | join(liked_by, (_, ((rid, who), (liker, _))) -> ((rid, who, liker), ())) | distinct; -- Sums: move each value into the multiplicity (join_function) and let `count` add them up. let scored = pairs | map(((rid, who, liker), ()) -> (liker, (rid, who))) - | join(popular, (_, (rid, who), p) -> ((rid, who), p)) + | join(popular, (_, ((rid, who), p)) -> ((rid, who), p)) | join_function((key, p) -> ((key, ()), 0, p) enlist) | count; let authors = written | map(((rid, _, who), ()) -> ((rid, who), ())) | distinct; -let score = scored + ((authors - (authors | join(scored, (key, (), _) -> (key, ())))) | map((key, ()) -> (key, 0))); +let score = scored + ((authors - (authors | join(scored, (key, ((), _)) -> (key, ())))) | map((key, ()) -> (key, 0))); -- Top k: `reduce` sees a key's values in sort order and keeps the first k (copies capped -- at k), emitting (rank, value). Changes below the top k produce no output. let ranked = score | map(((rid, who), s) -> (rid, (s neg, who))) diff --git a/interactive/bench/ldbc/authored/plans/bi07.ddp b/interactive/bench/ldbc/authored/plans/bi07.ddp index 8df437483..00fb64e94 100644 --- a/interactive/bench/ldbc/authored/plans/bi07.ddp +++ b/interactive/bench/ldbc/authored/plans/bi07.ddp @@ -11,13 +11,13 @@ let replies = import "graph::message.by_parent" : (int, int); -- -- Messages carrying the request's tag (one tag; (message, tag) is a set). let marked = request | map(((rid, name), ()) -> (name, rid)) - | join(tag_id, (_, rid, t) -> (t, rid)) - | join(tagged, (_, rid, m) -> ((rid, m), ())); -let answers = marked | map(((rid, m), ()) -> (m, rid)) | join(replies, (_, rid, r) -> ((rid, r), ())); -let fresh = answers - (answers | join(marked, (key, (), ()) -> (key, ()))); + | join(tag_id, (_, (rid, t)) -> (t, rid)) + | join(tagged, (_, (rid, m)) -> ((rid, m), ())); +let answers = marked | map(((rid, m), ()) -> (m, rid)) | join(replies, (_, (rid, r)) -> ((rid, r), ())); +let fresh = answers - (answers | join(marked, (key, ((), ())) -> (key, ()))); let counts = fresh | map(((rid, r), ()) -> (r, rid)) - | join(tags_of, (_, rid, t) -> (t, rid)) - | join(tag, (_, rid, (name, _)) -> ((rid, name), ())) | count; + | join(tags_of, (_, (rid, t)) -> (t, rid)) + | join(tag, (_, (rid, (name, _))) -> ((rid, name), ())) | count; -- Top k: `reduce` sees a key's values in sort order and keeps the first k (copies capped -- at k), emitting (rank, value). Changes below the top k produce no output. let ranked = counts | map(((rid, name), n) -> (rid, (n neg, name))) diff --git a/interactive/bench/ldbc/authored/plans/bi08.ddp b/interactive/bench/ldbc/authored/plans/bi08.ddp index de883a9f3..db9d3c95c 100644 --- a/interactive/bench/ldbc/authored/plans/bi08.ddp +++ b/interactive/bench/ldbc/authored/plans/bi08.ddp @@ -11,12 +11,12 @@ let tag_id = import "graph::tag.by_name" : (List(int), int); -- (nam let tagged = import "graph::mtag.by_tag" : (int, int); -- (tag, message) let fans_of = import "graph::interest.by_tag" : (int, int); -- (tag, person) -let tag_rid = request | map(((rid, name, _, _), ()) -> (name, rid)) | join(tag_id, (_, rid, t) -> (t, rid)); -let fans = tag_rid | join(fans_of, (_, rid, who) -> ((rid, who), 100)); +let tag_rid = request | map(((rid, name, _, _), ()) -> (name, rid)) | join(tag_id, (_, (rid, t)) -> (t, rid)); +let fans = tag_rid | join(fans_of, (_, (rid, who)) -> ((rid, who), 100)); let window = request | map(((rid, _, start, end), ()) -> (rid, (start, end))); -let recent = tag_rid | join(tagged, (_, rid, m) -> (m, rid)) - | join(message, (_, rid, (_, created, who, _, _, _, _, _, _, _, _, _, _)) -> (rid, (who, created))) - | join(window, (rid, (who, created), (start, end)) -> ((rid, who), (created, start, end))) +let recent = tag_rid | join(tagged, (_, (rid, m)) -> (m, rid)) + | join(message, (_, (rid, (_, created, who, _, _, _, _, _, _, _, _, _, _))) -> (rid, (who, created))) + | join(window, (rid, ((who, created), (start, end))) -> ((rid, who), (created, start, end))) | filter((_, (created, start, end)) -> ((created, start) gt, (created, end) lt) all) | map((key, _) -> (key, 1)); -- Sums: move each value into the multiplicity (join_function) and let `count` add them up. @@ -24,13 +24,13 @@ let score = (fans + recent) | join_function((key, p) -> ((key, ()), 0, p) enli -- Friends' points: for each scored person, the points of their scored friends. let theirs = score | map(((rid, f), s) -> (f, (rid, s))) - | join(friend, (_, (rid, s), (p, _)) -> ((rid, p), s)) - | join(score, (key, s, _) -> (key, s)); + | join(friend, (_, ((rid, s), (p, _))) -> ((rid, p), s)) + | join(score, (key, (s, _)) -> (key, s)); let summed = theirs | join_function((key, s) -> ((key, ()), 0, s) enlist) | count; -let friends = summed + ((score - (score | join(summed, (key, s, _) -> (key, s)))) | map((key, _) -> (key, 0))); +let friends = summed + ((score - (score | join(summed, (key, (s, _)) -> (key, s)))) | map((key, _) -> (key, 0))); -- Top k: `reduce` sees a key's values in sort order and keeps the first k (copies capped -- at k), emitting (rank, value). Changes below the top k produce no output. -let ranked = score | join(friends, ((rid, who), s, fs) -> (rid, (((s, fs) add neg, who), (s, fs)))) +let ranked = score | join(friends, ((rid, who), (s, fs)) -> (rid, (((s, fs) add neg, who), (s, fs)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 100) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 100) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 100) min in let before = (end, count) sub in let keep = (((100, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((_, who), (s, fs)))) -> ((rid, rank), (who, s, fs))); diff --git a/interactive/bench/ldbc/authored/plans/bi09.ddp b/interactive/bench/ldbc/authored/plans/bi09.ddp index 1087c9b16..5bed250cb 100644 --- a/interactive/bench/ldbc/authored/plans/bi09.ddp +++ b/interactive/bench/ldbc/authored/plans/bi09.ddp @@ -18,31 +18,31 @@ let thread = import "graph::message.by_root" : (int, int); -- (root, message let byday = message | map((id, (kind, created, who, _, _, _, _, _, _, _, _, _, day)) -> (day, (id, kind, created, who))); let days = byday | map((day, _) -> (day, ())) | distinct | map((day, ()) -> ((), day)); let windays = request | map(((rid, start, end), ()) -> ((), (rid, start, end))) - | join(days, (_, (rid, start, end), day) -> (day, (rid, start, end))) + | join(days, (_, ((rid, start, end), day)) -> (day, (rid, start, end))) | filter((day, (_, start, end)) -> ((day, ((start, 86400000) div, 86400000) mul) ge, (day, end) le) all); -let posts = windays | join(byday, (_, (rid, start, end), (id, kind, created, who)) -> ((rid, id), (kind, created, who, start, end))) +let posts = windays | join(byday, (_, ((rid, start, end), (id, kind, created, who))) -> ((rid, id), (kind, created, who, start, end))) | filter((_, (kind, created, _, start, end)) -> ((kind, 0) eq, (created, start) ge, (created, end) le) all) | map(((rid, id), (_, _, who, _, _)) -> ((rid, id), who)); -- Thread sizes per post, and messages after the request's end, by their post. let size = thread | map((root, _) -> (root, ())) | count; let late = request | map(((rid, _, end), ()) -> ((), (rid, end))) - | join(days, (_, (rid, end), day) -> (day, (rid, end))) + | join(days, (_, ((rid, end), day)) -> (day, (rid, end))) | filter((day, (_, end)) -> (day, ((end, 86400000) div, 86400000) mul) ge) - | join(byday, (_, (rid, end), (id, _, created, _)) -> (id, (rid, created, end))) + | join(byday, (_, ((rid, end), (id, _, created, _))) -> (id, (rid, created, end))) | filter((_, (_, created, end)) -> (created, end) gt) - | join(root_of, (_, (rid, _, _), root) -> ((rid, root), -1)); + | join(root_of, (_, ((rid, _, _), root)) -> ((rid, root), -1)); -- Per author: threads started, and messages (thread sizes minus the late messages). let started = posts | map(((rid, _), who) -> ((rid, who), ())) | count; -let parts = (posts | map(((rid, id), who) -> (id, (rid, who))) | join(size, (_, (rid, who), n) -> ((rid, who), n))) - + (late | join(posts, ((rid, _), minus, who) -> ((rid, who), minus))); +let parts = (posts | map(((rid, id), who) -> (id, (rid, who))) | join(size, (_, ((rid, who), n)) -> ((rid, who), n))) + + (late | join(posts, ((rid, _), (minus, who)) -> ((rid, who), minus))); -- Sums: move each value into the multiplicity (join_function) and let `count` add them up. let msgs = parts | join_function((key, n) -> ((key, ()), 0, n) enlist) | count; -- Top k: `reduce` sees a key's values in sort order and keeps the first k (copies capped -- at k), emitting (rank, value). Changes below the top k produce no output. -let ranked = started | join(msgs, ((rid, who), t, m) -> (who, (rid, t, m))) - | join(person, (who, (rid, t, m), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, ((m neg, who), (first, last, t)))) +let ranked = started | join(msgs, ((rid, who), (t, m)) -> (who, (rid, t, m))) + | join(person, (who, ((rid, t, m), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((m neg, who), (first, last, t)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 100) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 100) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 100) min in let before = (end, count) sub in let keep = (((100, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((neg, who), (first, last, t)))) -> ((rid, rank), (who, first, last, t, neg neg))); diff --git a/interactive/bench/ldbc/authored/plans/bi10.ddp b/interactive/bench/ldbc/authored/plans/bi10.ddp index 196e248fc..c83f3d109 100644 --- a/interactive/bench/ldbc/authored/plans/bi10.ddp +++ b/interactive/bench/ldbc/authored/plans/bi10.ddp @@ -22,7 +22,7 @@ let starts = request | map(((rid, who, _, _, _, most), ()) -> ((rid, most, who, search: { let grow = search | filter(((_, most, _, d), ()) -> (d, most) lt) | map(((rid, most, node, d), ()) -> (node, (rid, most, d))) - | join(friend, (_, (rid, most, d), (next, _)) -> ((rid, most, next, (d, 1) add), ())); + | join(friend, (_, ((rid, most, d), (next, _))) -> ((rid, most, next, (d, 1) add), ())); set search = (starts + grow) | map(((rid, most, node, d), ()) -> ((rid, most, node), d)) | min | map(((rid, most, node), d) -> ((rid, most, node, d), ())); } @@ -30,24 +30,24 @@ search: { -- People in range whose city is in the country. let asked = request | map(((rid, who, country, _, least, _), ()) -> (rid, (who, country, least))); let people = search::search | map(((rid, _, node, d), ()) -> (rid, (node, d))) - | join(asked, (rid, (node, d), (who, country, least)) -> (node, (rid, d, who, country, least))) + | join(asked, (rid, ((node, d), (who, country, least))) -> (node, (rid, d, who, country, least))) | filter((node, (_, d, who, _, least)) -> ((d, least) ge, (node, who) ne) all) - | join(person, (node, (rid, _, _, country, _), (_, _, _, _, _, _, _, city, _, _, _)) -> (city, (rid, node, country))) - | join(place, (_, (rid, node, country), (_, _, up)) -> (up, (rid, node, country))) - | join(place, (_, (rid, node, country), (kname, _, _)) -> ((rid, node), (country, kname))) + | join(person, (node, ((rid, _, _, country, _), (_, _, _, _, _, _, _, city, _, _, _))) -> (city, (rid, node, country))) + | join(place, (_, ((rid, node, country), (_, _, up))) -> (up, (rid, node, country))) + | join(place, (_, ((rid, node, country), (kname, _, _))) -> ((rid, node), (country, kname))) | filter((_, (country, kname)) -> (country, kname) eq) | map((key, _) -> (key, ())); -- The class's messages, by author, kept if the author qualifies; then all their tags. let inclass = request | map(((rid, _, _, cls, _, _), ()) -> (cls, rid)) - | join(class_id, (_, rid, c) -> (c, rid)) - | join(in_class, (_, rid, t) -> (t, rid)) - | join(tagged, (_, rid, m) -> ((rid, m), ())) | distinct + | join(class_id, (_, (rid, c)) -> (c, rid)) + | join(in_class, (_, (rid, t)) -> (t, rid)) + | join(tagged, (_, (rid, m)) -> ((rid, m), ())) | distinct | map(((rid, m), ()) -> (m, rid)) - | join(message, (m, rid, (_, _, who, _, _, _, _, _, _, _, _, _, _)) -> ((rid, who), m)) - | join(people, ((rid, who), m, ()) -> (m, (rid, who))); -let counts = inclass | join(tags_of, (_, (rid, who), t) -> (t, (rid, who))) - | join(tag, (_, (rid, who), (name, _)) -> ((rid, who, name), ())) | count; + | join(message, (m, (rid, (_, _, who, _, _, _, _, _, _, _, _, _, _))) -> ((rid, who), m)) + | join(people, ((rid, who), (m, ())) -> (m, (rid, who))); +let counts = inclass | join(tags_of, (_, ((rid, who), t)) -> (t, (rid, who))) + | join(tag, (_, ((rid, who), (name, _))) -> ((rid, who, name), ())) | count; -- Top k: `reduce` sees a key's values in sort order and keeps the first k (copies capped -- at k), emitting (rank, value). Changes below the top k produce no output. let ranked = counts | map(((rid, who, name), n) -> (rid, (n neg, name, who))) diff --git a/interactive/bench/ldbc/authored/plans/bi11.ddp b/interactive/bench/ldbc/authored/plans/bi11.ddp index 855e06bb6..c413291bd 100644 --- a/interactive/bench/ldbc/authored/plans/bi11.ddp +++ b/interactive/bench/ldbc/authored/plans/bi11.ddp @@ -10,29 +10,29 @@ let friend = import "graph::friend" : (int, (int, int)); -- -- Each request's residents, keyed by person, carrying the request and its window: -- country name -> country -> its cities -> the people living there. let residents = request | map(((rid, country, start, end), ()) -> (country, (rid, start, end))) - | join(place_by_name, (_, req, country) -> (country, req)) - | join(place_by_parent, (_, req, city) -> (city, req)) - | join(person_by_city, (_, req, who) -> (who, req)); + | join(place_by_name, (_, (req, country)) -> (country, req)) + | join(place_by_parent, (_, (req, city)) -> (city, req)) + | join(person_by_city, (_, (req, who)) -> (who, req)); -- Each request's friendships among its residents, made inside the window. let member = residents | map((who, (rid, _, _)) -> ((rid, who), ())); -let inside = residents | join(friend, (a, (rid, start, end), (b, t)) -> ((rid, a, b, t, start, end), ())) +let inside = residents | join(friend, (a, ((rid, start, end), (b, t))) -> ((rid, a, b, t, start, end), ())) | filter(((_, _, _, t, start, end), ()) -> ((t, start) ge, (t, end) le) all) | map(((rid, a, b, _, _, _), ()) -> ((rid, b), a)) - | join(member, ((rid, b), a, ()) -> ((rid, a, b), ())); + | join(member, ((rid, b), (a, ())) -> ((rid, a, b), ())); -- Triangles a < b < c: each friendship oriented a < b, so the path a-b-c is built -- only in increasing order, and closed by the friendship a-c. let oriented = inside | filter(((_, a, b), ()) -> (a, b) lt); let by_first = oriented | map(((rid, a, b), ()) -> ((rid, a), b)); let by_second = oriented | map(((rid, a, b), ()) -> ((rid, b), a)); -let triangles = by_second | join(by_first, ((rid, b), a, c) -> ((rid, a, c), b)) - | join(oriented, ((rid, _, _), _, ()) -> (rid, ())); +let triangles = by_second | join(by_first, ((rid, b), (a, c)) -> ((rid, a, c), b)) + | join(oriented, ((rid, _, _), (_, ())) -> (rid, ())); -- One row per request, zero when there are no triangles. let counted = triangles | count; let none = request | map(((rid, _, _, _), ()) -> (rid, ())); -let zero = none - (none | join(counted, (rid, (), _) -> (rid, ()))); +let zero = none - (none | join(counted, (rid, ((), _)) -> (rid, ()))); let answer = (counted | map((rid, n) -> ((rid, 0), (n)))) + (zero | map((rid, ()) -> ((rid, 0), (0)))); output "answer" = answer | arrange; diff --git a/interactive/bench/ldbc/authored/plans/bi12.ddp b/interactive/bench/ldbc/authored/plans/bi12.ddp index 528503f5a..c6360c67d 100644 --- a/interactive/bench/ldbc/authored/plans/bi12.ddp +++ b/interactive/bench/ldbc/authored/plans/bi12.ddp @@ -11,22 +11,22 @@ let root = import "graph::message.root" : (int, int); -- (me -- Messages with text, keyed by their thread's language (the root post's language). let texts = message | filter((_, (_, _, _, _, text, _, _, _, _, _, _, _, _)) -> (text len, 0) gt) | map((id, (_, created, who, _, _, _, length, _, _, _, _, _, _)) -> (id, (who, created, length))); -let bylang = texts | join(root, (id, row, top) -> (top, (id, row))) - | join(message, (_, (id, (who, created, length)), (_, _, _, _, _, _, _, lang, _, _, _, _, _)) -> (lang, (id, who, created, length))); +let bylang = texts | join(root, (id, (row, top)) -> (top, (id, row))) + | join(message, (_, ((id, (who, created, length)), (_, _, _, _, _, _, _, lang, _, _, _, _, _))) -> (lang, (id, who, created, length))); -- Each person's qualifying messages. A message has one thread language and a request -- names each language once, so every message matches at most once: no distinct needed. let chosen = request | filter(((_, _, _, _, on), ()) -> (on, 0) ne) | map(((rid, start, max, lang, _), ()) -> (lang, (rid, start, max))) - | join(bylang, (_, (rid, start, max), (id, who, created, length)) -> ((rid, who, id), (created, start, length, max))) + | join(bylang, (_, ((rid, start, max), (id, who, created, length))) -> ((rid, who, id), (created, start, length, max))) | filter((_, (created, start, length, max)) -> ((created, start) gt, (length, max) lt) all) | map((key, _) -> (key, ())); -- Everyone gets a count (zero by default); then people per count. let rids = request | map(((rid, _, _, _, _), ()) -> ((), rid)) | map((u, rid) -> ((u, rid), ())) | distinct | map(((u, rid), ()) -> (u, rid)); -let everyone = rids | join(person | map((id, _) -> ((), id)), (_, rid, who) -> ((rid, who), 0)); +let everyone = rids | join(person | map((id, _) -> ((), id)), (_, (rid, who)) -> ((rid, who), 0)); let counted = chosen | map(((rid, who, _), ()) -> ((rid, who), ())) | count; -let perhead = counted + (everyone - (everyone | join(counted, (key, z, _) -> (key, z)))); +let perhead = counted + (everyone - (everyone | join(counted, (key, (z, _)) -> (key, z)))); let people = perhead | map(((rid, _), n) -> ((rid, n), ())) | count; let ranked = people | map(((rid, n), p) -> (rid, (p neg, n neg))) | collect | flatmap((k, entries) -> (k, ((0, entries len) range, entries) zip) cap_list); diff --git a/interactive/bench/ldbc/authored/plans/bi13.ddp b/interactive/bench/ldbc/authored/plans/bi13.ddp index c405220e6..8903a301d 100644 --- a/interactive/bench/ldbc/authored/plans/bi13.ddp +++ b/interactive/bench/ldbc/authored/plans/bi13.ddp @@ -14,32 +14,32 @@ let person_by_city = import "graph::person.by_city" : (int, int); -- Candidates: residents (country name -> country -> cities -> people) created before -- the end, with their creation date and month. let cands = request | map(((rid, country, end, endmonth), ()) -> (country, (rid, end, endmonth))) - | join(place_by_name, (_, req, country) -> (country, req)) - | join(place_by_parent, (_, req, city) -> (city, req)) - | join(person_by_city, (_, req, who) -> (who, req)) - | join(person, (who, (rid, end, endmonth), (_, _, _, _, created, _, _, _, _, _, month)) -> ((rid, who), (end, endmonth, created, month))) + | join(place_by_name, (_, (req, country)) -> (country, req)) + | join(place_by_parent, (_, (req, city)) -> (city, req)) + | join(person_by_city, (_, (req, who)) -> (who, req)) + | join(person, (who, ((rid, end, endmonth), (_, _, _, _, created, _, _, _, _, _, month))) -> ((rid, who), (end, endmonth, created, month))) | filter((_, (end, _, created, _)) -> (created, end) lt); -- Messages between their creation and the end; zombies post fewer than one per month. let posts = cands | map(((rid, who), (end, _, created, _)) -> (who, (rid, end, created))) - | join(wrote, (who, req, m) -> (m, (who, req))) - | join(message, (_, (who, (rid, end, since)), (_, at, _, _, _, _, _, _, _, _, _, _, _)) -> ((rid, who), (at, since, end))) + | join(wrote, (who, (req, m)) -> (m, (who, req))) + | join(message, (_, ((who, (rid, end, since)), (_, at, _, _, _, _, _, _, _, _, _, _, _))) -> ((rid, who), (at, since, end))) | filter((_, (at, since, end)) -> ((at, since) ge, (at, end) le) all) | map((key, _) -> (key, 1)); let wrote_n = posts | map((key, _) -> (key, ())) | count; -let counts = wrote_n + ((cands - (cands | join(wrote_n, (key, v, _) -> (key, v)))) | map((key, _) -> (key, 0))); -let zombies = cands | join(counts, (key, (end, endmonth, _, month), n) -> (key, (end, endmonth, month, n))) +let counts = wrote_n + ((cands - (cands | join(wrote_n, (key, (v, _)) -> (key, v)))) | map((key, _) -> (key, 0))); +let zombies = cands | join(counts, (key, ((end, endmonth, _, month), n)) -> (key, (end, endmonth, month, n))) | filter((_, (_, endmonth, month, n)) -> (n, ((endmonth, month) sub, 1) add) lt) | map(((rid, who), (end, _, _, _)) -> ((rid, who), end)); -- Likes on zombies' messages by people created before the end, flagged if the liker is a zombie. let likes_z = zombies | map(((rid, who), end) -> (who, (rid, end))) - | join(wrote, (who, req, m) -> (m, (who, req))) - | join(liked, (_, (who, (rid, end)), (liker, _)) -> (liker, (rid, who, end))) - | join(person, (liker, (rid, who, end), (_, _, _, _, created, _, _, _, _, _, _)) -> ((rid, liker), (who, created, end))) + | join(wrote, (who, (req, m)) -> (m, (who, req))) + | join(liked, (_, ((who, (rid, end)), (liker, _))) -> (liker, (rid, who, end))) + | join(person, (liker, ((rid, who, end), (_, _, _, _, created, _, _, _, _, _, _))) -> ((rid, liker), (who, created, end))) | filter((_, (_, created, end)) -> (created, end) lt) | map(((rid, liker), (who, _, _)) -> ((rid, liker), who)); -let known = likes_z | join(zombies, (key, who, _) -> (key, who)); +let known = likes_z | join(zombies, (key, (who, _)) -> (key, who)); -- Likes per zombie (kind 0) and likes from zombies (kind 1), pivoted into (total, zombie); -- a zombie without likes gets (0, 0). let tally = ((likes_z | map(((rid, _), who) -> ((rid, who, 0), ()))) + (known | map(((rid, _), who) -> ((rid, who, 1), ())))) | count; diff --git a/interactive/bench/ldbc/authored/plans/bi14.ddp b/interactive/bench/ldbc/authored/plans/bi14.ddp index 5ed48ee91..edb1b14dd 100644 --- a/interactive/bench/ldbc/authored/plans/bi14.ddp +++ b/interactive/bench/ldbc/authored/plans/bi14.ddp @@ -14,36 +14,36 @@ let person_by_city = import "graph::person.by_city" : (int, int); -- Residents of country 1 with their city and its name; residents of country 2 -- (country name -> country -> cities -> people). let cities1 = request | map(((rid, c1, _), ()) -> (c1, rid)) - | join(place_by_name, (_, rid, country) -> (country, rid)) - | join(place_by_parent, (_, rid, city) -> (city, rid)) - | join(place, (city, rid, (cname, _, _)) -> (city, (rid, cname))); -let in1 = cities1 | join(person_by_city, (city, (rid, cname), p1) -> (p1, (rid, city, cname))); + | join(place_by_name, (_, (rid, country)) -> (country, rid)) + | join(place_by_parent, (_, (rid, city)) -> (city, rid)) + | join(place, (city, (rid, (cname, _, _))) -> (city, (rid, cname))); +let in1 = cities1 | join(person_by_city, (city, ((rid, cname), p1)) -> (p1, (rid, city, cname))); let in2 = request | map(((rid, _, c2), ()) -> (c2, rid)) - | join(place_by_name, (_, rid, country) -> (country, rid)) - | join(place_by_parent, (_, rid, city) -> (city, rid)) - | join(person_by_city, (_, rid, p2) -> ((rid, p2), ())); + | join(place_by_name, (_, (rid, country)) -> (country, rid)) + | join(place_by_parent, (_, (rid, city)) -> (city, rid)) + | join(person_by_city, (_, (rid, p2)) -> ((rid, p2), ())); -- Candidate pairs: friends p1 in country 1 and p2 in country 2, ((p1, p2), (rid, city, city name)). -let pairs = in1 | join(friend, (p1, (rid, city, cname), (p2, _)) -> ((rid, p2), (p1, city, cname))) - | join(in2, ((rid, p2), (p1, city, cname), ()) -> ((p1, p2), (rid, city, cname))); +let pairs = in1 | join(friend, (p1, ((rid, city, cname), (p2, _))) -> ((rid, p2), (p1, city, cname))) + | join(in2, ((rid, p2), ((p1, city, cname), ())) -> ((p1, p2), (rid, city, cname))); -- Who replied to whom, and who liked whose messages (each direction, once). let replied = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 1) eq) | map((_, (_, _, who, _, _, _, _, _, _, parent, _, _, _)) -> (parent, who)) - | join(message, (_, who, (_, _, to, _, _, _, _, _, _, _, _, _, _)) -> ((who, to), ())) | distinct; -let fan = likes | join(message, (_, (who, _), (_, _, to, _, _, _, _, _, _, _, _, _, _)) -> ((who, to), ())) | distinct; + | join(message, (_, (who, (_, _, to, _, _, _, _, _, _, _, _, _, _))) -> ((who, to), ())) | distinct; +let fan = likes | join(message, (_, ((who, _), (_, _, to, _, _, _, _, _, _, _, _, _, _))) -> ((who, to), ())) | distinct; -- Points per pair. let flip = pairs | map(((p1, p2), v) -> ((p2, p1), v)); let points = (pairs | map((pair, v) -> (pair, (v, 0)))) - + (pairs | join(replied, (pair, v, ()) -> (pair, (v, 4)))) - + (flip | join(replied, ((p2, p1), v, ()) -> ((p1, p2), (v, 1)))) - + (pairs | join(fan, (pair, v, ()) -> (pair, (v, 10)))) - + (flip | join(fan, ((p2, p1), v, ()) -> ((p1, p2), (v, 1)))); + + (pairs | join(replied, (pair, (v, ())) -> (pair, (v, 4)))) + + (flip | join(replied, ((p2, p1), (v, ())) -> ((p1, p2), (v, 1)))) + + (pairs | join(fan, (pair, (v, ())) -> (pair, (v, 10)))) + + (flip | join(fan, ((p2, p1), (v, ())) -> ((p1, p2), (v, 1)))); -- Sums: move each value into the multiplicity (join_function) and let `count` add them up. let summed = points | join_function(((p1, p2), ((rid, city, cname), s)) -> (((rid, city, cname, p1, p2), ()), 0, s) enlist) | count; let keys = pairs | map(((p1, p2), (rid, city, cname)) -> ((rid, city, cname, p1, p2), ())); -let scored = summed + ((keys - (keys | join(summed, (key, (), _) -> (key, ())))) | map((key, ()) -> (key, 0))); +let scored = summed + ((keys - (keys | join(summed, (key, ((), _)) -> (key, ())))) | map((key, ()) -> (key, 0))); -- The best pair per city, then the top 100 overall. -- Top k: `reduce` sees a key's values in sort order and keeps the first k (copies capped diff --git a/interactive/bench/ldbc/authored/plans/bi15.ddp b/interactive/bench/ldbc/authored/plans/bi15.ddp index de033728f..acb61bb78 100644 --- a/interactive/bench/ldbc/authored/plans/bi15.ddp +++ b/interactive/bench/ldbc/authored/plans/bi15.ddp @@ -14,11 +14,11 @@ let friend = import "graph::friend" : (int, (int, int)); -- (person -- Only friendships the search crosses are looked up here, so no friendship filter. let replies = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 1) eq) | map((child, (_, _, from, _, _, _, _, _, _, parent, _, _, _)) -> (parent, (child, from))); -let reply = replies | join(message, (_, (child, from), (kind, _, to, _, _, _, _, _, _, _, _, _, _)) -> (child, (from, to, ((kind, 0) eq, 2, 1) select))) +let reply = replies | join(message, (_, ((child, from), (kind, _, to, _, _, _, _, _, _, _, _, _, _))) -> (child, (from, to, ((kind, 0) eq, 2, 1) select))) | filter((_, (from, to, _)) -> (from, to) ne); -let dated = reply | join(root, (_, row, top) -> (top, row)) - | join(message, (_, row, (_, _, _, _, _, _, _, _, f, _, _, _, _)) -> (f, row)) - | join(forum, (_, (from, to, score), (_, when, _)) -> ((from, to), (when, score))); +let dated = reply | join(root, (_, (row, top)) -> (top, row)) + | join(message, (_, (row, (_, _, _, _, _, _, _, _, f, _, _, _, _))) -> (f, row)) + | join(forum, (_, ((from, to, score), (_, when, _))) -> ((from, to), (when, score))); let scored = dated + (dated | map(((from, to), v) -> ((to, from), v))); let window = request | map(((rid, _, _, start, end), ()) -> (rid, (start, end))); @@ -30,29 +30,29 @@ search: { -- The best meeting so far: least cost0 + cost1 over nodes both sides reached. let side0 = search | filter(((_, side, _, _), ()) -> (side, 0) eq) | map(((rid, _, node, c), ()) -> ((rid, node), c)); let side1 = search | filter(((_, side, _, _), ()) -> (side, 1) eq) | map(((rid, _, node, c), ()) -> ((rid, node), c)); - let best = side0 | join(side1, ((rid, _), c0, c1) -> (rid, (c0, c1) add)) | min; + let best = side0 | join(side1, ((rid, _), (c0, c1)) -> (rid, (c0, c1) add)) | min; -- A row keeps growing until its request has met, then only while 2·cost < best. let by_rid = search | map(((rid, side, node, c), ()) -> (rid, (side, node, c))); let met = best | map((rid, _) -> (rid, ())); - let open = by_rid - (by_rid | join(met, (rid, row, ()) -> (rid, row))); - let near = by_rid | join(best, (rid, (side, node, c), b) -> (rid, (side, node, c, b))) + let open = by_rid - (by_rid | join(met, (rid, (row, ())) -> (rid, row))); + let near = by_rid | join(best, (rid, ((side, node, c), b)) -> (rid, (side, node, c, b))) | filter((_, (_, _, c, b)) -> ((c, c) add, b) lt) | map((rid, (side, node, c, _)) -> (rid, (side, node, c))); let steps = (open + near) | map((rid, (side, node, c)) -> (node, (rid, side, c))) - | join(friend, (node, (rid, side, c), (next, _)) -> ((rid, node, next), (side, c))); + | join(friend, (node, ((rid, side, c), (next, _))) -> ((rid, node, next), (side, c))); -- Cost only the friendships being crossed: sum scores inside the request's window. let wanted = steps | map((key, _) -> (key, ())) | distinct; let inside = wanted | map(((rid, a, b), ()) -> ((a, b), rid)) - | join(scored, (pair, rid, (when, score)) -> (rid, (pair, when, score))) - | join(window, (rid, ((a, b), when, score), (start, end)) -> ((rid, a, b), (when, score, start, end))) + | join(scored, (pair, (rid, (when, score))) -> (rid, (pair, when, score))) + | join(window, (rid, (((a, b), when, score), (start, end))) -> ((rid, a, b), (when, score, start, end))) | filter((_, (when, _, start, end)) -> ((when, start) ge, (when, end) le) all) | map((key, (_, score, _, _)) -> (key, score)); let summed = inside | join_function((key, score) -> ((key, ()), (0, 0), score) enlist) | count; - let scores = summed + ((wanted - (wanted | join(summed, (key, (), _) -> (key, ())))) | map((key, ()) -> (key, 0))); + let scores = summed + ((wanted - (wanted | join(summed, (key, ((), _)) -> (key, ())))) | map((key, ()) -> (key, 0))); let cost = scores | map((key, s) -> (key, (2.0, (s, 2) add to_float) div)); - let grow = steps | join(cost, ((rid, _, next), (side, c), w) -> ((rid, side, next, (c, w) add), ())); + let grow = steps | join(cost, ((rid, _, next), ((side, c), w)) -> ((rid, side, next, (c, w) add), ())); set search = (starts + search + grow) | map(((rid, side, node, c), ()) -> ((rid, side, node), c)) | min | map(((rid, side, node), c) -> ((rid, side, node, c), ())); @@ -60,10 +60,10 @@ search: { let side0 = search::search | filter(((_, side, _, _), ()) -> (side, 0) eq) | map(((rid, _, node, c), ()) -> ((rid, node), c)); let side1 = search::search | filter(((_, side, _, _), ()) -> (side, 1) eq) | map(((rid, _, node, c), ()) -> ((rid, node), c)); -let best = side0 | join(side1, ((rid, _), c0, c1) -> (rid, (c0, c1) add)) | min; +let best = side0 | join(side1, ((rid, _), (c0, c1)) -> (rid, (c0, c1) add)) | min; let asked = request | map(((rid, _, _, _, _), ()) -> (rid, ())); -let never = asked - (asked | join(best, (rid, (), _) -> (rid, ()))); +let never = asked - (asked | join(best, (rid, ((), _)) -> (rid, ()))); let answer = (best | map((rid, c) -> ((rid, 0), (c)))) + (never | map((rid, ()) -> ((rid, 0), (-1 to_float)))); output "answer" = answer | arrange; diff --git a/interactive/bench/ldbc/authored/plans/bi16.ddp b/interactive/bench/ldbc/authored/plans/bi16.ddp index 0a6d8a2fb..4e698411d 100644 --- a/interactive/bench/ldbc/authored/plans/bi16.ddp +++ b/interactive/bench/ldbc/authored/plans/bi16.ddp @@ -10,9 +10,9 @@ let friend = import "graph::friend" : (int, (int, int)); -- (person -- The two (tag, day) groups of a request, as side 0 and side 1. let sides = (request | map(((rid, ta, da, _, _, k), ()) -> (ta, (rid, 0, da, k)))) + (request | map(((rid, _, _, tb, db, k), ()) -> (tb, (rid, 1, db, k)))); -let posts = sides | join(tag_by_name, (_, side, t) -> (t, side)) - | join(mtag_by_tag, (_, side, m) -> (m, side)) - | join(message, (m, (rid, s, d, k), (_, _, who, _, _, _, _, _, _, _, _, _, day)) -> ((rid, s, who, m, k), (d, day))) +let posts = sides | join(tag_by_name, (_, (side, t)) -> (t, side)) + | join(mtag_by_tag, (_, (side, m)) -> (m, side)) + | join(message, (m, ((rid, s, d, k), (_, _, who, _, _, _, _, _, _, _, _, _, day))) -> ((rid, s, who, m, k), (d, day))) | filter((_, (d, day)) -> (d, day) eq) | map((key, _) -> (key, ())); -- one tag per side; (message, tag) is a set @@ -20,13 +20,13 @@ let posts = sides | join(tag_by_name, (_, side, t) -> (t, side)) let counts = posts | map(((rid, s, who, _, k), ()) -> ((rid, s, who, k), ())) | count; let members = counts | map(((rid, s, who, _), _) -> ((rid, s, who), ())); let degree = members | map(((rid, s, who), ()) -> (who, (rid, s))) - | join(friend, (who, (rid, s), (f, _)) -> ((rid, s, f), who)) - | join(members, ((rid, s, _), who, ()) -> ((rid, s, who), ())) + | join(friend, (who, ((rid, s), (f, _))) -> ((rid, s, f), who)) + | join(members, ((rid, s, _), (who, ())) -> ((rid, s, who), ())) | count; let sized = counts | map(((rid, s, who, k), n) -> ((rid, s, who), (n, k))); -let social = sized | join(degree, (key, (n, k), d) -> (key, (n, k, d))) +let social = sized | join(degree, (key, ((n, k), d)) -> (key, (n, k, d))) | filter((_, (_, k, d)) -> (d, k) le) | map((key, (n, _, _)) -> (key, n)); -let loners = (sized - (sized | join(degree, (key, v, _) -> (key, v)))) | map((key, (n, _)) -> (key, n)); +let loners = (sized - (sized | join(degree, (key, (v, _)) -> (key, v)))) | map((key, (n, _)) -> (key, n)); let kept = (social + loners) | map(((rid, s, who), n) -> ((rid, who), (s, n))); -- People in both groups, ranked by messages on both days. @@ -34,7 +34,7 @@ let a = kept | filter((_, (s, _)) -> (s, 0) eq) | map((key, (_, n)) -> (ke let b = kept | filter((_, (s, _)) -> (s, 1) eq) | map((key, (_, n)) -> (key, n)); -- Top k: `reduce` sees a key's values in sort order and keeps the first k (copies capped -- at k), emitting (rank, value). Changes below the top k produce no output. -let ranked = a | join(b, ((rid, who), na, nb) -> (rid, (((na, nb) add neg, who), (na, nb)))) +let ranked = a | join(b, ((rid, who), (na, nb)) -> (rid, (((na, nb) add neg, who), (na, nb)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((_, who), (na, nb)))) -> ((rid, rank), (who, na, nb))); diff --git a/interactive/bench/ldbc/authored/plans/bi17.ddp b/interactive/bench/ldbc/authored/plans/bi17.ddp index f5f212a88..db5fd9dce 100644 --- a/interactive/bench/ldbc/authored/plans/bi17.ddp +++ b/interactive/bench/ldbc/authored/plans/bi17.ddp @@ -16,27 +16,27 @@ let member = import "graph::member.by_forum" : (int, (int, int)); -- (forum, -- message's forum is its thread's (the root post's). Tag names are unique and -- (message, tag) pairs are a set, so each message appears once. let tagged = request | map(((rid, name, _), ()) -> (name, rid)) - | join(tag_by_name, (_, rid, t) -> (t, rid)) - | join(mtag_by_tag, (_, rid, m) -> (m, rid)) - | join(message, (m, rid, (_, created, author, _, _, _, _, _, _, parent, _, _, _)) -> (m, (rid, author, created, parent))) - | join(root, (m, row, top) -> (top, (m, row))) - | join(message, (_, (m, (rid, author, created, parent)), (_, _, _, _, _, _, _, _, f, _, _, _, _)) -> ((rid, m, author, created, parent, f), ())); + | join(tag_by_name, (_, (rid, t)) -> (t, rid)) + | join(mtag_by_tag, (_, (rid, m)) -> (m, rid)) + | join(message, (m, (rid, (_, created, author, _, _, _, _, _, _, parent, _, _, _))) -> (m, (rid, author, created, parent))) + | join(root, (m, (row, top)) -> (top, (m, row))) + | join(message, (_, ((m, (rid, author, created, parent)), (_, _, _, _, _, _, _, _, f, _, _, _, _))) -> ((rid, m, author, created, parent, f), ())); -- Memberships, but only of forums holding one of the request's tagged messages. let forums = tagged | map(((rid, _, _, _, _, f), ()) -> ((f, rid), ())) | distinct | map(((f, rid), ()) -> (f, rid)); -let members = forums | join(member, (f, rid, (who, _)) -> ((rid, f, who), ())); +let members = forums | join(member, (f, (rid, (who, _))) -> ((rid, f, who), ())); -- Discussions: a tagged message2 with a tagged reply by someone else. For each, the -- candidate forum1s: forums (other than forum2) where both authors are members. let by_mid = tagged | map(((rid, m, author, created, _, f), ()) -> ((rid, m), (author, created, f))); let answers = tagged | map(((rid, _, author, _, parent, _), ()) -> ((rid, parent), author)); -let discuss = by_mid | join(answers, ((rid, m), (a2, created, f2), a3) -> ((rid, m, a2, created, f2, a3), ())) +let discuss = by_mid | join(answers, ((rid, m), ((a2, created, f2), a3)) -> ((rid, m, a2, created, f2, a3), ())) | filter(((_, _, a2, _, _, a3), ()) -> (a2, a3) ne); let in_forum = members | map(((rid, f, who), ()) -> ((rid, who), f)); let candidate = discuss | map(((rid, m, a2, created, f2, a3), ()) -> ((rid, a2), (m, created, f2, a3))) - | join(in_forum, ((rid, _), (m, created, f2, a3), f1) -> ((rid, f1, a3), (m, created, f2))) + | join(in_forum, ((rid, _), ((m, created, f2, a3), f1)) -> ((rid, f1, a3), (m, created, f2))) | filter(((_, f1, _), (_, _, f2)) -> (f1, f2) ne) - | join(members, ((rid, f1, _), (m, created, f2), ()) -> ((rid, f1), (m, created, f2))); + | join(members, ((rid, f1, _), ((m, created, f2), ())) -> ((rid, f1), (m, created, f2))); -- Each person's earliest tagged message per forum: (rid, forum1) -> (person1, first). let earliest = tagged | map(((rid, _, author, created, _, f), ()) -> ((rid, f, author), created)) | min @@ -44,10 +44,10 @@ let earliest = tagged | map(((rid, _, author, created, _, f), ()) -> ((rid, f, -- person1 posted in forum1 more than delta hours before message2, and is not in forum2. let delta = request | map(((rid, _, hours), ()) -> (rid, (hours, 3600000) mul)); -let spread = earliest | join(candidate, ((rid, _), (p1, first), (m, created, f2)) -> (rid, (p1, first, m, created, f2))) - | join(delta, (rid, (p1, first, m, created, f2), wait) -> ((rid, f2, p1), (m, created, (first, wait) add))) +let spread = earliest | join(candidate, ((rid, _), ((p1, first), (m, created, f2))) -> (rid, (p1, first, m, created, f2))) + | join(delta, (rid, ((p1, first, m, created, f2), wait)) -> ((rid, f2, p1), (m, created, (first, wait) add))) | filter((_, (_, created, after)) -> (after, created) lt); -let spread_ok = spread - (spread | join(members, (key, value, ()) -> (key, value))); +let spread_ok = spread - (spread | join(members, (key, (value, ())) -> (key, value))); -- Count distinct message2 per person1 and rank. let counts = spread_ok | map(((rid, _, p1), (m, _, _)) -> ((rid, p1, m), ())) | distinct diff --git a/interactive/bench/ldbc/authored/plans/bi18.ddp b/interactive/bench/ldbc/authored/plans/bi18.ddp index df0a0396f..464b7fa55 100644 --- a/interactive/bench/ldbc/authored/plans/bi18.ddp +++ b/interactive/bench/ldbc/authored/plans/bi18.ddp @@ -10,18 +10,18 @@ let friend = import "graph::friend" : (int, (int, int)); -- -- People interested in the tag: (rid, person). let fans = request | map(((rid, name), ()) -> (name, rid)) - | join(tag_by_name, (_, rid, t) -> (t, rid)) - | join(interest_by_tag, (_, rid, who) -> ((rid, who), ())); -- one tag; (person, tag) is a set + | join(tag_by_name, (_, (rid, t)) -> (t, rid)) + | join(interest_by_tag, (_, (rid, who)) -> ((rid, who), ())); -- one tag; (person, tag) is a set -- Each fan's friends, as both ends of a path: (rid, shared friend) -> fan. -let reach = fans | map(((rid, who), ()) -> (who, rid)) | join(friend, (who, rid, (mid, _)) -> ((rid, mid), who)); -let paths = reach | join(reach, ((rid, _), one, other) -> ((rid, one, other), ())) +let reach = fans | map(((rid, who), ()) -> (who, rid)) | join(friend, (who, (rid, (mid, _))) -> ((rid, mid), who)); +let paths = reach | join(reach, ((rid, _), (one, other)) -> ((rid, one, other), ())) | filter(((_, one, other), ()) -> (one, other) ne); -- Drop pairs who are already friends (both are fans, so `reach` holds their -- friendships), count shared friends, rank. let known = reach | map(((rid, mid), who) -> ((rid, who, mid), ())); -let fresh = paths - (paths | join(known, (key, (), ()) -> (key, ()))); +let fresh = paths - (paths | join(known, (key, ((), ())) -> (key, ()))); let pairs = fresh | count; -- Top k: `reduce` sees a key's values in sort order and keeps the first k (copies capped -- at k), emitting (rank, value). Changes below the top k produce no output. diff --git a/interactive/bench/ldbc/authored/plans/bi19.ddp b/interactive/bench/ldbc/authored/plans/bi19.ddp index 73a0b1c7e..4ce44fb6b 100644 --- a/interactive/bench/ldbc/authored/plans/bi19.ddp +++ b/interactive/bench/ldbc/authored/plans/bi19.ddp @@ -12,30 +12,30 @@ let person_by_city = import "graph::person.by_city" : (int, int); -- (city, -- 1 + the number of k in 1..39 with k(k-1) + 1 > n. let answered = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 1) eq) | map((_, (_, _, who, _, _, _, _, _, _, parent, _, _, _)) -> (parent, who)) - | join(message, (_, who, (_, _, to, _, _, _, _, _, _, _, _, _, _)) -> (who, to)) + | join(message, (_, (who, (_, _, to, _, _, _, _, _, _, _, _, _, _))) -> (who, to)) | filter((who, to) -> (who, to) ne); let talks = (answered | map((a, b) -> ((a, b), ()))) + (answered | map((a, b) -> ((b, a), ()))); let weighted = talks | count - | join(pals, (pair, n, ()) -> (pair, n)) + | join(pals, (pair, (n, ())) -> (pair, n)) | map(((a, b), n) -> (a, (b, (((0, n), (1, 40) range) fold (((acc, n), k) -> ((acc, ((((k, (k, 1) sub) mul, 1) add, n) gt, 1, 0) select) add, n)) .0, 1) add))); -- One search per (request, source in city 1). let starts = request | map(((rid, c1, _), ()) -> (c1, rid)) - | join(person_by_city, (_, rid, who) -> ((rid, who, who, 0), ())); + | join(person_by_city, (_, (rid, who)) -> ((rid, who, who, 0), ())); search: { let grow = search | map(((rid, source, node, cost), ()) -> (node, (rid, source, cost))) - | join(weighted, (_, (rid, source, cost), (next, w)) -> ((rid, source, next, (cost, w) add), ())); + | join(weighted, (_, ((rid, source, cost), (next, w))) -> ((rid, source, next, (cost, w) add), ())); set search = (starts + grow) | map(((rid, source, node, cost), ()) -> ((rid, source, node), cost)) | min | map(((rid, source, node), cost) -> ((rid, source, node, cost), ())); } -- Paths that end in city 2, and the least cost among them. let ends = search::search | map(((rid, source, node, cost), ()) -> (node, (rid, source, cost))) - | join(person, (node, (rid, source, cost), (_, _, _, _, _, _, _, city, _, _, _)) -> ((rid, city), (source, node, cost))) - | join(request | map(((rid, _, c2), ()) -> ((rid, c2), ())), ((rid, _), row, ()) -> (rid, row)); + | join(person, (node, ((rid, source, cost), (_, _, _, _, _, _, _, city, _, _, _))) -> ((rid, city), (source, node, cost))) + | join(request | map(((rid, _, c2), ()) -> ((rid, c2), ())), ((rid, _), (row, ())) -> (rid, row)); let best = ends | map((rid, (_, _, cost)) -> (rid, cost)) | min; -let ranked = ends | join(best, (rid, (source, node, cost), least) -> (rid, (source, node, cost, least))) +let ranked = ends | join(best, (rid, ((source, node, cost), least)) -> (rid, (source, node, cost, least))) | filter((_, (_, _, cost, least)) -> (cost, least) eq) | map((rid, (source, node, cost, _)) -> (rid, ((source, node), cost))) | collect | flatmap((k, entries) -> (k, ((0, entries len) range, entries) zip) cap_list); diff --git a/interactive/bench/ldbc/authored/plans/bi20.ddp b/interactive/bench/ldbc/authored/plans/bi20.ddp index a8b96c182..581725a2c 100644 --- a/interactive/bench/ldbc/authored/plans/bi20.ddp +++ b/interactive/bench/ldbc/authored/plans/bi20.ddp @@ -11,8 +11,8 @@ let work_by_org = import "graph::work.by_org" : (int, (int, int)); -- ( -- Classmate friendships, weighted by the closest pair of graduation years: start from -- each friendship and look up both people's schools. -let mates = friend | join(study_by_person, (a, (b, _), (at, ya)) -> (b, (a, at, ya))) - | join(study_by_person, (b, (a, at, ya), (bt, yb)) -> ((a, b), (at, bt, (ya, yb) sub))) +let mates = friend | join(study_by_person, (a, ((b, _), (at, ya))) -> (b, (a, at, ya))) + | join(study_by_person, (b, ((a, at, ya), (bt, yb))) -> ((a, b), (at, bt, (ya, yb) sub))) | filter(((a, b), (at, bt, _)) -> ((at, bt) eq, (a, b) ne) all) | map((pair, (_, _, d)) -> (pair, ((d, 0) lt, d neg, d) select)) | map((pair, gap) -> (pair, (gap, 1) add)) | min @@ -20,26 +20,26 @@ let mates = friend | join(study_by_person, (a, (b, _), (at, ya)) -> (b, (a, a -- The company's employees, and requests whose person is not one of them. let staff = request | map(((rid, name, _), ()) -> (name, rid)) - | join(org_by_name, (_, rid, at) -> (at, rid)) - | join(work_by_org, (_, rid, (who, _)) -> ((rid, who), ())) | distinct; + | join(org_by_name, (_, (rid, at)) -> (at, rid)) + | join(work_by_org, (_, (rid, (who, _))) -> ((rid, who), ())) | distinct; let asked = request | map(((rid, _, who), ()) -> ((rid, who), ())); -let outside = asked - (asked | join(staff, (key, (), ()) -> (key, ()))); +let outside = asked - (asked | join(staff, (key, ((), ())) -> (key, ()))); -- Shortest paths from the person, then the employees at the least distance. let starts = outside | map(((rid, who), ()) -> ((rid, who, who, 0), ())); search: { let grow = search | map(((rid, from, node, cost), ()) -> (node, (rid, from, cost))) - | join(mates, (_, (rid, from, cost), (next, w)) -> ((rid, from, next, (cost, w) add), ())); + | join(mates, (_, ((rid, from, cost), (next, w))) -> ((rid, from, next, (cost, w) add), ())); set search = (starts + grow) | map(((rid, from, node, cost), ()) -> ((rid, from, node), cost)) | min | map(((rid, from, node), cost) -> ((rid, from, node, cost), ())); } let reached = search::search | map(((rid, from, node, cost), ()) -> ((rid, node), (from, cost))) - | join(staff, ((rid, node), (from, cost), ()) -> (rid, (node, from, cost))) + | join(staff, ((rid, node), ((from, cost), ())) -> (rid, (node, from, cost))) | filter((_, (node, from, _)) -> (node, from) ne); let best = reached | map((rid, (_, _, cost)) -> (rid, cost)) | min; -- Top k: `reduce` sees a key's values in sort order and keeps the first k (copies capped -- at k), emitting (rank, value). Changes below the top k produce no output. -let ranked = reached | join(best, (rid, (node, _, cost), least) -> (rid, (node, cost, least))) +let ranked = reached | join(best, (rid, ((node, _, cost), least)) -> (rid, (node, cost, least))) | filter((_, (_, cost, least)) -> (cost, least) eq) | map((rid, (node, cost, _)) -> (rid, (cost, node))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); diff --git a/interactive/bench/ldbc/authored/plans/ic01.ddp b/interactive/bench/ldbc/authored/plans/ic01.ddp index bd676e4e0..8bf33b12f 100644 --- a/interactive/bench/ldbc/authored/plans/ic01.ddp +++ b/interactive/bench/ldbc/authored/plans/ic01.ddp @@ -15,42 +15,42 @@ let language = import "graph::language.by_person" : (int, List(int)); -- The circle with distances: least hops (1 to 3) to each person, not the person. let hop1 = request | map(((rid, who, name), ()) -> (who, (rid, who, name))) - | join(friend, (_, req, (f, _)) -> (f, (req, 1))); -let hop2 = hop1 | join(friend, (_, (req, _), (g, _)) -> (g, (req, 2))); -let hop3 = hop2 | join(friend, (_, (req, _), (h, _)) -> (h, (req, 3))); + | join(friend, (_, (req, (f, _))) -> (f, (req, 1))); +let hop2 = hop1 | join(friend, (_, ((req, _), (g, _))) -> (g, (req, 2))); +let hop3 = hop2 | join(friend, (_, ((req, _), (h, _))) -> (h, (req, 3))); let circle = (hop1 + hop2 + hop3) | filter((node, ((_, who, _), _)) -> (node, who) ne) | map((node, ((rid, _, name), d)) -> ((rid, node, name), d)) | min; -- Those with the first name, with their city's name: ((rid, node), (d, profile)). let found = circle | map(((rid, node, name), d) -> (node, (rid, name, d))) - | join(person, (node, (rid, name, d), (first, last, gender, birthday, created, ip, browser, home, _, _, _)) -> + | join(person, (node, ((rid, name, d), (first, last, gender, birthday, created, ip, browser, home, _, _, _))) -> ((first, name) eq, (home, (rid, node, d, last, gender, birthday, created, ip, browser)))) | filter((named, _) -> named) | map((_, row) -> row) - | join(place, (_, (rid, node, d, last, gender, birthday, created, ip, browser), (town, _, _)) -> + | join(place, (_, ((rid, node, d, last, gender, birthday, created, ip, browser), (town, _, _))) -> ((rid, node), (d, (last, gender, birthday, created, ip, browser, town)))); -- For the people found: sorted emails, languages, and (organisation, year, place) for -- study and work. let who = found | map(((rid, node), _) -> (node, rid)); -let emails = who | join(email, (node, rid, value) -> ((rid, node), value)) | collect; -let langs = who | join(language, (node, rid, value) -> ((rid, node), value)) | collect; -let school = who | join(study, (node, rid, (at, year)) -> (at, (rid, node, year))) - | join(org, (_, (rid, node, year), (name, _, at)) -> (at, (rid, node, name, year))) - | join(place, (_, (rid, node, name, year), (where, _, _)) -> ((rid, node), (name, year, where))) | collect; -let jobs = who | join(work, (node, rid, (at, year)) -> (at, (rid, node, year))) - | join(org, (_, (rid, node, year), (name, _, at)) -> (at, (rid, node, name, year))) - | join(place, (_, (rid, node, name, year), (where, _, _)) -> ((rid, node), (name, year, where))) | collect; +let emails = who | join(email, (node, (rid, value)) -> ((rid, node), value)) | collect; +let langs = who | join(language, (node, (rid, value)) -> ((rid, node), value)) | collect; +let school = who | join(study, (node, (rid, (at, year))) -> (at, (rid, node, year))) + | join(org, (_, ((rid, node, year), (name, _, at))) -> (at, (rid, node, name, year))) + | join(place, (_, ((rid, node, name, year), (where, _, _))) -> ((rid, node), (name, year, where))) | collect; +let jobs = who | join(work, (node, (rid, (at, year))) -> (at, (rid, node, year))) + | join(org, (_, ((rid, node, year), (name, _, at))) -> (at, (rid, node, name, year))) + | join(place, (_, ((rid, node, name, year), (where, _, _))) -> ((rid, node), (name, year, where))) | collect; -- Attach a list, or an empty one for people without entries. -let with1 = (found | join(emails, (key, row, list) -> (key, (row, list)))) - + ((found - (found | join(emails, (key, row, _) -> (key, row)))) | map((key, row) -> (key, (row, [] : List(List(int)))))); -let with2 = (with1 | join(langs, (key, (row, e), list) -> (key, (row, e, list)))) - + ((with1 - (with1 | join(langs, (key, row, _) -> (key, row)))) | map((key, (row, e)) -> (key, (row, e, [] : List(List(int)))))); -let with3 = (with2 | join(school, (key, (row, e, l), list) -> (key, (row, e, l, list)))) - + ((with2 - (with2 | join(school, (key, row, _) -> (key, row)))) | map((key, (row, e, l)) -> (key, (row, e, l, [] : List((List(int), int, List(int))))))); -let with4 = (with3 | join(jobs, (key, (row, e, l, s), list) -> (key, (row, e, l, s, list)))) - + ((with3 - (with3 | join(jobs, (key, row, _) -> (key, row)))) | map((key, (row, e, l, s)) -> (key, (row, e, l, s, [] : List((List(int), int, List(int))))))); +let with1 = (found | join(emails)) + + ((found - (found | join(emails, (key, (row, _)) -> (key, row)))) | map((key, row) -> (key, (row, [] : List(List(int)))))); +let with2 = (with1 | join(langs, (key, ((row, e), list)) -> (key, (row, e, list)))) + + ((with1 - (with1 | join(langs, (key, (row, _)) -> (key, row)))) | map((key, (row, e)) -> (key, (row, e, [] : List(List(int)))))); +let with3 = (with2 | join(school, (key, ((row, e, l), list)) -> (key, (row, e, l, list)))) + + ((with2 - (with2 | join(school, (key, (row, _)) -> (key, row)))) | map((key, (row, e, l)) -> (key, (row, e, l, [] : List((List(int), int, List(int))))))); +let with4 = (with3 | join(jobs, (key, ((row, e, l, s), list)) -> (key, (row, e, l, s, list)))) + + ((with3 - (with3 | join(jobs, (key, (row, _)) -> (key, row)))) | map((key, (row, e, l, s)) -> (key, (row, e, l, s, [] : List((List(int), int, List(int))))))); -- Top k: `reduce` sees a key's values in sort order and keeps the first k (copies capped -- at k), emitting (rank, value). Changes below the top k produce no output. diff --git a/interactive/bench/ldbc/authored/plans/ic02.ddp b/interactive/bench/ldbc/authored/plans/ic02.ddp index df5509008..55a58a661 100644 --- a/interactive/bench/ldbc/authored/plans/ic02.ddp +++ b/interactive/bench/ldbc/authored/plans/ic02.ddp @@ -20,20 +20,20 @@ let newest = by_author | map((author, (neg, _)) -> (author, neg)) | min; -- The person's friends and the request's cut-off. let friends = request | map(((rid, who, max), ()) -> (who, (rid, max))) - | join(friend, (_, req, (f, _)) -> (f, req)); -let ahead = friends | join(newest, (f, (rid, max), neg) -> (f, (rid, max, neg neg))); + | join(friend, (_, (req, (f, _))) -> (f, req)); +let ahead = friends | join(newest, (f, ((rid, max), neg)) -> (f, (rid, max, neg neg))); let fast = ahead | filter((_, (_, max, last)) -> (last, max) lt) - | join(latest, (f, (rid, _, _), (neg, id)) -> (rid, ((neg, id), f))); + | join(latest, (f, ((rid, _, _), (neg, id))) -> (rid, ((neg, id), f))); let slow = ahead | filter((_, (_, max, last)) -> (last, max) ge) - | join(by_author, (f, (rid, max, _), (neg, id)) -> (rid, ((neg, id), f, max))) + | join(by_author, (f, ((rid, max, _), (neg, id))) -> (rid, ((neg, id), f, max))) | filter((_, ((neg, _), _, max)) -> (neg neg, max) lt) | map((rid, (order, f, _)) -> (rid, (order, f))); let top = (fast + slow) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); -- Only now fetch the 20 messages' text and their authors' names. let answer = top | map((rid, (rank, ((neg, id), f))) -> (id, (rid, rank, neg neg, f))) - | join(message, (id, (rid, rank, created, f), (_, _, _, _, content, image, _, _, _, _, _, _, _)) -> + | join(message, (id, ((rid, rank, created, f), (_, _, _, _, content, image, _, _, _, _, _, _, _))) -> (f, (rid, rank, id, ((content len, 0) gt, content, image) select, created))) - | join(person, (f, (rid, rank, id, body, created), (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, rank), (f, first, last, id, body, created))); + | join(person, (f, ((rid, rank, id, body, created), (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, rank), (f, first, last, id, body, created))); output "answer" = answer | arrange; diff --git a/interactive/bench/ldbc/authored/plans/ic03.ddp b/interactive/bench/ldbc/authored/plans/ic03.ddp index ed4ec2574..3abb0b0d1 100644 --- a/interactive/bench/ldbc/authored/plans/ic03.ddp +++ b/interactive/bench/ldbc/authored/plans/ic03.ddp @@ -14,41 +14,41 @@ let by_name = import "graph::place.by_name" : (List(int), int); -- The request with X and Y as country ids ("Country" is the place type): (rid, (person, x, y, start, end)). let asked = request | map(((rid, who, x, y, start, days), ()) -> (x, (rid, who, y, start, (start, (days, 86400000) mul) add))) - | join(by_name, (_, req, x) -> (x, req)) - | join(place, (x, (rid, who, y, start, end), (_, kind, _)) -> ((kind, [67,111,117,110,116,114,121]) eq, (y, (rid, who, x, start, end)))) + | join(by_name, (_, (req, x)) -> (x, req)) + | join(place, (x, ((rid, who, y, start, end), (_, kind, _))) -> ((kind, [67,111,117,110,116,114,121]) eq, (y, (rid, who, x, start, end)))) | filter((country, _) -> country) | map((_, row) -> row) - | join(by_name, (_, req, y) -> (y, req)) - | join(place, (y, (rid, who, x, start, end), (_, kind, _)) -> ((kind, [67,111,117,110,116,114,121]) eq, (rid, (who, x, y, start, end)))) + | join(by_name, (_, (req, y)) -> (y, req)) + | join(place, (y, ((rid, who, x, start, end), (_, kind, _))) -> ((kind, [67,111,117,110,116,114,121]) eq, (rid, (who, x, y, start, end)))) | filter((country, _) -> country) | map((_, row) -> row); -- Messages sent from X or from Y inside the window: ((rid, author), (in X, in Y)). let sent_from = message | map((_, (_, created, author, country, _, _, _, _, _, _, _, _, _)) -> (country, (author, created))); let window = (asked | map((rid, (_, x, _, start, end)) -> (x, (rid, start, end, 1, 0)))) + (asked | map((rid, (_, _, y, start, end)) -> (y, (rid, start, end, 0, 1)))); -let visits = window | join(sent_from, (_, (rid, start, end, ix, iy), (author, created)) -> ((rid, author), (created, start, end, ix, iy))) +let visits = window | join(sent_from, (_, ((rid, start, end, ix, iy), (author, created))) -> ((rid, author), (created, start, end, ix, iy))) | filter((_, (created, start, end, _, _)) -> ((created, start) ge, (created, end) lt) all) | map((key, (_, _, _, ix, iy)) -> (key, (ix, iy))); -- The circle (one or two hops, not the person), living outside both X and Y. let hop1 = asked | map((rid, (who, x, y, _, _)) -> (who, (rid, who, x, y))) - | join(friend, (_, req, (f, _)) -> (f, req)); -let hop2 = hop1 | join(friend, (_, req, (g, _)) -> (g, req)); + | join(friend, (_, (req, (f, _))) -> (f, req)); +let hop2 = hop1 | join(friend, (_, (req, (g, _))) -> (g, req)); let circle = (hop1 + hop2) | filter((node, (_, who, _, _)) -> (node, who) ne) - | join(person, (node, (rid, _, x, y), (_, _, _, _, _, _, _, city, _, _, _)) -> (city, (rid, node, x, y))) - | join(place, (_, (rid, node, x, y), (_, _, home)) -> ((rid, node), (x, y, home))) + | join(person, (node, ((rid, _, x, y), (_, _, _, _, _, _, _, city, _, _, _))) -> (city, (rid, node, x, y))) + | join(place, (_, ((rid, node, x, y), (_, _, home))) -> ((rid, node), (x, y, home))) | filter((_, (x, y, home)) -> ((home, x) ne, (home, y) ne) all) | map((key, _) -> (key, ())) | distinct; -- Per person in the circle: messages from X and from Y. A count of zero has no row, -- so "both positive" is a join of the two counts. -let seen = visits | join(circle, (key, (ix, iy), ()) -> (key, (ix, iy))); +let seen = visits | join(circle, (key, ((ix, iy), ())) -> (key, (ix, iy))); let in_x = seen | filter((_, (ix, _)) -> (ix, 1) eq) | map((key, _) -> (key, ())) | count; let in_y = seen | filter((_, (_, iy)) -> (iy, 1) eq) | map((key, _) -> (key, ())) | count; -let counts = in_x | join(in_y, (key, nx, ny) -> (key, (nx, ny))); +let counts = in_x | join(in_y); -- Top k: `reduce` sees a key's values in sort order and keeps the first k (copies capped -- at k), emitting (rank, value). Changes below the top k produce no output. let ranked = counts | map(((rid, node), (nx, ny)) -> (node, (rid, nx, ny))) - | join(person, (node, (rid, nx, ny), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, (((nx, ny) add neg, node), (first, last, nx, ny)))) + | join(person, (node, ((rid, nx, ny), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, (((nx, ny) add neg, node), (first, last, nx, ny)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((_, node), (first, last, nx, ny)))) -> ((rid, rank), (node, first, last, nx, ny, (nx, ny) add))); diff --git a/interactive/bench/ldbc/authored/plans/ic04.ddp b/interactive/bench/ldbc/authored/plans/ic04.ddp index 7e1b43f9b..8c4934bb9 100644 --- a/interactive/bench/ldbc/authored/plans/ic04.ddp +++ b/interactive/bench/ldbc/authored/plans/ic04.ddp @@ -12,16 +12,16 @@ let mtag = import "graph::mtag.by_tag" : (int, int); -- (tag, message -- Friends' posts with their tags: ((rid, tag), (tag name, created, window start, window end)). let posts = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 0) eq) | map((id, (_, created, who, _, _, _, _, _, _, _, _, _, _)) -> (who, (id, created))); -let tags = mtag | join(tag, (t, m, (name, _)) -> (m, (t, name))); +let tags = mtag | join(tag, (t, (m, (name, _))) -> (m, (t, name))); let seen = request | map(((rid, who, start, days), ()) -> (who, (rid, start, (start, (days, 86400000) mul) add))) - | join(friend, (_, req, (f, _)) -> (f, req)) - | join(posts, (_, (rid, start, end), (m, created)) -> (m, (rid, created, start, end))) - | join(tags, (_, (rid, created, start, end), (t, name)) -> ((rid, t), (name, created, start, end))); + | join(friend, (_, (req, (f, _))) -> (f, req)) + | join(posts, (_, ((rid, start, end), (m, created))) -> (m, (rid, created, start, end))) + | join(tags, (_, ((rid, created, start, end), (t, name))) -> ((rid, t), (name, created, start, end))); -- Tags seen before the window, and the window's posts whose tag is new. let before = seen | filter((_, (_, created, start, _)) -> (created, start) lt) | map((key, _) -> (key, ())) | distinct; let during = seen | filter((_, (_, created, start, end)) -> ((created, start) ge, (created, end) lt) all); -let fresh = during - (during | join(before, (key, row, ()) -> (key, row))); +let fresh = during - (during | join(before, (key, (row, ())) -> (key, row))); let counts = fresh | map(((rid, _), (name, _, _, _)) -> ((rid, name), ())) | count; -- Top k: `reduce` sees a key's values in sort order and keeps the first k (copies capped -- at k), emitting (rank, value). Changes below the top k produce no output. diff --git a/interactive/bench/ldbc/authored/plans/ic05.ddp b/interactive/bench/ldbc/authored/plans/ic05.ddp index 3cb97dc57..b72194c0d 100644 --- a/interactive/bench/ldbc/authored/plans/ic05.ddp +++ b/interactive/bench/ldbc/authored/plans/ic05.ddp @@ -14,15 +14,15 @@ let member = import "graph::member.by_person" : (int, (int, int)); -- (per -- The circle (one or two hops, not the person), with the request's date. let hop1 = request | map(((rid, who, date), ()) -> (who, (rid, who, date))) - | join(person, (who, req, _) -> (who, req)) - | join(friend, (_, req, (f, _)) -> (f, req)); -let hop2 = hop1 | join(friend, (_, req, (g, _)) -> (g, req)); + | join(person, (who, (req, _)) -> (who, req)) + | join(friend, (_, (req, (f, _))) -> (f, req)); +let hop2 = hop1 | join(friend, (_, (req, (g, _))) -> (g, req)); let circle = (hop1 + hop2) | filter((node, (_, who, _)) -> (node, who) ne) | map((node, (rid, _, date)) -> ((node, rid, date), ())) | distinct | map(((node, rid, date), ()) -> (node, (rid, date))); -- Their memberships that began after the date: ((node, forum), rid). -let after = circle | join(member, (node, (rid, date), (f, since)) -> ((node, f), (rid, date, since))) +let after = circle | join(member, (node, ((rid, date), (f, since))) -> ((node, f), (rid, date, since))) | filter((_, (_, date, since)) -> (since, date) gt) | map((key, (rid, _, _)) -> (key, rid)); @@ -32,7 +32,7 @@ let posted = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> -- Forums with posts by circle members who joined after the date: ((rid, forum), posts). -- Sums: move each value into the multiplicity (join_function) and let `count` add them up. -let busy = after | join(posted, ((_, f), rid, n) -> ((rid, f), n)) +let busy = after | join(posted, ((_, f), (rid, n)) -> ((rid, f), n)) | join_function((key, n) -> ((key, ()), 0, n) enlist) | count; -- Forums joined after the date with no such posts: from each person's 40 smallest-id @@ -41,18 +41,18 @@ let first = member | map((who, (_, since)) -> (who, since)) | min; -- Top k: `reduce` sees a key's values in sort order and keeps the first k (copies capped -- at k), emitting (rank, value). Changes below the top k produce no output. let lowest = member | map((who, (f, _)) -> (who, f)) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 40) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 40) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 40) min in let before = (end, count) sub in let keep = (((40, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten) | map((who, (_, f)) -> (who, f)); -let timed = circle | join(first, (node, (rid, date), since) -> (node, (rid, date, since))); +let timed = circle | join(first, (node, ((rid, date), since)) -> (node, (rid, date, since))); let fast = timed | filter((_, (_, date, since)) -> (since, date) gt) - | join(lowest, (_, (rid, _, _), f) -> ((rid, f), ())); + | join(lowest, (_, ((rid, _, _), f)) -> ((rid, f), ())); let slow = timed | filter((_, (_, date, since)) -> (since, date) le) | map((node, (rid, _, _)) -> ((node, rid), ())) - | join(after | map(((node, f), rid) -> ((node, rid), f)), ((_, rid), (), f) -> ((rid, f), ())); + | join(after | map(((node, f), rid) -> ((node, rid), f)), ((_, rid), ((), f)) -> ((rid, f), ())); let idle = (fast + slow) | distinct; -let quiet = (idle - (idle | join(busy, (key, (), _) -> (key, ())))) | map((key, ()) -> (key, 0)); +let quiet = (idle - (idle | join(busy, (key, ((), _)) -> (key, ())))) | map((key, ()) -> (key, 0)); -- Rank by posts (most first), then forum id. let ranked = (busy + quiet) | map(((rid, f), n) -> (f, (rid, n))) - | join(forum, (f, (rid, n), (title, _, _)) -> (rid, ((n neg, f), title))) + | join(forum, (f, ((rid, n), (title, _, _))) -> (rid, ((n neg, f), title))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((neg, _), title))) -> ((rid, rank), (title, neg neg))); diff --git a/interactive/bench/ldbc/authored/plans/ic06.ddp b/interactive/bench/ldbc/authored/plans/ic06.ddp index 34e00674b..87c01c9af 100644 --- a/interactive/bench/ldbc/authored/plans/ic06.ddp +++ b/interactive/bench/ldbc/authored/plans/ic06.ddp @@ -13,29 +13,29 @@ let tagged = import "graph::mtag.by_tag" : (int, int); -- (tag let tags_of = import "graph::mtag.by_message" : (int, int); -- (message, tag) -- The circle: everyone one or two hops from the person, except the person. -let hop1 = request | map(((rid, who, _), ()) -> (who, (rid, who))) | join(person, (who, req, _) -> (who, req)) - | join(friend, (_, (rid, who), (f, _)) -> (f, (rid, who))); -let hop2 = hop1 | join(friend, (_, (rid, who), (g, _)) -> (g, (rid, who))); +let hop1 = request | map(((rid, who, _), ()) -> (who, (rid, who))) | join(person, (who, (req, _)) -> (who, req)) + | join(friend, (_, ((rid, who), (f, _))) -> (f, (rid, who))); +let hop2 = hop1 | join(friend, (_, ((rid, who), (g, _))) -> (g, (rid, who))); let circle = (hop1 + hop2) | filter((node, (_, who)) -> (node, who) ne) | map((node, (rid, _)) -> ((rid, node), ())) | distinct; -- The tag's posts by people in the circle: ((rid, post), the tag). -- Tag names are unique and (message, tag) pairs are a set, so each message appears once. let with_it = request | map(((rid, _, name), ()) -> (name, rid)) - | join(by_name, (_, rid, t) -> (t, rid)) - | join(tagged, (t, rid, m) -> (m, (rid, t))) - | join(message, (m, (rid, t), (kind, _, author, _, _, _, _, _, _, _, _, _, _)) -> ((rid, author), (kind, m, t))) + | join(by_name, (_, (rid, t)) -> (t, rid)) + | join(tagged, (t, (rid, m)) -> (m, (rid, t))) + | join(message, (m, ((rid, t), (kind, _, author, _, _, _, _, _, _, _, _, _, _))) -> ((rid, author), (kind, m, t))) | filter((_, (kind, _, _)) -> (kind, 0) eq) - | join(circle, ((rid, _), (_, m, t), ()) -> (m, (rid, t))); + | join(circle, ((rid, _), ((_, m, t), ())) -> (m, (rid, t))); -- Count the other tags on those posts, and rank by count, then name. -let others = with_it | join(tags_of, (_, (rid, t), other) -> ((rid, t, other), ())) +let others = with_it | join(tags_of, (_, ((rid, t), other)) -> ((rid, t, other), ())) | filter(((_, t, other), ()) -> (t, other) ne) | map(((rid, _, other), ()) -> ((rid, other), ())) | count; -- Top k: `reduce` sees a key's values in sort order and keeps the first k (copies capped -- at k), emitting (rank, value). Changes below the top k produce no output. let ranked = others | map(((rid, other), n) -> (other, (rid, n))) - | join(tag, (_, (rid, n), (name, _)) -> (rid, (n neg, name))) + | join(tag, (_, ((rid, n), (name, _))) -> (rid, (n neg, name))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, (neg, other))) -> ((rid, rank), (other, neg neg))); diff --git a/interactive/bench/ldbc/authored/plans/ic07.ddp b/interactive/bench/ldbc/authored/plans/ic07.ddp index 877826ce9..cae69c2c8 100644 --- a/interactive/bench/ldbc/authored/plans/ic07.ddp +++ b/interactive/bench/ldbc/authored/plans/ic07.ddp @@ -10,20 +10,20 @@ let pals = import "graph::friend.pair" : ((int, int), ()); -- ((perso -- Each liker's latest like on one of the person's messages (ties: smaller message id). let latest = request | map(((rid, who), ()) -> (who, rid)) - | join(wrote, (who, rid, m) -> (m, (rid, who))) - | join(likes, (m, (rid, who), (liker, when)) -> ((rid, who, liker), (when neg, m))) + | join(wrote, (who, (rid, m)) -> (m, (rid, who))) + | join(likes, (m, ((rid, who), (liker, when))) -> ((rid, who, liker), (when neg, m))) | min; -- The liked message's text and age, and whether the liker is a friend. let rows = latest | map(((rid, who, liker), (neg, m)) -> (m, (rid, who, liker, neg neg))) - | join(message, (m, (rid, who, liker, when), (_, created, _, _, text, image, _, _, _, _, _, _, _)) -> + | join(message, (m, ((rid, who, liker, when), (_, created, _, _, text, image, _, _, _, _, _, _, _))) -> ((who, liker), (rid, liker, when, m, ((text len, 0) gt, text, image) select, (when, created) sub))); -let known = rows | join(pals, (pair, row, ()) -> (pair, row)); +let known = rows | join(pals, (pair, (row, ())) -> (pair, row)); let marked = (known | map((_, row) -> (row, 0))) + ((rows - known) | map((_, row) -> (row, 1))); -- Top k: `reduce` sees a key's values in sort order and keeps the first k (copies capped -- at k), emitting (rank, value). Changes below the top k produce no output. let ranked = marked | map(((rid, liker, when, m, body, age), fresh) -> (liker, (rid, when, m, body, age, fresh))) - | join(person, (liker, (rid, when, m, body, age, fresh), (first, last, _, _, _, _, _, _, _, _, _)) -> + | join(person, (liker, ((rid, when, m, body, age, fresh), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((when neg, liker), (first, last, m, body, (age, 60000) div, fresh)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((neg, liker), (first, last, m, body, minutes, fresh)))) -> diff --git a/interactive/bench/ldbc/authored/plans/ic08.ddp b/interactive/bench/ldbc/authored/plans/ic08.ddp index d04c3f779..063480c89 100644 --- a/interactive/bench/ldbc/authored/plans/ic08.ddp +++ b/interactive/bench/ldbc/authored/plans/ic08.ddp @@ -11,15 +11,15 @@ let replies = import "graph::message.by_parent" : (int, int); -- (pare -- Top k: `reduce` sees a key's values in sort order and keeps the first k (copies capped -- at k), emitting (rank, value). Changes below the top k produce no output. let top = request | map(((rid, who), ()) -> (who, rid)) - | join(wrote, (_, rid, op) -> (op, rid)) - | join(replies, (_, rid, id) -> (id, rid)) - | join(message, (id, rid, (kind, created, who, _, _, _, _, _, _, _, _, _, _)) -> (rid, (kind, (created neg, id), who))) + | join(wrote, (_, (rid, op)) -> (op, rid)) + | join(replies, (_, (rid, id)) -> (id, rid)) + | join(message, (id, (rid, (kind, created, who, _, _, _, _, _, _, _, _, _, _))) -> (rid, (kind, (created neg, id), who))) | filter((_, (kind, _, _)) -> (kind, 1) eq) | map((rid, (_, order, who)) -> (rid, (order, who))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = top | map((rid, (rank, ((neg, id), who))) -> (id, (rid, rank, neg neg, who))) - | join(message, (id, (rid, rank, created, who), (_, _, _, _, content, _, _, _, _, _, _, _, _)) -> (who, (rid, rank, id, content, created))) - | join(person, (who, (rid, rank, id, content, created), (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, rank), (who, first, last, created, id, content))); + | join(message, (id, ((rid, rank, created, who), (_, _, _, _, content, _, _, _, _, _, _, _, _))) -> (who, (rid, rank, id, content, created))) + | join(person, (who, ((rid, rank, id, content, created), (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, rank), (who, first, last, created, id, content))); output "answer" = answer | arrange; diff --git a/interactive/bench/ldbc/authored/plans/ic09.ddp b/interactive/bench/ldbc/authored/plans/ic09.ddp index 53087ee81..22b5e8c2f 100644 --- a/interactive/bench/ldbc/authored/plans/ic09.ddp +++ b/interactive/bench/ldbc/authored/plans/ic09.ddp @@ -19,19 +19,19 @@ let latest = by_author | reduce((_, vals) -> let positive = (vals map ((_, d) let newest = by_author | map((author, (neg, _)) -> (author, neg)) | min; -- The circle (one or two hops, not the person) and the request's cut-off. -let hop1 = request | map(((rid, who, max), ()) -> (who, (rid, who, max))) | join(person, (who, req, _) -> (who, req)) - | join(friend, (_, req, (f, _)) -> (f, req)); -let hop2 = hop1 | join(friend, (_, req, (g, _)) -> (g, req)); +let hop1 = request | map(((rid, who, max), ()) -> (who, (rid, who, max))) | join(person, (who, (req, _)) -> (who, req)) + | join(friend, (_, (req, (f, _))) -> (f, req)); +let hop2 = hop1 | join(friend, (_, (req, (g, _))) -> (g, req)); let friends = (hop1 + hop2) | filter((node, (_, who, _)) -> (node, who) ne) | map((node, (rid, _, max)) -> ((node, rid, max), ())) | distinct | map(((node, rid, max), ()) -> (node, (rid, max))); -let ahead = friends | join(newest, (f, (rid, max), neg) -> (f, (rid, max, neg neg))); +let ahead = friends | join(newest, (f, ((rid, max), neg)) -> (f, (rid, max, neg neg))); let fast = ahead | filter((_, (_, max, last)) -> (last, max) lt) - | join(latest, (f, (rid, _, _), (neg, id)) -> (rid, ((neg, id), f))); + | join(latest, (f, ((rid, _, _), (neg, id))) -> (rid, ((neg, id), f))); -- A friend on the slow path keeps only their own 20 newest before the date, ranked where -- their messages already are, so the request's final ranking sees at most 20 per friend. let slow = ahead | filter((_, (_, max, last)) -> (last, max) ge) - | join(by_author, (f, (rid, max, _), (neg, id)) -> ((rid, f), ((neg, id), max))) + | join(by_author, (f, ((rid, max, _), (neg, id))) -> ((rid, f), ((neg, id), max))) | filter((_, ((neg, _), max)) -> (neg neg, max) lt) | map((key, (order, _)) -> (key, order)) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten) @@ -40,8 +40,8 @@ let top = (fast + slow) | reduce((_, vals) -> let positive = (vals map ((_, -- Only now fetch the 20 messages' text and their authors' names. let answer = top | map((rid, (rank, ((neg, id), f))) -> (id, (rid, rank, neg neg, f))) - | join(message, (id, (rid, rank, created, f), (_, _, _, _, content, image, _, _, _, _, _, _, _)) -> + | join(message, (id, ((rid, rank, created, f), (_, _, _, _, content, image, _, _, _, _, _, _, _))) -> (f, (rid, rank, id, ((content len, 0) gt, content, image) select, created))) - | join(person, (f, (rid, rank, id, body, created), (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, rank), (f, first, last, id, body, created))); + | join(person, (f, ((rid, rank, id, body, created), (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, rank), (f, first, last, id, body, created))); output "answer" = answer | arrange; diff --git a/interactive/bench/ldbc/authored/plans/ic10.ddp b/interactive/bench/ldbc/authored/plans/ic10.ddp index ce4eae13e..55b0b51ce 100644 --- a/interactive/bench/ldbc/authored/plans/ic10.ddp +++ b/interactive/bench/ldbc/authored/plans/ic10.ddp @@ -14,17 +14,17 @@ let interest = import "graph::interest.by_person" : (int, int); -- (pers -- Least hops to each person within two, keeping those at exactly two. let hop1 = request | map(((rid, who, month), ()) -> (who, (rid, who, month))) - | join(friend, (_, req, (f, _)) -> (f, (req, 1))); -let hop2 = hop1 | join(friend, (_, (req, _), (g, _)) -> (g, (req, 2))); + | join(friend, (_, (req, (f, _))) -> (f, (req, 1))); +let hop2 = hop1 | join(friend, (_, ((req, _), (g, _))) -> (g, (req, 2))); let twohop = (hop1 + hop2) | filter((node, ((_, who, _), _)) -> (node, who) ne) | map((node, ((rid, who, month), d)) -> ((rid, who, month, node), d)) | min | filter((_, d) -> (d, 2) eq); -- Born in the window, with gender and city. let people = person | map((id, (first, last, gender, _, _, _, _, home, bmonth, bday, _)) -> (home, (id, first, last, gender, bmonth, bday))) - | join(place, (_, (id, first, last, gender, bmonth, bday), (town, _, _)) -> (id, (first, last, gender, town, bmonth, bday))); + | join(place, (_, ((id, first, last, gender, bmonth, bday), (town, _, _))) -> (id, (first, last, gender, town, bmonth, bday))); let cands = twohop | map(((rid, who, month, node), _) -> (node, (rid, who, month))) - | join(people, (node, (rid, who, month), (first, last, gender, town, bmonth, bday)) -> + | join(people, (node, ((rid, who, month), (first, last, gender, town, bmonth, bday))) -> ((rid, who, node), (first, last, gender, town, month, bmonth, bday))) | filter((_, (_, _, _, _, month, bmonth, bday)) -> (((bmonth, month) eq, (bday, 21) ge) all, ((bmonth, ((month, 12) eq, 1, (month, 1) add) select) eq, (bday, 22) lt) all) any) @@ -34,20 +34,20 @@ let cands = twohop | map(((rid, who, month, node), _) -> (node, (rid, who, mont let posts = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 0) eq) | map((id, (_, _, who, _, _, _, _, _, _, _, _, _, _)) -> (who, id)); let likes = interest | map((p, t) -> ((p, t), ())); -let wrote = cands | map(((rid, who, node), _) -> (node, (rid, who))) | join(posts, (node, (rid, who), m) -> ((rid, who, node, m), ())); +let wrote = cands | map(((rid, who, node), _) -> (node, (rid, who))) | join(posts, (node, ((rid, who), m)) -> ((rid, who, node, m), ())); let liked = wrote | map(((rid, who, node, m), ()) -> (m, (rid, who, node))) - | join(tags_of, (m, (rid, who, node), t) -> ((who, t), (rid, node, m))) - | join(likes, ((who, _), (rid, node, m), ()) -> ((rid, who, node, m), ())) + | join(tags_of, (m, ((rid, who, node), t)) -> ((who, t), (rid, node, m))) + | join(likes, ((who, _), ((rid, node, m), ())) -> ((rid, who, node, m), ())) | distinct; let points = (liked | map(((rid, who, node, _), ()) -> ((rid, who, node), 1))) + ((wrote - liked) | map(((rid, who, node, _), ()) -> ((rid, who, node), -1))); -- Sums: move each value into the multiplicity (join_function) and let `count` add them up. let summed = points | join_function((key, p) -> ((key, ()), 0, p) enlist) | count; let keys = cands | map((key, _) -> (key, ())); -let score = summed + ((keys - (keys | join(summed, (key, (), _) -> (key, ())))) | map((key, ()) -> (key, 0))); +let score = summed + ((keys - (keys | join(summed, (key, ((), _)) -> (key, ())))) | map((key, ()) -> (key, 0))); -- Top k: `reduce` sees a key's values in sort order and keeps the first k (copies capped -- at k), emitting (rank, value). Changes below the top k produce no output. -let ranked = score | join(cands, ((rid, _, node), s, (first, last, gender, town)) -> (rid, ((s neg, node), (first, last, s, gender, town)))) +let ranked = score | join(cands, ((rid, _, node), (s, (first, last, gender, town))) -> (rid, ((s neg, node), (first, last, s, gender, town)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((_, node), (first, last, s, gender, town)))) -> ((rid, rank), (node, first, last, s, gender, town))); diff --git a/interactive/bench/ldbc/authored/plans/ic11.ddp b/interactive/bench/ldbc/authored/plans/ic11.ddp index 9d9c68c1e..df49d7b81 100644 --- a/interactive/bench/ldbc/authored/plans/ic11.ddp +++ b/interactive/bench/ldbc/authored/plans/ic11.ddp @@ -11,18 +11,18 @@ let place = import "graph::place" : (int, (List(int), List(int), int)); -- The circle: one or two hops from the person, not the person. let hop1 = request | map(((rid, who, country, year), ()) -> (who, (rid, who, country, year))) - | join(person, (who, req, _) -> (who, req)) - | join(friend, (_, req, (f, _)) -> (f, req)); -let hop2 = hop1 | join(friend, (_, req, (g, _)) -> (g, req)); + | join(person, (who, (req, _)) -> (who, req)) + | join(friend, (_, (req, (f, _))) -> (f, req)); +let hop2 = hop1 | join(friend, (_, (req, (g, _))) -> (g, req)); let circle = (hop1 + hop2) | filter((node, (_, who, _, _)) -> (node, who) ne) | map((node, (rid, _, country, year)) -> ((node, rid, country, year), ())) | distinct; -- Their jobs before the year at companies located in the country. let jobs = circle | map(((node, rid, country, year), ()) -> (node, (rid, country, year))) - | join(work, (node, (rid, country, year), (at, since)) -> (at, (rid, node, country, year, since))) + | join(work, (node, ((rid, country, year), (at, since))) -> (at, (rid, node, country, year, since))) | filter((_, (_, _, _, year, since)) -> (since, year) lt) - | join(org, (_, (rid, node, country, _, since), (company, _, located)) -> (located, ((rid, since, node), (company, country)))) - | join(place, (_, (key, (company, country)), (there, _, _)) -> (key, (company, country, there))) + | join(org, (_, ((rid, node, country, _, since), (company, _, located))) -> (located, ((rid, since, node), (company, country)))) + | join(place, (_, ((key, (company, country)), (there, _, _))) -> (key, (company, country, there))) | filter((_, (_, country, there)) -> (country, there) eq) | map((key, (company, _, _)) -> (key, company)); @@ -31,7 +31,7 @@ let named = jobs | collect | flatmap((k, entries) -> (k, ((0, entries len) rang -- Top k: `reduce` sees a key's values in sort order and keeps the first k (copies capped -- at k), emitting (rank, value). Changes below the top k produce no output. let ranked = named | map(((rid, since, node), (pos, company)) -> (node, (rid, since, pos, company))) - | join(person, (node, (rid, since, pos, company), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, ((since, node, pos neg), (first, last, company)))) + | join(person, (node, ((rid, since, pos, company), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((since, node, pos neg), (first, last, company)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((since, node, _), (first, last, company)))) -> ((rid, rank), (node, first, last, company, since))); diff --git a/interactive/bench/ldbc/authored/plans/ic12.ddp b/interactive/bench/ldbc/authored/plans/ic12.ddp index 262e39246..e5f9f95fb 100644 --- a/interactive/bench/ldbc/authored/plans/ic12.ddp +++ b/interactive/bench/ldbc/authored/plans/ic12.ddp @@ -17,7 +17,7 @@ let mtag = import "graph::mtag.by_tag" : (int, int); -- (tag, let selves = tagclass | map((id, _) -> ((id, id), ())); below: { let step = below | map(((low, high), ()) -> (high, low)) - | join(tagclass, (_, low, (_, parent)) -> (low, parent)) + | join(tagclass, (_, (low, (_, parent))) -> (low, parent)) | filter((_, parent) -> (parent, 0) ge) | map((low, higher) -> ((low, higher), ())); set below = selves + step | distinct; @@ -25,12 +25,12 @@ below: { -- Tagged posts in the requested class: ((rid, post), tag name). let intag = request | map(((rid, _, name), ()) -> (name, rid)) - | join(classes, (_, rid, high) -> (high, rid)) - | join(below::below | map(((low, high), ()) -> (high, low)), (_, rid, low) -> (low, rid)) - | join(by_class, (_, rid, t) -> (t, rid)) - | join(tag, (t, rid, (name, _)) -> (t, (rid, name))) - | join(mtag, (_, (rid, name), m) -> (m, (rid, name))) - | join(message, (m, (rid, name), (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> ((rid, m, name), kind)) + | join(classes, (_, (rid, high)) -> (high, rid)) + | join(below::below | map(((low, high), ()) -> (high, low)), (_, (rid, low)) -> (low, rid)) + | join(by_class, (_, (rid, t)) -> (t, rid)) + | join(tag, (t, (rid, (name, _))) -> (t, (rid, name))) + | join(mtag, (_, ((rid, name), m)) -> (m, (rid, name))) + | join(message, (m, ((rid, name), (kind, _, _, _, _, _, _, _, _, _, _, _, _))) -> ((rid, m, name), kind)) | filter((_, kind) -> (kind, 0) eq) | map((key, _) -> (key, ())); -- each tag reached once; names unique @@ -38,9 +38,9 @@ let intag = request | map(((rid, _, name), ()) -> (name, rid)) let reply = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 1) eq) | map((id, (_, _, who, _, _, _, _, _, _, parent, _, _, _)) -> (who, (id, parent))); let hits = request | map(((rid, who, _), ()) -> (who, rid)) - | join(friend, (_, rid, (f, _)) -> (f, rid)) - | join(reply, (f, rid, (id, parent)) -> ((rid, parent), (f, id))) - | join(intag | map(((rid, m, name), ()) -> ((rid, m), name)), ((rid, _), (f, id), name) -> ((rid, f), (id, name))); + | join(friend, (_, (rid, (f, _))) -> (f, rid)) + | join(reply, (f, (rid, (id, parent))) -> ((rid, parent), (f, id))) + | join(intag | map(((rid, m, name), ()) -> ((rid, m), name)), ((rid, _), ((f, id), name)) -> ((rid, f), (id, name))); -- Per friend: distinct replies counted, distinct tag names listed. let counts = hits | map(((rid, f), (id, _)) -> ((rid, f, id), ())) | distinct @@ -49,8 +49,8 @@ let tagset = hits | map(((rid, f), (_, name)) -> ((rid, f, name), ())) | distin | map(((rid, f, name), ()) -> ((rid, f), name)) | collect; -- Top k: `reduce` sees a key's values in sort order and keeps the first k (copies capped -- at k), emitting (rank, value). Changes below the top k produce no output. -let ranked = counts | join(tagset, ((rid, f), n, list) -> (f, (rid, n, list))) - | join(person, (f, (rid, n, list), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, ((n neg, f), (first, last, list)))) +let ranked = counts | join(tagset, ((rid, f), (n, list)) -> (f, (rid, n, list))) + | join(person, (f, ((rid, n, list), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((n neg, f), (first, last, list)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((neg, f), (first, last, list)))) -> ((rid, rank), (f, first, last, list, neg neg))); diff --git a/interactive/bench/ldbc/authored/plans/ic13.ddp b/interactive/bench/ldbc/authored/plans/ic13.ddp index 2374257a9..584279a78 100644 --- a/interactive/bench/ldbc/authored/plans/ic13.ddp +++ b/interactive/bench/ldbc/authored/plans/ic13.ddp @@ -13,17 +13,17 @@ search: { -- The best meeting so far: a node both sides reached, with the least total hops. let side0 = search | filter(((_, side, _, _), ()) -> (side, 0) eq) | map(((rid, _, node, d), ()) -> ((rid, node), d)); let side1 = search | filter(((_, side, _, _), ()) -> (side, 1) eq) | map(((rid, _, node, d), ()) -> ((rid, node), d)); - let best = side0 | join(side1, ((rid, _), d0, d1) -> (rid, (d0, d1) add)) | min; + let best = side0 | join(side1, ((rid, _), (d0, d1)) -> (rid, (d0, d1) add)) | min; -- A row keeps growing until its request has met, then only while 2·hops < best. let by_rid = search | map(((rid, side, node, d), ()) -> (rid, (side, node, d))); let met = best | map((rid, _) -> (rid, ())); - let open = by_rid - (by_rid | join(met, (rid, row, ()) -> (rid, row))); - let near = by_rid | join(best, (rid, (side, node, d), b) -> (rid, (side, node, d, b))) + let open = by_rid - (by_rid | join(met, (rid, (row, ())) -> (rid, row))); + let near = by_rid | join(best, (rid, ((side, node, d), b)) -> (rid, (side, node, d, b))) | filter((_, (_, _, d, b)) -> ((d, d) add, b) lt) | map((rid, (side, node, d, _)) -> (rid, (side, node, d))); let grow = (open + near) | map((rid, (side, node, d)) -> (node, (rid, side, d))) - | join(friend, (_, (rid, side, d), (next, _)) -> ((rid, side, next, (d, 1) add), ())); + | join(friend, (_, ((rid, side, d), (next, _))) -> ((rid, side, next, (d, 1) add), ())); set search = (starts + search + grow) | map(((rid, side, node, d), ()) -> ((rid, side, node), d)) | min | map(((rid, side, node), d) -> ((rid, side, node, d), ())); @@ -31,11 +31,11 @@ search: { let side0 = search::search | filter(((_, side, _, _), ()) -> (side, 0) eq) | map(((rid, _, node, d), ()) -> ((rid, node), d)); let side1 = search::search | filter(((_, side, _, _), ()) -> (side, 1) eq) | map(((rid, _, node, d), ()) -> ((rid, node), d)); -let best = side0 | join(side1, ((rid, _), d0, d1) -> (rid, (d0, d1) add)) | min; +let best = side0 | join(side1, ((rid, _), (d0, d1)) -> (rid, (d0, d1) add)) | min; -- Requests whose two sides never met get -1. let asked = request | map(((rid, _, _), ()) -> (rid, ())); -let never = asked - (asked | join(best, (rid, (), _) -> (rid, ()))); +let never = asked - (asked | join(best, (rid, ((), _)) -> (rid, ()))); let answer = (best | map((rid, d) -> ((rid, 0), (d)))) + (never | map((rid, ()) -> ((rid, 0), (-1)))); output "answer" = answer | arrange; diff --git a/interactive/bench/ldbc/authored/plans/ic14.ddp b/interactive/bench/ldbc/authored/plans/ic14.ddp index 4cfd2599a..c779db57a 100644 --- a/interactive/bench/ldbc/authored/plans/ic14.ddp +++ b/interactive/bench/ldbc/authored/plans/ic14.ddp @@ -12,11 +12,11 @@ let pals = import "graph::friend.pair" : ((int, int), ()); -- ((perso -- up in the friendships keyed by pair. let answered = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 1) eq) | map((_, (_, _, who, _, _, _, _, _, _, parent, _, _, _)) -> (parent, who)) - | join(message, (_, who, (_, _, to, _, _, _, _, _, _, _, _, _, _)) -> (who, to)) + | join(message, (_, (who, (_, _, to, _, _, _, _, _, _, _, _, _, _))) -> (who, to)) | filter((who, to) -> (who, to) ne); let talks = (answered | map((a, b) -> ((a, b), ()))) + (answered | map((a, b) -> ((b, a), ()))); let weighted = talks | count - | join(pals, (pair, n, ()) -> (pair, n)) + | join(pals, (pair, (n, ())) -> (pair, n)) | map(((a, b), n) -> (a, (b, (((0, n), (1, 40) range) fold (((acc, n), k) -> ((acc, ((((k, (k, 1) sub) mul, 1) add, n) gt, 1, 0) select) add, n)) .0, 1) add))); @@ -24,36 +24,36 @@ let weighted = talks | count let starts = request | map(((rid, from, _), ()) -> ((rid, from), 0)); dist: { let grow = dist | map(((rid, node, cost), ()) -> (node, (rid, cost))) - | join(weighted, (_, (rid, cost), (next, w)) -> ((rid, next), (cost, w) add)); + | join(weighted, (_, ((rid, cost), (next, w))) -> ((rid, next), (cost, w) add)); set dist = (starts + grow) | min | map(((rid, node), cost) -> ((rid, node, cost), ())); } let best = dist::dist | map(((rid, node, cost), ()) -> ((rid, node), cost)); -- Every predecessor that achieves a node's least cost (all of them, so ties survive). let pred = dist::dist | map(((rid, u, cost), ()) -> (u, (rid, cost))) - | join(weighted, (u, (rid, cost), (v, w)) -> ((rid, v), (u, (cost, w) add))) - | join(best, ((rid, v), (u, via), least) -> ((rid, v), (u, via, least))) + | join(weighted, (u, ((rid, cost), (v, w))) -> ((rid, v), (u, (cost, w) add))) + | join(best, ((rid, v), ((u, via), least)) -> ((rid, v), (u, via, least))) | filter((_, (_, via, least)) -> (via, least) eq) | map((key, (u, _, _)) -> (key, u)); -- The nodes on some cheapest path to `to`: walk the predecessors back from it. -let targets = request | map(((rid, _, to), ()) -> ((rid, to), ())) | join(best, (key, (), _) -> (key, ())); +let targets = request | map(((rid, _, to), ()) -> ((rid, to), ())) | join(best, (key, ((), _)) -> (key, ())); back: { - let up = back | join(pred, ((rid, _), (), u) -> ((rid, u), ())); + let up = back | join(pred, ((rid, _), ((), u)) -> ((rid, u), ())); set back = (targets + up) | distinct; } -- From `from`, the smallest next node that is still on a cheapest path: the -- lexicographically smallest of the cheapest paths. -let next = pred | join(back::back, ((rid, v), u, ()) -> ((rid, u), v)) | min; -let first = request | map(((rid, from, _), ()) -> ((rid, from), ())) | join(back::back, ((rid, from), (), ()) -> ((rid, from, from enlist), ())); +let next = pred | join(back::back, ((rid, v), (u, ())) -> ((rid, u), v)) | min; +let first = request | map(((rid, from, _), ()) -> ((rid, from), ())) | join(back::back, ((rid, from), ((), ())) -> ((rid, from, from enlist), ())); walk: { let step = walk | map(((rid, node, path), ()) -> ((rid, node), path)) - | join(next, ((rid, _), path, v) -> ((rid, v, (path, v enlist) append), ())); + | join(next, ((rid, _), (path, v)) -> ((rid, v, (path, v enlist) append), ())); set walk = first + step; } let answer = walk::walk | map(((rid, node, path), ()) -> ((rid, node), path)) - | join(targets, ((rid, node), path, ()) -> ((rid, node), path)) - | join(best, ((rid, _), path, cost) -> ((rid, 0), (path, cost))); + | join(targets, ((rid, node), (path, ())) -> ((rid, node), path)) + | join(best, ((rid, _), (path, cost)) -> ((rid, 0), (path, cost))); output "answer" = answer | arrange; diff --git a/interactive/bench/ldbc/authored/plans/is01.ddp b/interactive/bench/ldbc/authored/plans/is01.ddp index 6167e2b8d..7177805bb 100644 --- a/interactive/bench/ldbc/authored/plans/is01.ddp +++ b/interactive/bench/ldbc/authored/plans/is01.ddp @@ -4,7 +4,7 @@ let request = input 0 : ((int, int), ()); -- (rid, person) let person = import "graph::person" : (int, (List(int), List(int), List(int), int, int, List(int), List(int), int, int, int, int)); let answer = request | map(((rid, who), ()) -> (who, rid)) - | join(person, (_, rid, (first, last, gender, birthday, created, ip, browser, city, _, _, _)) -> + | join(person, (_, (rid, (first, last, gender, birthday, created, ip, browser, city, _, _, _))) -> ((rid, 0), (first, last, birthday, ip, browser, city, gender, created))); output "answer" = answer | arrange; diff --git a/interactive/bench/ldbc/authored/plans/is02.ddp b/interactive/bench/ldbc/authored/plans/is02.ddp index 74a67cadf..743943fd5 100644 --- a/interactive/bench/ldbc/authored/plans/is02.ddp +++ b/interactive/bench/ldbc/authored/plans/is02.ddp @@ -13,20 +13,20 @@ let root_of = import "graph::message.root" : (int, int); -- The person's messages, newest first (ties by larger id), keeping only ids and times to rank. let mine = request | map(((rid, who), ()) -> (who, rid)) - | join(by_creator, (_, rid, id) -> (id, rid)) - | join(message, (id, rid, (_, created, _, _, _, _, _, _, _, _, _, _, _)) -> (id, (rid, created))) - | join(root_of, (id, (rid, created), root) -> (rid, ((created neg, id neg), (id, created, root)))); + | join(by_creator, (_, (rid, id)) -> (id, rid)) + | join(message, (id, (rid, (_, created, _, _, _, _, _, _, _, _, _, _, _))) -> (id, (rid, created))) + | join(root_of, (id, ((rid, created), root)) -> (rid, ((created neg, id neg), (id, created, root)))); -- Top k: `reduce` sees a key's values in sort order and keeps the first k (copies capped -- at k), emitting (rank, value). Changes below the top k produce no output. let top = mine | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); -- Only now fetch the ten messages' text, their root posts' authors, and those authors' names. let answer = top | map((rid, (rank, (_, (id, created, root)))) -> (id, (rid, rank, created, root))) - | join(message, (id, (rid, rank, created, root), (_, _, _, _, content, image, _, _, _, _, _, _, _)) -> + | join(message, (id, ((rid, rank, created, root), (_, _, _, _, content, image, _, _, _, _, _, _, _))) -> (root, (rid, rank, id, ((content len, 0) gt, content, image) select, created))) - | join(message, (root, (rid, rank, id, body, created), (_, _, op, _, _, _, _, _, _, _, _, _, _)) -> + | join(message, (root, ((rid, rank, id, body, created), (_, _, op, _, _, _, _, _, _, _, _, _, _))) -> (op, (rid, rank, id, body, created, root))) - | join(person, (op, (rid, rank, id, body, created, root), (first, last, _, _, _, _, _, _, _, _, _)) -> + | join(person, (op, ((rid, rank, id, body, created, root), (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, rank), (id, body, created, root, op, first, last))); output "answer" = answer | arrange; diff --git a/interactive/bench/ldbc/authored/plans/is03.ddp b/interactive/bench/ldbc/authored/plans/is03.ddp index a4fb749f4..8da395636 100644 --- a/interactive/bench/ldbc/authored/plans/is03.ddp +++ b/interactive/bench/ldbc/authored/plans/is03.ddp @@ -5,8 +5,8 @@ let person = import "graph::person" : (int, (List(int), List(int), List(int), let friend = import "graph::friend" : (int, (int, int)); -- (person, (friend, since)) let ranked = request | map(((rid, who), ()) -> (who, rid)) - | join(friend, (_, rid, (f, since)) -> (f, (rid, since))) - | join(person, (f, (rid, since), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, ((since neg, f), (first, last)))) + | join(friend, (_, (rid, (f, since))) -> (f, (rid, since))) + | join(person, (f, ((rid, since), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((since neg, f), (first, last)))) | collect | flatmap((k, entries) -> (k, ((0, entries len) range, entries) zip) cap_list); let answer = ranked | map((rid, (rank, ((neg, f), (first, last)))) -> ((rid, rank), (f, first, last, neg neg))); diff --git a/interactive/bench/ldbc/authored/plans/is04.ddp b/interactive/bench/ldbc/authored/plans/is04.ddp index 3348b592f..62123ee3b 100644 --- a/interactive/bench/ldbc/authored/plans/is04.ddp +++ b/interactive/bench/ldbc/authored/plans/is04.ddp @@ -4,7 +4,7 @@ let request = input 0 : ((int, int), ()); -- (rid, message) let message = import "graph::message" : (int, (int, int, int, int, List(int), List(int), int, List(int), int, int, int, int, int)); let answer = request | map(((rid, m), ()) -> (m, rid)) - | join(message, (_, rid, (_, created, _, _, text, image, _, _, _, _, _, _, _)) -> + | join(message, (_, (rid, (_, created, _, _, text, image, _, _, _, _, _, _, _))) -> ((rid, 0), (created, ((text len, 0) gt, text, image) select))); output "answer" = answer | arrange; diff --git a/interactive/bench/ldbc/authored/plans/is05.ddp b/interactive/bench/ldbc/authored/plans/is05.ddp index b0e7208dd..8eaade3ed 100644 --- a/interactive/bench/ldbc/authored/plans/is05.ddp +++ b/interactive/bench/ldbc/authored/plans/is05.ddp @@ -5,7 +5,7 @@ let person = import "graph::person" : (int, (List(int), List(int), List(int), let message = import "graph::message" : (int, (int, int, int, int, List(int), List(int), int, List(int), int, int, int, int, int)); let answer = request | map(((rid, m), ()) -> (m, rid)) - | join(message, (_, rid, (_, _, who, _, _, _, _, _, _, _, _, _, _)) -> (who, rid)) - | join(person, (who, rid, (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, 0), (who, first, last))); + | join(message, (_, (rid, (_, _, who, _, _, _, _, _, _, _, _, _, _))) -> (who, rid)) + | join(person, (who, (rid, (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, 0), (who, first, last))); output "answer" = answer | arrange; diff --git a/interactive/bench/ldbc/authored/plans/is06.ddp b/interactive/bench/ldbc/authored/plans/is06.ddp index a1aa362b2..f0faf5a44 100644 --- a/interactive/bench/ldbc/authored/plans/is06.ddp +++ b/interactive/bench/ldbc/authored/plans/is06.ddp @@ -8,10 +8,10 @@ let forum = import "graph::forum" : (int, (List(int), int, int)); let root_of = import "graph::message.root" : (int, int); let answer = request | map(((rid, m), ()) -> (m, rid)) - | join(root_of, (_, rid, root) -> (root, rid)) - | join(message, (_, rid, (_, _, _, _, _, _, _, _, f, _, _, _, _)) -> (f, rid)) - | join(forum, (f, rid, (title, _, moderator)) -> (moderator, (rid, f, title))) - | join(person, (moderator, (rid, f, title), (first, last, _, _, _, _, _, _, _, _, _)) -> + | join(root_of, (_, (rid, root)) -> (root, rid)) + | join(message, (_, (rid, (_, _, _, _, _, _, _, _, f, _, _, _, _))) -> (f, rid)) + | join(forum, (f, (rid, (title, _, moderator))) -> (moderator, (rid, f, title))) + | join(person, (moderator, ((rid, f, title), (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, 0), (f, title, moderator, first, last))); output "answer" = answer | arrange; diff --git a/interactive/bench/ldbc/authored/plans/is07.ddp b/interactive/bench/ldbc/authored/plans/is07.ddp index 0e88a18c8..ef2260627 100644 --- a/interactive/bench/ldbc/authored/plans/is07.ddp +++ b/interactive/bench/ldbc/authored/plans/is07.ddp @@ -9,21 +9,21 @@ let friend = import "graph::friend" : (int, (int, int)); -- (person, (frie -- The request's replies: ((replier, author of the message), (rid, comment, content, created)). let reply = request | map(((rid, m), ()) -> (m, rid)) - | join(message, (m, rid, (_, _, op, _, _, _, _, _, _, _, _, _, _)) -> (m, (rid, op))) - | join(by_parent, (_, (rid, op), id) -> (id, (rid, op))) - | join(message, (id, (rid, op), (kind, created, who, _, text, _, _, _, _, _, _, _, _)) -> + | join(message, (m, (rid, (_, _, op, _, _, _, _, _, _, _, _, _, _))) -> (m, (rid, op))) + | join(by_parent, (_, ((rid, op), id)) -> (id, (rid, op))) + | join(message, (id, ((rid, op), (kind, created, who, _, text, _, _, _, _, _, _, _, _))) -> (kind, ((who, op), (rid, id, text, created)))) | filter((kind, _) -> (kind, 1) eq) | map((_, row) -> row); -- Known: the replier and the author are friends (never true of oneself). let known = reply | map(((who, op), row) -> (who, (op, row))) - | join(friend, (who, (op, row), (f, _)) -> ((f, op) eq, ((who, op), row))) + | join(friend, (who, ((op, row), (f, _))) -> ((f, op) eq, ((who, op), row))) | filter((same, _) -> same) | map((_, (pair, row)) -> ((pair, row), ())) | distinct | map(((pair, row), ()) -> (pair, row)); let marked = (known | map(((who, _), row) -> (who, (row, 1)))) + ((reply - known) | map(((who, _), row) -> (who, (row, 0)))); -let ranked = marked | join(person, (who, ((rid, id, text, created), k), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, ((created neg, who), (id, text, first, last, k)))) +let ranked = marked | join(person, (who, (((rid, id, text, created), k), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((created neg, who), (id, text, first, last, k)))) | collect | flatmap((k, entries) -> (k, ((0, entries len) range, entries) zip) cap_list); let answer = ranked | map((rid, (rank, ((neg, who), (id, text, first, last, k)))) -> ((rid, rank), (id, text, neg neg, who, first, last, k))); diff --git a/interactive/bench/ldbc/bi11.ddp b/interactive/bench/ldbc/bi11.ddp index 856d9325e..aa3966f53 100644 --- a/interactive/bench/ldbc/bi11.ddp +++ b/interactive/bench/ldbc/bi11.ddp @@ -4,15 +4,15 @@ let request = input 0 : ((int, List(int), int, int), ()); let resident = import "graph::resident" : (List(int), int); let edge = import "graph::edge" : (int, (int, int)); let people = request | map(((rid, country, start, end), ()) -> (country, (rid, start, end))) - | join(resident, (_, request, person) -> (person, request)); -let candidates = people | join(edge, (p, (rid, start, end), (q, created)) -> ((rid, p, q, created, start, end), ())) + | join(resident, (_, (request, person)) -> (person, request)); +let candidates = people | join(edge, (p, ((rid, start, end), (q, created))) -> ((rid, p, q, created, start, end), ())) | filter(((_, p, q, created, start, end), ()) -> ((p, q) lt, (created, start) ge, (created, end) le) all); let membership = people | map((p, (rid, _, _)) -> ((rid, p), ())) | distinct; let pairs = candidates | map(((rid, p, q, _, _, _), ()) -> ((rid, q), p)) - | join(membership, ((rid, q), p, ()) -> ((rid, p, q), ())) | distinct; + | join(membership, ((rid, q), (p, ())) -> ((rid, p, q), ())) | distinct; let paths = pairs | map(((rid, p, q), ()) -> ((rid, q), p)) - | join(pairs | map(((rid, p, q), ()) -> ((rid, p), q)), ((rid, x), a, b) -> ((rid, a, b), x)); -let counts = paths | join(pairs, ((rid, _, _), _, _) -> (rid, ())) | count; + | join(pairs | map(((rid, p, q), ()) -> ((rid, p), q)), ((rid, x), (a, b)) -> ((rid, a, b), x)); +let counts = paths | join(pairs, ((rid, _, _), (_, _)) -> (rid, ())) | count; let zero = request | map(((rid, _, _, _), ()) -> (rid, 0)); -let missing = zero - (zero | join(counts, (rid, _, _) -> (rid, 0))); +let missing = zero - (zero | join(counts, (rid, (_, _)) -> (rid, 0))); output "answer" = (counts + missing) | map((rid, n) -> ((rid, 0), (n))); diff --git a/interactive/bench/ldbc/bi5.ddp b/interactive/bench/ldbc/bi5.ddp index 52b73b7ef..5603bd17c 100644 --- a/interactive/bench/ldbc/bi5.ddp +++ b/interactive/bench/ldbc/bi5.ddp @@ -5,16 +5,16 @@ let tagged = import "graph::tagged" : (int, List(int)); let message = import "graph::message" : (int, (int, int, int)); let likes = import "graph::likes" : (int, int); let selected = request | map(((rid, name), ()) -> (name, rid)) - | join(tagged | map((m, name) -> (name, m)), (_, rid, m) -> (m, rid)) - | join(message, (m, rid, (_, creator, _)) -> ((rid, m, creator), ())) | distinct; + | join(tagged | map((m, name) -> (name, m)), (_, (rid, m)) -> (m, rid)) + | join(message, (m, (rid, (_, creator, _))) -> ((rid, m, creator), ())) | distinct; -- Contributions are (messages, replies, likes). Collect+fold expresses a -- sum without introducing a new reducer or optimizer in this benchmark. let own = selected | map(((rid, _, creator), ()) -> ((rid, creator), (1, 0, 0))); let liked = selected | map(((rid, m, creator), ()) -> (m, (rid, creator))) - | join(likes, (_, (rid, creator), _) -> ((rid, creator), (0, 0, 1))); + | join(likes, (_, ((rid, creator), _)) -> ((rid, creator), (0, 0, 1))); let comments = message | filter((_, (kind, _, _)) -> (kind, 1) eq) | map((id, (_, _, parent)) -> (parent, id)); let replied = selected | map(((rid, m, creator), ()) -> (m, (rid, creator))) - | join(comments, (_, (rid, creator), _) -> ((rid, creator), (0, 1, 0))); + | join(comments, (_, ((rid, creator), _)) -> ((rid, creator), (0, 1, 0))); let totals = (own + liked + replied) | collect | map((k, vs) -> (k, ((0, 0, 0), vs) fold (((a0, a1, a2), (x0, x1, x2)) -> ((a0, x0) add, (a1, x1) add, (a2, x2) add)))); let scores = totals | map(((rid, creator), t) -> let (m, r, l) = t in (rid, (((m, (2, r) mul) add, (10, l) mul) add neg, creator, t))); diff --git a/interactive/bench/ldbc/graph.ddp b/interactive/bench/ldbc/graph.ddp index 2b3e47b86..4ef637ca2 100644 --- a/interactive/bench/ldbc/graph.ddp +++ b/interactive/bench/ldbc/graph.ddp @@ -20,9 +20,9 @@ output "message" = message | map(((id, kind, creator, parent), ()) -> (id, (kind output "likes" = likes | map(((person, message), ()) -> (message, person)) | arrange; let tags = tag | map(((id, name), ()) -> (id, name)); output "tagged" = message_tag | map(((message, tag), ()) -> (tag, message)) - | join(tags, (_, message, name) -> (message, name)) | arrange; + | join(tags, (_, (message, name)) -> (message, name)) | arrange; let places = place | map(((id, name, parent), ()) -> (id, (name, parent))); -- City -> country is one parent step in SNB. output "resident" = person | map(((id, _, _, city), ()) -> (city, id)) - | join(places, (_, person, (_, parent)) -> (parent, person)) - | join(places, (_, person, (name, _)) -> (name, person)) | arrange; + | join(places, (_, (person, (_, parent))) -> (parent, person)) + | join(places, (_, (person, (name, _))) -> (name, person)) | arrange; diff --git a/interactive/bench/ldbc/ic6.ddp b/interactive/bench/ldbc/ic6.ddp index 6d2ad7cb8..691e8c38d 100644 --- a/interactive/bench/ldbc/ic6.ddp +++ b/interactive/bench/ldbc/ic6.ddp @@ -6,17 +6,17 @@ let message = import "graph::message" : (int, (int, int, int)); let tagged = import "graph::tagged" : (int, List(int)); -- Reachable rows: (request_id, origin, author, tag_name), deduplicated across paths. let one = request | map(((rid, pid, tag), ()) -> (pid, (rid, tag))) - | join(edge, (pid, (rid, tag), (q, _)) -> ((rid, pid, q, tag), ())); + | join(edge, (pid, ((rid, tag), (q, _))) -> ((rid, pid, q, tag), ())); let two = one | map(((rid, origin, author, tag), ()) -> (author, (rid, origin, tag))) - | join(edge, (_, (rid, origin, tag), (q, _)) -> ((rid, origin, q, tag), ())); + | join(edge, (_, ((rid, origin, tag), (q, _))) -> ((rid, origin, q, tag), ())); let authors = (one + two) | filter(((_, origin, author, _), ()) -> (origin, author) ne) | distinct; let posts = message | filter((_, (kind, _, _)) -> (kind, 0) eq) | map((id, (_, creator, _)) -> (creator, id)); let candidates = authors | map(((rid, _, author, tag), ()) -> (author, (rid, tag))) - | join(posts, (_, req, m) -> (m, req)); -let selected = candidates | join(tagged, (m, (rid, tag), name) -> ((rid, m, tag, name), ())) + | join(posts, (_, (req, m)) -> (m, req)); +let selected = candidates | join(tagged, (m, ((rid, tag), name)) -> ((rid, m, tag, name), ())) | filter(((_, _, tag, name), ()) -> (tag, name) eq) | map(((rid, m, tag, _), ()) -> ((rid, m, tag), ())) | distinct; let counts = selected | map(((rid, m, tag), ()) -> (m, (rid, tag))) - | join(tagged, (_, (rid, tag), name) -> ((rid, name, tag), ())) + | join(tagged, (_, ((rid, tag), name)) -> ((rid, name, tag), ())) | filter(((_, name, tag), ()) -> (name, tag) ne) | map(((rid, name, _), ()) -> ((rid, name), ())) | count; output "answer" = counts | map(((rid, name), n) -> (rid, (n neg, name))) | collect | flatmap((k, vs) -> (k, ((0, vs len) range, vs) zip) cap_list) | filter((_, (pos, _)) -> (pos, 10) lt) diff --git a/interactive/bench/ldbc/interactive_v2/graph.py b/interactive/bench/ldbc/interactive_v2/graph.py index 90069e744..a077a865f 100644 --- a/interactive/bench/ldbc/interactive_v2/graph.py +++ b/interactive/bench/ldbc/interactive_v2/graph.py @@ -49,13 +49,13 @@ def killed(name, field, dead): """Rows of `_in` whose `field` is in the unit-valued collection `dead`.""" b = binders(SCHEMA[name]) key = b[list(SCHEMA[name]).index(field)] - return f"({name}_in | map({row(name)} -> ({key}, {tup(name)})) | join({dead}, (_, r, ()) -> (r, ())))" + return f"({name}_in | map({row(name)} -> ({key}, {tup(name)})) | join({dead}, (_, (r, ())) -> (r, ())))" def killed_pair(name, dead): """Rows of `_in` whose first two fields are an explicit delete target in `dead`.""" b = binders(SCHEMA[name]) - return f"({name}_in | map({row(name)} -> (({b[0]}, {b[1]}), {tup(name)})) | join({dead}, (_, r, ()) -> (r, ())))" + return f"({name}_in | map({row(name)} -> (({b[0]}, {b[1]}), {tup(name)})) | join({dead}, (_, (r, ())) -> (r, ())))" def prefix(text, n): @@ -74,20 +74,20 @@ def cascade_program(export_live=False): -- People deleted, and the forums they moderated: Walls and Albums are deleted, Groups lose their moderator. let dead_p = del_person | map(((id, _), ()) -> (id, ())) | distinct; let owned = forum_in | map(((id, title, created, moderator), ()) -> (moderator, (id, title, created))) - | join(dead_p, (moderator, (id, title, created), ()) -> (id, (title, created, moderator))); + | join(dead_p, (moderator, ((id, title, created), ())) -> (id, (title, created, moderator))); let dead_f = ((del_forum | map(((id, _), ()) -> (id, ()))) + (owned | filter((_, (title, _, _)) -> {walls}) | map((id, _) -> (id, ())))) | distinct; let groups = owned | filter((_, (title, _, _)) -> {group}); -let kept = groups - (groups | join(dead_f, (id, v, ()) -> (id, v))); +let kept = groups - (groups | join(dead_f, (id, (v, ())) -> (id, v))); -- Messages deleted: explicit targets, those of deleted people, the posts of deleted forums, -- and every reply below any of them. let seeds = (del_message | map(((id, _), ()) -> (id, ()))) - + (message_in | map({msg} -> ({m[3]}, {m[0]})) | join(dead_p, (_, id, ()) -> (id, ()))) - + (message_in | filter({msg} -> ({m[1]}, 0) eq) | map({msg} -> ({m[9]}, {m[0]})) | join(dead_f, (_, id, ()) -> (id, ()))); + + (message_in | map({msg} -> ({m[3]}, {m[0]})) | join(dead_p, (_, (id, ())) -> (id, ()))) + + (message_in | filter({msg} -> ({m[1]}, 0) eq) | map({msg} -> ({m[9]}, {m[0]})) | join(dead_f, (_, (id, ())) -> (id, ()))); let children = message_in | filter({msg} -> ({m[1]}, 1) eq) | map({msg} -> ({m[10]}, {m[0]})); dead_m: {{ - let kids = dead_m | join(children, (_, (), child) -> (child, ())); + let kids = dead_m | join(children, (_, ((), child)) -> (child, ())); set dead_m = (seeds + kids) | distinct; }} let dead_msg = dead_m::dead_m; diff --git a/interactive/bench/ldbc/interactive_v2/oneoff/ic01.oneoff.ddp b/interactive/bench/ldbc/interactive_v2/oneoff/ic01.oneoff.ddp index c545d4ebe..386ae2c92 100644 --- a/interactive/bench/ldbc/interactive_v2/oneoff/ic01.oneoff.ddp +++ b/interactive/bench/ldbc/interactive_v2/oneoff/ic01.oneoff.ddp @@ -8,35 +8,35 @@ let work = import "graph::work.by_person" : (int, (int, int)); let email = import "graph::email.by_person" : (int, List(int)); let language = import "graph::language.by_person" : (int, List(int)); let hop1 = request | map(((rid, who, name), ()) -> (who, (rid, who, name))) - | join(friend, (_, req, (f, _)) -> (f, (req, 1))); -let hop2 = hop1 | join(friend, (_, (req, _), (g, _)) -> (g, (req, 2))); -let hop3 = hop2 | join(friend, (_, (req, _), (h, _)) -> (h, (req, 3))); + | join(friend, (_, (req, (f, _))) -> (f, (req, 1))); +let hop2 = hop1 | join(friend, (_, ((req, _), (g, _))) -> (g, (req, 2))); +let hop3 = hop2 | join(friend, (_, ((req, _), (h, _))) -> (h, (req, 3))); let circle = (hop1 + hop2 + hop3) | filter((node, ((_, who, _), _)) -> (node, who) ne) | map((node, ((rid, _, name), d)) -> ((rid, node, name), d)) | min; let found = circle | map(((rid, node, name), d) -> (node, (rid, name, d))) - | join(person, (node, (rid, name, d), (first, last, gender, birthday, created, ip, browser, home, _, _, _)) -> + | join(person, (node, ((rid, name, d), (first, last, gender, birthday, created, ip, browser, home, _, _, _))) -> ((first, name) eq, (home, (rid, node, d, last, gender, birthday, created, ip, browser)))) | filter((named, _) -> named) | map((_, row) -> row) - | join(place, (_, (rid, node, d, last, gender, birthday, created, ip, browser), (town, _, _)) -> + | join(place, (_, ((rid, node, d, last, gender, birthday, created, ip, browser), (town, _, _))) -> ((rid, node), (d, (last, gender, birthday, created, ip, browser, town)))); let who = found | map(((rid, node), _) -> (node, rid)); -let emails = who | join(email, (node, rid, value) -> ((rid, node), value)) | collect; -let langs = who | join(language, (node, rid, value) -> ((rid, node), value)) | collect; -let school = who | join(study, (node, rid, (at, year)) -> (at, (rid, node, year))) - | join(org, (_, (rid, node, year), (name, _, at)) -> (at, (rid, node, name, year))) - | join(place, (_, (rid, node, name, year), (where, _, _)) -> ((rid, node), (name, year, where))) | collect; -let jobs = who | join(work, (node, rid, (at, year)) -> (at, (rid, node, year))) - | join(org, (_, (rid, node, year), (name, _, at)) -> (at, (rid, node, name, year))) - | join(place, (_, (rid, node, name, year), (where, _, _)) -> ((rid, node), (name, year, where))) | collect; -let with1 = (found | join(emails, (key, row, list) -> (key, (row, list)))) - + ((found - (found | join(emails, (key, row, _) -> (key, row)))) | map((key, row) -> (key, (row, [] : List(List(int)))))); -let with2 = (with1 | join(langs, (key, (row, e), list) -> (key, (row, e, list)))) - + ((with1 - (with1 | join(langs, (key, row, _) -> (key, row)))) | map((key, (row, e)) -> (key, (row, e, [] : List(List(int)))))); -let with3 = (with2 | join(school, (key, (row, e, l), list) -> (key, (row, e, l, list)))) - + ((with2 - (with2 | join(school, (key, row, _) -> (key, row)))) | map((key, (row, e, l)) -> (key, (row, e, l, [] : List((List(int), int, List(int))))))); -let with4 = (with3 | join(jobs, (key, (row, e, l, s), list) -> (key, (row, e, l, s, list)))) - + ((with3 - (with3 | join(jobs, (key, row, _) -> (key, row)))) | map((key, (row, e, l, s)) -> (key, (row, e, l, s, [] : List((List(int), int, List(int))))))); +let emails = who | join(email, (node, (rid, value)) -> ((rid, node), value)) | collect; +let langs = who | join(language, (node, (rid, value)) -> ((rid, node), value)) | collect; +let school = who | join(study, (node, (rid, (at, year))) -> (at, (rid, node, year))) + | join(org, (_, ((rid, node, year), (name, _, at))) -> (at, (rid, node, name, year))) + | join(place, (_, ((rid, node, name, year), (where, _, _))) -> ((rid, node), (name, year, where))) | collect; +let jobs = who | join(work, (node, (rid, (at, year))) -> (at, (rid, node, year))) + | join(org, (_, ((rid, node, year), (name, _, at))) -> (at, (rid, node, name, year))) + | join(place, (_, ((rid, node, name, year), (where, _, _))) -> ((rid, node), (name, year, where))) | collect; +let with1 = (found | join(emails)) + + ((found - (found | join(emails, (key, (row, _)) -> (key, row)))) | map((key, row) -> (key, (row, [] : List(List(int)))))); +let with2 = (with1 | join(langs, (key, ((row, e), list)) -> (key, (row, e, list)))) + + ((with1 - (with1 | join(langs, (key, (row, _)) -> (key, row)))) | map((key, (row, e)) -> (key, (row, e, [] : List(List(int)))))); +let with3 = (with2 | join(school, (key, ((row, e, l), list)) -> (key, (row, e, l, list)))) + + ((with2 - (with2 | join(school, (key, (row, _)) -> (key, row)))) | map((key, (row, e, l)) -> (key, (row, e, l, [] : List((List(int), int, List(int))))))); +let with4 = (with3 | join(jobs, (key, ((row, e, l, s), list)) -> (key, (row, e, l, s, list)))) + + ((with3 - (with3 | join(jobs, (key, (row, _)) -> (key, row)))) | map((key, (row, e, l, s)) -> (key, (row, e, l, s, [] : List((List(int), int, List(int))))))); let ranked = with4 | map(((rid, node), ((d, (last, gender, birthday, created, ip, browser, town)), e, l, s, w)) -> (rid, ((d, last, node), (gender, birthday, created, ip, browser, town, e, l, s, w)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); diff --git a/interactive/bench/ldbc/interactive_v2/oneoff/ic02.oneoff.ddp b/interactive/bench/ldbc/interactive_v2/oneoff/ic02.oneoff.ddp index 3b1d8b099..791378c9b 100644 --- a/interactive/bench/ldbc/interactive_v2/oneoff/ic02.oneoff.ddp +++ b/interactive/bench/ldbc/interactive_v2/oneoff/ic02.oneoff.ddp @@ -6,17 +6,17 @@ let by_author = import "ic2s::by_author" : (int, (int, int)); let latest = import "ic2s::latest" : (int, (int, int)); let newest = import "ic2s::newest" : (int, int); let friends = request | map(((rid, who, max), ()) -> (who, (rid, max))) - | join(friend, (_, req, (f, _)) -> (f, req)); -let ahead = friends | join(newest, (f, (rid, max), neg) -> (f, (rid, max, neg neg))); + | join(friend, (_, (req, (f, _))) -> (f, req)); +let ahead = friends | join(newest, (f, ((rid, max), neg)) -> (f, (rid, max, neg neg))); let fast = ahead | filter((_, (_, max, last)) -> (last, max) lt) - | join(latest, (f, (rid, _, _), (neg, id)) -> (rid, ((neg, id), f))); + | join(latest, (f, ((rid, _, _), (neg, id))) -> (rid, ((neg, id), f))); let slow = ahead | filter((_, (_, max, last)) -> (last, max) ge) - | join(by_author, (f, (rid, max, _), (neg, id)) -> (rid, ((neg, id), f, max))) + | join(by_author, (f, ((rid, max, _), (neg, id))) -> (rid, ((neg, id), f, max))) | filter((_, ((neg, _), _, max)) -> (neg neg, max) lt) | map((rid, (order, f, _)) -> (rid, (order, f))); let top = (fast + slow) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = top | map((rid, (rank, ((neg, id), f))) -> (id, (rid, rank, neg neg, f))) - | join(message, (id, (rid, rank, created, f), (_, _, _, _, content, image, _, _, _, _, _, _, _)) -> + | join(message, (id, ((rid, rank, created, f), (_, _, _, _, content, image, _, _, _, _, _, _, _))) -> (f, (rid, rank, id, ((content len, 0) gt, content, image) select, created))) - | join(person, (f, (rid, rank, id, body, created), (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, rank), (f, first, last, id, body, created))); + | join(person, (f, ((rid, rank, id, body, created), (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, rank), (f, first, last, id, body, created))); output "answer" = answer | arrange; diff --git a/interactive/bench/ldbc/interactive_v2/oneoff/ic03.oneoff.ddp b/interactive/bench/ldbc/interactive_v2/oneoff/ic03.oneoff.ddp index f64374b2e..17b4070f7 100644 --- a/interactive/bench/ldbc/interactive_v2/oneoff/ic03.oneoff.ddp +++ b/interactive/bench/ldbc/interactive_v2/oneoff/ic03.oneoff.ddp @@ -6,31 +6,31 @@ let place = import "graph::place" : (int, (List(int), List(int), int)); let by_name = import "graph::place.by_name" : (List(int), int); let sent_from = import "ic3s::sent_from" : (int, (int, int)); let asked = request | map(((rid, who, x, y, start, days), ()) -> (x, (rid, who, y, start, (start, (days, 86400000) mul) add))) - | join(by_name, (_, req, x) -> (x, req)) - | join(place, (x, (rid, who, y, start, end), (_, kind, _)) -> ((kind, [67,111,117,110,116,114,121]) eq, (y, (rid, who, x, start, end)))) + | join(by_name, (_, (req, x)) -> (x, req)) + | join(place, (x, ((rid, who, y, start, end), (_, kind, _))) -> ((kind, [67,111,117,110,116,114,121]) eq, (y, (rid, who, x, start, end)))) | filter((country, _) -> country) | map((_, row) -> row) - | join(by_name, (_, req, y) -> (y, req)) - | join(place, (y, (rid, who, x, start, end), (_, kind, _)) -> ((kind, [67,111,117,110,116,114,121]) eq, (rid, (who, x, y, start, end)))) + | join(by_name, (_, (req, y)) -> (y, req)) + | join(place, (y, ((rid, who, x, start, end), (_, kind, _))) -> ((kind, [67,111,117,110,116,114,121]) eq, (rid, (who, x, y, start, end)))) | filter((country, _) -> country) | map((_, row) -> row); let window = (asked | map((rid, (_, x, _, start, end)) -> (x, (rid, start, end, 1, 0)))) + (asked | map((rid, (_, _, y, start, end)) -> (y, (rid, start, end, 0, 1)))); -let visits = window | join(sent_from, (_, (rid, start, end, ix, iy), (author, created)) -> ((rid, author), (created, start, end, ix, iy))) +let visits = window | join(sent_from, (_, ((rid, start, end, ix, iy), (author, created))) -> ((rid, author), (created, start, end, ix, iy))) | filter((_, (created, start, end, _, _)) -> ((created, start) ge, (created, end) lt) all) | map((key, (_, _, _, ix, iy)) -> (key, (ix, iy))); let hop1 = asked | map((rid, (who, x, y, _, _)) -> (who, (rid, who, x, y))) - | join(friend, (_, req, (f, _)) -> (f, req)); -let hop2 = hop1 | join(friend, (_, req, (g, _)) -> (g, req)); + | join(friend, (_, (req, (f, _))) -> (f, req)); +let hop2 = hop1 | join(friend, (_, (req, (g, _))) -> (g, req)); let circle = (hop1 + hop2) | filter((node, (_, who, _, _)) -> (node, who) ne) - | join(person, (node, (rid, _, x, y), (_, _, _, _, _, _, _, city, _, _, _)) -> (city, (rid, node, x, y))) - | join(place, (_, (rid, node, x, y), (_, _, home)) -> ((rid, node), (x, y, home))) + | join(person, (node, ((rid, _, x, y), (_, _, _, _, _, _, _, city, _, _, _))) -> (city, (rid, node, x, y))) + | join(place, (_, ((rid, node, x, y), (_, _, home))) -> ((rid, node), (x, y, home))) | filter((_, (x, y, home)) -> ((home, x) ne, (home, y) ne) all) | map((key, _) -> (key, ())) | distinct; -let seen = visits | join(circle, (key, (ix, iy), ()) -> (key, (ix, iy))); +let seen = visits | join(circle, (key, ((ix, iy), ())) -> (key, (ix, iy))); let in_x = seen | filter((_, (ix, _)) -> (ix, 1) eq) | map((key, _) -> (key, ())) | count; let in_y = seen | filter((_, (_, iy)) -> (iy, 1) eq) | map((key, _) -> (key, ())) | count; -let counts = in_x | join(in_y, (key, nx, ny) -> (key, (nx, ny))); +let counts = in_x | join(in_y); let ranked = counts | map(((rid, node), (nx, ny)) -> (node, (rid, nx, ny))) - | join(person, (node, (rid, nx, ny), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, (((nx, ny) add neg, node), (first, last, nx, ny)))) + | join(person, (node, ((rid, nx, ny), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, (((nx, ny) add neg, node), (first, last, nx, ny)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((_, node), (first, last, nx, ny)))) -> ((rid, rank), (node, first, last, nx, ny, (nx, ny) add))); output "answer" = answer | arrange; diff --git a/interactive/bench/ldbc/interactive_v2/oneoff/ic04.oneoff.ddp b/interactive/bench/ldbc/interactive_v2/oneoff/ic04.oneoff.ddp index f6c04ba27..a2fadc520 100644 --- a/interactive/bench/ldbc/interactive_v2/oneoff/ic04.oneoff.ddp +++ b/interactive/bench/ldbc/interactive_v2/oneoff/ic04.oneoff.ddp @@ -6,12 +6,12 @@ let mtag = import "graph::mtag.by_tag" : (int, int); let posts = import "ic4s::posts" : (int, (int, int)); let tags = import "ic4s::tags" : (int, (int, List(int))); let seen = request | map(((rid, who, start, days), ()) -> (who, (rid, start, (start, (days, 86400000) mul) add))) - | join(friend, (_, req, (f, _)) -> (f, req)) - | join(posts, (_, (rid, start, end), (m, created)) -> (m, (rid, created, start, end))) - | join(tags, (_, (rid, created, start, end), (t, name)) -> ((rid, t), (name, created, start, end))); + | join(friend, (_, (req, (f, _))) -> (f, req)) + | join(posts, (_, ((rid, start, end), (m, created))) -> (m, (rid, created, start, end))) + | join(tags, (_, ((rid, created, start, end), (t, name))) -> ((rid, t), (name, created, start, end))); let before = seen | filter((_, (_, created, start, _)) -> (created, start) lt) | map((key, _) -> (key, ())) | distinct; let during = seen | filter((_, (_, created, start, end)) -> ((created, start) ge, (created, end) lt) all); -let fresh = during - (during | join(before, (key, row, ()) -> (key, row))); +let fresh = during - (during | join(before, (key, (row, ())) -> (key, row))); let counts = fresh | map(((rid, _), (name, _, _, _)) -> ((rid, name), ())) | count; let ranked = counts | map(((rid, name), n) -> (rid, (n neg, name))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); diff --git a/interactive/bench/ldbc/interactive_v2/oneoff/ic04.standing.ddp b/interactive/bench/ldbc/interactive_v2/oneoff/ic04.standing.ddp index 0e0872924..10f5db118 100644 --- a/interactive/bench/ldbc/interactive_v2/oneoff/ic04.standing.ddp +++ b/interactive/bench/ldbc/interactive_v2/oneoff/ic04.standing.ddp @@ -4,6 +4,6 @@ let tag = import "graph::tag" : (int, (List(int), int)); let mtag = import "graph::mtag.by_tag" : (int, int); let posts = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 0) eq) | map((id, (_, created, who, _, _, _, _, _, _, _, _, _, _)) -> (who, (id, created))); -let tags = mtag | join(tag, (t, m, (name, _)) -> (m, (t, name))); +let tags = mtag | join(tag, (t, (m, (name, _))) -> (m, (t, name))); output "posts" = posts | arrange; output "tags" = tags | arrange; diff --git a/interactive/bench/ldbc/interactive_v2/oneoff/ic05.oneoff.ddp b/interactive/bench/ldbc/interactive_v2/oneoff/ic05.oneoff.ddp index 224f04cf9..698413f4e 100644 --- a/interactive/bench/ldbc/interactive_v2/oneoff/ic05.oneoff.ddp +++ b/interactive/bench/ldbc/interactive_v2/oneoff/ic05.oneoff.ddp @@ -8,27 +8,27 @@ let posted = import "ic5s::posted" : ((int, int), int); let first = import "ic5s::first" : (int, int); let lowest = import "ic5s::lowest" : (int, int); let hop1 = request | map(((rid, who, date), ()) -> (who, (rid, who, date))) - | join(person, (who, req, _) -> (who, req)) - | join(friend, (_, req, (f, _)) -> (f, req)); -let hop2 = hop1 | join(friend, (_, req, (g, _)) -> (g, req)); + | join(person, (who, (req, _)) -> (who, req)) + | join(friend, (_, (req, (f, _))) -> (f, req)); +let hop2 = hop1 | join(friend, (_, (req, (g, _))) -> (g, req)); let circle = (hop1 + hop2) | filter((node, (_, who, _)) -> (node, who) ne) | map((node, (rid, _, date)) -> ((node, rid, date), ())) | distinct | map(((node, rid, date), ()) -> (node, (rid, date))); -let after = circle | join(member, (node, (rid, date), (f, since)) -> ((node, f), (rid, date, since))) +let after = circle | join(member, (node, ((rid, date), (f, since))) -> ((node, f), (rid, date, since))) | filter((_, (_, date, since)) -> (since, date) gt) | map((key, (rid, _, _)) -> (key, rid)); -let busy = after | join(posted, ((_, f), rid, n) -> ((rid, f), n)) +let busy = after | join(posted, ((_, f), (rid, n)) -> ((rid, f), n)) | join_function((key, n) -> ((key, ()), 0, n) enlist) | count; -let timed = circle | join(first, (node, (rid, date), since) -> (node, (rid, date, since))); +let timed = circle | join(first, (node, ((rid, date), since)) -> (node, (rid, date, since))); let fast = timed | filter((_, (_, date, since)) -> (since, date) gt) - | join(lowest, (_, (rid, _, _), f) -> ((rid, f), ())); + | join(lowest, (_, ((rid, _, _), f)) -> ((rid, f), ())); let slow = timed | filter((_, (_, date, since)) -> (since, date) le) | map((node, (rid, _, _)) -> ((node, rid), ())) - | join(after | map(((node, f), rid) -> ((node, rid), f)), ((_, rid), (), f) -> ((rid, f), ())); + | join(after | map(((node, f), rid) -> ((node, rid), f)), ((_, rid), ((), f)) -> ((rid, f), ())); let idle = (fast + slow) | distinct; -let quiet = (idle - (idle | join(busy, (key, (), _) -> (key, ())))) | map((key, ()) -> (key, 0)); +let quiet = (idle - (idle | join(busy, (key, ((), _)) -> (key, ())))) | map((key, ()) -> (key, 0)); let ranked = (busy + quiet) | map(((rid, f), n) -> (f, (rid, n))) - | join(forum, (f, (rid, n), (title, _, _)) -> (rid, ((n neg, f), title))) + | join(forum, (f, ((rid, n), (title, _, _))) -> (rid, ((n neg, f), title))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((neg, _), title))) -> ((rid, rank), (title, neg neg))); output "answer" = answer | arrange; diff --git a/interactive/bench/ldbc/interactive_v2/oneoff/ic06.oneoff.ddp b/interactive/bench/ldbc/interactive_v2/oneoff/ic06.oneoff.ddp index 62ec1c766..7f963db76 100644 --- a/interactive/bench/ldbc/interactive_v2/oneoff/ic06.oneoff.ddp +++ b/interactive/bench/ldbc/interactive_v2/oneoff/ic06.oneoff.ddp @@ -6,22 +6,22 @@ let tag = import "graph::tag" : (int, (List(int), int)); let by_name = import "graph::tag.by_name" : (List(int), int); let tagged = import "graph::mtag.by_tag" : (int, int); let tags_of = import "graph::mtag.by_message" : (int, int); -let hop1 = request | map(((rid, who, _), ()) -> (who, (rid, who))) | join(person, (who, req, _) -> (who, req)) - | join(friend, (_, (rid, who), (f, _)) -> (f, (rid, who))); -let hop2 = hop1 | join(friend, (_, (rid, who), (g, _)) -> (g, (rid, who))); +let hop1 = request | map(((rid, who, _), ()) -> (who, (rid, who))) | join(person, (who, (req, _)) -> (who, req)) + | join(friend, (_, ((rid, who), (f, _))) -> (f, (rid, who))); +let hop2 = hop1 | join(friend, (_, ((rid, who), (g, _))) -> (g, (rid, who))); let circle = (hop1 + hop2) | filter((node, (_, who)) -> (node, who) ne) | map((node, (rid, _)) -> ((rid, node), ())) | distinct; let with_it = request | map(((rid, _, name), ()) -> (name, rid)) - | join(by_name, (_, rid, t) -> (t, rid)) - | join(tagged, (t, rid, m) -> (m, (rid, t))) - | join(message, (m, (rid, t), (kind, _, author, _, _, _, _, _, _, _, _, _, _)) -> ((rid, author), (kind, m, t))) + | join(by_name, (_, (rid, t)) -> (t, rid)) + | join(tagged, (t, (rid, m)) -> (m, (rid, t))) + | join(message, (m, ((rid, t), (kind, _, author, _, _, _, _, _, _, _, _, _, _))) -> ((rid, author), (kind, m, t))) | filter((_, (kind, _, _)) -> (kind, 0) eq) - | join(circle, ((rid, _), (_, m, t), ()) -> (m, (rid, t))); -let others = with_it | join(tags_of, (_, (rid, t), other) -> ((rid, t, other), ())) + | join(circle, ((rid, _), ((_, m, t), ())) -> (m, (rid, t))); +let others = with_it | join(tags_of, (_, ((rid, t), other)) -> ((rid, t, other), ())) | filter(((_, t, other), ()) -> (t, other) ne) | map(((rid, _, other), ()) -> ((rid, other), ())) | count; let ranked = others | map(((rid, other), n) -> (other, (rid, n))) - | join(tag, (_, (rid, n), (name, _)) -> (rid, (n neg, name))) + | join(tag, (_, ((rid, n), (name, _))) -> (rid, (n neg, name))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, (neg, other))) -> ((rid, rank), (other, neg neg))); output "answer" = answer | arrange; diff --git a/interactive/bench/ldbc/interactive_v2/oneoff/ic07.oneoff.ddp b/interactive/bench/ldbc/interactive_v2/oneoff/ic07.oneoff.ddp index 8e65f0467..596764bba 100644 --- a/interactive/bench/ldbc/interactive_v2/oneoff/ic07.oneoff.ddp +++ b/interactive/bench/ldbc/interactive_v2/oneoff/ic07.oneoff.ddp @@ -5,16 +5,16 @@ let wrote = import "graph::message.by_creator" : (int, int); let likes = import "graph::likes.by_message" : (int, (int, int)); let pals = import "graph::friend.pair" : ((int, int), ()); let latest = request | map(((rid, who), ()) -> (who, rid)) - | join(wrote, (who, rid, m) -> (m, (rid, who))) - | join(likes, (m, (rid, who), (liker, when)) -> ((rid, who, liker), (when neg, m))) + | join(wrote, (who, (rid, m)) -> (m, (rid, who))) + | join(likes, (m, ((rid, who), (liker, when))) -> ((rid, who, liker), (when neg, m))) | min; let rows = latest | map(((rid, who, liker), (neg, m)) -> (m, (rid, who, liker, neg neg))) - | join(message, (m, (rid, who, liker, when), (_, created, _, _, text, image, _, _, _, _, _, _, _)) -> + | join(message, (m, ((rid, who, liker, when), (_, created, _, _, text, image, _, _, _, _, _, _, _))) -> ((who, liker), (rid, liker, when, m, ((text len, 0) gt, text, image) select, (when, created) sub))); -let known = rows | join(pals, (pair, row, ()) -> (pair, row)); +let known = rows | join(pals, (pair, (row, ())) -> (pair, row)); let marked = (known | map((_, row) -> (row, 0))) + ((rows - known) | map((_, row) -> (row, 1))); let ranked = marked | map(((rid, liker, when, m, body, age), fresh) -> (liker, (rid, when, m, body, age, fresh))) - | join(person, (liker, (rid, when, m, body, age, fresh), (first, last, _, _, _, _, _, _, _, _, _)) -> + | join(person, (liker, ((rid, when, m, body, age, fresh), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((when neg, liker), (first, last, m, body, (age, 60000) div, fresh)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((neg, liker), (first, last, m, body, minutes, fresh)))) -> diff --git a/interactive/bench/ldbc/interactive_v2/oneoff/ic08.oneoff.ddp b/interactive/bench/ldbc/interactive_v2/oneoff/ic08.oneoff.ddp index b0ffd15e5..f3b4b973d 100644 --- a/interactive/bench/ldbc/interactive_v2/oneoff/ic08.oneoff.ddp +++ b/interactive/bench/ldbc/interactive_v2/oneoff/ic08.oneoff.ddp @@ -4,13 +4,13 @@ let message = import "graph::message" : (int, (int, int, int, int, List(int), Li let wrote = import "graph::message.by_creator" : (int, int); let replies = import "graph::message.by_parent" : (int, int); let top = request | map(((rid, who), ()) -> (who, rid)) - | join(wrote, (_, rid, op) -> (op, rid)) - | join(replies, (_, rid, id) -> (id, rid)) - | join(message, (id, rid, (kind, created, who, _, _, _, _, _, _, _, _, _, _)) -> (rid, (kind, (created neg, id), who))) + | join(wrote, (_, (rid, op)) -> (op, rid)) + | join(replies, (_, (rid, id)) -> (id, rid)) + | join(message, (id, (rid, (kind, created, who, _, _, _, _, _, _, _, _, _, _))) -> (rid, (kind, (created neg, id), who))) | filter((_, (kind, _, _)) -> (kind, 1) eq) | map((rid, (_, order, who)) -> (rid, (order, who))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = top | map((rid, (rank, ((neg, id), who))) -> (id, (rid, rank, neg neg, who))) - | join(message, (id, (rid, rank, created, who), (_, _, _, _, content, _, _, _, _, _, _, _, _)) -> (who, (rid, rank, id, content, created))) - | join(person, (who, (rid, rank, id, content, created), (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, rank), (who, first, last, created, id, content))); + | join(message, (id, ((rid, rank, created, who), (_, _, _, _, content, _, _, _, _, _, _, _, _))) -> (who, (rid, rank, id, content, created))) + | join(person, (who, ((rid, rank, id, content, created), (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, rank), (who, first, last, created, id, content))); output "answer" = answer | arrange; diff --git a/interactive/bench/ldbc/interactive_v2/oneoff/ic09.oneoff.ddp b/interactive/bench/ldbc/interactive_v2/oneoff/ic09.oneoff.ddp index 9857a0dd4..06142aea9 100644 --- a/interactive/bench/ldbc/interactive_v2/oneoff/ic09.oneoff.ddp +++ b/interactive/bench/ldbc/interactive_v2/oneoff/ic09.oneoff.ddp @@ -5,24 +5,24 @@ let friend = import "graph::friend" : (int, (int, int)); let by_author = import "ic9s::by_author" : (int, (int, int)); let latest = import "ic9s::latest" : (int, (int, int)); let newest = import "ic9s::newest" : (int, int); -let hop1 = request | map(((rid, who, max), ()) -> (who, (rid, who, max))) | join(person, (who, req, _) -> (who, req)) - | join(friend, (_, req, (f, _)) -> (f, req)); -let hop2 = hop1 | join(friend, (_, req, (g, _)) -> (g, req)); +let hop1 = request | map(((rid, who, max), ()) -> (who, (rid, who, max))) | join(person, (who, (req, _)) -> (who, req)) + | join(friend, (_, (req, (f, _))) -> (f, req)); +let hop2 = hop1 | join(friend, (_, (req, (g, _))) -> (g, req)); let friends = (hop1 + hop2) | filter((node, (_, who, _)) -> (node, who) ne) | map((node, (rid, _, max)) -> ((node, rid, max), ())) | distinct | map(((node, rid, max), ()) -> (node, (rid, max))); -let ahead = friends | join(newest, (f, (rid, max), neg) -> (f, (rid, max, neg neg))); +let ahead = friends | join(newest, (f, ((rid, max), neg)) -> (f, (rid, max, neg neg))); let fast = ahead | filter((_, (_, max, last)) -> (last, max) lt) - | join(latest, (f, (rid, _, _), (neg, id)) -> (rid, ((neg, id), f))); + | join(latest, (f, ((rid, _, _), (neg, id))) -> (rid, ((neg, id), f))); let slow = ahead | filter((_, (_, max, last)) -> (last, max) ge) - | join(by_author, (f, (rid, max, _), (neg, id)) -> ((rid, f), ((neg, id), max))) + | join(by_author, (f, ((rid, max, _), (neg, id))) -> ((rid, f), ((neg, id), max))) | filter((_, ((neg, _), max)) -> (neg neg, max) lt) | map((key, (order, _)) -> (key, order)) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten) | map(((rid, f), (_, order)) -> (rid, (order, f))); let top = (fast + slow) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = top | map((rid, (rank, ((neg, id), f))) -> (id, (rid, rank, neg neg, f))) - | join(message, (id, (rid, rank, created, f), (_, _, _, _, content, image, _, _, _, _, _, _, _)) -> + | join(message, (id, ((rid, rank, created, f), (_, _, _, _, content, image, _, _, _, _, _, _, _))) -> (f, (rid, rank, id, ((content len, 0) gt, content, image) select, created))) - | join(person, (f, (rid, rank, id, body, created), (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, rank), (f, first, last, id, body, created))); + | join(person, (f, ((rid, rank, id, body, created), (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, rank), (f, first, last, id, body, created))); output "answer" = answer | arrange; diff --git a/interactive/bench/ldbc/interactive_v2/oneoff/ic10.oneoff.ddp b/interactive/bench/ldbc/interactive_v2/oneoff/ic10.oneoff.ddp index f69df8bb4..eec7a189b 100644 --- a/interactive/bench/ldbc/interactive_v2/oneoff/ic10.oneoff.ddp +++ b/interactive/bench/ldbc/interactive_v2/oneoff/ic10.oneoff.ddp @@ -9,28 +9,28 @@ let people = import "ic10s::people" : (int, (List(int), List(int), List(int), Li let posts = import "ic10s::posts" : (int, int); let likes = import "ic10s::likes" : ((int, int), ()); let hop1 = request | map(((rid, who, month), ()) -> (who, (rid, who, month))) - | join(friend, (_, req, (f, _)) -> (f, (req, 1))); -let hop2 = hop1 | join(friend, (_, (req, _), (g, _)) -> (g, (req, 2))); + | join(friend, (_, (req, (f, _))) -> (f, (req, 1))); +let hop2 = hop1 | join(friend, (_, ((req, _), (g, _))) -> (g, (req, 2))); let twohop = (hop1 + hop2) | filter((node, ((_, who, _), _)) -> (node, who) ne) | map((node, ((rid, who, month), d)) -> ((rid, who, month, node), d)) | min | filter((_, d) -> (d, 2) eq); let cands = twohop | map(((rid, who, month, node), _) -> (node, (rid, who, month))) - | join(people, (node, (rid, who, month), (first, last, gender, town, bmonth, bday)) -> + | join(people, (node, ((rid, who, month), (first, last, gender, town, bmonth, bday))) -> ((rid, who, node), (first, last, gender, town, month, bmonth, bday))) | filter((_, (_, _, _, _, month, bmonth, bday)) -> (((bmonth, month) eq, (bday, 21) ge) all, ((bmonth, ((month, 12) eq, 1, (month, 1) add) select) eq, (bday, 22) lt) all) any) | map((key, (first, last, gender, town, _, _, _)) -> (key, (first, last, gender, town))); -let wrote = cands | map(((rid, who, node), _) -> (node, (rid, who))) | join(posts, (node, (rid, who), m) -> ((rid, who, node, m), ())); +let wrote = cands | map(((rid, who, node), _) -> (node, (rid, who))) | join(posts, (node, ((rid, who), m)) -> ((rid, who, node, m), ())); let liked = wrote | map(((rid, who, node, m), ()) -> (m, (rid, who, node))) - | join(tags_of, (m, (rid, who, node), t) -> ((who, t), (rid, node, m))) - | join(likes, ((who, _), (rid, node, m), ()) -> ((rid, who, node, m), ())) + | join(tags_of, (m, ((rid, who, node), t)) -> ((who, t), (rid, node, m))) + | join(likes, ((who, _), ((rid, node, m), ())) -> ((rid, who, node, m), ())) | distinct; let points = (liked | map(((rid, who, node, _), ()) -> ((rid, who, node), 1))) + ((wrote - liked) | map(((rid, who, node, _), ()) -> ((rid, who, node), -1))); let summed = points | join_function((key, p) -> ((key, ()), 0, p) enlist) | count; let keys = cands | map((key, _) -> (key, ())); -let score = summed + ((keys - (keys | join(summed, (key, (), _) -> (key, ())))) | map((key, ()) -> (key, 0))); -let ranked = score | join(cands, ((rid, _, node), s, (first, last, gender, town)) -> (rid, ((s neg, node), (first, last, s, gender, town)))) +let score = summed + ((keys - (keys | join(summed, (key, ((), _)) -> (key, ())))) | map((key, ()) -> (key, 0))); +let ranked = score | join(cands, ((rid, _, node), (s, (first, last, gender, town))) -> (rid, ((s neg, node), (first, last, s, gender, town)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((_, node), (first, last, s, gender, town)))) -> ((rid, rank), (node, first, last, s, gender, town))); output "answer" = answer | arrange; diff --git a/interactive/bench/ldbc/interactive_v2/oneoff/ic10.standing.ddp b/interactive/bench/ldbc/interactive_v2/oneoff/ic10.standing.ddp index ea2164a62..b845d360a 100644 --- a/interactive/bench/ldbc/interactive_v2/oneoff/ic10.standing.ddp +++ b/interactive/bench/ldbc/interactive_v2/oneoff/ic10.standing.ddp @@ -5,7 +5,7 @@ let place = import "graph::place" : (int, (List(int), List(int), int)); let tags_of = import "graph::mtag.by_message" : (int, int); let interest = import "graph::interest.by_person" : (int, int); let people = person | map((id, (first, last, gender, _, _, _, _, home, bmonth, bday, _)) -> (home, (id, first, last, gender, bmonth, bday))) - | join(place, (_, (id, first, last, gender, bmonth, bday), (town, _, _)) -> (id, (first, last, gender, town, bmonth, bday))); + | join(place, (_, ((id, first, last, gender, bmonth, bday), (town, _, _))) -> (id, (first, last, gender, town, bmonth, bday))); let posts = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 0) eq) | map((id, (_, _, who, _, _, _, _, _, _, _, _, _, _)) -> (who, id)); let likes = interest | map((p, t) -> ((p, t), ())); diff --git a/interactive/bench/ldbc/interactive_v2/oneoff/ic11.oneoff.ddp b/interactive/bench/ldbc/interactive_v2/oneoff/ic11.oneoff.ddp index aaaf1a5c2..a1f40c49e 100644 --- a/interactive/bench/ldbc/interactive_v2/oneoff/ic11.oneoff.ddp +++ b/interactive/bench/ldbc/interactive_v2/oneoff/ic11.oneoff.ddp @@ -5,21 +5,21 @@ let work = import "graph::work.by_person" : (int, (int, int)); let org = import "graph::org" : (int, (List(int), List(int), int)); let place = import "graph::place" : (int, (List(int), List(int), int)); let hop1 = request | map(((rid, who, country, year), ()) -> (who, (rid, who, country, year))) - | join(person, (who, req, _) -> (who, req)) - | join(friend, (_, req, (f, _)) -> (f, req)); -let hop2 = hop1 | join(friend, (_, req, (g, _)) -> (g, req)); + | join(person, (who, (req, _)) -> (who, req)) + | join(friend, (_, (req, (f, _))) -> (f, req)); +let hop2 = hop1 | join(friend, (_, (req, (g, _))) -> (g, req)); let circle = (hop1 + hop2) | filter((node, (_, who, _, _)) -> (node, who) ne) | map((node, (rid, _, country, year)) -> ((node, rid, country, year), ())) | distinct; let jobs = circle | map(((node, rid, country, year), ()) -> (node, (rid, country, year))) - | join(work, (node, (rid, country, year), (at, since)) -> (at, (rid, node, country, year, since))) + | join(work, (node, ((rid, country, year), (at, since))) -> (at, (rid, node, country, year, since))) | filter((_, (_, _, _, year, since)) -> (since, year) lt) - | join(org, (_, (rid, node, country, _, since), (company, _, located)) -> (located, ((rid, since, node), (company, country)))) - | join(place, (_, (key, (company, country)), (there, _, _)) -> (key, (company, country, there))) + | join(org, (_, ((rid, node, country, _, since), (company, _, located))) -> (located, ((rid, since, node), (company, country)))) + | join(place, (_, ((key, (company, country)), (there, _, _))) -> (key, (company, country, there))) | filter((_, (_, country, there)) -> (country, there) eq) | map((key, (company, _, _)) -> (key, company)); let named = jobs | collect | flatmap((k, entries) -> (k, ((0, entries len) range, entries) zip) cap_list); let ranked = named | map(((rid, since, node), (pos, company)) -> (node, (rid, since, pos, company))) - | join(person, (node, (rid, since, pos, company), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, ((since, node, pos neg), (first, last, company)))) + | join(person, (node, ((rid, since, pos, company), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((since, node, pos neg), (first, last, company)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((since, node, _), (first, last, company)))) -> ((rid, rank), (node, first, last, company, since))); output "answer" = answer | arrange; diff --git a/interactive/bench/ldbc/interactive_v2/oneoff/ic12.oneoff.ddp b/interactive/bench/ldbc/interactive_v2/oneoff/ic12.oneoff.ddp index fa58195e1..00671b5ea 100644 --- a/interactive/bench/ldbc/interactive_v2/oneoff/ic12.oneoff.ddp +++ b/interactive/bench/ldbc/interactive_v2/oneoff/ic12.oneoff.ddp @@ -10,24 +10,24 @@ let mtag = import "graph::mtag.by_tag" : (int, int); let below_hl = import "ic12s::below_hl" : (int, int); let reply = import "ic12s::reply" : (int, (int, int)); let intag = request | map(((rid, _, name), ()) -> (name, rid)) - | join(classes, (_, rid, high) -> (high, rid)) - | join(below_hl, (_, rid, low) -> (low, rid)) - | join(by_class, (_, rid, t) -> (t, rid)) - | join(tag, (t, rid, (name, _)) -> (t, (rid, name))) - | join(mtag, (_, (rid, name), m) -> (m, (rid, name))) - | join(message, (m, (rid, name), (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> ((rid, m, name), kind)) + | join(classes, (_, (rid, high)) -> (high, rid)) + | join(below_hl, (_, (rid, low)) -> (low, rid)) + | join(by_class, (_, (rid, t)) -> (t, rid)) + | join(tag, (t, (rid, (name, _))) -> (t, (rid, name))) + | join(mtag, (_, ((rid, name), m)) -> (m, (rid, name))) + | join(message, (m, ((rid, name), (kind, _, _, _, _, _, _, _, _, _, _, _, _))) -> ((rid, m, name), kind)) | filter((_, kind) -> (kind, 0) eq) | map((key, _) -> (key, ())); let hits = request | map(((rid, who, _), ()) -> (who, rid)) - | join(friend, (_, rid, (f, _)) -> (f, rid)) - | join(reply, (f, rid, (id, parent)) -> ((rid, parent), (f, id))) - | join(intag | map(((rid, m, name), ()) -> ((rid, m), name)), ((rid, _), (f, id), name) -> ((rid, f), (id, name))); + | join(friend, (_, (rid, (f, _))) -> (f, rid)) + | join(reply, (f, (rid, (id, parent))) -> ((rid, parent), (f, id))) + | join(intag | map(((rid, m, name), ()) -> ((rid, m), name)), ((rid, _), ((f, id), name)) -> ((rid, f), (id, name))); let counts = hits | map(((rid, f), (id, _)) -> ((rid, f, id), ())) | distinct | map(((rid, f, _), ()) -> ((rid, f), ())) | count; let tagset = hits | map(((rid, f), (_, name)) -> ((rid, f, name), ())) | distinct | map(((rid, f, name), ()) -> ((rid, f), name)) | collect; -let ranked = counts | join(tagset, ((rid, f), n, list) -> (f, (rid, n, list))) - | join(person, (f, (rid, n, list), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, ((n neg, f), (first, last, list)))) +let ranked = counts | join(tagset, ((rid, f), (n, list)) -> (f, (rid, n, list))) + | join(person, (f, ((rid, n, list), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((n neg, f), (first, last, list)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((neg, f), (first, last, list)))) -> ((rid, rank), (f, first, last, list, neg neg))); output "answer" = answer | arrange; diff --git a/interactive/bench/ldbc/interactive_v2/oneoff/ic12.standing.ddp b/interactive/bench/ldbc/interactive_v2/oneoff/ic12.standing.ddp index 260e4c61e..16ff8b603 100644 --- a/interactive/bench/ldbc/interactive_v2/oneoff/ic12.standing.ddp +++ b/interactive/bench/ldbc/interactive_v2/oneoff/ic12.standing.ddp @@ -9,7 +9,7 @@ let mtag = import "graph::mtag.by_tag" : (int, int); let selves = tagclass | map((id, _) -> ((id, id), ())); below: { let step = below | map(((low, high), ()) -> (high, low)) - | join(tagclass, (_, low, (_, parent)) -> (low, parent)) + | join(tagclass, (_, (low, (_, parent))) -> (low, parent)) | filter((_, parent) -> (parent, 0) ge) | map((low, higher) -> ((low, higher), ())); set below = selves + step | distinct; diff --git a/interactive/bench/ldbc/interactive_v2/oneoff/ic13.oneoff.ddp b/interactive/bench/ldbc/interactive_v2/oneoff/ic13.oneoff.ddp index 38d6d8062..2f149a9ac 100644 --- a/interactive/bench/ldbc/interactive_v2/oneoff/ic13.oneoff.ddp +++ b/interactive/bench/ldbc/interactive_v2/oneoff/ic13.oneoff.ddp @@ -6,25 +6,25 @@ search: { let side0 = search | filter(((_, side, _, _), ()) -> (side, 0) eq) | map(((rid, _, node, d), ()) -> ((rid, node), d)); let side1 = search | filter(((_, side, _, _), ()) -> (side, 1) eq) | map(((rid, _, node, d), ()) -> ((rid, node), d)); - let best = side0 | join(side1, ((rid, _), d0, d1) -> (rid, (d0, d1) add)) | min; + let best = side0 | join(side1, ((rid, _), (d0, d1)) -> (rid, (d0, d1) add)) | min; let by_rid = search | map(((rid, side, node, d), ()) -> (rid, (side, node, d))); let met = best | map((rid, _) -> (rid, ())); - let open = by_rid - (by_rid | join(met, (rid, row, ()) -> (rid, row))); - let near = by_rid | join(best, (rid, (side, node, d), b) -> (rid, (side, node, d, b))) + let open = by_rid - (by_rid | join(met, (rid, (row, ())) -> (rid, row))); + let near = by_rid | join(best, (rid, ((side, node, d), b)) -> (rid, (side, node, d, b))) | filter((_, (_, _, d, b)) -> ((d, d) add, b) lt) | map((rid, (side, node, d, _)) -> (rid, (side, node, d))); let grow = (open + near) | map((rid, (side, node, d)) -> (node, (rid, side, d))) - | join(friend, (_, (rid, side, d), (next, _)) -> ((rid, side, next, (d, 1) add), ())); + | join(friend, (_, ((rid, side, d), (next, _))) -> ((rid, side, next, (d, 1) add), ())); set search = (starts + search + grow) | map(((rid, side, node, d), ()) -> ((rid, side, node), d)) | min | map(((rid, side, node), d) -> ((rid, side, node, d), ())); } let side0 = search::search | filter(((_, side, _, _), ()) -> (side, 0) eq) | map(((rid, _, node, d), ()) -> ((rid, node), d)); let side1 = search::search | filter(((_, side, _, _), ()) -> (side, 1) eq) | map(((rid, _, node, d), ()) -> ((rid, node), d)); -let best = side0 | join(side1, ((rid, _), d0, d1) -> (rid, (d0, d1) add)) | min; +let best = side0 | join(side1, ((rid, _), (d0, d1)) -> (rid, (d0, d1) add)) | min; let asked = request | map(((rid, _, _), ()) -> (rid, ())); -let never = asked - (asked | join(best, (rid, (), _) -> (rid, ()))); +let never = asked - (asked | join(best, (rid, ((), _)) -> (rid, ()))); let answer = (best | map((rid, d) -> ((rid, 0), (d)))) + (never | map((rid, ()) -> ((rid, 0), (-1)))); output "answer" = answer | arrange; diff --git a/interactive/bench/ldbc/interactive_v2/oneoff/ic14.oneoff.ddp b/interactive/bench/ldbc/interactive_v2/oneoff/ic14.oneoff.ddp index 40c3cba8b..a2ee15c66 100644 --- a/interactive/bench/ldbc/interactive_v2/oneoff/ic14.oneoff.ddp +++ b/interactive/bench/ldbc/interactive_v2/oneoff/ic14.oneoff.ddp @@ -5,28 +5,28 @@ let weighted = import "ic14s::weighted" : (int, (int, int)); let starts = request | map(((rid, from, _), ()) -> ((rid, from), 0)); dist: { let grow = dist | map(((rid, node, cost), ()) -> (node, (rid, cost))) - | join(weighted, (_, (rid, cost), (next, w)) -> ((rid, next), (cost, w) add)); + | join(weighted, (_, ((rid, cost), (next, w))) -> ((rid, next), (cost, w) add)); set dist = (starts + grow) | min | map(((rid, node), cost) -> ((rid, node, cost), ())); } let best = dist::dist | map(((rid, node, cost), ()) -> ((rid, node), cost)); let pred = dist::dist | map(((rid, u, cost), ()) -> (u, (rid, cost))) - | join(weighted, (u, (rid, cost), (v, w)) -> ((rid, v), (u, (cost, w) add))) - | join(best, ((rid, v), (u, via), least) -> ((rid, v), (u, via, least))) + | join(weighted, (u, ((rid, cost), (v, w))) -> ((rid, v), (u, (cost, w) add))) + | join(best, ((rid, v), ((u, via), least)) -> ((rid, v), (u, via, least))) | filter((_, (_, via, least)) -> (via, least) eq) | map((key, (u, _, _)) -> (key, u)); -let targets = request | map(((rid, _, to), ()) -> ((rid, to), ())) | join(best, (key, (), _) -> (key, ())); +let targets = request | map(((rid, _, to), ()) -> ((rid, to), ())) | join(best, (key, ((), _)) -> (key, ())); back: { - let up = back | join(pred, ((rid, _), (), u) -> ((rid, u), ())); + let up = back | join(pred, ((rid, _), ((), u)) -> ((rid, u), ())); set back = (targets + up) | distinct; } -let next = pred | join(back::back, ((rid, v), u, ()) -> ((rid, u), v)) | min; -let first = request | map(((rid, from, _), ()) -> ((rid, from), ())) | join(back::back, ((rid, from), (), ()) -> ((rid, from, from enlist), ())); +let next = pred | join(back::back, ((rid, v), (u, ())) -> ((rid, u), v)) | min; +let first = request | map(((rid, from, _), ()) -> ((rid, from), ())) | join(back::back, ((rid, from), ((), ())) -> ((rid, from, from enlist), ())); walk: { let step = walk | map(((rid, node, path), ()) -> ((rid, node), path)) - | join(next, ((rid, _), path, v) -> ((rid, v, (path, v enlist) append), ())); + | join(next, ((rid, _), (path, v)) -> ((rid, v, (path, v enlist) append), ())); set walk = first + step; } let answer = walk::walk | map(((rid, node, path), ()) -> ((rid, node), path)) - | join(targets, ((rid, node), path, ()) -> ((rid, node), path)) - | join(best, ((rid, _), path, cost) -> ((rid, 0), (path, cost))); + | join(targets, ((rid, node), (path, ())) -> ((rid, node), path)) + | join(best, ((rid, _), (path, cost)) -> ((rid, 0), (path, cost))); output "answer" = answer | arrange; diff --git a/interactive/bench/ldbc/interactive_v2/oneoff/ic14.standing.ddp b/interactive/bench/ldbc/interactive_v2/oneoff/ic14.standing.ddp index cc784fcda..093ee18ed 100644 --- a/interactive/bench/ldbc/interactive_v2/oneoff/ic14.standing.ddp +++ b/interactive/bench/ldbc/interactive_v2/oneoff/ic14.standing.ddp @@ -2,11 +2,11 @@ let message = import "graph::message" : (int, (int, int, int, int, List(int), Li let pals = import "graph::friend.pair" : ((int, int), ()); let answered = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 1) eq) | map((_, (_, _, who, _, _, _, _, _, _, parent, _, _, _)) -> (parent, who)) - | join(message, (_, who, (_, _, to, _, _, _, _, _, _, _, _, _, _)) -> (who, to)) + | join(message, (_, (who, (_, _, to, _, _, _, _, _, _, _, _, _, _))) -> (who, to)) | filter((who, to) -> (who, to) ne); let talks = (answered | map((a, b) -> ((a, b), ()))) + (answered | map((a, b) -> ((b, a), ()))); let weighted = talks | count - | join(pals, (pair, n, ()) -> (pair, n)) + | join(pals, (pair, (n, ())) -> (pair, n)) | map(((a, b), n) -> (a, (b, (((0, n), (1, 40) range) fold (((acc, n), k) -> ((acc, ((((k, (k, 1) sub) mul, 1) add, n) gt, 1, 0) select) add, n)) .0, 1) add))); output "weighted" = weighted | arrange; diff --git a/interactive/bench/ldbc/is3.ddp b/interactive/bench/ldbc/is3.ddp index 9d7ca740c..a7218940f 100644 --- a/interactive/bench/ldbc/is3.ddp +++ b/interactive/bench/ldbc/is3.ddp @@ -3,7 +3,7 @@ let request = input 0 : ((int, int), ()); let edge = import "graph::edge" : (int, (int, int)); let person = import "graph::person" : (int, (List(int), List(int), int)); let friends = request | map(((rid, pid), ()) -> (pid, rid)) - | join(edge, (_, rid, (q, created)) -> (q, (rid, created))) - | join(person, (q, (rid, created), (first, last, _)) -> (rid, (created neg, q, first, last))); + | join(edge, (_, (rid, (q, created))) -> (q, (rid, created))) + | join(person, (q, ((rid, created), (first, last, _))) -> (rid, (created neg, q, first, last))); output "answer" = friends | collect | flatmap((k, vs) -> (k, ((0, vs len) range, vs) zip) cap_list) | map((rid, (pos, (negc, q, first, last))) -> ((rid, pos), (q, first, last, negc neg))); diff --git a/interactive/bench/ldbc/snb/indexes.py b/interactive/bench/ldbc/snb/indexes.py index 2f3879a62..ee4489e8c 100644 --- a/interactive/bench/ldbc/snb/indexes.py +++ b/interactive/bench/ldbc/snb/indexes.py @@ -96,8 +96,8 @@ def graph_program(): lines.append('''-- Friendships in both directions, between people who exist: (person, (friend, since)). let exists = person | map(((id, _, _, _, _, _, _, _, _, _, _, _), ()) -> (id, ())) | distinct; let either = (knows | map(((a, b, t), ()) -> (a, (b, t)))) + (knows | map(((a, b, t), ()) -> (b, (a, t)))); -let friends = either | join(exists, (a, (b, t), ()) -> (b, (a, t))) - | join(exists, (b, (a, t), ()) -> ((a, b, t), ())) | distinct +let friends = either | join(exists, (a, ((b, t), ())) -> (b, (a, t))) + | join(exists, (b, ((a, t), ())) -> ((a, b, t), ())) | distinct | map(((a, b, t), ()) -> (a, (b, t))); output "friend" = friends | arrange; output "friend.pair" = friends | map((a, (b, _)) -> ((a, b), ())) | arrange; @@ -108,7 +108,7 @@ def graph_program(): | map(((id, _, _, _, _, _, _, _, _, _, parent, _, _, _), ()) -> (parent, id)); thread: { let below = thread | map(((mid, root), ()) -> (mid, root)) - | join(replies, (_, root, child) -> ((child, root), ())); + | join(replies, (_, (root, child)) -> ((child, root), ())); set thread = (posts + below) | distinct; } output "message.root" = thread::thread | map(((mid, root), ()) -> (mid, root)) | arrange; diff --git a/interactive/bench/ldbc/snb/rel.py b/interactive/bench/ldbc/snb/rel.py index 03f4dc5ed..b6c7abd0c 100644 --- a/interactive/bench/ldbc/snb/rel.py +++ b/interactive/bench/ldbc/snb/rel.py @@ -327,7 +327,7 @@ def keyed(source, fields, keys, keep=True): else: keyed_right = f'({keyed(rhs,right.fields,on.values(),False)} | distinct)' out = 'left' - matched = f'{keyed_left} | join({keyed_right}, (_, left, right) -> ({out}, ()))' + matched = f'{keyed_left} | join({keyed_right}, (_, (left, right)) -> ({out}, ()))' code = f'{l} - ({matched})' if op=='anti' else matched elif op == 'group': source,keys,aggs = args @@ -355,7 +355,7 @@ def keyed(source, fields, keys, keep=True): if kind == 'sum': # count omits zero totals, but SQL-style grouping keeps # a zero sum while the input group remains nonempty. - missing = f'({groups} - ({groups} | join({aggregate}, (key, _, _) -> (key, ()))))' + missing = f'({groups} - ({groups} | join({aggregate}, (key, (_, _)) -> (key, ()))))' present = aggregate+'_present' self.lines.append(f'let {present} = {aggregate} + ({missing} | map((key, ()) -> (key, 0)));') aggregate = present @@ -364,7 +364,7 @@ def keyed(source, fields, keys, keep=True): code = f'{aggregate} | map((key, total) -> (key, (total)))' else: values = product([*(f'left.{j}' for j in range(i)), 'right']) - code = f'{joined} | join({aggregate}, (key, left, right) -> (key, {values}))' + code = f'{joined} | join({aggregate}, (key, (left, right)) -> (key, {values}))' self.lines.append(f'let {stage} = {code};') joined = stage fields = product([*outs, *(f'totals.{i}' for i in range(len(aggs)))]) @@ -424,7 +424,7 @@ def keyed(source, fields, keys, keep=True): combined = list(seed.fields)+['next_'+x for x in nextrel.fields] joined = product(refs[x] for x in combined) step_refs = positions(combined) - self.lines.append(f'{name}_loop: {{\n let proposals = {keyed("state",seed.fields,on)} | join({keyed(n,nextrel.fields,on.values())}, (_, left, right) -> ({joined}, ())) | filter((row, ()) -> {scalar(condition,step_refs)}) | map((row, ()) -> ({projection(select.values(),step_refs)}, ()));\n set state = ({s} + proposals) | map((row, ()) -> ({k}, {b})) | min | map((key, best) -> ({product(restore[x] for x in seed.fields)}, ()));\n}}') + self.lines.append(f'{name}_loop: {{\n let proposals = {keyed("state",seed.fields,on)} | join({keyed(n,nextrel.fields,on.values())}, (_, (left, right)) -> ({joined}, ())) | filter((row, ()) -> {scalar(condition,step_refs)}) | map((row, ()) -> ({projection(select.values(),step_refs)}, ()));\n set state = ({s} + proposals) | map((row, ()) -> ({k}, {b})) | min | map((key, best) -> ({product(restore[x] for x in seed.fields)}, ()));\n}}') code = f'{name}_loop::state' else: raise ValueError(op) self.lines.append(f'-- {name}: ({", ".join(r.fields)})\nlet {name} = {code};') diff --git a/interactive/bench/txn/maintained.ddp b/interactive/bench/txn/maintained.ddp index 8c3831129..0f9b061b4 100644 --- a/interactive/bench/txn/maintained.ddp +++ b/interactive/bench/txn/maintained.ddp @@ -20,13 +20,13 @@ let wid = wr_all | map(((wid, k), v) -> (wid, (k, v))); res: { -- writes of transactions not (currently) rolled back: anti-join on id. - let wr_bad = wid | join(rollback, (wid, kv, ()) -> (wid, kv)); + let wr_bad = wid | join(rollback, (wid, (kv, ())) -> (wid, kv)); let writes = wid + (wr_bad | negate); -- each read paired with every surviving prior write to its key ... let wk = writes | map((wid, (k, v)) -> (k, (wid, v))); let rk = reads | map(((rid, k), e) -> (k, (rid, e))); - let pairs = rk | join(wk, (k, (rid, e), (wid, v)) -> ((k, rid, e), (wid, v))) + let pairs = rk | join(wk, (k, ((rid, e), (wid, v))) -> ((k, rid, e), (wid, v))) | filter(((_, rid, _), (wid, _)) -> (wid, rid) lt); -- ... and the LATEST prior write wins: min over (-wid, v). let best = pairs | map((kre, (wid, v)) -> (kre, (wid neg, v))) | min; @@ -38,14 +38,14 @@ res: { let seen = best | map(((k, rid, _), _) -> ((rid, k), ())); let rkk = reads | map((rk, _) -> (rk, ())); let unseen = rkk + (seen | negate); - let bad_absent = unseen | join(reads, ((rid, _), (), e) -> ((rid, e), ())) + let bad_absent = unseen | join(reads, ((rid, _), ((), e)) -> ((rid, e), ())) | filter(((_, e), ()) -> (e, -1) ne) | map(((rid, _), ()) -> (rid, ())); set rollback = (bad_seen + bad_absent) | distinct; } let rb = res::rollback; -let wbad = wid | join(rb, (wid, kv, ()) -> (wid, kv)); +let wbad = wid | join(rb, (wid, (kv, ())) -> (wid, kv)); let committed = wid + (wbad | negate); -- ((k), (-wid, v)): each key's latest committed write. diff --git a/interactive/bench/txn/resolve.ddp b/interactive/bench/txn/resolve.ddp index 83251e78e..837d29012 100644 --- a/interactive/bench/txn/resolve.ddp +++ b/interactive/bench/txn/resolve.ddp @@ -20,17 +20,17 @@ let wid = wr_all | map(((wid, k), v) -> (wid, (k, v))); let rk = reads | map(((rid, k), e) -> (k, (rid, e))); -- Each read's store row: a prior write older than any of this epoch's. -let base = rk | join(store, (k, (rid, e), (sw, v)) -> ((k, rid, e), (sw neg, v))); +let base = rk | join(store, (k, ((rid, e), (sw, v))) -> ((k, rid, e), (sw neg, v))); res: { -- this epoch's writes of transactions not (currently) rolled back. - let wr_bad = wid | join(rollback, (wid, kv, ()) -> (wid, kv)); + let wr_bad = wid | join(rollback, (wid, (kv, ())) -> (wid, kv)); let writes = wid + (wr_bad | negate); -- each read paired with every surviving prior write to its key, the store's -- among them; the LATEST wins: min over (-wid, v). let wk = writes | map((wid, (k, v)) -> (k, (wid, v))); - let pairs = rk | join(wk, (k, (rid, e), (wid, v)) -> ((k, rid, e), (wid neg, v))) + let pairs = rk | join(wk, (k, ((rid, e), (wid, v))) -> ((k, rid, e), (wid neg, v))) | filter(((_, rid, _), (nw, _)) -> (nw neg, rid) lt); let best = (pairs + base) | min; @@ -41,20 +41,20 @@ res: { let seen = best | map(((k, rid, _), _) -> ((rid, k), ())); let rkk = reads | map((rk, _) -> (rk, ())); let unseen = rkk + (seen | negate); - let bad_absent = unseen | join(reads, ((rid, _), (), e) -> ((rid, e), ())) + let bad_absent = unseen | join(reads, ((rid, _), ((), e)) -> ((rid, e), ())) | filter(((_, e), ()) -> (e, -1) ne) | map(((rid, _), ()) -> (rid, ())); set rollback = (bad_seen + bad_absent) | distinct; } let rb = res::rollback; -let wbad = wid | join(rb, (wid, kv, ()) -> (wid, kv)); +let wbad = wid | join(rb, (wid, (kv, ())) -> (wid, kv)); let committed = wid + (wbad | negate); -- ((k), (-wid, v)): each written key's latest committed write this epoch. let latest = committed | map((wid, (k, v)) -> (k, (wid neg, v))) | min; let new_rows = latest | map((k, (nw, v)) -> (k, (nw neg, v))); -let old_rows = latest | join(store, (k, _, (sw, sv)) -> (k, (sw, sv))); +let old_rows = latest | join(store, (k, (_, (sw, sv))) -> (k, (sw, sv))); output "rollback" = rb; output "delta" = new_rows + (old_rows | negate); diff --git a/interactive/bench/txn/resolve_by_key.ddp b/interactive/bench/txn/resolve_by_key.ddp index a028c9655..7be348b37 100644 --- a/interactive/bench/txn/resolve_by_key.ddp +++ b/interactive/bench/txn/resolve_by_key.ddp @@ -14,10 +14,10 @@ let wid = wr_all | map(((wid, k), v) -> (wid, (k, v))); -- ((k), (tag, id, value)): tag 1 a read and its expected value, tag 0 a write. let read_entries = reads | map(((rid, k), e) -> (k, (1, rid, e))); -- The store row of each key read, as a write older than any of this epoch's. -let store_entries = read_entries | join(store, (k, _, (sw, v)) -> (k, (0, sw, v))); +let store_entries = read_entries | join(store, (k, (_, (sw, v))) -> (k, (0, sw, v))); res: { - let wr_bad = wid | join(rollback, (wid, kv, ()) -> (wid, kv)); + let wr_bad = wid | join(rollback, (wid, (kv, ())) -> (wid, kv)); let writes = wid + (wr_bad | negate); let write_entries = writes | map((wid, (k, v)) -> (k, (0, wid, v))); @@ -37,12 +37,12 @@ res: { } let rb = res::rollback; -let wbad = wid | join(rb, (wid, kv, ()) -> (wid, kv)); +let wbad = wid | join(rb, (wid, (kv, ())) -> (wid, kv)); let committed = wid + (wbad | negate); let latest = committed | map((wid, (k, v)) -> (k, (wid neg, v))) | min; let new_rows = latest | map((k, (nw, v)) -> (k, (nw neg, v))); -let old_rows = latest | join(store, (k, _, (sw, sv)) -> (k, (sw, sv))); +let old_rows = latest | join(store, (k, (_, (sw, sv))) -> (k, (sw, sv))); output "rollback" = rb; output "delta" = new_rows + (old_rows | negate); diff --git a/interactive/examples/aoc2023/day01/part1.ddp b/interactive/examples/aoc2023/day01/part1.ddp index 5199d6c5c..642e08e63 100644 --- a/interactive/examples/aoc2023/day01/part1.ddp +++ b/interactive/examples/aoc2023/day01/part1.ddp @@ -9,7 +9,7 @@ let digits = input 0 : ((int, int, int), ()) let first = digits | min | map((line, (_, d)) -> (line, d)); let last = digits | map((line, (pos, d)) -> (line, (pos neg, d))) | min | map((line, (_, d)) -> (line, d)); -let calib = first | join(last, (line, f, l) -> (line, ((f, 10) mul, l) add)); +let calib = first | join(last, (line, (f, l)) -> (line, ((f, 10) mul, l) add)); -- (line, value) output "result" = calib diff --git a/interactive/examples/aoc2023/day01/part2.ddp b/interactive/examples/aoc2023/day01/part2.ddp index 1c11b8e17..9b2e7f3aa 100644 --- a/interactive/examples/aoc2023/day01/part2.ddp +++ b/interactive/examples/aoc2023/day01/part2.ddp @@ -7,13 +7,13 @@ let chars = input 0 : ((int, int, int), ()) | map(((line, pos, c), _) -> ((line, -- ((line, pos), code) let s2 = chars | map(((line, p), c0) -> ((line, (p, 1) add), (p, c0))) - | join(chars, ((line, _), (p, c0), c1) -> ((line, p), (c0, c1))); + | join(chars, ((line, _), ((p, c0), c1)) -> ((line, p), (c0, c1))); let s3 = s2 | map(((line, p), (c0, c1)) -> ((line, (p, 2) add), (p, c0, c1))) - | join(chars, ((line, _), (p, c0, c1), c2) -> ((line, p), (c0, c1, c2))); + | join(chars, ((line, _), ((p, c0, c1), c2)) -> ((line, p), (c0, c1, c2))); let s4 = s3 | map(((line, p), (c0, c1, c2)) -> ((line, (p, 3) add), (p, c0, c1, c2))) - | join(chars, ((line, _), (p, c0, c1, c2), c3) -> ((line, p), (c0, c1, c2, c3))); + | join(chars, ((line, _), ((p, c0, c1, c2), c3)) -> ((line, p), (c0, c1, c2, c3))); let s5 = s4 | map(((line, p), (c0, c1, c2, c3)) -> ((line, (p, 4) add), (p, c0, c1, c2, c3))) - | join(chars, ((line, _), (p, c0, c1, c2, c3), c4) -> ((line, p), (c0, c1, c2, c3, c4))); + | join(chars, ((line, _), ((p, c0, c1, c2, c3), c4)) -> ((line, p), (c0, c1, c2, c3, c4))); -- sk: ((line, pos), c0..c{k-1}) let digits = chars | filter((_, c) -> ((c, 48) ge, (c, 57) le) all) @@ -35,7 +35,7 @@ let findings = digits + w_zero + w_one + w_two + w_three + w_four + w_five + w_s let first = findings | min | map((line, (_, d)) -> (line, d)); let last = findings | map((line, (pos, d)) -> (line, (pos neg, d))) | min | map((line, (_, d)) -> (line, d)); -let calib = first | join(last, (line, f, l) -> (line, ((f, 10) mul, l) add)); +let calib = first | join(last, (line, (f, l)) -> (line, ((f, 10) mul, l) add)); output "result" = calib | map((_, v) -> ((), v)) | collect diff --git a/interactive/examples/aoc2023/day02/part2.ddp b/interactive/examples/aoc2023/day02/part2.ddp index 7fc3aec0e..ed1938968 100644 --- a/interactive/examples/aoc2023/day02/part2.ddp +++ b/interactive/examples/aoc2023/day02/part2.ddp @@ -8,8 +8,8 @@ let maxr = sets | map(((game, _, _, r, _), _) -> (game, (1000000, r) sub)) | min let maxb = sets | map(((game, _, _, _, b), _) -> (game, (1000000, b) sub)) | min | map((game, x) -> (game, (1000000, x) sub)); let power = maxg - | join(maxr, (game, a, b) -> (game, (a, b) mul)) - | join(maxb, (game, a, b) -> (game, (a, b) mul)); + | join(maxr, (game, (a, b)) -> (game, (a, b) mul)) + | join(maxb, (game, (a, b)) -> (game, (a, b) mul)); output "result" = power | map((_, p) -> ((), p)) | collect diff --git a/interactive/examples/aoc2023/day03/part1.ddp b/interactive/examples/aoc2023/day03/part1.ddp index 25ca92f0e..64a9e81c4 100644 --- a/interactive/examples/aoc2023/day03/part1.ddp +++ b/interactive/examples/aoc2023/day03/part1.ddp @@ -7,23 +7,23 @@ let numerals = cells | filter((_, c) -> ((c, 48) ge, (c, 57) le) all) | map((cel let symbols = cells | filter((_, c) -> (((c, 48) ge, (c, 57) le) all, 0) eq); -- run starts: digit cells with no digit to the left -let has_left = numerals | map(((r, col), d) -> ((r, (col, 1) add), d)) | join(numerals, (cell, _, d) -> (cell, d)); +let has_left = numerals | map(((r, col), d) -> ((r, (col, 1) add), d)) | join(numerals, (cell, (_, d)) -> (cell, d)); let starts = numerals - has_left; -- grow runs rightward: key (row, start, end_exclusive, value) runs: { let grow = run | map(((r, s, e, v), _) -> ((r, e), (s, v))) - | join(numerals, ((r, e), (s, v), d) -> ((r, s, (e, 1) add, ((v, 10) mul, d) add), ())); + | join(numerals, ((r, e), ((s, v), d)) -> ((r, s, (e, 1) add, ((v, 10) mul, d) add), ())); set run = (starts | map(((r, col), d) -> ((r, col, (col, 1) add, d), ()))) + grow | distinct; } let extendable = runs::run | map(((r, s, e, v), _) -> ((r, e), (s, v))) - | join(numerals, ((r, e), (s, v), _) -> ((r, s, e, v), ())); + | join(numerals, ((r, e), ((s, v), _)) -> ((r, s, e, v), ())); let maximal = runs::run - extendable; -- key (row, start, end, value) -- label each digit cell with its run start: key (row, col, start) labels: { - let ext = label | map(((r, col, s), _) -> ((r, (col, 1) add), s)) | join(numerals, ((r, col), s, _) -> ((r, col, s), ())); + let ext = label | map(((r, col, s), _) -> ((r, (col, 1) add), s)) | join(numerals, ((r, col), (s, _)) -> ((r, col, s), ())); set label = (starts | map(((r, col), _) -> ((r, col, col), ()))) + ext | distinct; } @@ -32,10 +32,10 @@ let symnbrs = symbols | flatmap(((r, col), _) -> ((r, col), [(-1, -1), (-1, 0), (-1, 1), (0, -1), (0, 0), (0, 1), (1, -1), (1, 0), (1, 1)]) cap_list map (((r, col), (dr, dc)) -> (((r, dr) add, (col, dc) add), ()))) | distinct; let active_runs = labels::label - | map(((r, col, s), _) -> ((r, col), s)) | join(symnbrs, ((r, _), s, _) -> ((r, s), ())) | distinct; + | map(((r, col, s), _) -> ((r, col), s)) | join(symnbrs, ((r, _), (s, _)) -> ((r, s), ())) | distinct; -- key (row, start) of runs adjacent to a symbol -let part_numbers = maximal | map(((r, s, _, v), _) -> ((r, s), v)) | join(active_runs, (run, v, _) -> (run, v)); +let part_numbers = maximal | map(((r, s, _, v), _) -> ((r, s), v)) | join(active_runs, (run, (v, _)) -> (run, v)); -- ((row, start), value) output "result" = part_numbers diff --git a/interactive/examples/aoc2023/day03/part2.ddp b/interactive/examples/aoc2023/day03/part2.ddp index 8fb5dd723..f3dfcad62 100644 --- a/interactive/examples/aoc2023/day03/part2.ddp +++ b/interactive/examples/aoc2023/day03/part2.ddp @@ -6,22 +6,22 @@ let cells = input 0 : ((int, int, int), ()) | filter(((_, _, c), _) -> (c, 46 let numerals = cells | filter((_, c) -> ((c, 48) ge, (c, 57) le) all) | map((cell, c) -> (cell, (c, 48) sub)); let gears = cells | filter((_, c) -> (c, 42) eq); -let has_left = numerals | map(((r, col), d) -> ((r, (col, 1) add), d)) | join(numerals, (cell, _, d) -> (cell, d)); +let has_left = numerals | map(((r, col), d) -> ((r, (col, 1) add), d)) | join(numerals, (cell, (_, d)) -> (cell, d)); let starts = numerals - has_left; runs: { let grow = run | map(((r, s, e, v), _) -> ((r, e), (s, v))) - | join(numerals, ((r, e), (s, v), d) -> ((r, s, (e, 1) add, ((v, 10) mul, d) add), ())); + | join(numerals, ((r, e), ((s, v), d)) -> ((r, s, (e, 1) add, ((v, 10) mul, d) add), ())); set run = (starts | map(((r, col), d) -> ((r, col, (col, 1) add, d), ()))) + grow | distinct; } let extendable = runs::run | map(((r, s, e, v), _) -> ((r, e), (s, v))) - | join(numerals, ((r, e), (s, v), _) -> ((r, s, e, v), ())); + | join(numerals, ((r, e), ((s, v), _)) -> ((r, s, e, v), ())); let maximal = runs::run - extendable; let partvals = maximal | map(((r, s, _, v), _) -> ((r, s), v)); -- ((row, start), value) labels: { - let ext = label | map(((r, col, s), _) -> ((r, (col, 1) add), s)) | join(numerals, ((r, col), s, _) -> ((r, col, s), ())); + let ext = label | map(((r, col, s), _) -> ((r, (col, 1) add), s)) | join(numerals, ((r, col), (s, _)) -> ((r, col, s), ())); set label = (starts | map(((r, col), _) -> ((r, col, col), ()))) + ext | distinct; } let labelcells = labels::label | map(((r, col, s), _) -> ((r, col), s)); @@ -31,13 +31,13 @@ let gearnbrs = gears -- (nbr_cell, gear) let pairs = gearnbrs - | join(labelcells, ((pr, _), (gr, gc), ps) -> ((gr, gc, pr, ps), ())) + | join(labelcells, ((pr, _), ((gr, gc), ps)) -> ((gr, gc, pr, ps), ())) | distinct; -- key (gear_r, gear_c, part_row, part_start) let gearvals = pairs | map(((gr, gc, pr, ps), _) -> ((pr, ps), (gr, gc))) - | join(partvals, (_, gear, v) -> (gear, v)); + | join(partvals, (_, (gear, v)) -> (gear, v)); -- (gear, part value) output "result" = gearvals diff --git a/interactive/examples/aoc2023/day04/part2.ddp b/interactive/examples/aoc2023/day04/part2.ddp index a7a703c85..6067f46e6 100644 --- a/interactive/examples/aoc2023/day04/part2.ddp +++ b/interactive/examples/aoc2023/day04/part2.ddp @@ -5,14 +5,14 @@ let numbers = input 0 : ((int, int, int, int), ()); let winners = numbers | filter(((_, side, _, _), _) -> (side, 0) eq) | map(((card, _, _, value), _) -> ((card, value), ())); let ours = numbers | filter(((_, side, _, _), _) -> (side, 1) eq) | map(((card, _, _, value), _) -> ((card, value), ())); -let matches = ours | join(winners, (k, _, _) -> (k, ())) | map(((card, _), _) -> (card, ())) | count; +let matches = ours | join(winners, (k, (_, _)) -> (k, ())) | map(((card, _), _) -> (card, ())) | count; -- (card, m): number of matching (ours, winners) pairs let cards = numbers | map(((card, _, _, _), _) -> (card, ())) | distinct; -- prize targets: key (card, target) for target in card+1 .. card+m wt: { - let step = t | map(((card, target), _) -> (card, target)) | join(matches, (card, target, m) -> ((card, (target, 1) add, m), ())) + let step = t | map(((card, target), _) -> (card, target)) | join(matches, (card, (target, m)) -> ((card, (target, 1) add, m), ())) | filter(((card, target, m), _) -> (target, (card, m) add) le) | map(((card, target, _), _) -> ((card, target), ())); set t = (matches | filter((_, m) -> (m, 1) ge) | map((card, _) -> ((card, (card, 1) add), ()))) + step | distinct; } @@ -20,7 +20,7 @@ let wtflat = wt::t | map(((card, target), _) -> (card, target)); -- instances: multiplicity of (card, ()) = copies of that card insts: { - let spawned = inst | join(wtflat, (_, _, target) -> (target, ())); + let spawned = inst | join(wtflat, (_, (_, target)) -> (target, ())); set inst = cards + spawned | arrange; } diff --git a/interactive/examples/aoc2023/day05/part1.ddp b/interactive/examples/aoc2023/day05/part1.ddp index 3ede2b7ac..e776d3408 100644 --- a/interactive/examples/aoc2023/day05/part1.ddp +++ b/interactive/examples/aoc2023/day05/part1.ddp @@ -9,7 +9,7 @@ let entries = input 1 : ((int, int, int, int), ()) | map(((stage, dst, src, len) chain: { let cur = active | filter(((stage, _), _) -> (stage, 6) le); let cands = cur | map(((stage, idx), _) -> (stage, idx)) - | join(entries, (stage, idx, (dst, src, len)) -> ((stage, idx, dst, src, len), ())) + | join(entries, (stage, (idx, (dst, src, len))) -> ((stage, idx, dst, src, len), ())) | filter(((_, idx, _, src, len), _) -> ((src, idx) le, (idx, (src, len) add) lt) all); -- key (stage, idx, dst, src, len) let mapped = cands | map(((stage, idx, dst, src, _), _) -> (((stage, 1) add, ((idx, dst) add, src) sub), ())); diff --git a/interactive/examples/aoc2023/day05/part2.ddp b/interactive/examples/aoc2023/day05/part2.ddp index f78bb4614..e595d8473 100644 --- a/interactive/examples/aoc2023/day05/part2.ddp +++ b/interactive/examples/aoc2023/day05/part2.ddp @@ -11,7 +11,7 @@ ranges: { -- key (stage, s, e) let isect = cur | map(((stage, s, e), _) -> (stage, (s, e))) - | join(entries, (stage, (s, e), (dst, src, len)) -> ((stage, s, e, dst, src, len), ())) + | join(entries, (stage, ((s, e), (dst, src, len))) -> ((stage, s, e, dst, src, len), ())) | map(((stage, s, e, dst, src, len), _) -> ((stage, s, e, ((s, src) gt, s, src) select, ((e, (src, len) add) lt, e, (src, len) add) select, @@ -30,7 +30,7 @@ ranges: { -- (I, cs) let pairs = cand | map(((stage, s, e, h), _) -> ((stage, s, e), h)) - | join(csets, ((stage, s, e), h, cs) -> ((stage, s, e, h), cs)) + | join(csets, ((stage, s, e), (h, cs)) -> ((stage, s, e, h), cs)) | filter(((_, _, _, h), cs) -> (cs, h) ge); -- ((I, h), cs) for cs >= h diff --git a/interactive/examples/aoc2023/day06/part1.ddp b/interactive/examples/aoc2023/day06/part1.ddp index 812742f0e..4bc672d0d 100644 --- a/interactive/examples/aoc2023/day06/part1.ddp +++ b/interactive/examples/aoc2023/day06/part1.ddp @@ -13,7 +13,7 @@ cands: { let wins = cands::c | map(((t, h), ()) -> (t, h)) - | join(races, (t, h, d) -> ((t, h, d), ())) + | join(races, (t, (h, d)) -> ((t, h, d), ())) | filter(((t, h, d), ()) -> ((h, (t, h) sub) mul, d) gt) | map(((t, _, _), ()) -> (t, ())) | count; -- (time, n_wins) diff --git a/interactive/examples/aoc2023/day07/part1.ddp b/interactive/examples/aoc2023/day07/part1.ddp index 0f8fc5025..a670bad1e 100644 --- a/interactive/examples/aoc2023/day07/part1.ddp +++ b/interactive/examples/aoc2023/day07/part1.ddp @@ -15,7 +15,7 @@ let hands = input 0 : ((int, int, int, int, int, int, int), ()) | map(((id, c1, ((), (id, t, v1, v2, v3, v4, v5, bid))); -- ((), (id, t, v1..v5, bid)) -let less = hands | join(hands, (_, (_, l1, l2, l3, l4, l5, l6, _), (rid, r1, r2, r3, r4, r5, r6, _)) -> (( +let less = hands | join(hands, (_, ((_, l1, l2, l3, l4, l5, l6, _), (rid, r1, r2, r3, r4, r5, r6, _))) -> (( ((l1, r1) lt, 1, ((l1, r1) gt, 0, ((l2, r2) lt, 1, ((l2, r2) gt, 0, ((l3, r3) lt, 1, ((l3, r3) gt, 0, @@ -31,7 +31,7 @@ let ranks = less | filter(((m, _), ()) -> (m, 1) eq) | map(((_, id), ()) -> (id, let bids = hands | map((_, (id, _, _, _, _, _, _, bid)) -> (id, bid)); -- ranks misses the weakest hand (zero smaller); add every bid once instead. -let above = ranks | join(bids, (id, n, bid) -> (id, (n, bid) mul)); +let above = ranks | join(bids, (id, (n, bid)) -> (id, (n, bid) mul)); output "result" = above + bids | map((_, x) -> ((), x)) | collect diff --git a/interactive/examples/aoc2023/day07/part2.ddp b/interactive/examples/aoc2023/day07/part2.ddp index 50f3bf59f..fff841a82 100644 --- a/interactive/examples/aoc2023/day07/part2.ddp +++ b/interactive/examples/aoc2023/day07/part2.ddp @@ -36,7 +36,7 @@ let hands = stage | map((_, (id, pc2, j, g1, v1, v2, v3, v4, v5, bid)) -> ((), (id, ((pc2, (gj, (gj, 1) sub) mul) add, (g1, (g1, 1) sub) mul) sub, v1, v2, v3, v4, v5, bid))); -- ((), (id, type2, v1..v5, bid)) -let less = hands | join(hands, (_, (_, l1, l2, l3, l4, l5, l6, _), (rid, r1, r2, r3, r4, r5, r6, _)) -> (( +let less = hands | join(hands, (_, ((_, l1, l2, l3, l4, l5, l6, _), (rid, r1, r2, r3, r4, r5, r6, _))) -> (( ((l1, r1) lt, 1, ((l1, r1) gt, 0, ((l2, r2) lt, 1, ((l2, r2) gt, 0, ((l3, r3) lt, 1, ((l3, r3) gt, 0, @@ -47,7 +47,7 @@ let less = hands | join(hands, (_, (_, l1, l2, l3, l4, l5, l6, _), (rid, r1, r2, let ranks = less | filter(((m, _), ()) -> (m, 1) eq) | map(((_, id), ()) -> (id, ())) | count; let bids = hands | map((_, (id, _, _, _, _, _, _, bid)) -> (id, bid)); -let above = ranks | join(bids, (id, n, bid) -> (id, (n, bid) mul)); +let above = ranks | join(bids, (id, (n, bid)) -> (id, (n, bid) mul)); output "result" = above + bids | map((_, x) -> ((), x)) | collect diff --git a/interactive/examples/aoc2023/day10/part1.ddp b/interactive/examples/aoc2023/day10/part1.ddp index 4aba9e274..0810d4cb2 100644 --- a/interactive/examples/aoc2023/day10/part1.ddp +++ b/interactive/examples/aoc2023/day10/part1.ddp @@ -24,7 +24,7 @@ let symm = e1 + e2 + r1 + r2 + s4 | count | filter((_, n) -> (n, 2) eq) -- ((r1, c1), (r2, c2)), mutual links only reach: { - let next = r | join(symm, (_, (), rc) -> (rc, ())); + let next = r | join(symm, (_, ((), rc)) -> (rc, ())); set r = (scell | map(((r, c), _) -> ((r, c), ()))) + next | distinct; } diff --git a/interactive/examples/aoc2023/day10/part2.ddp b/interactive/examples/aoc2023/day10/part2.ddp index 9b947bd2d..10dec6eea 100644 --- a/interactive/examples/aoc2023/day10/part2.ddp +++ b/interactive/examples/aoc2023/day10/part2.ddp @@ -24,22 +24,22 @@ let symm = e1 + e2 + r1 + r2 + s4 | count | filter((_, n) -> (n, 2) eq) | map(((r, c, r2, c2), _) -> ((r, c), (r2, c2))); reach: { - let next = r | join(symm, (_, (), rc) -> (rc, ())); + let next = r | join(symm, (_, ((), rc)) -> (rc, ())); set r = (scell | map(((r, c), _) -> ((r, c), ()))) + next | distinct; } let loop = reach::r; -- key (r, c) of loop cells (incl. S) -- loop cells whose ORIGINAL symbol toggles parity (S resolves to '???': never) -let toggles = loop | join(symbols, (rc, _, s) -> (rc, s)) +let toggles = loop | join(symbols, (rc, (_, s)) -> (rc, s)) | filter((_, s) -> ((s, 74) eq, (s, 45) eq, (s, 124) eq, (s, 70) eq) any) | map((rc, _) -> (rc, ())); status: { - let cur = st | map(((r, c, e), _) -> ((r, c), e)) | join(symbols, (rc, e, _) -> (rc, e)); + let cur = st | map(((r, c, e), _) -> ((r, c), e)) | join(symbols, (rc, (e, _)) -> (rc, e)); -- in-grid status cells ((r, c), e) - let tog = cur | join(toggles, ((r, c), e, _) -> (((r, 1) add, (c, 1) add, (1, e) sub), ())); - let matched = cur | join(toggles, (rc, e, _) -> (rc, e)); + let tog = cur | join(toggles, ((r, c), (e, _)) -> (((r, 1) add, (c, 1) add, (1, e) sub), ())); + let matched = cur | join(toggles, (rc, (e, _)) -> (rc, e)); let nontog = (cur - matched) | map(((r, c), e) -> (((r, 1) add, (c, 1) add, e), ())); set st = (symbols | filter(((r, c), _) -> ((r, 1) eq, (c, 1) eq) any) | map(((r, c), _) -> ((r, c, 0), ()))) + tog + nontog | distinct; } @@ -49,10 +49,10 @@ let enclosed = status::st | filter(((_, _, e), _) -> (e, 1) eq) | map(((r, c, _) -- pipe = reach minus S cells with no symm out-edge (slt keeps those countable) let ssrc = symm | map((rc, _) -> (rc, ())) | distinct; let scellk = scell | map(((r, c), _) -> ((r, c), ())); -let isoS = scellk - (scellk | join(ssrc, (rc, _, _) -> (rc, ()))); +let isoS = scellk - (scellk | join(ssrc, (rc, (_, _)) -> (rc, ()))); let pipe = loop - isoS; -- set-minus needs an anti-join: bare `-` is multiset subtraction and lets -- non-enclosed pipe cells cancel unrelated enclosed cells in the count. -let enc_on_pipe = enclosed | join(pipe, (rc, _, _) -> (rc, ())); +let enc_on_pipe = enclosed | join(pipe, (rc, (_, _)) -> (rc, ())); output "result" = (enclosed - enc_on_pipe) | map(_ -> ((), ())) | count | arrange | inspect(part2); diff --git a/interactive/examples/aoc2023/day11/part1.ddp b/interactive/examples/aoc2023/day11/part1.ddp index 0c6ab425c..4ff0fd5e5 100644 --- a/interactive/examples/aoc2023/day11/part1.ddp +++ b/interactive/examples/aoc2023/day11/part1.ddp @@ -16,15 +16,15 @@ let gk = g | map(((r, c), _) -> ((), (r, c))); let rgk = row_gaps | map((r, _) -> ((), r)); let cgk = col_gaps | map((c, _) -> ((), c)); -let rcnt = gk | join(rgk, (_, (r, c), gr) -> ((r, c), gr)) | filter(((r, _), gr) -> (gr, r) lt) | map((rc, _) -> (rc, ())) | count; -let radj = (rcnt | map((rc, n) -> (rc, (1, n) mul))) + ((g - (g | join(rcnt, (rc, _, _) -> (rc, ())))) | map((rc, _) -> (rc, 0))); -let ccnt = gk | join(cgk, (_, (r, c), gc) -> ((r, c), gc)) | filter(((_, c), gc) -> (gc, c) lt) | map((rc, _) -> (rc, ())) | count; -let cadj = (ccnt | map((rc, n) -> (rc, (1, n) mul))) + ((g - (g | join(ccnt, (rc, _, _) -> (rc, ())))) | map((rc, _) -> (rc, 0))); +let rcnt = gk | join(rgk, (_, ((r, c), gr)) -> ((r, c), gr)) | filter(((r, _), gr) -> (gr, r) lt) | map((rc, _) -> (rc, ())) | count; +let radj = (rcnt | map((rc, n) -> (rc, (1, n) mul))) + ((g - (g | join(rcnt, (rc, (_, _)) -> (rc, ())))) | map((rc, _) -> (rc, 0))); +let ccnt = gk | join(cgk, (_, ((r, c), gc)) -> ((r, c), gc)) | filter(((_, c), gc) -> (gc, c) lt) | map((rc, _) -> (rc, ())) | count; +let cadj = (ccnt | map((rc, n) -> (rc, (1, n) mul))) + ((g - (g | join(ccnt, (rc, (_, _)) -> (rc, ())))) | map((rc, _) -> (rc, 0))); -let gal = radj | join(cadj, ((r, c), dr, dc) -> (((r, dr) add, (c, dc) add), ())); +let gal = radj | join(cadj, ((r, c), (dr, dc)) -> (((r, dr) add, (c, dc) add), ())); let gs = gal | map(((r, c), _) -> ((), (r, c))); -let contrib = gs | join(gs, (_, (r1, c1), (r2, c2)) -> ((), +let contrib = gs | join(gs, (_, ((r1, c1), (r2, c2))) -> ((), (((r1, r2) lt, ((r1, r2) eq, (c1, c2) lt) all) any, (((r1, r2) lt, (r2, r1) sub, (r1, r2) sub) select, ((c1, c2) lt, (c2, c1) sub, (c1, c2) sub) select) add, diff --git a/interactive/examples/aoc2023/day11/part2.ddp b/interactive/examples/aoc2023/day11/part2.ddp index 37e0296e3..d6b105815 100644 --- a/interactive/examples/aoc2023/day11/part2.ddp +++ b/interactive/examples/aoc2023/day11/part2.ddp @@ -16,15 +16,15 @@ let gk = g | map(((r, c), _) -> ((), (r, c))); let rgk = row_gaps | map((r, _) -> ((), r)); let cgk = col_gaps | map((c, _) -> ((), c)); -let rcnt = gk | join(rgk, (_, (r, c), gr) -> ((r, c), gr)) | filter(((r, _), gr) -> (gr, r) lt) | map((rc, _) -> (rc, ())) | count; -let radj = (rcnt | map((rc, n) -> (rc, (999999, n) mul))) + ((g - (g | join(rcnt, (rc, _, _) -> (rc, ())))) | map((rc, _) -> (rc, 0))); -let ccnt = gk | join(cgk, (_, (r, c), gc) -> ((r, c), gc)) | filter(((_, c), gc) -> (gc, c) lt) | map((rc, _) -> (rc, ())) | count; -let cadj = (ccnt | map((rc, n) -> (rc, (999999, n) mul))) + ((g - (g | join(ccnt, (rc, _, _) -> (rc, ())))) | map((rc, _) -> (rc, 0))); +let rcnt = gk | join(rgk, (_, ((r, c), gr)) -> ((r, c), gr)) | filter(((r, _), gr) -> (gr, r) lt) | map((rc, _) -> (rc, ())) | count; +let radj = (rcnt | map((rc, n) -> (rc, (999999, n) mul))) + ((g - (g | join(rcnt, (rc, (_, _)) -> (rc, ())))) | map((rc, _) -> (rc, 0))); +let ccnt = gk | join(cgk, (_, ((r, c), gc)) -> ((r, c), gc)) | filter(((_, c), gc) -> (gc, c) lt) | map((rc, _) -> (rc, ())) | count; +let cadj = (ccnt | map((rc, n) -> (rc, (999999, n) mul))) + ((g - (g | join(ccnt, (rc, (_, _)) -> (rc, ())))) | map((rc, _) -> (rc, 0))); -let gal = radj | join(cadj, ((r, c), dr, dc) -> (((r, dr) add, (c, dc) add), ())); +let gal = radj | join(cadj, ((r, c), (dr, dc)) -> (((r, dr) add, (c, dc) add), ())); let gs = gal | map(((r, c), _) -> ((), (r, c))); -let contrib = gs | join(gs, (_, (r1, c1), (r2, c2)) -> ((), +let contrib = gs | join(gs, (_, ((r1, c1), (r2, c2))) -> ((), (((r1, r2) lt, ((r1, r2) eq, (c1, c2) lt) all) any, (((r1, r2) lt, (r2, r1) sub, (r1, r2) sub) select, ((c1, c2) lt, (c2, c1) sub, (c1, c2) sub) select) add, diff --git a/interactive/examples/aoc2023/day13/part1.ddp b/interactive/examples/aoc2023/day13/part1.ddp index cce27eef3..386b334aa 100644 --- a/interactive/examples/aoc2023/day13/part1.ddp +++ b/interactive/examples/aoc2023/day13/part1.ddp @@ -13,11 +13,11 @@ let clines = cells | map(((b, r, c), s) -> ((b, c), (r, s))) | collect; -- row axes ------------------------------------------------------------ let rcand = cells | map(((b, r, _), _) -> ((b, r), ())) | distinct; -- (b, a) let rlk = rlines | map(((b, x), l) -> (b, (x, l))); -- (b, (x, L)) -let t1 = rcand | map(((b, a), _) -> (b, a)) | join(rlk, (b, a, (x, l)) -> ((b, a, x), l)); +let t1 = rcand | map(((b, a), _) -> (b, a)) | join(rlk, (b, (a, (x, l))) -> ((b, a, x), l)); -- ((b, a, x), Lx) let t2 = t1 | map(((b, a, x), l) -> ((b, (((2, a) mul, 1) sub, x) sub), (a, x, l))); -- ((b, y), (a, x, Lx)) -let rviol = t2 | join(rlines, ((b, y), (a, x, lx), ly) -> ((b, a), ((x, y) lt, ((lx, ly) eq, 0) eq) all)) +let rviol = t2 | join(rlines, ((b, y), ((a, x, lx), ly)) -> ((b, a), ((x, y) lt, ((lx, ly) eq, 0) eq) all)) | filter((_, m) -> (m, 1) eq) | map((k, _) -> (k, ())) | distinct; -- (b, a) with a mismatched pair let rvalid = rcand - rviol; @@ -26,9 +26,9 @@ let rscore = rvalid | map(((_, a), _) -> ((), (100, (a, 1) sub) mul)); -- column axes --------------------------------------------------------- let ccand = cells | map(((b, _, c), _) -> ((b, c), ())) | distinct; let clk = clines | map(((b, x), l) -> (b, (x, l))); -let s1 = ccand | map(((b, a), _) -> (b, a)) | join(clk, (b, a, (x, l)) -> ((b, a, x), l)); +let s1 = ccand | map(((b, a), _) -> (b, a)) | join(clk, (b, (a, (x, l))) -> ((b, a, x), l)); let s2 = s1 | map(((b, a, x), l) -> ((b, (((2, a) mul, 1) sub, x) sub), (a, x, l))); -let cviol = s2 | join(clines, ((b, y), (a, x, lx), ly) -> ((b, a), ((x, y) lt, ((lx, ly) eq, 0) eq) all)) +let cviol = s2 | join(clines, ((b, y), ((a, x, lx), ly)) -> ((b, a), ((x, y) lt, ((lx, ly) eq, 0) eq) all)) | filter((_, m) -> (m, 1) eq) | map((k, _) -> (k, ())) | distinct; let cvalid = ccand - cviol; let cscore = cvalid | map(((_, a), _) -> ((), (a, 1) sub)); diff --git a/interactive/examples/aoc2023/day13/part2.ddp b/interactive/examples/aoc2023/day13/part2.ddp index 5ea47e30b..1c1992be9 100644 --- a/interactive/examples/aoc2023/day13/part2.ddp +++ b/interactive/examples/aoc2023/day13/part2.ddp @@ -8,9 +8,9 @@ let cb = cells | map(((b, r, c), s) -> (b, (r, c, s))); -- (b, (r, c, s)) -- row axes let rcand = cells | map(((b, r, _), _) -> ((b, r), ())) | distinct; let u1 = rcand | map(((b, a), _) -> (b, a)) - | join(cb, (b, a, (x, c, s)) -> ((b, (((2, a) mul, 1) sub, x) sub, c), (a, x, s))); + | join(cb, (b, (a, (x, c, s))) -> ((b, (((2, a) mul, 1) sub, x) sub, c), (a, x, s))); -- ((b, y, c), (a, x, s)) -let rmis = u1 | join(cells, ((b, y, _), (a, x, s), s2) -> ((b, a), ((x, y) lt, ((s, s2) eq, 0) eq) all)) +let rmis = u1 | join(cells, ((b, y, _), ((a, x, s), s2)) -> ((b, a), ((x, y) lt, ((s, s2) eq, 0) eq) all)) | filter((_, m) -> (m, 1) eq) | map((k, _) -> (k, ())) | count | filter((_, n) -> (n, 1) eq); -- ((b, a), 1): exactly one smudge let rscore = rmis | map(((_, a), _) -> ((), (100, (a, 1) sub) mul)); @@ -18,9 +18,9 @@ let rscore = rmis | map(((_, a), _) -> ((), (100, (a, 1) sub) mul)); -- column axes let ccand = cells | map(((b, _, c), _) -> ((b, c), ())) | distinct; let v1 = ccand | map(((b, a), _) -> (b, a)) - | join(cb, (b, a, (r, x, s)) -> ((b, r, (((2, a) mul, 1) sub, x) sub), (a, x, s))); + | join(cb, (b, (a, (r, x, s))) -> ((b, r, (((2, a) mul, 1) sub, x) sub), (a, x, s))); -- ((b, r, y), (a, x, s)) -let cmis = v1 | join(cells, ((b, _, y), (a, x, s), s2) -> ((b, a), ((x, y) lt, ((s, s2) eq, 0) eq) all)) +let cmis = v1 | join(cells, ((b, _, y), ((a, x, s), s2)) -> ((b, a), ((x, y) lt, ((s, s2) eq, 0) eq) all)) | filter((_, m) -> (m, 1) eq) | map((k, _) -> (k, ())) | count | filter((_, n) -> (n, 1) eq); let cscore = cmis | map(((_, a), _) -> ((), (a, 1) sub)); diff --git a/interactive/examples/aoc2023/day14/part1.ddp b/interactive/examples/aoc2023/day14/part1.ddp index a2cb7014a..c352492fe 100644 --- a/interactive/examples/aoc2023/day14/part1.ddp +++ b/interactive/examples/aoc2023/day14/part1.ddp @@ -14,14 +14,14 @@ let cubcol = cubesN | map(((r, c), _) -> (c, r)); -- (c, rc) -- nearest cube above each rock let b = os | map(((r, c), _) -> (c, r)) - | join(cubcol, (c, r, rc) -> ((c, r), rc)) + | join(cubcol, (c, (r, rc)) -> ((c, r), rc)) | filter(((_, r), rc) -> (rc, r) lt) | map(((c, r), rc) -> ((r, c), (1000000, rc) sub)) | min | map((rc, v) -> (rc, (1000000, v) sub)); -- ((r, c), b) let seg = b | map(((r, c), b) -> ((c, b), r)); -- ((c, b), r) -let cnt = seg | join(seg, ((c, b), r, r2) -> ((c, b, r), r2)) +let cnt = seg | join(seg, ((c, b), (r, r2)) -> ((c, b, r), r2)) | filter(((_, _, r), r2) -> (r2, r) le) | map((k, _) -> (k, ())) | count; -- ((c, b, r), n): rank incl. self let rolled = cnt | map(((c, b, _), n) -> (((b, n) add, c), ())); -- (newr, c) @@ -29,6 +29,6 @@ let rolled = cnt | map(((c, b, _), n) -> (((b, n) add, c), ())); -- (newr, c) let maxr = cells | map(((r, _), _) -> ((), (1000000, r) sub)) | min | map((_, v) -> ((), (1000000, v) sub)); output "result" = rolled | map(((r, _), _) -> ((), r)) - | join(maxr, (_, r, m) -> ((), ((1, m) add, r) sub)) + | join(maxr, (_, (r, m)) -> ((), ((1, m) add, r) sub)) | collect | map((_, ps) -> ((), ps fold_add)) | arrange | inspect(part1); diff --git a/interactive/examples/aoc2023/day14/part2.ddp b/interactive/examples/aoc2023/day14/part2.ddp index 52c8c89b1..e930364a3 100644 --- a/interactive/examples/aoc2023/day14/part2.ddp +++ b/interactive/examples/aoc2023/day14/part2.ddp @@ -17,9 +17,9 @@ let maxc = cells | map(((_, c), _) -> ((), (1000000, c) sub)) | min -- sentinel cubes on all four borders let cubesX = cubes + (allcols | map((c, _) -> ((0, c), ()))) - + (allcols | map((c, _) -> ((), c)) | join(maxr, (_, c, m) -> (((m, 1) add, c), ()))) + + (allcols | map((c, _) -> ((), c)) | join(maxr, (_, (c, m)) -> (((m, 1) add, c), ()))) + (allrows | map((r, _) -> ((r, 0), ()))) - + (allrows | map((r, _) -> ((), r)) | join(maxc, (_, r, m) -> ((r, (m, 1) add), ()))); + + (allrows | map((r, _) -> ((), r)) | join(maxc, (_, (r, m)) -> ((r, (m, 1) add), ()))); let cubcol = cubesX | map(((r, c), _) -> (c, r)); -- (c, rc) let cubrow = cubesX | map(((r, c), _) -> (r, c)); -- (r, cc) @@ -30,49 +30,49 @@ spin: { -- roll north: blocker = nearest cube above; rank among segment peers let bn = cur | map(((k, r, c), _) -> (c, (k, r))) - | join(cubcol, (c, (k, r), rc) -> ((c, k, r), rc)) + | join(cubcol, (c, ((k, r), rc)) -> ((c, k, r), rc)) | filter(((_, _, r), rc) -> (rc, r) lt) | map(((c, k, r), rc) -> ((k, r, c), (1000000, rc) sub)) | min | map((x, v) -> (x, (1000000, v) sub)); -- ((k, r, c), b) let segn = bn | map(((k, r, c), b) -> ((k, c, b), r)); -- ((k, c, b), r) - let osn = segn | join(segn, ((k, c, b), r, r2) -> ((k, c, b, r), r2)) + let osn = segn | join(segn, ((k, c, b), (r, r2)) -> ((k, c, b, r), r2)) | filter(((_, _, _, r), r2) -> (r2, r) le) | map((x, _) -> (x, ())) | count | map(((k, c, b, _), n) -> ((k, (b, n) add, c), ())); -- (k, r', c) -- roll west let bw = osn | map(((k, r, c), _) -> (r, (k, c))) - | join(cubrow, (r, (k, c), cc) -> ((r, k, c), cc)) + | join(cubrow, (r, ((k, c), cc)) -> ((r, k, c), cc)) | filter(((_, _, c), cc) -> (cc, c) lt) | map(((r, k, c), cc) -> ((k, r, c), (1000000, cc) sub)) | min | map((x, v) -> (x, (1000000, v) sub)); -- ((k, r, c), b) let segw = bw | map(((k, r, c), b) -> ((k, r, b), c)); -- ((k, r, b), c) - let osw = segw | join(segw, ((k, r, b), c, c2) -> ((k, r, b, c), c2)) + let osw = segw | join(segw, ((k, r, b), (c, c2)) -> ((k, r, b, c), c2)) | filter(((_, _, _, c), c2) -> (c2, c) le) | map((x, _) -> (x, ())) | count | map(((k, r, b, _), n) -> ((k, r, (b, n) add), ())); -- (k, r, c') -- roll south: blocker = nearest cube below; rank counts peers >= r let bs = osw | map(((k, r, c), _) -> (c, (k, r))) - | join(cubcol, (c, (k, r), rc) -> ((c, k, r), rc)) + | join(cubcol, (c, ((k, r), rc)) -> ((c, k, r), rc)) | filter(((_, _, r), rc) -> (rc, r) gt) | map(((c, k, r), rc) -> ((k, r, c), rc)) | min; -- ((k, r, c), b) let segs = bs | map(((k, r, c), b) -> ((k, c, b), r)); -- ((k, c, b), r) - let oss = segs | join(segs, ((k, c, b), r, r2) -> ((k, c, b, r), r2)) + let oss = segs | join(segs, ((k, c, b), (r, r2)) -> ((k, c, b, r), r2)) | filter(((_, _, _, r), r2) -> (r2, r) ge) | map((x, _) -> (x, ())) | count | map(((k, c, b, _), n) -> ((k, (b, n) sub, c), ())); -- roll east let be = oss | map(((k, r, c), _) -> (r, (k, c))) - | join(cubrow, (r, (k, c), cc) -> ((r, k, c), cc)) + | join(cubrow, (r, ((k, c), cc)) -> ((r, k, c), cc)) | filter(((_, _, c), cc) -> (cc, c) gt) | map(((r, k, c), cc) -> ((k, r, c), cc)) | min; -- ((k, r, c), b) let sege = be | map(((k, r, c), b) -> ((k, r, b), c)); -- ((k, r, b), c) - let ose = sege | join(sege, ((k, r, b), c, c2) -> ((k, r, b, c), c2)) + let ose = sege | join(sege, ((k, r, b), (c, c2)) -> ((k, r, b, c), c2)) | filter(((_, _, _, c), c2) -> (c2, c) ge) | map((x, _) -> (x, ())) | count | map(((k, r, b, _), n) -> (((k, 1) add, r, (b, n) sub), ())); -- (k+1, r, c'): next round @@ -82,6 +82,6 @@ spin: { let final = spin::os | filter(((k, _, _), _) -> (k, 141) eq); output "result" = final | map(((_, r, _), _) -> ((), r)) - | join(maxr, (_, r, m) -> ((), ((1, m) add, r) sub)) + | join(maxr, (_, (r, m)) -> ((), ((1, m) add, r) sub)) | collect | map((_, ps) -> ((), ps fold_add)) | arrange | inspect(part2); diff --git a/interactive/examples/aoc2023/day15/part1.ddp b/interactive/examples/aoc2023/day15/part1.ddp index 50299487d..18ce87261 100644 --- a/interactive/examples/aoc2023/day15/part1.ddp +++ b/interactive/examples/aoc2023/day15/part1.ddp @@ -12,14 +12,14 @@ let ktab = chars | map(_ -> ((), ())) | distinct hash: { let t = st | map(((r, i, h), _) -> ((r, (i, 1) add), h)) - | join(chars, ((r, i), h, c) -> ((), (r, i, ((h, c) add, 17) mul))); + | join(chars, ((r, i), (h, c)) -> ((), (r, i, ((h, c) add, 17) mul))); -- ((), (r, i, t)) - let next = t | join(ktab, (_, (r, i, t), k) -> ((r, i, (t, (256, k) mul) sub), ())) + let next = t | join(ktab, (_, ((r, i, t), k)) -> ((r, i, (t, (256, k) mul) sub), ())) | filter(((_, _, t), _) -> ((t, 0) ge, (t, 256) lt) all); set st = (lens | map((r, _) -> ((r, 0, 0), ()))) + next | distinct; } output "result" = hash::st | map(((r, i, h), _) -> (r, (i, h))) - | join(lens, (r, (i, h), n) -> (r, (i, h, n))) | filter((_, (i, _, n)) -> (i, n) eq) + | join(lens, (r, ((i, h), n)) -> (r, (i, h, n))) | filter((_, (i, _, n)) -> (i, n) eq) | map((_, (_, h, _)) -> ((), h)) | collect | map((_, ps) -> ((), ps fold_add)) | arrange | inspect(part1); diff --git a/interactive/examples/aoc2023/day15/part2.ddp b/interactive/examples/aoc2023/day15/part2.ddp index b4a9f4c1c..6d5f2c6ac 100644 --- a/interactive/examples/aoc2023/day15/part2.ddp +++ b/interactive/examples/aoc2023/day15/part2.ddp @@ -12,9 +12,9 @@ let dels = opsk | filter((_, (_, f)) -> (f, 0) eq) | map((l, (r, _)) -> (l, (100 | map((l, v) -> (l, (1000000, v) sub)); -- ((l1,l2), last delete r) -let after = opsk | join(dels, (l, (r, f), d) -> (l, (r, f, d))) | filter((_, (r, _, d)) -> (r, d) gt) +let after = opsk | join(dels, (l, ((r, f), d)) -> (l, (r, f, d))) | filter((_, (r, _, d)) -> (r, d) gt) | map((l, (r, f, _)) -> (l, (r, f))); -let nodel = opsk - (opsk | join(dels, (l, rf, _) -> (l, rf))); +let nodel = opsk - (opsk | join(dels, (l, (rf, _)) -> (l, rf))); let finals = after + nodel; -- ((l1,l2), (r, op)) let last = finals | map((l, (r, f)) -> (l, ((1000000, r) sub, f))) | min | map((l, (_, f)) -> (l, f)); @@ -23,14 +23,14 @@ let present = last | filter((_, f) -> (f, 0) gt); -- ((l1,l2), foca -- box = HASH(label): two rounds of *17 mod 256 (relational modulo) let t1 = present | map(((l1, l2), f) -> ((), (l1, l2, f))); -- ((), (l1,l2,focal)) -let h1 = t1 | join(ktab, (_, (l1, l2, f), k) -> ((l1, l2), (f, ((l1, 17) mul, (256, k) mul) sub))) +let h1 = t1 | join(ktab, (_, ((l1, l2, f), k)) -> ((l1, l2), (f, ((l1, 17) mul, (256, k) mul) sub))) | filter((_, (_, h)) -> ((h, 0) ge, (h, 256) lt) all); -- ((l1,l2), (focal, h1)) let t2 = h1 | map(((l1, l2), (f, h)) -> ((), (l1, l2, f, ((h, l2) add, 17) mul))); -let hh = t2 | join(ktab, (_, (l1, l2, f, t), k) -> ((l1, l2), (f, (t, (256, k) mul) sub))) +let hh = t2 | join(ktab, (_, ((l1, l2, f, t), k)) -> ((l1, l2), (f, (t, (256, k) mul) sub))) | filter((_, (_, h)) -> ((h, 0) ge, (h, 256) lt) all); -- ((l1,l2), (focal, box)) -let boxed = hh | join(first, (_, (f, bx), m) -> (bx, (m, f))); -- (box, (minr, focal)) -let rank = boxed | join(boxed, (bx, (m, f), (m2, _)) -> ((bx, m, f), m2)) +let boxed = hh | join(first, (_, ((f, bx), m)) -> (bx, (m, f))); -- (box, (minr, focal)) +let rank = boxed | join(boxed, (bx, ((m, f), (m2, _))) -> ((bx, m, f), m2)) | filter(((_, m, _), m2) -> (m2, m) le) | map((x, _) -> (x, ())) | count; -- ((box, minr, focal), slot) output "result" = rank | map(((bx, _, f), n) -> ((), (((1, bx) add, n) mul, f) mul)) diff --git a/interactive/examples/aoc2023/day16/part1.ddp b/interactive/examples/aoc2023/day16/part1.ddp index e8c0dbdee..7a56a82f9 100644 --- a/interactive/examples/aoc2023/day16/part1.ddp +++ b/interactive/examples/aoc2023/day16/part1.ddp @@ -14,12 +14,12 @@ let shift = cells | map(_ -> ((), ())) | distinct | flatmap(_ -> [ -- ((dir, sym), (dr, dc, newdir)) beam: { - let onsym = l | map(((r, c, d), _) -> ((r, c), d)) | join(cells, (rc, d, s) -> ((d, s), rc)); + let onsym = l | map(((r, c, d), _) -> ((r, c), d)) | join(cells, (rc, (d, s)) -> ((d, s), rc)); -- ((dir, sym), (r, c)) - let next = onsym | join(shift, (_, (r, c), (dr, dc, nd)) -> (((r, dr) add, (c, dc) add, nd), ())); + let next = onsym | join(shift, (_, ((r, c), (dr, dc, nd))) -> (((r, dr) add, (c, dc) add, nd), ())); set l = (cells | filter(((r, c), _) -> ((r, 1) eq, (c, 1) eq) all) | map(_ -> ((1, 1, 0), ()))) + next | distinct; } output "result" = beam::l | map(((r, c, _), _) -> ((r, c), ())) - | join(cells, (rc, _, _) -> (rc, ())) | distinct + | join(cells, (rc, (_, _)) -> (rc, ())) | distinct | map(_ -> ((), ())) | count | arrange | inspect(part1); diff --git a/interactive/examples/aoc2023/day16/part2.ddp b/interactive/examples/aoc2023/day16/part2.ddp index 6b621e840..fb6695260 100644 --- a/interactive/examples/aoc2023/day16/part2.ddp +++ b/interactive/examples/aoc2023/day16/part2.ddp @@ -19,22 +19,22 @@ let maxc = cells | map(((_, c), _) -> ((), (1000000, c) sub)) | min | map((_, v) -> ((), (1000000, v) sub)); let minr = cells | map(((r, _), _) -> ((), r)) | min; -let seeds = (rows | map((r, _) -> ((), r)) | join(minc, (_, r, m) -> ((r, m, 0, 0, r), ()))) - + (rows | map((r, _) -> ((), r)) | join(maxc, (_, r, m) -> ((r, m, 1, 1, r), ()))) - + (cols | map((c, _) -> ((), c)) | join(minr, (_, c, m) -> ((m, c, 3, 3, c), ()))) - + (cols | map((c, _) -> ((), c)) | join(maxc, (_, c, m) -> ((m, c, 2, 2, c), ()))); +let seeds = (rows | map((r, _) -> ((), r)) | join(minc, (_, (r, m)) -> ((r, m, 0, 0, r), ()))) + + (rows | map((r, _) -> ((), r)) | join(maxc, (_, (r, m)) -> ((r, m, 1, 1, r), ()))) + + (cols | map((c, _) -> ((), c)) | join(minr, (_, (c, m)) -> ((m, c, 3, 3, c), ()))) + + (cols | map((c, _) -> ((), c)) | join(maxc, (_, (c, m)) -> ((m, c, 2, 2, c), ()))); -- key (r, c, dir, sd, si) beam: { let onsym = l | map(((r, c, d, sd, si), _) -> ((r, c), (d, sd, si))) - | join(cells, ((r, c), (d, sd, si), s) -> ((d, s), (r, c, sd, si))); + | join(cells, ((r, c), ((d, sd, si), s)) -> ((d, s), (r, c, sd, si))); -- ((dir, sym), (r, c, sd, si)) - let next = onsym | join(shift, (_, (r, c, sd, si), (dr, dc, nd)) -> (((r, dr) add, (c, dc) add, nd, sd, si), ())); + let next = onsym | join(shift, (_, ((r, c, sd, si), (dr, dc, nd))) -> (((r, dr) add, (c, dc) add, nd, sd, si), ())); set l = seeds + next | distinct; } let energized = beam::l | map(((r, c, _, sd, si), _) -> ((r, c), (sd, si))) - | join(cells, ((r, c), (sd, si), _) -> ((r, c, sd, si), ())) | distinct + | join(cells, ((r, c), ((sd, si), _)) -> ((r, c, sd, si), ())) | distinct | map(((_, _, sd, si), _) -> ((sd, si), ())) | count; -- ((sd, si), n) diff --git a/interactive/examples/aoc2023/day17/part1.ddp b/interactive/examples/aoc2023/day17/part1.ddp index 2ca2397d8..ebe1229ae 100644 --- a/interactive/examples/aoc2023/day17/part1.ddp +++ b/interactive/examples/aoc2023/day17/part1.ddp @@ -9,13 +9,13 @@ let seeds = (cells | filter(((r, c), _) -> ((r, 1) eq, (c, 1) eq) all) | map(_ - dijk: { let straight = mc | filter(((_, _, _, _, s), _) -> (s, 3) lt) | map(((r, c, dr, dc, s), h) -> (((r, dr) add, (c, dc) add), (dr, dc, s, h))) - | join(cells, ((r, c), (dr, dc, s, h), w) -> ((r, c, dr, dc, (s, 1) add), (h, w) add)); + | join(cells, ((r, c), ((dr, dc, s, h), w)) -> ((r, c, dr, dc, (s, 1) add), (h, w) add)); let turna = mc | map(((r, c, dr, dc, _), h) -> (((r, dc) add, (c, dr) add), (dc, dr, h))) - | join(cells, ((r, c), (dr, dc, h), w) -> ((r, c, dr, dc, 1), (h, w) add)); + | join(cells, ((r, c), ((dr, dc, h), w)) -> ((r, c, dr, dc, 1), (h, w) add)); let turnb = mc | map(((r, c, dr, dc, _), h) -> (((r, dc) sub, (c, dr) sub), ((0, dc) sub, (0, dr) sub, h))) - | join(cells, ((r, c), (dr, dc, h), w) -> ((r, c, dr, dc, 1), (h, w) add)); + | join(cells, ((r, c), ((dr, dc, h), w)) -> ((r, c, dr, dc, 1), (h, w) add)); set mc = seeds + straight + turna + turnb | min; } @@ -23,9 +23,9 @@ let maxr = cells | map(((r, _), _) -> ((), (1000000, r) sub)) | min | map((_, v) -> ((), (1000000, v) sub)); let maxc = cells | map(((_, c), _) -> ((), (1000000, c) sub)) | min | map((_, v) -> ((), (1000000, v) sub)); -let corner = maxr | join(maxc, (_, a, b) -> ((), (a, b))); +let corner = maxr | join(maxc, (_, (a, b)) -> ((), (a, b))); output "result" = dijk::mc | map(((r, c, _, _, _), h) -> ((), (r, c, h))) - | join(corner, (_, (r, c, h), (mr, mc)) -> ((), ((r, mr) eq, (c, mc) eq, h))) + | join(corner, (_, ((r, c, h), (mr, mc))) -> ((), ((r, mr) eq, (c, mc) eq, h))) | filter((_, (a, b, _)) -> (a, b) all) | map((_, (_, _, h)) -> ((), h)) | min | arrange | inspect(part1); diff --git a/interactive/examples/aoc2023/day17/part2.ddp b/interactive/examples/aoc2023/day17/part2.ddp index 82c19b05c..e8adcc99a 100644 --- a/interactive/examples/aoc2023/day17/part2.ddp +++ b/interactive/examples/aoc2023/day17/part2.ddp @@ -9,13 +9,13 @@ let seeds = (cells | filter(((r, c), _) -> ((r, 1) eq, (c, 1) eq) all) | map(_ - dijk: { let straight = mc | filter(((_, _, _, _, s), _) -> (s, 10) lt) | map(((r, c, dr, dc, s), h) -> (((r, dr) add, (c, dc) add), (dr, dc, s, h))) - | join(cells, ((r, c), (dr, dc, s, h), w) -> ((r, c, dr, dc, (s, 1) add), (h, w) add)); + | join(cells, ((r, c), ((dr, dc, s, h), w)) -> ((r, c, dr, dc, (s, 1) add), (h, w) add)); let turna = mc | filter(((_, _, _, _, s), _) -> (s, 4) ge) | map(((r, c, dr, dc, _), h) -> (((r, dc) add, (c, dr) add), (dc, dr, h))) - | join(cells, ((r, c), (dr, dc, h), w) -> ((r, c, dr, dc, 1), (h, w) add)); + | join(cells, ((r, c), ((dr, dc, h), w)) -> ((r, c, dr, dc, 1), (h, w) add)); let turnb = mc | filter(((_, _, _, _, s), _) -> (s, 4) ge) | map(((r, c, dr, dc, _), h) -> (((r, dc) sub, (c, dr) sub), ((0, dc) sub, (0, dr) sub, h))) - | join(cells, ((r, c), (dr, dc, h), w) -> ((r, c, dr, dc, 1), (h, w) add)); + | join(cells, ((r, c), ((dr, dc, h), w)) -> ((r, c, dr, dc, 1), (h, w) add)); set mc = seeds + straight + turna + turnb | min; } @@ -23,9 +23,9 @@ let maxr = cells | map(((r, _), _) -> ((), (1000000, r) sub)) | min | map((_, v) -> ((), (1000000, v) sub)); let maxc = cells | map(((_, c), _) -> ((), (1000000, c) sub)) | min | map((_, v) -> ((), (1000000, v) sub)); -let corner = maxr | join(maxc, (_, a, b) -> ((), (a, b))); +let corner = maxr | join(maxc, (_, (a, b)) -> ((), (a, b))); output "result" = dijk::mc | filter(((_, _, _, _, s), _) -> (s, 4) ge) | map(((r, c, _, _, s), h) -> ((), (r, c, h, s))) - | join(corner, (_, (r, c, h, s), (mr, mc)) -> ((), ((r, mr) eq, (c, mc) eq, h, s))) + | join(corner, (_, ((r, c, h, s), (mr, mc))) -> ((), ((r, mr) eq, (c, mc) eq, h, s))) | filter((_, (a, b, _, _)) -> (a, b) all) | map((_, (_, _, h, _)) -> ((), h)) | min | arrange | inspect(part2); diff --git a/interactive/examples/aoc2023/day18/part1.ddp b/interactive/examples/aoc2023/day18/part1.ddp index 60fffd518..875337c46 100644 --- a/interactive/examples/aoc2023/day18/part1.ddp +++ b/interactive/examples/aoc2023/day18/part1.ddp @@ -9,16 +9,16 @@ let instr = input 0 : ((int, int, int, int, int, int, int, int, int), ()) | map( path: { let step = p | map(((i, r, c), ()) -> (i, (r, c))) - | join(instr, (i, (r, c), (dr, dc, n)) -> (((i, 1) add, (r, (dr, n) mul) add, (c, (dc, n) mul) add), ())); + | join(instr, (i, ((r, c), (dr, dc, n))) -> (((i, 1) add, (r, (dr, n) mul) add, (c, (dc, n) mul) add), ())); set p = (instr | filter((line, _) -> (line, 1) eq) | map(_ -> ((1, 0, 0), ()))) + step | distinct; } let terms = path::p | map(((i, r, c), ()) -> ((i, 1) add, (r, c))) | join(path::p | map(((i, r, c), ()) -> (i, (r, c))), - (_, (r1, c1), (r2, c2)) -> ((), ((r1, r2) add, (c1, c2) sub) mul)); + (_, ((r1, c1), (r2, c2))) -> ((), ((r1, r2) add, (c1, c2) sub) mul)); let s = terms | collect | map((_, vs) -> ((), vs fold_add)) | map((_, x) -> ((), ((x, 0) lt, x neg, x) select)); let perim = instr | map((_, (_, _, n)) -> ((), n)) | collect | map((_, vs) -> ((), vs fold_add)); -let n = s | join(perim, (_, a, b) -> ((), (a, b) add)); +let n = s | join(perim, (_, (a, b)) -> ((), (a, b) add)); -- ((), |S| + P), even; halve by binary decomposition (no division) let powtab = instr | map(_ -> ((), ())) | distinct @@ -27,7 +27,7 @@ let powtab = instr | map(_ -> ((), ())) | distinct halve: { let next = st | map(((i, r, q), ()) -> (i, (r, q))) - | join(powtab, (i, (r, q), p) -> + | join(powtab, (i, ((r, q), p)) -> let t = (2, p) mul in let ge = (r, t) ge in (((i, 1) add, (ge, (r, t) sub, r) select, (ge, (q, p) add, q) select), ())); diff --git a/interactive/examples/aoc2023/day18/part2.ddp b/interactive/examples/aoc2023/day18/part2.ddp index af2911ea6..1632b651f 100644 --- a/interactive/examples/aoc2023/day18/part2.ddp +++ b/interactive/examples/aoc2023/day18/part2.ddp @@ -15,16 +15,16 @@ let instr = input 0 : ((int, int, int, int, int, int, int, int, int), ()) | map( path: { let step = p | map(((i, r, c), ()) -> (i, (r, c))) - | join(instr, (i, (r, c), (dr, dc, n)) -> (((i, 1) add, (r, (dr, n) mul) add, (c, (dc, n) mul) add), ())); + | join(instr, (i, ((r, c), (dr, dc, n))) -> (((i, 1) add, (r, (dr, n) mul) add, (c, (dc, n) mul) add), ())); set p = (instr | filter((line, _) -> (line, 1) eq) | map(_ -> ((1, 0, 0), ()))) + step | distinct; } let terms = path::p | map(((i, r, c), ()) -> ((i, 1) add, (r, c))) | join(path::p | map(((i, r, c), ()) -> (i, (r, c))), - (_, (r1, c1), (r2, c2)) -> ((), ((r1, r2) add, (c1, c2) sub) mul)); + (_, ((r1, c1), (r2, c2))) -> ((), ((r1, r2) add, (c1, c2) sub) mul)); let s = terms | collect | map((_, vs) -> ((), vs fold_add)) | map((_, x) -> ((), ((x, 0) lt, x neg, x) select)); let perim = instr | map((_, (_, _, n)) -> ((), n)) | collect | map((_, vs) -> ((), vs fold_add)); -let n = s | join(perim, (_, a, b) -> ((), (a, b) add)); +let n = s | join(perim, (_, (a, b)) -> ((), (a, b) add)); -- ((), |S| + P), even; halve by binary decomposition (no division) let powtab = instr | map(_ -> ((), ())) | distinct @@ -33,7 +33,7 @@ let powtab = instr | map(_ -> ((), ())) | distinct halve: { let next = st | map(((i, r, q), ()) -> (i, (r, q))) - | join(powtab, (i, (r, q), p) -> + | join(powtab, (i, ((r, q), p)) -> let t = (2, p) mul in let ge = (r, t) ge in (((i, 1) add, (ge, (r, t) sub, r) select, (ge, (q, p) add, q) select), ())); diff --git a/interactive/examples/aoc2023/day19/part1.ddp b/interactive/examples/aoc2023/day19/part1.ddp index d59d8d5d9..7903cf4c9 100644 --- a/interactive/examples/aoc2023/day19/part1.ddp +++ b/interactive/examples/aoc2023/day19/part1.ddp @@ -9,7 +9,7 @@ let parts = input 1 : ((int, int, int, int), ()) | map(((x, m, a, s), ()) -> ((0 move: { let mtch = mv | map(((st, x, m, a, s), ()) -> (st, (x, m, a, s))) - | join(rulesbys, (st, (x, m, a, s), (prio, f, cmp, v, nx)) -> + | join(rulesbys, (st, ((x, m, a, s), (prio, f, cmp, v, nx))) -> let sel = ((f, 0) eq, x, ((f, 1) eq, m, ((f, 2) eq, a, s) select) select) select in ((st, x, m, a, s), (((cmp, 0) eq, (sel, v) lt, (sel, v) gt) select, prio, nx))) | filter((_, (hit, _, _)) -> (hit, 1) eq) | map((k, (_, prio, nx)) -> (k, (prio, nx))) diff --git a/interactive/examples/aoc2023/day19/part2.ddp b/interactive/examples/aoc2023/day19/part2.ddp index ccdcf8760..f63d5a669 100644 --- a/interactive/examples/aoc2023/day19/part2.ddp +++ b/interactive/examples/aoc2023/day19/part2.ddp @@ -8,7 +8,7 @@ let rules = input 0 : ((int, int, int, int, int, int), ()) | map(((st, prio, f, flow: { let cur = reg | map(((st, prio, xl, xh, ml, mh, al, ah, sl, sh), ()) -> ((st, prio), (xl, xh, ml, mh, al, ah, sl, sh))); - let b1 = cur | join(rules, (_, (xl, xh, ml, mh, al, ah, sl, sh), (f, cmp, v, nx)) -> + let b1 = cur | join(rules, (_, ((xl, xh, ml, mh, al, ah, sl, sh), (f, cmp, v, nx))) -> let isgt = (cmp, 1) eq in let islt = (cmp, 0) eq in let up = (v, 1) add in @@ -22,7 +22,7 @@ flow: { (((f, 2) eq, islt) all, (dn, ah) min, ah) select, (((f, 3) eq, isgt) all, (up, sl) max, sl) select, (((f, 3) eq, islt) all, (dn, sh) min, sh) select), ())); - let b2 = cur | join(rules, ((st, prio), (xl, xh, ml, mh, al, ah, sl, sh), (f, cmp, v, _)) -> + let b2 = cur | join(rules, ((st, prio), ((xl, xh, ml, mh, al, ah, sl, sh), (f, cmp, v, _))) -> let isgt = (cmp, 1) eq in let islt = (cmp, 0) eq in ((st, (prio, 1) add, diff --git a/interactive/examples/aoc2023/day22/part1.ddp b/interactive/examples/aoc2023/day22/part1.ddp index c275bdb10..75942329e 100644 --- a/interactive/examples/aoc2023/day22/part1.ddp +++ b/interactive/examples/aoc2023/day22/part1.ddp @@ -9,7 +9,7 @@ let bricks = lines | filter(((_, x1, y1, z1, x2, y2, z2), _) -> ((x1, x2) le, (y -- (r, (xlo, ylo, zlo, xhi, yhi, zhi)) let bu = bricks | map((r, (xl, yl, zl, xh, yh, zh)) -> ((), (r, xl, yl, zl, xh, yh, zh))); -let ovl = bu | join(bu, (_, (r1, xl1, yl1, zl1, xh1, yh1, zh1), (r2, xl2, yl2, zl2, xh2, yh2, zh2)) -> +let ovl = bu | join(bu, (_, ((r1, xl1, yl1, zl1, xh1, yh1, zh1), (r2, xl2, yl2, zl2, xh2, yh2, zh2))) -> ((r1, r2), ( (((xl1, xl2) max, (xh1, xh2) min) le, ((yl1, yl2) max, (yh1, yh2) min) le, @@ -20,22 +20,22 @@ let ovl = bu | join(bu, (_, (r1, xl1, yl1, zl1, xh1, yh1, zh1), (r2, xl2, yl2, z -- resting z per brick, stored negated (mz = BIG - z) so `min` computes max rests: { - let lifted = rest | join(ovl, (_, mz, (up, h)) -> (up, (mz, h) sub)); + let lifted = rest | join(ovl, (_, (mz, (up, h))) -> (up, (mz, h) sub)); set rest = (bricks | map((r, _) -> (r, 999999))) + lifted | min; } let rest = rests::rest; -- (r, BIG - resting z), BIG = 1000000 -- direct supports: rest_top(lower) + 1 == rest_z(upper) -let s1 = ovl | join(rest, (lo, (up, h), mz) -> (up, (lo, (mz, h) sub))); +let s1 = ovl | join(rest, (lo, ((up, h), mz)) -> (up, (lo, (mz, h) sub))); -- (upper, (lower, required mz_upper)) -let sup = s1 | join(rest, (up, (lo, req), mz) -> ((lo, up, (req, mz) eq), ())) +let sup = s1 | join(rest, (up, ((lo, req), mz)) -> ((lo, up, (req, mz) eq), ())) | filter(((_, _, hit), _) -> (hit, 1) eq) | map(((lo, up, _), _) -> ((lo, up), ())); -- (lower, upper) let uniq2 = sup | map(((_, up), _) -> (up, ())) | count | filter((_, n) -> (n, 1) eq) | map((k, _) -> (k, ())); -let critical = sup | map(((lo, up), _) -> (up, lo)) | join(uniq2, (_, lo, ()) -> (lo, ())) | distinct; +let critical = sup | map(((lo, up), _) -> (up, lo)) | join(uniq2, (_, (lo, ())) -> (lo, ())) | distinct; let total = lines | map(_ -> ((), ())) | count; -- ALL lines, per the slt -output "result" = total | join(critical | map(_ -> ((), ())) | count, (_, a, b) -> ((), (a, b) sub)) +output "result" = total | join(critical | map(_ -> ((), ())) | count, (_, (a, b)) -> ((), (a, b) sub)) | arrange | inspect(part1); diff --git a/interactive/examples/aoc2023/day22/part2.ddp b/interactive/examples/aoc2023/day22/part2.ddp index 596a353d0..5024b2ebc 100644 --- a/interactive/examples/aoc2023/day22/part2.ddp +++ b/interactive/examples/aoc2023/day22/part2.ddp @@ -8,7 +8,7 @@ let bricks = input 0 : ((int, int, int, int, int, int, int), ()) | filter(((_, x -- (r, (xlo, ylo, zlo, xhi, yhi, zhi)) let bu = bricks | map((r, (xl, yl, zl, xh, yh, zh)) -> ((), (r, xl, yl, zl, xh, yh, zh))); -let ovl = bu | join(bu, (_, (r1, xl1, yl1, zl1, xh1, yh1, zh1), (r2, xl2, yl2, zl2, xh2, yh2, zh2)) -> +let ovl = bu | join(bu, (_, ((r1, xl1, yl1, zl1, xh1, yh1, zh1), (r2, xl2, yl2, zl2, xh2, yh2, zh2))) -> ((r1, r2), ( (((xl1, xl2) max, (xh1, xh2) min) le, ((yl1, yl2) max, (yh1, yh2) min) le, @@ -19,16 +19,16 @@ let ovl = bu | join(bu, (_, (r1, xl1, yl1, zl1, xh1, yh1, zh1), (r2, xl2, yl2, z -- resting z per brick, stored negated (mz = BIG - z) so `min` computes max rests: { - let lifted = rest | join(ovl, (_, mz, (up, h)) -> (up, (mz, h) sub)); + let lifted = rest | join(ovl, (_, (mz, (up, h))) -> (up, (mz, h) sub)); set rest = (bricks | map((r, _) -> (r, 999999))) + lifted | min; } let rest = rests::rest; -- (r, BIG - resting z), BIG = 1000000 -- direct supports: rest_top(lower) + 1 == rest_z(upper) -let s1 = ovl | join(rest, (lo, (up, h), mz) -> (up, (lo, (mz, h) sub))); +let s1 = ovl | join(rest, (lo, ((up, h), mz)) -> (up, (lo, (mz, h) sub))); -- (upper, (lower, required mz_upper)) -let sup = s1 | join(rest, (up, (lo, req), mz) -> ((lo, up, (req, mz) eq), ())) +let sup = s1 | join(rest, (up, ((lo, req), mz)) -> ((lo, up, (req, mz) eq), ())) | filter(((_, _, hit), _) -> (hit, 1) eq) | map(((lo, up, _), _) -> ((lo, up), ())); -- (lower, upper) @@ -36,15 +36,15 @@ let uniq2 = sup | map(((_, up), _) -> (up, ())) | count | filter((_, n) -> (n, 1 let supL = sup | map(((lo, up), _) -> (lo, up)); -- (lower, upper) let supcnt = sup | map(((_, up), _) -> (up, ())) | count; -- (b, #supporters) -let seed = sup | map(((lo, up), _) -> (up, lo)) | join(uniq2, (b, a, ()) -> ((a, b), ())); +let seed = sup | map(((lo, up), _) -> (up, lo)) | join(uniq2, (b, (a, ())) -> ((a, b), ())); -- (a, b): b uniquely supported by a trans: { - let fell = st | map(((a, b), _) -> (b, a)) | join(supL, (_, a, c) -> ((a, c), ())) + let fell = st | map(((a, b), _) -> (b, a)) | join(supL, (_, (a, c)) -> ((a, c), ())) | map((k, _) -> (k, ())) | count; -- ((a, b), #supporters of b falling with a) let full = fell | map(((a, b), n) -> (b, (a, n))) - | join(supcnt, (b, (a, n), m) -> ((a, b, (n, m) eq), ())) + | join(supcnt, (b, ((a, n), m)) -> ((a, b, (n, m) eq), ())) | filter(((_, _, hit), _) -> (hit, 1) eq) | map(((a, b, _), _) -> ((a, b), ())); set st = seed + full | distinct; } diff --git a/interactive/examples/programs/kcore.ddp b/interactive/examples/programs/kcore.ddp index 9516146e3..db44789c5 100644 --- a/interactive/examples/programs/kcore.ddp +++ b/interactive/examples/programs/kcore.ddp @@ -18,7 +18,7 @@ peel: { -- Edges whose source is not alive get removed. -- semijoin active with alive_set, negate to get removals. - let kept = active | join(alive_set, (k, v, _) -> (k, v)); + let kept = active | join(alive_set, (k, (v, _)) -> (k, v)); let newly_removed = active - kept; set removals = removals - newly_removed | arrange; diff --git a/interactive/examples/programs/reach.ddp b/interactive/examples/programs/reach.ddp index a1a9fbb44..cebaaab3e 100644 --- a/interactive/examples/programs/reach.ddp +++ b/interactive/examples/programs/reach.ddp @@ -6,7 +6,7 @@ let edges = input 0 : ((int, int), ()) | map(((src, dst), ()) -> (src, dst)); let roots = input 1 : (int, ()) | map((n, ()) -> (n, ())); reach: { - let proposals = reach | join(edges, (_, _, dst) -> (dst, ())); + let proposals = reach | join(edges, (_, (_, dst)) -> (dst, ())); set reach = roots + proposals | distinct; } diff --git a/interactive/examples/programs/scc.ddp b/interactive/examples/programs/scc.ddp index 7026c7902..8864bbb19 100644 --- a/interactive/examples/programs/scc.ddp +++ b/interactive/examples/programs/scc.ddp @@ -10,24 +10,24 @@ outer: { fwd: { let nodes = edges | map((_, dst) -> (dst, dst)) | delay((_, n) -> (0, 0, (256, n call bit_length) mul)); let labels = proposals + nodes | min; - set proposals = labels | join(scc, (_, label, dst) -> (dst, label)); + set proposals = labels | join(scc, (_, (label, dst)) -> (dst, label)); } let trim_fwd = edges - | join(fwd::labels, (src, dst, label) -> (dst, (src, label))) - | join(fwd::labels, (dst, (src, label), other) -> (dst, (src, label, other))) + | join(fwd::labels, (src, (dst, label)) -> (dst, (src, label))) + | join(fwd::labels, (dst, ((src, label), other)) -> (dst, (src, label, other))) | filter((_, (_, a, b)) -> (a, b) eq) | map((dst, (src, _, _)) -> (dst, src)); bwd: { let nodes = trans | map((_, dst) -> (dst, dst)) | delay((_, n) -> (0, 0, (256, n call bit_length) mul)); let labels = proposals + nodes | min; - set proposals = labels | join(trim_fwd, (_, label, dst) -> (dst, label)); + set proposals = labels | join(trim_fwd, (_, (label, dst)) -> (dst, label)); } let trim_bwd = trans - | join(bwd::labels, (src, dst, label) -> (dst, (src, label))) - | join(bwd::labels, (dst, (src, label), other) -> (dst, (src, label, other))) + | join(bwd::labels, (src, (dst, label)) -> (dst, (src, label))) + | join(bwd::labels, (dst, ((src, label), other)) -> (dst, (src, label, other))) | filter((_, (_, a, b)) -> (a, b) eq) | map((dst, (src, _, _)) -> (dst, src)); diff --git a/interactive/examples/programs/tour.ddp b/interactive/examples/programs/tour.ddp index 9db1febb6..d7c8d3e97 100644 --- a/interactive/examples/programs/tour.ddp +++ b/interactive/examples/programs/tour.ddp @@ -36,7 +36,7 @@ let uniq = edges | map((a, _) -> (a, ())) | distinct; -- Iteration: reach, with concat (+) and distinct. reach: { - let proposals = reach | join(edges, (_, _, dst) -> (dst, ())); + let proposals = reach | join(edges, (_, (_, dst)) -> (dst, ())); set reach = roots + proposals | distinct; } diff --git a/interactive/examples/server/programs/access.ddp b/interactive/examples/server/programs/access.ddp index cfdfc6fb4..ca68e5a85 100644 --- a/interactive/examples/server/programs/access.ddp +++ b/interactive/examples/server/programs/access.ddp @@ -16,7 +16,7 @@ let members = groups | map((grp, who) -> (grp inject Group, who)); let barred = barring::barred | map((who, doc) -> ((who, doc), ())); barring: { var barred : (Member, int); -- member -> doc: expanded blocked list. - let beyond = barred | join(members, (_grp, doc, who) -> (who, doc)); + let beyond = barred | join(members, (_grp, (doc, who)) -> (who, doc)); set barred = blocks | map((doc, who) -> (who, doc)) + beyond | distinct; } @@ -25,7 +25,7 @@ output "access" = sharing::allowed; sharing: { var allowed : ((Member, int), Access); -- (member, doc) -> strongest set allowed = offered - - (offered | join(barred, (pair, acc, ()) -> (pair, acc))) + - (offered | join(barred, (pair, (acc, ())) -> (pair, acc))) | min; let offered = grants | map((doc, (who, acc)) -> ((who, doc), acc)) @@ -33,5 +33,5 @@ sharing: { let inherited = allowed | map(((who, doc), acc) -> (who, (doc, acc))) - | join(members, (_grp, (doc, acc), who) -> ((who, doc), acc)); + | join(members, (_grp, ((doc, acc), who)) -> ((who, doc), acc)); } diff --git a/interactive/examples/server/programs/arrange_idem.ddp b/interactive/examples/server/programs/arrange_idem.ddp index ddc8f35e0..fe9754af3 100644 --- a/interactive/examples/server/programs/arrange_idem.ddp +++ b/interactive/examples/server/programs/arrange_idem.ddp @@ -35,19 +35,19 @@ let arr = node | filter((_, (op, _)) -> (op, 1) eq) | map((id, (_, cs)) -> (id, eqsat: { -- congruence closure: group nodes by (op ++ canonicalized children) - let canon = pc | join(eq, (_, (id, op, pos), cls) -> ((id, op), (pos, cls))); -- ((id,op), (pos,class)) + let canon = pc | join(eq, (_, ((id, op, pos), cls)) -> ((id, op), (pos, cls))); -- ((id,op), (pos,class)) let bsig = canon | collect | map(((id, op), cs) -> ((op, cs), id)); -- signature ; id let reps = bsig | min; -- signature ; min id - let cong = bsig | join(reps, (_, id, rep) -> (id, rep)); -- id ; representative + let cong = bsig | join(reps, (_, (id, rep)) -> (id, rep)); -- id ; representative -- arrange-idempotence rule: - let arranged = producing | join(eq, (_, _, cls) -> (cls, ())) | distinct; -- arranged class ; - let arr_cls = arr | map((a, c) -> (c, a)) | join(eq, (_, a, cls) -> (a, cls)); -- arrange_id ; child class - let rule = arr_cls | map((a, cls) -> (cls, a)) | join(arranged, (cls, a, _) -> (a, cls)); -- arrange_id ; child class + let arranged = producing | join(eq, (_, (_, cls)) -> (cls, ())) | distinct; -- arranged class ; + let arr_cls = arr | map((a, c) -> (c, a)) | join(eq, (_, (a, cls)) -> (a, cls)); -- arrange_id ; child class + let rule = arr_cls | map((a, cls) -> (cls, a)) | join(arranged, (cls, (a, _)) -> (a, cls)); -- arrange_id ; child class -- congruence edges + rule edges, symmetrized, then one min-label hop let edges = cong + (cong | map((a, b) -> (b, a))) + rule + (rule | map((a, b) -> (b, a))); - let prop = eq | join(edges, (_, l, n) -> (n, l)); + let prop = eq | join(edges, (_, (l, n)) -> (n, l)); set eq = elems + prop | min; } diff --git a/interactive/examples/server/programs/reach_import.ddp b/interactive/examples/server/programs/reach_import.ddp index 1abec6f41..a4e322485 100644 --- a/interactive/examples/server/programs/reach_import.ddp +++ b/interactive/examples/server/programs/reach_import.ddp @@ -8,7 +8,7 @@ let edges = input "edges" : (int, int); let roots = input "roots" : (int, ()); reach: { - let proposals = reach | join(edges, (_, _, dst) -> (dst, ())); + let proposals = reach | join(edges, (_, (_, dst)) -> (dst, ())); set reach = roots + proposals | distinct; } diff --git a/interactive/server/demo/txn.txt b/interactive/server/demo/txn.txt index 368c0f4c4..9d67ec6d5 100644 --- a/interactive/server/demo/txn.txt +++ b/interactive/server/demo/txn.txt @@ -26,13 +26,13 @@ let wr_all = intents | filter(((_, r, _), _) -> (r, 0) eq) | map(((wid, _, k), ( res: { -- writes of transactions not (currently) rolled back: anti-join on id. let wr_id = wr_all | map(((wid, k), (v)) -> ((wid), (k, v))); - let wr_bad = wr_id | join(rollback, (wid, kv, ()) -> (wid, kv)); + let wr_bad = wr_id | join(rollback, (wid, (kv, ())) -> (wid, kv)); let writes = wr_id + (wr_bad | negate); -- each read paired with every surviving prior write to its key ... let wk = writes | map(((wid), (k, v)) -> ((k), (wid, v))); let rk = reads | map(((rid, k), (e)) -> ((k), (rid, e))); - let pairs = rk | join(wk, ((k), (rid, e), (wid, v)) -> ((k, rid, e), (wid, v))) + let pairs = rk | join(wk, ((k), ((rid, e), (wid, v))) -> ((k, rid, e), (wid, v))) | filter(((_, rid, _), (wid, _)) -> (wid, rid) lt); -- ... and the LATEST prior write wins: min over (-wid, v). let best = pairs | map((kre, (wid, v)) -> (kre, (wid neg, v))) | min; @@ -44,7 +44,7 @@ res: { let seen = best | map(((k, rid, _), _) -> ((rid, k), ())); let rkk = reads | map((rk, _) -> (rk, ())); let unseen = rkk + (seen | negate); - let bad_absent = unseen | join(reads, ((rid, _), (), (e)) -> ((rid, e), ())) + let bad_absent = unseen | join(reads, ((rid, _), ((), (e))) -> ((rid, e), ())) | filter(((_, e), ()) -> (e, -1) ne) | map(((rid, _), ()) -> ((rid), ())); set rollback = (bad_seen + bad_absent) | distinct; @@ -52,7 +52,7 @@ res: { let rb = res::rollback; let wid = wr_all | map(((wid, k), (v)) -> ((wid), (k, v))); -let wbad = wid | join(rb, (wid, kv, ()) -> (wid, kv)); +let wbad = wid | join(rb, (wid, (kv, ())) -> (wid, kv)); let committed = wid + (wbad | negate); output "rollback" = rb; diff --git a/interactive/server/diag.ddp b/interactive/server/diag.ddp index e6fc43b37..6f51f622d 100644 --- a/interactive/server/diag.ddp +++ b/interactive/server/diag.ddp @@ -13,7 +13,7 @@ let placed = operator | filter((_, (program, _, _, _)) -> (program, "diag") ne) | map((key, (program, statement, _, _)) -> (key, (program, statement))); -- Nanoseconds scheduled: by live operators where they are, plus by operators that have shut down. -let timed = time | join(placed, (_, (), place) -> (place, ())) +let timed = time | join(placed, (_, ((), place)) -> (place, ())) + (retired | filter(((_, program, _, _), ()) -> (program, "diag") ne) | map(((_, program, statement, _), ()) -> ((program, statement), ()))); @@ -21,7 +21,7 @@ output "time_by_statement" = timed | count; output "time_by_program" = timed | map(((program, _), ()) -> (program, ())) | count; -- Updates the arrangements hold, by where they are. -output "records_by_statement" = records | join(placed, (_, (), place) -> (place, ())) | count; +output "records_by_statement" = records | join(placed, (_, ((), place)) -> (place, ())) | count; -- Live operators per program. output "operators_by_program" = placed | map((_, (program, _)) -> (program, ())) | count; diff --git a/interactive/server/src/cmd.rs b/interactive/server/src/cmd.rs index 66c62b159..69fcb81ad 100644 --- a/interactive/server/src/cmd.rs +++ b/interactive/server/src/cmd.rs @@ -1591,7 +1591,7 @@ mod tests { #[test] fn inputs_are_fed_by_name() { let catalog = Catalog::default(); - let program = "let e = input \"edges\" : ((int), (int));\nlet r = input \"roots\" : ((int), ());\noutput \"reach\" = r | join(e, (_, (), d) -> (d, ()));"; + let program = "let e = input \"edges\" : ((int), (int));\nlet r = input \"roots\" : ((int), ());\noutput \"reach\" = r | join(e, (_, ((), d)) -> (d, ()));"; let load = Cmd::Load { width: 1, id_hint: "g".into(), bindings: BTreeMap::new(), program: program.into(), explain: None }; prepare(load, &catalog).expect("the program lowers"); let mut p = LineParser::new(catalog); @@ -1608,7 +1608,7 @@ mod tests { #[test] fn loads_bind_inputs_to_traces() { let catalog = Catalog::default(); - let text = "let edges = input \"edges\" : ((int), (int));\nlet roots = input \"roots\" : ((int), ());\noutput \"hop\" = roots | join(edges, (_, (), d) -> ((d), ()));"; + let text = "let edges = input \"edges\" : ((int), (int));\nlet roots = input \"roots\" : ((int), ());\noutput \"hop\" = roots | join(edges, (_, ((), d)) -> ((d), ()));"; let bindings: BTreeMap = [("edges".to_string(), "g::edges".to_string())].into(); let load = Cmd::Load { width: 1, id_hint: "q".into(), bindings, program: text.into(), explain: None }; let Ok(PreparedCommand::Server(ServerCommand::Install { program, .. })) = prepare(load, &catalog) else { panic!("the program lowers") }; diff --git a/interactive/server/tests/multiworker.rs b/interactive/server/tests/multiworker.rs index 13c1d37d8..5b2ded4d5 100644 --- a/interactive/server/tests/multiworker.rs +++ b/interactive/server/tests/multiworker.rs @@ -358,7 +358,7 @@ fn assert_shared_import_requests(workers: usize) { &mut reader, name, &format!( - "{name} load {name} begin\nlet q = input 0 : ((int, int), ());\nlet e = import \"graph::edges\";\noutput \"answer\" = (q | map(((rid, n), ()) -> ((n), (rid)))) | join(e, (_, (rid), (node)) -> ((rid), (node))); + "{name} load {name} begin\nlet q = input 0 : ((int, int), ());\nlet e = import \"graph::edges\";\noutput \"answer\" = (q | map(((rid, n), ()) -> ((n), (rid)))) | join(e, (_, ((rid), (node))) -> ((rid), (node))); \n{name} end-load\n" ), ); @@ -723,7 +723,7 @@ fn refused_loads_do_not_redirect_named_inputs() { fn loads_bind_inputs_to_published_outputs() { let (server, mut writer, mut reader) = start_server(2); request(&mut writer, &mut reader, "g", "g load g begin\nlet e = input \"e\" : ((int), (int));\noutput \"edges\" = e | arrange;\ng end-load\n"); - request(&mut writer, &mut reader, "q", "q load q edges=g::edges begin\nlet edges = input \"edges\" : ((int), (int));\nlet roots = input \"roots\" : ((int), ());\noutput \"hop\" = roots | join(edges, (_, (), d) -> ((d), ()));\nq end-load\n"); + request(&mut writer, &mut reader, "q", "q load q edges=g::edges begin\nlet edges = input \"edges\" : ((int), (int));\nlet roots = input \"roots\" : ((int), ());\noutput \"hop\" = roots | join(edges, (_, ((), d)) -> ((d), ()));\nq end-load\n"); request(&mut writer, &mut reader, "f", "f feed g e time=0 begin\n1 val=2\n1 val=3\n4 val=5\nf end-feed\n"); request(&mut writer, &mut reader, "r", "r feed q roots time=0 1\n"); seal_inputs(&mut writer, &mut reader, &["g e", "q roots"], 1); diff --git a/interactive/src/backend/corgi.rs b/interactive/src/backend/corgi.rs index 5495d2834..3c2282574 100644 --- a/interactive/src/backend/corgi.rs +++ b/interactive/src/backend/corgi.rs @@ -353,7 +353,7 @@ impl Backend for CorgiBackend { // The proxy-join seam drives the backend blockwise under the driver's fuel; the backend // compiles the projection once, now, at both inputs' shapes, and emits corgi columns // directly as `CorgiContainer`s — column-native, no row round-trip. - let backend = CorgiJoinBackend::new(projection, [k, v0, v1]); + let backend = CorgiJoinBackend::new(&projection.on_join_columns(), [k, v0, v1]); // An import is read as of its time. Reading one input as of a time `s` or both gives the // same output when the other's times are at or beyond `s` (`(t0 ∨ s) ∨ t1 = t0 ∨ t1` // then), as every program time is; two imports share one time. So the join reads both diff --git a/interactive/src/explain/decouple.rs b/interactive/src/explain/decouple.rs index 7fbfb40eb..415cdac88 100644 --- a/interactive/src/explain/decouple.rs +++ b/interactive/src/explain/decouple.rs @@ -31,9 +31,9 @@ //! ## Reading the method contracts //! //! Each method returns a projection/predicate that the rule applies to a row of -//! a stated shape. Inputs are addressed by **slot**: after a binary `join`, the -//! row exposes `$0` = the join key, `$1` = the left input's value, `$2` = the -//! right input's value; before a join, `$0`/`$1` are key/value. A method's doc +//! a stated shape. For a binary `join`, the three names `$0`, `$1`, and `$2` in these +//! contracts refer to the key, left value, and right value in the row `($0, ($1, $2))`. +//! For a single input, `$0` and `$1` name the key and value. A method's doc //! gives the *input row shape it expects* and the *output row shape it //! produces*, both in the `(K, …)` notation above; an implementer needs only //! those shapes, not the positions. Sizes arrive as parameters: `*_len`/`*_depth` @@ -275,7 +275,7 @@ mod mem { for (lk, lv) in l { for (rk, rv) in r { if lk == rk { - out.push(pair(&p, vec![lk.clone(), lv.clone(), rv.clone()])); + out.push(pair(&p, vec![lk.clone(), Value::Prod(vec![lv.clone(), rv.clone()])])); } } } @@ -369,7 +369,7 @@ mod value_contract { #[test] fn join_demands_both_inputs() { // proj: K_out = K, V_out = V_L. - let p = proj("let (k, l, r) = input in (k, l.0)"); + let p = proj("let (k, (l, r)) = input in (k, l.0)"); let left = side(vec![(one(5), env(7, &[]))], 0); let right = side(vec![(one(5), env(9, &[]))], 0); let dep = vec![(one(5), env(7, &[1]))]; // (K_out=5 ; V_out=7, q=1) diff --git a/interactive/src/explain/mod.rs b/interactive/src/explain/mod.rs index 55b44b6d8..8621ff0d9 100644 --- a/interactive/src/explain/mod.rs +++ b/interactive/src/explain/mod.rs @@ -321,11 +321,11 @@ fn resolve(root: &Scope, path: &[usize], r: &Ref) -> Target { // its own rows, whose values carry the chain (and `q`) after the user's value. A user lambda must // see the user's row, not explain's, so explain calls it on the user's row as it sits in explain's. -/// A slot of `arity` rows, `(k, v)` or `(k, v, r)`, whose body is `body` applied to the results +/// A slot of `arity` rows, `(k, v)` or `(k, (v, r))`, whose body is `body` applied to the results /// of the user's lambdas in `calls`, each called on an argument written over those rows (explain's /// spelling of the user's row). See [`Lambda::compose`]: the user's text is never parsed again. fn slot(arity: usize, calls: &[(&Lambda, String)], body: impl FnOnce(&[String]) -> String) -> Lambda { - let pattern = if arity == 2 { "(k, v)" } else { "(k, v, r)" }; + let pattern = if arity == 2 { "(k, v)" } else { "(k, (v, r))" }; Lambda::compose(pattern, calls, body) } /// A projection of explain's own, from the lambda text of its key and value. @@ -641,7 +641,7 @@ impl RowModel for Val { let mut val = vec![var(0), f(1, 0), f(2, 0)]; // K, V_L, V_R val.extend(fidx(1, 1, 1 + l_len)); // chain_L val.extend(fidx(2, 1, 1 + r_len)); // chain_R - slot(3, &[(proj, tup(vec![var(0), var(1), var(2)]))], |out| { + slot(3, &[(proj, tup(vec![var(0), tup(vec![var(1), var(2)])]))], |out| { val.push(fld(out[0].clone(), 1)); // V_out (pair-produced) tup(vec![fld(out[0].clone(), 0), tup(val)]) }) @@ -1076,7 +1076,7 @@ impl<'a> Reverse<'a> { let rt = resolve(self.orig, path, right); let ls = self.side(<); let rs = self.side(&rt); - let projection = slot(3, &[(projection, tup(vec![var(0), f(1, 0), f(2, 0)]))], |out| out[0].clone()); + let projection = slot(3, &[(projection, tup(vec![var(0), tup(vec![f(1, 0), f(2, 0)])]))], |out| out[0].clone()); // Keep the coordinates of the scopes both sides share with this one: dropping more // than one side needs only widens its demand, which is still sound. let shared = shared_len(path, target_path(<)).min(shared_len(path, target_path(&rt))); diff --git a/interactive/src/lambda.rs b/interactive/src/lambda.rs index fe49db3a5..dbe36b781 100644 --- a/interactive/src/lambda.rs +++ b/interactive/src/lambda.rs @@ -35,8 +35,8 @@ pub(crate) mod print; pub(crate) mod normalize; /// A slot of the IR: a corgi program as written, `let pat = input in body`, and its graph, whose -/// input is the tuple of the slot's rows (`(key, val)` for a linear op, `(key, left val, right -/// val)` for a join). The parser binds it where it is written, in the scope of its DDIR program: +/// input is the tuple of the slot's rows (`(key, val)` for a linear op, `(key, (left val, right +/// val))` for a join). The parser binds it where it is written, in the scope of its DDIR program: /// the registered functions and the base words `call` reaches, and the program's /// `enum` declarations. A parse error is kept, and reported where the program is compiled /// or run. A composed lambda's graph is authoritative: its readable `src` is @@ -84,6 +84,10 @@ impl Lambda { if let Some(e) = error { lambda.graph = Err(e); } lambda } + /// Apply a join projection to the three columns supplied by the arrangements. + pub(crate) fn on_join_columns(&self) -> Self { + Self::compose("(k, l, r)", &[(self, "(k, (l, r))".into())], |out| out[0].clone()) + } /// The graph, or the parse error naming the program. pub(crate) fn graph(&self) -> Result<&std::sync::Arc>, String> { self.graph.as_ref().map_err(|e| format!("corgi program `{}`: {e}", self.src)) diff --git a/interactive/src/lower.rs b/interactive/src/lower.rs index 0aba661f7..f5dd87fbd 100644 --- a/interactive/src/lower.rs +++ b/interactive/src/lower.rs @@ -636,7 +636,7 @@ mod tree_tests { let producer = "min"; let text = format!(r#" let rows = input 0 : (int, ()) | {producer}; - output "join" = rows | join(rows, (k, _, _) -> (k, ())); + output "join" = rows | join(rows, (k, (_, _)) -> (k, ())); output "half" = rows | lift | half_join(rows, 0, (k, _, _) -> (k, ())); output "reduce" = rows | min; output "distinct" = rows | distinct; @@ -655,7 +655,7 @@ mod tree_tests { fn arranges_consumers_of_whole_record_distinct() { let program = lower_tree(parse(r#" let rows = input 0 : (int, int) | distinct; - output "join" = rows | join(rows, (k, a, _) -> (k, a)); + output "join" = rows | join(rows, (k, (a, _)) -> (k, a)); output "half" = rows | lift | half_join(rows, 0, (k, a, _) -> (k, a)); output "reduce" = rows | min; output "distinct" = rows | distinct; @@ -739,7 +739,7 @@ mod tree_tests { let edges = input 0 | map(((src, dst), ()) -> (src, dst));\n\ let roots = input 1 | map(((n), ()) -> (n, ()));\n\ reach: {\n\ - let proposals = reach | join(edges, (_, _, dst) -> (dst, ()));\n\ + let proposals = reach | join(edges, (_, (_, dst)) -> (dst, ()));\n\ set reach = roots + proposals | distinct;\n\ }\n\ output \"result\" = reach::reach | map(_ -> ((), ())) | arrange | inspect(total);"; @@ -1003,8 +1003,8 @@ mod optimize_tests { let src = "\ let a = input 0 | map(((a, b), ()) -> (a, b));\n\ let b = input 0 | map(((a, b), ()) -> (b, a));\n\ - let j1 = a | join(b, (_, x, y) -> (x, y));\n\ - let j2 = a | join(b, (_, x, y) -> (y, x));\n\ + let j1 = a | join(b, (_, (x, y)) -> (x, y));\n\ + let j2 = a | join(b, (_, (x, y)) -> (y, x));\n\ output \"result\" = j1 + j2;"; let mut p = lower_tree(parse(src)); crate::passes::optimize(&mut p); @@ -1018,7 +1018,7 @@ mod optimize_tests { let src = "\ let a = input 0 | map(((a, b), ()) -> (a, b)) | distinct;\n\ let b = input 0 | map(((a, b), ()) -> (a, b));\n\ - output \"result\" = a | join(b, (_, x, _) -> (x, ()));"; + output \"result\" = a | join(b, (_, (x, _)) -> (x, ()));"; let mut p = lower_tree(parse(src)); crate::passes::optimize(&mut p); // No Arrange may target a Reduce item. @@ -1050,7 +1050,7 @@ mod optimize_tests { /// written; others keep `input{n}`, whatever `let` they are bound to. #[test] fn inputs_take_positions_and_names() { - let p = lower_tree(parse("let e = input \"edges\" : ((int), (int));\nlet roots = input : ((int), ());\noutput \"out\" = roots | join(e, (_, (), d) -> (d, ()));")); + let p = lower_tree(parse("let e = input \"edges\" : ((int), (int));\nlet roots = input : ((int), ());\noutput \"out\" = roots | join(e, (_, ((), d)) -> (d, ()));")); let named: Vec<_> = p.root.imports.iter().map(|i| (i.name.as_str(), i.from.clone())).collect(); assert!(named.contains(&("edges", st::Source::Input(0))), "{named:?}"); assert!(named.contains(&("input1", st::Source::Input(1))), "{named:?}"); diff --git a/interactive/src/parse/pipe.rs b/interactive/src/parse/pipe.rs index 267a728e3..8798cc662 100644 --- a/interactive/src/parse/pipe.rs +++ b/interactive/src/parse/pipe.rs @@ -33,7 +33,8 @@ //! //! - `| map((k, v) -> (k', v'))` — reshape each row to the `(key, val)` the lambda gives. //! - `| value((k, v) -> v')` — replace the value, keeping the key. -//! - `| join(other, (k, v, v1) -> (k', v'))` — equijoin on the key. +//! - `| join(other)` — equijoin on the key, yielding `(k, (v, v1))`. +//! - `| join(other, f)` — project the joined row, as in `| join(other) | map(f)`. //! - `| min` / `| distinct` / `| count` / `| collect` — reduce; `distinct` keeps each `(key, val)` //! record once, and `collect` is NEST (gather a key's values into a `List`). //! - `| sum` — sum integer values per key, weighted by their signed diffs. A zero total has no row. @@ -113,7 +114,7 @@ //! # Lambdas //! //! A slot runs `let pat = input in body` in corgi's ML syntax, with `input` the tuple of the -//! slot's rows: `(key, val)` for a linear op, `(key, left val, right val)` for a join, and +//! slot's rows: `(key, val)` for a linear op, `(key, (left val, right val))` for a join, and //! `(key, vals)` for `reduce`. For example, `map(((src, dst), ()) -> (src, dst))` keys each row //! of pairs by its first field. Operators are postfix (`(a, b) add`, `xs len`, //! `(mask, a, b) select`); `x.N` is a tuple's field, `(e)` a one-field tuple and `()` unit; @@ -676,7 +677,19 @@ impl Parser { let val = self.slot(); Expr::Map(Box::new(lhs), Lambda::compose("(k, v)", &[(&val, "(k, v)".into())], |r| format!("(k, {})", r[0]))) }, - Token::Join => { self.next(); self.expect(&Token::LParen); let r = self.parse_pipe_expr(); self.expect(&Token::Comma); let p = self.lambda(); self.expect(&Token::RParen); Expr::Join(Box::new(lhs), Box::new(r), p) }, + Token::Join => { + self.next(); + self.expect(&Token::LParen); + let rhs = self.parse_pipe_expr(); + let projection = if *self.peek() == Token::Comma { + self.next(); + self.lambda() + } else { + Lambda::new("let row = input in row".into(), &corgi::Scope::default()) + }; + self.expect(&Token::RParen); + Expr::Join(Box::new(lhs), Box::new(rhs), projection) + }, Token::Min => { self.next(); Expr::Reduce(Box::new(lhs), Reducer::Min) }, Token::Distinct => { self.next(); Expr::Reduce(Box::new(lhs), Reducer::Distinct) }, Token::Count => { self.next(); Expr::Reduce(Box::new(lhs), Reducer::Count) }, diff --git a/interactive/src/parse/pipe/fragment.rs b/interactive/src/parse/pipe/fragment.rs index 31dda7f3d..f02a3202a 100644 --- a/interactive/src/parse/pipe/fragment.rs +++ b/interactive/src/parse/pipe/fragment.rs @@ -456,8 +456,8 @@ impl Fragment { s.ingest(&atom.val, "r".into()); let body = s.row("k".into(), s.key.clone()); let pat = if i == 1 && direct { - format!("(k : {}, v : {}, r : {})", first.key.shape_pattern(), first.val.shape_pattern(), atom.val.shape_pattern()) - } else { format!("(k, v, r : {})", atom.val.shape_pattern()) }; + format!("(k : {}, (v : {}, r : {}))", first.key.shape_pattern(), first.val.shape_pattern(), atom.val.shape_pattern()) + } else { format!("(k, (v, r : {}))", atom.val.shape_pattern()) }; expr = Expr::Join(Box::new(expr), Box::new(atom.source.clone()), s.lambda(&pat, body)); let target = self.atoms.get(i + 1).map_or(Target::Output, |a| Target::Key(&a.key)); expr = s.checkpoint(expr, target, "(k, v)"); diff --git a/interactive/src/passes/arrangements_tests.rs b/interactive/src/passes/arrangements_tests.rs index 4627a7206..1171f0edb 100644 --- a/interactive/src/passes/arrangements_tests.rs +++ b/interactive/src/passes/arrangements_tests.rs @@ -169,7 +169,7 @@ fn batched_statements_keep_sibling_names_and_dependencies() { let b = ::x | map((k, v) -> (k, v neg)); let c = a + b; } - third: { let a = first::c | join(second::c, (k, a, b) -> (k, (a, b) add)); } + third: { let a = first::c | join(second::c, (k, (a, b)) -> (k, (a, b) add)); } output "first" = first::c; output "second" = second::c; output "third" = third::a; @@ -194,7 +194,7 @@ fn variant_inputs_and_half_joins_stop_motion_before_fusion() { let idx = mapped | arrange; } let mixed = variant + ::rows; - let joined = mixed | join(::rows, (k, l, r) -> (k, l)); + let joined = mixed | join(::rows, (k, (l, r)) -> (k, l)); let reduced = joined | min; set x = ::rows; } @@ -215,7 +215,7 @@ fn variant_inputs_and_half_joins_stop_motion_before_fusion() { "#, hoist(), 0); let Item::Sub(child) = p.root.items.last().unwrap() else { panic!() }; assert!(matches!(child.items[0], Item::Op(Node::HalfJoin { .. }))); - for (input, removed) in [("::x", 0), ("::x | join(::x, (k, l, r) -> (k, l))", 1)] { + for (input, removed) in [("::x", 0), ("::x | join(::x, (k, (l, r)) -> (k, l))", 1)] { let p = rewrite(&format!(r#" let x = input 0 : (int, int); child: {{ diff --git a/interactive/src/passes/projection.rs b/interactive/src/passes/projection.rs index a190c3da2..a6e43b650 100644 --- a/interactive/src/passes/projection.rs +++ b/interactive/src/passes/projection.rs @@ -546,7 +546,7 @@ fn build_scope( builder.edge(id, key); } } - builder.lambda(&output, &[left, right], projection); + builder.lambda(&output, &[left, right], &projection.on_join_columns()); } Node::HalfJoin { stream, diff --git a/interactive/src/plan.rs b/interactive/src/plan.rs index 72cad883c..4a7c8f58b 100644 --- a/interactive/src/plan.rs +++ b/interactive/src/plan.rs @@ -443,7 +443,7 @@ fn printed_node(node: &Node, names: &Names, shapes: &ScopeShapes, depth: usize, format!("{} | linear [ {} ]", names.of(input), printed.join(" | ")) } Node::Join { left, right, projection } => { - let input = shapes.of(left).zip(shapes.of(right)).map(|(l, r)| Shape::Prod(vec![l.key, l.val, r.val])); + let input = shapes.of(left).zip(shapes.of(right)).map(|(l, r)| Shape::Prod(vec![l.key, Shape::Prod(vec![l.val, r.val])])); format!("{} | join({}, {})", names.of(left), names.of(right), slots.lambda(projection, input, &format!("{at}/join"))) } Node::HalfJoin { stream, arrangement, strict, projection } => { diff --git a/interactive/src/plan/compare.rs b/interactive/src/plan/compare.rs index af1d09478..fe7d3f57a 100644 --- a/interactive/src/plan/compare.rs +++ b/interactive/src/plan/compare.rs @@ -60,7 +60,7 @@ fn input_shapes(node: &Node, shapes: &ScopeShapes) -> Option> } Node::Join { left, right, .. } => { let (l, r) = (shapes.of(left)?, shapes.of(right)?); - vec![Shape::Prod(vec![l.key, l.val, r.val])] + vec![Shape::Prod(vec![l.key, Shape::Prod(vec![l.val, r.val])])] } Node::HalfJoin { stream, arrangement, .. } => { let (s, a) = (shapes.of(stream)?, shapes.of(arrangement)?); diff --git a/interactive/src/plan/ddp.rs b/interactive/src/plan/ddp.rs index b9a28312f..ce9d3dc96 100644 --- a/interactive/src/plan/ddp.rs +++ b/interactive/src/plan/ddp.rs @@ -410,7 +410,7 @@ impl Printer<'_> { Node::Concat(refs) => refs.iter().map(|r| names.of(r)).collect::>().join(" + "), Node::Arrange(r) => format!("{} | arrange", names.of(r)), Node::Join { left, right, projection } => { - let input = shapes.of(left).zip(shapes.of(right)).map(|(l, r)| Shape::Prod(vec![l.key, l.val, r.val])); + let input = shapes.of(left).zip(shapes.of(right)).map(|(l, r)| Shape::Prod(vec![l.key, Shape::Prod(vec![l.val, r.val])])); format!("{} | join({}, {})", names.of(left), names.of(right), self.lambda(projection, input)?) } Node::HalfJoin { stream, arrangement, strict, projection } => { @@ -548,7 +548,7 @@ mod tests { let x = input 0 : (int, int); let result_idx = x | map((k, v) -> (v, k)); let intermediate = x | filter((k, _) -> (k, 0) gt); - let result = intermediate | map((k, v) -> (k, v)) | join(result_idx, (k, x, y) -> (k, (x, y))); + let result = intermediate | map((k, v) -> (k, v)) | join(result_idx); output "out" = result; "#); assert!(output.contains("let result_idx =")); @@ -662,7 +662,7 @@ mod tests { let early = rows | map((k, v) -> (v, k)); let late = rows | filter((k, v) -> (k, v) lt); let between = rows | map((k, v) -> (k, v neg)); - child: { let pair = ::early | join(::late, (k, a, b) -> (k, (a, b))); } + child: { let pair = ::early | join(::late); } output "out" = child::pair; output "between" = between;"#; let printed = roundtrip(text); diff --git a/interactive/src/scope_ir.rs b/interactive/src/scope_ir.rs index ea47e6ed9..bd2cad888 100644 --- a/interactive/src/scope_ir.rs +++ b/interactive/src/scope_ir.rs @@ -116,7 +116,7 @@ pub enum Node { Linear { input: Ref, ops: Vec }, Concat(Vec), Arrange(Ref), - /// `projection` is a lambda over `(key, left val, right val)` giving `(key', val')`. + /// `projection` is a lambda over `(key, (left val, right val))` giving `(key', val')`. Join { left: Ref, right: Ref, projection: Lambda }, /// Each update of `stream`, `(key, (val, time))` with `time` a time as data (as `lift` writes /// it), meets the rows of `arrangement` at its key whose times are less than the update's own diff --git a/interactive/src/server/read.rs b/interactive/src/server/read.rs index f694b9ecb..52605c8d5 100644 --- a/interactive/src/server/read.rs +++ b/interactive/src/server/read.rs @@ -169,7 +169,7 @@ impl Published { }); let keys = CorgiBackend::arrange(stream.as_collection()); let target = Arr { arranged: self.0.import_named(scope, "PeekImport"), since: point.clone() }; - let project = crate::ir::Lambda::new("let (k, _, v) = input in (k, v)".into(), &corgi::Scope::default()); + let project = crate::ir::Lambda::new("let (k, (_, v)) = input in (k, v)".into(), &corgi::Scope::default()); CorgiBackend::join(keys, target, &project, ((shape.0.clone(), corgi::Shape::Unit), shape.clone())) } } diff --git a/interactive/src/shapes.rs b/interactive/src/shapes.rs index 631d20dc1..914bb2859 100644 --- a/interactive/src/shapes.rs +++ b/interactive/src/shapes.rs @@ -339,7 +339,7 @@ fn infer_node(node: &st::Node, shapes: &ScopeShapes, level: Level, at: &str, pro st::Node::Join { left, right, projection } => { let (l, r) = (shapes.of(left)?, shapes.of(right)?); if l.key != r.key { return fail(format!("join keys {} and {}", l.key, r.key)); } - match pair_shape(projection, &[l.key, l.val, r.val]) { + match pair_shape(projection, &[l.key, Shape::Prod(vec![l.val, r.val])]) { Ok((k, v)) => Some(CollShape::new(k, v, level)), Err(e) => fail(format!("join: {e}")), } @@ -383,7 +383,7 @@ mod tests { let edges = input 0 : ((int, int), ()) | map(((src, dst), ()) -> ((src), (dst))); let roots = input 1 : ((int), ()) | map(((n), ()) -> ((n), ())); reach: { - let proposals = reach | join(edges, (_, _, dst) -> (dst, ())); + let proposals = reach | join(edges, (_, (_, dst)) -> (dst, ())); set reach = roots + proposals | distinct; } output "result" = reach::reach | map(_ -> ((), ())) | arrange; diff --git a/interactive/tests/bitemporal.ddp b/interactive/tests/bitemporal.ddp index 0856583b2..80c7ecb96 100644 --- a/interactive/tests/bitemporal.ddp +++ b/interactive/tests/bitemporal.ddp @@ -7,15 +7,15 @@ let owners = input 1 : (int, int); let links = input 2 : ((int, int), ()) | map(((a, b), ()) -> (a, b)); let balance = deposits | count; -let by_owner = balance | join(owners, (acct, bal, owner) -> (owner, (bal, acct))); +let by_owner = balance | join(owners, (acct, (bal, owner)) -> (owner, (bal, acct))); let roots = balance | filter((_, bal) -> (bal, 0) gt) | map((acct, _) -> (acct, ())); reach: { - let proposals = reach | join(links, (_, _, dst) -> (dst, ())); + let proposals = reach | join(links, (_, (_, dst)) -> (dst, ())); set reach = roots + proposals | distinct; } output "balance" = balance; -output "owner_total" = deposits | join(owners, (acct, (), owner) -> (owner, ())) | count; +output "owner_total" = deposits | join(owners, (acct, ((), owner)) -> (owner, ())) | count; output "owner_min" = by_owner | min; output "reach" = reach::reach; diff --git a/interactive/tests/corgi_backend.rs b/interactive/tests/corgi_backend.rs index 20aec0255..a7fa1d144 100644 --- a/interactive/tests/corgi_backend.rs +++ b/interactive/tests/corgi_backend.rs @@ -306,3 +306,39 @@ fn reregistering_a_function_replaces_it() { assert!(std::sync::Arc::ptr_eq(&second, &interactive::lambda::lookup("reregistered").unwrap())); assert!(std::panic::catch_unwind(|| register("min", vec![], corgi::Shape::Int, Ok)).is_err()); } + +#[test] +fn join_projects_the_same_row_as_map() { + use interactive::server::{host_time, Server}; + + timely::execute_directly(|worker| { + let source = r#" + let l = input 0 : (int, int); + let r = input 1 : (int, (int, int)); + let pairs = l | join(r); + output "pairs" = pairs; + output "identity" = l | join(r, row -> row); + output "mapped" = pairs | map((k, (x, (y, z))) -> (y, (z, k, x))); + output "projected" = l | join(r, (k, (x, (y, z))) -> (y, (z, k, x))); + "#; + let mut server = Server::new(); + let mut program = lower::lower_tree(parse::pipe::parse(source)); + interactive::passes::optimize(&mut program); + server.install(worker, "p", &program).unwrap(); + server.feed("p", 0, "1 val=10 diff=2\n2 val=20", 0).unwrap(); + server.feed("p", 1, "1 val=tuple(3,4) diff=3", 0).unwrap(); + server.feed("p", 0, "1 val=10 diff=-1", 1).unwrap(); + for input in [0, 1] { server.advance_input("p", input, &host_time(2)).unwrap(); } + for (t, diff) in [(0, 6), (1, 3)] { + for (name, key, val) in [ + ("pairs", "1", "tuple(10,tuple(3,4))"), + ("identity", "1", "tuple(10,tuple(3,4))"), + ("mapped", "3", "tuple(4,1,10)"), + ("projected", "3", "tuple(4,1,10)"), + ] { + let rows = server.snapshot_as_of(worker, &format!("p::{name}"), &host_time(t)).unwrap().rows(); + assert_eq!(rows, vec![(key.into(), val.into(), diff)], "{name} at {t}"); + } + } + }); +} diff --git a/interactive/tests/dedup.rs b/interactive/tests/dedup.rs index 04b20b7bd..a9fd00f9d 100644 --- a/interactive/tests/dedup.rs +++ b/interactive/tests/dedup.rs @@ -30,7 +30,7 @@ fn nested_recursive_scopes_share_only_with_the_same_inputs() { {name}: {{ let seed_{name} = {input} | map((k_{name}, v_{name}) -> (k_{name}, ())); inner_{name}: {{ - let step_{name} = reach_{name} | join({input}, (k_{name}, v_{name}, w_{name}) -> (w_{name}, ())); + let step_{name} = reach_{name} | join({input}, (k_{name}, (v_{name}, w_{name})) -> (w_{name}, ())); set reach_{name} = (seed_{name} + step_{name}) | distinct; }} let out_{name} = inner_{name}::reach_{name}; diff --git a/interactive/tests/diag.rs b/interactive/tests/diag.rs index 6a7537684..11893e129 100644 --- a/interactive/tests/diag.rs +++ b/interactive/tests/diag.rs @@ -13,7 +13,7 @@ const DIAG: &str = include_str!("../server/diag.ddp"); const PAIRS: &str = " let edges = input 0 : ((int, int), ()); let forward = edges | map(((a, b), ()) -> (a, b)); -let pairs = forward | join(forward, (_, x, y) -> ((x, y), ())); +let pairs = forward | join(forward, (_, (x, y)) -> ((x, y), ())); output \"NAME\" = pairs | distinct; "; diff --git a/interactive/tests/difftest.rs b/interactive/tests/difftest.rs index e26612454..2ebc2d1dc 100644 --- a/interactive/tests/difftest.rs +++ b/interactive/tests/difftest.rs @@ -135,7 +135,7 @@ struct Let { enum Op { /// An op whose text is fixed, and the shape after it (to know when it may be dropped). Plain { text: String, same_shape: bool }, - /// `join(other, (k, l, r) -> body)`, kept in parts so a delta join can swap the sides. + /// `join(other, (k, (l, r)) -> body)`, kept in parts so a delta join can swap the sides. Join { other: String, k: String, l: String, r: String, body: String }, } @@ -259,7 +259,7 @@ fn render_let(l: &Let, delta: bool, indent: &str) -> String { "(({text} | lift | half_join({other}, 0, ({k}, {lp}, {r}) -> {body})) + ({other} | lift | half_join({text}, 1, ({k}, {r}, {lp}) -> {body}))) | delay((_, (_, t)) -> t) | map((k, (v, _)) -> (k, v))" ); } else { - text = format!("{text} | join({other}, ({k}, {lp}, {r}) -> {body})"); + text = format!("{text} | join({other}, ({k}, ({lp}, {r})) -> {body})"); } } } @@ -1530,7 +1530,7 @@ let roots = input 1 : (int, ()); left: { var reached : (int, ()); let edges_idx = ::edges | arrange; - let step = reached | join(edges_idx, (_, (), dst) -> (dst, ())); + let step = reached | join(edges_idx, (_, ((), dst)) -> (dst, ())); set reached = (::roots + step) | distinct; let rows = ::edges + edges_idx; } @@ -1540,7 +1540,7 @@ outer: { right: { var reached : (int, ()); let edges_idx = ::edges | arrange; - let step = reached | join(edges_idx, (_, (), dst) -> (dst, ())); + let step = reached | join(edges_idx, (_, ((), dst)) -> (dst, ())); set reached = (outer::starts + step) | distinct; } set again = right::reached | filter((_, ()) -> 0); @@ -1581,9 +1581,9 @@ outer: { let flat = filtered | flatmap((k, src) -> (k, (0, src)) enlist) | map((dst, (_, src)) -> (src, dst)); let doubled = flat + flat; let once = doubled + (flat | negate); - let paired = once | join(::edges, (src, dst, _) -> (src, dst)); + let paired = once | join(::edges, (src, (dst, _)) -> (src, dst)); let reduced = paired | min; - let step = reached | join(reduced, (_, (), dst) -> (dst, ())); + let step = reached | join(reduced, (_, ((), dst)) -> (dst, ())); set reached = (::roots + outer::again + step) | distinct; } set again = inner::reached | filter((_, ()) -> 0); diff --git a/interactive/tests/explain.rs b/interactive/tests/explain.rs index 1d09f6a98..570b9450a 100644 --- a/interactive/tests/explain.rs +++ b/interactive/tests/explain.rs @@ -30,21 +30,21 @@ const SCC_ROW: &str = r#" fwd: { let nodes = edges | map((_, dst) -> (dst, dst)) | delay((_, (n)) -> (0, 0, (256, n call bit_length) mul)); let labels = proposals + nodes | min; - set proposals = labels | join(scc, (_, label, dst) -> (dst, label)); + set proposals = labels | join(scc, (_, (label, dst)) -> (dst, label)); } let trim_fwd = edges - | join(fwd::labels, ((src), (dst), (l)) -> ((dst), (src, l))) - | join(fwd::labels, (dst, (src, l), (m)) -> (dst, (src, l, m))) + | join(fwd::labels, ((src), ((dst), (l))) -> ((dst), (src, l))) + | join(fwd::labels, (dst, ((src, l), (m))) -> (dst, (src, l, m))) | filter((_, (_, l, m)) -> (l, m) eq) | map((dst, (src, _, _)) -> (dst, (src))); bwd: { let nodes = trans | map((_, dst) -> (dst, dst)) | delay((_, (n)) -> (0, 0, (256, n call bit_length) mul)); let labels = proposals + nodes | min; - set proposals = labels | join(trim_fwd, (_, label, dst) -> (dst, label)); + set proposals = labels | join(trim_fwd, (_, (label, dst)) -> (dst, label)); } let trim_bwd = trans - | join(bwd::labels, ((src), (dst), (l)) -> ((dst), (src, l))) - | join(bwd::labels, (dst, (src, l), (m)) -> (dst, (src, l, m))) + | join(bwd::labels, ((src), ((dst), (l))) -> ((dst), (src, l))) + | join(bwd::labels, (dst, ((src, l), (m))) -> (dst, (src, l, m))) | filter((_, (_, l, m)) -> (l, m) eq) | map((dst, (src, _, _)) -> (dst, (src))); set trim = trim_bwd - edges; @@ -62,7 +62,7 @@ const TC_ROW: &str = r#" | map(((a, b), ()) -> ((a), (b))); set more = tc | map((a, b) -> (b, a)) - | join(edges, (_, a, c) -> (a, c)); + | join(edges, (_, (a, c)) -> (a, c)); } output "result" = outer::tc; "#; @@ -72,7 +72,7 @@ const REACH_ROW: &str = r#" let edges = input 0 : ((int, int), ()) | map(((src, dst), ()) -> ((src), (dst))); let roots = input 1 : ((int), ()) | map(((n), ()) -> ((n), ())); reach: { - let proposals = reach | join(edges, (_, _, dst) -> (dst, ())); + let proposals = reach | join(edges, (_, (_, dst)) -> (dst, ())); set reach = roots + proposals | distinct; } output "result" = reach::reach; @@ -404,7 +404,7 @@ fn unread_loop_explanations_sufficient() { let edges = input 0 : ((int, int), ()) | map(((src, dst), ()) -> ((src), (dst))); let roots = edges | map((src, _) -> (src, ())); reach: { - let step = x | join(edges, (_, (), dst) -> (dst, ())); + let step = x | join(edges, (_, ((), dst)) -> (dst, ())); set x = (roots + step) | distinct; } output "result" = edges; @@ -424,12 +424,12 @@ fn sibling_scope_explanations_sufficient() { let edges = input 0 : ((int, int), ()) | map(((src, dst), ()) -> ((src), (dst))); let roots = edges | map((src, _) -> (src, ())); reach: { - let step = x | join(edges, (_, (), dst) -> (dst, ())); + let step = x | join(edges, (_, ((), dst)) -> (dst, ())); set x = (roots + step) | distinct; } let reached = reach::x; again: { - let step = y | join(edges, (_, (), dst) -> (dst, ())); + let step = y | join(edges, (_, ((), dst)) -> (dst, ())); set y = (reached + step) | distinct; } output "result" = again::y; @@ -448,7 +448,7 @@ fn flatmap_of_a_parent_explanations_sufficient() { let roots = edges | map((src, _) -> (src, ())); reach: { let hops = edges | flatmap((k, (dst)) -> (k, (dst enlist, (dst, 1) add enlist) append) cap_list) | map((src, dst) -> (src, (dst))); - let step = x | join(hops, (_, (), (dst)) -> ((dst), ())); + let step = x | join(hops, (_, ((), (dst))) -> ((dst), ())); set x = (roots + step) | distinct; } output "result" = reach::x; @@ -467,7 +467,7 @@ fn negated_fanout_explanations_sufficient() { let i0 = input 0 : ((int, int), ()); let r2 = i0 | negate | map(((a, b), ()) -> (b, b)); let r3 = r2 | map((x, y) -> (y, x)); - output "result" = r3 | join(r2, (k, l, r) -> (k, (l, r))); + output "result" = r3 | join(r2); "#; let sizes = assert_all_rows_sufficient(NEGATED, &[vec![(row(&[3, 0]), row(&[]))]]); assert!(!sizes.is_empty()); @@ -482,7 +482,7 @@ fn negated_fanout_explanations_sufficient() { fn min_then_count_explanations_sufficient() { const MIN_COUNT: &str = r#" let i0 = input 0 : (int, int); - s: { let step = x | join(i0, (k, l, e) -> (e, l)); set x = (i0 + step) | min; } + s: { let step = x | join(i0, (k, (l, e)) -> (e, l)); set x = (i0 + step) | min; } output "result" = s::x | map((k, l) -> ((l), ())) | count; "#; let sizes = assert_all_rows_sufficient(MIN_COUNT, &[vec![(int(2), int(4)), (int(4), int(1))]]); @@ -623,7 +623,7 @@ const LAMBDAS: &str = r#" let edges = input 0 : ((int, int), ()) | map(((src, dst), ()) -> ((src), (dst))) | filter((_, (d)) -> (d, d) eq); let roots = input 1 : ((int), ()) | map(((n), ()) -> ((n), ())); reach: { - let proposals = reach | join(edges, (_, _, dst) -> (dst, ())); + let proposals = reach | join(edges, (_, (_, dst)) -> (dst, ())); set reach = roots + proposals | distinct; } output "result" = reach::reach | map((k, _) -> (k, k)) | flatmap(((a), (b)) -> let l = (a enlist, b enlist) append in ((a), ((0, l len) range, l) zip) cap_list); @@ -648,7 +648,7 @@ const SCALARS: &str = r#" inner: { set low = (flat | filter((_, x) -> (x, 15) lt)) + (low | filter((k, _) -> (k, k) eq)) | min; } - output "result" = inner::low | join(counted, (k, l, c) -> (k, (l, c))); + output "result" = inner::low | join(counted); "#; fn scalar_pairs() -> Vec<(Row, Row)> { @@ -689,7 +689,7 @@ const REACH_DELAY: &str = r#" let roots = input 1 : ((int), ()) | map(((n), ()) -> ((n), ())); reach: { let seeds = roots | delay(((n), ()) -> (0, (256, n call bit_length) mul)); - let proposals = reach | join(edges, (_, _, dst) -> (dst, ())); + let proposals = reach | join(edges, (_, (_, dst)) -> (dst, ())); set reach = seeds + proposals | distinct; } output "result" = reach::reach; @@ -706,7 +706,7 @@ const BARE_TIMES: &str = r#" let edges = input 0 : ((int, int), ()) | map(((src, dst), ()) -> (src, dst)); let degree = edges | join_function((src, _) -> ((src, ()), 0, 1) enlist) | count; let later = edges | delay((_, dst) -> (dst, 100) rem) | map((src, dst) -> (dst, src)); - output "result" = later | join(degree, (dst, src, n) -> (src, (dst, n))); + output "result" = later | join(degree, (dst, (src, n)) -> (src, (dst, n))); "#; #[test] @@ -720,7 +720,7 @@ const MIN_LOOP: &str = r#" outer: { let nodes = edges | map((_, dst) -> (dst, dst)); let labels = proposals + nodes | min; - set proposals = labels | join(edges, (_, label, dst) -> (dst, label)); + set proposals = labels | join(edges, (_, (label, dst)) -> (dst, label)); } output "result" = outer::labels; "#; @@ -739,11 +739,11 @@ const SCC_ONE_SCOPE: &str = r#" fwd: { let nodes = edges | map((_, dst) -> (dst, dst)); let labels = proposals + nodes | min; - set proposals = labels | join(scc, (_, label, dst) -> (dst, label)); + set proposals = labels | join(scc, (_, (label, dst)) -> (dst, label)); } let trim_fwd = edges - | join(fwd::labels, ((src), (dst), (l)) -> ((dst), (src, l))) - | join(fwd::labels, (dst, (src, l), (m)) -> (dst, (src, l, m))) + | join(fwd::labels, ((src), ((dst), (l))) -> ((dst), (src, l))) + | join(fwd::labels, (dst, ((src, l), (m))) -> (dst, (src, l, m))) | filter((_, (_, l, m)) -> (l, m) eq) | map((dst, (src, _, _)) -> (dst, (src))); set trim = trim_fwd | filter(((n), _) -> (n, 1000000) gt); @@ -759,11 +759,11 @@ const SCC_ONE_SCOPE_NEGATED: &str = r#" fwd: { let nodes = edges | map((_, dst) -> (dst, dst)); let labels = proposals + nodes | min; - set proposals = labels | join(scc, (_, label, dst) -> (dst, label)); + set proposals = labels | join(scc, (_, (label, dst)) -> (dst, label)); } let trim_fwd = edges - | join(fwd::labels, ((src), (dst), (l)) -> ((dst), (src, l))) - | join(fwd::labels, (dst, (src, l), (m)) -> (dst, (src, l, m))) + | join(fwd::labels, ((src), ((dst), (l))) -> ((dst), (src, l))) + | join(fwd::labels, (dst, ((src, l), (m))) -> (dst, (src, l, m))) | filter((_, (_, l, m)) -> (l, m) eq) | map((dst, (src, _, _)) -> (dst, (src))); set trim = trim_fwd - edges; diff --git a/interactive/tests/fragments.rs b/interactive/tests/fragments.rs index 21f638a37..5403f9c28 100644 --- a/interactive/tests/fragments.rs +++ b/interactive/tests/fragments.rs @@ -278,7 +278,7 @@ fn direct_first_join_reads_the_key_and_carries_it_when_rekeyed() { let source = format!("{projection:?}"); let graph = corgi::parse_ml_in(&source[1..source.len()-1], &corgi::Scope::default()).unwrap(); let got = corgi::eval_graph(&graph, corgi::Value::Prod(vec![ - corgi::Value::int([10]), corgi::Value::int([20]), corgi::Value::int([30]), + corgi::Value::int([10]), corgi::Value::Prod(vec![corgi::Value::int([20]), corgi::Value::int([30])]), ])); // y becomes the key. x leaves the key and must survive in the value with z. assert_eq!(got, corgi::Value::Prod(vec![corgi::Value::int([20]), diff --git a/interactive/tests/half_join.rs b/interactive/tests/half_join.rs index 8648d9d88..20ad1c56a 100644 --- a/interactive/tests/half_join.rs +++ b/interactive/tests/half_join.rs @@ -15,12 +15,12 @@ const PROGRAM: &str = r#" let ab = a | lift | half_join(b, 0, (k, x, y) -> (k, (x, y))); let ba = b | lift | half_join(a, 1, (k, y, x) -> (k, (x, y))); output "delta" = (ab + ba) | delay((_, (_, t)) -> t) | map((k, (xy, _)) -> (k, xy)); - output "join" = a | join(b, (k, x, y) -> (k, (x, y))); + output "join" = a | join(b); -- Reachability from `a`'s keys along `b`'s edges, joining by each method. let roots = a | map((k, _) -> (k, ())); joined: { - let step = reach | join(b, (_, (), dst) -> (dst, ())); + let step = reach | join(b, (_, ((), dst)) -> (dst, ())); set reach = roots + step | distinct; } halved: { diff --git a/interactive/tests/labels.rs b/interactive/tests/labels.rs index 9999b0acb..599aadf50 100644 --- a/interactive/tests/labels.rs +++ b/interactive/tests/labels.rs @@ -10,10 +10,10 @@ use interactive::{lower, parse}; const SRC: &str = " let edges = input 0 : ((int) ; (int)); -let pairs = edges | join(edges, (k, a, b) -> (a, b)); +let pairs = edges | join(edges, (k, (a, b)) -> (a, b)); let counts = pairs | count; reach: { - let step = reach | join(edges, (k, (), b) -> (b, ())); + let step = reach | join(edges, (k, ((), b)) -> (b, ())); set reach = (edges | map((a, _) -> (a, ()))) + step | distinct; } output \"counts\" = counts | map((k, n) -> (k, n)); diff --git a/interactive/tests/pass_roundtrip.rs b/interactive/tests/pass_roundtrip.rs index 4aeb15be8..0ec7e8707 100644 --- a/interactive/tests/pass_roundtrip.rs +++ b/interactive/tests/pass_roundtrip.rs @@ -12,7 +12,7 @@ const REPROS: [(&str, &str); 8] = [ ("F2", r#"let rows = input 0 : (int, int); s: { - let step = x | join(rows, (k,l,r) -> (k,l)); + let step = x | join(rows, (k,(l,r)) -> (k,l)); set x = (rows + step) | min; } let dead = rows | map((k,v) -> (v,k)); @@ -32,7 +32,7 @@ const REPROS: [(&str, &str); 8] = [ var x : (int, int); inner: { let mapped = ::rows | map((k, v) -> (v, k)); - let joined = mapped | join(::rows, (k, l, r) -> (k, l)); + let joined = mapped | join(::rows, (k, (l, r)) -> (k, l)); let reduced = joined | min; } set x = inner::reduced; @@ -42,7 +42,7 @@ const REPROS: [(&str, &str); 8] = [ ("local arrangements survive ancestor reuse without structural cleanup (seed 20864)", r#"let rows = input 0 : (int, int); let twice = rows | arrange | arrange; - let joined = rows | join(twice, (k, l, r) -> (k, l)); + let joined = rows | join(twice, (k, (l, r)) -> (k, l)); child: { let idx = ::rows | arrange; var x : (int, int); set x = idx; } output "root" = joined; output "child" = child::x;"#), diff --git a/interactive/tests/plan_pairs/cases/access_shared.ddp b/interactive/tests/plan_pairs/cases/access_shared.ddp index 9187bba61..b917f029a 100644 --- a/interactive/tests/plan_pairs/cases/access_shared.ddp +++ b/interactive/tests/plan_pairs/cases/access_shared.ddp @@ -17,7 +17,7 @@ let members = groups | map((grp, who) -> (grp inject Group, who)) | arrange; let barred = barring::barred | map((who, doc) -> ((who, doc), ())); barring: { var barred : (Member, int); -- member -> doc: expanded blocked list. - let beyond = barred | join(members, (_grp, doc, who) -> (who, doc)); + let beyond = barred | join(members, (_grp, (doc, who)) -> (who, doc)); set barred = blocks | map((doc, who) -> (who, doc)) + beyond | distinct; } @@ -26,7 +26,7 @@ output "access" = sharing::allowed; sharing: { var allowed : ((Member, int), Access); -- (member, doc) -> strongest set allowed = offered - - (offered | join(barred, (pair, acc, ()) -> (pair, acc))) + - (offered | join(barred, (pair, (acc, ())) -> (pair, acc))) | min; let offered = grants | map((doc, (who, acc)) -> ((who, doc), acc)) @@ -34,5 +34,5 @@ sharing: { let inherited = allowed | map(((who, doc), acc) -> (who, (doc, acc))) - | join(members, (_grp, (doc, acc), who) -> ((who, doc), acc)); + | join(members, (_grp, ((doc, acc), who)) -> ((who, doc), acc)); } diff --git a/interactive/tests/plan_pairs/cases/nested.ddp b/interactive/tests/plan_pairs/cases/nested.ddp index 3ff03be75..0f8d5bbb9 100644 --- a/interactive/tests/plan_pairs/cases/nested.ddp +++ b/interactive/tests/plan_pairs/cases/nested.ddp @@ -4,7 +4,7 @@ let seeds = input 1 : (int, ()); outer: { let starts = seeds + again; inner: { - let step = reached | join(edges, (_, (), dst) -> (dst, ())); + let step = reached | join(edges, (_, ((), dst)) -> (dst, ())); set reached = starts + step | distinct; } set again = inner::reached | filter((k, ()) -> 0); diff --git a/interactive/tests/plan_pairs/expected/access_shared.ddp b/interactive/tests/plan_pairs/expected/access_shared.ddp index 2849db449..3e37b3a5a 100644 --- a/interactive/tests/plan_pairs/expected/access_shared.ddp +++ b/interactive/tests/plan_pairs/expected/access_shared.ddp @@ -12,7 +12,7 @@ barring: { -- Time lanes: epoch, then 1 scope coordinates; this scope enters at zero. var barred : (Member, int); let barred_idx = barred | arrange; - let beyond = barred_idx | join(::members, (_grp, doc, who) -> (who, doc)); + let beyond = barred_idx | join(::members, (_grp, (doc, who)) -> (who, doc)); let barred_rows = ::blocks_map + beyond; let barred_map = barred_rows | map((k, v) -> ((k, v), ())); let barred_idx2 = barred_map | arrange; @@ -27,10 +27,10 @@ sharing: { var allowed : ((Member, int), Access); let inherited_map = allowed | map(((who, doc), acc) -> (who, (doc, acc))); let inherited_idx = inherited_map | arrange; - let inherited = inherited_idx | join(::members, (_grp, (doc, acc), who) -> ((who, doc), acc)); + let inherited = inherited_idx | join(::members, (_grp, ((doc, acc), who)) -> ((who, doc), acc)); let offered = ::grants_map + inherited; let offered_idx = offered | arrange; - let allowed_join = offered_idx | join(::barred_idx, (pair, acc, ()) -> (pair, acc)); + let allowed_join = offered_idx | join(::barred_idx, (pair, (acc, ())) -> (pair, acc)); let allowed_neg = allowed_join | negate; let allowed_rows = offered + allowed_neg; let allowed_idx = allowed_rows | arrange; diff --git a/interactive/tests/plan_pairs/expected/access_shared.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/access_shared.lowered--optimized.diff index d31a8e110..8be6eb9b3 100644 --- a/interactive/tests/plan_pairs/expected/access_shared.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/access_shared.lowered--optimized.diff @@ -13,12 +13,12 @@ var barred : (Member, int); let barred_idx = barred | arrange; - let members_idx = ::members | arrange; -- let beyond = barred_idx | join(members_idx, (_grp, doc, who) -> (who, doc)); +- let beyond = barred_idx | join(members_idx, (_grp, (doc, who)) -> (who, doc)); - let barred_map = ::blocks | map((doc, who) -> (who, doc)); - let barred_rows = barred_map + beyond; - let barred_map2 = barred_rows | map((k, v) -> ((k, v), ())); - let barred_idx2 = barred_map2 | arrange; -+ let beyond = barred_idx | join(::members, (_grp, doc, who) -> (who, doc)); ++ let beyond = barred_idx | join(::members, (_grp, (doc, who)) -> (who, doc)); + let barred_rows = ::blocks_map + beyond; + let barred_map = barred_rows | map((k, v) -> ((k, v), ())); + let barred_idx2 = barred_map | arrange; @@ -36,15 +36,15 @@ let inherited_map = allowed | map(((who, doc), acc) -> (who, (doc, acc))); let inherited_idx = inherited_map | arrange; - let members_idx = ::members | arrange; -- let inherited = inherited_idx | join(members_idx, (_grp, (doc, acc), who) -> ((who, doc), acc)); +- let inherited = inherited_idx | join(members_idx, (_grp, ((doc, acc), who)) -> ((who, doc), acc)); - let offered_map = ::grants | map((doc, (who, acc)) -> ((who, doc), acc)); - let offered = offered_map + inherited; -+ let inherited = inherited_idx | join(::members, (_grp, (doc, acc), who) -> ((who, doc), acc)); ++ let inherited = inherited_idx | join(::members, (_grp, ((doc, acc), who)) -> ((who, doc), acc)); + let offered = ::grants_map + inherited; let offered_idx = offered | arrange; - let barred_idx = ::barred | arrange; -- let allowed_join = offered_idx | join(barred_idx, (pair, acc, ()) -> (pair, acc)); -+ let allowed_join = offered_idx | join(::barred_idx, (pair, acc, ()) -> (pair, acc)); +- let allowed_join = offered_idx | join(barred_idx, (pair, (acc, ())) -> (pair, acc)); ++ let allowed_join = offered_idx | join(::barred_idx, (pair, (acc, ())) -> (pair, acc)); let allowed_neg = allowed_join | negate; let allowed_rows = offered + allowed_neg; let allowed_idx = allowed_rows | arrange; diff --git a/interactive/tests/plan_pairs/expected/access_shared.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/access_shared.parsed--lowered.diff index c07210acd..27d49b837 100644 --- a/interactive/tests/plan_pairs/expected/access_shared.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/access_shared.parsed--lowered.diff @@ -15,7 +15,7 @@ + var barred : (Member, int); + let barred_idx = barred | arrange; + let members_idx = ::members | arrange; -+ let beyond = barred_idx | join(members_idx, (_grp, doc, who) -> (who, doc)); ++ let beyond = barred_idx | join(members_idx, (_grp, (doc, who)) -> (who, doc)); + let barred_map = ::blocks | map((doc, who) -> (who, doc)); + let barred_rows = barred_map + beyond; + let barred_map2 = barred_rows | map((k, v) -> ((k, v), ())); @@ -27,7 +27,7 @@ let barred = barring::barred | map((who, doc) -> ((who, doc), ())); -barring: { - var barred : (Member, int); -- let beyond = barred | join(members, (_grp, doc, who) -> (who, doc)); +- let beyond = barred | join(members, (_grp, (doc, who)) -> (who, doc)); - set barred = (blocks | map((doc, who) -> (who, doc))) + beyond | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); +sharing: { + -- Time lanes: epoch, then 1 scope coordinates; this scope enters at zero. @@ -35,12 +35,12 @@ + let inherited_map = allowed | map(((who, doc), acc) -> (who, (doc, acc))); + let inherited_idx = inherited_map | arrange; + let members_idx = ::members | arrange; -+ let inherited = inherited_idx | join(members_idx, (_grp, (doc, acc), who) -> ((who, doc), acc)); ++ let inherited = inherited_idx | join(members_idx, (_grp, ((doc, acc), who)) -> ((who, doc), acc)); + let offered_map = ::grants | map((doc, (who, acc)) -> ((who, doc), acc)); + let offered = offered_map + inherited; + let offered_idx = offered | arrange; + let barred_idx = ::barred | arrange; -+ let allowed_join = offered_idx | join(barred_idx, (pair, acc, ()) -> (pair, acc)); ++ let allowed_join = offered_idx | join(barred_idx, (pair, (acc, ())) -> (pair, acc)); + let allowed_neg = allowed_join | negate; + let allowed_rows = offered + allowed_neg; + let allowed_idx = allowed_rows | arrange; @@ -51,7 +51,7 @@ output "access" = sharing::allowed; -sharing: { - var allowed : ((Member, int), Access); -- set allowed = offered + (offered | join(barred, (pair, acc, ()) -> (pair, acc)) | negate) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); +- set allowed = offered + (offered | join(barred, (pair, (acc, ())) -> (pair, acc)) | negate) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); - let offered = (grants | map((doc, (who, acc)) -> ((who, doc), acc))) + inherited; -- let inherited = allowed | map(((who, doc), acc) -> (who, (doc, acc))) | join(members, (_grp, (doc, acc), who) -> ((who, doc), acc)); +- let inherited = allowed | map(((who, doc), acc) -> (who, (doc, acc))) | join(members, (_grp, ((doc, acc), who)) -> ((who, doc), acc)); -} diff --git a/interactive/tests/plan_pairs/expected/access_shared.parsed.ddp b/interactive/tests/plan_pairs/expected/access_shared.parsed.ddp index df600708d..9f36e131c 100644 --- a/interactive/tests/plan_pairs/expected/access_shared.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/access_shared.parsed.ddp @@ -7,13 +7,13 @@ let members = groups | map((grp, who) -> (grp inject Member::Group, who)) | arra let barred = barring::barred | map((who, doc) -> ((who, doc), ())); barring: { var barred : (Member, int); - let beyond = barred | join(members, (_grp, doc, who) -> (who, doc)); + let beyond = barred | join(members, (_grp, (doc, who)) -> (who, doc)); set barred = (blocks | map((doc, who) -> (who, doc))) + beyond | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); } output "access" = sharing::allowed; sharing: { var allowed : ((Member, int), Access); - set allowed = offered + (offered | join(barred, (pair, acc, ()) -> (pair, acc)) | negate) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); + set allowed = offered + (offered | join(barred, (pair, (acc, ())) -> (pair, acc)) | negate) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); let offered = (grants | map((doc, (who, acc)) -> ((who, doc), acc))) + inherited; - let inherited = allowed | map(((who, doc), acc) -> (who, (doc, acc))) | join(members, (_grp, (doc, acc), who) -> ((who, doc), acc)); + let inherited = allowed | map(((who, doc), acc) -> (who, (doc, acc))) | join(members, (_grp, ((doc, acc), who)) -> ((who, doc), acc)); } diff --git a/interactive/tests/plan_pairs/expected/explain.ddp b/interactive/tests/plan_pairs/expected/explain.ddp index ac2a3c288..93a0ce9e5 100644 --- a/interactive/tests/plan_pairs/expected/explain.ddp +++ b/interactive/tests/plan_pairs/expected/explain.ddp @@ -23,7 +23,7 @@ explain: { let demand_input0 = demand_set_0; let item_map = demand_set_0 | map((k, v) -> ((k, v), ())); let item_idx = item_map | arrange; - let item_join = ::input0_map_idx | join(item_idx, (k, v, r) -> (k, ())); + let item_join = ::input0_map_idx | join(item_idx, (k, (v, r)) -> (k, ())); let item_map2 = item_join | map((k, v) -> (k.0, k.1)); let item_map3 = item_map2 | map((a, b) -> ((a, (b)), ())); let item_idx2 = item_map3 | arrange; @@ -35,7 +35,7 @@ explain: { let item_map6 = item_reduce2 | map((k, v) -> (k.0, k.1)); let item_rows = ::rows_map2_map_idx_reduce_map + item_map6; let item_idx4 = item_rows | arrange; - let item_join2 = ::query_map_idx_reduce_map_map_idx | join(item_idx4, (k, v, r) -> (k, (r.0, v.1))); + let item_join2 = ::query_map_idx_reduce_map_map_idx | join(item_idx4, (k, (v, r)) -> (k, (r.0, v.1))); let item_map7 = item_join2 | map((k, v) -> ((k, v), ())); let item_idx5 = item_map7 | arrange; let item_reduce3 = item_idx5 | reduce((_, vals) -> vals map ((v, _) -> (v, 1))); @@ -45,10 +45,10 @@ explain: { let item_map9 = item_reduce4 | map(arg -> let _call0 = let input = arg in let (k, v) = input in (k.0, k.1) in (_call0.0, ((), ()))); let item_idx6 = item_map9 | arrange; let item_idx7 = item_map8 | arrange; - let item_join3 = item_idx6 | join(item_idx7, arg -> let _call0 = let input = arg in let (k, v, r) = input in (r.0, (r.1, v.1)) in (_call0.0, (_call0.1.0, ()))); + let item_join3 = item_idx6 | join(item_idx7, arg -> let _call0 = let input = arg in let (k, (v, r)) = input in (r.0, (r.1, v.1)) in (_call0.0, (_call0.1.0, ()))); let item_map10 = item_join3 | map((a, (b, _)) -> ((a, b), ())); let item_idx8 = item_map10 | arrange; - let item_join4 = item_idx8 | join(::input0_map_idx_reduce_map_map2_idx, (k, v, r) -> (k, ())); + let item_join4 = item_idx8 | join(::input0_map_idx_reduce_map_map2_idx, (k, (v, r)) -> (k, ())); let item_map11 = item_join4 | map((k, v) -> (k.0, k.1)); let item_rows3 = demand_set_0 + item_map11; let item_map12 = item_rows3 | map((k, v) -> ((k, v), ())); diff --git a/interactive/tests/plan_pairs/expected/explain.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/explain.lowered--optimized.diff index f22755d5b..918f4e388 100644 --- a/interactive/tests/plan_pairs/expected/explain.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/explain.lowered--optimized.diff @@ -37,7 +37,7 @@ - let item_map5 = item_map4 | map((k, v) -> ((k, v), ())); - let item_idx2 = item_map5 | arrange; - let item_reduce2 = item_idx2 | reduce((_, vals) -> vals map ((v, _) -> (v, 1))); -+ let item_join = ::input0_map_idx | join(item_idx, (k, v, r) -> (k, ())); ++ let item_join = ::input0_map_idx | join(item_idx, (k, (v, r)) -> (k, ())); + let item_map2 = item_join | map((k, v) -> (k.0, k.1)); + let item_map3 = item_map2 | map((a, b) -> ((a, (b)), ())); + let item_idx2 = item_map3 | arrange; @@ -61,7 +61,7 @@ - let item_map14 = demand_set_0 | map((k, v) -> ((k, v), ())); - let item_idx5 = item_map13 | arrange; - let item_idx6 = item_map14 | arrange; -- let item_join = item_idx5 | join(item_idx6, (k, v, r) -> (k, ())); +- let item_join = item_idx5 | join(item_idx6, (k, (v, r)) -> (k, ())); - let item_map15 = item_join | map((k, v) -> (k.0, k.1)); - let item_map16 = item_map15 | map((k, v) -> (k, (v))); - let item_map17 = item_map16 | map((k, v) -> ((k, v), ())); @@ -94,7 +94,7 @@ - let item_map31 = item_map30 | map((k, v) -> (k, ((), v.1))); - let item_idx13 = item_map31 | arrange; - let item_idx14 = item_rows | arrange; -- let item_join2 = item_idx13 | join(item_idx14, (k, v, r) -> (k, (r.0, v.1))); +- let item_join2 = item_idx13 | join(item_idx14, (k, (v, r)) -> (k, (r.0, v.1))); - let item_map32 = item_join2 | map((k, v) -> (k, (v.0, v.1))); - let item_map33 = item_map32 | map((k, v) -> ((k, v), ())); - let item_idx15 = item_map33 | arrange; @@ -109,14 +109,14 @@ - let item_map38 = item_map36 | map((k, v) -> (k, ((), v.1))); - let item_idx17 = item_map38 | arrange; - let item_idx18 = item_map37 | arrange; -- let item_join3 = item_idx17 | join(item_idx18, (k, v, r) -> (r.0, (r.1, v.1))); +- let item_join3 = item_idx17 | join(item_idx18, (k, (v, r)) -> (r.0, (r.1, v.1))); - let item_map39 = item_join3 | map((k, v) -> (k, (v.0, v.1))); - let item_map40 = item_map39 | map((k, v) -> (k, v.0)); - let item_map41 = item_map40 | map((k, v) -> ((k, v), ())); - let item_map42 = item_map2 | map((k, v) -> ((k, v), ())); - let item_idx19 = item_map41 | arrange; - let item_idx20 = item_map42 | arrange; -- let item_join4 = item_idx19 | join(item_idx20, (k, v, r) -> (k, ())); +- let item_join4 = item_idx19 | join(item_idx20, (k, (v, r)) -> (k, ())); - let item_map43 = item_join4 | map((k, v) -> (k.0, k.1)); - let item_rows3 = demand_set_0 + item_map43; - let item_map44 = item_rows3 | map((k, v) -> ((k, v), ())); @@ -126,7 +126,7 @@ - set demand_set_0 = item_map45; + let item_rows = ::rows_map2_map_idx_reduce_map + item_map6; + let item_idx4 = item_rows | arrange; -+ let item_join2 = ::query_map_idx_reduce_map_map_idx | join(item_idx4, (k, v, r) -> (k, (r.0, v.1))); ++ let item_join2 = ::query_map_idx_reduce_map_map_idx | join(item_idx4, (k, (v, r)) -> (k, (r.0, v.1))); + let item_map7 = item_join2 | map((k, v) -> ((k, v), ())); + let item_idx5 = item_map7 | arrange; + let item_reduce3 = item_idx5 | reduce((_, vals) -> vals map ((v, _) -> (v, 1))); @@ -136,10 +136,10 @@ + let item_map9 = item_reduce4 | map((k, v) -> (k.0, k.1)); + let item_idx6 = item_map9 | arrange; + let item_idx7 = item_map8 | arrange; -+ let item_join3 = item_idx6 | join(item_idx7, (k, v, r) -> (r.0, (r.1, v.1))); ++ let item_join3 = item_idx6 | join(item_idx7, (k, (v, r)) -> (r.0, (r.1, v.1))); + let item_map10 = item_join3 | map((a, (b, _)) -> ((a, b), ())); + let item_idx8 = item_map10 | arrange; -+ let item_join4 = item_idx8 | join(::input0_map_idx_reduce_map_map2_idx, (k, v, r) -> (k, ())); ++ let item_join4 = item_idx8 | join(::input0_map_idx_reduce_map_map2_idx, (k, (v, r)) -> (k, ())); + let item_map11 = item_join4 | map((k, v) -> (k.0, k.1)); + let item_rows3 = demand_set_0 + item_map11; + let item_map12 = item_rows3 | map((k, v) -> ((k, v), ())); diff --git a/interactive/tests/plan_pairs/expected/explain.lowered.ddp b/interactive/tests/plan_pairs/expected/explain.lowered.ddp index aff5dab95..337c5e367 100644 --- a/interactive/tests/plan_pairs/expected/explain.lowered.ddp +++ b/interactive/tests/plan_pairs/expected/explain.lowered.ddp @@ -32,7 +32,7 @@ explain: { let item_map14 = demand_set_0 | map((k, v) -> ((k, v), ())); let item_idx5 = item_map13 | arrange; let item_idx6 = item_map14 | arrange; - let item_join = item_idx5 | join(item_idx6, (k, v, r) -> (k, ())); + let item_join = item_idx5 | join(item_idx6, (k, (v, r)) -> (k, ())); let item_map15 = item_join | map((k, v) -> (k.0, k.1)); let item_map16 = item_map15 | map((k, v) -> (k, (v))); let item_map17 = item_map16 | map((k, v) -> ((k, v), ())); @@ -65,7 +65,7 @@ explain: { let item_map31 = item_map30 | map((k, v) -> (k, ((), v.1))); let item_idx13 = item_map31 | arrange; let item_idx14 = item_rows | arrange; - let item_join2 = item_idx13 | join(item_idx14, (k, v, r) -> (k, (r.0, v.1))); + let item_join2 = item_idx13 | join(item_idx14, (k, (v, r)) -> (k, (r.0, v.1))); let item_map32 = item_join2 | map((k, v) -> (k, (v.0, v.1))); let item_map33 = item_map32 | map((k, v) -> ((k, v), ())); let item_idx15 = item_map33 | arrange; @@ -80,14 +80,14 @@ explain: { let item_map38 = item_map36 | map((k, v) -> (k, ((), v.1))); let item_idx17 = item_map38 | arrange; let item_idx18 = item_map37 | arrange; - let item_join3 = item_idx17 | join(item_idx18, (k, v, r) -> (r.0, (r.1, v.1))); + let item_join3 = item_idx17 | join(item_idx18, (k, (v, r)) -> (r.0, (r.1, v.1))); let item_map39 = item_join3 | map((k, v) -> (k, (v.0, v.1))); let item_map40 = item_map39 | map((k, v) -> (k, v.0)); let item_map41 = item_map40 | map((k, v) -> ((k, v), ())); let item_map42 = item_map2 | map((k, v) -> ((k, v), ())); let item_idx19 = item_map41 | arrange; let item_idx20 = item_map42 | arrange; - let item_join4 = item_idx19 | join(item_idx20, (k, v, r) -> (k, ())); + let item_join4 = item_idx19 | join(item_idx20, (k, (v, r)) -> (k, ())); let item_map43 = item_join4 | map((k, v) -> (k.0, k.1)); let item_rows3 = demand_set_0 + item_map43; let item_map44 = item_rows3 | map((k, v) -> ((k, v), ())); diff --git a/interactive/tests/plan_pairs/expected/explain.optimized--projected.diff b/interactive/tests/plan_pairs/expected/explain.optimized--projected.diff index d621b29ff..2f520dc0c 100644 --- a/interactive/tests/plan_pairs/expected/explain.optimized--projected.diff +++ b/interactive/tests/plan_pairs/expected/explain.optimized--projected.diff @@ -8,8 +8,8 @@ + let item_map9 = item_reduce4 | map(arg -> let _call0 = let input = arg in let (k, v) = input in (k.0, k.1) in (_call0.0, ((), ()))); let item_idx6 = item_map9 | arrange; let item_idx7 = item_map8 | arrange; -- let item_join3 = item_idx6 | join(item_idx7, (k, v, r) -> (r.0, (r.1, v.1))); -+ let item_join3 = item_idx6 | join(item_idx7, arg -> let _call0 = let input = arg in let (k, v, r) = input in (r.0, (r.1, v.1)) in (_call0.0, (_call0.1.0, ()))); +- let item_join3 = item_idx6 | join(item_idx7, (k, (v, r)) -> (r.0, (r.1, v.1))); ++ let item_join3 = item_idx6 | join(item_idx7, arg -> let _call0 = let input = arg in let (k, (v, r)) = input in (r.0, (r.1, v.1)) in (_call0.0, (_call0.1.0, ()))); let item_map10 = item_join3 | map((a, (b, _)) -> ((a, b), ())); let item_idx8 = item_map10 | arrange; - let item_join4 = item_idx8 | join(::input0_map_idx_reduce_map_map2_idx, (k, v, r) -> (k, ())); + let item_join4 = item_idx8 | join(::input0_map_idx_reduce_map_map2_idx, (k, (v, r)) -> (k, ())); diff --git a/interactive/tests/plan_pairs/expected/ldbc_cascade.ddp b/interactive/tests/plan_pairs/expected/ldbc_cascade.ddp index 1bd4cd505..3dff03c7d 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_cascade.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_cascade.ddp @@ -27,7 +27,7 @@ let dead_p_idx = dead_p_map | arrange; let dead_p = dead_p_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); let owned_map = forum_in | map(((id, title, created, moderator), ()) -> (moderator, (id, title, created))); let owned_idx = owned_map | arrange; -let owned = owned_idx | join(dead_p, (moderator, (id, title, created), ()) -> (id, (title, created, moderator))); +let owned = owned_idx | join(dead_p, (moderator, ((id, title, created), ())) -> (id, (title, created, moderator))); let dead_f_map = del_forum | map(((id, _), ()) -> (id, ())); let dead_f_rows = owned | filter((_, (title, _, _)) -> ((((0, (title len, 6) min) range, title) gather, "Album ") eq, (((0, (title len, 5) min) range, title) gather, "Wall ") eq) any) | map((id, _) -> (id, ())); let dead_f_rows2 = dead_f_map + dead_f_rows; @@ -35,16 +35,16 @@ let dead_f_idx = dead_f_rows2 | arrange; let dead_f = dead_f_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); let groups = owned | filter((_, (title, _, _)) -> (((0, (title len, 6) min) range, title) gather, "Group ") eq); let groups_idx = groups | arrange; -let kept_join = groups_idx | join(dead_f, (id, v, ()) -> (id, v)); +let kept_join = groups_idx | join(dead_f, (id, (v, ())) -> (id, v)); let kept_neg = kept_join | negate; let kept = groups + kept_neg; let seeds_map = del_message | map(((id, _), ()) -> (id, ())); let seeds_map2 = message_in | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (creator, id)); let seeds_idx = seeds_map2 | arrange; -let seeds_join = seeds_idx | join(dead_p, (_, id, ()) -> (id, ())); +let seeds_join = seeds_idx | join(dead_p, (_, (id, ())) -> (id, ())); let seeds_rows = message_in | filter(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (kind, 0) eq) | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (forum, id)); let seeds_idx2 = seeds_rows | arrange; -let seeds_join2 = seeds_idx2 | join(dead_f, (_, id, ()) -> (id, ())); +let seeds_join2 = seeds_idx2 | join(dead_f, (_, (id, ())) -> (id, ())); let seeds = seeds_map + seeds_join + seeds_join2; let children = message_in | filter(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (kind, 1) eq) | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (parent, id)); let children_idx = children | arrange; @@ -52,7 +52,7 @@ dead_m: { -- Time lanes: epoch, then 1 scope coordinates; this scope enters at zero. var dead_m; let dead_m_idx = dead_m | arrange; - let kids = dead_m_idx | join(::children_idx, (_, (), child) -> (child, ())); + let kids = dead_m_idx | join(::children_idx, (_, ((), child)) -> (child, ())); let dead_m_rows = ::seeds + kids; let dead_m_idx2 = dead_m_rows | arrange; let dead_m_reduce = dead_m_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); @@ -70,7 +70,7 @@ let gone_likes_idx = gone_likes_map | arrange; let gone_likes = gone_likes_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); let person_map = person_in | map(((id, first, last, gender, birthday, created, ip, browser, city, birthmonth, birthdaynum, creationmonth), ()) -> (id, (id, first, last, gender, birthday, created, ip, browser, city, birthmonth, birthdaynum, creationmonth))); let person_idx = person_map | arrange; -let person_join = person_idx | join(dead_p, (_, r, ()) -> (r, ())); +let person_join = person_idx | join(dead_p, (_, (r, ())) -> (r, ())); let person_idx2 = person_join | arrange; let person_reduce = person_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); let person_neg = person_reduce | negate; @@ -78,14 +78,14 @@ let person = person_in + person_neg; let message_map = message_in | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (id, (id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day))); let message_idx = message_map | arrange; let dead_msg_idx = dead_msg | arrange; -let message_join = message_idx | join(dead_msg_idx, (_, r, ()) -> (r, ())); +let message_join = message_idx | join(dead_msg_idx, (_, (r, ())) -> (r, ())); let message_idx2 = message_join | arrange; let message_reduce = message_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); let message_neg = message_reduce | negate; let message = message_in + message_neg; let forum_map = forum_in | map(((id, title, created, moderator), ()) -> (id, (id, title, created, moderator))); let forum_idx = forum_map | arrange; -let forum_join = forum_idx | join(dead_f, (_, r, ()) -> (r, ())); +let forum_join = forum_idx | join(dead_f, (_, (r, ())) -> (r, ())); let forum_map2 = kept | map((id, (title, created, moderator)) -> ((id, title, created, moderator), ())); let forum_rows = forum_join + forum_map2; let forum_idx2 = forum_rows | arrange; @@ -95,13 +95,13 @@ let forum_map3 = kept | map((id, (title, created, _)) -> ((id, title, created, - let forum = forum_in + forum_neg + forum_map3; let knows_map = knows_in | map(((src, dst, created), ()) -> ((src, dst), (src, dst, created))); let knows_idx = knows_map | arrange; -let knows_join = knows_idx | join(gone_knows, (_, r, ()) -> (r, ())); +let knows_join = knows_idx | join(gone_knows, (_, (r, ())) -> (r, ())); let knows_map2 = knows_in | map(((src, dst, created), ()) -> (src, (src, dst, created))); let knows_idx2 = knows_map2 | arrange; -let knows_join2 = knows_idx2 | join(dead_p, (_, r, ()) -> (r, ())); +let knows_join2 = knows_idx2 | join(dead_p, (_, (r, ())) -> (r, ())); let knows_map3 = knows_in | map(((src, dst, created), ()) -> (dst, (src, dst, created))); let knows_idx3 = knows_map3 | arrange; -let knows_join3 = knows_idx3 | join(dead_p, (_, r, ()) -> (r, ())); +let knows_join3 = knows_idx3 | join(dead_p, (_, (r, ())) -> (r, ())); let knows_rows = knows_join + knows_join2 + knows_join3; let knows_idx4 = knows_rows | arrange; let knows_reduce = knows_idx4 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); @@ -109,13 +109,13 @@ let knows_neg = knows_reduce | negate; let knows = knows_in + knows_neg; let member_map = member_in | map(((forum, person, created), ()) -> ((forum, person), (forum, person, created))); let member_idx = member_map | arrange; -let member_join = member_idx | join(gone_member, (_, r, ()) -> (r, ())); +let member_join = member_idx | join(gone_member, (_, (r, ())) -> (r, ())); let member_map2 = member_in | map(((forum, person, created), ()) -> (forum, (forum, person, created))); let member_idx2 = member_map2 | arrange; -let member_join2 = member_idx2 | join(dead_f, (_, r, ()) -> (r, ())); +let member_join2 = member_idx2 | join(dead_f, (_, (r, ())) -> (r, ())); let member_map3 = member_in | map(((forum, person, created), ()) -> (person, (forum, person, created))); let member_idx3 = member_map3 | arrange; -let member_join3 = member_idx3 | join(dead_p, (_, r, ()) -> (r, ())); +let member_join3 = member_idx3 | join(dead_p, (_, (r, ())) -> (r, ())); let member_rows = member_join + member_join2 + member_join3; let member_idx4 = member_rows | arrange; let member_reduce = member_idx4 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); @@ -123,34 +123,34 @@ let member_neg = member_reduce | negate; let member = member_in + member_neg; let interest_map = interest_in | map(((person, tag), ()) -> (person, (person, tag))); let interest_idx = interest_map | arrange; -let interest_join = interest_idx | join(dead_p, (_, r, ()) -> (r, ())); +let interest_join = interest_idx | join(dead_p, (_, (r, ())) -> (r, ())); let interest_idx2 = interest_join | arrange; let interest_reduce = interest_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); let interest_neg = interest_reduce | negate; let interest = interest_in + interest_neg; let mtag_map = mtag_in | map(((message, tag), ()) -> (message, (message, tag))); let mtag_idx = mtag_map | arrange; -let mtag_join = mtag_idx | join(dead_msg_idx, (_, r, ()) -> (r, ())); +let mtag_join = mtag_idx | join(dead_msg_idx, (_, (r, ())) -> (r, ())); let mtag_idx2 = mtag_join | arrange; let mtag_reduce = mtag_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); let mtag_neg = mtag_reduce | negate; let mtag = mtag_in + mtag_neg; let ftag_map = ftag_in | map(((forum, tag), ()) -> (forum, (forum, tag))); let ftag_idx = ftag_map | arrange; -let ftag_join = ftag_idx | join(dead_f, (_, r, ()) -> (r, ())); +let ftag_join = ftag_idx | join(dead_f, (_, (r, ())) -> (r, ())); let ftag_idx2 = ftag_join | arrange; let ftag_reduce = ftag_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); let ftag_neg = ftag_reduce | negate; let ftag = ftag_in + ftag_neg; let likes_map = likes_in | map(((person, message, created), ()) -> ((person, message), (person, message, created))); let likes_idx = likes_map | arrange; -let likes_join = likes_idx | join(gone_likes, (_, r, ()) -> (r, ())); +let likes_join = likes_idx | join(gone_likes, (_, (r, ())) -> (r, ())); let likes_map2 = likes_in | map(((person, message, created), ()) -> (person, (person, message, created))); let likes_idx2 = likes_map2 | arrange; -let likes_join2 = likes_idx2 | join(dead_p, (_, r, ()) -> (r, ())); +let likes_join2 = likes_idx2 | join(dead_p, (_, (r, ())) -> (r, ())); let likes_map3 = likes_in | map(((person, message, created), ()) -> (message, (person, message, created))); let likes_idx3 = likes_map3 | arrange; -let likes_join3 = likes_idx3 | join(dead_msg_idx, (_, r, ()) -> (r, ())); +let likes_join3 = likes_idx3 | join(dead_msg_idx, (_, (r, ())) -> (r, ())); let likes_rows = likes_join + likes_join2 + likes_join3; let likes_idx4 = likes_rows | arrange; let likes_reduce = likes_idx4 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); @@ -158,28 +158,28 @@ let likes_neg = likes_reduce | negate; let likes = likes_in + likes_neg; let study_map = study_in | map(((person, org, year), ()) -> (person, (person, org, year))); let study_idx = study_map | arrange; -let study_join = study_idx | join(dead_p, (_, r, ()) -> (r, ())); +let study_join = study_idx | join(dead_p, (_, (r, ())) -> (r, ())); let study_idx2 = study_join | arrange; let study_reduce = study_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); let study_neg = study_reduce | negate; let study = study_in + study_neg; let work_map = work_in | map(((person, org, year), ()) -> (person, (person, org, year))); let work_idx = work_map | arrange; -let work_join = work_idx | join(dead_p, (_, r, ()) -> (r, ())); +let work_join = work_idx | join(dead_p, (_, (r, ())) -> (r, ())); let work_idx2 = work_join | arrange; let work_reduce = work_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); let work_neg = work_reduce | negate; let work = work_in + work_neg; let email_map = email_in | map(((person, value), ()) -> (person, (person, value))); let email_idx = email_map | arrange; -let email_join = email_idx | join(dead_p, (_, r, ()) -> (r, ())); +let email_join = email_idx | join(dead_p, (_, (r, ())) -> (r, ())); let email_idx2 = email_join | arrange; let email_reduce = email_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); let email_neg = email_reduce | negate; let email = email_in + email_neg; let language_map = language_in | map(((person, value), ()) -> (person, (person, value))); let language_idx = language_map | arrange; -let language_join = language_idx | join(dead_p, (_, r, ()) -> (r, ())); +let language_join = language_idx | join(dead_p, (_, (r, ())) -> (r, ())); let language_idx2 = language_join | arrange; let language_reduce = language_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); let language_neg = language_reduce | negate; @@ -191,9 +191,9 @@ let either_map = knows | map(((a, b, t), ()) -> (a, (b, t))); let either_map2 = knows | map(((a, b, t), ()) -> (b, (a, t))); let either = either_map + either_map2; let either_idx = either | arrange; -let friends_join = either_idx | join(exists, (a, (b, t), ()) -> (b, (a, t))); +let friends_join = either_idx | join(exists, (a, ((b, t), ())) -> (b, (a, t))); let friends_idx = friends_join | arrange; -let friends_join2 = friends_idx | join(exists, (b, (a, t), ()) -> ((a, b, t), ())); +let friends_join2 = friends_idx | join(exists, (b, ((a, t), ())) -> ((a, b, t), ())); let friends_idx2 = friends_join2 | arrange; let friends_reduce = friends_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); let friends = friends_reduce | map(((a, b, t), ()) -> (a, (b, t))); @@ -205,7 +205,7 @@ thread: { var thread; let below_map = thread | map(((mid, root), ()) -> (mid, root)); let below_idx = below_map | arrange; - let below = below_idx | join(::replies_idx, (_, root, child) -> ((child, root), ())); + let below = below_idx | join(::replies_idx, (_, (root, child)) -> ((child, root), ())); let thread_rows = ::posts + below; let thread_idx = thread_rows | arrange; let thread_reduce = thread_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_cascade.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/ldbc_cascade.lowered--optimized.diff index d93816951..073b229b3 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_cascade.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_cascade.lowered--optimized.diff @@ -13,7 +13,7 @@ let dead_p = dead_p_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); @@ -33,10 +29,9 @@ let owned_idx = owned_map | arrange; - let owned = owned_idx | join(dead_p, (moderator, (id, title, created), ()) -> (id, (title, created, moderator))); + let owned = owned_idx | join(dead_p, (moderator, ((id, title, created), ())) -> (id, (title, created, moderator))); let dead_f_map = del_forum | map(((id, _), ()) -> (id, ())); -let dead_f_filter = owned | filter((_, (title, _, _)) -> ((((0, (title len, 6) min) range, title) gather, "Album ") eq, (((0, (title len, 5) min) range, title) gather, "Wall ") eq) any); -let dead_f_map2 = dead_f_filter | map((id, _) -> (id, ())); @@ -28,13 +28,13 @@ @@ -47,19 +42,17 @@ let seeds_map2 = message_in | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (creator, id)); let seeds_idx = seeds_map2 | arrange; - let seeds_join = seeds_idx | join(dead_p, (_, id, ()) -> (id, ())); + let seeds_join = seeds_idx | join(dead_p, (_, (id, ())) -> (id, ())); -let seeds_filter = message_in | filter(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (kind, 0) eq); -let seeds_map3 = seeds_filter | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (forum, id)); -let seeds_idx2 = seeds_map3 | arrange; +let seeds_rows = message_in | filter(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (kind, 0) eq) | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (forum, id)); +let seeds_idx2 = seeds_rows | arrange; - let seeds_join2 = seeds_idx2 | join(dead_f, (_, id, ()) -> (id, ())); + let seeds_join2 = seeds_idx2 | join(dead_f, (_, (id, ())) -> (id, ())); let seeds = seeds_map + seeds_join + seeds_join2; -let children_filter = message_in | filter(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (kind, 1) eq); -let children = children_filter | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (parent, id)); @@ -45,8 +45,8 @@ var dead_m; let dead_m_idx = dead_m | arrange; - let children_idx = ::children | arrange; -- let kids = dead_m_idx | join(children_idx, (_, (), child) -> (child, ())); -+ let kids = dead_m_idx | join(::children_idx, (_, (), child) -> (child, ())); +- let kids = dead_m_idx | join(children_idx, (_, ((), child)) -> (child, ())); ++ let kids = dead_m_idx | join(::children_idx, (_, ((), child)) -> (child, ())); let dead_m_rows = ::seeds + kids; let dead_m_idx2 = dead_m_rows | arrange; let dead_m_reduce = dead_m_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); @@ -55,18 +55,18 @@ let mtag_map = mtag_in | map(((message, tag), ()) -> (message, (message, tag))); let mtag_idx = mtag_map | arrange; -let dead_msg_idx2 = dead_msg | arrange; --let mtag_join = mtag_idx | join(dead_msg_idx2, (_, r, ()) -> (r, ())); -+let mtag_join = mtag_idx | join(dead_msg_idx, (_, r, ()) -> (r, ())); +-let mtag_join = mtag_idx | join(dead_msg_idx2, (_, (r, ())) -> (r, ())); ++let mtag_join = mtag_idx | join(dead_msg_idx, (_, (r, ())) -> (r, ())); let mtag_idx2 = mtag_join | arrange; let mtag_reduce = mtag_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); let mtag_neg = mtag_reduce | negate; @@ -158,8 +150,7 @@ - let likes_join2 = likes_idx2 | join(dead_p, (_, r, ()) -> (r, ())); + let likes_join2 = likes_idx2 | join(dead_p, (_, (r, ())) -> (r, ())); let likes_map3 = likes_in | map(((person, message, created), ()) -> (message, (person, message, created))); let likes_idx3 = likes_map3 | arrange; -let dead_msg_idx3 = dead_msg | arrange; --let likes_join3 = likes_idx3 | join(dead_msg_idx3, (_, r, ()) -> (r, ())); -+let likes_join3 = likes_idx3 | join(dead_msg_idx, (_, r, ()) -> (r, ())); +-let likes_join3 = likes_idx3 | join(dead_msg_idx3, (_, (r, ())) -> (r, ())); ++let likes_join3 = likes_idx3 | join(dead_msg_idx, (_, (r, ())) -> (r, ())); let likes_rows = likes_join + likes_join2 + likes_join3; let likes_idx4 = likes_rows | arrange; let likes_reduce = likes_idx4 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); @@ -87,8 +87,8 @@ let below_map = thread | map(((mid, root), ()) -> (mid, root)); let below_idx = below_map | arrange; - let replies_idx = ::replies | arrange; -- let below = below_idx | join(replies_idx, (_, root, child) -> ((child, root), ())); -+ let below = below_idx | join(::replies_idx, (_, root, child) -> ((child, root), ())); +- let below = below_idx | join(replies_idx, (_, (root, child)) -> ((child, root), ())); ++ let below = below_idx | join(::replies_idx, (_, (root, child)) -> ((child, root), ())); let thread_rows = ::posts + below; let thread_idx = thread_rows | arrange; let thread_reduce = thread_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_cascade.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_cascade.parsed--lowered.diff index 7bac72e8a..13d84d56a 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_cascade.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_cascade.parsed--lowered.diff @@ -10,37 +10,37 @@ let del_member = input 21 : ((int, int, int), ()); let del_likes = input 22 : ((int, int, int), ()); -let dead_p = del_person | map(((id, _), ()) -> (id, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); --let owned = forum_in | map(((id, title, created, moderator), ()) -> (moderator, (id, title, created))) | join(dead_p, (moderator, (id, title, created), ()) -> (id, (title, created, moderator))); +-let owned = forum_in | map(((id, title, created, moderator), ()) -> (moderator, (id, title, created))) | join(dead_p, (moderator, ((id, title, created), ())) -> (id, (title, created, moderator))); -let dead_f = (del_forum | map(((id, _), ()) -> (id, ()))) + (owned | filter((_, (title, _, _)) -> ((((0, (title len, 6) min) range, title) gather, "Album ") eq, (((0, (title len, 5) min) range, title) gather, "Wall ") eq) any) | map((id, _) -> (id, ()))) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); -let groups = owned | filter((_, (title, _, _)) -> (((0, (title len, 6) min) range, title) gather, "Group ") eq); --let kept = groups + (groups | join(dead_f, (id, v, ()) -> (id, v)) | negate); --let seeds = (del_message | map(((id, _), ()) -> (id, ()))) + (message_in | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (creator, id)) | join(dead_p, (_, id, ()) -> (id, ()))) + (message_in | filter(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (kind, 0) eq) | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (forum, id)) | join(dead_f, (_, id, ()) -> (id, ()))); +-let kept = groups + (groups | join(dead_f, (id, (v, ())) -> (id, v)) | negate); +-let seeds = (del_message | map(((id, _), ()) -> (id, ()))) + (message_in | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (creator, id)) | join(dead_p, (_, (id, ())) -> (id, ()))) + (message_in | filter(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (kind, 0) eq) | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (forum, id)) | join(dead_f, (_, (id, ())) -> (id, ()))); -let children = message_in | filter(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (kind, 1) eq) | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (parent, id)); -dead_m: { -- let kids = dead_m | join(children, (_, (), child) -> (child, ())); +- let kids = dead_m | join(children, (_, ((), child)) -> (child, ())); - set dead_m = seeds + kids | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); -} -let dead_msg = dead_m::dead_m; -let gone_knows = del_knows | map(((a, b, _), ()) -> ((a, b), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); -let gone_member = del_member | map(((f, p, _), ()) -> ((f, p), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); -let gone_likes = del_likes | map(((p, x, _), ()) -> ((p, x), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); --let person = person_in + (person_in | map(((id, first, last, gender, birthday, created, ip, browser, city, birthmonth, birthdaynum, creationmonth), ()) -> (id, (id, first, last, gender, birthday, created, ip, browser, city, birthmonth, birthdaynum, creationmonth))) | join(dead_p, (_, r, ()) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); --let message = message_in + (message_in | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (id, (id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day))) | join(dead_msg, (_, r, ()) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); --let forum = forum_in + ((forum_in | map(((id, title, created, moderator), ()) -> (id, (id, title, created, moderator))) | join(dead_f, (_, r, ()) -> (r, ()))) + (kept | map((id, (title, created, moderator)) -> ((id, title, created, moderator), ()))) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate) + (kept | map((id, (title, created, _)) -> ((id, title, created, -1), ()))); --let knows = knows_in + ((knows_in | map(((src, dst, created), ()) -> ((src, dst), (src, dst, created))) | join(gone_knows, (_, r, ()) -> (r, ()))) + (knows_in | map(((src, dst, created), ()) -> (src, (src, dst, created))) | join(dead_p, (_, r, ()) -> (r, ()))) + (knows_in | map(((src, dst, created), ()) -> (dst, (src, dst, created))) | join(dead_p, (_, r, ()) -> (r, ()))) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); --let member = member_in + ((member_in | map(((forum, person, created), ()) -> ((forum, person), (forum, person, created))) | join(gone_member, (_, r, ()) -> (r, ()))) + (member_in | map(((forum, person, created), ()) -> (forum, (forum, person, created))) | join(dead_f, (_, r, ()) -> (r, ()))) + (member_in | map(((forum, person, created), ()) -> (person, (forum, person, created))) | join(dead_p, (_, r, ()) -> (r, ()))) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); +-let person = person_in + (person_in | map(((id, first, last, gender, birthday, created, ip, browser, city, birthmonth, birthdaynum, creationmonth), ()) -> (id, (id, first, last, gender, birthday, created, ip, browser, city, birthmonth, birthdaynum, creationmonth))) | join(dead_p, (_, (r, ())) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); +-let message = message_in + (message_in | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (id, (id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day))) | join(dead_msg, (_, (r, ())) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); +-let forum = forum_in + ((forum_in | map(((id, title, created, moderator), ()) -> (id, (id, title, created, moderator))) | join(dead_f, (_, (r, ())) -> (r, ()))) + (kept | map((id, (title, created, moderator)) -> ((id, title, created, moderator), ()))) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate) + (kept | map((id, (title, created, _)) -> ((id, title, created, -1), ()))); +-let knows = knows_in + ((knows_in | map(((src, dst, created), ()) -> ((src, dst), (src, dst, created))) | join(gone_knows, (_, (r, ())) -> (r, ()))) + (knows_in | map(((src, dst, created), ()) -> (src, (src, dst, created))) | join(dead_p, (_, (r, ())) -> (r, ()))) + (knows_in | map(((src, dst, created), ()) -> (dst, (src, dst, created))) | join(dead_p, (_, (r, ())) -> (r, ()))) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); +-let member = member_in + ((member_in | map(((forum, person, created), ()) -> ((forum, person), (forum, person, created))) | join(gone_member, (_, (r, ())) -> (r, ()))) + (member_in | map(((forum, person, created), ()) -> (forum, (forum, person, created))) | join(dead_f, (_, (r, ())) -> (r, ()))) + (member_in | map(((forum, person, created), ()) -> (person, (forum, person, created))) | join(dead_p, (_, (r, ())) -> (r, ()))) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); let tag = tag_in; let tagclass = tagclass_in; let place = place_in; let org = org_in; --let interest = interest_in + (interest_in | map(((person, tag), ()) -> (person, (person, tag))) | join(dead_p, (_, r, ()) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); --let mtag = mtag_in + (mtag_in | map(((message, tag), ()) -> (message, (message, tag))) | join(dead_msg, (_, r, ()) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); --let ftag = ftag_in + (ftag_in | map(((forum, tag), ()) -> (forum, (forum, tag))) | join(dead_f, (_, r, ()) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); --let likes = likes_in + ((likes_in | map(((person, message, created), ()) -> ((person, message), (person, message, created))) | join(gone_likes, (_, r, ()) -> (r, ()))) + (likes_in | map(((person, message, created), ()) -> (person, (person, message, created))) | join(dead_p, (_, r, ()) -> (r, ()))) + (likes_in | map(((person, message, created), ()) -> (message, (person, message, created))) | join(dead_msg, (_, r, ()) -> (r, ()))) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); --let study = study_in + (study_in | map(((person, org, year), ()) -> (person, (person, org, year))) | join(dead_p, (_, r, ()) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); --let work = work_in + (work_in | map(((person, org, year), ()) -> (person, (person, org, year))) | join(dead_p, (_, r, ()) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); --let email = email_in + (email_in | map(((person, value), ()) -> (person, (person, value))) | join(dead_p, (_, r, ()) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); --let language = language_in + (language_in | map(((person, value), ()) -> (person, (person, value))) | join(dead_p, (_, r, ()) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); +-let interest = interest_in + (interest_in | map(((person, tag), ()) -> (person, (person, tag))) | join(dead_p, (_, (r, ())) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); +-let mtag = mtag_in + (mtag_in | map(((message, tag), ()) -> (message, (message, tag))) | join(dead_msg, (_, (r, ())) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); +-let ftag = ftag_in + (ftag_in | map(((forum, tag), ()) -> (forum, (forum, tag))) | join(dead_f, (_, (r, ())) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); +-let likes = likes_in + ((likes_in | map(((person, message, created), ()) -> ((person, message), (person, message, created))) | join(gone_likes, (_, (r, ())) -> (r, ()))) + (likes_in | map(((person, message, created), ()) -> (person, (person, message, created))) | join(dead_p, (_, (r, ())) -> (r, ()))) + (likes_in | map(((person, message, created), ()) -> (message, (person, message, created))) | join(dead_msg, (_, (r, ())) -> (r, ()))) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); +-let study = study_in + (study_in | map(((person, org, year), ()) -> (person, (person, org, year))) | join(dead_p, (_, (r, ())) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); +-let work = work_in + (work_in | map(((person, org, year), ()) -> (person, (person, org, year))) | join(dead_p, (_, (r, ())) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); +-let email = email_in + (email_in | map(((person, value), ()) -> (person, (person, value))) | join(dead_p, (_, (r, ())) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); +-let language = language_in + (language_in | map(((person, value), ()) -> (person, (person, value))) | join(dead_p, (_, (r, ())) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); -output "person" = person | map(((id, first, last, gender, birthday, created, ip, browser, city, birthmonth, birthdaynum, creationmonth), ()) -> (id, (first, last, gender, birthday, created, ip, browser, city, birthmonth, birthdaynum, creationmonth))) | arrange; -output "person.by_city" = person | map(((id, first, last, gender, birthday, created, ip, browser, city, birthmonth, birthdaynum, creationmonth), ()) -> (city, id)) | arrange; -output "message" = message | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (id, (kind, created, creator, country, content, image, length, language, forum, parent, year, month, day))) | arrange; @@ -77,7 +77,7 @@ -output "language.by_person" = language | map(((person, value), ()) -> (person, value)) | arrange; -let exists = person | map(((id, _, _, _, _, _, _, _, _, _, _, _), ()) -> (id, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); -let either = (knows | map(((a, b, t), ()) -> (a, (b, t)))) + (knows | map(((a, b, t), ()) -> (b, (a, t)))); --let friends = either | join(exists, (a, (b, t), ()) -> (b, (a, t))) | join(exists, (b, (a, t), ()) -> ((a, b, t), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((a, b, t), ()) -> (a, (b, t))); +-let friends = either | join(exists, (a, ((b, t), ())) -> (b, (a, t))) | join(exists, (b, ((a, t), ())) -> ((a, b, t), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((a, b, t), ()) -> (a, (b, t))); -output "friend" = friends | arrange; -output "friend.pair" = friends | map((a, (b, _)) -> ((a, b), ())) | arrange; -let posts = message | filter(((_, kind, _, _, _, _, _, _, _, _, _, _, _, _), ()) -> (kind, 0) eq) | map(((id, _, _, _, _, _, _, _, _, _, _, _, _, _), ()) -> ((id, id), ())); @@ -87,7 +87,7 @@ +let dead_p = dead_p_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); +let owned_map = forum_in | map(((id, title, created, moderator), ()) -> (moderator, (id, title, created))); +let owned_idx = owned_map | arrange; -+let owned = owned_idx | join(dead_p, (moderator, (id, title, created), ()) -> (id, (title, created, moderator))); ++let owned = owned_idx | join(dead_p, (moderator, ((id, title, created), ())) -> (id, (title, created, moderator))); +let dead_f_map = del_forum | map(((id, _), ()) -> (id, ())); +let dead_f_filter = owned | filter((_, (title, _, _)) -> ((((0, (title len, 6) min) range, title) gather, "Album ") eq, (((0, (title len, 5) min) range, title) gather, "Wall ") eq) any); +let dead_f_map2 = dead_f_filter | map((id, _) -> (id, ())); @@ -96,17 +96,17 @@ +let dead_f = dead_f_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); +let groups = owned | filter((_, (title, _, _)) -> (((0, (title len, 6) min) range, title) gather, "Group ") eq); +let groups_idx = groups | arrange; -+let kept_join = groups_idx | join(dead_f, (id, v, ()) -> (id, v)); ++let kept_join = groups_idx | join(dead_f, (id, (v, ())) -> (id, v)); +let kept_neg = kept_join | negate; +let kept = groups + kept_neg; +let seeds_map = del_message | map(((id, _), ()) -> (id, ())); +let seeds_map2 = message_in | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (creator, id)); +let seeds_idx = seeds_map2 | arrange; -+let seeds_join = seeds_idx | join(dead_p, (_, id, ()) -> (id, ())); ++let seeds_join = seeds_idx | join(dead_p, (_, (id, ())) -> (id, ())); +let seeds_filter = message_in | filter(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (kind, 0) eq); +let seeds_map3 = seeds_filter | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (forum, id)); +let seeds_idx2 = seeds_map3 | arrange; -+let seeds_join2 = seeds_idx2 | join(dead_f, (_, id, ()) -> (id, ())); ++let seeds_join2 = seeds_idx2 | join(dead_f, (_, (id, ())) -> (id, ())); +let seeds = seeds_map + seeds_join + seeds_join2; +let children_filter = message_in | filter(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (kind, 1) eq); +let children = children_filter | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (parent, id)); @@ -115,7 +115,7 @@ + var dead_m; + let dead_m_idx = dead_m | arrange; + let children_idx = ::children | arrange; -+ let kids = dead_m_idx | join(children_idx, (_, (), child) -> (child, ())); ++ let kids = dead_m_idx | join(children_idx, (_, ((), child)) -> (child, ())); + let dead_m_rows = ::seeds + kids; + let dead_m_idx2 = dead_m_rows | arrange; + let dead_m_reduce = dead_m_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); @@ -133,7 +133,7 @@ +let gone_likes = gone_likes_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); +let person_map = person_in | map(((id, first, last, gender, birthday, created, ip, browser, city, birthmonth, birthdaynum, creationmonth), ()) -> (id, (id, first, last, gender, birthday, created, ip, browser, city, birthmonth, birthdaynum, creationmonth))); +let person_idx = person_map | arrange; -+let person_join = person_idx | join(dead_p, (_, r, ()) -> (r, ())); ++let person_join = person_idx | join(dead_p, (_, (r, ())) -> (r, ())); +let person_idx2 = person_join | arrange; +let person_reduce = person_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); +let person_neg = person_reduce | negate; @@ -141,14 +141,14 @@ +let message_map = message_in | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (id, (id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day))); +let message_idx = message_map | arrange; +let dead_msg_idx = dead_msg | arrange; -+let message_join = message_idx | join(dead_msg_idx, (_, r, ()) -> (r, ())); ++let message_join = message_idx | join(dead_msg_idx, (_, (r, ())) -> (r, ())); +let message_idx2 = message_join | arrange; +let message_reduce = message_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); +let message_neg = message_reduce | negate; +let message = message_in + message_neg; +let forum_map = forum_in | map(((id, title, created, moderator), ()) -> (id, (id, title, created, moderator))); +let forum_idx = forum_map | arrange; -+let forum_join = forum_idx | join(dead_f, (_, r, ()) -> (r, ())); ++let forum_join = forum_idx | join(dead_f, (_, (r, ())) -> (r, ())); +let forum_map2 = kept | map((id, (title, created, moderator)) -> ((id, title, created, moderator), ())); +let forum_rows = forum_join + forum_map2; +let forum_idx2 = forum_rows | arrange; @@ -158,13 +158,13 @@ +let forum = forum_in + forum_neg + forum_map3; +let knows_map = knows_in | map(((src, dst, created), ()) -> ((src, dst), (src, dst, created))); +let knows_idx = knows_map | arrange; -+let knows_join = knows_idx | join(gone_knows, (_, r, ()) -> (r, ())); ++let knows_join = knows_idx | join(gone_knows, (_, (r, ())) -> (r, ())); +let knows_map2 = knows_in | map(((src, dst, created), ()) -> (src, (src, dst, created))); +let knows_idx2 = knows_map2 | arrange; -+let knows_join2 = knows_idx2 | join(dead_p, (_, r, ()) -> (r, ())); ++let knows_join2 = knows_idx2 | join(dead_p, (_, (r, ())) -> (r, ())); +let knows_map3 = knows_in | map(((src, dst, created), ()) -> (dst, (src, dst, created))); +let knows_idx3 = knows_map3 | arrange; -+let knows_join3 = knows_idx3 | join(dead_p, (_, r, ()) -> (r, ())); ++let knows_join3 = knows_idx3 | join(dead_p, (_, (r, ())) -> (r, ())); +let knows_rows = knows_join + knows_join2 + knows_join3; +let knows_idx4 = knows_rows | arrange; +let knows_reduce = knows_idx4 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); @@ -172,13 +172,13 @@ +let knows = knows_in + knows_neg; +let member_map = member_in | map(((forum, person, created), ()) -> ((forum, person), (forum, person, created))); +let member_idx = member_map | arrange; -+let member_join = member_idx | join(gone_member, (_, r, ()) -> (r, ())); ++let member_join = member_idx | join(gone_member, (_, (r, ())) -> (r, ())); +let member_map2 = member_in | map(((forum, person, created), ()) -> (forum, (forum, person, created))); +let member_idx2 = member_map2 | arrange; -+let member_join2 = member_idx2 | join(dead_f, (_, r, ()) -> (r, ())); ++let member_join2 = member_idx2 | join(dead_f, (_, (r, ())) -> (r, ())); +let member_map3 = member_in | map(((forum, person, created), ()) -> (person, (forum, person, created))); +let member_idx3 = member_map3 | arrange; -+let member_join3 = member_idx3 | join(dead_p, (_, r, ()) -> (r, ())); ++let member_join3 = member_idx3 | join(dead_p, (_, (r, ())) -> (r, ())); +let member_rows = member_join + member_join2 + member_join3; +let member_idx4 = member_rows | arrange; +let member_reduce = member_idx4 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); @@ -186,7 +186,7 @@ +let member = member_in + member_neg; +let interest_map = interest_in | map(((person, tag), ()) -> (person, (person, tag))); +let interest_idx = interest_map | arrange; -+let interest_join = interest_idx | join(dead_p, (_, r, ()) -> (r, ())); ++let interest_join = interest_idx | join(dead_p, (_, (r, ())) -> (r, ())); +let interest_idx2 = interest_join | arrange; +let interest_reduce = interest_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); +let interest_neg = interest_reduce | negate; @@ -194,28 +194,28 @@ +let mtag_map = mtag_in | map(((message, tag), ()) -> (message, (message, tag))); +let mtag_idx = mtag_map | arrange; +let dead_msg_idx2 = dead_msg | arrange; -+let mtag_join = mtag_idx | join(dead_msg_idx2, (_, r, ()) -> (r, ())); ++let mtag_join = mtag_idx | join(dead_msg_idx2, (_, (r, ())) -> (r, ())); +let mtag_idx2 = mtag_join | arrange; +let mtag_reduce = mtag_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); +let mtag_neg = mtag_reduce | negate; +let mtag = mtag_in + mtag_neg; +let ftag_map = ftag_in | map(((forum, tag), ()) -> (forum, (forum, tag))); +let ftag_idx = ftag_map | arrange; -+let ftag_join = ftag_idx | join(dead_f, (_, r, ()) -> (r, ())); ++let ftag_join = ftag_idx | join(dead_f, (_, (r, ())) -> (r, ())); +let ftag_idx2 = ftag_join | arrange; +let ftag_reduce = ftag_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); +let ftag_neg = ftag_reduce | negate; +let ftag = ftag_in + ftag_neg; +let likes_map = likes_in | map(((person, message, created), ()) -> ((person, message), (person, message, created))); +let likes_idx = likes_map | arrange; -+let likes_join = likes_idx | join(gone_likes, (_, r, ()) -> (r, ())); ++let likes_join = likes_idx | join(gone_likes, (_, (r, ())) -> (r, ())); +let likes_map2 = likes_in | map(((person, message, created), ()) -> (person, (person, message, created))); +let likes_idx2 = likes_map2 | arrange; -+let likes_join2 = likes_idx2 | join(dead_p, (_, r, ()) -> (r, ())); ++let likes_join2 = likes_idx2 | join(dead_p, (_, (r, ())) -> (r, ())); +let likes_map3 = likes_in | map(((person, message, created), ()) -> (message, (person, message, created))); +let likes_idx3 = likes_map3 | arrange; +let dead_msg_idx3 = dead_msg | arrange; -+let likes_join3 = likes_idx3 | join(dead_msg_idx3, (_, r, ()) -> (r, ())); ++let likes_join3 = likes_idx3 | join(dead_msg_idx3, (_, (r, ())) -> (r, ())); +let likes_rows = likes_join + likes_join2 + likes_join3; +let likes_idx4 = likes_rows | arrange; +let likes_reduce = likes_idx4 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); @@ -223,28 +223,28 @@ +let likes = likes_in + likes_neg; +let study_map = study_in | map(((person, org, year), ()) -> (person, (person, org, year))); +let study_idx = study_map | arrange; -+let study_join = study_idx | join(dead_p, (_, r, ()) -> (r, ())); ++let study_join = study_idx | join(dead_p, (_, (r, ())) -> (r, ())); +let study_idx2 = study_join | arrange; +let study_reduce = study_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); +let study_neg = study_reduce | negate; +let study = study_in + study_neg; +let work_map = work_in | map(((person, org, year), ()) -> (person, (person, org, year))); +let work_idx = work_map | arrange; -+let work_join = work_idx | join(dead_p, (_, r, ()) -> (r, ())); ++let work_join = work_idx | join(dead_p, (_, (r, ())) -> (r, ())); +let work_idx2 = work_join | arrange; +let work_reduce = work_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); +let work_neg = work_reduce | negate; +let work = work_in + work_neg; +let email_map = email_in | map(((person, value), ()) -> (person, (person, value))); +let email_idx = email_map | arrange; -+let email_join = email_idx | join(dead_p, (_, r, ()) -> (r, ())); ++let email_join = email_idx | join(dead_p, (_, (r, ())) -> (r, ())); +let email_idx2 = email_join | arrange; +let email_reduce = email_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); +let email_neg = email_reduce | negate; +let email = email_in + email_neg; +let language_map = language_in | map(((person, value), ()) -> (person, (person, value))); +let language_idx = language_map | arrange; -+let language_join = language_idx | join(dead_p, (_, r, ()) -> (r, ())); ++let language_join = language_idx | join(dead_p, (_, (r, ())) -> (r, ())); +let language_idx2 = language_join | arrange; +let language_reduce = language_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); +let language_neg = language_reduce | negate; @@ -256,9 +256,9 @@ +let either_map2 = knows | map(((a, b, t), ()) -> (b, (a, t))); +let either = either_map + either_map2; +let either_idx = either | arrange; -+let friends_join = either_idx | join(exists, (a, (b, t), ()) -> (b, (a, t))); ++let friends_join = either_idx | join(exists, (a, ((b, t), ())) -> (b, (a, t))); +let friends_idx = friends_join | arrange; -+let friends_join2 = friends_idx | join(exists, (b, (a, t), ()) -> ((a, b, t), ())); ++let friends_join2 = friends_idx | join(exists, (b, ((a, t), ())) -> ((a, b, t), ())); +let friends_idx2 = friends_join2 | arrange; +let friends_reduce = friends_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); +let friends = friends_reduce | map(((a, b, t), ()) -> (a, (b, t))); @@ -267,14 +267,14 @@ +let replies_filter = message | filter(((_, kind, _, _, _, _, _, _, _, _, _, _, _, _), ()) -> (kind, 1) eq); +let replies = replies_filter | map(((id, _, _, _, _, _, _, _, _, _, parent, _, _, _), ()) -> (parent, id)); thread: { -- let below = thread | map(((mid, root), ()) -> (mid, root)) | join(replies, (_, root, child) -> ((child, root), ())); +- let below = thread | map(((mid, root), ()) -> (mid, root)) | join(replies, (_, (root, child)) -> ((child, root), ())); - set thread = posts + below | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); + -- Time lanes: epoch, then 1 scope coordinates; this scope enters at zero. + var thread; + let below_map = thread | map(((mid, root), ()) -> (mid, root)); + let below_idx = below_map | arrange; + let replies_idx = ::replies | arrange; -+ let below = below_idx | join(replies_idx, (_, root, child) -> ((child, root), ())); ++ let below = below_idx | join(replies_idx, (_, (root, child)) -> ((child, root), ())); + let thread_rows = ::posts + below; + let thread_idx = thread_rows | arrange; + let thread_reduce = thread_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_cascade.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_cascade.parsed.ddp index baba077f6..dfd74e8d4 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_cascade.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_cascade.parsed.ddp @@ -22,37 +22,37 @@ let del_knows = input 20 : ((int, int, int), ()); let del_member = input 21 : ((int, int, int), ()); let del_likes = input 22 : ((int, int, int), ()); let dead_p = del_person | map(((id, _), ()) -> (id, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); -let owned = forum_in | map(((id, title, created, moderator), ()) -> (moderator, (id, title, created))) | join(dead_p, (moderator, (id, title, created), ()) -> (id, (title, created, moderator))); +let owned = forum_in | map(((id, title, created, moderator), ()) -> (moderator, (id, title, created))) | join(dead_p, (moderator, ((id, title, created), ())) -> (id, (title, created, moderator))); let dead_f = (del_forum | map(((id, _), ()) -> (id, ()))) + (owned | filter((_, (title, _, _)) -> ((((0, (title len, 6) min) range, title) gather, "Album ") eq, (((0, (title len, 5) min) range, title) gather, "Wall ") eq) any) | map((id, _) -> (id, ()))) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); let groups = owned | filter((_, (title, _, _)) -> (((0, (title len, 6) min) range, title) gather, "Group ") eq); -let kept = groups + (groups | join(dead_f, (id, v, ()) -> (id, v)) | negate); -let seeds = (del_message | map(((id, _), ()) -> (id, ()))) + (message_in | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (creator, id)) | join(dead_p, (_, id, ()) -> (id, ()))) + (message_in | filter(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (kind, 0) eq) | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (forum, id)) | join(dead_f, (_, id, ()) -> (id, ()))); +let kept = groups + (groups | join(dead_f, (id, (v, ())) -> (id, v)) | negate); +let seeds = (del_message | map(((id, _), ()) -> (id, ()))) + (message_in | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (creator, id)) | join(dead_p, (_, (id, ())) -> (id, ()))) + (message_in | filter(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (kind, 0) eq) | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (forum, id)) | join(dead_f, (_, (id, ())) -> (id, ()))); let children = message_in | filter(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (kind, 1) eq) | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (parent, id)); dead_m: { - let kids = dead_m | join(children, (_, (), child) -> (child, ())); + let kids = dead_m | join(children, (_, ((), child)) -> (child, ())); set dead_m = seeds + kids | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); } let dead_msg = dead_m::dead_m; let gone_knows = del_knows | map(((a, b, _), ()) -> ((a, b), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); let gone_member = del_member | map(((f, p, _), ()) -> ((f, p), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); let gone_likes = del_likes | map(((p, x, _), ()) -> ((p, x), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); -let person = person_in + (person_in | map(((id, first, last, gender, birthday, created, ip, browser, city, birthmonth, birthdaynum, creationmonth), ()) -> (id, (id, first, last, gender, birthday, created, ip, browser, city, birthmonth, birthdaynum, creationmonth))) | join(dead_p, (_, r, ()) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); -let message = message_in + (message_in | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (id, (id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day))) | join(dead_msg, (_, r, ()) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); -let forum = forum_in + ((forum_in | map(((id, title, created, moderator), ()) -> (id, (id, title, created, moderator))) | join(dead_f, (_, r, ()) -> (r, ()))) + (kept | map((id, (title, created, moderator)) -> ((id, title, created, moderator), ()))) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate) + (kept | map((id, (title, created, _)) -> ((id, title, created, -1), ()))); -let knows = knows_in + ((knows_in | map(((src, dst, created), ()) -> ((src, dst), (src, dst, created))) | join(gone_knows, (_, r, ()) -> (r, ()))) + (knows_in | map(((src, dst, created), ()) -> (src, (src, dst, created))) | join(dead_p, (_, r, ()) -> (r, ()))) + (knows_in | map(((src, dst, created), ()) -> (dst, (src, dst, created))) | join(dead_p, (_, r, ()) -> (r, ()))) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); -let member = member_in + ((member_in | map(((forum, person, created), ()) -> ((forum, person), (forum, person, created))) | join(gone_member, (_, r, ()) -> (r, ()))) + (member_in | map(((forum, person, created), ()) -> (forum, (forum, person, created))) | join(dead_f, (_, r, ()) -> (r, ()))) + (member_in | map(((forum, person, created), ()) -> (person, (forum, person, created))) | join(dead_p, (_, r, ()) -> (r, ()))) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); +let person = person_in + (person_in | map(((id, first, last, gender, birthday, created, ip, browser, city, birthmonth, birthdaynum, creationmonth), ()) -> (id, (id, first, last, gender, birthday, created, ip, browser, city, birthmonth, birthdaynum, creationmonth))) | join(dead_p, (_, (r, ())) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); +let message = message_in + (message_in | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (id, (id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day))) | join(dead_msg, (_, (r, ())) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); +let forum = forum_in + ((forum_in | map(((id, title, created, moderator), ()) -> (id, (id, title, created, moderator))) | join(dead_f, (_, (r, ())) -> (r, ()))) + (kept | map((id, (title, created, moderator)) -> ((id, title, created, moderator), ()))) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate) + (kept | map((id, (title, created, _)) -> ((id, title, created, -1), ()))); +let knows = knows_in + ((knows_in | map(((src, dst, created), ()) -> ((src, dst), (src, dst, created))) | join(gone_knows, (_, (r, ())) -> (r, ()))) + (knows_in | map(((src, dst, created), ()) -> (src, (src, dst, created))) | join(dead_p, (_, (r, ())) -> (r, ()))) + (knows_in | map(((src, dst, created), ()) -> (dst, (src, dst, created))) | join(dead_p, (_, (r, ())) -> (r, ()))) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); +let member = member_in + ((member_in | map(((forum, person, created), ()) -> ((forum, person), (forum, person, created))) | join(gone_member, (_, (r, ())) -> (r, ()))) + (member_in | map(((forum, person, created), ()) -> (forum, (forum, person, created))) | join(dead_f, (_, (r, ())) -> (r, ()))) + (member_in | map(((forum, person, created), ()) -> (person, (forum, person, created))) | join(dead_p, (_, (r, ())) -> (r, ()))) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); let tag = tag_in; let tagclass = tagclass_in; let place = place_in; let org = org_in; -let interest = interest_in + (interest_in | map(((person, tag), ()) -> (person, (person, tag))) | join(dead_p, (_, r, ()) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); -let mtag = mtag_in + (mtag_in | map(((message, tag), ()) -> (message, (message, tag))) | join(dead_msg, (_, r, ()) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); -let ftag = ftag_in + (ftag_in | map(((forum, tag), ()) -> (forum, (forum, tag))) | join(dead_f, (_, r, ()) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); -let likes = likes_in + ((likes_in | map(((person, message, created), ()) -> ((person, message), (person, message, created))) | join(gone_likes, (_, r, ()) -> (r, ()))) + (likes_in | map(((person, message, created), ()) -> (person, (person, message, created))) | join(dead_p, (_, r, ()) -> (r, ()))) + (likes_in | map(((person, message, created), ()) -> (message, (person, message, created))) | join(dead_msg, (_, r, ()) -> (r, ()))) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); -let study = study_in + (study_in | map(((person, org, year), ()) -> (person, (person, org, year))) | join(dead_p, (_, r, ()) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); -let work = work_in + (work_in | map(((person, org, year), ()) -> (person, (person, org, year))) | join(dead_p, (_, r, ()) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); -let email = email_in + (email_in | map(((person, value), ()) -> (person, (person, value))) | join(dead_p, (_, r, ()) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); -let language = language_in + (language_in | map(((person, value), ()) -> (person, (person, value))) | join(dead_p, (_, r, ()) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); +let interest = interest_in + (interest_in | map(((person, tag), ()) -> (person, (person, tag))) | join(dead_p, (_, (r, ())) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); +let mtag = mtag_in + (mtag_in | map(((message, tag), ()) -> (message, (message, tag))) | join(dead_msg, (_, (r, ())) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); +let ftag = ftag_in + (ftag_in | map(((forum, tag), ()) -> (forum, (forum, tag))) | join(dead_f, (_, (r, ())) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); +let likes = likes_in + ((likes_in | map(((person, message, created), ()) -> ((person, message), (person, message, created))) | join(gone_likes, (_, (r, ())) -> (r, ()))) + (likes_in | map(((person, message, created), ()) -> (person, (person, message, created))) | join(dead_p, (_, (r, ())) -> (r, ()))) + (likes_in | map(((person, message, created), ()) -> (message, (person, message, created))) | join(dead_msg, (_, (r, ())) -> (r, ()))) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); +let study = study_in + (study_in | map(((person, org, year), ()) -> (person, (person, org, year))) | join(dead_p, (_, (r, ())) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); +let work = work_in + (work_in | map(((person, org, year), ()) -> (person, (person, org, year))) | join(dead_p, (_, (r, ())) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); +let email = email_in + (email_in | map(((person, value), ()) -> (person, (person, value))) | join(dead_p, (_, (r, ())) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); +let language = language_in + (language_in | map(((person, value), ()) -> (person, (person, value))) | join(dead_p, (_, (r, ())) -> (r, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | negate); output "person" = person | map(((id, first, last, gender, birthday, created, ip, browser, city, birthmonth, birthdaynum, creationmonth), ()) -> (id, (first, last, gender, birthday, created, ip, browser, city, birthmonth, birthdaynum, creationmonth))) | arrange; output "person.by_city" = person | map(((id, first, last, gender, birthday, created, ip, browser, city, birthmonth, birthdaynum, creationmonth), ()) -> (city, id)) | arrange; output "message" = message | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (id, (kind, created, creator, country, content, image, length, language, forum, parent, year, month, day))) | arrange; @@ -89,13 +89,13 @@ output "email.by_person" = email | map(((person, value), ()) -> (person, value)) output "language.by_person" = language | map(((person, value), ()) -> (person, value)) | arrange; let exists = person | map(((id, _, _, _, _, _, _, _, _, _, _, _), ()) -> (id, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); let either = (knows | map(((a, b, t), ()) -> (a, (b, t)))) + (knows | map(((a, b, t), ()) -> (b, (a, t)))); -let friends = either | join(exists, (a, (b, t), ()) -> (b, (a, t))) | join(exists, (b, (a, t), ()) -> ((a, b, t), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((a, b, t), ()) -> (a, (b, t))); +let friends = either | join(exists, (a, ((b, t), ())) -> (b, (a, t))) | join(exists, (b, ((a, t), ())) -> ((a, b, t), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((a, b, t), ()) -> (a, (b, t))); output "friend" = friends | arrange; output "friend.pair" = friends | map((a, (b, _)) -> ((a, b), ())) | arrange; let posts = message | filter(((_, kind, _, _, _, _, _, _, _, _, _, _, _, _), ()) -> (kind, 0) eq) | map(((id, _, _, _, _, _, _, _, _, _, _, _, _, _), ()) -> ((id, id), ())); let replies = message | filter(((_, kind, _, _, _, _, _, _, _, _, _, _, _, _), ()) -> (kind, 1) eq) | map(((id, _, _, _, _, _, _, _, _, _, parent, _, _, _), ()) -> (parent, id)); thread: { - let below = thread | map(((mid, root), ()) -> (mid, root)) | join(replies, (_, root, child) -> ((child, root), ())); + let below = thread | map(((mid, root), ()) -> (mid, root)) | join(replies, (_, (root, child)) -> ((child, root), ())); set thread = posts + below | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); } output "message.root" = thread::thread | map(((mid, root), ()) -> (mid, root)) | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_graph.ddp b/interactive/tests/plan_pairs/expected/ldbc_graph.ddp index eabc26c1b..5ac982903 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_graph.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_graph.ddp @@ -23,9 +23,9 @@ let either_map = knows | map(((a, b, t), ()) -> (a, (b, t))); let either_map2 = knows | map(((a, b, t), ()) -> (b, (a, t))); let either = either_map + either_map2; let either_idx = either | arrange; -let friends_join = either_idx | join(exists, (a, (b, t), ()) -> (b, (a, t))); +let friends_join = either_idx | join(exists, (a, ((b, t), ())) -> (b, (a, t))); let friends_idx = friends_join | arrange; -let friends_join2 = friends_idx | join(exists, (b, (a, t), ()) -> ((a, b, t), ())); +let friends_join2 = friends_idx | join(exists, (b, ((a, t), ())) -> ((a, b, t), ())); let friends_idx2 = friends_join2 | arrange; let friends_reduce = friends_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); let friends = friends_reduce | map(((a, b, t), ()) -> (a, (b, t))); @@ -37,7 +37,7 @@ thread: { var thread; let below_map = thread | map(((mid, root), ()) -> (mid, root)); let below_idx = below_map | arrange; - let below = below_idx | join(::replies_idx, (_, root, child) -> ((child, root), ())); + let below = below_idx | join(::replies_idx, (_, (root, child)) -> ((child, root), ())); let thread_rows = ::posts + below; let thread_idx = thread_rows | arrange; let thread_reduce = thread_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_graph.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/ldbc_graph.lowered--optimized.diff index 2bf2d8eae..10ae35f3f 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_graph.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_graph.lowered--optimized.diff @@ -17,8 +17,8 @@ let below_map = thread | map(((mid, root), ()) -> (mid, root)); let below_idx = below_map | arrange; - let replies_idx = ::replies | arrange; -- let below = below_idx | join(replies_idx, (_, root, child) -> ((child, root), ())); -+ let below = below_idx | join(::replies_idx, (_, root, child) -> ((child, root), ())); +- let below = below_idx | join(replies_idx, (_, (root, child)) -> ((child, root), ())); ++ let below = below_idx | join(::replies_idx, (_, (root, child)) -> ((child, root), ())); let thread_rows = ::posts + below; let thread_idx = thread_rows | arrange; let thread_reduce = thread_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_graph.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_graph.parsed--lowered.diff index df4000056..326f58084 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_graph.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_graph.parsed--lowered.diff @@ -55,7 +55,7 @@ -output "language.by_person" = language | map(((person, value), ()) -> (person, value)) | arrange; -let exists = person | map(((id, _, _, _, _, _, _, _, _, _, _, _), ()) -> (id, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); -let either = (knows | map(((a, b, t), ()) -> (a, (b, t)))) + (knows | map(((a, b, t), ()) -> (b, (a, t)))); --let friends = either | join(exists, (a, (b, t), ()) -> (b, (a, t))) | join(exists, (b, (a, t), ()) -> ((a, b, t), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((a, b, t), ()) -> (a, (b, t))); +-let friends = either | join(exists, (a, ((b, t), ())) -> (b, (a, t))) | join(exists, (b, ((a, t), ())) -> ((a, b, t), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((a, b, t), ()) -> (a, (b, t))); -output "friend" = friends | arrange; -output "friend.pair" = friends | map((a, (b, _)) -> ((a, b), ())) | arrange; -let posts = message | filter(((_, kind, _, _, _, _, _, _, _, _, _, _, _, _), ()) -> (kind, 0) eq) | map(((id, _, _, _, _, _, _, _, _, _, _, _, _, _), ()) -> ((id, id), ())); @@ -67,9 +67,9 @@ +let either_map2 = knows | map(((a, b, t), ()) -> (b, (a, t))); +let either = either_map + either_map2; +let either_idx = either | arrange; -+let friends_join = either_idx | join(exists, (a, (b, t), ()) -> (b, (a, t))); ++let friends_join = either_idx | join(exists, (a, ((b, t), ())) -> (b, (a, t))); +let friends_idx = friends_join | arrange; -+let friends_join2 = friends_idx | join(exists, (b, (a, t), ()) -> ((a, b, t), ())); ++let friends_join2 = friends_idx | join(exists, (b, ((a, t), ())) -> ((a, b, t), ())); +let friends_idx2 = friends_join2 | arrange; +let friends_reduce = friends_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); +let friends = friends_reduce | map(((a, b, t), ()) -> (a, (b, t))); @@ -78,14 +78,14 @@ +let replies_filter = message | filter(((_, kind, _, _, _, _, _, _, _, _, _, _, _, _), ()) -> (kind, 1) eq); +let replies = replies_filter | map(((id, _, _, _, _, _, _, _, _, _, parent, _, _, _), ()) -> (parent, id)); thread: { -- let below = thread | map(((mid, root), ()) -> (mid, root)) | join(replies, (_, root, child) -> ((child, root), ())); +- let below = thread | map(((mid, root), ()) -> (mid, root)) | join(replies, (_, (root, child)) -> ((child, root), ())); - set thread = posts + below | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); + -- Time lanes: epoch, then 1 scope coordinates; this scope enters at zero. + var thread; + let below_map = thread | map(((mid, root), ()) -> (mid, root)); + let below_idx = below_map | arrange; + let replies_idx = ::replies | arrange; -+ let below = below_idx | join(replies_idx, (_, root, child) -> ((child, root), ())); ++ let below = below_idx | join(replies_idx, (_, (root, child)) -> ((child, root), ())); + let thread_rows = ::posts + below; + let thread_idx = thread_rows | arrange; + let thread_reduce = thread_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_graph.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_graph.parsed.ddp index d59fe08d7..62cc33d19 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_graph.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_graph.parsed.ddp @@ -51,13 +51,13 @@ let language = input 16 : ((int, List(int)), ()); output "language.by_person" = language | map(((person, value), ()) -> (person, value)) | arrange; let exists = person | map(((id, _, _, _, _, _, _, _, _, _, _, _), ()) -> (id, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); let either = (knows | map(((a, b, t), ()) -> (a, (b, t)))) + (knows | map(((a, b, t), ()) -> (b, (a, t)))); -let friends = either | join(exists, (a, (b, t), ()) -> (b, (a, t))) | join(exists, (b, (a, t), ()) -> ((a, b, t), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((a, b, t), ()) -> (a, (b, t))); +let friends = either | join(exists, (a, ((b, t), ())) -> (b, (a, t))) | join(exists, (b, ((a, t), ())) -> ((a, b, t), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((a, b, t), ()) -> (a, (b, t))); output "friend" = friends | arrange; output "friend.pair" = friends | map((a, (b, _)) -> ((a, b), ())) | arrange; let posts = message | filter(((_, kind, _, _, _, _, _, _, _, _, _, _, _, _), ()) -> (kind, 0) eq) | map(((id, _, _, _, _, _, _, _, _, _, _, _, _, _), ()) -> ((id, id), ())); let replies = message | filter(((_, kind, _, _, _, _, _, _, _, _, _, _, _, _), ()) -> (kind, 1) eq) | map(((id, _, _, _, _, _, _, _, _, _, parent, _, _, _), ()) -> (parent, id)); thread: { - let below = thread | map(((mid, root), ()) -> (mid, root)) | join(replies, (_, root, child) -> ((child, root), ())); + let below = thread | map(((mid, root), ()) -> (mid, root)) | join(replies, (_, (root, child)) -> ((child, root), ())); set thread = posts + below | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); } output "message.root" = thread::thread | map(((mid, root), ()) -> (mid, root)) | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic01_oneoff.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic01_oneoff.ddp index 96012db65..6d146dc33 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic01_oneoff.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic01_oneoff.ddp @@ -12,11 +12,11 @@ let language = import "graph::language.by_person" : (int, List(int)); let hop1_map = request | map(((rid, who, name), ()) -> (who, (rid, who, name))); let hop1_idx = hop1_map | arrange; let friend_idx = friend | arrange; -let hop1 = hop1_idx | join(friend_idx, (_, req, (f, _)) -> (f, (req, 1))); +let hop1 = hop1_idx | join(friend_idx, (_, (req, (f, _))) -> (f, (req, 1))); let hop1_idx2 = hop1 | arrange; -let hop2 = hop1_idx2 | join(friend_idx, (_, (req, _), (g, _)) -> (g, (req, 2))); +let hop2 = hop1_idx2 | join(friend_idx, (_, ((req, _), (g, _))) -> (g, (req, 2))); let hop2_idx = hop2 | arrange; -let hop3 = hop2_idx | join(friend_idx, (_, (req, _), (h, _)) -> (h, (req, 3))); +let hop3 = hop2_idx | join(friend_idx, (_, ((req, _), (h, _))) -> (h, (req, 3))); let circle_rows = hop1 + hop2 + hop3; let circle_rows2 = circle_rows | filter((node, ((_, who, _), _)) -> (node, who) ne) | map((node, ((rid, _, name), d)) -> ((rid, node, name), d)); let circle_idx = circle_rows2 | arrange; @@ -24,62 +24,62 @@ let circle = circle_idx | reduce(limit 1, (_, vals) -> let first = vals head in let found_map = circle | map(((rid, node, name), d) -> (node, (rid, name, d))); let found_idx = found_map | arrange; let person_idx = person | arrange; -let found_join = found_idx | join(person_idx, (node, (rid, name, d), (first, last, gender, birthday, created, ip, browser, home, _, _, _)) -> ((first, name) eq, (home, (rid, node, d, last, gender, birthday, created, ip, browser)))); +let found_join = found_idx | join(person_idx, (node, ((rid, name, d), (first, last, gender, birthday, created, ip, browser, home, _, _, _))) -> ((first, name) eq, (home, (rid, node, d, last, gender, birthday, created, ip, browser)))); let found_rows = found_join | filter((named, _) -> named) | map((_, row) -> row); let found_idx2 = found_rows | arrange; let place_idx = place | arrange; -let found = found_idx2 | join(place_idx, (_, (rid, node, d, last, gender, birthday, created, ip, browser), (town, _, _)) -> ((rid, node), (d, (last, gender, birthday, created, ip, browser, town)))); +let found = found_idx2 | join(place_idx, (_, ((rid, node, d, last, gender, birthday, created, ip, browser), (town, _, _))) -> ((rid, node), (d, (last, gender, birthday, created, ip, browser, town)))); let who = found | map(((rid, node), _) -> (node, rid)); let who_idx = who | arrange; let email_idx = email | arrange; -let emails_join = who_idx | join(email_idx, (node, rid, value) -> ((rid, node), value)); +let emails_join = who_idx | join(email_idx, (node, (rid, value)) -> ((rid, node), value)); let emails_idx = emails_join | arrange; let emails = emails_idx | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); let language_idx = language | arrange; -let langs_join = who_idx | join(language_idx, (node, rid, value) -> ((rid, node), value)); +let langs_join = who_idx | join(language_idx, (node, (rid, value)) -> ((rid, node), value)); let langs_idx = langs_join | arrange; let langs = langs_idx | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); let study_idx = study | arrange; -let school_join = who_idx | join(study_idx, (node, rid, (at, year)) -> (at, (rid, node, year))); +let school_join = who_idx | join(study_idx, (node, (rid, (at, year))) -> (at, (rid, node, year))); let school_idx = school_join | arrange; let org_idx = org | arrange; -let school_join2 = school_idx | join(org_idx, (_, (rid, node, year), (name, _, at)) -> (at, (rid, node, name, year))); +let school_join2 = school_idx | join(org_idx, (_, ((rid, node, year), (name, _, at))) -> (at, (rid, node, name, year))); let school_idx2 = school_join2 | arrange; -let school_join3 = school_idx2 | join(place_idx, (_, (rid, node, name, year), (where, _, _)) -> ((rid, node), (name, year, where))); +let school_join3 = school_idx2 | join(place_idx, (_, ((rid, node, name, year), (where, _, _))) -> ((rid, node), (name, year, where))); let school_idx3 = school_join3 | arrange; let school = school_idx3 | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); let work_idx = work | arrange; -let jobs_join = who_idx | join(work_idx, (node, rid, (at, year)) -> (at, (rid, node, year))); +let jobs_join = who_idx | join(work_idx, (node, (rid, (at, year))) -> (at, (rid, node, year))); let jobs_idx = jobs_join | arrange; -let jobs_join2 = jobs_idx | join(org_idx, (_, (rid, node, year), (name, _, at)) -> (at, (rid, node, name, year))); +let jobs_join2 = jobs_idx | join(org_idx, (_, ((rid, node, year), (name, _, at))) -> (at, (rid, node, name, year))); let jobs_idx2 = jobs_join2 | arrange; -let jobs_join3 = jobs_idx2 | join(place_idx, (_, (rid, node, name, year), (where, _, _)) -> ((rid, node), (name, year, where))); +let jobs_join3 = jobs_idx2 | join(place_idx, (_, ((rid, node, name, year), (where, _, _))) -> ((rid, node), (name, year, where))); let jobs_idx3 = jobs_join3 | arrange; let jobs = jobs_idx3 | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); let found_idx3 = found | arrange; -let with1_join = found_idx3 | join(emails, (key, row, list) -> (key, (row, list))); -let with1_join2 = found_idx3 | join(emails, (key, row, _) -> (key, row)); +let with1_join = found_idx3 | join(emails, row -> row); +let with1_join2 = found_idx3 | join(emails, (key, (row, _)) -> (key, row)); let with1_neg = with1_join2 | negate; let with1_rows = found + with1_neg; let with1_map = with1_rows | map((key, row) -> (key, (row, [] : List(List(int))))); let with1 = with1_join + with1_map; let with1_idx = with1 | arrange; -let with2_join = with1_idx | join(langs, (key, (row, e), list) -> (key, (row, e, list))); -let with2_join2 = with1_idx | join(langs, (key, row, _) -> (key, row)); +let with2_join = with1_idx | join(langs, (key, ((row, e), list)) -> (key, (row, e, list))); +let with2_join2 = with1_idx | join(langs, (key, (row, _)) -> (key, row)); let with2_neg = with2_join2 | negate; let with2_rows = with1 + with2_neg; let with2_map = with2_rows | map((key, (row, e)) -> (key, (row, e, [] : List(List(int))))); let with2 = with2_join + with2_map; let with2_idx = with2 | arrange; -let with3_join = with2_idx | join(school, (key, (row, e, l), list) -> (key, (row, e, l, list))); -let with3_join2 = with2_idx | join(school, (key, row, _) -> (key, row)); +let with3_join = with2_idx | join(school, (key, ((row, e, l), list)) -> (key, (row, e, l, list))); +let with3_join2 = with2_idx | join(school, (key, (row, _)) -> (key, row)); let with3_neg = with3_join2 | negate; let with3_rows = with2 + with3_neg; let with3_map = with3_rows | map((key, (row, e, l)) -> (key, (row, e, l, [] : List((List(int), int, List(int)))))); let with3 = with3_join + with3_map; let with3_idx = with3 | arrange; -let with4_join = with3_idx | join(jobs, (key, (row, e, l, s), list) -> (key, (row, e, l, s, list))); -let with4_join2 = with3_idx | join(jobs, (key, row, _) -> (key, row)); +let with4_join = with3_idx | join(jobs, (key, ((row, e, l, s), list)) -> (key, (row, e, l, s, list))); +let with4_join2 = with3_idx | join(jobs, (key, (row, _)) -> (key, row)); let with4_neg = with4_join2 | negate; let with4_rows = with3 + with4_neg; let with4_map = with4_rows | map((key, (row, e, l, s)) -> (key, (row, e, l, s, [] : List((List(int), int, List(int)))))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic01_oneoff.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/ldbc_ic01_oneoff.lowered--optimized.diff index 11e2797e7..b06da07ac 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic01_oneoff.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic01_oneoff.lowered--optimized.diff @@ -2,15 +2,15 @@ +++ optimized @@ -14,23 +14,19 @@ let friend_idx = friend | arrange; - let hop1 = hop1_idx | join(friend_idx, (_, req, (f, _)) -> (f, (req, 1))); + let hop1 = hop1_idx | join(friend_idx, (_, (req, (f, _))) -> (f, (req, 1))); let hop1_idx2 = hop1 | arrange; -let friend_idx2 = friend | arrange; --let hop2 = hop1_idx2 | join(friend_idx2, (_, (req, _), (g, _)) -> (g, (req, 2))); -+let hop2 = hop1_idx2 | join(friend_idx, (_, (req, _), (g, _)) -> (g, (req, 2))); +-let hop2 = hop1_idx2 | join(friend_idx2, (_, ((req, _), (g, _))) -> (g, (req, 2))); ++let hop2 = hop1_idx2 | join(friend_idx, (_, ((req, _), (g, _))) -> (g, (req, 2))); let hop2_idx = hop2 | arrange; -let friend_idx3 = friend | arrange; --let hop3 = hop2_idx | join(friend_idx3, (_, (req, _), (h, _)) -> (h, (req, 3))); -+let hop3 = hop2_idx | join(friend_idx, (_, (req, _), (h, _)) -> (h, (req, 3))); +-let hop3 = hop2_idx | join(friend_idx3, (_, ((req, _), (h, _))) -> (h, (req, 3))); ++let hop3 = hop2_idx | join(friend_idx, (_, ((req, _), (h, _))) -> (h, (req, 3))); let circle_rows = hop1 + hop2 + hop3; -let circle_filter = circle_rows | filter((node, ((_, who, _), _)) -> (node, who) ne); -let circle_map = circle_filter | map((node, ((rid, _, name), d)) -> ((rid, node, name), d)); @@ -21,84 +21,84 @@ let found_map = circle | map(((rid, node, name), d) -> (node, (rid, name, d))); let found_idx = found_map | arrange; let person_idx = person | arrange; - let found_join = found_idx | join(person_idx, (node, (rid, name, d), (first, last, gender, birthday, created, ip, browser, home, _, _, _)) -> ((first, name) eq, (home, (rid, node, d, last, gender, birthday, created, ip, browser)))); + let found_join = found_idx | join(person_idx, (node, ((rid, name, d), (first, last, gender, birthday, created, ip, browser, home, _, _, _))) -> ((first, name) eq, (home, (rid, node, d, last, gender, birthday, created, ip, browser)))); -let found_filter = found_join | filter((named, _) -> named); -let found_map2 = found_filter | map((_, row) -> row); -let found_idx2 = found_map2 | arrange; +let found_rows = found_join | filter((named, _) -> named) | map((_, row) -> row); +let found_idx2 = found_rows | arrange; let place_idx = place | arrange; - let found = found_idx2 | join(place_idx, (_, (rid, node, d, last, gender, birthday, created, ip, browser), (town, _, _)) -> ((rid, node), (d, (last, gender, birthday, created, ip, browser, town)))); + let found = found_idx2 | join(place_idx, (_, ((rid, node, d, last, gender, birthday, created, ip, browser), (town, _, _))) -> ((rid, node), (d, (last, gender, birthday, created, ip, browser, town)))); let who = found | map(((rid, node), _) -> (node, rid)); @@ -39,61 +35,51 @@ - let emails_join = who_idx | join(email_idx, (node, rid, value) -> ((rid, node), value)); + let emails_join = who_idx | join(email_idx, (node, (rid, value)) -> ((rid, node), value)); let emails_idx = emails_join | arrange; let emails = emails_idx | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); -let who_idx2 = who | arrange; let language_idx = language | arrange; --let langs_join = who_idx2 | join(language_idx, (node, rid, value) -> ((rid, node), value)); -+let langs_join = who_idx | join(language_idx, (node, rid, value) -> ((rid, node), value)); +-let langs_join = who_idx2 | join(language_idx, (node, (rid, value)) -> ((rid, node), value)); ++let langs_join = who_idx | join(language_idx, (node, (rid, value)) -> ((rid, node), value)); let langs_idx = langs_join | arrange; let langs = langs_idx | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); -let who_idx3 = who | arrange; let study_idx = study | arrange; --let school_join = who_idx3 | join(study_idx, (node, rid, (at, year)) -> (at, (rid, node, year))); -+let school_join = who_idx | join(study_idx, (node, rid, (at, year)) -> (at, (rid, node, year))); +-let school_join = who_idx3 | join(study_idx, (node, (rid, (at, year))) -> (at, (rid, node, year))); ++let school_join = who_idx | join(study_idx, (node, (rid, (at, year))) -> (at, (rid, node, year))); let school_idx = school_join | arrange; let org_idx = org | arrange; - let school_join2 = school_idx | join(org_idx, (_, (rid, node, year), (name, _, at)) -> (at, (rid, node, name, year))); + let school_join2 = school_idx | join(org_idx, (_, ((rid, node, year), (name, _, at))) -> (at, (rid, node, name, year))); let school_idx2 = school_join2 | arrange; -let place_idx2 = place | arrange; --let school_join3 = school_idx2 | join(place_idx2, (_, (rid, node, name, year), (where, _, _)) -> ((rid, node), (name, year, where))); -+let school_join3 = school_idx2 | join(place_idx, (_, (rid, node, name, year), (where, _, _)) -> ((rid, node), (name, year, where))); +-let school_join3 = school_idx2 | join(place_idx2, (_, ((rid, node, name, year), (where, _, _))) -> ((rid, node), (name, year, where))); ++let school_join3 = school_idx2 | join(place_idx, (_, ((rid, node, name, year), (where, _, _))) -> ((rid, node), (name, year, where))); let school_idx3 = school_join3 | arrange; let school = school_idx3 | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); -let who_idx4 = who | arrange; let work_idx = work | arrange; --let jobs_join = who_idx4 | join(work_idx, (node, rid, (at, year)) -> (at, (rid, node, year))); -+let jobs_join = who_idx | join(work_idx, (node, rid, (at, year)) -> (at, (rid, node, year))); +-let jobs_join = who_idx4 | join(work_idx, (node, (rid, (at, year))) -> (at, (rid, node, year))); ++let jobs_join = who_idx | join(work_idx, (node, (rid, (at, year))) -> (at, (rid, node, year))); let jobs_idx = jobs_join | arrange; -let org_idx2 = org | arrange; --let jobs_join2 = jobs_idx | join(org_idx2, (_, (rid, node, year), (name, _, at)) -> (at, (rid, node, name, year))); -+let jobs_join2 = jobs_idx | join(org_idx, (_, (rid, node, year), (name, _, at)) -> (at, (rid, node, name, year))); +-let jobs_join2 = jobs_idx | join(org_idx2, (_, ((rid, node, year), (name, _, at))) -> (at, (rid, node, name, year))); ++let jobs_join2 = jobs_idx | join(org_idx, (_, ((rid, node, year), (name, _, at))) -> (at, (rid, node, name, year))); let jobs_idx2 = jobs_join2 | arrange; -let place_idx3 = place | arrange; --let jobs_join3 = jobs_idx2 | join(place_idx3, (_, (rid, node, name, year), (where, _, _)) -> ((rid, node), (name, year, where))); -+let jobs_join3 = jobs_idx2 | join(place_idx, (_, (rid, node, name, year), (where, _, _)) -> ((rid, node), (name, year, where))); +-let jobs_join3 = jobs_idx2 | join(place_idx3, (_, ((rid, node, name, year), (where, _, _))) -> ((rid, node), (name, year, where))); ++let jobs_join3 = jobs_idx2 | join(place_idx, (_, ((rid, node, name, year), (where, _, _))) -> ((rid, node), (name, year, where))); let jobs_idx3 = jobs_join3 | arrange; let jobs = jobs_idx3 | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); let found_idx3 = found | arrange; - let with1_join = found_idx3 | join(emails, (key, row, list) -> (key, (row, list))); + let with1_join = found_idx3 | join(emails, row -> row); -let found_idx4 = found | arrange; --let with1_join2 = found_idx4 | join(emails, (key, row, _) -> (key, row)); -+let with1_join2 = found_idx3 | join(emails, (key, row, _) -> (key, row)); +-let with1_join2 = found_idx4 | join(emails, (key, (row, _)) -> (key, row)); ++let with1_join2 = found_idx3 | join(emails, (key, (row, _)) -> (key, row)); let with1_neg = with1_join2 | negate; let with1_rows = found + with1_neg; let with1_map = with1_rows | map((key, row) -> (key, (row, [] : List(List(int))))); let with1 = with1_join + with1_map; let with1_idx = with1 | arrange; - let with2_join = with1_idx | join(langs, (key, (row, e), list) -> (key, (row, e, list))); + let with2_join = with1_idx | join(langs, (key, ((row, e), list)) -> (key, (row, e, list))); -let with1_idx2 = with1 | arrange; --let with2_join2 = with1_idx2 | join(langs, (key, row, _) -> (key, row)); -+let with2_join2 = with1_idx | join(langs, (key, row, _) -> (key, row)); +-let with2_join2 = with1_idx2 | join(langs, (key, (row, _)) -> (key, row)); ++let with2_join2 = with1_idx | join(langs, (key, (row, _)) -> (key, row)); let with2_neg = with2_join2 | negate; let with2_rows = with1 + with2_neg; let with2_map = with2_rows | map((key, (row, e)) -> (key, (row, e, [] : List(List(int))))); let with2 = with2_join + with2_map; let with2_idx = with2 | arrange; - let with3_join = with2_idx | join(school, (key, (row, e, l), list) -> (key, (row, e, l, list))); + let with3_join = with2_idx | join(school, (key, ((row, e, l), list)) -> (key, (row, e, l, list))); -let with2_idx2 = with2 | arrange; --let with3_join2 = with2_idx2 | join(school, (key, row, _) -> (key, row)); -+let with3_join2 = with2_idx | join(school, (key, row, _) -> (key, row)); +-let with3_join2 = with2_idx2 | join(school, (key, (row, _)) -> (key, row)); ++let with3_join2 = with2_idx | join(school, (key, (row, _)) -> (key, row)); let with3_neg = with3_join2 | negate; let with3_rows = with2 + with3_neg; let with3_map = with3_rows | map((key, (row, e, l)) -> (key, (row, e, l, [] : List((List(int), int, List(int)))))); let with3 = with3_join + with3_map; let with3_idx = with3 | arrange; - let with4_join = with3_idx | join(jobs, (key, (row, e, l, s), list) -> (key, (row, e, l, s, list))); + let with4_join = with3_idx | join(jobs, (key, ((row, e, l, s), list)) -> (key, (row, e, l, s, list))); -let with3_idx2 = with3 | arrange; --let with4_join2 = with3_idx2 | join(jobs, (key, row, _) -> (key, row)); -+let with4_join2 = with3_idx | join(jobs, (key, row, _) -> (key, row)); +-let with4_join2 = with3_idx2 | join(jobs, (key, (row, _)) -> (key, row)); ++let with4_join2 = with3_idx | join(jobs, (key, (row, _)) -> (key, row)); let with4_neg = with4_join2 | negate; let with4_rows = with3 + with4_neg; let with4_map = with4_rows | map((key, (row, e, l, s)) -> (key, (row, e, l, s, [] : List((List(int), int, List(int)))))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic01_oneoff.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic01_oneoff.parsed--lowered.diff index f64309125..a22b00993 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic01_oneoff.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic01_oneoff.parsed--lowered.diff @@ -10,21 +10,21 @@ let work = import "graph::work.by_person" : (int, (int, int)); let email = import "graph::email.by_person" : (int, List(int)); let language = import "graph::language.by_person" : (int, List(int)); --let hop1 = request | map(((rid, who, name), ()) -> (who, (rid, who, name))) | join(friend, (_, req, (f, _)) -> (f, (req, 1))); --let hop2 = hop1 | join(friend, (_, (req, _), (g, _)) -> (g, (req, 2))); --let hop3 = hop2 | join(friend, (_, (req, _), (h, _)) -> (h, (req, 3))); +-let hop1 = request | map(((rid, who, name), ()) -> (who, (rid, who, name))) | join(friend, (_, (req, (f, _))) -> (f, (req, 1))); +-let hop2 = hop1 | join(friend, (_, ((req, _), (g, _))) -> (g, (req, 2))); +-let hop3 = hop2 | join(friend, (_, ((req, _), (h, _))) -> (h, (req, 3))); -let circle = hop1 + hop2 + hop3 | filter((node, ((_, who, _), _)) -> (node, who) ne) | map((node, ((rid, _, name), d)) -> ((rid, node, name), d)) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); --let found = circle | map(((rid, node, name), d) -> (node, (rid, name, d))) | join(person, (node, (rid, name, d), (first, last, gender, birthday, created, ip, browser, home, _, _, _)) -> ((first, name) eq, (home, (rid, node, d, last, gender, birthday, created, ip, browser)))) | filter((named, _) -> named) | map((_, row) -> row) | join(place, (_, (rid, node, d, last, gender, birthday, created, ip, browser), (town, _, _)) -> ((rid, node), (d, (last, gender, birthday, created, ip, browser, town)))); +-let found = circle | map(((rid, node, name), d) -> (node, (rid, name, d))) | join(person, (node, ((rid, name, d), (first, last, gender, birthday, created, ip, browser, home, _, _, _))) -> ((first, name) eq, (home, (rid, node, d, last, gender, birthday, created, ip, browser)))) | filter((named, _) -> named) | map((_, row) -> row) | join(place, (_, ((rid, node, d, last, gender, birthday, created, ip, browser), (town, _, _))) -> ((rid, node), (d, (last, gender, birthday, created, ip, browser, town)))); +let hop1_map = request | map(((rid, who, name), ()) -> (who, (rid, who, name))); +let hop1_idx = hop1_map | arrange; +let friend_idx = friend | arrange; -+let hop1 = hop1_idx | join(friend_idx, (_, req, (f, _)) -> (f, (req, 1))); ++let hop1 = hop1_idx | join(friend_idx, (_, (req, (f, _))) -> (f, (req, 1))); +let hop1_idx2 = hop1 | arrange; +let friend_idx2 = friend | arrange; -+let hop2 = hop1_idx2 | join(friend_idx2, (_, (req, _), (g, _)) -> (g, (req, 2))); ++let hop2 = hop1_idx2 | join(friend_idx2, (_, ((req, _), (g, _))) -> (g, (req, 2))); +let hop2_idx = hop2 | arrange; +let friend_idx3 = friend | arrange; -+let hop3 = hop2_idx | join(friend_idx3, (_, (req, _), (h, _)) -> (h, (req, 3))); ++let hop3 = hop2_idx | join(friend_idx3, (_, ((req, _), (h, _))) -> (h, (req, 3))); +let circle_rows = hop1 + hop2 + hop3; +let circle_filter = circle_rows | filter((node, ((_, who, _), _)) -> (node, who) ne); +let circle_map = circle_filter | map((node, ((rid, _, name), d)) -> ((rid, node, name), d)); @@ -33,82 +33,82 @@ +let found_map = circle | map(((rid, node, name), d) -> (node, (rid, name, d))); +let found_idx = found_map | arrange; +let person_idx = person | arrange; -+let found_join = found_idx | join(person_idx, (node, (rid, name, d), (first, last, gender, birthday, created, ip, browser, home, _, _, _)) -> ((first, name) eq, (home, (rid, node, d, last, gender, birthday, created, ip, browser)))); ++let found_join = found_idx | join(person_idx, (node, ((rid, name, d), (first, last, gender, birthday, created, ip, browser, home, _, _, _))) -> ((first, name) eq, (home, (rid, node, d, last, gender, birthday, created, ip, browser)))); +let found_filter = found_join | filter((named, _) -> named); +let found_map2 = found_filter | map((_, row) -> row); +let found_idx2 = found_map2 | arrange; +let place_idx = place | arrange; -+let found = found_idx2 | join(place_idx, (_, (rid, node, d, last, gender, birthday, created, ip, browser), (town, _, _)) -> ((rid, node), (d, (last, gender, birthday, created, ip, browser, town)))); ++let found = found_idx2 | join(place_idx, (_, ((rid, node, d, last, gender, birthday, created, ip, browser), (town, _, _))) -> ((rid, node), (d, (last, gender, birthday, created, ip, browser, town)))); let who = found | map(((rid, node), _) -> (node, rid)); --let emails = who | join(email, (node, rid, value) -> ((rid, node), value)) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); --let langs = who | join(language, (node, rid, value) -> ((rid, node), value)) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); --let school = who | join(study, (node, rid, (at, year)) -> (at, (rid, node, year))) | join(org, (_, (rid, node, year), (name, _, at)) -> (at, (rid, node, name, year))) | join(place, (_, (rid, node, name, year), (where, _, _)) -> ((rid, node), (name, year, where))) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); --let jobs = who | join(work, (node, rid, (at, year)) -> (at, (rid, node, year))) | join(org, (_, (rid, node, year), (name, _, at)) -> (at, (rid, node, name, year))) | join(place, (_, (rid, node, name, year), (where, _, _)) -> ((rid, node), (name, year, where))) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); --let with1 = (found | join(emails, (key, row, list) -> (key, (row, list)))) + (found + (found | join(emails, (key, row, _) -> (key, row)) | negate) | map((key, row) -> (key, (row, [] : List(List(int)))))); --let with2 = (with1 | join(langs, (key, (row, e), list) -> (key, (row, e, list)))) + (with1 + (with1 | join(langs, (key, row, _) -> (key, row)) | negate) | map((key, (row, e)) -> (key, (row, e, [] : List(List(int)))))); --let with3 = (with2 | join(school, (key, (row, e, l), list) -> (key, (row, e, l, list)))) + (with2 + (with2 | join(school, (key, row, _) -> (key, row)) | negate) | map((key, (row, e, l)) -> (key, (row, e, l, [] : List((List(int), int, List(int))))))); --let with4 = (with3 | join(jobs, (key, (row, e, l, s), list) -> (key, (row, e, l, s, list)))) + (with3 + (with3 | join(jobs, (key, row, _) -> (key, row)) | negate) | map((key, (row, e, l, s)) -> (key, (row, e, l, s, [] : List((List(int), int, List(int))))))); +-let emails = who | join(email, (node, (rid, value)) -> ((rid, node), value)) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); +-let langs = who | join(language, (node, (rid, value)) -> ((rid, node), value)) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); +-let school = who | join(study, (node, (rid, (at, year))) -> (at, (rid, node, year))) | join(org, (_, ((rid, node, year), (name, _, at))) -> (at, (rid, node, name, year))) | join(place, (_, ((rid, node, name, year), (where, _, _))) -> ((rid, node), (name, year, where))) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); +-let jobs = who | join(work, (node, (rid, (at, year))) -> (at, (rid, node, year))) | join(org, (_, ((rid, node, year), (name, _, at))) -> (at, (rid, node, name, year))) | join(place, (_, ((rid, node, name, year), (where, _, _))) -> ((rid, node), (name, year, where))) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); +-let with1 = (found | join(emails, row -> row)) + (found + (found | join(emails, (key, (row, _)) -> (key, row)) | negate) | map((key, row) -> (key, (row, [] : List(List(int)))))); +-let with2 = (with1 | join(langs, (key, ((row, e), list)) -> (key, (row, e, list)))) + (with1 + (with1 | join(langs, (key, (row, _)) -> (key, row)) | negate) | map((key, (row, e)) -> (key, (row, e, [] : List(List(int)))))); +-let with3 = (with2 | join(school, (key, ((row, e, l), list)) -> (key, (row, e, l, list)))) + (with2 + (with2 | join(school, (key, (row, _)) -> (key, row)) | negate) | map((key, (row, e, l)) -> (key, (row, e, l, [] : List((List(int), int, List(int))))))); +-let with4 = (with3 | join(jobs, (key, ((row, e, l, s), list)) -> (key, (row, e, l, s, list)))) + (with3 + (with3 | join(jobs, (key, (row, _)) -> (key, row)) | negate) | map((key, (row, e, l, s)) -> (key, (row, e, l, s, [] : List((List(int), int, List(int))))))); -let ranked = with4 | map(((rid, node), ((d, (last, gender, birthday, created, ip, browser, town)), e, l, s, w)) -> (rid, ((d, last, node), (gender, birthday, created, ip, browser, town, e, l, s, w)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); +let who_idx = who | arrange; +let email_idx = email | arrange; -+let emails_join = who_idx | join(email_idx, (node, rid, value) -> ((rid, node), value)); ++let emails_join = who_idx | join(email_idx, (node, (rid, value)) -> ((rid, node), value)); +let emails_idx = emails_join | arrange; +let emails = emails_idx | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); +let who_idx2 = who | arrange; +let language_idx = language | arrange; -+let langs_join = who_idx2 | join(language_idx, (node, rid, value) -> ((rid, node), value)); ++let langs_join = who_idx2 | join(language_idx, (node, (rid, value)) -> ((rid, node), value)); +let langs_idx = langs_join | arrange; +let langs = langs_idx | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); +let who_idx3 = who | arrange; +let study_idx = study | arrange; -+let school_join = who_idx3 | join(study_idx, (node, rid, (at, year)) -> (at, (rid, node, year))); ++let school_join = who_idx3 | join(study_idx, (node, (rid, (at, year))) -> (at, (rid, node, year))); +let school_idx = school_join | arrange; +let org_idx = org | arrange; -+let school_join2 = school_idx | join(org_idx, (_, (rid, node, year), (name, _, at)) -> (at, (rid, node, name, year))); ++let school_join2 = school_idx | join(org_idx, (_, ((rid, node, year), (name, _, at))) -> (at, (rid, node, name, year))); +let school_idx2 = school_join2 | arrange; +let place_idx2 = place | arrange; -+let school_join3 = school_idx2 | join(place_idx2, (_, (rid, node, name, year), (where, _, _)) -> ((rid, node), (name, year, where))); ++let school_join3 = school_idx2 | join(place_idx2, (_, ((rid, node, name, year), (where, _, _))) -> ((rid, node), (name, year, where))); +let school_idx3 = school_join3 | arrange; +let school = school_idx3 | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); +let who_idx4 = who | arrange; +let work_idx = work | arrange; -+let jobs_join = who_idx4 | join(work_idx, (node, rid, (at, year)) -> (at, (rid, node, year))); ++let jobs_join = who_idx4 | join(work_idx, (node, (rid, (at, year))) -> (at, (rid, node, year))); +let jobs_idx = jobs_join | arrange; +let org_idx2 = org | arrange; -+let jobs_join2 = jobs_idx | join(org_idx2, (_, (rid, node, year), (name, _, at)) -> (at, (rid, node, name, year))); ++let jobs_join2 = jobs_idx | join(org_idx2, (_, ((rid, node, year), (name, _, at))) -> (at, (rid, node, name, year))); +let jobs_idx2 = jobs_join2 | arrange; +let place_idx3 = place | arrange; -+let jobs_join3 = jobs_idx2 | join(place_idx3, (_, (rid, node, name, year), (where, _, _)) -> ((rid, node), (name, year, where))); ++let jobs_join3 = jobs_idx2 | join(place_idx3, (_, ((rid, node, name, year), (where, _, _))) -> ((rid, node), (name, year, where))); +let jobs_idx3 = jobs_join3 | arrange; +let jobs = jobs_idx3 | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); +let found_idx3 = found | arrange; -+let with1_join = found_idx3 | join(emails, (key, row, list) -> (key, (row, list))); ++let with1_join = found_idx3 | join(emails, row -> row); +let found_idx4 = found | arrange; -+let with1_join2 = found_idx4 | join(emails, (key, row, _) -> (key, row)); ++let with1_join2 = found_idx4 | join(emails, (key, (row, _)) -> (key, row)); +let with1_neg = with1_join2 | negate; +let with1_rows = found + with1_neg; +let with1_map = with1_rows | map((key, row) -> (key, (row, [] : List(List(int))))); +let with1 = with1_join + with1_map; +let with1_idx = with1 | arrange; -+let with2_join = with1_idx | join(langs, (key, (row, e), list) -> (key, (row, e, list))); ++let with2_join = with1_idx | join(langs, (key, ((row, e), list)) -> (key, (row, e, list))); +let with1_idx2 = with1 | arrange; -+let with2_join2 = with1_idx2 | join(langs, (key, row, _) -> (key, row)); ++let with2_join2 = with1_idx2 | join(langs, (key, (row, _)) -> (key, row)); +let with2_neg = with2_join2 | negate; +let with2_rows = with1 + with2_neg; +let with2_map = with2_rows | map((key, (row, e)) -> (key, (row, e, [] : List(List(int))))); +let with2 = with2_join + with2_map; +let with2_idx = with2 | arrange; -+let with3_join = with2_idx | join(school, (key, (row, e, l), list) -> (key, (row, e, l, list))); ++let with3_join = with2_idx | join(school, (key, ((row, e, l), list)) -> (key, (row, e, l, list))); +let with2_idx2 = with2 | arrange; -+let with3_join2 = with2_idx2 | join(school, (key, row, _) -> (key, row)); ++let with3_join2 = with2_idx2 | join(school, (key, (row, _)) -> (key, row)); +let with3_neg = with3_join2 | negate; +let with3_rows = with2 + with3_neg; +let with3_map = with3_rows | map((key, (row, e, l)) -> (key, (row, e, l, [] : List((List(int), int, List(int)))))); +let with3 = with3_join + with3_map; +let with3_idx = with3 | arrange; -+let with4_join = with3_idx | join(jobs, (key, (row, e, l, s), list) -> (key, (row, e, l, s, list))); ++let with4_join = with3_idx | join(jobs, (key, ((row, e, l, s), list)) -> (key, (row, e, l, s, list))); +let with3_idx2 = with3 | arrange; -+let with4_join2 = with3_idx2 | join(jobs, (key, row, _) -> (key, row)); ++let with4_join2 = with3_idx2 | join(jobs, (key, (row, _)) -> (key, row)); +let with4_neg = with4_join2 | negate; +let with4_rows = with3 + with4_neg; +let with4_map = with4_rows | map((key, (row, e, l, s)) -> (key, (row, e, l, s, [] : List((List(int), int, List(int)))))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic01_oneoff.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic01_oneoff.parsed.ddp index 31536a5b0..d494fe7e2 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic01_oneoff.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic01_oneoff.parsed.ddp @@ -7,20 +7,20 @@ let study = import "graph::study.by_person" : (int, (int, int)); let work = import "graph::work.by_person" : (int, (int, int)); let email = import "graph::email.by_person" : (int, List(int)); let language = import "graph::language.by_person" : (int, List(int)); -let hop1 = request | map(((rid, who, name), ()) -> (who, (rid, who, name))) | join(friend, (_, req, (f, _)) -> (f, (req, 1))); -let hop2 = hop1 | join(friend, (_, (req, _), (g, _)) -> (g, (req, 2))); -let hop3 = hop2 | join(friend, (_, (req, _), (h, _)) -> (h, (req, 3))); +let hop1 = request | map(((rid, who, name), ()) -> (who, (rid, who, name))) | join(friend, (_, (req, (f, _))) -> (f, (req, 1))); +let hop2 = hop1 | join(friend, (_, ((req, _), (g, _))) -> (g, (req, 2))); +let hop3 = hop2 | join(friend, (_, ((req, _), (h, _))) -> (h, (req, 3))); let circle = hop1 + hop2 + hop3 | filter((node, ((_, who, _), _)) -> (node, who) ne) | map((node, ((rid, _, name), d)) -> ((rid, node, name), d)) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); -let found = circle | map(((rid, node, name), d) -> (node, (rid, name, d))) | join(person, (node, (rid, name, d), (first, last, gender, birthday, created, ip, browser, home, _, _, _)) -> ((first, name) eq, (home, (rid, node, d, last, gender, birthday, created, ip, browser)))) | filter((named, _) -> named) | map((_, row) -> row) | join(place, (_, (rid, node, d, last, gender, birthday, created, ip, browser), (town, _, _)) -> ((rid, node), (d, (last, gender, birthday, created, ip, browser, town)))); +let found = circle | map(((rid, node, name), d) -> (node, (rid, name, d))) | join(person, (node, ((rid, name, d), (first, last, gender, birthday, created, ip, browser, home, _, _, _))) -> ((first, name) eq, (home, (rid, node, d, last, gender, birthday, created, ip, browser)))) | filter((named, _) -> named) | map((_, row) -> row) | join(place, (_, ((rid, node, d, last, gender, birthday, created, ip, browser), (town, _, _))) -> ((rid, node), (d, (last, gender, birthday, created, ip, browser, town)))); let who = found | map(((rid, node), _) -> (node, rid)); -let emails = who | join(email, (node, rid, value) -> ((rid, node), value)) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); -let langs = who | join(language, (node, rid, value) -> ((rid, node), value)) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); -let school = who | join(study, (node, rid, (at, year)) -> (at, (rid, node, year))) | join(org, (_, (rid, node, year), (name, _, at)) -> (at, (rid, node, name, year))) | join(place, (_, (rid, node, name, year), (where, _, _)) -> ((rid, node), (name, year, where))) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); -let jobs = who | join(work, (node, rid, (at, year)) -> (at, (rid, node, year))) | join(org, (_, (rid, node, year), (name, _, at)) -> (at, (rid, node, name, year))) | join(place, (_, (rid, node, name, year), (where, _, _)) -> ((rid, node), (name, year, where))) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); -let with1 = (found | join(emails, (key, row, list) -> (key, (row, list)))) + (found + (found | join(emails, (key, row, _) -> (key, row)) | negate) | map((key, row) -> (key, (row, [] : List(List(int)))))); -let with2 = (with1 | join(langs, (key, (row, e), list) -> (key, (row, e, list)))) + (with1 + (with1 | join(langs, (key, row, _) -> (key, row)) | negate) | map((key, (row, e)) -> (key, (row, e, [] : List(List(int)))))); -let with3 = (with2 | join(school, (key, (row, e, l), list) -> (key, (row, e, l, list)))) + (with2 + (with2 | join(school, (key, row, _) -> (key, row)) | negate) | map((key, (row, e, l)) -> (key, (row, e, l, [] : List((List(int), int, List(int))))))); -let with4 = (with3 | join(jobs, (key, (row, e, l, s), list) -> (key, (row, e, l, s, list)))) + (with3 + (with3 | join(jobs, (key, row, _) -> (key, row)) | negate) | map((key, (row, e, l, s)) -> (key, (row, e, l, s, [] : List((List(int), int, List(int))))))); +let emails = who | join(email, (node, (rid, value)) -> ((rid, node), value)) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); +let langs = who | join(language, (node, (rid, value)) -> ((rid, node), value)) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); +let school = who | join(study, (node, (rid, (at, year))) -> (at, (rid, node, year))) | join(org, (_, ((rid, node, year), (name, _, at))) -> (at, (rid, node, name, year))) | join(place, (_, ((rid, node, name, year), (where, _, _))) -> ((rid, node), (name, year, where))) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); +let jobs = who | join(work, (node, (rid, (at, year))) -> (at, (rid, node, year))) | join(org, (_, ((rid, node, year), (name, _, at))) -> (at, (rid, node, name, year))) | join(place, (_, ((rid, node, name, year), (where, _, _))) -> ((rid, node), (name, year, where))) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); +let with1 = (found | join(emails, row -> row)) + (found + (found | join(emails, (key, (row, _)) -> (key, row)) | negate) | map((key, row) -> (key, (row, [] : List(List(int)))))); +let with2 = (with1 | join(langs, (key, ((row, e), list)) -> (key, (row, e, list)))) + (with1 + (with1 | join(langs, (key, (row, _)) -> (key, row)) | negate) | map((key, (row, e)) -> (key, (row, e, [] : List(List(int)))))); +let with3 = (with2 | join(school, (key, ((row, e, l), list)) -> (key, (row, e, l, list)))) + (with2 + (with2 | join(school, (key, (row, _)) -> (key, row)) | negate) | map((key, (row, e, l)) -> (key, (row, e, l, [] : List((List(int), int, List(int))))))); +let with4 = (with3 | join(jobs, (key, ((row, e, l, s), list)) -> (key, (row, e, l, s, list)))) + (with3 + (with3 | join(jobs, (key, (row, _)) -> (key, row)) | negate) | map((key, (row, e, l, s)) -> (key, (row, e, l, s, [] : List((List(int), int, List(int))))))); let ranked = with4 | map(((rid, node), ((d, (last, gender, birthday, created, ip, browser, town)), e, l, s, w)) -> (rid, ((d, last, node), (gender, birthday, created, ip, browser, town, e, l, s, w)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((d, last, node), (gender, birthday, created, ip, browser, town, e, l, s, w)))) -> ((rid, rank), (node, last, d, birthday, created, gender, browser, ip, e, l, town, s, w))); output "answer" = answer | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic02_oneoff.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic02_oneoff.ddp index 703c70f9b..71e05657f 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic02_oneoff.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic02_oneoff.ddp @@ -10,18 +10,18 @@ let newest = import "ic2s::newest" : (int, int); let friends_map = request | map(((rid, who, max), ()) -> (who, (rid, max))); let friends_idx = friends_map | arrange; let friend_idx = friend | arrange; -let friends = friends_idx | join(friend_idx, (_, req, (f, _)) -> (f, req)); +let friends = friends_idx | join(friend_idx, (_, (req, (f, _))) -> (f, req)); let friends_idx2 = friends | arrange; let newest_idx = newest | arrange; -let ahead = friends_idx2 | join(newest_idx, (f, (rid, max), neg) -> (f, (rid, max, neg neg))); +let ahead = friends_idx2 | join(newest_idx, (f, ((rid, max), neg)) -> (f, (rid, max, neg neg))); let fast_filter = ahead | filter((_, (_, max, last)) -> (last, max) lt); let fast_idx = fast_filter | arrange; let latest_idx = latest | arrange; -let fast = fast_idx | join(latest_idx, (f, (rid, _, _), (neg, id)) -> (rid, ((neg, id), f))); +let fast = fast_idx | join(latest_idx, (f, ((rid, _, _), (neg, id))) -> (rid, ((neg, id), f))); let slow_filter = ahead | filter((_, (_, max, last)) -> (last, max) ge); let slow_idx = slow_filter | arrange; let by_author_idx = by_author | arrange; -let slow_join = slow_idx | join(by_author_idx, (f, (rid, max, _), (neg, id)) -> (rid, ((neg, id), f, max))); +let slow_join = slow_idx | join(by_author_idx, (f, ((rid, max, _), (neg, id))) -> (rid, ((neg, id), f, max))); let slow = slow_join | filter((_, ((neg, _), _, max)) -> (neg neg, max) lt) | map((rid, (order, f, _)) -> (rid, (order, f))); let top_rows = fast + slow; let top_idx = top_rows | arrange; @@ -29,10 +29,10 @@ let top = top_idx | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, let answer_map = top | map((rid, (rank, ((neg, id), f))) -> (id, (rid, rank, neg neg, f))); let answer_idx = answer_map | arrange; let message_idx = message | arrange; -let answer_join = answer_idx | join(message_idx, (id, (rid, rank, created, f), (_, _, _, _, content, image, _, _, _, _, _, _, _)) -> (f, (rid, rank, id, ((content len, 0) gt, content, image) select, created))); +let answer_join = answer_idx | join(message_idx, (id, ((rid, rank, created, f), (_, _, _, _, content, image, _, _, _, _, _, _, _))) -> (f, (rid, rank, id, ((content len, 0) gt, content, image) select, created))); let answer_idx2 = answer_join | arrange; let person_idx = person | arrange; -let answer = answer_idx2 | join(person_idx, (f, (rid, rank, id, body, created), (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, rank), (f, first, last, id, body, created))); +let answer = answer_idx2 | join(person_idx, (f, ((rid, rank, id, body, created), (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, rank), (f, first, last, id, body, created))); let answer_idx3 = answer | arrange; -- Renderer: publish the existing arrangement. output "answer" = answer_idx3; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic02_oneoff.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/ldbc_ic02_oneoff.lowered--optimized.diff index bc992df1c..4345b5c63 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic02_oneoff.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic02_oneoff.lowered--optimized.diff @@ -3,7 +3,7 @@ @@ -22,8 +22,7 @@ let slow_idx = slow_filter | arrange; let by_author_idx = by_author | arrange; - let slow_join = slow_idx | join(by_author_idx, (f, (rid, max, _), (neg, id)) -> (rid, ((neg, id), f, max))); + let slow_join = slow_idx | join(by_author_idx, (f, ((rid, max, _), (neg, id))) -> (rid, ((neg, id), f, max))); -let slow_filter2 = slow_join | filter((_, ((neg, _), _, max)) -> (neg neg, max) lt); -let slow = slow_filter2 | map((rid, (order, f, _)) -> (rid, (order, f))); +let slow = slow_join | filter((_, ((neg, _), _, max)) -> (neg neg, max) lt) | map((rid, (order, f, _)) -> (rid, (order, f))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic02_oneoff.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic02_oneoff.parsed--lowered.diff index 2f2053745..0aa20fbfd 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic02_oneoff.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic02_oneoff.parsed--lowered.diff @@ -10,28 +10,28 @@ let by_author = import "ic2s::by_author" : (int, (int, int)); let latest = import "ic2s::latest" : (int, (int, int)); let newest = import "ic2s::newest" : (int, int); --let friends = request | map(((rid, who, max), ()) -> (who, (rid, max))) | join(friend, (_, req, (f, _)) -> (f, req)); --let ahead = friends | join(newest, (f, (rid, max), neg) -> (f, (rid, max, neg neg))); --let fast = ahead | filter((_, (_, max, last)) -> (last, max) lt) | join(latest, (f, (rid, _, _), (neg, id)) -> (rid, ((neg, id), f))); --let slow = ahead | filter((_, (_, max, last)) -> (last, max) ge) | join(by_author, (f, (rid, max, _), (neg, id)) -> (rid, ((neg, id), f, max))) | filter((_, ((neg, _), _, max)) -> (neg neg, max) lt) | map((rid, (order, f, _)) -> (rid, (order, f))); +-let friends = request | map(((rid, who, max), ()) -> (who, (rid, max))) | join(friend, (_, (req, (f, _))) -> (f, req)); +-let ahead = friends | join(newest, (f, ((rid, max), neg)) -> (f, (rid, max, neg neg))); +-let fast = ahead | filter((_, (_, max, last)) -> (last, max) lt) | join(latest, (f, ((rid, _, _), (neg, id))) -> (rid, ((neg, id), f))); +-let slow = ahead | filter((_, (_, max, last)) -> (last, max) ge) | join(by_author, (f, ((rid, max, _), (neg, id))) -> (rid, ((neg, id), f, max))) | filter((_, ((neg, _), _, max)) -> (neg neg, max) lt) | map((rid, (order, f, _)) -> (rid, (order, f))); -let top = fast + slow | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); --let answer = top | map((rid, (rank, ((neg, id), f))) -> (id, (rid, rank, neg neg, f))) | join(message, (id, (rid, rank, created, f), (_, _, _, _, content, image, _, _, _, _, _, _, _)) -> (f, (rid, rank, id, ((content len, 0) gt, content, image) select, created))) | join(person, (f, (rid, rank, id, body, created), (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, rank), (f, first, last, id, body, created))); +-let answer = top | map((rid, (rank, ((neg, id), f))) -> (id, (rid, rank, neg neg, f))) | join(message, (id, ((rid, rank, created, f), (_, _, _, _, content, image, _, _, _, _, _, _, _))) -> (f, (rid, rank, id, ((content len, 0) gt, content, image) select, created))) | join(person, (f, ((rid, rank, id, body, created), (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, rank), (f, first, last, id, body, created))); -output "answer" = answer | arrange; +let friends_map = request | map(((rid, who, max), ()) -> (who, (rid, max))); +let friends_idx = friends_map | arrange; +let friend_idx = friend | arrange; -+let friends = friends_idx | join(friend_idx, (_, req, (f, _)) -> (f, req)); ++let friends = friends_idx | join(friend_idx, (_, (req, (f, _))) -> (f, req)); +let friends_idx2 = friends | arrange; +let newest_idx = newest | arrange; -+let ahead = friends_idx2 | join(newest_idx, (f, (rid, max), neg) -> (f, (rid, max, neg neg))); ++let ahead = friends_idx2 | join(newest_idx, (f, ((rid, max), neg)) -> (f, (rid, max, neg neg))); +let fast_filter = ahead | filter((_, (_, max, last)) -> (last, max) lt); +let fast_idx = fast_filter | arrange; +let latest_idx = latest | arrange; -+let fast = fast_idx | join(latest_idx, (f, (rid, _, _), (neg, id)) -> (rid, ((neg, id), f))); ++let fast = fast_idx | join(latest_idx, (f, ((rid, _, _), (neg, id))) -> (rid, ((neg, id), f))); +let slow_filter = ahead | filter((_, (_, max, last)) -> (last, max) ge); +let slow_idx = slow_filter | arrange; +let by_author_idx = by_author | arrange; -+let slow_join = slow_idx | join(by_author_idx, (f, (rid, max, _), (neg, id)) -> (rid, ((neg, id), f, max))); ++let slow_join = slow_idx | join(by_author_idx, (f, ((rid, max, _), (neg, id))) -> (rid, ((neg, id), f, max))); +let slow_filter2 = slow_join | filter((_, ((neg, _), _, max)) -> (neg neg, max) lt); +let slow = slow_filter2 | map((rid, (order, f, _)) -> (rid, (order, f))); +let top_rows = fast + slow; @@ -40,10 +40,10 @@ +let answer_map = top | map((rid, (rank, ((neg, id), f))) -> (id, (rid, rank, neg neg, f))); +let answer_idx = answer_map | arrange; +let message_idx = message | arrange; -+let answer_join = answer_idx | join(message_idx, (id, (rid, rank, created, f), (_, _, _, _, content, image, _, _, _, _, _, _, _)) -> (f, (rid, rank, id, ((content len, 0) gt, content, image) select, created))); ++let answer_join = answer_idx | join(message_idx, (id, ((rid, rank, created, f), (_, _, _, _, content, image, _, _, _, _, _, _, _))) -> (f, (rid, rank, id, ((content len, 0) gt, content, image) select, created))); +let answer_idx2 = answer_join | arrange; +let person_idx = person | arrange; -+let answer = answer_idx2 | join(person_idx, (f, (rid, rank, id, body, created), (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, rank), (f, first, last, id, body, created))); ++let answer = answer_idx2 | join(person_idx, (f, ((rid, rank, id, body, created), (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, rank), (f, first, last, id, body, created))); +let answer_idx3 = answer | arrange; +-- Renderer: publish the existing arrangement. +output "answer" = answer_idx3; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic02_oneoff.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic02_oneoff.parsed.ddp index e6723d097..74eba1e4b 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic02_oneoff.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic02_oneoff.parsed.ddp @@ -5,10 +5,10 @@ let friend = import "graph::friend" : (int, (int, int)); let by_author = import "ic2s::by_author" : (int, (int, int)); let latest = import "ic2s::latest" : (int, (int, int)); let newest = import "ic2s::newest" : (int, int); -let friends = request | map(((rid, who, max), ()) -> (who, (rid, max))) | join(friend, (_, req, (f, _)) -> (f, req)); -let ahead = friends | join(newest, (f, (rid, max), neg) -> (f, (rid, max, neg neg))); -let fast = ahead | filter((_, (_, max, last)) -> (last, max) lt) | join(latest, (f, (rid, _, _), (neg, id)) -> (rid, ((neg, id), f))); -let slow = ahead | filter((_, (_, max, last)) -> (last, max) ge) | join(by_author, (f, (rid, max, _), (neg, id)) -> (rid, ((neg, id), f, max))) | filter((_, ((neg, _), _, max)) -> (neg neg, max) lt) | map((rid, (order, f, _)) -> (rid, (order, f))); +let friends = request | map(((rid, who, max), ()) -> (who, (rid, max))) | join(friend, (_, (req, (f, _))) -> (f, req)); +let ahead = friends | join(newest, (f, ((rid, max), neg)) -> (f, (rid, max, neg neg))); +let fast = ahead | filter((_, (_, max, last)) -> (last, max) lt) | join(latest, (f, ((rid, _, _), (neg, id))) -> (rid, ((neg, id), f))); +let slow = ahead | filter((_, (_, max, last)) -> (last, max) ge) | join(by_author, (f, ((rid, max, _), (neg, id))) -> (rid, ((neg, id), f, max))) | filter((_, ((neg, _), _, max)) -> (neg neg, max) lt) | map((rid, (order, f, _)) -> (rid, (order, f))); let top = fast + slow | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); -let answer = top | map((rid, (rank, ((neg, id), f))) -> (id, (rid, rank, neg neg, f))) | join(message, (id, (rid, rank, created, f), (_, _, _, _, content, image, _, _, _, _, _, _, _)) -> (f, (rid, rank, id, ((content len, 0) gt, content, image) select, created))) | join(person, (f, (rid, rank, id, body, created), (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, rank), (f, first, last, id, body, created))); +let answer = top | map((rid, (rank, ((neg, id), f))) -> (id, (rid, rank, neg neg, f))) | join(message, (id, ((rid, rank, created, f), (_, _, _, _, content, image, _, _, _, _, _, _, _))) -> (f, (rid, rank, id, ((content len, 0) gt, content, image) select, created))) | join(person, (f, ((rid, rank, id, body, created), (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, rank), (f, first, last, id, body, created))); output "answer" = answer | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.ddp index 009352ab7..538acf1d3 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.ddp @@ -9,51 +9,51 @@ let sent_from = import "ic3s::sent_from" : (int, (int, int)); let asked_map = request | map(((rid, who, x, y, start, days), ()) -> (x, (rid, who, y, start, (start, (days, 86400000) mul) add))); let asked_idx = asked_map | arrange; let by_name_idx = by_name | arrange; -let asked_join = asked_idx | join(by_name_idx, (_, req, x) -> (x, req)); +let asked_join = asked_idx | join(by_name_idx, (_, (req, x)) -> (x, req)); let asked_idx2 = asked_join | arrange; let place_idx = place | arrange; -let asked_join2 = asked_idx2 | join(place_idx, (x, (rid, who, y, start, end), (_, kind, _)) -> ((kind, [67,111,117,110,116,114,121]) eq, (y, (rid, who, x, start, end)))); +let asked_join2 = asked_idx2 | join(place_idx, (x, ((rid, who, y, start, end), (_, kind, _))) -> ((kind, [67,111,117,110,116,114,121]) eq, (y, (rid, who, x, start, end)))); let asked_rows = asked_join2 | filter((country, _) -> country) | map((_, row) -> row); let asked_idx3 = asked_rows | arrange; -let asked_join3 = asked_idx3 | join(by_name_idx, (_, req, y) -> (y, req)); +let asked_join3 = asked_idx3 | join(by_name_idx, (_, (req, y)) -> (y, req)); let asked_idx4 = asked_join3 | arrange; -let asked_join4 = asked_idx4 | join(place_idx, (y, (rid, who, x, start, end), (_, kind, _)) -> ((kind, [67,111,117,110,116,114,121]) eq, (rid, (who, x, y, start, end)))); +let asked_join4 = asked_idx4 | join(place_idx, (y, ((rid, who, x, start, end), (_, kind, _))) -> ((kind, [67,111,117,110,116,114,121]) eq, (rid, (who, x, y, start, end)))); let asked = asked_join4 | filter((country, _) -> country) | map((_, row) -> row); let window_map = asked | map((rid, (_, x, _, start, end)) -> (x, (rid, start, end, 1, 0))); let window_map2 = asked | map((rid, (_, _, y, start, end)) -> (y, (rid, start, end, 0, 1))); let window = window_map + window_map2; let window_idx = window | arrange; let sent_from_idx = sent_from | arrange; -let visits_join = window_idx | join(sent_from_idx, (_, (rid, start, end, ix, iy), (author, created)) -> ((rid, author), (created, start, end, ix, iy))); +let visits_join = window_idx | join(sent_from_idx, (_, ((rid, start, end, ix, iy), (author, created))) -> ((rid, author), (created, start, end, ix, iy))); let visits = visits_join | filter((_, (created, start, end, _, _)) -> ((created, start) ge, (created, end) lt) all) | map((key, (_, _, _, ix, iy)) -> (key, (ix, iy))); let hop1_map = asked | map((rid, (who, x, y, _, _)) -> (who, (rid, who, x, y))); let hop1_idx = hop1_map | arrange; let friend_idx = friend | arrange; -let hop1 = hop1_idx | join(friend_idx, (_, req, (f, _)) -> (f, req)); +let hop1 = hop1_idx | join(friend_idx, (_, (req, (f, _))) -> (f, req)); let hop1_idx2 = hop1 | arrange; -let hop2 = hop1_idx2 | join(friend_idx, (_, req, (g, _)) -> (g, req)); +let hop2 = hop1_idx2 | join(friend_idx, (_, (req, (g, _))) -> (g, req)); let circle_rows = hop1 + hop2; let circle_filter = circle_rows | filter((node, (_, who, _, _)) -> (node, who) ne); let circle_idx = circle_filter | arrange; let person_idx = person | arrange; -let circle_join = circle_idx | join(person_idx, (node, (rid, _, x, y), (_, _, _, _, _, _, _, city, _, _, _)) -> (city, (rid, node, x, y))); +let circle_join = circle_idx | join(person_idx, (node, ((rid, _, x, y), (_, _, _, _, _, _, _, city, _, _, _))) -> (city, (rid, node, x, y))); let circle_idx2 = circle_join | arrange; -let circle_join2 = circle_idx2 | join(place_idx, (_, (rid, node, x, y), (_, _, home)) -> ((rid, node), (x, y, home))); +let circle_join2 = circle_idx2 | join(place_idx, (_, ((rid, node, x, y), (_, _, home))) -> ((rid, node), (x, y, home))); let circle_rows2 = circle_join2 | filter((_, (x, y, home)) -> ((home, x) ne, (home, y) ne) all) | map((key, _) -> (key, ())); let circle_idx3 = circle_rows2 | arrange; let circle = circle_idx3 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); let visits_idx = visits | arrange; -let seen = visits_idx | join(circle, (key, (ix, iy), ()) -> (key, (ix, iy))); +let seen = visits_idx | join(circle, (key, ((ix, iy), ())) -> (key, (ix, iy))); let in_x_rows = seen | filter((_, (ix, _)) -> (ix, 1) eq) | map((key, _) -> (key, ())); let in_x_idx = in_x_rows | arrange; let in_x = in_x_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); let in_y_rows = seen | filter((_, (_, iy)) -> (iy, 1) eq) | map((key, _) -> (key, ())); let in_y_idx = in_y_rows | arrange; let in_y = in_y_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); -let counts = in_x | join(in_y, (key, nx, ny) -> (key, (nx, ny))); +let counts = in_x | join(in_y, row -> row); let ranked_map = counts | map(((rid, node), (nx, ny)) -> (node, (rid, nx, ny))); let ranked_idx = ranked_map | arrange; -let ranked_join = ranked_idx | join(person_idx, (node, (rid, nx, ny), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, (((nx, ny) add neg, node), (first, last, nx, ny)))); +let ranked_join = ranked_idx | join(person_idx, (node, ((rid, nx, ny), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, (((nx, ny) add neg, node), (first, last, nx, ny)))); let ranked_idx2 = ranked_join | arrange; let ranked = ranked_idx2 | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((_, node), (first, last, nx, ny)))) -> ((rid, rank), (node, first, last, nx, ny, (nx, ny) add))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.lowered--optimized.diff index e0cb7952e..2ab01679d 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.lowered--optimized.diff @@ -11,56 +11,56 @@ @@ -14,58 +13,47 @@ let asked_idx2 = asked_join | arrange; let place_idx = place | arrange; - let asked_join2 = asked_idx2 | join(place_idx, (x, (rid, who, y, start, end), (_, kind, _)) -> ((kind, [67,111,117,110,116,114,121]) eq, (y, (rid, who, x, start, end)))); + let asked_join2 = asked_idx2 | join(place_idx, (x, ((rid, who, y, start, end), (_, kind, _))) -> ((kind, [67,111,117,110,116,114,121]) eq, (y, (rid, who, x, start, end)))); -let asked_filter = asked_join2 | filter((country, _) -> country); -let asked_map2 = asked_filter | map((_, row) -> row); -let asked_idx3 = asked_map2 | arrange; -let by_name_idx2 = by_name | arrange; --let asked_join3 = asked_idx3 | join(by_name_idx2, (_, req, y) -> (y, req)); +-let asked_join3 = asked_idx3 | join(by_name_idx2, (_, (req, y)) -> (y, req)); +let asked_rows = asked_join2 | filter((country, _) -> country) | map((_, row) -> row); +let asked_idx3 = asked_rows | arrange; -+let asked_join3 = asked_idx3 | join(by_name_idx, (_, req, y) -> (y, req)); ++let asked_join3 = asked_idx3 | join(by_name_idx, (_, (req, y)) -> (y, req)); let asked_idx4 = asked_join3 | arrange; -let place_idx2 = place | arrange; --let asked_join4 = asked_idx4 | join(place_idx2, (y, (rid, who, x, start, end), (_, kind, _)) -> ((kind, [67,111,117,110,116,114,121]) eq, (rid, (who, x, y, start, end)))); +-let asked_join4 = asked_idx4 | join(place_idx2, (y, ((rid, who, x, start, end), (_, kind, _))) -> ((kind, [67,111,117,110,116,114,121]) eq, (rid, (who, x, y, start, end)))); -let asked_filter2 = asked_join4 | filter((country, _) -> country); -let asked = asked_filter2 | map((_, row) -> row); -+let asked_join4 = asked_idx4 | join(place_idx, (y, (rid, who, x, start, end), (_, kind, _)) -> ((kind, [67,111,117,110,116,114,121]) eq, (rid, (who, x, y, start, end)))); ++let asked_join4 = asked_idx4 | join(place_idx, (y, ((rid, who, x, start, end), (_, kind, _))) -> ((kind, [67,111,117,110,116,114,121]) eq, (rid, (who, x, y, start, end)))); +let asked = asked_join4 | filter((country, _) -> country) | map((_, row) -> row); let window_map = asked | map((rid, (_, x, _, start, end)) -> (x, (rid, start, end, 1, 0))); let window_map2 = asked | map((rid, (_, _, y, start, end)) -> (y, (rid, start, end, 0, 1))); let window = window_map + window_map2; let window_idx = window | arrange; let sent_from_idx = sent_from | arrange; - let visits_join = window_idx | join(sent_from_idx, (_, (rid, start, end, ix, iy), (author, created)) -> ((rid, author), (created, start, end, ix, iy))); + let visits_join = window_idx | join(sent_from_idx, (_, ((rid, start, end, ix, iy), (author, created))) -> ((rid, author), (created, start, end, ix, iy))); -let visits_filter = visits_join | filter((_, (created, start, end, _, _)) -> ((created, start) ge, (created, end) lt) all); -let visits = visits_filter | map((key, (_, _, _, ix, iy)) -> (key, (ix, iy))); +let visits = visits_join | filter((_, (created, start, end, _, _)) -> ((created, start) ge, (created, end) lt) all) | map((key, (_, _, _, ix, iy)) -> (key, (ix, iy))); let hop1_map = asked | map((rid, (who, x, y, _, _)) -> (who, (rid, who, x, y))); let hop1_idx = hop1_map | arrange; let friend_idx = friend | arrange; - let hop1 = hop1_idx | join(friend_idx, (_, req, (f, _)) -> (f, req)); + let hop1 = hop1_idx | join(friend_idx, (_, (req, (f, _))) -> (f, req)); let hop1_idx2 = hop1 | arrange; -let friend_idx2 = friend | arrange; --let hop2 = hop1_idx2 | join(friend_idx2, (_, req, (g, _)) -> (g, req)); -+let hop2 = hop1_idx2 | join(friend_idx, (_, req, (g, _)) -> (g, req)); +-let hop2 = hop1_idx2 | join(friend_idx2, (_, (req, (g, _))) -> (g, req)); ++let hop2 = hop1_idx2 | join(friend_idx, (_, (req, (g, _))) -> (g, req)); let circle_rows = hop1 + hop2; let circle_filter = circle_rows | filter((node, (_, who, _, _)) -> (node, who) ne); let circle_idx = circle_filter | arrange; let person_idx = person | arrange; - let circle_join = circle_idx | join(person_idx, (node, (rid, _, x, y), (_, _, _, _, _, _, _, city, _, _, _)) -> (city, (rid, node, x, y))); + let circle_join = circle_idx | join(person_idx, (node, ((rid, _, x, y), (_, _, _, _, _, _, _, city, _, _, _))) -> (city, (rid, node, x, y))); let circle_idx2 = circle_join | arrange; -let place_idx3 = place | arrange; --let circle_join2 = circle_idx2 | join(place_idx3, (_, (rid, node, x, y), (_, _, home)) -> ((rid, node), (x, y, home))); +-let circle_join2 = circle_idx2 | join(place_idx3, (_, ((rid, node, x, y), (_, _, home))) -> ((rid, node), (x, y, home))); -let circle_filter2 = circle_join2 | filter((_, (x, y, home)) -> ((home, x) ne, (home, y) ne) all); -let circle_map = circle_filter2 | map((key, _) -> (key, ())); -let circle_idx3 = circle_map | arrange; -+let circle_join2 = circle_idx2 | join(place_idx, (_, (rid, node, x, y), (_, _, home)) -> ((rid, node), (x, y, home))); ++let circle_join2 = circle_idx2 | join(place_idx, (_, ((rid, node, x, y), (_, _, home))) -> ((rid, node), (x, y, home))); +let circle_rows2 = circle_join2 | filter((_, (x, y, home)) -> ((home, x) ne, (home, y) ne) all) | map((key, _) -> (key, ())); +let circle_idx3 = circle_rows2 | arrange; let circle = circle_idx3 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); let visits_idx = visits | arrange; - let seen = visits_idx | join(circle, (key, (ix, iy), ()) -> (key, (ix, iy))); + let seen = visits_idx | join(circle, (key, ((ix, iy), ())) -> (key, (ix, iy))); -let in_x_filter = seen | filter((_, (ix, _)) -> (ix, 1) eq); -let in_x_map = in_x_filter | map((key, _) -> (key, ())); -let in_x_idx = in_x_map | arrange; @@ -73,12 +73,12 @@ +let in_y_rows = seen | filter((_, (_, iy)) -> (iy, 1) eq) | map((key, _) -> (key, ())); +let in_y_idx = in_y_rows | arrange; let in_y = in_y_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); - let counts = in_x | join(in_y, (key, nx, ny) -> (key, (nx, ny))); + let counts = in_x | join(in_y, row -> row); let ranked_map = counts | map(((rid, node), (nx, ny)) -> (node, (rid, nx, ny))); let ranked_idx = ranked_map | arrange; -let person_idx2 = person | arrange; --let ranked_join = ranked_idx | join(person_idx2, (node, (rid, nx, ny), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, (((nx, ny) add neg, node), (first, last, nx, ny)))); -+let ranked_join = ranked_idx | join(person_idx, (node, (rid, nx, ny), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, (((nx, ny) add neg, node), (first, last, nx, ny)))); +-let ranked_join = ranked_idx | join(person_idx2, (node, ((rid, nx, ny), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, (((nx, ny) add neg, node), (first, last, nx, ny)))); ++let ranked_join = ranked_idx | join(person_idx, (node, ((rid, nx, ny), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, (((nx, ny) add neg, node), (first, last, nx, ny)))); let ranked_idx2 = ranked_join | arrange; let ranked = ranked_idx2 | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((_, node), (first, last, nx, ny)))) -> ((rid, rank), (node, first, last, nx, ny, (nx, ny) add))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.parsed--lowered.diff index 3e47b604a..44a699a12 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.parsed--lowered.diff @@ -10,30 +10,30 @@ let place = import "graph::place" : (int, (List(int), List(int), int)); let by_name = import "graph::place.by_name" : (List(int), int); let sent_from = import "ic3s::sent_from" : (int, (int, int)); --let asked = request | map(((rid, who, x, y, start, days), ()) -> (x, (rid, who, y, start, (start, (days, 86400000) mul) add))) | join(by_name, (_, req, x) -> (x, req)) | join(place, (x, (rid, who, y, start, end), (_, kind, _)) -> ((kind, [67,111,117,110,116,114,121]) eq, (y, (rid, who, x, start, end)))) | filter((country, _) -> country) | map((_, row) -> row) | join(by_name, (_, req, y) -> (y, req)) | join(place, (y, (rid, who, x, start, end), (_, kind, _)) -> ((kind, [67,111,117,110,116,114,121]) eq, (rid, (who, x, y, start, end)))) | filter((country, _) -> country) | map((_, row) -> row); +-let asked = request | map(((rid, who, x, y, start, days), ()) -> (x, (rid, who, y, start, (start, (days, 86400000) mul) add))) | join(by_name, (_, (req, x)) -> (x, req)) | join(place, (x, ((rid, who, y, start, end), (_, kind, _))) -> ((kind, [67,111,117,110,116,114,121]) eq, (y, (rid, who, x, start, end)))) | filter((country, _) -> country) | map((_, row) -> row) | join(by_name, (_, (req, y)) -> (y, req)) | join(place, (y, ((rid, who, x, start, end), (_, kind, _))) -> ((kind, [67,111,117,110,116,114,121]) eq, (rid, (who, x, y, start, end)))) | filter((country, _) -> country) | map((_, row) -> row); -let window = (asked | map((rid, (_, x, _, start, end)) -> (x, (rid, start, end, 1, 0)))) + (asked | map((rid, (_, _, y, start, end)) -> (y, (rid, start, end, 0, 1)))); --let visits = window | join(sent_from, (_, (rid, start, end, ix, iy), (author, created)) -> ((rid, author), (created, start, end, ix, iy))) | filter((_, (created, start, end, _, _)) -> ((created, start) ge, (created, end) lt) all) | map((key, (_, _, _, ix, iy)) -> (key, (ix, iy))); --let hop1 = asked | map((rid, (who, x, y, _, _)) -> (who, (rid, who, x, y))) | join(friend, (_, req, (f, _)) -> (f, req)); --let hop2 = hop1 | join(friend, (_, req, (g, _)) -> (g, req)); --let circle = hop1 + hop2 | filter((node, (_, who, _, _)) -> (node, who) ne) | join(person, (node, (rid, _, x, y), (_, _, _, _, _, _, _, city, _, _, _)) -> (city, (rid, node, x, y))) | join(place, (_, (rid, node, x, y), (_, _, home)) -> ((rid, node), (x, y, home))) | filter((_, (x, y, home)) -> ((home, x) ne, (home, y) ne) all) | map((key, _) -> (key, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); --let seen = visits | join(circle, (key, (ix, iy), ()) -> (key, (ix, iy))); +-let visits = window | join(sent_from, (_, ((rid, start, end, ix, iy), (author, created))) -> ((rid, author), (created, start, end, ix, iy))) | filter((_, (created, start, end, _, _)) -> ((created, start) ge, (created, end) lt) all) | map((key, (_, _, _, ix, iy)) -> (key, (ix, iy))); +-let hop1 = asked | map((rid, (who, x, y, _, _)) -> (who, (rid, who, x, y))) | join(friend, (_, (req, (f, _))) -> (f, req)); +-let hop2 = hop1 | join(friend, (_, (req, (g, _))) -> (g, req)); +-let circle = hop1 + hop2 | filter((node, (_, who, _, _)) -> (node, who) ne) | join(person, (node, ((rid, _, x, y), (_, _, _, _, _, _, _, city, _, _, _))) -> (city, (rid, node, x, y))) | join(place, (_, ((rid, node, x, y), (_, _, home))) -> ((rid, node), (x, y, home))) | filter((_, (x, y, home)) -> ((home, x) ne, (home, y) ne) all) | map((key, _) -> (key, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); +-let seen = visits | join(circle, (key, ((ix, iy), ())) -> (key, (ix, iy))); -let in_x = seen | filter((_, (ix, _)) -> (ix, 1) eq) | map((key, _) -> (key, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); -let in_y = seen | filter((_, (_, iy)) -> (iy, 1) eq) | map((key, _) -> (key, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +let asked_map = request | map(((rid, who, x, y, start, days), ()) -> (x, (rid, who, y, start, (start, (days, 86400000) mul) add))); +let asked_idx = asked_map | arrange; +let by_name_idx = by_name | arrange; -+let asked_join = asked_idx | join(by_name_idx, (_, req, x) -> (x, req)); ++let asked_join = asked_idx | join(by_name_idx, (_, (req, x)) -> (x, req)); +let asked_idx2 = asked_join | arrange; +let place_idx = place | arrange; -+let asked_join2 = asked_idx2 | join(place_idx, (x, (rid, who, y, start, end), (_, kind, _)) -> ((kind, [67,111,117,110,116,114,121]) eq, (y, (rid, who, x, start, end)))); ++let asked_join2 = asked_idx2 | join(place_idx, (x, ((rid, who, y, start, end), (_, kind, _))) -> ((kind, [67,111,117,110,116,114,121]) eq, (y, (rid, who, x, start, end)))); +let asked_filter = asked_join2 | filter((country, _) -> country); +let asked_map2 = asked_filter | map((_, row) -> row); +let asked_idx3 = asked_map2 | arrange; +let by_name_idx2 = by_name | arrange; -+let asked_join3 = asked_idx3 | join(by_name_idx2, (_, req, y) -> (y, req)); ++let asked_join3 = asked_idx3 | join(by_name_idx2, (_, (req, y)) -> (y, req)); +let asked_idx4 = asked_join3 | arrange; +let place_idx2 = place | arrange; -+let asked_join4 = asked_idx4 | join(place_idx2, (y, (rid, who, x, start, end), (_, kind, _)) -> ((kind, [67,111,117,110,116,114,121]) eq, (rid, (who, x, y, start, end)))); ++let asked_join4 = asked_idx4 | join(place_idx2, (y, ((rid, who, x, start, end), (_, kind, _))) -> ((kind, [67,111,117,110,116,114,121]) eq, (rid, (who, x, y, start, end)))); +let asked_filter2 = asked_join4 | filter((country, _) -> country); +let asked = asked_filter2 | map((_, row) -> row); +let window_map = asked | map((rid, (_, x, _, start, end)) -> (x, (rid, start, end, 1, 0))); @@ -41,30 +41,30 @@ +let window = window_map + window_map2; +let window_idx = window | arrange; +let sent_from_idx = sent_from | arrange; -+let visits_join = window_idx | join(sent_from_idx, (_, (rid, start, end, ix, iy), (author, created)) -> ((rid, author), (created, start, end, ix, iy))); ++let visits_join = window_idx | join(sent_from_idx, (_, ((rid, start, end, ix, iy), (author, created))) -> ((rid, author), (created, start, end, ix, iy))); +let visits_filter = visits_join | filter((_, (created, start, end, _, _)) -> ((created, start) ge, (created, end) lt) all); +let visits = visits_filter | map((key, (_, _, _, ix, iy)) -> (key, (ix, iy))); +let hop1_map = asked | map((rid, (who, x, y, _, _)) -> (who, (rid, who, x, y))); +let hop1_idx = hop1_map | arrange; +let friend_idx = friend | arrange; -+let hop1 = hop1_idx | join(friend_idx, (_, req, (f, _)) -> (f, req)); ++let hop1 = hop1_idx | join(friend_idx, (_, (req, (f, _))) -> (f, req)); +let hop1_idx2 = hop1 | arrange; +let friend_idx2 = friend | arrange; -+let hop2 = hop1_idx2 | join(friend_idx2, (_, req, (g, _)) -> (g, req)); ++let hop2 = hop1_idx2 | join(friend_idx2, (_, (req, (g, _))) -> (g, req)); +let circle_rows = hop1 + hop2; +let circle_filter = circle_rows | filter((node, (_, who, _, _)) -> (node, who) ne); +let circle_idx = circle_filter | arrange; +let person_idx = person | arrange; -+let circle_join = circle_idx | join(person_idx, (node, (rid, _, x, y), (_, _, _, _, _, _, _, city, _, _, _)) -> (city, (rid, node, x, y))); ++let circle_join = circle_idx | join(person_idx, (node, ((rid, _, x, y), (_, _, _, _, _, _, _, city, _, _, _))) -> (city, (rid, node, x, y))); +let circle_idx2 = circle_join | arrange; +let place_idx3 = place | arrange; -+let circle_join2 = circle_idx2 | join(place_idx3, (_, (rid, node, x, y), (_, _, home)) -> ((rid, node), (x, y, home))); ++let circle_join2 = circle_idx2 | join(place_idx3, (_, ((rid, node, x, y), (_, _, home))) -> ((rid, node), (x, y, home))); +let circle_filter2 = circle_join2 | filter((_, (x, y, home)) -> ((home, x) ne, (home, y) ne) all); +let circle_map = circle_filter2 | map((key, _) -> (key, ())); +let circle_idx3 = circle_map | arrange; +let circle = circle_idx3 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); +let visits_idx = visits | arrange; -+let seen = visits_idx | join(circle, (key, (ix, iy), ()) -> (key, (ix, iy))); ++let seen = visits_idx | join(circle, (key, ((ix, iy), ())) -> (key, (ix, iy))); +let in_x_filter = seen | filter((_, (ix, _)) -> (ix, 1) eq); +let in_x_map = in_x_filter | map((key, _) -> (key, ())); +let in_x_idx = in_x_map | arrange; @@ -73,12 +73,12 @@ +let in_y_map = in_y_filter | map((key, _) -> (key, ())); +let in_y_idx = in_y_map | arrange; +let in_y = in_y_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); - let counts = in_x | join(in_y, (key, nx, ny) -> (key, (nx, ny))); --let ranked = counts | map(((rid, node), (nx, ny)) -> (node, (rid, nx, ny))) | join(person, (node, (rid, nx, ny), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, (((nx, ny) add neg, node), (first, last, nx, ny)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); + let counts = in_x | join(in_y, row -> row); +-let ranked = counts | map(((rid, node), (nx, ny)) -> (node, (rid, nx, ny))) | join(person, (node, ((rid, nx, ny), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, (((nx, ny) add neg, node), (first, last, nx, ny)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); +let ranked_map = counts | map(((rid, node), (nx, ny)) -> (node, (rid, nx, ny))); +let ranked_idx = ranked_map | arrange; +let person_idx2 = person | arrange; -+let ranked_join = ranked_idx | join(person_idx2, (node, (rid, nx, ny), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, (((nx, ny) add neg, node), (first, last, nx, ny)))); ++let ranked_join = ranked_idx | join(person_idx2, (node, ((rid, nx, ny), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, (((nx, ny) add neg, node), (first, last, nx, ny)))); +let ranked_idx2 = ranked_join | arrange; +let ranked = ranked_idx2 | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((_, node), (first, last, nx, ny)))) -> ((rid, rank), (node, first, last, nx, ny, (nx, ny) add))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.parsed.ddp index c729f29bb..8d673f21e 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.parsed.ddp @@ -5,16 +5,16 @@ let friend = import "graph::friend" : (int, (int, int)); let place = import "graph::place" : (int, (List(int), List(int), int)); let by_name = import "graph::place.by_name" : (List(int), int); let sent_from = import "ic3s::sent_from" : (int, (int, int)); -let asked = request | map(((rid, who, x, y, start, days), ()) -> (x, (rid, who, y, start, (start, (days, 86400000) mul) add))) | join(by_name, (_, req, x) -> (x, req)) | join(place, (x, (rid, who, y, start, end), (_, kind, _)) -> ((kind, [67,111,117,110,116,114,121]) eq, (y, (rid, who, x, start, end)))) | filter((country, _) -> country) | map((_, row) -> row) | join(by_name, (_, req, y) -> (y, req)) | join(place, (y, (rid, who, x, start, end), (_, kind, _)) -> ((kind, [67,111,117,110,116,114,121]) eq, (rid, (who, x, y, start, end)))) | filter((country, _) -> country) | map((_, row) -> row); +let asked = request | map(((rid, who, x, y, start, days), ()) -> (x, (rid, who, y, start, (start, (days, 86400000) mul) add))) | join(by_name, (_, (req, x)) -> (x, req)) | join(place, (x, ((rid, who, y, start, end), (_, kind, _))) -> ((kind, [67,111,117,110,116,114,121]) eq, (y, (rid, who, x, start, end)))) | filter((country, _) -> country) | map((_, row) -> row) | join(by_name, (_, (req, y)) -> (y, req)) | join(place, (y, ((rid, who, x, start, end), (_, kind, _))) -> ((kind, [67,111,117,110,116,114,121]) eq, (rid, (who, x, y, start, end)))) | filter((country, _) -> country) | map((_, row) -> row); let window = (asked | map((rid, (_, x, _, start, end)) -> (x, (rid, start, end, 1, 0)))) + (asked | map((rid, (_, _, y, start, end)) -> (y, (rid, start, end, 0, 1)))); -let visits = window | join(sent_from, (_, (rid, start, end, ix, iy), (author, created)) -> ((rid, author), (created, start, end, ix, iy))) | filter((_, (created, start, end, _, _)) -> ((created, start) ge, (created, end) lt) all) | map((key, (_, _, _, ix, iy)) -> (key, (ix, iy))); -let hop1 = asked | map((rid, (who, x, y, _, _)) -> (who, (rid, who, x, y))) | join(friend, (_, req, (f, _)) -> (f, req)); -let hop2 = hop1 | join(friend, (_, req, (g, _)) -> (g, req)); -let circle = hop1 + hop2 | filter((node, (_, who, _, _)) -> (node, who) ne) | join(person, (node, (rid, _, x, y), (_, _, _, _, _, _, _, city, _, _, _)) -> (city, (rid, node, x, y))) | join(place, (_, (rid, node, x, y), (_, _, home)) -> ((rid, node), (x, y, home))) | filter((_, (x, y, home)) -> ((home, x) ne, (home, y) ne) all) | map((key, _) -> (key, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); -let seen = visits | join(circle, (key, (ix, iy), ()) -> (key, (ix, iy))); +let visits = window | join(sent_from, (_, ((rid, start, end, ix, iy), (author, created))) -> ((rid, author), (created, start, end, ix, iy))) | filter((_, (created, start, end, _, _)) -> ((created, start) ge, (created, end) lt) all) | map((key, (_, _, _, ix, iy)) -> (key, (ix, iy))); +let hop1 = asked | map((rid, (who, x, y, _, _)) -> (who, (rid, who, x, y))) | join(friend, (_, (req, (f, _))) -> (f, req)); +let hop2 = hop1 | join(friend, (_, (req, (g, _))) -> (g, req)); +let circle = hop1 + hop2 | filter((node, (_, who, _, _)) -> (node, who) ne) | join(person, (node, ((rid, _, x, y), (_, _, _, _, _, _, _, city, _, _, _))) -> (city, (rid, node, x, y))) | join(place, (_, ((rid, node, x, y), (_, _, home))) -> ((rid, node), (x, y, home))) | filter((_, (x, y, home)) -> ((home, x) ne, (home, y) ne) all) | map((key, _) -> (key, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); +let seen = visits | join(circle, (key, ((ix, iy), ())) -> (key, (ix, iy))); let in_x = seen | filter((_, (ix, _)) -> (ix, 1) eq) | map((key, _) -> (key, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); let in_y = seen | filter((_, (_, iy)) -> (iy, 1) eq) | map((key, _) -> (key, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); -let counts = in_x | join(in_y, (key, nx, ny) -> (key, (nx, ny))); -let ranked = counts | map(((rid, node), (nx, ny)) -> (node, (rid, nx, ny))) | join(person, (node, (rid, nx, ny), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, (((nx, ny) add neg, node), (first, last, nx, ny)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); +let counts = in_x | join(in_y, row -> row); +let ranked = counts | map(((rid, node), (nx, ny)) -> (node, (rid, nx, ny))) | join(person, (node, ((rid, nx, ny), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, (((nx, ny) add neg, node), (first, last, nx, ny)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((_, node), (first, last, nx, ny)))) -> ((rid, rank), (node, first, last, nx, ny, (nx, ny) add))); output "answer" = answer | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.ddp index 7ef9a210a..35faa11aa 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.ddp @@ -7,19 +7,19 @@ let tags = import "ic4s::tags" : (int, (int, List(int))); let seen_map = request | map(((rid, who, start, days), ()) -> (who, (rid, start, (start, (days, 86400000) mul) add))); let seen_idx = seen_map | arrange; let friend_idx = friend | arrange; -let seen_join = seen_idx | join(friend_idx, (_, req, (f, _)) -> (f, req)); +let seen_join = seen_idx | join(friend_idx, (_, (req, (f, _))) -> (f, req)); let seen_idx2 = seen_join | arrange; let posts_idx = posts | arrange; -let seen_join2 = seen_idx2 | join(posts_idx, (_, (rid, start, end), (m, created)) -> (m, (rid, created, start, end))); +let seen_join2 = seen_idx2 | join(posts_idx, (_, ((rid, start, end), (m, created))) -> (m, (rid, created, start, end))); let seen_idx3 = seen_join2 | arrange; let tags_idx = tags | arrange; -let seen = seen_idx3 | join(tags_idx, (_, (rid, created, start, end), (t, name)) -> ((rid, t), (name, created, start, end))); +let seen = seen_idx3 | join(tags_idx, (_, ((rid, created, start, end), (t, name))) -> ((rid, t), (name, created, start, end))); let before_rows = seen | filter((_, (_, created, start, _)) -> (created, start) lt) | map((key, _) -> (key, ())); let before_idx = before_rows | arrange; let before = before_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); let during = seen | filter((_, (_, created, start, end)) -> ((created, start) ge, (created, end) lt) all); let during_idx = during | arrange; -let fresh_join = during_idx | join(before, arg -> let _call0 = let input = arg in let (key, row, ()) = input in (key, row) in ((_call0.0.0, ()), (_call0.1.0, (), (), ()))); +let fresh_join = during_idx | join(before, arg -> let _call0 = let input = arg in let (key, (row, ())) = input in (key, row) in ((_call0.0.0, ()), (_call0.1.0, (), (), ()))); let fresh_map = during | map(arg -> let _call0 = let input = arg in input in ((_call0.0.0, ()), (_call0.1.0, (), (), ()))); let fresh_neg = fresh_join | negate; let fresh = fresh_map + fresh_neg; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.lowered--optimized.diff index 34b756169..967d5227e 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.lowered--optimized.diff @@ -14,7 +14,7 @@ @@ -17,9 +14,8 @@ let seen_idx3 = seen_join2 | arrange; let tags_idx = tags | arrange; - let seen = seen_idx3 | join(tags_idx, (_, (rid, created, start, end), (t, name)) -> ((rid, t), (name, created, start, end))); + let seen = seen_idx3 | join(tags_idx, (_, ((rid, created, start, end), (t, name))) -> ((rid, t), (name, created, start, end))); -let before_filter = seen | filter((_, (_, created, start, _)) -> (created, start) lt); -let before_map = before_filter | map((key, _) -> (key, ())); -let before_idx = before_map | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.optimized--projected.diff b/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.optimized--projected.diff index 8f7c454e0..b6015c418 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.optimized--projected.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.optimized--projected.diff @@ -4,8 +4,8 @@ let before = before_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); let during = seen | filter((_, (_, created, start, end)) -> ((created, start) ge, (created, end) lt) all); let during_idx = during | arrange; --let fresh_join = during_idx | join(before, (key, row, ()) -> (key, row)); -+let fresh_join = during_idx | join(before, arg -> let _call0 = let input = arg in let (key, row, ()) = input in (key, row) in ((_call0.0.0, ()), (_call0.1.0, (), (), ()))); +-let fresh_join = during_idx | join(before, (key, (row, ())) -> (key, row)); ++let fresh_join = during_idx | join(before, arg -> let _call0 = let input = arg in let (key, (row, ())) = input in (key, row) in ((_call0.0.0, ()), (_call0.1.0, (), (), ()))); +let fresh_map = during | map(arg -> let _call0 = let input = arg in input in ((_call0.0.0, ()), (_call0.1.0, (), (), ()))); let fresh_neg = fresh_join | negate; -let fresh = during + fresh_neg; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.parsed--lowered.diff index 14bf22e22..fe8689260 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.parsed--lowered.diff @@ -10,28 +10,28 @@ let mtag = import "graph::mtag.by_tag" : (int, int); let posts = import "ic4s::posts" : (int, (int, int)); let tags = import "ic4s::tags" : (int, (int, List(int))); --let seen = request | map(((rid, who, start, days), ()) -> (who, (rid, start, (start, (days, 86400000) mul) add))) | join(friend, (_, req, (f, _)) -> (f, req)) | join(posts, (_, (rid, start, end), (m, created)) -> (m, (rid, created, start, end))) | join(tags, (_, (rid, created, start, end), (t, name)) -> ((rid, t), (name, created, start, end))); +-let seen = request | map(((rid, who, start, days), ()) -> (who, (rid, start, (start, (days, 86400000) mul) add))) | join(friend, (_, (req, (f, _))) -> (f, req)) | join(posts, (_, ((rid, start, end), (m, created))) -> (m, (rid, created, start, end))) | join(tags, (_, ((rid, created, start, end), (t, name))) -> ((rid, t), (name, created, start, end))); -let before = seen | filter((_, (_, created, start, _)) -> (created, start) lt) | map((key, _) -> (key, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); +let seen_map = request | map(((rid, who, start, days), ()) -> (who, (rid, start, (start, (days, 86400000) mul) add))); +let seen_idx = seen_map | arrange; +let friend_idx = friend | arrange; -+let seen_join = seen_idx | join(friend_idx, (_, req, (f, _)) -> (f, req)); ++let seen_join = seen_idx | join(friend_idx, (_, (req, (f, _))) -> (f, req)); +let seen_idx2 = seen_join | arrange; +let posts_idx = posts | arrange; -+let seen_join2 = seen_idx2 | join(posts_idx, (_, (rid, start, end), (m, created)) -> (m, (rid, created, start, end))); ++let seen_join2 = seen_idx2 | join(posts_idx, (_, ((rid, start, end), (m, created))) -> (m, (rid, created, start, end))); +let seen_idx3 = seen_join2 | arrange; +let tags_idx = tags | arrange; -+let seen = seen_idx3 | join(tags_idx, (_, (rid, created, start, end), (t, name)) -> ((rid, t), (name, created, start, end))); ++let seen = seen_idx3 | join(tags_idx, (_, ((rid, created, start, end), (t, name))) -> ((rid, t), (name, created, start, end))); +let before_filter = seen | filter((_, (_, created, start, _)) -> (created, start) lt); +let before_map = before_filter | map((key, _) -> (key, ())); +let before_idx = before_map | arrange; +let before = before_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); let during = seen | filter((_, (_, created, start, end)) -> ((created, start) ge, (created, end) lt) all); --let fresh = during + (during | join(before, (key, row, ()) -> (key, row)) | negate); +-let fresh = during + (during | join(before, (key, (row, ())) -> (key, row)) | negate); -let counts = fresh | map(((rid, _), (name, _, _, _)) -> ((rid, name), ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); -let ranked = counts | map(((rid, name), n) -> (rid, (n neg, name))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); +let during_idx = during | arrange; -+let fresh_join = during_idx | join(before, (key, row, ()) -> (key, row)); ++let fresh_join = during_idx | join(before, (key, (row, ())) -> (key, row)); +let fresh_neg = fresh_join | negate; +let fresh = during + fresh_neg; +let counts_map = fresh | map(((rid, _), (name, _, _, _)) -> ((rid, name), ())); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.parsed.ddp index b91593950..b99333028 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic04_oneoff.parsed.ddp @@ -5,10 +5,10 @@ let tag = import "graph::tag" : (int, (List(int), int)); let mtag = import "graph::mtag.by_tag" : (int, int); let posts = import "ic4s::posts" : (int, (int, int)); let tags = import "ic4s::tags" : (int, (int, List(int))); -let seen = request | map(((rid, who, start, days), ()) -> (who, (rid, start, (start, (days, 86400000) mul) add))) | join(friend, (_, req, (f, _)) -> (f, req)) | join(posts, (_, (rid, start, end), (m, created)) -> (m, (rid, created, start, end))) | join(tags, (_, (rid, created, start, end), (t, name)) -> ((rid, t), (name, created, start, end))); +let seen = request | map(((rid, who, start, days), ()) -> (who, (rid, start, (start, (days, 86400000) mul) add))) | join(friend, (_, (req, (f, _))) -> (f, req)) | join(posts, (_, ((rid, start, end), (m, created))) -> (m, (rid, created, start, end))) | join(tags, (_, ((rid, created, start, end), (t, name))) -> ((rid, t), (name, created, start, end))); let before = seen | filter((_, (_, created, start, _)) -> (created, start) lt) | map((key, _) -> (key, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); let during = seen | filter((_, (_, created, start, end)) -> ((created, start) ge, (created, end) lt) all); -let fresh = during + (during | join(before, (key, row, ()) -> (key, row)) | negate); +let fresh = during + (during | join(before, (key, (row, ())) -> (key, row)) | negate); let counts = fresh | map(((rid, _), (name, _, _, _)) -> ((rid, name), ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); let ranked = counts | map(((rid, name), n) -> (rid, (n neg, name))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, (neg, name))) -> ((rid, rank), (name, neg neg))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic04_standing.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic04_standing.ddp index 6439534ce..6c606e31e 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic04_standing.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic04_standing.ddp @@ -6,7 +6,7 @@ let mtag = import "graph::mtag.by_tag" : (int, int); let posts = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 0) eq) | map((id, (_, created, who, _, _, _, _, _, _, _, _, _, _)) -> (who, (id, created))); let mtag_idx = mtag | arrange; let tag_idx = tag | arrange; -let tags = mtag_idx | join(tag_idx, (t, m, (name, _)) -> (m, (t, name))); +let tags = mtag_idx | join(tag_idx, (t, (m, (name, _))) -> (m, (t, name))); let posts_idx = posts | arrange; let tags_idx = tags | arrange; -- Renderer: publish the existing arrangement. diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic04_standing.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/ldbc_ic04_standing.lowered--optimized.diff index 4d697215a..051e8aed3 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic04_standing.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic04_standing.lowered--optimized.diff @@ -12,4 +12,4 @@ +let posts = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 0) eq) | map((id, (_, created, who, _, _, _, _, _, _, _, _, _, _)) -> (who, (id, created))); let mtag_idx = mtag | arrange; let tag_idx = tag | arrange; - let tags = mtag_idx | join(tag_idx, (t, m, (name, _)) -> (m, (t, name))); + let tags = mtag_idx | join(tag_idx, (t, (m, (name, _))) -> (m, (t, name))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic04_standing.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic04_standing.parsed--lowered.diff index b6a53a507..e12d6274a 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic04_standing.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic04_standing.parsed--lowered.diff @@ -8,14 +8,14 @@ let tag = import "graph::tag" : (int, (List(int), int)); let mtag = import "graph::mtag.by_tag" : (int, int); -let posts = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 0) eq) | map((id, (_, created, who, _, _, _, _, _, _, _, _, _, _)) -> (who, (id, created))); --let tags = mtag | join(tag, (t, m, (name, _)) -> (m, (t, name))); +-let tags = mtag | join(tag, (t, (m, (name, _))) -> (m, (t, name))); -output "posts" = posts | arrange; -output "tags" = tags | arrange; +let posts_filter = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 0) eq); +let posts = posts_filter | map((id, (_, created, who, _, _, _, _, _, _, _, _, _, _)) -> (who, (id, created))); +let mtag_idx = mtag | arrange; +let tag_idx = tag | arrange; -+let tags = mtag_idx | join(tag_idx, (t, m, (name, _)) -> (m, (t, name))); ++let tags = mtag_idx | join(tag_idx, (t, (m, (name, _))) -> (m, (t, name))); +let posts_idx = posts | arrange; +let tags_idx = tags | arrange; +-- Renderer: publish the existing arrangement. diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic04_standing.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic04_standing.parsed.ddp index 1d3cc38ae..c13448bfd 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic04_standing.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic04_standing.parsed.ddp @@ -3,6 +3,6 @@ let friend = import "graph::friend" : (int, (int, int)); let tag = import "graph::tag" : (int, (List(int), int)); let mtag = import "graph::mtag.by_tag" : (int, int); let posts = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 0) eq) | map((id, (_, created, who, _, _, _, _, _, _, _, _, _, _)) -> (who, (id, created))); -let tags = mtag | join(tag, (t, m, (name, _)) -> (m, (t, name))); +let tags = mtag | join(tag, (t, (m, (name, _))) -> (m, (t, name))); output "posts" = posts | arrange; output "tags" = tags | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic05_oneoff.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic05_oneoff.ddp index e64eda23c..2bbfc18fb 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic05_oneoff.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic05_oneoff.ddp @@ -11,12 +11,12 @@ let lowest = import "ic5s::lowest" : (int, int); let hop1_map = request | map(((rid, who, date), ()) -> (who, (rid, who, date))); let hop1_idx = hop1_map | arrange; let person_idx = person | arrange; -let hop1_join = hop1_idx | join(person_idx, (who, req, _) -> (who, req)); +let hop1_join = hop1_idx | join(person_idx, (who, (req, _)) -> (who, req)); let hop1_idx2 = hop1_join | arrange; let friend_idx = friend | arrange; -let hop1 = hop1_idx2 | join(friend_idx, (_, req, (f, _)) -> (f, req)); +let hop1 = hop1_idx2 | join(friend_idx, (_, (req, (f, _))) -> (f, req)); let hop1_idx3 = hop1 | arrange; -let hop2 = hop1_idx3 | join(friend_idx, (_, req, (g, _)) -> (g, req)); +let hop2 = hop1_idx3 | join(friend_idx, (_, (req, (g, _))) -> (g, req)); let circle_rows = hop1 + hop2; let circle_rows2 = circle_rows | filter((node, (_, who, _)) -> (node, who) ne) | map((node, (rid, _, date)) -> ((node, rid, date), ())); let circle_idx = circle_rows2 | arrange; @@ -24,29 +24,29 @@ let circle_reduce = circle_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1 let circle = circle_reduce | map(((node, rid, date), ()) -> (node, (rid, date))); let circle_idx2 = circle | arrange; let member_idx = member | arrange; -let after_join = circle_idx2 | join(member_idx, (node, (rid, date), (f, since)) -> ((node, f), (rid, date, since))); +let after_join = circle_idx2 | join(member_idx, (node, ((rid, date), (f, since))) -> ((node, f), (rid, date, since))); let after = after_join | filter((_, (_, date, since)) -> (since, date) gt) | map((key, (rid, _, _)) -> (key, rid)); let after_idx = after | arrange; let posted_idx = posted | arrange; -let busy_join = after_idx | join(posted_idx, ((_, f), rid, n) -> ((rid, f), n)); +let busy_join = after_idx | join(posted_idx, ((_, f), (rid, n)) -> ((rid, f), n)); let busy_rows = busy_join | join_function((key, n) -> ((key, ()), 0, n) enlist); let busy_idx = busy_rows | arrange; let busy = busy_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); let first_idx = first | arrange; -let timed = circle_idx2 | join(first_idx, (node, (rid, date), since) -> (node, (rid, date, since))); +let timed = circle_idx2 | join(first_idx, (node, ((rid, date), since)) -> (node, (rid, date, since))); let fast_filter = timed | filter((_, (_, date, since)) -> (since, date) gt); let fast_idx = fast_filter | arrange; let lowest_idx = lowest | arrange; -let fast = fast_idx | join(lowest_idx, (_, (rid, _, _), f) -> ((rid, f), ())); +let fast = fast_idx | join(lowest_idx, (_, ((rid, _, _), f)) -> ((rid, f), ())); let slow_rows = timed | filter((_, (_, date, since)) -> (since, date) le) | map((node, (rid, _, _)) -> ((node, rid), ())); let slow_idx = slow_rows | arrange; let slow_map = after | map(((node, f), rid) -> ((node, rid), f)); let slow_idx2 = slow_map | arrange; -let slow = slow_idx | join(slow_idx2, ((_, rid), (), f) -> ((rid, f), ())); +let slow = slow_idx | join(slow_idx2, ((_, rid), ((), f)) -> ((rid, f), ())); let idle_rows = fast + slow; let idle_idx = idle_rows | arrange; let idle = idle_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); -let quiet_join = idle | join(busy, (key, (), _) -> (key, ())); +let quiet_join = idle | join(busy, (key, ((), _)) -> (key, ())); let quiet_neg = quiet_join | negate; let quiet_rows = idle + quiet_neg; let quiet = quiet_rows | map((key, ()) -> (key, 0)); @@ -54,7 +54,7 @@ let ranked_rows = busy + quiet; let ranked_map = ranked_rows | map(((rid, f), n) -> (f, (rid, n))); let ranked_idx = ranked_map | arrange; let forum_idx = forum | arrange; -let ranked_join = ranked_idx | join(forum_idx, (f, (rid, n), (title, _, _)) -> (rid, ((n neg, f), title))); +let ranked_join = ranked_idx | join(forum_idx, (f, ((rid, n), (title, _, _))) -> (rid, ((n neg, f), title))); let ranked_idx2 = ranked_join | arrange; let ranked = ranked_idx2 | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((neg, _), title))) -> ((rid, rank), (title, neg neg))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic05_oneoff.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/ldbc_ic05_oneoff.lowered--optimized.diff index c8148d993..425f0a10d 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic05_oneoff.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic05_oneoff.lowered--optimized.diff @@ -10,11 +10,11 @@ let member = import "graph::member.by_person" : (int, (int, int)); @@ -17,37 +16,32 @@ let friend_idx = friend | arrange; - let hop1 = hop1_idx2 | join(friend_idx, (_, req, (f, _)) -> (f, req)); + let hop1 = hop1_idx2 | join(friend_idx, (_, (req, (f, _))) -> (f, req)); let hop1_idx3 = hop1 | arrange; -let friend_idx2 = friend | arrange; --let hop2 = hop1_idx3 | join(friend_idx2, (_, req, (g, _)) -> (g, req)); -+let hop2 = hop1_idx3 | join(friend_idx, (_, req, (g, _)) -> (g, req)); +-let hop2 = hop1_idx3 | join(friend_idx2, (_, (req, (g, _))) -> (g, req)); ++let hop2 = hop1_idx3 | join(friend_idx, (_, (req, (g, _))) -> (g, req)); let circle_rows = hop1 + hop2; -let circle_filter = circle_rows | filter((node, (_, who, _)) -> (node, who) ne); -let circle_map = circle_filter | map((node, (rid, _, date)) -> ((node, rid, date), ())); @@ -25,24 +25,24 @@ let circle = circle_reduce | map(((node, rid, date), ()) -> (node, (rid, date))); let circle_idx2 = circle | arrange; let member_idx = member | arrange; - let after_join = circle_idx2 | join(member_idx, (node, (rid, date), (f, since)) -> ((node, f), (rid, date, since))); + let after_join = circle_idx2 | join(member_idx, (node, ((rid, date), (f, since))) -> ((node, f), (rid, date, since))); -let after_filter = after_join | filter((_, (_, date, since)) -> (since, date) gt); -let after = after_filter | map((key, (rid, _, _)) -> (key, rid)); +let after = after_join | filter((_, (_, date, since)) -> (since, date) gt) | map((key, (rid, _, _)) -> (key, rid)); let after_idx = after | arrange; let posted_idx = posted | arrange; - let busy_join = after_idx | join(posted_idx, ((_, f), rid, n) -> ((rid, f), n)); + let busy_join = after_idx | join(posted_idx, ((_, f), (rid, n)) -> ((rid, f), n)); let busy_rows = busy_join | join_function((key, n) -> ((key, ()), 0, n) enlist); let busy_idx = busy_rows | arrange; let busy = busy_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); -let circle_idx3 = circle | arrange; let first_idx = first | arrange; --let timed = circle_idx3 | join(first_idx, (node, (rid, date), since) -> (node, (rid, date, since))); -+let timed = circle_idx2 | join(first_idx, (node, (rid, date), since) -> (node, (rid, date, since))); +-let timed = circle_idx3 | join(first_idx, (node, ((rid, date), since)) -> (node, (rid, date, since))); ++let timed = circle_idx2 | join(first_idx, (node, ((rid, date), since)) -> (node, (rid, date, since))); let fast_filter = timed | filter((_, (_, date, since)) -> (since, date) gt); let fast_idx = fast_filter | arrange; let lowest_idx = lowest | arrange; - let fast = fast_idx | join(lowest_idx, (_, (rid, _, _), f) -> ((rid, f), ())); + let fast = fast_idx | join(lowest_idx, (_, ((rid, _, _), f)) -> ((rid, f), ())); -let slow_filter = timed | filter((_, (_, date, since)) -> (since, date) le); -let slow_map = slow_filter | map((node, (rid, _, _)) -> ((node, rid), ())); -let slow_idx = slow_map | arrange; @@ -52,6 +52,6 @@ +let slow_idx = slow_rows | arrange; +let slow_map = after | map(((node, f), rid) -> ((node, rid), f)); +let slow_idx2 = slow_map | arrange; - let slow = slow_idx | join(slow_idx2, ((_, rid), (), f) -> ((rid, f), ())); + let slow = slow_idx | join(slow_idx2, ((_, rid), ((), f)) -> ((rid, f), ())); let idle_rows = fast + slow; let idle_idx = idle_rows | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic05_oneoff.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic05_oneoff.parsed--lowered.diff index 84d10febc..358873ba5 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic05_oneoff.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic05_oneoff.parsed--lowered.diff @@ -10,27 +10,27 @@ let posted = import "ic5s::posted" : ((int, int), int); let first = import "ic5s::first" : (int, int); let lowest = import "ic5s::lowest" : (int, int); --let hop1 = request | map(((rid, who, date), ()) -> (who, (rid, who, date))) | join(person, (who, req, _) -> (who, req)) | join(friend, (_, req, (f, _)) -> (f, req)); --let hop2 = hop1 | join(friend, (_, req, (g, _)) -> (g, req)); +-let hop1 = request | map(((rid, who, date), ()) -> (who, (rid, who, date))) | join(person, (who, (req, _)) -> (who, req)) | join(friend, (_, (req, (f, _))) -> (f, req)); +-let hop2 = hop1 | join(friend, (_, (req, (g, _))) -> (g, req)); -let circle = hop1 + hop2 | filter((node, (_, who, _)) -> (node, who) ne) | map((node, (rid, _, date)) -> ((node, rid, date), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((node, rid, date), ()) -> (node, (rid, date))); --let after = circle | join(member, (node, (rid, date), (f, since)) -> ((node, f), (rid, date, since))) | filter((_, (_, date, since)) -> (since, date) gt) | map((key, (rid, _, _)) -> (key, rid)); --let busy = after | join(posted, ((_, f), rid, n) -> ((rid, f), n)) | join_function((key, n) -> ((key, ()), 0, n) enlist) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); --let timed = circle | join(first, (node, (rid, date), since) -> (node, (rid, date, since))); --let fast = timed | filter((_, (_, date, since)) -> (since, date) gt) | join(lowest, (_, (rid, _, _), f) -> ((rid, f), ())); --let slow = timed | filter((_, (_, date, since)) -> (since, date) le) | map((node, (rid, _, _)) -> ((node, rid), ())) | join(after | map(((node, f), rid) -> ((node, rid), f)), ((_, rid), (), f) -> ((rid, f), ())); +-let after = circle | join(member, (node, ((rid, date), (f, since))) -> ((node, f), (rid, date, since))) | filter((_, (_, date, since)) -> (since, date) gt) | map((key, (rid, _, _)) -> (key, rid)); +-let busy = after | join(posted, ((_, f), (rid, n)) -> ((rid, f), n)) | join_function((key, n) -> ((key, ()), 0, n) enlist) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +-let timed = circle | join(first, (node, ((rid, date), since)) -> (node, (rid, date, since))); +-let fast = timed | filter((_, (_, date, since)) -> (since, date) gt) | join(lowest, (_, ((rid, _, _), f)) -> ((rid, f), ())); +-let slow = timed | filter((_, (_, date, since)) -> (since, date) le) | map((node, (rid, _, _)) -> ((node, rid), ())) | join(after | map(((node, f), rid) -> ((node, rid), f)), ((_, rid), ((), f)) -> ((rid, f), ())); -let idle = fast + slow | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); --let quiet = idle + (idle | join(busy, (key, (), _) -> (key, ())) | negate) | map((key, ()) -> (key, 0)); --let ranked = busy + quiet | map(((rid, f), n) -> (f, (rid, n))) | join(forum, (f, (rid, n), (title, _, _)) -> (rid, ((n neg, f), title))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); +-let quiet = idle + (idle | join(busy, (key, ((), _)) -> (key, ())) | negate) | map((key, ()) -> (key, 0)); +-let ranked = busy + quiet | map(((rid, f), n) -> (f, (rid, n))) | join(forum, (f, ((rid, n), (title, _, _))) -> (rid, ((n neg, f), title))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); +let hop1_map = request | map(((rid, who, date), ()) -> (who, (rid, who, date))); +let hop1_idx = hop1_map | arrange; +let person_idx = person | arrange; -+let hop1_join = hop1_idx | join(person_idx, (who, req, _) -> (who, req)); ++let hop1_join = hop1_idx | join(person_idx, (who, (req, _)) -> (who, req)); +let hop1_idx2 = hop1_join | arrange; +let friend_idx = friend | arrange; -+let hop1 = hop1_idx2 | join(friend_idx, (_, req, (f, _)) -> (f, req)); ++let hop1 = hop1_idx2 | join(friend_idx, (_, (req, (f, _))) -> (f, req)); +let hop1_idx3 = hop1 | arrange; +let friend_idx2 = friend | arrange; -+let hop2 = hop1_idx3 | join(friend_idx2, (_, req, (g, _)) -> (g, req)); ++let hop2 = hop1_idx3 | join(friend_idx2, (_, (req, (g, _))) -> (g, req)); +let circle_rows = hop1 + hop2; +let circle_filter = circle_rows | filter((node, (_, who, _)) -> (node, who) ne); +let circle_map = circle_filter | map((node, (rid, _, date)) -> ((node, rid, date), ())); @@ -39,32 +39,32 @@ +let circle = circle_reduce | map(((node, rid, date), ()) -> (node, (rid, date))); +let circle_idx2 = circle | arrange; +let member_idx = member | arrange; -+let after_join = circle_idx2 | join(member_idx, (node, (rid, date), (f, since)) -> ((node, f), (rid, date, since))); ++let after_join = circle_idx2 | join(member_idx, (node, ((rid, date), (f, since))) -> ((node, f), (rid, date, since))); +let after_filter = after_join | filter((_, (_, date, since)) -> (since, date) gt); +let after = after_filter | map((key, (rid, _, _)) -> (key, rid)); +let after_idx = after | arrange; +let posted_idx = posted | arrange; -+let busy_join = after_idx | join(posted_idx, ((_, f), rid, n) -> ((rid, f), n)); ++let busy_join = after_idx | join(posted_idx, ((_, f), (rid, n)) -> ((rid, f), n)); +let busy_rows = busy_join | join_function((key, n) -> ((key, ()), 0, n) enlist); +let busy_idx = busy_rows | arrange; +let busy = busy_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +let circle_idx3 = circle | arrange; +let first_idx = first | arrange; -+let timed = circle_idx3 | join(first_idx, (node, (rid, date), since) -> (node, (rid, date, since))); ++let timed = circle_idx3 | join(first_idx, (node, ((rid, date), since)) -> (node, (rid, date, since))); +let fast_filter = timed | filter((_, (_, date, since)) -> (since, date) gt); +let fast_idx = fast_filter | arrange; +let lowest_idx = lowest | arrange; -+let fast = fast_idx | join(lowest_idx, (_, (rid, _, _), f) -> ((rid, f), ())); ++let fast = fast_idx | join(lowest_idx, (_, ((rid, _, _), f)) -> ((rid, f), ())); +let slow_filter = timed | filter((_, (_, date, since)) -> (since, date) le); +let slow_map = slow_filter | map((node, (rid, _, _)) -> ((node, rid), ())); +let slow_idx = slow_map | arrange; +let slow_map2 = after | map(((node, f), rid) -> ((node, rid), f)); +let slow_idx2 = slow_map2 | arrange; -+let slow = slow_idx | join(slow_idx2, ((_, rid), (), f) -> ((rid, f), ())); ++let slow = slow_idx | join(slow_idx2, ((_, rid), ((), f)) -> ((rid, f), ())); +let idle_rows = fast + slow; +let idle_idx = idle_rows | arrange; +let idle = idle_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); -+let quiet_join = idle | join(busy, (key, (), _) -> (key, ())); ++let quiet_join = idle | join(busy, (key, ((), _)) -> (key, ())); +let quiet_neg = quiet_join | negate; +let quiet_rows = idle + quiet_neg; +let quiet = quiet_rows | map((key, ()) -> (key, 0)); @@ -72,7 +72,7 @@ +let ranked_map = ranked_rows | map(((rid, f), n) -> (f, (rid, n))); +let ranked_idx = ranked_map | arrange; +let forum_idx = forum | arrange; -+let ranked_join = ranked_idx | join(forum_idx, (f, (rid, n), (title, _, _)) -> (rid, ((n neg, f), title))); ++let ranked_join = ranked_idx | join(forum_idx, (f, ((rid, n), (title, _, _))) -> (rid, ((n neg, f), title))); +let ranked_idx2 = ranked_join | arrange; +let ranked = ranked_idx2 | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((neg, _), title))) -> ((rid, rank), (title, neg neg))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic05_oneoff.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic05_oneoff.parsed.ddp index 1ff12778e..a2bf2c443 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic05_oneoff.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic05_oneoff.parsed.ddp @@ -7,16 +7,16 @@ let member = import "graph::member.by_person" : (int, (int, int)); let posted = import "ic5s::posted" : ((int, int), int); let first = import "ic5s::first" : (int, int); let lowest = import "ic5s::lowest" : (int, int); -let hop1 = request | map(((rid, who, date), ()) -> (who, (rid, who, date))) | join(person, (who, req, _) -> (who, req)) | join(friend, (_, req, (f, _)) -> (f, req)); -let hop2 = hop1 | join(friend, (_, req, (g, _)) -> (g, req)); +let hop1 = request | map(((rid, who, date), ()) -> (who, (rid, who, date))) | join(person, (who, (req, _)) -> (who, req)) | join(friend, (_, (req, (f, _))) -> (f, req)); +let hop2 = hop1 | join(friend, (_, (req, (g, _))) -> (g, req)); let circle = hop1 + hop2 | filter((node, (_, who, _)) -> (node, who) ne) | map((node, (rid, _, date)) -> ((node, rid, date), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((node, rid, date), ()) -> (node, (rid, date))); -let after = circle | join(member, (node, (rid, date), (f, since)) -> ((node, f), (rid, date, since))) | filter((_, (_, date, since)) -> (since, date) gt) | map((key, (rid, _, _)) -> (key, rid)); -let busy = after | join(posted, ((_, f), rid, n) -> ((rid, f), n)) | join_function((key, n) -> ((key, ()), 0, n) enlist) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); -let timed = circle | join(first, (node, (rid, date), since) -> (node, (rid, date, since))); -let fast = timed | filter((_, (_, date, since)) -> (since, date) gt) | join(lowest, (_, (rid, _, _), f) -> ((rid, f), ())); -let slow = timed | filter((_, (_, date, since)) -> (since, date) le) | map((node, (rid, _, _)) -> ((node, rid), ())) | join(after | map(((node, f), rid) -> ((node, rid), f)), ((_, rid), (), f) -> ((rid, f), ())); +let after = circle | join(member, (node, ((rid, date), (f, since))) -> ((node, f), (rid, date, since))) | filter((_, (_, date, since)) -> (since, date) gt) | map((key, (rid, _, _)) -> (key, rid)); +let busy = after | join(posted, ((_, f), (rid, n)) -> ((rid, f), n)) | join_function((key, n) -> ((key, ()), 0, n) enlist) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +let timed = circle | join(first, (node, ((rid, date), since)) -> (node, (rid, date, since))); +let fast = timed | filter((_, (_, date, since)) -> (since, date) gt) | join(lowest, (_, ((rid, _, _), f)) -> ((rid, f), ())); +let slow = timed | filter((_, (_, date, since)) -> (since, date) le) | map((node, (rid, _, _)) -> ((node, rid), ())) | join(after | map(((node, f), rid) -> ((node, rid), f)), ((_, rid), ((), f)) -> ((rid, f), ())); let idle = fast + slow | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); -let quiet = idle + (idle | join(busy, (key, (), _) -> (key, ())) | negate) | map((key, ()) -> (key, 0)); -let ranked = busy + quiet | map(((rid, f), n) -> (f, (rid, n))) | join(forum, (f, (rid, n), (title, _, _)) -> (rid, ((n neg, f), title))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); +let quiet = idle + (idle | join(busy, (key, ((), _)) -> (key, ())) | negate) | map((key, ()) -> (key, 0)); +let ranked = busy + quiet | map(((rid, f), n) -> (f, (rid, n))) | join(forum, (f, ((rid, n), (title, _, _))) -> (rid, ((n neg, f), title))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((neg, _), title))) -> ((rid, rank), (title, neg neg))); output "answer" = answer | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic06_oneoff.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic06_oneoff.ddp index ec714d587..6aa8b9e11 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic06_oneoff.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic06_oneoff.ddp @@ -11,12 +11,12 @@ let tags_of = import "graph::mtag.by_message" : (int, int); let hop1_map = request | map(((rid, who, _), ()) -> (who, (rid, who))); let hop1_idx = hop1_map | arrange; let person_idx = person | arrange; -let hop1_join = hop1_idx | join(person_idx, (who, req, _) -> (who, req)); +let hop1_join = hop1_idx | join(person_idx, (who, (req, _)) -> (who, req)); let hop1_idx2 = hop1_join | arrange; let friend_idx = friend | arrange; -let hop1 = hop1_idx2 | join(friend_idx, (_, (rid, who), (f, _)) -> (f, (rid, who))); +let hop1 = hop1_idx2 | join(friend_idx, (_, ((rid, who), (f, _))) -> (f, (rid, who))); let hop1_idx3 = hop1 | arrange; -let hop2 = hop1_idx3 | join(friend_idx, (_, (rid, who), (g, _)) -> (g, (rid, who))); +let hop2 = hop1_idx3 | join(friend_idx, (_, ((rid, who), (g, _))) -> (g, (rid, who))); let circle_rows = hop1 + hop2; let circle_rows2 = circle_rows | filter((node, (_, who)) -> (node, who) ne) | map((node, (rid, _)) -> ((rid, node), ())); let circle_idx = circle_rows2 | arrange; @@ -24,26 +24,26 @@ let circle = circle_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); let with_it_map = request | map(((rid, _, name), ()) -> (name, rid)); let with_it_idx = with_it_map | arrange; let by_name_idx = by_name | arrange; -let with_it_join = with_it_idx | join(by_name_idx, (_, rid, t) -> (t, rid)); +let with_it_join = with_it_idx | join(by_name_idx, (_, (rid, t)) -> (t, rid)); let with_it_idx2 = with_it_join | arrange; let tagged_idx = tagged | arrange; -let with_it_join2 = with_it_idx2 | join(tagged_idx, (t, rid, m) -> (m, (rid, t))); +let with_it_join2 = with_it_idx2 | join(tagged_idx, (t, (rid, m)) -> (m, (rid, t))); let with_it_idx3 = with_it_join2 | arrange; let message_idx = message | arrange; -let with_it_join3 = with_it_idx3 | join(message_idx, (m, (rid, t), (kind, _, author, _, _, _, _, _, _, _, _, _, _)) -> ((rid, author), (kind, m, t))); +let with_it_join3 = with_it_idx3 | join(message_idx, (m, ((rid, t), (kind, _, author, _, _, _, _, _, _, _, _, _, _))) -> ((rid, author), (kind, m, t))); let with_it_filter = with_it_join3 | filter((_, (kind, _, _)) -> (kind, 0) eq); let with_it_idx4 = with_it_filter | arrange; -let with_it = with_it_idx4 | join(circle, ((rid, _), (_, m, t), ()) -> (m, (rid, t))); +let with_it = with_it_idx4 | join(circle, ((rid, _), ((_, m, t), ())) -> (m, (rid, t))); let with_it_idx5 = with_it | arrange; let tags_of_idx = tags_of | arrange; -let others_join = with_it_idx5 | join(tags_of_idx, (_, (rid, t), other) -> ((rid, t, other), ())); +let others_join = with_it_idx5 | join(tags_of_idx, (_, ((rid, t), other)) -> ((rid, t, other), ())); let others_rows = others_join | filter(((_, t, other), ()) -> (t, other) ne) | map(((rid, _, other), ()) -> ((rid, other), ())); let others_idx = others_rows | arrange; let others = others_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); let ranked_map = others | map(((rid, other), n) -> (other, (rid, n))); let ranked_idx = ranked_map | arrange; let tag_idx = tag | arrange; -let ranked_join = ranked_idx | join(tag_idx, (_, (rid, n), (name, _)) -> (rid, (n neg, name))); +let ranked_join = ranked_idx | join(tag_idx, (_, ((rid, n), (name, _))) -> (rid, (n neg, name))); let ranked_idx2 = ranked_join | arrange; let ranked = ranked_idx2 | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, (neg, other))) -> ((rid, rank), (other, neg neg))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic06_oneoff.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/ldbc_ic06_oneoff.lowered--optimized.diff index 2bdd24233..95b6f3ec8 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic06_oneoff.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic06_oneoff.lowered--optimized.diff @@ -2,11 +2,11 @@ +++ optimized @@ -16,12 +16,10 @@ let friend_idx = friend | arrange; - let hop1 = hop1_idx2 | join(friend_idx, (_, (rid, who), (f, _)) -> (f, (rid, who))); + let hop1 = hop1_idx2 | join(friend_idx, (_, ((rid, who), (f, _))) -> (f, (rid, who))); let hop1_idx3 = hop1 | arrange; -let friend_idx2 = friend | arrange; --let hop2 = hop1_idx3 | join(friend_idx2, (_, (rid, who), (g, _)) -> (g, (rid, who))); -+let hop2 = hop1_idx3 | join(friend_idx, (_, (rid, who), (g, _)) -> (g, (rid, who))); +-let hop2 = hop1_idx3 | join(friend_idx2, (_, ((rid, who), (g, _))) -> (g, (rid, who))); ++let hop2 = hop1_idx3 | join(friend_idx, (_, ((rid, who), (g, _))) -> (g, (rid, who))); let circle_rows = hop1 + hop2; -let circle_filter = circle_rows | filter((node, (_, who)) -> (node, who) ne); -let circle_map = circle_filter | map((node, (rid, _)) -> ((rid, node), ())); @@ -19,7 +19,7 @@ @@ -39,9 +37,8 @@ let with_it_idx5 = with_it | arrange; let tags_of_idx = tags_of | arrange; - let others_join = with_it_idx5 | join(tags_of_idx, (_, (rid, t), other) -> ((rid, t, other), ())); + let others_join = with_it_idx5 | join(tags_of_idx, (_, ((rid, t), other)) -> ((rid, t, other), ())); -let others_filter = others_join | filter(((_, t, other), ()) -> (t, other) ne); -let others_map = others_filter | map(((rid, _, other), ()) -> ((rid, other), ())); -let others_idx = others_map | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic06_oneoff.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic06_oneoff.parsed--lowered.diff index a7fceac62..498e978ca 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic06_oneoff.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic06_oneoff.parsed--lowered.diff @@ -10,22 +10,22 @@ let by_name = import "graph::tag.by_name" : (List(int), int); let tagged = import "graph::mtag.by_tag" : (int, int); let tags_of = import "graph::mtag.by_message" : (int, int); --let hop1 = request | map(((rid, who, _), ()) -> (who, (rid, who))) | join(person, (who, req, _) -> (who, req)) | join(friend, (_, (rid, who), (f, _)) -> (f, (rid, who))); --let hop2 = hop1 | join(friend, (_, (rid, who), (g, _)) -> (g, (rid, who))); +-let hop1 = request | map(((rid, who, _), ()) -> (who, (rid, who))) | join(person, (who, (req, _)) -> (who, req)) | join(friend, (_, ((rid, who), (f, _))) -> (f, (rid, who))); +-let hop2 = hop1 | join(friend, (_, ((rid, who), (g, _))) -> (g, (rid, who))); -let circle = hop1 + hop2 | filter((node, (_, who)) -> (node, who) ne) | map((node, (rid, _)) -> ((rid, node), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); --let with_it = request | map(((rid, _, name), ()) -> (name, rid)) | join(by_name, (_, rid, t) -> (t, rid)) | join(tagged, (t, rid, m) -> (m, (rid, t))) | join(message, (m, (rid, t), (kind, _, author, _, _, _, _, _, _, _, _, _, _)) -> ((rid, author), (kind, m, t))) | filter((_, (kind, _, _)) -> (kind, 0) eq) | join(circle, ((rid, _), (_, m, t), ()) -> (m, (rid, t))); --let others = with_it | join(tags_of, (_, (rid, t), other) -> ((rid, t, other), ())) | filter(((_, t, other), ()) -> (t, other) ne) | map(((rid, _, other), ()) -> ((rid, other), ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); --let ranked = others | map(((rid, other), n) -> (other, (rid, n))) | join(tag, (_, (rid, n), (name, _)) -> (rid, (n neg, name))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); +-let with_it = request | map(((rid, _, name), ()) -> (name, rid)) | join(by_name, (_, (rid, t)) -> (t, rid)) | join(tagged, (t, (rid, m)) -> (m, (rid, t))) | join(message, (m, ((rid, t), (kind, _, author, _, _, _, _, _, _, _, _, _, _))) -> ((rid, author), (kind, m, t))) | filter((_, (kind, _, _)) -> (kind, 0) eq) | join(circle, ((rid, _), ((_, m, t), ())) -> (m, (rid, t))); +-let others = with_it | join(tags_of, (_, ((rid, t), other)) -> ((rid, t, other), ())) | filter(((_, t, other), ()) -> (t, other) ne) | map(((rid, _, other), ()) -> ((rid, other), ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +-let ranked = others | map(((rid, other), n) -> (other, (rid, n))) | join(tag, (_, ((rid, n), (name, _))) -> (rid, (n neg, name))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); +let hop1_map = request | map(((rid, who, _), ()) -> (who, (rid, who))); +let hop1_idx = hop1_map | arrange; +let person_idx = person | arrange; -+let hop1_join = hop1_idx | join(person_idx, (who, req, _) -> (who, req)); ++let hop1_join = hop1_idx | join(person_idx, (who, (req, _)) -> (who, req)); +let hop1_idx2 = hop1_join | arrange; +let friend_idx = friend | arrange; -+let hop1 = hop1_idx2 | join(friend_idx, (_, (rid, who), (f, _)) -> (f, (rid, who))); ++let hop1 = hop1_idx2 | join(friend_idx, (_, ((rid, who), (f, _))) -> (f, (rid, who))); +let hop1_idx3 = hop1 | arrange; +let friend_idx2 = friend | arrange; -+let hop2 = hop1_idx3 | join(friend_idx2, (_, (rid, who), (g, _)) -> (g, (rid, who))); ++let hop2 = hop1_idx3 | join(friend_idx2, (_, ((rid, who), (g, _))) -> (g, (rid, who))); +let circle_rows = hop1 + hop2; +let circle_filter = circle_rows | filter((node, (_, who)) -> (node, who) ne); +let circle_map = circle_filter | map((node, (rid, _)) -> ((rid, node), ())); @@ -34,19 +34,19 @@ +let with_it_map = request | map(((rid, _, name), ()) -> (name, rid)); +let with_it_idx = with_it_map | arrange; +let by_name_idx = by_name | arrange; -+let with_it_join = with_it_idx | join(by_name_idx, (_, rid, t) -> (t, rid)); ++let with_it_join = with_it_idx | join(by_name_idx, (_, (rid, t)) -> (t, rid)); +let with_it_idx2 = with_it_join | arrange; +let tagged_idx = tagged | arrange; -+let with_it_join2 = with_it_idx2 | join(tagged_idx, (t, rid, m) -> (m, (rid, t))); ++let with_it_join2 = with_it_idx2 | join(tagged_idx, (t, (rid, m)) -> (m, (rid, t))); +let with_it_idx3 = with_it_join2 | arrange; +let message_idx = message | arrange; -+let with_it_join3 = with_it_idx3 | join(message_idx, (m, (rid, t), (kind, _, author, _, _, _, _, _, _, _, _, _, _)) -> ((rid, author), (kind, m, t))); ++let with_it_join3 = with_it_idx3 | join(message_idx, (m, ((rid, t), (kind, _, author, _, _, _, _, _, _, _, _, _, _))) -> ((rid, author), (kind, m, t))); +let with_it_filter = with_it_join3 | filter((_, (kind, _, _)) -> (kind, 0) eq); +let with_it_idx4 = with_it_filter | arrange; -+let with_it = with_it_idx4 | join(circle, ((rid, _), (_, m, t), ()) -> (m, (rid, t))); ++let with_it = with_it_idx4 | join(circle, ((rid, _), ((_, m, t), ())) -> (m, (rid, t))); +let with_it_idx5 = with_it | arrange; +let tags_of_idx = tags_of | arrange; -+let others_join = with_it_idx5 | join(tags_of_idx, (_, (rid, t), other) -> ((rid, t, other), ())); ++let others_join = with_it_idx5 | join(tags_of_idx, (_, ((rid, t), other)) -> ((rid, t, other), ())); +let others_filter = others_join | filter(((_, t, other), ()) -> (t, other) ne); +let others_map = others_filter | map(((rid, _, other), ()) -> ((rid, other), ())); +let others_idx = others_map | arrange; @@ -54,7 +54,7 @@ +let ranked_map = others | map(((rid, other), n) -> (other, (rid, n))); +let ranked_idx = ranked_map | arrange; +let tag_idx = tag | arrange; -+let ranked_join = ranked_idx | join(tag_idx, (_, (rid, n), (name, _)) -> (rid, (n neg, name))); ++let ranked_join = ranked_idx | join(tag_idx, (_, ((rid, n), (name, _))) -> (rid, (n neg, name))); +let ranked_idx2 = ranked_join | arrange; +let ranked = ranked_idx2 | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, (neg, other))) -> ((rid, rank), (other, neg neg))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic06_oneoff.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic06_oneoff.parsed.ddp index 91b70d414..a792c7fd7 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic06_oneoff.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic06_oneoff.parsed.ddp @@ -6,11 +6,11 @@ let tag = import "graph::tag" : (int, (List(int), int)); let by_name = import "graph::tag.by_name" : (List(int), int); let tagged = import "graph::mtag.by_tag" : (int, int); let tags_of = import "graph::mtag.by_message" : (int, int); -let hop1 = request | map(((rid, who, _), ()) -> (who, (rid, who))) | join(person, (who, req, _) -> (who, req)) | join(friend, (_, (rid, who), (f, _)) -> (f, (rid, who))); -let hop2 = hop1 | join(friend, (_, (rid, who), (g, _)) -> (g, (rid, who))); +let hop1 = request | map(((rid, who, _), ()) -> (who, (rid, who))) | join(person, (who, (req, _)) -> (who, req)) | join(friend, (_, ((rid, who), (f, _))) -> (f, (rid, who))); +let hop2 = hop1 | join(friend, (_, ((rid, who), (g, _))) -> (g, (rid, who))); let circle = hop1 + hop2 | filter((node, (_, who)) -> (node, who) ne) | map((node, (rid, _)) -> ((rid, node), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); -let with_it = request | map(((rid, _, name), ()) -> (name, rid)) | join(by_name, (_, rid, t) -> (t, rid)) | join(tagged, (t, rid, m) -> (m, (rid, t))) | join(message, (m, (rid, t), (kind, _, author, _, _, _, _, _, _, _, _, _, _)) -> ((rid, author), (kind, m, t))) | filter((_, (kind, _, _)) -> (kind, 0) eq) | join(circle, ((rid, _), (_, m, t), ()) -> (m, (rid, t))); -let others = with_it | join(tags_of, (_, (rid, t), other) -> ((rid, t, other), ())) | filter(((_, t, other), ()) -> (t, other) ne) | map(((rid, _, other), ()) -> ((rid, other), ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); -let ranked = others | map(((rid, other), n) -> (other, (rid, n))) | join(tag, (_, (rid, n), (name, _)) -> (rid, (n neg, name))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); +let with_it = request | map(((rid, _, name), ()) -> (name, rid)) | join(by_name, (_, (rid, t)) -> (t, rid)) | join(tagged, (t, (rid, m)) -> (m, (rid, t))) | join(message, (m, ((rid, t), (kind, _, author, _, _, _, _, _, _, _, _, _, _))) -> ((rid, author), (kind, m, t))) | filter((_, (kind, _, _)) -> (kind, 0) eq) | join(circle, ((rid, _), ((_, m, t), ())) -> (m, (rid, t))); +let others = with_it | join(tags_of, (_, ((rid, t), other)) -> ((rid, t, other), ())) | filter(((_, t, other), ()) -> (t, other) ne) | map(((rid, _, other), ()) -> ((rid, other), ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +let ranked = others | map(((rid, other), n) -> (other, (rid, n))) | join(tag, (_, ((rid, n), (name, _))) -> (rid, (n neg, name))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, (neg, other))) -> ((rid, rank), (other, neg neg))); output "answer" = answer | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic07_oneoff.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic07_oneoff.ddp index 5ea61650a..361ab5fce 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic07_oneoff.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic07_oneoff.ddp @@ -9,19 +9,19 @@ let pals = import "graph::friend.pair" : ((int, int), ()); let latest_map = request | map(((rid, who), ()) -> (who, rid)); let latest_idx = latest_map | arrange; let wrote_idx = wrote | arrange; -let latest_join = latest_idx | join(wrote_idx, (who, rid, m) -> (m, (rid, who))); +let latest_join = latest_idx | join(wrote_idx, (who, (rid, m)) -> (m, (rid, who))); let latest_idx2 = latest_join | arrange; let likes_idx = likes | arrange; -let latest_join2 = latest_idx2 | join(likes_idx, (m, (rid, who), (liker, when)) -> ((rid, who, liker), (when neg, m))); +let latest_join2 = latest_idx2 | join(likes_idx, (m, ((rid, who), (liker, when))) -> ((rid, who, liker), (when neg, m))); let latest_idx3 = latest_join2 | arrange; let latest = latest_idx3 | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); let rows_map = latest | map(((rid, who, liker), (neg, m)) -> (m, (rid, who, liker, neg neg))); let rows_idx = rows_map | arrange; let message_idx = message | arrange; -let rows = rows_idx | join(message_idx, (m, (rid, who, liker, when), (_, created, _, _, text, image, _, _, _, _, _, _, _)) -> ((who, liker), (rid, liker, when, m, ((text len, 0) gt, text, image) select, (when, created) sub))); +let rows = rows_idx | join(message_idx, (m, ((rid, who, liker, when), (_, created, _, _, text, image, _, _, _, _, _, _, _))) -> ((who, liker), (rid, liker, when, m, ((text len, 0) gt, text, image) select, (when, created) sub))); let rows_idx2 = rows | arrange; let pals_idx = pals | arrange; -let known = rows_idx2 | join(pals_idx, arg -> let _call0 = let input = arg in let (pair, row, ()) = input in (pair, row) in (((), ()), (_call0.1.0, _call0.1.1, _call0.1.2, _call0.1.3, _call0.1.4, _call0.1.5))); +let known = rows_idx2 | join(pals_idx, arg -> let _call0 = let input = arg in let (pair, (row, ())) = input in (pair, row) in (((), ()), (_call0.1.0, _call0.1.1, _call0.1.2, _call0.1.3, _call0.1.4, _call0.1.5))); let marked_map = known | map((_, row) -> (row, 0)); let marked_map2 = rows | map(arg -> let _call0 = let input = arg in input in (((), ()), (_call0.1.0, _call0.1.1, _call0.1.2, _call0.1.3, _call0.1.4, _call0.1.5))); let marked_neg = known | negate; @@ -31,7 +31,7 @@ let marked = marked_map + marked_map3; let ranked_map = marked | map(((rid, liker, when, m, body, age), fresh) -> (liker, (rid, when, m, body, age, fresh))); let ranked_idx = ranked_map | arrange; let person_idx = person | arrange; -let ranked_join = ranked_idx | join(person_idx, (liker, (rid, when, m, body, age, fresh), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, ((when neg, liker), (first, last, m, body, (age, 60000) div, fresh)))); +let ranked_join = ranked_idx | join(person_idx, (liker, ((rid, when, m, body, age, fresh), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((when neg, liker), (first, last, m, body, (age, 60000) div, fresh)))); let ranked_idx2 = ranked_join | arrange; let ranked = ranked_idx2 | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((neg, liker), (first, last, m, body, minutes, fresh)))) -> ((rid, rank), (liker, first, last, neg neg, m, body, minutes, fresh))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic07_oneoff.optimized--projected.diff b/interactive/tests/plan_pairs/expected/ldbc_ic07_oneoff.optimized--projected.diff index c90d26d3c..d2cd56bce 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic07_oneoff.optimized--projected.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic07_oneoff.optimized--projected.diff @@ -1,11 +1,11 @@ --- optimized +++ projected @@ -21,12 +21,13 @@ - let rows = rows_idx | join(message_idx, (m, (rid, who, liker, when), (_, created, _, _, text, image, _, _, _, _, _, _, _)) -> ((who, liker), (rid, liker, when, m, ((text len, 0) gt, text, image) select, (when, created) sub))); + let rows = rows_idx | join(message_idx, (m, ((rid, who, liker, when), (_, created, _, _, text, image, _, _, _, _, _, _, _))) -> ((who, liker), (rid, liker, when, m, ((text len, 0) gt, text, image) select, (when, created) sub))); let rows_idx2 = rows | arrange; let pals_idx = pals | arrange; --let known = rows_idx2 | join(pals_idx, (pair, row, ()) -> (pair, row)); -+let known = rows_idx2 | join(pals_idx, arg -> let _call0 = let input = arg in let (pair, row, ()) = input in (pair, row) in (((), ()), (_call0.1.0, _call0.1.1, _call0.1.2, _call0.1.3, _call0.1.4, _call0.1.5))); +-let known = rows_idx2 | join(pals_idx, (pair, (row, ())) -> (pair, row)); ++let known = rows_idx2 | join(pals_idx, arg -> let _call0 = let input = arg in let (pair, (row, ())) = input in (pair, row) in (((), ()), (_call0.1.0, _call0.1.1, _call0.1.2, _call0.1.3, _call0.1.4, _call0.1.5))); let marked_map = known | map((_, row) -> (row, 0)); +let marked_map2 = rows | map(arg -> let _call0 = let input = arg in input in (((), ()), (_call0.1.0, _call0.1.1, _call0.1.2, _call0.1.3, _call0.1.4, _call0.1.5))); let marked_neg = known | negate; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic07_oneoff.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic07_oneoff.parsed--lowered.diff index 76027da16..a57d81f6f 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic07_oneoff.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic07_oneoff.parsed--lowered.diff @@ -9,27 +9,27 @@ let wrote = import "graph::message.by_creator" : (int, int); let likes = import "graph::likes.by_message" : (int, (int, int)); let pals = import "graph::friend.pair" : ((int, int), ()); --let latest = request | map(((rid, who), ()) -> (who, rid)) | join(wrote, (who, rid, m) -> (m, (rid, who))) | join(likes, (m, (rid, who), (liker, when)) -> ((rid, who, liker), (when neg, m))) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); --let rows = latest | map(((rid, who, liker), (neg, m)) -> (m, (rid, who, liker, neg neg))) | join(message, (m, (rid, who, liker, when), (_, created, _, _, text, image, _, _, _, _, _, _, _)) -> ((who, liker), (rid, liker, when, m, ((text len, 0) gt, text, image) select, (when, created) sub))); --let known = rows | join(pals, (pair, row, ()) -> (pair, row)); +-let latest = request | map(((rid, who), ()) -> (who, rid)) | join(wrote, (who, (rid, m)) -> (m, (rid, who))) | join(likes, (m, ((rid, who), (liker, when))) -> ((rid, who, liker), (when neg, m))) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); +-let rows = latest | map(((rid, who, liker), (neg, m)) -> (m, (rid, who, liker, neg neg))) | join(message, (m, ((rid, who, liker, when), (_, created, _, _, text, image, _, _, _, _, _, _, _))) -> ((who, liker), (rid, liker, when, m, ((text len, 0) gt, text, image) select, (when, created) sub))); +-let known = rows | join(pals, (pair, (row, ())) -> (pair, row)); -let marked = (known | map((_, row) -> (row, 0))) + (rows + (known | negate) | map((_, row) -> (row, 1))); --let ranked = marked | map(((rid, liker, when, m, body, age), fresh) -> (liker, (rid, when, m, body, age, fresh))) | join(person, (liker, (rid, when, m, body, age, fresh), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, ((when neg, liker), (first, last, m, body, (age, 60000) div, fresh)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); +-let ranked = marked | map(((rid, liker, when, m, body, age), fresh) -> (liker, (rid, when, m, body, age, fresh))) | join(person, (liker, ((rid, when, m, body, age, fresh), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((when neg, liker), (first, last, m, body, (age, 60000) div, fresh)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); +let latest_map = request | map(((rid, who), ()) -> (who, rid)); +let latest_idx = latest_map | arrange; +let wrote_idx = wrote | arrange; -+let latest_join = latest_idx | join(wrote_idx, (who, rid, m) -> (m, (rid, who))); ++let latest_join = latest_idx | join(wrote_idx, (who, (rid, m)) -> (m, (rid, who))); +let latest_idx2 = latest_join | arrange; +let likes_idx = likes | arrange; -+let latest_join2 = latest_idx2 | join(likes_idx, (m, (rid, who), (liker, when)) -> ((rid, who, liker), (when neg, m))); ++let latest_join2 = latest_idx2 | join(likes_idx, (m, ((rid, who), (liker, when))) -> ((rid, who, liker), (when neg, m))); +let latest_idx3 = latest_join2 | arrange; +let latest = latest_idx3 | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); +let rows_map = latest | map(((rid, who, liker), (neg, m)) -> (m, (rid, who, liker, neg neg))); +let rows_idx = rows_map | arrange; +let message_idx = message | arrange; -+let rows = rows_idx | join(message_idx, (m, (rid, who, liker, when), (_, created, _, _, text, image, _, _, _, _, _, _, _)) -> ((who, liker), (rid, liker, when, m, ((text len, 0) gt, text, image) select, (when, created) sub))); ++let rows = rows_idx | join(message_idx, (m, ((rid, who, liker, when), (_, created, _, _, text, image, _, _, _, _, _, _, _))) -> ((who, liker), (rid, liker, when, m, ((text len, 0) gt, text, image) select, (when, created) sub))); +let rows_idx2 = rows | arrange; +let pals_idx = pals | arrange; -+let known = rows_idx2 | join(pals_idx, (pair, row, ()) -> (pair, row)); ++let known = rows_idx2 | join(pals_idx, (pair, (row, ())) -> (pair, row)); +let marked_map = known | map((_, row) -> (row, 0)); +let marked_neg = known | negate; +let marked_rows = rows + marked_neg; @@ -38,7 +38,7 @@ +let ranked_map = marked | map(((rid, liker, when, m, body, age), fresh) -> (liker, (rid, when, m, body, age, fresh))); +let ranked_idx = ranked_map | arrange; +let person_idx = person | arrange; -+let ranked_join = ranked_idx | join(person_idx, (liker, (rid, when, m, body, age, fresh), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, ((when neg, liker), (first, last, m, body, (age, 60000) div, fresh)))); ++let ranked_join = ranked_idx | join(person_idx, (liker, ((rid, when, m, body, age, fresh), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((when neg, liker), (first, last, m, body, (age, 60000) div, fresh)))); +let ranked_idx2 = ranked_join | arrange; +let ranked = ranked_idx2 | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((neg, liker), (first, last, m, body, minutes, fresh)))) -> ((rid, rank), (liker, first, last, neg neg, m, body, minutes, fresh))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic07_oneoff.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic07_oneoff.parsed.ddp index 2abed6852..0a6ebc742 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic07_oneoff.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic07_oneoff.parsed.ddp @@ -4,10 +4,10 @@ let message = import "graph::message" : (int, (int, int, int, int, List(int), Li let wrote = import "graph::message.by_creator" : (int, int); let likes = import "graph::likes.by_message" : (int, (int, int)); let pals = import "graph::friend.pair" : ((int, int), ()); -let latest = request | map(((rid, who), ()) -> (who, rid)) | join(wrote, (who, rid, m) -> (m, (rid, who))) | join(likes, (m, (rid, who), (liker, when)) -> ((rid, who, liker), (when neg, m))) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); -let rows = latest | map(((rid, who, liker), (neg, m)) -> (m, (rid, who, liker, neg neg))) | join(message, (m, (rid, who, liker, when), (_, created, _, _, text, image, _, _, _, _, _, _, _)) -> ((who, liker), (rid, liker, when, m, ((text len, 0) gt, text, image) select, (when, created) sub))); -let known = rows | join(pals, (pair, row, ()) -> (pair, row)); +let latest = request | map(((rid, who), ()) -> (who, rid)) | join(wrote, (who, (rid, m)) -> (m, (rid, who))) | join(likes, (m, ((rid, who), (liker, when))) -> ((rid, who, liker), (when neg, m))) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); +let rows = latest | map(((rid, who, liker), (neg, m)) -> (m, (rid, who, liker, neg neg))) | join(message, (m, ((rid, who, liker, when), (_, created, _, _, text, image, _, _, _, _, _, _, _))) -> ((who, liker), (rid, liker, when, m, ((text len, 0) gt, text, image) select, (when, created) sub))); +let known = rows | join(pals, (pair, (row, ())) -> (pair, row)); let marked = (known | map((_, row) -> (row, 0))) + (rows + (known | negate) | map((_, row) -> (row, 1))); -let ranked = marked | map(((rid, liker, when, m, body, age), fresh) -> (liker, (rid, when, m, body, age, fresh))) | join(person, (liker, (rid, when, m, body, age, fresh), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, ((when neg, liker), (first, last, m, body, (age, 60000) div, fresh)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); +let ranked = marked | map(((rid, liker, when, m, body, age), fresh) -> (liker, (rid, when, m, body, age, fresh))) | join(person, (liker, ((rid, when, m, body, age, fresh), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((when neg, liker), (first, last, m, body, (age, 60000) div, fresh)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((neg, liker), (first, last, m, body, minutes, fresh)))) -> ((rid, rank), (liker, first, last, neg neg, m, body, minutes, fresh))); output "answer" = answer | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic08_oneoff.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic08_oneoff.ddp index f6de7e88a..9a7119777 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic08_oneoff.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic08_oneoff.ddp @@ -8,22 +8,22 @@ let replies = import "graph::message.by_parent" : (int, int); let top_map = request | map(((rid, who), ()) -> (who, rid)); let top_idx = top_map | arrange; let wrote_idx = wrote | arrange; -let top_join = top_idx | join(wrote_idx, (_, rid, op) -> (op, rid)); +let top_join = top_idx | join(wrote_idx, (_, (rid, op)) -> (op, rid)); let top_idx2 = top_join | arrange; let replies_idx = replies | arrange; -let top_join2 = top_idx2 | join(replies_idx, (_, rid, id) -> (id, rid)); +let top_join2 = top_idx2 | join(replies_idx, (_, (rid, id)) -> (id, rid)); let top_idx3 = top_join2 | arrange; let message_idx = message | arrange; -let top_join3 = top_idx3 | join(message_idx, (id, rid, (kind, created, who, _, _, _, _, _, _, _, _, _, _)) -> (rid, (kind, (created neg, id), who))); +let top_join3 = top_idx3 | join(message_idx, (id, (rid, (kind, created, who, _, _, _, _, _, _, _, _, _, _))) -> (rid, (kind, (created neg, id), who))); let top_rows = top_join3 | filter((_, (kind, _, _)) -> (kind, 1) eq) | map((rid, (_, order, who)) -> (rid, (order, who))); let top_idx4 = top_rows | arrange; let top = top_idx4 | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer_map = top | map((rid, (rank, ((neg, id), who))) -> (id, (rid, rank, neg neg, who))); let answer_idx = answer_map | arrange; -let answer_join = answer_idx | join(message_idx, (id, (rid, rank, created, who), (_, _, _, _, content, _, _, _, _, _, _, _, _)) -> (who, (rid, rank, id, content, created))); +let answer_join = answer_idx | join(message_idx, (id, ((rid, rank, created, who), (_, _, _, _, content, _, _, _, _, _, _, _, _))) -> (who, (rid, rank, id, content, created))); let answer_idx2 = answer_join | arrange; let person_idx = person | arrange; -let answer = answer_idx2 | join(person_idx, (who, (rid, rank, id, content, created), (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, rank), (who, first, last, created, id, content))); +let answer = answer_idx2 | join(person_idx, (who, ((rid, rank, id, content, created), (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, rank), (who, first, last, created, id, content))); let answer_idx3 = answer | arrange; -- Renderer: publish the existing arrangement. output "answer" = answer_idx3; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic08_oneoff.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/ldbc_ic08_oneoff.lowered--optimized.diff index dc67315f9..f0b32566e 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic08_oneoff.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic08_oneoff.lowered--optimized.diff @@ -3,7 +3,7 @@ @@ -15,14 +15,12 @@ let top_idx3 = top_join2 | arrange; let message_idx = message | arrange; - let top_join3 = top_idx3 | join(message_idx, (id, rid, (kind, created, who, _, _, _, _, _, _, _, _, _, _)) -> (rid, (kind, (created neg, id), who))); + let top_join3 = top_idx3 | join(message_idx, (id, (rid, (kind, created, who, _, _, _, _, _, _, _, _, _, _))) -> (rid, (kind, (created neg, id), who))); -let top_filter = top_join3 | filter((_, (kind, _, _)) -> (kind, 1) eq); -let top_map2 = top_filter | map((rid, (_, order, who)) -> (rid, (order, who))); -let top_idx4 = top_map2 | arrange; @@ -13,8 +13,8 @@ let answer_map = top | map((rid, (rank, ((neg, id), who))) -> (id, (rid, rank, neg neg, who))); let answer_idx = answer_map | arrange; -let message_idx2 = message | arrange; --let answer_join = answer_idx | join(message_idx2, (id, (rid, rank, created, who), (_, _, _, _, content, _, _, _, _, _, _, _, _)) -> (who, (rid, rank, id, content, created))); -+let answer_join = answer_idx | join(message_idx, (id, (rid, rank, created, who), (_, _, _, _, content, _, _, _, _, _, _, _, _)) -> (who, (rid, rank, id, content, created))); +-let answer_join = answer_idx | join(message_idx2, (id, ((rid, rank, created, who), (_, _, _, _, content, _, _, _, _, _, _, _, _))) -> (who, (rid, rank, id, content, created))); ++let answer_join = answer_idx | join(message_idx, (id, ((rid, rank, created, who), (_, _, _, _, content, _, _, _, _, _, _, _, _))) -> (who, (rid, rank, id, content, created))); let answer_idx2 = answer_join | arrange; let person_idx = person | arrange; - let answer = answer_idx2 | join(person_idx, (who, (rid, rank, id, content, created), (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, rank), (who, first, last, created, id, content))); + let answer = answer_idx2 | join(person_idx, (who, ((rid, rank, id, content, created), (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, rank), (who, first, last, created, id, content))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic08_oneoff.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic08_oneoff.parsed--lowered.diff index 5d082e0ed..4b6ee23bd 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic08_oneoff.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic08_oneoff.parsed--lowered.diff @@ -8,19 +8,19 @@ let message = import "graph::message" : (int, (int, int, int, int, List(int), List(int), int, List(int), int, int, int, int, int)); let wrote = import "graph::message.by_creator" : (int, int); let replies = import "graph::message.by_parent" : (int, int); --let top = request | map(((rid, who), ()) -> (who, rid)) | join(wrote, (_, rid, op) -> (op, rid)) | join(replies, (_, rid, id) -> (id, rid)) | join(message, (id, rid, (kind, created, who, _, _, _, _, _, _, _, _, _, _)) -> (rid, (kind, (created neg, id), who))) | filter((_, (kind, _, _)) -> (kind, 1) eq) | map((rid, (_, order, who)) -> (rid, (order, who))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); --let answer = top | map((rid, (rank, ((neg, id), who))) -> (id, (rid, rank, neg neg, who))) | join(message, (id, (rid, rank, created, who), (_, _, _, _, content, _, _, _, _, _, _, _, _)) -> (who, (rid, rank, id, content, created))) | join(person, (who, (rid, rank, id, content, created), (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, rank), (who, first, last, created, id, content))); +-let top = request | map(((rid, who), ()) -> (who, rid)) | join(wrote, (_, (rid, op)) -> (op, rid)) | join(replies, (_, (rid, id)) -> (id, rid)) | join(message, (id, (rid, (kind, created, who, _, _, _, _, _, _, _, _, _, _))) -> (rid, (kind, (created neg, id), who))) | filter((_, (kind, _, _)) -> (kind, 1) eq) | map((rid, (_, order, who)) -> (rid, (order, who))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); +-let answer = top | map((rid, (rank, ((neg, id), who))) -> (id, (rid, rank, neg neg, who))) | join(message, (id, ((rid, rank, created, who), (_, _, _, _, content, _, _, _, _, _, _, _, _))) -> (who, (rid, rank, id, content, created))) | join(person, (who, ((rid, rank, id, content, created), (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, rank), (who, first, last, created, id, content))); -output "answer" = answer | arrange; +let top_map = request | map(((rid, who), ()) -> (who, rid)); +let top_idx = top_map | arrange; +let wrote_idx = wrote | arrange; -+let top_join = top_idx | join(wrote_idx, (_, rid, op) -> (op, rid)); ++let top_join = top_idx | join(wrote_idx, (_, (rid, op)) -> (op, rid)); +let top_idx2 = top_join | arrange; +let replies_idx = replies | arrange; -+let top_join2 = top_idx2 | join(replies_idx, (_, rid, id) -> (id, rid)); ++let top_join2 = top_idx2 | join(replies_idx, (_, (rid, id)) -> (id, rid)); +let top_idx3 = top_join2 | arrange; +let message_idx = message | arrange; -+let top_join3 = top_idx3 | join(message_idx, (id, rid, (kind, created, who, _, _, _, _, _, _, _, _, _, _)) -> (rid, (kind, (created neg, id), who))); ++let top_join3 = top_idx3 | join(message_idx, (id, (rid, (kind, created, who, _, _, _, _, _, _, _, _, _, _))) -> (rid, (kind, (created neg, id), who))); +let top_filter = top_join3 | filter((_, (kind, _, _)) -> (kind, 1) eq); +let top_map2 = top_filter | map((rid, (_, order, who)) -> (rid, (order, who))); +let top_idx4 = top_map2 | arrange; @@ -28,10 +28,10 @@ +let answer_map = top | map((rid, (rank, ((neg, id), who))) -> (id, (rid, rank, neg neg, who))); +let answer_idx = answer_map | arrange; +let message_idx2 = message | arrange; -+let answer_join = answer_idx | join(message_idx2, (id, (rid, rank, created, who), (_, _, _, _, content, _, _, _, _, _, _, _, _)) -> (who, (rid, rank, id, content, created))); ++let answer_join = answer_idx | join(message_idx2, (id, ((rid, rank, created, who), (_, _, _, _, content, _, _, _, _, _, _, _, _))) -> (who, (rid, rank, id, content, created))); +let answer_idx2 = answer_join | arrange; +let person_idx = person | arrange; -+let answer = answer_idx2 | join(person_idx, (who, (rid, rank, id, content, created), (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, rank), (who, first, last, created, id, content))); ++let answer = answer_idx2 | join(person_idx, (who, ((rid, rank, id, content, created), (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, rank), (who, first, last, created, id, content))); +let answer_idx3 = answer | arrange; +-- Renderer: publish the existing arrangement. +output "answer" = answer_idx3; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic08_oneoff.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic08_oneoff.parsed.ddp index 7a559ec15..0ee542d68 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic08_oneoff.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic08_oneoff.parsed.ddp @@ -3,6 +3,6 @@ let person = import "graph::person" : (int, (List(int), List(int), List(int), in let message = import "graph::message" : (int, (int, int, int, int, List(int), List(int), int, List(int), int, int, int, int, int)); let wrote = import "graph::message.by_creator" : (int, int); let replies = import "graph::message.by_parent" : (int, int); -let top = request | map(((rid, who), ()) -> (who, rid)) | join(wrote, (_, rid, op) -> (op, rid)) | join(replies, (_, rid, id) -> (id, rid)) | join(message, (id, rid, (kind, created, who, _, _, _, _, _, _, _, _, _, _)) -> (rid, (kind, (created neg, id), who))) | filter((_, (kind, _, _)) -> (kind, 1) eq) | map((rid, (_, order, who)) -> (rid, (order, who))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); -let answer = top | map((rid, (rank, ((neg, id), who))) -> (id, (rid, rank, neg neg, who))) | join(message, (id, (rid, rank, created, who), (_, _, _, _, content, _, _, _, _, _, _, _, _)) -> (who, (rid, rank, id, content, created))) | join(person, (who, (rid, rank, id, content, created), (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, rank), (who, first, last, created, id, content))); +let top = request | map(((rid, who), ()) -> (who, rid)) | join(wrote, (_, (rid, op)) -> (op, rid)) | join(replies, (_, (rid, id)) -> (id, rid)) | join(message, (id, (rid, (kind, created, who, _, _, _, _, _, _, _, _, _, _))) -> (rid, (kind, (created neg, id), who))) | filter((_, (kind, _, _)) -> (kind, 1) eq) | map((rid, (_, order, who)) -> (rid, (order, who))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); +let answer = top | map((rid, (rank, ((neg, id), who))) -> (id, (rid, rank, neg neg, who))) | join(message, (id, ((rid, rank, created, who), (_, _, _, _, content, _, _, _, _, _, _, _, _))) -> (who, (rid, rank, id, content, created))) | join(person, (who, ((rid, rank, id, content, created), (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, rank), (who, first, last, created, id, content))); output "answer" = answer | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic09_oneoff.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic09_oneoff.ddp index 59ac64fd5..81c579d90 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic09_oneoff.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic09_oneoff.ddp @@ -10,12 +10,12 @@ let newest = import "ic9s::newest" : (int, int); let hop1_map = request | map(((rid, who, max), ()) -> (who, (rid, who, max))); let hop1_idx = hop1_map | arrange; let person_idx = person | arrange; -let hop1_join = hop1_idx | join(person_idx, (who, req, _) -> (who, req)); +let hop1_join = hop1_idx | join(person_idx, (who, (req, _)) -> (who, req)); let hop1_idx2 = hop1_join | arrange; let friend_idx = friend | arrange; -let hop1 = hop1_idx2 | join(friend_idx, (_, req, (f, _)) -> (f, req)); +let hop1 = hop1_idx2 | join(friend_idx, (_, (req, (f, _))) -> (f, req)); let hop1_idx3 = hop1 | arrange; -let hop2 = hop1_idx3 | join(friend_idx, (_, req, (g, _)) -> (g, req)); +let hop2 = hop1_idx3 | join(friend_idx, (_, (req, (g, _))) -> (g, req)); let friends_rows = hop1 + hop2; let friends_rows2 = friends_rows | filter((node, (_, who, _)) -> (node, who) ne) | map((node, (rid, _, max)) -> ((node, rid, max), ())); let friends_idx = friends_rows2 | arrange; @@ -23,15 +23,15 @@ let friends_reduce = friends_idx | reduce((_, vals) -> vals map (((), _) -> ((), let friends = friends_reduce | map(((node, rid, max), ()) -> (node, (rid, max))); let friends_idx2 = friends | arrange; let newest_idx = newest | arrange; -let ahead = friends_idx2 | join(newest_idx, (f, (rid, max), neg) -> (f, (rid, max, neg neg))); +let ahead = friends_idx2 | join(newest_idx, (f, ((rid, max), neg)) -> (f, (rid, max, neg neg))); let fast_filter = ahead | filter((_, (_, max, last)) -> (last, max) lt); let fast_idx = fast_filter | arrange; let latest_idx = latest | arrange; -let fast = fast_idx | join(latest_idx, (f, (rid, _, _), (neg, id)) -> (rid, ((neg, id), f))); +let fast = fast_idx | join(latest_idx, (f, ((rid, _, _), (neg, id))) -> (rid, ((neg, id), f))); let slow_filter = ahead | filter((_, (_, max, last)) -> (last, max) ge); let slow_idx = slow_filter | arrange; let by_author_idx = by_author | arrange; -let slow_join = slow_idx | join(by_author_idx, (f, (rid, max, _), (neg, id)) -> ((rid, f), ((neg, id), max))); +let slow_join = slow_idx | join(by_author_idx, (f, ((rid, max, _), (neg, id))) -> ((rid, f), ((neg, id), max))); let slow_rows = slow_join | filter((_, ((neg, _), max)) -> (neg neg, max) lt) | map((key, (order, _)) -> (key, order)); let slow_idx2 = slow_rows | arrange; let slow_reduce = slow_idx2 | reduce((_, a) -> let v0 = (a map ((_, a) -> (a, 0) gt), (0, a len) range) zip filter in let v1 = (((0, (v0 len, 20) min) range, v0) gather, a) gather in let v2 = (v1, v1 map ((_, a) -> (a, 20) min) scan_add) zip map ((a, b) -> let v0 = (a.1, 20) min in let v1 = (((20, (b, v0) sub) sub, 0) max, v0) min in ((v1, 0) gt, (a.0, v1))) filter in (v2, v2 map ((_, a) -> a) scan_add) zip map ((a, b) -> (a.0, ((b, a.1) sub, b) range) cap_list map ((a, b) -> ((b, a), 1))) flatten map ((a, b) -> (((), a.1), b))); @@ -42,9 +42,9 @@ let top = top_idx | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, let answer_map = top | map((rid, (rank, ((neg, id), f))) -> (id, (rid, rank, neg neg, f))); let answer_idx = answer_map | arrange; let message_idx = message | arrange; -let answer_join = answer_idx | join(message_idx, (id, (rid, rank, created, f), (_, _, _, _, content, image, _, _, _, _, _, _, _)) -> (f, (rid, rank, id, ((content len, 0) gt, content, image) select, created))); +let answer_join = answer_idx | join(message_idx, (id, ((rid, rank, created, f), (_, _, _, _, content, image, _, _, _, _, _, _, _))) -> (f, (rid, rank, id, ((content len, 0) gt, content, image) select, created))); let answer_idx2 = answer_join | arrange; -let answer = answer_idx2 | join(person_idx, (f, (rid, rank, id, body, created), (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, rank), (f, first, last, id, body, created))); +let answer = answer_idx2 | join(person_idx, (f, ((rid, rank, id, body, created), (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, rank), (f, first, last, id, body, created))); let answer_idx3 = answer | arrange; -- Renderer: publish the existing arrangement. output "answer" = answer_idx3; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic09_oneoff.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/ldbc_ic09_oneoff.lowered--optimized.diff index 9e7c8ab1c..47d9c20bb 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic09_oneoff.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic09_oneoff.lowered--optimized.diff @@ -2,11 +2,11 @@ +++ optimized @@ -15,12 +15,10 @@ let friend_idx = friend | arrange; - let hop1 = hop1_idx2 | join(friend_idx, (_, req, (f, _)) -> (f, req)); + let hop1 = hop1_idx2 | join(friend_idx, (_, (req, (f, _))) -> (f, req)); let hop1_idx3 = hop1 | arrange; -let friend_idx2 = friend | arrange; --let hop2 = hop1_idx3 | join(friend_idx2, (_, req, (g, _)) -> (g, req)); -+let hop2 = hop1_idx3 | join(friend_idx, (_, req, (g, _)) -> (g, req)); +-let hop2 = hop1_idx3 | join(friend_idx2, (_, (req, (g, _))) -> (g, req)); ++let hop2 = hop1_idx3 | join(friend_idx, (_, (req, (g, _))) -> (g, req)); let friends_rows = hop1 + hop2; -let friends_filter = friends_rows | filter((node, (_, who, _)) -> (node, who) ne); -let friends_map = friends_filter | map((node, (rid, _, max)) -> ((node, rid, max), ())); @@ -19,7 +19,7 @@ @@ -34,9 +32,8 @@ let slow_idx = slow_filter | arrange; let by_author_idx = by_author | arrange; - let slow_join = slow_idx | join(by_author_idx, (f, (rid, max, _), (neg, id)) -> ((rid, f), ((neg, id), max))); + let slow_join = slow_idx | join(by_author_idx, (f, ((rid, max, _), (neg, id))) -> ((rid, f), ((neg, id), max))); -let slow_filter2 = slow_join | filter((_, ((neg, _), max)) -> (neg neg, max) lt); -let slow_map = slow_filter2 | map((key, (order, _)) -> (key, order)); -let slow_idx2 = slow_map | arrange; @@ -30,11 +30,11 @@ let top_rows = fast + slow; @@ -47,8 +44,7 @@ let message_idx = message | arrange; - let answer_join = answer_idx | join(message_idx, (id, (rid, rank, created, f), (_, _, _, _, content, image, _, _, _, _, _, _, _)) -> (f, (rid, rank, id, ((content len, 0) gt, content, image) select, created))); + let answer_join = answer_idx | join(message_idx, (id, ((rid, rank, created, f), (_, _, _, _, content, image, _, _, _, _, _, _, _))) -> (f, (rid, rank, id, ((content len, 0) gt, content, image) select, created))); let answer_idx2 = answer_join | arrange; -let person_idx2 = person | arrange; --let answer = answer_idx2 | join(person_idx2, (f, (rid, rank, id, body, created), (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, rank), (f, first, last, id, body, created))); -+let answer = answer_idx2 | join(person_idx, (f, (rid, rank, id, body, created), (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, rank), (f, first, last, id, body, created))); +-let answer = answer_idx2 | join(person_idx2, (f, ((rid, rank, id, body, created), (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, rank), (f, first, last, id, body, created))); ++let answer = answer_idx2 | join(person_idx, (f, ((rid, rank, id, body, created), (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, rank), (f, first, last, id, body, created))); let answer_idx3 = answer | arrange; -- Renderer: publish the existing arrangement. output "answer" = answer_idx3; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic09_oneoff.optimized--projected.diff b/interactive/tests/plan_pairs/expected/ldbc_ic09_oneoff.optimized--projected.diff index 4ad8801b5..3cdfb530c 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic09_oneoff.optimized--projected.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic09_oneoff.optimized--projected.diff @@ -1,7 +1,7 @@ --- optimized +++ projected @@ -34,7 +34,7 @@ - let slow_join = slow_idx | join(by_author_idx, (f, (rid, max, _), (neg, id)) -> ((rid, f), ((neg, id), max))); + let slow_join = slow_idx | join(by_author_idx, (f, ((rid, max, _), (neg, id))) -> ((rid, f), ((neg, id), max))); let slow_rows = slow_join | filter((_, ((neg, _), max)) -> (neg neg, max) lt) | map((key, (order, _)) -> (key, order)); let slow_idx2 = slow_rows | arrange; -let slow_reduce = slow_idx2 | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic09_oneoff.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic09_oneoff.parsed--lowered.diff index 7d22743da..a62554c91 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic09_oneoff.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic09_oneoff.parsed--lowered.diff @@ -10,25 +10,25 @@ let by_author = import "ic9s::by_author" : (int, (int, int)); let latest = import "ic9s::latest" : (int, (int, int)); let newest = import "ic9s::newest" : (int, int); --let hop1 = request | map(((rid, who, max), ()) -> (who, (rid, who, max))) | join(person, (who, req, _) -> (who, req)) | join(friend, (_, req, (f, _)) -> (f, req)); --let hop2 = hop1 | join(friend, (_, req, (g, _)) -> (g, req)); +-let hop1 = request | map(((rid, who, max), ()) -> (who, (rid, who, max))) | join(person, (who, (req, _)) -> (who, req)) | join(friend, (_, (req, (f, _))) -> (f, req)); +-let hop2 = hop1 | join(friend, (_, (req, (g, _))) -> (g, req)); -let friends = hop1 + hop2 | filter((node, (_, who, _)) -> (node, who) ne) | map((node, (rid, _, max)) -> ((node, rid, max), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((node, rid, max), ()) -> (node, (rid, max))); --let ahead = friends | join(newest, (f, (rid, max), neg) -> (f, (rid, max, neg neg))); --let fast = ahead | filter((_, (_, max, last)) -> (last, max) lt) | join(latest, (f, (rid, _, _), (neg, id)) -> (rid, ((neg, id), f))); --let slow = ahead | filter((_, (_, max, last)) -> (last, max) ge) | join(by_author, (f, (rid, max, _), (neg, id)) -> ((rid, f), ((neg, id), max))) | filter((_, ((neg, _), max)) -> (neg neg, max) lt) | map((key, (order, _)) -> (key, order)) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten) | map(((rid, f), (_, order)) -> (rid, (order, f))); +-let ahead = friends | join(newest, (f, ((rid, max), neg)) -> (f, (rid, max, neg neg))); +-let fast = ahead | filter((_, (_, max, last)) -> (last, max) lt) | join(latest, (f, ((rid, _, _), (neg, id))) -> (rid, ((neg, id), f))); +-let slow = ahead | filter((_, (_, max, last)) -> (last, max) ge) | join(by_author, (f, ((rid, max, _), (neg, id))) -> ((rid, f), ((neg, id), max))) | filter((_, ((neg, _), max)) -> (neg neg, max) lt) | map((key, (order, _)) -> (key, order)) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten) | map(((rid, f), (_, order)) -> (rid, (order, f))); -let top = fast + slow | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); --let answer = top | map((rid, (rank, ((neg, id), f))) -> (id, (rid, rank, neg neg, f))) | join(message, (id, (rid, rank, created, f), (_, _, _, _, content, image, _, _, _, _, _, _, _)) -> (f, (rid, rank, id, ((content len, 0) gt, content, image) select, created))) | join(person, (f, (rid, rank, id, body, created), (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, rank), (f, first, last, id, body, created))); +-let answer = top | map((rid, (rank, ((neg, id), f))) -> (id, (rid, rank, neg neg, f))) | join(message, (id, ((rid, rank, created, f), (_, _, _, _, content, image, _, _, _, _, _, _, _))) -> (f, (rid, rank, id, ((content len, 0) gt, content, image) select, created))) | join(person, (f, ((rid, rank, id, body, created), (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, rank), (f, first, last, id, body, created))); -output "answer" = answer | arrange; +let hop1_map = request | map(((rid, who, max), ()) -> (who, (rid, who, max))); +let hop1_idx = hop1_map | arrange; +let person_idx = person | arrange; -+let hop1_join = hop1_idx | join(person_idx, (who, req, _) -> (who, req)); ++let hop1_join = hop1_idx | join(person_idx, (who, (req, _)) -> (who, req)); +let hop1_idx2 = hop1_join | arrange; +let friend_idx = friend | arrange; -+let hop1 = hop1_idx2 | join(friend_idx, (_, req, (f, _)) -> (f, req)); ++let hop1 = hop1_idx2 | join(friend_idx, (_, (req, (f, _))) -> (f, req)); +let hop1_idx3 = hop1 | arrange; +let friend_idx2 = friend | arrange; -+let hop2 = hop1_idx3 | join(friend_idx2, (_, req, (g, _)) -> (g, req)); ++let hop2 = hop1_idx3 | join(friend_idx2, (_, (req, (g, _))) -> (g, req)); +let friends_rows = hop1 + hop2; +let friends_filter = friends_rows | filter((node, (_, who, _)) -> (node, who) ne); +let friends_map = friends_filter | map((node, (rid, _, max)) -> ((node, rid, max), ())); @@ -37,15 +37,15 @@ +let friends = friends_reduce | map(((node, rid, max), ()) -> (node, (rid, max))); +let friends_idx2 = friends | arrange; +let newest_idx = newest | arrange; -+let ahead = friends_idx2 | join(newest_idx, (f, (rid, max), neg) -> (f, (rid, max, neg neg))); ++let ahead = friends_idx2 | join(newest_idx, (f, ((rid, max), neg)) -> (f, (rid, max, neg neg))); +let fast_filter = ahead | filter((_, (_, max, last)) -> (last, max) lt); +let fast_idx = fast_filter | arrange; +let latest_idx = latest | arrange; -+let fast = fast_idx | join(latest_idx, (f, (rid, _, _), (neg, id)) -> (rid, ((neg, id), f))); ++let fast = fast_idx | join(latest_idx, (f, ((rid, _, _), (neg, id))) -> (rid, ((neg, id), f))); +let slow_filter = ahead | filter((_, (_, max, last)) -> (last, max) ge); +let slow_idx = slow_filter | arrange; +let by_author_idx = by_author | arrange; -+let slow_join = slow_idx | join(by_author_idx, (f, (rid, max, _), (neg, id)) -> ((rid, f), ((neg, id), max))); ++let slow_join = slow_idx | join(by_author_idx, (f, ((rid, max, _), (neg, id))) -> ((rid, f), ((neg, id), max))); +let slow_filter2 = slow_join | filter((_, ((neg, _), max)) -> (neg neg, max) lt); +let slow_map = slow_filter2 | map((key, (order, _)) -> (key, order)); +let slow_idx2 = slow_map | arrange; @@ -57,10 +57,10 @@ +let answer_map = top | map((rid, (rank, ((neg, id), f))) -> (id, (rid, rank, neg neg, f))); +let answer_idx = answer_map | arrange; +let message_idx = message | arrange; -+let answer_join = answer_idx | join(message_idx, (id, (rid, rank, created, f), (_, _, _, _, content, image, _, _, _, _, _, _, _)) -> (f, (rid, rank, id, ((content len, 0) gt, content, image) select, created))); ++let answer_join = answer_idx | join(message_idx, (id, ((rid, rank, created, f), (_, _, _, _, content, image, _, _, _, _, _, _, _))) -> (f, (rid, rank, id, ((content len, 0) gt, content, image) select, created))); +let answer_idx2 = answer_join | arrange; +let person_idx2 = person | arrange; -+let answer = answer_idx2 | join(person_idx2, (f, (rid, rank, id, body, created), (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, rank), (f, first, last, id, body, created))); ++let answer = answer_idx2 | join(person_idx2, (f, ((rid, rank, id, body, created), (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, rank), (f, first, last, id, body, created))); +let answer_idx3 = answer | arrange; +-- Renderer: publish the existing arrangement. +output "answer" = answer_idx3; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic09_oneoff.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic09_oneoff.parsed.ddp index 48a20ea94..7eeebdaab 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic09_oneoff.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic09_oneoff.parsed.ddp @@ -5,12 +5,12 @@ let friend = import "graph::friend" : (int, (int, int)); let by_author = import "ic9s::by_author" : (int, (int, int)); let latest = import "ic9s::latest" : (int, (int, int)); let newest = import "ic9s::newest" : (int, int); -let hop1 = request | map(((rid, who, max), ()) -> (who, (rid, who, max))) | join(person, (who, req, _) -> (who, req)) | join(friend, (_, req, (f, _)) -> (f, req)); -let hop2 = hop1 | join(friend, (_, req, (g, _)) -> (g, req)); +let hop1 = request | map(((rid, who, max), ()) -> (who, (rid, who, max))) | join(person, (who, (req, _)) -> (who, req)) | join(friend, (_, (req, (f, _))) -> (f, req)); +let hop2 = hop1 | join(friend, (_, (req, (g, _))) -> (g, req)); let friends = hop1 + hop2 | filter((node, (_, who, _)) -> (node, who) ne) | map((node, (rid, _, max)) -> ((node, rid, max), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((node, rid, max), ()) -> (node, (rid, max))); -let ahead = friends | join(newest, (f, (rid, max), neg) -> (f, (rid, max, neg neg))); -let fast = ahead | filter((_, (_, max, last)) -> (last, max) lt) | join(latest, (f, (rid, _, _), (neg, id)) -> (rid, ((neg, id), f))); -let slow = ahead | filter((_, (_, max, last)) -> (last, max) ge) | join(by_author, (f, (rid, max, _), (neg, id)) -> ((rid, f), ((neg, id), max))) | filter((_, ((neg, _), max)) -> (neg neg, max) lt) | map((key, (order, _)) -> (key, order)) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten) | map(((rid, f), (_, order)) -> (rid, (order, f))); +let ahead = friends | join(newest, (f, ((rid, max), neg)) -> (f, (rid, max, neg neg))); +let fast = ahead | filter((_, (_, max, last)) -> (last, max) lt) | join(latest, (f, ((rid, _, _), (neg, id))) -> (rid, ((neg, id), f))); +let slow = ahead | filter((_, (_, max, last)) -> (last, max) ge) | join(by_author, (f, ((rid, max, _), (neg, id))) -> ((rid, f), ((neg, id), max))) | filter((_, ((neg, _), max)) -> (neg neg, max) lt) | map((key, (order, _)) -> (key, order)) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten) | map(((rid, f), (_, order)) -> (rid, (order, f))); let top = fast + slow | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); -let answer = top | map((rid, (rank, ((neg, id), f))) -> (id, (rid, rank, neg neg, f))) | join(message, (id, (rid, rank, created, f), (_, _, _, _, content, image, _, _, _, _, _, _, _)) -> (f, (rid, rank, id, ((content len, 0) gt, content, image) select, created))) | join(person, (f, (rid, rank, id, body, created), (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, rank), (f, first, last, id, body, created))); +let answer = top | map((rid, (rank, ((neg, id), f))) -> (id, (rid, rank, neg neg, f))) | join(message, (id, ((rid, rank, created, f), (_, _, _, _, content, image, _, _, _, _, _, _, _))) -> (f, (rid, rank, id, ((content len, 0) gt, content, image) select, created))) | join(person, (f, ((rid, rank, id, body, created), (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, rank), (f, first, last, id, body, created))); output "answer" = answer | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic10_oneoff.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic10_oneoff.ddp index 3c4637201..3e0bf0b4a 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic10_oneoff.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic10_oneoff.ddp @@ -9,9 +9,9 @@ let likes = import "ic10s::likes" : ((int, int), ()); let hop1_map = request | map(((rid, who, month), ()) -> (who, (rid, who, month))); let hop1_idx = hop1_map | arrange; let friend_idx = friend | arrange; -let hop1 = hop1_idx | join(friend_idx, (_, req, (f, _)) -> (f, (req, 1))); +let hop1 = hop1_idx | join(friend_idx, (_, (req, (f, _))) -> (f, (req, 1))); let hop1_idx2 = hop1 | arrange; -let hop2 = hop1_idx2 | join(friend_idx, (_, (req, _), (g, _)) -> (g, (req, 2))); +let hop2 = hop1_idx2 | join(friend_idx, (_, ((req, _), (g, _))) -> (g, (req, 2))); let twohop_rows = hop1 + hop2; let twohop_rows2 = twohop_rows | filter((node, ((_, who, _), _)) -> (node, who) ne) | map((node, ((rid, who, month), d)) -> ((rid, who, month, node), d)); let twohop_idx = twohop_rows2 | arrange; @@ -19,19 +19,19 @@ let twohop_reduce = twohop_idx | reduce(limit 1, (_, vals) -> let first = vals h let cands_rows = twohop_reduce | filter((_, d) -> (d, 2) eq) | map(((rid, who, month, node), _) -> (node, (rid, who, month))); let cands_idx = cands_rows | arrange; let people_idx = people | arrange; -let cands_join = cands_idx | join(people_idx, (node, (rid, who, month), (first, last, gender, town, bmonth, bday)) -> ((rid, who, node), (first, last, gender, town, month, bmonth, bday))); +let cands_join = cands_idx | join(people_idx, (node, ((rid, who, month), (first, last, gender, town, bmonth, bday))) -> ((rid, who, node), (first, last, gender, town, month, bmonth, bday))); let cands = cands_join | filter((_, (_, _, _, _, month, bmonth, bday)) -> (((bmonth, month) eq, (bday, 21) ge) all, ((bmonth, ((month, 12) eq, 1, (month, 1) add) select) eq, (bday, 22) lt) all) any) | map((key, (first, last, gender, town, _, _, _)) -> (key, (first, last, gender, town))); let wrote_map = cands | map(((rid, who, node), _) -> (node, (rid, who))); let wrote_idx = wrote_map | arrange; let posts_idx = posts | arrange; -let wrote = wrote_idx | join(posts_idx, (node, (rid, who), m) -> ((rid, who, node, m), ())); +let wrote = wrote_idx | join(posts_idx, (node, ((rid, who), m)) -> ((rid, who, node, m), ())); let liked_map = wrote | map(((rid, who, node, m), ()) -> (m, (rid, who, node))); let liked_idx = liked_map | arrange; let tags_of_idx = tags_of | arrange; -let liked_join = liked_idx | join(tags_of_idx, (m, (rid, who, node), t) -> ((who, t), (rid, node, m))); +let liked_join = liked_idx | join(tags_of_idx, (m, ((rid, who, node), t)) -> ((who, t), (rid, node, m))); let liked_idx2 = liked_join | arrange; let likes_idx = likes | arrange; -let liked_join2 = liked_idx2 | join(likes_idx, ((who, _), (rid, node, m), ()) -> ((rid, who, node, m), ())); +let liked_join2 = liked_idx2 | join(likes_idx, ((who, _), ((rid, node, m), ())) -> ((rid, who, node, m), ())); let liked_idx3 = liked_join2 | arrange; let liked = liked_idx3 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); let points_map = liked | map(((rid, who, node, _), ()) -> ((rid, who, node), 1)); @@ -45,14 +45,14 @@ let summed_idx = summed_rows | arrange; let summed = summed_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); let keys = cands | map((key, _) -> (key, ())); let keys_idx = keys | arrange; -let score_join = keys_idx | join(summed, (key, (), _) -> (key, ())); +let score_join = keys_idx | join(summed, (key, ((), _)) -> (key, ())); let score_neg = score_join | negate; let score_rows = keys + score_neg; let score_map = score_rows | map((key, ()) -> (key, 0)); let score = summed + score_map; let score_idx = score | arrange; let cands_idx2 = cands | arrange; -let ranked_join = score_idx | join(cands_idx2, ((rid, _, node), s, (first, last, gender, town)) -> (rid, ((s neg, node), (first, last, s, gender, town)))); +let ranked_join = score_idx | join(cands_idx2, ((rid, _, node), (s, (first, last, gender, town))) -> (rid, ((s neg, node), (first, last, s, gender, town)))); let ranked_idx = ranked_join | arrange; let ranked = ranked_idx | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((_, node), (first, last, s, gender, town)))) -> ((rid, rank), (node, first, last, s, gender, town))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic10_oneoff.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/ldbc_ic10_oneoff.lowered--optimized.diff index 39451088a..625d24a76 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic10_oneoff.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic10_oneoff.lowered--optimized.diff @@ -15,11 +15,11 @@ let likes = import "ic10s::likes" : ((int, int), ()); @@ -15,20 +11,16 @@ let friend_idx = friend | arrange; - let hop1 = hop1_idx | join(friend_idx, (_, req, (f, _)) -> (f, (req, 1))); + let hop1 = hop1_idx | join(friend_idx, (_, (req, (f, _))) -> (f, (req, 1))); let hop1_idx2 = hop1 | arrange; -let friend_idx2 = friend | arrange; --let hop2 = hop1_idx2 | join(friend_idx2, (_, (req, _), (g, _)) -> (g, (req, 2))); -+let hop2 = hop1_idx2 | join(friend_idx, (_, (req, _), (g, _)) -> (g, (req, 2))); +-let hop2 = hop1_idx2 | join(friend_idx2, (_, ((req, _), (g, _))) -> (g, (req, 2))); ++let hop2 = hop1_idx2 | join(friend_idx, (_, ((req, _), (g, _))) -> (g, (req, 2))); let twohop_rows = hop1 + hop2; -let twohop_filter = twohop_rows | filter((node, ((_, who, _), _)) -> (node, who) ne); -let twohop_map = twohop_filter | map((node, ((rid, who, month), d)) -> ((rid, who, month, node), d)); @@ -33,7 +33,7 @@ +let cands_rows = twohop_reduce | filter((_, d) -> (d, 2) eq) | map(((rid, who, month, node), _) -> (node, (rid, who, month))); +let cands_idx = cands_rows | arrange; let people_idx = people | arrange; - let cands_join = cands_idx | join(people_idx, (node, (rid, who, month), (first, last, gender, town, bmonth, bday)) -> ((rid, who, node), (first, last, gender, town, month, bmonth, bday))); + let cands_join = cands_idx | join(people_idx, (node, ((rid, who, month), (first, last, gender, town, bmonth, bday))) -> ((rid, who, node), (first, last, gender, town, month, bmonth, bday))); -let cands_filter = cands_join | filter((_, (_, _, _, _, month, bmonth, bday)) -> (((bmonth, month) eq, (bday, 21) ge) all, ((bmonth, ((month, 12) eq, 1, (month, 1) add) select) eq, (bday, 22) lt) all) any); -let cands = cands_filter | map((key, (first, last, gender, town, _, _, _)) -> (key, (first, last, gender, town))); +let cands = cands_join | filter((_, (_, _, _, _, month, bmonth, bday)) -> (((bmonth, month) eq, (bday, 21) ge) all, ((bmonth, ((month, 12) eq, 1, (month, 1) add) select) eq, (bday, 22) lt) all) any) | map((key, (first, last, gender, town, _, _, _)) -> (key, (first, last, gender, town))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic10_oneoff.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic10_oneoff.parsed--lowered.diff index 7f6b1ce1a..d0ced800e 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic10_oneoff.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic10_oneoff.parsed--lowered.diff @@ -10,21 +10,21 @@ let people = import "ic10s::people" : (int, (List(int), List(int), List(int), List(int), int, int)); let posts = import "ic10s::posts" : (int, int); let likes = import "ic10s::likes" : ((int, int), ()); --let hop1 = request | map(((rid, who, month), ()) -> (who, (rid, who, month))) | join(friend, (_, req, (f, _)) -> (f, (req, 1))); --let hop2 = hop1 | join(friend, (_, (req, _), (g, _)) -> (g, (req, 2))); +-let hop1 = request | map(((rid, who, month), ()) -> (who, (rid, who, month))) | join(friend, (_, (req, (f, _))) -> (f, (req, 1))); +-let hop2 = hop1 | join(friend, (_, ((req, _), (g, _))) -> (g, (req, 2))); -let twohop = hop1 + hop2 | filter((node, ((_, who, _), _)) -> (node, who) ne) | map((node, ((rid, who, month), d)) -> ((rid, who, month, node), d)) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist) | filter((_, d) -> (d, 2) eq); --let cands = twohop | map(((rid, who, month, node), _) -> (node, (rid, who, month))) | join(people, (node, (rid, who, month), (first, last, gender, town, bmonth, bday)) -> ((rid, who, node), (first, last, gender, town, month, bmonth, bday))) | filter((_, (_, _, _, _, month, bmonth, bday)) -> (((bmonth, month) eq, (bday, 21) ge) all, ((bmonth, ((month, 12) eq, 1, (month, 1) add) select) eq, (bday, 22) lt) all) any) | map((key, (first, last, gender, town, _, _, _)) -> (key, (first, last, gender, town))); --let wrote = cands | map(((rid, who, node), _) -> (node, (rid, who))) | join(posts, (node, (rid, who), m) -> ((rid, who, node, m), ())); --let liked = wrote | map(((rid, who, node, m), ()) -> (m, (rid, who, node))) | join(tags_of, (m, (rid, who, node), t) -> ((who, t), (rid, node, m))) | join(likes, ((who, _), (rid, node, m), ()) -> ((rid, who, node, m), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); +-let cands = twohop | map(((rid, who, month, node), _) -> (node, (rid, who, month))) | join(people, (node, ((rid, who, month), (first, last, gender, town, bmonth, bday))) -> ((rid, who, node), (first, last, gender, town, month, bmonth, bday))) | filter((_, (_, _, _, _, month, bmonth, bday)) -> (((bmonth, month) eq, (bday, 21) ge) all, ((bmonth, ((month, 12) eq, 1, (month, 1) add) select) eq, (bday, 22) lt) all) any) | map((key, (first, last, gender, town, _, _, _)) -> (key, (first, last, gender, town))); +-let wrote = cands | map(((rid, who, node), _) -> (node, (rid, who))) | join(posts, (node, ((rid, who), m)) -> ((rid, who, node, m), ())); +-let liked = wrote | map(((rid, who, node, m), ()) -> (m, (rid, who, node))) | join(tags_of, (m, ((rid, who, node), t)) -> ((who, t), (rid, node, m))) | join(likes, ((who, _), ((rid, node, m), ())) -> ((rid, who, node, m), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); -let points = (liked | map(((rid, who, node, _), ()) -> ((rid, who, node), 1))) + (wrote + (liked | negate) | map(((rid, who, node, _), ()) -> ((rid, who, node), -1))); -let summed = points | join_function((key, p) -> ((key, ()), 0, p) enlist) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +let hop1_map = request | map(((rid, who, month), ()) -> (who, (rid, who, month))); +let hop1_idx = hop1_map | arrange; +let friend_idx = friend | arrange; -+let hop1 = hop1_idx | join(friend_idx, (_, req, (f, _)) -> (f, (req, 1))); ++let hop1 = hop1_idx | join(friend_idx, (_, (req, (f, _))) -> (f, (req, 1))); +let hop1_idx2 = hop1 | arrange; +let friend_idx2 = friend | arrange; -+let hop2 = hop1_idx2 | join(friend_idx2, (_, (req, _), (g, _)) -> (g, (req, 2))); ++let hop2 = hop1_idx2 | join(friend_idx2, (_, ((req, _), (g, _))) -> (g, (req, 2))); +let twohop_rows = hop1 + hop2; +let twohop_filter = twohop_rows | filter((node, ((_, who, _), _)) -> (node, who) ne); +let twohop_map = twohop_filter | map((node, ((rid, who, month), d)) -> ((rid, who, month, node), d)); @@ -34,20 +34,20 @@ +let cands_map = twohop | map(((rid, who, month, node), _) -> (node, (rid, who, month))); +let cands_idx = cands_map | arrange; +let people_idx = people | arrange; -+let cands_join = cands_idx | join(people_idx, (node, (rid, who, month), (first, last, gender, town, bmonth, bday)) -> ((rid, who, node), (first, last, gender, town, month, bmonth, bday))); ++let cands_join = cands_idx | join(people_idx, (node, ((rid, who, month), (first, last, gender, town, bmonth, bday))) -> ((rid, who, node), (first, last, gender, town, month, bmonth, bday))); +let cands_filter = cands_join | filter((_, (_, _, _, _, month, bmonth, bday)) -> (((bmonth, month) eq, (bday, 21) ge) all, ((bmonth, ((month, 12) eq, 1, (month, 1) add) select) eq, (bday, 22) lt) all) any); +let cands = cands_filter | map((key, (first, last, gender, town, _, _, _)) -> (key, (first, last, gender, town))); +let wrote_map = cands | map(((rid, who, node), _) -> (node, (rid, who))); +let wrote_idx = wrote_map | arrange; +let posts_idx = posts | arrange; -+let wrote = wrote_idx | join(posts_idx, (node, (rid, who), m) -> ((rid, who, node, m), ())); ++let wrote = wrote_idx | join(posts_idx, (node, ((rid, who), m)) -> ((rid, who, node, m), ())); +let liked_map = wrote | map(((rid, who, node, m), ()) -> (m, (rid, who, node))); +let liked_idx = liked_map | arrange; +let tags_of_idx = tags_of | arrange; -+let liked_join = liked_idx | join(tags_of_idx, (m, (rid, who, node), t) -> ((who, t), (rid, node, m))); ++let liked_join = liked_idx | join(tags_of_idx, (m, ((rid, who, node), t)) -> ((who, t), (rid, node, m))); +let liked_idx2 = liked_join | arrange; +let likes_idx = likes | arrange; -+let liked_join2 = liked_idx2 | join(likes_idx, ((who, _), (rid, node, m), ()) -> ((rid, who, node, m), ())); ++let liked_join2 = liked_idx2 | join(likes_idx, ((who, _), ((rid, node, m), ())) -> ((rid, who, node, m), ())); +let liked_idx3 = liked_join2 | arrange; +let liked = liked_idx3 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); +let points_map = liked | map(((rid, who, node, _), ()) -> ((rid, who, node), 1)); @@ -59,17 +59,17 @@ +let summed_idx = summed_rows | arrange; +let summed = summed_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); let keys = cands | map((key, _) -> (key, ())); --let score = summed + (keys + (keys | join(summed, (key, (), _) -> (key, ())) | negate) | map((key, ()) -> (key, 0))); --let ranked = score | join(cands, ((rid, _, node), s, (first, last, gender, town)) -> (rid, ((s neg, node), (first, last, s, gender, town)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); +-let score = summed + (keys + (keys | join(summed, (key, ((), _)) -> (key, ())) | negate) | map((key, ()) -> (key, 0))); +-let ranked = score | join(cands, ((rid, _, node), (s, (first, last, gender, town))) -> (rid, ((s neg, node), (first, last, s, gender, town)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); +let keys_idx = keys | arrange; -+let score_join = keys_idx | join(summed, (key, (), _) -> (key, ())); ++let score_join = keys_idx | join(summed, (key, ((), _)) -> (key, ())); +let score_neg = score_join | negate; +let score_rows = keys + score_neg; +let score_map = score_rows | map((key, ()) -> (key, 0)); +let score = summed + score_map; +let score_idx = score | arrange; +let cands_idx2 = cands | arrange; -+let ranked_join = score_idx | join(cands_idx2, ((rid, _, node), s, (first, last, gender, town)) -> (rid, ((s neg, node), (first, last, s, gender, town)))); ++let ranked_join = score_idx | join(cands_idx2, ((rid, _, node), (s, (first, last, gender, town))) -> (rid, ((s neg, node), (first, last, s, gender, town)))); +let ranked_idx = ranked_join | arrange; +let ranked = ranked_idx | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((_, node), (first, last, s, gender, town)))) -> ((rid, rank), (node, first, last, s, gender, town))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic10_oneoff.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic10_oneoff.parsed.ddp index 08849901a..1d731ea74 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic10_oneoff.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic10_oneoff.parsed.ddp @@ -8,16 +8,16 @@ let interest = import "graph::interest.by_person" : (int, int); let people = import "ic10s::people" : (int, (List(int), List(int), List(int), List(int), int, int)); let posts = import "ic10s::posts" : (int, int); let likes = import "ic10s::likes" : ((int, int), ()); -let hop1 = request | map(((rid, who, month), ()) -> (who, (rid, who, month))) | join(friend, (_, req, (f, _)) -> (f, (req, 1))); -let hop2 = hop1 | join(friend, (_, (req, _), (g, _)) -> (g, (req, 2))); +let hop1 = request | map(((rid, who, month), ()) -> (who, (rid, who, month))) | join(friend, (_, (req, (f, _))) -> (f, (req, 1))); +let hop2 = hop1 | join(friend, (_, ((req, _), (g, _))) -> (g, (req, 2))); let twohop = hop1 + hop2 | filter((node, ((_, who, _), _)) -> (node, who) ne) | map((node, ((rid, who, month), d)) -> ((rid, who, month, node), d)) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist) | filter((_, d) -> (d, 2) eq); -let cands = twohop | map(((rid, who, month, node), _) -> (node, (rid, who, month))) | join(people, (node, (rid, who, month), (first, last, gender, town, bmonth, bday)) -> ((rid, who, node), (first, last, gender, town, month, bmonth, bday))) | filter((_, (_, _, _, _, month, bmonth, bday)) -> (((bmonth, month) eq, (bday, 21) ge) all, ((bmonth, ((month, 12) eq, 1, (month, 1) add) select) eq, (bday, 22) lt) all) any) | map((key, (first, last, gender, town, _, _, _)) -> (key, (first, last, gender, town))); -let wrote = cands | map(((rid, who, node), _) -> (node, (rid, who))) | join(posts, (node, (rid, who), m) -> ((rid, who, node, m), ())); -let liked = wrote | map(((rid, who, node, m), ()) -> (m, (rid, who, node))) | join(tags_of, (m, (rid, who, node), t) -> ((who, t), (rid, node, m))) | join(likes, ((who, _), (rid, node, m), ()) -> ((rid, who, node, m), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); +let cands = twohop | map(((rid, who, month, node), _) -> (node, (rid, who, month))) | join(people, (node, ((rid, who, month), (first, last, gender, town, bmonth, bday))) -> ((rid, who, node), (first, last, gender, town, month, bmonth, bday))) | filter((_, (_, _, _, _, month, bmonth, bday)) -> (((bmonth, month) eq, (bday, 21) ge) all, ((bmonth, ((month, 12) eq, 1, (month, 1) add) select) eq, (bday, 22) lt) all) any) | map((key, (first, last, gender, town, _, _, _)) -> (key, (first, last, gender, town))); +let wrote = cands | map(((rid, who, node), _) -> (node, (rid, who))) | join(posts, (node, ((rid, who), m)) -> ((rid, who, node, m), ())); +let liked = wrote | map(((rid, who, node, m), ()) -> (m, (rid, who, node))) | join(tags_of, (m, ((rid, who, node), t)) -> ((who, t), (rid, node, m))) | join(likes, ((who, _), ((rid, node, m), ())) -> ((rid, who, node, m), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); let points = (liked | map(((rid, who, node, _), ()) -> ((rid, who, node), 1))) + (wrote + (liked | negate) | map(((rid, who, node, _), ()) -> ((rid, who, node), -1))); let summed = points | join_function((key, p) -> ((key, ()), 0, p) enlist) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); let keys = cands | map((key, _) -> (key, ())); -let score = summed + (keys + (keys | join(summed, (key, (), _) -> (key, ())) | negate) | map((key, ()) -> (key, 0))); -let ranked = score | join(cands, ((rid, _, node), s, (first, last, gender, town)) -> (rid, ((s neg, node), (first, last, s, gender, town)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); +let score = summed + (keys + (keys | join(summed, (key, ((), _)) -> (key, ())) | negate) | map((key, ()) -> (key, 0))); +let ranked = score | join(cands, ((rid, _, node), (s, (first, last, gender, town))) -> (rid, ((s neg, node), (first, last, s, gender, town)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((_, node), (first, last, s, gender, town)))) -> ((rid, rank), (node, first, last, s, gender, town))); output "answer" = answer | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic10_standing.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic10_standing.ddp index 697d6ee13..3118650e0 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic10_standing.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic10_standing.ddp @@ -7,7 +7,7 @@ let interest = import "graph::interest.by_person" : (int, int); let people_map = person | map((id, (first, last, gender, _, _, _, _, home, bmonth, bday, _)) -> (home, (id, first, last, gender, bmonth, bday))); let people_idx = people_map | arrange; let place_idx = place | arrange; -let people = people_idx | join(place_idx, (_, (id, first, last, gender, bmonth, bday), (town, _, _)) -> (id, (first, last, gender, town, bmonth, bday))); +let people = people_idx | join(place_idx, (_, ((id, first, last, gender, bmonth, bday), (town, _, _))) -> (id, (first, last, gender, town, bmonth, bday))); let posts = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 0) eq) | map((id, (_, _, who, _, _, _, _, _, _, _, _, _, _)) -> (who, id)); let likes = interest | map((p, t) -> ((p, t), ())); let people_idx2 = people | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic10_standing.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/ldbc_ic10_standing.lowered--optimized.diff index a30233b3a..2cf4cfcdb 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic10_standing.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic10_standing.lowered--optimized.diff @@ -11,7 +11,7 @@ let people_map = person | map((id, (first, last, gender, _, _, _, _, home, bmonth, bday, _)) -> (home, (id, first, last, gender, bmonth, bday))); let people_idx = people_map | arrange; let place_idx = place | arrange; - let people = people_idx | join(place_idx, (_, (id, first, last, gender, bmonth, bday), (town, _, _)) -> (id, (first, last, gender, town, bmonth, bday))); + let people = people_idx | join(place_idx, (_, ((id, first, last, gender, bmonth, bday), (town, _, _))) -> (id, (first, last, gender, town, bmonth, bday))); -let posts_filter = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 0) eq); -let posts = posts_filter | map((id, (_, _, who, _, _, _, _, _, _, _, _, _, _)) -> (who, id)); +let posts = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 0) eq) | map((id, (_, _, who, _, _, _, _, _, _, _, _, _, _)) -> (who, id)); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic10_standing.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic10_standing.parsed--lowered.diff index 0c15102e6..6c5a50030 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic10_standing.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic10_standing.parsed--lowered.diff @@ -9,12 +9,12 @@ let place = import "graph::place" : (int, (List(int), List(int), int)); let tags_of = import "graph::mtag.by_message" : (int, int); let interest = import "graph::interest.by_person" : (int, int); --let people = person | map((id, (first, last, gender, _, _, _, _, home, bmonth, bday, _)) -> (home, (id, first, last, gender, bmonth, bday))) | join(place, (_, (id, first, last, gender, bmonth, bday), (town, _, _)) -> (id, (first, last, gender, town, bmonth, bday))); +-let people = person | map((id, (first, last, gender, _, _, _, _, home, bmonth, bday, _)) -> (home, (id, first, last, gender, bmonth, bday))) | join(place, (_, ((id, first, last, gender, bmonth, bday), (town, _, _))) -> (id, (first, last, gender, town, bmonth, bday))); -let posts = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 0) eq) | map((id, (_, _, who, _, _, _, _, _, _, _, _, _, _)) -> (who, id)); +let people_map = person | map((id, (first, last, gender, _, _, _, _, home, bmonth, bday, _)) -> (home, (id, first, last, gender, bmonth, bday))); +let people_idx = people_map | arrange; +let place_idx = place | arrange; -+let people = people_idx | join(place_idx, (_, (id, first, last, gender, bmonth, bday), (town, _, _)) -> (id, (first, last, gender, town, bmonth, bday))); ++let people = people_idx | join(place_idx, (_, ((id, first, last, gender, bmonth, bday), (town, _, _))) -> (id, (first, last, gender, town, bmonth, bday))); +let posts_filter = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 0) eq); +let posts = posts_filter | map((id, (_, _, who, _, _, _, _, _, _, _, _, _, _)) -> (who, id)); let likes = interest | map((p, t) -> ((p, t), ())); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic10_standing.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic10_standing.parsed.ddp index bbb45625d..24ca78249 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic10_standing.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic10_standing.parsed.ddp @@ -4,7 +4,7 @@ let friend = import "graph::friend" : (int, (int, int)); let place = import "graph::place" : (int, (List(int), List(int), int)); let tags_of = import "graph::mtag.by_message" : (int, int); let interest = import "graph::interest.by_person" : (int, int); -let people = person | map((id, (first, last, gender, _, _, _, _, home, bmonth, bday, _)) -> (home, (id, first, last, gender, bmonth, bday))) | join(place, (_, (id, first, last, gender, bmonth, bday), (town, _, _)) -> (id, (first, last, gender, town, bmonth, bday))); +let people = person | map((id, (first, last, gender, _, _, _, _, home, bmonth, bday, _)) -> (home, (id, first, last, gender, bmonth, bday))) | join(place, (_, ((id, first, last, gender, bmonth, bday), (town, _, _))) -> (id, (first, last, gender, town, bmonth, bday))); let posts = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 0) eq) | map((id, (_, _, who, _, _, _, _, _, _, _, _, _, _)) -> (who, id)); let likes = interest | map((p, t) -> ((p, t), ())); output "people" = people | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic11_oneoff.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic11_oneoff.ddp index 767a68f23..602c30283 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic11_oneoff.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic11_oneoff.ddp @@ -9,12 +9,12 @@ let place = import "graph::place" : (int, (List(int), List(int), int)); let hop1_map = request | map(((rid, who, country, year), ()) -> (who, (rid, who, country, year))); let hop1_idx = hop1_map | arrange; let person_idx = person | arrange; -let hop1_join = hop1_idx | join(person_idx, (who, req, _) -> (who, req)); +let hop1_join = hop1_idx | join(person_idx, (who, (req, _)) -> (who, req)); let hop1_idx2 = hop1_join | arrange; let friend_idx = friend | arrange; -let hop1 = hop1_idx2 | join(friend_idx, (_, req, (f, _)) -> (f, req)); +let hop1 = hop1_idx2 | join(friend_idx, (_, (req, (f, _))) -> (f, req)); let hop1_idx3 = hop1 | arrange; -let hop2 = hop1_idx3 | join(friend_idx, (_, req, (g, _)) -> (g, req)); +let hop2 = hop1_idx3 | join(friend_idx, (_, (req, (g, _))) -> (g, req)); let circle_rows = hop1 + hop2; let circle_rows2 = circle_rows | filter((node, (_, who, _, _)) -> (node, who) ne) | map((node, (rid, _, country, year)) -> ((node, rid, country, year), ())); let circle_idx = circle_rows2 | arrange; @@ -22,20 +22,20 @@ let circle = circle_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); let jobs_map = circle | map(((node, rid, country, year), ()) -> (node, (rid, country, year))); let jobs_idx = jobs_map | arrange; let work_idx = work | arrange; -let jobs_join = jobs_idx | join(work_idx, (node, (rid, country, year), (at, since)) -> (at, (rid, node, country, year, since))); +let jobs_join = jobs_idx | join(work_idx, (node, ((rid, country, year), (at, since))) -> (at, (rid, node, country, year, since))); let jobs_filter = jobs_join | filter((_, (_, _, _, year, since)) -> (since, year) lt); let jobs_idx2 = jobs_filter | arrange; let org_idx = org | arrange; -let jobs_join2 = jobs_idx2 | join(org_idx, (_, (rid, node, country, _, since), (company, _, located)) -> (located, ((rid, since, node), (company, country)))); +let jobs_join2 = jobs_idx2 | join(org_idx, (_, ((rid, node, country, _, since), (company, _, located))) -> (located, ((rid, since, node), (company, country)))); let jobs_idx3 = jobs_join2 | arrange; let place_idx = place | arrange; -let jobs_join3 = jobs_idx3 | join(place_idx, (_, (key, (company, country)), (there, _, _)) -> (key, (company, country, there))); +let jobs_join3 = jobs_idx3 | join(place_idx, (_, ((key, (company, country)), (there, _, _))) -> (key, (company, country, there))); let jobs = jobs_join3 | filter((_, (_, country, there)) -> (country, there) eq) | map((key, (company, _, _)) -> (key, company)); let jobs_idx4 = jobs | arrange; let named_reduce = jobs_idx4 | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); let ranked_rows = named_reduce | flatmap((k, entries) -> (k, ((0, entries len) range, entries) zip) cap_list) | map(((rid, since, node), (pos, company)) -> (node, (rid, since, pos, company))); let ranked_idx = ranked_rows | arrange; -let ranked_join = ranked_idx | join(person_idx, (node, (rid, since, pos, company), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, ((since, node, pos neg), (first, last, company)))); +let ranked_join = ranked_idx | join(person_idx, (node, ((rid, since, pos, company), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((since, node, pos neg), (first, last, company)))); let ranked_idx2 = ranked_join | arrange; let ranked = ranked_idx2 | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((since, node, _), (first, last, company)))) -> ((rid, rank), (node, first, last, company, since))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic11_oneoff.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/ldbc_ic11_oneoff.lowered--optimized.diff index 82d199ffd..4114f9d51 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic11_oneoff.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic11_oneoff.lowered--optimized.diff @@ -2,11 +2,11 @@ +++ optimized @@ -14,12 +14,10 @@ let friend_idx = friend | arrange; - let hop1 = hop1_idx2 | join(friend_idx, (_, req, (f, _)) -> (f, req)); + let hop1 = hop1_idx2 | join(friend_idx, (_, (req, (f, _))) -> (f, req)); let hop1_idx3 = hop1 | arrange; -let friend_idx2 = friend | arrange; --let hop2 = hop1_idx3 | join(friend_idx2, (_, req, (g, _)) -> (g, req)); -+let hop2 = hop1_idx3 | join(friend_idx, (_, req, (g, _)) -> (g, req)); +-let hop2 = hop1_idx3 | join(friend_idx2, (_, (req, (g, _))) -> (g, req)); ++let hop2 = hop1_idx3 | join(friend_idx, (_, (req, (g, _))) -> (g, req)); let circle_rows = hop1 + hop2; -let circle_filter = circle_rows | filter((node, (_, who, _, _)) -> (node, who) ne); -let circle_map = circle_filter | map((node, (rid, _, country, year)) -> ((node, rid, country, year), ())); @@ -19,7 +19,7 @@ @@ -32,15 +30,12 @@ let jobs_idx3 = jobs_join2 | arrange; let place_idx = place | arrange; - let jobs_join3 = jobs_idx3 | join(place_idx, (_, (key, (company, country)), (there, _, _)) -> (key, (company, country, there))); + let jobs_join3 = jobs_idx3 | join(place_idx, (_, ((key, (company, country)), (there, _, _))) -> (key, (company, country, there))); -let jobs_filter2 = jobs_join3 | filter((_, (_, country, there)) -> (country, there) eq); -let jobs = jobs_filter2 | map((key, (company, _, _)) -> (key, company)); +let jobs = jobs_join3 | filter((_, (_, country, there)) -> (country, there) eq) | map((key, (company, _, _)) -> (key, company)); @@ -29,10 +29,10 @@ -let ranked_map = named | map(((rid, since, node), (pos, company)) -> (node, (rid, since, pos, company))); -let ranked_idx = ranked_map | arrange; -let person_idx2 = person | arrange; --let ranked_join = ranked_idx | join(person_idx2, (node, (rid, since, pos, company), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, ((since, node, pos neg), (first, last, company)))); +-let ranked_join = ranked_idx | join(person_idx2, (node, ((rid, since, pos, company), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((since, node, pos neg), (first, last, company)))); +let ranked_rows = named_reduce | flatmap((k, entries) -> (k, ((0, entries len) range, entries) zip) cap_list) | map(((rid, since, node), (pos, company)) -> (node, (rid, since, pos, company))); +let ranked_idx = ranked_rows | arrange; -+let ranked_join = ranked_idx | join(person_idx, (node, (rid, since, pos, company), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, ((since, node, pos neg), (first, last, company)))); ++let ranked_join = ranked_idx | join(person_idx, (node, ((rid, since, pos, company), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((since, node, pos neg), (first, last, company)))); let ranked_idx2 = ranked_join | arrange; let ranked = ranked_idx2 | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((since, node, _), (first, last, company)))) -> ((rid, rank), (node, first, last, company, since))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic11_oneoff.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic11_oneoff.parsed--lowered.diff index dab5d2102..84503ed77 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic11_oneoff.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic11_oneoff.parsed--lowered.diff @@ -9,22 +9,22 @@ let work = import "graph::work.by_person" : (int, (int, int)); let org = import "graph::org" : (int, (List(int), List(int), int)); let place = import "graph::place" : (int, (List(int), List(int), int)); --let hop1 = request | map(((rid, who, country, year), ()) -> (who, (rid, who, country, year))) | join(person, (who, req, _) -> (who, req)) | join(friend, (_, req, (f, _)) -> (f, req)); --let hop2 = hop1 | join(friend, (_, req, (g, _)) -> (g, req)); +-let hop1 = request | map(((rid, who, country, year), ()) -> (who, (rid, who, country, year))) | join(person, (who, (req, _)) -> (who, req)) | join(friend, (_, (req, (f, _))) -> (f, req)); +-let hop2 = hop1 | join(friend, (_, (req, (g, _))) -> (g, req)); -let circle = hop1 + hop2 | filter((node, (_, who, _, _)) -> (node, who) ne) | map((node, (rid, _, country, year)) -> ((node, rid, country, year), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); --let jobs = circle | map(((node, rid, country, year), ()) -> (node, (rid, country, year))) | join(work, (node, (rid, country, year), (at, since)) -> (at, (rid, node, country, year, since))) | filter((_, (_, _, _, year, since)) -> (since, year) lt) | join(org, (_, (rid, node, country, _, since), (company, _, located)) -> (located, ((rid, since, node), (company, country)))) | join(place, (_, (key, (company, country)), (there, _, _)) -> (key, (company, country, there))) | filter((_, (_, country, there)) -> (country, there) eq) | map((key, (company, _, _)) -> (key, company)); +-let jobs = circle | map(((node, rid, country, year), ()) -> (node, (rid, country, year))) | join(work, (node, ((rid, country, year), (at, since))) -> (at, (rid, node, country, year, since))) | filter((_, (_, _, _, year, since)) -> (since, year) lt) | join(org, (_, ((rid, node, country, _, since), (company, _, located))) -> (located, ((rid, since, node), (company, country)))) | join(place, (_, ((key, (company, country)), (there, _, _))) -> (key, (company, country, there))) | filter((_, (_, country, there)) -> (country, there) eq) | map((key, (company, _, _)) -> (key, company)); -let named = jobs | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist) | flatmap((k, entries) -> (k, ((0, entries len) range, entries) zip) cap_list); --let ranked = named | map(((rid, since, node), (pos, company)) -> (node, (rid, since, pos, company))) | join(person, (node, (rid, since, pos, company), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, ((since, node, pos neg), (first, last, company)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); +-let ranked = named | map(((rid, since, node), (pos, company)) -> (node, (rid, since, pos, company))) | join(person, (node, ((rid, since, pos, company), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((since, node, pos neg), (first, last, company)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); +let hop1_map = request | map(((rid, who, country, year), ()) -> (who, (rid, who, country, year))); +let hop1_idx = hop1_map | arrange; +let person_idx = person | arrange; -+let hop1_join = hop1_idx | join(person_idx, (who, req, _) -> (who, req)); ++let hop1_join = hop1_idx | join(person_idx, (who, (req, _)) -> (who, req)); +let hop1_idx2 = hop1_join | arrange; +let friend_idx = friend | arrange; -+let hop1 = hop1_idx2 | join(friend_idx, (_, req, (f, _)) -> (f, req)); ++let hop1 = hop1_idx2 | join(friend_idx, (_, (req, (f, _))) -> (f, req)); +let hop1_idx3 = hop1 | arrange; +let friend_idx2 = friend | arrange; -+let hop2 = hop1_idx3 | join(friend_idx2, (_, req, (g, _)) -> (g, req)); ++let hop2 = hop1_idx3 | join(friend_idx2, (_, (req, (g, _))) -> (g, req)); +let circle_rows = hop1 + hop2; +let circle_filter = circle_rows | filter((node, (_, who, _, _)) -> (node, who) ne); +let circle_map = circle_filter | map((node, (rid, _, country, year)) -> ((node, rid, country, year), ())); @@ -33,14 +33,14 @@ +let jobs_map = circle | map(((node, rid, country, year), ()) -> (node, (rid, country, year))); +let jobs_idx = jobs_map | arrange; +let work_idx = work | arrange; -+let jobs_join = jobs_idx | join(work_idx, (node, (rid, country, year), (at, since)) -> (at, (rid, node, country, year, since))); ++let jobs_join = jobs_idx | join(work_idx, (node, ((rid, country, year), (at, since))) -> (at, (rid, node, country, year, since))); +let jobs_filter = jobs_join | filter((_, (_, _, _, year, since)) -> (since, year) lt); +let jobs_idx2 = jobs_filter | arrange; +let org_idx = org | arrange; -+let jobs_join2 = jobs_idx2 | join(org_idx, (_, (rid, node, country, _, since), (company, _, located)) -> (located, ((rid, since, node), (company, country)))); ++let jobs_join2 = jobs_idx2 | join(org_idx, (_, ((rid, node, country, _, since), (company, _, located))) -> (located, ((rid, since, node), (company, country)))); +let jobs_idx3 = jobs_join2 | arrange; +let place_idx = place | arrange; -+let jobs_join3 = jobs_idx3 | join(place_idx, (_, (key, (company, country)), (there, _, _)) -> (key, (company, country, there))); ++let jobs_join3 = jobs_idx3 | join(place_idx, (_, ((key, (company, country)), (there, _, _))) -> (key, (company, country, there))); +let jobs_filter2 = jobs_join3 | filter((_, (_, country, there)) -> (country, there) eq); +let jobs = jobs_filter2 | map((key, (company, _, _)) -> (key, company)); +let jobs_idx4 = jobs | arrange; @@ -49,7 +49,7 @@ +let ranked_map = named | map(((rid, since, node), (pos, company)) -> (node, (rid, since, pos, company))); +let ranked_idx = ranked_map | arrange; +let person_idx2 = person | arrange; -+let ranked_join = ranked_idx | join(person_idx2, (node, (rid, since, pos, company), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, ((since, node, pos neg), (first, last, company)))); ++let ranked_join = ranked_idx | join(person_idx2, (node, ((rid, since, pos, company), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((since, node, pos neg), (first, last, company)))); +let ranked_idx2 = ranked_join | arrange; +let ranked = ranked_idx2 | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((since, node, _), (first, last, company)))) -> ((rid, rank), (node, first, last, company, since))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic11_oneoff.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic11_oneoff.parsed.ddp index a689e32ea..0f57f3ef2 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic11_oneoff.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic11_oneoff.parsed.ddp @@ -4,11 +4,11 @@ let friend = import "graph::friend" : (int, (int, int)); let work = import "graph::work.by_person" : (int, (int, int)); let org = import "graph::org" : (int, (List(int), List(int), int)); let place = import "graph::place" : (int, (List(int), List(int), int)); -let hop1 = request | map(((rid, who, country, year), ()) -> (who, (rid, who, country, year))) | join(person, (who, req, _) -> (who, req)) | join(friend, (_, req, (f, _)) -> (f, req)); -let hop2 = hop1 | join(friend, (_, req, (g, _)) -> (g, req)); +let hop1 = request | map(((rid, who, country, year), ()) -> (who, (rid, who, country, year))) | join(person, (who, (req, _)) -> (who, req)) | join(friend, (_, (req, (f, _))) -> (f, req)); +let hop2 = hop1 | join(friend, (_, (req, (g, _))) -> (g, req)); let circle = hop1 + hop2 | filter((node, (_, who, _, _)) -> (node, who) ne) | map((node, (rid, _, country, year)) -> ((node, rid, country, year), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); -let jobs = circle | map(((node, rid, country, year), ()) -> (node, (rid, country, year))) | join(work, (node, (rid, country, year), (at, since)) -> (at, (rid, node, country, year, since))) | filter((_, (_, _, _, year, since)) -> (since, year) lt) | join(org, (_, (rid, node, country, _, since), (company, _, located)) -> (located, ((rid, since, node), (company, country)))) | join(place, (_, (key, (company, country)), (there, _, _)) -> (key, (company, country, there))) | filter((_, (_, country, there)) -> (country, there) eq) | map((key, (company, _, _)) -> (key, company)); +let jobs = circle | map(((node, rid, country, year), ()) -> (node, (rid, country, year))) | join(work, (node, ((rid, country, year), (at, since))) -> (at, (rid, node, country, year, since))) | filter((_, (_, _, _, year, since)) -> (since, year) lt) | join(org, (_, ((rid, node, country, _, since), (company, _, located))) -> (located, ((rid, since, node), (company, country)))) | join(place, (_, ((key, (company, country)), (there, _, _))) -> (key, (company, country, there))) | filter((_, (_, country, there)) -> (country, there) eq) | map((key, (company, _, _)) -> (key, company)); let named = jobs | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist) | flatmap((k, entries) -> (k, ((0, entries len) range, entries) zip) cap_list); -let ranked = named | map(((rid, since, node), (pos, company)) -> (node, (rid, since, pos, company))) | join(person, (node, (rid, since, pos, company), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, ((since, node, pos neg), (first, last, company)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); +let ranked = named | map(((rid, since, node), (pos, company)) -> (node, (rid, since, pos, company))) | join(person, (node, ((rid, since, pos, company), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((since, node, pos neg), (first, last, company)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((since, node, _), (first, last, company)))) -> ((rid, rank), (node, first, last, company, since))); output "answer" = answer | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic12_oneoff.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic12_oneoff.ddp index d29e379bb..438613ce5 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic12_oneoff.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic12_oneoff.ddp @@ -13,33 +13,33 @@ let reply = import "ic12s::reply" : (int, (int, int)); let intag_map = request | map(((rid, _, name), ()) -> (name, rid)); let intag_idx = intag_map | arrange; let classes_idx = classes | arrange; -let intag_join = intag_idx | join(classes_idx, (_, rid, high) -> (high, rid)); +let intag_join = intag_idx | join(classes_idx, (_, (rid, high)) -> (high, rid)); let intag_idx2 = intag_join | arrange; let below_hl_idx = below_hl | arrange; -let intag_join2 = intag_idx2 | join(below_hl_idx, (_, rid, low) -> (low, rid)); +let intag_join2 = intag_idx2 | join(below_hl_idx, (_, (rid, low)) -> (low, rid)); let intag_idx3 = intag_join2 | arrange; let by_class_idx = by_class | arrange; -let intag_join3 = intag_idx3 | join(by_class_idx, (_, rid, t) -> (t, rid)); +let intag_join3 = intag_idx3 | join(by_class_idx, (_, (rid, t)) -> (t, rid)); let intag_idx4 = intag_join3 | arrange; let tag_idx = tag | arrange; -let intag_join4 = intag_idx4 | join(tag_idx, (t, rid, (name, _)) -> (t, (rid, name))); +let intag_join4 = intag_idx4 | join(tag_idx, (t, (rid, (name, _))) -> (t, (rid, name))); let intag_idx5 = intag_join4 | arrange; let mtag_idx = mtag | arrange; -let intag_join5 = intag_idx5 | join(mtag_idx, (_, (rid, name), m) -> (m, (rid, name))); +let intag_join5 = intag_idx5 | join(mtag_idx, (_, ((rid, name), m)) -> (m, (rid, name))); let intag_idx6 = intag_join5 | arrange; let message_idx = message | arrange; -let intag_join6 = intag_idx6 | join(message_idx, (m, (rid, name), (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> ((rid, m, name), kind)); +let intag_join6 = intag_idx6 | join(message_idx, (m, ((rid, name), (kind, _, _, _, _, _, _, _, _, _, _, _, _))) -> ((rid, m, name), kind)); let hits_map = request | map(((rid, who, _), ()) -> (who, rid)); let hits_idx = hits_map | arrange; let friend_idx = friend | arrange; -let hits_join = hits_idx | join(friend_idx, (_, rid, (f, _)) -> (f, rid)); +let hits_join = hits_idx | join(friend_idx, (_, (rid, (f, _))) -> (f, rid)); let hits_idx2 = hits_join | arrange; let reply_idx = reply | arrange; -let hits_join2 = hits_idx2 | join(reply_idx, (f, rid, (id, parent)) -> ((rid, parent), (f, id))); +let hits_join2 = hits_idx2 | join(reply_idx, (f, (rid, (id, parent))) -> ((rid, parent), (f, id))); let hits_idx3 = hits_join2 | arrange; let hits_rows = intag_join6 | filter((_, kind) -> (kind, 0) eq) | map(((a, b, c), _) -> ((a, b), c)); let hits_idx4 = hits_rows | arrange; -let hits = hits_idx3 | join(hits_idx4, ((rid, _), (f, id), name) -> ((rid, f), (id, name))); +let hits = hits_idx3 | join(hits_idx4, ((rid, _), ((f, id), name)) -> ((rid, f), (id, name))); let counts_map = hits | map(((rid, f), (id, _)) -> ((rid, f, id), ())); let counts_idx = counts_map | arrange; let counts_reduce = counts_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); @@ -52,10 +52,10 @@ let tagset_reduce = tagset_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1 let tagset_map2 = tagset_reduce | map(((rid, f, name), ()) -> ((rid, f), name)); let tagset_idx2 = tagset_map2 | arrange; let tagset = tagset_idx2 | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); -let ranked_join = counts | join(tagset, ((rid, f), n, list) -> (f, (rid, n, list))); +let ranked_join = counts | join(tagset, ((rid, f), (n, list)) -> (f, (rid, n, list))); let ranked_idx = ranked_join | arrange; let person_idx = person | arrange; -let ranked_join2 = ranked_idx | join(person_idx, (f, (rid, n, list), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, ((n neg, f), (first, last, list)))); +let ranked_join2 = ranked_idx | join(person_idx, (f, ((rid, n, list), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((n neg, f), (first, last, list)))); let ranked_idx2 = ranked_join2 | arrange; let ranked = ranked_idx2 | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((neg, f), (first, last, list)))) -> ((rid, rank), (f, first, last, list, neg neg))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic12_oneoff.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/ldbc_ic12_oneoff.lowered--optimized.diff index 45a66d793..e3e634629 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic12_oneoff.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic12_oneoff.lowered--optimized.diff @@ -11,7 +11,7 @@ @@ -30,8 +29,6 @@ let intag_idx6 = intag_join5 | arrange; let message_idx = message | arrange; - let intag_join6 = intag_idx6 | join(message_idx, (m, (rid, name), (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> ((rid, m, name), kind)); + let intag_join6 = intag_idx6 | join(message_idx, (m, ((rid, name), (kind, _, _, _, _, _, _, _, _, _, _, _, _))) -> ((rid, m, name), kind)); -let intag_filter = intag_join6 | filter((_, kind) -> (kind, 0) eq); -let intag = intag_filter | map((key, _) -> (key, ())); let hits_map = request | map(((rid, who, _), ()) -> (who, rid)); @@ -19,12 +19,12 @@ let friend_idx = friend | arrange; @@ -40,8 +37,8 @@ let reply_idx = reply | arrange; - let hits_join2 = hits_idx2 | join(reply_idx, (f, rid, (id, parent)) -> ((rid, parent), (f, id))); + let hits_join2 = hits_idx2 | join(reply_idx, (f, (rid, (id, parent))) -> ((rid, parent), (f, id))); let hits_idx3 = hits_join2 | arrange; -let hits_map2 = intag | map(((rid, m, name), ()) -> ((rid, m), name)); -let hits_idx4 = hits_map2 | arrange; +let hits_rows = intag_join6 | filter((_, kind) -> (kind, 0) eq) | map(((a, b, c), _) -> ((a, b), c)); +let hits_idx4 = hits_rows | arrange; - let hits = hits_idx3 | join(hits_idx4, ((rid, _), (f, id), name) -> ((rid, f), (id, name))); + let hits = hits_idx3 | join(hits_idx4, ((rid, _), ((f, id), name)) -> ((rid, f), (id, name))); let counts_map = hits | map(((rid, f), (id, _)) -> ((rid, f, id), ())); let counts_idx = counts_map | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic12_oneoff.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic12_oneoff.parsed--lowered.diff index 37d72bae7..ca16e38f2 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic12_oneoff.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic12_oneoff.parsed--lowered.diff @@ -10,43 +10,43 @@ let mtag = import "graph::mtag.by_tag" : (int, int); let below_hl = import "ic12s::below_hl" : (int, int); let reply = import "ic12s::reply" : (int, (int, int)); --let intag = request | map(((rid, _, name), ()) -> (name, rid)) | join(classes, (_, rid, high) -> (high, rid)) | join(below_hl, (_, rid, low) -> (low, rid)) | join(by_class, (_, rid, t) -> (t, rid)) | join(tag, (t, rid, (name, _)) -> (t, (rid, name))) | join(mtag, (_, (rid, name), m) -> (m, (rid, name))) | join(message, (m, (rid, name), (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> ((rid, m, name), kind)) | filter((_, kind) -> (kind, 0) eq) | map((key, _) -> (key, ())); --let hits = request | map(((rid, who, _), ()) -> (who, rid)) | join(friend, (_, rid, (f, _)) -> (f, rid)) | join(reply, (f, rid, (id, parent)) -> ((rid, parent), (f, id))) | join(intag | map(((rid, m, name), ()) -> ((rid, m), name)), ((rid, _), (f, id), name) -> ((rid, f), (id, name))); +-let intag = request | map(((rid, _, name), ()) -> (name, rid)) | join(classes, (_, (rid, high)) -> (high, rid)) | join(below_hl, (_, (rid, low)) -> (low, rid)) | join(by_class, (_, (rid, t)) -> (t, rid)) | join(tag, (t, (rid, (name, _))) -> (t, (rid, name))) | join(mtag, (_, ((rid, name), m)) -> (m, (rid, name))) | join(message, (m, ((rid, name), (kind, _, _, _, _, _, _, _, _, _, _, _, _))) -> ((rid, m, name), kind)) | filter((_, kind) -> (kind, 0) eq) | map((key, _) -> (key, ())); +-let hits = request | map(((rid, who, _), ()) -> (who, rid)) | join(friend, (_, (rid, (f, _))) -> (f, rid)) | join(reply, (f, (rid, (id, parent))) -> ((rid, parent), (f, id))) | join(intag | map(((rid, m, name), ()) -> ((rid, m), name)), ((rid, _), ((f, id), name)) -> ((rid, f), (id, name))); -let counts = hits | map(((rid, f), (id, _)) -> ((rid, f, id), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((rid, f, _), ()) -> ((rid, f), ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); -let tagset = hits | map(((rid, f), (_, name)) -> ((rid, f, name), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((rid, f, name), ()) -> ((rid, f), name)) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); --let ranked = counts | join(tagset, ((rid, f), n, list) -> (f, (rid, n, list))) | join(person, (f, (rid, n, list), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, ((n neg, f), (first, last, list)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); +-let ranked = counts | join(tagset, ((rid, f), (n, list)) -> (f, (rid, n, list))) | join(person, (f, ((rid, n, list), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((n neg, f), (first, last, list)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); +let intag_map = request | map(((rid, _, name), ()) -> (name, rid)); +let intag_idx = intag_map | arrange; +let classes_idx = classes | arrange; -+let intag_join = intag_idx | join(classes_idx, (_, rid, high) -> (high, rid)); ++let intag_join = intag_idx | join(classes_idx, (_, (rid, high)) -> (high, rid)); +let intag_idx2 = intag_join | arrange; +let below_hl_idx = below_hl | arrange; -+let intag_join2 = intag_idx2 | join(below_hl_idx, (_, rid, low) -> (low, rid)); ++let intag_join2 = intag_idx2 | join(below_hl_idx, (_, (rid, low)) -> (low, rid)); +let intag_idx3 = intag_join2 | arrange; +let by_class_idx = by_class | arrange; -+let intag_join3 = intag_idx3 | join(by_class_idx, (_, rid, t) -> (t, rid)); ++let intag_join3 = intag_idx3 | join(by_class_idx, (_, (rid, t)) -> (t, rid)); +let intag_idx4 = intag_join3 | arrange; +let tag_idx = tag | arrange; -+let intag_join4 = intag_idx4 | join(tag_idx, (t, rid, (name, _)) -> (t, (rid, name))); ++let intag_join4 = intag_idx4 | join(tag_idx, (t, (rid, (name, _))) -> (t, (rid, name))); +let intag_idx5 = intag_join4 | arrange; +let mtag_idx = mtag | arrange; -+let intag_join5 = intag_idx5 | join(mtag_idx, (_, (rid, name), m) -> (m, (rid, name))); ++let intag_join5 = intag_idx5 | join(mtag_idx, (_, ((rid, name), m)) -> (m, (rid, name))); +let intag_idx6 = intag_join5 | arrange; +let message_idx = message | arrange; -+let intag_join6 = intag_idx6 | join(message_idx, (m, (rid, name), (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> ((rid, m, name), kind)); ++let intag_join6 = intag_idx6 | join(message_idx, (m, ((rid, name), (kind, _, _, _, _, _, _, _, _, _, _, _, _))) -> ((rid, m, name), kind)); +let intag_filter = intag_join6 | filter((_, kind) -> (kind, 0) eq); +let intag = intag_filter | map((key, _) -> (key, ())); +let hits_map = request | map(((rid, who, _), ()) -> (who, rid)); +let hits_idx = hits_map | arrange; +let friend_idx = friend | arrange; -+let hits_join = hits_idx | join(friend_idx, (_, rid, (f, _)) -> (f, rid)); ++let hits_join = hits_idx | join(friend_idx, (_, (rid, (f, _))) -> (f, rid)); +let hits_idx2 = hits_join | arrange; +let reply_idx = reply | arrange; -+let hits_join2 = hits_idx2 | join(reply_idx, (f, rid, (id, parent)) -> ((rid, parent), (f, id))); ++let hits_join2 = hits_idx2 | join(reply_idx, (f, (rid, (id, parent))) -> ((rid, parent), (f, id))); +let hits_idx3 = hits_join2 | arrange; +let hits_map2 = intag | map(((rid, m, name), ()) -> ((rid, m), name)); +let hits_idx4 = hits_map2 | arrange; -+let hits = hits_idx3 | join(hits_idx4, ((rid, _), (f, id), name) -> ((rid, f), (id, name))); ++let hits = hits_idx3 | join(hits_idx4, ((rid, _), ((f, id), name)) -> ((rid, f), (id, name))); +let counts_map = hits | map(((rid, f), (id, _)) -> ((rid, f, id), ())); +let counts_idx = counts_map | arrange; +let counts_reduce = counts_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); @@ -59,10 +59,10 @@ +let tagset_map2 = tagset_reduce | map(((rid, f, name), ()) -> ((rid, f), name)); +let tagset_idx2 = tagset_map2 | arrange; +let tagset = tagset_idx2 | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); -+let ranked_join = counts | join(tagset, ((rid, f), n, list) -> (f, (rid, n, list))); ++let ranked_join = counts | join(tagset, ((rid, f), (n, list)) -> (f, (rid, n, list))); +let ranked_idx = ranked_join | arrange; +let person_idx = person | arrange; -+let ranked_join2 = ranked_idx | join(person_idx, (f, (rid, n, list), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, ((n neg, f), (first, last, list)))); ++let ranked_join2 = ranked_idx | join(person_idx, (f, ((rid, n, list), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((n neg, f), (first, last, list)))); +let ranked_idx2 = ranked_join2 | arrange; +let ranked = ranked_idx2 | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((neg, f), (first, last, list)))) -> ((rid, rank), (f, first, last, list, neg neg))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic12_oneoff.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic12_oneoff.parsed.ddp index 01fd3d9eb..6405f167e 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic12_oneoff.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic12_oneoff.parsed.ddp @@ -9,10 +9,10 @@ let classes = import "graph::tagclass.by_name" : (List(int), int); let mtag = import "graph::mtag.by_tag" : (int, int); let below_hl = import "ic12s::below_hl" : (int, int); let reply = import "ic12s::reply" : (int, (int, int)); -let intag = request | map(((rid, _, name), ()) -> (name, rid)) | join(classes, (_, rid, high) -> (high, rid)) | join(below_hl, (_, rid, low) -> (low, rid)) | join(by_class, (_, rid, t) -> (t, rid)) | join(tag, (t, rid, (name, _)) -> (t, (rid, name))) | join(mtag, (_, (rid, name), m) -> (m, (rid, name))) | join(message, (m, (rid, name), (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> ((rid, m, name), kind)) | filter((_, kind) -> (kind, 0) eq) | map((key, _) -> (key, ())); -let hits = request | map(((rid, who, _), ()) -> (who, rid)) | join(friend, (_, rid, (f, _)) -> (f, rid)) | join(reply, (f, rid, (id, parent)) -> ((rid, parent), (f, id))) | join(intag | map(((rid, m, name), ()) -> ((rid, m), name)), ((rid, _), (f, id), name) -> ((rid, f), (id, name))); +let intag = request | map(((rid, _, name), ()) -> (name, rid)) | join(classes, (_, (rid, high)) -> (high, rid)) | join(below_hl, (_, (rid, low)) -> (low, rid)) | join(by_class, (_, (rid, t)) -> (t, rid)) | join(tag, (t, (rid, (name, _))) -> (t, (rid, name))) | join(mtag, (_, ((rid, name), m)) -> (m, (rid, name))) | join(message, (m, ((rid, name), (kind, _, _, _, _, _, _, _, _, _, _, _, _))) -> ((rid, m, name), kind)) | filter((_, kind) -> (kind, 0) eq) | map((key, _) -> (key, ())); +let hits = request | map(((rid, who, _), ()) -> (who, rid)) | join(friend, (_, (rid, (f, _))) -> (f, rid)) | join(reply, (f, (rid, (id, parent))) -> ((rid, parent), (f, id))) | join(intag | map(((rid, m, name), ()) -> ((rid, m), name)), ((rid, _), ((f, id), name)) -> ((rid, f), (id, name))); let counts = hits | map(((rid, f), (id, _)) -> ((rid, f, id), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((rid, f, _), ()) -> ((rid, f), ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); let tagset = hits | map(((rid, f), (_, name)) -> ((rid, f, name), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((rid, f, name), ()) -> ((rid, f), name)) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); -let ranked = counts | join(tagset, ((rid, f), n, list) -> (f, (rid, n, list))) | join(person, (f, (rid, n, list), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, ((n neg, f), (first, last, list)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); +let ranked = counts | join(tagset, ((rid, f), (n, list)) -> (f, (rid, n, list))) | join(person, (f, ((rid, n, list), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((n neg, f), (first, last, list)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((neg, f), (first, last, list)))) -> ((rid, rank), (f, first, last, list, neg neg))); output "answer" = answer | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic12_standing.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic12_standing.ddp index 073d44aa3..bfbc9a5ae 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic12_standing.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic12_standing.ddp @@ -8,7 +8,7 @@ below: { var below; let step_map = below | map(((low, high), ()) -> (high, low)); let step_idx = step_map | arrange; - let step_join = step_idx | join(::tagclass, (_, low, (_, parent)) -> (low, parent)); + let step_join = step_idx | join(::tagclass, (_, (low, (_, parent))) -> (low, parent)); let step = step_join | filter((_, parent) -> (parent, 0) ge) | map((low, higher) -> ((low, higher), ())); let below_rows = ::selves + step; let below_idx = below_rows | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic12_standing.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/ldbc_ic12_standing.lowered--optimized.diff index fcae73f9d..e4a66f8de 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic12_standing.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic12_standing.lowered--optimized.diff @@ -18,10 +18,10 @@ let step_map = below | map(((low, high), ()) -> (high, low)); let step_idx = step_map | arrange; - let tagclass_idx = ::tagclass | arrange; -- let step_join = step_idx | join(tagclass_idx, (_, low, (_, parent)) -> (low, parent)); +- let step_join = step_idx | join(tagclass_idx, (_, (low, (_, parent))) -> (low, parent)); - let step_filter = step_join | filter((_, parent) -> (parent, 0) ge); - let step = step_filter | map((low, higher) -> ((low, higher), ())); -+ let step_join = step_idx | join(::tagclass, (_, low, (_, parent)) -> (low, parent)); ++ let step_join = step_idx | join(::tagclass, (_, (low, (_, parent))) -> (low, parent)); + let step = step_join | filter((_, parent) -> (parent, 0) ge) | map((low, higher) -> ((low, higher), ())); let below_rows = ::selves + step; let below_idx = below_rows | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic12_standing.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic12_standing.parsed--lowered.diff index 9b0d44260..8d52c51bc 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic12_standing.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic12_standing.parsed--lowered.diff @@ -10,14 +10,14 @@ let mtag = import "graph::mtag.by_tag" : (int, int); let selves = tagclass | map((id, _) -> ((id, id), ())); below: { -- let step = below | map(((low, high), ()) -> (high, low)) | join(tagclass, (_, low, (_, parent)) -> (low, parent)) | filter((_, parent) -> (parent, 0) ge) | map((low, higher) -> ((low, higher), ())); +- let step = below | map(((low, high), ()) -> (high, low)) | join(tagclass, (_, (low, (_, parent))) -> (low, parent)) | filter((_, parent) -> (parent, 0) ge) | map((low, higher) -> ((low, higher), ())); - set below = selves + step | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); + -- Time lanes: epoch, then 1 scope coordinates; this scope enters at zero. + var below; + let step_map = below | map(((low, high), ()) -> (high, low)); + let step_idx = step_map | arrange; + let tagclass_idx = ::tagclass | arrange; -+ let step_join = step_idx | join(tagclass_idx, (_, low, (_, parent)) -> (low, parent)); ++ let step_join = step_idx | join(tagclass_idx, (_, (low, (_, parent))) -> (low, parent)); + let step_filter = step_join | filter((_, parent) -> (parent, 0) ge); + let step = step_filter | map((low, higher) -> ((low, higher), ())); + let below_rows = ::selves + step; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic12_standing.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic12_standing.parsed.ddp index 43bf9efd8..8fb4b96c4 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic12_standing.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic12_standing.parsed.ddp @@ -8,7 +8,7 @@ let classes = import "graph::tagclass.by_name" : (List(int), int); let mtag = import "graph::mtag.by_tag" : (int, int); let selves = tagclass | map((id, _) -> ((id, id), ())); below: { - let step = below | map(((low, high), ()) -> (high, low)) | join(tagclass, (_, low, (_, parent)) -> (low, parent)) | filter((_, parent) -> (parent, 0) ge) | map((low, higher) -> ((low, higher), ())); + let step = below | map(((low, high), ()) -> (high, low)) | join(tagclass, (_, (low, (_, parent))) -> (low, parent)) | filter((_, parent) -> (parent, 0) ge) | map((low, higher) -> ((low, higher), ())); set below = selves + step | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); } let below_hl = below::below | map(((low, high), ()) -> (high, low)); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic13_oneoff.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic13_oneoff.ddp index 679ecf31d..00b10022c 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic13_oneoff.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic13_oneoff.ddp @@ -12,22 +12,22 @@ search: { let side1 = search | filter(((_, side, _, _), ()) -> (side, 1) eq) | map(((rid, _, node, d), ()) -> ((rid, node), d)); let side0_idx = side0 | arrange; let side1_idx = side1 | arrange; - let best_join = side0_idx | join(side1_idx, ((rid, _), d0, d1) -> (rid, (d0, d1) add)); + let best_join = side0_idx | join(side1_idx, ((rid, _), (d0, d1)) -> (rid, (d0, d1) add)); let best_idx = best_join | arrange; let best = best_idx | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); let by_rid = search | map(((rid, side, node, d), ()) -> (rid, (side, node, d))); let met = best | map((rid, _) -> (rid, ())); let by_rid_idx = by_rid | arrange; let met_idx = met | arrange; - let open_join = by_rid_idx | join(met_idx, (rid, row, ()) -> (rid, row)); + let open_join = by_rid_idx | join(met_idx, (rid, (row, ())) -> (rid, row)); let open_neg = open_join | negate; let open = by_rid + open_neg; - let near_join = by_rid_idx | join(best, (rid, (side, node, d), b) -> (rid, (side, node, d, b))); + let near_join = by_rid_idx | join(best, (rid, ((side, node, d), b)) -> (rid, (side, node, d, b))); let near = near_join | filter((_, (_, _, d, b)) -> ((d, d) add, b) lt) | map((rid, (side, node, d, _)) -> (rid, (side, node, d))); let grow_rows = open + near; let grow_map = grow_rows | map((rid, (side, node, d)) -> (node, (rid, side, d))); let grow_idx = grow_map | arrange; - let grow = grow_idx | join(::friend, (_, (rid, side, d), (next, _)) -> ((rid, side, next, (d, 1) add), ())); + let grow = grow_idx | join(::friend, (_, ((rid, side, d), (next, _))) -> ((rid, side, next, (d, 1) add), ())); let search_rows = ::starts + search + grow; let search_map = search_rows | map(((rid, side, node, d), ()) -> ((rid, side, node), d)); let search_idx = search_map | arrange; @@ -39,12 +39,12 @@ let side0 = search::search | filter(((_, side, _, _), ()) -> (side, 0) eq) | map let side1 = search::search | filter(((_, side, _, _), ()) -> (side, 1) eq) | map(((rid, _, node, d), ()) -> ((rid, node), d)); let side0_idx = side0 | arrange; let side1_idx = side1 | arrange; -let best_join = side0_idx | join(side1_idx, ((rid, _), d0, d1) -> (rid, (d0, d1) add)); +let best_join = side0_idx | join(side1_idx, ((rid, _), (d0, d1)) -> (rid, (d0, d1) add)); let best_idx = best_join | arrange; let best = best_idx | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); let asked = request | map(((rid, _, _), ()) -> (rid, ())); let asked_idx = asked | arrange; -let never_join = asked_idx | join(best, (rid, (), _) -> (rid, ())); +let never_join = asked_idx | join(best, (rid, ((), _)) -> (rid, ())); let never_neg = never_join | negate; let never = asked + never_neg; let answer_map = best | map((rid, d) -> ((rid, 0), (d))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic13_oneoff.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/ldbc_ic13_oneoff.lowered--optimized.diff index 0c55c2117..618951a5f 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic13_oneoff.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic13_oneoff.lowered--optimized.diff @@ -12,23 +12,23 @@ + let side1 = search | filter(((_, side, _, _), ()) -> (side, 1) eq) | map(((rid, _, node, d), ()) -> ((rid, node), d)); let side0_idx = side0 | arrange; let side1_idx = side1 | arrange; - let best_join = side0_idx | join(side1_idx, ((rid, _), d0, d1) -> (rid, (d0, d1) add)); + let best_join = side0_idx | join(side1_idx, ((rid, _), (d0, d1)) -> (rid, (d0, d1) add)); @@ -24,15 +22,12 @@ - let open_join = by_rid_idx | join(met_idx, (rid, row, ()) -> (rid, row)); + let open_join = by_rid_idx | join(met_idx, (rid, (row, ())) -> (rid, row)); let open_neg = open_join | negate; let open = by_rid + open_neg; - let by_rid_idx2 = by_rid | arrange; -- let near_join = by_rid_idx2 | join(best, (rid, (side, node, d), b) -> (rid, (side, node, d, b))); +- let near_join = by_rid_idx2 | join(best, (rid, ((side, node, d), b)) -> (rid, (side, node, d, b))); - let near_filter = near_join | filter((_, (_, _, d, b)) -> ((d, d) add, b) lt); - let near = near_filter | map((rid, (side, node, d, _)) -> (rid, (side, node, d))); -+ let near_join = by_rid_idx | join(best, (rid, (side, node, d), b) -> (rid, (side, node, d, b))); ++ let near_join = by_rid_idx | join(best, (rid, ((side, node, d), b)) -> (rid, (side, node, d, b))); + let near = near_join | filter((_, (_, _, d, b)) -> ((d, d) add, b) lt) | map((rid, (side, node, d, _)) -> (rid, (side, node, d))); let grow_rows = open + near; let grow_map = grow_rows | map((rid, (side, node, d)) -> (node, (rid, side, d))); let grow_idx = grow_map | arrange; - let friend_idx = ::friend | arrange; -- let grow = grow_idx | join(friend_idx, (_, (rid, side, d), (next, _)) -> ((rid, side, next, (d, 1) add), ())); -+ let grow = grow_idx | join(::friend, (_, (rid, side, d), (next, _)) -> ((rid, side, next, (d, 1) add), ())); +- let grow = grow_idx | join(friend_idx, (_, ((rid, side, d), (next, _))) -> ((rid, side, next, (d, 1) add), ())); ++ let grow = grow_idx | join(::friend, (_, ((rid, side, d), (next, _))) -> ((rid, side, next, (d, 1) add), ())); let search_rows = ::starts + search + grow; let search_map = search_rows | map(((rid, side, node, d), ()) -> ((rid, side, node), d)); let search_idx = search_map | arrange; @@ -44,4 +44,4 @@ +let side1 = search::search | filter(((_, side, _, _), ()) -> (side, 1) eq) | map(((rid, _, node, d), ()) -> ((rid, node), d)); let side0_idx = side0 | arrange; let side1_idx = side1 | arrange; - let best_join = side0_idx | join(side1_idx, ((rid, _), d0, d1) -> (rid, (d0, d1) add)); + let best_join = side0_idx | join(side1_idx, ((rid, _), (d0, d1)) -> (rid, (d0, d1) add)); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic13_oneoff.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic13_oneoff.parsed--lowered.diff index c506c1ed9..a75da0d46 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic13_oneoff.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic13_oneoff.parsed--lowered.diff @@ -12,7 +12,7 @@ search: { - let side0 = search | filter(((_, side, _, _), ()) -> (side, 0) eq) | map(((rid, _, node, d), ()) -> ((rid, node), d)); - let side1 = search | filter(((_, side, _, _), ()) -> (side, 1) eq) | map(((rid, _, node, d), ()) -> ((rid, node), d)); -- let best = side0 | join(side1, ((rid, _), d0, d1) -> (rid, (d0, d1) add)) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); +- let best = side0 | join(side1, ((rid, _), (d0, d1)) -> (rid, (d0, d1) add)) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); + -- Time lanes: epoch, then 1 scope coordinates; this scope enters at zero. + var search; + let side0_filter = search | filter(((_, side, _, _), ()) -> (side, 0) eq); @@ -21,29 +21,29 @@ + let side1 = side1_filter | map(((rid, _, node, d), ()) -> ((rid, node), d)); + let side0_idx = side0 | arrange; + let side1_idx = side1 | arrange; -+ let best_join = side0_idx | join(side1_idx, ((rid, _), d0, d1) -> (rid, (d0, d1) add)); ++ let best_join = side0_idx | join(side1_idx, ((rid, _), (d0, d1)) -> (rid, (d0, d1) add)); + let best_idx = best_join | arrange; + let best = best_idx | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); let by_rid = search | map(((rid, side, node, d), ()) -> (rid, (side, node, d))); let met = best | map((rid, _) -> (rid, ())); -- let open = by_rid + (by_rid | join(met, (rid, row, ()) -> (rid, row)) | negate); -- let near = by_rid | join(best, (rid, (side, node, d), b) -> (rid, (side, node, d, b))) | filter((_, (_, _, d, b)) -> ((d, d) add, b) lt) | map((rid, (side, node, d, _)) -> (rid, (side, node, d))); -- let grow = open + near | map((rid, (side, node, d)) -> (node, (rid, side, d))) | join(friend, (_, (rid, side, d), (next, _)) -> ((rid, side, next, (d, 1) add), ())); +- let open = by_rid + (by_rid | join(met, (rid, (row, ())) -> (rid, row)) | negate); +- let near = by_rid | join(best, (rid, ((side, node, d), b)) -> (rid, (side, node, d, b))) | filter((_, (_, _, d, b)) -> ((d, d) add, b) lt) | map((rid, (side, node, d, _)) -> (rid, (side, node, d))); +- let grow = open + near | map((rid, (side, node, d)) -> (node, (rid, side, d))) | join(friend, (_, ((rid, side, d), (next, _))) -> ((rid, side, next, (d, 1) add), ())); - set search = starts + search + grow | map(((rid, side, node, d), ()) -> ((rid, side, node), d)) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist) | map(((rid, side, node), d) -> ((rid, side, node, d), ())); + let by_rid_idx = by_rid | arrange; + let met_idx = met | arrange; -+ let open_join = by_rid_idx | join(met_idx, (rid, row, ()) -> (rid, row)); ++ let open_join = by_rid_idx | join(met_idx, (rid, (row, ())) -> (rid, row)); + let open_neg = open_join | negate; + let open = by_rid + open_neg; + let by_rid_idx2 = by_rid | arrange; -+ let near_join = by_rid_idx2 | join(best, (rid, (side, node, d), b) -> (rid, (side, node, d, b))); ++ let near_join = by_rid_idx2 | join(best, (rid, ((side, node, d), b)) -> (rid, (side, node, d, b))); + let near_filter = near_join | filter((_, (_, _, d, b)) -> ((d, d) add, b) lt); + let near = near_filter | map((rid, (side, node, d, _)) -> (rid, (side, node, d))); + let grow_rows = open + near; + let grow_map = grow_rows | map((rid, (side, node, d)) -> (node, (rid, side, d))); + let grow_idx = grow_map | arrange; + let friend_idx = ::friend | arrange; -+ let grow = grow_idx | join(friend_idx, (_, (rid, side, d), (next, _)) -> ((rid, side, next, (d, 1) add), ())); ++ let grow = grow_idx | join(friend_idx, (_, ((rid, side, d), (next, _))) -> ((rid, side, next, (d, 1) add), ())); + let search_rows = ::starts + search + grow; + let search_map = search_rows | map(((rid, side, node, d), ()) -> ((rid, side, node), d)); + let search_idx = search_map | arrange; @@ -53,22 +53,22 @@ } -let side0 = search::search | filter(((_, side, _, _), ()) -> (side, 0) eq) | map(((rid, _, node, d), ()) -> ((rid, node), d)); -let side1 = search::search | filter(((_, side, _, _), ()) -> (side, 1) eq) | map(((rid, _, node, d), ()) -> ((rid, node), d)); --let best = side0 | join(side1, ((rid, _), d0, d1) -> (rid, (d0, d1) add)) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); +-let best = side0 | join(side1, ((rid, _), (d0, d1)) -> (rid, (d0, d1) add)) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); +let side0_filter = search::search | filter(((_, side, _, _), ()) -> (side, 0) eq); +let side0 = side0_filter | map(((rid, _, node, d), ()) -> ((rid, node), d)); +let side1_filter = search::search | filter(((_, side, _, _), ()) -> (side, 1) eq); +let side1 = side1_filter | map(((rid, _, node, d), ()) -> ((rid, node), d)); +let side0_idx = side0 | arrange; +let side1_idx = side1 | arrange; -+let best_join = side0_idx | join(side1_idx, ((rid, _), d0, d1) -> (rid, (d0, d1) add)); ++let best_join = side0_idx | join(side1_idx, ((rid, _), (d0, d1)) -> (rid, (d0, d1) add)); +let best_idx = best_join | arrange; +let best = best_idx | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); let asked = request | map(((rid, _, _), ()) -> (rid, ())); --let never = asked + (asked | join(best, (rid, (), _) -> (rid, ())) | negate); +-let never = asked + (asked | join(best, (rid, ((), _)) -> (rid, ())) | negate); -let answer = (best | map((rid, d) -> ((rid, 0), (d)))) + (never | map((rid, ()) -> ((rid, 0), (-1)))); -output "answer" = answer | arrange; +let asked_idx = asked | arrange; -+let never_join = asked_idx | join(best, (rid, (), _) -> (rid, ())); ++let never_join = asked_idx | join(best, (rid, ((), _)) -> (rid, ())); +let never_neg = never_join | negate; +let never = asked + never_neg; +let answer_map = best | map((rid, d) -> ((rid, 0), (d))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic13_oneoff.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic13_oneoff.parsed.ddp index b51f7c549..da86ad1c4 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic13_oneoff.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic13_oneoff.parsed.ddp @@ -4,18 +4,18 @@ let starts = (request | map(((rid, from, _), ()) -> ((rid, 0, from, 0), ()))) + search: { let side0 = search | filter(((_, side, _, _), ()) -> (side, 0) eq) | map(((rid, _, node, d), ()) -> ((rid, node), d)); let side1 = search | filter(((_, side, _, _), ()) -> (side, 1) eq) | map(((rid, _, node, d), ()) -> ((rid, node), d)); - let best = side0 | join(side1, ((rid, _), d0, d1) -> (rid, (d0, d1) add)) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); + let best = side0 | join(side1, ((rid, _), (d0, d1)) -> (rid, (d0, d1) add)) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); let by_rid = search | map(((rid, side, node, d), ()) -> (rid, (side, node, d))); let met = best | map((rid, _) -> (rid, ())); - let open = by_rid + (by_rid | join(met, (rid, row, ()) -> (rid, row)) | negate); - let near = by_rid | join(best, (rid, (side, node, d), b) -> (rid, (side, node, d, b))) | filter((_, (_, _, d, b)) -> ((d, d) add, b) lt) | map((rid, (side, node, d, _)) -> (rid, (side, node, d))); - let grow = open + near | map((rid, (side, node, d)) -> (node, (rid, side, d))) | join(friend, (_, (rid, side, d), (next, _)) -> ((rid, side, next, (d, 1) add), ())); + let open = by_rid + (by_rid | join(met, (rid, (row, ())) -> (rid, row)) | negate); + let near = by_rid | join(best, (rid, ((side, node, d), b)) -> (rid, (side, node, d, b))) | filter((_, (_, _, d, b)) -> ((d, d) add, b) lt) | map((rid, (side, node, d, _)) -> (rid, (side, node, d))); + let grow = open + near | map((rid, (side, node, d)) -> (node, (rid, side, d))) | join(friend, (_, ((rid, side, d), (next, _))) -> ((rid, side, next, (d, 1) add), ())); set search = starts + search + grow | map(((rid, side, node, d), ()) -> ((rid, side, node), d)) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist) | map(((rid, side, node), d) -> ((rid, side, node, d), ())); } let side0 = search::search | filter(((_, side, _, _), ()) -> (side, 0) eq) | map(((rid, _, node, d), ()) -> ((rid, node), d)); let side1 = search::search | filter(((_, side, _, _), ()) -> (side, 1) eq) | map(((rid, _, node, d), ()) -> ((rid, node), d)); -let best = side0 | join(side1, ((rid, _), d0, d1) -> (rid, (d0, d1) add)) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); +let best = side0 | join(side1, ((rid, _), (d0, d1)) -> (rid, (d0, d1) add)) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); let asked = request | map(((rid, _, _), ()) -> (rid, ())); -let never = asked + (asked | join(best, (rid, (), _) -> (rid, ())) | negate); +let never = asked + (asked | join(best, (rid, ((), _)) -> (rid, ())) | negate); let answer = (best | map((rid, d) -> ((rid, 0), (d)))) + (never | map((rid, ()) -> ((rid, 0), (-1)))); output "answer" = answer | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic14_oneoff.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic14_oneoff.ddp index 55d86150c..8b65545c0 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic14_oneoff.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic14_oneoff.ddp @@ -8,7 +8,7 @@ dist: { var dist; let grow_map = dist | map(((rid, node, cost), ()) -> (node, (rid, cost))); let grow_idx = grow_map | arrange; - let grow = grow_idx | join(::weighted, (_, (rid, cost), (next, w)) -> ((rid, next), (cost, w) add)); + let grow = grow_idx | join(::weighted, (_, ((rid, cost), (next, w))) -> ((rid, next), (cost, w) add)); let dist_rows = ::starts + grow; let dist_idx = dist_rows | arrange; let dist_reduce = dist_idx | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); @@ -19,47 +19,47 @@ let best = dist::dist | map(((rid, node, cost), ()) -> ((rid, node), cost)); let pred_map = dist::dist | map(((rid, u, cost), ()) -> (u, (rid, cost))); let pred_idx = pred_map | arrange; let weighted_idx = weighted | arrange; -let pred_join = pred_idx | join(weighted_idx, (u, (rid, cost), (v, w)) -> ((rid, v), (u, (cost, w) add))); +let pred_join = pred_idx | join(weighted_idx, (u, ((rid, cost), (v, w))) -> ((rid, v), (u, (cost, w) add))); let pred_idx2 = pred_join | arrange; let best_idx = best | arrange; -let pred_join2 = pred_idx2 | join(best_idx, ((rid, v), (u, via), least) -> ((rid, v), (u, via, least))); +let pred_join2 = pred_idx2 | join(best_idx, ((rid, v), ((u, via), least)) -> ((rid, v), (u, via, least))); let pred = pred_join2 | filter((_, (_, via, least)) -> (via, least) eq) | map((key, (u, _, _)) -> (key, u)); let pred_idx3 = pred | arrange; let targets_map = request | map(((rid, _, to), ()) -> ((rid, to), ())); let targets_idx = targets_map | arrange; -let targets = targets_idx | join(best_idx, (key, (), _) -> (key, ())); +let targets = targets_idx | join(best_idx, (key, ((), _)) -> (key, ())); back: { -- Time lanes: epoch, then 1 scope coordinates; this scope enters at zero. var back; let back_idx = back | arrange; - let up = back_idx | join(::pred_idx3, ((rid, _), (), u) -> ((rid, u), ())); + let up = back_idx | join(::pred_idx3, ((rid, _), ((), u)) -> ((rid, u), ())); let back_rows = ::targets + up; let back_idx2 = back_rows | arrange; let back_reduce = back_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); set back = back_reduce; } let next_idx = back::back | arrange; -let next_join = pred_idx3 | join(next_idx, ((rid, v), u, ()) -> ((rid, u), v)); +let next_join = pred_idx3 | join(next_idx, ((rid, v), (u, ())) -> ((rid, u), v)); let next_idx2 = next_join | arrange; let next = next_idx2 | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); let first_map = request | map(((rid, from, _), ()) -> ((rid, from), ())); let first_idx = first_map | arrange; -let first = first_idx | join(next_idx, ((rid, from), (), ()) -> ((rid, from, from enlist), ())); +let first = first_idx | join(next_idx, ((rid, from), ((), ())) -> ((rid, from, from enlist), ())); walk: { -- Time lanes: epoch, then 1 scope coordinates; this scope enters at zero. var walk; let step_map = walk | map(((rid, node, path), ()) -> ((rid, node), path)); let step_idx = step_map | arrange; - let step = step_idx | join(::next, ((rid, _), path, v) -> ((rid, v, (path, v enlist) append), ())); + let step = step_idx | join(::next, ((rid, _), (path, v)) -> ((rid, v, (path, v enlist) append), ())); let walk_rows = ::first + step; set walk = walk_rows; } let answer_map = walk::walk | map(((rid, node, path), ()) -> ((rid, node), path)); let answer_idx = answer_map | arrange; let targets_idx2 = targets | arrange; -let answer_join = answer_idx | join(targets_idx2, ((rid, node), path, ()) -> ((rid, node), path)); +let answer_join = answer_idx | join(targets_idx2, ((rid, node), (path, ())) -> ((rid, node), path)); let answer_idx2 = answer_join | arrange; -let answer = answer_idx2 | join(best_idx, ((rid, _), path, cost) -> ((rid, 0), (path, cost))); +let answer = answer_idx2 | join(best_idx, ((rid, _), (path, cost)) -> ((rid, 0), (path, cost))); let answer_idx3 = answer | arrange; -- Renderer: publish the existing arrangement. output "answer" = answer_idx3; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic14_oneoff.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/ldbc_ic14_oneoff.lowered--optimized.diff index bd7f36fca..8f471dc53 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic14_oneoff.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic14_oneoff.lowered--optimized.diff @@ -14,15 +14,15 @@ let grow_map = dist | map(((rid, node, cost), ()) -> (node, (rid, cost))); let grow_idx = grow_map | arrange; - let weighted_idx = ::weighted | arrange; -- let grow = grow_idx | join(weighted_idx, (_, (rid, cost), (next, w)) -> ((rid, next), (cost, w) add)); -+ let grow = grow_idx | join(::weighted, (_, (rid, cost), (next, w)) -> ((rid, next), (cost, w) add)); +- let grow = grow_idx | join(weighted_idx, (_, ((rid, cost), (next, w))) -> ((rid, next), (cost, w) add)); ++ let grow = grow_idx | join(::weighted, (_, ((rid, cost), (next, w))) -> ((rid, next), (cost, w) add)); let dist_rows = ::starts + grow; let dist_idx = dist_rows | arrange; let dist_reduce = dist_idx | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); @@ -26,39 +23,34 @@ let pred_idx2 = pred_join | arrange; let best_idx = best | arrange; - let pred_join2 = pred_idx2 | join(best_idx, ((rid, v), (u, via), least) -> ((rid, v), (u, via, least))); + let pred_join2 = pred_idx2 | join(best_idx, ((rid, v), ((u, via), least)) -> ((rid, v), (u, via, least))); -let pred_filter = pred_join2 | filter((_, (_, via, least)) -> (via, least) eq); -let pred = pred_filter | map((key, (u, _, _)) -> (key, u)); +let pred = pred_join2 | filter((_, (_, via, least)) -> (via, least) eq) | map((key, (u, _, _)) -> (key, u)); @@ -30,15 +30,15 @@ let targets_map = request | map(((rid, _, to), ()) -> ((rid, to), ())); let targets_idx = targets_map | arrange; -let best_idx2 = best | arrange; --let targets = targets_idx | join(best_idx2, (key, (), _) -> (key, ())); -+let targets = targets_idx | join(best_idx, (key, (), _) -> (key, ())); +-let targets = targets_idx | join(best_idx2, (key, ((), _)) -> (key, ())); ++let targets = targets_idx | join(best_idx, (key, ((), _)) -> (key, ())); back: { -- Time lanes: epoch, then 1 scope coordinates; this scope enters at zero. var back; let back_idx = back | arrange; - let pred_idx = ::pred | arrange; -- let up = back_idx | join(pred_idx, ((rid, _), (), u) -> ((rid, u), ())); -+ let up = back_idx | join(::pred_idx3, ((rid, _), (), u) -> ((rid, u), ())); +- let up = back_idx | join(pred_idx, ((rid, _), ((), u)) -> ((rid, u), ())); ++ let up = back_idx | join(::pred_idx3, ((rid, _), ((), u)) -> ((rid, u), ())); let back_rows = ::targets + up; let back_idx2 = back_rows | arrange; let back_reduce = back_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); @@ -46,32 +46,32 @@ } -let pred_idx3 = pred | arrange; let next_idx = back::back | arrange; - let next_join = pred_idx3 | join(next_idx, ((rid, v), u, ()) -> ((rid, u), v)); + let next_join = pred_idx3 | join(next_idx, ((rid, v), (u, ())) -> ((rid, u), v)); let next_idx2 = next_join | arrange; let next = next_idx2 | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); let first_map = request | map(((rid, from, _), ()) -> ((rid, from), ())); let first_idx = first_map | arrange; -let first_idx2 = back::back | arrange; --let first = first_idx | join(first_idx2, ((rid, from), (), ()) -> ((rid, from, from enlist), ())); -+let first = first_idx | join(next_idx, ((rid, from), (), ()) -> ((rid, from, from enlist), ())); +-let first = first_idx | join(first_idx2, ((rid, from), ((), ())) -> ((rid, from, from enlist), ())); ++let first = first_idx | join(next_idx, ((rid, from), ((), ())) -> ((rid, from, from enlist), ())); walk: { -- Time lanes: epoch, then 1 scope coordinates; this scope enters at zero. var walk; let step_map = walk | map(((rid, node, path), ()) -> ((rid, node), path)); let step_idx = step_map | arrange; - let next_idx = ::next | arrange; -- let step = step_idx | join(next_idx, ((rid, _), path, v) -> ((rid, v, (path, v enlist) append), ())); -+ let step = step_idx | join(::next, ((rid, _), path, v) -> ((rid, v, (path, v enlist) append), ())); +- let step = step_idx | join(next_idx, ((rid, _), (path, v)) -> ((rid, v, (path, v enlist) append), ())); ++ let step = step_idx | join(::next, ((rid, _), (path, v)) -> ((rid, v, (path, v enlist) append), ())); let walk_rows = ::first + step; set walk = walk_rows; } @@ -67,8 +59,7 @@ let targets_idx2 = targets | arrange; - let answer_join = answer_idx | join(targets_idx2, ((rid, node), path, ()) -> ((rid, node), path)); + let answer_join = answer_idx | join(targets_idx2, ((rid, node), (path, ())) -> ((rid, node), path)); let answer_idx2 = answer_join | arrange; -let best_idx3 = best | arrange; --let answer = answer_idx2 | join(best_idx3, ((rid, _), path, cost) -> ((rid, 0), (path, cost))); -+let answer = answer_idx2 | join(best_idx, ((rid, _), path, cost) -> ((rid, 0), (path, cost))); +-let answer = answer_idx2 | join(best_idx3, ((rid, _), (path, cost)) -> ((rid, 0), (path, cost))); ++let answer = answer_idx2 | join(best_idx, ((rid, _), (path, cost)) -> ((rid, 0), (path, cost))); let answer_idx3 = answer | arrange; -- Renderer: publish the existing arrangement. output "answer" = answer_idx3; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic14_oneoff.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic14_oneoff.parsed--lowered.diff index d5d7d4e92..8f649f6e8 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic14_oneoff.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic14_oneoff.parsed--lowered.diff @@ -9,14 +9,14 @@ let weighted = import "ic14s::weighted" : (int, (int, int)); let starts = request | map(((rid, from, _), ()) -> ((rid, from), 0)); dist: { -- let grow = dist | map(((rid, node, cost), ()) -> (node, (rid, cost))) | join(weighted, (_, (rid, cost), (next, w)) -> ((rid, next), (cost, w) add)); +- let grow = dist | map(((rid, node, cost), ()) -> (node, (rid, cost))) | join(weighted, (_, ((rid, cost), (next, w))) -> ((rid, next), (cost, w) add)); - set dist = starts + grow | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist) | map(((rid, node), cost) -> ((rid, node, cost), ())); + -- Time lanes: epoch, then 1 scope coordinates; this scope enters at zero. + var dist; + let grow_map = dist | map(((rid, node, cost), ()) -> (node, (rid, cost))); + let grow_idx = grow_map | arrange; + let weighted_idx = ::weighted | arrange; -+ let grow = grow_idx | join(weighted_idx, (_, (rid, cost), (next, w)) -> ((rid, next), (cost, w) add)); ++ let grow = grow_idx | join(weighted_idx, (_, ((rid, cost), (next, w))) -> ((rid, next), (cost, w) add)); + let dist_rows = ::starts + grow; + let dist_idx = dist_rows | arrange; + let dist_reduce = dist_idx | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); @@ -24,66 +24,66 @@ + set dist = dist_map; } let best = dist::dist | map(((rid, node, cost), ()) -> ((rid, node), cost)); --let pred = dist::dist | map(((rid, u, cost), ()) -> (u, (rid, cost))) | join(weighted, (u, (rid, cost), (v, w)) -> ((rid, v), (u, (cost, w) add))) | join(best, ((rid, v), (u, via), least) -> ((rid, v), (u, via, least))) | filter((_, (_, via, least)) -> (via, least) eq) | map((key, (u, _, _)) -> (key, u)); --let targets = request | map(((rid, _, to), ()) -> ((rid, to), ())) | join(best, (key, (), _) -> (key, ())); +-let pred = dist::dist | map(((rid, u, cost), ()) -> (u, (rid, cost))) | join(weighted, (u, ((rid, cost), (v, w))) -> ((rid, v), (u, (cost, w) add))) | join(best, ((rid, v), ((u, via), least)) -> ((rid, v), (u, via, least))) | filter((_, (_, via, least)) -> (via, least) eq) | map((key, (u, _, _)) -> (key, u)); +-let targets = request | map(((rid, _, to), ()) -> ((rid, to), ())) | join(best, (key, ((), _)) -> (key, ())); +let pred_map = dist::dist | map(((rid, u, cost), ()) -> (u, (rid, cost))); +let pred_idx = pred_map | arrange; +let weighted_idx = weighted | arrange; -+let pred_join = pred_idx | join(weighted_idx, (u, (rid, cost), (v, w)) -> ((rid, v), (u, (cost, w) add))); ++let pred_join = pred_idx | join(weighted_idx, (u, ((rid, cost), (v, w))) -> ((rid, v), (u, (cost, w) add))); +let pred_idx2 = pred_join | arrange; +let best_idx = best | arrange; -+let pred_join2 = pred_idx2 | join(best_idx, ((rid, v), (u, via), least) -> ((rid, v), (u, via, least))); ++let pred_join2 = pred_idx2 | join(best_idx, ((rid, v), ((u, via), least)) -> ((rid, v), (u, via, least))); +let pred_filter = pred_join2 | filter((_, (_, via, least)) -> (via, least) eq); +let pred = pred_filter | map((key, (u, _, _)) -> (key, u)); +let targets_map = request | map(((rid, _, to), ()) -> ((rid, to), ())); +let targets_idx = targets_map | arrange; +let best_idx2 = best | arrange; -+let targets = targets_idx | join(best_idx2, (key, (), _) -> (key, ())); ++let targets = targets_idx | join(best_idx2, (key, ((), _)) -> (key, ())); back: { -- let up = back | join(pred, ((rid, _), (), u) -> ((rid, u), ())); +- let up = back | join(pred, ((rid, _), ((), u)) -> ((rid, u), ())); - set back = targets + up | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); + -- Time lanes: epoch, then 1 scope coordinates; this scope enters at zero. + var back; + let back_idx = back | arrange; + let pred_idx = ::pred | arrange; -+ let up = back_idx | join(pred_idx, ((rid, _), (), u) -> ((rid, u), ())); ++ let up = back_idx | join(pred_idx, ((rid, _), ((), u)) -> ((rid, u), ())); + let back_rows = ::targets + up; + let back_idx2 = back_rows | arrange; + let back_reduce = back_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); + set back = back_reduce; } --let next = pred | join(back::back, ((rid, v), u, ()) -> ((rid, u), v)) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); --let first = request | map(((rid, from, _), ()) -> ((rid, from), ())) | join(back::back, ((rid, from), (), ()) -> ((rid, from, from enlist), ())); +-let next = pred | join(back::back, ((rid, v), (u, ())) -> ((rid, u), v)) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); +-let first = request | map(((rid, from, _), ()) -> ((rid, from), ())) | join(back::back, ((rid, from), ((), ())) -> ((rid, from, from enlist), ())); +let pred_idx3 = pred | arrange; +let next_idx = back::back | arrange; -+let next_join = pred_idx3 | join(next_idx, ((rid, v), u, ()) -> ((rid, u), v)); ++let next_join = pred_idx3 | join(next_idx, ((rid, v), (u, ())) -> ((rid, u), v)); +let next_idx2 = next_join | arrange; +let next = next_idx2 | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); +let first_map = request | map(((rid, from, _), ()) -> ((rid, from), ())); +let first_idx = first_map | arrange; +let first_idx2 = back::back | arrange; -+let first = first_idx | join(first_idx2, ((rid, from), (), ()) -> ((rid, from, from enlist), ())); ++let first = first_idx | join(first_idx2, ((rid, from), ((), ())) -> ((rid, from, from enlist), ())); walk: { -- let step = walk | map(((rid, node, path), ()) -> ((rid, node), path)) | join(next, ((rid, _), path, v) -> ((rid, v, (path, v enlist) append), ())); +- let step = walk | map(((rid, node, path), ()) -> ((rid, node), path)) | join(next, ((rid, _), (path, v)) -> ((rid, v, (path, v enlist) append), ())); - set walk = first + step; + -- Time lanes: epoch, then 1 scope coordinates; this scope enters at zero. + var walk; + let step_map = walk | map(((rid, node, path), ()) -> ((rid, node), path)); + let step_idx = step_map | arrange; + let next_idx = ::next | arrange; -+ let step = step_idx | join(next_idx, ((rid, _), path, v) -> ((rid, v, (path, v enlist) append), ())); ++ let step = step_idx | join(next_idx, ((rid, _), (path, v)) -> ((rid, v, (path, v enlist) append), ())); + let walk_rows = ::first + step; + set walk = walk_rows; } --let answer = walk::walk | map(((rid, node, path), ()) -> ((rid, node), path)) | join(targets, ((rid, node), path, ()) -> ((rid, node), path)) | join(best, ((rid, _), path, cost) -> ((rid, 0), (path, cost))); +-let answer = walk::walk | map(((rid, node, path), ()) -> ((rid, node), path)) | join(targets, ((rid, node), (path, ())) -> ((rid, node), path)) | join(best, ((rid, _), (path, cost)) -> ((rid, 0), (path, cost))); -output "answer" = answer | arrange; +let answer_map = walk::walk | map(((rid, node, path), ()) -> ((rid, node), path)); +let answer_idx = answer_map | arrange; +let targets_idx2 = targets | arrange; -+let answer_join = answer_idx | join(targets_idx2, ((rid, node), path, ()) -> ((rid, node), path)); ++let answer_join = answer_idx | join(targets_idx2, ((rid, node), (path, ())) -> ((rid, node), path)); +let answer_idx2 = answer_join | arrange; +let best_idx3 = best | arrange; -+let answer = answer_idx2 | join(best_idx3, ((rid, _), path, cost) -> ((rid, 0), (path, cost))); ++let answer = answer_idx2 | join(best_idx3, ((rid, _), (path, cost)) -> ((rid, 0), (path, cost))); +let answer_idx3 = answer | arrange; +-- Renderer: publish the existing arrangement. +output "answer" = answer_idx3; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic14_oneoff.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic14_oneoff.parsed.ddp index f01258071..315970002 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic14_oneoff.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic14_oneoff.parsed.ddp @@ -4,21 +4,21 @@ let pals = import "graph::friend.pair" : ((int, int), ()); let weighted = import "ic14s::weighted" : (int, (int, int)); let starts = request | map(((rid, from, _), ()) -> ((rid, from), 0)); dist: { - let grow = dist | map(((rid, node, cost), ()) -> (node, (rid, cost))) | join(weighted, (_, (rid, cost), (next, w)) -> ((rid, next), (cost, w) add)); + let grow = dist | map(((rid, node, cost), ()) -> (node, (rid, cost))) | join(weighted, (_, ((rid, cost), (next, w))) -> ((rid, next), (cost, w) add)); set dist = starts + grow | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist) | map(((rid, node), cost) -> ((rid, node, cost), ())); } let best = dist::dist | map(((rid, node, cost), ()) -> ((rid, node), cost)); -let pred = dist::dist | map(((rid, u, cost), ()) -> (u, (rid, cost))) | join(weighted, (u, (rid, cost), (v, w)) -> ((rid, v), (u, (cost, w) add))) | join(best, ((rid, v), (u, via), least) -> ((rid, v), (u, via, least))) | filter((_, (_, via, least)) -> (via, least) eq) | map((key, (u, _, _)) -> (key, u)); -let targets = request | map(((rid, _, to), ()) -> ((rid, to), ())) | join(best, (key, (), _) -> (key, ())); +let pred = dist::dist | map(((rid, u, cost), ()) -> (u, (rid, cost))) | join(weighted, (u, ((rid, cost), (v, w))) -> ((rid, v), (u, (cost, w) add))) | join(best, ((rid, v), ((u, via), least)) -> ((rid, v), (u, via, least))) | filter((_, (_, via, least)) -> (via, least) eq) | map((key, (u, _, _)) -> (key, u)); +let targets = request | map(((rid, _, to), ()) -> ((rid, to), ())) | join(best, (key, ((), _)) -> (key, ())); back: { - let up = back | join(pred, ((rid, _), (), u) -> ((rid, u), ())); + let up = back | join(pred, ((rid, _), ((), u)) -> ((rid, u), ())); set back = targets + up | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); } -let next = pred | join(back::back, ((rid, v), u, ()) -> ((rid, u), v)) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); -let first = request | map(((rid, from, _), ()) -> ((rid, from), ())) | join(back::back, ((rid, from), (), ()) -> ((rid, from, from enlist), ())); +let next = pred | join(back::back, ((rid, v), (u, ())) -> ((rid, u), v)) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); +let first = request | map(((rid, from, _), ()) -> ((rid, from), ())) | join(back::back, ((rid, from), ((), ())) -> ((rid, from, from enlist), ())); walk: { - let step = walk | map(((rid, node, path), ()) -> ((rid, node), path)) | join(next, ((rid, _), path, v) -> ((rid, v, (path, v enlist) append), ())); + let step = walk | map(((rid, node, path), ()) -> ((rid, node), path)) | join(next, ((rid, _), (path, v)) -> ((rid, v, (path, v enlist) append), ())); set walk = first + step; } -let answer = walk::walk | map(((rid, node, path), ()) -> ((rid, node), path)) | join(targets, ((rid, node), path, ()) -> ((rid, node), path)) | join(best, ((rid, _), path, cost) -> ((rid, 0), (path, cost))); +let answer = walk::walk | map(((rid, node, path), ()) -> ((rid, node), path)) | join(targets, ((rid, node), (path, ())) -> ((rid, node), path)) | join(best, ((rid, _), (path, cost)) -> ((rid, 0), (path, cost))); output "answer" = answer | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic14_standing.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic14_standing.ddp index 7ea9f852a..15640ab33 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic14_standing.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic14_standing.ddp @@ -5,7 +5,7 @@ let pals = import "graph::friend.pair" : ((int, int), ()); let answered_rows = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 1) eq) | map((_, (_, _, who, _, _, _, _, _, _, parent, _, _, _)) -> (parent, who)); let answered_idx = answered_rows | arrange; let message_idx = message | arrange; -let answered_join = answered_idx | join(message_idx, (_, who, (_, _, to, _, _, _, _, _, _, _, _, _, _)) -> (who, to)); +let answered_join = answered_idx | join(message_idx, (_, (who, (_, _, to, _, _, _, _, _, _, _, _, _, _))) -> (who, to)); let answered = answered_join | filter((who, to) -> (who, to) ne); let talks_map = answered | map((a, b) -> ((a, b), ())); let talks_map2 = answered | map((a, b) -> ((b, a), ())); @@ -13,7 +13,7 @@ let talks = talks_map + talks_map2; let talks_idx = talks | arrange; let weighted_reduce = talks_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); let pals_idx = pals | arrange; -let weighted_join = weighted_reduce | join(pals_idx, (pair, n, ()) -> (pair, n)); +let weighted_join = weighted_reduce | join(pals_idx, (pair, (n, ())) -> (pair, n)); let weighted = weighted_join | map(((a, b), n) -> (a, (b, (((0, n), (1, 40) range) fold (((acc, n), k) -> ((acc, ((((k, (k, 1) sub) mul, 1) add, n) gt, 1, 0) select) add, n)) .0, 1) add))); let weighted_idx = weighted | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic14_standing.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/ldbc_ic14_standing.lowered--optimized.diff index b63f2011e..9238d725d 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic14_standing.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic14_standing.lowered--optimized.diff @@ -10,5 +10,5 @@ +let answered_rows = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 1) eq) | map((_, (_, _, who, _, _, _, _, _, _, parent, _, _, _)) -> (parent, who)); +let answered_idx = answered_rows | arrange; let message_idx = message | arrange; - let answered_join = answered_idx | join(message_idx, (_, who, (_, _, to, _, _, _, _, _, _, _, _, _, _)) -> (who, to)); + let answered_join = answered_idx | join(message_idx, (_, (who, (_, _, to, _, _, _, _, _, _, _, _, _, _))) -> (who, to)); let answered = answered_join | filter((who, to) -> (who, to) ne); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic14_standing.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic14_standing.parsed--lowered.diff index 926ece7ae..49f7fda28 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic14_standing.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic14_standing.parsed--lowered.diff @@ -5,14 +5,14 @@ +-- Trace imports use provider arrangements; history advances to G = (0, []). let message = import "graph::message" : (int, (int, int, int, int, List(int), List(int), int, List(int), int, int, int, int, int)); let pals = import "graph::friend.pair" : ((int, int), ()); --let answered = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 1) eq) | map((_, (_, _, who, _, _, _, _, _, _, parent, _, _, _)) -> (parent, who)) | join(message, (_, who, (_, _, to, _, _, _, _, _, _, _, _, _, _)) -> (who, to)) | filter((who, to) -> (who, to) ne); +-let answered = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 1) eq) | map((_, (_, _, who, _, _, _, _, _, _, parent, _, _, _)) -> (parent, who)) | join(message, (_, (who, (_, _, to, _, _, _, _, _, _, _, _, _, _))) -> (who, to)) | filter((who, to) -> (who, to) ne); -let talks = (answered | map((a, b) -> ((a, b), ()))) + (answered | map((a, b) -> ((b, a), ()))); --let weighted = talks | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter) | join(pals, (pair, n, ()) -> (pair, n)) | map(((a, b), n) -> (a, (b, (((0, n), (1, 40) range) fold (((acc, n), k) -> +-let weighted = talks | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter) | join(pals, (pair, (n, ())) -> (pair, n)) | map(((a, b), n) -> (a, (b, (((0, n), (1, 40) range) fold (((acc, n), k) -> +let answered_filter = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 1) eq); +let answered_map = answered_filter | map((_, (_, _, who, _, _, _, _, _, _, parent, _, _, _)) -> (parent, who)); +let answered_idx = answered_map | arrange; +let message_idx = message | arrange; -+let answered_join = answered_idx | join(message_idx, (_, who, (_, _, to, _, _, _, _, _, _, _, _, _, _)) -> (who, to)); ++let answered_join = answered_idx | join(message_idx, (_, (who, (_, _, to, _, _, _, _, _, _, _, _, _, _))) -> (who, to)); +let answered = answered_join | filter((who, to) -> (who, to) ne); +let talks_map = answered | map((a, b) -> ((a, b), ())); +let talks_map2 = answered | map((a, b) -> ((b, a), ())); @@ -20,7 +20,7 @@ +let talks_idx = talks | arrange; +let weighted_reduce = talks_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); +let pals_idx = pals | arrange; -+let weighted_join = weighted_reduce | join(pals_idx, (pair, n, ()) -> (pair, n)); ++let weighted_join = weighted_reduce | join(pals_idx, (pair, (n, ())) -> (pair, n)); +let weighted = weighted_join | map(((a, b), n) -> (a, (b, (((0, n), (1, 40) range) fold (((acc, n), k) -> ((acc, ((((k, (k, 1) sub) mul, 1) add, n) gt, 1, 0) select) add, n)) .0, 1) add))); -output "weighted" = weighted | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic14_standing.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic14_standing.parsed.ddp index a252755e4..f461b9ec9 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic14_standing.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic14_standing.parsed.ddp @@ -1,7 +1,7 @@ let message = import "graph::message" : (int, (int, int, int, int, List(int), List(int), int, List(int), int, int, int, int, int)); let pals = import "graph::friend.pair" : ((int, int), ()); -let answered = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 1) eq) | map((_, (_, _, who, _, _, _, _, _, _, parent, _, _, _)) -> (parent, who)) | join(message, (_, who, (_, _, to, _, _, _, _, _, _, _, _, _, _)) -> (who, to)) | filter((who, to) -> (who, to) ne); +let answered = message | filter((_, (kind, _, _, _, _, _, _, _, _, _, _, _, _)) -> (kind, 1) eq) | map((_, (_, _, who, _, _, _, _, _, _, parent, _, _, _)) -> (parent, who)) | join(message, (_, (who, (_, _, to, _, _, _, _, _, _, _, _, _, _))) -> (who, to)) | filter((who, to) -> (who, to) ne); let talks = (answered | map((a, b) -> ((a, b), ()))) + (answered | map((a, b) -> ((b, a), ()))); -let weighted = talks | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter) | join(pals, (pair, n, ()) -> (pair, n)) | map(((a, b), n) -> (a, (b, (((0, n), (1, 40) range) fold (((acc, n), k) -> +let weighted = talks | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter) | join(pals, (pair, (n, ())) -> (pair, n)) | map(((a, b), n) -> (a, (b, (((0, n), (1, 40) range) fold (((acc, n), k) -> ((acc, ((((k, (k, 1) sub) mul, 1) add, n) gt, 1, 0) select) add, n)) .0, 1) add))); output "weighted" = weighted | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_is01.ddp b/interactive/tests/plan_pairs/expected/ldbc_is01.ddp index c4de7b330..7c7150050 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_is01.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_is01.ddp @@ -5,7 +5,7 @@ let person = import "graph::person" : (int, (List(int), List(int), List(int), in let answer_map = request | map(((rid, who), ()) -> (who, rid)); let answer_idx = answer_map | arrange; let person_idx = person | arrange; -let answer = answer_idx | join(person_idx, (_, rid, (first, last, gender, birthday, created, ip, browser, city, _, _, _)) -> ((rid, 0), (first, last, birthday, ip, browser, city, gender, created))); +let answer = answer_idx | join(person_idx, (_, (rid, (first, last, gender, birthday, created, ip, browser, city, _, _, _))) -> ((rid, 0), (first, last, birthday, ip, browser, city, gender, created))); let answer_idx2 = answer | arrange; -- Renderer: publish the existing arrangement. output "answer" = answer_idx2; diff --git a/interactive/tests/plan_pairs/expected/ldbc_is01.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_is01.parsed--lowered.diff index 884638de4..fba3751f3 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_is01.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_is01.parsed--lowered.diff @@ -5,12 +5,12 @@ +-- Trace imports use provider arrangements; history advances to G = (0, []). let request = input 0 : ((int, int), ()); let person = import "graph::person" : (int, (List(int), List(int), List(int), int, int, List(int), List(int), int, int, int, int)); --let answer = request | map(((rid, who), ()) -> (who, rid)) | join(person, (_, rid, (first, last, gender, birthday, created, ip, browser, city, _, _, _)) -> ((rid, 0), (first, last, birthday, ip, browser, city, gender, created))); +-let answer = request | map(((rid, who), ()) -> (who, rid)) | join(person, (_, (rid, (first, last, gender, birthday, created, ip, browser, city, _, _, _))) -> ((rid, 0), (first, last, birthday, ip, browser, city, gender, created))); -output "answer" = answer | arrange; +let answer_map = request | map(((rid, who), ()) -> (who, rid)); +let answer_idx = answer_map | arrange; +let person_idx = person | arrange; -+let answer = answer_idx | join(person_idx, (_, rid, (first, last, gender, birthday, created, ip, browser, city, _, _, _)) -> ((rid, 0), (first, last, birthday, ip, browser, city, gender, created))); ++let answer = answer_idx | join(person_idx, (_, (rid, (first, last, gender, birthday, created, ip, browser, city, _, _, _))) -> ((rid, 0), (first, last, birthday, ip, browser, city, gender, created))); +let answer_idx2 = answer | arrange; +-- Renderer: publish the existing arrangement. +output "answer" = answer_idx2; diff --git a/interactive/tests/plan_pairs/expected/ldbc_is01.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_is01.parsed.ddp index 49d074eba..64a498dcb 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_is01.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_is01.parsed.ddp @@ -1,4 +1,4 @@ let request = input 0 : ((int, int), ()); let person = import "graph::person" : (int, (List(int), List(int), List(int), int, int, List(int), List(int), int, int, int, int)); -let answer = request | map(((rid, who), ()) -> (who, rid)) | join(person, (_, rid, (first, last, gender, birthday, created, ip, browser, city, _, _, _)) -> ((rid, 0), (first, last, birthday, ip, browser, city, gender, created))); +let answer = request | map(((rid, who), ()) -> (who, rid)) | join(person, (_, (rid, (first, last, gender, birthday, created, ip, browser, city, _, _, _))) -> ((rid, 0), (first, last, birthday, ip, browser, city, gender, created))); output "answer" = answer | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_is02.ddp b/interactive/tests/plan_pairs/expected/ldbc_is02.ddp index c9a7c520a..057e83a99 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_is02.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_is02.ddp @@ -8,23 +8,23 @@ let root_of = import "graph::message.root" : (int, int); let mine_map = request | map(((rid, who), ()) -> (who, rid)); let mine_idx = mine_map | arrange; let by_creator_idx = by_creator | arrange; -let mine_join = mine_idx | join(by_creator_idx, (_, rid, id) -> (id, rid)); +let mine_join = mine_idx | join(by_creator_idx, (_, (rid, id)) -> (id, rid)); let mine_idx2 = mine_join | arrange; let message_idx = message | arrange; -let mine_join2 = mine_idx2 | join(message_idx, (id, rid, (_, created, _, _, _, _, _, _, _, _, _, _, _)) -> (id, (rid, created))); +let mine_join2 = mine_idx2 | join(message_idx, (id, (rid, (_, created, _, _, _, _, _, _, _, _, _, _, _))) -> (id, (rid, created))); let mine_idx3 = mine_join2 | arrange; let root_of_idx = root_of | arrange; -let mine = mine_idx3 | join(root_of_idx, (id, (rid, created), root) -> (rid, ((created neg, id neg), (id, created, root)))); +let mine = mine_idx3 | join(root_of_idx, (id, ((rid, created), root)) -> (rid, ((created neg, id neg), (id, created, root)))); let mine_idx4 = mine | arrange; let top = mine_idx4 | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer_map = top | map((rid, (rank, (_, (id, created, root)))) -> (id, (rid, rank, created, root))); let answer_idx = answer_map | arrange; -let answer_join = answer_idx | join(message_idx, (id, (rid, rank, created, root), (_, _, _, _, content, image, _, _, _, _, _, _, _)) -> (root, (rid, rank, id, ((content len, 0) gt, content, image) select, created))); +let answer_join = answer_idx | join(message_idx, (id, ((rid, rank, created, root), (_, _, _, _, content, image, _, _, _, _, _, _, _))) -> (root, (rid, rank, id, ((content len, 0) gt, content, image) select, created))); let answer_idx2 = answer_join | arrange; -let answer_join2 = answer_idx2 | join(message_idx, (root, (rid, rank, id, body, created), (_, _, op, _, _, _, _, _, _, _, _, _, _)) -> (op, (rid, rank, id, body, created, root))); +let answer_join2 = answer_idx2 | join(message_idx, (root, ((rid, rank, id, body, created), (_, _, op, _, _, _, _, _, _, _, _, _, _))) -> (op, (rid, rank, id, body, created, root))); let answer_idx3 = answer_join2 | arrange; let person_idx = person | arrange; -let answer = answer_idx3 | join(person_idx, (op, (rid, rank, id, body, created, root), (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, rank), (id, body, created, root, op, first, last))); +let answer = answer_idx3 | join(person_idx, (op, ((rid, rank, id, body, created, root), (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, rank), (id, body, created, root, op, first, last))); let answer_idx4 = answer | arrange; -- Renderer: publish the existing arrangement. output "answer" = answer_idx4; diff --git a/interactive/tests/plan_pairs/expected/ldbc_is02.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/ldbc_is02.lowered--optimized.diff index 937f46c04..0a81916ab 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_is02.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_is02.lowered--optimized.diff @@ -5,12 +5,12 @@ let answer_map = top | map((rid, (rank, (_, (id, created, root)))) -> (id, (rid, rank, created, root))); let answer_idx = answer_map | arrange; -let message_idx2 = message | arrange; --let answer_join = answer_idx | join(message_idx2, (id, (rid, rank, created, root), (_, _, _, _, content, image, _, _, _, _, _, _, _)) -> (root, (rid, rank, id, ((content len, 0) gt, content, image) select, created))); -+let answer_join = answer_idx | join(message_idx, (id, (rid, rank, created, root), (_, _, _, _, content, image, _, _, _, _, _, _, _)) -> (root, (rid, rank, id, ((content len, 0) gt, content, image) select, created))); +-let answer_join = answer_idx | join(message_idx2, (id, ((rid, rank, created, root), (_, _, _, _, content, image, _, _, _, _, _, _, _))) -> (root, (rid, rank, id, ((content len, 0) gt, content, image) select, created))); ++let answer_join = answer_idx | join(message_idx, (id, ((rid, rank, created, root), (_, _, _, _, content, image, _, _, _, _, _, _, _))) -> (root, (rid, rank, id, ((content len, 0) gt, content, image) select, created))); let answer_idx2 = answer_join | arrange; -let message_idx3 = message | arrange; --let answer_join2 = answer_idx2 | join(message_idx3, (root, (rid, rank, id, body, created), (_, _, op, _, _, _, _, _, _, _, _, _, _)) -> (op, (rid, rank, id, body, created, root))); -+let answer_join2 = answer_idx2 | join(message_idx, (root, (rid, rank, id, body, created), (_, _, op, _, _, _, _, _, _, _, _, _, _)) -> (op, (rid, rank, id, body, created, root))); +-let answer_join2 = answer_idx2 | join(message_idx3, (root, ((rid, rank, id, body, created), (_, _, op, _, _, _, _, _, _, _, _, _, _))) -> (op, (rid, rank, id, body, created, root))); ++let answer_join2 = answer_idx2 | join(message_idx, (root, ((rid, rank, id, body, created), (_, _, op, _, _, _, _, _, _, _, _, _, _))) -> (op, (rid, rank, id, body, created, root))); let answer_idx3 = answer_join2 | arrange; let person_idx = person | arrange; - let answer = answer_idx3 | join(person_idx, (op, (rid, rank, id, body, created, root), (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, rank), (id, body, created, root, op, first, last))); + let answer = answer_idx3 | join(person_idx, (op, ((rid, rank, id, body, created, root), (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, rank), (id, body, created, root, op, first, last))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_is02.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_is02.parsed--lowered.diff index 3a824488c..3289011f8 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_is02.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_is02.parsed--lowered.diff @@ -8,32 +8,32 @@ let message = import "graph::message" : (int, (int, int, int, int, List(int), List(int), int, List(int), int, int, int, int, int)); let by_creator = import "graph::message.by_creator" : (int, int); let root_of = import "graph::message.root" : (int, int); --let mine = request | map(((rid, who), ()) -> (who, rid)) | join(by_creator, (_, rid, id) -> (id, rid)) | join(message, (id, rid, (_, created, _, _, _, _, _, _, _, _, _, _, _)) -> (id, (rid, created))) | join(root_of, (id, (rid, created), root) -> (rid, ((created neg, id neg), (id, created, root)))); +-let mine = request | map(((rid, who), ()) -> (who, rid)) | join(by_creator, (_, (rid, id)) -> (id, rid)) | join(message, (id, (rid, (_, created, _, _, _, _, _, _, _, _, _, _, _))) -> (id, (rid, created))) | join(root_of, (id, ((rid, created), root)) -> (rid, ((created neg, id neg), (id, created, root)))); -let top = mine | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); --let answer = top | map((rid, (rank, (_, (id, created, root)))) -> (id, (rid, rank, created, root))) | join(message, (id, (rid, rank, created, root), (_, _, _, _, content, image, _, _, _, _, _, _, _)) -> (root, (rid, rank, id, ((content len, 0) gt, content, image) select, created))) | join(message, (root, (rid, rank, id, body, created), (_, _, op, _, _, _, _, _, _, _, _, _, _)) -> (op, (rid, rank, id, body, created, root))) | join(person, (op, (rid, rank, id, body, created, root), (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, rank), (id, body, created, root, op, first, last))); +-let answer = top | map((rid, (rank, (_, (id, created, root)))) -> (id, (rid, rank, created, root))) | join(message, (id, ((rid, rank, created, root), (_, _, _, _, content, image, _, _, _, _, _, _, _))) -> (root, (rid, rank, id, ((content len, 0) gt, content, image) select, created))) | join(message, (root, ((rid, rank, id, body, created), (_, _, op, _, _, _, _, _, _, _, _, _, _))) -> (op, (rid, rank, id, body, created, root))) | join(person, (op, ((rid, rank, id, body, created, root), (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, rank), (id, body, created, root, op, first, last))); -output "answer" = answer | arrange; +let mine_map = request | map(((rid, who), ()) -> (who, rid)); +let mine_idx = mine_map | arrange; +let by_creator_idx = by_creator | arrange; -+let mine_join = mine_idx | join(by_creator_idx, (_, rid, id) -> (id, rid)); ++let mine_join = mine_idx | join(by_creator_idx, (_, (rid, id)) -> (id, rid)); +let mine_idx2 = mine_join | arrange; +let message_idx = message | arrange; -+let mine_join2 = mine_idx2 | join(message_idx, (id, rid, (_, created, _, _, _, _, _, _, _, _, _, _, _)) -> (id, (rid, created))); ++let mine_join2 = mine_idx2 | join(message_idx, (id, (rid, (_, created, _, _, _, _, _, _, _, _, _, _, _))) -> (id, (rid, created))); +let mine_idx3 = mine_join2 | arrange; +let root_of_idx = root_of | arrange; -+let mine = mine_idx3 | join(root_of_idx, (id, (rid, created), root) -> (rid, ((created neg, id neg), (id, created, root)))); ++let mine = mine_idx3 | join(root_of_idx, (id, ((rid, created), root)) -> (rid, ((created neg, id neg), (id, created, root)))); +let mine_idx4 = mine | arrange; +let top = mine_idx4 | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); +let answer_map = top | map((rid, (rank, (_, (id, created, root)))) -> (id, (rid, rank, created, root))); +let answer_idx = answer_map | arrange; +let message_idx2 = message | arrange; -+let answer_join = answer_idx | join(message_idx2, (id, (rid, rank, created, root), (_, _, _, _, content, image, _, _, _, _, _, _, _)) -> (root, (rid, rank, id, ((content len, 0) gt, content, image) select, created))); ++let answer_join = answer_idx | join(message_idx2, (id, ((rid, rank, created, root), (_, _, _, _, content, image, _, _, _, _, _, _, _))) -> (root, (rid, rank, id, ((content len, 0) gt, content, image) select, created))); +let answer_idx2 = answer_join | arrange; +let message_idx3 = message | arrange; -+let answer_join2 = answer_idx2 | join(message_idx3, (root, (rid, rank, id, body, created), (_, _, op, _, _, _, _, _, _, _, _, _, _)) -> (op, (rid, rank, id, body, created, root))); ++let answer_join2 = answer_idx2 | join(message_idx3, (root, ((rid, rank, id, body, created), (_, _, op, _, _, _, _, _, _, _, _, _, _))) -> (op, (rid, rank, id, body, created, root))); +let answer_idx3 = answer_join2 | arrange; +let person_idx = person | arrange; -+let answer = answer_idx3 | join(person_idx, (op, (rid, rank, id, body, created, root), (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, rank), (id, body, created, root, op, first, last))); ++let answer = answer_idx3 | join(person_idx, (op, ((rid, rank, id, body, created, root), (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, rank), (id, body, created, root, op, first, last))); +let answer_idx4 = answer | arrange; +-- Renderer: publish the existing arrangement. +output "answer" = answer_idx4; diff --git a/interactive/tests/plan_pairs/expected/ldbc_is02.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_is02.parsed.ddp index 58b530cd8..268084338 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_is02.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_is02.parsed.ddp @@ -3,7 +3,7 @@ let person = import "graph::person" : (int, (List(int), List(int), List(int), in let message = import "graph::message" : (int, (int, int, int, int, List(int), List(int), int, List(int), int, int, int, int, int)); let by_creator = import "graph::message.by_creator" : (int, int); let root_of = import "graph::message.root" : (int, int); -let mine = request | map(((rid, who), ()) -> (who, rid)) | join(by_creator, (_, rid, id) -> (id, rid)) | join(message, (id, rid, (_, created, _, _, _, _, _, _, _, _, _, _, _)) -> (id, (rid, created))) | join(root_of, (id, (rid, created), root) -> (rid, ((created neg, id neg), (id, created, root)))); +let mine = request | map(((rid, who), ()) -> (who, rid)) | join(by_creator, (_, (rid, id)) -> (id, rid)) | join(message, (id, (rid, (_, created, _, _, _, _, _, _, _, _, _, _, _))) -> (id, (rid, created))) | join(root_of, (id, ((rid, created), root)) -> (rid, ((created neg, id neg), (id, created, root)))); let top = mine | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 10) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 10) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 10) min in let before = (end, count) sub in let keep = (((10, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); -let answer = top | map((rid, (rank, (_, (id, created, root)))) -> (id, (rid, rank, created, root))) | join(message, (id, (rid, rank, created, root), (_, _, _, _, content, image, _, _, _, _, _, _, _)) -> (root, (rid, rank, id, ((content len, 0) gt, content, image) select, created))) | join(message, (root, (rid, rank, id, body, created), (_, _, op, _, _, _, _, _, _, _, _, _, _)) -> (op, (rid, rank, id, body, created, root))) | join(person, (op, (rid, rank, id, body, created, root), (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, rank), (id, body, created, root, op, first, last))); +let answer = top | map((rid, (rank, (_, (id, created, root)))) -> (id, (rid, rank, created, root))) | join(message, (id, ((rid, rank, created, root), (_, _, _, _, content, image, _, _, _, _, _, _, _))) -> (root, (rid, rank, id, ((content len, 0) gt, content, image) select, created))) | join(message, (root, ((rid, rank, id, body, created), (_, _, op, _, _, _, _, _, _, _, _, _, _))) -> (op, (rid, rank, id, body, created, root))) | join(person, (op, ((rid, rank, id, body, created, root), (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, rank), (id, body, created, root, op, first, last))); output "answer" = answer | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_is03.ddp b/interactive/tests/plan_pairs/expected/ldbc_is03.ddp index bd66b0392..97f89a4c1 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_is03.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_is03.ddp @@ -6,10 +6,10 @@ let friend = import "graph::friend" : (int, (int, int)); let ranked_map = request | map(((rid, who), ()) -> (who, rid)); let ranked_idx = ranked_map | arrange; let friend_idx = friend | arrange; -let ranked_join = ranked_idx | join(friend_idx, (_, rid, (f, since)) -> (f, (rid, since))); +let ranked_join = ranked_idx | join(friend_idx, (_, (rid, (f, since))) -> (f, (rid, since))); let ranked_idx2 = ranked_join | arrange; let person_idx = person | arrange; -let ranked_join2 = ranked_idx2 | join(person_idx, (f, (rid, since), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, ((since neg, f), (first, last)))); +let ranked_join2 = ranked_idx2 | join(person_idx, (f, ((rid, since), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((since neg, f), (first, last)))); let ranked_idx3 = ranked_join2 | arrange; let ranked_reduce = ranked_idx3 | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); let answer = ranked_reduce | flatmap((k, entries) -> (k, ((0, entries len) range, entries) zip) cap_list) | map((rid, (rank, ((neg, f), (first, last)))) -> ((rid, rank), (f, first, last, neg neg))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_is03.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/ldbc_is03.lowered--optimized.diff index ef763c8b0..a4742f5a9 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_is03.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_is03.lowered--optimized.diff @@ -1,7 +1,7 @@ --- lowered +++ optimized @@ -12,8 +12,7 @@ - let ranked_join2 = ranked_idx2 | join(person_idx, (f, (rid, since), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, ((since neg, f), (first, last)))); + let ranked_join2 = ranked_idx2 | join(person_idx, (f, ((rid, since), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((since neg, f), (first, last)))); let ranked_idx3 = ranked_join2 | arrange; let ranked_reduce = ranked_idx3 | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); -let ranked = ranked_reduce | flatmap((k, entries) -> (k, ((0, entries len) range, entries) zip) cap_list); diff --git a/interactive/tests/plan_pairs/expected/ldbc_is03.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_is03.parsed--lowered.diff index f27ca031f..d1e5921bc 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_is03.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_is03.parsed--lowered.diff @@ -6,14 +6,14 @@ let request = input 0 : ((int, int), ()); let person = import "graph::person" : (int, (List(int), List(int), List(int), int, int, List(int), List(int), int, int, int, int)); let friend = import "graph::friend" : (int, (int, int)); --let ranked = request | map(((rid, who), ()) -> (who, rid)) | join(friend, (_, rid, (f, since)) -> (f, (rid, since))) | join(person, (f, (rid, since), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, ((since neg, f), (first, last)))) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist) | flatmap((k, entries) -> (k, ((0, entries len) range, entries) zip) cap_list); +-let ranked = request | map(((rid, who), ()) -> (who, rid)) | join(friend, (_, (rid, (f, since))) -> (f, (rid, since))) | join(person, (f, ((rid, since), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((since neg, f), (first, last)))) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist) | flatmap((k, entries) -> (k, ((0, entries len) range, entries) zip) cap_list); +let ranked_map = request | map(((rid, who), ()) -> (who, rid)); +let ranked_idx = ranked_map | arrange; +let friend_idx = friend | arrange; -+let ranked_join = ranked_idx | join(friend_idx, (_, rid, (f, since)) -> (f, (rid, since))); ++let ranked_join = ranked_idx | join(friend_idx, (_, (rid, (f, since))) -> (f, (rid, since))); +let ranked_idx2 = ranked_join | arrange; +let person_idx = person | arrange; -+let ranked_join2 = ranked_idx2 | join(person_idx, (f, (rid, since), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, ((since neg, f), (first, last)))); ++let ranked_join2 = ranked_idx2 | join(person_idx, (f, ((rid, since), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((since neg, f), (first, last)))); +let ranked_idx3 = ranked_join2 | arrange; +let ranked_reduce = ranked_idx3 | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); +let ranked = ranked_reduce | flatmap((k, entries) -> (k, ((0, entries len) range, entries) zip) cap_list); diff --git a/interactive/tests/plan_pairs/expected/ldbc_is03.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_is03.parsed.ddp index 1601d86b0..f7e88985c 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_is03.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_is03.parsed.ddp @@ -1,6 +1,6 @@ let request = input 0 : ((int, int), ()); let person = import "graph::person" : (int, (List(int), List(int), List(int), int, int, List(int), List(int), int, int, int, int)); let friend = import "graph::friend" : (int, (int, int)); -let ranked = request | map(((rid, who), ()) -> (who, rid)) | join(friend, (_, rid, (f, since)) -> (f, (rid, since))) | join(person, (f, (rid, since), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, ((since neg, f), (first, last)))) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist) | flatmap((k, entries) -> (k, ((0, entries len) range, entries) zip) cap_list); +let ranked = request | map(((rid, who), ()) -> (who, rid)) | join(friend, (_, (rid, (f, since))) -> (f, (rid, since))) | join(person, (f, ((rid, since), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((since neg, f), (first, last)))) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist) | flatmap((k, entries) -> (k, ((0, entries len) range, entries) zip) cap_list); let answer = ranked | map((rid, (rank, ((neg, f), (first, last)))) -> ((rid, rank), (f, first, last, neg neg))); output "answer" = answer | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_is04.ddp b/interactive/tests/plan_pairs/expected/ldbc_is04.ddp index c17e1ad19..c9dd88a85 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_is04.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_is04.ddp @@ -5,7 +5,7 @@ let message = import "graph::message" : (int, (int, int, int, int, List(int), Li let answer_map = request | map(((rid, m), ()) -> (m, rid)); let answer_idx = answer_map | arrange; let message_idx = message | arrange; -let answer = answer_idx | join(message_idx, (_, rid, (_, created, _, _, text, image, _, _, _, _, _, _, _)) -> ((rid, 0), (created, ((text len, 0) gt, text, image) select))); +let answer = answer_idx | join(message_idx, (_, (rid, (_, created, _, _, text, image, _, _, _, _, _, _, _))) -> ((rid, 0), (created, ((text len, 0) gt, text, image) select))); let answer_idx2 = answer | arrange; -- Renderer: publish the existing arrangement. output "answer" = answer_idx2; diff --git a/interactive/tests/plan_pairs/expected/ldbc_is04.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_is04.parsed--lowered.diff index ff621ea75..8df8f241b 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_is04.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_is04.parsed--lowered.diff @@ -5,12 +5,12 @@ +-- Trace imports use provider arrangements; history advances to G = (0, []). let request = input 0 : ((int, int), ()); let message = import "graph::message" : (int, (int, int, int, int, List(int), List(int), int, List(int), int, int, int, int, int)); --let answer = request | map(((rid, m), ()) -> (m, rid)) | join(message, (_, rid, (_, created, _, _, text, image, _, _, _, _, _, _, _)) -> ((rid, 0), (created, ((text len, 0) gt, text, image) select))); +-let answer = request | map(((rid, m), ()) -> (m, rid)) | join(message, (_, (rid, (_, created, _, _, text, image, _, _, _, _, _, _, _))) -> ((rid, 0), (created, ((text len, 0) gt, text, image) select))); -output "answer" = answer | arrange; +let answer_map = request | map(((rid, m), ()) -> (m, rid)); +let answer_idx = answer_map | arrange; +let message_idx = message | arrange; -+let answer = answer_idx | join(message_idx, (_, rid, (_, created, _, _, text, image, _, _, _, _, _, _, _)) -> ((rid, 0), (created, ((text len, 0) gt, text, image) select))); ++let answer = answer_idx | join(message_idx, (_, (rid, (_, created, _, _, text, image, _, _, _, _, _, _, _))) -> ((rid, 0), (created, ((text len, 0) gt, text, image) select))); +let answer_idx2 = answer | arrange; +-- Renderer: publish the existing arrangement. +output "answer" = answer_idx2; diff --git a/interactive/tests/plan_pairs/expected/ldbc_is04.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_is04.parsed.ddp index 1a13b4bb5..0ce9c485c 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_is04.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_is04.parsed.ddp @@ -1,4 +1,4 @@ let request = input 0 : ((int, int), ()); let message = import "graph::message" : (int, (int, int, int, int, List(int), List(int), int, List(int), int, int, int, int, int)); -let answer = request | map(((rid, m), ()) -> (m, rid)) | join(message, (_, rid, (_, created, _, _, text, image, _, _, _, _, _, _, _)) -> ((rid, 0), (created, ((text len, 0) gt, text, image) select))); +let answer = request | map(((rid, m), ()) -> (m, rid)) | join(message, (_, (rid, (_, created, _, _, text, image, _, _, _, _, _, _, _))) -> ((rid, 0), (created, ((text len, 0) gt, text, image) select))); output "answer" = answer | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_is05.ddp b/interactive/tests/plan_pairs/expected/ldbc_is05.ddp index 9172fc792..d40cc6c70 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_is05.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_is05.ddp @@ -6,10 +6,10 @@ let message = import "graph::message" : (int, (int, int, int, int, List(int), Li let answer_map = request | map(((rid, m), ()) -> (m, rid)); let answer_idx = answer_map | arrange; let message_idx = message | arrange; -let answer_join = answer_idx | join(message_idx, (_, rid, (_, _, who, _, _, _, _, _, _, _, _, _, _)) -> (who, rid)); +let answer_join = answer_idx | join(message_idx, (_, (rid, (_, _, who, _, _, _, _, _, _, _, _, _, _))) -> (who, rid)); let answer_idx2 = answer_join | arrange; let person_idx = person | arrange; -let answer = answer_idx2 | join(person_idx, (who, rid, (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, 0), (who, first, last))); +let answer = answer_idx2 | join(person_idx, (who, (rid, (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, 0), (who, first, last))); let answer_idx3 = answer | arrange; -- Renderer: publish the existing arrangement. output "answer" = answer_idx3; diff --git a/interactive/tests/plan_pairs/expected/ldbc_is05.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_is05.parsed--lowered.diff index a91541d9f..65d2d4f31 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_is05.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_is05.parsed--lowered.diff @@ -6,15 +6,15 @@ let request = input 0 : ((int, int), ()); let person = import "graph::person" : (int, (List(int), List(int), List(int), int, int, List(int), List(int), int, int, int, int)); let message = import "graph::message" : (int, (int, int, int, int, List(int), List(int), int, List(int), int, int, int, int, int)); --let answer = request | map(((rid, m), ()) -> (m, rid)) | join(message, (_, rid, (_, _, who, _, _, _, _, _, _, _, _, _, _)) -> (who, rid)) | join(person, (who, rid, (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, 0), (who, first, last))); +-let answer = request | map(((rid, m), ()) -> (m, rid)) | join(message, (_, (rid, (_, _, who, _, _, _, _, _, _, _, _, _, _))) -> (who, rid)) | join(person, (who, (rid, (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, 0), (who, first, last))); -output "answer" = answer | arrange; +let answer_map = request | map(((rid, m), ()) -> (m, rid)); +let answer_idx = answer_map | arrange; +let message_idx = message | arrange; -+let answer_join = answer_idx | join(message_idx, (_, rid, (_, _, who, _, _, _, _, _, _, _, _, _, _)) -> (who, rid)); ++let answer_join = answer_idx | join(message_idx, (_, (rid, (_, _, who, _, _, _, _, _, _, _, _, _, _))) -> (who, rid)); +let answer_idx2 = answer_join | arrange; +let person_idx = person | arrange; -+let answer = answer_idx2 | join(person_idx, (who, rid, (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, 0), (who, first, last))); ++let answer = answer_idx2 | join(person_idx, (who, (rid, (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, 0), (who, first, last))); +let answer_idx3 = answer | arrange; +-- Renderer: publish the existing arrangement. +output "answer" = answer_idx3; diff --git a/interactive/tests/plan_pairs/expected/ldbc_is05.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_is05.parsed.ddp index 4fe9617ec..4fdda8fd9 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_is05.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_is05.parsed.ddp @@ -1,5 +1,5 @@ let request = input 0 : ((int, int), ()); let person = import "graph::person" : (int, (List(int), List(int), List(int), int, int, List(int), List(int), int, int, int, int)); let message = import "graph::message" : (int, (int, int, int, int, List(int), List(int), int, List(int), int, int, int, int, int)); -let answer = request | map(((rid, m), ()) -> (m, rid)) | join(message, (_, rid, (_, _, who, _, _, _, _, _, _, _, _, _, _)) -> (who, rid)) | join(person, (who, rid, (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, 0), (who, first, last))); +let answer = request | map(((rid, m), ()) -> (m, rid)) | join(message, (_, (rid, (_, _, who, _, _, _, _, _, _, _, _, _, _))) -> (who, rid)) | join(person, (who, (rid, (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, 0), (who, first, last))); output "answer" = answer | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_is06.ddp b/interactive/tests/plan_pairs/expected/ldbc_is06.ddp index 1e8a40034..c7db7e9d1 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_is06.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_is06.ddp @@ -8,16 +8,16 @@ let root_of = import "graph::message.root" : (int, int); let answer_map = request | map(((rid, m), ()) -> (m, rid)); let answer_idx = answer_map | arrange; let root_of_idx = root_of | arrange; -let answer_join = answer_idx | join(root_of_idx, (_, rid, root) -> (root, rid)); +let answer_join = answer_idx | join(root_of_idx, (_, (rid, root)) -> (root, rid)); let answer_idx2 = answer_join | arrange; let message_idx = message | arrange; -let answer_join2 = answer_idx2 | join(message_idx, (_, rid, (_, _, _, _, _, _, _, _, f, _, _, _, _)) -> (f, rid)); +let answer_join2 = answer_idx2 | join(message_idx, (_, (rid, (_, _, _, _, _, _, _, _, f, _, _, _, _))) -> (f, rid)); let answer_idx3 = answer_join2 | arrange; let forum_idx = forum | arrange; -let answer_join3 = answer_idx3 | join(forum_idx, (f, rid, (title, _, moderator)) -> (moderator, (rid, f, title))); +let answer_join3 = answer_idx3 | join(forum_idx, (f, (rid, (title, _, moderator))) -> (moderator, (rid, f, title))); let answer_idx4 = answer_join3 | arrange; let person_idx = person | arrange; -let answer = answer_idx4 | join(person_idx, (moderator, (rid, f, title), (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, 0), (f, title, moderator, first, last))); +let answer = answer_idx4 | join(person_idx, (moderator, ((rid, f, title), (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, 0), (f, title, moderator, first, last))); let answer_idx5 = answer | arrange; -- Renderer: publish the existing arrangement. output "answer" = answer_idx5; diff --git a/interactive/tests/plan_pairs/expected/ldbc_is06.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_is06.parsed--lowered.diff index 37aefc16c..94c24e1fc 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_is06.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_is06.parsed--lowered.diff @@ -8,21 +8,21 @@ let message = import "graph::message" : (int, (int, int, int, int, List(int), List(int), int, List(int), int, int, int, int, int)); let forum = import "graph::forum" : (int, (List(int), int, int)); let root_of = import "graph::message.root" : (int, int); --let answer = request | map(((rid, m), ()) -> (m, rid)) | join(root_of, (_, rid, root) -> (root, rid)) | join(message, (_, rid, (_, _, _, _, _, _, _, _, f, _, _, _, _)) -> (f, rid)) | join(forum, (f, rid, (title, _, moderator)) -> (moderator, (rid, f, title))) | join(person, (moderator, (rid, f, title), (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, 0), (f, title, moderator, first, last))); +-let answer = request | map(((rid, m), ()) -> (m, rid)) | join(root_of, (_, (rid, root)) -> (root, rid)) | join(message, (_, (rid, (_, _, _, _, _, _, _, _, f, _, _, _, _))) -> (f, rid)) | join(forum, (f, (rid, (title, _, moderator))) -> (moderator, (rid, f, title))) | join(person, (moderator, ((rid, f, title), (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, 0), (f, title, moderator, first, last))); -output "answer" = answer | arrange; +let answer_map = request | map(((rid, m), ()) -> (m, rid)); +let answer_idx = answer_map | arrange; +let root_of_idx = root_of | arrange; -+let answer_join = answer_idx | join(root_of_idx, (_, rid, root) -> (root, rid)); ++let answer_join = answer_idx | join(root_of_idx, (_, (rid, root)) -> (root, rid)); +let answer_idx2 = answer_join | arrange; +let message_idx = message | arrange; -+let answer_join2 = answer_idx2 | join(message_idx, (_, rid, (_, _, _, _, _, _, _, _, f, _, _, _, _)) -> (f, rid)); ++let answer_join2 = answer_idx2 | join(message_idx, (_, (rid, (_, _, _, _, _, _, _, _, f, _, _, _, _))) -> (f, rid)); +let answer_idx3 = answer_join2 | arrange; +let forum_idx = forum | arrange; -+let answer_join3 = answer_idx3 | join(forum_idx, (f, rid, (title, _, moderator)) -> (moderator, (rid, f, title))); ++let answer_join3 = answer_idx3 | join(forum_idx, (f, (rid, (title, _, moderator))) -> (moderator, (rid, f, title))); +let answer_idx4 = answer_join3 | arrange; +let person_idx = person | arrange; -+let answer = answer_idx4 | join(person_idx, (moderator, (rid, f, title), (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, 0), (f, title, moderator, first, last))); ++let answer = answer_idx4 | join(person_idx, (moderator, ((rid, f, title), (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, 0), (f, title, moderator, first, last))); +let answer_idx5 = answer | arrange; +-- Renderer: publish the existing arrangement. +output "answer" = answer_idx5; diff --git a/interactive/tests/plan_pairs/expected/ldbc_is06.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_is06.parsed.ddp index c182bbb34..7d72131a3 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_is06.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_is06.parsed.ddp @@ -3,5 +3,5 @@ let person = import "graph::person" : (int, (List(int), List(int), List(int), in let message = import "graph::message" : (int, (int, int, int, int, List(int), List(int), int, List(int), int, int, int, int, int)); let forum = import "graph::forum" : (int, (List(int), int, int)); let root_of = import "graph::message.root" : (int, int); -let answer = request | map(((rid, m), ()) -> (m, rid)) | join(root_of, (_, rid, root) -> (root, rid)) | join(message, (_, rid, (_, _, _, _, _, _, _, _, f, _, _, _, _)) -> (f, rid)) | join(forum, (f, rid, (title, _, moderator)) -> (moderator, (rid, f, title))) | join(person, (moderator, (rid, f, title), (first, last, _, _, _, _, _, _, _, _, _)) -> ((rid, 0), (f, title, moderator, first, last))); +let answer = request | map(((rid, m), ()) -> (m, rid)) | join(root_of, (_, (rid, root)) -> (root, rid)) | join(message, (_, (rid, (_, _, _, _, _, _, _, _, f, _, _, _, _))) -> (f, rid)) | join(forum, (f, (rid, (title, _, moderator))) -> (moderator, (rid, f, title))) | join(person, (moderator, ((rid, f, title), (first, last, _, _, _, _, _, _, _, _, _))) -> ((rid, 0), (f, title, moderator, first, last))); output "answer" = answer | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_is07.ddp b/interactive/tests/plan_pairs/expected/ldbc_is07.ddp index c23e8da70..b31405ba1 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_is07.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_is07.ddp @@ -8,17 +8,17 @@ let friend = import "graph::friend" : (int, (int, int)); let reply_map = request | map(((rid, m), ()) -> (m, rid)); let reply_idx = reply_map | arrange; let message_idx = message | arrange; -let reply_join = reply_idx | join(message_idx, (m, rid, (_, _, op, _, _, _, _, _, _, _, _, _, _)) -> (m, (rid, op))); +let reply_join = reply_idx | join(message_idx, (m, (rid, (_, _, op, _, _, _, _, _, _, _, _, _, _))) -> (m, (rid, op))); let reply_idx2 = reply_join | arrange; let by_parent_idx = by_parent | arrange; -let reply_join2 = reply_idx2 | join(by_parent_idx, (_, (rid, op), id) -> (id, (rid, op))); +let reply_join2 = reply_idx2 | join(by_parent_idx, (_, ((rid, op), id)) -> (id, (rid, op))); let reply_idx3 = reply_join2 | arrange; -let reply_join3 = reply_idx3 | join(message_idx, (id, (rid, op), (kind, created, who, _, text, _, _, _, _, _, _, _, _)) -> (kind, ((who, op), (rid, id, text, created)))); +let reply_join3 = reply_idx3 | join(message_idx, (id, ((rid, op), (kind, created, who, _, text, _, _, _, _, _, _, _, _))) -> (kind, ((who, op), (rid, id, text, created)))); let reply = reply_join3 | filter((kind, _) -> (kind, 1) eq) | map((_, row) -> row); let known_map = reply | map(((who, op), row) -> (who, (op, row))); let known_idx = known_map | arrange; let friend_idx = friend | arrange; -let known_join = known_idx | join(friend_idx, (who, (op, row), (f, _)) -> ((f, op) eq, ((who, op), row))); +let known_join = known_idx | join(friend_idx, (who, ((op, row), (f, _))) -> ((f, op) eq, ((who, op), row))); let known_rows = known_join | filter((same, _) -> same) | map((_, (pair, row)) -> ((pair, row), ())); let known_idx2 = known_rows | arrange; let known_reduce = known_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); @@ -31,7 +31,7 @@ let marked_map3 = marked_rows | map(((who, _), row) -> (who, (row, 0))); let marked = marked_map + marked_map3; let marked_idx = marked | arrange; let person_idx = person | arrange; -let ranked_join = marked_idx | join(person_idx, (who, ((rid, id, text, created), k), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, ((created neg, who), (id, text, first, last, k)))); +let ranked_join = marked_idx | join(person_idx, (who, (((rid, id, text, created), k), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((created neg, who), (id, text, first, last, k)))); let ranked_idx = ranked_join | arrange; let ranked_reduce = ranked_idx | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); let answer = ranked_reduce | flatmap((k, entries) -> (k, ((0, entries len) range, entries) zip) cap_list) | map((rid, (rank, ((neg, who), (id, text, first, last, k)))) -> ((rid, rank), (id, text, neg neg, who, first, last, k))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_is07.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/ldbc_is07.lowered--optimized.diff index f9718c148..3509da2b7 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_is07.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_is07.lowered--optimized.diff @@ -2,18 +2,18 @@ +++ optimized @@ -13,17 +13,14 @@ let by_parent_idx = by_parent | arrange; - let reply_join2 = reply_idx2 | join(by_parent_idx, (_, (rid, op), id) -> (id, (rid, op))); + let reply_join2 = reply_idx2 | join(by_parent_idx, (_, ((rid, op), id)) -> (id, (rid, op))); let reply_idx3 = reply_join2 | arrange; -let message_idx2 = message | arrange; --let reply_join3 = reply_idx3 | join(message_idx2, (id, (rid, op), (kind, created, who, _, text, _, _, _, _, _, _, _, _)) -> (kind, ((who, op), (rid, id, text, created)))); +-let reply_join3 = reply_idx3 | join(message_idx2, (id, ((rid, op), (kind, created, who, _, text, _, _, _, _, _, _, _, _))) -> (kind, ((who, op), (rid, id, text, created)))); -let reply_filter = reply_join3 | filter((kind, _) -> (kind, 1) eq); -let reply = reply_filter | map((_, row) -> row); -+let reply_join3 = reply_idx3 | join(message_idx, (id, (rid, op), (kind, created, who, _, text, _, _, _, _, _, _, _, _)) -> (kind, ((who, op), (rid, id, text, created)))); ++let reply_join3 = reply_idx3 | join(message_idx, (id, ((rid, op), (kind, created, who, _, text, _, _, _, _, _, _, _, _))) -> (kind, ((who, op), (rid, id, text, created)))); +let reply = reply_join3 | filter((kind, _) -> (kind, 1) eq) | map((_, row) -> row); let known_map = reply | map(((who, op), row) -> (who, (op, row))); let known_idx = known_map | arrange; let friend_idx = friend | arrange; - let known_join = known_idx | join(friend_idx, (who, (op, row), (f, _)) -> ((f, op) eq, ((who, op), row))); + let known_join = known_idx | join(friend_idx, (who, ((op, row), (f, _))) -> ((f, op) eq, ((who, op), row))); -let known_filter = known_join | filter((same, _) -> same); -let known_map2 = known_filter | map((_, (pair, row)) -> ((pair, row), ())); -let known_idx2 = known_map2 | arrange; @@ -23,7 +23,7 @@ let known = known_reduce | map(((pair, row), ()) -> (pair, row)); let marked_map = known | map(((who, _), row) -> (who, (row, 1))); @@ -36,8 +33,7 @@ - let ranked_join = marked_idx | join(person_idx, (who, ((rid, id, text, created), k), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, ((created neg, who), (id, text, first, last, k)))); + let ranked_join = marked_idx | join(person_idx, (who, (((rid, id, text, created), k), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((created neg, who), (id, text, first, last, k)))); let ranked_idx = ranked_join | arrange; let ranked_reduce = ranked_idx | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); -let ranked = ranked_reduce | flatmap((k, entries) -> (k, ((0, entries len) range, entries) zip) cap_list); diff --git a/interactive/tests/plan_pairs/expected/ldbc_is07.optimized--projected.diff b/interactive/tests/plan_pairs/expected/ldbc_is07.optimized--projected.diff index 0891b264a..bae82304e 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_is07.optimized--projected.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_is07.optimized--projected.diff @@ -17,4 +17,4 @@ +let marked = marked_map + marked_map3; let marked_idx = marked | arrange; let person_idx = person | arrange; - let ranked_join = marked_idx | join(person_idx, (who, ((rid, id, text, created), k), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, ((created neg, who), (id, text, first, last, k)))); + let ranked_join = marked_idx | join(person_idx, (who, (((rid, id, text, created), k), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((created neg, who), (id, text, first, last, k)))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_is07.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_is07.parsed--lowered.diff index 2430be463..88708f3cf 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_is07.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_is07.parsed--lowered.diff @@ -8,26 +8,26 @@ let message = import "graph::message" : (int, (int, int, int, int, List(int), List(int), int, List(int), int, int, int, int, int)); let by_parent = import "graph::message.by_parent" : (int, int); let friend = import "graph::friend" : (int, (int, int)); --let reply = request | map(((rid, m), ()) -> (m, rid)) | join(message, (m, rid, (_, _, op, _, _, _, _, _, _, _, _, _, _)) -> (m, (rid, op))) | join(by_parent, (_, (rid, op), id) -> (id, (rid, op))) | join(message, (id, (rid, op), (kind, created, who, _, text, _, _, _, _, _, _, _, _)) -> (kind, ((who, op), (rid, id, text, created)))) | filter((kind, _) -> (kind, 1) eq) | map((_, row) -> row); --let known = reply | map(((who, op), row) -> (who, (op, row))) | join(friend, (who, (op, row), (f, _)) -> ((f, op) eq, ((who, op), row))) | filter((same, _) -> same) | map((_, (pair, row)) -> ((pair, row), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((pair, row), ()) -> (pair, row)); +-let reply = request | map(((rid, m), ()) -> (m, rid)) | join(message, (m, (rid, (_, _, op, _, _, _, _, _, _, _, _, _, _))) -> (m, (rid, op))) | join(by_parent, (_, ((rid, op), id)) -> (id, (rid, op))) | join(message, (id, ((rid, op), (kind, created, who, _, text, _, _, _, _, _, _, _, _))) -> (kind, ((who, op), (rid, id, text, created)))) | filter((kind, _) -> (kind, 1) eq) | map((_, row) -> row); +-let known = reply | map(((who, op), row) -> (who, (op, row))) | join(friend, (who, ((op, row), (f, _))) -> ((f, op) eq, ((who, op), row))) | filter((same, _) -> same) | map((_, (pair, row)) -> ((pair, row), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((pair, row), ()) -> (pair, row)); -let marked = (known | map(((who, _), row) -> (who, (row, 1)))) + (reply + (known | negate) | map(((who, _), row) -> (who, (row, 0)))); --let ranked = marked | join(person, (who, ((rid, id, text, created), k), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, ((created neg, who), (id, text, first, last, k)))) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist) | flatmap((k, entries) -> (k, ((0, entries len) range, entries) zip) cap_list); +-let ranked = marked | join(person, (who, (((rid, id, text, created), k), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((created neg, who), (id, text, first, last, k)))) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist) | flatmap((k, entries) -> (k, ((0, entries len) range, entries) zip) cap_list); +let reply_map = request | map(((rid, m), ()) -> (m, rid)); +let reply_idx = reply_map | arrange; +let message_idx = message | arrange; -+let reply_join = reply_idx | join(message_idx, (m, rid, (_, _, op, _, _, _, _, _, _, _, _, _, _)) -> (m, (rid, op))); ++let reply_join = reply_idx | join(message_idx, (m, (rid, (_, _, op, _, _, _, _, _, _, _, _, _, _))) -> (m, (rid, op))); +let reply_idx2 = reply_join | arrange; +let by_parent_idx = by_parent | arrange; -+let reply_join2 = reply_idx2 | join(by_parent_idx, (_, (rid, op), id) -> (id, (rid, op))); ++let reply_join2 = reply_idx2 | join(by_parent_idx, (_, ((rid, op), id)) -> (id, (rid, op))); +let reply_idx3 = reply_join2 | arrange; +let message_idx2 = message | arrange; -+let reply_join3 = reply_idx3 | join(message_idx2, (id, (rid, op), (kind, created, who, _, text, _, _, _, _, _, _, _, _)) -> (kind, ((who, op), (rid, id, text, created)))); ++let reply_join3 = reply_idx3 | join(message_idx2, (id, ((rid, op), (kind, created, who, _, text, _, _, _, _, _, _, _, _))) -> (kind, ((who, op), (rid, id, text, created)))); +let reply_filter = reply_join3 | filter((kind, _) -> (kind, 1) eq); +let reply = reply_filter | map((_, row) -> row); +let known_map = reply | map(((who, op), row) -> (who, (op, row))); +let known_idx = known_map | arrange; +let friend_idx = friend | arrange; -+let known_join = known_idx | join(friend_idx, (who, (op, row), (f, _)) -> ((f, op) eq, ((who, op), row))); ++let known_join = known_idx | join(friend_idx, (who, ((op, row), (f, _))) -> ((f, op) eq, ((who, op), row))); +let known_filter = known_join | filter((same, _) -> same); +let known_map2 = known_filter | map((_, (pair, row)) -> ((pair, row), ())); +let known_idx2 = known_map2 | arrange; @@ -40,7 +40,7 @@ +let marked = marked_map + marked_map2; +let marked_idx = marked | arrange; +let person_idx = person | arrange; -+let ranked_join = marked_idx | join(person_idx, (who, ((rid, id, text, created), k), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, ((created neg, who), (id, text, first, last, k)))); ++let ranked_join = marked_idx | join(person_idx, (who, (((rid, id, text, created), k), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((created neg, who), (id, text, first, last, k)))); +let ranked_idx = ranked_join | arrange; +let ranked_reduce = ranked_idx | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); +let ranked = ranked_reduce | flatmap((k, entries) -> (k, ((0, entries len) range, entries) zip) cap_list); diff --git a/interactive/tests/plan_pairs/expected/ldbc_is07.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_is07.parsed.ddp index db5857540..c612880c1 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_is07.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_is07.parsed.ddp @@ -3,9 +3,9 @@ let person = import "graph::person" : (int, (List(int), List(int), List(int), in let message = import "graph::message" : (int, (int, int, int, int, List(int), List(int), int, List(int), int, int, int, int, int)); let by_parent = import "graph::message.by_parent" : (int, int); let friend = import "graph::friend" : (int, (int, int)); -let reply = request | map(((rid, m), ()) -> (m, rid)) | join(message, (m, rid, (_, _, op, _, _, _, _, _, _, _, _, _, _)) -> (m, (rid, op))) | join(by_parent, (_, (rid, op), id) -> (id, (rid, op))) | join(message, (id, (rid, op), (kind, created, who, _, text, _, _, _, _, _, _, _, _)) -> (kind, ((who, op), (rid, id, text, created)))) | filter((kind, _) -> (kind, 1) eq) | map((_, row) -> row); -let known = reply | map(((who, op), row) -> (who, (op, row))) | join(friend, (who, (op, row), (f, _)) -> ((f, op) eq, ((who, op), row))) | filter((same, _) -> same) | map((_, (pair, row)) -> ((pair, row), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((pair, row), ()) -> (pair, row)); +let reply = request | map(((rid, m), ()) -> (m, rid)) | join(message, (m, (rid, (_, _, op, _, _, _, _, _, _, _, _, _, _))) -> (m, (rid, op))) | join(by_parent, (_, ((rid, op), id)) -> (id, (rid, op))) | join(message, (id, ((rid, op), (kind, created, who, _, text, _, _, _, _, _, _, _, _))) -> (kind, ((who, op), (rid, id, text, created)))) | filter((kind, _) -> (kind, 1) eq) | map((_, row) -> row); +let known = reply | map(((who, op), row) -> (who, (op, row))) | join(friend, (who, ((op, row), (f, _))) -> ((f, op) eq, ((who, op), row))) | filter((same, _) -> same) | map((_, (pair, row)) -> ((pair, row), ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv) | map(((pair, row), ()) -> (pair, row)); let marked = (known | map(((who, _), row) -> (who, (row, 1)))) + (reply + (known | negate) | map(((who, _), row) -> (who, (row, 0)))); -let ranked = marked | join(person, (who, ((rid, id, text, created), k), (first, last, _, _, _, _, _, _, _, _, _)) -> (rid, ((created neg, who), (id, text, first, last, k)))) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist) | flatmap((k, entries) -> (k, ((0, entries len) range, entries) zip) cap_list); +let ranked = marked | join(person, (who, (((rid, id, text, created), k), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, ((created neg, who), (id, text, first, last, k)))) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist) | flatmap((k, entries) -> (k, ((0, entries len) range, entries) zip) cap_list); let answer = ranked | map((rid, (rank, ((neg, who), (id, text, first, last, k)))) -> ((rid, rank), (id, text, neg neg, who, first, last, k))); output "answer" = answer | arrange; diff --git a/interactive/tests/plan_pairs/expected/nested.ddp b/interactive/tests/plan_pairs/expected/nested.ddp index f262f978a..8ed4b3f8c 100644 --- a/interactive/tests/plan_pairs/expected/nested.ddp +++ b/interactive/tests/plan_pairs/expected/nested.ddp @@ -10,7 +10,7 @@ outer: { -- Time lanes: epoch, then 2 scope coordinates; this scope enters at zero. var reached; let reached_idx = reached | arrange; - let step = reached_idx | join(::edges_idx, (_, (), dst) -> (dst, ())); + let step = reached_idx | join(::edges_idx, (_, ((), dst)) -> (dst, ())); let reached_rows = outer::starts + step; let reached_idx2 = reached_rows | arrange; let reached_reduce = reached_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); diff --git a/interactive/tests/plan_pairs/expected/nested.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/nested.lowered--optimized.diff index 229fea5e8..0c091bbc0 100644 --- a/interactive/tests/plan_pairs/expected/nested.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/nested.lowered--optimized.diff @@ -13,8 +13,8 @@ var reached; let reached_idx = reached | arrange; - let edges_idx = ::edges | arrange; -- let step = reached_idx | join(edges_idx, (_, (), dst) -> (dst, ())); -+ let step = reached_idx | join(::edges_idx, (_, (), dst) -> (dst, ())); +- let step = reached_idx | join(edges_idx, (_, ((), dst)) -> (dst, ())); ++ let step = reached_idx | join(::edges_idx, (_, ((), dst)) -> (dst, ())); let reached_rows = outer::starts + step; let reached_idx2 = reached_rows | arrange; let reached_reduce = reached_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); diff --git a/interactive/tests/plan_pairs/expected/nested.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/nested.parsed--lowered.diff index aad36163c..f422a59be 100644 --- a/interactive/tests/plan_pairs/expected/nested.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/nested.parsed--lowered.diff @@ -10,13 +10,13 @@ + var again; + let starts = ::seeds + again; inner: { -- let step = reached | join(edges, (_, (), dst) -> (dst, ())); +- let step = reached | join(edges, (_, ((), dst)) -> (dst, ())); - set reached = starts + step | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); + -- Time lanes: epoch, then 2 scope coordinates; this scope enters at zero. + var reached; + let reached_idx = reached | arrange; + let edges_idx = ::edges | arrange; -+ let step = reached_idx | join(edges_idx, (_, (), dst) -> (dst, ())); ++ let step = reached_idx | join(edges_idx, (_, ((), dst)) -> (dst, ())); + let reached_rows = outer::starts + step; + let reached_idx2 = reached_rows | arrange; + let reached_reduce = reached_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); diff --git a/interactive/tests/plan_pairs/expected/nested.parsed.ddp b/interactive/tests/plan_pairs/expected/nested.parsed.ddp index 7656698b2..c7efd6ecf 100644 --- a/interactive/tests/plan_pairs/expected/nested.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/nested.parsed.ddp @@ -3,7 +3,7 @@ let seeds = input 1 : (int, ()); outer: { let starts = seeds + again; inner: { - let step = reached | join(edges, (_, (), dst) -> (dst, ())); + let step = reached | join(edges, (_, ((), dst)) -> (dst, ())); set reached = starts + step | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); } set again = inner::reached | filter((k, ()) -> 0); diff --git a/interactive/tests/plan_pairs/expected/server_access.ddp b/interactive/tests/plan_pairs/expected/server_access.ddp index 364c5ed87..d0813caaf 100644 --- a/interactive/tests/plan_pairs/expected/server_access.ddp +++ b/interactive/tests/plan_pairs/expected/server_access.ddp @@ -12,7 +12,7 @@ barring: { -- Time lanes: epoch, then 1 scope coordinates; this scope enters at zero. var barred : (Member, int); let barred_idx = barred | arrange; - let beyond = barred_idx | join(::members_idx, (_grp, doc, who) -> (who, doc)); + let beyond = barred_idx | join(::members_idx, (_grp, (doc, who)) -> (who, doc)); let barred_rows = ::blocks_map + beyond; let barred_map = barred_rows | map((k, v) -> ((k, v), ())); let barred_idx2 = barred_map | arrange; @@ -27,10 +27,10 @@ sharing: { var allowed : ((Member, int), Access); let inherited_map = allowed | map(((who, doc), acc) -> (who, (doc, acc))); let inherited_idx = inherited_map | arrange; - let inherited = inherited_idx | join(::members_idx, (_grp, (doc, acc), who) -> ((who, doc), acc)); + let inherited = inherited_idx | join(::members_idx, (_grp, ((doc, acc), who)) -> ((who, doc), acc)); let offered = ::grants_map + inherited; let offered_idx = offered | arrange; - let allowed_join = offered_idx | join(::barred_idx, (pair, acc, ()) -> (pair, acc)); + let allowed_join = offered_idx | join(::barred_idx, (pair, (acc, ())) -> (pair, acc)); let allowed_neg = allowed_join | negate; let allowed_rows = offered + allowed_neg; let allowed_idx = allowed_rows | arrange; diff --git a/interactive/tests/plan_pairs/expected/server_access.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/server_access.lowered--optimized.diff index 5d526c637..f5a5ed89a 100644 --- a/interactive/tests/plan_pairs/expected/server_access.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/server_access.lowered--optimized.diff @@ -13,12 +13,12 @@ var barred : (Member, int); let barred_idx = barred | arrange; - let members_idx = ::members | arrange; -- let beyond = barred_idx | join(members_idx, (_grp, doc, who) -> (who, doc)); +- let beyond = barred_idx | join(members_idx, (_grp, (doc, who)) -> (who, doc)); - let barred_map = ::blocks | map((doc, who) -> (who, doc)); - let barred_rows = barred_map + beyond; - let barred_map2 = barred_rows | map((k, v) -> ((k, v), ())); - let barred_idx2 = barred_map2 | arrange; -+ let beyond = barred_idx | join(::members_idx, (_grp, doc, who) -> (who, doc)); ++ let beyond = barred_idx | join(::members_idx, (_grp, (doc, who)) -> (who, doc)); + let barred_rows = ::blocks_map + beyond; + let barred_map = barred_rows | map((k, v) -> ((k, v), ())); + let barred_idx2 = barred_map | arrange; @@ -36,15 +36,15 @@ let inherited_map = allowed | map(((who, doc), acc) -> (who, (doc, acc))); let inherited_idx = inherited_map | arrange; - let members_idx = ::members | arrange; -- let inherited = inherited_idx | join(members_idx, (_grp, (doc, acc), who) -> ((who, doc), acc)); +- let inherited = inherited_idx | join(members_idx, (_grp, ((doc, acc), who)) -> ((who, doc), acc)); - let offered_map = ::grants | map((doc, (who, acc)) -> ((who, doc), acc)); - let offered = offered_map + inherited; -+ let inherited = inherited_idx | join(::members_idx, (_grp, (doc, acc), who) -> ((who, doc), acc)); ++ let inherited = inherited_idx | join(::members_idx, (_grp, ((doc, acc), who)) -> ((who, doc), acc)); + let offered = ::grants_map + inherited; let offered_idx = offered | arrange; - let barred_idx = ::barred | arrange; -- let allowed_join = offered_idx | join(barred_idx, (pair, acc, ()) -> (pair, acc)); -+ let allowed_join = offered_idx | join(::barred_idx, (pair, acc, ()) -> (pair, acc)); +- let allowed_join = offered_idx | join(barred_idx, (pair, (acc, ())) -> (pair, acc)); ++ let allowed_join = offered_idx | join(::barred_idx, (pair, (acc, ())) -> (pair, acc)); let allowed_neg = allowed_join | negate; let allowed_rows = offered + allowed_neg; let allowed_idx = allowed_rows | arrange; diff --git a/interactive/tests/plan_pairs/expected/server_access.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/server_access.parsed--lowered.diff index f64d65b2c..bf2b69433 100644 --- a/interactive/tests/plan_pairs/expected/server_access.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/server_access.parsed--lowered.diff @@ -13,7 +13,7 @@ + var barred : (Member, int); + let barred_idx = barred | arrange; + let members_idx = ::members | arrange; -+ let beyond = barred_idx | join(members_idx, (_grp, doc, who) -> (who, doc)); ++ let beyond = barred_idx | join(members_idx, (_grp, (doc, who)) -> (who, doc)); + let barred_map = ::blocks | map((doc, who) -> (who, doc)); + let barred_rows = barred_map + beyond; + let barred_map2 = barred_rows | map((k, v) -> ((k, v), ())); @@ -25,7 +25,7 @@ let barred = barring::barred | map((who, doc) -> ((who, doc), ())); -barring: { - var barred : (Member, int); -- let beyond = barred | join(members, (_grp, doc, who) -> (who, doc)); +- let beyond = barred | join(members, (_grp, (doc, who)) -> (who, doc)); - set barred = (blocks | map((doc, who) -> (who, doc))) + beyond | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); +sharing: { + -- Time lanes: epoch, then 1 scope coordinates; this scope enters at zero. @@ -33,12 +33,12 @@ + let inherited_map = allowed | map(((who, doc), acc) -> (who, (doc, acc))); + let inherited_idx = inherited_map | arrange; + let members_idx = ::members | arrange; -+ let inherited = inherited_idx | join(members_idx, (_grp, (doc, acc), who) -> ((who, doc), acc)); ++ let inherited = inherited_idx | join(members_idx, (_grp, ((doc, acc), who)) -> ((who, doc), acc)); + let offered_map = ::grants | map((doc, (who, acc)) -> ((who, doc), acc)); + let offered = offered_map + inherited; + let offered_idx = offered | arrange; + let barred_idx = ::barred | arrange; -+ let allowed_join = offered_idx | join(barred_idx, (pair, acc, ()) -> (pair, acc)); ++ let allowed_join = offered_idx | join(barred_idx, (pair, (acc, ())) -> (pair, acc)); + let allowed_neg = allowed_join | negate; + let allowed_rows = offered + allowed_neg; + let allowed_idx = allowed_rows | arrange; @@ -49,7 +49,7 @@ output "access" = sharing::allowed; -sharing: { - var allowed : ((Member, int), Access); -- set allowed = offered + (offered | join(barred, (pair, acc, ()) -> (pair, acc)) | negate) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); +- set allowed = offered + (offered | join(barred, (pair, (acc, ())) -> (pair, acc)) | negate) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); - let offered = (grants | map((doc, (who, acc)) -> ((who, doc), acc))) + inherited; -- let inherited = allowed | map(((who, doc), acc) -> (who, (doc, acc))) | join(members, (_grp, (doc, acc), who) -> ((who, doc), acc)); +- let inherited = allowed | map(((who, doc), acc) -> (who, (doc, acc))) | join(members, (_grp, ((doc, acc), who)) -> ((who, doc), acc)); -} diff --git a/interactive/tests/plan_pairs/expected/server_access.parsed.ddp b/interactive/tests/plan_pairs/expected/server_access.parsed.ddp index 4fb45749f..516f2f64f 100644 --- a/interactive/tests/plan_pairs/expected/server_access.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/server_access.parsed.ddp @@ -7,13 +7,13 @@ let members = groups | map((grp, who) -> (grp inject Member::Group, who)); let barred = barring::barred | map((who, doc) -> ((who, doc), ())); barring: { var barred : (Member, int); - let beyond = barred | join(members, (_grp, doc, who) -> (who, doc)); + let beyond = barred | join(members, (_grp, (doc, who)) -> (who, doc)); set barred = (blocks | map((doc, who) -> (who, doc))) + beyond | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); } output "access" = sharing::allowed; sharing: { var allowed : ((Member, int), Access); - set allowed = offered + (offered | join(barred, (pair, acc, ()) -> (pair, acc)) | negate) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); + set allowed = offered + (offered | join(barred, (pair, (acc, ())) -> (pair, acc)) | negate) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); let offered = (grants | map((doc, (who, acc)) -> ((who, doc), acc))) + inherited; - let inherited = allowed | map(((who, doc), acc) -> (who, (doc, acc))) | join(members, (_grp, (doc, acc), who) -> ((who, doc), acc)); + let inherited = allowed | map(((who, doc), acc) -> (who, (doc, acc))) | join(members, (_grp, ((doc, acc), who)) -> ((who, doc), acc)); } diff --git a/interactive/tests/plan_pairs/expected/server_arrange_idem.ddp b/interactive/tests/plan_pairs/expected/server_arrange_idem.ddp index 9373816c5..6a2ee5caf 100644 --- a/interactive/tests/plan_pairs/expected/server_arrange_idem.ddp +++ b/interactive/tests/plan_pairs/expected/server_arrange_idem.ddp @@ -14,25 +14,25 @@ eqsat: { -- Time lanes: epoch, then 1 scope coordinates; this scope enters at zero. var eq; let eq_idx = eq | arrange; - let canon = ::pc_idx | join(eq_idx, (_, (id, op, pos), cls) -> ((id, op), (pos, cls))); + let canon = ::pc_idx | join(eq_idx, (_, ((id, op, pos), cls)) -> ((id, op), (pos, cls))); let canon_idx = canon | arrange; let bsig_reduce = canon_idx | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); let bsig = bsig_reduce | map(((id, op), cs) -> ((op, cs), id)); let bsig_idx = bsig | arrange; let reps = bsig_idx | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); - let cong = bsig_idx | join(reps, (_, id, rep) -> (id, rep)); - let arranged_join = ::producing_idx | join(eq_idx, (_, _, cls) -> (cls, ())); + let cong = bsig_idx | join(reps, (_, (id, rep)) -> (id, rep)); + let arranged_join = ::producing_idx | join(eq_idx, (_, (_, cls)) -> (cls, ())); let arranged_idx = arranged_join | arrange; let arranged = arranged_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); - let arr_cls = ::arr_map_idx | join(eq_idx, (_, a, cls) -> (a, cls)); + let arr_cls = ::arr_map_idx | join(eq_idx, (_, (a, cls)) -> (a, cls)); let rule_map = arr_cls | map((a, cls) -> (cls, a)); let rule_idx = rule_map | arrange; - let rule = rule_idx | join(arranged, (cls, a, _) -> (a, cls)); + let rule = rule_idx | join(arranged, (cls, (a, _)) -> (a, cls)); let edges_map = cong | map((a, b) -> (b, a)); let edges_map2 = rule | map((a, b) -> (b, a)); let edges = cong + edges_map + rule + edges_map2; let edges_idx = edges | arrange; - let prop = eq_idx | join(edges_idx, (_, l, n) -> (n, l)); + let prop = eq_idx | join(edges_idx, (_, (l, n)) -> (n, l)); let eq_rows = ::elems + prop; let eq_idx2 = eq_rows | arrange; let eq_reduce = eq_idx2 | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); diff --git a/interactive/tests/plan_pairs/expected/server_arrange_idem.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/server_arrange_idem.lowered--optimized.diff index cde420b54..7360bec67 100644 --- a/interactive/tests/plan_pairs/expected/server_arrange_idem.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/server_arrange_idem.lowered--optimized.diff @@ -28,37 +28,37 @@ var eq; - let pc_idx = ::pc | arrange; let eq_idx = eq | arrange; -- let canon = pc_idx | join(eq_idx, (_, (id, op, pos), cls) -> ((id, op), (pos, cls))); -+ let canon = ::pc_idx | join(eq_idx, (_, (id, op, pos), cls) -> ((id, op), (pos, cls))); +- let canon = pc_idx | join(eq_idx, (_, ((id, op, pos), cls)) -> ((id, op), (pos, cls))); ++ let canon = ::pc_idx | join(eq_idx, (_, ((id, op, pos), cls)) -> ((id, op), (pos, cls))); let canon_idx = canon | arrange; let bsig_reduce = canon_idx | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); let bsig = bsig_reduce | map(((id, op), cs) -> ((op, cs), id)); let bsig_idx = bsig | arrange; let reps = bsig_idx | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); - let bsig_idx2 = bsig | arrange; -- let cong = bsig_idx2 | join(reps, (_, id, rep) -> (id, rep)); +- let cong = bsig_idx2 | join(reps, (_, (id, rep)) -> (id, rep)); - let producing_idx = ::producing | arrange; - let eq_idx2 = eq | arrange; -- let arranged_join = producing_idx | join(eq_idx2, (_, _, cls) -> (cls, ())); -+ let cong = bsig_idx | join(reps, (_, id, rep) -> (id, rep)); -+ let arranged_join = ::producing_idx | join(eq_idx, (_, _, cls) -> (cls, ())); +- let arranged_join = producing_idx | join(eq_idx2, (_, (_, cls)) -> (cls, ())); ++ let cong = bsig_idx | join(reps, (_, (id, rep)) -> (id, rep)); ++ let arranged_join = ::producing_idx | join(eq_idx, (_, (_, cls)) -> (cls, ())); let arranged_idx = arranged_join | arrange; let arranged = arranged_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); - let arr_cls_map = ::arr | map((a, c) -> (c, a)); - let arr_cls_idx = arr_cls_map | arrange; - let eq_idx3 = eq | arrange; -- let arr_cls = arr_cls_idx | join(eq_idx3, (_, a, cls) -> (a, cls)); -+ let arr_cls = ::arr_map_idx | join(eq_idx, (_, a, cls) -> (a, cls)); +- let arr_cls = arr_cls_idx | join(eq_idx3, (_, (a, cls)) -> (a, cls)); ++ let arr_cls = ::arr_map_idx | join(eq_idx, (_, (a, cls)) -> (a, cls)); let rule_map = arr_cls | map((a, cls) -> (cls, a)); let rule_idx = rule_map | arrange; - let rule = rule_idx | join(arranged, (cls, a, _) -> (a, cls)); + let rule = rule_idx | join(arranged, (cls, (a, _)) -> (a, cls)); let edges_map = cong | map((a, b) -> (b, a)); let edges_map2 = rule | map((a, b) -> (b, a)); let edges = cong + edges_map + rule + edges_map2; - let eq_idx4 = eq | arrange; let edges_idx = edges | arrange; -- let prop = eq_idx4 | join(edges_idx, (_, l, n) -> (n, l)); -+ let prop = eq_idx | join(edges_idx, (_, l, n) -> (n, l)); +- let prop = eq_idx4 | join(edges_idx, (_, (l, n)) -> (n, l)); ++ let prop = eq_idx | join(edges_idx, (_, (l, n)) -> (n, l)); let eq_rows = ::elems + prop; - let eq_idx5 = eq_rows | arrange; - let eq_reduce = eq_idx5 | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); diff --git a/interactive/tests/plan_pairs/expected/server_arrange_idem.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/server_arrange_idem.parsed--lowered.diff index 07da8367f..ec1314c01 100644 --- a/interactive/tests/plan_pairs/expected/server_arrange_idem.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/server_arrange_idem.parsed--lowered.diff @@ -20,46 +20,46 @@ +let arr_filter = node | filter((_, (op, _)) -> (op, 1) eq); +let arr = arr_filter | map((id, (_, cs)) -> (id, (0, cs) get)); eqsat: { -- let canon = pc | join(eq, (_, (id, op, pos), cls) -> ((id, op), (pos, cls))); +- let canon = pc | join(eq, (_, ((id, op, pos), cls)) -> ((id, op), (pos, cls))); - let bsig = canon | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist) | map(((id, op), cs) -> ((op, cs), id)); - let reps = bsig | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); -- let cong = bsig | join(reps, (_, id, rep) -> (id, rep)); -- let arranged = producing | join(eq, (_, _, cls) -> (cls, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); -- let arr_cls = arr | map((a, c) -> (c, a)) | join(eq, (_, a, cls) -> (a, cls)); -- let rule = arr_cls | map((a, cls) -> (cls, a)) | join(arranged, (cls, a, _) -> (a, cls)); +- let cong = bsig | join(reps, (_, (id, rep)) -> (id, rep)); +- let arranged = producing | join(eq, (_, (_, cls)) -> (cls, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); +- let arr_cls = arr | map((a, c) -> (c, a)) | join(eq, (_, (a, cls)) -> (a, cls)); +- let rule = arr_cls | map((a, cls) -> (cls, a)) | join(arranged, (cls, (a, _)) -> (a, cls)); - let edges = cong + (cong | map((a, b) -> (b, a))) + rule + (rule | map((a, b) -> (b, a))); -- let prop = eq | join(edges, (_, l, n) -> (n, l)); +- let prop = eq | join(edges, (_, (l, n)) -> (n, l)); - set eq = elems + prop | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); + -- Time lanes: epoch, then 1 scope coordinates; this scope enters at zero. + var eq; + let pc_idx = ::pc | arrange; + let eq_idx = eq | arrange; -+ let canon = pc_idx | join(eq_idx, (_, (id, op, pos), cls) -> ((id, op), (pos, cls))); ++ let canon = pc_idx | join(eq_idx, (_, ((id, op, pos), cls)) -> ((id, op), (pos, cls))); + let canon_idx = canon | arrange; + let bsig_reduce = canon_idx | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); + let bsig = bsig_reduce | map(((id, op), cs) -> ((op, cs), id)); + let bsig_idx = bsig | arrange; + let reps = bsig_idx | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); + let bsig_idx2 = bsig | arrange; -+ let cong = bsig_idx2 | join(reps, (_, id, rep) -> (id, rep)); ++ let cong = bsig_idx2 | join(reps, (_, (id, rep)) -> (id, rep)); + let producing_idx = ::producing | arrange; + let eq_idx2 = eq | arrange; -+ let arranged_join = producing_idx | join(eq_idx2, (_, _, cls) -> (cls, ())); ++ let arranged_join = producing_idx | join(eq_idx2, (_, (_, cls)) -> (cls, ())); + let arranged_idx = arranged_join | arrange; + let arranged = arranged_idx | reduce((_, vals) -> vals map (((), _) -> ((), 1))); + let arr_cls_map = ::arr | map((a, c) -> (c, a)); + let arr_cls_idx = arr_cls_map | arrange; + let eq_idx3 = eq | arrange; -+ let arr_cls = arr_cls_idx | join(eq_idx3, (_, a, cls) -> (a, cls)); ++ let arr_cls = arr_cls_idx | join(eq_idx3, (_, (a, cls)) -> (a, cls)); + let rule_map = arr_cls | map((a, cls) -> (cls, a)); + let rule_idx = rule_map | arrange; -+ let rule = rule_idx | join(arranged, (cls, a, _) -> (a, cls)); ++ let rule = rule_idx | join(arranged, (cls, (a, _)) -> (a, cls)); + let edges_map = cong | map((a, b) -> (b, a)); + let edges_map2 = rule | map((a, b) -> (b, a)); + let edges = cong + edges_map + rule + edges_map2; + let eq_idx4 = eq | arrange; + let edges_idx = edges | arrange; -+ let prop = eq_idx4 | join(edges_idx, (_, l, n) -> (n, l)); ++ let prop = eq_idx4 | join(edges_idx, (_, (l, n)) -> (n, l)); + let eq_rows = ::elems + prop; + let eq_idx5 = eq_rows | arrange; + let eq_reduce = eq_idx5 | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); diff --git a/interactive/tests/plan_pairs/expected/server_arrange_idem.parsed.ddp b/interactive/tests/plan_pairs/expected/server_arrange_idem.parsed.ddp index 7da097c67..49740997f 100644 --- a/interactive/tests/plan_pairs/expected/server_arrange_idem.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/server_arrange_idem.parsed.ddp @@ -4,15 +4,15 @@ let pc = node | map((id, (op, cs)) -> ((id, op), cs)) | flatmap((k, cs) -> (k, ( let producing = (node | filter((_, (op, _)) -> (op, 1) eq) | map((id, _) -> (id, ()))) + (node | filter((_, (op, _)) -> (op, 2) eq) | map((id, _) -> (id, ()))); let arr = node | filter((_, (op, _)) -> (op, 1) eq) | map((id, (_, cs)) -> (id, (0, cs) get)); eqsat: { - let canon = pc | join(eq, (_, (id, op, pos), cls) -> ((id, op), (pos, cls))); + let canon = pc | join(eq, (_, ((id, op, pos), cls)) -> ((id, op), (pos, cls))); let bsig = canon | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist) | map(((id, op), cs) -> ((op, cs), id)); let reps = bsig | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); - let cong = bsig | join(reps, (_, id, rep) -> (id, rep)); - let arranged = producing | join(eq, (_, _, cls) -> (cls, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); - let arr_cls = arr | map((a, c) -> (c, a)) | join(eq, (_, a, cls) -> (a, cls)); - let rule = arr_cls | map((a, cls) -> (cls, a)) | join(arranged, (cls, a, _) -> (a, cls)); + let cong = bsig | join(reps, (_, (id, rep)) -> (id, rep)); + let arranged = producing | join(eq, (_, (_, cls)) -> (cls, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); + let arr_cls = arr | map((a, c) -> (c, a)) | join(eq, (_, (a, cls)) -> (a, cls)); + let rule = arr_cls | map((a, cls) -> (cls, a)) | join(arranged, (cls, (a, _)) -> (a, cls)); let edges = cong + (cong | map((a, b) -> (b, a))) + rule + (rule | map((a, b) -> (b, a))); - let prop = eq | join(edges, (_, l, n) -> (n, l)); + let prop = eq | join(edges, (_, (l, n)) -> (n, l)); set eq = elems + prop | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); } output "classes" = eqsat::eq | map((n, c) -> (n, c)) | arrange | inspect(classes); diff --git a/interactive/tests/plan_pairs/expected/server_reach_import.ddp b/interactive/tests/plan_pairs/expected/server_reach_import.ddp index 4d7d374c5..abddfb8f0 100644 --- a/interactive/tests/plan_pairs/expected/server_reach_import.ddp +++ b/interactive/tests/plan_pairs/expected/server_reach_import.ddp @@ -6,7 +6,7 @@ reach: { -- Time lanes: epoch, then 1 scope coordinates; this scope enters at zero. var reach; let reach_idx = reach | arrange; - let proposals = reach_idx | join(::edges_idx, (_, _, dst) -> (dst, ())); + let proposals = reach_idx | join(::edges_idx, (_, (_, dst)) -> (dst, ())); let reach_rows = ::roots + proposals; let reach_idx2 = reach_rows | arrange; let reach_reduce = reach_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); diff --git a/interactive/tests/plan_pairs/expected/server_reach_import.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/server_reach_import.lowered--optimized.diff index 8fc1aaf2b..93c10c1d8 100644 --- a/interactive/tests/plan_pairs/expected/server_reach_import.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/server_reach_import.lowered--optimized.diff @@ -10,8 +10,8 @@ var reach; let reach_idx = reach | arrange; - let edges_idx = ::edges | arrange; -- let proposals = reach_idx | join(edges_idx, (_, _, dst) -> (dst, ())); -+ let proposals = reach_idx | join(::edges_idx, (_, _, dst) -> (dst, ())); +- let proposals = reach_idx | join(edges_idx, (_, (_, dst)) -> (dst, ())); ++ let proposals = reach_idx | join(::edges_idx, (_, (_, dst)) -> (dst, ())); let reach_rows = ::roots + proposals; let reach_idx2 = reach_rows | arrange; let reach_reduce = reach_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); diff --git a/interactive/tests/plan_pairs/expected/server_reach_import.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/server_reach_import.parsed--lowered.diff index 48c5b9855..50962bbf8 100644 --- a/interactive/tests/plan_pairs/expected/server_reach_import.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/server_reach_import.parsed--lowered.diff @@ -5,13 +5,13 @@ let edges = input "edges" : (int, int); let roots = input "roots" : (int, ()); reach: { -- let proposals = reach | join(edges, (_, _, dst) -> (dst, ())); +- let proposals = reach | join(edges, (_, (_, dst)) -> (dst, ())); - set reach = roots + proposals | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); + -- Time lanes: epoch, then 1 scope coordinates; this scope enters at zero. + var reach; + let reach_idx = reach | arrange; + let edges_idx = ::edges | arrange; -+ let proposals = reach_idx | join(edges_idx, (_, _, dst) -> (dst, ())); ++ let proposals = reach_idx | join(edges_idx, (_, (_, dst)) -> (dst, ())); + let reach_rows = ::roots + proposals; + let reach_idx2 = reach_rows | arrange; + let reach_reduce = reach_idx2 | reduce((_, vals) -> vals map (((), _) -> ((), 1))); diff --git a/interactive/tests/plan_pairs/expected/server_reach_import.parsed.ddp b/interactive/tests/plan_pairs/expected/server_reach_import.parsed.ddp index 8bafe4a2b..9e252f75e 100644 --- a/interactive/tests/plan_pairs/expected/server_reach_import.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/server_reach_import.parsed.ddp @@ -1,7 +1,7 @@ let edges = input "edges" : (int, int); let roots = input "roots" : (int, ()); reach: { - let proposals = reach | join(edges, (_, _, dst) -> (dst, ())); + let proposals = reach | join(edges, (_, (_, dst)) -> (dst, ())); set reach = roots + proposals | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); } output "result" = reach::reach | inspect(reach); diff --git a/interactive/tests/plan_pairs/expected/txn_maintained.ddp b/interactive/tests/plan_pairs/expected/txn_maintained.ddp index ffcccafff..51ddfc551 100644 --- a/interactive/tests/plan_pairs/expected/txn_maintained.ddp +++ b/interactive/tests/plan_pairs/expected/txn_maintained.ddp @@ -11,12 +11,12 @@ res: { -- Time lanes: epoch, then 1 scope coordinates; this scope enters at zero. var rollback; let rollback_idx = rollback | arrange; - let wr_bad = ::wid_idx | join(rollback_idx, (wid, kv, ()) -> (wid, kv)); + let wr_bad = ::wid_idx | join(rollback_idx, (wid, (kv, ())) -> (wid, kv)); let writes_neg = wr_bad | negate; let writes = ::wid + writes_neg; let wk = writes | map((wid, (k, v)) -> (k, (wid, v))); let wk_idx = wk | arrange; - let pairs_join = ::rk_idx | join(wk_idx, (k, (rid, e), (wid, v)) -> ((k, rid, e), (wid, v))); + let pairs_join = ::rk_idx | join(wk_idx, (k, ((rid, e), (wid, v))) -> ((k, rid, e), (wid, v))); let best_rows = pairs_join | filter(((_, rid, _), (wid, _)) -> (wid, rid) lt) | map((kre, (wid, v)) -> (kre, (wid neg, v))); let best_idx = best_rows | arrange; let best = best_idx | reduce(limit 1, (_, a) -> (((), a head.0.1), 1) enlist); @@ -24,7 +24,7 @@ res: { let unseen_rows = best | map(((k, rid, _), _) -> ((rid, k), ())) | negate; let unseen = ::rkk + unseen_rows; let unseen_idx = unseen | arrange; - let bad_absent_join = unseen_idx | join(::reads_idx, ((rid, _), (), e) -> ((rid, e), ())); + let bad_absent_join = unseen_idx | join(::reads_idx, ((rid, _), ((), e)) -> ((rid, e), ())); let bad_absent = bad_absent_join | filter(((_, e), ()) -> (e, -1) ne) | map(((rid, _), ()) -> (rid, ())); let rollback_rows = bad_seen + bad_absent; let rollback_idx2 = rollback_rows | arrange; @@ -33,7 +33,7 @@ res: { } let rb = res::rollback; let rb_idx = rb | arrange; -let wbad = wid_idx | join(rb_idx, (wid, kv, ()) -> (wid, kv)); +let wbad = wid_idx | join(rb_idx, (wid, (kv, ())) -> (wid, kv)); let committed_neg = wbad | negate; let committed = wid + committed_neg; let current_map = committed | map((wid, (k, v)) -> (k, (wid neg, v))); diff --git a/interactive/tests/plan_pairs/expected/txn_maintained.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/txn_maintained.lowered--optimized.diff index 288df0fbc..0ac6ee20a 100644 --- a/interactive/tests/plan_pairs/expected/txn_maintained.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/txn_maintained.lowered--optimized.diff @@ -20,19 +20,19 @@ var rollback; - let wid_idx = ::wid | arrange; let rollback_idx = rollback | arrange; -- let wr_bad = wid_idx | join(rollback_idx, (wid, kv, ()) -> (wid, kv)); -+ let wr_bad = ::wid_idx | join(rollback_idx, (wid, kv, ()) -> (wid, kv)); +- let wr_bad = wid_idx | join(rollback_idx, (wid, (kv, ())) -> (wid, kv)); ++ let wr_bad = ::wid_idx | join(rollback_idx, (wid, (kv, ())) -> (wid, kv)); let writes_neg = wr_bad | negate; let writes = ::wid + writes_neg; let wk = writes | map((wid, (k, v)) -> (k, (wid, v))); - let rk = ::reads | map(((rid, k), e) -> (k, (rid, e))); - let rk_idx = rk | arrange; let wk_idx = wk | arrange; -- let pairs_join = rk_idx | join(wk_idx, (k, (rid, e), (wid, v)) -> ((k, rid, e), (wid, v))); +- let pairs_join = rk_idx | join(wk_idx, (k, ((rid, e), (wid, v))) -> ((k, rid, e), (wid, v))); - let pairs = pairs_join | filter(((_, rid, _), (wid, _)) -> (wid, rid) lt); - let best_map = pairs | map((kre, (wid, v)) -> (kre, (wid neg, v))); - let best_idx = best_map | arrange; -+ let pairs_join = ::rk_idx | join(wk_idx, (k, (rid, e), (wid, v)) -> ((k, rid, e), (wid, v))); ++ let pairs_join = ::rk_idx | join(wk_idx, (k, ((rid, e), (wid, v))) -> ((k, rid, e), (wid, v))); + let best_rows = pairs_join | filter(((_, rid, _), (wid, _)) -> (wid, rid) lt) | map((kre, (wid, v)) -> (kre, (wid neg, v))); + let best_idx = best_rows | arrange; let best = best_idx | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); @@ -47,10 +47,10 @@ + let unseen = ::rkk + unseen_rows; let unseen_idx = unseen | arrange; - let reads_idx = ::reads | arrange; -- let bad_absent_join = unseen_idx | join(reads_idx, ((rid, _), (), e) -> ((rid, e), ())); +- let bad_absent_join = unseen_idx | join(reads_idx, ((rid, _), ((), e)) -> ((rid, e), ())); - let bad_absent_filter = bad_absent_join | filter(((_, e), ()) -> (e, -1) ne); - let bad_absent = bad_absent_filter | map(((rid, _), ()) -> (rid, ())); -+ let bad_absent_join = unseen_idx | join(::reads_idx, ((rid, _), (), e) -> ((rid, e), ())); ++ let bad_absent_join = unseen_idx | join(::reads_idx, ((rid, _), ((), e)) -> ((rid, e), ())); + let bad_absent = bad_absent_join | filter(((_, e), ()) -> (e, -1) ne) | map(((rid, _), ()) -> (rid, ())); let rollback_rows = bad_seen + bad_absent; let rollback_idx2 = rollback_rows | arrange; @@ -60,7 +60,7 @@ let rb = res::rollback; -let wid_idx = wid | arrange; let rb_idx = rb | arrange; - let wbad = wid_idx | join(rb_idx, (wid, kv, ()) -> (wid, kv)); + let wbad = wid_idx | join(rb_idx, (wid, (kv, ())) -> (wid, kv)); let committed_neg = wbad | negate; @@ -48,12 +40,11 @@ let current_idx = current_map | arrange; diff --git a/interactive/tests/plan_pairs/expected/txn_maintained.optimized--projected.diff b/interactive/tests/plan_pairs/expected/txn_maintained.optimized--projected.diff index 9c9c43af8..121ba5195 100644 --- a/interactive/tests/plan_pairs/expected/txn_maintained.optimized--projected.diff +++ b/interactive/tests/plan_pairs/expected/txn_maintained.optimized--projected.diff @@ -1,7 +1,7 @@ --- optimized +++ projected @@ -19,7 +19,7 @@ - let pairs_join = ::rk_idx | join(wk_idx, (k, (rid, e), (wid, v)) -> ((k, rid, e), (wid, v))); + let pairs_join = ::rk_idx | join(wk_idx, (k, ((rid, e), (wid, v))) -> ((k, rid, e), (wid, v))); let best_rows = pairs_join | filter(((_, rid, _), (wid, _)) -> (wid, rid) lt) | map((kre, (wid, v)) -> (kre, (wid neg, v))); let best_idx = best_rows | arrange; - let best = best_idx | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); diff --git a/interactive/tests/plan_pairs/expected/txn_maintained.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/txn_maintained.parsed--lowered.diff index 7bf31ccdc..fbbc43ee9 100644 --- a/interactive/tests/plan_pairs/expected/txn_maintained.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/txn_maintained.parsed--lowered.diff @@ -11,24 +11,24 @@ +let wr_all = wr_all_filter | map(((wid, _, k), v) -> ((wid, k), v)); let wid = wr_all | map(((wid, k), v) -> (wid, (k, v))); res: { -- let wr_bad = wid | join(rollback, (wid, kv, ()) -> (wid, kv)); +- let wr_bad = wid | join(rollback, (wid, (kv, ())) -> (wid, kv)); - let writes = wid + (wr_bad | negate); + -- Time lanes: epoch, then 1 scope coordinates; this scope enters at zero. + var rollback; + let wid_idx = ::wid | arrange; + let rollback_idx = rollback | arrange; -+ let wr_bad = wid_idx | join(rollback_idx, (wid, kv, ()) -> (wid, kv)); ++ let wr_bad = wid_idx | join(rollback_idx, (wid, (kv, ())) -> (wid, kv)); + let writes_neg = wr_bad | negate; + let writes = ::wid + writes_neg; let wk = writes | map((wid, (k, v)) -> (k, (wid, v))); - let rk = reads | map(((rid, k), e) -> (k, (rid, e))); -- let pairs = rk | join(wk, (k, (rid, e), (wid, v)) -> ((k, rid, e), (wid, v))) | filter(((_, rid, _), (wid, _)) -> (wid, rid) lt); +- let pairs = rk | join(wk, (k, ((rid, e), (wid, v))) -> ((k, rid, e), (wid, v))) | filter(((_, rid, _), (wid, _)) -> (wid, rid) lt); - let best = pairs | map((kre, (wid, v)) -> (kre, (wid neg, v))) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); - let bad_seen = best | filter(((_, _, e), (_, v)) -> (v, e) ne) | map(((_, rid, _), _) -> (rid, ())); + let rk = ::reads | map(((rid, k), e) -> (k, (rid, e))); + let rk_idx = rk | arrange; + let wk_idx = wk | arrange; -+ let pairs_join = rk_idx | join(wk_idx, (k, (rid, e), (wid, v)) -> ((k, rid, e), (wid, v))); ++ let pairs_join = rk_idx | join(wk_idx, (k, ((rid, e), (wid, v))) -> ((k, rid, e), (wid, v))); + let pairs = pairs_join | filter(((_, rid, _), (wid, _)) -> (wid, rid) lt); + let best_map = pairs | map((kre, (wid, v)) -> (kre, (wid neg, v))); + let best_idx = best_map | arrange; @@ -38,14 +38,14 @@ let seen = best | map(((k, rid, _), _) -> ((rid, k), ())); - let rkk = reads | map((rk, _) -> (rk, ())); - let unseen = rkk + (seen | negate); -- let bad_absent = unseen | join(reads, ((rid, _), (), e) -> ((rid, e), ())) | filter(((_, e), ()) -> (e, -1) ne) | map(((rid, _), ()) -> (rid, ())); +- let bad_absent = unseen | join(reads, ((rid, _), ((), e)) -> ((rid, e), ())) | filter(((_, e), ()) -> (e, -1) ne) | map(((rid, _), ()) -> (rid, ())); - set rollback = bad_seen + bad_absent | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); + let rkk = ::reads | map((rk, _) -> (rk, ())); + let unseen_neg = seen | negate; + let unseen = rkk + unseen_neg; + let unseen_idx = unseen | arrange; + let reads_idx = ::reads | arrange; -+ let bad_absent_join = unseen_idx | join(reads_idx, ((rid, _), (), e) -> ((rid, e), ())); ++ let bad_absent_join = unseen_idx | join(reads_idx, ((rid, _), ((), e)) -> ((rid, e), ())); + let bad_absent_filter = bad_absent_join | filter(((_, e), ()) -> (e, -1) ne); + let bad_absent = bad_absent_filter | map(((rid, _), ()) -> (rid, ())); + let rollback_rows = bad_seen + bad_absent; @@ -54,12 +54,12 @@ + set rollback = rollback_reduce; } let rb = res::rollback; --let wbad = wid | join(rb, (wid, kv, ()) -> (wid, kv)); +-let wbad = wid | join(rb, (wid, (kv, ())) -> (wid, kv)); -let committed = wid + (wbad | negate); -let current = committed | map((wid, (k, v)) -> (k, (wid neg, v))) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); +let wid_idx = wid | arrange; +let rb_idx = rb | arrange; -+let wbad = wid_idx | join(rb_idx, (wid, kv, ()) -> (wid, kv)); ++let wbad = wid_idx | join(rb_idx, (wid, (kv, ())) -> (wid, kv)); +let committed_neg = wbad | negate; +let committed = wid + committed_neg; +let current_map = committed | map((wid, (k, v)) -> (k, (wid neg, v))); diff --git a/interactive/tests/plan_pairs/expected/txn_maintained.parsed.ddp b/interactive/tests/plan_pairs/expected/txn_maintained.parsed.ddp index f5e8f9e78..687d2df38 100644 --- a/interactive/tests/plan_pairs/expected/txn_maintained.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/txn_maintained.parsed.ddp @@ -3,21 +3,21 @@ let reads = intents | filter(((_, r, _), _) -> (r, 1) eq) | map(((rid, _, k), e) let wr_all = intents | filter(((_, r, _), _) -> (r, 0) eq) | map(((wid, _, k), v) -> ((wid, k), v)); let wid = wr_all | map(((wid, k), v) -> (wid, (k, v))); res: { - let wr_bad = wid | join(rollback, (wid, kv, ()) -> (wid, kv)); + let wr_bad = wid | join(rollback, (wid, (kv, ())) -> (wid, kv)); let writes = wid + (wr_bad | negate); let wk = writes | map((wid, (k, v)) -> (k, (wid, v))); let rk = reads | map(((rid, k), e) -> (k, (rid, e))); - let pairs = rk | join(wk, (k, (rid, e), (wid, v)) -> ((k, rid, e), (wid, v))) | filter(((_, rid, _), (wid, _)) -> (wid, rid) lt); + let pairs = rk | join(wk, (k, ((rid, e), (wid, v))) -> ((k, rid, e), (wid, v))) | filter(((_, rid, _), (wid, _)) -> (wid, rid) lt); let best = pairs | map((kre, (wid, v)) -> (kre, (wid neg, v))) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); let bad_seen = best | filter(((_, _, e), (_, v)) -> (v, e) ne) | map(((_, rid, _), _) -> (rid, ())); let seen = best | map(((k, rid, _), _) -> ((rid, k), ())); let rkk = reads | map((rk, _) -> (rk, ())); let unseen = rkk + (seen | negate); - let bad_absent = unseen | join(reads, ((rid, _), (), e) -> ((rid, e), ())) | filter(((_, e), ()) -> (e, -1) ne) | map(((rid, _), ()) -> (rid, ())); + let bad_absent = unseen | join(reads, ((rid, _), ((), e)) -> ((rid, e), ())) | filter(((_, e), ()) -> (e, -1) ne) | map(((rid, _), ()) -> (rid, ())); set rollback = bad_seen + bad_absent | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); } let rb = res::rollback; -let wbad = wid | join(rb, (wid, kv, ()) -> (wid, kv)); +let wbad = wid | join(rb, (wid, (kv, ())) -> (wid, kv)); let committed = wid + (wbad | negate); let current = committed | map((wid, (k, v)) -> (k, (wid neg, v))) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); output "rollback" = rb; diff --git a/interactive/tests/plan_pairs/expected/txn_resolve.ddp b/interactive/tests/plan_pairs/expected/txn_resolve.ddp index b49fc38d1..9620afb36 100644 --- a/interactive/tests/plan_pairs/expected/txn_resolve.ddp +++ b/interactive/tests/plan_pairs/expected/txn_resolve.ddp @@ -10,17 +10,17 @@ let wid_idx = wid | arrange; let rk = reads | map(((rid, k), e) -> (k, (rid, e))); let rk_idx = rk | arrange; let store_idx = store | arrange; -let base = rk_idx | join(store_idx, (k, (rid, e), (sw, v)) -> ((k, rid, e), (sw neg, v))); +let base = rk_idx | join(store_idx, (k, ((rid, e), (sw, v))) -> ((k, rid, e), (sw neg, v))); res: { -- Time lanes: epoch, then 1 scope coordinates; this scope enters at zero. var rollback; let rollback_idx = rollback | arrange; - let wr_bad = ::wid_idx | join(rollback_idx, (wid, kv, ()) -> (wid, kv)); + let wr_bad = ::wid_idx | join(rollback_idx, (wid, (kv, ())) -> (wid, kv)); let writes_neg = wr_bad | negate; let writes = ::wid + writes_neg; let wk = writes | map((wid, (k, v)) -> (k, (wid, v))); let wk_idx = wk | arrange; - let pairs_join = ::rk_idx | join(wk_idx, (k, (rid, e), (wid, v)) -> ((k, rid, e), (wid neg, v))); + let pairs_join = ::rk_idx | join(wk_idx, (k, ((rid, e), (wid, v))) -> ((k, rid, e), (wid neg, v))); let pairs = pairs_join | filter(((_, rid, _), (nw, _)) -> (nw neg, rid) lt); let best_rows = pairs + ::base; let best_idx = best_rows | arrange; @@ -29,7 +29,7 @@ res: { let unseen_rows = best | map(((k, rid, _), _) -> ((rid, k), ())) | negate; let unseen = ::rkk + unseen_rows; let unseen_idx = unseen | arrange; - let bad_absent_join = unseen_idx | join(::reads_idx, ((rid, _), (), e) -> ((rid, e), ())); + let bad_absent_join = unseen_idx | join(::reads_idx, ((rid, _), ((), e)) -> ((rid, e), ())); let bad_absent = bad_absent_join | filter(((_, e), ()) -> (e, -1) ne) | map(((rid, _), ()) -> (rid, ())); let rollback_rows = bad_seen + bad_absent; let rollback_idx2 = rollback_rows | arrange; @@ -38,14 +38,14 @@ res: { } let rb = res::rollback; let rb_idx = rb | arrange; -let wbad = wid_idx | join(rb_idx, (wid, kv, ()) -> (wid, kv)); +let wbad = wid_idx | join(rb_idx, (wid, (kv, ())) -> (wid, kv)); let committed_neg = wbad | negate; let committed = wid + committed_neg; let latest_map = committed | map((wid, (k, v)) -> (k, (wid neg, v))); let latest_idx = latest_map | arrange; let latest = latest_idx | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); let new_rows = latest | map((k, (nw, v)) -> (k, (nw neg, v))); -let old_rows = latest | join(store_idx, (k, _, (sw, sv)) -> (k, (sw, sv))); +let old_rows = latest | join(store_idx, (k, (_, (sw, sv))) -> (k, (sw, sv))); let delta_neg = old_rows | negate; let delta_rows = new_rows + delta_neg; -- Renderer: build an export arrangement. diff --git a/interactive/tests/plan_pairs/expected/txn_resolve.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/txn_resolve.lowered--optimized.diff index 44ef19311..6467f0061 100644 --- a/interactive/tests/plan_pairs/expected/txn_resolve.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/txn_resolve.lowered--optimized.diff @@ -23,15 +23,15 @@ var rollback; - let wid_idx = ::wid | arrange; let rollback_idx = rollback | arrange; -- let wr_bad = wid_idx | join(rollback_idx, (wid, kv, ()) -> (wid, kv)); -+ let wr_bad = ::wid_idx | join(rollback_idx, (wid, kv, ()) -> (wid, kv)); +- let wr_bad = wid_idx | join(rollback_idx, (wid, (kv, ())) -> (wid, kv)); ++ let wr_bad = ::wid_idx | join(rollback_idx, (wid, (kv, ())) -> (wid, kv)); let writes_neg = wr_bad | negate; let writes = ::wid + writes_neg; let wk = writes | map((wid, (k, v)) -> (k, (wid, v))); - let rk_idx = ::rk | arrange; let wk_idx = wk | arrange; -- let pairs_join = rk_idx | join(wk_idx, (k, (rid, e), (wid, v)) -> ((k, rid, e), (wid neg, v))); -+ let pairs_join = ::rk_idx | join(wk_idx, (k, (rid, e), (wid, v)) -> ((k, rid, e), (wid neg, v))); +- let pairs_join = rk_idx | join(wk_idx, (k, ((rid, e), (wid, v))) -> ((k, rid, e), (wid neg, v))); ++ let pairs_join = ::rk_idx | join(wk_idx, (k, ((rid, e), (wid, v))) -> ((k, rid, e), (wid neg, v))); let pairs = pairs_join | filter(((_, rid, _), (nw, _)) -> (nw neg, rid) lt); let best_rows = pairs + ::base; let best_idx = best_rows | arrange; @@ -47,10 +47,10 @@ + let unseen = ::rkk + unseen_rows; let unseen_idx = unseen | arrange; - let reads_idx = ::reads | arrange; -- let bad_absent_join = unseen_idx | join(reads_idx, ((rid, _), (), e) -> ((rid, e), ())); +- let bad_absent_join = unseen_idx | join(reads_idx, ((rid, _), ((), e)) -> ((rid, e), ())); - let bad_absent_filter = bad_absent_join | filter(((_, e), ()) -> (e, -1) ne); - let bad_absent = bad_absent_filter | map(((rid, _), ()) -> (rid, ())); -+ let bad_absent_join = unseen_idx | join(::reads_idx, ((rid, _), (), e) -> ((rid, e), ())); ++ let bad_absent_join = unseen_idx | join(::reads_idx, ((rid, _), ((), e)) -> ((rid, e), ())); + let bad_absent = bad_absent_join | filter(((_, e), ()) -> (e, -1) ne) | map(((rid, _), ()) -> (rid, ())); let rollback_rows = bad_seen + bad_absent; let rollback_idx2 = rollback_rows | arrange; @@ -60,15 +60,15 @@ let rb = res::rollback; -let wid_idx = wid | arrange; let rb_idx = rb | arrange; - let wbad = wid_idx | join(rb_idx, (wid, kv, ()) -> (wid, kv)); + let wbad = wid_idx | join(rb_idx, (wid, (kv, ())) -> (wid, kv)); let committed_neg = wbad | negate; @@ -53,8 +45,7 @@ let latest_idx = latest_map | arrange; let latest = latest_idx | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); let new_rows = latest | map((k, (nw, v)) -> (k, (nw neg, v))); -let store_idx2 = store | arrange; --let old_rows = latest | join(store_idx2, (k, _, (sw, sv)) -> (k, (sw, sv))); -+let old_rows = latest | join(store_idx, (k, _, (sw, sv)) -> (k, (sw, sv))); +-let old_rows = latest | join(store_idx2, (k, (_, (sw, sv))) -> (k, (sw, sv))); ++let old_rows = latest | join(store_idx, (k, (_, (sw, sv))) -> (k, (sw, sv))); let delta_neg = old_rows | negate; let delta_rows = new_rows + delta_neg; -- Renderer: build an export arrangement. diff --git a/interactive/tests/plan_pairs/expected/txn_resolve.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/txn_resolve.parsed--lowered.diff index f82a906df..5b4b13d33 100644 --- a/interactive/tests/plan_pairs/expected/txn_resolve.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/txn_resolve.parsed--lowered.diff @@ -13,27 +13,27 @@ +let wr_all = wr_all_filter | map(((wid, _, k), v) -> ((wid, k), v)); let wid = wr_all | map(((wid, k), v) -> (wid, (k, v))); let rk = reads | map(((rid, k), e) -> (k, (rid, e))); --let base = rk | join(store, (k, (rid, e), (sw, v)) -> ((k, rid, e), (sw neg, v))); +-let base = rk | join(store, (k, ((rid, e), (sw, v))) -> ((k, rid, e), (sw neg, v))); +let rk_idx = rk | arrange; +let store_idx = store | arrange; -+let base = rk_idx | join(store_idx, (k, (rid, e), (sw, v)) -> ((k, rid, e), (sw neg, v))); ++let base = rk_idx | join(store_idx, (k, ((rid, e), (sw, v))) -> ((k, rid, e), (sw neg, v))); res: { -- let wr_bad = wid | join(rollback, (wid, kv, ()) -> (wid, kv)); +- let wr_bad = wid | join(rollback, (wid, (kv, ())) -> (wid, kv)); - let writes = wid + (wr_bad | negate); + -- Time lanes: epoch, then 1 scope coordinates; this scope enters at zero. + var rollback; + let wid_idx = ::wid | arrange; + let rollback_idx = rollback | arrange; -+ let wr_bad = wid_idx | join(rollback_idx, (wid, kv, ()) -> (wid, kv)); ++ let wr_bad = wid_idx | join(rollback_idx, (wid, (kv, ())) -> (wid, kv)); + let writes_neg = wr_bad | negate; + let writes = ::wid + writes_neg; let wk = writes | map((wid, (k, v)) -> (k, (wid, v))); -- let pairs = rk | join(wk, (k, (rid, e), (wid, v)) -> ((k, rid, e), (wid neg, v))) | filter(((_, rid, _), (nw, _)) -> (nw neg, rid) lt); +- let pairs = rk | join(wk, (k, ((rid, e), (wid, v))) -> ((k, rid, e), (wid neg, v))) | filter(((_, rid, _), (nw, _)) -> (nw neg, rid) lt); - let best = pairs + base | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); - let bad_seen = best | filter(((_, _, e), (_, v)) -> (v, e) ne) | map(((_, rid, _), _) -> (rid, ())); + let rk_idx = ::rk | arrange; + let wk_idx = wk | arrange; -+ let pairs_join = rk_idx | join(wk_idx, (k, (rid, e), (wid, v)) -> ((k, rid, e), (wid neg, v))); ++ let pairs_join = rk_idx | join(wk_idx, (k, ((rid, e), (wid, v))) -> ((k, rid, e), (wid neg, v))); + let pairs = pairs_join | filter(((_, rid, _), (nw, _)) -> (nw neg, rid) lt); + let best_rows = pairs + ::base; + let best_idx = best_rows | arrange; @@ -43,14 +43,14 @@ let seen = best | map(((k, rid, _), _) -> ((rid, k), ())); - let rkk = reads | map((rk, _) -> (rk, ())); - let unseen = rkk + (seen | negate); -- let bad_absent = unseen | join(reads, ((rid, _), (), e) -> ((rid, e), ())) | filter(((_, e), ()) -> (e, -1) ne) | map(((rid, _), ()) -> (rid, ())); +- let bad_absent = unseen | join(reads, ((rid, _), ((), e)) -> ((rid, e), ())) | filter(((_, e), ()) -> (e, -1) ne) | map(((rid, _), ()) -> (rid, ())); - set rollback = bad_seen + bad_absent | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); + let rkk = ::reads | map((rk, _) -> (rk, ())); + let unseen_neg = seen | negate; + let unseen = rkk + unseen_neg; + let unseen_idx = unseen | arrange; + let reads_idx = ::reads | arrange; -+ let bad_absent_join = unseen_idx | join(reads_idx, ((rid, _), (), e) -> ((rid, e), ())); ++ let bad_absent_join = unseen_idx | join(reads_idx, ((rid, _), ((), e)) -> ((rid, e), ())); + let bad_absent_filter = bad_absent_join | filter(((_, e), ()) -> (e, -1) ne); + let bad_absent = bad_absent_filter | map(((rid, _), ()) -> (rid, ())); + let rollback_rows = bad_seen + bad_absent; @@ -59,21 +59,21 @@ + set rollback = rollback_reduce; } let rb = res::rollback; --let wbad = wid | join(rb, (wid, kv, ()) -> (wid, kv)); +-let wbad = wid | join(rb, (wid, (kv, ())) -> (wid, kv)); -let committed = wid + (wbad | negate); -let latest = committed | map((wid, (k, v)) -> (k, (wid neg, v))) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); +let wid_idx = wid | arrange; +let rb_idx = rb | arrange; -+let wbad = wid_idx | join(rb_idx, (wid, kv, ()) -> (wid, kv)); ++let wbad = wid_idx | join(rb_idx, (wid, (kv, ())) -> (wid, kv)); +let committed_neg = wbad | negate; +let committed = wid + committed_neg; +let latest_map = committed | map((wid, (k, v)) -> (k, (wid neg, v))); +let latest_idx = latest_map | arrange; +let latest = latest_idx | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); let new_rows = latest | map((k, (nw, v)) -> (k, (nw neg, v))); --let old_rows = latest | join(store, (k, _, (sw, sv)) -> (k, (sw, sv))); +-let old_rows = latest | join(store, (k, (_, (sw, sv))) -> (k, (sw, sv))); +let store_idx2 = store | arrange; -+let old_rows = latest | join(store_idx2, (k, _, (sw, sv)) -> (k, (sw, sv))); ++let old_rows = latest | join(store_idx2, (k, (_, (sw, sv))) -> (k, (sw, sv))); +let delta_neg = old_rows | negate; +let delta_rows = new_rows + delta_neg; +-- Renderer: build an export arrangement. diff --git a/interactive/tests/plan_pairs/expected/txn_resolve.parsed.ddp b/interactive/tests/plan_pairs/expected/txn_resolve.parsed.ddp index 49a47a375..9cb339de1 100644 --- a/interactive/tests/plan_pairs/expected/txn_resolve.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/txn_resolve.parsed.ddp @@ -4,25 +4,25 @@ let reads = intents | filter(((_, r, _), _) -> (r, 1) eq) | map(((rid, _, k), e) let wr_all = intents | filter(((_, r, _), _) -> (r, 0) eq) | map(((wid, _, k), v) -> ((wid, k), v)); let wid = wr_all | map(((wid, k), v) -> (wid, (k, v))); let rk = reads | map(((rid, k), e) -> (k, (rid, e))); -let base = rk | join(store, (k, (rid, e), (sw, v)) -> ((k, rid, e), (sw neg, v))); +let base = rk | join(store, (k, ((rid, e), (sw, v))) -> ((k, rid, e), (sw neg, v))); res: { - let wr_bad = wid | join(rollback, (wid, kv, ()) -> (wid, kv)); + let wr_bad = wid | join(rollback, (wid, (kv, ())) -> (wid, kv)); let writes = wid + (wr_bad | negate); let wk = writes | map((wid, (k, v)) -> (k, (wid, v))); - let pairs = rk | join(wk, (k, (rid, e), (wid, v)) -> ((k, rid, e), (wid neg, v))) | filter(((_, rid, _), (nw, _)) -> (nw neg, rid) lt); + let pairs = rk | join(wk, (k, ((rid, e), (wid, v))) -> ((k, rid, e), (wid neg, v))) | filter(((_, rid, _), (nw, _)) -> (nw neg, rid) lt); let best = pairs + base | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); let bad_seen = best | filter(((_, _, e), (_, v)) -> (v, e) ne) | map(((_, rid, _), _) -> (rid, ())); let seen = best | map(((k, rid, _), _) -> ((rid, k), ())); let rkk = reads | map((rk, _) -> (rk, ())); let unseen = rkk + (seen | negate); - let bad_absent = unseen | join(reads, ((rid, _), (), e) -> ((rid, e), ())) | filter(((_, e), ()) -> (e, -1) ne) | map(((rid, _), ()) -> (rid, ())); + let bad_absent = unseen | join(reads, ((rid, _), ((), e)) -> ((rid, e), ())) | filter(((_, e), ()) -> (e, -1) ne) | map(((rid, _), ()) -> (rid, ())); set rollback = bad_seen + bad_absent | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); } let rb = res::rollback; -let wbad = wid | join(rb, (wid, kv, ()) -> (wid, kv)); +let wbad = wid | join(rb, (wid, (kv, ())) -> (wid, kv)); let committed = wid + (wbad | negate); let latest = committed | map((wid, (k, v)) -> (k, (wid neg, v))) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); let new_rows = latest | map((k, (nw, v)) -> (k, (nw neg, v))); -let old_rows = latest | join(store, (k, _, (sw, sv)) -> (k, (sw, sv))); +let old_rows = latest | join(store, (k, (_, (sw, sv))) -> (k, (sw, sv))); output "rollback" = rb; output "delta" = new_rows + (old_rows | negate); diff --git a/interactive/tests/plan_pairs/expected/txn_resolve_by_key.ddp b/interactive/tests/plan_pairs/expected/txn_resolve_by_key.ddp index 6166833e4..7b47c6596 100644 --- a/interactive/tests/plan_pairs/expected/txn_resolve_by_key.ddp +++ b/interactive/tests/plan_pairs/expected/txn_resolve_by_key.ddp @@ -7,12 +7,12 @@ let wid_idx = wid | arrange; let read_entries = intents | filter(((_, r, _), _) -> (r, 1) eq) | map(((a, _, b), c) -> (b, (1, a, c))); let read_entries_idx = read_entries | arrange; let store_idx = store | arrange; -let store_entries = read_entries_idx | join(store_idx, (k, _, (sw, v)) -> (k, (0, sw, v))); +let store_entries = read_entries_idx | join(store_idx, (k, (_, (sw, v))) -> (k, (0, sw, v))); res: { -- Time lanes: epoch, then 1 scope coordinates; this scope enters at zero. var rollback; let rollback_idx = rollback | arrange; - let wr_bad = ::wid_idx | join(rollback_idx, (wid, kv, ()) -> (wid, kv)); + let wr_bad = ::wid_idx | join(rollback_idx, (wid, (kv, ())) -> (wid, kv)); let writes_neg = wr_bad | negate; let writes = ::wid + writes_neg; let write_entries = writes | map((wid, (k, v)) -> (k, (0, wid, v))); @@ -34,14 +34,14 @@ res: { } let rb = res::rollback; let rb_idx = rb | arrange; -let wbad = wid_idx | join(rb_idx, (wid, kv, ()) -> (wid, kv)); +let wbad = wid_idx | join(rb_idx, (wid, (kv, ())) -> (wid, kv)); let committed_neg = wbad | negate; let committed = wid + committed_neg; let latest_map = committed | map((wid, (k, v)) -> (k, (wid neg, v))); let latest_idx = latest_map | arrange; let latest = latest_idx | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); let new_rows = latest | map((k, (nw, v)) -> (k, (nw neg, v))); -let old_rows = latest | join(store_idx, (k, _, (sw, sv)) -> (k, (sw, sv))); +let old_rows = latest | join(store_idx, (k, (_, (sw, sv))) -> (k, (sw, sv))); let delta_neg = old_rows | negate; let delta_rows = new_rows + delta_neg; -- Renderer: build an export arrangement. diff --git a/interactive/tests/plan_pairs/expected/txn_resolve_by_key.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/txn_resolve_by_key.lowered--optimized.diff index ad3c66e2f..702cf93d9 100644 --- a/interactive/tests/plan_pairs/expected/txn_resolve_by_key.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/txn_resolve_by_key.lowered--optimized.diff @@ -15,14 +15,14 @@ +let read_entries = intents | filter(((_, r, _), _) -> (r, 1) eq) | map(((a, _, b), c) -> (b, (1, a, c))); let read_entries_idx = read_entries | arrange; let store_idx = store | arrange; - let store_entries = read_entries_idx | join(store_idx, (k, _, (sw, v)) -> (k, (0, sw, v))); + let store_entries = read_entries_idx | join(store_idx, (k, (_, (sw, v))) -> (k, (0, sw, v))); res: { -- Time lanes: epoch, then 1 scope coordinates; this scope enters at zero. var rollback; - let wid_idx = ::wid | arrange; let rollback_idx = rollback | arrange; -- let wr_bad = wid_idx | join(rollback_idx, (wid, kv, ()) -> (wid, kv)); -+ let wr_bad = ::wid_idx | join(rollback_idx, (wid, kv, ()) -> (wid, kv)); +- let wr_bad = wid_idx | join(rollback_idx, (wid, (kv, ())) -> (wid, kv)); ++ let wr_bad = ::wid_idx | join(rollback_idx, (wid, (kv, ())) -> (wid, kv)); let writes_neg = wr_bad | negate; let writes = ::wid + writes_neg; let write_entries = writes | map((wid, (k, v)) -> (k, (0, wid, v))); @@ -32,15 +32,15 @@ let rb = res::rollback; -let wid_idx = wid | arrange; let rb_idx = rb | arrange; - let wbad = wid_idx | join(rb_idx, (wid, kv, ()) -> (wid, kv)); + let wbad = wid_idx | join(rb_idx, (wid, (kv, ())) -> (wid, kv)); let committed_neg = wbad | negate; @@ -46,8 +41,7 @@ let latest_idx = latest_map | arrange; let latest = latest_idx | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); let new_rows = latest | map((k, (nw, v)) -> (k, (nw neg, v))); -let store_idx2 = store | arrange; --let old_rows = latest | join(store_idx2, (k, _, (sw, sv)) -> (k, (sw, sv))); -+let old_rows = latest | join(store_idx, (k, _, (sw, sv)) -> (k, (sw, sv))); +-let old_rows = latest | join(store_idx2, (k, (_, (sw, sv))) -> (k, (sw, sv))); ++let old_rows = latest | join(store_idx, (k, (_, (sw, sv))) -> (k, (sw, sv))); let delta_neg = old_rows | negate; let delta_rows = new_rows + delta_neg; -- Renderer: build an export arrangement. diff --git a/interactive/tests/plan_pairs/expected/txn_resolve_by_key.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/txn_resolve_by_key.parsed--lowered.diff index 1460cc13d..aa20e728b 100644 --- a/interactive/tests/plan_pairs/expected/txn_resolve_by_key.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/txn_resolve_by_key.parsed--lowered.diff @@ -13,18 +13,18 @@ +let wr_all = wr_all_filter | map(((wid, _, k), v) -> ((wid, k), v)); let wid = wr_all | map(((wid, k), v) -> (wid, (k, v))); let read_entries = reads | map(((rid, k), e) -> (k, (1, rid, e))); --let store_entries = read_entries | join(store, (k, _, (sw, v)) -> (k, (0, sw, v))); +-let store_entries = read_entries | join(store, (k, (_, (sw, v))) -> (k, (0, sw, v))); +let read_entries_idx = read_entries | arrange; +let store_idx = store | arrange; -+let store_entries = read_entries_idx | join(store_idx, (k, _, (sw, v)) -> (k, (0, sw, v))); ++let store_entries = read_entries_idx | join(store_idx, (k, (_, (sw, v))) -> (k, (0, sw, v))); res: { -- let wr_bad = wid | join(rollback, (wid, kv, ()) -> (wid, kv)); +- let wr_bad = wid | join(rollback, (wid, (kv, ())) -> (wid, kv)); - let writes = wid + (wr_bad | negate); + -- Time lanes: epoch, then 1 scope coordinates; this scope enters at zero. + var rollback; + let wid_idx = ::wid | arrange; + let rollback_idx = rollback | arrange; -+ let wr_bad = wid_idx | join(rollback_idx, (wid, kv, ()) -> (wid, kv)); ++ let wr_bad = wid_idx | join(rollback_idx, (wid, (kv, ())) -> (wid, kv)); + let writes_neg = wr_bad | negate; + let writes = ::wid + writes_neg; let write_entries = writes | map((wid, (k, v)) -> (k, (0, wid, v))); @@ -46,21 +46,21 @@ + set rollback = rollback_reduce; } let rb = res::rollback; --let wbad = wid | join(rb, (wid, kv, ()) -> (wid, kv)); +-let wbad = wid | join(rb, (wid, (kv, ())) -> (wid, kv)); -let committed = wid + (wbad | negate); -let latest = committed | map((wid, (k, v)) -> (k, (wid neg, v))) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); +let wid_idx = wid | arrange; +let rb_idx = rb | arrange; -+let wbad = wid_idx | join(rb_idx, (wid, kv, ()) -> (wid, kv)); ++let wbad = wid_idx | join(rb_idx, (wid, (kv, ())) -> (wid, kv)); +let committed_neg = wbad | negate; +let committed = wid + committed_neg; +let latest_map = committed | map((wid, (k, v)) -> (k, (wid neg, v))); +let latest_idx = latest_map | arrange; +let latest = latest_idx | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); let new_rows = latest | map((k, (nw, v)) -> (k, (nw neg, v))); --let old_rows = latest | join(store, (k, _, (sw, sv)) -> (k, (sw, sv))); +-let old_rows = latest | join(store, (k, (_, (sw, sv))) -> (k, (sw, sv))); +let store_idx2 = store | arrange; -+let old_rows = latest | join(store_idx2, (k, _, (sw, sv)) -> (k, (sw, sv))); ++let old_rows = latest | join(store_idx2, (k, (_, (sw, sv))) -> (k, (sw, sv))); +let delta_neg = old_rows | negate; +let delta_rows = new_rows + delta_neg; +-- Renderer: build an export arrangement. diff --git a/interactive/tests/plan_pairs/expected/txn_resolve_by_key.parsed.ddp b/interactive/tests/plan_pairs/expected/txn_resolve_by_key.parsed.ddp index 5f959e354..5a54935c6 100644 --- a/interactive/tests/plan_pairs/expected/txn_resolve_by_key.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/txn_resolve_by_key.parsed.ddp @@ -4,9 +4,9 @@ let reads = intents | filter(((_, r, _), _) -> (r, 1) eq) | map(((rid, _, k), e) let wr_all = intents | filter(((_, r, _), _) -> (r, 0) eq) | map(((wid, _, k), v) -> ((wid, k), v)); let wid = wr_all | map(((wid, k), v) -> (wid, (k, v))); let read_entries = reads | map(((rid, k), e) -> (k, (1, rid, e))); -let store_entries = read_entries | join(store, (k, _, (sw, v)) -> (k, (0, sw, v))); +let store_entries = read_entries | join(store, (k, (_, (sw, v))) -> (k, (0, sw, v))); res: { - let wr_bad = wid | join(rollback, (wid, kv, ()) -> (wid, kv)); + let wr_bad = wid | join(rollback, (wid, (kv, ())) -> (wid, kv)); let writes = wid + (wr_bad | negate); let write_entries = writes | map((wid, (k, v)) -> (k, (0, wid, v))); let bad = read_entries + store_entries + write_entries | reduce((_, vals) -> let entries = vals map ((e, _) -> e) in @@ -21,10 +21,10 @@ res: { set rollback = bad | map((_, rid) -> (rid, ())) | map((k, v) -> ((k, v), ())) | reduce((_, vals) -> vals map (((), _) -> ((), 1))) | map((kv, ()) -> kv); } let rb = res::rollback; -let wbad = wid | join(rb, (wid, kv, ()) -> (wid, kv)); +let wbad = wid | join(rb, (wid, (kv, ())) -> (wid, kv)); let committed = wid + (wbad | negate); let latest = committed | map((wid, (k, v)) -> (k, (wid neg, v))) | reduce(limit 1, (_, vals) -> let first = vals head in (first.0, 1) enlist); let new_rows = latest | map((k, (nw, v)) -> (k, (nw neg, v))); -let old_rows = latest | join(store, (k, _, (sw, sv)) -> (k, (sw, sv))); +let old_rows = latest | join(store, (k, (_, (sw, sv))) -> (k, (sw, sv))); output "rollback" = rb; output "delta" = new_rows + (old_rows | negate); diff --git a/interactive/tests/plan_pairs/generated/cascade.ddp b/interactive/tests/plan_pairs/generated/cascade.ddp index 42d406637..fa0b9897f 100644 --- a/interactive/tests/plan_pairs/generated/cascade.ddp +++ b/interactive/tests/plan_pairs/generated/cascade.ddp @@ -25,20 +25,20 @@ let del_likes = input 22 : ((int, int, int), ()); -- People deleted, and the forums they moderated: Walls and Albums are deleted, Groups lose their moderator. let dead_p = del_person | map(((id, _), ()) -> (id, ())) | distinct; let owned = forum_in | map(((id, title, created, moderator), ()) -> (moderator, (id, title, created))) - | join(dead_p, (moderator, (id, title, created), ()) -> (id, (title, created, moderator))); + | join(dead_p, (moderator, ((id, title, created), ())) -> (id, (title, created, moderator))); let dead_f = ((del_forum | map(((id, _), ()) -> (id, ()))) + (owned | filter((_, (title, _, _)) -> ((((0, (title len, 6) min) range, title) gather, "Album ") eq, (((0, (title len, 5) min) range, title) gather, "Wall ") eq) any) | map((id, _) -> (id, ())))) | distinct; let groups = owned | filter((_, (title, _, _)) -> (((0, (title len, 6) min) range, title) gather, "Group ") eq); -let kept = groups - (groups | join(dead_f, (id, v, ()) -> (id, v))); +let kept = groups - (groups | join(dead_f, (id, (v, ())) -> (id, v))); -- Messages deleted: explicit targets, those of deleted people, the posts of deleted forums, -- and every reply below any of them. let seeds = (del_message | map(((id, _), ()) -> (id, ()))) - + (message_in | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (creator, id)) | join(dead_p, (_, id, ()) -> (id, ()))) - + (message_in | filter(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (kind, 0) eq) | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (forum, id)) | join(dead_f, (_, id, ()) -> (id, ()))); + + (message_in | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (creator, id)) | join(dead_p, (_, (id, ())) -> (id, ()))) + + (message_in | filter(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (kind, 0) eq) | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (forum, id)) | join(dead_f, (_, (id, ())) -> (id, ()))); let children = message_in | filter(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (kind, 1) eq) | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (parent, id)); dead_m: { - let kids = dead_m | join(children, (_, (), child) -> (child, ())); + let kids = dead_m | join(children, (_, ((), child)) -> (child, ())); set dead_m = (seeds + kids) | distinct; } let dead_msg = dead_m::dead_m; @@ -46,24 +46,24 @@ let gone_knows = del_knows | map(((a, b, _), ()) -> ((a, b), ())) | distinct; let gone_member = del_member | map(((f, p, _), ()) -> ((f, p), ())) | distinct; let gone_likes = del_likes | map(((p, x, _), ()) -> ((p, x), ())) | distinct; -let person = person_in - (((person_in | map(((id, first, last, gender, birthday, created, ip, browser, city, birthmonth, birthdaynum, creationmonth), ()) -> (id, (id, first, last, gender, birthday, created, ip, browser, city, birthmonth, birthdaynum, creationmonth))) | join(dead_p, (_, r, ()) -> (r, ())))) | distinct); -let message = message_in - (((message_in | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (id, (id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day))) | join(dead_msg, (_, r, ()) -> (r, ())))) | distinct); -let forum = forum_in - (((forum_in | map(((id, title, created, moderator), ()) -> (id, (id, title, created, moderator))) | join(dead_f, (_, r, ()) -> (r, ()))) + (kept | map((id, (title, created, moderator)) -> ((id, title, created, moderator), ())))) | distinct) +let person = person_in - (((person_in | map(((id, first, last, gender, birthday, created, ip, browser, city, birthmonth, birthdaynum, creationmonth), ()) -> (id, (id, first, last, gender, birthday, created, ip, browser, city, birthmonth, birthdaynum, creationmonth))) | join(dead_p, (_, (r, ())) -> (r, ())))) | distinct); +let message = message_in - (((message_in | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (id, (id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day))) | join(dead_msg, (_, (r, ())) -> (r, ())))) | distinct); +let forum = forum_in - (((forum_in | map(((id, title, created, moderator), ()) -> (id, (id, title, created, moderator))) | join(dead_f, (_, (r, ())) -> (r, ()))) + (kept | map((id, (title, created, moderator)) -> ((id, title, created, moderator), ())))) | distinct) + (kept | map((id, (title, created, _)) -> ((id, title, created, -1), ()))); -let knows = knows_in - (((knows_in | map(((src, dst, created), ()) -> ((src, dst), (src, dst, created))) | join(gone_knows, (_, r, ()) -> (r, ()))) + (knows_in | map(((src, dst, created), ()) -> (src, (src, dst, created))) | join(dead_p, (_, r, ()) -> (r, ()))) + (knows_in | map(((src, dst, created), ()) -> (dst, (src, dst, created))) | join(dead_p, (_, r, ()) -> (r, ())))) | distinct); -let member = member_in - (((member_in | map(((forum, person, created), ()) -> ((forum, person), (forum, person, created))) | join(gone_member, (_, r, ()) -> (r, ()))) + (member_in | map(((forum, person, created), ()) -> (forum, (forum, person, created))) | join(dead_f, (_, r, ()) -> (r, ()))) + (member_in | map(((forum, person, created), ()) -> (person, (forum, person, created))) | join(dead_p, (_, r, ()) -> (r, ())))) | distinct); +let knows = knows_in - (((knows_in | map(((src, dst, created), ()) -> ((src, dst), (src, dst, created))) | join(gone_knows, (_, (r, ())) -> (r, ()))) + (knows_in | map(((src, dst, created), ()) -> (src, (src, dst, created))) | join(dead_p, (_, (r, ())) -> (r, ()))) + (knows_in | map(((src, dst, created), ()) -> (dst, (src, dst, created))) | join(dead_p, (_, (r, ())) -> (r, ())))) | distinct); +let member = member_in - (((member_in | map(((forum, person, created), ()) -> ((forum, person), (forum, person, created))) | join(gone_member, (_, (r, ())) -> (r, ()))) + (member_in | map(((forum, person, created), ()) -> (forum, (forum, person, created))) | join(dead_f, (_, (r, ())) -> (r, ()))) + (member_in | map(((forum, person, created), ()) -> (person, (forum, person, created))) | join(dead_p, (_, (r, ())) -> (r, ())))) | distinct); let tag = tag_in; let tagclass = tagclass_in; let place = place_in; let org = org_in; -let interest = interest_in - (((interest_in | map(((person, tag), ()) -> (person, (person, tag))) | join(dead_p, (_, r, ()) -> (r, ())))) | distinct); -let mtag = mtag_in - (((mtag_in | map(((message, tag), ()) -> (message, (message, tag))) | join(dead_msg, (_, r, ()) -> (r, ())))) | distinct); -let ftag = ftag_in - (((ftag_in | map(((forum, tag), ()) -> (forum, (forum, tag))) | join(dead_f, (_, r, ()) -> (r, ())))) | distinct); -let likes = likes_in - (((likes_in | map(((person, message, created), ()) -> ((person, message), (person, message, created))) | join(gone_likes, (_, r, ()) -> (r, ()))) + (likes_in | map(((person, message, created), ()) -> (person, (person, message, created))) | join(dead_p, (_, r, ()) -> (r, ()))) + (likes_in | map(((person, message, created), ()) -> (message, (person, message, created))) | join(dead_msg, (_, r, ()) -> (r, ())))) | distinct); -let study = study_in - (((study_in | map(((person, org, year), ()) -> (person, (person, org, year))) | join(dead_p, (_, r, ()) -> (r, ())))) | distinct); -let work = work_in - (((work_in | map(((person, org, year), ()) -> (person, (person, org, year))) | join(dead_p, (_, r, ()) -> (r, ())))) | distinct); -let email = email_in - (((email_in | map(((person, value), ()) -> (person, (person, value))) | join(dead_p, (_, r, ()) -> (r, ())))) | distinct); -let language = language_in - (((language_in | map(((person, value), ()) -> (person, (person, value))) | join(dead_p, (_, r, ()) -> (r, ())))) | distinct); +let interest = interest_in - (((interest_in | map(((person, tag), ()) -> (person, (person, tag))) | join(dead_p, (_, (r, ())) -> (r, ())))) | distinct); +let mtag = mtag_in - (((mtag_in | map(((message, tag), ()) -> (message, (message, tag))) | join(dead_msg, (_, (r, ())) -> (r, ())))) | distinct); +let ftag = ftag_in - (((ftag_in | map(((forum, tag), ()) -> (forum, (forum, tag))) | join(dead_f, (_, (r, ())) -> (r, ())))) | distinct); +let likes = likes_in - (((likes_in | map(((person, message, created), ()) -> ((person, message), (person, message, created))) | join(gone_likes, (_, (r, ())) -> (r, ()))) + (likes_in | map(((person, message, created), ()) -> (person, (person, message, created))) | join(dead_p, (_, (r, ())) -> (r, ()))) + (likes_in | map(((person, message, created), ()) -> (message, (person, message, created))) | join(dead_msg, (_, (r, ())) -> (r, ())))) | distinct); +let study = study_in - (((study_in | map(((person, org, year), ()) -> (person, (person, org, year))) | join(dead_p, (_, (r, ())) -> (r, ())))) | distinct); +let work = work_in - (((work_in | map(((person, org, year), ()) -> (person, (person, org, year))) | join(dead_p, (_, (r, ())) -> (r, ())))) | distinct); +let email = email_in - (((email_in | map(((person, value), ()) -> (person, (person, value))) | join(dead_p, (_, (r, ())) -> (r, ())))) | distinct); +let language = language_in - (((language_in | map(((person, value), ()) -> (person, (person, value))) | join(dead_p, (_, (r, ())) -> (r, ())))) | distinct); output "person" = person | map(((id, first, last, gender, birthday, created, ip, browser, city, birthmonth, birthdaynum, creationmonth), ()) -> (id, (first, last, gender, birthday, created, ip, browser, city, birthmonth, birthdaynum, creationmonth))) | arrange; output "person.by_city" = person | map(((id, first, last, gender, birthday, created, ip, browser, city, birthmonth, birthdaynum, creationmonth), ()) -> (city, id)) | arrange; output "message" = message | map(((id, kind, created, creator, country, content, image, length, language, forum, parent, year, month, day), ()) -> (id, (kind, created, creator, country, content, image, length, language, forum, parent, year, month, day))) | arrange; @@ -101,8 +101,8 @@ output "language.by_person" = language | map(((person, value), ()) -> (person, v -- Friendships in both directions, between people who exist: (person, (friend, since)). let exists = person | map(((id, _, _, _, _, _, _, _, _, _, _, _), ()) -> (id, ())) | distinct; let either = (knows | map(((a, b, t), ()) -> (a, (b, t)))) + (knows | map(((a, b, t), ()) -> (b, (a, t)))); -let friends = either | join(exists, (a, (b, t), ()) -> (b, (a, t))) - | join(exists, (b, (a, t), ()) -> ((a, b, t), ())) | distinct +let friends = either | join(exists, (a, ((b, t), ())) -> (b, (a, t))) + | join(exists, (b, ((a, t), ())) -> ((a, b, t), ())) | distinct | map(((a, b, t), ()) -> (a, (b, t))); output "friend" = friends | arrange; output "friend.pair" = friends | map((a, (b, _)) -> ((a, b), ())) | arrange; @@ -113,7 +113,7 @@ let replies = message | filter(((_, kind, _, _, _, _, _, _, _, _, _, _, _, _), ( | map(((id, _, _, _, _, _, _, _, _, _, parent, _, _, _), ()) -> (parent, id)); thread: { let below = thread | map(((mid, root), ()) -> (mid, root)) - | join(replies, (_, root, child) -> ((child, root), ())); + | join(replies, (_, (root, child)) -> ((child, root), ())); set thread = (posts + below) | distinct; } output "message.root" = thread::thread | map(((mid, root), ()) -> (mid, root)) | arrange; diff --git a/interactive/tests/plan_pairs/generated/graph.ddp b/interactive/tests/plan_pairs/generated/graph.ddp index a711c6669..2629fba93 100644 --- a/interactive/tests/plan_pairs/generated/graph.ddp +++ b/interactive/tests/plan_pairs/generated/graph.ddp @@ -52,8 +52,8 @@ output "language.by_person" = language | map(((person, value), ()) -> (person, v -- Friendships in both directions, between people who exist: (person, (friend, since)). let exists = person | map(((id, _, _, _, _, _, _, _, _, _, _, _), ()) -> (id, ())) | distinct; let either = (knows | map(((a, b, t), ()) -> (a, (b, t)))) + (knows | map(((a, b, t), ()) -> (b, (a, t)))); -let friends = either | join(exists, (a, (b, t), ()) -> (b, (a, t))) - | join(exists, (b, (a, t), ()) -> ((a, b, t), ())) | distinct +let friends = either | join(exists, (a, ((b, t), ())) -> (b, (a, t))) + | join(exists, (b, ((a, t), ())) -> ((a, b, t), ())) | distinct | map(((a, b, t), ()) -> (a, (b, t))); output "friend" = friends | arrange; output "friend.pair" = friends | map((a, (b, _)) -> ((a, b), ())) | arrange; @@ -64,7 +64,7 @@ let replies = message | filter(((_, kind, _, _, _, _, _, _, _, _, _, _, _, _), ( | map(((id, _, _, _, _, _, _, _, _, _, parent, _, _, _), ()) -> (parent, id)); thread: { let below = thread | map(((mid, root), ()) -> (mid, root)) - | join(replies, (_, root, child) -> ((child, root), ())); + | join(replies, (_, (root, child)) -> ((child, root), ())); set thread = (posts + below) | distinct; } output "message.root" = thread::thread | map(((mid, root), ()) -> (mid, root)) | arrange; diff --git a/interactive/tests/programs/cc_records.ddp b/interactive/tests/programs/cc_records.ddp index ed1e5e553..287e30cf5 100644 --- a/interactive/tests/programs/cc_records.ddp +++ b/interactive/tests/programs/cc_records.ddp @@ -11,7 +11,7 @@ let nodes = edges | map((a, _) -> (a, ())) | distinct; -- starting from its own name as a label, and -- maintaining the least label it has seen. cc: { - let props = labels | join(edges, (_, lbl, dst) -> (dst, lbl)); + let props = labels | join(edges, (_, (lbl, dst)) -> (dst, lbl)); set labels = nodes | map((n, _) -> (n, n)) + props | min; } diff --git a/interactive/tests/programs/cc_stages.ddp b/interactive/tests/programs/cc_stages.ddp index d202ac8a0..a8596ba1a 100644 --- a/interactive/tests/programs/cc_stages.ddp +++ b/interactive/tests/programs/cc_stages.ddp @@ -9,7 +9,7 @@ let nodes = edges | map((a, _) -> (a, ())) | distinct; -- each node starts with its own name as a label and keeps the least label it has seen cc: { - let props = labels | join(edges, (_, lbl, dst) -> (dst, lbl)); + let props = labels | join(edges, (_, (lbl, dst)) -> (dst, lbl)); set labels = nodes | map((n, _) -> (n, n)) + props | min; } diff --git a/interactive/tests/programs/f64_math.ddp b/interactive/tests/programs/f64_math.ddp index 1300a37f4..7f3320505 100644 --- a/interactive/tests/programs/f64_math.ddp +++ b/interactive/tests/programs/f64_math.ddp @@ -36,7 +36,7 @@ let spread = xs | flatmap((k, (x, y)) -> let l = (x call fabs call fsqrt enlist, -- A join whose projection runs a row at a time. let scale = input 1 : ((int, int), ()) | map(((a, b), ()) -> (a, b)); -let scaled = xs | join(scale, (k, (x, y), n) -> (k, ((x, n) call fpowi, (y call fabs, n to_float) call fpow call fint))); +let scaled = xs | join(scale, (k, ((x, y), n)) -> (k, ((x, n) call fpowi, (y call fabs, n to_float) call fpow call fint))); output "unary" = unary | arrange | inspect(total); output "binary" = binary | arrange | inspect(total); diff --git a/interactive/tests/programs/join_fallback.ddp b/interactive/tests/programs/join_fallback.ddp index 650a90cfe..62b85f371 100644 --- a/interactive/tests/programs/join_fallback.ddp +++ b/interactive/tests/programs/join_fallback.ddp @@ -5,6 +5,6 @@ enum L = L int; let left = input 0 : ((int, int), ()) | map(((a, b), ()) -> (a, b)); let right = input 1 : ((int, int), ()) | map(((a, b), ()) -> (a, b)); -let out = left | join(right, (k, l, r) -> (k, ((l, r) add inject L match (L (s -> s)), r))); +let out = left | join(right, (k, (l, r)) -> (k, ((l, r) add inject L match (L (s -> s)), r))); output "result" = out | arrange | inspect(total); diff --git a/interactive/tests/programs/pair_keys.ddp b/interactive/tests/programs/pair_keys.ddp index 9f047cd78..59157c1c5 100644 --- a/interactive/tests/programs/pair_keys.ddp +++ b/interactive/tests/programs/pair_keys.ddp @@ -4,6 +4,6 @@ let left = input 0 : ((int, int, int), ()) | map(((a, b, c), ()) -> ((a, b), c)); let right = input 1 : ((int, int, int), ()) | map(((a, b, c), ()) -> ((a, b), c)); -let out = left | join(right, (k, l, r) -> (k, (l, r) add)); +let out = left | join(right, (k, (l, r)) -> (k, (l, r) add)); output "result" = out | arrange | inspect(total); diff --git a/interactive/tests/programs/registered.ddp b/interactive/tests/programs/registered.ddp index 5a056b6c2..8d82d2270 100644 --- a/interactive/tests/programs/registered.ddp +++ b/interactive/tests/programs/registered.ddp @@ -26,7 +26,7 @@ let sevens = cells | map((n, ()) -> (n, (() call seven, n) add)); let doubled = cells | map((n, ()) -> (n, n call double)); -- A join whose projection calls a registered function on both sides. -let joined = cells | join(described, (n, (), ((m), _)) -> (n, (n, m) call blend)); +let joined = cells | join(described, (n, ((), ((m), _))) -> (n, (n, m) call blend)); output "cells" = cells | arrange | inspect(total); output "blended" = blended | arrange | inspect(total); diff --git a/interactive/tests/root_var.rs b/interactive/tests/root_var.rs index 63ef42356..7f5a313ce 100644 --- a/interactive/tests/root_var.rs +++ b/interactive/tests/root_var.rs @@ -28,7 +28,7 @@ fn run(src: &'static str, epochs: Vec>) -> Vec<(Vec<(String, S const REMEMBER: &str = r#" let x = input 0 : (int, ()); var seen : (int, ()); - let fresh = x - (x | join(seen, (k, (), ()) -> (k, ()))); + let fresh = x - (x | join(seen, (k, ((), ())) -> (k, ()))); set seen = seen + fresh; output "seen" = seen; output "fresh" = fresh; diff --git a/interactive/tests/server_arranged_imports.rs b/interactive/tests/server_arranged_imports.rs index a7e0a0b0f..9123e9ff8 100644 --- a/interactive/tests/server_arranged_imports.rs +++ b/interactive/tests/server_arranged_imports.rs @@ -19,10 +19,10 @@ fn consumer() -> String { let edges = import "prod::edges"; let roots = input 0 : ((int), ()) | map(((n), ()) -> (n, ())); r: {{ - let step = reach | join(edges, (_, _, dst) -> (dst, ())); + let step = reach | join(edges, (_, (_, dst)) -> (dst, ())); set reach = roots + step | distinct; }} - output "hop" = roots | join(edges, (n, _, dst) -> (dst, n)); + output "hop" = roots | join(edges, (n, (_, dst)) -> (dst, n)); output "degree" = edges | count; output "reach" = r::reach; output "flipped" = edges | map((src, dst) -> (dst, src)); diff --git a/interactive/tests/server_late_import.rs b/interactive/tests/server_late_import.rs index 98494b54a..929757f82 100644 --- a/interactive/tests/server_late_import.rs +++ b/interactive/tests/server_late_import.rs @@ -25,7 +25,7 @@ const CONSUMER: &str = r#" let seeds = import "producer::seeds"; let edges = import "producer::edges"; s: { - let step = x | join(edges, ((_, b), (), ()) -> (((b, 5) rem, b), ())); + let step = x | join(edges, ((_, b), ((), ())) -> (((b, 5) rem, b), ())); set x = (seeds + step) | distinct; } output "result" = s::x; @@ -101,9 +101,9 @@ const ARRANGED_PRODUCER: &str = r#" const ARRANGED_CONSUMER: &str = r#" let pairs = import "producer::pairs"; let weights = import "producer::weights"; - let r = pairs | join(pairs, ((a, b), (), _) -> ((b, (a, b) min) eq, 0)) | distinct; + let r = pairs | join(pairs, ((a, b), ((), _)) -> ((b, (a, b) min) eq, 0)) | distinct; s: { - let step = x | join(weights, (k, v, _) -> (k, v)); + let step = x | join(weights, (k, (v, _)) -> (k, v)); set x = (r + step) | min; } output "result" = s::x; @@ -165,7 +165,7 @@ const DELTA_CONSUMER: &str = r#" + (pairs | lift | half_join(pairs, 1, ((a, b), (), _) -> ((b, (a, b) min) eq, 0)))) | delay((_, (_, t)) -> t) | map((k, (v, _)) -> (k, v)) | distinct; s: { - let step = x | join(weights, (k, v, _) -> (k, v)); + let step = x | join(weights, (k, (v, _)) -> (k, v)); set x = (r + step) | min; } output "result" = s::x; @@ -247,7 +247,7 @@ const TRIPLE_JOIN: &str = r#" let a = import "producer::a"; let b = import "producer::b"; let c = import "producer::c"; - output "r" = a | join(b, (k, x, y) -> (k, (x, y))) | join(c, (k, (x, y), z) -> (k, (x, y, z))); + output "r" = a | join(b) | join(c, (k, ((x, y), z)) -> (k, (x, y, z))); "#; /// The same join as a delta join: one path per import, each strict against the imports before it. diff --git a/interactive/tests/server_shapes.rs b/interactive/tests/server_shapes.rs index d533c1fa8..1752b6063 100644 --- a/interactive/tests/server_shapes.rs +++ b/interactive/tests/server_shapes.rs @@ -100,7 +100,7 @@ fn conflicting_shapes_reject_the_install() { timely::execute_directly(move |worker| { let mut server = Server::new(); // Join keys of different shapes. - let join = r#"let a = input 0 : ((int), ()); let b = input 1 : ((int, int), ()); output "j" = a | join(b, (k, _, _) -> (k, ()));"#; + let join = r#"let a = input 0 : ((int), ()); let b = input 1 : ((int, int), ()); output "j" = a | join(b, (k, (_, _)) -> (k, ()));"#; let err = server.install(worker, "join", &program(join)).unwrap_err(); assert!(err.contains("join keys"), "{err}"); // A field of an integer, through an import whose shape is published. diff --git a/interactive/tests/shared_arrangements.rs b/interactive/tests/shared_arrangements.rs index 2911928b9..4fc65d509 100644 --- a/interactive/tests/shared_arrangements.rs +++ b/interactive/tests/shared_arrangements.rs @@ -94,7 +94,7 @@ const REACH: &str = r#" let edges = input 0 : ((int, int), ()) | map(((src, dst), ()) -> (src, dst)) EDGES; let roots = edges | map((src, _) -> (src, ())) | distinct | filter((n, ()) -> (n, 3) lt); reach: { - let proposals = reach | join(edges, (_, _, dst) -> (dst, ())); + let proposals = reach | join(edges, (_, (_, dst)) -> (dst, ())); set reach = roots + proposals | distinct; } output "result" = reach::reach; diff --git a/interactive/tests/tritemporal.rs b/interactive/tests/tritemporal.rs index f8aedcdee..fa98c3e5d 100644 --- a/interactive/tests/tritemporal.rs +++ b/interactive/tests/tritemporal.rs @@ -27,28 +27,28 @@ const PROGRAM: &str = r#" let degrees = scc | map((src, _) -> (src, ())) | count; fwd: { let nodes = edges | map((_, dst) -> (dst, dst)) - | join(allowed, (dst, label, ()) -> (dst, label)) + | join(allowed, (dst, (label, ())) -> (dst, label)) | delay((_, n) -> (0, 0, 0, 0, (256, n call bit_length) mul)); let labels = proposals + nodes | min; - set proposals = labels | join(scc, (_, label, dst) -> (dst, label)) - | join(allowed, (dst, label, ()) -> (dst, label)); + set proposals = labels | join(scc, (_, (label, dst)) -> (dst, label)) + | join(allowed, (dst, (label, ())) -> (dst, label)); } let trim_fwd = edges - | join(fwd::labels, (src, dst, label) -> (dst, (src, label))) - | join(fwd::labels, (dst, (src, label), other) -> (dst, (src, label, other))) + | join(fwd::labels, (src, (dst, label)) -> (dst, (src, label))) + | join(fwd::labels, (dst, ((src, label), other)) -> (dst, (src, label, other))) | filter((_, (_, a, b)) -> (a, b) eq) | map((dst, (src, _, _)) -> (dst, src)); bwd: { let nodes = trans | map((_, dst) -> (dst, dst)) - | join(allowed, (dst, label, ()) -> (dst, label)) + | join(allowed, (dst, (label, ())) -> (dst, label)) | delay((_, n) -> (0, 0, 0, 0, (256, n call bit_length) mul)); let labels = proposals + nodes | min; - set proposals = labels | join(trim_fwd, (_, label, dst) -> (dst, label)) - | join(allowed, (dst, label, ()) -> (dst, label)); + set proposals = labels | join(trim_fwd, (_, (label, dst)) -> (dst, label)) + | join(allowed, (dst, (label, ())) -> (dst, label)); } let trim_bwd = trans - | join(bwd::labels, (src, dst, label) -> (dst, (src, label))) - | join(bwd::labels, (dst, (src, label), other) -> (dst, (src, label, other))) + | join(bwd::labels, (src, (dst, label)) -> (dst, (src, label))) + | join(bwd::labels, (dst, ((src, label), other)) -> (dst, (src, label, other))) | filter((_, (_, a, b)) -> (a, b) eq) | map((dst, (src, _, _)) -> (dst, src)); set trim = trim_bwd - edges; From f5eae0eb247504f42f1680f332d52c5c8ff52808 Mon Sep 17 00:00:00 2001 From: Frank McSherry Date: Fri, 20 Nov 2026 15:22:20 -0500 Subject: [PATCH 2/2] DDIR: print identity joins without a projection and explain the join binder --- interactive/README.md | 3 ++- interactive/src/lambda.rs | 5 +++++ interactive/src/plan.rs | 3 ++- interactive/src/plan/ddp.rs | 20 +++++++++++++++++++ interactive/src/plan/ddp/parsed.rs | 1 + interactive/src/shapes.rs | 9 ++++++++- .../plan_pairs/expected/ldbc_ic01_oneoff.ddp | 2 +- .../ldbc_ic01_oneoff.lowered--optimized.diff | 2 +- .../ldbc_ic01_oneoff.parsed--lowered.diff | 4 ++-- .../expected/ldbc_ic01_oneoff.parsed.ddp | 2 +- .../plan_pairs/expected/ldbc_ic03_oneoff.ddp | 2 +- .../ldbc_ic03_oneoff.lowered--optimized.diff | 2 +- .../ldbc_ic03_oneoff.parsed--lowered.diff | 2 +- .../expected/ldbc_ic03_oneoff.parsed.ddp | 2 +- 14 files changed, 47 insertions(+), 12 deletions(-) diff --git a/interactive/README.md b/interactive/README.md index 26476b02c..ea09a1bb4 100644 --- a/interactive/README.md +++ b/interactive/README.md @@ -66,7 +66,8 @@ let blocks = input 2 : (int, Member); -- doc -> blocked member The shape of records is load bearing when we perform joins and other data-parallel operations. The pair structure `(key, val)` calls out first how the data are organized, and then any associated values. For each pair of matches `(k, l)` and `(k, r)`, `left | join(right)` produces `(k, (l, r))`. -You can project that row inside the join: `left | join(right, f)` computes the same collection as `left | join(right) | map(f)`. +`left | join(right, f)` computes the same collection as `left | join(right) | map(f)`. +Passing `f` to the join lets it emit the projected fields directly. A separate `map` can require the full joined row as an intermediate result. Existing three-field join patterns migrate from `(k, l, r) -> …` to `(k, (l, r)) -> …`. A currently unfortunate amount of symbolic load is in shuffling around fields of collection elements. diff --git a/interactive/src/lambda.rs b/interactive/src/lambda.rs index dbe36b781..34c00b493 100644 --- a/interactive/src/lambda.rs +++ b/interactive/src/lambda.rs @@ -84,6 +84,11 @@ impl Lambda { if let Some(e) = error { lambda.graph = Err(e); } lambda } + /// A bare join's implicit projection. Check the whole graph so an unused shape + /// assertion or other computation is never discarded by printing it as a bare join. + pub(crate) fn returns_input(&self) -> bool { + self.graph().is_ok_and(|g| g.node_count() == 1 && matches!(g.nodes().next(), Some(corgi::NodeRef::Input))) + } /// Apply a join projection to the three columns supplied by the arrangements. pub(crate) fn on_join_columns(&self) -> Self { Self::compose("(k, l, r)", &[(self, "(k, (l, r))".into())], |out| out[0].clone()) diff --git a/interactive/src/plan.rs b/interactive/src/plan.rs index 4a7c8f58b..d9de5be0f 100644 --- a/interactive/src/plan.rs +++ b/interactive/src/plan.rs @@ -411,6 +411,7 @@ fn node(node: &Node, names: &Names) -> String { Node::Linear { input, ops } => format!("{} | linear [ {} ]", names.of(input), ops.iter().map(linear).collect::>().join(" | ")), Node::Concat(refs) => format!("concat({})", refs.iter().map(|r| names.of(r)).collect::>().join(", ")), Node::Arrange(input) => format!("{} | arrange", names.of(input)), + Node::Join { left, right, projection } if projection.returns_input() => format!("{} | join({})", names.of(left), names.of(right)), Node::Join { left, right, projection } => format!("{} | join({}, {})", names.of(left), names.of(right), source_lambda(projection)), Node::HalfJoin { stream, arrangement, strict, projection } => format!("{} | half_join({}, strict={strict}, {})", names.of(stream), names.of(arrangement), source_lambda(projection)), Node::Reduce { input, reducer: r, limit } => format!("{} | {}", names.of(input), reducer(r, *limit)), @@ -442,7 +443,7 @@ fn printed_node(node: &Node, names: &Names, shapes: &ScopeShapes, depth: usize, let Node::Linear { input, .. } = node else { unreachable!() }; format!("{} | linear [ {} ]", names.of(input), printed.join(" | ")) } - Node::Join { left, right, projection } => { + Node::Join { left, right, projection } if !projection.returns_input() => { let input = shapes.of(left).zip(shapes.of(right)).map(|(l, r)| Shape::Prod(vec![l.key, Shape::Prod(vec![l.val, r.val])])); format!("{} | join({}, {})", names.of(left), names.of(right), slots.lambda(projection, input, &format!("{at}/join"))) } diff --git a/interactive/src/plan/ddp.rs b/interactive/src/plan/ddp.rs index ce9d3dc96..3c3f31732 100644 --- a/interactive/src/plan/ddp.rs +++ b/interactive/src/plan/ddp.rs @@ -409,6 +409,7 @@ impl Printer<'_> { } Node::Concat(refs) => refs.iter().map(|r| names.of(r)).collect::>().join(" + "), Node::Arrange(r) => format!("{} | arrange", names.of(r)), + Node::Join { left, right, projection } if projection.returns_input() => format!("{} | join({})", names.of(left), names.of(right)), Node::Join { left, right, projection } => { let input = shapes.of(left).zip(shapes.of(right)).map(|(l, r)| Shape::Prod(vec![l.key, Shape::Prod(vec![l.val, r.val])])); format!("{} | join({}, {})", names.of(left), names.of(right), self.lambda(projection, input)?) @@ -542,6 +543,25 @@ mod tests { "#); } + #[test] + fn identity_joins_print_without_a_projection() { + for (args, explicit) in [ + ("b", false), + ("b, x -> x", false), + ("b, (k, (l, r)) -> (k, (r, l))", true), + ] { + let text = format!(r#"let a = input 0 : (int, int); let b = input 1 : (int, int); + output "out" = a | join({args});"#); + let report = plan(&text); + for stage in DdpStage::ALL { + let printed = report.emit_ddp_at(stage).unwrap().text; + let join_args = printed.split_once("| join(").unwrap().1.split_once(';').unwrap().0; + assert_eq!(join_args.contains(" -> "), explicit, "{stage:?}: {printed}"); + assert!(report.compare_final(&plan(&printed)).is_empty(), "{stage:?}: {printed}"); + } + } + } + #[test] fn fused_names_and_generated_names_do_not_collide() { let output = roundtrip(r#" diff --git a/interactive/src/plan/ddp/parsed.rs b/interactive/src/plan/ddp/parsed.rs index 23185e9c4..8669a8c7c 100644 --- a/interactive/src/plan/ddp/parsed.rs +++ b/interactive/src/plan/ddp/parsed.rs @@ -69,6 +69,7 @@ impl Printer<'_> { }; format!("{} | {word}({})", self.expression(input)?, self.parsed_lambda(l)?) } + Expr::Join(left, right, l) if l.returns_input() => format!("{} | join({})", self.expression(left)?, self.expression(right)?), Expr::Join(left, right, l) => format!("{} | join({}, {})", self.expression(left)?, self.expression(right)?, self.parsed_lambda(l)?), Expr::HalfJoin(stream, arrangement, strict, l) => format!("{} | half_join({}, {}, {})", self.expression(stream)?, self.expression(arrangement)?, u8::from(*strict), self.parsed_lambda(l)?), Expr::Reduce(input, reducer) => { diff --git a/interactive/src/shapes.rs b/interactive/src/shapes.rs index 914bb2859..312574432 100644 --- a/interactive/src/shapes.rs +++ b/interactive/src/shapes.rs @@ -341,7 +341,7 @@ fn infer_node(node: &st::Node, shapes: &ScopeShapes, level: Level, at: &str, pro if l.key != r.key { return fail(format!("join keys {} and {}", l.key, r.key)); } match pair_shape(projection, &[l.key, Shape::Prod(vec![l.val, r.val])]) { Ok((k, v)) => Some(CollShape::new(k, v, level)), - Err(e) => fail(format!("join: {e}")), + Err(e) => fail(format!("join: {e}. The projection takes (key, (left, right)) as its input.")), } } st::Node::HalfJoin { stream, arrangement, projection, .. } => { @@ -403,6 +403,13 @@ mod tests { } } + #[test] + fn old_join_pattern_reports_the_new_binder() { + let (_, problems) = shapes_of(r#"let a = input 0 : (int, int); + output "out" = a | join(a, (k, l, r) -> (k, r));"#); + assert!(problems.conflicts.iter().any(|e| e.contains("3 fields") && e.contains("(key, (left, right))")), "{problems:?}"); + } + #[test] fn declared_sources_shape_every_collection() { let (shapes, problems) = shapes_of(REACH); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic01_oneoff.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic01_oneoff.ddp index 6d146dc33..5b693239d 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic01_oneoff.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic01_oneoff.ddp @@ -57,7 +57,7 @@ let jobs_join3 = jobs_idx2 | join(place_idx, (_, ((rid, node, name, year), (wher let jobs_idx3 = jobs_join3 | arrange; let jobs = jobs_idx3 | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); let found_idx3 = found | arrange; -let with1_join = found_idx3 | join(emails, row -> row); +let with1_join = found_idx3 | join(emails); let with1_join2 = found_idx3 | join(emails, (key, (row, _)) -> (key, row)); let with1_neg = with1_join2 | negate; let with1_rows = found + with1_neg; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic01_oneoff.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/ldbc_ic01_oneoff.lowered--optimized.diff index b06da07ac..caf7e2556 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic01_oneoff.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic01_oneoff.lowered--optimized.diff @@ -68,7 +68,7 @@ let jobs_idx3 = jobs_join3 | arrange; let jobs = jobs_idx3 | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); let found_idx3 = found | arrange; - let with1_join = found_idx3 | join(emails, row -> row); + let with1_join = found_idx3 | join(emails); -let found_idx4 = found | arrange; -let with1_join2 = found_idx4 | join(emails, (key, (row, _)) -> (key, row)); +let with1_join2 = found_idx3 | join(emails, (key, (row, _)) -> (key, row)); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic01_oneoff.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic01_oneoff.parsed--lowered.diff index a22b00993..c38f8bce1 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic01_oneoff.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic01_oneoff.parsed--lowered.diff @@ -44,7 +44,7 @@ -let langs = who | join(language, (node, (rid, value)) -> ((rid, node), value)) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); -let school = who | join(study, (node, (rid, (at, year))) -> (at, (rid, node, year))) | join(org, (_, ((rid, node, year), (name, _, at))) -> (at, (rid, node, name, year))) | join(place, (_, ((rid, node, name, year), (where, _, _))) -> ((rid, node), (name, year, where))) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); -let jobs = who | join(work, (node, (rid, (at, year))) -> (at, (rid, node, year))) | join(org, (_, ((rid, node, year), (name, _, at))) -> (at, (rid, node, name, year))) | join(place, (_, ((rid, node, name, year), (where, _, _))) -> ((rid, node), (name, year, where))) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); --let with1 = (found | join(emails, row -> row)) + (found + (found | join(emails, (key, (row, _)) -> (key, row)) | negate) | map((key, row) -> (key, (row, [] : List(List(int)))))); +-let with1 = (found | join(emails)) + (found + (found | join(emails, (key, (row, _)) -> (key, row)) | negate) | map((key, row) -> (key, (row, [] : List(List(int)))))); -let with2 = (with1 | join(langs, (key, ((row, e), list)) -> (key, (row, e, list)))) + (with1 + (with1 | join(langs, (key, (row, _)) -> (key, row)) | negate) | map((key, (row, e)) -> (key, (row, e, [] : List(List(int)))))); -let with3 = (with2 | join(school, (key, ((row, e, l), list)) -> (key, (row, e, l, list)))) + (with2 + (with2 | join(school, (key, (row, _)) -> (key, row)) | negate) | map((key, (row, e, l)) -> (key, (row, e, l, [] : List((List(int), int, List(int))))))); -let with4 = (with3 | join(jobs, (key, ((row, e, l, s), list)) -> (key, (row, e, l, s, list)))) + (with3 + (with3 | join(jobs, (key, (row, _)) -> (key, row)) | negate) | map((key, (row, e, l, s)) -> (key, (row, e, l, s, [] : List((List(int), int, List(int))))))); @@ -82,7 +82,7 @@ +let jobs_idx3 = jobs_join3 | arrange; +let jobs = jobs_idx3 | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); +let found_idx3 = found | arrange; -+let with1_join = found_idx3 | join(emails, row -> row); ++let with1_join = found_idx3 | join(emails); +let found_idx4 = found | arrange; +let with1_join2 = found_idx4 | join(emails, (key, (row, _)) -> (key, row)); +let with1_neg = with1_join2 | negate; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic01_oneoff.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic01_oneoff.parsed.ddp index d494fe7e2..d75ea59a9 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic01_oneoff.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic01_oneoff.parsed.ddp @@ -17,7 +17,7 @@ let emails = who | join(email, (node, (rid, value)) -> ((rid, node), value)) | r let langs = who | join(language, (node, (rid, value)) -> ((rid, node), value)) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); let school = who | join(study, (node, (rid, (at, year))) -> (at, (rid, node, year))) | join(org, (_, ((rid, node, year), (name, _, at))) -> (at, (rid, node, name, year))) | join(place, (_, ((rid, node, name, year), (where, _, _))) -> ((rid, node), (name, year, where))) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); let jobs = who | join(work, (node, (rid, (at, year))) -> (at, (rid, node, year))) | join(org, (_, ((rid, node, year), (name, _, at))) -> (at, (rid, node, name, year))) | join(place, (_, ((rid, node, name, year), (where, _, _))) -> ((rid, node), (name, year, where))) | reduce((_, vals) -> let lists = vals map ((v, d) -> (v, (0, d) range) cap_list map ((v, _) -> v)) in let flat = lists flatten in (flat, 1) enlist); -let with1 = (found | join(emails, row -> row)) + (found + (found | join(emails, (key, (row, _)) -> (key, row)) | negate) | map((key, row) -> (key, (row, [] : List(List(int)))))); +let with1 = (found | join(emails)) + (found + (found | join(emails, (key, (row, _)) -> (key, row)) | negate) | map((key, row) -> (key, (row, [] : List(List(int)))))); let with2 = (with1 | join(langs, (key, ((row, e), list)) -> (key, (row, e, list)))) + (with1 + (with1 | join(langs, (key, (row, _)) -> (key, row)) | negate) | map((key, (row, e)) -> (key, (row, e, [] : List(List(int)))))); let with3 = (with2 | join(school, (key, ((row, e, l), list)) -> (key, (row, e, l, list)))) + (with2 + (with2 | join(school, (key, (row, _)) -> (key, row)) | negate) | map((key, (row, e, l)) -> (key, (row, e, l, [] : List((List(int), int, List(int))))))); let with4 = (with3 | join(jobs, (key, ((row, e, l, s), list)) -> (key, (row, e, l, s, list)))) + (with3 + (with3 | join(jobs, (key, (row, _)) -> (key, row)) | negate) | map((key, (row, e, l, s)) -> (key, (row, e, l, s, [] : List((List(int), int, List(int))))))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.ddp index 538acf1d3..6066a7150 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.ddp @@ -50,7 +50,7 @@ let in_x = in_x_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let in_y_rows = seen | filter((_, (_, iy)) -> (iy, 1) eq) | map((key, _) -> (key, ())); let in_y_idx = in_y_rows | arrange; let in_y = in_y_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); -let counts = in_x | join(in_y, row -> row); +let counts = in_x | join(in_y); let ranked_map = counts | map(((rid, node), (nx, ny)) -> (node, (rid, nx, ny))); let ranked_idx = ranked_map | arrange; let ranked_join = ranked_idx | join(person_idx, (node, ((rid, nx, ny), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, (((nx, ny) add neg, node), (first, last, nx, ny)))); diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.lowered--optimized.diff b/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.lowered--optimized.diff index 2ab01679d..7b34de9a4 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.lowered--optimized.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.lowered--optimized.diff @@ -73,7 +73,7 @@ +let in_y_rows = seen | filter((_, (_, iy)) -> (iy, 1) eq) | map((key, _) -> (key, ())); +let in_y_idx = in_y_rows | arrange; let in_y = in_y_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); - let counts = in_x | join(in_y, row -> row); + let counts = in_x | join(in_y); let ranked_map = counts | map(((rid, node), (nx, ny)) -> (node, (rid, nx, ny))); let ranked_idx = ranked_map | arrange; -let person_idx2 = person | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.parsed--lowered.diff b/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.parsed--lowered.diff index 44a699a12..88cf06531 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.parsed--lowered.diff +++ b/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.parsed--lowered.diff @@ -73,7 +73,7 @@ +let in_y_map = in_y_filter | map((key, _) -> (key, ())); +let in_y_idx = in_y_map | arrange; +let in_y = in_y_idx | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); - let counts = in_x | join(in_y, row -> row); + let counts = in_x | join(in_y); -let ranked = counts | map(((rid, node), (nx, ny)) -> (node, (rid, nx, ny))) | join(person, (node, ((rid, nx, ny), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, (((nx, ny) add neg, node), (first, last, nx, ny)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); +let ranked_map = counts | map(((rid, node), (nx, ny)) -> (node, (rid, nx, ny))); +let ranked_idx = ranked_map | arrange; diff --git a/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.parsed.ddp b/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.parsed.ddp index 8d673f21e..482619a57 100644 --- a/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.parsed.ddp +++ b/interactive/tests/plan_pairs/expected/ldbc_ic03_oneoff.parsed.ddp @@ -14,7 +14,7 @@ let circle = hop1 + hop2 | filter((node, (_, who, _, _)) -> (node, who) ne) | jo let seen = visits | join(circle, (key, ((ix, iy), ())) -> (key, (ix, iy))); let in_x = seen | filter((_, (ix, _)) -> (ix, 1) eq) | map((key, _) -> (key, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); let in_y = seen | filter((_, (_, iy)) -> (iy, 1) eq) | map((key, _) -> (key, ())) | reduce((_, vals) -> let diffs = vals map ((_, d) -> d) in let n = diffs fold_add in (n, (n, 1)) enlist filter); -let counts = in_x | join(in_y, row -> row); +let counts = in_x | join(in_y); let ranked = counts | map(((rid, node), (nx, ny)) -> (node, (rid, nx, ny))) | join(person, (node, ((rid, nx, ny), (first, last, _, _, _, _, _, _, _, _, _))) -> (rid, (((nx, ny) add neg, node), (first, last, nx, ny)))) | reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten); let answer = ranked | map((rid, (rank, ((_, node), (first, last, nx, ny)))) -> ((rid, rank), (node, first, last, nx, ny, (nx, ny) add))); output "answer" = answer | arrange;