Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions interactive/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,10 @@ let blocks = input 2 : (int, Member); -- doc -> blocked member

The shape of records is load bearing when we perform joins and other data-parallel operations.
The pair structure `(key, val)` calls out first how the data are organized, and then any associated values.
For each pair of matches `(k, l)` and `(k, r)`, `left | join(right)` produces `(k, (l, r))`.
`left | join(right, f)` computes the same collection as `left | join(right) | map(f)`.
Passing `f` to the join lets it emit the projected fields directly. A separate `map` can require the full joined row as an intermediate result.
Existing three-field join patterns migrate from `(k, l, r) -> …` to `(k, (l, r)) -> …`.

A currently unfortunate amount of symbolic load is in shuffling around fields of collection elements.
The `map` operator applies to each record, and the `->` fragment is an expression converting the input to the output.
Expand Down Expand Up @@ -115,7 +119,7 @@ To see what a variable is about to become, bind the `set` value with a `let` and
let barred = barring::barred | map((who, doc) -> ((who, doc), ()));
barring: {
var barred : (Member, int); -- member -> doc: expanded blocked list.
let beyond = barred | join(members, (_grp, doc, who) -> (who, doc));
let beyond = barred | join(members, (_grp, (doc, who)) -> (who, doc));
set barred = blocks | map((doc, who) -> (who, doc)) + beyond | distinct;
}
```
Expand Down Expand Up @@ -145,15 +149,15 @@ output "access" = sharing::allowed;
sharing: {
var allowed : ((Member, int), Access); -- (member, doc) -> strongest
set allowed = offered
- (offered | join(barred, (pair, acc, ()) -> (pair, acc)))
- (offered | join(barred, (pair, (acc, ())) -> (pair, acc)))
| min;

let offered = grants | map((doc, (who, acc)) -> ((who, doc), acc))
+ inherited;

let inherited = allowed
| map(((who, doc), acc) -> (who, (doc, acc)))
| join(members, (_grp, (doc, acc), who) -> ((who, doc), acc));
| join(members, (_grp, ((doc, acc), who)) -> ((who, doc), acc));
}
```

Expand Down
2 changes: 1 addition & 1 deletion interactive/bench/ldbc/authored/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ Two conventions apply throughout:
## Idioms

- **Keyed collections.** Keep a collection as `(key, value)` by the key it is
joined on: `friend` is `(person, friend)`, so `join(friend, (_, req, f) -> …)`
joined on: `friend` is `(person, friend)`, so `join(friend, (_, (req, f)) -> …)`
reads as "for each friend". Map back to `((fields), ())` only where a step needs
a whole-row key.
- **Destructuring.** Lambdas name what they take apart:
Expand Down
6 changes: 3 additions & 3 deletions interactive/bench/ldbc/authored/plans/bi01.ddp
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,12 @@ let per_day = classified | join_function(((day, year, kind, cat), (_, length)) -
let cutoff = request | map(((rid, end), ()) -> (rid, (end, ((end, 86400000) div, 86400000) mul)));
let whole = cutoff | map((rid, (_, day)) -> ((), (rid, day)))
| join(per_day | map(((day, year, kind, cat, c), v) -> ((), (day, year, kind, cat, c, v))),
(_, (rid, cut), (day, year, kind, cat, c, v)) -> ((rid, cut, day), (year, kind, cat, c, v)))
(_, ((rid, cut), (day, year, kind, cat, c, v))) -> ((rid, cut, day), (year, kind, cat, c, v)))
| filter(((_, cut, day), _) -> (day, cut) lt)
| join_function(((rid, _, _), (year, kind, cat, c, v)) -> (((rid, year, kind, cat, c), ()), 0, v) enlist);
let part = cutoff | map((rid, (end, day)) -> (day, (rid, end)))
| join(classified | map(((day, year, kind, cat), (created, length)) -> (day, (year, kind, cat, created, length))),
(_, (rid, end), (year, kind, cat, created, length)) -> ((rid, year, kind, cat), (created, end, length)))
(_, ((rid, end), (year, kind, cat, created, length))) -> ((rid, year, kind, cat), (created, end, length)))
| filter((_, (created, end, _)) -> (created, end) lt)
| join_function(((rid, year, kind, cat), (_, _, length)) ->
((((rid, year, kind, cat, 0), ()), 0, 1) enlist, (((rid, year, kind, cat, 1), ()), 0, length) enlist) append);
Expand All @@ -42,7 +42,7 @@ let groups = sums | map(((rid, year, kind, cat, c), v) -> ((rid, year, kind, cat
let total = sums | filter(((_, _, _, _, c), _) -> (c, 0) eq)
| join_function(((rid, _, _, _, _), n) -> ((rid, ()), 0, n) enlist) | count;
let ranked = groups | map(((rid, year, kind, cat), (n, s)) -> (rid, (year, kind, cat, n, s)))
| join(total, (rid, (year, kind, cat, n, s), all) -> (rid, ((year neg, kind, cat), (n, s, all))))
| join(total, (rid, ((year, kind, cat, n, s), all)) -> (rid, ((year neg, kind, cat), (n, s, all))))
| collect | flatmap((k, entries) -> (k, ((0, entries len) range, entries) zip) cap_list);
let answer = ranked | map((rid, (rank, ((neg_year, kind, cat), (n, s, all)))) -> ((rid, rank),
(neg_year neg, kind, cat, n, (s to_float, n to_float) div, s, ((n, 100) mul to_float, all to_float) div)));
Expand Down
10 changes: 5 additions & 5 deletions interactive/bench/ldbc/authored/plans/bi02.ddp
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,18 @@ let tagged = import "graph::mtag.by_tag" : (int, int); -
let class_id = import "graph::tagclass.by_name" : (List(int), int); -- (name, id)

-- ((class, day), (tag name, created)) for every tagged message, and each class's days.
let byday = tag | join(tagged, (_, (name, cls), m) -> (m, (cls, name)))
| join(message, (_, (cls, name), (_, created, _, _, _, _, _, _, _, _, _, _, day)) -> ((cls, day), (name, created)));
let byday = tag | join(tagged, (_, ((name, cls), m)) -> (m, (cls, name)))
| join(message, (_, ((cls, name), (_, created, _, _, _, _, _, _, _, _, _, _, day))) -> ((cls, day), (name, created)));
let days = byday | map(((cls, day), _) -> ((cls, day), ())) | distinct | map(((cls, day), ()) -> (cls, day));

-- The request's class and its 200 window days (days with a message of the class).
let wanted = request | map(((rid, date, name), ()) -> (name, (rid, date)))
| join(class_id, (_, (rid, date), cls) -> (cls, (rid, date)))
| join(days, (cls, (rid, date), day) -> ((cls, day), (rid, date)))
| join(class_id, (_, ((rid, date), cls)) -> (cls, (rid, date)))
| join(days, (cls, ((rid, date), day)) -> ((cls, day), (rid, date)))
| filter(((_, day), (_, date)) -> ((day, ((date, 86400000) div, 86400000) mul) ge, (day, (date, 17280000000) add) lt) all);

-- Count each tag's messages in the two halves.
let halves = wanted | join(byday, (_, (rid, date), (name, created)) -> ((rid, name), (created, date)))
let halves = wanted | join(byday, (_, ((rid, date), (name, created))) -> ((rid, name), (created, date)))
| filter((_, (created, date)) -> ((created, date) ge, (created, (date, 17280000000) add) lt) all)
| map((key, (created, date)) -> (key, ((created, (date, 8640000000) add) lt, (1, 0), (0, 1)) select));
let counts = halves | map(((rid, name), (a, _)) -> ((rid, name, a), ())) | count
Expand Down
22 changes: 11 additions & 11 deletions interactive/bench/ldbc/authored/plans/bi03.ddp
Original file line number Diff line number Diff line change
Expand Up @@ -18,24 +18,24 @@ let root_of = import "graph::message.root" : (int, int);

-- Forums moderated by someone living in a city of the country.
let local = request | map(((rid, _, country), ()) -> (country, rid))
| join(place_id, (_, rid, country) -> (country, rid))
| join(within, (_, rid, city) -> (city, rid))
| join(residents, (_, rid, who) -> (who, rid))
| join(moderates, (_, rid, f) -> ((rid, f), ()));
| join(place_id, (_, (rid, country)) -> (country, rid))
| join(within, (_, (rid, city)) -> (city, rid))
| join(residents, (_, (rid, who)) -> (who, rid))
| join(moderates, (_, (rid, f)) -> ((rid, f), ()));

-- Messages tagged with a tag of the class (once per message), and their threads' forums.
let inclass = request | map(((rid, cls, _), ()) -> (cls, rid))
| join(class_id, (_, rid, c) -> (c, rid))
| join(in_class, (_, rid, t) -> (t, rid))
| join(tagged, (_, rid, m) -> ((rid, m), ())) | distinct;
| join(class_id, (_, (rid, c)) -> (c, rid))
| join(in_class, (_, (rid, t)) -> (t, rid))
| join(tagged, (_, (rid, m)) -> ((rid, m), ())) | distinct;
let counts = inclass | map(((rid, m), ()) -> (m, rid))
| join(root_of, (_, rid, root) -> (root, rid))
| join(message, (_, rid, (_, _, _, _, _, _, _, _, f, _, _, _, _)) -> ((rid, f), ()))
| join(local, (key, (), ()) -> (key, ())) | count;
| join(root_of, (_, (rid, root)) -> (root, rid))
| join(message, (_, (rid, (_, _, _, _, _, _, _, _, f, _, _, _, _))) -> ((rid, f), ()))
| join(local, (key, ((), ())) -> (key, ())) | count;
-- Top k: `reduce` sees a key's values in sort order and keeps the first k (copies capped
-- at k), emitting (rank, value). Changes below the top k produce no output.
let ranked = counts | map(((rid, f), n) -> (f, (rid, n)))
| join(forum, (f, (rid, n), (title, created, moderator)) -> (rid, ((n neg, f), (title, created, moderator))))
| join(forum, (f, ((rid, n), (title, created, moderator))) -> (rid, ((n neg, f), (title, created, moderator))))
| reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 20) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 20) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 20) min in let before = (end, count) sub in let keep = (((20, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten);
let answer = ranked | map((rid, (rank, ((neg, f), (title, created, moderator)))) -> ((rid, rank), (f, title, created, moderator, neg neg)));

Expand Down
22 changes: 11 additions & 11 deletions interactive/bench/ldbc/authored/plans/bi04.ddp
Original file line number Diff line number Diff line change
Expand Up @@ -18,33 +18,33 @@ let posts_in = import "graph::message.by_forum" : (int, int);
let thread = import "graph::message.by_root" : (int, int); -- (root, message)

-- Each person's country, and each forum's popularity (negated, so min is max).
let country_of = place | join(residents, (_, (_, _, country), who) -> (who, country));
let popularity = country_of | join(joins, (_, country, (f, _)) -> ((f, country), ())) | count
let country_of = place | join(residents, (_, ((_, _, country), who)) -> (who, country));
let popularity = country_of | join(joins, (_, (country, (f, _))) -> ((f, country), ())) | count
| map(((f, _), n) -> (f, n neg)) | min;

-- The request's top 100 forums created after its date.
-- Top k: `reduce` sees a key's values in sort order and keeps the first k (copies capped
-- at k), emitting (rank, value). Changes below the top k produce no output.
let top = request | map(((rid, date), ()) -> ((), (rid, date)))
| join(popularity | join(forum, (f, neg, (_, when, _)) -> ((), (f, neg, when))),
(_, (rid, date), (f, neg, when)) -> (rid, ((neg, f), (date, when))))
| join(popularity | join(forum, (f, (neg, (_, when, _))) -> ((), (f, neg, when))),
(_, ((rid, date), (f, neg, when))) -> (rid, ((neg, f), (date, when))))
| filter((_, (_, (date, when))) -> (when, date) gt)
| map((rid, (order, _)) -> (rid, order))
| reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 100) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 100) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 100) min in let before = (end, count) sub in let keep = (((100, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten)
| map((rid, (_, (_, f))) -> (f, rid));

-- Their members, and how many messages each person posted in them (every message in
-- a thread rooted at one of the forum's posts).
let joined = top | join(members, (_, rid, (who, _)) -> ((rid, who), ())) | distinct;
let posted = top | join(posts_in, (_, rid, post) -> (post, rid))
| join(thread, (_, rid, mid) -> (mid, rid))
| join(message, (_, rid, (_, _, author, _, _, _, _, _, _, _, _, _, _)) -> ((rid, author), 1));
let wrote = posted | join(joined, (key, _, ()) -> (key, ())) | count;
let counts = wrote + ((joined - (joined | join(wrote, (key, (), _) -> (key, ())))) | map((key, ()) -> (key, 0)));
let joined = top | join(members, (_, (rid, (who, _))) -> ((rid, who), ())) | distinct;
let posted = top | join(posts_in, (_, (rid, post)) -> (post, rid))
| join(thread, (_, (rid, mid)) -> (mid, rid))
| join(message, (_, (rid, (_, _, author, _, _, _, _, _, _, _, _, _, _))) -> ((rid, author), 1));
let wrote = posted | join(joined, (key, (_, ())) -> (key, ())) | count;
let counts = wrote + ((joined - (joined | join(wrote, (key, ((), _)) -> (key, ())))) | map((key, ()) -> (key, 0)));

-- Rank members by messages (most first), then id.
let ranked = counts | map(((rid, who), n) -> (who, (rid, n)))
| join(person, (who, (rid, n), (first, last, _, _, created, _, _, _, _, _, _)) -> (rid, ((n neg, who), (first, last, created, n))))
| join(person, (who, ((rid, n), (first, last, _, _, created, _, _, _, _, _, _))) -> (rid, ((n neg, who), (first, last, created, n))))
| reduce((_, vals) -> let positive = (vals map ((_, d) -> (d, 0) gt), (0, vals len) range) zip filter in let indices = ((0, (positive len, 100) min) range, positive) gather in let prefix = (indices, vals) gather in let ends = prefix map ((_, d) -> (d, 100) min) scan_add in let chosen = (prefix, ends) zip map (((item, d), end) -> let count = (d, 100) min in let before = (end, count) sub in let keep = (((100, before) sub, 0) max, count) min in ((keep, 0) gt, (item, keep))) filter in let ends = chosen map ((_, d) -> d) scan_add in (chosen, ends) zip map (((item, d), end) -> (item, ((end, d) sub, end) range) cap_list map ((item, rank) -> ((rank, item), 1))) flatten);
let answer = ranked | map((rid, (rank, ((_, who), (first, last, created, n)))) -> ((rid, rank), (who, first, last, created, n)));

Expand Down
10 changes: 5 additions & 5 deletions interactive/bench/ldbc/authored/plans/bi05.ddp
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,14 @@ let liked_by = import "graph::likes.by_message" : (int, (int, int)); -- (
-- The request's tagged messages and their authors: (message, (rid, author)).
-- Tag names are unique and (message, tag) pairs are a set, so each message appears once.
let written = request | map(((rid, name), ()) -> (name, rid))
| join(tag_id, (_, rid, tag) -> (tag, rid))
| join(tagged, (_, rid, m) -> (m, rid))
| join(message, (m, rid, (_, _, author, _, _, _, _, _, _, _, _, _, _)) -> (m, (rid, author)));
| join(tag_id, (_, (rid, tag)) -> (tag, rid))
| join(tagged, (_, (rid, m)) -> (m, rid))
| join(message, (m, (rid, (_, _, author, _, _, _, _, _, _, _, _, _, _))) -> (m, (rid, author)));

-- Count each author's tagged messages (kind 0), replies to them (1) and likes on them (2).
let as_message = written | map((_, (rid, author)) -> ((rid, author, 0), ()));
let as_like = written | join(liked_by, (_, (rid, author), _) -> ((rid, author, 2), ()));
let as_reply = written | join(replies, (_, (rid, author), _) -> ((rid, author, 1), ()));
let as_like = written | join(liked_by, (_, ((rid, author), _)) -> ((rid, author, 2), ()));
let as_reply = written | join(replies, (_, ((rid, author), _)) -> ((rid, author, 1), ()));
let tally = (as_message + as_reply + as_like) | count;

-- Pivot the (at most three) counts per author into (messages, replies, likes).
Expand Down
14 changes: 7 additions & 7 deletions interactive/bench/ldbc/authored/plans/bi06.ddp
Original file line number Diff line number Diff line change
Expand Up @@ -12,22 +12,22 @@ let liked_by = import "graph::likes.by_message" : (int, (int, int)); -- (

-- The request's tagged messages and their authors (one tag; (message, tag) is a set).
let written = request | map(((rid, name), ()) -> (name, rid))
| join(tag_id, (_, rid, t) -> (t, rid))
| join(tagged, (_, rid, m) -> (m, rid))
| join(message, (m, rid, (_, _, who, _, _, _, _, _, _, _, _, _, _)) -> ((rid, m, who), ()));
| join(tag_id, (_, (rid, t)) -> (t, rid))
| join(tagged, (_, (rid, m)) -> (m, rid))
| join(message, (m, (rid, (_, _, who, _, _, _, _, _, _, _, _, _, _))) -> ((rid, m, who), ()));

-- Popularity: likes received on all of a person's messages.
let popular = liked_by | join(message, (_, _, (_, _, who, _, _, _, _, _, _, _, _, _, _)) -> (who, ())) | count;
let popular = liked_by | join(message, (_, (_, (_, _, who, _, _, _, _, _, _, _, _, _, _))) -> (who, ())) | count;

-- Each (author, liker) pair once, scored by the liker's popularity; zero for the rest.
let pairs = written | map(((rid, m, who), ()) -> (m, (rid, who)))
| join(liked_by, (_, (rid, who), (liker, _)) -> ((rid, who, liker), ())) | distinct;
| join(liked_by, (_, ((rid, who), (liker, _))) -> ((rid, who, liker), ())) | distinct;
-- Sums: move each value into the multiplicity (join_function) and let `count` add them up.
let scored = pairs | map(((rid, who, liker), ()) -> (liker, (rid, who)))
| join(popular, (_, (rid, who), p) -> ((rid, who), p))
| join(popular, (_, ((rid, who), p)) -> ((rid, who), p))
| join_function((key, p) -> ((key, ()), 0, p) enlist) | count;
let authors = written | map(((rid, _, who), ()) -> ((rid, who), ())) | distinct;
let score = scored + ((authors - (authors | join(scored, (key, (), _) -> (key, ())))) | map((key, ()) -> (key, 0)));
let score = scored + ((authors - (authors | join(scored, (key, ((), _)) -> (key, ())))) | map((key, ()) -> (key, 0)));
-- Top k: `reduce` sees a key's values in sort order and keeps the first k (copies capped
-- at k), emitting (rank, value). Changes below the top k produce no output.
let ranked = score | map(((rid, who), s) -> (rid, (s neg, who)))
Expand Down
Loading
Loading