fix(solid): bound the reads of documents outside the user's storages

docs/boundaries.md

@@ -338,6 +338,21 @@ the [pod fence](#the-pod-fence) that fetch passes the credential gate
   ([data-model.md](data-model.md#where-an-instances-documents-are)), and
   what such a document says of itself never makes it writable. Such a
   read is held to `FOREIGN_LIMITS` too.
+- **Outside the user's storages**, the reads share a budget in a page
+  session (`FOREIGN_READS_BUDGET`, `sharedReadBudget` in
+  [readLimits.ts](../packages/solid/src/client/readLimits.ts)): four
+  times `FOREIGN_LIMITS` in total, on top of the limits on each read.
+  A catalog may list a thousand decks, each with two documents as
+  large as a release. The reads as no one share one budget; the reads
+  in a registered instance elsewhere have one for each server, so
+  reads anywhere else never spend it. As with releases read from a
+  link ([deck-library.md](deck-library.md#from-a-link)), a document
+  counts once, by its largest read, and reading it again comes out of
+  an allowance of its own (`FOREIGN_REREADS`) while that lasts. Past
+  the budget, such a read fails with a `ReadLimitError` until the page
+  is loaded again, without asking the server. Only reads count: a
+  write to a registered instance goes out whatever is spent. The
+  storages and the folders entered on this page aren't part of it.
 - **A read past its limits** fails with a `ReadLimitError`. So does
   a document the app reads into a dataset whose Turtle text has too
   many bases, a base that's too long or not plain, or too many

docs/data-model.md

@@ -897,12 +897,15 @@ instance listing one stays valid on real servers
 ([foreignData.integration.test.ts](../e2e/pod/src/foreignData.integration.test.ts)).
 
 A catalog is data anyone may have written, so the app reads at most
-`MAX_CATALOG_LINKS` (1,000) decks of it, in the deck list, the
-statistics and the format update as in the Studio
+`MAX_CATALOG_LINKS` (1,000) decks of it, in the deck list and its
+arrangement, the statistics and the format update as in the Studio
 ([catalogLinks.ts](../packages/solid/src/catalogLinks.ts)). Past that,
 it reads none of them and says why (`catalogTooManyLinks`). The same
 limit holds for its links to drafts and releases
-([Drafts and releases](#drafts-and-releases)).
+([Drafts and releases](#drafts-and-releases)). The deck list works out
+the study counts of three decks at a time (`getStudyCounts` in
+[study.ts](../packages/application/src/study.ts)), each reading the
+deck's two documents, so a long list never reads all of them at once.
 
 The catalog is written when an instance is created, and by the [format
 update](migrations.md) for an instance made before there were catalogs;

packages/application/src/atOnce.test.ts

@@ -3,7 +3,7 @@
 // SPDX-License-Identifier: MIT
 
 import { describe, expect, it } from "vitest";
-import { mapAtOnce, READS_AT_ONCE } from "./atOnce";
+import { inTurns, mapAtOnce, READS_AT_ONCE } from "./atOnce";
 
 /** `each` that counts how many run at once, each waiting a turn of the event loop. */
 function counted<T>() {
@@ -51,3 +51,41 @@ describe("mapAtOnce", () => {
     expect(started).toEqual([0, 1]);
   });
 });
+
+describe("inTurns", () => {
+  it("runs all the work asked of it, READS_AT_ONCE at most at once, in the order asked", async () => {
+    const turn = inTurns();
+    const { each, most } = counted<number>();
+    const started: number[] = [];
+    const items = Array.from({ length: 40 }, (_, at) => at);
+    const done = await Promise.all(
+      items.map((item) =>
+        turn(() => {
+          started.push(item);
+          return each(item);
+        }),
+      ),
+    );
+    expect(done).toEqual(items);
+    expect(started).toEqual(items);
+    expect(most()).toBe(READS_AT_ONCE);
+  });
+
+  it("ends a turn whose work fails, and lets the next start", async () => {
+    const turn = inTurns(1);
+    const failure = new Error("unreadable");
+    const first = turn(async () => {
+      throw failure;
+    });
+    const second = turn(async () => "read");
+    await expect(first).rejects.toBe(failure);
+    await expect(second).resolves.toBe("read");
+    // Every turn is free again.
+    const { each, most } = counted<string>();
+    await expect(Promise.all(["a", "b"].map((item) => turn(() => each(item))))).resolves.toEqual([
+      "a",
+      "b",
+    ]);
+    expect(most()).toBe(1);
+  });
+});

packages/application/src/atOnce.ts

@@ -33,3 +33,25 @@ export async function mapAtOnce<T, R>(
   await Promise.all(Array.from({ length: Math.min(atOnce, items.length) }, worker));
   return results;
 }
+
+/**
+ * Work taken in turns, `atOnce` at most at once, the rest waiting in the
+ * order asked: whoever asks, a list as long as anyone may write never
+ * runs all of its work together. A turn ends when its work does, failed
+ * or not.
+ */
+export function inTurns(atOnce = READS_AT_ONCE): <T>(work: () => Promise<T>) => Promise<T> {
+  let running = 0;
+  const waiting: (() => void)[] = [];
+  return async (work) => {
+    if (running >= atOnce) await new Promise<void>((start) => waiting.push(start));
+    else running++;
+    try {
+      return await work();
+    } finally {
+      const next = waiting.shift();
+      if (next === undefined) running--;
+      else next();
+    }
+  };
+}

packages/application/src/digestStore.test.ts

@@ -33,6 +33,31 @@ describe("the instance digest", () => {
     });
   });
 
+  it("counts three decks at once at most, however many are asked for, each reading its two documents", async () => {
+    const { deps, useCases } = setup();
+    const read = deps.deckRepository.readCardsSince;
+    let reading = 0;
+    let most = 0;
+    deps.deckRepository.readCardsSince = vi.fn(async (...args: Parameters<typeof read>) => {
+      most = Math.max(most, ++reading);
+      await new Promise((resolve) => setTimeout(resolve, 0));
+      reading--;
+      return read(...args);
+    });
+    const decks = Array.from({ length: 200 }, (_, at) => ({
+      ...deck,
+      url: `${deck.url}-${at}`,
+      cardsDocumentUrl: `${deck.cardsDocumentUrl}-${at}`,
+      reviewsDocumentUrl: `${deck.reviewsDocumentUrl}-${at}`,
+    }));
+    const counts = await Promise.all(
+      decks.map((one) => useCases.getStudyCounts(instance.url, one, now)),
+    );
+    expect(counts).toHaveLength(200);
+    expect(read).toHaveBeenCalledTimes(200);
+    expect(most).toBe(3);
+  });
+
   it("counts from the schedule while neither document changed", async () => {
     const { deps, useCases } = setup(await learned());
     await expect(useCases.getStudyCounts(instance.url, deck, now)).resolves.toEqual({

packages/application/src/study.ts

@@ -16,6 +16,7 @@ import {
 } from "@solid-memo/domain/scheduling";
 import { effectiveStates } from "@solid-memo/domain/fsrs";
 import type { ResponseTimes } from "@solid-memo/domain/answer";
+import { inTurns, READS_AT_ONCE } from "./atOnce";
 import type { DeckRepository, ReviewStateRepository } from "./ports";
 import type { PreferenceStore } from "./preferenceStore";
 import type { AnswerStore } from "./answerStore";
@@ -34,7 +35,9 @@ export interface StudyUseCases {
    * digest is brought up to date). Each document is read as the user
    * only when it is the instance's (isInInstancePod), as no one
    * otherwise, and only a deck whose documents both are has its
-   * schedule kept in the digest.
+   * schedule kept in the digest. STUDY_COUNTS_AT_ONCE decks are counted
+   * at once, whoever asks, the others waiting their turn: a deck list
+   * asks for every deck's, and a catalog may list a thousand.
    */
   getStudyCounts(
     instanceUrl: string,
@@ -127,6 +130,9 @@ export interface ReviewUndo {
   answer: { id: string; studyDay: string };
 }
 
+/** How many decks' study counts are worked out at once: each reads two documents together. */
+const STUDY_COUNTS_AT_ONCE = READS_AT_ONCE / 2;
+
 /**
  * Study (docs/srs.md): a deck's queue for today and how many prompts it
  * holds, each answer graded by the instance's scheduler and logged, an
@@ -148,6 +154,7 @@ export function createStudyUseCases({
   answerStore: Pick<AnswerStore, "grade" | "withdraw" | "removeDay">;
   random: () => number;
 }): StudyUseCases {
+  const countsTurn = inTurns(STUDY_COUNTS_AT_ONCE);
   async function studyCountsAndSchedule(instanceUrl: string, deck: Deck, now: Date) {
     const [prefs, digest, asNoOne] = await Promise.all([
       preferenceStore.get(instanceUrl),
@@ -215,7 +222,7 @@ export function createStudyUseCases({
       return { ...queue, prefs, states: reviews.sm2, fsrsStates: reviews.fsrs };
     },
     getStudyCounts(instanceUrl, deck, now) {
-      return studyCountsAndSchedule(instanceUrl, deck, now);
+      return countsTurn(() => studyCountsAndSchedule(instanceUrl, deck, now));
     },
     refreshStudyDigest: digestStore.refresh,
     recordReview(instanceUrl, deck, prompt, quality, now, times = {}, known) {

packages/composition/src/appUseCases.test.ts

@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: MIT
 
-import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, onTestFinished, vi } from "vitest";
 import { IDBFactory } from "fake-indexeddb";
 import { SITE } from "@solid-memo/vocab/tooling/sources";
 import { shapesFetch } from "@solid-memo/vocab/tooling/testing/shapeSources";
@@ -71,6 +71,24 @@ vi.mock(
 vi.mock("@solid-memo/solid/solidReleasePublisher", recorded("createSolidReleasePublisher"));
 vi.mock("@solid-memo/solid/solidInstanceGraphReader", recorded("createSolidInstanceGraphReader"));
 
+/** A budget for the reads outside the user's storages, small enough for a test to spend, while `on`. */
+const smallBudget = vi.hoisted(() => ({
+  on: false,
+  limits: { bytes: 1024 * 1024, quads: 30, characters: 1024 * 1024 },
+}));
+vi.mock("@solid-memo/solid/client/readLimits", async (importOriginal) => {
+  const original = await importOriginal<typeof import("@solid-memo/solid/client/readLimits")>();
+  return {
+    ...original,
+    sharedReadBudget: (options: Parameters<typeof original.sharedReadBudget>[0]) =>
+      original.sharedReadBudget(
+        smallBudget.on
+          ? { ...options, budget: smallBudget.limits, rereads: smallBudget.limits }
+          : options,
+      ),
+  };
+});
+
 /** The login: who is logged in, and the addresses requested with their credentials. */
 const login = vi.hoisted(() => ({
   webId: undefined as string | undefined,
@@ -752,6 +770,98 @@ describe("createAppUseCases", () => {
     expect(await useCases.listCards(deckAt(`${alice}memo/decks/deck-1.ttl`))).toEqual([]);
   });
 
+  it("reads the documents of many decks outside the user's storages within a budget, one of each server for an instance they registered, and sends their writes there whatever is spent", async () => {
+    smallBudget.on = true;
+    onTestFinished(() => {
+      smallBudget.on = false;
+    });
+    const alice = "https://alice.example/";
+    const shared = "https://shared.example/study/";
+    login.webId = `${alice}profile/card#me`;
+    const guest = createAppUseCases({ ...options, indexedDB: undefined });
+    await guest.startGuest("My study");
+    const [instance] = await guest.listInstances(GUEST_SESSION);
+    const made = await guest.createDeck(instance!.url, { en: "Capitals" });
+    const entry = `${alice}memo/catalog.ttl#deck-1`;
+    const deckAt = (url: string) => ({
+      ...made,
+      url: entry,
+      cardsDocumentUrl: url,
+      reviewsDocumentUrl: url,
+    });
+    const documents: Record<string, string> = {
+      [`${alice}profile/card`]: `<${login.webId}>
+        <http://www.w3.org/ns/pim/space#storage> <${alice}>;
+        <http://www.w3.org/ns/solid/terms#privateTypeIndex> <${alice}settings/private.ttl>.`,
+      // An instance on another server, registered as the user's.
+      [`${alice}settings/private.ttl`]: `<#memo>
+        a <http://www.w3.org/ns/solid/terms#TypeRegistration>;
+        <http://www.w3.org/ns/solid/terms#forClass> <https://solid-memo.com/ns/vocab/v1.ttl#Instance>;
+        <http://www.w3.org/ns/solid/terms#instanceContainer> <${shared}>.`,
+    };
+    const asked: { url: string; method: string }[] = [];
+    vi.stubGlobal(
+      "fetch",
+      vi.fn((input: RequestInfo | URL, init?: RequestInit) => {
+        const url = String(input instanceof Request ? input.url : input).replace(/#.*/, "");
+        const elsewhere = ["https://elsewhere.example/", shared].some((at) => url.startsWith(at));
+        if (!url.startsWith(alice) && !elsewhere) return servedSiteNetwork(input, init);
+        asked.push({ url, method: init?.method ?? "GET" });
+        // Each deck's document well within the limits of one read: twelve triples.
+        const triples = Array.from({ length: 11 }, (_, at) => `<#c${at}> <#p> "${at}".`);
+        const body =
+          documents[url] ??
+          [`<${url}> <http://purl.org/dc/terms/isPartOf> <${entry}>.`, ...triples].join("\n");
+        const response = new Response(body, { headers: { "Content-Type": "text/turtle" } });
+        Object.defineProperty(response, "url", { value: url });
+        return Promise.resolve(response);
+      }),
+    );
+    const useCases = createAppUseCases({ ...options, indexedDB: undefined });
+    /** Lists the cards of the decks whose documents are at `urls`, one after another: whether each was read. */
+    const listed = async (urls: string[]) => {
+      const read: boolean[] = [];
+      for (const url of urls)
+        read.push(
+          await useCases.listCards(deckAt(url)).then(
+            () => true,
+            (error: unknown) => {
+              expect(error).toMatchObject({ name: "ReadLimitError" });
+              return false;
+            },
+          ),
+        );
+      return read;
+    };
+    const decks = (at: string) => Array.from({ length: 5 }, (_, deck) => `${at}deck-${deck}.ttl`);
+
+    const foreign = decks("https://elsewhere.example/");
+    expect(await listed(foreign)).toEqual([true, true, false, false, false]);
+    // Past the budget, the server is not asked.
+    expect(asked.filter(({ url }) => url === foreign[4])).toEqual([]);
+    // Nor when a deck's cards or states are copied from there: those reads count in the same budget.
+    for (const name of ["createSolidDeckRepository", "createSolidReviewStateRepository"])
+      await expect(given.get(name)!.publicFetch!(foreign[4]!)).rejects.toMatchObject({
+        name: "ReadLimitError",
+        limits: smallBudget.limits,
+      });
+    expect(asked.filter(({ url }) => url === foreign[4])).toEqual([]);
+    const registered = decks(`${shared}decks/`);
+    expect(await listed(registered)).toEqual([true, true, false, false, false]);
+    expect(asked.filter(({ url }) => url === registered[4])).toEqual([]);
+    expect(login.sent).toContain(registered[0]);
+    // A write to the instance registered elsewhere goes out all the same.
+    const write = await given.get("createSolidDeckRepository")!.fetch!(registered[4]!, {
+      method: "PUT",
+      body: "",
+    });
+    expect(write.ok).toBe(true);
+    expect(asked).toContainEqual({ url: registered[4], method: "PUT" });
+    // The user's own storage has no such budget.
+    for (let deck = 0; deck < 5; deck++)
+      expect(await useCases.listCards(deckAt(`${alice}memo/decks/deck-${deck}.ttl`))).toEqual([]);
+  });
+
   it("lets a guest delete their instance, which is in their own storage", async () => {
     const guest = createAppUseCases({ ...options, indexedDB: undefined });
     await guest.startGuest("My study");