diff --git a/docs/boundaries.md b/docs/boundaries.md index 10d8dd74..9e6406df 100644 --- a/docs/boundaries.md +++ b/docs/boundaries.md @@ -338,6 +338,21 @@ the [pod fence](#the-pod-fence) that fetch passes the credential gate ([data-model.md](data-model.md#where-an-instances-documents-are)), and what such a document says of itself never makes it writable. Such a read is held to `FOREIGN_LIMITS` too. +- **Outside the user's storages**, the reads share a budget in a page + session (`FOREIGN_READS_BUDGET`, `sharedReadBudget` in + [readLimits.ts](../packages/solid/src/client/readLimits.ts)): four + times `FOREIGN_LIMITS` in total, on top of the limits on each read. + A catalog may list a thousand decks, each with two documents as + large as a release. The reads as no one share one budget; the reads + in a registered instance elsewhere have one for each server, so + reads anywhere else never spend it. As with releases read from a + link ([deck-library.md](deck-library.md#from-a-link)), a document + counts once, by its largest read, and reading it again comes out of + an allowance of its own (`FOREIGN_REREADS`) while that lasts. Past + the budget, such a read fails with a `ReadLimitError` until the page + is loaded again, without asking the server. Only reads count: a + write to a registered instance goes out whatever is spent. The + storages and the folders entered on this page aren't part of it. - **A read past its limits** fails with a `ReadLimitError`. So does a document the app reads into a dataset whose Turtle text has too many bases, a base that's too long or not plain, or too many diff --git a/docs/data-model.md b/docs/data-model.md index 37e785de..d63c2ea4 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -897,12 +897,15 @@ instance listing one stays valid on real servers ([foreignData.integration.test.ts](../e2e/pod/src/foreignData.integration.test.ts)). A catalog is data anyone may have written, so the app reads at most -`MAX_CATALOG_LINKS` (1,000) decks of it, in the deck list, the -statistics and the format update as in the Studio +`MAX_CATALOG_LINKS` (1,000) decks of it, in the deck list and its +arrangement, the statistics and the format update as in the Studio ([catalogLinks.ts](../packages/solid/src/catalogLinks.ts)). Past that, it reads none of them and says why (`catalogTooManyLinks`). The same limit holds for its links to drafts and releases -([Drafts and releases](#drafts-and-releases)). +([Drafts and releases](#drafts-and-releases)). The deck list works out +the study counts of three decks at a time (`getStudyCounts` in +[study.ts](../packages/application/src/study.ts)), each reading the +deck's two documents, so a long list never reads all of them at once. The catalog is written when an instance is created, and by the [format update](migrations.md) for an instance made before there were catalogs; diff --git a/packages/application/src/atOnce.test.ts b/packages/application/src/atOnce.test.ts index e74a1e2a..093511b4 100644 --- a/packages/application/src/atOnce.test.ts +++ b/packages/application/src/atOnce.test.ts @@ -3,7 +3,7 @@ // SPDX-License-Identifier: MIT import { describe, expect, it } from "vitest"; -import { mapAtOnce, READS_AT_ONCE } from "./atOnce"; +import { inTurns, mapAtOnce, READS_AT_ONCE } from "./atOnce"; /** `each` that counts how many run at once, each waiting a turn of the event loop. */ function counted() { @@ -51,3 +51,41 @@ describe("mapAtOnce", () => { expect(started).toEqual([0, 1]); }); }); + +describe("inTurns", () => { + it("runs all the work asked of it, READS_AT_ONCE at most at once, in the order asked", async () => { + const turn = inTurns(); + const { each, most } = counted(); + const started: number[] = []; + const items = Array.from({ length: 40 }, (_, at) => at); + const done = await Promise.all( + items.map((item) => + turn(() => { + started.push(item); + return each(item); + }), + ), + ); + expect(done).toEqual(items); + expect(started).toEqual(items); + expect(most()).toBe(READS_AT_ONCE); + }); + + it("ends a turn whose work fails, and lets the next start", async () => { + const turn = inTurns(1); + const failure = new Error("unreadable"); + const first = turn(async () => { + throw failure; + }); + const second = turn(async () => "read"); + await expect(first).rejects.toBe(failure); + await expect(second).resolves.toBe("read"); + // Every turn is free again. + const { each, most } = counted(); + await expect(Promise.all(["a", "b"].map((item) => turn(() => each(item))))).resolves.toEqual([ + "a", + "b", + ]); + expect(most()).toBe(1); + }); +}); diff --git a/packages/application/src/atOnce.ts b/packages/application/src/atOnce.ts index 9b3820f6..0ad419d9 100644 --- a/packages/application/src/atOnce.ts +++ b/packages/application/src/atOnce.ts @@ -33,3 +33,25 @@ export async function mapAtOnce( await Promise.all(Array.from({ length: Math.min(atOnce, items.length) }, worker)); return results; } + +/** + * Work taken in turns, `atOnce` at most at once, the rest waiting in the + * order asked: whoever asks, a list as long as anyone may write never + * runs all of its work together. A turn ends when its work does, failed + * or not. + */ +export function inTurns(atOnce = READS_AT_ONCE): (work: () => Promise) => Promise { + let running = 0; + const waiting: (() => void)[] = []; + return async (work) => { + if (running >= atOnce) await new Promise((start) => waiting.push(start)); + else running++; + try { + return await work(); + } finally { + const next = waiting.shift(); + if (next === undefined) running--; + else next(); + } + }; +} diff --git a/packages/application/src/digestStore.test.ts b/packages/application/src/digestStore.test.ts index 9ef5c53a..b0e233fa 100644 --- a/packages/application/src/digestStore.test.ts +++ b/packages/application/src/digestStore.test.ts @@ -33,6 +33,31 @@ describe("the instance digest", () => { }); }); + it("counts three decks at once at most, however many are asked for, each reading its two documents", async () => { + const { deps, useCases } = setup(); + const read = deps.deckRepository.readCardsSince; + let reading = 0; + let most = 0; + deps.deckRepository.readCardsSince = vi.fn(async (...args: Parameters) => { + most = Math.max(most, ++reading); + await new Promise((resolve) => setTimeout(resolve, 0)); + reading--; + return read(...args); + }); + const decks = Array.from({ length: 200 }, (_, at) => ({ + ...deck, + url: `${deck.url}-${at}`, + cardsDocumentUrl: `${deck.cardsDocumentUrl}-${at}`, + reviewsDocumentUrl: `${deck.reviewsDocumentUrl}-${at}`, + })); + const counts = await Promise.all( + decks.map((one) => useCases.getStudyCounts(instance.url, one, now)), + ); + expect(counts).toHaveLength(200); + expect(read).toHaveBeenCalledTimes(200); + expect(most).toBe(3); + }); + it("counts from the schedule while neither document changed", async () => { const { deps, useCases } = setup(await learned()); await expect(useCases.getStudyCounts(instance.url, deck, now)).resolves.toEqual({ diff --git a/packages/application/src/study.ts b/packages/application/src/study.ts index d1e5dcc1..22ce8674 100644 --- a/packages/application/src/study.ts +++ b/packages/application/src/study.ts @@ -16,6 +16,7 @@ import { } from "@solid-memo/domain/scheduling"; import { effectiveStates } from "@solid-memo/domain/fsrs"; import type { ResponseTimes } from "@solid-memo/domain/answer"; +import { inTurns, READS_AT_ONCE } from "./atOnce"; import type { DeckRepository, ReviewStateRepository } from "./ports"; import type { PreferenceStore } from "./preferenceStore"; import type { AnswerStore } from "./answerStore"; @@ -34,7 +35,9 @@ export interface StudyUseCases { * digest is brought up to date). Each document is read as the user * only when it is the instance's (isInInstancePod), as no one * otherwise, and only a deck whose documents both are has its - * schedule kept in the digest. + * schedule kept in the digest. STUDY_COUNTS_AT_ONCE decks are counted + * at once, whoever asks, the others waiting their turn: a deck list + * asks for every deck's, and a catalog may list a thousand. */ getStudyCounts( instanceUrl: string, @@ -127,6 +130,9 @@ export interface ReviewUndo { answer: { id: string; studyDay: string }; } +/** How many decks' study counts are worked out at once: each reads two documents together. */ +const STUDY_COUNTS_AT_ONCE = READS_AT_ONCE / 2; + /** * Study (docs/srs.md): a deck's queue for today and how many prompts it * holds, each answer graded by the instance's scheduler and logged, an @@ -148,6 +154,7 @@ export function createStudyUseCases({ answerStore: Pick; random: () => number; }): StudyUseCases { + const countsTurn = inTurns(STUDY_COUNTS_AT_ONCE); async function studyCountsAndSchedule(instanceUrl: string, deck: Deck, now: Date) { const [prefs, digest, asNoOne] = await Promise.all([ preferenceStore.get(instanceUrl), @@ -215,7 +222,7 @@ export function createStudyUseCases({ return { ...queue, prefs, states: reviews.sm2, fsrsStates: reviews.fsrs }; }, getStudyCounts(instanceUrl, deck, now) { - return studyCountsAndSchedule(instanceUrl, deck, now); + return countsTurn(() => studyCountsAndSchedule(instanceUrl, deck, now)); }, refreshStudyDigest: digestStore.refresh, recordReview(instanceUrl, deck, prompt, quality, now, times = {}, known) { diff --git a/packages/composition/src/appUseCases.test.ts b/packages/composition/src/appUseCases.test.ts index 94dd342c..148440c8 100644 --- a/packages/composition/src/appUseCases.test.ts +++ b/packages/composition/src/appUseCases.test.ts @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: MIT -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, onTestFinished, vi } from "vitest"; import { IDBFactory } from "fake-indexeddb"; import { SITE } from "@solid-memo/vocab/tooling/sources"; import { shapesFetch } from "@solid-memo/vocab/tooling/testing/shapeSources"; @@ -71,6 +71,24 @@ vi.mock( vi.mock("@solid-memo/solid/solidReleasePublisher", recorded("createSolidReleasePublisher")); vi.mock("@solid-memo/solid/solidInstanceGraphReader", recorded("createSolidInstanceGraphReader")); +/** A budget for the reads outside the user's storages, small enough for a test to spend, while `on`. */ +const smallBudget = vi.hoisted(() => ({ + on: false, + limits: { bytes: 1024 * 1024, quads: 30, characters: 1024 * 1024 }, +})); +vi.mock("@solid-memo/solid/client/readLimits", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + sharedReadBudget: (options: Parameters[0]) => + original.sharedReadBudget( + smallBudget.on + ? { ...options, budget: smallBudget.limits, rereads: smallBudget.limits } + : options, + ), + }; +}); + /** The login: who is logged in, and the addresses requested with their credentials. */ const login = vi.hoisted(() => ({ webId: undefined as string | undefined, @@ -752,6 +770,98 @@ describe("createAppUseCases", () => { expect(await useCases.listCards(deckAt(`${alice}memo/decks/deck-1.ttl`))).toEqual([]); }); + it("reads the documents of many decks outside the user's storages within a budget, one of each server for an instance they registered, and sends their writes there whatever is spent", async () => { + smallBudget.on = true; + onTestFinished(() => { + smallBudget.on = false; + }); + const alice = "https://alice.example/"; + const shared = "https://shared.example/study/"; + login.webId = `${alice}profile/card#me`; + const guest = createAppUseCases({ ...options, indexedDB: undefined }); + await guest.startGuest("My study"); + const [instance] = await guest.listInstances(GUEST_SESSION); + const made = await guest.createDeck(instance!.url, { en: "Capitals" }); + const entry = `${alice}memo/catalog.ttl#deck-1`; + const deckAt = (url: string) => ({ + ...made, + url: entry, + cardsDocumentUrl: url, + reviewsDocumentUrl: url, + }); + const documents: Record = { + [`${alice}profile/card`]: `<${login.webId}> + <${alice}>; + <${alice}settings/private.ttl>.`, + // An instance on another server, registered as the user's. + [`${alice}settings/private.ttl`]: `<#memo> + a ; + ; + <${shared}>.`, + }; + const asked: { url: string; method: string }[] = []; + vi.stubGlobal( + "fetch", + vi.fn((input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input instanceof Request ? input.url : input).replace(/#.*/, ""); + const elsewhere = ["https://elsewhere.example/", shared].some((at) => url.startsWith(at)); + if (!url.startsWith(alice) && !elsewhere) return servedSiteNetwork(input, init); + asked.push({ url, method: init?.method ?? "GET" }); + // Each deck's document well within the limits of one read: twelve triples. + const triples = Array.from({ length: 11 }, (_, at) => `<#c${at}> <#p> "${at}".`); + const body = + documents[url] ?? + [`<${url}> <${entry}>.`, ...triples].join("\n"); + const response = new Response(body, { headers: { "Content-Type": "text/turtle" } }); + Object.defineProperty(response, "url", { value: url }); + return Promise.resolve(response); + }), + ); + const useCases = createAppUseCases({ ...options, indexedDB: undefined }); + /** Lists the cards of the decks whose documents are at `urls`, one after another: whether each was read. */ + const listed = async (urls: string[]) => { + const read: boolean[] = []; + for (const url of urls) + read.push( + await useCases.listCards(deckAt(url)).then( + () => true, + (error: unknown) => { + expect(error).toMatchObject({ name: "ReadLimitError" }); + return false; + }, + ), + ); + return read; + }; + const decks = (at: string) => Array.from({ length: 5 }, (_, deck) => `${at}deck-${deck}.ttl`); + + const foreign = decks("https://elsewhere.example/"); + expect(await listed(foreign)).toEqual([true, true, false, false, false]); + // Past the budget, the server is not asked. + expect(asked.filter(({ url }) => url === foreign[4])).toEqual([]); + // Nor when a deck's cards or states are copied from there: those reads count in the same budget. + for (const name of ["createSolidDeckRepository", "createSolidReviewStateRepository"]) + await expect(given.get(name)!.publicFetch!(foreign[4]!)).rejects.toMatchObject({ + name: "ReadLimitError", + limits: smallBudget.limits, + }); + expect(asked.filter(({ url }) => url === foreign[4])).toEqual([]); + const registered = decks(`${shared}decks/`); + expect(await listed(registered)).toEqual([true, true, false, false, false]); + expect(asked.filter(({ url }) => url === registered[4])).toEqual([]); + expect(login.sent).toContain(registered[0]); + // A write to the instance registered elsewhere goes out all the same. + const write = await given.get("createSolidDeckRepository")!.fetch!(registered[4]!, { + method: "PUT", + body: "", + }); + expect(write.ok).toBe(true); + expect(asked).toContainEqual({ url: registered[4], method: "PUT" }); + // The user's own storage has no such budget. + for (let deck = 0; deck < 5; deck++) + expect(await useCases.listCards(deckAt(`${alice}memo/decks/deck-${deck}.ttl`))).toEqual([]); + }); + it("lets a guest delete their instance, which is in their own storage", async () => { const guest = createAppUseCases({ ...options, indexedDB: undefined }); await guest.startGuest("My study"); diff --git a/packages/composition/src/appUseCases.ts b/packages/composition/src/appUseCases.ts index 2e63620c..5bf620dd 100644 --- a/packages/composition/src/appUseCases.ts +++ b/packages/composition/src/appUseCases.ts @@ -43,7 +43,12 @@ import { createSolidRepairRepository } from "@solid-memo/solid/solidRepairReposi import { createSolidReviewStateRepository } from "@solid-memo/solid/solidReviewStateRepository"; import { createSolidStorageGateway } from "@solid-memo/solid/solidStorageGateway"; import { grantPublicRead } from "@solid-memo/solid/client/access"; -import { FOREIGN_LIMITS, limitedFetch } from "@solid-memo/solid/client/readLimits"; +import { + FOREIGN_LIMITS, + FOREIGN_READS_BUDGET, + FOREIGN_REREADS, + sharedReadBudget, +} from "@solid-memo/solid/client/readLimits"; import { getProfileStorages } from "@solid-memo/solid/client/resources"; import { createSessionClient } from "@solid-memo/solid/client/session"; import { createSolidWebIdDocumentRepository } from "@solid-memo/solid/solidWebIdDocumentRepository"; @@ -120,6 +125,29 @@ export function createAppUseCases({ ); /** A fetch as no one (authFetch.ts), whose reads have a deadline as the user's do. */ const noOneFetch = withReadDeadline(anonymousFetch); + /** + * What is read outside the user's storages as no one, each document + * under FOREIGN_LIMITS and all of them within FOREIGN_READS_BUDGET + * together in a page session: an instance anyone may make may list a + * thousand decks, each with two documents as large as a release. + */ + const foreignReads = sharedReadBudget({ + limits: FOREIGN_LIMITS, + budget: FOREIGN_READS_BUDGET, + rereads: FOREIGN_REREADS, + }); + /** + * What is read as the user in an instance they registered outside their + * storages, as foreignReads is, within a budget of each server's own, + * so reads elsewhere never spend it; what is written there is sent + * whatever is spent. + */ + const instanceReads = sharedReadBudget({ + limits: FOREIGN_LIMITS, + budget: FOREIGN_READS_BUDGET, + rereads: FOREIGN_REREADS, + scope: (url) => new URL(url).origin, + }); const podFetch = createPodFence(writeFence.fetch); /** The release publisher's alone, for a release's own access control: the pod fence lets it write a document's ACL or ACR. */ const shareFetch = createPodFence(writeFence.fetch, { accessControl: true }); @@ -131,7 +159,8 @@ export function createAppUseCases({ * anywhere else as no one, writing nothing there; a guest's pod is * this browser's. What it reads outside the user's storages, in an * instance they registered elsewhere too, is read under - * FOREIGN_LIMITS. + * FOREIGN_LIMITS, within a budget for the page session (foreignReads, + * instanceReads). */ const dataFetch = createPodFence( routedFetch({ @@ -139,8 +168,8 @@ export function createAppUseCases({ local: writeFence.fetch, remote: createOwnPodGate({ fetch: writeFence.fetch, - instanceFetch: limitedFetch(writeFence.fetch, FOREIGN_LIMITS), - publicFetch: limitedFetch(noOneFetch, FOREIGN_LIMITS), + instanceFetch: instanceReads(writeFence.fetch), + publicFetch: foreignReads(noOneFetch), webId: session.webId, ownFolders, revision: () => folderRevision, @@ -280,7 +309,7 @@ export function createAppUseCases({ fetch: dataFetch, // A deck's cards copied into a draft outside their pod are read as no one, as the gate reads elsewhere; // never from a guest's pod, which is this browser's. - publicFetch: createPodFence(limitedFetch(noOneFetch, FOREIGN_LIMITS)), + publicFetch: createPodFence(foreignReads(noOneFetch)), checkWrite, now, randomId, @@ -298,7 +327,7 @@ export function createAppUseCases({ fetch: dataFetch, // A deck's states copied into another deck outside their pod are read as no one, as its cards are; // never from a guest's pod, which is this browser's. - publicFetch: createPodFence(limitedFetch(noOneFetch, FOREIGN_LIMITS)), + publicFetch: createPodFence(foreignReads(noOneFetch)), checkWrite, }), shapeValidator, diff --git a/packages/solid-memo-client/src/resources.ts b/packages/solid-memo-client/src/resources.ts index f8f7b769..0c633122 100644 --- a/packages/solid-memo-client/src/resources.ts +++ b/packages/solid-memo-client/src/resources.ts @@ -20,7 +20,7 @@ import { import { limitsOf, namesIriUnder, - NO_SIZE_LIMITS, + limitsPast, quadTally, type ReadsSpent, refusalOf, @@ -186,7 +186,7 @@ function collector( refused = true; quads.length = 0; // The tally is past limits only under some; the declarations, under none too. - reject(new ReadLimitError(url, limits ?? NO_SIZE_LIMITS)); + reject(new ReadLimitError(url, limitsPast(limits, spent))); }; return { quads, diff --git a/packages/solid/src/catalogLinks.ts b/packages/solid/src/catalogLinks.ts index a06bf99f..2a432e59 100644 --- a/packages/solid/src/catalogLinks.ts +++ b/packages/solid/src/catalogLinks.ts @@ -23,7 +23,7 @@ const LINK_ATTEMPTS = 3; export { MAX_CATALOG_LINKS }; /** `links`, unless there are more than MAX_CATALOG_LINKS of them: catalogTooManyLinks. */ -export function withinCatalogLimit(links: T[]): T[] { +export function withinCatalogLimit(links: L): L { if (links.length > MAX_CATALOG_LINKS) throw new AppError("catalogTooManyLinks", { limit: MAX_CATALOG_LINKS }); return links; diff --git a/packages/solid/src/client/linearDataset.ts b/packages/solid/src/client/linearDataset.ts index 6b0e878a..d1368653 100644 --- a/packages/solid/src/client/linearDataset.ts +++ b/packages/solid/src/client/linearDataset.ts @@ -16,7 +16,7 @@ import { limitsOf, namesIriUnder, spentOf, - NO_SIZE_LIMITS, + limitsPast, pathKeys, quadTally, type ReadsSpent, @@ -163,7 +163,7 @@ function collectingTurtleParser(collected: Collected, url: string): Parser { collected.quads = quads; const refuse = () => { // The tally is past limits only under some; the declarations, under none too. - collected.refused = new ReadLimitError(url, collected.limits ?? NO_SIZE_LIMITS); + collected.refused = new ReadLimitError(url, limitsPast(collected.limits, collected.spent)); quads.length = 0; for (const callback of failed) callback(collected.refused); }; diff --git a/packages/solid/src/client/readLimits.test.ts b/packages/solid/src/client/readLimits.test.ts index d1af81bd..2f35184b 100644 --- a/packages/solid/src/client/readLimits.test.ts +++ b/packages/solid/src/client/readLimits.test.ts @@ -17,12 +17,13 @@ import { quadTally, refusalOf, RELEASE_LIMITS, + sharedReadBudget, spentOf, termKeyLength, textChunks, writeTurtle, } from "./readLimits"; -import { parseQuads } from "./resources"; +import { getSolidDatasetLinear, parseQuads } from "./resources"; const URL_ = "https://other.example/release.ttl"; const LIMITS = { bytes: 10, quads: 5, characters: 100 }; @@ -193,6 +194,131 @@ describe("limitedFetch", () => { }); }); +describe("sharedReadBudget", () => { + const BUDGET = { bytes: 20, quads: 10, characters: 200 }; + const REREADS = { bytes: 10, quads: 5, characters: 100 }; + const budgeted = () => sharedReadBudget({ limits: LIMITS, budget: BUDGET, rereads: REREADS }); + /** Whether a read of `url` through `fetch` is taken in whole; one that is not names `past`, the limits it went past. */ + const reads = async (fetch: typeof globalThis.fetch, url: string | Request, past = LIMITS) => { + const response = await fetch(url); + try { + await response.text(); + return true; + } catch { + expect(refusalOf(response)).toMatchObject({ limits: past }); + return false; + } + }; + + it("reads each document under its own limits", async () => { + const fetch = budgeted()(serving("0123456789".repeat(2)).fetch); + expect(await reads(fetch, URL_)).toBe(false); + }); + + it("reads documents within the budget together, through every fetch it made, then refuses any read without asking", async () => { + const budget = budgeted(); + const { fetch: seven } = serving("0123456"); + const first = budget(seven); + const second = budget(seven); + expect(await reads(first, "https://other.example/a.ttl")).toBe(true); + expect(await reads(second, "https://other.example/b.ttl")).toBe(true); + // Within the document's limits, past what the budget leaves: the budget is named. + expect(await reads(first, "https://other.example/c.ttl", BUDGET)).toBe(false); + const asked = vi.fn(seven); + const request = new Request("https://other.example/d.ttl"); + await expect(budget(asked)(request)).rejects.toMatchObject({ + name: "ReadLimitError", + url: request.url, + limits: BUDGET, + }); + await expect(budget(asked)("https://other.example/e.ttl")).rejects.toMatchObject({ + limits: BUDGET, + }); + expect(asked).not.toHaveBeenCalled(); + }); + + it("names the limits a Content-Length says the body is past: the document's own, or else the budget", async () => { + const budget = budgeted(); + const seven = serving("0123456").fetch; + expect(await reads(budget(seven), "https://other.example/a.ttl")).toBe(true); + expect(await reads(budget(seven), "https://other.example/b.ttl")).toBe(true); + const said = (length: string) => serving("0123456", { headers: { "Content-Length": length } }); + await expect(budget(said("11").fetch)("https://other.example/c.ttl")).rejects.toMatchObject({ + limits: LIMITS, + }); + await expect(budget(said("7").fetch)("https://other.example/c.ttl")).rejects.toMatchObject({ + limits: BUDGET, + }); + }); + + it("counts a document read again up to its largest read out of the rereads, and whole once they are spent", async () => { + const fetch = budgeted()(serving("0123456").fetch); + const url = "https://other.example/a.ttl"; + // 7 counted, then 7 and 3 of the rereads, then 4 and 7 counted: 18 of 20. + for (let read = 0; read < 4; read++) expect(await reads(fetch, url)).toBe(true); + expect(await reads(fetch, url, BUDGET)).toBe(false); + }); + + it("holds the documents of many decks, each within its limits, to the budget together", async () => { + // Each deck's document as large as a read of one takes in; together, past the budget. + const limits = { bytes: 1024 * 1024, quads: 400, characters: 1024 * 1024 }; + const budget = { bytes: 4 * 1024 * 1024, quads: 1_000, characters: 4 * 1024 * 1024 }; + const turtle = Array.from({ length: 400 }, (_, at) => `<#c${at}> <#front> "card ${at}" .`); + const fetch = sharedReadBudget({ limits, budget, rereads: budget })(async (input) => { + const response = new Response(turtle.join("\n"), { + headers: { "Content-Type": "text/turtle" }, + }); + Object.defineProperty(response, "url", { value: String(input) }); + return response; + }); + const deck = (at: number) => + getSolidDatasetLinear(`https://other.example/memo/decks/deck-${at}.ttl`, { fetch }); + await expect(deck(0)).resolves.toBeDefined(); + await expect(deck(1)).resolves.toBeDefined(); + await expect(deck(2)).rejects.toMatchObject({ name: "ReadLimitError", limits: budget }); + await expect(deck(3)).rejects.toMatchObject({ name: "ReadLimitError", limits: budget }); + }); + + it("sends a write once the budget is spent, its answer under the limits of one read", async () => { + const budget = budgeted(); + const { fetch: seven } = serving("0123456"); + const fetch = budget(seven); + for (const name of ["a", "b"]) await reads(fetch, `https://other.example/${name}.ttl`); + await reads(fetch, "https://other.example/c.ttl", BUDGET); + await expect(fetch("https://other.example/d.ttl")).rejects.toMatchObject({ limits: BUDGET }); + const asked = vi.fn(seven); + for (const method of ["PUT", "patch", "DELETE"]) { + const response = await budget(asked)("https://other.example/d.ttl", { method }); + expect(limitsOf(response)).toEqual(LIMITS); + expect(await response.text()).toBe("0123456"); + } + const written = await budget(asked)( + new Request("https://other.example/d.ttl", { method: "POST" }), + ); + expect(written.status).toBe(200); + expect(asked).toHaveBeenCalledTimes(4); + }); + + it("keeps a budget for each scope a read's address is in", async () => { + const budget = sharedReadBudget({ + limits: LIMITS, + budget: BUDGET, + rereads: REREADS, + scope: (url) => new URL(url).origin, + }); + const fetch = budget(serving("0123456").fetch); + for (const name of ["a", "b"]) await reads(fetch, `https://other.example/${name}.ttl`); + await reads(fetch, "https://other.example/c.ttl", BUDGET); + await expect(fetch("https://other.example/d.ttl")).rejects.toMatchObject({ limits: BUDGET }); + expect(await reads(fetch, "https://own.example/a.ttl")).toBe(true); + }); + + it("gives an answer with no body as it is, counting nothing", async () => { + const fetch = budgeted()(serving(null).fetch); + for (let read = 0; read < 30; read++) expect((await fetch(URL_)).status).toBe(304); + }); +}); + describe("textChunks", () => { it("gives a text in parts that are the text again, each at most one character past a part's length", () => { const text = "0123456789".repeat(1000); diff --git a/packages/solid/src/client/readLimits.ts b/packages/solid/src/client/readLimits.ts index bde72714..ee88ea8d 100644 --- a/packages/solid/src/client/readLimits.ts +++ b/packages/solid/src/client/readLimits.ts @@ -31,7 +31,8 @@ export const RELEASE_LIMITS: ReadLimits = { * holds a deck's cards as an instance's largest documents do. An * instance anyone may make can be added by its address or registered by * another app, a catalog may name a deck's documents on any server, and - * the app keeps every triple of what it reads. The user's storages are + * the app keeps every triple of what it reads; all of them together + * are held to FOREIGN_READS_BUDGET too. The user's storages are * read with no limits but on their declarations and nesting * (writeTurtle) and on the length of each term (MAX_TERM_CHARACTERS). */ @@ -122,11 +123,31 @@ export interface ReadsSpent { characters: number; } +/** + * The limits a read went past, `bytes` more taken in, for each tally + * that stands for more than one set of them (sharedReadBudget); those + * it is read under for any other. + */ +const PAST = new WeakMap ReadLimits>(); + /** Nothing taken in yet. */ export function noneSpent(): ReadsSpent { return { bytes: 0, quads: 0, characters: 0 }; } +/** + * The limits to name in the ReadLimitError of a read past `limits`, with + * `bytes` more taken in: the ones the tally names (PAST), or + * `limits`, or NO_SIZE_LIMITS for a read with none. + */ +export function limitsPast( + limits: ReadLimits | undefined, + spent: ReadsSpent | undefined, + bytes = 0, +): ReadLimits { + return (spent === undefined ? undefined : PAST.get(spent)?.(bytes)) ?? limits ?? NO_SIZE_LIMITS; +} + /** Each response limitedFetch gave: the limits it is read under, what is spent of them, and how it went past them, once it did. */ const READS = new WeakMap< Response, @@ -155,9 +176,10 @@ export function limitedFetch( if (response.body === null || NULL_BODY_STATUSES.has(response.status)) return response; const { url } = response; const tally = spent ?? noneSpent(); - if (tally.bytes + Number(response.headers.get("Content-Length")) > limits.bytes) { + const said = Number(response.headers.get("Content-Length")); + if (tally.bytes + said > limits.bytes) { await response.body.cancel(); - throw new ReadLimitError(url, limits); + throw new ReadLimitError(url, limitsPast(limits, tally, said)); } const read: { limits: ReadLimits; spent: ReadsSpent; refused?: ReadLimitError } = { limits, @@ -168,7 +190,7 @@ export function limitedFetch( transform(chunk, controller) { tally.bytes += chunk.byteLength; if (tally.bytes <= limits.bytes) return controller.enqueue(chunk); - read.refused = new ReadLimitError(url, limits); + read.refused = new ReadLimitError(url, limitsPast(limits, tally)); controller.error(read.refused); }, }), @@ -185,6 +207,120 @@ export function limitedFetch( }; } +/** + * The most the reads of documents outside the user's storages take in + * together in a page session (foreignReads), on top of FOREIGN_LIMITS on + * each: four times what one release may hold, so many times the largest + * release in the library. A catalog may name a thousand decks, each with + * two documents as large as a release, and the app holds every triple of + * what it reads while it reads it. Once read, a document counts once, by + * its largest read, while reads of it again come out of FOREIGN_REREADS. + */ +export const FOREIGN_READS_BUDGET: ReadLimits = { + bytes: 4 * RELEASE_LIMITS.bytes, + quads: 4 * RELEASE_LIMITS.quads, + characters: 4 * RELEASE_LIMITS.characters, +}; + +/** + * How much reads of documents already read may take in a page session + * without counting in FOREIGN_READS_BUDGET, up to each one's largest + * read: twice that budget. Past it they count whole, or many decks + * naming one document could read it again and again without limit. + */ +export const FOREIGN_REREADS: ReadLimits = { + bytes: 2 * FOREIGN_READS_BUDGET.bytes, + quads: 2 * FOREIGN_READS_BUDGET.quads, + characters: 2 * FOREIGN_READS_BUDGET.characters, +}; + +const SPENT_KEYS = ["bytes", "quads", "characters"] as const; + +/** The methods that read, which a read budget counts; any other request is sent as it is asked. */ +const BUDGETED_METHODS = new Set(["GET", "HEAD"]); + +/** What one budget of sharedReadBudget has spent so far. */ +interface BudgetSpent { + /** What the reads have taken in of the budget. */ + session: ReadsSpent; + /** What is left of `rereads`. */ + free: ReadsSpent; + /** The largest read of each document so far. */ + largest: Map; +} + +/** + * Reads under one budget together, as the reads of releases from a link + * are (solidDeckLibrary.ts): what it gives makes a fetch whose answers + * are each read under `limits` (limitedFetch), and its reads (GET and + * HEAD), through every fetch it made, within `budget` too. With `scope`, + * each scope it names a read's address by has a budget of its own. A + * document counts in the budget by its largest read; what a read of it + * again takes within that comes out of `rereads` while that lasts. Once + * the budget is spent, a read under way fails as it goes on, and any + * after it fails at once, without asking the server, with a + * ReadLimitError naming the budget, until the page is loaded again. A + * request that writes is sent whatever is spent, its answer under + * `limits` alone. + */ +export function sharedReadBudget({ + limits, + budget, + rereads, + scope = () => "", +}: { + limits: ReadLimits; + budget: ReadLimits; + rereads: ReadLimits; + /** The budget a read of an address counts in: one for all of them unless given. */ + scope?: (url: string) => string; +}): (fetch: typeof globalThis.fetch) => typeof globalThis.fetch { + const budgets = new Map(); + const budgetOf = (url: string): BudgetSpent => { + const key = scope(url); + const found = budgets.get(key); + if (found !== undefined) return found; + const made: BudgetSpent = { session: noneSpent(), free: { ...rereads }, largest: new Map() }; + budgets.set(key, made); + return made; + }; + return (fetch) => async (input, init) => { + const url = input instanceof Request ? input.url : String(input); + const method = ( + init?.method ?? (input instanceof Request ? input.method : "GET") + ).toUpperCase(); + if (!BUDGETED_METHODS.has(method)) return limitedFetch(fetch, limits)(input, init); + const { session, free, largest } = budgetOf(url); + if (SPENT_KEYS.some((key) => session[key] > budget[key])) throw new ReadLimitError(url, budget); + const most = largest.get(url) ?? noneSpent(); + largest.set(url, most); + const document = noneSpent(); + // limitedFetch compares one tally with `limits`: this read's own, or what the budget leaves of them, whichever is + // more, so it is past them once either is past its limit, and names the budget when it is the one past. + const over = (tally: ReadsSpent, bounds: ReadLimits, bytes: number) => + SPENT_KEYS.some((key) => tally[key] + (key === "bytes" ? bytes : 0) > bounds[key]); + const spent = {} as ReadsSpent; + PAST.set(spent, (bytes) => + over(session, budget, bytes) && !over(document, limits, bytes) ? budget : limits, + ); + for (const key of SPENT_KEYS) { + Object.defineProperty(spent, key, { + get: () => Math.max(document[key], limits[key] - budget[key] + session[key]), + set: (value: number) => { + const taken = value - spent[key]; + const within = Math.max(0, Math.min(most[key], document[key] + taken) - document[key]); + const reread = Math.min(within, free[key]); + free[key] -= reread; + document[key] += taken; + most[key] = Math.max(most[key], document[key]); + session[key] += taken - reread; + }, + }); + } + return limitedFetch(fetch, limits, spent)(input, init); + }; +} + /** The limits a response is read under: limitedFetch's, or none. */ export function limitsOf(response: Response): ReadLimits | undefined { return READS.get(response)?.limits; diff --git a/packages/solid/src/deckTree.test.ts b/packages/solid/src/deckTree.test.ts index 78327f04..8096637d 100644 --- a/packages/solid/src/deckTree.test.ts +++ b/packages/solid/src/deckTree.test.ts @@ -3,7 +3,7 @@ // SPDX-License-Identifier: MIT import { describe, expect, it, vi } from "vitest"; -import type { Deck } from "@solid-memo/domain/deck"; +import { type Deck, MAX_CATALOG_LINKS } from "@solid-memo/domain/deck"; import { decksOf, nodeId, type DeckTree, type TreeNode } from "@solid-memo/domain/deckTree"; import { SHAPE_SOURCES, shapesFetch } from "@solid-memo/vocab/tooling/testing/shapeSources"; import { createShaclShapeValidator } from "./shaclShapeValidator"; @@ -75,6 +75,24 @@ describe("the deck tree in the pod", () => { expect(shape(tree.children)).toEqual(["b", { "group-1": ["c", "a"] }]); }); + it("refuses a catalog with more decks than MAX_CATALOG_LINKS, as catalogTooManyLinks, to read or edit", async () => { + const decks = (count: number) => + Array.from({ length: count }, (_, i) => deckTtl(`d${i}`)).join(""); + const most = fakePod(); + await most.put(CATALOG, PREFIXES + decks(MAX_CATALOG_LINKS)); + expect(decksOf((await repositoryOn(most.fetch).readDeckTree(INSTANCE)).children)).toHaveLength( + MAX_CATALOG_LINKS, + ); + const past = fakePod(); + await past.put(CATALOG, PREFIXES + decks(MAX_CATALOG_LINKS + 1)); + const tooMany = { code: "catalogTooManyLinks", vars: { limit: MAX_CATALOG_LINKS } }; + await expect(repositoryOn(past.fetch).readDeckTree(INSTANCE)).rejects.toMatchObject(tooMany); + await expect( + repositoryOn(past.fetch).editDeckTree(INSTANCE, { kind: "removeGroup", group: group.url }), + ).rejects.toMatchObject(tooMany); + expect(writes(past)).toEqual([]); + }); + it("is edited in one checked write of the groups and the catalog, If-Match the read, and reads back as written", async () => { const pod = await podWith(); const checkWrite = vi.fn(async () => undefined); diff --git a/packages/solid/src/solidDeckRepository.ts b/packages/solid/src/solidDeckRepository.ts index 56d8b49e..c63ba88c 100644 --- a/packages/solid/src/solidDeckRepository.ts +++ b/packages/solid/src/solidDeckRepository.ts @@ -44,7 +44,12 @@ import { documentUrlOf, fragmentIdOf } from "@solid-memo/domain/subjectUrl"; import { isDeckOutdated, isOutdated, upgradeCard, upgradeDeck } from "@solid-memo/domain/migration"; import { ABSENT_VERSION } from "@solid-memo/domain/studyDigest"; import { CARD_V7 } from "@solid-memo/vocab/descriptors.generated"; -import { applyDeckTreeEdit, buildTree, treeChanges } from "@solid-memo/domain/deckTree"; +import { + applyDeckTreeEdit, + buildTree, + type StoredLayout, + treeChanges, +} from "@solid-memo/domain/deckTree"; import { getSolidDatasetOrNull, PreconditionFailedError, @@ -121,6 +126,17 @@ export interface SolidDeckRepositoryDeps { */ const TREE_ATTEMPTS = 3; +/** + * The arrangement the catalog states (toStoredLayout), or + * catalogTooManyLinks past MAX_CATALOG_LINKS decks, as listDecks gives: + * each deck's documents are read by whoever shows the tree. + */ +function cappedLayout(dataset: Dataset, catalogUrl: string): StoredLayout { + const layout = toStoredLayout(dataset, catalogUrl); + withinCatalogLimit(layout.decks); + return layout; +} + export function createSolidDeckRepository({ fetch, publicFetch = fetch, @@ -263,14 +279,14 @@ export function createSolidDeckRepository({ async readDeckTree(instanceUrl) { const catalogUrl = catalogUrlOf(instanceUrl); const dataset = await getSolidDatasetOrNull(catalogUrl, fetch); - return buildTree(toStoredLayout(dataset ?? createSolidDataset(), catalogUrl)); + return buildTree(cappedLayout(dataset ?? createSolidDataset(), catalogUrl)); }, async editDeckTree(instanceUrl, edit) { const catalogUrl = catalogUrlOf(instanceUrl); for (let attempt = 1; ; attempt++) { const dataset = (await getSolidDatasetOrNull(catalogUrl, fetch)) ?? createSolidDataset(); - const stored = toStoredLayout(dataset, catalogUrl); + const stored = cappedLayout(dataset, catalogUrl); const before = buildTree(stored); const after = applyDeckTreeEdit(before, edit); if (after === before) return after;