From f674c3a8c68a44b7a698ffb132d6a2d122bce6c4 Mon Sep 17 00:00:00 2001 From: Anton Wiklund Date: Sun, 8 Nov 2026 21:56:55 +0100 Subject: [PATCH] fix(solid): bound the reads of documents outside the user's storages An instance outside the user's storages could list many decks, and the deck list then read every deck's cards and reviews documents at once, each held only to its own limits. Nothing bounded the total, so such a catalog could take up the tab's memory. Reads outside the user's storages now count in a budget for the page session on top of the limits on each document, counted as the reads of releases from a link are: a document once, by its largest read, and reads of it again out of an allowance of their own. The reads as no one share one budget, and an instance the user registered on another server has one for each server, so reads elsewhere never take from it. Past the budget, such reads fail with a read error until the page is loaded again, and a read that goes past the budget names the budget in its error, not the limits on each document. Writes are not counted, so the user's writes to an instance they registered elsewhere go out whatever is spent. The deck arrangement refuses a catalog with more decks than MAX_CATALOG_LINKS, as the deck list already did, and the study counts are worked out a few decks at a time, whoever asks for them. Refs: GHSA-9q65-rfrp-5gj6 --- docs/boundaries.md | 15 ++ docs/data-model.md | 9 +- packages/application/src/atOnce.test.ts | 40 +++++- packages/application/src/atOnce.ts | 22 +++ packages/application/src/digestStore.test.ts | 25 ++++ packages/application/src/study.ts | 11 +- packages/composition/src/appUseCases.test.ts | 112 ++++++++++++++- packages/composition/src/appUseCases.ts | 41 +++++- packages/solid-memo-client/src/resources.ts | 4 +- packages/solid/src/catalogLinks.ts | 2 +- packages/solid/src/client/linearDataset.ts | 4 +- packages/solid/src/client/readLimits.test.ts | 128 ++++++++++++++++- packages/solid/src/client/readLimits.ts | 144 ++++++++++++++++++- packages/solid/src/deckTree.test.ts | 20 ++- packages/solid/src/solidDeckRepository.ts | 22 ++- 15 files changed, 572 insertions(+), 27 deletions(-) diff --git a/docs/boundaries.md b/docs/boundaries.md index 10d8dd748..9e6406df2 100644 --- a/docs/boundaries.md +++ b/docs/boundaries.md @@ -338,6 +338,21 @@ the [pod fence](#the-pod-fence) that fetch passes the credential gate ([data-model.md](data-model.md#where-an-instances-documents-are)), and what such a document says of itself never makes it writable. Such a read is held to `FOREIGN_LIMITS` too. +- **Outside the user's storages**, the reads share a budget in a page + session (`FOREIGN_READS_BUDGET`, `sharedReadBudget` in + [readLimits.ts](../packages/solid/src/client/readLimits.ts)): four + times `FOREIGN_LIMITS` in total, on top of the limits on each read. + A catalog may list a thousand decks, each with two documents as + large as a release. The reads as no one share one budget; the reads + in a registered instance elsewhere have one for each server, so + reads anywhere else never spend it. As with releases read from a + link ([deck-library.md](deck-library.md#from-a-link)), a document + counts once, by its largest read, and reading it again comes out of + an allowance of its own (`FOREIGN_REREADS`) while that lasts. Past + the budget, such a read fails with a `ReadLimitError` until the page + is loaded again, without asking the server. Only reads count: a + write to a registered instance goes out whatever is spent. The + storages and the folders entered on this page aren't part of it. - **A read past its limits** fails with a `ReadLimitError`. So does a document the app reads into a dataset whose Turtle text has too many bases, a base that's too long or not plain, or too many diff --git a/docs/data-model.md b/docs/data-model.md index 37e785dec..d63c2ea4a 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -897,12 +897,15 @@ instance listing one stays valid on real servers ([foreignData.integration.test.ts](../e2e/pod/src/foreignData.integration.test.ts)). A catalog is data anyone may have written, so the app reads at most -`MAX_CATALOG_LINKS` (1,000) decks of it, in the deck list, the -statistics and the format update as in the Studio +`MAX_CATALOG_LINKS` (1,000) decks of it, in the deck list and its +arrangement, the statistics and the format update as in the Studio ([catalogLinks.ts](../packages/solid/src/catalogLinks.ts)). Past that, it reads none of them and says why (`catalogTooManyLinks`). The same limit holds for its links to drafts and releases -([Drafts and releases](#drafts-and-releases)). +([Drafts and releases](#drafts-and-releases)). The deck list works out +the study counts of three decks at a time (`getStudyCounts` in +[study.ts](../packages/application/src/study.ts)), each reading the +deck's two documents, so a long list never reads all of them at once. The catalog is written when an instance is created, and by the [format update](migrations.md) for an instance made before there were catalogs; diff --git a/packages/application/src/atOnce.test.ts b/packages/application/src/atOnce.test.ts index e74a1e2a9..093511b4f 100644 --- a/packages/application/src/atOnce.test.ts +++ b/packages/application/src/atOnce.test.ts @@ -3,7 +3,7 @@ // SPDX-License-Identifier: MIT import { describe, expect, it } from "vitest"; -import { mapAtOnce, READS_AT_ONCE } from "./atOnce"; +import { inTurns, mapAtOnce, READS_AT_ONCE } from "./atOnce"; /** `each` that counts how many run at once, each waiting a turn of the event loop. */ function counted() { @@ -51,3 +51,41 @@ describe("mapAtOnce", () => { expect(started).toEqual([0, 1]); }); }); + +describe("inTurns", () => { + it("runs all the work asked of it, READS_AT_ONCE at most at once, in the order asked", async () => { + const turn = inTurns(); + const { each, most } = counted(); + const started: number[] = []; + const items = Array.from({ length: 40 }, (_, at) => at); + const done = await Promise.all( + items.map((item) => + turn(() => { + started.push(item); + return each(item); + }), + ), + ); + expect(done).toEqual(items); + expect(started).toEqual(items); + expect(most()).toBe(READS_AT_ONCE); + }); + + it("ends a turn whose work fails, and lets the next start", async () => { + const turn = inTurns(1); + const failure = new Error("unreadable"); + const first = turn(async () => { + throw failure; + }); + const second = turn(async () => "read"); + await expect(first).rejects.toBe(failure); + await expect(second).resolves.toBe("read"); + // Every turn is free again. + const { each, most } = counted(); + await expect(Promise.all(["a", "b"].map((item) => turn(() => each(item))))).resolves.toEqual([ + "a", + "b", + ]); + expect(most()).toBe(1); + }); +}); diff --git a/packages/application/src/atOnce.ts b/packages/application/src/atOnce.ts index 9b3820f66..0ad419d9d 100644 --- a/packages/application/src/atOnce.ts +++ b/packages/application/src/atOnce.ts @@ -33,3 +33,25 @@ export async function mapAtOnce( await Promise.all(Array.from({ length: Math.min(atOnce, items.length) }, worker)); return results; } + +/** + * Work taken in turns, `atOnce` at most at once, the rest waiting in the + * order asked: whoever asks, a list as long as anyone may write never + * runs all of its work together. A turn ends when its work does, failed + * or not. + */ +export function inTurns(atOnce = READS_AT_ONCE): (work: () => Promise) => Promise { + let running = 0; + const waiting: (() => void)[] = []; + return async (work) => { + if (running >= atOnce) await new Promise((start) => waiting.push(start)); + else running++; + try { + return await work(); + } finally { + const next = waiting.shift(); + if (next === undefined) running--; + else next(); + } + }; +} diff --git a/packages/application/src/digestStore.test.ts b/packages/application/src/digestStore.test.ts index 9ef5c53a4..b0e233fa8 100644 --- a/packages/application/src/digestStore.test.ts +++ b/packages/application/src/digestStore.test.ts @@ -33,6 +33,31 @@ describe("the instance digest", () => { }); }); + it("counts three decks at once at most, however many are asked for, each reading its two documents", async () => { + const { deps, useCases } = setup(); + const read = deps.deckRepository.readCardsSince; + let reading = 0; + let most = 0; + deps.deckRepository.readCardsSince = vi.fn(async (...args: Parameters) => { + most = Math.max(most, ++reading); + await new Promise((resolve) => setTimeout(resolve, 0)); + reading--; + return read(...args); + }); + const decks = Array.from({ length: 200 }, (_, at) => ({ + ...deck, + url: `${deck.url}-${at}`, + cardsDocumentUrl: `${deck.cardsDocumentUrl}-${at}`, + reviewsDocumentUrl: `${deck.reviewsDocumentUrl}-${at}`, + })); + const counts = await Promise.all( + decks.map((one) => useCases.getStudyCounts(instance.url, one, now)), + ); + expect(counts).toHaveLength(200); + expect(read).toHaveBeenCalledTimes(200); + expect(most).toBe(3); + }); + it("counts from the schedule while neither document changed", async () => { const { deps, useCases } = setup(await learned()); await expect(useCases.getStudyCounts(instance.url, deck, now)).resolves.toEqual({ diff --git a/packages/application/src/study.ts b/packages/application/src/study.ts index d1e5dcc1b..22ce86745 100644 --- a/packages/application/src/study.ts +++ b/packages/application/src/study.ts @@ -16,6 +16,7 @@ import { } from "@solid-memo/domain/scheduling"; import { effectiveStates } from "@solid-memo/domain/fsrs"; import type { ResponseTimes } from "@solid-memo/domain/answer"; +import { inTurns, READS_AT_ONCE } from "./atOnce"; import type { DeckRepository, ReviewStateRepository } from "./ports"; import type { PreferenceStore } from "./preferenceStore"; import type { AnswerStore } from "./answerStore"; @@ -34,7 +35,9 @@ export interface StudyUseCases { * digest is brought up to date). Each document is read as the user * only when it is the instance's (isInInstancePod), as no one * otherwise, and only a deck whose documents both are has its - * schedule kept in the digest. + * schedule kept in the digest. STUDY_COUNTS_AT_ONCE decks are counted + * at once, whoever asks, the others waiting their turn: a deck list + * asks for every deck's, and a catalog may list a thousand. */ getStudyCounts( instanceUrl: string, @@ -127,6 +130,9 @@ export interface ReviewUndo { answer: { id: string; studyDay: string }; } +/** How many decks' study counts are worked out at once: each reads two documents together. */ +const STUDY_COUNTS_AT_ONCE = READS_AT_ONCE / 2; + /** * Study (docs/srs.md): a deck's queue for today and how many prompts it * holds, each answer graded by the instance's scheduler and logged, an @@ -148,6 +154,7 @@ export function createStudyUseCases({ answerStore: Pick; random: () => number; }): StudyUseCases { + const countsTurn = inTurns(STUDY_COUNTS_AT_ONCE); async function studyCountsAndSchedule(instanceUrl: string, deck: Deck, now: Date) { const [prefs, digest, asNoOne] = await Promise.all([ preferenceStore.get(instanceUrl), @@ -215,7 +222,7 @@ export function createStudyUseCases({ return { ...queue, prefs, states: reviews.sm2, fsrsStates: reviews.fsrs }; }, getStudyCounts(instanceUrl, deck, now) { - return studyCountsAndSchedule(instanceUrl, deck, now); + return countsTurn(() => studyCountsAndSchedule(instanceUrl, deck, now)); }, refreshStudyDigest: digestStore.refresh, recordReview(instanceUrl, deck, prompt, quality, now, times = {}, known) { diff --git a/packages/composition/src/appUseCases.test.ts b/packages/composition/src/appUseCases.test.ts index 94dd342c9..148440c80 100644 --- a/packages/composition/src/appUseCases.test.ts +++ b/packages/composition/src/appUseCases.test.ts @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: MIT -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, onTestFinished, vi } from "vitest"; import { IDBFactory } from "fake-indexeddb"; import { SITE } from "@solid-memo/vocab/tooling/sources"; import { shapesFetch } from "@solid-memo/vocab/tooling/testing/shapeSources"; @@ -71,6 +71,24 @@ vi.mock( vi.mock("@solid-memo/solid/solidReleasePublisher", recorded("createSolidReleasePublisher")); vi.mock("@solid-memo/solid/solidInstanceGraphReader", recorded("createSolidInstanceGraphReader")); +/** A budget for the reads outside the user's storages, small enough for a test to spend, while `on`. */ +const smallBudget = vi.hoisted(() => ({ + on: false, + limits: { bytes: 1024 * 1024, quads: 30, characters: 1024 * 1024 }, +})); +vi.mock("@solid-memo/solid/client/readLimits", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + sharedReadBudget: (options: Parameters[0]) => + original.sharedReadBudget( + smallBudget.on + ? { ...options, budget: smallBudget.limits, rereads: smallBudget.limits } + : options, + ), + }; +}); + /** The login: who is logged in, and the addresses requested with their credentials. */ const login = vi.hoisted(() => ({ webId: undefined as string | undefined, @@ -752,6 +770,98 @@ describe("createAppUseCases", () => { expect(await useCases.listCards(deckAt(`${alice}memo/decks/deck-1.ttl`))).toEqual([]); }); + it("reads the documents of many decks outside the user's storages within a budget, one of each server for an instance they registered, and sends their writes there whatever is spent", async () => { + smallBudget.on = true; + onTestFinished(() => { + smallBudget.on = false; + }); + const alice = "https://alice.example/"; + const shared = "https://shared.example/study/"; + login.webId = `${alice}profile/card#me`; + const guest = createAppUseCases({ ...options, indexedDB: undefined }); + await guest.startGuest("My study"); + const [instance] = await guest.listInstances(GUEST_SESSION); + const made = await guest.createDeck(instance!.url, { en: "Capitals" }); + const entry = `${alice}memo/catalog.ttl#deck-1`; + const deckAt = (url: string) => ({ + ...made, + url: entry, + cardsDocumentUrl: url, + reviewsDocumentUrl: url, + }); + const documents: Record = { + [`${alice}profile/card`]: `<${login.webId}> + <${alice}>; + <${alice}settings/private.ttl>.`, + // An instance on another server, registered as the user's. + [`${alice}settings/private.ttl`]: `<#memo> + a ; + ; + <${shared}>.`, + }; + const asked: { url: string; method: string }[] = []; + vi.stubGlobal( + "fetch", + vi.fn((input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input instanceof Request ? input.url : input).replace(/#.*/, ""); + const elsewhere = ["https://elsewhere.example/", shared].some((at) => url.startsWith(at)); + if (!url.startsWith(alice) && !elsewhere) return servedSiteNetwork(input, init); + asked.push({ url, method: init?.method ?? "GET" }); + // Each deck's document well within the limits of one read: twelve triples. + const triples = Array.from({ length: 11 }, (_, at) => `<#c${at}> <#p> "${at}".`); + const body = + documents[url] ?? + [`<${url}> <${entry}>.`, ...triples].join("\n"); + const response = new Response(body, { headers: { "Content-Type": "text/turtle" } }); + Object.defineProperty(response, "url", { value: url }); + return Promise.resolve(response); + }), + ); + const useCases = createAppUseCases({ ...options, indexedDB: undefined }); + /** Lists the cards of the decks whose documents are at `urls`, one after another: whether each was read. */ + const listed = async (urls: string[]) => { + const read: boolean[] = []; + for (const url of urls) + read.push( + await useCases.listCards(deckAt(url)).then( + () => true, + (error: unknown) => { + expect(error).toMatchObject({ name: "ReadLimitError" }); + return false; + }, + ), + ); + return read; + }; + const decks = (at: string) => Array.from({ length: 5 }, (_, deck) => `${at}deck-${deck}.ttl`); + + const foreign = decks("https://elsewhere.example/"); + expect(await listed(foreign)).toEqual([true, true, false, false, false]); + // Past the budget, the server is not asked. + expect(asked.filter(({ url }) => url === foreign[4])).toEqual([]); + // Nor when a deck's cards or states are copied from there: those reads count in the same budget. + for (const name of ["createSolidDeckRepository", "createSolidReviewStateRepository"]) + await expect(given.get(name)!.publicFetch!(foreign[4]!)).rejects.toMatchObject({ + name: "ReadLimitError", + limits: smallBudget.limits, + }); + expect(asked.filter(({ url }) => url === foreign[4])).toEqual([]); + const registered = decks(`${shared}decks/`); + expect(await listed(registered)).toEqual([true, true, false, false, false]); + expect(asked.filter(({ url }) => url === registered[4])).toEqual([]); + expect(login.sent).toContain(registered[0]); + // A write to the instance registered elsewhere goes out all the same. + const write = await given.get("createSolidDeckRepository")!.fetch!(registered[4]!, { + method: "PUT", + body: "", + }); + expect(write.ok).toBe(true); + expect(asked).toContainEqual({ url: registered[4], method: "PUT" }); + // The user's own storage has no such budget. + for (let deck = 0; deck < 5; deck++) + expect(await useCases.listCards(deckAt(`${alice}memo/decks/deck-${deck}.ttl`))).toEqual([]); + }); + it("lets a guest delete their instance, which is in their own storage", async () => { const guest = createAppUseCases({ ...options, indexedDB: undefined }); await guest.startGuest("My study"); diff --git a/packages/composition/src/appUseCases.ts b/packages/composition/src/appUseCases.ts index 2e63620c8..5bf620ddf 100644 --- a/packages/composition/src/appUseCases.ts +++ b/packages/composition/src/appUseCases.ts @@ -43,7 +43,12 @@ import { createSolidRepairRepository } from "@solid-memo/solid/solidRepairReposi import { createSolidReviewStateRepository } from "@solid-memo/solid/solidReviewStateRepository"; import { createSolidStorageGateway } from "@solid-memo/solid/solidStorageGateway"; import { grantPublicRead } from "@solid-memo/solid/client/access"; -import { FOREIGN_LIMITS, limitedFetch } from "@solid-memo/solid/client/readLimits"; +import { + FOREIGN_LIMITS, + FOREIGN_READS_BUDGET, + FOREIGN_REREADS, + sharedReadBudget, +} from "@solid-memo/solid/client/readLimits"; import { getProfileStorages } from "@solid-memo/solid/client/resources"; import { createSessionClient } from "@solid-memo/solid/client/session"; import { createSolidWebIdDocumentRepository } from "@solid-memo/solid/solidWebIdDocumentRepository"; @@ -120,6 +125,29 @@ export function createAppUseCases({ ); /** A fetch as no one (authFetch.ts), whose reads have a deadline as the user's do. */ const noOneFetch = withReadDeadline(anonymousFetch); + /** + * What is read outside the user's storages as no one, each document + * under FOREIGN_LIMITS and all of them within FOREIGN_READS_BUDGET + * together in a page session: an instance anyone may make may list a + * thousand decks, each with two documents as large as a release. + */ + const foreignReads = sharedReadBudget({ + limits: FOREIGN_LIMITS, + budget: FOREIGN_READS_BUDGET, + rereads: FOREIGN_REREADS, + }); + /** + * What is read as the user in an instance they registered outside their + * storages, as foreignReads is, within a budget of each server's own, + * so reads elsewhere never spend it; what is written there is sent + * whatever is spent. + */ + const instanceReads = sharedReadBudget({ + limits: FOREIGN_LIMITS, + budget: FOREIGN_READS_BUDGET, + rereads: FOREIGN_REREADS, + scope: (url) => new URL(url).origin, + }); const podFetch = createPodFence(writeFence.fetch); /** The release publisher's alone, for a release's own access control: the pod fence lets it write a document's ACL or ACR. */ const shareFetch = createPodFence(writeFence.fetch, { accessControl: true }); @@ -131,7 +159,8 @@ export function createAppUseCases({ * anywhere else as no one, writing nothing there; a guest's pod is * this browser's. What it reads outside the user's storages, in an * instance they registered elsewhere too, is read under - * FOREIGN_LIMITS. + * FOREIGN_LIMITS, within a budget for the page session (foreignReads, + * instanceReads). */ const dataFetch = createPodFence( routedFetch({ @@ -139,8 +168,8 @@ export function createAppUseCases({ local: writeFence.fetch, remote: createOwnPodGate({ fetch: writeFence.fetch, - instanceFetch: limitedFetch(writeFence.fetch, FOREIGN_LIMITS), - publicFetch: limitedFetch(noOneFetch, FOREIGN_LIMITS), + instanceFetch: instanceReads(writeFence.fetch), + publicFetch: foreignReads(noOneFetch), webId: session.webId, ownFolders, revision: () => folderRevision, @@ -280,7 +309,7 @@ export function createAppUseCases({ fetch: dataFetch, // A deck's cards copied into a draft outside their pod are read as no one, as the gate reads elsewhere; // never from a guest's pod, which is this browser's. - publicFetch: createPodFence(limitedFetch(noOneFetch, FOREIGN_LIMITS)), + publicFetch: createPodFence(foreignReads(noOneFetch)), checkWrite, now, randomId, @@ -298,7 +327,7 @@ export function createAppUseCases({ fetch: dataFetch, // A deck's states copied into another deck outside their pod are read as no one, as its cards are; // never from a guest's pod, which is this browser's. - publicFetch: createPodFence(limitedFetch(noOneFetch, FOREIGN_LIMITS)), + publicFetch: createPodFence(foreignReads(noOneFetch)), checkWrite, }), shapeValidator, diff --git a/packages/solid-memo-client/src/resources.ts b/packages/solid-memo-client/src/resources.ts index f8f7b7698..0c6331228 100644 --- a/packages/solid-memo-client/src/resources.ts +++ b/packages/solid-memo-client/src/resources.ts @@ -20,7 +20,7 @@ import { import { limitsOf, namesIriUnder, - NO_SIZE_LIMITS, + limitsPast, quadTally, type ReadsSpent, refusalOf, @@ -186,7 +186,7 @@ function collector( refused = true; quads.length = 0; // The tally is past limits only under some; the declarations, under none too. - reject(new ReadLimitError(url, limits ?? NO_SIZE_LIMITS)); + reject(new ReadLimitError(url, limitsPast(limits, spent))); }; return { quads, diff --git a/packages/solid/src/catalogLinks.ts b/packages/solid/src/catalogLinks.ts index a06bf99f2..2a432e596 100644 --- a/packages/solid/src/catalogLinks.ts +++ b/packages/solid/src/catalogLinks.ts @@ -23,7 +23,7 @@ const LINK_ATTEMPTS = 3; export { MAX_CATALOG_LINKS }; /** `links`, unless there are more than MAX_CATALOG_LINKS of them: catalogTooManyLinks. */ -export function withinCatalogLimit(links: T[]): T[] { +export function withinCatalogLimit(links: L): L { if (links.length > MAX_CATALOG_LINKS) throw new AppError("catalogTooManyLinks", { limit: MAX_CATALOG_LINKS }); return links; diff --git a/packages/solid/src/client/linearDataset.ts b/packages/solid/src/client/linearDataset.ts index 6b0e878a6..d13686532 100644 --- a/packages/solid/src/client/linearDataset.ts +++ b/packages/solid/src/client/linearDataset.ts @@ -16,7 +16,7 @@ import { limitsOf, namesIriUnder, spentOf, - NO_SIZE_LIMITS, + limitsPast, pathKeys, quadTally, type ReadsSpent, @@ -163,7 +163,7 @@ function collectingTurtleParser(collected: Collected, url: string): Parser { collected.quads = quads; const refuse = () => { // The tally is past limits only under some; the declarations, under none too. - collected.refused = new ReadLimitError(url, collected.limits ?? NO_SIZE_LIMITS); + collected.refused = new ReadLimitError(url, limitsPast(collected.limits, collected.spent)); quads.length = 0; for (const callback of failed) callback(collected.refused); }; diff --git a/packages/solid/src/client/readLimits.test.ts b/packages/solid/src/client/readLimits.test.ts index d1af81bd5..2f35184b5 100644 --- a/packages/solid/src/client/readLimits.test.ts +++ b/packages/solid/src/client/readLimits.test.ts @@ -17,12 +17,13 @@ import { quadTally, refusalOf, RELEASE_LIMITS, + sharedReadBudget, spentOf, termKeyLength, textChunks, writeTurtle, } from "./readLimits"; -import { parseQuads } from "./resources"; +import { getSolidDatasetLinear, parseQuads } from "./resources"; const URL_ = "https://other.example/release.ttl"; const LIMITS = { bytes: 10, quads: 5, characters: 100 }; @@ -193,6 +194,131 @@ describe("limitedFetch", () => { }); }); +describe("sharedReadBudget", () => { + const BUDGET = { bytes: 20, quads: 10, characters: 200 }; + const REREADS = { bytes: 10, quads: 5, characters: 100 }; + const budgeted = () => sharedReadBudget({ limits: LIMITS, budget: BUDGET, rereads: REREADS }); + /** Whether a read of `url` through `fetch` is taken in whole; one that is not names `past`, the limits it went past. */ + const reads = async (fetch: typeof globalThis.fetch, url: string | Request, past = LIMITS) => { + const response = await fetch(url); + try { + await response.text(); + return true; + } catch { + expect(refusalOf(response)).toMatchObject({ limits: past }); + return false; + } + }; + + it("reads each document under its own limits", async () => { + const fetch = budgeted()(serving("0123456789".repeat(2)).fetch); + expect(await reads(fetch, URL_)).toBe(false); + }); + + it("reads documents within the budget together, through every fetch it made, then refuses any read without asking", async () => { + const budget = budgeted(); + const { fetch: seven } = serving("0123456"); + const first = budget(seven); + const second = budget(seven); + expect(await reads(first, "https://other.example/a.ttl")).toBe(true); + expect(await reads(second, "https://other.example/b.ttl")).toBe(true); + // Within the document's limits, past what the budget leaves: the budget is named. + expect(await reads(first, "https://other.example/c.ttl", BUDGET)).toBe(false); + const asked = vi.fn(seven); + const request = new Request("https://other.example/d.ttl"); + await expect(budget(asked)(request)).rejects.toMatchObject({ + name: "ReadLimitError", + url: request.url, + limits: BUDGET, + }); + await expect(budget(asked)("https://other.example/e.ttl")).rejects.toMatchObject({ + limits: BUDGET, + }); + expect(asked).not.toHaveBeenCalled(); + }); + + it("names the limits a Content-Length says the body is past: the document's own, or else the budget", async () => { + const budget = budgeted(); + const seven = serving("0123456").fetch; + expect(await reads(budget(seven), "https://other.example/a.ttl")).toBe(true); + expect(await reads(budget(seven), "https://other.example/b.ttl")).toBe(true); + const said = (length: string) => serving("0123456", { headers: { "Content-Length": length } }); + await expect(budget(said("11").fetch)("https://other.example/c.ttl")).rejects.toMatchObject({ + limits: LIMITS, + }); + await expect(budget(said("7").fetch)("https://other.example/c.ttl")).rejects.toMatchObject({ + limits: BUDGET, + }); + }); + + it("counts a document read again up to its largest read out of the rereads, and whole once they are spent", async () => { + const fetch = budgeted()(serving("0123456").fetch); + const url = "https://other.example/a.ttl"; + // 7 counted, then 7 and 3 of the rereads, then 4 and 7 counted: 18 of 20. + for (let read = 0; read < 4; read++) expect(await reads(fetch, url)).toBe(true); + expect(await reads(fetch, url, BUDGET)).toBe(false); + }); + + it("holds the documents of many decks, each within its limits, to the budget together", async () => { + // Each deck's document as large as a read of one takes in; together, past the budget. + const limits = { bytes: 1024 * 1024, quads: 400, characters: 1024 * 1024 }; + const budget = { bytes: 4 * 1024 * 1024, quads: 1_000, characters: 4 * 1024 * 1024 }; + const turtle = Array.from({ length: 400 }, (_, at) => `<#c${at}> <#front> "card ${at}" .`); + const fetch = sharedReadBudget({ limits, budget, rereads: budget })(async (input) => { + const response = new Response(turtle.join("\n"), { + headers: { "Content-Type": "text/turtle" }, + }); + Object.defineProperty(response, "url", { value: String(input) }); + return response; + }); + const deck = (at: number) => + getSolidDatasetLinear(`https://other.example/memo/decks/deck-${at}.ttl`, { fetch }); + await expect(deck(0)).resolves.toBeDefined(); + await expect(deck(1)).resolves.toBeDefined(); + await expect(deck(2)).rejects.toMatchObject({ name: "ReadLimitError", limits: budget }); + await expect(deck(3)).rejects.toMatchObject({ name: "ReadLimitError", limits: budget }); + }); + + it("sends a write once the budget is spent, its answer under the limits of one read", async () => { + const budget = budgeted(); + const { fetch: seven } = serving("0123456"); + const fetch = budget(seven); + for (const name of ["a", "b"]) await reads(fetch, `https://other.example/${name}.ttl`); + await reads(fetch, "https://other.example/c.ttl", BUDGET); + await expect(fetch("https://other.example/d.ttl")).rejects.toMatchObject({ limits: BUDGET }); + const asked = vi.fn(seven); + for (const method of ["PUT", "patch", "DELETE"]) { + const response = await budget(asked)("https://other.example/d.ttl", { method }); + expect(limitsOf(response)).toEqual(LIMITS); + expect(await response.text()).toBe("0123456"); + } + const written = await budget(asked)( + new Request("https://other.example/d.ttl", { method: "POST" }), + ); + expect(written.status).toBe(200); + expect(asked).toHaveBeenCalledTimes(4); + }); + + it("keeps a budget for each scope a read's address is in", async () => { + const budget = sharedReadBudget({ + limits: LIMITS, + budget: BUDGET, + rereads: REREADS, + scope: (url) => new URL(url).origin, + }); + const fetch = budget(serving("0123456").fetch); + for (const name of ["a", "b"]) await reads(fetch, `https://other.example/${name}.ttl`); + await reads(fetch, "https://other.example/c.ttl", BUDGET); + await expect(fetch("https://other.example/d.ttl")).rejects.toMatchObject({ limits: BUDGET }); + expect(await reads(fetch, "https://own.example/a.ttl")).toBe(true); + }); + + it("gives an answer with no body as it is, counting nothing", async () => { + const fetch = budgeted()(serving(null).fetch); + for (let read = 0; read < 30; read++) expect((await fetch(URL_)).status).toBe(304); + }); +}); + describe("textChunks", () => { it("gives a text in parts that are the text again, each at most one character past a part's length", () => { const text = "0123456789".repeat(1000); diff --git a/packages/solid/src/client/readLimits.ts b/packages/solid/src/client/readLimits.ts index bde72714c..ee88ea8da 100644 --- a/packages/solid/src/client/readLimits.ts +++ b/packages/solid/src/client/readLimits.ts @@ -31,7 +31,8 @@ export const RELEASE_LIMITS: ReadLimits = { * holds a deck's cards as an instance's largest documents do. An * instance anyone may make can be added by its address or registered by * another app, a catalog may name a deck's documents on any server, and - * the app keeps every triple of what it reads. The user's storages are + * the app keeps every triple of what it reads; all of them together + * are held to FOREIGN_READS_BUDGET too. The user's storages are * read with no limits but on their declarations and nesting * (writeTurtle) and on the length of each term (MAX_TERM_CHARACTERS). */ @@ -122,11 +123,31 @@ export interface ReadsSpent { characters: number; } +/** + * The limits a read went past, `bytes` more taken in, for each tally + * that stands for more than one set of them (sharedReadBudget); those + * it is read under for any other. + */ +const PAST = new WeakMap ReadLimits>(); + /** Nothing taken in yet. */ export function noneSpent(): ReadsSpent { return { bytes: 0, quads: 0, characters: 0 }; } +/** + * The limits to name in the ReadLimitError of a read past `limits`, with + * `bytes` more taken in: the ones the tally names (PAST), or + * `limits`, or NO_SIZE_LIMITS for a read with none. + */ +export function limitsPast( + limits: ReadLimits | undefined, + spent: ReadsSpent | undefined, + bytes = 0, +): ReadLimits { + return (spent === undefined ? undefined : PAST.get(spent)?.(bytes)) ?? limits ?? NO_SIZE_LIMITS; +} + /** Each response limitedFetch gave: the limits it is read under, what is spent of them, and how it went past them, once it did. */ const READS = new WeakMap< Response, @@ -155,9 +176,10 @@ export function limitedFetch( if (response.body === null || NULL_BODY_STATUSES.has(response.status)) return response; const { url } = response; const tally = spent ?? noneSpent(); - if (tally.bytes + Number(response.headers.get("Content-Length")) > limits.bytes) { + const said = Number(response.headers.get("Content-Length")); + if (tally.bytes + said > limits.bytes) { await response.body.cancel(); - throw new ReadLimitError(url, limits); + throw new ReadLimitError(url, limitsPast(limits, tally, said)); } const read: { limits: ReadLimits; spent: ReadsSpent; refused?: ReadLimitError } = { limits, @@ -168,7 +190,7 @@ export function limitedFetch( transform(chunk, controller) { tally.bytes += chunk.byteLength; if (tally.bytes <= limits.bytes) return controller.enqueue(chunk); - read.refused = new ReadLimitError(url, limits); + read.refused = new ReadLimitError(url, limitsPast(limits, tally)); controller.error(read.refused); }, }), @@ -185,6 +207,120 @@ export function limitedFetch( }; } +/** + * The most the reads of documents outside the user's storages take in + * together in a page session (foreignReads), on top of FOREIGN_LIMITS on + * each: four times what one release may hold, so many times the largest + * release in the library. A catalog may name a thousand decks, each with + * two documents as large as a release, and the app holds every triple of + * what it reads while it reads it. Once read, a document counts once, by + * its largest read, while reads of it again come out of FOREIGN_REREADS. + */ +export const FOREIGN_READS_BUDGET: ReadLimits = { + bytes: 4 * RELEASE_LIMITS.bytes, + quads: 4 * RELEASE_LIMITS.quads, + characters: 4 * RELEASE_LIMITS.characters, +}; + +/** + * How much reads of documents already read may take in a page session + * without counting in FOREIGN_READS_BUDGET, up to each one's largest + * read: twice that budget. Past it they count whole, or many decks + * naming one document could read it again and again without limit. + */ +export const FOREIGN_REREADS: ReadLimits = { + bytes: 2 * FOREIGN_READS_BUDGET.bytes, + quads: 2 * FOREIGN_READS_BUDGET.quads, + characters: 2 * FOREIGN_READS_BUDGET.characters, +}; + +const SPENT_KEYS = ["bytes", "quads", "characters"] as const; + +/** The methods that read, which a read budget counts; any other request is sent as it is asked. */ +const BUDGETED_METHODS = new Set(["GET", "HEAD"]); + +/** What one budget of sharedReadBudget has spent so far. */ +interface BudgetSpent { + /** What the reads have taken in of the budget. */ + session: ReadsSpent; + /** What is left of `rereads`. */ + free: ReadsSpent; + /** The largest read of each document so far. */ + largest: Map; +} + +/** + * Reads under one budget together, as the reads of releases from a link + * are (solidDeckLibrary.ts): what it gives makes a fetch whose answers + * are each read under `limits` (limitedFetch), and its reads (GET and + * HEAD), through every fetch it made, within `budget` too. With `scope`, + * each scope it names a read's address by has a budget of its own. A + * document counts in the budget by its largest read; what a read of it + * again takes within that comes out of `rereads` while that lasts. Once + * the budget is spent, a read under way fails as it goes on, and any + * after it fails at once, without asking the server, with a + * ReadLimitError naming the budget, until the page is loaded again. A + * request that writes is sent whatever is spent, its answer under + * `limits` alone. + */ +export function sharedReadBudget({ + limits, + budget, + rereads, + scope = () => "", +}: { + limits: ReadLimits; + budget: ReadLimits; + rereads: ReadLimits; + /** The budget a read of an address counts in: one for all of them unless given. */ + scope?: (url: string) => string; +}): (fetch: typeof globalThis.fetch) => typeof globalThis.fetch { + const budgets = new Map(); + const budgetOf = (url: string): BudgetSpent => { + const key = scope(url); + const found = budgets.get(key); + if (found !== undefined) return found; + const made: BudgetSpent = { session: noneSpent(), free: { ...rereads }, largest: new Map() }; + budgets.set(key, made); + return made; + }; + return (fetch) => async (input, init) => { + const url = input instanceof Request ? input.url : String(input); + const method = ( + init?.method ?? (input instanceof Request ? input.method : "GET") + ).toUpperCase(); + if (!BUDGETED_METHODS.has(method)) return limitedFetch(fetch, limits)(input, init); + const { session, free, largest } = budgetOf(url); + if (SPENT_KEYS.some((key) => session[key] > budget[key])) throw new ReadLimitError(url, budget); + const most = largest.get(url) ?? noneSpent(); + largest.set(url, most); + const document = noneSpent(); + // limitedFetch compares one tally with `limits`: this read's own, or what the budget leaves of them, whichever is + // more, so it is past them once either is past its limit, and names the budget when it is the one past. + const over = (tally: ReadsSpent, bounds: ReadLimits, bytes: number) => + SPENT_KEYS.some((key) => tally[key] + (key === "bytes" ? bytes : 0) > bounds[key]); + const spent = {} as ReadsSpent; + PAST.set(spent, (bytes) => + over(session, budget, bytes) && !over(document, limits, bytes) ? budget : limits, + ); + for (const key of SPENT_KEYS) { + Object.defineProperty(spent, key, { + get: () => Math.max(document[key], limits[key] - budget[key] + session[key]), + set: (value: number) => { + const taken = value - spent[key]; + const within = Math.max(0, Math.min(most[key], document[key] + taken) - document[key]); + const reread = Math.min(within, free[key]); + free[key] -= reread; + document[key] += taken; + most[key] = Math.max(most[key], document[key]); + session[key] += taken - reread; + }, + }); + } + return limitedFetch(fetch, limits, spent)(input, init); + }; +} + /** The limits a response is read under: limitedFetch's, or none. */ export function limitsOf(response: Response): ReadLimits | undefined { return READS.get(response)?.limits; diff --git a/packages/solid/src/deckTree.test.ts b/packages/solid/src/deckTree.test.ts index 78327f041..8096637d6 100644 --- a/packages/solid/src/deckTree.test.ts +++ b/packages/solid/src/deckTree.test.ts @@ -3,7 +3,7 @@ // SPDX-License-Identifier: MIT import { describe, expect, it, vi } from "vitest"; -import type { Deck } from "@solid-memo/domain/deck"; +import { type Deck, MAX_CATALOG_LINKS } from "@solid-memo/domain/deck"; import { decksOf, nodeId, type DeckTree, type TreeNode } from "@solid-memo/domain/deckTree"; import { SHAPE_SOURCES, shapesFetch } from "@solid-memo/vocab/tooling/testing/shapeSources"; import { createShaclShapeValidator } from "./shaclShapeValidator"; @@ -75,6 +75,24 @@ describe("the deck tree in the pod", () => { expect(shape(tree.children)).toEqual(["b", { "group-1": ["c", "a"] }]); }); + it("refuses a catalog with more decks than MAX_CATALOG_LINKS, as catalogTooManyLinks, to read or edit", async () => { + const decks = (count: number) => + Array.from({ length: count }, (_, i) => deckTtl(`d${i}`)).join(""); + const most = fakePod(); + await most.put(CATALOG, PREFIXES + decks(MAX_CATALOG_LINKS)); + expect(decksOf((await repositoryOn(most.fetch).readDeckTree(INSTANCE)).children)).toHaveLength( + MAX_CATALOG_LINKS, + ); + const past = fakePod(); + await past.put(CATALOG, PREFIXES + decks(MAX_CATALOG_LINKS + 1)); + const tooMany = { code: "catalogTooManyLinks", vars: { limit: MAX_CATALOG_LINKS } }; + await expect(repositoryOn(past.fetch).readDeckTree(INSTANCE)).rejects.toMatchObject(tooMany); + await expect( + repositoryOn(past.fetch).editDeckTree(INSTANCE, { kind: "removeGroup", group: group.url }), + ).rejects.toMatchObject(tooMany); + expect(writes(past)).toEqual([]); + }); + it("is edited in one checked write of the groups and the catalog, If-Match the read, and reads back as written", async () => { const pod = await podWith(); const checkWrite = vi.fn(async () => undefined); diff --git a/packages/solid/src/solidDeckRepository.ts b/packages/solid/src/solidDeckRepository.ts index 56d8b49e8..c63ba88c9 100644 --- a/packages/solid/src/solidDeckRepository.ts +++ b/packages/solid/src/solidDeckRepository.ts @@ -44,7 +44,12 @@ import { documentUrlOf, fragmentIdOf } from "@solid-memo/domain/subjectUrl"; import { isDeckOutdated, isOutdated, upgradeCard, upgradeDeck } from "@solid-memo/domain/migration"; import { ABSENT_VERSION } from "@solid-memo/domain/studyDigest"; import { CARD_V7 } from "@solid-memo/vocab/descriptors.generated"; -import { applyDeckTreeEdit, buildTree, treeChanges } from "@solid-memo/domain/deckTree"; +import { + applyDeckTreeEdit, + buildTree, + type StoredLayout, + treeChanges, +} from "@solid-memo/domain/deckTree"; import { getSolidDatasetOrNull, PreconditionFailedError, @@ -121,6 +126,17 @@ export interface SolidDeckRepositoryDeps { */ const TREE_ATTEMPTS = 3; +/** + * The arrangement the catalog states (toStoredLayout), or + * catalogTooManyLinks past MAX_CATALOG_LINKS decks, as listDecks gives: + * each deck's documents are read by whoever shows the tree. + */ +function cappedLayout(dataset: Dataset, catalogUrl: string): StoredLayout { + const layout = toStoredLayout(dataset, catalogUrl); + withinCatalogLimit(layout.decks); + return layout; +} + export function createSolidDeckRepository({ fetch, publicFetch = fetch, @@ -263,14 +279,14 @@ export function createSolidDeckRepository({ async readDeckTree(instanceUrl) { const catalogUrl = catalogUrlOf(instanceUrl); const dataset = await getSolidDatasetOrNull(catalogUrl, fetch); - return buildTree(toStoredLayout(dataset ?? createSolidDataset(), catalogUrl)); + return buildTree(cappedLayout(dataset ?? createSolidDataset(), catalogUrl)); }, async editDeckTree(instanceUrl, edit) { const catalogUrl = catalogUrlOf(instanceUrl); for (let attempt = 1; ; attempt++) { const dataset = (await getSolidDatasetOrNull(catalogUrl, fetch)) ?? createSolidDataset(); - const stored = toStoredLayout(dataset, catalogUrl); + const stored = cappedLayout(dataset, catalogUrl); const before = buildTree(stored); const after = applyDeckTreeEdit(before, edit); if (after === before) return after;