Before going public: an AI tool gets read and write unless it names delete, uploads have limits, and the review's other fixes

.gitignore

@@ -15,3 +15,9 @@ cdk.out/
 apps/ios/ios/
 apps/ios/.expo/
 .vercel
+
+# Claude Code: worktrees and personal settings
+.claude/worktrees/
+.claude/settings.local.json
+*.tsbuildinfo
+coverage/

apps/web/app/api/v1/digest/send/route.ts

@@ -1,5 +1,11 @@
-import { json, route } from "@/lib/api";
+import { hitRateLimit } from "@pluck/db";
+import { ApiError, json, route } from "@/lib/api";
 import { sendDigest } from "@/lib/library";
 
-// "Send me one now" from Settings: the digest as it would look today, whatever the day.
-export const POST = route({ kind: "write" }, async ({ caller }) => json({ sent: await sendDigest(caller.userId, { force: true }) }));
+// "Send me one now" from Settings: the digest as it would look today, whatever the day. Five a day: each is an email
+// we pay for, and a key could otherwise send thousands.
+export const POST = route({ kind: "write" }, async ({ caller }) => {
+  const r = await hitRateLimit(`digest-send:${caller.userId}`, 5, 24 * 60 * 60);
+  if (!r.ok) throw new ApiError("rate_limited", "That's five today. Try again tomorrow.", 429, { retryAfter: r.retryAfter });
+  return json({ sent: await sendDigest(caller.userId, { force: true }) });
+});

apps/web/app/api/v1/items/[id]/cover/route.test.ts

@@ -0,0 +1,24 @@
+import { twoUsers } from "@pluck/db/testing";
+import { expect, test } from "vitest";
+import { newKey } from "@/lib/library";
+import { PUT } from "./route";
+
+test("a cover over 2 MB is refused, by its content-length or, without one, while it is read", async () => {
+  const { a } = await twoUsers();
+  const { key } = await newKey(a, "extension");
+  const id = "0".repeat(24); // refused before the item is looked up
+  const put = (body: BodyInit, headers: Record<string, string> = {}) =>
+    PUT(
+      new Request(`http://x/api/v1/items/${id}/cover`, {
+        method: "PUT", body, headers: { authorization: `Bearer ${key}`, "content-type": "image/png", ...headers }, duplex: "half",
+      } as RequestInit), // why: `duplex` (needed for a stream body) is not in the DOM RequestInit type yet
+      { params: Promise.resolve({ id }) },
+    );
+
+  const refused = async (res: Response) => [res.status, ((await res.json()) as { error: { message: string } }).error.message];
+
+  expect(await refused(await put("tiny", { "content-length": "2000001" }))).toEqual([400, "That image is over 2 MB."]);
+  const chunk = new Uint8Array(500_000);
+  const endless = new ReadableStream<Uint8Array>({ pull: (c) => c.enqueue(chunk) }); // no content-length
+  expect(await refused(await put(endless))).toEqual([400, "That image is over 2 MB."]);
+});

apps/web/app/api/v1/items/[id]/cover/route.ts

@@ -11,12 +11,31 @@ export const GET = route<{ id: string }>({ kind: "read" }, async ({ req, caller,
 
 const COVER_TYPES = new Set(["image/jpeg", "image/png", "image/webp"]);
 const MAX_COVER = 2_000_000;
+const tooBig = () => new ApiError("bad_request", "That image is over 2 MB.", 400);
+
+// Stops reading at the limit: content-length can be missing or wrong.
+async function readUpTo(req: Request, max: number): Promise<Uint8Array | null> {
+  const chunks: Uint8Array[] = [];
+  let size = 0;
+  const reader = req.body?.getReader();
+  while (reader) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    if ((size += value.byteLength) > max) {
+      await reader.cancel();
+      return null;
+    }
+    chunks.push(value);
+  }
+  return Buffer.concat(chunks);
+}
 
 // The extension sends a screenshot of the tab when the page has no image of its own.
 export const PUT = route<{ id: string }>({ kind: "write" }, async ({ req, caller, params }) => {
   const type = req.headers.get("content-type")?.split(";")[0]?.trim() ?? "";
   if (!COVER_TYPES.has(type)) throw new ApiError("bad_request", "Send a JPEG, PNG or WebP image.", 400);
-  const body = new Uint8Array(await req.arrayBuffer());
-  if (body.byteLength > MAX_COVER) throw new ApiError("bad_request", "That image is over 2 MB.", 400);
+  if (Number(req.headers.get("content-length")) > MAX_COVER) throw tooBig();
+  const body = await readUpTo(req, MAX_COVER);
+  if (!body) throw tooBig();
   return json({ item: await setCoverFromApp(caller.userId, params.id, body, type) });
 });

apps/web/app/signin/page.tsx

@@ -4,14 +4,15 @@ import { cookies } from "next/headers";
 import { redirect } from "next/navigation";
 import { TRY_COOKIE } from "@/lib/api";
 import { currentUser } from "@/lib/auth";
+import { samePath } from "@/lib/same-site";
 import { SignInForm } from "./SignInForm";
 
 export async function generateMetadata(): Promise<Metadata> {
   return { title: (await getTranslations("app.meta"))("signin") };
 }
 
 // Only same-site paths, so a crafted link cannot bounce people elsewhere after sign-in.
-const safeNext = (n?: string) => (n && n.startsWith("/") && !n.startsWith("//") ? n : "/inbox");
+const safeNext = (n?: string) => samePath(n, "/inbox");
 
 // Sent here by the OAuth provider (an AI tool connecting): after the email link signs you in, carry on
 // with the same authorization request. The signing fields only matter to this page, so they are dropped.

apps/web/instrumentation.ts

@@ -9,6 +9,10 @@ export function register() {
   if (secret === "dev-only-secret-change-me-in-production-0123456789" || secret.length < 32)
     throw new Error("BETTER_AUTH_SECRET is the example one or shorter than 32 characters: set 32+ random bytes (openssl rand -base64 32).");
   if (process.env.CRON_SECRET === "dev-cron-secret") throw new Error("CRON_SECRET is the example one: set a random value (openssl rand -hex 32).");
+  // The two switches for tests (docs/self-host.md): on a public server one opens the private network to page fetches,
+  // the other lets anyone guess sign-in codes without limit.
+  if (process.env.ALLOW_PRIVATE_FETCH === "1") throw new Error("ALLOW_PRIVATE_FETCH is for local development and tests only: unset it.");
+  if (process.env.AUTH_RATE_LIMIT === "off") throw new Error("AUTH_RATE_LIMIT=off is for local development and tests only: unset it.");
 }
 
 // Errors thrown while rendering a page or running a route that did not catch them (route handlers in lib/api.ts

apps/web/lib/auth-guard.test.ts

@@ -31,6 +31,19 @@ test("only our own apps can ask for the account scope", async () => {
   expect((await register("Pluck for iPhone", "io.getpluck.app:/oauth", [`${BASE}/api/v1`], "offline_access library:read library:write account")).status).toBe(201);
 });
 
+test("an AI tool that names no scope gets read and write, never delete or the account", async () => {
+  const { client_id } = (await (await register("Example AI", "https://ai.example.test/cb")).json()) as { client_id: string };
+  const authorize = (scope?: string) =>
+    auth.handler(new Request(`${BASE}/api/auth/oauth2/authorize?${new URLSearchParams({
+      response_type: "code", client_id, redirect_uri: "https://ai.example.test/cb", state: "s", code_challenge_method: "S256",
+      code_challenge: "E9Melhoa2OwvFrEMTJguCHaoeK1t8URWbuGJSstw-cM", resource: `${BASE}/mcp`, ...(scope ? { scope } : {}),
+    })}`, { headers: { "x-forwarded-for": `198.51.100.${++ip}` } }));
+  const scopeOf = (res: Response) => new URL(res.headers.get("location") ?? "", BASE).searchParams.get("scope");
+  expect(scopeOf(await authorize())).toBe("offline_access library:read library:write"); // on to sign-in, narrowed
+  expect(scopeOf(await authorize("library:read library:delete"))).toBe("library:read library:delete"); // named, so the consent screen shows it
+  expect((await authorize("library:read account")).status).toBe(400);
+});
+
 test("sign-in links are limited per address, whatever network asks: one a minute, six an hour", async () => {
   const email = `limit-${Date.now()}@example.test`;
   vi.useFakeTimers({ toFake: ["Date"] });

apps/web/lib/auth.ts

@@ -28,6 +28,8 @@ export const API_RESOURCE = `${ORIGIN}/api/v1`;
 export const LIBRARY_SCOPES = ["library:read", "library:write", "library:delete"] as const;
 // Account settings a browser session has (connected AI tools, export, deleting it all), for our own apps only.
 export const ACCOUNT_SCOPE = "account";
+// What an AI tool gets when it names no scope, and all the connector advertises: never delete (CLAUDE.md) or the account.
+export const AI_TOOL_SCOPES = ["offline_access", "library:read", "library:write"];
 // Where the extension and the iPhone app take their sign-in back. Codes sent here never leave our apps.
 export const APP_REDIRECT = "io.getpluck.app:/oauth";
 export const FIRST_PARTY_REDIRECTS = [`${ORIGIN}/extension/connected`, APP_REDIRECT];
@@ -70,6 +72,15 @@ const guard = createAuthMiddleware(async (ctx) => {
     if ((wantsApi || wantsAccount || /pluck/i.test(String(body.client_name ?? ""))) && !firstParty(body.redirect_uris))
       throw new APIError("BAD_REQUEST", { error: "invalid_client_metadata", error_description: "Only Pluck's own apps can register for the Pluck API or use its name." });
   }
+  // Better Auth stores every registered client with every scope Pluck has, whatever it asked for, and gives an authorize
+  // with no scope all of them. So the line is drawn here: an AI tool, or any client not ours, gets read and write unless
+  // it names delete itself, and never the account.
+  if (ctx.path === "/oauth2/authorize" && !firstParty([redirectOf(ctx)])) {
+    const asked = String(ctx.query?.scope ?? "").split(" ").filter(Boolean);
+    if (asked.includes(ACCOUNT_SCOPE))
+      throw new APIError("BAD_REQUEST", { error: "invalid_scope", error_description: "The account scope is for Pluck's own apps." });
+    if (!asked.length) return { context: { query: { ...ctx.query, scope: AI_TOOL_SCOPES.join(" ") } } };
+  }
   const email = String((ctx.body as { email?: unknown } | undefined)?.email ?? "").trim().toLowerCase();
   // The review address gets no link: the form asks for its password instead.
   if (ctx.path === "/sign-in/magic-link" && email === REVIEW?.email) return { password: true };
@@ -142,6 +153,8 @@ export const auth = betterAuth({
       consentPage: "/oauth/consent",
       resource: MCP_RESOURCE,
       scopes: ["openid", "profile", "email", "offline_access", ...LIBRARY_SCOPES, ACCOUNT_SCOPE],
+      // Clients ask for what is advertised (the MCP spec says scopes_supported), so delete and the account stay unlisted.
+      advertisedMetadata: { scopes_supported: ["openid", "profile", "email", ...AI_TOOL_SCOPES] },
       accessTokenExpiresIn: 60 * 60,
       refreshTokenExpiresIn: 90 * DAY,
       // CIMD first; dynamic registration as the fallback for clients that do not speak CIMD yet.

apps/web/lib/capture.ts

@@ -24,10 +24,13 @@ const toResult = (x: Extracted): ExtractionResult => ({
   canonicalUrl: x.canonicalUrl, lang: x.lang, wordCount: x.wordCount, markdown: x.markdown, extractor: x.extractor,
 });
 
+// Pictures only: an SVG can carry script, and a self-host may serve its blob store from its own domain.
+const picture = (contentType: string) => /^image\/(jpeg|png|webp|gif|avif)\b/i.test(contentType);
+
 // Raw page, file and cover to the blob store. Covers go through the same guarded fetch as pages.
 export async function storeBlobs(userId: string, itemId: string, x: Pick<Extracted, "kind" | "raw" | "coverUrl">): Promise<void> {
   const update: { coverKey?: string; htmlKey?: string } = {};
-  if (x.raw) {
+  if (x.raw && (x.kind !== "image" || picture(x.raw.contentType))) {
     const key = x.kind === "pdf" ? keys.file(userId, itemId, "pdf") : x.kind === "image" ? keys.cover(userId, itemId) : keys.html(userId, itemId);
     await putObject(key, x.raw.body, x.raw.contentType);
     if (x.kind === "image") update.coverKey = key;
@@ -36,7 +39,7 @@ export async function storeBlobs(userId: string, itemId: string, x: Pick<Extract
   if (x.coverUrl && !update.coverKey) {
     try {
       const img = await guardedFetch(x.coverUrl, { maxBytes: 5_000_000, accept: "image/*", signal: AbortSignal.timeout(8000) });
-      if (img.contentType.startsWith("image/")) {
+      if (picture(img.contentType)) {
         await putObject(keys.cover(userId, itemId), img.body, img.contentType);
         update.coverKey = keys.cover(userId, itemId);
       }

apps/web/lib/library/account.ts

@@ -1,10 +1,10 @@
-import { deletePrefix, getObject, putObject, signedPutUrl, signedUrl } from "@pluck/blob";
+import { TooLarge, deleteObject, deletePrefix, getObject, putObject, signedPutUrl, signedUrl } from "@pluck/blob";
 import {
   aiConnections, applyExtraction, claimOnce, countItems, createApiKey, disconnectAi, claimImportStart, createImport, deleteAccountData, enqueue, exportData, finishExport, getImport, getMe, importLink,
   latestExport, recountTags, listApiKeys, revokeApiKey, startExport, updateImport,
 } from "@pluck/db";
 import { extractFromHtml, parseImport, type ImportSource } from "@pluck/extract";
-import { normalizeUrl, type AiConnection, type ApiKey, type ImportStatus } from "@pluck/shared-schemas";
+import { MAX_IMPORT_BYTES, normalizeUrl, type AiConnection, type ApiKey, type ImportStatus } from "@pluck/shared-schemas";
 import { strToU8, zipSync } from "fflate";
 import { sendEmail } from "../email";
 import { exportEmail, welcomeEmail } from "../emails";
@@ -82,9 +82,14 @@ export async function runImport(userId: string, importId: string): Promise<void>
   if (!record || record.state !== "running") return;
   let entries;
   try {
-    entries = parseImport(record.source, new Uint8Array(await getObject(record.blobKey)));
+    entries = parseImport(record.source, new Uint8Array(await getObject(record.blobKey, { maxBytes: MAX_IMPORT_BYTES })));
   } catch (e) {
-    await updateImport(importId, { state: "failed", error: "That file could not be read. Check it is the export from the right service." });
+    const tooLarge = e instanceof TooLarge;
+    if (tooLarge) await deleteObject(record.blobKey);
+    await updateImport(importId, {
+      state: "failed",
+      error: tooLarge ? "That file is over 500 MB." : "That file could not be read. Check it is the export from the right service.",
+    });
     log("warn", "import unreadable", { userId, importId, error: String(e) });
     return;
   }

apps/web/lib/library/email-in.ts

@@ -1,4 +1,4 @@
-import { deleteObject, getObject, keys, listKeys, moveObject } from "@pluck/blob";
+import { TooLarge, deleteObject, getObject, keys, listKeys, moveObject } from "@pluck/blob";
 import { aliasOf, claimOnce, createEmailItem, enqueue, hitRateLimit, regenerateAlias, setRawEmailKey, userForAlias } from "@pluck/db";
 import { parseEmail } from "@pluck/extract";
 import { log } from "../log";
@@ -31,13 +31,19 @@ export async function pollInbound(): Promise<number> {
 }
 
 export async function ingestInbound(key: string): Promise<"created" | "duplicate" | "dropped"> {
-  const raw = await getObject(key);
   const drop = async (why: string) => {
     log("warn", "email-in dropped", { key, why });
     await deleteObject(key);
     return "dropped" as const;
   };
-  if (raw.length > MAX_BYTES) return drop("too large");
+  // Refused before it is read: SES itself accepts messages up to 40 MB.
+  let raw: Buffer;
+  try {
+    raw = await getObject(key, { maxBytes: MAX_BYTES });
+  } catch (e) {
+    if (e instanceof TooLarge) return drop("too large");
+    throw e;
+  }
   const mail = await parseEmail(raw).catch(() => null);
   if (!mail) return drop("unreadable"); // retrying will not make it parse
   if (mail.spam) return drop("spam or virus verdict");

apps/web/lib/library/files.ts

@@ -1,7 +1,7 @@
-import { getObject, keys, signedPutUrl } from "@pluck/blob";
-import { applyExtraction, createFileItem, enqueue, getItem, setBlobKeys } from "@pluck/db";
+import { TooLarge, deleteObject, getObject, keys, signedPutUrl } from "@pluck/blob";
+import { applyExtraction, createFileItem, enqueue, getItem, recordExtractionFailure, setBlobKeys } from "@pluck/db";
 import { extractFile } from "@pluck/extract";
-import { FILE_TYPES, type CreateFileItemBody, type Item } from "@pluck/shared-schemas";
+import { FILE_TYPES, MAX_FILE_BYTES, type CreateFileItemBody, type Item } from "@pluck/shared-schemas";
 
 const keyFor = (userId: string, itemId: string, contentType: CreateFileItemBody["contentType"]) =>
   contentType === "application/pdf" ? keys.file(userId, itemId, "pdf") : keys.file(userId, itemId, FILE_TYPES[contentType]);
@@ -21,7 +21,15 @@ export async function finishFile(userId: string, itemId: string, key: string, co
   const item = await getItem(userId, itemId);
   if (!item || item.extraction.state !== "pending") return;
   // shortcut: an upload that never arrives leaves the item pending once the job gives up (five tries, about 15 minutes); mark it failed if that shows up.
-  const body = await getObject(key); // missing yet: throws, and the job tries again later
+  let body: Buffer;
+  try {
+    body = await getObject(key, { maxBytes: MAX_FILE_BYTES }); // missing yet: throws, and the job tries again later
+  } catch (e) {
+    if (!(e instanceof TooLarge)) throw e;
+    await recordExtractionFailure(userId, itemId, "That file is over 50 MB.", true);
+    await deleteObject(key);
+    return;
+  }
   const x = await extractFile({ body, contentType, fileName: name });
   if (x.kind === "image") await setBlobKeys(userId, itemId, { coverKey: key });
   await applyExtraction(userId, itemId, {

apps/web/lib/same-site.test.ts

@@ -0,0 +1,8 @@
+import { expect, test } from "vitest";
+import { samePath } from "./same-site";
+
+test("a path on this site goes through; anything a browser reads as another site does not", () => {
+  for (const ok of ["/inbox", "/items/abc?x=1#top", "/lists/a/b", "/try/claim"]) expect(samePath(ok, "/inbox")).toBe(ok);
+  for (const bad of [undefined, "", "inbox", "//evil.com", "/\\evil.com", "/\t/evil.com", "/\\/evil.com", "https://evil.com", "/a\u0000b", "/a\nb", "\\\\evil.com"])
+    expect(samePath(bad, "/inbox")).toBe("/inbox");
+});

apps/web/lib/same-site.ts

@@ -0,0 +1,8 @@
+// Where a link may send someone after signing in: a path on this site and nowhere else. "/\evil.com" and "/\t/evil.com"
+// are other sites to a browser (a backslash is a slash, a tab is dropped), so any backslash or control character is
+// refused, and what is left must resolve to this origin.
+export function samePath(next: string | undefined, fallback: string): string {
+  if (!next || !/^\/(?![/\\])/.test(next) || /[\\\u0000-\u001f\u007f]/.test(next)) return fallback;
+  const base = "https://pluck.invalid";
+  return new URL(next, base).origin === base ? next : fallback;
+}

packages/blob/src/index.test.ts

@@ -1,5 +1,5 @@
 import { expect, test } from "vitest";
-import { deletePrefix, getObject, putObject, signedUrl } from "./index.ts";
+import { TooLarge, deletePrefix, getObject, putObject, signedUrl } from "./index.ts";
 
 test("put, read back, sign and delete against MinIO", async () => {
   const prefix = `test/${Date.now()}/`;
@@ -9,3 +9,11 @@ test("put, read back, sign and delete against MinIO", async () => {
   expect(await res.text()).toBe("hello");
   expect(await deletePrefix(prefix)).toBe(1);
 });
+
+test("getObject refuses an object over maxBytes", async () => {
+  const prefix = `test/${Date.now()}-max/`;
+  await putObject(`${prefix}a.txt`, "hello", "text/plain");
+  expect((await getObject(`${prefix}a.txt`, { maxBytes: 5 })).toString()).toBe("hello");
+  await expect(getObject(`${prefix}a.txt`, { maxBytes: 4 })).rejects.toBeInstanceOf(TooLarge);
+  await deletePrefix(prefix);
+});

packages/blob/src/index.ts

@@ -33,8 +33,15 @@ export async function putObject(key: string, body: Buffer | string, contentType:
   await s3().send(new PutObjectCommand({ Bucket: bucket(), Key: key, Body: body, ContentType: contentType }));
 }
 
-export async function getObject(key: string): Promise<Buffer> {
+export class TooLarge extends Error {}
+
+// `maxBytes` refuses an object before its body is read: a signed PUT URL takes whatever size is sent to it.
+export async function getObject(key: string, { maxBytes = Infinity } = {}): Promise<Buffer> {
   const res = await s3().send(new GetObjectCommand({ Bucket: bucket(), Key: key }));
+  if ((res.ContentLength ?? 0) > maxBytes) {
+    await res.Body?.transformToWebStream().cancel();
+    throw new TooLarge(`${key} is ${res.ContentLength} bytes, over ${maxBytes}`);
+  }
   return Buffer.from(await res.Body!.transformToByteArray());
 }
 

packages/extract/src/fetch.test.ts

@@ -5,7 +5,7 @@ afterEach(() => {
   delete process.env.ALLOW_PRIVATE_FETCH;
 });
 
-test.each(["127.0.0.1", "10.1.2.3", "172.20.0.1", "192.168.1.1", "169.254.169.254", "::1", "fd00::1", "::ffff:127.0.0.1"])("%s is private", (ip) => {
+test.each(["127.0.0.1", "10.1.2.3", "172.20.0.1", "192.168.1.1", "169.254.169.254", "192.0.0.8", "240.1.2.3", "::1", "fd00::1", "::ffff:127.0.0.1", "64:ff9b::7f00:1", "2002:7f00:1::"])("%s is private", (ip) => {
   expect(isPrivateAddress(ip)).toBe(true);
 });