From 5b2f8cb072650ce40f22a4883e9d0cbac9019871 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 7 Oct 2026 15:57:18 -0700 Subject: [PATCH 1/2] fix(docs): publish full LLM text as a static asset --- apps/docs/.gitignore | 4 +++ apps/docs/app/llms-full.txt/route.ts | 33 ----------------- apps/docs/next.config.ts | 11 ++++++ apps/docs/package.json | 7 ++-- apps/docs/scripts/generate-llms-full.ts | 47 +++++++++++++++++++++++++ apps/docs/turbo.json | 8 +++++ 6 files changed, 74 insertions(+), 36 deletions(-) delete mode 100644 apps/docs/app/llms-full.txt/route.ts create mode 100644 apps/docs/scripts/generate-llms-full.ts create mode 100644 apps/docs/turbo.json diff --git a/apps/docs/.gitignore b/apps/docs/.gitignore index d4dabf71e70..7770540b041 100644 --- a/apps/docs/.gitignore +++ b/apps/docs/.gitignore @@ -41,3 +41,7 @@ next-env.d.ts # fumadocs generates .source dirs anywhere a source.config sits **/.source/ + +# Build-generated documentation asset +/public/llms-full.txt +/public/.llms-full-*/ diff --git a/apps/docs/app/llms-full.txt/route.ts b/apps/docs/app/llms-full.txt/route.ts deleted file mode 100644 index 51d3da16431..00000000000 --- a/apps/docs/app/llms-full.txt/route.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { getLLMText } from '@/lib/llms' -import { source } from '@/lib/source' - -export const dynamic = 'force-static' - -export async function GET() { - const pages = source.getPages().filter((page) => Boolean(page?.data && page.url)) - const encoder = new TextEncoder() - let nextPage = 0 - let hasContent = false - let cancelled = false - const body = new ReadableStream({ - async pull(controller) { - while (nextPage < pages.length) { - const page = pages[nextPage] - nextPage += 1 - const text = await getLLMText(page) - if (cancelled) return - if (!text) continue - controller.enqueue(encoder.encode(`${hasContent ? '\n\n---\n\n' : ''}${text}`)) - hasContent = true - return - } - controller.close() - }, - cancel() { - cancelled = true - }, - }) - return new Response(body, { - headers: { 'Content-Type': 'text/plain; charset=utf-8' }, - }) -} diff --git a/apps/docs/next.config.ts b/apps/docs/next.config.ts index da86d15eec4..1d26662f05b 100644 --- a/apps/docs/next.config.ts +++ b/apps/docs/next.config.ts @@ -16,6 +16,17 @@ const config: NextConfig = { webpackMemoryOptimizations: true, webpackBuildWorker: true, }, + async headers() { + return [ + { + source: '/llms-full.txt', + headers: [ + { key: 'Content-Type', value: 'text/plain; charset=utf-8' }, + { key: 'Cache-Control', value: 'public, max-age=0, s-maxage=86400, must-revalidate' }, + ], + }, + ] + }, async redirects() { return DOCS_REDIRECTS }, diff --git a/apps/docs/package.json b/apps/docs/package.json index 032dce986c5..4e8f5f0d6b9 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -4,10 +4,10 @@ "private": true, "license": "Apache-2.0", "scripts": { - "dev": "bun run dev:cache:cap && next dev --port 3001", + "dev": "bun run dev:cache:cap && fumadocs-mdx && bun run llms-full:generate && next dev --port 3001", "dev:cache:cap": "bun run ../../scripts/prune-turbopack-cache.ts", "dev:clean": "rm -rf .next/dev/cache", - "build": "fumadocs-mdx && NODE_OPTIONS='--max-old-space-size=8192' next build", + "build": "fumadocs-mdx && bun run llms-full:generate && NODE_OPTIONS='--max-old-space-size=8192' next build", "start": "next start", "postinstall": "fumadocs-mdx", "test": "vitest run", @@ -15,7 +15,8 @@ "lint": "biome check --write --unsafe .", "lint:check": "biome check .", "format": "biome format --write .", - "format:check": "biome format ." + "format:check": "biome format .", + "llms-full:generate": "bun scripts/generate-llms-full.ts" }, "dependencies": { "@sim/db": "workspace:*", diff --git a/apps/docs/scripts/generate-llms-full.ts b/apps/docs/scripts/generate-llms-full.ts new file mode 100644 index 00000000000..88e8ad32875 --- /dev/null +++ b/apps/docs/scripts/generate-llms-full.ts @@ -0,0 +1,47 @@ +import { createWriteStream } from 'node:fs' +import { mkdtemp, rename, rm } from 'node:fs/promises' +import path from 'node:path' +import { Readable } from 'node:stream' +import { pipeline } from 'node:stream/promises' +import { createLogger } from '@sim/logger' +import { compareStrings } from '@sim/utils/string' +import { plugin } from 'bun' +import { createMdxPlugin } from 'fumadocs-mdx/bun' + +const MAX_LLM_TEXT_BYTES = 64 * 1024 * 1024 +const logger = createLogger('DocsLlmsFullAsset') + +await plugin(createMdxPlugin({ disableMetaFile: true })) + +const { source } = await import('@/lib/source') +const { getLLMText } = await import('@/lib/llms') +const publicDirectory = path.resolve('public') +const temporaryDirectory = await mkdtemp(path.join(publicDirectory, '.llms-full-')) +const temporaryFile = path.join(temporaryDirectory, 'llms-full.txt') + +async function* generateText(): AsyncGenerator { + let hasContent = false + let bytes = 0 + const pages = [...source.getPages()].sort((left, right) => compareStrings(left.url, right.url)) + for (const page of pages) { + if (!page?.data || !page.url) continue + const text = await getLLMText(page) + if (!text) continue + const chunk = `${hasContent ? '\n\n---\n\n' : ''}${text}` + bytes += Buffer.byteLength(chunk) + if (bytes > MAX_LLM_TEXT_BYTES) { + throw new Error('Full documentation text exceeds the static asset size budget') + } + yield chunk + hasContent = true + } + if (!hasContent) throw new Error('No documentation text was generated') +} + +try { + await pipeline(Readable.from(generateText()), createWriteStream(temporaryFile)) + await rename(temporaryFile, path.join(publicDirectory, 'llms-full.txt')) + logger.info('Generated full documentation static asset') +} finally { + await rm(temporaryDirectory, { recursive: true, force: true }) +} diff --git a/apps/docs/turbo.json b/apps/docs/turbo.json new file mode 100644 index 00000000000..9eaa4dd5d8b --- /dev/null +++ b/apps/docs/turbo.json @@ -0,0 +1,8 @@ +{ + "extends": ["//"], + "tasks": { + "build": { + "outputs": ["$TURBO_EXTENDS$", "public/llms-full.txt"] + } + } +} From beb4958eaa9413025b347e4ecb182dd313083e22 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 7 Oct 2026 16:18:05 -0700 Subject: [PATCH 2/2] fix(docs): regenerate full text during development --- apps/docs/package.json | 6 ++-- apps/docs/scripts/watch-llms-full.ts | 51 ++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 apps/docs/scripts/watch-llms-full.ts diff --git a/apps/docs/package.json b/apps/docs/package.json index 4e8f5f0d6b9..138b0e2b06e 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -4,10 +4,10 @@ "private": true, "license": "Apache-2.0", "scripts": { - "dev": "bun run dev:cache:cap && fumadocs-mdx && bun run llms-full:generate && next dev --port 3001", + "dev": "bun run dev:cache:cap && bun run llms-full:generate && bunx concurrently --kill-others --names Next,LLMs \"next dev --port 3001\" \"bun scripts/watch-llms-full.ts\"", "dev:cache:cap": "bun run ../../scripts/prune-turbopack-cache.ts", "dev:clean": "rm -rf .next/dev/cache", - "build": "fumadocs-mdx && bun run llms-full:generate && NODE_OPTIONS='--max-old-space-size=8192' next build", + "build": "bun run llms-full:generate && NODE_OPTIONS='--max-old-space-size=8192' next build", "start": "next start", "postinstall": "fumadocs-mdx", "test": "vitest run", @@ -16,7 +16,7 @@ "lint:check": "biome check .", "format": "biome format --write .", "format:check": "biome format .", - "llms-full:generate": "bun scripts/generate-llms-full.ts" + "llms-full:generate": "fumadocs-mdx && bun scripts/generate-llms-full.ts" }, "dependencies": { "@sim/db": "workspace:*", diff --git a/apps/docs/scripts/watch-llms-full.ts b/apps/docs/scripts/watch-llms-full.ts new file mode 100644 index 00000000000..5e67acd7b99 --- /dev/null +++ b/apps/docs/scripts/watch-llms-full.ts @@ -0,0 +1,51 @@ +import { watch } from 'node:fs' +import { createLogger } from '@sim/logger' +import { spawn } from 'bun' +import { OPENAPI_SPEC_FILES } from '@/lib/openapi-specs' + +const logger = createLogger('DocsLlmsFullWatcher') +const WATCHED_ROOT_FILES = new Set(['source.config.ts', ...OPENAPI_SPEC_FILES]) + +let debounceTimer: ReturnType | undefined +let pending = false +let running = false + +async function regenerate(): Promise { + running = true + try { + while (pending) { + pending = false + const child = spawn(['bun', 'run', 'llms-full:generate'], { + stdin: 'ignore', + stdout: 'inherit', + stderr: 'inherit', + }) + const exitCode = await child.exited + if (exitCode !== 0) { + logger.warn('Documentation asset generation failed; retaining the previous asset', { + exitCode, + }) + } + } + } finally { + running = false + } +} + +function scheduleGeneration(): void { + pending = true + if (running) return + if (debounceTimer) clearTimeout(debounceTimer) + debounceTimer = setTimeout(() => { + debounceTimer = undefined + void regenerate() + }, 250) +} + +for (const directory of ['content', 'lib']) { + watch(directory, { recursive: true }, scheduleGeneration) +} + +watch('.', (_, changedPath) => { + if (changedPath && WATCHED_ROOT_FILES.has(changedPath.toString())) scheduleGeneration() +})