From 4151093ab5c37bbc12acabebf8777c94186f50c7 Mon Sep 17 00:00:00 2001 From: Rajat Date: Thu, 5 Nov 2026 20:59:55 +0530 Subject: [PATCH] feat(sites): serve Markdown for AI agents - Add Markdown page routes, `.md` URLs, and `/llms.txt` - Include agent instructions in site configuration and Git sync --- apps/api/src/db/schema.ts | 2 + apps/api/src/pages/git-export.ts | 3 + apps/api/src/pages/git-import.ts | 3 + apps/api/src/pages/queries.ts | 1 + apps/api/src/public/routes.ts | 14 +- .../docs/developers/pages-from-git.mdx | 23 +- .../agent-markdown/[[...path]]/route.ts | 35 +++ .../app/sites/[teamId]/llms.txt/route.ts | 19 ++ apps/sites/lib/api.ts | 3 + apps/sites/lib/edge-cache.test.ts | 16 + apps/sites/lib/edge-cache.ts | 8 + apps/sites/lib/markdown-negotiation.test.ts | 46 +++ apps/sites/lib/markdown-negotiation.ts | 46 +++ apps/sites/lib/markdown.test.ts | 100 ++++++ apps/sites/lib/markdown.ts | 294 ++++++++++++++++++ apps/sites/proxy.test.ts | 51 +++ apps/sites/proxy.ts | 37 +++ docs/site-delivery.md | 8 + packages/api-contract/src/schemas/public.ts | 2 + packages/api-contract/src/schemas/settings.ts | 3 + packages/page-builder/schema/site.v1.json | 5 + .../page-builder/src/schema/files.test.ts | 4 +- packages/page-builder/src/schema/files.ts | 14 +- packages/page-builder/src/schema/index.ts | 1 + .../page-builder/src/schema/markdown.test.ts | 70 +++++ packages/page-builder/src/schema/markdown.ts | 150 +++++++++ 26 files changed, 952 insertions(+), 6 deletions(-) create mode 100644 apps/sites/app/sites/[teamId]/agent-markdown/[[...path]]/route.ts create mode 100644 apps/sites/app/sites/[teamId]/llms.txt/route.ts create mode 100644 apps/sites/lib/markdown-negotiation.test.ts create mode 100644 apps/sites/lib/markdown-negotiation.ts create mode 100644 apps/sites/lib/markdown.test.ts create mode 100644 apps/sites/lib/markdown.ts create mode 100644 packages/page-builder/src/schema/markdown.test.ts create mode 100644 packages/page-builder/src/schema/markdown.ts diff --git a/apps/api/src/db/schema.ts b/apps/api/src/db/schema.ts index abe1dfca..08abef8e 100644 --- a/apps/api/src/db/schema.ts +++ b/apps/api/src/db/schema.ts @@ -757,6 +757,8 @@ export type SiteReleaseSnapshot = { * team's selected theme applies. */ theme?: Record; redirects?: { from: string; to: string; permanent?: boolean }[]; + /** `site.json`'s Markdown for AI agents, served in `/llms.txt`. */ + agentInstructions?: string; }; /** "preview": built from a pull request's head; never activated. */ diff --git a/apps/api/src/pages/git-export.ts b/apps/api/src/pages/git-export.ts index d31404bb..55f50ff5 100644 --- a/apps/api/src/pages/git-export.ts +++ b/apps/api/src/pages/git-export.ts @@ -51,6 +51,9 @@ export async function exportPagesFiles(teamId: string, directory: string) { footer: release.site.footer, ...(release.site.theme ? { theme: release.site.theme as any } : {}), ...(release.site.redirects ? { redirects: release.site.redirects } : {}), + ...(release.site.agentInstructions + ? { agentInstructions: release.site.agentInstructions } + : {}), }), }); return files; diff --git a/apps/api/src/pages/git-import.ts b/apps/api/src/pages/git-import.ts index 99cfe755..1f2b6f28 100644 --- a/apps/api/src/pages/git-import.ts +++ b/apps/api/src/pages/git-import.ts @@ -559,6 +559,9 @@ export async function createPagesRelease( ? { theme: manifest.site.theme as unknown as Record } : {}), ...(manifest.site.redirects ? { redirects: manifest.site.redirects } : {}), + ...(manifest.site.agentInstructions + ? { agentInstructions: manifest.site.agentInstructions } + : {}), } : { header: liveSite.header, footer: liveSite.footer }; diff --git a/apps/api/src/pages/queries.ts b/apps/api/src/pages/queries.ts index 4d460734..c025d282 100644 --- a/apps/api/src/pages/queries.ts +++ b/apps/api/src/pages/queries.ts @@ -377,6 +377,7 @@ export async function listPublishedPages(teamId: string) { const release = await getActiveRelease(teamId); return (release?.pages ?? []).map((entry) => ({ slug: entry.slug, + title: entry.revision.content.title || entry.name, robotsAllowed: entry.revision.content.robotsAllowed !== false, // A revision is created when its content first appears, so this only // moves when the page itself changes. diff --git a/apps/api/src/public/routes.ts b/apps/api/src/public/routes.ts index 33167302..1e87d45e 100644 --- a/apps/api/src/public/routes.ts +++ b/apps/api/src/public/routes.ts @@ -343,6 +343,7 @@ const impl = s.router(contract.public, { codeInjectionHead, codeInjectionBody, redirects: release?.site.redirects ?? [], + agentInstructions: release?.site.agentInstructions ?? null, canonicalHost: domains?.canonicalHost ?? null, hideFromSearch: domains?.hideFromSearch ?? false, }, @@ -418,6 +419,7 @@ const impl = s.router(contract.public, { const items: { kind: "page" | "blog" | "doc" | "changelog"; path: string; + title: string | null; lastmod: string | null; }[] = [ ...pagesList @@ -425,27 +427,36 @@ const impl = s.router(contract.public, { .map((p) => ({ kind: "page" as const, path: `/${p.slug}`, + title: p.title, lastmod: (p.updatedAt ?? p.publishedAt)?.toISOString() ?? null, })), ...blogs.map((d) => ({ kind: "blog" as const, path: `/blog/${d.slug}`, + title: d.title, lastmod: (d.updatedAt ?? d.publishedAt)?.toISOString() ?? null, })), ...docs.map((d) => ({ kind: "doc" as const, path: `/docs/${d.slug}`, + title: d.title, lastmod: (d.updatedAt ?? d.publishedAt)?.toISOString() ?? null, })), ]; const apiReference = await readApiReference(team.id); if (apiReference) items.push( - { kind: "doc", path: `/docs/${apiReference.mount}`, lastmod: null }, + { + kind: "doc", + path: `/docs/${apiReference.mount}`, + title: apiReference.info.title, + lastmod: null, + }, ...apiReference.groups.flatMap((group) => group.operations.map((operation) => ({ kind: "doc" as const, path: `/docs/${operation.route}`, + title: operation.title, lastmod: null, })), ), @@ -461,6 +472,7 @@ const impl = s.router(contract.public, { items.push({ kind: "changelog" as const, path: "/changelog", + title: "Changelog", lastmod: lastmod?.toISOString() ?? null, }); } diff --git a/apps/docs/content/docs/developers/pages-from-git.mdx b/apps/docs/content/docs/developers/pages-from-git.mdx index d2d73d95..34ccdf30 100644 --- a/apps/docs/content/docs/developers/pages-from-git.mdx +++ b/apps/docs/content/docs/developers/pages-from-git.mdx @@ -34,7 +34,7 @@ Everything FrontLit syncs lives in one site folder, `frontlit/` by default: ```text frontlit/ -├─ site.json # header, footer, theme and redirects (optional) +├─ site.json # header, footer, theme, redirects, agent instructions (optional) ├─ docs/ # Markdown docs → /docs/... └─ pages/ ├─ index.json # / @@ -90,10 +90,27 @@ A page file: "$schema": "https://frontlit.app/schema/site.v1.json", "header": { "links": [{ "label": "Pricing", "href": "/pricing" }], "ctaLabel": "Sign up", "ctaHref": "/signup" }, "footer": { "tagline": "The front office of your SaaS." }, - "redirects": [{ "from": "/plans", "to": "/pricing" }] + "redirects": [{ "from": "/plans", "to": "/pricing" }], + "agentInstructions": "Use this site for questions about Acme's API and pricing." } ``` +`agentInstructions` is optional Markdown for AI agents: what the site is for +and when to use it. It shows as its own section of the site's `/llms.txt`. + +## Markdown for AI agents + +Every site also serves its public content as Markdown: + +- Request any page, docs page, blog post or the changelog with + `Accept: text/markdown`, or add `.md` to its URL (`/docs/setup.md`, + `/index.md` for the homepage). Browsers keep getting HTML. +- A missing page requested as Markdown returns a 404 with a short Markdown + body that links to `/llms.txt` and `/sitemap.xml`. +- `/llms.txt` lists the site's pages, docs, blog posts, changelog and OpenAPI + spec with links to their `.md` URLs. Pages with `"robotsAllowed": false` + are left out, and the list is empty while the site is hidden from search. + ## How a push is published - Each push to the publishing branch is published as one **release**: every @@ -132,7 +149,7 @@ Marketing pages live in `frontlit/pages/*.json` and publish on merge. - File name = URL (`pricing.json` → `/pricing`, `index.json` → `/`). Flat folder only. - Every file starts with `"$schema": "https://frontlit.app/schema/page.v1.json"`; validate against it before committing. `frontlit/site.json` (schema - `site.v1.json`) holds the header, footer, theme and redirects. + `site.v1.json`) holds the header, footer, theme, redirects and agent instructions. - A page is `{ title, description, layout: [{ name, settings? }] }`. Only set settings that differ from the block defaults listed in the schema. - Rich text is Markdown. Media is `{ "url", "alt" }` with an `https://` URL or diff --git a/apps/sites/app/sites/[teamId]/agent-markdown/[[...path]]/route.ts b/apps/sites/app/sites/[teamId]/agent-markdown/[[...path]]/route.ts new file mode 100644 index 00000000..e1fd22df --- /dev/null +++ b/apps/sites/app/sites/[teamId]/agent-markdown/[[...path]]/route.ts @@ -0,0 +1,35 @@ +import { getSettings } from "@/lib/api"; +import { notFoundMarkdown, renderMarkdown } from "@/lib/markdown"; +import { markdownUrl } from "@/lib/markdown-negotiation"; + +/** + * Markdown for AI agents. The proxy rewrites here for `Accept: + * text/markdown` and `.md` (see lib/markdown.ts); `?md=url` marks a + * `.md` request so redirects stay on `.md` URLs. + */ +export async function GET( + request: Request, + { params }: { params: Promise<{ teamId: string; path?: string[] }> }, +) { + const { teamId, path: segments = [] } = await params; + const path = `/${segments.join("/")}`; + const headers = { "content-type": "text/markdown; charset=utf-8" }; + const markdown = await renderMarkdown(teamId, path); + if (markdown) return new Response(markdown, { headers }); + + // Redirects from site.json only apply where no page exists. + const settings = await getSettings(teamId); + const match = settings?.redirects?.find( + (redirect) => (redirect.from.replace(/\/+$/, "") || "/") === path, + ); + if (match) { + const viaUrl = new URL(request.url).searchParams.get("md") === "url"; + const location = + viaUrl && match.to.startsWith("/") ? markdownUrl(match.to) : match.to; + return new Response(null, { + status: match.permanent === false ? 307 : 308, + headers: { location }, + }); + } + return new Response(notFoundMarkdown(path), { status: 404, headers }); +} diff --git a/apps/sites/app/sites/[teamId]/llms.txt/route.ts b/apps/sites/app/sites/[teamId]/llms.txt/route.ts new file mode 100644 index 00000000..47c340cb --- /dev/null +++ b/apps/sites/app/sites/[teamId]/llms.txt/route.ts @@ -0,0 +1,19 @@ +import { getApiReference, getSettings, getSitemap } from "@/lib/api"; +import { buildLlmsTxt } from "@/lib/markdown"; + +/** `/llms.txt`: an index of the site's public content for AI agents. */ +export async function GET( + _request: Request, + { params }: { params: Promise<{ teamId: string }> }, +) { + const { teamId } = await params; + const [entries, settings, reference] = await Promise.all([ + getSitemap(teamId), + getSettings(teamId), + getApiReference(teamId), + ]); + return new Response( + buildLlmsTxt(settings, settings?.hideFromSearch ? [] : entries, Boolean(reference)), + { headers: { "content-type": "text/plain; charset=utf-8" } }, + ); +} diff --git a/apps/sites/lib/api.ts b/apps/sites/lib/api.ts index de385aac..493f6574 100644 --- a/apps/sites/lib/api.ts +++ b/apps/sites/lib/api.ts @@ -94,6 +94,8 @@ export interface PublicSettings { codeInjectionHead: string | null; codeInjectionBody: string | null; redirects?: { from: string; to: string; permanent?: boolean }[]; + /** site.json's Markdown for AI agents, shown in `/llms.txt`. */ + agentInstructions?: string | null; /** Host links, canonical URLs and feeds point at. */ canonicalHost: string | null; /** Pre-launch: noindex everywhere, robots disallow all. */ @@ -187,6 +189,7 @@ export function getPreviewPage( export interface SitemapEntry { kind: "page" | "blog" | "doc" | "changelog"; path: string; + title?: string | null; lastmod: string | null; } diff --git a/apps/sites/lib/edge-cache.test.ts b/apps/sites/lib/edge-cache.test.ts index 7f53a2b7..acd0b74a 100644 --- a/apps/sites/lib/edge-cache.test.ts +++ b/apps/sites/lib/edge-cache.test.ts @@ -97,4 +97,20 @@ describe("edgeCacheHeaders", () => { ).not.toEqual(privateOnly); } }); + + it("never stores Markdown chosen by Accept", () => { + for (const mode of ["none", "ttl", "purge"] as const) + expect(edgeCacheHeaders("team_a", "/docs/x", mode, 60, undefined, true)).toEqual({ + "Cache-Control": "private, no-store", + }); + }); + + it("tags llms.txt with the sitemap and docs", () => { + expect(edgeTags("team_a", "/llms.txt")).toEqual([ + "team:team_a", + "shell:team_a", + "sitemap:team_a", + "docs:team_a", + ]); + }); }); diff --git a/apps/sites/lib/edge-cache.ts b/apps/sites/lib/edge-cache.ts index 95bcbd68..b5849ae3 100644 --- a/apps/sites/lib/edge-cache.ts +++ b/apps/sites/lib/edge-cache.ts @@ -41,6 +41,9 @@ export function edgeTags(teamId: string, pathname: string): string[] { if (path === "/changelog") tags.push(`page:${teamId}:changelog`); } else if (path === "/sitemap.xml") { tags.push(`sitemap:${teamId}`); + } else if (path === "/llms.txt") { + // Lists everything in the sitemap, plus the API reference. + tags.push(`sitemap:${teamId}`, `docs:${teamId}`); } else if ( !["robots.txt", "favicon.ico", "opengraph-image", "embed"].includes(first) ) { @@ -70,9 +73,14 @@ export function edgeCacheHeaders( mode: EdgeCacheMode = edgeCacheMode(), ttl = Number(process.env.EDGE_CACHE_TTL) || 60, rsc = { rscHeader: false, rscParam: false }, + /** Markdown chosen by `Accept`, at a URL that also serves HTML. Never + * stored, so a cache that ignores `Vary: Accept` can't hand Markdown to + * browsers. Agents can cache the `.md` URL instead. */ + markdownByAccept = false, ): Record | null { if (isPrivatePath(pathname)) return { "Cache-Control": "private, no-store" }; if (isMixedRscRequest(rsc)) return { "Cache-Control": "private, no-store" }; + if (markdownByAccept) return { "Cache-Control": "private, no-store" }; if (mode === "none") return null; const maxAge = mode === "ttl" ? ttl : 31_536_000; const headers: Record = { diff --git a/apps/sites/lib/markdown-negotiation.test.ts b/apps/sites/lib/markdown-negotiation.test.ts new file mode 100644 index 00000000..22099e7f --- /dev/null +++ b/apps/sites/lib/markdown-negotiation.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; +import { + isNegotiablePath, + markdownSourcePath, + markdownUrl, + prefersMarkdown, +} from "./markdown-negotiation"; + +describe("prefersMarkdown", () => { + it("serves Markdown only when asked for at least as strongly as HTML", () => { + expect(prefersMarkdown("text/markdown")).toBe(true); + expect(prefersMarkdown("text/markdown, text/html;q=0.9")).toBe(true); + expect(prefersMarkdown("text/html, text/markdown")).toBe(true); + expect(prefersMarkdown("text/html, text/markdown;q=0.5")).toBe(false); + expect(prefersMarkdown("text/markdown;q=0")).toBe(false); + }); + + it("keeps browsers on HTML", () => { + expect( + prefersMarkdown( + "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,*/*;q=0.8", + ), + ).toBe(false); + expect(prefersMarkdown("*/*")).toBe(false); + expect(prefersMarkdown(null)).toBe(false); + }); +}); + +describe("markdown paths", () => { + it("maps .md URLs to their page", () => { + expect(markdownSourcePath("/docs/intro.md")).toBe("/docs/intro"); + expect(markdownSourcePath("/index.md")).toBe("/"); + expect(markdownSourcePath("/docs/intro")).toBeNull(); + expect(markdownUrl("/")).toBe("/index.md"); + expect(markdownUrl("/blog/post")).toBe("/blog/post.md"); + }); + + it("negotiates pages only", () => { + expect(isNegotiablePath("/")).toBe(true); + expect(isNegotiablePath("/docs/intro")).toBe(true); + expect(isNegotiablePath("/sitemap.xml")).toBe(false); + expect(isNegotiablePath("/blog/rss.xml")).toBe(false); + expect(isNegotiablePath("/help/tickets/t_1")).toBe(false); + expect(isNegotiablePath("/embed/chat")).toBe(false); + }); +}); diff --git a/apps/sites/lib/markdown-negotiation.ts b/apps/sites/lib/markdown-negotiation.ts new file mode 100644 index 00000000..55880d6a --- /dev/null +++ b/apps/sites/lib/markdown-negotiation.ts @@ -0,0 +1,46 @@ +/** + * When a request gets Markdown instead of HTML (see lib/markdown.ts). Kept + * free of imports: the proxy runs it on every request. + */ + +/** Route segment, under `/sites/[teamId]`, of the Markdown handler. */ +export const MARKDOWN_ROUTE = "agent-markdown"; + +/** True when the Accept header ranks Markdown at least as high as HTML. + * Browsers never list `text/markdown`, so they keep getting HTML. */ +export function prefersMarkdown(accept: string | null): boolean { + if (!accept) return false; + let markdown = 0; + let html = 0; + for (const part of accept.split(",")) { + const [type = "", ...params] = part.split(";").map((value) => value.trim()); + const qParam = params.find((param) => param.startsWith("q=")); + const q = qParam ? Number(qParam.slice(2)) : 1; + if (Number.isNaN(q)) continue; + const mediaType = type.toLowerCase(); + if (mediaType === "text/markdown" || mediaType === "text/x-markdown") + markdown = Math.max(markdown, q); + else if (mediaType === "text/html") html = Math.max(html, q); + } + return markdown > 0 && markdown >= html; +} + +/** Paths whose response depends on Accept: pages, not files or private + * routes. `.md` URLs, feeds and assets have one representation. */ +export function isNegotiablePath(pathname: string): boolean { + if (/\.[a-z0-9]+$/i.test(pathname)) return false; + return !/^\/(_preview|embed|help\/tickets|opengraph-image)(\/|$)/.test(pathname); +} + +/** `/docs/intro.md` → `/docs/intro`; `/index.md` → `/`. Null when the path + * isn't a `.md` URL. */ +export function markdownSourcePath(pathname: string): string | null { + if (!pathname.endsWith(".md")) return null; + const path = pathname.slice(0, -".md".length); + return path === "/index" || path === "" ? "/" : path; +} + +/** The `.md` URL of a public path, for links agents follow. */ +export function markdownUrl(path: string): string { + return path === "/" ? "/index.md" : `${path.replace(/\/+$/, "")}.md`; +} diff --git a/apps/sites/lib/markdown.test.ts b/apps/sites/lib/markdown.test.ts new file mode 100644 index 00000000..26d23012 --- /dev/null +++ b/apps/sites/lib/markdown.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it, vi } from "vitest"; + +vi.mock("./api", () => ({})); +vi.mock("./docs-tree", () => ({})); + +const { buildLlmsTxt, notFoundMarkdown, tiptapToMarkdown } = await import("./markdown"); + +describe("buildLlmsTxt", () => { + const settings = { + title: "Acme", + subtitle: "Widgets for everyone", + canonicalHost: "acme.com", + agentInstructions: "Use this site for questions about Acme widgets.", + } as any; + + it("lists public content as .md links", () => { + const text = buildLlmsTxt( + settings, + [ + { kind: "page", path: "/", title: "Home", lastmod: null }, + { kind: "doc", path: "/docs/intro", title: "Intro", lastmod: null }, + { kind: "blog", path: "/blog/hello", title: "Hello", lastmod: null }, + { kind: "changelog", path: "/changelog", title: "Changelog", lastmod: null }, + ], + true, + ); + expect(text).toBe(`# Acme + +> Widgets for everyone + +## Agent instructions + +Use this site for questions about Acme widgets. + +## Pages + +- [Home](https://acme.com/index.md) + +## Docs + +- [Intro](https://acme.com/docs/intro.md) + +## Blog + +- [Hello](https://acme.com/blog/hello.md) + +## Changelog + +- [Changelog](https://acme.com/changelog.md) + +## API + +- [OpenAPI spec](https://acme.com/openapi.json) +`); + }); + + it("lists a path once when a builder page sits on the changelog", () => { + const text = buildLlmsTxt( + settings, + [ + { kind: "page", path: "/changelog", title: "Changelog", lastmod: null }, + { kind: "changelog", path: "/changelog", title: "Changelog", lastmod: null }, + ], + false, + ); + expect(text.match(/changelog\.md/g)).toHaveLength(1); + expect(text).not.toContain("## Changelog"); + }); + + it("omits empty sections", () => { + expect(buildLlmsTxt({ ...settings, agentInstructions: null }, [], false)).toBe( + "# Acme\n\n> Widgets for everyone\n", + ); + }); +}); + +describe("markdown bodies", () => { + it("links a missing page to the site indexes", () => { + const body = notFoundMarkdown("/nope"); + expect(body).toContain("/llms.txt"); + expect(body).toContain("/sitemap.xml"); + expect(body.length).toBeGreaterThan(20); + }); + + it("serializes TipTap content", () => { + expect( + tiptapToMarkdown({ + type: "doc", + content: [ + { + type: "heading", + attrs: { level: 2 }, + content: [{ type: "text", text: "Setup" }], + }, + { type: "paragraph", content: [{ type: "text", text: "Install it." }] }, + ], + }), + ).toBe("## Setup\n\nInstall it."); + }); +}); diff --git a/apps/sites/lib/markdown.ts b/apps/sites/lib/markdown.ts new file mode 100644 index 00000000..7e7c3945 --- /dev/null +++ b/apps/sites/lib/markdown.ts @@ -0,0 +1,294 @@ +import type { ApiRichText } from "@frontlit/api-contract"; +import { layoutToMarkdown } from "@frontlit/page-builder/schema"; +import { serializeDocsMarkdown } from "@frontlit/text-editor/docs"; +import { + getApiOperation, + getApiReference, + getArticleBySlug, + getDocumentationBySlug, + getPageBySlug, + getSettings, + listArticles, + listProductUpdates, + type PublicSettings, + type SitemapEntry, +} from "./api"; +import { loadDocsTree } from "./docs-tree"; +import { markdownUrl } from "./markdown-negotiation"; + +/** + * Markdown versions of public pages for AI agents (`Accept: text/markdown` + * or `.md`), plus `/llms.txt`. The proxy decides when a request wants + * Markdown (lib/markdown-negotiation.ts); everything here is keyed by path + * alone, so it caches like HTML. + */ + +/** TipTap JSON → Markdown. Content the docs serializer can't read (rare + * blog formatting) falls back to its plain text. */ +export function tiptapToMarkdown(content: unknown): string { + if (!content) return ""; + try { + return serializeDocsMarkdown(content).trim(); + } catch { + return plainText(content).trim(); + } +} + +function plainText(node: unknown): string { + if (!node || typeof node !== "object") return ""; + const { text, content, type } = node as { + text?: string; + content?: unknown[]; + type?: string; + }; + if (typeof text === "string") return text; + const inner = (content ?? []).map(plainText).join(""); + return ["paragraph", "heading", "listItem", "codeBlock", "blockquote"].includes( + type ?? "", + ) + ? `${inner}\n\n` + : inner; +} + +function richText(value: ApiRichText | undefined): string { + if (!value) return ""; + return value.kind === "text" ? value.text : tiptapToMarkdown(value.content); +} + +const date = (value: string | null | undefined) => (value ? value.slice(0, 10) : ""); + +function document(title: string, ...parts: (string | null | undefined)[]): string { + return `${[`# ${title}`, ...parts].filter((part) => part?.trim()).join("\n\n")}\n`; +} + +export function notFoundMarkdown(path: string): string { + return document( + "Page not found", + `Nothing is published at \`${path}\`.`, + "- [llms.txt](/llms.txt): an index of this site's pages, docs and posts\n- [sitemap.xml](/sitemap.xml): every public URL", + ); +} + +async function blogIndex(teamId: string) { + const { items } = await listArticles(teamId, { itemsPerPage: 1000 }); + return items + .map( + (post) => + `- [${post.title || post.slug}](${markdownUrl(`/blog/${post.slug}`)})${post.publishedAt ? ` (${date(post.publishedAt)})` : ""}${post.excerpt ? `: ${post.excerpt}` : ""}`, + ) + .join("\n"); +} + +async function changelogFeed(teamId: string) { + const { items } = await listProductUpdates(teamId, { itemsPerPage: 1000 }); + return items + .map((update) => { + const label = [ + update.meta.version, + date(update.meta.releaseDate ?? update.publishedAt), + ] + .filter(Boolean) + .join(", "); + return [ + `## ${update.title || update.slug}${label ? ` (${label})` : ""}`, + tiptapToMarkdown(update.content), + ] + .filter(Boolean) + .join("\n\n"); + }) + .join("\n\n"); +} + +async function docsIndex(teamId: string) { + const collections = await loadDocsTree(teamId); + return collections + .map((collection) => { + const items = collection.api + ? [ + `- [${collection.name}](${markdownUrl(collection.api.href)})`, + ...collection.api.groups.flatMap((group) => + group.items.map( + (item) => + `- [${item.method ? `${item.method.toUpperCase()} ` : ""}${item.title || item.slug}](${markdownUrl(`/docs/${item.slug}`)})`, + ), + ), + ] + : collection.items.map( + (item) => + `- [${item.title || item.slug}](${markdownUrl(`/docs/${item.slug}`)})${item.excerpt ? `: ${item.excerpt}` : ""}`, + ); + return [`## ${collection.name}`, items.join("\n")].join("\n\n"); + }) + .join("\n\n"); +} + +/** A builder page, with FrontLit's content feed in its slot (or appended + * when the page doesn't place one). */ +async function pageMarkdown( + teamId: string, + slug: string, + fallbackTitle: string, + feed?: () => Promise, +): Promise { + const page = await getPageBySlug(teamId, slug); + if (!page && !feed) return null; + const feedMarkdown = feed ? await feed() : ""; + let placed = false; + const body = layoutToMarkdown(page?.layout ?? [], () => { + placed = true; + return feedMarkdown; + }); + const settings = await getSettings(teamId); + const title = + page?.title || (slug === "" ? settings?.title : page?.name) || fallbackTitle; + return document( + title, + (page?.description ?? (slug === "" ? settings?.subtitle : null)) + ? `> ${page?.description ?? settings?.subtitle}` + : "", + body, + placed ? "" : feedMarkdown, + ); +} + +async function docMarkdown(teamId: string, slug: string): Promise { + const reference = await getApiReference(teamId); + if (reference && slug === reference.mount) { + return document( + reference.info.title, + reference.info.version ? `Version ${reference.info.version}` : "", + richText(reference.info.description), + "[OpenAPI spec](/openapi.json)", + ...reference.groups.map((group) => + [ + `## ${group.name}`, + group.operations + .map( + (operation) => + `- [${operation.method.toUpperCase()} ${operation.path}](${markdownUrl(`/docs/${operation.route}`)}): ${operation.title}`, + ) + .join("\n"), + ].join("\n\n"), + ), + ); + } + if (reference && slug.startsWith(`${reference.mount}/`)) { + const found = await getApiOperation(teamId, slug); + if (!found) return null; + const { operation } = found; + const parameters = operation.parameters.map( + (param) => + `- \`${param.name}\` (${param.in}${param.required ? ", required" : ""})${param.description ? `: ${param.description}` : ""}`, + ); + return document( + operation.title, + `\`${operation.method.toUpperCase()} ${operation.path}\`${operation.deprecated ? " (deprecated)" : ""}`, + richText(operation.description), + parameters.length ? `## Parameters\n\n${parameters.join("\n")}` : "", + operation.responses.length + ? `## Responses\n\n${operation.responses + .map( + (response) => + `- \`${response.status}\`${response.description ? `: ${response.description}` : ""}`, + ) + .join("\n")}` + : "", + "Full schemas: [OpenAPI spec](/openapi.json)", + ); + } + const doc = await getDocumentationBySlug(teamId, slug); + if (!doc) return null; + return document( + doc.title || slug, + doc.excerpt ? `> ${doc.excerpt}` : "", + tiptapToMarkdown(doc.content), + ); +} + +async function collectionMarkdown(teamId: string, sectionSlug: string) { + const collection = (await loadDocsTree(teamId)).find( + (item) => item.slug === sectionSlug, + ); + if (!collection) return null; + return document( + collection.name, + collection.items + .map( + (item) => + `- [${item.title || item.slug}](${markdownUrl(`/docs/${item.slug}`)})${item.excerpt ? `: ${item.excerpt}` : ""}`, + ) + .join("\n"), + ); +} + +/** The Markdown for a public path, or null when nothing is published there. */ +export async function renderMarkdown( + teamId: string, + path: string, +): Promise { + const segments = path.split("/").filter(Boolean).map(decodeURIComponent); + const [first = "", ...rest] = segments; + if (first === "docs") { + if (rest.length === 0) + return pageMarkdown(teamId, "docs", "Docs", () => docsIndex(teamId)); + if (rest[0] === "collections" && rest.length === 2) + return collectionMarkdown(teamId, rest[1]!); + return docMarkdown(teamId, rest.join("/")); + } + if (first === "blog") { + if (rest.length === 0) + return pageMarkdown(teamId, "blog", "Blog", () => blogIndex(teamId)); + if (rest.length > 1) return null; + const post = await getArticleBySlug(teamId, rest[0]!); + if (!post) return null; + return document( + post.title || post.slug, + post.publishedAt ? `Published ${date(post.publishedAt)}` : "", + tiptapToMarkdown(post.content), + ); + } + if (first === "changelog" && rest.length === 0) + return pageMarkdown(teamId, "changelog", "Changelog", () => changelogFeed(teamId)); + return pageMarkdown(teamId, segments.join("/"), segments.at(-1) ?? "Home"); +} + +/** `/llms.txt` (llmstxt.org): the site's title, summary, agent + * instructions and links to the Markdown of every public page. Built from + * the same read as the sitemap, so hidden pages stay out. */ +export function buildLlmsTxt( + settings: PublicSettings | null, + entries: SitemapEntry[], + hasOpenApi: boolean, +): string { + const host = settings?.canonicalHost ?? "localhost"; + const url = (path: string) => `https://${host}${markdownUrl(path)}`; + const line = (entry: SitemapEntry) => + `- [${entry.title || entry.path}](${url(entry.path)})`; + // A builder page at /changelog also comes back as the changelog entry; + // list each path once. + const seen = new Set(); + const of = (kind: SitemapEntry["kind"]) => + entries + .filter((e) => e.kind === kind && !seen.has(e.path) && seen.add(e.path)) + .map(line); + const sections: [string, string[]][] = [ + ["Pages", of("page")], + ["Docs", of("doc")], + ["Blog", of("blog")], + ["Changelog", of("changelog")], + ]; + if (hasOpenApi) + sections.push(["API", [`- [OpenAPI spec](https://${host}/openapi.json)`]]); + return `${[ + `# ${settings?.title || host}`, + settings?.subtitle ? `> ${settings.subtitle}` : "", + settings?.agentInstructions + ? `## Agent instructions\n\n${settings.agentInstructions}` + : "", + ...sections + .filter(([, lines]) => lines.length) + .map(([name, lines]) => `## ${name}\n\n${lines.join("\n")}`), + ] + .filter(Boolean) + .join("\n\n")}\n`; +} diff --git a/apps/sites/proxy.test.ts b/apps/sites/proxy.test.ts index 5334fd3d..ca11099a 100644 --- a/apps/sites/proxy.test.ts +++ b/apps/sites/proxy.test.ts @@ -62,6 +62,57 @@ describe("proxy middleware", () => { ); }); + describe("markdown for agents", () => { + const resolve = (host: string) => + ( + api.resolveHost as unknown as { + mockResolvedValueOnce: (val: unknown) => void; + } + ).mockResolvedValueOnce({ + teamId: "team_md", + canonicalHost: host, + redirect: false, + hideFromSearch: false, + }); + const rewrite = (response: Response) => + new URL(response.headers.get("x-middleware-rewrite")!); + + it("rewrites Accept: text/markdown to the Markdown route and varies on Accept", async () => { + resolve("md1.localhost"); + const response = await proxy( + new NextRequest("http://md1.localhost/docs/intro", { + headers: { host: "md1.localhost", accept: "text/markdown" }, + }), + ); + expect(rewrite(response).pathname).toBe( + "/sites/team_md/agent-markdown/docs/intro", + ); + expect(response.headers.get("vary")).toBe("Accept"); + }); + + it("keeps browsers on HTML", async () => { + resolve("md2.localhost"); + const response = await proxy( + new NextRequest("http://md2.localhost/docs/intro", { + headers: { host: "md2.localhost", accept: "text/html,*/*;q=0.8" }, + }), + ); + expect(rewrite(response).pathname).toBe("/sites/team_md/docs/intro"); + }); + + it("serves .md as Markdown", async () => { + resolve("md3.localhost"); + const response = await proxy( + new NextRequest("http://md3.localhost/index.md", { + headers: { host: "md3.localhost" }, + }), + ); + expect(rewrite(response).pathname).toBe("/sites/team_md/agent-markdown"); + expect(rewrite(response).searchParams.get("md")).toBe("url"); + expect(response.headers.get("vary")).toBeNull(); + }); + }); + it("301-redirects a non-canonical host to the canonical host over the forwarded scheme", async () => { const mockResolveHost = api.resolveHost as unknown as { mockResolvedValueOnce: (val: unknown) => void; diff --git a/apps/sites/proxy.ts b/apps/sites/proxy.ts index 4c142620..1c19115b 100644 --- a/apps/sites/proxy.ts +++ b/apps/sites/proxy.ts @@ -2,6 +2,12 @@ import { NextRequest, NextResponse } from "next/server"; import { resolveHost } from "./lib/api"; import { decideHost, lookupHost, normalizeHost } from "./lib/host"; import { edgeCacheHeaders } from "./lib/edge-cache"; +import { + isNegotiablePath, + MARKDOWN_ROUTE, + markdownSourcePath, + prefersMarkdown, +} from "./lib/markdown-negotiation"; export async function proxy(request: NextRequest) { // Direct access to the internal /sites rewrite segment is never allowed. @@ -69,6 +75,34 @@ export async function proxy(request: NextRequest) { return response; } + // Markdown for AI agents: `.md`, or `Accept: text/markdown` on a + // page URL (which then varies on Accept). + const pathname = request.nextUrl.pathname; + const negotiable = isNegotiablePath(pathname); + const mdPath = markdownSourcePath(pathname); + const markdownByAccept = negotiable && prefersMarkdown(request.headers.get("accept")); + if (mdPath !== null || markdownByAccept) { + const url = request.nextUrl.clone(); + const source = mdPath ?? pathname; + url.pathname = `/sites/${decision.teamId}/${MARKDOWN_ROUTE}${source === "/" ? "" : source}`; + url.search = mdPath !== null ? "?md=url" : ""; + const response = NextResponse.rewrite(url); + if (markdownByAccept) response.headers.set("Vary", "Accept"); + if (resolved!.hideFromSearch || host !== resolved!.canonicalHost) + response.headers.set("X-Robots-Tag", "noindex"); + const edge = edgeCacheHeaders( + decision.teamId, + source, + undefined, + undefined, + undefined, + markdownByAccept, + ); + for (const [name, value] of Object.entries(edge ?? {})) + response.headers.set(name, value); + return response; + } + const url = request.nextUrl.clone(); // Not `_sites` — a leading underscore makes Next.js treat the folder as // a private, non-routable segment, which would silently 404 everything. @@ -81,6 +115,9 @@ export async function proxy(request: NextRequest) { // The rendered response depends only on team and path, never on request // headers, so it can be cached and shared (docs/site-delivery.md §4.2). + // HTML pages can't add `Vary: Accept` here: Next's page runtime + // replaces Vary. Markdown by Accept is never stored by shared caches + // (edge-cache.ts), so a cached page is always HTML. const response = NextResponse.rewrite(url); // Search visibility: a hidden (pre-launch) site is noindex everywhere; a // non-canonical host that still serves (subdomain with redirect off) is diff --git a/docs/site-delivery.md b/docs/site-delivery.md index 077009bf..397dde9c 100644 --- a/docs/site-delivery.md +++ b/docs/site-delivery.md @@ -302,6 +302,14 @@ to an RSC payload, and one crafted request is enough to do it. The proxy needs both signals to tell, so `skipProxyUrlNormalize` is on (Next strips them before the proxy by default). +**Markdown for agents.** A page URL requested with `Accept: text/markdown` +returns Markdown with `Vary: Accept`. That response is never cacheable +(`private, no-store`, in every mode), so a cache that ignores `Vary` only +ever stores the HTML. Next's page runtime replaces `Vary` on HTML pages, so +the HTML can't carry `Accept` itself. Agents that want a cacheable copy use +the `.md` URL, which has its own cache key and the page's tags. +`/llms.txt` is tagged with `sitemap` and `docs`. + **Static assets.** `/_next/static/*` is content-hashed and already `immutable`; any layer caches it. An optional `ASSET_PREFIX` serves it from a separate host. diff --git a/packages/api-contract/src/schemas/public.ts b/packages/api-contract/src/schemas/public.ts index d30bb249..ef1de0c2 100644 --- a/packages/api-contract/src/schemas/public.ts +++ b/packages/api-contract/src/schemas/public.ts @@ -45,6 +45,8 @@ export const sitemapEntryKindSchema = z.enum(["page", "blog", "doc", "changelog" export const sitemapEntrySchema = z.object({ kind: sitemapEntryKindSchema, path: z.string(), + /** Page or document title, for link lists like `/llms.txt`. */ + title: z.string().nullable().optional(), lastmod: z.string().nullable(), }); diff --git a/packages/api-contract/src/schemas/settings.ts b/packages/api-contract/src/schemas/settings.ts index e2a0e81b..807ae76a 100644 --- a/packages/api-contract/src/schemas/settings.ts +++ b/packages/api-contract/src/schemas/settings.ts @@ -44,6 +44,9 @@ export const publicTeamSettingsSchema = z.object({ z.object({ from: z.string(), to: z.string(), permanent: z.boolean().optional() }), ) .optional(), + /** From the live release's site.json: Markdown for AI agents, shown in + * the site's `/llms.txt`. */ + agentInstructions: z.string().nullable().optional(), /** Host the site's links, canonical URLs and feeds point at; null while * the site has no public host. */ canonicalHost: z.string().nullable(), diff --git a/packages/page-builder/schema/site.v1.json b/packages/page-builder/schema/site.v1.json index 5f579d57..951e8761 100644 --- a/packages/page-builder/schema/site.v1.json +++ b/packages/page-builder/schema/site.v1.json @@ -3813,6 +3813,11 @@ ], "additionalProperties": false } + }, + "agentInstructions": { + "type": "string", + "maxLength": 10000, + "description": "Markdown for AI agents: what this site is for and when to use it. Shown in /llms.txt." } }, "additionalProperties": false diff --git a/packages/page-builder/src/schema/files.test.ts b/packages/page-builder/src/schema/files.test.ts index 0115dfb5..d71f0a59 100644 --- a/packages/page-builder/src/schema/files.test.ts +++ b/packages/page-builder/src/schema/files.test.ts @@ -207,17 +207,19 @@ describe("site files", () => { } }); - it("round-trips theme, chrome and redirects", () => { + it("round-trips theme, chrome, redirects and agent instructions", () => { const text = serializeSiteFile({ theme: themes[0].theme, header: { ...blockDefaults.header, ctaLabel: "Sign up" }, footer: blockDefaults.footer, redirects: [{ from: "/old", to: "/new", permanent: true }], + agentInstructions: "Use for questions about Acme.", }); const parsed = JSON.parse(text); expect(parsed.header).toEqual({ ctaLabel: "Sign up" }); expect(parsed.footer).toBeUndefined(); expect(parsed.redirects).toEqual([{ from: "/old", to: "/new" }]); + expect(parsed.agentInstructions).toBe("Use for questions about Acme."); expect(serializeSiteFile(parseSiteFile(text))).toBe(text); }); diff --git a/packages/page-builder/src/schema/files.ts b/packages/page-builder/src/schema/files.ts index e4659ea6..d933ccb1 100644 --- a/packages/page-builder/src/schema/files.ts +++ b/packages/page-builder/src/schema/files.ts @@ -73,6 +73,9 @@ export interface SiteDocument { header?: Record; footer?: Record; redirects?: Redirect[]; + /** Markdown for AI agents: what the site is for and when to use it. + * Served as its own section of `/llms.txt`. */ + agentInstructions?: string; } const fileSettings = blockSettingsSchemas(fileKinds); @@ -363,6 +366,13 @@ export const siteFileSchema = z .optional() .describe("Site footer, shown on every page."), redirects: z.array(redirectSchema).optional(), + agentInstructions: z + .string() + .max(10_000) + .optional() + .describe( + "Markdown for AI agents: what this site is for and when to use it. Shown in /llms.txt.", + ), }) .strict(); @@ -371,12 +381,13 @@ export function parseSiteFile(source: string | unknown): SiteDocument { const result = siteFileSchema.safeParse(parseJson(source)); if (!result.success) throw new SchemaValidationError(issuesToDiagnostics(result.error)); - const { theme, header, footer, redirects } = result.data; + const { theme, header, footer, redirects, agentInstructions } = result.data; const site: SiteDocument = {}; if (theme) site.theme = theme as ThemeStyle; if (header) site.header = canonicalizeSettings("header", header, ["header"]); if (footer) site.footer = canonicalizeSettings("footer", footer, ["footer"]); if (redirects) site.redirects = redirects as Redirect[]; + if (agentInstructions?.trim()) site.agentInstructions = agentInstructions.trim(); return site; } @@ -409,6 +420,7 @@ export function serializeSiteFile(site: SiteDocument): string { header: header && Object.keys(header).length ? header : undefined, footer: footer && Object.keys(footer).length ? footer : undefined, redirects, + agentInstructions: site.agentInstructions || undefined, }; return `${JSON.stringify(file, null, 2)}\n`; } diff --git a/packages/page-builder/src/schema/index.ts b/packages/page-builder/src/schema/index.ts index 06fb395d..e2aa5c05 100644 --- a/packages/page-builder/src/schema/index.ts +++ b/packages/page-builder/src/schema/index.ts @@ -10,4 +10,5 @@ export { type BlockName, } from "./blocks"; export { exportKinds, fileKinds, fileMediaSchema, type FieldKinds } from "./fields"; +export { layoutToMarkdown } from "./markdown"; export { themeStyleSchema } from "./theme-style"; diff --git a/packages/page-builder/src/schema/markdown.test.ts b/packages/page-builder/src/schema/markdown.test.ts new file mode 100644 index 00000000..18c2d58a --- /dev/null +++ b/packages/page-builder/src/schema/markdown.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from "vitest"; +import { layoutToMarkdown } from "./markdown"; + +const paragraph = (text: string) => ({ + type: "doc", + content: [{ type: "paragraph", content: [{ type: "text", text }] }], +}); + +describe("layoutToMarkdown", () => { + it("renders headings, text, buttons as links and cards as lists", () => { + const markdown = layoutToMarkdown( + [ + { name: "header", settings: { title: "Acme" } }, + { + name: "hero", + settings: { + title: "Ship faster", + description: paragraph("One tool for it all."), + buttonCaption: "Start", + buttonAction: "/signup", + secondaryButtonCaption: "", + secondaryButtonAction: "", + }, + }, + { name: "media", settings: {} }, + { + name: "grid", + settings: { + title: "Features", + items: [{ title: "Fast", description: "Very." }], + }, + }, + { + name: "faq", + settings: { title: "FAQ", items: [{ question: "Why?", answer: "Because." }] }, + }, + { name: "data-slot", settings: { slot: "content-feed" } }, + { name: "footer", settings: {} }, + ], + () => "FEED", + ); + expect(markdown).toBe(`## Ship faster + +One tool for it all. + +[Start](/signup) + +## Features + +- **Fast**: Very. + +## FAQ + +### Why? + +Because. + +FEED`); + }); + + it("uses block defaults like the renderer", () => { + expect(layoutToMarkdown([{ name: "hero" }])).toContain( + "## Run your whole business", + ); + }); + + it("skips blocks it can't read", () => { + expect(layoutToMarkdown([{ name: "nope", settings: {} }])).toBe(""); + }); +}); diff --git a/packages/page-builder/src/schema/markdown.ts b/packages/page-builder/src/schema/markdown.ts new file mode 100644 index 00000000..14ed2c6b --- /dev/null +++ b/packages/page-builder/src/schema/markdown.ts @@ -0,0 +1,150 @@ +import { type BlockName, DATA_SLOT_BLOCK } from "./blocks"; +import { fileBlockDefaults, type StoredBlock, toFileSettings } from "./files"; + +/** Blocks with nothing to read: chrome repeats on every page, and the rest + * are decorative or interactive only. */ +const SKIPPED = new Set(["header", "footer", "media", "marquee", "embed"]); + +type Settings = Record; + +const text = (value: unknown) => (typeof value === "string" ? value.trim() : ""); + +function link(caption: unknown, href: unknown): string { + const label = text(caption); + if (!label) return ""; + const target = text(href); + return target ? `[${label}](${target})` : label; +} + +function heading(level: number, value: unknown): string { + const title = text(value); + return title ? `${"#".repeat(level)} ${title}` : ""; +} + +function links(...pairs: [unknown, unknown][]): string { + return pairs + .map(([caption, href]) => link(caption, href)) + .filter(Boolean) + .join(" · "); +} + +function blockMarkdown(name: string, s: Settings): string[] { + switch (name) { + case "hero": + return [ + text(s.preTitle), + heading(2, s.title), + text(s.description), + text(s.command) ? `\`\`\`sh\n${text(s.command)}\n\`\`\`` : "", + links( + [s.buttonCaption, s.buttonAction], + [s.secondaryButtonCaption, s.secondaryButtonAction], + ), + text(s.smallPrint), + ]; + case "content": + return [heading(2, s.title), text(s.body)]; + case "rich-text": + return [text(s.text)]; + case "featured": + return [ + heading(2, s.title), + text(s.description), + (s.items ?? []) + .map((item: Settings) => + [`- **${text(item.title)}**`, text(item.description)] + .filter(Boolean) + .join(": "), + ) + .join("\n"), + ]; + case "grid": + return [ + heading(2, s.title), + text(s.description), + (s.items ?? []) + .map((item: Settings) => + [ + `- **${text(item.title)}**`, + [text(item.description), link(item.buttonCaption, item.buttonAction)] + .filter(Boolean) + .join(" "), + ] + .filter(Boolean) + .join(": "), + ) + .join("\n"), + link(s.buttonCaption, s.buttonAction), + ]; + case "pricing": + return [ + heading(2, s.title), + text(s.description), + ...(s.tiers ?? []).map((tier: Settings) => + [ + heading(3, tier.name), + [text(tier.price), text(tier.priceYearly)].filter(Boolean).join(" or "), + text(tier.description), + (tier.features ?? []).map((f: string) => `- ${text(f)}`).join("\n"), + link(tier.buttonCaption, tier.buttonAction), + ] + .filter(Boolean) + .join("\n\n"), + ), + ]; + case "faq": + return [ + heading(2, s.title), + ...(s.items ?? []).map((item: Settings) => + [heading(3, item.question), text(item.answer)].filter(Boolean).join("\n\n"), + ), + ]; + case "banner": + return [ + heading(2, s.title), + text(s.description), + links( + [s.buttonCaption, s.buttonAction], + [s.secondaryButtonCaption, s.secondaryButtonAction], + ), + ]; + case "email-form": + case "support-form": + return [heading(2, s.title), text(s.description)]; + case "code-snippet": + return [text(s.code) ? `\`\`\`\n${text(s.code)}\n\`\`\`` : "", text(s.caption)]; + default: + return []; + } +} + +/** + * A published page layout as Markdown, for agents (`Accept: text/markdown` + * and `.md`): headings, text, buttons as links and cards as lists. + * `slot` renders FrontLit's content feed (the blog, docs or changelog list) + * where the page places it. Blocks that fail to read are skipped. + */ +export function layoutToMarkdown( + layout: StoredBlock[], + slot: () => string = () => "", +): string { + const parts: string[] = []; + for (const block of layout) { + if (block.name === DATA_SLOT_BLOCK) { + parts.push(slot()); + continue; + } + if (SKIPPED.has(block.name)) continue; + let settings: Settings; + try { + settings = { + ...fileBlockDefaults[block.name as BlockName], + ...toFileSettings(block.name, block.settings), + }; + } catch { + continue; + } + parts.push(...blockMarkdown(block.name, settings)); + } + return parts.filter((part) => part.trim()).join("\n\n"); +}