Skip to content

Commit eaababa

Browse files
codercatdevcodercatdev
committed
feat(site): add x-markdown-tokens, Content-Signal, fast HTML-to-MD conversion, and WebMCP AbortSignal support
1 parent 7f6ed62 commit eaababa

6 files changed

Lines changed: 161 additions & 37 deletions

File tree

‎apps/site/src/components/AjAnimatedMascot.astro‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ export interface Props {
4242
mouth?: AjMouthViseme;
4343
caption?: string;
4444
cls?: string;
45+
size?: number;
4546
showYarnBall?: boolean;
4647
showPivots?: boolean;
4748
interactive?: boolean;

‎apps/site/src/env.d.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ declare namespace astroHTML.JSX {
4040
interface FormHTMLAttributes {
4141
toolname?: string;
4242
tooldescription?: string;
43+
toolautosubmit?: string;
4344
}
4445
interface InputHTMLAttributes {
4546
toolparamdescription?: string;

‎apps/site/src/layouts/BaseLayout.astro‎

Lines changed: 94 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -169,13 +169,20 @@ const graph = buildGraph([
169169
{/* WebMCP API: Expose client-side tools to browser AI agents */}
170170
<script is:inline>
171171
(function() {
172-
const mc = document.modelContext || (typeof navigator !== "undefined" && navigator.modelContext);
173-
if (mc && typeof mc.registerTool === "function") {
174-
try {
175-
const controller = new AbortController();
176-
mc.registerTool({
172+
var registered = false;
173+
var controller = typeof AbortController !== "undefined" ? new AbortController() : null;
174+
175+
function registerCodingCatWebMcp() {
176+
if (registered) return;
177+
var mc = document.modelContext || (typeof navigator !== "undefined" && navigator.modelContext);
178+
if (!mc || typeof mc.registerTool !== "function") return;
179+
registered = true;
180+
var opts = controller ? { signal: controller.signal } : undefined;
181+
182+
var tools = [
183+
{
177184
name: "search_content",
178-
description: "Search CodingCat.dev tutorials, articles, podcasts, guests, and sponsors",
185+
description: "Search CodingCat.dev courses, podcasts, blog posts, guests, and sponsors",
179186
annotations: {
180187
readOnlyHint: true,
181188
untrustedContentHint: true,
@@ -185,25 +192,100 @@ const graph = buildGraph([
185192
properties: {
186193
query: {
187194
type: "string",
188-
description: "Search keyword, technical topic, or library name (e.g. Astro, React, Sanity, TypeScript)",
195+
description: "Search keyword, technical topic, or library name (for example: Astro, React, Sanity, TypeScript, Cloudflare)",
189196
},
190197
},
191198
required: ["query"],
192199
},
193-
execute: async ({ query }) => {
200+
execute: async function(input) {
194201
try {
195-
const res = await fetch("/api/search?q=" + encodeURIComponent(query));
202+
var q = (input && input.query) ? String(input.query) : "";
203+
var res = await fetch("/api/search?q=" + encodeURIComponent(q));
196204
if (!res.ok) throw new Error("Search failed with status " + res.status);
197205
return await res.json();
198206
} catch (err) {
199207
return { error: String(err) };
200208
}
201209
},
202-
}, { signal: controller.signal });
203-
} catch (e) {
204-
console.debug("WebMCP registration error:", e);
210+
},
211+
{
212+
name: "ask_agent_aj",
213+
description: "Ask Agent AJ technical questions about CodingCat.dev courses, podcast episodes, blog posts, and transcripts",
214+
annotations: {
215+
readOnlyHint: true,
216+
untrustedContentHint: true,
217+
},
218+
inputSchema: {
219+
type: "object",
220+
properties: {
221+
prompt: {
222+
type: "string",
223+
description: "Natural language question for Agent AJ",
224+
},
225+
},
226+
required: ["prompt"],
227+
},
228+
execute: async function(input) {
229+
try {
230+
var prompt = (input && input.prompt) ? String(input.prompt) : "";
231+
var res = await fetch("/api/chat", {
232+
method: "POST",
233+
headers: { "Content-Type": "application/json" },
234+
body: JSON.stringify({ messages: [{ role: "user", content: prompt }] }),
235+
});
236+
if (!res.ok) throw new Error("Chat failed with status " + res.status);
237+
return await res.json();
238+
} catch (err) {
239+
return { error: String(err) };
240+
}
241+
},
242+
},
243+
{
244+
name: "browse_pillar",
245+
description: "Navigate to a primary CodingCat.dev content pillar (courses, podcasts, blog, topics, or sponsorships)",
246+
annotations: {
247+
readOnlyHint: true,
248+
untrustedContentHint: false,
249+
},
250+
inputSchema: {
251+
type: "object",
252+
properties: {
253+
pillar: {
254+
type: "string",
255+
enum: ["courses", "podcasts", "blog", "topics", "sponsorships"],
256+
description: "Target content pillar to open",
257+
},
258+
},
259+
required: ["pillar"],
260+
},
261+
execute: async function(input) {
262+
var pillar = (input && input.pillar) ? String(input.pillar) : "courses";
263+
var target = "/" + pillar.replace(/^\/+/, "");
264+
window.location.assign(target);
265+
return { navigatedTo: target };
266+
},
267+
},
268+
];
269+
270+
for (var i = 0; i < tools.length; i++) {
271+
try {
272+
Promise.resolve(mc.registerTool(tools[i], opts)).catch(function() {});
273+
} catch (e) {
274+
console.debug("WebMCP registration error:", e);
275+
}
205276
}
206277
}
278+
279+
registerCodingCatWebMcp();
280+
if (typeof document !== "undefined") {
281+
document.addEventListener("DOMContentLoaded", registerCodingCatWebMcp, { once: true });
282+
}
283+
if (typeof window !== "undefined") {
284+
window.addEventListener("load", registerCodingCatWebMcp, { once: true });
285+
window.addEventListener("pagehide", function() {
286+
if (controller) controller.abort();
287+
}, { once: true });
288+
}
207289
})();
208290
</script>
209291
{/* Per-page head extras: rel=prev/next, feed alternates, preloads. */}

‎apps/site/src/lib/html-to-markdown.ts‎

Lines changed: 22 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -13,27 +13,8 @@ export function htmlToMarkdown(html: string, pageUrl?: string): string {
1313
const titleMatch = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
1414
const pageTitle = titleMatch ? decodeEntities(titleMatch[1].trim()) : "";
1515

16-
// 2. Strip noise elements: scripts, styles, SVGs, modals, nav, header, footer
17-
let text = html
18-
.replace(/<!--[\s\S]*?-->/g, "")
19-
.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, "")
20-
.replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, "")
21-
.replace(/<svg\b[^<]*(?:(?!<\/svg>)<[^<]*)*<\/svg>/gi, "")
22-
.replace(/<dialog\b[^<]*(?:(?!<\/dialog>)<[^<]*)*<\/dialog>/gi, "")
23-
.replace(
24-
/<aside\b[^>]*id=["']aj-agent-drawer["'][^>]*>[\s\S]*?<\/aside>/gi,
25-
"",
26-
)
27-
.replace(
28-
/<div[^>]*class=["'][^"']*transcript-widget[^"']*["'][^>]*>[\s\S]*?<\/footer>\s*<\/div>\s*<\/div>\s*<\/div>/gi,
29-
"",
30-
)
31-
.replace(/<button\b[^<]*(?:(?!<\/button>)<[^<]*)*<\/button>/gi, "")
32-
.replace(/<nav\b[^<]*(?:(?!<\/nav>)<[^<]*)*<\/nav>/gi, "")
33-
.replace(/<header\b[^<]*(?:(?!<\/header>)<[^<]*)*<\/header>/gi, "")
34-
.replace(/<footer\b[^<]*(?:(?!<\/footer>)<[^<]*)*<\/footer>/gi, "");
35-
36-
// 3. Focus on <main> or <article> if available
16+
// 2. Focus on <main> or <article> first so we skip <head>, <header>, <footer>, and modal drawers
17+
let text = html;
3718
const mainMatch = text.match(/<main[^>]*>([\s\S]*?)<\/main>/i);
3819
if (mainMatch) {
3920
text = mainMatch[1];
@@ -49,6 +30,26 @@ export function htmlToMarkdown(html: string, pageUrl?: string): string {
4930
}
5031
}
5132

33+
// 3. Strip noise elements inside main content: scripts, styles, SVGs, modals, nav, buttons
34+
text = text
35+
.replace(/<!--[\s\S]*?-->/g, "")
36+
.replace(/<script\b[\s\S]*?<\/script>/gi, "")
37+
.replace(/<style\b[\s\S]*?<\/style>/gi, "")
38+
.replace(/<svg\b[\s\S]*?<\/svg>/gi, "")
39+
.replace(/<dialog\b[\s\S]*?<\/dialog>/gi, "")
40+
.replace(
41+
/<aside\b[^>]*id=["']aj-agent-drawer["'][^>]*>[\s\S]*?<\/aside>/gi,
42+
"",
43+
)
44+
.replace(
45+
/<div[^>]*class=["'][^"']*transcript-widget[^"']*["'][^>]*>[\s\S]*?<\/footer>\s*<\/div>\s*<\/div>\s*<\/div>/gi,
46+
"",
47+
)
48+
.replace(/<button\b[\s\S]*?<\/button>/gi, "")
49+
.replace(/<nav\b[\s\S]*?<\/nav>/gi, "")
50+
.replace(/<header\b[\s\S]*?<\/header>/gi, "")
51+
.replace(/<footer\b[\s\S]*?<\/footer>/gi, "");
52+
5253
// 4. Code blocks (<pre><code ...>...</code></pre>)
5354
text = text.replace(
5455
/<pre[^>]*><code[^>]*class=["'][^"']*language-([a-z0-9_-]+)[^"']*["'][^>]*>([\s\S]*?)<\/code><\/pre>/gi,

‎apps/site/src/lib/sanity/context.ts‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,11 +53,32 @@ export function createSanityContext({
5353
perspectiveParam: url.searchParams.get(PERSPECTIVE_PARAM),
5454
});
5555

56+
const loadCache = new Map<string, Promise<unknown>>();
57+
const publishedCache = new Map<string, Promise<unknown>>();
58+
5659
return {
5760
config,
5861
preview,
59-
loadQuery: (query, params) => loadQuery(config, preview, query, params),
60-
fetchPublished: (query, params) => fetchPublished(config, query, params),
62+
loadQuery: (query, params) => {
63+
const key = `${query}::${params ? JSON.stringify(params) : ""}`;
64+
const hit = loadCache.get(key);
65+
if (hit) {
66+
return hit as Promise<ClientReturn<typeof query>>;
67+
}
68+
const p = loadQuery(config, preview, query, params);
69+
loadCache.set(key, p);
70+
return p;
71+
},
72+
fetchPublished: (query, params) => {
73+
const key = `${query}::${params ? JSON.stringify(params) : ""}`;
74+
const hit = publishedCache.get(key);
75+
if (hit) {
76+
return hit as Promise<ClientReturn<typeof query>>;
77+
}
78+
const p = fetchPublished(config, query, params);
79+
publishedCache.set(key, p);
80+
return p;
81+
},
6182
urlForImage: (source) => urlForImage(config, source),
6283
resolveOpenGraphImage: (image, width, height) =>
6384
resolveOpenGraphImage(config, image, width, height),

‎apps/site/src/middleware.ts‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,8 @@ export const onRequest = defineMiddleware(async (context, next) => {
4545
(pathname.endsWith(".md") || pathname.endsWith("/index.md"));
4646
const acceptMarkdown = context.request.headers
4747
.get("Accept")
48-
?.includes("text/markdown");
48+
?.toLowerCase()
49+
.includes("text/markdown");
4950
const wantsMarkdown = isMarkdownUrl || Boolean(acceptMarkdown);
5051

5152
let response: Response;
@@ -101,8 +102,12 @@ export const onRequest = defineMiddleware(async (context, next) => {
101102
"frame-ancestors 'self' https://*.sanity.studio https://*.sanity.io https://sanity.io http://localhost:3333",
102103
);
103104

104-
// AIPREF / AI Preference Headers
105+
// AIPREF / AI Preference & Cloudflare Markdown for Agents Headers
105106
response.headers.set("Content-Usage", "train-ai=y, search=y");
107+
response.headers.set(
108+
"Content-Signal",
109+
"ai-train=yes, search=yes, ai-input=yes",
110+
);
106111

107112
// Always emit Link headers (RFC 8288) for AI agent discoverability
108113
response.headers.set(
@@ -123,9 +128,11 @@ export const onRequest = defineMiddleware(async (context, next) => {
123128
pathname.replace(/\/index\.md$/, "").replace(/\.md$/, "") || "/";
124129
const canonicalUrl = `${origin}${canonicalPath}`;
125130
const markdown = htmlToMarkdown(html, canonicalUrl);
131+
const estimatedTokens = Math.max(1, Math.ceil(markdown.length / 4));
126132

127133
const headers = new Headers(response.headers);
128134
headers.set("content-type", "text/markdown; charset=utf-8");
135+
headers.set("x-markdown-tokens", String(estimatedTokens));
129136
headers.delete("content-length");
130137

131138
trackEdgeRequest({
@@ -144,6 +151,17 @@ export const onRequest = defineMiddleware(async (context, next) => {
144151
});
145152
}
146153

154+
if (
155+
contentType.includes("text/markdown") &&
156+
!response.headers.has("x-markdown-tokens")
157+
) {
158+
const cloneText = await response.clone().text();
159+
response.headers.set(
160+
"x-markdown-tokens",
161+
String(Math.max(1, Math.ceil(cloneText.length / 4))),
162+
);
163+
}
164+
147165
if (!pathname.startsWith("/ingest") && pathname !== "/api/event") {
148166
trackEdgeRequest({
149167
env: env as unknown as Record<string, unknown>,

0 commit comments

Comments
 (0)