diff --git a/docs/analytics.html b/docs/analytics.html index 188d9fe..8acee0a 100644 --- a/docs/analytics.html +++ b/docs/analytics.html @@ -23,6 +23,24 @@ .empty { color: #f87171; } .error { color: #f87171; padding: 24px; text-align: center; } @media (max-width: 768px) { .chart-row { grid-template-columns: 1fr; } } + + /* Filter bar styles */ + .filter-row { display: flex; flex-wrap: wrap; align-items: center; gap: 6px; margin-bottom: 16px; } + .filter-row .spacer { flex: 1; } + .filter-row .divider { width: 1px; height: 20px; background: #334155; margin: 0 6px; } + .filter-row .pill { + display: inline-flex; align-items: center; gap: 4px; + padding: 5px 12px; border-radius: 20px; border: 1px solid #334155; + background: #1e293b; color: #94a3b8; font-size: 0.78rem; font-weight: 500; + cursor: pointer; transition: all 0.15s ease; user-select: none; + } + .filter-row .pill:hover { border-color: #3b82f6; color: #e2e8f0; } + .filter-row .pill.active { background: #3b82f6; border-color: #3b82f6; color: #ffffff; } + .filter-row .pill .count { opacity: 0.7; font-size: 0.72rem; } + .tool-badge { + display: inline-block; padding: 2px 8px; border-radius: 10px; + font-size: 0.7rem; font-weight: 500; background: #334155; color: #94a3b8; + } @@ -33,7 +51,7 @@

Pathfinder Analytics

-

Queries per Day (7d)

+

Queries per Day (7d)

@@ -42,9 +60,13 @@

Queries by Source

+ +

Top Queries (7d)

- +
QueryCountAvg ResultsAvg Score
QueryToolCountAvg ResultsAvg Score
@@ -65,6 +87,12 @@

Analytics Token

diff --git a/src/__tests__/analytics-endpoints.test.ts b/src/__tests__/analytics-endpoints.test.ts index 67d6956..951761f 100644 --- a/src/__tests__/analytics-endpoints.test.ts +++ b/src/__tests__/analytics-endpoints.test.ts @@ -38,10 +38,11 @@ vi.mock("../config.js", () => ({ hasBashSemanticSearch: vi.fn().mockReturnValue(false), })); -import { getAnalyticsConfig } from "../config.js"; +import { getAnalyticsConfig, getConfig } from "../config.js"; import { analyticsAuth } from "../server.js"; const mockGetAnalyticsConfigFn = vi.mocked(getAnalyticsConfig); +const mockGetConfigFn = vi.mocked(getConfig); function mockRes() { const json = vi.fn(); @@ -272,6 +273,71 @@ describe("analyticsAuth middleware", () => { expect(res.status).toHaveBeenCalledWith(403); }); + + it("skips token check in development mode", () => { + mockGetAnalyticsConfigFn.mockReturnValue({ + enabled: true, + log_queries: true, + retention_days: 90, + token: "secret", + }); + // Override getConfig to return nodeEnv: "development" + mockGetConfigFn.mockReturnValue({ + port: 3001, + databaseUrl: "pglite:///tmp/test", + openaiApiKey: "", + githubToken: "", + githubWebhookSecret: "", + nodeEnv: "development", + logLevel: "info", + cloneDir: "/tmp/test", + slackBotToken: "", + slackSigningSecret: "", + discordBotToken: "", + discordPublicKey: "", + notionToken: "", + }); + const res = mockRes(); + const next = vi.fn(); + + // No auth header — should still pass in dev mode + analyticsAuth({ headers: {} } as never, res as never, next); + + expect(next).toHaveBeenCalled(); + expect(res.status).not.toHaveBeenCalled(); + }); + + it("requires token in non-dev mode even when analytics is enabled", () => { + mockGetAnalyticsConfigFn.mockReturnValue({ + enabled: true, + log_queries: true, + retention_days: 90, + token: "secret", + }); + // Explicitly set nodeEnv to "production" + mockGetConfigFn.mockReturnValue({ + port: 3001, + databaseUrl: "pglite:///tmp/test", + openaiApiKey: "", + githubToken: "", + githubWebhookSecret: "", + nodeEnv: "production", + logLevel: "info", + cloneDir: "/tmp/test", + slackBotToken: "", + slackSigningSecret: "", + discordBotToken: "", + discordPublicKey: "", + notionToken: "", + }); + const res = mockRes(); + const next = vi.fn(); + + analyticsAuth({ headers: {} } as never, res as never, next); + + expect(next).not.toHaveBeenCalled(); + expect(res.status).toHaveBeenCalledWith(401); + }); }); // --------------------------------------------------------------------------- diff --git a/src/__tests__/analytics.test.ts b/src/__tests__/analytics.test.ts index 29a36cf..59d9eef 100644 --- a/src/__tests__/analytics.test.ts +++ b/src/__tests__/analytics.test.ts @@ -11,6 +11,7 @@ import { getAnalyticsSummary, getTopQueries, getEmptyQueries, + getToolCounts, cleanupOldQueryLogs, } from "../db/analytics.js"; import type { QueryLogEntry } from "../db/analytics.js"; @@ -217,17 +218,19 @@ describe("p95 computation edge cases", () => { // --------------------------------------------------------------------------- describe("getTopQueries", () => { - it("returns top queries with frequency and avg stats", async () => { + it("returns top queries with frequency, tool_name, and avg stats", async () => { mockQuery.mockResolvedValueOnce({ rows: [ { query_text: "install guide", + tool_name: "search-docs", count: 42, avg_result_count: "3.5", avg_top_score: "0.88", }, { query_text: "api reference", + tool_name: "search-code", count: 30, avg_result_count: "5.0", avg_top_score: null, @@ -239,9 +242,11 @@ describe("getTopQueries", () => { expect(result).toHaveLength(2); expect(result[0].query_text).toBe("install guide"); + expect(result[0].tool_name).toBe("search-docs"); expect(result[0].count).toBe(42); expect(result[0].avg_result_count).toBeCloseTo(3.5); expect(result[0].avg_top_score).toBeCloseTo(0.88); + expect(result[1].tool_name).toBe("search-code"); expect(result[1].avg_top_score).toBeNull(); }); @@ -250,7 +255,9 @@ describe("getTopQueries", () => { await getTopQueries(30, 10); const [, params] = mockQuery.mock.calls[0]; - expect(params).toEqual([30, 10]); + // No filter params, so just days and limit + expect(params).toContain(30); + expect(params).toContain(10); }); it("excludes redacted queries", async () => { @@ -384,3 +391,169 @@ describe("cleanupOldQueryLogs error handling", () => { await expect(cleanupOldQueryLogs(90)).rejects.toThrow("disk full"); }); }); + +// --------------------------------------------------------------------------- +// getToolCounts +// --------------------------------------------------------------------------- + +describe("getToolCounts", () => { + it("returns counts grouped by tool type prefix", async () => { + mockQuery.mockResolvedValueOnce({ + rows: [ + { tool_type: "search", count: 3033 }, + { tool_type: "explore", count: 755 }, + ], + }); + + const result = await getToolCounts(7); + + expect(result).toHaveLength(2); + expect(result[0]).toEqual({ tool_type: "search", count: 3033 }); + expect(result[1]).toEqual({ tool_type: "explore", count: 755 }); + + const [sql, params] = mockQuery.mock.calls[0]; + expect(sql).toContain("split_part(tool_name"); + expect(params).toEqual([7]); + }); + + it("returns empty array when no queries exist", async () => { + mockQuery.mockResolvedValueOnce({ rows: [] }); + const result = await getToolCounts(7); + expect(result).toEqual([]); + }); +}); + +// --------------------------------------------------------------------------- +// Filter support (tool_type, source) +// --------------------------------------------------------------------------- + +describe("getAnalyticsSummary with filters", () => { + function mockSummaryQueries() { + mockQuery + .mockResolvedValueOnce({ rows: [{ count: 500 }] }) // total + .mockResolvedValueOnce({ + rows: [{ total: 100, empty: 5, avg_latency: 50 }], + }) // 7d summary + .mockResolvedValueOnce({ rows: [] }) // latency rows + .mockResolvedValueOnce({ rows: [] }) // by source + .mockResolvedValueOnce({ rows: [] }); // per day + } + + it("passes tool_type filter as LIKE clause to all queries", async () => { + mockSummaryQueries(); + await getAnalyticsSummary({ tool_type: "search" }); + + // All 5 queries should have the filter + for (let i = 0; i < 5; i++) { + const [sql, params] = mockQuery.mock.calls[i]; + expect(sql).toContain("tool_name LIKE"); + expect(params).toContain("search"); + } + }); + + it("passes source filter as exact match to all queries", async () => { + mockSummaryQueries(); + await getAnalyticsSummary({ source: "docs" }); + + for (let i = 0; i < 5; i++) { + const [sql, params] = mockQuery.mock.calls[i]; + expect(sql).toContain("source_name ="); + expect(params).toContain("docs"); + } + }); + + it("passes both filters when provided", async () => { + mockSummaryQueries(); + await getAnalyticsSummary({ tool_type: "search", source: "docs" }); + + const [sql, params] = mockQuery.mock.calls[0]; + expect(sql).toContain("tool_name LIKE"); + expect(sql).toContain("source_name ="); + expect(params).toContain("search"); + expect(params).toContain("docs"); + }); + + it("omits filter clauses when no filter provided", async () => { + mockSummaryQueries(); + await getAnalyticsSummary({}); + + const [sql] = mockQuery.mock.calls[0]; + expect(sql).not.toContain("tool_name LIKE"); + expect(sql).not.toContain("source_name ="); + }); +}); + +describe("getTopQueries with filters", () => { + it("includes tool_type filter in WHERE clause", async () => { + mockQuery.mockResolvedValueOnce({ rows: [] }); + await getTopQueries(7, 50, { tool_type: "explore" }); + + const [sql, params] = mockQuery.mock.calls[0]; + expect(sql).toContain("tool_name LIKE"); + expect(params).toContain("explore"); + }); + + it("includes source filter in WHERE clause", async () => { + mockQuery.mockResolvedValueOnce({ rows: [] }); + await getTopQueries(7, 50, { source: "ag-ui-docs" }); + + const [sql, params] = mockQuery.mock.calls[0]; + expect(sql).toContain("source_name ="); + expect(params).toContain("ag-ui-docs"); + }); +}); + +describe("getEmptyQueries with filters", () => { + it("includes tool_type filter in WHERE clause", async () => { + mockQuery.mockResolvedValueOnce({ rows: [] }); + await getEmptyQueries(7, 50, { tool_type: "search" }); + + const [sql, params] = mockQuery.mock.calls[0]; + expect(sql).toContain("tool_name LIKE"); + expect(params).toContain("search"); + }); +}); + +// --------------------------------------------------------------------------- +// avg_result_count null handling +// --------------------------------------------------------------------------- + +describe("getTopQueries null avg_result_count", () => { + it("returns null avg_result_count when SQL returns null (backfilled data)", async () => { + mockQuery.mockResolvedValueOnce({ + rows: [ + { + query_text: "cat /INDEX.md", + tool_name: "explore-docs", + count: 17, + avg_result_count: null, + avg_top_score: null, + }, + ], + }); + + const result = await getTopQueries(7, 50); + + expect(result[0].avg_result_count).toBeNull(); + expect(result[0].avg_top_score).toBeNull(); + }); + + it("returns 0 avg_result_count when SQL returns '0' (real zero-result queries)", async () => { + mockQuery.mockResolvedValueOnce({ + rows: [ + { + query_text: "nonexistent", + tool_name: "search-docs", + count: 5, + avg_result_count: "0", + avg_top_score: "0.0", + }, + ], + }); + + const result = await getTopQueries(7, 50); + + expect(result[0].avg_result_count).toBe(0); + expect(result[0].avg_top_score).toBe(0); + }); +}); diff --git a/src/db/analytics.ts b/src/db/analytics.ts index 9757f5e..75e792e 100644 --- a/src/db/analytics.ts +++ b/src/db/analytics.ts @@ -27,8 +27,9 @@ export interface AnalyticsSummary { export interface TopQuery { query_text: string; + tool_name: string; count: number; - avg_result_count: number; + avg_result_count: number | null; avg_top_score: number | null; } @@ -40,6 +41,16 @@ export interface EmptyQuery { last_seen: string; } +export interface ToolCount { + tool_type: string; + count: number; +} + +export interface AnalyticsFilter { + tool_type?: string; + source?: string; +} + // --------------------------------------------------------------------------- // Write // --------------------------------------------------------------------------- @@ -69,6 +80,41 @@ export async function logQuery( ); } +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** + * Build WHERE clause fragments and params for tool_type and source filters. + * Returns { clauses: string[], params: any[], nextIdx: number }. + */ +function buildFilterClauses( + filter: AnalyticsFilter, + startIdx: number = 1, +): { clauses: string[]; params: unknown[]; nextIdx: number } { + const clauses: string[] = []; + const params: unknown[] = []; + let idx = startIdx; + + if (filter.tool_type) { + clauses.push(`tool_name LIKE $${idx} || '-%'`); + params.push(filter.tool_type); + idx++; + } + if (filter.source) { + clauses.push(`source_name = $${idx}`); + params.push(filter.source); + idx++; + } + + return { clauses, params, nextIdx: idx }; +} + +function whereAnd(baseClauses: string[], filterClauses: string[]): string { + const all = [...baseClauses, ...filterClauses]; + return all.length > 0 ? "WHERE " + all.join(" AND ") : ""; +} + // --------------------------------------------------------------------------- // Read // --------------------------------------------------------------------------- @@ -88,43 +134,74 @@ function computeP95(latencies: number[]): number { /** * Get a summary of analytics data. */ -export async function getAnalyticsSummary(): Promise { +export async function getAnalyticsSummary( + filter: AnalyticsFilter = {}, +): Promise { const pool = getPool(); - // Note: percentile_cont is NOT used in SQL because PGlite doesn't support it. - // Instead, we fetch latencies and compute p95 in application code. - const [totalRes, summary7dRes, latencyRes, bySourceRes, perDayRes] = - await Promise.all([ - pool.query("SELECT count(*)::int AS count FROM query_log"), - pool.query(` - SELECT - count(*)::int AS total, - count(*) FILTER (WHERE result_count = 0)::int AS empty, - COALESCE(avg(latency_ms)::int, 0) AS avg_latency - FROM query_log - WHERE created_at > NOW() - INTERVAL '7 days' - `), - pool.query(` - SELECT latency_ms FROM query_log - WHERE created_at > NOW() - INTERVAL '7 days' - ORDER BY latency_ms - `), - pool.query(` - SELECT source_name, count(*)::int AS count - FROM query_log - WHERE source_name IS NOT NULL - AND created_at > NOW() - INTERVAL '7 days' - GROUP BY source_name - ORDER BY count DESC - `), - pool.query(` - SELECT date_trunc('day', created_at)::date::text AS day, count(*)::int AS count - FROM query_log - WHERE created_at > NOW() - INTERVAL '7 days' - GROUP BY day - ORDER BY day - `), - ]); + const { clauses: fc, params: fp } = buildFilterClauses(filter); + + // Total queries (all time, filtered) + const totalWhere = whereAnd([], fc); + const totalRes = await pool.query( + `SELECT count(*)::int AS count FROM query_log ${totalWhere}`, + fp, + ); + + // 7d summary (filtered) - exclude backfilled rows from latency/empty stats + const { clauses: fc2, params: fp2, nextIdx: _ } = buildFilterClauses(filter); + const base7d = ["created_at > NOW() - INTERVAL '7 days'"]; + const summary7dWhere = whereAnd(base7d, fc2); + const summary7dRes = await pool.query( + `SELECT + count(*)::int AS total, + count(*) FILTER (WHERE result_count = 0)::int AS empty, + COALESCE(avg(latency_ms) FILTER (WHERE latency_ms >= 0)::int, 0) AS avg_latency + FROM query_log + ${summary7dWhere}`, + fp2, + ); + + // Latencies for p95 (exclude backfilled rows with latency_ms=-1) + const { clauses: fc3, params: fp3 } = buildFilterClauses(filter); + const latencyBase = [ + "created_at > NOW() - INTERVAL '7 days'", + "latency_ms >= 0", + ]; + const latencyWhere = whereAnd(latencyBase, fc3); + const latencyRes = await pool.query( + `SELECT latency_ms FROM query_log ${latencyWhere} ORDER BY latency_ms`, + fp3, + ); + + // By source (7d, filtered) + const { clauses: fc4, params: fp4 } = buildFilterClauses(filter); + const sourceBase = [ + "source_name IS NOT NULL", + "created_at > NOW() - INTERVAL '7 days'", + ]; + const sourceWhere = whereAnd(sourceBase, fc4); + const bySourceRes = await pool.query( + `SELECT source_name, count(*)::int AS count + FROM query_log + ${sourceWhere} + GROUP BY source_name + ORDER BY count DESC`, + fp4, + ); + + // Per day (7d, filtered) + const { clauses: fc5, params: fp5 } = buildFilterClauses(filter); + const dayBase = ["created_at > NOW() - INTERVAL '7 days'"]; + const dayWhere = whereAnd(dayBase, fc5); + const perDayRes = await pool.query( + `SELECT date_trunc('day', created_at)::date::text AS day, count(*)::int AS count + FROM query_log + ${dayWhere} + GROUP BY day + ORDER BY day`, + fp5, + ); const totalQueries = totalRes.rows[0]?.count ?? 0; const s = summary7dRes.rows[0] ?? {}; @@ -157,33 +234,45 @@ export async function getAnalyticsSummary(): Promise { /** * Get top queries by frequency over the last N days. + * Now includes tool_name in the grouping. */ export async function getTopQueries( days: number = 7, limit: number = 50, + filter: AnalyticsFilter = {}, ): Promise { const pool = getPool(); + + const { clauses: fc, params: fp, nextIdx } = buildFilterClauses(filter); + const baseClauses = [ + `created_at > NOW() - INTERVAL '1 day' * $${nextIdx}`, + "query_text != ''", + ]; + const where = whereAnd(baseClauses, fc); + const { rows } = await pool.query( - ` - SELECT - query_text, - count(*)::int AS count, - avg(result_count)::real AS avg_result_count, - avg(top_score)::real AS avg_top_score - FROM query_log - WHERE created_at > NOW() - INTERVAL '1 day' * $1 - AND query_text != '' - GROUP BY query_text - ORDER BY count DESC - LIMIT $2 - `, - [days, limit], + `SELECT + query_text, + tool_name, + count(*)::int AS count, + avg(result_count) FILTER (WHERE result_count >= 0)::real AS avg_result_count, + avg(top_score) FILTER (WHERE top_score IS NOT NULL)::real AS avg_top_score + FROM query_log + ${where} + GROUP BY query_text, tool_name + ORDER BY count DESC + LIMIT $${nextIdx + 1}`, + [...fp, days, limit], ); return rows.map((r: Record) => ({ query_text: r.query_text as string, + tool_name: r.tool_name as string, count: r.count as number, - avg_result_count: parseFloat(r.avg_result_count as string) || 0, + avg_result_count: + r.avg_result_count != null + ? parseFloat(r.avg_result_count as string) + : null, avg_top_score: r.avg_top_score != null ? parseFloat(r.avg_top_score as string) : null, })); @@ -195,25 +284,31 @@ export async function getTopQueries( export async function getEmptyQueries( days: number = 7, limit: number = 50, + filter: AnalyticsFilter = {}, ): Promise { const pool = getPool(); + + const { clauses: fc, params: fp, nextIdx } = buildFilterClauses(filter); + const baseClauses = [ + "result_count = 0", + `created_at > NOW() - INTERVAL '1 day' * $${nextIdx}`, + "query_text != ''", + ]; + const where = whereAnd(baseClauses, fc); + const { rows } = await pool.query( - ` - SELECT - query_text, - tool_name, - source_name, - count(*)::int AS count, - max(created_at)::text AS last_seen - FROM query_log - WHERE result_count = 0 - AND created_at > NOW() - INTERVAL '1 day' * $1 - AND query_text != '' - GROUP BY query_text, tool_name, source_name - ORDER BY count DESC - LIMIT $2 - `, - [days, limit], + `SELECT + query_text, + tool_name, + source_name, + count(*)::int AS count, + max(created_at)::text AS last_seen + FROM query_log + ${where} + GROUP BY query_text, tool_name, source_name + ORDER BY count DESC + LIMIT $${nextIdx + 1}`, + [...fp, days, limit], ); return rows.map((r: Record) => ({ @@ -225,6 +320,28 @@ export async function getEmptyQueries( })); } +/** + * Get query counts grouped by tool type prefix (e.g. "search", "explore"). + */ +export async function getToolCounts(days: number = 7): Promise { + const pool = getPool(); + const { rows } = await pool.query( + `SELECT + split_part(tool_name, '-', 1) AS tool_type, + count(*)::int AS count + FROM query_log + WHERE created_at > NOW() - INTERVAL '1 day' * $1 + GROUP BY tool_type + ORDER BY count DESC`, + [days], + ); + + return rows.map((r: Record) => ({ + tool_type: r.tool_type as string, + count: r.count as number, + })); +} + // --------------------------------------------------------------------------- // Cleanup // --------------------------------------------------------------------------- diff --git a/src/server.ts b/src/server.ts index b0ff05b..83c766a 100644 --- a/src/server.ts +++ b/src/server.ts @@ -39,7 +39,9 @@ import { getAnalyticsSummary, getTopQueries, getEmptyQueries, + getToolCounts, } from "./db/analytics.js"; +import type { AnalyticsFilter } from "./db/analytics.js"; import path from "node:path"; import { fileURLToPath } from "node:url"; @@ -635,13 +637,21 @@ export function analyticsAuth( next: express.NextFunction, ): void { const analyticsCfg = getAnalyticsConfig(); - const token = getAnalyticsToken(); + const config = getConfig(); if (!analyticsCfg?.enabled) { res.status(404).json({ error: "Analytics not enabled" }); return; } + // Skip token check in development mode + if (config.nodeEnv === "development") { + next(); + return; + } + + const token = getAnalyticsToken(); + if (!token) { // Should not happen — getAnalyticsToken auto-generates if needed next(); @@ -665,12 +675,20 @@ export function analyticsAuth( next(); } +function parseAnalyticsFilter(req: Request): AnalyticsFilter { + const filter: AnalyticsFilter = {}; + if (req.query.tool_type) filter.tool_type = req.query.tool_type as string; + if (req.query.source) filter.source = req.query.source as string; + return filter; +} + app.get( "/api/analytics/summary", analyticsAuth, - async (_req: Request, res: Response) => { + async (req: Request, res: Response) => { try { - const summary = await getAnalyticsSummary(); + const filter = parseAnalyticsFilter(req); + const summary = await getAnalyticsSummary(filter); res.json(summary); } catch (err) { console.error("[analytics] Summary query failed:", err); @@ -686,7 +704,8 @@ app.get( try { const days = parseInt(req.query.days as string) || 7; const limit = parseInt(req.query.limit as string) || 50; - const queries = await getTopQueries(days, Math.min(limit, 200)); + const filter = parseAnalyticsFilter(req); + const queries = await getTopQueries(days, Math.min(limit, 200), filter); res.json(queries); } catch (err) { console.error("[analytics] Top queries failed:", err); @@ -702,7 +721,8 @@ app.get( try { const days = parseInt(req.query.days as string) || 7; const limit = parseInt(req.query.limit as string) || 50; - const queries = await getEmptyQueries(days, Math.min(limit, 200)); + const filter = parseAnalyticsFilter(req); + const queries = await getEmptyQueries(days, Math.min(limit, 200), filter); res.json(queries); } catch (err) { console.error("[analytics] Empty queries failed:", err); @@ -711,6 +731,26 @@ app.get( }, ); +app.get( + "/api/analytics/tool-counts", + analyticsAuth, + async (req: Request, res: Response) => { + try { + const days = parseInt(req.query.days as string) || 7; + const counts = await getToolCounts(days); + res.json(counts); + } catch (err) { + console.error("[analytics] Tool counts failed:", err); + res.status(500).json({ error: "Failed to fetch tool counts" }); + } + }, +); + +app.get("/api/analytics/auth-mode", (_req: Request, res: Response) => { + const config = getConfig(); + res.json({ dev: config.nodeEnv === "development" }); +}); + app.get("/analytics", (_req: Request, res: Response) => { if (!getAnalyticsConfig()?.enabled) { res.status(404).json({ error: "Analytics not enabled" });