feat(core,server): record whether each reply answered the question

.changeset/answered-signal.md

@@ -0,0 +1,6 @@
+---
+'@chatcops/core': minor
+'@chatcops/server': minor
+---
+
+Each finished reply is saved with `metadata.answered`, and `metadata.unansweredReason` (`refusal`, `no_retrieval` or `no_citation`) when it didn't answer, for an answer rate and a list of questions the knowledge doesn't cover. `checkAnswered()` in `@chatcops/core` is the same check.

api/core.api.md

@@ -49,6 +49,22 @@ export interface AnalyticsStats {
     eventCounts: Record<string, number>;
 }
 
+export interface AnswerCheck {
+    reply: string;
+    retrieval?: {
+        chunks: number;
+        minScore?: number;
+        topScore?: number;
+    };
+    cited?: number;
+    phrases?: string[];
+}
+
+export interface AnswerVerdict {
+    answered: boolean;
+    reason?: UnansweredReason;
+}
+
 export interface AppendOptions {
     expectedVersion?: number;
     metadata?: Record<string, unknown>;
@@ -170,6 +186,8 @@ export interface ChatTool {
     emits?(result: ToolResult): MessagePart[];
 }
 
+export declare function checkAnswered(input: AnswerCheck): AnswerVerdict;
+
 export declare class CitationParser {
     constructor(count: number);
     push(delta: string): {
@@ -939,6 +957,8 @@ export interface TurnResult {
     error?: unknown;
 }
 
+export type UnansweredReason = 'refusal' | 'no_retrieval' | 'no_citation';
+
 export declare function uniqueChunks(chunks: RetrievedChunk[]): RetrievedChunk[];
 
 export declare function validateTools(tools: readonly ToolDefinition[]): void;

packages/core/src/analytics/index.ts

@@ -4,3 +4,5 @@ export { BufferedEventSink, statsFromSink } from './sink.js';
 export { ANALYTICS_EVENTS } from './names.js';
 export { redactPII } from './redact.js';
 export type { RedactOptions } from './redact.js';
+export { checkAnswered } from './unanswered.js';
+export type { AnswerCheck, AnswerVerdict, UnansweredReason } from './unanswered.js';

packages/core/src/analytics/unanswered.ts

@@ -0,0 +1,80 @@
+/** Why a reply counts as not answering the question. */
+export type UnansweredReason = 'refusal' | 'no_retrieval' | 'no_citation';
+
+export interface AnswerCheck {
+  /** The reply's text. */
+  reply: string;
+  /**
+   * What retrieval found for this reply, when a knowledge source that
+   * retrieves chunks ran. `minScore` only applies to `topScore` from the same
+   * retriever, since scores aren't comparable across retrievers.
+   */
+  retrieval?: { chunks: number; topScore?: number; minScore?: number };
+  /** How many pages the reply cited. */
+  cited?: number;
+  /** More phrases that mean "I don't know", matched case-insensitively. */
+  phrases?: string[];
+}
+
+export interface AnswerVerdict {
+  answered: boolean;
+  reason?: UnansweredReason;
+}
+
+// The ways a reply says it doesn't know, in the languages the widget ships. A
+// reply is checked near its start, where a refusal usually is, so a hedge late
+// in a real answer doesn't count.
+const REFUSALS = [
+  "i don't know",
+  'i do not know',
+  "i'm not sure",
+  'i am not sure',
+  "i don't have information",
+  "i don't have any information",
+  'i do not have information',
+  "i don't have details",
+  "i couldn't find",
+  'i could not find',
+  "i can't find",
+  "i'm unable to find",
+  'no lo sé',
+  'no tengo información',
+  'no estoy seguro',
+  'no estoy segura',
+  'no encontré',
+  'je ne sais pas',
+  "je n'ai pas d'information",
+  'je ne suis pas sûr',
+  "je n'ai pas trouvé",
+  'ich weiß nicht',
+  'ich habe keine informationen',
+  'ich bin mir nicht sicher',
+  'não sei',
+  'não tenho informações',
+  'não tenho certeza',
+  'não encontrei',
+];
+const REFUSAL_WINDOW = 300;
+const MIN_WORDS_FOR_CITATION = 20;
+
+/**
+ * Whether a reply answered the visitor's question, from free signals only
+ * (F-11): the reply says it doesn't know, retrieval found nothing, or
+ * retrieval found pages the reply didn't cite. It's a heuristic for an answer
+ * rate and a list of questions to fix, not a judgement of quality.
+ */
+export function checkAnswered(input: AnswerCheck): AnswerVerdict {
+  const start = input.reply.slice(0, REFUSAL_WINDOW).toLowerCase().replace(/[‘’]/g, "'");
+  const phrases = [...REFUSALS, ...(input.phrases ?? []).map((phrase) => phrase.toLowerCase())];
+  if (phrases.some((phrase) => phrase && start.includes(phrase))) return { answered: false, reason: 'refusal' };
+
+  const retrieval = input.retrieval;
+  if (retrieval) {
+    const tooWeak =
+      retrieval.minScore !== undefined && retrieval.topScore !== undefined && retrieval.topScore < retrieval.minScore;
+    if (retrieval.chunks === 0 || tooWeak) return { answered: false, reason: 'no_retrieval' };
+    const words = input.reply.trim().split(/\s+/).filter(Boolean).length;
+    if ((input.cited ?? 0) === 0 && words > MIN_WORDS_FOR_CITATION) return { answered: false, reason: 'no_citation' };
+  }
+  return { answered: true };
+}

packages/core/src/index.ts

@@ -180,8 +180,8 @@ export { createRateLimiter } from './limits/rate-limit.js';
 export type { RateLimitRule, RateLimitResult, RateLimiterOptions } from './limits/rate-limit.js';
 
 // Analytics
-export { AnalyticsCollector, BufferedEventSink, statsFromSink, ANALYTICS_EVENTS, redactPII } from './analytics/index.js';
-export type { AnalyticsStats, RedactOptions } from './analytics/index.js';
+export { AnalyticsCollector, BufferedEventSink, statsFromSink, ANALYTICS_EVENTS, redactPII, checkAnswered } from './analytics/index.js';
+export type { AnalyticsStats, RedactOptions, AnswerCheck, AnswerVerdict, UnansweredReason } from './analytics/index.js';
 
 // i18n
 export { getLocale, getAvailableLocales } from './i18n/index.js';

packages/core/tests/analytics/unanswered.test.ts

@@ -0,0 +1,50 @@
+import { describe, expect, it } from 'vitest';
+import { checkAnswered } from '../../src/analytics/unanswered.js';
+
+const long = 'Our Team plan costs twenty dollars per seat each month and includes shared inboxes, priority support, usage reports and single sign-on for every member.';
+
+describe('checkAnswered', () => {
+  it('counts a plain reply as answered when no knowledge ran', () => {
+    expect(checkAnswered({ reply: 'Hi! How can I help?' })).toEqual({ answered: true });
+  });
+
+  it.each([
+    "I don't know the answer to that.",
+    'I’m not sure, but you could ask support.',
+    "Sorry, I couldn't find anything about refunds.",
+    'No tengo información sobre eso.',
+    "Je n'ai pas d'information à ce sujet.",
+    'Ich bin mir nicht sicher.',
+    'Não encontrei nada sobre isso.',
+  ])('flags a refusal: %s', (reply) => {
+    expect(checkAnswered({ reply })).toEqual({ answered: false, reason: 'refusal' });
+  });
+
+  it('looks for a refusal near the start only, not a hedge late in a real answer', () => {
+    const reply = `${long.repeat(3)} Beyond that I don't know of other plans.`;
+    expect(checkAnswered({ reply })).toEqual({ answered: true });
+  });
+
+  it('takes extra phrases, in any case', () => {
+    expect(checkAnswered({ reply: 'That is outside what I cover.', phrases: ['OUTSIDE WHAT I COVER'] })).toEqual({ answered: false, reason: 'refusal' });
+    expect(checkAnswered({ reply: 'Fine.', phrases: [''] })).toEqual({ answered: true });
+  });
+
+  it('flags a reply when retrieval found nothing', () => {
+    expect(checkAnswered({ reply: 'Here is what I know.', retrieval: { chunks: 0 } })).toEqual({ answered: false, reason: 'no_retrieval' });
+  });
+
+  it('flags a reply when the best chunk scores under minScore', () => {
+    expect(checkAnswered({ reply: 'Yes.', retrieval: { chunks: 3, topScore: 0.2, minScore: 0.5 }, cited: 1 })).toEqual({ answered: false, reason: 'no_retrieval' });
+    expect(checkAnswered({ reply: 'Yes.', retrieval: { chunks: 3, topScore: 0.7, minScore: 0.5 }, cited: 1 })).toEqual({ answered: true });
+  });
+
+  it('flags a long reply that cites none of the pages retrieval found', () => {
+    expect(checkAnswered({ reply: long, retrieval: { chunks: 2 }, cited: 0 })).toEqual({ answered: false, reason: 'no_citation' });
+  });
+
+  it('accepts a long reply that cites, and a short one that does not', () => {
+    expect(checkAnswered({ reply: long, retrieval: { chunks: 2 }, cited: 1 })).toEqual({ answered: true });
+    expect(checkAnswered({ reply: 'You are welcome!', retrieval: { chunks: 2 } })).toEqual({ answered: true });
+  });
+});

packages/server/src/handler.ts

@@ -4,6 +4,7 @@ import {
   ConversationManager,
   AnalyticsCollector,
   ANALYTICS_EVENTS,
+  checkAnswered,
   BufferedEventSink,
   statsFromSink,
   type AnalyticsStats,
@@ -337,6 +338,7 @@ export function createChatHandler(config: ChatCopsServerConfig) {
     // changes per request goes in the context after it.
     const context: string[] = [];
     let sources: RetrievedChunk[] = [];
+    const retrieving = config.knowledge?.some((source) => source.retrieve) ?? false;
     if (config.knowledge?.length) {
       const found = await knowledgeContext(config.knowledge, req.message, historyBefore(conversationMessages, req), signal);
       if (found.context) context.push(found.context);
@@ -396,8 +398,11 @@ export function createChatHandler(config: ChatCopsServerConfig) {
           });
         pendingWebhooks.add(delivery);
       },
-      onPersist: async (message, { final }) => {
-        const complete = message.status === 'complete';
+      onPersist: async (persisted, { final }) => {
+        const complete = persisted.status === 'complete';
+        // A finished reply records whether it answered, for an answer rate
+        // and a list of questions the knowledge doesn't cover (F-11).
+        const message = final && complete && persisted.content ? withAnswerVerdict(persisted, sources, retrieving) : persisted;
         if (!saved) {
           if (final && !message.content && !message.parts?.length) return;
           // A reply the visitor walked away from is kept, marked `aborted`,
@@ -621,3 +626,15 @@ function findLastIndex<T>(array: T[], predicate: (item: T) => boolean): number {
   return -1;
 }
 
+/** The reply with `answered` (and `unansweredReason` when it didn't) in its metadata. */
+function withAnswerVerdict(message: ChatMessage, sources: RetrievedChunk[], retrieving: boolean): ChatMessage {
+  const verdict = checkAnswered({
+    reply: message.content,
+    ...(retrieving ? { retrieval: { chunks: sources.length } } : {}),
+    cited: message.parts?.filter((part) => part.type === 'source').length ?? 0,
+  });
+  return {
+    ...message,
+    metadata: { ...message.metadata, answered: verdict.answered, ...(verdict.reason ? { unansweredReason: verdict.reason } : {}) },
+  };
+}

packages/server/tests/handler-answered.test.ts

@@ -0,0 +1,54 @@
+import { memoryStorage, type KnowledgeSource, type RetrievedChunk } from '@chatcops/core';
+import { scriptedProvider, textStep } from '@chatcops/core/testing/provider';
+import { describe, expect, it } from 'vitest';
+import { createChatHandler } from '../src/handler.js';
+
+const chunk = (id: string, text: string): RetrievedChunk => ({
+  id,
+  text,
+  score: 1,
+  source: { id: `https://example.test/${id}`, url: `https://example.test/${id}`, title: `Page ${id}` },
+});
+const retrieving = (chunks: RetrievedChunk[]): KnowledgeSource => ({ type: 'test', retrieve: async () => chunks, getContext: async () => '' });
+const long = 'Our Team plan costs twenty dollars per seat each month and includes shared inboxes, priority support, usage reports and single sign-on for every member.';
+
+async function reply(options: { provider: ReturnType<typeof scriptedProvider>; knowledge?: KnowledgeSource[] }) {
+  const storage = memoryStorage();
+  const { handleChat, cleanup } = createChatHandler({ allowAnonymous: true, systemPrompt: 'Be brief.', storage, ...options });
+  for await (const _ of handleChat({ conversationId: 'c1', message: 'How much is Team?' }, { protocol: 'v1' } as never)) {
+    // Only what's stored matters here.
+  }
+  cleanup();
+  return (await storage.conversations.get('c1'))?.messages.at(-1);
+}
+
+describe('answered verdict on saved replies', () => {
+  it('marks a cited reply as answered', async () => {
+    const provider = scriptedProvider(
+      [[
+        { type: 'text-delta', delta: long },
+        { type: 'source', index: 0 },
+        { type: 'finish', reason: 'stop', usage: { inputTokens: 1, outputTokens: 1 }, parts: [{ type: 'text', text: long }] },
+      ]],
+      { citations: 'native' }
+    );
+    const saved = await reply({ provider, knowledge: [retrieving([chunk('a', 'Team is $20.')])] });
+    expect(saved?.metadata).toMatchObject({ answered: true });
+    expect(saved?.metadata).not.toHaveProperty('unansweredReason');
+  });
+
+  it('flags a long reply that cites none of the retrieved pages', async () => {
+    const saved = await reply({ provider: scriptedProvider([textStep(long)]), knowledge: [retrieving([chunk('a', 'Team is $20.')])] });
+    expect(saved?.metadata).toMatchObject({ answered: false, unansweredReason: 'no_citation' });
+  });
+
+  it('flags a reply when retrieval found nothing', async () => {
+    const saved = await reply({ provider: scriptedProvider([textStep('Hmm.')]), knowledge: [retrieving([])] });
+    expect(saved?.metadata).toMatchObject({ answered: false, unansweredReason: 'no_retrieval' });
+  });
+
+  it('flags a refusal without any knowledge, and marks a plain reply answered', async () => {
+    expect((await reply({ provider: scriptedProvider([textStep("I don't know that one.")]) }))?.metadata).toMatchObject({ answered: false, unansweredReason: 'refusal' });
+    expect((await reply({ provider: scriptedProvider([textStep('Team is $20 a month.')]) }))?.metadata).toMatchObject({ answered: true });
+  });
+});

website/src/content/docs/core/analytics.mdx

@@ -174,6 +174,25 @@ ChatCops.on('message', (...args: unknown[]) => {
 // conversation.started, message.sent, message.received, lead.captured
 ```
 
+## Whether a reply answered
+
+Each finished reply is saved with `metadata.answered` (`true` or `false`), and `metadata.unansweredReason` when it's `false`, so you can work out an answer rate and find the questions your knowledge doesn't cover. A reply counts as not answered when:
+
+- it says it doesn't know near its start ("I don't know", "I couldn't find", and the same in Spanish, French, German and Portuguese): `refusal`
+- a knowledge source that retrieves pages ran and found nothing: `no_retrieval`
+- retrieval found pages, but a reply longer than 20 words cited none of them: `no_citation`
+
+It's a heuristic built from those free signals, not a judgement of quality. `checkAnswered` from `@chatcops/core` is the same check, if you want to run it yourself:
+
+```typescript
+import { checkAnswered } from '@chatcops/core';
+
+checkAnswered({ reply: "Sorry, I couldn't find that.", retrieval: { chunks: 0 } });
+// { answered: false, reason: 'refusal' }
+```
+
+It also takes `phrases` (more ways your bot says it doesn't know) and, in `retrieval`, a `topScore` with a `minScore` to count weak retrieval as nothing found.
+
 ## Redacting personal data
 
 `redactPII()` takes personal data out of text, so that what a visitor typed can be kept without it. Each kind is replaced with a tag: