From 97212f7ab3d1d520de01564949b07f2958280ad4 Mon Sep 17 00:00:00 2001 From: anurag629 Date: Wed, 4 Nov 2026 14:08:54 +0530 Subject: [PATCH] feat(core,server): record whether each reply answered the question --- .changeset/answered-signal.md | 6 ++ api/core.api.md | 20 +++++ packages/core/src/analytics/index.ts | 2 + packages/core/src/analytics/unanswered.ts | 80 +++++++++++++++++++ packages/core/src/index.ts | 4 +- .../core/tests/analytics/unanswered.test.ts | 50 ++++++++++++ packages/server/src/handler.ts | 21 ++++- .../server/tests/handler-answered.test.ts | 54 +++++++++++++ website/src/content/docs/core/analytics.mdx | 19 +++++ 9 files changed, 252 insertions(+), 4 deletions(-) create mode 100644 .changeset/answered-signal.md create mode 100644 packages/core/src/analytics/unanswered.ts create mode 100644 packages/core/tests/analytics/unanswered.test.ts create mode 100644 packages/server/tests/handler-answered.test.ts diff --git a/.changeset/answered-signal.md b/.changeset/answered-signal.md new file mode 100644 index 00000000..7bf70858 --- /dev/null +++ b/.changeset/answered-signal.md @@ -0,0 +1,6 @@ +--- +'@chatcops/core': minor +'@chatcops/server': minor +--- + +Each finished reply is saved with `metadata.answered`, and `metadata.unansweredReason` (`refusal`, `no_retrieval` or `no_citation`) when it didn't answer, for an answer rate and a list of questions the knowledge doesn't cover. `checkAnswered()` in `@chatcops/core` is the same check. diff --git a/api/core.api.md b/api/core.api.md index 3b72544a..8bdc90e3 100644 --- a/api/core.api.md +++ b/api/core.api.md @@ -49,6 +49,22 @@ export interface AnalyticsStats { eventCounts: Record; } +export interface AnswerCheck { + reply: string; + retrieval?: { + chunks: number; + minScore?: number; + topScore?: number; + }; + cited?: number; + phrases?: string[]; +} + +export interface AnswerVerdict { + answered: boolean; + reason?: UnansweredReason; +} + export interface AppendOptions { expectedVersion?: number; metadata?: Record; @@ -170,6 +186,8 @@ export interface ChatTool { emits?(result: ToolResult): MessagePart[]; } +export declare function checkAnswered(input: AnswerCheck): AnswerVerdict; + export declare class CitationParser { constructor(count: number); push(delta: string): { @@ -939,6 +957,8 @@ export interface TurnResult { error?: unknown; } +export type UnansweredReason = 'refusal' | 'no_retrieval' | 'no_citation'; + export declare function uniqueChunks(chunks: RetrievedChunk[]): RetrievedChunk[]; export declare function validateTools(tools: readonly ToolDefinition[]): void; diff --git a/packages/core/src/analytics/index.ts b/packages/core/src/analytics/index.ts index b8f531df..f1a918f4 100644 --- a/packages/core/src/analytics/index.ts +++ b/packages/core/src/analytics/index.ts @@ -4,3 +4,5 @@ export { BufferedEventSink, statsFromSink } from './sink.js'; export { ANALYTICS_EVENTS } from './names.js'; export { redactPII } from './redact.js'; export type { RedactOptions } from './redact.js'; +export { checkAnswered } from './unanswered.js'; +export type { AnswerCheck, AnswerVerdict, UnansweredReason } from './unanswered.js'; diff --git a/packages/core/src/analytics/unanswered.ts b/packages/core/src/analytics/unanswered.ts new file mode 100644 index 00000000..dba6fa1e --- /dev/null +++ b/packages/core/src/analytics/unanswered.ts @@ -0,0 +1,80 @@ +/** Why a reply counts as not answering the question. */ +export type UnansweredReason = 'refusal' | 'no_retrieval' | 'no_citation'; + +export interface AnswerCheck { + /** The reply's text. */ + reply: string; + /** + * What retrieval found for this reply, when a knowledge source that + * retrieves chunks ran. `minScore` only applies to `topScore` from the same + * retriever, since scores aren't comparable across retrievers. + */ + retrieval?: { chunks: number; topScore?: number; minScore?: number }; + /** How many pages the reply cited. */ + cited?: number; + /** More phrases that mean "I don't know", matched case-insensitively. */ + phrases?: string[]; +} + +export interface AnswerVerdict { + answered: boolean; + reason?: UnansweredReason; +} + +// The ways a reply says it doesn't know, in the languages the widget ships. A +// reply is checked near its start, where a refusal usually is, so a hedge late +// in a real answer doesn't count. +const REFUSALS = [ + "i don't know", + 'i do not know', + "i'm not sure", + 'i am not sure', + "i don't have information", + "i don't have any information", + 'i do not have information', + "i don't have details", + "i couldn't find", + 'i could not find', + "i can't find", + "i'm unable to find", + 'no lo sé', + 'no tengo información', + 'no estoy seguro', + 'no estoy segura', + 'no encontré', + 'je ne sais pas', + "je n'ai pas d'information", + 'je ne suis pas sûr', + "je n'ai pas trouvé", + 'ich weiß nicht', + 'ich habe keine informationen', + 'ich bin mir nicht sicher', + 'não sei', + 'não tenho informações', + 'não tenho certeza', + 'não encontrei', +]; +const REFUSAL_WINDOW = 300; +const MIN_WORDS_FOR_CITATION = 20; + +/** + * Whether a reply answered the visitor's question, from free signals only + * (F-11): the reply says it doesn't know, retrieval found nothing, or + * retrieval found pages the reply didn't cite. It's a heuristic for an answer + * rate and a list of questions to fix, not a judgement of quality. + */ +export function checkAnswered(input: AnswerCheck): AnswerVerdict { + const start = input.reply.slice(0, REFUSAL_WINDOW).toLowerCase().replace(/[‘’]/g, "'"); + const phrases = [...REFUSALS, ...(input.phrases ?? []).map((phrase) => phrase.toLowerCase())]; + if (phrases.some((phrase) => phrase && start.includes(phrase))) return { answered: false, reason: 'refusal' }; + + const retrieval = input.retrieval; + if (retrieval) { + const tooWeak = + retrieval.minScore !== undefined && retrieval.topScore !== undefined && retrieval.topScore < retrieval.minScore; + if (retrieval.chunks === 0 || tooWeak) return { answered: false, reason: 'no_retrieval' }; + const words = input.reply.trim().split(/\s+/).filter(Boolean).length; + if ((input.cited ?? 0) === 0 && words > MIN_WORDS_FOR_CITATION) return { answered: false, reason: 'no_citation' }; + } + return { answered: true }; +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 0b5c0d0b..c64fd4f6 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -180,8 +180,8 @@ export { createRateLimiter } from './limits/rate-limit.js'; export type { RateLimitRule, RateLimitResult, RateLimiterOptions } from './limits/rate-limit.js'; // Analytics -export { AnalyticsCollector, BufferedEventSink, statsFromSink, ANALYTICS_EVENTS, redactPII } from './analytics/index.js'; -export type { AnalyticsStats, RedactOptions } from './analytics/index.js'; +export { AnalyticsCollector, BufferedEventSink, statsFromSink, ANALYTICS_EVENTS, redactPII, checkAnswered } from './analytics/index.js'; +export type { AnalyticsStats, RedactOptions, AnswerCheck, AnswerVerdict, UnansweredReason } from './analytics/index.js'; // i18n export { getLocale, getAvailableLocales } from './i18n/index.js'; diff --git a/packages/core/tests/analytics/unanswered.test.ts b/packages/core/tests/analytics/unanswered.test.ts new file mode 100644 index 00000000..21dcab74 --- /dev/null +++ b/packages/core/tests/analytics/unanswered.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from 'vitest'; +import { checkAnswered } from '../../src/analytics/unanswered.js'; + +const long = 'Our Team plan costs twenty dollars per seat each month and includes shared inboxes, priority support, usage reports and single sign-on for every member.'; + +describe('checkAnswered', () => { + it('counts a plain reply as answered when no knowledge ran', () => { + expect(checkAnswered({ reply: 'Hi! How can I help?' })).toEqual({ answered: true }); + }); + + it.each([ + "I don't know the answer to that.", + 'I’m not sure, but you could ask support.', + "Sorry, I couldn't find anything about refunds.", + 'No tengo información sobre eso.', + "Je n'ai pas d'information à ce sujet.", + 'Ich bin mir nicht sicher.', + 'Não encontrei nada sobre isso.', + ])('flags a refusal: %s', (reply) => { + expect(checkAnswered({ reply })).toEqual({ answered: false, reason: 'refusal' }); + }); + + it('looks for a refusal near the start only, not a hedge late in a real answer', () => { + const reply = `${long.repeat(3)} Beyond that I don't know of other plans.`; + expect(checkAnswered({ reply })).toEqual({ answered: true }); + }); + + it('takes extra phrases, in any case', () => { + expect(checkAnswered({ reply: 'That is outside what I cover.', phrases: ['OUTSIDE WHAT I COVER'] })).toEqual({ answered: false, reason: 'refusal' }); + expect(checkAnswered({ reply: 'Fine.', phrases: [''] })).toEqual({ answered: true }); + }); + + it('flags a reply when retrieval found nothing', () => { + expect(checkAnswered({ reply: 'Here is what I know.', retrieval: { chunks: 0 } })).toEqual({ answered: false, reason: 'no_retrieval' }); + }); + + it('flags a reply when the best chunk scores under minScore', () => { + expect(checkAnswered({ reply: 'Yes.', retrieval: { chunks: 3, topScore: 0.2, minScore: 0.5 }, cited: 1 })).toEqual({ answered: false, reason: 'no_retrieval' }); + expect(checkAnswered({ reply: 'Yes.', retrieval: { chunks: 3, topScore: 0.7, minScore: 0.5 }, cited: 1 })).toEqual({ answered: true }); + }); + + it('flags a long reply that cites none of the pages retrieval found', () => { + expect(checkAnswered({ reply: long, retrieval: { chunks: 2 }, cited: 0 })).toEqual({ answered: false, reason: 'no_citation' }); + }); + + it('accepts a long reply that cites, and a short one that does not', () => { + expect(checkAnswered({ reply: long, retrieval: { chunks: 2 }, cited: 1 })).toEqual({ answered: true }); + expect(checkAnswered({ reply: 'You are welcome!', retrieval: { chunks: 2 } })).toEqual({ answered: true }); + }); +}); diff --git a/packages/server/src/handler.ts b/packages/server/src/handler.ts index 1d04c6e2..e272d36b 100644 --- a/packages/server/src/handler.ts +++ b/packages/server/src/handler.ts @@ -4,6 +4,7 @@ import { ConversationManager, AnalyticsCollector, ANALYTICS_EVENTS, + checkAnswered, BufferedEventSink, statsFromSink, type AnalyticsStats, @@ -337,6 +338,7 @@ export function createChatHandler(config: ChatCopsServerConfig) { // changes per request goes in the context after it. const context: string[] = []; let sources: RetrievedChunk[] = []; + const retrieving = config.knowledge?.some((source) => source.retrieve) ?? false; if (config.knowledge?.length) { const found = await knowledgeContext(config.knowledge, req.message, historyBefore(conversationMessages, req), signal); if (found.context) context.push(found.context); @@ -396,8 +398,11 @@ export function createChatHandler(config: ChatCopsServerConfig) { }); pendingWebhooks.add(delivery); }, - onPersist: async (message, { final }) => { - const complete = message.status === 'complete'; + onPersist: async (persisted, { final }) => { + const complete = persisted.status === 'complete'; + // A finished reply records whether it answered, for an answer rate + // and a list of questions the knowledge doesn't cover (F-11). + const message = final && complete && persisted.content ? withAnswerVerdict(persisted, sources, retrieving) : persisted; if (!saved) { if (final && !message.content && !message.parts?.length) return; // A reply the visitor walked away from is kept, marked `aborted`, @@ -621,3 +626,15 @@ function findLastIndex(array: T[], predicate: (item: T) => boolean): number { return -1; } +/** The reply with `answered` (and `unansweredReason` when it didn't) in its metadata. */ +function withAnswerVerdict(message: ChatMessage, sources: RetrievedChunk[], retrieving: boolean): ChatMessage { + const verdict = checkAnswered({ + reply: message.content, + ...(retrieving ? { retrieval: { chunks: sources.length } } : {}), + cited: message.parts?.filter((part) => part.type === 'source').length ?? 0, + }); + return { + ...message, + metadata: { ...message.metadata, answered: verdict.answered, ...(verdict.reason ? { unansweredReason: verdict.reason } : {}) }, + }; +} diff --git a/packages/server/tests/handler-answered.test.ts b/packages/server/tests/handler-answered.test.ts new file mode 100644 index 00000000..dbf36c1e --- /dev/null +++ b/packages/server/tests/handler-answered.test.ts @@ -0,0 +1,54 @@ +import { memoryStorage, type KnowledgeSource, type RetrievedChunk } from '@chatcops/core'; +import { scriptedProvider, textStep } from '@chatcops/core/testing/provider'; +import { describe, expect, it } from 'vitest'; +import { createChatHandler } from '../src/handler.js'; + +const chunk = (id: string, text: string): RetrievedChunk => ({ + id, + text, + score: 1, + source: { id: `https://example.test/${id}`, url: `https://example.test/${id}`, title: `Page ${id}` }, +}); +const retrieving = (chunks: RetrievedChunk[]): KnowledgeSource => ({ type: 'test', retrieve: async () => chunks, getContext: async () => '' }); +const long = 'Our Team plan costs twenty dollars per seat each month and includes shared inboxes, priority support, usage reports and single sign-on for every member.'; + +async function reply(options: { provider: ReturnType; knowledge?: KnowledgeSource[] }) { + const storage = memoryStorage(); + const { handleChat, cleanup } = createChatHandler({ allowAnonymous: true, systemPrompt: 'Be brief.', storage, ...options }); + for await (const _ of handleChat({ conversationId: 'c1', message: 'How much is Team?' }, { protocol: 'v1' } as never)) { + // Only what's stored matters here. + } + cleanup(); + return (await storage.conversations.get('c1'))?.messages.at(-1); +} + +describe('answered verdict on saved replies', () => { + it('marks a cited reply as answered', async () => { + const provider = scriptedProvider( + [[ + { type: 'text-delta', delta: long }, + { type: 'source', index: 0 }, + { type: 'finish', reason: 'stop', usage: { inputTokens: 1, outputTokens: 1 }, parts: [{ type: 'text', text: long }] }, + ]], + { citations: 'native' } + ); + const saved = await reply({ provider, knowledge: [retrieving([chunk('a', 'Team is $20.')])] }); + expect(saved?.metadata).toMatchObject({ answered: true }); + expect(saved?.metadata).not.toHaveProperty('unansweredReason'); + }); + + it('flags a long reply that cites none of the retrieved pages', async () => { + const saved = await reply({ provider: scriptedProvider([textStep(long)]), knowledge: [retrieving([chunk('a', 'Team is $20.')])] }); + expect(saved?.metadata).toMatchObject({ answered: false, unansweredReason: 'no_citation' }); + }); + + it('flags a reply when retrieval found nothing', async () => { + const saved = await reply({ provider: scriptedProvider([textStep('Hmm.')]), knowledge: [retrieving([])] }); + expect(saved?.metadata).toMatchObject({ answered: false, unansweredReason: 'no_retrieval' }); + }); + + it('flags a refusal without any knowledge, and marks a plain reply answered', async () => { + expect((await reply({ provider: scriptedProvider([textStep("I don't know that one.")]) }))?.metadata).toMatchObject({ answered: false, unansweredReason: 'refusal' }); + expect((await reply({ provider: scriptedProvider([textStep('Team is $20 a month.')]) }))?.metadata).toMatchObject({ answered: true }); + }); +}); diff --git a/website/src/content/docs/core/analytics.mdx b/website/src/content/docs/core/analytics.mdx index d44a12fc..f833f0aa 100644 --- a/website/src/content/docs/core/analytics.mdx +++ b/website/src/content/docs/core/analytics.mdx @@ -174,6 +174,25 @@ ChatCops.on('message', (...args: unknown[]) => { // conversation.started, message.sent, message.received, lead.captured ``` +## Whether a reply answered + +Each finished reply is saved with `metadata.answered` (`true` or `false`), and `metadata.unansweredReason` when it's `false`, so you can work out an answer rate and find the questions your knowledge doesn't cover. A reply counts as not answered when: + +- it says it doesn't know near its start ("I don't know", "I couldn't find", and the same in Spanish, French, German and Portuguese): `refusal` +- a knowledge source that retrieves pages ran and found nothing: `no_retrieval` +- retrieval found pages, but a reply longer than 20 words cited none of them: `no_citation` + +It's a heuristic built from those free signals, not a judgement of quality. `checkAnswered` from `@chatcops/core` is the same check, if you want to run it yourself: + +```typescript +import { checkAnswered } from '@chatcops/core'; + +checkAnswered({ reply: "Sorry, I couldn't find that.", retrieval: { chunks: 0 } }); +// { answered: false, reason: 'refusal' } +``` + +It also takes `phrases` (more ways your bot says it doesn't know) and, in `retrieval`, a `topScore` with a `minScore` to count weak retrieval as nothing found. + ## Redacting personal data `redactPII()` takes personal data out of text, so that what a visitor typed can be kept without it. Each kind is replaced with a tag: