From c07a5ccb3e674dee334cccf45328ba8872c31fc0 Mon Sep 17 00:00:00 2001 From: Hemant <104012423+hemant838@users.noreply.github.com> Date: Sun, 22 Feb 2026 04:41:49 +0530 Subject: [PATCH] fix(chat): handle prompt too long errors by truncating files and limiting history Prevent context window overflow by truncating large file contents (default 100K chars) and capping message history (default 50 messages). When a context window error still occurs, detect provider-specific error messages and show a user-friendly message with actionable guidance. Fixes #911 Co-Authored-By: Claude Opus 4.6 --- packages/shared/src/env.server.ts | 2 + .../web/src/app/api/(server)/chat/route.ts | 25 ++++--- packages/web/src/features/chat/agent.ts | 18 ++++- .../components/chatThread/errorBanner.tsx | 3 +- packages/web/src/features/chat/tools.ts | 20 +++--- packages/web/src/features/chat/utils.test.ts | 72 ++++++++++++++++++- packages/web/src/features/chat/utils.ts | 42 +++++++++++ 7 files changed, 161 insertions(+), 21 deletions(-) diff --git a/packages/shared/src/env.server.ts b/packages/shared/src/env.server.ts index b2ce25998..c0be68abf 100644 --- a/packages/shared/src/env.server.ts +++ b/packages/shared/src/env.server.ts @@ -231,6 +231,8 @@ export const env = createEnv({ SOURCEBOT_CHAT_MODEL_TEMPERATURE: numberSchema.default(0.3), SOURCEBOT_CHAT_MAX_STEP_COUNT: numberSchema.default(20), + SOURCEBOT_CHAT_FILE_MAX_CHARACTERS: numberSchema.default(100_000), + SOURCEBOT_CHAT_MAX_MESSAGE_HISTORY: numberSchema.default(50), DEBUG_WRITE_CHAT_MESSAGES_TO_FILE: booleanSchema.default('false'), diff --git a/packages/web/src/app/api/(server)/chat/route.ts b/packages/web/src/app/api/(server)/chat/route.ts index 5ac0d4080..712d7825d 100644 --- a/packages/web/src/app/api/(server)/chat/route.ts +++ b/packages/web/src/app/api/(server)/chat/route.ts @@ -2,7 +2,7 @@ import { sew } from "@/actions"; import { _getConfiguredLanguageModelsFull, _getAISDKLanguageModelAndOptions, _updateChatMessages, _isOwnerOfChat } from "@/features/chat/actions"; import { createAgentStream } from "@/features/chat/agent"; import { additionalChatRequestParamsSchema, LanguageModelInfo, SBChatMessage, SearchScope } from "@/features/chat/types"; -import { getAnswerPartFromAssistantMessage, getLanguageModelKey } from "@/features/chat/utils"; +import { getAnswerPartFromAssistantMessage, getLanguageModelKey, isContextWindowError, CONTEXT_WINDOW_USER_MESSAGE } from "@/features/chat/utils"; import { apiHandler } from "@/lib/apiHandler"; import { ErrorCode } from "@/lib/errorCodes"; import { notFound, requestBodySchemaValidationError, ServiceError, serviceErrorResponse } from "@/lib/serviceError"; @@ -11,7 +11,7 @@ import { withOptionalAuthV2 } from "@/withAuthV2"; import { LanguageModelV2 as AISDKLanguageModelV2 } from "@ai-sdk/provider"; import * as Sentry from "@sentry/nextjs"; import { PrismaClient } from "@sourcebot/db"; -import { createLogger } from "@sourcebot/shared"; +import { createLogger, env } from "@sourcebot/shared"; import { captureEvent } from "@/lib/posthog"; import { createUIMessageStream, @@ -114,15 +114,17 @@ export const POST = apiHandler(async (req: NextRequest) => { return 'unknown error'; } - if (typeof error === 'string') { - return error; - } + const errorMessage = (() => { + if (typeof error === 'string') return error; + if (error instanceof Error) return error.message; + return JSON.stringify(error); + })(); - if (error instanceof Error) { - return error.message; + if (isContextWindowError(errorMessage)) { + return CONTEXT_WINDOW_USER_MESSAGE; } - return JSON.stringify(error); + return errorMessage; } }); @@ -203,6 +205,11 @@ export const createMessageStream = async ({ } }).filter(message => message !== undefined); + const maxMessages = env.SOURCEBOT_CHAT_MAX_MESSAGE_HISTORY; + const trimmedMessageHistory = messageHistory.length > maxMessages + ? messageHistory.slice(-maxMessages) + : messageHistory; + const stream = createUIMessageStream({ execute: async ({ writer }) => { writer.write({ @@ -238,7 +245,7 @@ export const createMessageStream = async ({ const researchStream = await createAgentStream({ model, providerOptions: modelProviderOptions, - inputMessages: messageHistory, + inputMessages: trimmedMessageHistory, inputSources: sources, selectedRepos: expandedRepos, onWriteSource: (source) => { diff --git a/packages/web/src/features/chat/agent.ts b/packages/web/src/features/chat/agent.ts index ec2a30758..ec11ef727 100644 --- a/packages/web/src/features/chat/agent.ts +++ b/packages/web/src/features/chat/agent.ts @@ -7,7 +7,7 @@ import { LanguageModel, ModelMessage, StopCondition, streamText } from "ai"; import { ANSWER_TAG, FILE_REFERENCE_PREFIX, toolNames } from "./constants"; import { createCodeSearchTool, findSymbolDefinitionsTool, findSymbolReferencesTool, listReposTool, listCommitsTool, readFilesTool } from "./tools"; import { Source } from "./types"; -import { addLineNumbers, fileReferenceToString } from "./utils"; +import { addLineNumbers, fileReferenceToString, truncateFileContent } from "./utils"; import _dedent from "dedent"; const dedent = _dedent.withOptions({ alignValues: true }); @@ -60,9 +60,20 @@ export const createAgentStream = async ({ })) ).filter((source) => source !== undefined); + const maxChars = env.SOURCEBOT_CHAT_FILE_MAX_CHARACTERS; + let anyFileTruncated = false; + const truncatedFileSources = resolvedFileSources.map((file) => { + const { content, wasTruncated } = truncateFileContent(file.source, maxChars); + if (wasTruncated) { + anyFileTruncated = true; + } + return { ...file, source: content }; + }); + const systemPrompt = createPrompt({ repos: selectedRepos, - files: resolvedFileSources, + files: truncatedFileSources, + filesWereTruncated: anyFileTruncated, }); const stream = streamText({ @@ -148,6 +159,7 @@ export const createAgentStream = async ({ const createPrompt = ({ files, repos, + filesWereTruncated, }: { files?: { path: string; @@ -157,6 +169,7 @@ const createPrompt = ({ revision: string; }[], repos: string[], + filesWereTruncated?: boolean, }) => { return dedent` You are a powerful agentic AI code assistant built into Sourcebot, the world's best code-intelligence platform. Your job is to help developers understand and navigate their large codebases. @@ -189,6 +202,7 @@ const createPrompt = ({ ${(files && files.length > 0) ? dedent` + ${filesWereTruncated ? `**Note:** Some files were truncated because they exceeded the character limit. Use the readFiles tool to retrieve specific sections if needed.` : ''} The user has mentioned the following files, which are automatically included for analysis. ${files?.map(file => ` diff --git a/packages/web/src/features/chat/components/chatThread/errorBanner.tsx b/packages/web/src/features/chat/components/chatThread/errorBanner.tsx index 2020e29f7..a0ef8ac0a 100644 --- a/packages/web/src/features/chat/components/chatThread/errorBanner.tsx +++ b/packages/web/src/features/chat/components/chatThread/errorBanner.tsx @@ -2,6 +2,7 @@ import { Button } from '@/components/ui/button'; import { serviceErrorSchema } from '@/lib/serviceError'; +import { CONTEXT_WINDOW_USER_MESSAGE } from '@/features/chat/utils'; import { AlertCircle, X } from "lucide-react"; import { useMemo } from 'react'; @@ -33,7 +34,7 @@ export const ErrorBanner = ({ error, isVisible, onClose }: ErrorBannerProps) =>
- Error occurred + {errorMessage === CONTEXT_WINDOW_USER_MESSAGE ? 'Context limit exceeded' : 'Error occurred'} {errorMessage} diff --git a/packages/web/src/features/chat/tools.ts b/packages/web/src/features/chat/tools.ts index 87a251214..8235a2542 100644 --- a/packages/web/src/features/chat/tools.ts +++ b/packages/web/src/features/chat/tools.ts @@ -4,7 +4,8 @@ import { InferToolInput, InferToolOutput, InferUITool, tool, ToolUIPart } from " import { isServiceError } from "@/lib/utils"; import { FileSourceResponse, getFileSource, listCommits } from '@/features/git'; import { findSearchBasedSymbolDefinitions, findSearchBasedSymbolReferences } from "../codeNav/api"; -import { addLineNumbers } from "./utils"; +import { addLineNumbers, truncateFileContent } from "./utils"; +import { env } from "@sourcebot/shared"; import { toolNames } from "./constants"; import { listReposQueryParamsSchema } from "@/lib/schemas"; import { ListReposQueryParams } from "@/lib/types"; @@ -123,13 +124,16 @@ export const readFilesTool = tool({ return firstError!; } - return (responses as FileSourceResponse[]).map((response) => ({ - path: response.path, - repository: response.repo, - language: response.language, - source: addLineNumbers(response.source), - revision, - })); + return (responses as FileSourceResponse[]).map((response) => { + const { content } = truncateFileContent(response.source, env.SOURCEBOT_CHAT_FILE_MAX_CHARACTERS); + return { + path: response.path, + repository: response.repo, + language: response.language, + source: addLineNumbers(content), + revision, + }; + }); } }); diff --git a/packages/web/src/features/chat/utils.test.ts b/packages/web/src/features/chat/utils.test.ts index 5c0932eb3..826bd5461 100644 --- a/packages/web/src/features/chat/utils.test.ts +++ b/packages/web/src/features/chat/utils.test.ts @@ -1,5 +1,5 @@ import { expect, test, vi } from 'vitest' -import { fileReferenceToString, getAnswerPartFromAssistantMessage, groupMessageIntoSteps, repairReferences } from './utils' +import { fileReferenceToString, getAnswerPartFromAssistantMessage, groupMessageIntoSteps, repairReferences, truncateFileContent, isContextWindowError, CONTEXT_WINDOW_USER_MESSAGE } from './utils' import { FILE_REFERENCE_REGEX, ANSWER_TAG } from './constants'; import { SBChatMessage, SBChatMessagePart } from './types'; @@ -351,3 +351,73 @@ test('repairReferences handles malformed inline code blocks', () => { const expected = 'See @file:{github.com/sourcebot-dev/sourcebot::packages/web/src/auth.ts} for details.'; expect(repairReferences(input)).toBe(expected); }); + +// truncateFileContent tests + +test('truncateFileContent returns content unchanged when under limit', () => { + const source = 'line 1\nline 2\nline 3'; + const result = truncateFileContent(source, 100); + expect(result.content).toBe(source); + expect(result.wasTruncated).toBe(false); +}); + +test('truncateFileContent returns content unchanged when exactly at limit', () => { + const source = 'abcde'; + const result = truncateFileContent(source, 5); + expect(result.content).toBe(source); + expect(result.wasTruncated).toBe(false); +}); + +test('truncateFileContent truncates at line boundary when over limit', () => { + const source = 'line 1\nline 2\nline 3\nline 4\nline 5'; + // Limit of 20 characters: "line 1\nline 2\nline 3" is 20 chars + const result = truncateFileContent(source, 15); + expect(result.wasTruncated).toBe(true); + expect(result.content).toContain('line 1\nline 2'); + expect(result.content).toContain('... [truncated:'); + expect(result.content).not.toContain('line 4'); +}); + +test('truncateFileContent includes line count information', () => { + const source = 'a\nb\nc\nd\ne'; + const result = truncateFileContent(source, 3); + expect(result.wasTruncated).toBe(true); + expect(result.content).toMatch(/showing \d+ of 5 lines/); +}); + +// isContextWindowError tests + +test('isContextWindowError detects OpenAI context length error', () => { + expect(isContextWindowError('This model\'s maximum context length is 128000 tokens')).toBe(true); +}); + +test('isContextWindowError detects Anthropic prompt too long error', () => { + expect(isContextWindowError('prompt is too long: 150000 tokens > 100000 maximum')).toBe(true); +}); + +test('isContextWindowError detects context_length_exceeded error', () => { + expect(isContextWindowError('context_length_exceeded')).toBe(true); +}); + +test('isContextWindowError detects token limit error', () => { + expect(isContextWindowError('Request exceeds the maximum number of tokens')).toBe(true); +}); + +test('isContextWindowError detects reduce the length error', () => { + expect(isContextWindowError('Please reduce the length of the messages')).toBe(true); +}); + +test('isContextWindowError detects request too large error', () => { + expect(isContextWindowError('request too large')).toBe(true); +}); + +test('isContextWindowError returns false for unrelated errors', () => { + expect(isContextWindowError('Internal server error')).toBe(false); + expect(isContextWindowError('Rate limit exceeded')).toBe(false); + expect(isContextWindowError('Invalid API key')).toBe(false); +}); + +test('CONTEXT_WINDOW_USER_MESSAGE is a non-empty string', () => { + expect(typeof CONTEXT_WINDOW_USER_MESSAGE).toBe('string'); + expect(CONTEXT_WINDOW_USER_MESSAGE.length).toBeGreaterThan(0); +}); diff --git a/packages/web/src/features/chat/utils.ts b/packages/web/src/features/chat/utils.ts index ca412618e..beaace3c9 100644 --- a/packages/web/src/features/chat/utils.ts +++ b/packages/web/src/features/chat/utils.ts @@ -176,6 +176,48 @@ export const addLineNumbers = (source: string, lineOffset = 1) => { return source.split('\n').map((line, index) => `${index + lineOffset}:${line}`).join('\n'); } +export const truncateFileContent = ( + source: string, + maxCharacters: number, +): { content: string; wasTruncated: boolean } => { + if (source.length <= maxCharacters) { + return { content: source, wasTruncated: false }; + } + + const cutoff = source.lastIndexOf('\n', maxCharacters); + const effectiveCutoff = cutoff > 0 ? cutoff : maxCharacters; + const truncated = source.substring(0, effectiveCutoff); + + const totalLines = source.split('\n').length; + const includedLines = truncated.split('\n').length; + + return { + content: truncated + `\n\n... [truncated: showing ${includedLines} of ${totalLines} lines]`, + wasTruncated: true, + }; +}; + +const CONTEXT_WINDOW_ERROR_PATTERNS = [ + /maximum context length/i, + /prompt is too long/i, + /context.?length.?exceeded/i, + /exceeds? the maximum.*tokens?/i, + /token.?limit/i, + /request.?too.?large/i, + /input.?too.?long/i, + /request payload size exceeds/i, + /max_tokens/i, + /reduce the length/i, +]; + +export const isContextWindowError = (errorMessage: string): boolean => { + return CONTEXT_WINDOW_ERROR_PATTERNS.some((pattern) => pattern.test(errorMessage)); +}; + +export const CONTEXT_WINDOW_USER_MESSAGE = + 'The conversation exceeded the model\'s context window limit. ' + + 'Try removing some attached files, starting a new conversation, or switching to a model with a larger context window.'; + export const createUIMessage = (text: string, mentions: MentionData[], selectedSearchScopes: SearchScope[]): CreateUIMessage => { // Converts applicable mentions into sources. const sources: Source[] = mentions