From 3e01e730af38b7ac68619f49343d3fcf49b24765 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 13 Oct 2026 11:31:29 -0700 Subject: [PATCH 1/3] improvement(chat): coalesce replayed transcript reloads and narrow text-only message reads --- .../resource-invalidation.ts | 4 +- .../stream/handle-resource-event.test.ts | 99 +++++++++++++- .../hooks/stream/handle-resource-event.ts | 26 +++- .../home/hooks/stream/stream-context.ts | 21 ++- apps/sim/hooks/queries/mothership-chats.ts | 31 +++++ .../lib/cleanup/chat-cleanup.integration.ts | 125 ++++++++++++++++++ apps/sim/lib/cleanup/chat-cleanup.test.ts | 2 +- apps/sim/lib/cleanup/chat-cleanup.ts | 16 ++- apps/sim/lib/mothership/chat/lifecycle.ts | 21 +++ .../lib/mothership/chat/process-contents.ts | 23 +--- .../chat/text-transcript.integration.ts | 121 +++++++++++++++++ .../lib/mothership/chat/text-transcript.ts | 54 ++++++++ packages/testing/src/mocks/database.mock.ts | 6 +- scripts/check-explicit-any.baseline.json | 2 +- 14 files changed, 510 insertions(+), 41 deletions(-) create mode 100644 apps/sim/lib/cleanup/chat-cleanup.integration.ts create mode 100644 apps/sim/lib/mothership/chat/text-transcript.integration.ts create mode 100644 apps/sim/lib/mothership/chat/text-transcript.ts diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-registry/resource-invalidation.ts b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-registry/resource-invalidation.ts index 66fdac714ee..b8db2ae2c5c 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-registry/resource-invalidation.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-registry/resource-invalidation.ts @@ -3,7 +3,7 @@ import type { MothershipResourceType } from '@/lib/mothership/resources/types' import { dashboardKeys } from '@/hooks/queries/dashboards' import { deploymentKeys, invalidateDeploymentQueries } from '@/hooks/queries/deployments' import { logKeys } from '@/hooks/queries/logs' -import { mothershipChatKeys } from '@/hooks/queries/mothership-chats' +import { mothershipChatKeys, refreshMothershipChatDetail } from '@/hooks/queries/mothership-chats' import { folderKeys } from '@/hooks/queries/utils/folder-keys' import { invalidateWorkflowLists } from '@/hooks/queries/utils/invalidate-workflow-lists' import { knowledgeKeys } from '@/hooks/queries/utils/knowledge-keys' @@ -59,7 +59,7 @@ const RESOURCE_INVALIDATORS: Record< }, task: (qc, wId, id) => { invalidate(qc, mothershipChatKeys.workspaceLists(wId)) - if (id) invalidate(qc, mothershipChatKeys.detail(id)) + if (id) refreshMothershipChatDetail(qc, id) }, log: (qc, wId, id) => { invalidate(qc, [...logKeys.lists(), wId]) diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event.test.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event.test.ts index 5049e9050d5..bfef657320c 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event.test.ts @@ -1,3 +1,4 @@ +import { createDeferred } from '@sim/testing/helpers/deferred' import { QueryClient, QueryObserver } from '@tanstack/react-query' import { beforeEach, describe, expect, it, vi } from 'vitest' import { resourceScopeKey } from '@/lib/core/resource-scope' @@ -26,7 +27,10 @@ vi.mock('@/hooks/queries/utils/workflow-cache', () => ({ import type { PersistedStreamEventEnvelope } from '@/lib/mothership/request/session/contract' import { handleResourceEvent } from '@/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event' -import type { StreamLoopContext } from '@/app/workspace/[workspaceId]/home/hooks/stream/stream-context' +import { + createStreamLoopContext, + type StreamLoopContext, +} from '@/app/workspace/[workspaceId]/home/hooks/stream/stream-context' import { makeStreamLoopDeps, ref, @@ -197,6 +201,99 @@ describe('live resource effects', () => { ) }) +describe('recovered resource events', () => { + const detailKey = mothershipChatKeys.detail('chat-1') + const tables = ['tbl-1', 'tbl-2', 'tbl-3'].map( + (id): MothershipResource => ({ type: 'table', id, title: id }) + ) + + function replayed(resource: MothershipResource, seq: number): ResourceEvent { + return { + ...removeEvent('file', resource.id), + seq, + payload: { op: 'upsert', replay: true, resource }, + } as ResourceEvent + } + + /** A mounted chat whose every read returns the resources stored when the read began. */ + function mountedChat(queryClient: QueryClient) { + const server = { stored: [] as MothershipResource[] } + const reads = vi.fn( + async (): Promise => ({ + id: 'chat-1', + title: null, + messages: [], + activeStreamId: 'stream-1', + resources: [...server.stored], + }) + ) + queryClient.setQueryData(detailKey, { + id: 'chat-1', + title: null, + messages: [], + activeStreamId: 'stream-1', + resources: [], + } satisfies MothershipChatHistory) + const unsubscribe = new QueryObserver(queryClient, { + queryKey: detailKey, + queryFn: reads, + staleTime: Number.POSITIVE_INFINITY, + }).subscribe(() => {}) + return { server, reads, unsubscribe } + } + + async function settledResources(queryClient: QueryClient) { + await vi.waitFor(() => expect(queryClient.isFetching({ queryKey: detailKey })).toBe(0)) + return queryClient.getQueryData(detailKey)?.resources + } + + it('reloads the transcript once for a whole batch replay, after its terminal flush', async () => { + const queryClient = new QueryClient() + const { server, reads, unsubscribe } = mountedChat(queryClient) + const ctx = createStreamLoopContext( + makeStreamLoopDeps({ + queryClient, + chatIdRef: ref('chat-1'), + options: { deferFlushes: true }, + }) + ) + server.stored = tables + + tables.forEach((table, index) => handleResourceEvent(ctx, replayed(table, index + 1))) + expect(reads).not.toHaveBeenCalled() + ctx.ops.forceFlush() + + expect(await settledResources(queryClient)).toEqual(tables) + expect(reads).toHaveBeenCalledOnce() + unsubscribe() + }) + + it('coalesces a reconnected tail burst into one read plus one trailing read that sees the last change', async () => { + const queryClient = new QueryClient() + const { server, reads, unsubscribe } = mountedChat(queryClient) + const deps = makeStreamLoopDeps({ queryClient, chatIdRef: ref('chat-1') }) + const ctx = createStreamLoopContext(deps) + const firstReadReleased = createDeferred() + reads.mockImplementationOnce(async () => { + const resources = [...server.stored] + await firstReadReleased.promise + return { id: 'chat-1', title: null, messages: [], activeStreamId: 'stream-1', resources } + }) + + server.stored = tables.slice(0, 1) + handleResourceEvent(ctx, replayed(tables[0], 1)) + server.stored = tables + handleResourceEvent(ctx, replayed(tables[1], 2)) + handleResourceEvent(ctx, replayed(tables[2], 3)) + firstReadReleased.resolve() + + await vi.waitFor(() => expect(reads).toHaveBeenCalledTimes(2)) + expect(await settledResources(queryClient)).toEqual(tables) + expect(reads).toHaveBeenCalledTimes(2) + unsubscribe() + }) +}) + describe('handleResourceEvent saved-view pins', () => { beforeEach(() => { useTableViewPinStore.getState().reset() diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event.ts index b6faa8383d8..0b895f780f1 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event.ts @@ -24,7 +24,11 @@ import type { MothershipResource, MothershipResourceType, } from '@/app/workspace/[workspaceId]/home/types' -import { type MothershipChatHistory, mothershipChatKeys } from '@/hooks/queries/mothership-chats' +import { + type MothershipChatHistory, + mothershipChatKeys, + refreshMothershipChatDetail, +} from '@/hooks/queries/mothership-chats' import { knowledgeKeys } from '@/hooks/queries/utils/knowledge-keys' import { removeWorkflowFromActiveCache } from '@/hooks/queries/utils/workflow-cache' import { useTableViewPinStore } from '@/stores/table/view-pin/store' @@ -63,6 +67,18 @@ function applyResourceEffect( } } +/** + * Recovery reloads the current panel state from the stored chat instead of + * repeating historical focus commands. A batch replay reloads once at its + * terminal flush; a reconnected live tail coalesces its reloads. + */ +function refreshRecoveredChatResources(ctx: StreamLoopContext): void { + const chatId = ctx.deps.chatIdRef.current + if (!chatId) return + if (ctx.deps.options.deferFlushes) ctx.state.deferredChatDetailRefreshes.add(chatId) + else refreshMothershipChatDetail(ctx.deps.queryClient, chatId) +} + /** * Applies a streamed resource upsert/remove to the mothership resource list, * reconciling it with any in-flight or just-completed file-preview handoff so a @@ -159,9 +175,7 @@ export function handleResourceEvent(ctx: StreamLoopContext, parsed: ResourceEven } } if (payload.replay || ctx.deps.options.deferFlushes) { - const chatId = ctx.deps.chatIdRef.current - if (chatId) - void queryClient.invalidateQueries({ queryKey: mothershipChatKeys.detail(chatId) }) + refreshRecoveredChatResources(ctx) return } if (ctx.deps.citedSourcesEnabled) return @@ -190,9 +204,7 @@ export function handleResourceEvent(ctx: StreamLoopContext, parsed: ResourceEven } if (payload.op === 'refresh') return if (payload.replay || ctx.deps.options.deferFlushes) { - /** Recovery refetches current panel state instead of repeating historical focus commands. */ - const chatId = ctx.deps.chatIdRef.current - if (chatId) void queryClient.invalidateQueries({ queryKey: mothershipChatKeys.detail(chatId) }) + refreshRecoveredChatResources(ctx) return } if (payload.op === 'clear_view') { diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/stream-context.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/stream-context.ts index de83519b5c7..b0fd0e2402b 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/stream-context.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/stream-context.ts @@ -22,7 +22,10 @@ import type { MothershipResource, MothershipResourceType, } from '@/app/workspace/[workspaceId]/home/types' -import type { MothershipChatHistory } from '@/hooks/queries/mothership-chats' +import { + type MothershipChatHistory, + refreshMothershipChatDetail, +} from '@/hooks/queries/mothership-chats' /** Minimum spacing between text-driven snapshot flushes (see flushText). */ const MIN_TEXT_FLUSH_INTERVAL_MS = 50 @@ -83,6 +86,8 @@ export interface StreamLoopState { scheduledTextFlushFrame: number | null /** Trailing timer for the min-interval text-flush gate (see flushText). */ scheduledTextFlushTimer: ReturnType | null + /** Chats a batch replay recovered resources for, reloaded once by its terminal flush. */ + deferredChatDetailRefreshes: Set } export interface StreamEventScope { @@ -198,7 +203,10 @@ export interface StreamLoopOps { isStale: () => boolean flush: () => void flushText: () => void - /** Real flush that bypasses `deferFlushes` — the batch-replay terminal flush. */ + /** + * Real flush that bypasses `deferFlushes` — the batch-replay terminal flush — then + * reloads each chat the replay recovered resources for. + */ forceFlush: () => void } @@ -234,6 +242,7 @@ export function createStreamLoopContext(deps: StreamLoopDeps): StreamLoopContext browserAgentRunIds: new Set(), scheduledTextFlushFrame: null, scheduledTextFlushTimer: null, + deferredChatDetailRefreshes: new Set(), } if (preserveState) { @@ -371,7 +380,13 @@ export function createStreamLoopContext(deps: StreamLoopDeps): StreamLoopContext flushAndStamp() }, flushText, - forceFlush: flush, + forceFlush: () => { + flush() + for (const chatId of state.deferredChatDetailRefreshes) { + refreshMothershipChatDetail(deps.queryClient, chatId) + } + state.deferredChatDetailRefreshes.clear() + }, } return { state, ops, deps } diff --git a/apps/sim/hooks/queries/mothership-chats.ts b/apps/sim/hooks/queries/mothership-chats.ts index f1d9a0693de..afb44b18092 100644 --- a/apps/sim/hooks/queries/mothership-chats.ts +++ b/apps/sim/hooks/queries/mothership-chats.ts @@ -109,6 +109,37 @@ export const mothershipChatKeys = { detail: (chatId: string | undefined) => [...mothershipChatKeys.details(), chatId ?? ''] as const, } +/** Per client, the chats with a reload in flight, and whether another was requested meanwhile. */ +const chatDetailRefreshes = new WeakMap>() + +/** + * Invalidates a chat's detail so a mounted transcript reloads, coalescing a burst of + * requests (a replayed or reconnected stream re-announces every resource it touched) + * into one reload in flight plus one trailing reload. The trailing reload starts after + * the last request, so the final read reflects every change requested meanwhile; a read + * already in flight may predate a request and is never trusted to include it. + */ +export function refreshMothershipChatDetail(queryClient: QueryClient, chatId: string): void { + let refreshes = chatDetailRefreshes.get(queryClient) + if (!refreshes) { + refreshes = new Map() + chatDetailRefreshes.set(queryClient, refreshes) + } + if (refreshes.has(chatId)) { + refreshes.set(chatId, true) + return + } + const inFlight = refreshes + inFlight.set(chatId, false) + void queryClient + .invalidateQueries({ queryKey: mothershipChatKeys.detail(chatId) }) + .finally(() => { + const requestedAgain = inFlight.get(chatId) === true + inFlight.delete(chatId) + if (requestedAgain) refreshMothershipChatDetail(queryClient, chatId) + }) +} + /** Shared by the `useMothershipChats` hook and the workspace sidebar prefetch. */ export const MOTHERSHIP_CHAT_LIST_STALE_TIME = 60 * 1000 export const MOTHERSHIP_CHAT_HISTORY_STALE_TIME = 30 * 1000 diff --git a/apps/sim/lib/cleanup/chat-cleanup.integration.ts b/apps/sim/lib/cleanup/chat-cleanup.integration.ts new file mode 100644 index 00000000000..8cf97ded8d2 --- /dev/null +++ b/apps/sim/lib/cleanup/chat-cleanup.integration.ts @@ -0,0 +1,125 @@ +/** Exercises chat cleanup's storage-key collection against real PostgreSQL message rows. */ +import { db } from '@sim/db' +import { copilotChats, copilotMessages, user } from '@sim/db/schema' +import { generateId } from '@sim/utils/id' +import { compareStrings } from '@sim/utils/string' +import { eq } from 'drizzle-orm' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { collectChatFiles } from '@/lib/cleanup/chat-cleanup' +import { inlineChatImageKey } from '@/lib/mothership/chat/inline-image-key' + +function attachment(key: string) { + return { id: generateId(), key, filename: 'x.png', media_type: 'image/png', size: 1 } +} + +describe('chat cleanup file collection in PostgreSQL', () => { + const userId = generateId() + + beforeAll(async () => { + const now = new Date() + await db.insert(user).values({ + id: userId, + name: 'Chat cleanup fixture', + email: `${userId}@fixture.test`, + emailVerified: true, + createdAt: now, + updatedAt: now, + }) + }) + + afterAll(async () => { + await db.delete(copilotChats).where(eq(copilotChats.userId, userId)) + await db.delete(user).where(eq(user.id, userId)) + await db.$client.end() + }) + + async function seedChat(contents: unknown[], deletedSeqs: number[] = []): Promise { + const [chat] = await db + .insert(copilotChats) + .values({ userId, type: 'mothership' }) + .returning({ id: copilotChats.id }) + await db.insert(copilotMessages).values( + contents.map((content, seq) => ({ + chatId: chat.id, + messageId: generateId(), + role: 'user', + seq, + content, + ...(deletedSeqs.includes(seq) ? { deletedAt: new Date() } : {}), + })) + ) + return chat.id + } + + it('collects attachment and inline image keys from every row, never from tool payloads', async () => { + const chatId = await seedChat( + [ + { + role: 'user', + content: 'look', + fileAttachments: [ + attachment('copilot/1234/legacy.png'), + attachment('assistant/org-1/user-1/u1/shared.png'), + attachment('workspace/ws-1/1700-abc-shared.png'), + 'not an attachment', + null, + { key: '' }, + ], + }, + { + role: 'assistant', + requestId: 'req-1', + content: 'Here: ![chart](files/chart.png)', + contentBlocks: [ + { type: 'text', content: '![other](/tmp/out.png)' }, + { + type: 'tool', + toolCall: { + id: 'tool-1', + name: 'edit_workflow', + state: 'success', + params: { + code: `![tool](files/tool.png)${'x'.repeat(3 * 1024 * 1024)}`, + fileAttachments: [attachment('copilot/1234/in-params.png')], + }, + }, + }, + ], + }, + { + role: 'user', + content: 'removed', + fileAttachments: [attachment('copilot/9/removed.png')], + }, + { role: 'assistant', content: '![x](files/x.png)', contentBlocks: 'not an array' }, + 'a bare string message', + ], + [2] + ) + const otherChatId = await seedChat([ + { role: 'user', content: 'other', fileAttachments: [attachment('copilot/5/other.png')] }, + ]) + + const files = await collectChatFiles([chatId, otherChatId]) + + expect([...files].sort((a, b) => compareStrings(a.key, b.key))).toEqual( + [ + { + key: 'assistant/org-1/user-1/u1/shared.png', + context: 'mothership', + chatId, + organizationId: 'org-1', + }, + { + key: inlineChatImageKey(chatId, 'req-1', 'files/chart.png'), + context: 'mothership', + chatId, + }, + { key: inlineChatImageKey(chatId, 'req-1', '/tmp/out.png'), context: 'mothership', chatId }, + { key: 'copilot/1234/legacy.png', context: 'copilot', chatId }, + { key: 'copilot/5/other.png', context: 'copilot', chatId: otherChatId }, + { key: 'copilot/9/removed.png', context: 'copilot', chatId }, + ].sort((a, b) => compareStrings(a.key, b.key)) + ) + }) +}) diff --git a/apps/sim/lib/cleanup/chat-cleanup.test.ts b/apps/sim/lib/cleanup/chat-cleanup.test.ts index ef054b4696b..d6a8596661a 100644 --- a/apps/sim/lib/cleanup/chat-cleanup.test.ts +++ b/apps/sim/lib/cleanup/chat-cleanup.test.ts @@ -32,7 +32,7 @@ async function purge( queueTableRows(copilotChats, []) queueTableRows( copilotMessages, - remaining.map((content) => ({ content })) + remaining.map((content) => ({ fileAttachments: content.fileAttachments ?? null })) ) await cleanup.execute() return Object.fromEntries( diff --git a/apps/sim/lib/cleanup/chat-cleanup.ts b/apps/sim/lib/cleanup/chat-cleanup.ts index ab9745dd093..f537f41f808 100644 --- a/apps/sim/lib/cleanup/chat-cleanup.ts +++ b/apps/sim/lib/cleanup/chat-cleanup.ts @@ -9,6 +9,7 @@ import { inlineChatImageKey, inlineChatImageReferences, } from '@/lib/mothership/chat/inline-image-key' +import { detoastedMessageContent, textMessageContent } from '@/lib/mothership/chat/text-transcript' import { SIM_AGENT_API_URL } from '@/lib/mothership/constants' import type { StorageContext } from '@/lib/uploads' import { isUsingCloudStorage, StorageService } from '@/lib/uploads' @@ -119,8 +120,12 @@ export async function collectChatFiles(chatIds: string[]): Promise { // Scan every message row for the chat (no deleted_at filter): this is a // deletion path collecting blob keys, so attachments on any row count. cleanupDb - .select({ content: copilotMessages.content, chatId: copilotMessages.chatId }) + .select({ + content: textMessageContent(['role', 'requestId', 'content', 'fileAttachments']), + chatId: copilotMessages.chatId, + }) .from(copilotMessages) + .crossJoinLateral(detoastedMessageContent) .where(inArray(copilotMessages.chatId, chunk)), ]) @@ -184,7 +189,7 @@ async function organizationAttachmentsStillReferenced(files: FileRef[]): Promise for (const chunk of chunkArray(keys, CHAT_FILE_COLLECT_CHUNK_SIZE)) { const wanted = new Set(chunk) const rows = await cleanupDb - .select({ content: copilotMessages.content }) + .select({ fileAttachments: sql`${copilotMessages.content} -> 'fileAttachments'` }) .from(copilotMessages) .innerJoin(copilotChats, eq(copilotChats.id, copilotMessages.chatId)) .where( @@ -198,10 +203,9 @@ async function organizationAttachmentsStillReferenced(files: FileRef[]): Promise ) ) ) - for (const { content } of rows) { - const attachments = isRecordLike(content) ? content.fileAttachments : undefined - if (!Array.isArray(attachments)) continue - for (const attachment of attachments) { + for (const { fileAttachments } of rows) { + if (!Array.isArray(fileAttachments)) continue + for (const attachment of fileAttachments) { if ( isRecordLike(attachment) && typeof attachment.key === 'string' && diff --git a/apps/sim/lib/mothership/chat/lifecycle.ts b/apps/sim/lib/mothership/chat/lifecycle.ts index 3d8bb3c0151..62258423ad0 100644 --- a/apps/sim/lib/mothership/chat/lifecycle.ts +++ b/apps/sim/lib/mothership/chat/lifecycle.ts @@ -22,6 +22,7 @@ import { type PersistedMessage, stripToolResultOutput, } from '@/lib/mothership/chat/persisted-message' +import { detoastedMessageContent, textMessageContent } from '@/lib/mothership/chat/text-transcript' import type { MothershipEffort } from '@/lib/mothership/model-options' import { assertActiveWorkspaceAccess, @@ -382,6 +383,26 @@ export async function getAccessibleCopilotChatWithMessages( return { ...chat, messages } } +/** + * The chat {@link getAccessibleCopilotChatWithMessages} loads, with each message reduced to + * its role and text ({@link textMessageContent}) for a reader that quotes the transcript. + */ +export async function getAccessibleCopilotChatWithTextMessages( + chatId: string, + userId: string +): Promise { + const chat = await getAccessibleCopilotChatDetail(chatId, userId) + if (!chat) return null + + const rows = await db + .select({ content: textMessageContent(['role', 'content']) }) + .from(copilotMessages) + .crossJoinLateral(detoastedMessageContent) + .where(and(eq(copilotMessages.chatId, chatId), isNull(copilotMessages.deletedAt))) + .orderBy(...TRANSCRIPT_ORDER) + return { ...chat, messages: rows.map((row) => row.content) } +} + /** * Resolve or create a copilot chat session. * If chatId is provided, loads the existing chat. Otherwise creates a new one. diff --git a/apps/sim/lib/mothership/chat/process-contents.ts b/apps/sim/lib/mothership/chat/process-contents.ts index ed39dd2d575..18d7a4fd78c 100644 --- a/apps/sim/lib/mothership/chat/process-contents.ts +++ b/apps/sim/lib/mothership/chat/process-contents.ts @@ -50,6 +50,7 @@ import { safeBrowserSelectionUrl, truncateSelectionText, } from '@/lib/mothership/chat/selection-context' +import { quoteChatTranscript } from '@/lib/mothership/chat/text-transcript' import { QueryLogs } from '@/lib/mothership/generated/tool-catalog-v1' import { canonicalWorkspaceFilePath } from '@/lib/mothership/vfs/path-utils' import { isBlockTypeAccessControlExempt } from '@/lib/permission-groups/block-access' @@ -550,8 +551,8 @@ async function processPastChatFromDb( currentWorkspaceId?: string ): Promise { try { - const { getAccessibleCopilotChatWithMessages } = await import('./lifecycle') - const chat = await getAccessibleCopilotChatWithMessages(chatId, userId) + const { getAccessibleCopilotChatWithTextMessages } = await import('./lifecycle') + const chat = await getAccessibleCopilotChatWithTextMessages(chatId, userId) if (!chat) { return null } @@ -567,23 +568,7 @@ async function processPastChatFromDb( } } } - const messages = Array.isArray(chat.messages) ? (chat as any).messages : [] - const content = messages - .map((m: any) => { - const role = m.role || 'user' - let text = '' - if (Array.isArray(m.contentBlocks) && m.contentBlocks.length > 0) { - text = m.contentBlocks - .filter((b: any) => b?.type === 'text') - .map((b: any) => String(b.content || '')) - .join('') - .trim() - } - if (!text && typeof m.content === 'string') text = m.content - return `${role}: ${text}`.trim() - }) - .filter((s: string) => s.length > 0) - .join('\n') + const content = quoteChatTranscript(chat.messages) logger.info('Processed past_chat context from DB', { chatId, length: content.length, diff --git a/apps/sim/lib/mothership/chat/text-transcript.integration.ts b/apps/sim/lib/mothership/chat/text-transcript.integration.ts new file mode 100644 index 00000000000..76900c953c4 --- /dev/null +++ b/apps/sim/lib/mothership/chat/text-transcript.integration.ts @@ -0,0 +1,121 @@ +/** Exercises the text-only transcript read against real PostgreSQL rows, multi-MB tool payloads included. */ +import { db } from '@sim/db' +import { copilotChats, copilotMessages, user } from '@sim/db/schema' +import { generateId } from '@sim/utils/id' +import { and, eq } from 'drizzle-orm' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { + getAccessibleCopilotChatWithMessages, + getAccessibleCopilotChatWithTextMessages, +} from '@/lib/mothership/chat/lifecycle' +import { quoteChatTranscript } from '@/lib/mothership/chat/text-transcript' + +const MULTI_MB_PARAMS = 'x'.repeat(3 * 1024 * 1024) + +function toolBlock(id: string) { + return { + type: 'tool', + toolCall: { id, name: 'edit_workflow', state: 'success', params: { code: MULTI_MB_PARAMS } }, + } +} + +describe('text-only chat transcript in PostgreSQL', () => { + const userId = generateId() + + beforeAll(async () => { + const now = new Date() + await db.insert(user).values({ + id: userId, + name: 'Text transcript fixture', + email: `${userId}@fixture.test`, + emailVerified: true, + createdAt: now, + updatedAt: now, + }) + }) + + afterAll(async () => { + await db.delete(copilotChats).where(eq(copilotChats.userId, userId)) + await db.delete(user).where(eq(user.id, userId)) + await db.$client.end() + }) + + async function seedChat(contents: unknown[]): Promise { + const [chat] = await db + .insert(copilotChats) + .values({ userId, type: 'mothership' }) + .returning({ id: copilotChats.id }) + await db.insert(copilotMessages).values( + contents.map((content, seq) => ({ + chatId: chat.id, + messageId: generateId(), + role: 'user', + seq, + content, + })) + ) + return chat.id + } + + it('quotes the same transcript as the full read without loading tool payloads', async () => { + const chatId = await seedChat([ + { id: 'm1', role: 'user', content: 'Build me a workflow', timestamp: '2026-01-01T00:00:00Z' }, + { + id: 'm2', + role: 'assistant', + content: 'fallback text', + requestId: 'req-1', + contentBlocks: [ + { type: 'text', content: 'Working on it. ' }, + toolBlock('tool-1'), + { type: 'text', content: 'Done.' }, + 'not a block', + { type: true, content: 'not text' }, + ], + }, + { role: 'assistant', content: 'only tools ran', contentBlocks: [toolBlock('tool-2')] }, + { role: 'assistant', content: 'blocks not an array', contentBlocks: 'oops' }, + { content: 'no role' }, + { role: 'assistant', contentBlocks: [{ type: 'text', content: ' ' }] }, + 'a bare string message', + [1, 2], + ]) + + const full = await getAccessibleCopilotChatWithMessages(chatId, userId) + const text = await getAccessibleCopilotChatWithTextMessages(chatId, userId) + + expect(full).not.toBeNull() + expect(text).not.toBeNull() + expect(quoteChatTranscript(text?.messages ?? [])).toBe( + quoteChatTranscript(full?.messages ?? []) + ) + expect(quoteChatTranscript(text?.messages ?? [])).toBe( + [ + 'user: Build me a workflow', + 'assistant: Working on it. Done.', + 'assistant: only tools ran', + 'assistant: blocks not an array', + 'user: no role', + 'assistant:', + 'user:', + 'user:', + ].join('\n') + ) + expect(JSON.stringify(text?.messages).length).toBeLessThan(10_000) + }) + + it('skips deleted messages and refuses a chat the caller does not own', async () => { + const chatId = await seedChat([ + { role: 'user', content: 'kept' }, + { role: 'user', content: 'deleted' }, + ]) + await db + .update(copilotMessages) + .set({ deletedAt: new Date() }) + .where(and(eq(copilotMessages.chatId, chatId), eq(copilotMessages.seq, 1))) + + const text = await getAccessibleCopilotChatWithTextMessages(chatId, userId) + expect(quoteChatTranscript(text?.messages ?? [])).toBe('user: kept') + expect(await getAccessibleCopilotChatWithTextMessages(chatId, generateId())).toBeNull() + }) +}) diff --git a/apps/sim/lib/mothership/chat/text-transcript.ts b/apps/sim/lib/mothership/chat/text-transcript.ts new file mode 100644 index 00000000000..77f911ea6e4 --- /dev/null +++ b/apps/sim/lib/mothership/chat/text-transcript.ts @@ -0,0 +1,54 @@ +import { copilotMessages } from '@sim/db/schema' +import { type SQL, sql } from 'drizzle-orm' + +/** + * Each `copilot_messages` row's `content`, detoasted once, as `detoasted.content` for a + * `crossJoinLateral`. Postgres detoasts a toasted column again for every expression that + * reads it, so a projection reading several fields of a multi-MB message reads this copy. + */ +export const detoastedMessageContent: SQL = sql`(select ${copilotMessages.content} #> '{}' as content offset 0) as detoasted` + +/** + * Each message's `content` reduced to `fields` plus its text `contentBlocks`, so a reader that + * needs only a transcript's text never loads its tool payloads (multi-MB `toolCall.params`). A + * field the message lacks, and a `contentBlocks` that is not an array, read as JSON null; a + * content that is not an object passes through unchanged. Requires + * {@link detoastedMessageContent} joined. + */ +export function textMessageContent(fields: readonly string[]): SQL { + const content = sql`detoasted.content` + // The strict path copies only the blocks it keeps, never the tool payloads it skips. + const textBlocks = sql`case when jsonb_typeof(${content} -> 'contentBlocks') = 'array' + then jsonb_path_query_array(${content}, 'strict $.contentBlocks[*] ? (@.type == "text")') end` + const entries = [ + ...fields.map((field) => sql`${field}::text, ${content} -> ${field}::text`), + sql`'contentBlocks', ${textBlocks}`, + ] + return sql`case when jsonb_typeof(${content}) = 'object' + then jsonb_build_object(${sql.join(entries, sql`, `)}) + else ${content} end` +} + +/** + * The transcript a `past_chat` context quotes: one `role: text` line per message, its text + * blocks preferred over its plain `content`. Reads only what {@link textMessageContent} keeps. + */ +export function quoteChatTranscript(messages: unknown[]): string { + return messages + .map((message) => { + const m = message as { role?: unknown; content?: unknown; contentBlocks?: unknown } + const role = m.role || 'user' + let text = '' + if (Array.isArray(m.contentBlocks) && m.contentBlocks.length > 0) { + text = m.contentBlocks + .filter((block) => block?.type === 'text') + .map((block) => String(block.content || '')) + .join('') + .trim() + } + if (!text && typeof m.content === 'string') text = m.content + return `${role}: ${text}`.trim() + }) + .filter((line) => line.length > 0) + .join('\n') +} diff --git a/packages/testing/src/mocks/database.mock.ts b/packages/testing/src/mocks/database.mock.ts index 6477fcff866..e76952e557c 100644 --- a/packages/testing/src/mocks/database.mock.ts +++ b/packages/testing/src/mocks/database.mock.ts @@ -163,7 +163,7 @@ function dequeueChainRows(tables: unknown[]): unknown[] | null { * * - `select().from().where()` → returns a builder with `.limit` / `.orderBy` / * `.returning` / `.groupBy` / `.for` terminals - * - `select().from().innerJoin()|leftJoin()` → returns the same where-builder + * - `select().from().innerJoin()|leftJoin()|crossJoinLateral()` → returns the same where-builder * - `insert().values().returning()` / `update().set().where()` / `delete().where()` * * Results resolve, in priority order: @@ -239,6 +239,7 @@ const asAlias = chainSpy() const forClause = chainSpy() const innerJoin = chainSpy() const leftJoin = chainSpy() +const crossJoinLateral = chainSpy() const insert = chainSpy() const update = chainSpy() const set = chainSpy() @@ -340,6 +341,8 @@ const joinBuilder = (tables: unknown[], fields: SelectedFields = {}): any => { builder.leftJoin = spyOrDefault(leftJoin, (table: unknown) => joinBuilder([...tables, table], fields) ) + // A lateral join reads a SQL expression, not a table, so the chain routes as before it. + builder.crossJoinLateral = spyOrDefault(crossJoinLateral, () => joinBuilder(tables, fields)) return builder } @@ -366,6 +369,7 @@ export const dbChainMockFns = { returning, innerJoin, leftJoin, + crossJoinLateral, groupBy, having, as: asAlias, diff --git a/scripts/check-explicit-any.baseline.json b/scripts/check-explicit-any.baseline.json index ca5f3557445..3c318b82e9f 100644 --- a/scripts/check-explicit-any.baseline.json +++ b/scripts/check-explicit-any.baseline.json @@ -232,7 +232,7 @@ "apps/sim/lib/logs/types.ts": 1, "apps/sim/lib/mcp/client.ts": 1, "apps/sim/lib/messaging/email/mailer.test.ts": 3, - "apps/sim/lib/mothership/chat/process-contents.ts": 5, + "apps/sim/lib/mothership/chat/process-contents.ts": 1, "apps/sim/lib/mothership/persistence/tool-confirm/tool-confirm.test.ts": 3, "apps/sim/lib/mothership/tool-executor/register-handlers.ts": 3, "apps/sim/lib/mothership/tools/server/user/get-credentials.ts": 2, From 2a577da79f3b6105c4afe82f4925b92d2c61f640 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 13 Oct 2026 11:38:02 -0700 Subject: [PATCH 2/3] improvement(chat): bound a hung transcript reload and tighten text-block filtering - a chat detail reload in flight holds back later refresh requests for at most 30s; past it the next request starts its own reload and the stuck one is dropped when it settles - quoteChatTranscript filters blocks with isRecordLike - simplify the refresh coalescer's map init --- .../stream/handle-resource-event.test.ts | 26 ++++++++++++ apps/sim/hooks/queries/mothership-chats.ts | 40 ++++++++++++------- .../lib/mothership/chat/text-transcript.ts | 5 ++- 3 files changed, 54 insertions(+), 17 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event.test.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event.test.ts index bfef657320c..039741201e2 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event.test.ts @@ -292,6 +292,32 @@ describe('recovered resource events', () => { expect(reads).toHaveBeenCalledTimes(2) unsubscribe() }) + + it('stops holding reloads behind one that never settles once 30 seconds pass', async () => { + const queryClient = new QueryClient() + const { server, reads, unsubscribe } = mountedChat(queryClient) + const ctx = createStreamLoopContext( + makeStreamLoopDeps({ queryClient, chatIdRef: ref('chat-1') }) + ) + reads.mockImplementationOnce(() => new Promise(() => {})) + vi.useFakeTimers({ toFake: ['Date'] }) + try { + handleResourceEvent(ctx, replayed(tables[0], 1)) + await vi.waitFor(() => expect(reads).toHaveBeenCalledOnce()) + handleResourceEvent(ctx, replayed(tables[1], 2)) + expect(reads).toHaveBeenCalledOnce() + + server.stored = tables + vi.setSystemTime(Date.now() + 30_000) + handleResourceEvent(ctx, replayed(tables[2], 3)) + + expect(await settledResources(queryClient)).toEqual(tables) + expect(reads).toHaveBeenCalledTimes(2) + } finally { + vi.useRealTimers() + unsubscribe() + } + }) }) describe('handleResourceEvent saved-view pins', () => { diff --git a/apps/sim/hooks/queries/mothership-chats.ts b/apps/sim/hooks/queries/mothership-chats.ts index afb44b18092..4b71c817bae 100644 --- a/apps/sim/hooks/queries/mothership-chats.ts +++ b/apps/sim/hooks/queries/mothership-chats.ts @@ -109,34 +109,44 @@ export const mothershipChatKeys = { detail: (chatId: string | undefined) => [...mothershipChatKeys.details(), chatId ?? ''] as const, } -/** Per client, the chats with a reload in flight, and whether another was requested meanwhile. */ -const chatDetailRefreshes = new WeakMap>() +/** + * How long a chat's reload in flight holds back later requests. Past it a hung fetch no longer + * blocks the chat: the next request starts its own reload, which replaces the stuck one. + */ +const CHAT_DETAIL_REFRESH_HOLD_MS = 30 * 1000 + +interface ChatDetailRefresh { + startedAt: number + requestedAgain: boolean +} + +/** Per client, the chats with a reload in flight. */ +const chatDetailRefreshes = new WeakMap>() /** * Invalidates a chat's detail so a mounted transcript reloads, coalescing a burst of * requests (a replayed or reconnected stream re-announces every resource it touched) * into one reload in flight plus one trailing reload. The trailing reload starts after * the last request, so the final read reflects every change requested meanwhile; a read - * already in flight may predate a request and is never trusted to include it. + * already in flight may predate a request and is never trusted to include it. A reload + * in flight longer than {@link CHAT_DETAIL_REFRESH_HOLD_MS} stops holding requests back. */ export function refreshMothershipChatDetail(queryClient: QueryClient, chatId: string): void { - let refreshes = chatDetailRefreshes.get(queryClient) - if (!refreshes) { - refreshes = new Map() - chatDetailRefreshes.set(queryClient, refreshes) - } - if (refreshes.has(chatId)) { - refreshes.set(chatId, true) + const refreshes = chatDetailRefreshes.get(queryClient) ?? new Map() + chatDetailRefreshes.set(queryClient, refreshes) + const inFlight = refreshes.get(chatId) + if (inFlight && Date.now() - inFlight.startedAt < CHAT_DETAIL_REFRESH_HOLD_MS) { + inFlight.requestedAgain = true return } - const inFlight = refreshes - inFlight.set(chatId, false) + const refresh: ChatDetailRefresh = { startedAt: Date.now(), requestedAgain: false } + refreshes.set(chatId, refresh) void queryClient .invalidateQueries({ queryKey: mothershipChatKeys.detail(chatId) }) .finally(() => { - const requestedAgain = inFlight.get(chatId) === true - inFlight.delete(chatId) - if (requestedAgain) refreshMothershipChatDetail(queryClient, chatId) + if (refreshes.get(chatId) !== refresh) return + refreshes.delete(chatId) + if (refresh.requestedAgain) refreshMothershipChatDetail(queryClient, chatId) }) } diff --git a/apps/sim/lib/mothership/chat/text-transcript.ts b/apps/sim/lib/mothership/chat/text-transcript.ts index 77f911ea6e4..204cf40410b 100644 --- a/apps/sim/lib/mothership/chat/text-transcript.ts +++ b/apps/sim/lib/mothership/chat/text-transcript.ts @@ -1,4 +1,5 @@ import { copilotMessages } from '@sim/db/schema' +import { isRecordLike } from '@sim/utils/object' import { type SQL, sql } from 'drizzle-orm' /** @@ -39,9 +40,9 @@ export function quoteChatTranscript(messages: unknown[]): string { const m = message as { role?: unknown; content?: unknown; contentBlocks?: unknown } const role = m.role || 'user' let text = '' - if (Array.isArray(m.contentBlocks) && m.contentBlocks.length > 0) { + if (Array.isArray(m.contentBlocks)) { text = m.contentBlocks - .filter((block) => block?.type === 'text') + .filter((block) => isRecordLike(block) && block.type === 'text') .map((block) => String(block.content || '')) .join('') .trim() From 73970dd9a7750d1cf62cd14058b5f23fb6d7458d Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 13 Oct 2026 11:51:54 -0700 Subject: [PATCH 3/3] fix(chat): cancel a stuck first transcript load before reloading, and skip malformed blocks - a reload past the hold cancels the stuck detail fetch first, since a query with no data joins its fetch in flight instead of replacing it - the text-block JSONPath runs silent, so a malformed block is skipped rather than failing the read - keep the non-array contentBlocks fixture out of the full-reader comparison, which cannot read it - shrink the unused-exports baseline now that collectChatFiles has an importer --- .../stream/handle-resource-event.test.ts | 76 +++++++++++-------- apps/sim/hooks/queries/mothership-chats.ts | 20 +++-- .../chat/text-transcript.integration.ts | 11 ++- .../lib/mothership/chat/text-transcript.ts | 5 +- scripts/check-unused-exports.baseline.json | 1 - 5 files changed, 69 insertions(+), 44 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event.test.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event.test.ts index 039741201e2..6360f9daa3a 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-resource-event.test.ts @@ -216,7 +216,7 @@ describe('recovered resource events', () => { } /** A mounted chat whose every read returns the resources stored when the read began. */ - function mountedChat(queryClient: QueryClient) { + function mountedChat(queryClient: QueryClient, { cached = true, hangFirstRead = false } = {}) { const server = { stored: [] as MothershipResource[] } const reads = vi.fn( async (): Promise => ({ @@ -227,13 +227,16 @@ describe('recovered resource events', () => { resources: [...server.stored], }) ) - queryClient.setQueryData(detailKey, { - id: 'chat-1', - title: null, - messages: [], - activeStreamId: 'stream-1', - resources: [], - } satisfies MothershipChatHistory) + if (hangFirstRead) reads.mockImplementationOnce(() => new Promise(() => {})) + if (cached) { + queryClient.setQueryData(detailKey, { + id: 'chat-1', + title: null, + messages: [], + activeStreamId: 'stream-1', + resources: [], + } satisfies MothershipChatHistory) + } const unsubscribe = new QueryObserver(queryClient, { queryKey: detailKey, queryFn: reads, @@ -293,31 +296,40 @@ describe('recovered resource events', () => { unsubscribe() }) - it('stops holding reloads behind one that never settles once 30 seconds pass', async () => { - const queryClient = new QueryClient() - const { server, reads, unsubscribe } = mountedChat(queryClient) - const ctx = createStreamLoopContext( - makeStreamLoopDeps({ queryClient, chatIdRef: ref('chat-1') }) - ) - reads.mockImplementationOnce(() => new Promise(() => {})) - vi.useFakeTimers({ toFake: ['Date'] }) - try { - handleResourceEvent(ctx, replayed(tables[0], 1)) - await vi.waitFor(() => expect(reads).toHaveBeenCalledOnce()) - handleResourceEvent(ctx, replayed(tables[1], 2)) - expect(reads).toHaveBeenCalledOnce() - - server.stored = tables - vi.setSystemTime(Date.now() + 30_000) - handleResourceEvent(ctx, replayed(tables[2], 3)) - - expect(await settledResources(queryClient)).toEqual(tables) - expect(reads).toHaveBeenCalledTimes(2) - } finally { - vi.useRealTimers() - unsubscribe() + it.each([ + { name: 'a loaded transcript', cached: true }, + { name: 'a transcript still on its first load', cached: false }, + ])( + 'stops holding reloads of $name behind one that never settles once 30 seconds pass', + async ({ cached }) => { + const queryClient = new QueryClient() + const { server, reads, unsubscribe } = mountedChat(queryClient, { + cached, + hangFirstRead: true, + }) + const ctx = createStreamLoopContext( + makeStreamLoopDeps({ queryClient, chatIdRef: ref('chat-1') }) + ) + vi.useFakeTimers({ toFake: ['Date'] }) + try { + handleResourceEvent(ctx, replayed(tables[0], 1)) + await vi.waitFor(() => expect(reads).toHaveBeenCalledOnce()) + handleResourceEvent(ctx, replayed(tables[1], 2)) + expect(reads).toHaveBeenCalledOnce() + + server.stored = tables + vi.setSystemTime(Date.now() + 30_000) + handleResourceEvent(ctx, replayed(tables[2], 3)) + + await vi.waitFor(() => expect(reads).toHaveBeenCalledTimes(2)) + expect(await settledResources(queryClient)).toEqual(tables) + expect(reads).toHaveBeenCalledTimes(2) + } finally { + vi.useRealTimers() + unsubscribe() + } } - }) + ) }) describe('handleResourceEvent saved-view pins', () => { diff --git a/apps/sim/hooks/queries/mothership-chats.ts b/apps/sim/hooks/queries/mothership-chats.ts index 4b71c817bae..d7ab043c632 100644 --- a/apps/sim/hooks/queries/mothership-chats.ts +++ b/apps/sim/hooks/queries/mothership-chats.ts @@ -141,13 +141,19 @@ export function refreshMothershipChatDetail(queryClient: QueryClient, chatId: st } const refresh: ChatDetailRefresh = { startedAt: Date.now(), requestedAgain: false } refreshes.set(chatId, refresh) - void queryClient - .invalidateQueries({ queryKey: mothershipChatKeys.detail(chatId) }) - .finally(() => { - if (refreshes.get(chatId) !== refresh) return - refreshes.delete(chatId) - if (refresh.requestedAgain) refreshMothershipChatDetail(queryClient, chatId) - }) + const queryKey = mothershipChatKeys.detail(chatId) + // A query with no data yet joins its fetch in flight instead of replacing it, so a stuck + // one is cancelled first. + const reload = inFlight + ? queryClient + .cancelQueries({ queryKey, exact: true }) + .then(() => queryClient.invalidateQueries({ queryKey })) + : queryClient.invalidateQueries({ queryKey }) + void reload.finally(() => { + if (refreshes.get(chatId) !== refresh) return + refreshes.delete(chatId) + if (refresh.requestedAgain) refreshMothershipChatDetail(queryClient, chatId) + }) } /** Shared by the `useMothershipChats` hook and the workspace sidebar prefetch. */ diff --git a/apps/sim/lib/mothership/chat/text-transcript.integration.ts b/apps/sim/lib/mothership/chat/text-transcript.integration.ts index 76900c953c4..df46362b3eb 100644 --- a/apps/sim/lib/mothership/chat/text-transcript.integration.ts +++ b/apps/sim/lib/mothership/chat/text-transcript.integration.ts @@ -74,7 +74,6 @@ describe('text-only chat transcript in PostgreSQL', () => { ], }, { role: 'assistant', content: 'only tools ran', contentBlocks: [toolBlock('tool-2')] }, - { role: 'assistant', content: 'blocks not an array', contentBlocks: 'oops' }, { content: 'no role' }, { role: 'assistant', contentBlocks: [{ type: 'text', content: ' ' }] }, 'a bare string message', @@ -94,7 +93,6 @@ describe('text-only chat transcript in PostgreSQL', () => { 'user: Build me a workflow', 'assistant: Working on it. Done.', 'assistant: only tools ran', - 'assistant: blocks not an array', 'user: no role', 'assistant:', 'user:', @@ -104,6 +102,15 @@ describe('text-only chat transcript in PostgreSQL', () => { expect(JSON.stringify(text?.messages).length).toBeLessThan(10_000) }) + it('quotes a message whose contentBlocks is not an array by its content', async () => { + const chatId = await seedChat([ + { role: 'assistant', content: 'blocks not an array', contentBlocks: 'oops' }, + ]) + + const text = await getAccessibleCopilotChatWithTextMessages(chatId, userId) + expect(quoteChatTranscript(text?.messages ?? [])).toBe('assistant: blocks not an array') + }) + it('skips deleted messages and refuses a chat the caller does not own', async () => { const chatId = await seedChat([ { role: 'user', content: 'kept' }, diff --git a/apps/sim/lib/mothership/chat/text-transcript.ts b/apps/sim/lib/mothership/chat/text-transcript.ts index 204cf40410b..770eaaccb29 100644 --- a/apps/sim/lib/mothership/chat/text-transcript.ts +++ b/apps/sim/lib/mothership/chat/text-transcript.ts @@ -18,9 +18,10 @@ export const detoastedMessageContent: SQL = sql`(select ${copilotMessages.conten */ export function textMessageContent(fields: readonly string[]): SQL { const content = sql`detoasted.content` - // The strict path copies only the blocks it keeps, never the tool payloads it skips. + // The strict path copies only the blocks it keeps, never the tool payloads it skips; `silent` + // skips a malformed block instead of failing the read. const textBlocks = sql`case when jsonb_typeof(${content} -> 'contentBlocks') = 'array' - then jsonb_path_query_array(${content}, 'strict $.contentBlocks[*] ? (@.type == "text")') end` + then jsonb_path_query_array(${content}, 'strict $.contentBlocks[*] ? (@.type == "text")', '{}', true) end` const entries = [ ...fields.map((field) => sql`${field}::text, ${content} -> ${field}::text`), sql`'contentBlocks', ${textBlocks}`, diff --git a/scripts/check-unused-exports.baseline.json b/scripts/check-unused-exports.baseline.json index 8225d5e84e9..3b6288eae5d 100644 --- a/scripts/check-unused-exports.baseline.json +++ b/scripts/check-unused-exports.baseline.json @@ -4221,7 +4221,6 @@ "apps/sim/lib/chunkers/index.ts#ChunkBudget", "apps/sim/lib/chunkers/utils.ts#assertPositiveSafeInteger", "apps/sim/lib/chunkers/utils.ts#iterateWordBoundaryChunkSpans", - "apps/sim/lib/cleanup/chat-cleanup.ts#collectChatFiles", "apps/sim/lib/cleanup/chat-cleanup.ts#deleteStorageFiles", "apps/sim/lib/collab-doc/collab-state.ts#CollabDocStateToken", "apps/sim/lib/colors/brightness.ts#isLightColor",