fix(knowledge): restore chunk retry metadata

Author: TheodoreSpeaks

apps/sim/app/api/knowledge/[id]/documents/[documentId]/chunks/route.test.ts

@@ -0,0 +1,63 @@
+/**
+ * @vitest-environment node
+ */
+
+import { authMockFns, createMockRequest } from '@sim/testing'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const mocks = vi.hoisted(() => ({
+  list: vi.fn(),
+  create: vi.fn(),
+  bulk: vi.fn(),
+}))
+
+vi.mock('@/lib/knowledge/application/chunks', () => ({
+  listKnowledgeChunks: {
+    operation: { id: 'knowledge.chunks.list' },
+    execute: mocks.list,
+  },
+  createKnowledgeChunk: {
+    operation: { id: 'knowledge.chunks.create' },
+    execute: mocks.create,
+  },
+  bulkUpdateKnowledgeChunks: {
+    operation: { id: 'knowledge.chunks.bulk' },
+    execute: mocks.bulk,
+  },
+}))
+
+vi.mock('@/app/api/knowledge/secret-provenance', () => ({
+  finalizeKnowledgePersistedResponse: vi.fn(),
+  finalizeKnowledgeProvenanceResponse: vi.fn(),
+  resolveKnowledgeWriteSecretProvenance: vi.fn(),
+}))
+
+import { KnowledgeDocumentNotReadyError } from '@/lib/knowledge/application/chunk-errors'
+import { GET } from '@/app/api/knowledge/[id]/documents/[documentId]/chunks/route'
+
+const params = () => ({
+  params: Promise.resolve({ id: 'knowledge-1', documentId: 'document-1' }),
+})
+
+describe('/api/knowledge/[id]/documents/[documentId]/chunks internal route composition', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    authMockFns.mockGetSession.mockResolvedValue({
+      user: { id: 'user-1' },
+      session: { id: 'session-1' },
+    })
+  })
+
+  it('preserves retry metadata when a document is still processing', async () => {
+    mocks.list.mockRejectedValueOnce(new KnowledgeDocumentNotReadyError('processing'))
+
+    const response = await GET(createMockRequest('GET'), params())
+
+    expect(response.status).toBe(400)
+    await expect(response.json()).resolves.toEqual({
+      error: 'Document is not ready for access',
+      details: 'Document status: processing',
+      retryAfter: 5,
+    })
+  })
+})

apps/sim/app/api/knowledge/[id]/documents/[documentId]/chunks/route.ts

@@ -54,7 +54,7 @@ export const GET = defineInternalJsonRoute({
   auth: internalKnowledgeSessionOrExecutorAuth,
   operation: knowledgeOperations.listChunks,
   rateLimit: internalRateLimits.none({ reason: 'Preserve existing internal chunk-list behavior' }),
-  errorPolicy: internalKnowledgeErrorPolicies.chunks,
+  errorPolicy: internalKnowledgeErrorPolicies.chunkList,
   mapInput: ({ params, query }) => ({
     knowledgeBaseId: params.id,
     documentId: params.documentId,

apps/sim/lib/knowledge/api/route-policies.ts

@@ -2,6 +2,7 @@ import {
   createInternalResourceConcealmentPolicy,
   createInternalSessionOrExecutorAuth,
   createV2ResourceConcealmentPolicy,
+  extendInternalErrorPolicy,
   type InternalErrorPolicy,
   internalErrorResponse,
   internalOrchestrationErrorPolicy,
@@ -11,6 +12,7 @@ import {
 import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
 import { KNOWLEDGE_DELEGATION_AUDIENCE } from '@/lib/knowledge/application/authorization'
 import { KnowledgeUsageLimitExceededError } from '@/lib/knowledge/application/billing'
+import { KnowledgeDocumentNotReadyError } from '@/lib/knowledge/application/chunk-errors'
 import { KnowledgeSearchProvenanceUnavailableError } from '@/lib/knowledge/application/search'
 import { KnowledgeDocumentUnsupportedMediaTypeError } from '@/lib/knowledge/application/upload-sessions'
 import { v2Error } from '@/app/api/v2/lib/response'
@@ -82,6 +84,19 @@ export const internalKnowledgeErrorPolicies = {
   chunks: concealKnowledgeBase(
     internalKnowledgeErrorPolicy('Failed to process knowledge chunk request')
   ),
+  chunkList: concealKnowledgeBase(
+    extendInternalErrorPolicy(
+      internalKnowledgeErrorPolicy('Failed to process knowledge chunk request'),
+      (error) =>
+        error instanceof KnowledgeDocumentNotReadyError
+          ? internalErrorResponse(400, {
+              error: 'Document is not ready for access',
+              details: `Document status: ${error.processingStatus}`,
+              retryAfter: error.processingStatus === 'processing' ? 5 : null,
+            })
+          : null
+    )
+  ),
   upsert: concealKnowledgeBase(internalKnowledgeUploadErrorPolicy),
   search: concealKnowledgeBase(internalKnowledgeSearchErrorPolicy),
   tags: concealKnowledgeBase(

apps/sim/lib/knowledge/application/chunk-errors.ts

@@ -0,0 +1,8 @@
+import { OrchestrationError } from '@/lib/core/orchestration/types'
+
+export class KnowledgeDocumentNotReadyError extends OrchestrationError {
+  constructor(readonly processingStatus: string) {
+    super('validation', `Document is not ready for access (status: ${processingStatus})`)
+    this.name = 'KnowledgeDocumentNotReadyError'
+  }
+}

apps/sim/lib/knowledge/application/chunks.test.ts

@@ -0,0 +1,79 @@
+/**
+ * @vitest-environment node
+ */
+
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const mocks = vi.hoisted(() => ({
+  resolveDocument: vi.fn(),
+  resolvePermission: vi.fn(),
+  queryChunks: vi.fn(),
+}))
+
+vi.mock('@sim/platform-authz/workspace', () => ({
+  permissionSatisfies: (actual: string | null, required: string) => {
+    const rank = { read: 1, write: 2, admin: 3 } as const
+    return (
+      actual !== null && rank[actual as keyof typeof rank] >= rank[required as keyof typeof rank]
+    )
+  },
+  resolveEffectiveWorkspacePermission: mocks.resolvePermission,
+}))
+
+vi.mock('@/lib/knowledge/application/contexts', () => ({
+  resolveCanonicalActiveKnowledgeDocumentContext: mocks.resolveDocument,
+  resolveActiveKnowledgeChunkContext: vi.fn(),
+}))
+
+vi.mock('@/lib/knowledge/chunks/service', () => ({
+  batchChunkOperation: vi.fn(),
+  createChunk: vi.fn(),
+  deleteChunk: vi.fn(),
+  queryChunks: mocks.queryChunks,
+  updateChunk: vi.fn(),
+}))
+
+vi.mock('@/lib/execution/durable-secret-provenance', () => ({
+  createDurableSecretProvenanceRegistry: vi.fn(),
+}))
+
+vi.mock('@/lib/knowledge/model-input-provenance', () => ({
+  runWithKnowledgeModelInputProvenance: vi.fn(),
+}))
+
+vi.mock('@/providers/utils', () => ({ calculateCost: vi.fn() }))
+
+import { KnowledgeDocumentNotReadyError } from '@/lib/knowledge/application/chunk-errors'
+import { listKnowledgeChunks } from '@/lib/knowledge/application/chunks'
+
+describe('knowledge chunk application use cases', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mocks.resolvePermission.mockResolvedValue('read')
+    mocks.resolveDocument.mockResolvedValue({
+      workspaceId: 'workspace-1',
+      workspaceOrganizationId: null,
+      allowPersonalApiKeys: true,
+      billedAccountUserId: 'billing-owner-1',
+      knowledgeBaseId: 'knowledge-1',
+      knowledgeBase: { id: 'knowledge-1' },
+      documentId: 'document-1',
+      document: { id: 'document-1', processingStatus: 'processing' },
+    })
+  })
+
+  it('returns a typed transient failure before querying chunks for a processing document', async () => {
+    const promise = listKnowledgeChunks.execute({
+      principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' },
+      input: { knowledgeBaseId: 'knowledge-1', documentId: 'document-1' },
+    })
+
+    await expect(promise).rejects.toBeInstanceOf(KnowledgeDocumentNotReadyError)
+    await expect(promise).rejects.toMatchObject({
+      code: 'validation',
+      processingStatus: 'processing',
+      message: 'Document is not ready for access (status: processing)',
+    })
+    expect(mocks.queryChunks).not.toHaveBeenCalled()
+  })
+})

apps/sim/lib/knowledge/application/chunks.ts

@@ -7,6 +7,7 @@ import {
 } from '@/lib/execution/durable-secret-provenance'
 import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case'
 import { resolveKnowledgeAttributedUserId } from '@/lib/knowledge/application/billing'
+import { KnowledgeDocumentNotReadyError } from '@/lib/knowledge/application/chunk-errors'
 import {
   type ActiveKnowledgeDocumentContext,
   resolveActiveKnowledgeChunkContext,
@@ -62,10 +63,7 @@ export interface BulkKnowledgeChunksInput extends KnowledgeDocumentChunkInput {
 
 function requireChunkReadable(context: ActiveKnowledgeDocumentContext): void {
   if (context.document.processingStatus !== 'completed') {
-    throw new OrchestrationError(
-      'validation',
-      `Document is not ready for access (status: ${context.document.processingStatus})`
-    )
+    throw new KnowledgeDocumentNotReadyError(context.document.processingStatus)
   }
 }