From e45dd3cf6a81e809d958bbbf1a81644447031f79 Mon Sep 17 00:00:00 2001 From: Jan Kubica Date: Tue, 7 Jul 2026 13:11:57 +0200 Subject: [PATCH] feat(ai-mistral): support document (PDF) input via document_url The Mistral text adapter already snake-cased a `document_url` part in messageToWire, but convertContentPartToMistral never produced one and threw for any `document` content part, and model-meta declared no `document` input modality. Add the document branch (mirroring the image path: hosted URLs pass through, inline bytes become a data: URL) and declare `document` input for the vision-capable models, matching how the other provider adapters expose PDF input. Verified against Mistral's API: /v1/models reports vision=true for these models, and document_url round-trips a PDF on mistral-small-latest and mistral-medium-latest (both hosted-URL and inline base64). --- .changeset/mistral-document-input.md | 5 ++ packages/ai-mistral/src/adapters/text.ts | 11 +++- packages/ai-mistral/src/model-meta.ts | 10 +-- .../ai-mistral/tests/document-input.test.ts | 64 +++++++++++++++++++ 4 files changed, 84 insertions(+), 6 deletions(-) create mode 100644 .changeset/mistral-document-input.md create mode 100644 packages/ai-mistral/tests/document-input.test.ts diff --git a/.changeset/mistral-document-input.md b/.changeset/mistral-document-input.md new file mode 100644 index 000000000..8a4406ccb --- /dev/null +++ b/.changeset/mistral-document-input.md @@ -0,0 +1,5 @@ +--- +'@tanstack/ai-mistral': minor +--- + +Support document (PDF) input for Mistral vision models via `document_url`. The text adapter now maps `document` content parts to Mistral's `document_url` format (hosted URLs pass through; inline bytes are wrapped in a `data:` URL, mirroring image handling), and `document` is declared as an input modality for the vision-capable models. Previously any `document` content part threw "Supported types: text, image". diff --git a/packages/ai-mistral/src/adapters/text.ts b/packages/ai-mistral/src/adapters/text.ts index 6e7865e8e..e081ce000 100644 --- a/packages/ai-mistral/src/adapters/text.ts +++ b/packages/ai-mistral/src/adapters/text.ts @@ -1019,8 +1019,17 @@ export class MistralTextAdapter< } } + if (part.type === 'document') { + const documentValue = part.source.value + const documentUrl = + part.source.type === 'data' && !documentValue.startsWith('data:') + ? `data:${part.source.mimeType};base64,${documentValue}` + : documentValue + return { type: 'document_url', documentUrl } + } + throw new Error( - `Mistral text adapter does not support content part of type '${(part as ContentPart).type}'. Supported types: text, image. Use a vision-capable model (pixtral-large-latest, pixtral-12b-2409, mistral-medium-latest, or mistral-small-latest) for images.`, + `Mistral text adapter does not support content part of type '${(part as ContentPart).type}'. Supported types: text, image, document. Use a vision-capable model (mistral-medium-latest or mistral-small-latest) for images and documents.`, ) } diff --git a/packages/ai-mistral/src/model-meta.ts b/packages/ai-mistral/src/model-meta.ts index 45ba0a56e..2a1b4cfbd 100644 --- a/packages/ai-mistral/src/model-meta.ts +++ b/packages/ai-mistral/src/model-meta.ts @@ -19,7 +19,7 @@ interface ModelMeta { output?: { normal: number } } supports: { - input: Array<'text' | 'image' | 'audio'> + input: Array<'text' | 'image' | 'audio' | 'document'> output: Array<'text'> endpoints: Array<'chat' | 'embeddings'> @@ -61,7 +61,7 @@ const MISTRAL_MEDIUM_LATEST = { output: { normal: 2 }, }, supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: ['chat'], features: ['streaming', 'tools', 'json_object', 'json_schema', 'vision'], @@ -77,7 +77,7 @@ const MISTRAL_SMALL_LATEST = { output: { normal: 0.3 }, }, supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: ['chat'], features: ['streaming', 'tools', 'json_object', 'json_schema', 'vision'], @@ -141,7 +141,7 @@ const PIXTRAL_LARGE_LATEST = { output: { normal: 6 }, }, supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: ['chat'], features: ['streaming', 'tools', 'json_object', 'json_schema', 'vision'], @@ -157,7 +157,7 @@ const PIXTRAL_12B_2409 = { output: { normal: 0.15 }, }, supports: { - input: ['text', 'image'], + input: ['text', 'image', 'document'], output: ['text'], endpoints: ['chat'], features: ['streaming', 'tools', 'json_object', 'vision'], diff --git a/packages/ai-mistral/tests/document-input.test.ts b/packages/ai-mistral/tests/document-input.test.ts new file mode 100644 index 000000000..3c25a7d4d --- /dev/null +++ b/packages/ai-mistral/tests/document-input.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from 'vitest' +import { createMistralText } from '../src/adapters/text' +import type { DocumentPart } from '@tanstack/ai' + +// `convertContentPartToMistral` is private; reach it directly so the test +// exercises the document mapping without standing up an HTTP mock. The +// `messageToWire` step then snake-cases `documentUrl` -> `document_url`. +const convertContentPart = ( + adapter: ReturnType, + part: DocumentPart, +): unknown => + ( + adapter as unknown as { + convertContentPartToMistral: (p: DocumentPart) => unknown + } + ).convertContentPartToMistral(part) + +describe('mistral document content parts', () => { + const adapter = createMistralText('mistral-small-latest', 'test-key') + + it('maps a hosted-URL document to a document_url part', () => { + const part: DocumentPart = { + type: 'document', + source: { type: 'url', value: 'https://example.com/contract.pdf' }, + } + + expect(convertContentPart(adapter, part)).toEqual({ + type: 'document_url', + documentUrl: 'https://example.com/contract.pdf', + }) + }) + + it('wraps inline base64 document bytes in a data: URL', () => { + const part: DocumentPart = { + type: 'document', + source: { + type: 'data', + value: 'JVBERi0xLjQ=', + mimeType: 'application/pdf', + }, + } + + expect(convertContentPart(adapter, part)).toEqual({ + type: 'document_url', + documentUrl: 'data:application/pdf;base64,JVBERi0xLjQ=', + }) + }) + + it('passes through a data: URL that is already formatted', () => { + const part: DocumentPart = { + type: 'document', + source: { + type: 'data', + value: 'data:application/pdf;base64,JVBERi0xLjQ=', + mimeType: 'application/pdf', + }, + } + + expect(convertContentPart(adapter, part)).toEqual({ + type: 'document_url', + documentUrl: 'data:application/pdf;base64,JVBERi0xLjQ=', + }) + }) +})