From 5d0744b1764f0558c6b2acb7f035e44749cc8de1 Mon Sep 17 00:00:00 2001 From: Rolando Santamaria Maso Date: Sat, 19 Sep 2026 14:00:48 +0200 Subject: [PATCH 1/2] feat: multimodal ContentPart message input (text + inline images) --- README.md | 15 +++++++ anthropic.go | 34 +++++++++++----- chat.go | 3 ++ content.go | 70 +++++++++++++++++++++++++++++++++ gemini.go | 23 +++++++++-- message.go | 36 ++++++++++++++++- multimodal_test.go | 98 ++++++++++++++++++++++++++++++++++++++++++++++ openai.go | 66 +++++++++++++++++++++++++++---- responses.go | 28 +++++++++++-- 9 files changed, 347 insertions(+), 26 deletions(-) create mode 100644 content.go create mode 100644 multimodal_test.go diff --git a/README.md b/README.md index 4a8c1fe..47ec3d3 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,21 @@ sdk := llm.New(llm.WithProvider("my-gateway", Requests and results are provider-neutral. Unknown message roles are rejected at the SDK boundary (never silently dropped or reinterpreted). +User messages may contain ordered text and inline image parts. `Content` remains +the backwards-compatible plain-text form; use `TextPart` and `ImagePart` when +an image is needed. Images are sent as bounded base64 data (up to +`MaxImageBytes`) to every supported provider format. + +```go +req := &llm.ChatRequest{Messages: []llm.Message{{ + Role: llm.RoleUser, + Parts: []llm.ContentPart{ + llm.TextPart("Describe this image:"), + llm.ImagePart("image/png", pngBytes), + }, +}}} +``` + ```go type ChatRequest struct { Model string // optional; ChatClient's model wins when both set diff --git a/anthropic.go b/anthropic.go index db36249..eae46c4 100644 --- a/anthropic.go +++ b/anthropic.go @@ -2,6 +2,7 @@ package llm import ( "context" + "encoding/base64" "encoding/json" "fmt" neturl "net/url" @@ -27,8 +28,9 @@ type anSysBlock struct { } type anBlock struct { - Type string `json:"type"` // "text" | "tool_use" | "tool_result" - Text string `json:"text,omitempty"` + Type string `json:"type"` // "text" | "tool_use" | "tool_result" + Text string `json:"text,omitempty"` + Source *anImageSource `json:"source,omitempty"` // CacheControl is set on user text blocks when Message.Cache is true. CacheControl *anCacheControl `json:"cache_control,omitempty"` // thinking (replayed assistant turns; must be the FIRST block and @@ -44,6 +46,12 @@ type anBlock struct { Result string `json:"content,omitempty"` } +type anImageSource struct { + Type string `json:"type"` + MediaType string `json:"media_type"` + Data string `json:"data"` +} + type anMessage struct { Role string `json:"role"` // "user" | "assistant" Content []anBlock `json:"content"` @@ -171,14 +179,22 @@ func buildAnthropicRequest(req *ChatRequest, model string, stream bool) ([]byte, case RoleSystem: out.System = append(out.System, anSysBlock{Type: "text", Text: m.Content}) case RoleUser: - blk := anBlock{Type: "text", Text: m.Content} - if m.Cache { - blk.CacheControl = &anCacheControl{Type: "ephemeral"} + blocks := make([]anBlock, 0, maxInt(1, len(m.Parts))) + if len(m.Parts) == 0 { + blocks = append(blocks, anBlock{Type: "text", Text: m.Content}) + } else { + for _, p := range m.Parts { + if p.Type == ContentPartImage { + blocks = append(blocks, anBlock{Type: "image", Source: &anImageSource{Type: "base64", MediaType: p.MIMEType, Data: base64.StdEncoding.EncodeToString(p.Image)}}) + } else { + blocks = append(blocks, anBlock{Type: "text", Text: p.Text}) + } + } } - out.Messages = append(out.Messages, anMessage{ - Role: "user", - Content: []anBlock{blk}, - }) + if m.Cache && len(blocks) > 0 && blocks[0].Type == "text" { + blocks[0].CacheControl = &anCacheControl{Type: "ephemeral"} + } + out.Messages = append(out.Messages, anMessage{Role: "user", Content: blocks}) case RoleAssistant: if m.ReasoningContent != "" && m.ThinkingSignature == "" { return nil, &ConfigError{Msg: fmt.Sprintf("message %d: Anthropic thinking replay requires ThinkingSignature", i)} diff --git a/chat.go b/chat.go index 3d558e0..66668d9 100644 --- a/chat.go +++ b/chat.go @@ -141,6 +141,9 @@ func (pc *providerClient) buildChatRequest(req *ChatRequest, model string, strea return nil, "", &ConfigError{Msg: fmt.Sprintf("message %d: unknown role %q", i, string(m.Role))} } } + if err := validateRequestContent(req.Messages); err != nil { + return nil, "", err + } if model == "" { model = req.Model } diff --git a/content.go b/content.go new file mode 100644 index 0000000..d94d8f3 --- /dev/null +++ b/content.go @@ -0,0 +1,70 @@ +package llm + +import ( + "fmt" + "mime" + "strings" +) + +// MaxRequestImageBytes bounds the aggregate inline image payload in one +// request, preventing many individually valid parts from causing an +// unexpectedly large allocation or HTTP body. +const MaxRequestImageBytes = 32 << 20 + +func validateRequestContent(messages []Message) error { + total := 0 + for i, m := range messages { + if err := validateMessageContent(m, i); err != nil { + return err + } + for _, p := range m.Parts { + total += len(p.Image) + if total > MaxRequestImageBytes { + return &ConfigError{Msg: fmt.Sprintf("request inline images exceed %d bytes", MaxRequestImageBytes)} + } + } + } + return nil +} + +func validateMessageContent(m Message, index int) error { + if len(m.Parts) == 0 { + return nil + } + if m.Content != "" { + return &ConfigError{Msg: fmt.Sprintf("message %d: Content and Parts cannot both be set", index)} + } + if m.Role != RoleUser { + return &ConfigError{Msg: fmt.Sprintf("message %d: content parts require user role", index)} + } + for j, p := range m.Parts { + switch p.Type { + case ContentPartText: + if p.Image != nil || p.MIMEType != "" { + return &ConfigError{Msg: fmt.Sprintf("message %d part %d: text part has image fields", index, j)} + } + case ContentPartImage: + if p.Text != "" { + return &ConfigError{Msg: fmt.Sprintf("message %d part %d: image part has text fields", index, j)} + } + media, params, err := mime.ParseMediaType(p.MIMEType) + if err != nil || len(params) != 0 || !supportedImageMIME[strings.ToLower(media)] || media != strings.ToLower(media) { + return &ConfigError{Msg: fmt.Sprintf("message %d part %d: MIMEType must be an image type", index, j)} + } + if len(p.Image) == 0 { + return &ConfigError{Msg: fmt.Sprintf("message %d part %d: image data is empty", index, j)} + } + if len(p.Image) > MaxImageBytes { + return &ConfigError{Msg: fmt.Sprintf("message %d part %d: image exceeds %d bytes", index, j, MaxImageBytes)} + } + default: + return &ConfigError{Msg: fmt.Sprintf("message %d part %d: unknown content part type %q", index, j, p.Type)} + } + } + return nil +} + +var supportedImageMIME = map[string]bool{ + "image/png": true, "image/jpeg": true, "image/jpg": true, + "image/gif": true, "image/webp": true, +} diff --git a/gemini.go b/gemini.go index dbce468..27357fb 100644 --- a/gemini.go +++ b/gemini.go @@ -2,6 +2,7 @@ package llm import ( "context" + "encoding/base64" "encoding/json" "fmt" neturl "net/url" @@ -27,10 +28,15 @@ type gmFnResp struct { type gmPart struct { Text string `json:"text,omitempty"` + InlineData *gmBlob `json:"inlineData,omitempty"` Thought bool `json:"thought,omitempty"` FunctionCall *gmFnCall `json:"functionCall,omitempty"` FunctionResponse *gmFnResp `json:"functionResponse,omitempty"` } +type gmBlob struct { + MIMEType string `json:"mimeType"` + Data string `json:"data"` +} type gmContent struct { Role string `json:"role,omitempty"` // "user" | "model" @@ -134,10 +140,19 @@ func buildGeminiRequest(req *ChatRequest, model string, stream bool) ([]byte, er case RoleSystem: // folded into systemInstruction case RoleUser: - out.Contents = append(out.Contents, gmContent{ - Role: "user", - Parts: []gmPart{{Text: m.Content}}, - }) + parts := make([]gmPart, 0, maxInt(1, len(m.Parts))) + if len(m.Parts) == 0 { + parts = append(parts, gmPart{Text: m.Content}) + } else { + for _, p := range m.Parts { + if p.Type == ContentPartImage { + parts = append(parts, gmPart{InlineData: &gmBlob{MIMEType: p.MIMEType, Data: base64.StdEncoding.EncodeToString(p.Image)}}) + } else { + parts = append(parts, gmPart{Text: p.Text}) + } + } + } + out.Contents = append(out.Contents, gmContent{Role: "user", Parts: parts}) case RoleAssistant: for _, tc := range m.ToolCalls { if tc.ID != "" { diff --git a/message.go b/message.go index 8925134..43457c7 100644 --- a/message.go +++ b/message.go @@ -22,6 +22,35 @@ import ( // Role enumerates canonical message roles. type Role string +// ContentPartType identifies the payload carried by ContentPart. +type ContentPartType string + +const ( + ContentPartText ContentPartType = "text" + ContentPartImage ContentPartType = "image" +) + +// ContentPart is one ordered text or inline-image segment in a message. +// Image data is raw bytes and is base64-encoded only by provider serializers. +// A part must contain exactly one supported payload: Text, or Image with a +// valid MIMEType. Inline images are bounded by MaxImageBytes. +type ContentPart struct { + Type ContentPartType + Text string + Image []byte + MIMEType string +} + +const MaxImageBytes = 10 << 20 + +// TextPart creates an ordered text content part. +func TextPart(text string) ContentPart { return ContentPart{Type: ContentPartText, Text: text} } + +// ImagePart creates an ordered inline-image content part. +func ImagePart(mimeType string, data []byte) ContentPart { + return ContentPart{Type: ContentPartImage, Image: data, MIMEType: mimeType} +} + const ( RoleSystem Role = "system" RoleUser Role = "user" @@ -55,8 +84,11 @@ type ToolCall struct { // Anthropic re-serializes a signed thinking block as the first content // block when ThinkingSignature is also set. type Message struct { - Role Role - Content string + Role Role + Content string + // Parts optionally carries ordered text and inline-image content. When + // empty, Content retains the legacy plain-text representation. + Parts []ContentPart ReasoningContent string // ThinkingSignature authenticates ReasoningContent for providers that // require thinking to be replayed verbatim (Anthropic signature, OpenAI diff --git a/multimodal_test.go b/multimodal_test.go new file mode 100644 index 0000000..38d249f --- /dev/null +++ b/multimodal_test.go @@ -0,0 +1,98 @@ +package llm + +import ( + "encoding/json" + "strings" + "testing" +) + +func multimodalRequest() *ChatRequest { + return &ChatRequest{Messages: []Message{{Role: RoleUser, Parts: []ContentPart{TextPart("before"), ImagePart("image/png", []byte{1, 2, 3}), TextPart("after")}}}} +} + +func TestMultimodalContentPartsOpenAI(t *testing.T) { + b := buildOpenAIRequest(ProviderConfig{}, multimodalRequest(), "gpt-4o", false, false) + if len(b.Messages) != 1 { + t.Fatalf("messages=%d", len(b.Messages)) + } + raw, err := json.Marshal(b.Messages[0]) + if err != nil { + t.Fatal(err) + } + got := string(raw) + if !strings.Contains(got, `"content":[{"type":"text"`) || !strings.Contains(got, `data:image/png;base64,AQID`) { + t.Fatalf("content=%s", got) + } +} + +func TestMultimodalContentPartsOtherFormats(t *testing.T) { + for _, format := range []Format{FormatAnthropic, FormatGemini} { + var body []byte + var err error + if format == FormatAnthropic { + body, err = buildAnthropicRequest(multimodalRequest(), "claude", false) + } else { + body, err = buildGeminiRequest(multimodalRequest(), "gemini", false) + } + if err != nil { + t.Fatal(err) + } + got := string(body) + if !strings.Contains(got, `"image/png"`) || !strings.Contains(got, `AQID`) || !strings.Contains(got, "before") || !strings.Contains(got, "after") { + t.Fatalf("%s body=%s", format, got) + } + } + _, input := buildResponsesInput(multimodalRequest()) + got, _ := json.Marshal(input) + if !strings.Contains(string(got), `"input_image"`) || !strings.Contains(string(got), `data:image/png;base64,AQID`) { + t.Fatalf("responses=%s", got) + } +} + +func TestMultimodalValidationBeforeNetwork(t *testing.T) { + pc := newProviderClient(ProviderConfig{Format: FormatOpenAI, BaseURL: "http://127.0.0.1:1"}, nil, nil) + for _, m := range []Message{ + {Role: RoleUser, Content: "legacy", Parts: []ContentPart{TextPart("part")}}, + {Role: RoleUser, Parts: []ContentPart{{Type: ContentPartImage, MIMEType: "text/plain", Image: []byte("x")}}}, + {Role: RoleUser, Parts: []ContentPart{{Type: ContentPartImage, MIMEType: "image/png; charset=utf-8", Text: "hidden", Image: []byte("x")}}}, + {Role: RoleUser, Parts: []ContentPart{{Type: ContentPartImage, MIMEType: "image/bmp", Image: []byte("x")}}}, + {Role: RoleSystem, Parts: []ContentPart{ImagePart("image/png", []byte("x"))}}, + {Role: RoleUser, Parts: []ContentPart{{Type: "bad", Text: "x"}}}, + } { + if _, _, err := pc.buildChatRequest(&ChatRequest{Messages: []Message{m}}, "gpt-4o", false); err == nil { + t.Fatalf("invalid content accepted: %+v", m) + } + } +} + +func TestMultimodalAggregateImageCap(t *testing.T) { + parts := make([]ContentPart, 0, 4) + for i := 0; i < 4; i++ { + parts = append(parts, ImagePart("image/png", make([]byte, 9<<20))) + } + pc := newProviderClient(ProviderConfig{Format: FormatOpenAI, BaseURL: "http://example.invalid"}, nil, nil) + if _, _, err := pc.buildChatRequest(&ChatRequest{Messages: []Message{{Role: RoleUser, Parts: parts}}}, "model", false); err == nil { + t.Fatal("aggregate image cap accepted oversized request") + } +} + +func TestPlainContentSerializationUnchanged(t *testing.T) { + b := buildOpenAIRequest(ProviderConfig{}, &ChatRequest{Messages: []Message{{Role: RoleUser, Content: "hello"}}}, "gpt-4o", false, false) + raw, _ := json.Marshal(b.Messages[0].Content) + if string(raw) != `"hello"` { + t.Fatalf("content=%s", raw) + } +} + +func TestMultimodalStreamRequestPath(t *testing.T) { + for _, format := range []Format{FormatOpenAI, FormatAnthropic, FormatGemini} { + pc := newProviderClient(ProviderConfig{Format: format, BaseURL: "http://example.invalid"}, nil, nil) + body, _, err := pc.buildChatRequest(multimodalRequest(), "model", true) + if err != nil { + t.Fatalf("%s: %v", format, err) + } + if !strings.Contains(string(body), "AQID") { + t.Fatalf("%s stream body omitted image", format) + } + } +} diff --git a/openai.go b/openai.go index 410c08b..4052359 100644 --- a/openai.go +++ b/openai.go @@ -1,6 +1,7 @@ package llm import ( + "encoding/base64" "encoding/json" "fmt" "strings" @@ -31,12 +32,53 @@ type oaMessage struct { Role string `json:"role"` // Content and ReasoningContent are pointers so the key can be present and // empty: nil omits it, a pointer to "" keeps it on the wire. - Content *string `json:"content"` // nil keeps JSON null for tool calls - ReasoningContent *string `json:"reasoning_content,omitempty"` - ToolCalls []oaToolCall `json:"tool_calls,omitempty"` - ToolCallID string `json:"tool_call_id,omitempty"` + Content *string `json:"content"` // nil keeps JSON null for tool calls + ContentParts []oaContentPart `json:"-"` + ReasoningContent *string `json:"reasoning_content,omitempty"` + ToolCalls []oaToolCall `json:"tool_calls,omitempty"` + ToolCallID string `json:"tool_call_id,omitempty"` } +func (m oaMessage) MarshalJSON() ([]byte, error) { + type alias oaMessage + if len(m.ContentParts) == 0 { + return json.Marshal(alias(m)) + } + return json.Marshal(struct { + Role string `json:"role"` + Content []oaContentPart `json:"content"` + ReasoningContent *string `json:"reasoning_content,omitempty"` + ToolCalls []oaToolCall `json:"tool_calls,omitempty"` + ToolCallID string `json:"tool_call_id,omitempty"` + }{m.Role, m.ContentParts, m.ReasoningContent, m.ToolCalls, m.ToolCallID}) +} + +type oaImageURL struct { + URL string `json:"url"` +} +type oaContentPart struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` + ImageURL *oaImageURL `json:"image_url,omitempty"` +} + +func openAIContent(m Message) any { + if len(m.Parts) == 0 { + return m.Content + } + parts := make([]oaContentPart, 0, len(m.Parts)) + for _, p := range m.Parts { + if p.Type == ContentPartImage { + parts = append(parts, oaContentPart{Type: "image_url", ImageURL: &oaImageURL{URL: "data:" + p.MIMEType + ";base64," + base64.StdEncoding.EncodeToString(p.Image)}}) + } else { + parts = append(parts, oaContentPart{Type: "text", Text: p.Text}) + } + } + return parts +} + +func openAIText(m Message) *string { return &m.Content } + type oaToolDef struct { Type string `json:"type"` // "function" Function json.RawMessage `json:"function"` @@ -111,8 +153,11 @@ func buildOpenAIRequest(cfg ProviderConfig, req *ChatRequest, model string, stre msgs = append(msgs, oaMessage{Role: "tool", Content: &c, ToolCallID: m.ToolCallID}) case RoleAssistant: om := oaMessage{Role: "assistant"} - c := m.Content - om.Content = &c + if len(m.Parts) == 0 { + om.Content = openAIText(m) + } else { + om.ContentParts = openAIContent(m).([]oaContentPart) + } // DeepSeek thinking mode requires the reasoning_content key on every // replayed assistant turn once the request carries tools — including // turns where the provider returned no reasoning of its own. Omitting @@ -143,8 +188,13 @@ func buildOpenAIRequest(cfg ProviderConfig, req *ChatRequest, model string, stre } msgs = append(msgs, om) default: // user - c := m.Content - msgs = append(msgs, oaMessage{Role: "user", Content: &c}) + om := oaMessage{Role: "user"} + if len(m.Parts) == 0 { + om.Content = openAIText(m) + } else { + om.ContentParts = openAIContent(m).([]oaContentPart) + } + msgs = append(msgs, om) } } out.Messages = msgs diff --git a/responses.go b/responses.go index 2021f05..d1aea57 100644 --- a/responses.go +++ b/responses.go @@ -1,6 +1,7 @@ package llm import ( + "encoding/base64" "encoding/json" "fmt" "strings" @@ -28,7 +29,28 @@ type rsTool struct { type rsEasyMessage struct { Role string `json:"role"` - Content string `json:"content"` + Content any `json:"content"` +} + +type rsInputContentPart struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` + ImageURL string `json:"image_url,omitempty"` +} + +func responsesContent(m Message) any { + if len(m.Parts) == 0 { + return m.Content + } + parts := make([]rsInputContentPart, 0, len(m.Parts)) + for _, p := range m.Parts { + if p.Type == ContentPartImage { + parts = append(parts, rsInputContentPart{Type: "input_image", ImageURL: "data:" + p.MIMEType + ";base64," + base64.StdEncoding.EncodeToString(p.Image)}) + } else { + parts = append(parts, rsInputContentPart{Type: "input_text", Text: p.Text}) + } + } + return parts } type rsReasoningItem struct { @@ -194,7 +216,7 @@ func buildResponsesInput(req *ChatRequest) (instructions string, input []any) { case RoleSystem: continue case RoleUser: - input = append(input, rsEasyMessage{Role: "user", Content: m.Content}) + input = append(input, rsEasyMessage{Role: "user", Content: responsesContent(m)}) case RoleAssistant: // Asymmetry with the chat-completions builder: this format replays // reasoning as an encrypted reasoning item gated on @@ -220,7 +242,7 @@ func buildResponsesInput(req *ChatRequest) (instructions string, input []any) { }) } if m.Content != "" { - input = append(input, rsEasyMessage{Role: "assistant", Content: m.Content}) + input = append(input, rsEasyMessage{Role: "assistant", Content: responsesContent(m)}) } case RoleTool: input = append(input, rsFunctionOutputItem{ From 26b2678591dc1a79ed63aea7a7c7f9357b6d25e4 Mon Sep 17 00:00:00 2001 From: Rolando Santamaria Maso Date: Sat, 19 Sep 2026 14:34:54 +0200 Subject: [PATCH 2/2] fix: review findings on multimodal input - normalize image/jpg alias to image/jpeg on the wire for all four formats (Anthropic and Gemini reject the informal alias) - reject empty text parts at validation (Anthropic/Gemini emit invalid degenerate blocks otherwise) - aggregate image-cap error carries the message index - README documents part rules, caps, and MIME normalization --- README.md | 8 ++++++-- anthropic.go | 2 +- content.go | 18 ++++++++++++++++-- gemini.go | 2 +- multimodal_test.go | 31 +++++++++++++++++++++++++++++++ openai.go | 2 +- responses.go | 2 +- 7 files changed, 57 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 47ec3d3..c8cf08d 100644 --- a/README.md +++ b/README.md @@ -107,8 +107,12 @@ Requests and results are provider-neutral. Unknown message roles are rejected at User messages may contain ordered text and inline image parts. `Content` remains the backwards-compatible plain-text form; use `TextPart` and `ImagePart` when -an image is needed. Images are sent as bounded base64 data (up to -`MaxImageBytes`) to every supported provider format. +an image is needed. Parts are user-role only and cannot be combined with +`Content` on the same message; empty text parts are rejected. Accepted image +types are png, jpeg, gif, and webp (the informal `image/jpg` alias is accepted +and normalized to `image/jpeg` on the wire). Per-image (`MaxImageBytes`, 10 MiB) +and per-request aggregate (`MaxRequestImageBytes`, 32 MiB) caps are enforced +before any network I/O. All validation failures are typed `*ConfigError`. ```go req := &llm.ChatRequest{Messages: []llm.Message{{ diff --git a/anthropic.go b/anthropic.go index eae46c4..0ec3df2 100644 --- a/anthropic.go +++ b/anthropic.go @@ -185,7 +185,7 @@ func buildAnthropicRequest(req *ChatRequest, model string, stream bool) ([]byte, } else { for _, p := range m.Parts { if p.Type == ContentPartImage { - blocks = append(blocks, anBlock{Type: "image", Source: &anImageSource{Type: "base64", MediaType: p.MIMEType, Data: base64.StdEncoding.EncodeToString(p.Image)}}) + blocks = append(blocks, anBlock{Type: "image", Source: &anImageSource{Type: "base64", MediaType: wireMIME(p.MIMEType), Data: base64.StdEncoding.EncodeToString(p.Image)}}) } else { blocks = append(blocks, anBlock{Type: "text", Text: p.Text}) } diff --git a/content.go b/content.go index d94d8f3..3c38eaf 100644 --- a/content.go +++ b/content.go @@ -20,7 +20,7 @@ func validateRequestContent(messages []Message) error { for _, p := range m.Parts { total += len(p.Image) if total > MaxRequestImageBytes { - return &ConfigError{Msg: fmt.Sprintf("request inline images exceed %d bytes", MaxRequestImageBytes)} + return &ConfigError{Msg: fmt.Sprintf("message %d: request inline images exceed %d bytes", i, MaxRequestImageBytes)} } } } @@ -40,6 +40,9 @@ func validateMessageContent(m Message, index int) error { for j, p := range m.Parts { switch p.Type { case ContentPartText: + if p.Text == "" { + return &ConfigError{Msg: fmt.Sprintf("message %d part %d: text part is empty", index, j)} + } if p.Image != nil || p.MIMEType != "" { return &ConfigError{Msg: fmt.Sprintf("message %d part %d: text part has image fields", index, j)} } @@ -65,6 +68,17 @@ func validateMessageContent(m Message, index int) error { } var supportedImageMIME = map[string]bool{ - "image/png": true, "image/jpeg": true, "image/jpg": true, + "image/png": true, "image/jpeg": true, "image/gif": true, "image/webp": true, } + +// wireMIME maps an accepted MIME type to the form providers expect on the +// wire. The informal image/jpg alias is accepted at the API boundary (some +// callers derive MIME from file extensions) but must not reach providers: +// Anthropic and Gemini reject it while image/jpeg is universally valid. +func wireMIME(m string) string { + if m == "image/jpg" { + return "image/jpeg" + } + return m +} diff --git a/gemini.go b/gemini.go index 27357fb..43ed7ff 100644 --- a/gemini.go +++ b/gemini.go @@ -146,7 +146,7 @@ func buildGeminiRequest(req *ChatRequest, model string, stream bool) ([]byte, er } else { for _, p := range m.Parts { if p.Type == ContentPartImage { - parts = append(parts, gmPart{InlineData: &gmBlob{MIMEType: p.MIMEType, Data: base64.StdEncoding.EncodeToString(p.Image)}}) + parts = append(parts, gmPart{InlineData: &gmBlob{MIMEType: wireMIME(p.MIMEType), Data: base64.StdEncoding.EncodeToString(p.Image)}}) } else { parts = append(parts, gmPart{Text: p.Text}) } diff --git a/multimodal_test.go b/multimodal_test.go index 38d249f..5201a54 100644 --- a/multimodal_test.go +++ b/multimodal_test.go @@ -96,3 +96,34 @@ func TestMultimodalStreamRequestPath(t *testing.T) { } } } + +func TestMultimodalJPGAliasNormalizedOnWire(t *testing.T) { + req := &ChatRequest{Messages: []Message{{Role: RoleUser, Parts: []ContentPart{ImagePart("image/jpg", []byte{1})}}}} + b, err := buildAnthropicRequest(req, "claude", false) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(b), "image/jpg") || !strings.Contains(string(b), "image/jpeg") { + t.Fatalf("anthropic media_type not normalized: %s", b) + } + g, err := buildGeminiRequest(req, "gemini", false) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(g), "image/jpg") || !strings.Contains(string(g), "image/jpeg") { + t.Fatalf("gemini mime_type not normalized: %s", g) + } + o := buildOpenAIRequest(ProviderConfig{}, req, "gpt-4o", false, false) + raw, _ := json.Marshal(o.Messages[0]) + if strings.Contains(string(raw), "image/jpg") || !strings.Contains(string(raw), "image/jpeg") { + t.Fatalf("openai data url not normalized: %s", raw) + } +} + +func TestMultimodalEmptyTextPartRejected(t *testing.T) { + pc := newProviderClient(ProviderConfig{Format: FormatOpenAI, BaseURL: "http://127.0.0.1:1"}, nil, nil) + _, _, err := pc.buildChatRequest(&ChatRequest{Messages: []Message{{Role: RoleUser, Parts: []ContentPart{ImagePart("image/png", []byte("x")), {Type: ContentPartText}}}}}, "m", false) + if err == nil || !strings.Contains(err.Error(), "text part is empty") { + t.Fatalf("empty text part accepted: %v", err) + } +} diff --git a/openai.go b/openai.go index 4052359..0bdc9b5 100644 --- a/openai.go +++ b/openai.go @@ -69,7 +69,7 @@ func openAIContent(m Message) any { parts := make([]oaContentPart, 0, len(m.Parts)) for _, p := range m.Parts { if p.Type == ContentPartImage { - parts = append(parts, oaContentPart{Type: "image_url", ImageURL: &oaImageURL{URL: "data:" + p.MIMEType + ";base64," + base64.StdEncoding.EncodeToString(p.Image)}}) + parts = append(parts, oaContentPart{Type: "image_url", ImageURL: &oaImageURL{URL: "data:" + wireMIME(p.MIMEType) + ";base64," + base64.StdEncoding.EncodeToString(p.Image)}}) } else { parts = append(parts, oaContentPart{Type: "text", Text: p.Text}) } diff --git a/responses.go b/responses.go index d1aea57..d11646b 100644 --- a/responses.go +++ b/responses.go @@ -45,7 +45,7 @@ func responsesContent(m Message) any { parts := make([]rsInputContentPart, 0, len(m.Parts)) for _, p := range m.Parts { if p.Type == ContentPartImage { - parts = append(parts, rsInputContentPart{Type: "input_image", ImageURL: "data:" + p.MIMEType + ";base64," + base64.StdEncoding.EncodeToString(p.Image)}) + parts = append(parts, rsInputContentPart{Type: "input_image", ImageURL: "data:" + wireMIME(p.MIMEType) + ";base64," + base64.StdEncoding.EncodeToString(p.Image)}) } else { parts = append(parts, rsInputContentPart{Type: "input_text", Text: p.Text}) }