diff --git a/js/plugins/google-genai/src/googleai/gemini.ts b/js/plugins/google-genai/src/googleai/gemini.ts index 775fa96871..38d5bf7d7a 100644 --- a/js/plugins/google-genai/src/googleai/gemini.ts +++ b/js/plugins/google-genai/src/googleai/gemini.ts @@ -65,6 +65,8 @@ import { removeClientOptionOverrides, } from './utils.js'; +const MAX_INLINE_MEDIA_BYTES = 1024 * 1024 * 100; // 100 MB + /** * See https://ai.google.dev/gemini-api/docs/safety-settings#safety-filters. */ @@ -576,11 +578,15 @@ export function defineModel( const middleware: ModelMiddleware[] = []; if (ref.info?.supports?.media) { - // the gemini api doesn't support downloading media from http(s) + // For Gemini 2.0, external URLs are not supported, so we must download. + // For newer models, we can pass the URL directly. + const supportsExternalUrls = !name.startsWith('gemini-2.0'); + middleware.push( downloadRequestMedia({ - maxBytes: 1024 * 1024 * 10, - // don't downlaod files that have been uploaded using the Files API + maxBytes: MAX_INLINE_MEDIA_BYTES, + // don't download files that have been uploaded using the Files API + // or external URLs supported by the model filter: (part) => { try { const url = new URL(part.media.url); @@ -594,6 +600,14 @@ export function defineModel( ].includes(url.hostname) ) return false; + + // If model supports external URLs, allow http/https URLs to pass through + if ( + supportsExternalUrls && + (url.protocol === 'https:' || url.protocol === 'http:') + ) { + return false; + } } catch {} return true; }, diff --git a/js/plugins/google-genai/tests/googleai/gemini_test.ts b/js/plugins/google-genai/tests/googleai/gemini_test.ts index 9202feacea..bae64a681a 100644 --- a/js/plugins/google-genai/tests/googleai/gemini_test.ts +++ b/js/plugins/google-genai/tests/googleai/gemini_test.ts @@ -378,6 +378,56 @@ describe('Google AI Gemini', () => { }); }); + describe('Media Handling', () => { + const imageUrl = 'https://example.com/image.png'; + + it('passes external URLs for non-Gemini 2.0 models', async () => { + const model = defineModel( + 'gemini-3-flash-preview', + defaultPluginOptions + ); + + fetchStub.callsFake(async (url: string | Request) => { + if (typeof url === 'string' && url === imageUrl) { + return new Response('image-data', { + headers: { 'Content-Type': 'image/png' }, + status: 200, + }); + } + return new Response(JSON.stringify(defaultApiResponse), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }); + + const request: GenerateRequest = { + messages: [ + { + role: 'user', + content: [{ media: { url: imageUrl, contentType: 'image/png' } }], + }, + ], + }; + + await model.run(request); + + // Verify image was NOT downloaded + assert.ok( + !fetchStub.calledWith(imageUrl), + 'Should NOT attempt to download image for Gemini 3.0' + ); + + // Verify API request contained fileData + const apiRequest: GenerateContentRequest = JSON.parse( + fetchStub.lastCall.args[1].body + ); + const part = apiRequest.contents[0].parts[0]; + assert.ok(part.fileData, 'Should be fileData'); + assert.strictEqual(part.fileData?.mimeType, 'image/png'); + assert.strictEqual(part.fileData?.fileUri, imageUrl); + }); + }); + describe('Error Handling', () => { it('throws if no candidates are returned', async () => { const model = defineModel('gemini-2.0-flash', defaultPluginOptions); diff --git a/js/testapps/basic-gemini/src/index.ts b/js/testapps/basic-gemini/src/index.ts index 40246ef2b9..1c5e8fb8da 100644 --- a/js/testapps/basic-gemini/src/index.ts +++ b/js/testapps/basic-gemini/src/index.ts @@ -745,3 +745,37 @@ async function downloadVideo(video: MediaPart, path: string) { Readable.from(videoDownloadResponse.body).pipe(fs.createWriteStream(path)); } + +// Test external URL with Gemini 2.0 (should download and inline) +ai.defineFlow('external-url-gemini-2.0', async () => { + const { text } = await ai.generate({ + model: googleAI.model('gemini-2.0-flash'), + prompt: [ + { text: 'Describe this image.' }, + { + media: { + url: 'https://storage.googleapis.com/generativeai-downloads/images/scones.jpg', + contentType: 'image/jpeg', + }, + }, + ], + }); + return text; +}); + +// Test external URL with Gemini 3.0 (should pass as fileUri) +ai.defineFlow('external-url-gemini-3.0', async () => { + const { text } = await ai.generate({ + model: googleAI.model('gemini-3-flash-preview'), + prompt: [ + { text: 'Describe this image.' }, + { + media: { + url: 'https://storage.googleapis.com/generativeai-downloads/images/scones.jpg', + contentType: 'image/jpeg', + }, + }, + ], + }); + return text; +});