diff --git a/ai-logic/firebase-ai-ondevice-interop/CHANGELOG.md b/ai-logic/firebase-ai-ondevice-interop/CHANGELOG.md index 18d392d7039..c618d823a53 100644 --- a/ai-logic/firebase-ai-ondevice-interop/CHANGELOG.md +++ b/ai-logic/firebase-ai-ondevice-interop/CHANGELOG.md @@ -1,5 +1,7 @@ # Unreleased +- [feature] Added support for on-device system instructions. + # 16.0.0-beta03 - [feature] Added `modelVersion` property to `GenerateContentResponse` (#8227) diff --git a/ai-logic/firebase-ai-ondevice-interop/api.txt b/ai-logic/firebase-ai-ondevice-interop/api.txt index 42f19306399..96efc52b99d 100644 --- a/ai-logic/firebase-ai-ondevice-interop/api.txt +++ b/ai-logic/firebase-ai-ondevice-interop/api.txt @@ -79,11 +79,12 @@ package com.google.firebase.ai.ondevice.interop { } public final class GenerateContentRequest { - ctor public GenerateContentRequest(com.google.firebase.ai.ondevice.interop.TextPart text, com.google.firebase.ai.ondevice.interop.ImagePart? image = null, Float? temperature = null, Integer? topK = null, Integer? seed = null, Integer? candidateCount = null, Integer? maxOutputTokens = null); + ctor public GenerateContentRequest(com.google.firebase.ai.ondevice.interop.TextPart text, com.google.firebase.ai.ondevice.interop.ImagePart? image = null, Float? temperature = null, Integer? topK = null, Integer? seed = null, Integer? candidateCount = null, Integer? maxOutputTokens = null, com.google.firebase.ai.ondevice.interop.TextPart? systemInstruction = null); method public Integer? getCandidateCount(); method public com.google.firebase.ai.ondevice.interop.ImagePart? getImage(); method public Integer? getMaxOutputTokens(); method public Integer? getSeed(); + method public com.google.firebase.ai.ondevice.interop.TextPart? getSystemInstruction(); method public Float? getTemperature(); method public com.google.firebase.ai.ondevice.interop.TextPart getText(); method public Integer? getTopK(); @@ -91,6 +92,7 @@ package com.google.firebase.ai.ondevice.interop { property public final com.google.firebase.ai.ondevice.interop.ImagePart? image; property public final Integer? maxOutputTokens; property public final Integer? seed; + property public final com.google.firebase.ai.ondevice.interop.TextPart? systemInstruction; property public final Float? temperature; property public final com.google.firebase.ai.ondevice.interop.TextPart text; property public final Integer? topK; diff --git a/ai-logic/firebase-ai-ondevice-interop/src/main/kotlin/com/google/firebase/ai/ondevice/interop/GenerateContentRequest.kt b/ai-logic/firebase-ai-ondevice-interop/src/main/kotlin/com/google/firebase/ai/ondevice/interop/GenerateContentRequest.kt index bfe2877887b..65ae961552d 100644 --- a/ai-logic/firebase-ai-ondevice-interop/src/main/kotlin/com/google/firebase/ai/ondevice/interop/GenerateContentRequest.kt +++ b/ai-logic/firebase-ai-ondevice-interop/src/main/kotlin/com/google/firebase/ai/ondevice/interop/GenerateContentRequest.kt @@ -30,6 +30,7 @@ package com.google.firebase.ai.ondevice.interop * @property candidateCount The number of candidates to generate. * @property maxOutputTokens Specifies the maximum number of tokens that can be generated in the * response. + * @property systemInstruction The system instructions for directing model behavior. */ public class GenerateContentRequest( public val text: TextPart, @@ -39,4 +40,5 @@ public class GenerateContentRequest( public val seed: Int? = null, public val candidateCount: Int? = null, public val maxOutputTokens: Int? = null, + public val systemInstruction: TextPart? = null, ) {} diff --git a/ai-logic/firebase-ai-ondevice/CHANGELOG.md b/ai-logic/firebase-ai-ondevice/CHANGELOG.md index 04dab651cfd..823a866de61 100644 --- a/ai-logic/firebase-ai-ondevice/CHANGELOG.md +++ b/ai-logic/firebase-ai-ondevice/CHANGELOG.md @@ -1,6 +1,7 @@ # Unreleased - [feature] Added `getOnDeviceModelName` to `GenerativeModel` (#8247) +- [feature] Added support for on-device system instructions. # 16.0.0-beta03 diff --git a/ai-logic/firebase-ai-ondevice/src/main/kotlin/com/google/firebase/ai/ondevice/Converters.kt b/ai-logic/firebase-ai-ondevice/src/main/kotlin/com/google/firebase/ai/ondevice/Converters.kt index 3b3e2454f9a..78fd67d2c4c 100644 --- a/ai-logic/firebase-ai-ondevice/src/main/kotlin/com/google/firebase/ai/ondevice/Converters.kt +++ b/ai-logic/firebase-ai-ondevice/src/main/kotlin/com/google/firebase/ai/ondevice/Converters.kt @@ -31,6 +31,7 @@ import com.google.mlkit.genai.prompt.GenerateContentRequest import com.google.mlkit.genai.prompt.ImagePart import com.google.mlkit.genai.prompt.ModelPreference import com.google.mlkit.genai.prompt.ModelReleaseStage +import com.google.mlkit.genai.prompt.SystemInstruction import com.google.mlkit.genai.prompt.TextPart import com.google.mlkit.genai.prompt.generationConfig import com.google.mlkit.genai.prompt.modelConfig @@ -74,6 +75,7 @@ internal fun com.google.firebase.ai.ondevice.interop.GenerateContentRequest.toMl candidateCount = this@toMlKit.candidateCount topK = this@toMlKit.topK seed = this@toMlKit.seed + this@toMlKit.systemInstruction?.let { systemInstruction = SystemInstruction(it.text) } } } catch (e: IllegalArgumentException) { throw FirebaseAIOnDeviceInvalidRequestException(e) diff --git a/ai-logic/firebase-ai-ondevice/src/test/kotlin/com/google/firebase/ai/ondevice/ConvertersTest.kt b/ai-logic/firebase-ai-ondevice/src/test/kotlin/com/google/firebase/ai/ondevice/ConvertersTest.kt index 92c51816ad7..0b4821b4428 100644 --- a/ai-logic/firebase-ai-ondevice/src/test/kotlin/com/google/firebase/ai/ondevice/ConvertersTest.kt +++ b/ai-logic/firebase-ai-ondevice/src/test/kotlin/com/google/firebase/ai/ondevice/ConvertersTest.kt @@ -101,7 +101,19 @@ internal class ConvertersTest { assertThat(mlKitRequest.topK).isEqualTo(3) assertThat(mlKitRequest.seed).isEqualTo(0) assertThat(mlKitRequest.candidateCount).isEqualTo(1) - assertThat(mlKitRequest.maxOutputTokens).isEqualTo(256) + assertThat(mlKitRequest.maxOutputTokens).isEqualTo(4096) + } + + @Test + fun `GenerateContentRequest toMlKit should convert systemInstruction correctly`() { + val interopRequest = + InteropGenerateContentRequest( + text = InteropTextPart("prompt"), + systemInstruction = InteropTextPart("system instruction") + ) + val mlKitRequest = interopRequest.toMlKit() + + assertThat(mlKitRequest.systemInstruction?.textString).isEqualTo("system instruction") } @Test diff --git a/ai-logic/firebase-ai/CHANGELOG.md b/ai-logic/firebase-ai/CHANGELOG.md index 3c8437ead69..e2d86f5af52 100644 --- a/ai-logic/firebase-ai/CHANGELOG.md +++ b/ai-logic/firebase-ai/CHANGELOG.md @@ -9,6 +9,7 @@ Note: The default location is now `global` instead of `us-central1` (no other functionality has changed). To continue using `us-central1`, specify `GenerativeBackend.agentPlatform(location = "us-central1"))`. +- [feature] Added support for on-device system instructions. # 17.14.0 diff --git a/ai-logic/firebase-ai/api.txt b/ai-logic/firebase-ai/api.txt index 9b02089f183..995f6724afe 100644 --- a/ai-logic/firebase-ai/api.txt +++ b/ai-logic/firebase-ai/api.txt @@ -150,11 +150,13 @@ package com.google.firebase.ai { ctor public OnDeviceConfig(com.google.firebase.ai.InferenceMode mode, Integer? maxOutputTokens = null, Float? temperature = null, Integer? topK = null, Integer? seed = null); ctor public OnDeviceConfig(com.google.firebase.ai.InferenceMode mode, Integer? maxOutputTokens = null, Float? temperature = null, Integer? topK = null, Integer? seed = null, int candidateCount = 1); ctor public OnDeviceConfig(com.google.firebase.ai.InferenceMode mode, Integer? maxOutputTokens = null, Float? temperature = null, Integer? topK = null, Integer? seed = null, int candidateCount = 1, com.google.firebase.ai.OnDeviceModelOption? modelOption = null); + ctor public OnDeviceConfig(com.google.firebase.ai.InferenceMode mode, Integer? maxOutputTokens = null, Float? temperature = null, Integer? topK = null, Integer? seed = null, int candidateCount = 1, com.google.firebase.ai.OnDeviceModelOption? modelOption = null, com.google.firebase.ai.type.Content? systemInstruction = null); method public int getCandidateCount(); method public Integer? getMaxOutputTokens(); method public com.google.firebase.ai.InferenceMode getMode(); method public com.google.firebase.ai.OnDeviceModelOption? getModelOption(); method public Integer? getSeed(); + method public com.google.firebase.ai.type.Content? getSystemInstruction(); method public Float? getTemperature(); method public Integer? getTopK(); property public final int candidateCount; @@ -162,6 +164,7 @@ package com.google.firebase.ai { property public final com.google.firebase.ai.InferenceMode mode; property public final com.google.firebase.ai.OnDeviceModelOption? modelOption; property public final Integer? seed; + property public final com.google.firebase.ai.type.Content? systemInstruction; property public final Float? temperature; property public final Integer? topK; field public static final com.google.firebase.ai.OnDeviceConfig.Companion Companion; diff --git a/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/GenerativeModel.kt b/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/GenerativeModel.kt index 389d54434c5..b659958d33f 100644 --- a/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/GenerativeModel.kt +++ b/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/GenerativeModel.kt @@ -356,14 +356,31 @@ internal constructor( @OptIn(PublicPreviewAPI::class) internal fun buildOnDeviceModelProvider( modelOption: OnDeviceModelOption? - ): GenerativeModelProvider = - onDeviceFactoryProvider?.let { + ): GenerativeModelProvider { + // if developer didn't pass a OnDeviceConfig.SystemInstruction, we use GenerativeModel.systemInstruction + val effectiveOnDeviceConfig = + if (onDeviceConfig.systemInstruction == null && systemInstruction != null) { + OnDeviceConfig( + mode = onDeviceConfig.mode, + maxOutputTokens = onDeviceConfig.maxOutputTokens, + temperature = onDeviceConfig.temperature, + topK = onDeviceConfig.topK, + seed = onDeviceConfig.seed, + candidateCount = onDeviceConfig.candidateCount, + modelOption = onDeviceConfig.modelOption, + systemInstruction = systemInstruction + ) + } else { + onDeviceConfig + } + return onDeviceFactoryProvider?.let { OnDeviceGenerativeModelProvider( it.newGenerativeModel(modelOption?.toInterop()), - onDeviceConfig + effectiveOnDeviceConfig ) } ?: MissingOnDeviceGenerativeModelProvider() + } @PublicPreviewAPI internal fun getModelProvider(): GenerativeModelProvider = diff --git a/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/OnDeviceConfig.kt b/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/OnDeviceConfig.kt index 7b613ebaf8c..8860fa1cc76 100644 --- a/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/OnDeviceConfig.kt +++ b/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/OnDeviceConfig.kt @@ -16,6 +16,7 @@ package com.google.firebase.ai +import com.google.firebase.ai.type.Content import com.google.firebase.ai.type.FirebaseAIException import com.google.firebase.ai.type.PublicPreviewAPI @@ -34,6 +35,7 @@ import com.google.firebase.ai.type.PublicPreviewAPI * @property candidateCount The number of generated responses to return. See [GenerationConfig] for * more detail. By default it's set to `1`. * @property modelOption Configuration for the on-device model selection and performance. + * @property systemInstruction Instructions that direct the model to behave in a certain way. */ @PublicPreviewAPI public class OnDeviceConfig @@ -45,7 +47,8 @@ constructor( public val topK: Int? = null, public val seed: Int? = null, public val candidateCount: Int = 1, - public val modelOption: OnDeviceModelOption? = null + public val modelOption: OnDeviceModelOption? = null, + public val systemInstruction: Content? = null ) { public companion object { diff --git a/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/generativemodel/OnDeviceGenerativeModelProvider.kt b/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/generativemodel/OnDeviceGenerativeModelProvider.kt index 5471db5dfdb..896dea968ff 100644 --- a/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/generativemodel/OnDeviceGenerativeModelProvider.kt +++ b/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/generativemodel/OnDeviceGenerativeModelProvider.kt @@ -215,6 +215,13 @@ internal class OnDeviceGenerativeModelProvider( ) } .firstOrNull() + val systemInstructionPart = + onDeviceConfig.systemInstruction + ?.parts + ?.filterIsInstance() + ?.joinToString("") { it.text } + ?.takeIf { it.isNotEmpty() } + ?.let { OnDeviceTextPart(it) } return OnDeviceGenerateContentRequest( text = OnDeviceTextPart(text), image = image?.let { OnDeviceImagePart(it.image) }, @@ -222,7 +229,8 @@ internal class OnDeviceGenerativeModelProvider( topK = onDeviceConfig.topK, seed = onDeviceConfig.seed, candidateCount = onDeviceConfig.candidateCount, - maxOutputTokens = onDeviceConfig.maxOutputTokens + maxOutputTokens = onDeviceConfig.maxOutputTokens, + systemInstruction = systemInstructionPart ) } diff --git a/ai-logic/firebase-ai/src/test/java/com/google/firebase/ai/generativeModel/OnDeviceGenerativeModelProviderTests.kt b/ai-logic/firebase-ai/src/test/java/com/google/firebase/ai/generativeModel/OnDeviceGenerativeModelProviderTests.kt index 9ef22433578..dec63179753 100644 --- a/ai-logic/firebase-ai/src/test/java/com/google/firebase/ai/generativeModel/OnDeviceGenerativeModelProviderTests.kt +++ b/ai-logic/firebase-ai/src/test/java/com/google/firebase/ai/generativeModel/OnDeviceGenerativeModelProviderTests.kt @@ -24,6 +24,7 @@ import com.google.firebase.ai.ondevice.interop.Candidate as OnDeviceCandidate import com.google.firebase.ai.ondevice.interop.CountTokensResponse as OnDeviceCountTokensResponse import com.google.firebase.ai.ondevice.interop.FinishReason as OnDeviceFinishReason import com.google.firebase.ai.ondevice.interop.FirebaseAIOnDeviceNotAvailableException +import com.google.firebase.ai.ondevice.interop.GenerateContentRequest as OnDeviceGenerateContentRequest import com.google.firebase.ai.ondevice.interop.GenerateContentResponse as OnDeviceGenerateContentResponse import com.google.firebase.ai.ondevice.interop.GenerativeModel as OnDeviceGenerativeModel import com.google.firebase.ai.toInterop @@ -37,6 +38,7 @@ import io.kotest.matchers.shouldBe import io.mockk.coEvery import io.mockk.every import io.mockk.mockk +import io.mockk.slot import kotlinx.coroutines.flow.first import kotlinx.coroutines.flow.flowOf import kotlinx.coroutines.runBlocking @@ -153,4 +155,29 @@ internal class OnDeviceGenerativeModelProviderTests { interopConfig.modelConfig?.preference shouldBe com.google.firebase.ai.ondevice.interop.ModelPreference.FAST } + + @Test + fun `generateContent passes systemInstruction in OnDeviceConfig to interop request`(): Unit = + runBlocking { + coEvery { onDeviceModel.isAvailable() } returns true + val systemInstruction = Content(parts = listOf(TextPart("system instruction rule"))) + val configWithSystemInstruction = + OnDeviceConfig( + mode = InferenceMode.ONLY_ON_DEVICE, + temperature = 0.5f, + systemInstruction = systemInstruction + ) + val providerWithSystemInstruction = + OnDeviceGenerativeModelProvider(onDeviceModel, configWithSystemInstruction) + + val requestSlot = slot() + coEvery { onDeviceModel.generateContent(capture(requestSlot)) } returns + OnDeviceGenerateContentResponse( + listOf(OnDeviceCandidate("generated text", OnDeviceFinishReason.STOP)) + ) + + providerWithSystemInstruction.generateContent(prompt) + + requestSlot.captured.systemInstruction?.text shouldBe "system instruction rule" + } } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 618f5ee8c6e..762d7b6a26e 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -28,7 +28,7 @@ firebaseAnnotations = "17.0.0" firebaseCommon = "22.0.1" firebaseComponents = "19.0.0" firebaseCrashlyticsGradle = "3.0.4" -genaiPrompt = "1.0.0-beta2" +genaiPrompt = "1.0.0-beta4" glide = "5.0.5" googleApiClient = "2.8.1" googleServices = "4.3.15"