diff --git a/agent/app/src/main/AndroidManifest.xml b/agent/app/src/main/AndroidManifest.xml index 699a2e4..7e32717 100644 --- a/agent/app/src/main/AndroidManifest.xml +++ b/agent/app/src/main/AndroidManifest.xml @@ -69,21 +69,6 @@ android:resource="@xml/file_paths" /> - - - - - - - , + ) { + /** + * Geocode a physical address string into its latitude and longitude coordinates. + * + * @param address The physical address to geocode (e.g., "1600 Amphitheatre Pkwy, Mountain View, + * CA"). + * @return The latitude and longitude coordinates of the address, or null if geocoding fails. + */ + suspend fun geocodeAddress(address: String): LatLng? { + if (!Geocoder.isPresent()) { + return null + } + + val geocoder = Geocoder(context) + + return withContext(Dispatchers.IO) { + try { + suspendCoroutine { continuation -> + geocoder.getFromLocationName( + address, + 1, + object : Geocoder.GeocodeListener { + override fun onGeocode(addresses: MutableList
) { + val location = addresses.firstOrNull() + if (location != null) { + continuation.resume( + LatLng(location.latitude, location.longitude), + ) + } else { + continuation.resume(null) + } + } + + override fun onError(errorMessage: String?) { + continuation.resume(null) + } + }, + ) + } + } catch (e: Exception) { + throw IllegalStateException(e.message, e) + } + } + } + + /** + * Retrieve the current latitude and longitude coordinates of the device. + * + * @return The current location coordinates of the device, or null if location is unavailable or + * permission is denied. + */ + @SuppressLint("MissingPermission") + suspend fun getCurrentLocation(): LatLng? = + withContext(Dispatchers.Default) { + // Check permissions + val hasFineLocation = + ContextCompat.checkSelfPermission( + context, + Manifest.permission.ACCESS_FINE_LOCATION, + ) == PackageManager.PERMISSION_GRANTED + + val hasCoarseLocation = + ContextCompat.checkSelfPermission( + context, + Manifest.permission.ACCESS_COARSE_LOCATION, + ) == PackageManager.PERMISSION_GRANTED + + if (!hasFineLocation && !hasCoarseLocation) { + throw IllegalStateException("Location permission is not granted") + } + + val locationManager = + context.getSystemService(Context.LOCATION_SERVICE) as LocationManager + + try { + // Try GPS Provider first + var location = + if (locationManager.isProviderEnabled(LocationManager.GPS_PROVIDER)) { + locationManager.getLastKnownLocation(LocationManager.GPS_PROVIDER) + } else { + null + } + + // Fallback to Network Provider if GPS is not available + if (location == null && + locationManager.isProviderEnabled(LocationManager.NETWORK_PROVIDER) + ) { + location = + locationManager.getLastKnownLocation(LocationManager.NETWORK_PROVIDER) + } + + if (location != null) { + LatLng(location.latitude, location.longitude) + } else { + null + } + } catch (e: Exception) { + throw IllegalStateException(e.message, e) + } + } + + /** + * Generates an image from a text prompt and returns the remote image URI. + * + * @param prompt The text prompt describing the image to generate (e.g., "futuristic cityscape at sunset"). + * @param aspectRatio Optional aspect ratio for the image (e.g., "16:9", "1:1"). + * @return A GeneratedImageResult containing the generated remote image URI. + */ + suspend fun generateImage( + prompt: String, + aspectRatio: String? = null, + ): GeneratedImageResult = + withContext(Dispatchers.IO) { + val apiKey = getOrFetchApiKey() + val requestPayload = buildImageGenerationPayload(prompt, aspectRatio) + val responseText = executeImageRequest(apiKey, requestPayload) + saveBase64ImageToCache(responseText, prompt) + } + + private suspend fun getOrFetchApiKey(): String { + val apiKey = + settingsDataStore.data + .first()[stringPreferencesKey("gemini_api_key")] + ?.takeIf { it.isNotBlank() } + ?: BuildConfig.GEMINI_API_KEY.takeIf { it.isNotBlank() } + if (apiKey.isNullOrBlank()) { + throw IllegalStateException( + "Gemini API key is not configured. Please set gemini_api_key in settings.", + ) + } + return apiKey + } + + private fun buildImageGenerationPayload( + prompt: String, + aspectRatio: String?, + ): JSONObject = + JSONObject().apply { + put( + "contents", + JSONArray().apply { + put( + JSONObject().apply { + put( + "parts", + JSONArray().apply { + put(JSONObject().apply { put("text", prompt) }) + }, + ) + }, + ) + }, + ) + put( + "generationConfig", + JSONObject().apply { + put("responseModalities", JSONArray().apply { put("IMAGE") }) + if (!aspectRatio.isNullOrBlank()) { + put( + "imageConfig", + JSONObject().apply { + put("aspectRatio", aspectRatio) + }, + ) + } + }, + ) + } + + private fun executeImageRequest( + apiKey: String, + requestJson: JSONObject, + ): String { + val endpointUrl = + "https://generativelanguage.googleapis.com/v1beta/models/gemini-3.1-flash-image-preview:generateContent?key=$apiKey" + val url = URL(endpointUrl) + val connection = + (url.openConnection() as HttpURLConnection).apply { + requestMethod = "POST" + setRequestProperty("Content-Type", "application/json") + doOutput = true + connectTimeout = 30000 + readTimeout = 60000 + } + + try { + connection.outputStream.use { os -> + os.write(requestJson.toString().toByteArray(Charsets.UTF_8)) + } + + val responseCode = connection.responseCode + if (responseCode != HttpURLConnection.HTTP_OK) { + val errorBody = + connection.errorStream?.bufferedReader()?.use { it.readText() } + ?: "HTTP $responseCode" + throw IllegalStateException( + "Image generation failed ($responseCode): $errorBody", + ) + } + + return connection.inputStream.bufferedReader().use { it.readText() } + } finally { + connection.disconnect() + } + } + + private fun saveBase64ImageToCache( + responseText: String, + prompt: String, + ): GeneratedImageResult { + val responseJson = JSONObject(responseText) + val candidates = responseJson.optJSONArray("candidates") + if (candidates == null || candidates.length() == 0) { + throw IllegalStateException( + "No candidates returned from Gemini image generation API", + ) + } + + val parts = + candidates + .getJSONObject(0) + .optJSONObject("content") + ?.optJSONArray("parts") + if (parts == null || parts.length() == 0) { + throw IllegalStateException( + "No parts returned in candidate content. Gemini response: $responseText", + ) + } + + val candidateData = + (0 until parts.length()) + .asSequence() + .mapNotNull { i -> + val part = parts.getJSONObject(i) + val inlineData = + part.optJSONObject("inlineData") + ?: part.optJSONObject("inline_data") + if (inlineData != null) { + val base64 = inlineData.optString("data") + val returnedMime = + inlineData + .optString("mimeType") + .takeIf { it.isNotBlank() } + ?: inlineData.optString("mime_type") + .takeIf { it.isNotBlank() } + ?: "image/png" + base64 to returnedMime + } else { + null + } + } + .firstOrNull() + + if (candidateData == null || candidateData.first.isBlank()) { + throw IllegalStateException( + "No inlineData image found in response parts. Gemini response: $responseText", + ) + } + + val (base64Data, mimeType) = candidateData + val imageBytes = Base64.decode(base64Data, Base64.DEFAULT) + val extension = + when { + mimeType.contains("jpeg") || mimeType.contains("jpg") -> "jpg" + else -> "png" + } + val cachedFile = + File( + context.cacheDir, + "generated_${UUID.randomUUID()}.$extension", + ) + cachedFile.writeBytes(imageBytes) + + val authority = "${context.packageName}.fileprovider" + val contentUri = FileProvider.getUriForFile(context, authority, cachedFile) + return GeneratedImageResult( + imageUri = contentUri.toString(), + mimeType = mimeType, + prompt = prompt, + ) + } + + /** + * Generates a list of AppFunctionMetadata representing these internal tools so the LLM provider + * can convert them to standard schemas. + */ + fun getInternalToolsMetadata(): List { + val latLngType = + AppFunctionObjectTypeMetadata( + properties = + mapOf( + "latitude" to AppFunctionDoubleTypeMetadata(isNullable = false), + "longitude" to AppFunctionDoubleTypeMetadata(isNullable = false), + ), + required = listOf("latitude", "longitude"), + isNullable = true, + qualifiedName = "com.example.appfunctions.agent.data.AgentInternalTools.LatLng", + ) + + val imageResultType = + AppFunctionObjectTypeMetadata( + properties = + mapOf( + "imageUri" to AppFunctionStringTypeMetadata(isNullable = false), + "mimeType" to AppFunctionStringTypeMetadata(isNullable = false), + "prompt" to AppFunctionStringTypeMetadata(isNullable = false), + ), + required = listOf("imageUri", "mimeType", "prompt"), + isNullable = false, + qualifiedName = "com.example.appfunctions.agent.data.AgentInternalTools.GeneratedImageResult", + ) + + val getCurrentLocationTool = + AppFunctionMetadata( + id = "getCurrentLocation", + packageName = INTERNAL_TOOL_PACKAGE, + isEnabled = true, + schema = null, + parameters = emptyList(), + response = + AppFunctionResponseMetadata( + valueType = latLngType, + description = "The current location coordinates of the device, or null.", + ), + components = AppFunctionComponentsMetadata(emptyMap()), + description = "Retrieve the current latitude and longitude coordinates of the device.", + deprecation = null, + ) + + val geocodeAddressTool = + AppFunctionMetadata( + id = "geocodeAddress", + packageName = INTERNAL_TOOL_PACKAGE, + isEnabled = true, + schema = null, + parameters = + listOf( + AppFunctionParameterMetadata( + name = "address", + isRequired = true, + dataType = AppFunctionStringTypeMetadata(isNullable = false), + description = "The physical address to geocode (e.g., '1600 Amphitheatre Pkwy, Mountain View, CA').", + ), + ), + response = + AppFunctionResponseMetadata( + valueType = latLngType, + description = "The latitude and longitude coordinates of the address, or null.", + ), + components = AppFunctionComponentsMetadata(emptyMap()), + description = "Geocode a physical address string into its latitude and longitude coordinates.", + deprecation = null, + ) + + val generateImageTool = + AppFunctionMetadata( + id = "generateImage", + packageName = INTERNAL_TOOL_PACKAGE, + isEnabled = true, + schema = null, + parameters = + listOf( + AppFunctionParameterMetadata( + name = "prompt", + isRequired = true, + dataType = AppFunctionStringTypeMetadata(isNullable = false), + description = "The text prompt describing the image to generate (e.g., 'futuristic cityscape at sunset').", + ), + AppFunctionParameterMetadata( + name = "aspectRatio", + isRequired = false, + dataType = AppFunctionStringTypeMetadata(isNullable = true), + description = "Optional aspect ratio for the image (e.g., '16:9', '1:1').", + ), + ), + response = + AppFunctionResponseMetadata( + valueType = imageResultType, + description = "A GeneratedImageResult containing the generated remote image URI.", + ), + components = AppFunctionComponentsMetadata(emptyMap()), + description = "Generates an image from a text prompt and returns the remote image URI.", + deprecation = null, + ) + + return listOf(getCurrentLocationTool, geocodeAddressTool, generateImageTool) + } + + /** Represents the latitude and longitude coordinates. */ + data class LatLng( + val latitude: Double, + val longitude: Double, + ) + + /** Represents the result of an image generation request. */ + data class GeneratedImageResult( + val imageUri: String, + val mimeType: String, + val prompt: String, + ) + + companion object { + const val INTERNAL_TOOL_PACKAGE = "com.example.appfunctions.agent.internal" + } + } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/BaseBuiltInAppFunctionService.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/BaseBuiltInAppFunctionService.kt deleted file mode 100644 index 2e1eb54..0000000 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/BaseBuiltInAppFunctionService.kt +++ /dev/null @@ -1,360 +0,0 @@ -/* - * Copyright 2026 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.example.appfunctions.agent.data - -import android.Manifest -import android.annotation.SuppressLint -import android.content.Context -import android.content.pm.PackageManager -import android.location.Address -import android.location.Geocoder -import android.location.LocationManager -import android.util.Base64 -import androidx.annotation.RequiresApi -import androidx.appfunctions.AppFunction -import androidx.appfunctions.AppFunctionSerializable -import androidx.appfunctions.AppFunctionService -import androidx.appfunctions.AppFunctionServiceEntryPoint -import androidx.core.content.ContextCompat -import androidx.core.content.FileProvider -import androidx.datastore.preferences.core.stringPreferencesKey -import com.example.appfunctions.agent.BuildConfig -import com.example.appfunctions.agent.di.settingsDataStore -import dagger.hilt.android.AndroidEntryPoint -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.flow.first -import kotlinx.coroutines.withContext -import org.json.JSONArray -import org.json.JSONObject -import java.io.File -import java.net.HttpURLConnection -import java.net.URL -import java.util.UUID -import kotlin.coroutines.resume -import kotlin.coroutines.suspendCoroutine - -/** Built-in AppFunctions for location and geocoding services. */ -@RequiresApi(36) -@AndroidEntryPoint -@AppFunctionServiceEntryPoint( - serviceName = "BuiltInAppFunctionService", - appFunctionXmlFileName = "builtin_app_function_service", -) -abstract class BaseBuiltInAppFunctionService : AppFunctionService() { - /** - * Geocode a physical address string into its latitude and longitude coordinates. - * - * @param address The physical address to geocode (e.g., "1600 Amphitheatre Pkwy, Mountain View, - * CA"). - * @return The latitude and longitude coordinates of the address, or null if geocoding fails. - */ - @AppFunction(isDescribedByKDoc = true) - suspend fun geocodeAddress(address: String): LatLng? { - if (!Geocoder.isPresent()) { - return null - } - - val geocoder = Geocoder(this) - - return withContext(Dispatchers.IO) { - try { - suspendCoroutine { continuation -> - geocoder.getFromLocationName( - address, - 1, - object : Geocoder.GeocodeListener { - override fun onGeocode(addresses: MutableList
) { - val location = addresses.firstOrNull() - if (location != null) { - continuation.resume( - LatLng(location.latitude, location.longitude), - ) - } else { - continuation.resume(null) - } - } - - override fun onError(errorMessage: String?) { - continuation.resume(null) - } - }, - ) - } - } catch (e: Exception) { - throw IllegalStateException(e.message, e) - } - } - } - - /** - * Retrieve the current latitude and longitude coordinates of the device. - * - * @return The current location coordinates of the device, or null if location is unavailable or - * permission is denied. - */ - @SuppressLint("MissingPermission") - @AppFunction(isDescribedByKDoc = true) - suspend fun getCurrentLocation(): LatLng? = - withContext(Dispatchers.Default) { - // Check permissions - val hasFineLocation = - ContextCompat.checkSelfPermission( - this@BaseBuiltInAppFunctionService, - Manifest.permission.ACCESS_FINE_LOCATION, - ) == PackageManager.PERMISSION_GRANTED - - val hasCoarseLocation = - ContextCompat.checkSelfPermission( - this@BaseBuiltInAppFunctionService, - Manifest.permission.ACCESS_COARSE_LOCATION, - ) == PackageManager.PERMISSION_GRANTED - - if (!hasFineLocation && !hasCoarseLocation) { - throw IllegalStateException("Location permission is not granted") - } - - val locationManager = - this@BaseBuiltInAppFunctionService.getSystemService(Context.LOCATION_SERVICE) as LocationManager - - try { - // Try GPS Provider first - var location = - if (locationManager.isProviderEnabled(LocationManager.GPS_PROVIDER)) { - locationManager.getLastKnownLocation(LocationManager.GPS_PROVIDER) - } else { - null - } - - // Fallback to Network Provider if GPS is not available - if (location == null && - locationManager.isProviderEnabled(LocationManager.NETWORK_PROVIDER) - ) { - location = - locationManager.getLastKnownLocation(LocationManager.NETWORK_PROVIDER) - } - - if (location != null) { - LatLng(location.latitude, location.longitude) - } else { - null - } - } catch (e: Exception) { - throw IllegalStateException(e.message, e) - } - } - - /** Represents the latitude and longitude coordinates. */ - @AppFunctionSerializable(isDescribedByKDoc = true) - data class LatLng( - /** The latitude coordinate. */ - val latitude: Double, - /** The longitude coordinate. */ - val longitude: Double, - ) - - /** - * Generates an image from a text prompt and returns the remote image URI. - * - * @param prompt The text prompt describing the image to generate (e.g., "futuristic cityscape at sunset"). - * @param aspectRatio Optional aspect ratio for the image (e.g., "16:9", "1:1"). - * @return A GeneratedImageResult containing the generated remote image URI. - */ - @AppFunction(isDescribedByKDoc = true) - suspend fun generateImage( - prompt: String, - aspectRatio: String? = null, - ): GeneratedImageResult = - withContext(Dispatchers.IO) { - val apiKey = getOrFetchApiKey() - val requestPayload = buildImageGenerationPayload(prompt, aspectRatio) - val responseText = executeImageRequest(apiKey, requestPayload) - saveBase64ImageToCache(responseText, prompt) - } - - private suspend fun getOrFetchApiKey(): String { - val apiKey = - settingsDataStore.data - .first()[stringPreferencesKey("gemini_api_key")] - ?.takeIf { it.isNotBlank() } - ?: BuildConfig.GEMINI_API_KEY.takeIf { it.isNotBlank() } - if (apiKey.isNullOrBlank()) { - throw IllegalStateException( - "Gemini API key is not configured. Please set gemini_api_key in settings.", - ) - } - return apiKey - } - - private fun buildImageGenerationPayload( - prompt: String, - aspectRatio: String?, - ): JSONObject = - JSONObject().apply { - put( - "contents", - JSONArray().apply { - put( - JSONObject().apply { - put( - "parts", - JSONArray().apply { - put(JSONObject().apply { put("text", prompt) }) - }, - ) - }, - ) - }, - ) - put( - "generationConfig", - JSONObject().apply { - put("responseModalities", JSONArray().apply { put("IMAGE") }) - if (!aspectRatio.isNullOrBlank()) { - put( - "imageConfig", - JSONObject().apply { - put("aspectRatio", aspectRatio) - }, - ) - } - }, - ) - } - - private fun executeImageRequest( - apiKey: String, - requestJson: JSONObject, - ): String { - val endpointUrl = - "https://generativelanguage.googleapis.com/v1beta/models/gemini-3.1-flash-image-preview:generateContent?key=$apiKey" - val url = URL(endpointUrl) - val connection = - (url.openConnection() as HttpURLConnection).apply { - requestMethod = "POST" - setRequestProperty("Content-Type", "application/json") - doOutput = true - connectTimeout = 30000 - readTimeout = 60000 - } - - try { - connection.outputStream.use { os -> - os.write(requestJson.toString().toByteArray(Charsets.UTF_8)) - } - - val responseCode = connection.responseCode - if (responseCode != HttpURLConnection.HTTP_OK) { - val errorBody = - connection.errorStream?.bufferedReader()?.use { it.readText() } - ?: "HTTP $responseCode" - throw IllegalStateException( - "Image generation failed ($responseCode): $errorBody", - ) - } - - return connection.inputStream.bufferedReader().use { it.readText() } - } finally { - connection.disconnect() - } - } - - private fun saveBase64ImageToCache( - responseText: String, - prompt: String, - ): GeneratedImageResult { - val responseJson = JSONObject(responseText) - val candidates = responseJson.optJSONArray("candidates") - if (candidates == null || candidates.length() == 0) { - throw IllegalStateException( - "No candidates returned from Gemini image generation API", - ) - } - - val parts = - candidates - .getJSONObject(0) - .optJSONObject("content") - ?.optJSONArray("parts") - if (parts == null || parts.length() == 0) { - throw IllegalStateException( - "No parts returned in candidate content. Gemini response: $responseText", - ) - } - - val candidateData = - (0 until parts.length()) - .asSequence() - .mapNotNull { i -> - val part = parts.getJSONObject(i) - val inlineData = - part.optJSONObject("inlineData") - ?: part.optJSONObject("inline_data") - if (inlineData != null) { - val base64 = inlineData.optString("data") - val returnedMime = - inlineData - .optString("mimeType") - .takeIf { it.isNotBlank() } - ?: inlineData.optString("mime_type") - .takeIf { it.isNotBlank() } - ?: "image/png" - base64 to returnedMime - } else { - null - } - } - .firstOrNull() - - if (candidateData == null || candidateData.first.isBlank()) { - throw IllegalStateException( - "No inlineData image found in response parts. Gemini response: $responseText", - ) - } - - val (base64Data, mimeType) = candidateData - val imageBytes = Base64.decode(base64Data, Base64.DEFAULT) - val extension = - when { - mimeType.contains("jpeg") || mimeType.contains("jpg") -> "jpg" - else -> "png" - } - val cachedFile = - File( - cacheDir, - "generated_${UUID.randomUUID()}.$extension", - ) - cachedFile.writeBytes(imageBytes) - - val authority = "$packageName.fileprovider" - val contentUri = FileProvider.getUriForFile(this, authority, cachedFile) - return GeneratedImageResult( - imageUri = contentUri.toString(), - mimeType = mimeType, - prompt = prompt, - ) - } - - /** Represents the result of an image generation request. */ - @AppFunctionSerializable - data class GeneratedImageResult( - /** The remote URI or URL of the generated image. */ - val imageUri: String, - /** The MIME type of the generated image. */ - val mimeType: String, - /** The original prompt used to generate the image. */ - val prompt: String, - ) -} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt index e551c4d..5e3b04d 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt @@ -27,6 +27,7 @@ import androidx.appfunctions.metadata.AppFunctionObjectTypeMetadata import androidx.appfunctions.metadata.AppFunctionParameterMetadata import androidx.appfunctions.metadata.AppFunctionReferenceTypeMetadata import androidx.core.content.FileProvider +import com.example.appfunctions.agent.data.AgentInternalTools import com.example.appfunctions.agent.data.LlmProviderName import com.example.appfunctions.agent.data.SettingsRepository import com.example.appfunctions.agent.data.db.entities.MessageAttachment @@ -84,6 +85,7 @@ class AgentOrchestrator private val convertInputToAppFunctionDataUseCase: ConvertInputToAppFunctionDataUseCase, private val executeAppFunctionUseCase: ExecuteAppFunctionUseCase, private val savePendingIntentUseCase: SavePendingIntentUseCase, + private val agentInternalTools: AgentInternalTools, ) { private val _status = MutableStateFlow(AgentStatus.Idle) @@ -129,7 +131,9 @@ class AgentOrchestrator } val disconnectedApps = settingsRepository.disconnectedApps.first() - val allTools = getAppFunctionsUseCase().first().values.flatten() + val allTools = + getAppFunctionsUseCase().first().values.flatten() + + agentInternalTools.getInternalToolsMetadata() val targetPackageName = message.targetPackageName val queryText = message.textContent @@ -363,6 +367,29 @@ class AgentOrchestrator for (toolCall in toolCalls) { _status.value = AgentStatus.InvokingTool(toolCall.functionId, toolCall.packageName) + if (toolCall.packageName == AgentInternalTools.INTERNAL_TOOL_PACKAGE) { + val executionResult = executeInternalTool(toolCall) + executionResult + .onSuccess { jsonString -> + results.add( + ToolOutput( + functionId = toolCall.functionId, + callId = toolCall.callId, + result = jsonString, + ), + ) + } + .onFailure { exception -> + completeMessageWithError( + message.messageId, + message.threadId, + "Internal tool execution failed: ${exception.message}", + ) + return ExecuteToolCallsResult.Error + } + continue + } + val matchingTool = tools.find { it.packageName == toolCall.packageName && it.id == toolCall.functionId @@ -648,6 +675,51 @@ class AgentOrchestrator return false } + private suspend fun executeInternalTool(toolCall: LlmResponsePart.ToolCall): Result { + return runCatching { + when (toolCall.functionId) { + "getCurrentLocation" -> { + val location = agentInternalTools.getCurrentLocation() + if (location != null) { + JSONObject().apply { + put("latitude", location.latitude) + put("longitude", location.longitude) + }.toString() + } else { + "null" + } + } + "geocodeAddress" -> { + val address = + toolCall.arguments["address"] as? String + ?: throw IllegalArgumentException("Missing required parameter 'address'") + val location = agentInternalTools.geocodeAddress(address) + if (location != null) { + JSONObject().apply { + put("latitude", location.latitude) + put("longitude", location.longitude) + }.toString() + } else { + "null" + } + } + "generateImage" -> { + val prompt = + toolCall.arguments["prompt"] as? String + ?: throw IllegalArgumentException("Missing required parameter 'prompt'") + val aspectRatio = toolCall.arguments["aspectRatio"] as? String + val result = agentInternalTools.generateImage(prompt, aspectRatio) + JSONObject().apply { + put("imageUri", result.imageUri) + put("mimeType", result.mimeType) + put("prompt", result.prompt) + }.toString() + } + else -> throw IllegalArgumentException("Unknown internal tool: ${toolCall.functionId}") + } + } + } + companion object { private val KNOWN_FILE_REFERENCE_PARAM_NAMES = setOf( diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCase.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCase.kt index 799331a..4daa7b4 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCase.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCase.kt @@ -106,7 +106,7 @@ class ConvertInputToAppFunctionDataUseCase .build() builder.setAppFunctionData(name, uriData) } else { - val objData = convertObject(dataType, value as Map, components) + val objData = toAppFunctionData(dataType, value, components) builder.setAppFunctionData(name, objData) } } @@ -126,7 +126,7 @@ class ConvertInputToAppFunctionDataUseCase .build() builder.setAppFunctionData(name, uriData) } else { - val objData = convertObject(objectType, value as Map, components) + val objData = toAppFunctionData(objectType, value, components) builder.setAppFunctionData(name, objData) } } @@ -134,15 +134,29 @@ class ConvertInputToAppFunctionDataUseCase } } - private fun convertObject( + private fun toAppFunctionData( objectType: AppFunctionObjectTypeMetadata, - values: Map, + value: Any, components: AppFunctionComponentsMetadata, ): AppFunctionData { + val mapValue = + if (value is String) { + val uriPropName = + objectType.properties.entries.find { isUriMetadata(it.value) }?.key + ?: objectType.properties.entries.find { it.value is AppFunctionStringTypeMetadata }?.key + ?: throw IllegalArgumentException( + "Cannot convert string value to object $objectType: " + + "no Uri or String property found", + ) + mapOf(uriPropName to value) + } else { + value as Map<*, *> + } + val builder = AppFunctionData.Builder(objectType, components) for ((propName, propType) in objectType.properties) { - val value = values[propName] ?: continue - setParameterValue(builder, propName, propType, value, components) + val propValue = mapValue[propName] ?: continue + setParameterValue(builder, propName, propType, propValue, components) } return builder.build() } @@ -180,7 +194,9 @@ class ConvertInputToAppFunctionDataUseCase } is AppFunctionObjectTypeMetadata -> { val dataList = - values.map { convertObject(itemType, it as Map, components) } + values.map { item -> + toAppFunctionData(itemType, item, components) + } builder.setAppFunctionDataList(name, dataList) } is AppFunctionReferenceTypeMetadata -> { @@ -189,10 +205,19 @@ class ConvertInputToAppFunctionDataUseCase components.dataTypes[referenceKey] as? AppFunctionObjectTypeMetadata if (objectType != null) { val dataList = - values.map { convertObject(objectType, it as Map, components) } + values.map { item -> + toAppFunctionData(objectType, item, components) + } builder.setAppFunctionDataList(name, dataList) } } } } + + private fun isUriMetadata(dataType: AppFunctionDataTypeMetadata?): Boolean { + if (dataType == null) return false + if (dataType is AppFunctionObjectTypeMetadata && dataType.qualifiedName == "android.net.Uri") return true + if (dataType is AppFunctionReferenceTypeMetadata && dataType.referenceDataType == "android.net.Uri") return true + return false + } } diff --git a/agent/app/src/main/res/xml/app_metadata.xml b/agent/app/src/main/res/xml/app_metadata.xml index 81cf42b..6e605d2 100644 --- a/agent/app/src/main/res/xml/app_metadata.xml +++ b/agent/app/src/main/res/xml/app_metadata.xml @@ -15,10 +15,5 @@ ~ limitations under the License. --> diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt index 179ffe6..93350b7 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt @@ -19,6 +19,7 @@ import android.content.Intent import androidx.appfunctions.AppFunctionData import androidx.appfunctions.metadata.AppFunctionMetadata import androidx.appfunctions.metadata.AppFunctionPackageMetadata +import com.example.appfunctions.agent.data.AgentInternalTools import com.example.appfunctions.agent.data.LlmModel import com.example.appfunctions.agent.data.LlmProviderName import com.example.appfunctions.agent.data.ServiceTier @@ -70,6 +71,7 @@ class AgentOrchestratorTest { private val convertInputToAppFunctionDataUseCase: ConvertInputToAppFunctionDataUseCase = mockk() private val context: android.content.Context = mockk(relaxed = true) private val savePendingIntentUseCase: SavePendingIntentUseCase = mockk(relaxed = true) + private val agentInternalTools: AgentInternalTools = mockk(relaxed = true) private lateinit var agentOrchestrator: AgentOrchestrator @@ -89,6 +91,7 @@ class AgentOrchestratorTest { convertInputToAppFunctionDataUseCase = convertInputToAppFunctionDataUseCase, executeAppFunctionUseCase = executeAppFunctionUseCase, savePendingIntentUseCase = savePendingIntentUseCase, + agentInternalTools = agentInternalTools, ) } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCaseTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCaseTest.kt index 478030b..8eefa45 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCaseTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCaseTest.kt @@ -266,4 +266,84 @@ class ConvertInputToAppFunctionDataUseCaseTest { val uriData = result.getAppFunctionData("wallpaperUri") assertEquals("content://com.example/file.jpg", uriData?.getString("uri")) } + + @Test + fun convert_customObjectTypeWithStringInput_buildsCustomObjectTypeWithEmbeddedUri() { + val uriObjectType = + AppFunctionObjectTypeMetadata( + properties = mapOf("uri" to AppFunctionStringTypeMetadata(false)), + required = listOf("uri"), + qualifiedName = "android.net.Uri", + isNullable = false, + ) + val customObjectType = + AppFunctionObjectTypeMetadata( + properties = + mapOf( + "uri" to uriObjectType, + "mimeType" to AppFunctionStringTypeMetadata(true), + ), + required = listOf("uri"), + qualifiedName = "com.example.chatapp.Attachment", + isNullable = false, + ) + val parameters = + listOf( + AppFunctionParameterMetadata( + name = "attachment", + isRequired = true, + dataType = customObjectType, + ), + ) + val inputs = mapOf("attachment" to "content://com.example/file.jpg") + + val result = useCase(parameters, components, inputs).getOrThrow() + + val attachmentData = result.getAppFunctionData("attachment") + val uriData = attachmentData?.getAppFunctionData("uri") + assertEquals("content://com.example/file.jpg", uriData?.getString("uri")) + } + + @Test + fun convert_customObjectTypeArrayWithStringInput_buildsCustomObjectTypeArrayWithEmbeddedUri() { + val uriObjectType = + AppFunctionObjectTypeMetadata( + properties = mapOf("uri" to AppFunctionStringTypeMetadata(false)), + required = listOf("uri"), + qualifiedName = "android.net.Uri", + isNullable = false, + ) + val customObjectType = + AppFunctionObjectTypeMetadata( + properties = + mapOf( + "uri" to uriObjectType, + "mimeType" to AppFunctionStringTypeMetadata(true), + ), + required = listOf("uri"), + qualifiedName = "com.example.chatapp.Attachment", + isNullable = false, + ) + val arrayType = + AppFunctionArrayTypeMetadata( + itemType = customObjectType, + isNullable = false, + ) + val parameters = + listOf( + AppFunctionParameterMetadata( + name = "attachments", + isRequired = true, + dataType = arrayType, + ), + ) + val inputs = mapOf("attachments" to listOf("content://com.example/file.jpg")) + + val result = useCase(parameters, components, inputs).getOrThrow() + + val dataList = result.getAppFunctionDataList("attachments") + assertEquals(1, dataList?.size) + val uriData = dataList?.first()?.getAppFunctionData("uri") + assertEquals("content://com.example/file.jpg", uriData?.getString("uri")) + } } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/ConnectedAppsViewModelTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/ConnectedAppsViewModelTest.kt index 16044b0..57fa44a 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/ConnectedAppsViewModelTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/ConnectedAppsViewModelTest.kt @@ -15,6 +15,7 @@ */ package com.example.appfunctions.agent.ui.screens.agentdemo +import androidx.lifecycle.ViewModel import com.example.appfunctions.agent.domain.appfunction.ConnectedAppInfo import com.example.appfunctions.agent.domain.appfunction.GetConnectedAppsUseCase import io.mockk.every @@ -50,6 +51,9 @@ class ConnectedAppsViewModelTest { @After fun tearDown() { + if (::viewModel.isInitialized) { + viewModel.clearForTest() + } Dispatchers.resetMain() } @@ -83,4 +87,10 @@ class ConnectedAppsViewModelTest { val disconnectedApps = fakeRepository.disconnectedApps.first() assertEquals(setOf("test.package"), disconnectedApps) } + + private fun ViewModel.clearForTest() { + val method = ViewModel::class.java.getDeclaredMethod("onCleared") + method.isAccessible = true + method.invoke(this) + } }