diff --git a/AGENTS.md b/AGENTS.md index c4e4171..2e644be 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,6 +18,6 @@ MOST IMPORTANT: 9. If debug compilation fails in your environment, resolve the issue before reporting it as complete. -10. For code changes only, compile only the code and do not perform a full build. +10. For code changes only, compile only the code with ./gradlew :app:compileDebugKotlin and omit always the lint check. 11. This app is production software and not a toy. diff --git a/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt b/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt index 0daa352..ea774d7 100644 --- a/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt +++ b/app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt @@ -146,6 +146,7 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { private var pendingScreenshotDelayMillis: Long = 0L private var pendingDelayedScreenshotRunnable: Runnable? = null + private var sawNonTermuxCommandSinceLastScreenshot: Boolean = false // App name to package mapper private lateinit var appNamePackageMapper: AppNamePackageMapper @@ -417,13 +418,17 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { pressEnterKey() } } + }.also { + if (command !is Command.TakeScreenshot && command !is Command.TermuxCommand) { + sawNonTermuxCommandSinceLastScreenshot = true + } } } private fun executeTakeScreenshotCommand(): Boolean { val delayMillis = pendingScreenshotDelayMillis pendingScreenshotDelayMillis = 0L - + val onlyTermuxContext = !sawNonTermuxCommandSinceLastScreenshot fun buildScreenInfoPayload(rawScreenInfo: String?): String? { val termuxOutput = TermuxOutputPreferences.consumeOutput(applicationContext)?.trim().orEmpty() if (termuxOutput.isBlank()) { @@ -435,7 +440,7 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { val captureAndRequestScreenshot = { val currentModel = GenerativeAiViewModelFactory.getCurrentModel() - if (!currentModel.supportsScreenshot) { + if (!currentModel.supportsScreenshot || onlyTermuxContext) { Log.d(TAG, "Command.TakeScreenshot: Model has no screenshot support, capturing screen info only.") showToast("Capturing screen info...", false) val screenInfo = buildScreenInfoPayload(captureScreenInformation()) @@ -445,6 +450,7 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { applicationContext, screenInfo ) + sawNonTermuxCommandSinceLastScreenshot = false } else { Log.d(TAG, "Command.TakeScreenshot: Capturing screen info and sending request broadcast to MainActivity.") showToast("Preparing screenshot...", false) @@ -457,6 +463,7 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { } applicationContext.sendBroadcast(intent) Log.d(TAG, "Sent broadcast ACTION_REQUEST_MEDIAPROJECTION_SCREENSHOT to MainActivity with screenInfo.") + sawNonTermuxCommandSinceLastScreenshot = false } } @@ -594,7 +601,7 @@ class ScreenOperatorAccessibilityService : AccessibilityService() { putExtra("com.termux.RUN_COMMAND_PATH", "/data/data/com.termux/files/usr/bin/bash") putExtra("com.termux.RUN_COMMAND_ARGUMENTS", arrayOf("-lc", trimmedCommand)) putExtra("com.termux.RUN_COMMAND_WORKDIR", "/data/data/com.termux/files/home") - putExtra("com.termux.RUN_COMMAND_BACKGROUND", true) + putExtra("com.termux.RUN_COMMAND_BACKGROUND", false) putExtra("com.termux.RUN_COMMAND_SESSION_ACTION", 0) putExtra("com.termux.RUN_COMMAND_RUNNER", "app-shell") putExtra("com.termux.RUN_COMMAND_PENDING_INTENT", pendingResultIntent) diff --git a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScreen.kt b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScreen.kt index abb9985..ebf09ff 100644 --- a/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScreen.kt +++ b/app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScreen.kt @@ -2,10 +2,11 @@ package com.google.ai.sample.feature.multimodal import android.app.Activity import android.content.Intent -import android.graphics.drawable.BitmapDrawable import android.net.Uri +import android.graphics.drawable.BitmapDrawable import android.provider.Settings import android.widget.Toast +import androidx.core.content.ContextCompat import androidx.activity.compose.BackHandler import androidx.activity.compose.rememberLauncherForActivityResult import androidx.activity.result.PickVisualMediaRequest @@ -110,6 +111,8 @@ import com.google.ai.sample.ScreenOperatorAccessibilityService import com.google.ai.sample.util.Command import com.google.ai.sample.util.SystemMessageEntry import com.google.ai.sample.util.SystemMessageEntryPreferences +import com.google.ai.sample.util.TermuxFeedbackPreferences +import com.google.ai.sample.util.TermuxOutputPreferences import com.google.ai.sample.util.UriSaver import com.google.ai.sample.util.shareTextFile import kotlinx.coroutines.Dispatchers @@ -177,6 +180,18 @@ fun PhotoReasoningScreen( ) { uri -> uri?.let { imageUris.add(it) } } + val termuxPermissionLauncher = rememberLauncherForActivityResult( + ActivityResultContracts.RequestPermission() + ) { isGranted -> + if (isGranted) { + TermuxFeedbackPreferences.resetPermissionDenialCount(context) + if (userQuestion.isNotBlank()) { + onReasonClicked(userQuestion, imageUris.toList()) + onUserQuestionChanged("") + imageUris.clear() + } + } + } LaunchedEffect(messages.size, commandExecutionStatus, detectedCommands.size) { val chatMessageCount = messages.size @@ -386,7 +401,11 @@ fun PhotoReasoningScreen( IconButton(onClick = { pickMedia.launch(PickVisualMediaRequest(ActivityResultContracts.PickVisualMedia.ImageAndVideo)) }, modifier = Modifier.padding(bottom = 4.dp)) { Icon(Icons.Rounded.Add, stringResource(R.string.add_image)) } - IconButton(onClick = onClearChatHistory, modifier = Modifier.padding(top = 4.dp).drawBehind { + IconButton(onClick = { + ScreenOperatorAccessibilityService.clearCommandQueue() + TermuxOutputPreferences.consumeOutput(context) + onClearChatHistory() + }, modifier = Modifier.padding(top = 4.dp).drawBehind { drawCircle(color = Color.Black, radius = size.minDimension / 2, style = androidx.compose.ui.graphics.drawscope.Stroke(width = 1.dp.toPx())) }) { Text("New", style = MaterialTheme.typography.labelMedium, color = MaterialTheme.colorScheme.primary) } } @@ -425,6 +444,29 @@ fun PhotoReasoningScreen( } if (userQuestion.isNotBlank()) { + val hasTermuxRunCommandPermission = ContextCompat.checkSelfPermission( + context, + "com.termux.permission.RUN_COMMAND" + ) == android.content.pm.PackageManager.PERMISSION_GRANTED + if (!hasTermuxRunCommandPermission) { + val denialCount = TermuxFeedbackPreferences.incrementPermissionDenialCount(context) + if (denialCount >= 3) { + Toast.makeText( + context, + "Enable Termux permissions in the Android settings", + Toast.LENGTH_LONG + ).show() + val appInfoIntent = Intent( + Settings.ACTION_APPLICATION_DETAILS_SETTINGS, + Uri.fromParts("package", context.packageName, null) + ) + context.startActivity(appInfoIntent) + } else { + termuxPermissionLauncher.launch("com.termux.permission.RUN_COMMAND") + } + return@IconButton + } + TermuxFeedbackPreferences.resetPermissionDenialCount(context) onReasonClicked(userQuestion, imageUris.toList()) onUserQuestionChanged("") imageUris.clear() diff --git a/app/src/main/kotlin/com/google/ai/sample/util/TermuxFeedbackPreferences.kt b/app/src/main/kotlin/com/google/ai/sample/util/TermuxFeedbackPreferences.kt index c9b1274..6b1fb1e 100644 --- a/app/src/main/kotlin/com/google/ai/sample/util/TermuxFeedbackPreferences.kt +++ b/app/src/main/kotlin/com/google/ai/sample/util/TermuxFeedbackPreferences.kt @@ -5,6 +5,7 @@ import android.content.Context object TermuxFeedbackPreferences { private const val PREF_NAME = "termux_feedback_prefs" private const val KEY_TERMUX_NOT_FOUND = "termux_not_found" + private const val KEY_TERMUX_PERMISSION_DENIAL_COUNT = "termux_permission_denial_count" fun markTermuxNotFound(context: Context) { context.getSharedPreferences(PREF_NAME, Context.MODE_PRIVATE) @@ -21,4 +22,18 @@ object TermuxFeedbackPreferences { } return value } + + fun incrementPermissionDenialCount(context: Context): Int { + val prefs = context.getSharedPreferences(PREF_NAME, Context.MODE_PRIVATE) + val updated = (prefs.getInt(KEY_TERMUX_PERMISSION_DENIAL_COUNT, 0) + 1).coerceAtMost(3) + prefs.edit().putInt(KEY_TERMUX_PERMISSION_DENIAL_COUNT, updated).apply() + return updated + } + + fun resetPermissionDenialCount(context: Context) { + context.getSharedPreferences(PREF_NAME, Context.MODE_PRIVATE) + .edit() + .putInt(KEY_TERMUX_PERMISSION_DENIAL_COUNT, 0) + .apply() + } }