diff --git a/android/samples/mobile-2/README.md b/android/samples/mobile-2/README.md index 0b845dfa6e..fcbec853cc 100644 --- a/android/samples/mobile-2/README.md +++ b/android/samples/mobile-2/README.md @@ -11,7 +11,6 @@ An Android Jetpack Compose chat client that connects to a TypeAgent agent-server - `registerClientAgent` with an inline action schema - Client-hosted `executeAction` callbacks - Inbound `appendDisplay`, `setDisplay`, `setDisplayInfo`, and command completion events - - Inbound `takeAction` client actions - Incremental assistant response streaming into a single bubble per `requestId`, honouring the SDK's `DisplayAppendMode` (`inline`, `block`, `temporary`, `step`) and `DisplayMessageKind` styling the same way the Electron shell does @@ -21,25 +20,30 @@ An Android Jetpack Compose chat client that connects to a TypeAgent agent-server ## Client-hosted Android agent After joining a conversation, the app registers `androidDevice` as a -client-hosted agent. Its alarm and timer schema is packaged in the APK and sent -inline to TypeAgent. TypeAgent translates or directly invokes the typed action, -then calls `executeAction` on the app over the existing WebSocket connection. +client-hosted agent. Its action schema is packaged in the APK and sent inline to +TypeAgent. TypeAgent translates or directly invokes the typed action, then calls +`executeAction` on the app over the existing WebSocket connection, and the app +reports success or failure back as the action result. -The app also retains its existing `takeAction` handlers for compatibility with -the static `androidMobile` agent. +This is the only path for device actions. The legacy fire-and-forget +`takeAction` path served by the server-side `androidMobile` agent has been +removed; the `clientio:` channel is still used, but only for display and user +interaction traffic. -| Client action | Android intent | Notes | +| Schema action | Android intent | Notes | |---|---|---| -| `set-alarm` | `AlarmClock.ACTION_SET_ALARM` | Opens the clock app so the user can confirm the alarm. | -| `set-timer` | `AlarmClock.ACTION_SET_TIMER` | Starts the countdown in the background (`EXTRA_SKIP_UI = true`) and confirms with a toast, so a chat request never yanks the user out of the conversation. Durations outside the documented 1..86400 second range are rejected rather than clamped. | +| `setAlarm` | `AlarmClock.ACTION_SET_ALARM` | Opens the clock app so the user can confirm the alarm. | +| `setTimer` | `AlarmClock.ACTION_SET_TIMER` | Starts the countdown in the background (`EXTRA_SKIP_UI = true`) and confirms with a toast, so a chat request never yanks the user out of the conversation. Durations outside the documented 1..86400 second range are rejected rather than clamped. | +| `searchNearby` | `Intent.ACTION_VIEW` with a `geo:0,0?q=` URI | Opens the device's maps app on a local search. The intent is implicit rather than pinned to Google Maps, so it resolves on any device with a maps app. | -Both actions require the app to be in the foreground: Android 10+ silently refuses +All actions require the app to be in the foreground: Android 10+ silently refuses background activity starts (no exception is thrown), so the app checks its own lifecycle state first and reports a failure rather than a false confirmation. -Both require the `com.android.alarm.permission.SET_ALARM` permission (declared in -the manifest, install-time only) and matching `` entries so -`resolveActivity` works under Android 11+ package visibility rules. +The clock actions require the `com.android.alarm.permission.SET_ALARM` permission +(declared in the manifest, install-time only). Every action needs a matching +`` entry so `resolveActivity` works under Android 11+ package +visibility rules. The registered client agent does not require installing the server-side `androidMobile` package. Use `@action` for a deterministic registration test: diff --git a/android/samples/mobile-2/app/src/main/assets/typeagent/androidDeviceSchema.ts b/android/samples/mobile-2/app/src/main/assets/typeagent/androidDeviceSchema.ts index 8e7965550e..f88ccac76b 100644 --- a/android/samples/mobile-2/app/src/main/assets/typeagent/androidDeviceSchema.ts +++ b/android/samples/mobile-2/app/src/main/assets/typeagent/androidDeviceSchema.ts @@ -1,8 +1,13 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -export type AndroidDeviceAction = SetAlarmAction | SetTimerAction; +export type AndroidDeviceAction = + | SetAlarmAction + | SetTimerAction + | SearchNearbyAction; +// Sets an alarm on the Android device. +// Use when the user asks to create or schedule an alarm for a time of day. export type SetAlarmAction = { actionName: "setAlarm"; parameters: { @@ -13,6 +18,8 @@ export type SetAlarmAction = { }; }; +// Starts a countdown timer on the Android device. +// Use when the user asks for a timer or countdown lasting a specified duration. export type SetTimerAction = { actionName: "setTimer"; parameters: { @@ -22,3 +29,15 @@ export type SetTimerAction = { durationInSeconds: number; }; }; + +// Opens the Android device's maps app on a search for places near the user. +// Use when the user asks to find, locate or show places nearby. +export type SearchNearbyAction = { + actionName: "searchNearby"; + parameters: { + // The original user request. + originalRequest: string; + // The kind of place to look for, for example "coffee shops". + searchTerm: string; + }; +}; diff --git a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/AndroidDeviceAgent.kt b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/AndroidDeviceAgent.kt index 741438c911..35e01c46ab 100644 --- a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/AndroidDeviceAgent.kt +++ b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/AndroidDeviceAgent.kt @@ -7,6 +7,8 @@ internal object AndroidDeviceAgent { const val NAME = "androidDevice" const val CHANNEL_NAME = "agent:$NAME" const val SCHEMA_ASSET = "typeagent/androidDeviceSchema.ts" + private const val AGENT_DESCRIPTION = + "Sets alarms and countdown timers, and searches for nearby places, on this Android device." fun createRegistrationParams( conversationId: String, @@ -16,12 +18,12 @@ internal object AndroidDeviceAgent { .put("format", "ts") .put("content", schemaContent) val schema = JSONObject() - .put("description", "Sets alarms and countdown timers on this Android device.") + .put("description", AGENT_DESCRIPTION) .put("schemaType", "AndroidDeviceAction") .put("schemaFile", schemaFile) val manifest = JSONObject() .put("emojiChar", "\u23F0") - .put("description", "Sets alarms and countdown timers on this Android device.") + .put("description", AGENT_DESCRIPTION) .put("defaultEnabled", true) .put("schemaDefaultEnabled", true) .put("actionDefaultEnabled", true) @@ -71,6 +73,14 @@ internal object AndroidDeviceAgent { AndroidDeviceActionParseResult.Success(AndroidDeviceAction.Timer(parsed)) } + "searchNearby" -> { + val parsed = parseSearchNearbyActionPayload(parameters) + ?: return AndroidDeviceActionParseResult.ActionError( + "Invalid searchNearby parameters." + ) + AndroidDeviceActionParseResult.Success(AndroidDeviceAction.SearchNearby(parsed)) + } + else -> AndroidDeviceActionParseResult.ActionError( "Unsupported Android agent action: $actionName" ) @@ -92,6 +102,7 @@ internal object AndroidDeviceAgent { internal sealed interface AndroidDeviceAction { data class Alarm(val action: SetAlarmAction) : AndroidDeviceAction data class Timer(val action: SetTimerAction) : AndroidDeviceAction + data class SearchNearby(val action: SearchNearbyAction) : AndroidDeviceAction } internal sealed interface AndroidDeviceActionParseResult { diff --git a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/MainActivity.kt b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/MainActivity.kt index b81b2dcb37..2645d65e0a 100644 --- a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/MainActivity.kt +++ b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/MainActivity.kt @@ -99,8 +99,11 @@ class MainActivity : ComponentActivity() { runOnUiThread { launchSetTimerIntent(action, completion) } } - override fun onSearchNearby(action: SearchNearbyAction) { - runOnUiThread { launchSearchNearbyIntent(action) } + override fun onSearchNearby( + action: SearchNearbyAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) { + runOnUiThread { launchSearchNearbyIntent(action, completion) } } }) webSocketManager.connect( @@ -154,7 +157,8 @@ class MainActivity : ComponentActivity() { } /** - * Handles `takeAction("set-timer", ...)` from the androidMobile agent. + * Handles the `setTimer` action of the registered `androidDevice` client + * agent. * * `EXTRA_SKIP_UI` is true so the clock app starts the countdown in the * background instead of coming to the foreground. The reference @@ -189,12 +193,22 @@ class MainActivity : ComponentActivity() { } /** - * Opens the device maps app on a local search. The intent is left implicit - * rather than pinned to `com.google.android.apps.maps` as TypeAgent's + * Handles the `searchNearby` action of the registered `androidDevice` + * client agent by opening the device maps app on a local search. + * + * Unlike the clock intents there is no `EXTRA_SKIP_UI` equivalent, so maps + * necessarily comes to the foreground; that makes the RESUMED guard in + * [launchExternalIntent] load-bearing. + * + * The intent is left implicit rather than pinned to + * `com.google.android.apps.maps` as TypeAgent's * `JavaScriptInterface.searchNearby` does, so it still resolves on devices * without Google Maps. */ - private fun launchSearchNearbyIntent(action: SearchNearbyAction) { + private fun launchSearchNearbyIntent( + action: SearchNearbyAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) { val intent = Intent(Intent.ACTION_VIEW, Uri.parse(buildGeoSearchUri(action.searchTerm))) launchExternalIntent( intent = intent, @@ -203,7 +217,8 @@ class MainActivity : ComponentActivity() { successMessage = "Searching nearby for ${action.searchTerm}", missingAppMessage = "No maps app is available on this device.", deniedMessage = "This app is not allowed to open the maps app.", - backgroundMessage = "Could not open maps while the app was in the background." + backgroundMessage = "Could not open maps while the app was in the background.", + completion = completion ) } @@ -233,7 +248,7 @@ class MainActivity : ComponentActivity() { missingAppMessage: String, deniedMessage: String, backgroundMessage: String, - completion: (AndroidDeviceExecutionResult) -> Unit = {} + completion: (AndroidDeviceExecutionResult) -> Unit ) { val target = intent.resolveActivity(packageManager) Log.d( diff --git a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/SearchNearbyActionParser.kt b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/SearchNearbyActionParser.kt index 55b8e15522..ad8325383c 100644 --- a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/SearchNearbyActionParser.kt +++ b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/SearchNearbyActionParser.kt @@ -19,9 +19,10 @@ private val controlCharRegex = Regex("""\p{Cntrl}""") private val whitespaceRunRegex = Regex("""\s+""") /** - * Parses `takeAction("search-nearby", ...)` from the androidMobile agent: - * `{ originalRequest: string; searchTerm: string }`. Re-validated here because - * `takeAction` is fire-and-forget and carries no schema guarantee over the wire. + * Parses the `parameters` of the `searchNearby` action declared by + * `androidDeviceSchema.ts`: `{ originalRequest: string; searchTerm: string }`. + * Re-validated here because the values are shaped by an LLM and reach + * `startActivity` unmodified. */ internal fun parseSearchNearbyActionPayload(data: Any?): SearchNearbyAction? { val payload = data as? JSONObject ?: return null diff --git a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/TimerActionParser.kt b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/TimerActionParser.kt index b4b3cedffb..20814fb8fd 100644 --- a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/TimerActionParser.kt +++ b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/TimerActionParser.kt @@ -26,16 +26,16 @@ private const val MAX_TIMER_SECONDS = 86_400L private const val MAX_ORIGINAL_REQUEST_CHARS = 256 /** - * Parses the payload of `takeAction("set-timer", ...)` emitted by the - * androidMobile agent's `SetTimerAction` (TypeAgent PR #2780): + * Parses the `parameters` of the `setTimer` action declared by + * `androidDeviceSchema.ts`: * * ```ts * parameters: { originalRequest: string; durationInSeconds: number } * ``` * - * The agent already floors the value and rejects non-positive durations, but - * this client re-validates because `takeAction` is fire-and-forget and carries - * no schema guarantee over the wire. + * The server-side dispatcher already validates against the schema, but this + * client re-validates because the value is shaped by an LLM and reaches + * `startActivity` unmodified. */ internal fun parseSetTimerActionPayload(data: Any?): SetTimerAction? { val payload = data as? JSONObject ?: return null diff --git a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/WebSocketManager.kt b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/WebSocketManager.kt index 9c688f3c2a..8c3e03c051 100644 --- a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/WebSocketManager.kt +++ b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/WebSocketManager.kt @@ -440,9 +440,6 @@ class WebSocketManager { TAG, "RPC call channel=$channelName method=$methodName argCount=${args.length()}" ) - if (methodName == "takeAction") { - Log.d(TAG, "RPC call raw takeAction args=$args") - } when { channelName.startsWith(CLIENT_IO_CHANNEL_PREFIX) -> handleClientIoCall(methodName, args) else -> Log.d(TAG, "Unhandled RPC call channel=$channelName method=$methodName") @@ -564,6 +561,8 @@ class WebSocketManager { when (action) { is AndroidDeviceAction.Alarm -> handler.onSetAlarm(action.action, completion) is AndroidDeviceAction.Timer -> handler.onSetTimer(action.action, completion) + is AndroidDeviceAction.SearchNearby -> + handler.onSearchNearby(action.action, completion) } } @@ -691,10 +690,6 @@ class WebSocketManager { } } - "takeAction" -> { - handleTakeActionCall(args) - } - else -> { val requestId = extractRequestId(args.opt(0)) logInboundEvent( @@ -706,101 +701,6 @@ class WebSocketManager { } } - private fun handleTakeActionCall(args: JSONArray) { - val requestId = extractRequestId(args.opt(0)) - val actionName = args.optString(1).orEmpty() - val actionData = args.optNullable(2) - logInboundEvent( - type = "take-action:$actionName", - requestId = requestId, - content = stringifyDisplayValue(actionData) - ) - Log.d( - TAG, - "takeAction received action=$actionName requestId=${requestId.orEmpty()} data=${stringifyDisplayValue(actionData)}" - ) - when (actionName) { - "set-alarm" -> handleSetAlarmAction(actionData) - "set-timer" -> handleSetTimerAction(actionData) - "search-nearby" -> handleSearchNearbyAction(actionData) - else -> Log.d(TAG, "takeAction ignored: unsupported action=$actionName") - } - } - - private fun handleSetAlarmAction(actionData: Any?) { - val alarm = parseSetAlarmActionPayload(actionData) - if (alarm == null) { - Log.e( - TAG, - "Invalid set-alarm payload: ${stringifyDisplayValue(actionData)}" - ) - return - } - val handler = requireClientActionHandler( - "set-alarm", - "hour=${alarm.hour} minute=${alarm.minute}" - ) ?: return - Log.d( - TAG, - "Dispatching set-alarm to client handler hour=${alarm.hour} minute=${alarm.minute}" - ) - handler.onSetAlarm(alarm) {} - } - - private fun handleSetTimerAction(actionData: Any?) { - val timer = parseSetTimerActionPayload(actionData) - if (timer == null) { - Log.e( - TAG, - "Invalid set-timer payload: ${stringifyDisplayValue(actionData)}" - ) - return - } - val handler = requireClientActionHandler( - "set-timer", - "durationInSeconds=${timer.durationInSeconds}" - ) ?: return - Log.d( - TAG, - "Dispatching set-timer to client handler durationInSeconds=${timer.durationInSeconds}" - ) - handler.onSetTimer(timer) {} - } - - private fun handleSearchNearbyAction(actionData: Any?) { - val search = parseSearchNearbyActionPayload(actionData) - if (search == null) { - Log.e( - TAG, - "Invalid search-nearby payload: ${stringifyDisplayValue(actionData)}" - ) - return - } - val handler = requireClientActionHandler( - "search-nearby", - "searchTerm=${search.searchTerm}" - ) ?: return - Log.d( - TAG, - "Dispatching search-nearby to client handler searchTerm=${search.searchTerm}" - ) - handler.onSearchNearby(search) - } - - private fun requireClientActionHandler( - actionName: String, - detail: String - ): ClientActionHandler? { - val handler = synchronized(lock) { clientActionHandler } - if (handler == null) { - Log.e( - TAG, - "$actionName parsed ($detail) but no client action handler is registered" - ) - } - return handler - } - private fun handleDisplayLogEvent(event: JSONObject) { val eventType = event.optString("type") when (eventType) { @@ -1474,7 +1374,10 @@ class WebSocketManager { completion: (AndroidDeviceExecutionResult) -> Unit ) - fun onSearchNearby(action: SearchNearbyAction) + fun onSearchNearby( + action: SearchNearbyAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) } } diff --git a/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/AndroidDeviceAgentTest.kt b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/AndroidDeviceAgentTest.kt index 653f61d7ef..2900d4357b 100644 --- a/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/AndroidDeviceAgentTest.kt +++ b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/AndroidDeviceAgentTest.kt @@ -67,6 +67,41 @@ class AndroidDeviceAgentTest { assertEquals(30, alarm.action.minute) } + @Test + fun parsesSearchNearbyExecuteAction() { + val parameters = JSONObject() + .put("originalRequest", "Find coffee shops near me") + .put("searchTerm", "coffee shops") + val action = JSONObject() + .put("actionName", "searchNearby") + .put("parameters", parameters) + val args = JSONArray().put(JSONObject().put("action", action)) + + val parsed = AndroidDeviceAgent.parseExecuteAction(args) + + assertTrue(parsed is AndroidDeviceActionParseResult.Success) + val search = (parsed as AndroidDeviceActionParseResult.Success).action + as AndroidDeviceAction.SearchNearby + assertEquals("coffee shops", search.action.searchTerm) + } + + @Test + fun classifiesInvalidSearchNearbyParametersAsActionError() { + val action = JSONObject() + .put("actionName", "searchNearby") + .put( + "parameters", + JSONObject() + .put("originalRequest", "Find something") + .put("searchTerm", " ") + ) + val args = JSONArray().put(JSONObject().put("action", action)) + + val parsed = AndroidDeviceAgent.parseExecuteAction(args) + + assertTrue(parsed is AndroidDeviceActionParseResult.ActionError) + } + @Test fun classifiesInvalidParametersAsActionError() { val action = JSONObject() @@ -87,7 +122,7 @@ class AndroidDeviceAgentTest { @Test fun classifiesUnsupportedActionAsActionError() { val action = JSONObject() - .put("actionName", "openMaps") + .put("actionName", "sendSms") .put("parameters", JSONObject()) val args = JSONArray().put(JSONObject().put("action", action))