From 2928311bb7276616a742881c086e10d7dc820542 Mon Sep 17 00:00:00 2001 From: Xiting Zhang Date: Thu, 10 Sep 2026 13:16:57 -0700 Subject: [PATCH 1/4] Add Voice Agents realtime client, samples, and tests aligned with .beta structure feature/azure-ai-projects/vnext already has the generated Voice Agents surface (.beta.agent_endpoint_conversations, .beta.agent_telephony, .beta.agents telephony binding/call/generate methods, .beta.voice_agent_web_socket) via PR #48950. This change adds the missing hand-written realtime WebSocket client and the surrounding samples/tests/docs to make Voice Agents a complete, usable feature, all under the .beta naming convention: - azure/ai/projects/_realtime.py and aio/_realtime.py: hand-written realtime WebSocket client (Realtime/AsyncRealtime, RealtimeConnection(Manager)), exposed as client.beta.realtime / async_client.beta.realtime via a new property on BetaOperations. Exported from azure.ai.projects.operations / azure.ai.projects.aio.operations (not the top-level package), matching how other .beta-only classes are exported. - 11 samples under samples/agents/voice/: CRUD lifecycle, guided generation, versioning, tools, live text/audio conversations, function-tool calling, and reading back persisted conversations/audio. - 14 test files under tests/agents/ and tests/foundry_features_header/: CRUD, telephony, telephony campaigns, conversations, realtime client (mocked + live), and telephony protocol tests. - Supporting test infra: conftest.py Foundry-Features/Accept sanitizers, test_base.py foundry_voice_model_name, and a NON_OPERATION_BETA_ATTRIBUTES exclusion in the generic .beta header-injection test (realtime is a hand-written WebSocket entry point, not a generated REST operation, so it can't be exercised by that generic mechanism -- it has its own dedicated header test in test_realtime_client.py). - PostEmitter.ps1: regression guard so a future tsp-client update can't silently overwrite _realtime.py's SDK client-identification fix (PR #48848) without failing the emit step. - pyproject.toml: optional ealtime extra (websockets / aiohttp). - CHANGELOG.md, README.md, .env.template, dev_requirements.txt updated. Renames applied while porting (vnext's TypeSpec commit is newer than the source branch this was ported from, and renamed several methods beyond just .beta nesting -- verified against vnext's own generated code and docs/public-methods.md): - agent_endpoint_conversations: dropped the "agent_conversation" infix and renamed *_content methods to download_* (e.g. get_agent_conversation_item -> get_item, get_agent_conversation_item_audio_content -> download_item_audio). - agent_telephony: dropped the redundant "telephony_" infix (e.g. create_telephony_call_job -> create_call_job, begin_validate_telephony_campaign -> begin_validate_campaign). - agents.generate_agent -> agents.generate, and telephony binding/call methods moved from top-level client.agents to client.beta.agents. Validated: black/pylint/mypy/pyright all clean. Full test suite: 219 passed/74 skipped/0 failed outside voice; voice+header suite 732 passed/49 skipped/8 failed (all 8 are brand-new tests with no recording yet, not a regression). Live-tested all 11 samples end-to-end against a real Foundry project with a real gpt-realtime deployment: real HTTP status codes, real multi-turn realtime conversations with byte-accurate audio sizing, real function-tool invocation, and real conversation/audio persistence+readback. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk/ai/azure-ai-projects/.env.template | 11 + sdk/ai/azure-ai-projects/CHANGELOG.md | 31 + sdk/ai/azure-ai-projects/PostEmitter.ps1 | 28 + sdk/ai/azure-ai-projects/README.md | 2 + .../azure/ai/projects/_realtime.py | 882 +++++++++++++++++ .../azure/ai/projects/aio/_realtime.py | 887 ++++++++++++++++++ .../ai/projects/aio/operations/_patch.py | 28 +- .../azure/ai/projects/operations/_patch.py | 28 +- sdk/ai/azure-ai-projects/dev_requirements.txt | 1 + sdk/ai/azure-ai-projects/pyproject.toml | 6 + .../agents/voice/sample_voice_agent_basic.py | 107 +++ .../voice/sample_voice_agent_basic_async.py | 73 ++ .../voice/sample_voice_agent_generate.py | 66 ++ ...ice_agent_live_audio_conversation_async.py | 469 +++++++++ .../sample_voice_agent_live_function_tool.py | 204 ++++ ...mple_voice_agent_live_text_conversation.py | 354 +++++++ ...oice_agent_live_text_conversation_async.py | 357 +++++++ .../sample_voice_agent_read_conversation.py | 86 ++ ...ple_voice_agent_read_conversation_audio.py | 136 +++ .../voice/sample_voice_agent_versions.py | 92 ++ .../voice/sample_voice_agent_with_tools.py | 146 +++ .../tests/agents/test_realtime_client.py | 395 ++++++++ .../agents/test_realtime_client_async.py | 349 +++++++ .../agents/test_voice_agent_conversations.py | 239 +++++ .../test_voice_agent_conversations_async.py | 229 +++++ .../tests/agents/test_voice_agent_crud.py | 199 ++++ .../agents/test_voice_agent_crud_async.py | 205 ++++ .../agents/test_voice_agent_realtime_live.py | 299 ++++++ .../test_voice_agent_realtime_live_async.py | 297 ++++++ .../agents/test_voice_agent_telephony.py | 310 ++++++ .../test_voice_agent_telephony_async.py | 313 ++++++ .../test_voice_agent_telephony_campaign.py | 238 +++++ ...st_voice_agent_telephony_campaign_async.py | 246 +++++ sdk/ai/azure-ai-projects/tests/conftest.py | 24 + .../foundry_features_header_test_base.py | 12 + .../test_agent_telephony_protocol.py | 123 +++ .../test_agent_telephony_protocol_async.py | 119 +++ ...ndry_features_header_on_beta_operations.py | 3 + ...eatures_header_on_beta_operations_async.py | 3 + sdk/ai/azure-ai-projects/tests/test_base.py | 1 + 40 files changed, 7596 insertions(+), 2 deletions(-) create mode 100644 sdk/ai/azure-ai-projects/azure/ai/projects/_realtime.py create mode 100644 sdk/ai/azure-ai-projects/azure/ai/projects/aio/_realtime.py create mode 100644 sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_basic.py create mode 100644 sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_basic_async.py create mode 100644 sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_generate.py create mode 100644 sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py create mode 100644 sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_function_tool.py create mode 100644 sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation.py create mode 100644 sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation_async.py create mode 100644 sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_read_conversation.py create mode 100644 sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_read_conversation_audio.py create mode 100644 sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_versions.py create mode 100644 sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_with_tools.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/test_realtime_client.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/test_realtime_client_async.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_conversations.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_conversations_async.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_crud.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_crud_async.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_realtime_live.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_realtime_live_async.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_telephony.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_telephony_async.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_telephony_campaign.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_telephony_campaign_async.py create mode 100644 sdk/ai/azure-ai-projects/tests/foundry_features_header/test_agent_telephony_protocol.py create mode 100644 sdk/ai/azure-ai-projects/tests/foundry_features_header/test_agent_telephony_protocol_async.py diff --git a/sdk/ai/azure-ai-projects/.env.template b/sdk/ai/azure-ai-projects/.env.template index 14effd0c4418..8e60ee147e0c 100644 --- a/sdk/ai/azure-ai-projects/.env.template +++ b/sdk/ai/azure-ai-projects/.env.template @@ -103,6 +103,13 @@ GITHUB_USERNAME= TEAMS_CONNECTION_NAME= TEAMS_CHANNEL_URL= +# Read by the samples under samples/agents/voice/ (model deployment name, agent name, model type, +# and a conversation ID for the read-conversation samples). Distinct from FOUNDRY_VOICE_MODEL_NAME below. +FOUNDRY_VOICE_MODEL= +FOUNDRY_VOICE_MODEL_TYPE= +FOUNDRY_VOICE_AGENT_NAME= +FOUNDRY_VOICE_CONVERSATION_ID= + ####################################################################### # # Used in tests @@ -116,6 +123,10 @@ AZURE_SKIP_LIVE_RECORDING=true #Used by hosted agent FOUNDRY_HOSTED_AGENT_NAME= +# Read by the recorded voice-agent CRUD, conversation, realtime-live, and telephony tests +# (tests/test_base.py and friends), not by any sample. +FOUNDRY_VOICE_MODEL_NAME= + # Used in Fine-tuning tests COMPLETED_OAI_MODEL_SFT_FINE_TUNING_JOB_ID= COMPLETED_OAI_MODEL_RFT_FINE_TUNING_JOB_ID= diff --git a/sdk/ai/azure-ai-projects/CHANGELOG.md b/sdk/ai/azure-ai-projects/CHANGELOG.md index 52fdc2188ab1..a418bd184149 100644 --- a/sdk/ai/azure-ai-projects/CHANGELOG.md +++ b/sdk/ai/azure-ai-projects/CHANGELOG.md @@ -2,8 +2,39 @@ ## 2.7.0 (Unreleased) +### Features Added + +* Added Voice Agents, a new agent kind for real-time, speech-to-speech conversational AI unified with the rest of the Agents API. Define a voice agent's model, audio, turn detection, greeting, and tools; manage it like any other agent; hold a live conversation with it over a WebSocket with barge-in and persisted conversation history/audio; and reach it through telephony (inbound bindings or outbound calls/campaigns). +* The core voice agent definition, as a new `kind="voice"` on `AgentDefinition`: + * Define a voice agent with `VoiceAgentDefinition`, configuring its model (`VoiceModelType`), audio input/output (`VoiceAgentAudioConfig`, `VoiceAgentAudioInputConfig`, `VoiceAgentAudioOutputConfig`), turn detection (`VoiceAgentTurnDetectionConfig` and its `VoiceAgentServerVadTurnDetection` / `VoiceAgentAzureSemanticVadTurnDetection` / `VoiceAgentAzureSemanticVadEnTurnDetection` / `VoiceAgentAzureSemanticVadMultilingualTurnDetection` variants), greeting (`VoiceAgentGreetingConfig` and its `VoiceAgentTemplateGreetingConfig` / `VoiceAgentLlmGeneratedGreetingConfig` variants), tools (`VoiceAgentTool`, `VoiceAgentFunctionTool`, `VoiceAgentMcpTool`, `VoiceAgentSystemTool` and its `VoiceAgentEndConversationSystemTool` variant, `VoiceAgentToolboxTool`), and avatar (`VoiceAgentAvatarConfig`). Manage it like any other agent through `project_client.agents` (`create_version`, `get`, `list`, `disable`/`enable`, `delete`). + * Added guided authoring via `project_client.beta.agents.generate(GenerateVoiceAgentRequest(kind=AgentKind.VOICE, ...))`, which returns a service-generated starter definition that can be edited afterward through the standard `create_version`/`update` flow. + * Added a new `client.beta.realtime` / `async_client.beta.realtime` entry point for realtime speech-to-speech streaming. Use `with client.beta.realtime.connect(agent_name=...) as connection:` to open a WebSocket connection, `connection.send(...)` to send strongly-typed client events (or use the `connection.response`, `connection.conversation.item`, and `connection.session` helpers), and iterate over `connection` to receive strongly-typed server events (`RealtimeServerEvent*`). Conversation items exchanged with `connection.conversation.item.create(...)` are `RealtimeConversationItemMessageSystem`, `RealtimeConversationItemMessageUser`, `RealtimeConversationItemMessageAssistant`, `RealtimeConversationItemFunctionCall`, `RealtimeConversationItemFunctionCallOutput`, `RealtimeMCPApprovalResponse`, or a raw `Mapping[str, Any]`. The new types `Realtime`, `RealtimeConnection`, and `RealtimeConnectionManager` (and their async equivalents `AsyncRealtime`, `AsyncRealtimeConnection`, `AsyncRealtimeConnectionManager`) are exported from `azure.ai.projects.operations` / `azure.ai.projects.aio.operations`. These WebSocket clients identify themselves to the service the same way the generated HTTP surface does, via a standard Azure SDK `User-Agent` header and an `x-ms-client-sdk` query parameter for paths where the header isn't forwarded, so service telemetry can attribute this traffic to the SDK; a caller-supplied `User-Agent` in `extra_headers` still takes precedence. Requires the optional `websockets` package for the sync client, or `aiohttp` for the async client. + * Added the `.beta.agent_endpoint_conversations` operation group for reading back persisted voice-agent conversation transcripts and audio, for agents created with `store=True`. + * Added the underlying `RealtimeConversationItem*`, `RealtimeMCP*`, `RealtimeResponseUsage`, and related realtime event/session models used by the voice agent WebSocket protocol. +* Telephony, WebRTC, and sub-agent consultation: + * Added telephony bindings so a voice agent can receive calls through Teams Phone or Twilio. `project_client.beta.agents.create_telephony_binding`/`get_telephony_binding`/`update_telephony_binding`/`delete_telephony_binding`/`list_telephony_bindings` manage the binding (`TelephonyBinding` and its `TeamsPhoneExtensionTelephonyBinding`/`TwilioTelephonyBinding` variants), and `list_telephony_calls`/`get_telephony_call`/`transfer_telephony_call`/`end_telephony_call`/`get_telephony_transfer_targets`/`replace_telephony_transfer_targets` manage in-progress and historical calls (`TelephonyCallRecord`, `TelephonyCallSummary`, `TelephonyCallTrace`, `TelephonyTransferTarget` and its `PSTNTelephonyTransferDestination`/`SipTelephonyTransferDestination`/`TeamsTelephonyTransferDestination` variants). + * Added outbound telephony call jobs and campaigns through the new `.beta.agent_telephony` operation group. `create_call_job`/`get_call_job`/`cancel_call_job` place and manage a single durable outbound call against a `TelephonyOutboundDestination` (`TelephonyCallJob`), and `create_campaign`/`get_campaign`/`cancel_campaign`/`pause_campaign`/`resume_campaign` manage a bulk outbound-calling campaign (`TelephonyCampaign`). A campaign's recipients are staged with `begin_import_campaign_recipients`, checked with `begin_validate_campaign`, and started with `begin_publish_campaign` - all long-running operations polled through `get_operation` (`TelephonyOperation`, `TelephonyOperationResource`, `TelephonyCampaignRecipientImport`). + * Added an optional WebRTC transport for realtime voice sessions (`VoiceAgentTransport.WEBRTC`), where only SDP signaling travels over the WebSocket connection while media flows peer-to-peer. The new `VoiceAgentClientEventRtcCallSdpCreate`, `VoiceAgentServerEventRtcCallSdpCreated`, and `VoiceAgentServerEventRtcCallError` events carry the signaling exchange. + * Added the `.beta.agent_endpoint_conversations.get_item_generated_audio`/`download_item_generated_audio` methods for reading back a conversation item's *generated* audio, a subordinate artifact that can differ from what the listener heard when playback was interrupted, returning `VoiceGeneratedItemAudioResponse`. + * Added sub-agent consultation, letting a voice agent consult sibling Foundry text agents as background specialists mid-conversation, through the new `subagent_config` property on `VoiceAgentDefinition` (`VoiceAgentSubagentConfig`, `VoiceAgentSubagent`, `VoiceAgentSubagentResponsePolicy`), and the new `session.subagent.started`/`session.subagent.completed`/`session.subagent.aborted` realtime server events. + * Added an optional `conversation_engine` property on `VoiceAgentDefinition` (`VoiceConversationEngine`, `VoiceHostedAgentConversationEngine`) to delegate a voice agent's conversation handling to another hosted agent instead of configuring a model directly. + +### Dependency update + +* Added an optional dependency on `websockets` (sync `client.beta.realtime`) and `aiohttp` (async `async_client.beta.realtime`), required only when using the new voice agent realtime streaming APIs. + ### Sample updates +* Added voice agent samples under `samples/agents/voice/`: + * `sample_voice_agent_basic.py` / `sample_voice_agent_basic_async.py` demonstrating the voice-agent management lifecycle: create, get, list, and delete. + * `sample_voice_agent_generate.py` demonstrating guided authoring of a voice agent via `.beta.agents.generate` with `kind="voice"`. + * `sample_voice_agent_with_tools.py` demonstrating a richer voice agent definition: audio configuration, turn detection, greeting, and tools. + * `sample_voice_agent_versions.py` demonstrating voice-agent versioning: creating, drafting, listing, and publishing versions. + * `sample_voice_agent_live_text_conversation.py` / `sample_voice_agent_live_text_conversation_async.py` demonstrating a live, typed conversation with a voice agent over `client.beta.realtime`/`async_client.beta.realtime`. + * `sample_voice_agent_live_audio_conversation_async.py` demonstrating a hands-free, bidirectional live audio conversation over `async_client.beta.realtime`. + * `sample_voice_agent_live_function_tool.py` demonstrating handling a client-executed function tool during a live voice-agent session. + * `sample_voice_agent_read_conversation.py` demonstrating reading a persisted voice conversation's transcript back via `.beta.agent_endpoint_conversations`. + * `sample_voice_agent_read_conversation_audio.py` demonstrating reading a persisted voice conversation's audio, both the merged whole-call recording and a single turn's segment, via `.beta.agent_endpoint_conversations`. * Added `sample_agent_web_iq.py` under `samples/agents/tools/`, demonstrating a Prompt Agent using the `WebIQPreviewTool`. ## 2.6.0 (2026-09-04) diff --git a/sdk/ai/azure-ai-projects/PostEmitter.ps1 b/sdk/ai/azure-ai-projects/PostEmitter.ps1 index 2783c2cfdbea..92bd83fd0910 100644 --- a/sdk/ai/azure-ai-projects/PostEmitter.ps1 +++ b/sdk/ai/azure-ai-projects/PostEmitter.ps1 @@ -120,6 +120,34 @@ $c = Get-Content $f -Raw $c = $c -replace ' if_match = prep_if_match\(etag, match_condition\)\r?\n if if_match is not None:\r?\n _headers\["If-Match"\] = _SERIALIZER\.header\("if_match", if_match, "str"\)', " if etag is not None:`r`n _headers[`"If-Match`"] = _SERIALIZER.header(`"if_match`", etag, `"str`")" Set-Content $f $c -NoNewline +# Regression guard: `_realtime.py` and `aio\_realtime.py` are hand-written files that are NOT +# `_patch.py`-named, so they aren't covered by the emitter's own "never touch _patch.py" guarantee -- +# nothing in the TypeSpec emitter is aware these files exist. They carry the SDK client-identification +# fix ported from the azure-ai-voicelive PR #48848 (a User-Agent header and x-ms-client-sdk query +# parameter, both derived from `_USER_AGENT = UserAgentPolicy(sdk_moniker=...)`, with a case-insensitive +# guard so a caller-supplied extra_headers User-Agent of any casing is honored instead of duplicated). +# If a future `tsp-client update` ever starts generating (and thus silently overwriting) a file at either +# of these paths, this fix would be lost with no other signal until someone happens to run the realtime +# test suite. Fail the emit step immediately instead, right after regeneration, rather than relying on +# that eventual test run. +$realtimeFiles = @('azure\ai\projects\_realtime.py', 'azure\ai\projects\aio\_realtime.py') +foreach ($f in $realtimeFiles) { + if (-not (Test-Path $f)) { + throw "PostEmitter safety check failed: '$f' is missing. This hand-written file (not tracked by the TypeSpec emitter) carries the SDK client-identification fix from PR #48848; if the emitter deleted or renamed it, restore it from git history before continuing." + } + $c = Get-Content $f -Raw + if ($c -notmatch 'UserAgentPolicy\(sdk_moniker=') { + throw "PostEmitter safety check failed: '$f' no longer defines _USER_AGENT via UserAgentPolicy(sdk_moniker=...). The SDK client-identification fix from PR #48848 appears to have been overwritten -- reinstate the User-Agent header + x-ms-client-sdk query param wiring." + } + if ($c -notmatch '_has_header_case_insensitive') { + throw "PostEmitter safety check failed: '$f' no longer guards the User-Agent header with _has_header_case_insensitive. A caller-supplied extra_headers User-Agent (in any casing) would be duplicated instead of honored -- reinstate the case-insensitive check." + } + if ($c -notmatch 'x-ms-client-sdk') { + throw "PostEmitter safety check failed: '$f' no longer sends the x-ms-client-sdk query parameter alongside the User-Agent header -- reinstate it so service telemetry can still attribute traffic on paths that don't forward the header." + } +} +Write-Host "PostEmitter safety check passed: SDK client-identification fix (PR #48848) is intact in both _realtime.py files." + # Finishing by running 'black' tool to format code. pip install black black --config ../../../eng/black-pyproject.toml . diff --git a/sdk/ai/azure-ai-projects/README.md b/sdk/ai/azure-ai-projects/README.md index 6879f2a5e466..24edf42b4cb6 100644 --- a/sdk/ai/azure-ai-projects/README.md +++ b/sdk/ai/azure-ai-projects/README.md @@ -4,6 +4,7 @@ The AI Projects client library is part of the Microsoft Foundry SDK, and provide resources in your [Microsoft Foundry](https://ai.azure.com/) Project. Use it to: * **Create and run Agents** using methods on the `.agents` client property. This includes **Hosted Agents**, which let you run your own containerized agent runtime while using Microsoft Foundry for managed hosting and scaling. +* **Build and run Voice Agents (preview)** for real-time, speech-to-speech conversational AI, reachable over a WebSocket (`.beta.realtime`) or telephony (`.beta.agents`, `.beta.agent_telephony`), with persisted conversation transcripts and audio through `.beta.agent_endpoint_conversations`. * **Enhance Agents with specialized tools and toolbox tools** such as: * Agent-to-Agent (A2A) * Azure AI Search @@ -191,6 +192,7 @@ The table below lists the operation groups supported by the client library, with | Sessions | [Manage hosted sessions](https://learn.microsoft.com/azure/foundry/agents/how-to/manage-hosted-sessions?pivots=python) | `samples/hosted_agents/` | | Skills (preview) | | `samples/skills/` | | Toolboxes | [Curate intent-based toolbox in Foundry](https://learn.microsoft.com/azure/foundry/agents/how-to/tools/toolbox?pivots=python) | `samples/hosted_agents/`, `samples/toolboxes/` | +| Voice agents (preview) | | `samples/agents/voice/` | ## Client-side tracing diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/_realtime.py b/sdk/ai/azure-ai-projects/azure/ai/projects/_realtime.py new file mode 100644 index 000000000000..879bb801c116 --- /dev/null +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/_realtime.py @@ -0,0 +1,882 @@ +# pylint: disable=networking-import-outside-azure-core-transport +# coding=utf-8 +# -------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------- +"""Hand-written sync realtime (WebSocket) streaming client for voice agents. + +This is the synchronous counterpart of :mod:`azure.ai.projects.aio._realtime`. See that +module's docstring for the full design rationale; the two modules are kept structurally +identical (sync method names drop the ``async``/``await`` keywords) so fixes/features land in +both at once. + +``websockets`` is required for this feature and is *not* a hard dependency of the package; it +is imported lazily so importing the SDK never fails when it is absent. +""" + +from __future__ import annotations + +import base64 +import json +from urllib.parse import quote, urlencode, urlparse +from typing import ( + Any, + Dict, + Iterator, + List, + Mapping, + Optional, + Protocol, + Tuple, + Type, + TYPE_CHECKING, + Union, + cast, +) + +from azure.core.pipeline.policies import UserAgentPolicy + +from . import models as _models +from .models._enums import _AgentDefinitionOptInKeys +from .models._patch import _FOUNDRY_FEATURES_HEADER_NAME, _has_header_case_insensitive +from ._utils.model_base import Model as _Model, SdkJSONEncoder +from ._version import VERSION + +# Scoped to just the voice-agent preview opt-in; callers connecting to other preview agent +# kinds through this same route can pass a broader value explicitly via ``foundry_features``. +_VOICE_AGENT_FEATURE_HEADER: str = _AgentDefinitionOptInKeys.VOICE_AGENTS_V1_PREVIEW.value + +# Identifies the SDK to the service on the WebSocket handshake, which otherwise falls back to +# the underlying `websockets` library's generic default (the generated HTTP surface gets this +# for free from the pipeline's own UserAgentPolicy; this hand-written client builds its own +# request instead, so it needs to opt in explicitly the same way). +_USER_AGENT: str = UserAgentPolicy(sdk_moniker=f"ai-projects/{VERSION}").user_agent + +if TYPE_CHECKING: + from websockets.sync.client import ClientConnection + from azure.core.credentials import TokenCredential + from ._configuration import AIProjectClientConfiguration + + +class _ConfigProvider(Protocol): + """Anything exposing the shared client configuration (endpoint, credential, etc.). + + :class:`~azure.ai.projects.AIProjectClient` and its ``.beta`` sub-client + (:class:`~azure.ai.projects.operations.BetaOperations`) both satisfy this: operation groups + are constructed with the same shared configuration instance as the top-level client, so + ``client.beta.realtime`` can reuse the endpoint/credential wiring without needing a + back-reference to the top-level client itself. + """ + + _config: "AIProjectClientConfiguration" + + +__all__ = [ + "Realtime", + "RealtimeConnection", + "RealtimeConnectionManager", + "ClientEvent", + "ConversationItem", + "ServerEvent", +] + +# Union of the client event models sendable over the connection, plus a raw mapping escape +# hatch for forward compatibility with event types not yet represented in the generated models. +ClientEvent = Union[ + _models.RealtimeClientEventConversationItemCreate, + _models.RealtimeClientEventConversationItemDelete, + _models.RealtimeClientEventConversationItemRetrieve, + _models.RealtimeClientEventConversationItemTruncate, + _models.RealtimeClientEventInputAudioBufferAppend, + _models.RealtimeClientEventInputAudioBufferClear, + _models.RealtimeClientEventInputAudioBufferCommit, + _models.RealtimeClientEventOutputAudioBufferClear, + _models.RealtimeClientEventResponseCancel, + _models.RealtimeClientEventResponseCreate, + _models.VoiceAgentClientEventRtcCallSdpCreate, + _models.VoiceAgentClientEventSessionAvatarConnect, + _models.VoiceAgentClientEventSessionUpdate, + str, + Mapping[str, Any], +] + +# The conversation item variants accepted by ``conversation.item.create``. +ConversationItem = Union[ + _models.RealtimeConversationItemMessageSystem, + _models.RealtimeConversationItemMessageUser, + _models.RealtimeConversationItemMessageAssistant, + _models.RealtimeConversationItemFunctionCall, + _models.RealtimeConversationItemFunctionCallOutput, + _models.RealtimeMCPApprovalResponse, + Mapping[str, Any], +] + +# Every server event ``type`` string mapped to its generated model, used to deserialize +# inbound frames into strongly-typed objects. Event types not represented by a dedicated +# generated model in this package (for example ``conversation.created``) are intentionally +# left out here and fall back to a plain ``dict``, as do any newly-added service events. +_SERVER_EVENT_TYPES: Dict[str, Type[_Model]] = { + "conversation.item.added": _models.RealtimeServerEventConversationItemAdded, + "conversation.item.created": _models.RealtimeServerEventConversationItemCreated, + "conversation.item.deleted": _models.RealtimeServerEventConversationItemDeleted, + "conversation.item.done": _models.RealtimeServerEventConversationItemDone, + "conversation.item.input_audio_transcription.completed": ( + _models.RealtimeServerEventConversationItemInputAudioTranscriptionCompleted + ), + "conversation.item.input_audio_transcription.delta": ( + _models.RealtimeServerEventConversationItemInputAudioTranscriptionDelta + ), + "conversation.item.input_audio_transcription.failed": ( + _models.RealtimeServerEventConversationItemInputAudioTranscriptionFailed + ), + "conversation.item.input_audio_transcription.segment": ( + _models.RealtimeServerEventConversationItemInputAudioTranscriptionSegment + ), + "conversation.item.retrieved": _models.RealtimeServerEventConversationItemRetrieved, + "conversation.item.truncated": _models.RealtimeServerEventConversationItemTruncated, + # Shared OpenAI-style Realtime error event (not voice-agent specific in this package). + "error": _models.RealtimeServerEventError, + "input_audio_buffer.cleared": _models.RealtimeServerEventInputAudioBufferCleared, + "input_audio_buffer.committed": _models.RealtimeServerEventInputAudioBufferCommitted, + "input_audio_buffer.speech_started": _models.RealtimeServerEventInputAudioBufferSpeechStarted, + "input_audio_buffer.speech_stopped": _models.RealtimeServerEventInputAudioBufferSpeechStopped, + "input_audio_buffer.timeout_triggered": (_models.RealtimeServerEventInputAudioBufferTimeoutTriggered), + "mcp_list_tools.completed": _models.RealtimeServerEventMCPListToolsCompleted, + "mcp_list_tools.failed": _models.RealtimeServerEventMCPListToolsFailed, + "mcp_list_tools.in_progress": _models.RealtimeServerEventMCPListToolsInProgress, + "output_audio_buffer.cleared": _models.RealtimeServerEventOutputAudioBufferCleared, + "rate_limits.updated": _models.RealtimeServerEventRateLimitsUpdated, + "response.animation_blendshapes.delta": (_models.VoiceAgentServerEventResponseAnimationBlendshapesDelta), + "response.animation_blendshapes.done": (_models.VoiceAgentServerEventResponseAnimationBlendshapesDone), + "response.animation_viseme.delta": _models.VoiceAgentServerEventResponseAnimationVisemeDelta, + "response.animation_viseme.done": _models.VoiceAgentServerEventResponseAnimationVisemeDone, + "response.audio_timestamp.delta": _models.VoiceAgentServerEventResponseAudioTimestampDelta, + "response.audio_timestamp.done": _models.VoiceAgentServerEventResponseAudioTimestampDone, + "response.content_part.added": _models.RealtimeServerEventResponseContentPartAdded, + "response.content_part.done": _models.RealtimeServerEventResponseContentPartDone, + "response.created": _models.RealtimeServerEventResponseCreated, + "response.done": _models.RealtimeServerEventResponseDone, + "response.function_call_arguments.delta": (_models.RealtimeServerEventResponseFunctionCallArgumentsDelta), + "response.function_call_arguments.done": (_models.RealtimeServerEventResponseFunctionCallArgumentsDone), + "response.mcp_call.completed": _models.RealtimeServerEventResponseMCPCallCompleted, + "response.mcp_call.failed": _models.RealtimeServerEventResponseMCPCallFailed, + "response.mcp_call.in_progress": _models.RealtimeServerEventResponseMCPCallInProgress, + "response.mcp_call_arguments.delta": _models.RealtimeServerEventResponseMCPCallArgumentsDelta, + "response.mcp_call_arguments.done": _models.RealtimeServerEventResponseMCPCallArgumentsDone, + "response.output_audio.delta": _models.RealtimeServerEventResponseAudioDelta, + "response.output_audio.done": _models.RealtimeServerEventResponseAudioDone, + "response.output_audio_transcript.delta": (_models.RealtimeServerEventResponseAudioTranscriptDelta), + "response.output_audio_transcript.done": (_models.RealtimeServerEventResponseAudioTranscriptDone), + "response.output_item.added": _models.RealtimeServerEventResponseOutputItemAdded, + "response.output_item.done": _models.RealtimeServerEventResponseOutputItemDone, + "response.output_text.delta": _models.RealtimeServerEventResponseTextDelta, + "response.output_text.done": _models.RealtimeServerEventResponseTextDone, + "response.video.delta": _models.VoiceAgentServerEventResponseVideoDelta, + "rtc.call.error": _models.VoiceAgentServerEventRtcCallError, + "rtc.call.sdp.created": _models.VoiceAgentServerEventRtcCallSdpCreated, + "session.avatar.connecting": _models.VoiceAgentServerEventSessionAvatarConnecting, + "session.avatar.switch_to_idle": _models.VoiceAgentServerEventSessionAvatarSwitchToIdle, + "session.avatar.switch_to_speaking": _models.VoiceAgentServerEventSessionAvatarSwitchToSpeaking, + "session.created": _models.RealtimeServerEventSessionCreated, + "session.subagent.aborted": _models.VoiceAgentServerEventSessionSubagentAborted, + "session.subagent.completed": _models.VoiceAgentServerEventSessionSubagentCompleted, + "session.subagent.started": _models.VoiceAgentServerEventSessionSubagentStarted, + "session.updated": _models.RealtimeServerEventSessionUpdated, + "warning": _models.VoiceAgentServerEventWarning, +} + +# Every generated server event model, for consumers that want a precise return type. +ServerEvent = Union[ + _models.RealtimeServerEventError, + _models.RealtimeServerEventResponseContentPartAdded, + _models.RealtimeServerEventConversationItemAdded, + _models.RealtimeServerEventConversationItemCreated, + _models.RealtimeServerEventConversationItemDeleted, + _models.RealtimeServerEventConversationItemDone, + _models.RealtimeServerEventConversationItemInputAudioTranscriptionCompleted, + _models.RealtimeServerEventConversationItemInputAudioTranscriptionDelta, + _models.RealtimeServerEventConversationItemInputAudioTranscriptionFailed, + _models.RealtimeServerEventConversationItemInputAudioTranscriptionSegment, + _models.RealtimeServerEventConversationItemRetrieved, + _models.RealtimeServerEventConversationItemTruncated, + _models.RealtimeServerEventInputAudioBufferCleared, + _models.RealtimeServerEventInputAudioBufferCommitted, + _models.RealtimeServerEventInputAudioBufferSpeechStarted, + _models.RealtimeServerEventInputAudioBufferSpeechStopped, + _models.RealtimeServerEventInputAudioBufferTimeoutTriggered, + _models.RealtimeServerEventMCPListToolsCompleted, + _models.RealtimeServerEventMCPListToolsFailed, + _models.RealtimeServerEventMCPListToolsInProgress, + _models.RealtimeServerEventOutputAudioBufferCleared, + _models.RealtimeServerEventRateLimitsUpdated, + _models.VoiceAgentServerEventResponseAnimationBlendshapesDelta, + _models.VoiceAgentServerEventResponseAnimationBlendshapesDone, + _models.VoiceAgentServerEventResponseAnimationVisemeDelta, + _models.VoiceAgentServerEventResponseAnimationVisemeDone, + _models.RealtimeServerEventResponseAudioDelta, + _models.RealtimeServerEventResponseAudioDone, + _models.VoiceAgentServerEventResponseAudioTimestampDelta, + _models.VoiceAgentServerEventResponseAudioTimestampDone, + _models.RealtimeServerEventResponseAudioTranscriptDelta, + _models.RealtimeServerEventResponseAudioTranscriptDone, + _models.RealtimeServerEventResponseContentPartDone, + _models.RealtimeServerEventResponseCreated, + _models.RealtimeServerEventResponseDone, + _models.RealtimeServerEventResponseFunctionCallArgumentsDelta, + _models.RealtimeServerEventResponseFunctionCallArgumentsDone, + _models.RealtimeServerEventResponseMCPCallArgumentsDelta, + _models.RealtimeServerEventResponseMCPCallArgumentsDone, + _models.RealtimeServerEventResponseMCPCallCompleted, + _models.RealtimeServerEventResponseMCPCallFailed, + _models.RealtimeServerEventResponseMCPCallInProgress, + _models.RealtimeServerEventResponseOutputItemAdded, + _models.RealtimeServerEventResponseOutputItemDone, + _models.RealtimeServerEventResponseTextDelta, + _models.RealtimeServerEventResponseTextDone, + _models.VoiceAgentServerEventResponseVideoDelta, + _models.VoiceAgentServerEventRtcCallError, + _models.VoiceAgentServerEventRtcCallSdpCreated, + _models.VoiceAgentServerEventSessionAvatarConnecting, + _models.VoiceAgentServerEventSessionAvatarSwitchToIdle, + _models.VoiceAgentServerEventSessionAvatarSwitchToSpeaking, + _models.RealtimeServerEventSessionCreated, + _models.VoiceAgentServerEventSessionSubagentAborted, + _models.VoiceAgentServerEventSessionSubagentCompleted, + _models.VoiceAgentServerEventSessionSubagentStarted, + _models.RealtimeServerEventSessionUpdated, + _models.VoiceAgentServerEventWarning, + Mapping[str, Any], +] + + +def _to_ws_url(endpoint: str, agent_name: str) -> str: + """Build the realtime WebSocket URL from the HTTPS project endpoint. + + Only the ``https://`` scheme is translated (to ``wss://``); any other scheme is left + unchanged so that :meth:`RealtimeConnectionManager.enter`'s ``wss://``-only check rejects + it with a clear error instead of silently producing an unencrypted ``ws://`` URL that would + also send the live Authorization token in plain text. + + :param str endpoint: The Foundry project endpoint (``https://.../api/projects/...``). + :param str agent_name: The name of the voice agent to connect to. + :return: A ``wss://`` URL targeting the realtime route. + :rtype: str + """ + base = endpoint.rstrip("/") + if base.startswith("https://"): + base = "wss://" + base[len("https://") :] + return f"{base}/agents/{quote(agent_name, safe='')}/endpoint/protocols/voice" + + +_DEFAULT_PORT_BY_SCHEME = {"http": 80, "https": 443, "ws": 80, "wss": 443} + + +def _normalized_authority(url: str) -> Tuple[str, Optional[int]]: + """Return a ``(hostname, port)`` tuple with the scheme's default port filled in. + + ``urlparse(...).port`` is ``None`` when a URL omits an explicit port, which would make + ``https://host/...`` and ``https://host:8443/...`` compare as equal on hostname alone. + Resolving the scheme's default port here lets callers compare authorities (not just + hostnames) so a same-host override on a different, non-default port is correctly rejected. + + :param str url: The URL to parse. + :return: A tuple of the lower-cased hostname (or empty string) and the resolved port + (or ``None`` if the scheme has no known default and none was specified). + :rtype: tuple[str, Optional[int]] + """ + parsed = urlparse(url) + port = parsed.port + if port is None: + port = _DEFAULT_PORT_BY_SCHEME.get((parsed.scheme or "").lower()) + return (parsed.hostname or "").lower(), port + + +def _assert_trusted_connection_url(connection_url: str, endpoint: str) -> None: + """Guard against attaching the caller's Entra bearer token to an untrusted host. + + ``connection_url`` is an escape hatch that lets a caller override the computed + scheme/host/path, but the Authorization header carrying the live credential's + token must never be sent to a host other than the configured Foundry project + endpoint: a caller-controlled or compromised URL could otherwise be used to + exfiltrate the token to an arbitrary host or port. + + :param str connection_url: The caller-supplied override URL. + :param str endpoint: The configured, trusted Foundry project endpoint. + :raises ValueError: If the override URL's host or port does not match the endpoint's. + """ + override_host, override_port = _normalized_authority(connection_url) + trusted_host, trusted_port = _normalized_authority(endpoint) + if not override_host or (override_host, override_port) != (trusted_host, trusted_port): + got = override_host or connection_url + if override_host and override_port: + got = f"{override_host}:{override_port}" + raise ValueError( + "The 'connection_url' override must target the same host and port as the configured " + f"Foundry project endpoint ('{trusted_host}:{trusted_port}') to avoid sending the " + f"Authorization token to an untrusted host; got '{got}'." + ) + + +class _BaseResource: # pylint: disable=too-few-public-methods + """Base helper that forwards typed helpers to the parent connection.""" + + def __init__(self, connection: "RealtimeConnection") -> None: + self._connection = connection + + def _send(self, event: ClientEvent) -> None: + self._connection.send(event) + + +class SessionResource(_BaseResource): + """Send ``session.*`` client events.""" + + def update( + self, + *, + session: Union["_models.VoiceAgentSessionUpdateConfig", Mapping[str, Any]], + event_id: Optional[str] = None, + ) -> None: + """Update the realtime session configuration. + + :keyword session: The session configuration to apply. + :paramtype session: ~azure.ai.projects.models.VoiceAgentSessionUpdateConfig or + Mapping[str, Any] + :keyword event_id: Optional client-generated ID used to identify this event. + :paramtype event_id: str or None + """ + self._send( + cast(Any, _models.VoiceAgentClientEventSessionUpdate)( + type=_models.RealtimeClientEventType.SESSION_UPDATE, + session=session, + event_id=event_id, + ) + ) + + def avatar_connect(self, *, client_sdp: str, event_id: Optional[str] = None) -> None: + """Negotiate an avatar media session over WebRTC. + + :keyword str client_sdp: The client's SDP offer for avatar media negotiation. + :keyword event_id: An optional client-generated event identifier. + :paramtype event_id: str or None + """ + self._send( + _models.VoiceAgentClientEventSessionAvatarConnect( + client_sdp=client_sdp, + event_id=event_id, + ) + ) + + +class InputAudioBufferResource(_BaseResource): + """Send ``input_audio_buffer.*`` client events.""" + + def append(self, *, audio: Union[str, bytes], event_id: Optional[str] = None) -> None: + """Append audio bytes to the input buffer. + + :keyword audio: Raw audio bytes, or an already base64-encoded string. + :paramtype audio: str or bytes + :keyword event_id: Optional client-generated ID used to identify this event. + :paramtype event_id: str or None + """ + if isinstance(audio, (bytes, bytearray)): + audio = base64.b64encode(bytes(audio)).decode("ascii") + self._send( + _models.RealtimeClientEventInputAudioBufferAppend( + audio=audio, + event_id=event_id, + ) + ) + + def commit(self, *, event_id: Optional[str] = None) -> None: + """Commit the buffered input audio as a user turn. + + :keyword event_id: Optional client-generated ID used to identify this event. + :paramtype event_id: str or None + """ + self._send(_models.RealtimeClientEventInputAudioBufferCommit(event_id=event_id)) + + def clear(self, *, event_id: Optional[str] = None) -> None: + """Discard any buffered input audio. + + :keyword event_id: Optional client-generated ID used to identify this event. + :paramtype event_id: str or None + """ + self._send(_models.RealtimeClientEventInputAudioBufferClear(event_id=event_id)) + + +class OutputAudioBufferResource(_BaseResource): # pylint: disable=too-few-public-methods + """Send ``output_audio_buffer.*`` client events.""" + + def clear(self, *, event_id: Optional[str] = None) -> None: + """Stop and clear any audio the service is currently playing back (barge-in). + + :keyword event_id: Optional client-generated ID used to identify this event. + :paramtype event_id: str or None + """ + self._send(_models.RealtimeClientEventOutputAudioBufferClear(event_id=event_id)) + + +class ConversationItemResource(_BaseResource): + """Send ``conversation.item.*`` client events.""" + + def create( + self, + *, + item: ConversationItem, + previous_item_id: Optional[str] = None, + event_id: Optional[str] = None, + ) -> None: + """Insert an item into the conversation. + + :keyword item: The conversation item to create. + :paramtype item: ~azure.ai.projects.models.RealtimeConversationItemMessageSystem or + ~azure.ai.projects.models.RealtimeConversationItemMessageUser or + ~azure.ai.projects.models.RealtimeConversationItemMessageAssistant or + ~azure.ai.projects.models.RealtimeConversationItemFunctionCall or + ~azure.ai.projects.models.RealtimeConversationItemFunctionCallOutput or + ~azure.ai.projects.models.RealtimeMCPApprovalResponse or Mapping[str, Any] + :keyword previous_item_id: The ID of the preceding item after which the new item will be + inserted. Default value is None. + :paramtype previous_item_id: str or None + :keyword event_id: Optional client-generated ID used to identify this event. + :paramtype event_id: str or None + """ + self._send( + cast(Any, _models.RealtimeClientEventConversationItemCreate)( + item=item, + previous_item_id=previous_item_id, + event_id=event_id, + ) + ) + + def delete(self, *, item_id: str, event_id: Optional[str] = None) -> None: + """Delete an item from the conversation. + + :keyword str item_id: The ID of the item to delete. + :keyword event_id: Optional client-generated ID used to identify this event. + :paramtype event_id: str or None + """ + self._send( + _models.RealtimeClientEventConversationItemDelete( + item_id=item_id, + event_id=event_id, + ) + ) + + def retrieve(self, *, item_id: str, event_id: Optional[str] = None) -> None: + """Ask the server to emit a ``conversation.item.retrieved`` event for an item. + + :keyword str item_id: The ID of the item to retrieve. + :keyword event_id: Optional client-generated ID used to identify this event. + :paramtype event_id: str or None + """ + self._send( + _models.RealtimeClientEventConversationItemRetrieve( + item_id=item_id, + event_id=event_id, + ) + ) + + def truncate(self, *, item_id: str, content_index: int, audio_end_ms: int, event_id: Optional[str] = None) -> None: + """Truncate a previously produced assistant audio item (used for barge-in). + + :keyword str item_id: The ID of the assistant message item to truncate. + :keyword int content_index: The index of the content part to truncate. Use ``0``. + :keyword int audio_end_ms: The point, in milliseconds, to truncate the audio to. + :keyword event_id: Optional client-generated ID used to identify this event. + :paramtype event_id: str or None + """ + self._send( + _models.RealtimeClientEventConversationItemTruncate( + item_id=item_id, + content_index=content_index, + audio_end_ms=audio_end_ms, + event_id=event_id, + ) + ) + + +class ConversationResource(_BaseResource): # pylint: disable=too-few-public-methods + """Send ``conversation.*`` client events.""" + + def __init__(self, connection: "RealtimeConnection") -> None: + super().__init__(connection) + self.item: ConversationItemResource = ConversationItemResource(connection) + + +class ResponseResource(_BaseResource): + """Send ``response.*`` client events.""" + + def create( + self, + *, + response: Optional[Union["_models.VoiceAgentResponseCreateParams", Mapping[str, Any]]] = None, + event_id: Optional[str] = None, + ) -> None: + """Ask the model to generate a response. + + :keyword response: Optional per-response overrides. Default value is None. + :paramtype response: ~azure.ai.projects.models.VoiceAgentResponseCreateParams or + Mapping[str, Any] or None + :keyword event_id: Optional client-generated ID used to identify this event. + :paramtype event_id: str or None + """ + self._send( + cast(Any, _models.RealtimeClientEventResponseCreate)( + response=response, + event_id=event_id, + ) + ) + + def cancel(self, *, response_id: Optional[str] = None, event_id: Optional[str] = None) -> None: + """Cancel an in-progress response. + + :keyword response_id: The ID of the response to cancel, if targeting a specific one. + Default value is None. + :paramtype response_id: str or None + :keyword event_id: Optional client-generated ID used to identify this event. + :paramtype event_id: str or None + """ + self._send( + _models.RealtimeClientEventResponseCancel( + response_id=response_id, + event_id=event_id, + ) + ) + + +class RealtimeConnection: # pylint: disable=too-many-instance-attributes + """An open realtime WebSocket connection to a voice agent. + + Iterate over the connection to receive strongly-typed server events, and use the + sub-namespaces to send strongly-typed client events:: + + with client.beta.realtime.connect(agent_name="my-agent") as conn: + for event in conn: + if event.type == RealtimeServerEventType.RESPONSE_DONE: + break + """ + + def __init__(self, connection: "ClientConnection") -> None: + self._connection = connection + self._closed = False + self.session: SessionResource = SessionResource(self) + self.input_audio_buffer: InputAudioBufferResource = InputAudioBufferResource(self) + self.output_audio_buffer: OutputAudioBufferResource = OutputAudioBufferResource(self) + self.conversation: ConversationResource = ConversationResource(self) + self.response: ResponseResource = ResponseResource(self) + + def __enter__(self) -> "RealtimeConnection": + return self + + def __exit__(self, *exc_details: Any) -> None: + self.close() + + def __repr__(self) -> str: + state = "closed" if self.closed else "open" + return f"" + + @property + def closed(self) -> bool: + """Whether the underlying WebSocket connection has been closed. + + :rtype: bool + """ + return self._closed + + def __iter__(self) -> Iterator[ServerEvent]: + return self._iter() + + def _iter(self) -> Iterator[ServerEvent]: + while True: + try: + yield self.recv() + except ConnectionResetError: + return + + def recv(self, *, timeout: Optional[float] = None) -> ServerEvent: + """Receive and parse the next server event. + + Known event types are returned as their strongly-typed + ``VoiceAgentServerEventXxx`` model. Event types not (yet) represented by a + generated model are returned as a plain ``dict`` for forward compatibility. + + :keyword timeout: Maximum time in seconds to wait for the next event. If ``None`` + (the default), block until an event is received. If no event arrives within + ``timeout`` seconds, raise :exc:`TimeoutError`. + :paramtype timeout: float or None + :return: The parsed server event. + :rtype: ~azure.ai.projects.ServerEvent + :raises ConnectionResetError: If the connection was closed by the server. + :raises TimeoutError: If ``timeout`` elapses before an event is received. + """ + from websockets.exceptions import ConnectionClosed # pylint: disable=import-outside-toplevel + + try: + raw = self._connection.recv(timeout=timeout) + except ConnectionClosed as exc: + self._closed = True + raise ConnectionResetError("The realtime connection was closed.") from exc + data = raw.decode("utf-8") if isinstance(raw, (bytes, bytearray)) else raw + payload: Dict[str, Any] = json.loads(data) + event_type = payload.get("type") + if not isinstance(event_type, str): + return payload + event_cls = _SERVER_EVENT_TYPES.get(event_type) + if event_cls is None: + return payload + return event_cls(payload) + + def send(self, event: ClientEvent) -> None: + """Send a client event over the connection. + + :param event: A strongly-typed client event, a ready-made mapping, or a raw JSON string. + :type event: ~azure.ai.projects.ClientEvent or str + :raises ValueError: If ``event`` is a ``str`` that is not valid JSON. + """ + if isinstance(event, str): + try: + json.loads(event) + except ValueError as exc: + raise ValueError(f"'event' is not valid JSON: {exc}") from exc + payload = event + else: + payload = json.dumps(event, cls=SdkJSONEncoder) + self._connection.send(payload) + + def close(self, *, code: int = 1000, reason: str = "") -> None: + """Close the connection. + + :keyword int code: The WebSocket close code. + :keyword str reason: The close reason. + """ + if self._closed: + return + try: + self._connection.close(code=code, reason=reason) + finally: + self._closed = True + + +class RealtimeConnectionManager: # pylint: disable=too-many-instance-attributes + """Context manager that opens a :class:`RealtimeConnection`. + + Returned by :meth:`Realtime.connect`; you normally use it as + ``with client.beta.realtime.connect(...) as conn:``. + """ + + def __init__( # pylint: disable=too-many-arguments + self, + *, + endpoint: str, + credential: "TokenCredential", + credential_scopes: List[str], + api_version: str, + agent_name: str, + foundry_features: str, + agent_session_id: Optional[str] = None, + agent_version_override: Optional[str] = None, + structured_inputs: Optional[str] = None, + connection_url: Optional[str] = None, + extra_query: Optional[Mapping[str, str]] = None, + extra_headers: Optional[Mapping[str, str]] = None, + **kwargs: Any, + ) -> None: + self._endpoint = endpoint + self._credential = credential + self._credential_scopes = credential_scopes + self._api_version = api_version + self._agent_name = agent_name + self._foundry_features = foundry_features + self._agent_session_id = agent_session_id + self._agent_version_override = agent_version_override + self._structured_inputs = structured_inputs + self._connection_url = connection_url + self._extra_query = dict(extra_query or {}) + self._extra_headers = dict(extra_headers or {}) + self._kwargs = kwargs + self._connection: Optional[RealtimeConnection] = None + + def __enter__(self) -> RealtimeConnection: + return self.enter() + + def enter(self) -> RealtimeConnection: # pylint: disable=too-many-locals + """Open the connection. + + :return: The live realtime connection. + :rtype: ~azure.ai.projects.RealtimeConnection + :raises RuntimeError: If ``websockets`` is not installed. + :raises ValueError: If the computed or supplied WebSocket URL does not use ``wss://``. + :raises ConnectionError: If the WebSocket upgrade handshake fails (for example, a + network error, DNS failure, or a non-101 response from the service). + """ + try: + from websockets.sync.client import connect as _ws_connect # pylint: disable=import-outside-toplevel + from websockets.typing import Subprotocol # pylint: disable=import-outside-toplevel + except ImportError as exc: # pragma: no cover - dependency guard + raise RuntimeError( + "The realtime client requires `websockets`. Install it with `pip install websockets`." + ) from exc + + # ``connection_url`` fully overrides the computed route (scheme/host/path). This is the + # escape hatch used to reach a specific data-plane host/path directly. + if self._connection_url is not None: + _assert_trusted_connection_url(self._connection_url, self._endpoint) + url = self._connection_url or _to_ws_url(self._endpoint, self._agent_name) + if not url.startswith("wss://"): + raise ValueError("The realtime WebSocket URL must use wss:// to protect credentials in transit.") + + params: Dict[str, str] = {"api-version": self._api_version, "x-ms-client-sdk": _USER_AGENT} + if self._agent_session_id is not None: + params["agent_session_id"] = self._agent_session_id + if self._agent_version_override is not None: + params["x-agent-version-override"] = self._agent_version_override + params.update(self._extra_query) + + if params: + # Preserve an existing query string on a `connection_url` override (for example a + # SAS-style `?sig=...`) instead of unconditionally appending a second `?`. + delimiter = "&" if urlparse(url).query else "?" + full_url = f"{url}{delimiter}{urlencode(params)}" + else: + full_url = url + + token = self._credential.get_token(*self._credential_scopes) + headers: Dict[str, str] = { + "Authorization": "Bearer " + token.token, + _FOUNDRY_FEATURES_HEADER_NAME: self._foundry_features, + } + if self._structured_inputs is not None: + headers["x-ms-voice-structured-inputs"] = self._structured_inputs + headers.update(self._extra_headers) + if not _has_header_case_insensitive(headers, "User-Agent"): + # Only set our default if the caller didn't supply their own (in any casing) -- + # a plain dict merge would otherwise leave both as separate keys (HTTP header names + # are case-insensitive, but Python dict keys are not), sending two User-Agent-like + # headers instead of cleanly honoring the caller's override. + headers["User-Agent"] = _USER_AGENT + + try: + # Force the "realtime" WebSocket subprotocol regardless of any caller-supplied + # override in ``self._kwargs``: the service requires this exact subprotocol, so + # silently accepting a different one here would just move the failure to a less + # clear error inside the handshake. Also disable ``websockets``' own + # ``user_agent_header`` default: unlike aiohttp, it is a wholly separate mechanism + # from ``additional_headers`` -- passing our own "User-Agent" there does not + # override it, so without this the connection would carry two distinct + # User-Agent-like values. + ws_connect_kwargs = dict(self._kwargs) + ws_connect_kwargs.pop("subprotocols", None) + connection = _ws_connect( + full_url, + additional_headers=headers, + subprotocols=[Subprotocol("realtime")], + user_agent_header=None, + **ws_connect_kwargs, + ) + except BaseException as exc: + if not isinstance(exc, Exception) or isinstance(exc, (ValueError, RuntimeError)): + raise + raise ConnectionError( + f"Failed to open the realtime WebSocket connection to voice agent " + f"'{self._agent_name}' at '{url}': {exc}" + ) from exc + self._connection = RealtimeConnection(connection) + return self._connection + + def __exit__(self, *exc_details: Any) -> None: + if self._connection is not None: + self._connection.close() + self._connection = None + + +class Realtime: # pylint: disable=too-few-public-methods + """Realtime streaming entry point, exposed as ``client.beta.realtime``. + + Follows the OpenAI Python realtime surface: obtain it from the HTTP client and open a + connection with :meth:`connect`:: + + from azure.ai.projects import AIProjectClient + from azure.identity import DefaultAzureCredential + + client = AIProjectClient(endpoint, DefaultAzureCredential()) + with client.beta.realtime.connect(agent_name="my-agent") as conn: + conn.input_audio_buffer.append(audio=chunk) + conn.input_audio_buffer.commit() + conn.response.create() + for event in conn: + if event.type == RealtimeServerEventType.RESPONSE_DONE: + break + + :param client: The object whose endpoint and credential are reused for the realtime + handshake -- either the top-level client or its ``.beta`` sub-client, since both share the + same underlying configuration. + :type client: ~azure.ai.projects.AIProjectClient or ~azure.ai.projects.operations.BetaOperations + """ + + def __init__(self, client: "_ConfigProvider") -> None: + self._config = client._config # pylint: disable=protected-access + + def connect( # pylint: disable=too-many-arguments + self, + *, + agent_name: str, + foundry_features: str = _VOICE_AGENT_FEATURE_HEADER, + agent_session_id: Optional[str] = None, + agent_version_override: Optional[str] = None, + structured_inputs: Optional[str] = None, + connection_url: Optional[str] = None, + api_version: Optional[str] = None, + credential_scopes: Optional[List[str]] = None, + extra_query: Optional[Mapping[str, str]] = None, + extra_headers: Optional[Mapping[str, str]] = None, + **kwargs: Any, + ) -> RealtimeConnectionManager: + """Open a realtime WebSocket connection to a voice agent. + + :keyword str agent_name: The name of the voice agent to connect to. + :keyword foundry_features: Preview opt-in value(s) for the ``Foundry-Features`` header. + Defaults to ``VoiceAgents=V1Preview``. Pass a comma-separated value to opt in to + additional preview features on the same request. + :paramtype foundry_features: str + :keyword agent_session_id: An optional identifier used to correlate the voice session. + Default value is None. + :paramtype agent_session_id: str or None + :keyword agent_version_override: Selects a specific version of the voice agent for this + session. Default value is None. + :paramtype agent_version_override: str or None + :keyword structured_inputs: A JSON object that maps structured-input names to their + values for this session. Default value is None. + :paramtype structured_inputs: str or None + :keyword connection_url: Full ``wss://`` URL that overrides the route computed + from the client endpoint. Query parameters are still appended. Default value is None. + :paramtype connection_url: str or None + :keyword api_version: Overrides the client's API version for the handshake. Default + value is None. + :paramtype api_version: str or None + :keyword credential_scopes: Overrides the client's token scopes for the handshake. + Default value is None. + :paramtype credential_scopes: list[str] or None + :keyword extra_query: Additional query-string parameters for the handshake. + :paramtype extra_query: Mapping[str, str] or None + :keyword extra_headers: Additional headers for the handshake. + :paramtype extra_headers: Mapping[str, str] or None + :return: A context manager yielding a :class:`RealtimeConnection`. + :rtype: ~azure.ai.projects.RealtimeConnectionManager + """ + return RealtimeConnectionManager( + endpoint=self._config.endpoint, + credential=self._config.credential, + credential_scopes=credential_scopes or self._config.credential_scopes, + api_version=api_version or self._config.api_version, + agent_name=agent_name, + foundry_features=foundry_features, + agent_session_id=agent_session_id, + agent_version_override=agent_version_override, + structured_inputs=structured_inputs, + connection_url=connection_url, + extra_query=extra_query, + extra_headers=extra_headers, + **kwargs, + ) diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_realtime.py b/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_realtime.py new file mode 100644 index 000000000000..9d8e26af0ff6 --- /dev/null +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_realtime.py @@ -0,0 +1,887 @@ +# pylint: disable=networking-import-outside-azure-core-transport +# coding=utf-8 +# -------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------- +"""Hand-written async realtime (WebSocket) streaming client for voice agents. + +Realtime uses a fundamentally different transport (a persistent WebSocket) than the +request/response HTTP surface generated from the service's TypeSpec definition, so it is +hand-written and exposed as the ``AIProjectClient.beta.realtime`` namespace. + +The connection ergonomics follow the OpenAI Python realtime client so that developers moving +between the libraries get a familiar surface: + +* :meth:`AsyncRealtime.connect` returns an async context manager. +* Entering the context yields an :class:`AsyncRealtimeConnection`. +* The connection is async-iterable over inbound, strongly-typed server events and exposes + sub-namespaces (``session``, ``input_audio_buffer``, ``output_audio_buffer``, + ``conversation``, ``response``) for sending strongly-typed outbound client events. + +Outbound and inbound events use the generated ``VoiceAgentClientEventXxx``/ +``VoiceAgentServerEventXxx`` models directly where one exists. ``send`` and ``recv`` still +accept/return plain ``dict`` objects as a forward-compatible fallback for any event ``type`` +the generated models don't yet know about (for example ``conversation.created``, which is a +valid event but does not (yet) have a dedicated generated model in this package). + +``aiohttp`` is required for this feature and is *not* a hard dependency of the package; it is +imported lazily so importing the SDK never fails when it is absent. +""" + +from __future__ import annotations + +import base64 +import json +from urllib.parse import quote, urlparse +from typing import ( + Any, + AsyncIterator, + cast, + Dict, + List, + Mapping, + Optional, + Protocol, + Tuple, + Type, + TYPE_CHECKING, + Union, +) + +from azure.core.pipeline.policies import UserAgentPolicy + +from .. import models as _models +from ..models._enums import _AgentDefinitionOptInKeys +from ..models._patch import _FOUNDRY_FEATURES_HEADER_NAME, _has_header_case_insensitive +from .._utils.model_base import Model as _Model, SdkJSONEncoder +from .._version import VERSION + +# Scoped to just the voice-agent preview opt-in; callers connecting to other preview agent +# kinds through this same route can pass a broader value explicitly via ``foundry_features``. +_VOICE_AGENT_FEATURE_HEADER: str = _AgentDefinitionOptInKeys.VOICE_AGENTS_V1_PREVIEW.value + +# Identifies the SDK to the service on the WebSocket handshake, which otherwise falls back to +# the underlying `aiohttp` library's generic default (the generated HTTP surface gets this for +# free from the pipeline's own UserAgentPolicy; this hand-written client builds its own request +# instead, so it needs to opt in explicitly the same way). +_USER_AGENT: str = UserAgentPolicy(sdk_moniker=f"ai-projects/{VERSION}").user_agent + +if TYPE_CHECKING: + from aiohttp import ClientSession, ClientWebSocketResponse + from azure.core.credentials_async import AsyncTokenCredential + from ._configuration import AIProjectClientConfiguration + + +class _ConfigProvider(Protocol): + """Anything exposing the shared client configuration (endpoint, credential, etc.). + + :class:`~azure.ai.projects.aio.AIProjectClient` and its ``.beta`` sub-client + (:class:`~azure.ai.projects.aio.operations.BetaOperations`) both satisfy this: operation + groups are constructed with the same shared configuration instance as the top-level client, + so ``async_client.beta.realtime`` can reuse the endpoint/credential wiring without needing a + back-reference to the top-level client itself. + """ + + _config: "AIProjectClientConfiguration" + + +__all__ = [ + "AsyncRealtime", + "AsyncRealtimeConnection", + "AsyncRealtimeConnectionManager", + "ClientEvent", + "ConversationItem", + "ServerEvent", +] + +# Union of the client event models sendable over the connection, plus a raw mapping escape +# hatch for forward compatibility with event types not yet represented in the generated models. +ClientEvent = Union[ + _models.RealtimeClientEventConversationItemCreate, + _models.RealtimeClientEventConversationItemDelete, + _models.RealtimeClientEventConversationItemRetrieve, + _models.RealtimeClientEventConversationItemTruncate, + _models.RealtimeClientEventInputAudioBufferAppend, + _models.RealtimeClientEventInputAudioBufferClear, + _models.RealtimeClientEventInputAudioBufferCommit, + _models.RealtimeClientEventOutputAudioBufferClear, + _models.RealtimeClientEventResponseCancel, + _models.RealtimeClientEventResponseCreate, + _models.VoiceAgentClientEventRtcCallSdpCreate, + _models.VoiceAgentClientEventSessionAvatarConnect, + _models.VoiceAgentClientEventSessionUpdate, + str, + Mapping[str, Any], +] + +# The conversation item variants accepted by ``conversation.item.create``. +ConversationItem = Union[ + _models.RealtimeConversationItemMessageSystem, + _models.RealtimeConversationItemMessageUser, + _models.RealtimeConversationItemMessageAssistant, + _models.RealtimeConversationItemFunctionCall, + _models.RealtimeConversationItemFunctionCallOutput, + _models.RealtimeMCPApprovalResponse, + Mapping[str, Any], +] + +# Every server event ``type`` string mapped to its generated model, used to deserialize +# inbound frames into strongly-typed objects. Event types not represented by a dedicated +# generated model in this package (for example ``conversation.created``) are intentionally +# left out here and fall back to a plain ``dict``, as do any newly-added service events. +_SERVER_EVENT_TYPES: Dict[str, Type[_Model]] = { + "conversation.item.added": _models.RealtimeServerEventConversationItemAdded, + "conversation.item.created": _models.RealtimeServerEventConversationItemCreated, + "conversation.item.deleted": _models.RealtimeServerEventConversationItemDeleted, + "conversation.item.done": _models.RealtimeServerEventConversationItemDone, + "conversation.item.input_audio_transcription.completed": ( + _models.RealtimeServerEventConversationItemInputAudioTranscriptionCompleted + ), + "conversation.item.input_audio_transcription.delta": ( + _models.RealtimeServerEventConversationItemInputAudioTranscriptionDelta + ), + "conversation.item.input_audio_transcription.failed": ( + _models.RealtimeServerEventConversationItemInputAudioTranscriptionFailed + ), + "conversation.item.input_audio_transcription.segment": ( + _models.RealtimeServerEventConversationItemInputAudioTranscriptionSegment + ), + "conversation.item.retrieved": _models.RealtimeServerEventConversationItemRetrieved, + "conversation.item.truncated": _models.RealtimeServerEventConversationItemTruncated, + # Shared OpenAI-style Realtime error event (not voice-agent specific in this package). + "error": _models.RealtimeServerEventError, + "input_audio_buffer.cleared": _models.RealtimeServerEventInputAudioBufferCleared, + "input_audio_buffer.committed": _models.RealtimeServerEventInputAudioBufferCommitted, + "input_audio_buffer.speech_started": _models.RealtimeServerEventInputAudioBufferSpeechStarted, + "input_audio_buffer.speech_stopped": _models.RealtimeServerEventInputAudioBufferSpeechStopped, + "input_audio_buffer.timeout_triggered": (_models.RealtimeServerEventInputAudioBufferTimeoutTriggered), + "mcp_list_tools.completed": _models.RealtimeServerEventMCPListToolsCompleted, + "mcp_list_tools.failed": _models.RealtimeServerEventMCPListToolsFailed, + "mcp_list_tools.in_progress": _models.RealtimeServerEventMCPListToolsInProgress, + "output_audio_buffer.cleared": _models.RealtimeServerEventOutputAudioBufferCleared, + "rate_limits.updated": _models.RealtimeServerEventRateLimitsUpdated, + "response.animation_blendshapes.delta": (_models.VoiceAgentServerEventResponseAnimationBlendshapesDelta), + "response.animation_blendshapes.done": (_models.VoiceAgentServerEventResponseAnimationBlendshapesDone), + "response.animation_viseme.delta": _models.VoiceAgentServerEventResponseAnimationVisemeDelta, + "response.animation_viseme.done": _models.VoiceAgentServerEventResponseAnimationVisemeDone, + "response.audio_timestamp.delta": _models.VoiceAgentServerEventResponseAudioTimestampDelta, + "response.audio_timestamp.done": _models.VoiceAgentServerEventResponseAudioTimestampDone, + "response.content_part.added": _models.RealtimeServerEventResponseContentPartAdded, + "response.content_part.done": _models.RealtimeServerEventResponseContentPartDone, + "response.created": _models.RealtimeServerEventResponseCreated, + "response.done": _models.RealtimeServerEventResponseDone, + "response.function_call_arguments.delta": (_models.RealtimeServerEventResponseFunctionCallArgumentsDelta), + "response.function_call_arguments.done": (_models.RealtimeServerEventResponseFunctionCallArgumentsDone), + "response.mcp_call.completed": _models.RealtimeServerEventResponseMCPCallCompleted, + "response.mcp_call.failed": _models.RealtimeServerEventResponseMCPCallFailed, + "response.mcp_call.in_progress": _models.RealtimeServerEventResponseMCPCallInProgress, + "response.mcp_call_arguments.delta": _models.RealtimeServerEventResponseMCPCallArgumentsDelta, + "response.mcp_call_arguments.done": _models.RealtimeServerEventResponseMCPCallArgumentsDone, + "response.output_audio.delta": _models.RealtimeServerEventResponseAudioDelta, + "response.output_audio.done": _models.RealtimeServerEventResponseAudioDone, + "response.output_audio_transcript.delta": (_models.RealtimeServerEventResponseAudioTranscriptDelta), + "response.output_audio_transcript.done": (_models.RealtimeServerEventResponseAudioTranscriptDone), + "response.output_item.added": _models.RealtimeServerEventResponseOutputItemAdded, + "response.output_item.done": _models.RealtimeServerEventResponseOutputItemDone, + "response.output_text.delta": _models.RealtimeServerEventResponseTextDelta, + "response.output_text.done": _models.RealtimeServerEventResponseTextDone, + "response.video.delta": _models.VoiceAgentServerEventResponseVideoDelta, + "rtc.call.error": _models.VoiceAgentServerEventRtcCallError, + "rtc.call.sdp.created": _models.VoiceAgentServerEventRtcCallSdpCreated, + "session.avatar.connecting": _models.VoiceAgentServerEventSessionAvatarConnecting, + "session.avatar.switch_to_idle": _models.VoiceAgentServerEventSessionAvatarSwitchToIdle, + "session.avatar.switch_to_speaking": _models.VoiceAgentServerEventSessionAvatarSwitchToSpeaking, + "session.created": _models.RealtimeServerEventSessionCreated, + "session.subagent.aborted": _models.VoiceAgentServerEventSessionSubagentAborted, + "session.subagent.completed": _models.VoiceAgentServerEventSessionSubagentCompleted, + "session.subagent.started": _models.VoiceAgentServerEventSessionSubagentStarted, + "session.updated": _models.RealtimeServerEventSessionUpdated, + "warning": _models.VoiceAgentServerEventWarning, +} + +# Every generated server event model, for consumers that want a precise return type. +ServerEvent = Union[ + _models.RealtimeServerEventError, + _models.RealtimeServerEventResponseContentPartAdded, + _models.RealtimeServerEventConversationItemAdded, + _models.RealtimeServerEventConversationItemCreated, + _models.RealtimeServerEventConversationItemDeleted, + _models.RealtimeServerEventConversationItemDone, + _models.RealtimeServerEventConversationItemInputAudioTranscriptionCompleted, + _models.RealtimeServerEventConversationItemInputAudioTranscriptionDelta, + _models.RealtimeServerEventConversationItemInputAudioTranscriptionFailed, + _models.RealtimeServerEventConversationItemInputAudioTranscriptionSegment, + _models.RealtimeServerEventConversationItemRetrieved, + _models.RealtimeServerEventConversationItemTruncated, + _models.RealtimeServerEventInputAudioBufferCleared, + _models.RealtimeServerEventInputAudioBufferCommitted, + _models.RealtimeServerEventInputAudioBufferSpeechStarted, + _models.RealtimeServerEventInputAudioBufferSpeechStopped, + _models.RealtimeServerEventInputAudioBufferTimeoutTriggered, + _models.RealtimeServerEventMCPListToolsCompleted, + _models.RealtimeServerEventMCPListToolsFailed, + _models.RealtimeServerEventMCPListToolsInProgress, + _models.RealtimeServerEventOutputAudioBufferCleared, + _models.RealtimeServerEventRateLimitsUpdated, + _models.VoiceAgentServerEventResponseAnimationBlendshapesDelta, + _models.VoiceAgentServerEventResponseAnimationBlendshapesDone, + _models.VoiceAgentServerEventResponseAnimationVisemeDelta, + _models.VoiceAgentServerEventResponseAnimationVisemeDone, + _models.RealtimeServerEventResponseAudioDelta, + _models.RealtimeServerEventResponseAudioDone, + _models.VoiceAgentServerEventResponseAudioTimestampDelta, + _models.VoiceAgentServerEventResponseAudioTimestampDone, + _models.RealtimeServerEventResponseAudioTranscriptDelta, + _models.RealtimeServerEventResponseAudioTranscriptDone, + _models.RealtimeServerEventResponseContentPartDone, + _models.RealtimeServerEventResponseCreated, + _models.RealtimeServerEventResponseDone, + _models.RealtimeServerEventResponseFunctionCallArgumentsDelta, + _models.RealtimeServerEventResponseFunctionCallArgumentsDone, + _models.RealtimeServerEventResponseMCPCallArgumentsDelta, + _models.RealtimeServerEventResponseMCPCallArgumentsDone, + _models.RealtimeServerEventResponseMCPCallCompleted, + _models.RealtimeServerEventResponseMCPCallFailed, + _models.RealtimeServerEventResponseMCPCallInProgress, + _models.RealtimeServerEventResponseOutputItemAdded, + _models.RealtimeServerEventResponseOutputItemDone, + _models.RealtimeServerEventResponseTextDelta, + _models.RealtimeServerEventResponseTextDone, + _models.VoiceAgentServerEventResponseVideoDelta, + _models.VoiceAgentServerEventRtcCallError, + _models.VoiceAgentServerEventRtcCallSdpCreated, + _models.VoiceAgentServerEventSessionAvatarConnecting, + _models.VoiceAgentServerEventSessionAvatarSwitchToIdle, + _models.VoiceAgentServerEventSessionAvatarSwitchToSpeaking, + _models.RealtimeServerEventSessionCreated, + _models.VoiceAgentServerEventSessionSubagentAborted, + _models.VoiceAgentServerEventSessionSubagentCompleted, + _models.VoiceAgentServerEventSessionSubagentStarted, + _models.RealtimeServerEventSessionUpdated, + _models.VoiceAgentServerEventWarning, + Mapping[str, Any], +] + + +def _to_ws_url(endpoint: str, agent_name: str) -> str: + """Build the realtime WebSocket URL from the HTTPS project endpoint. + + Only the ``https://`` scheme is translated (to ``wss://``); any other scheme is left + unchanged so that :meth:`AsyncRealtimeConnectionManager.enter`'s ``wss://``-only check + rejects it with a clear error instead of silently producing an unencrypted ``ws://`` URL + that would also send the live Authorization token in plain text. + + :param str endpoint: The Foundry project endpoint (``https://.../api/projects/...``). + :param str agent_name: The name of the voice agent to connect to. + :return: A ``wss://`` URL targeting the realtime route. + :rtype: str + """ + base = endpoint.rstrip("/") + if base.startswith("https://"): + base = "wss://" + base[len("https://") :] + return f"{base}/agents/{quote(agent_name, safe='')}/endpoint/protocols/voice" + + +_DEFAULT_PORT_BY_SCHEME = {"http": 80, "https": 443, "ws": 80, "wss": 443} + + +def _normalized_authority(url: str) -> Tuple[str, Optional[int]]: + """Return a ``(hostname, port)`` tuple with the scheme's default port filled in. + + ``urlparse(...).port`` is ``None`` when a URL omits an explicit port, which would make + ``https://host/...`` and ``https://host:8443/...`` compare as equal on hostname alone. + Resolving the scheme's default port here lets callers compare authorities (not just + hostnames) so a same-host override on a different, non-default port is correctly rejected. + + :param str url: The URL to parse. + :return: A tuple of the lower-cased hostname (or empty string) and the resolved port + (or ``None`` if the scheme has no known default and none was specified). + :rtype: tuple[str, Optional[int]] + """ + parsed = urlparse(url) + port = parsed.port + if port is None: + port = _DEFAULT_PORT_BY_SCHEME.get((parsed.scheme or "").lower()) + return (parsed.hostname or "").lower(), port + + +def _assert_trusted_connection_url(connection_url: str, endpoint: str) -> None: + """Guard against attaching the caller's Entra bearer token to an untrusted host. + + ``connection_url`` is an escape hatch that lets a caller override the computed + scheme/host/path, but the Authorization header carrying the live credential's + token must never be sent to a host other than the configured Foundry project + endpoint: a caller-controlled or compromised URL could otherwise be used to + exfiltrate the token to an arbitrary host or port. + + :param str connection_url: The caller-supplied override URL. + :param str endpoint: The configured, trusted Foundry project endpoint. + :raises ValueError: If the override URL's host or port does not match the endpoint's. + """ + override_host, override_port = _normalized_authority(connection_url) + trusted_host, trusted_port = _normalized_authority(endpoint) + if not override_host or (override_host, override_port) != (trusted_host, trusted_port): + got = override_host or connection_url + if override_host and override_port: + got = f"{override_host}:{override_port}" + raise ValueError( + "The 'connection_url' override must target the same host and port as the configured " + f"Foundry project endpoint ('{trusted_host}:{trusted_port}') to avoid sending the " + f"Authorization token to an untrusted host; got '{got}'." + ) + + +class _BaseResource: # pylint: disable=too-few-public-methods + """Base helper that forwards typed helpers to the parent connection.""" + + def __init__(self, connection: "AsyncRealtimeConnection") -> None: + self._connection = connection + + async def _send(self, event: ClientEvent) -> None: + await self._connection.send(event) + + +class SessionResource(_BaseResource): + """Send ``session.*`` client events.""" + + async def update( + self, + *, + session: Union["_models.VoiceAgentSessionUpdateConfig", Mapping[str, Any]], + event_id: Optional[str] = None, + ) -> None: + """Update the realtime session configuration. + + :keyword session: The session configuration to apply. + :paramtype session: ~azure.ai.projects.models.VoiceAgentSessionUpdateConfig or + Mapping[str, Any] + :keyword event_id: Optional client-generated ID used to identify this event. + :paramtype event_id: str or None + """ + await self._send( + cast(Any, _models.VoiceAgentClientEventSessionUpdate)( + type=_models.RealtimeClientEventType.SESSION_UPDATE, + session=session, + event_id=event_id, + ) + ) + + async def avatar_connect(self, *, client_sdp: str, event_id: Optional[str] = None) -> None: + """Negotiate an avatar media session over WebRTC. + + :keyword str client_sdp: The client's SDP offer for avatar media negotiation. + :keyword event_id: An optional client-generated event identifier. + :paramtype event_id: str or None + """ + await self._send( + _models.VoiceAgentClientEventSessionAvatarConnect( + client_sdp=client_sdp, + event_id=event_id, + ) + ) + + +class InputAudioBufferResource(_BaseResource): + """Send ``input_audio_buffer.*`` client events.""" + + async def append(self, *, audio: Union[str, bytes], event_id: Optional[str] = None) -> None: + """Append audio bytes to the input buffer. + + :keyword audio: Raw audio bytes, or an already base64-encoded string. + :paramtype audio: str or bytes + :keyword event_id: Optional client-generated ID used to identify this event. + :paramtype event_id: str or None + """ + if isinstance(audio, (bytes, bytearray)): + audio = base64.b64encode(bytes(audio)).decode("ascii") + await self._send( + _models.RealtimeClientEventInputAudioBufferAppend( + audio=audio, + event_id=event_id, + ) + ) + + async def commit(self, *, event_id: Optional[str] = None) -> None: + """Commit the buffered input audio as a user turn. + + :keyword event_id: Optional client-generated ID used to identify this event. + :paramtype event_id: str or None + """ + await self._send(_models.RealtimeClientEventInputAudioBufferCommit(event_id=event_id)) + + async def clear(self, *, event_id: Optional[str] = None) -> None: + """Discard any buffered input audio. + + :keyword event_id: Optional client-generated ID used to identify this event. + :paramtype event_id: str or None + """ + await self._send(_models.RealtimeClientEventInputAudioBufferClear(event_id=event_id)) + + +class OutputAudioBufferResource(_BaseResource): # pylint: disable=too-few-public-methods + """Send ``output_audio_buffer.*`` client events.""" + + async def clear(self, *, event_id: Optional[str] = None) -> None: + """Stop and clear any audio the service is currently playing back (barge-in). + + :keyword event_id: Optional client-generated ID used to identify this event. + :paramtype event_id: str or None + """ + await self._send(_models.RealtimeClientEventOutputAudioBufferClear(event_id=event_id)) + + +class ConversationItemResource(_BaseResource): + """Send ``conversation.item.*`` client events.""" + + async def create( + self, + *, + item: ConversationItem, + previous_item_id: Optional[str] = None, + event_id: Optional[str] = None, + ) -> None: + """Insert an item into the conversation. + + :keyword item: The conversation item to create. + :paramtype item: ~azure.ai.projects.models.RealtimeConversationItemMessageSystem or + ~azure.ai.projects.models.RealtimeConversationItemMessageUser or + ~azure.ai.projects.models.RealtimeConversationItemMessageAssistant or + ~azure.ai.projects.models.RealtimeConversationItemFunctionCall or + ~azure.ai.projects.models.RealtimeConversationItemFunctionCallOutput or + ~azure.ai.projects.models.RealtimeMCPApprovalResponse or Mapping[str, Any] + :keyword previous_item_id: The ID of the preceding item after which the new item will be + inserted. Default value is None. + :paramtype previous_item_id: str or None + :keyword event_id: Optional client-generated ID used to identify this event. + :paramtype event_id: str or None + """ + await self._send( + cast(Any, _models.RealtimeClientEventConversationItemCreate)( + item=item, + previous_item_id=previous_item_id, + event_id=event_id, + ) + ) + + async def delete(self, *, item_id: str, event_id: Optional[str] = None) -> None: + """Delete an item from the conversation. + + :keyword str item_id: The ID of the item to delete. + :keyword event_id: Optional client-generated ID used to identify this event. + :paramtype event_id: str or None + """ + await self._send( + _models.RealtimeClientEventConversationItemDelete( + item_id=item_id, + event_id=event_id, + ) + ) + + async def retrieve(self, *, item_id: str, event_id: Optional[str] = None) -> None: + """Ask the server to emit a ``conversation.item.retrieved`` event for an item. + + :keyword str item_id: The ID of the item to retrieve. + :keyword event_id: Optional client-generated ID used to identify this event. + :paramtype event_id: str or None + """ + await self._send( + _models.RealtimeClientEventConversationItemRetrieve( + item_id=item_id, + event_id=event_id, + ) + ) + + async def truncate( + self, *, item_id: str, content_index: int, audio_end_ms: int, event_id: Optional[str] = None + ) -> None: + """Truncate a previously produced assistant audio item (used for barge-in). + + :keyword str item_id: The ID of the assistant message item to truncate. + :keyword int content_index: The index of the content part to truncate. Use ``0``. + :keyword int audio_end_ms: The point, in milliseconds, to truncate the audio to. + :keyword event_id: Optional client-generated ID used to identify this event. + :paramtype event_id: str or None + """ + await self._send( + _models.RealtimeClientEventConversationItemTruncate( + item_id=item_id, + content_index=content_index, + audio_end_ms=audio_end_ms, + event_id=event_id, + ) + ) + + +class ConversationResource(_BaseResource): # pylint: disable=too-few-public-methods + """Send ``conversation.*`` client events.""" + + def __init__(self, connection: "AsyncRealtimeConnection") -> None: + super().__init__(connection) + self.item: ConversationItemResource = ConversationItemResource(connection) + + +class ResponseResource(_BaseResource): + """Send ``response.*`` client events.""" + + async def create( + self, + *, + response: Optional[Union["_models.VoiceAgentResponseCreateParams", Mapping[str, Any]]] = None, + event_id: Optional[str] = None, + ) -> None: + """Ask the model to generate a response. + + :keyword response: Optional per-response overrides. Default value is None. + :paramtype response: ~azure.ai.projects.models.VoiceAgentResponseCreateParams or + Mapping[str, Any] or None + :keyword event_id: Optional client-generated ID used to identify this event. + :paramtype event_id: str or None + """ + await self._send( + cast(Any, _models.RealtimeClientEventResponseCreate)( + response=response, + event_id=event_id, + ) + ) + + async def cancel(self, *, response_id: Optional[str] = None, event_id: Optional[str] = None) -> None: + """Cancel an in-progress response. + + :keyword response_id: The ID of the response to cancel, if targeting a specific one. + Default value is None. + :paramtype response_id: str or None + :keyword event_id: Optional client-generated ID used to identify this event. + :paramtype event_id: str or None + """ + await self._send( + _models.RealtimeClientEventResponseCancel( + response_id=response_id, + event_id=event_id, + ) + ) + + +class AsyncRealtimeConnection: # pylint: disable=too-many-instance-attributes + """An open realtime WebSocket connection to a voice agent. + + Iterate over the connection to receive strongly-typed server events, and use the + sub-namespaces to send strongly-typed client events:: + + async with client.beta.realtime.connect(agent_name="my-agent") as conn: + await conn.input_audio_buffer.append(audio=chunk) + await conn.input_audio_buffer.commit() + await conn.response.create() + async for event in conn: + if event.type == RealtimeServerEventType.RESPONSE_DONE: + break + """ + + def __init__(self, connection: "ClientWebSocketResponse", session: "ClientSession") -> None: + self._connection = connection + self._session = session + self.session: SessionResource = SessionResource(self) + self.input_audio_buffer: InputAudioBufferResource = InputAudioBufferResource(self) + self.output_audio_buffer: OutputAudioBufferResource = OutputAudioBufferResource(self) + self.conversation: ConversationResource = ConversationResource(self) + self.response: ResponseResource = ResponseResource(self) + + async def __aenter__(self) -> "AsyncRealtimeConnection": + return self + + async def __aexit__(self, *exc_details: Any) -> None: + await self.close() + + def __repr__(self) -> str: + state = "closed" if self.closed else "open" + return f"" + + @property + def closed(self) -> bool: + """Whether the underlying WebSocket connection has been closed. + + :rtype: bool + """ + return self._connection.closed + + def __aiter__(self) -> AsyncIterator[ServerEvent]: + return self._iter() + + async def _iter(self) -> AsyncIterator[ServerEvent]: + while True: + try: + yield await self.recv() + except ConnectionResetError: + return + + async def recv(self) -> ServerEvent: + """Receive and parse the next server event. + + Known event types are returned as their strongly-typed + ``VoiceAgentServerEventXxx`` model. Event types not (yet) represented by a + generated model are returned as a plain ``dict`` for forward compatibility. + + :return: The parsed server event. + :rtype: ~azure.ai.projects.aio.ServerEvent + :raises ConnectionResetError: If the connection was closed by the server. + """ + import aiohttp # pylint: disable=import-outside-toplevel + + msg = await self._connection.receive() + while msg.type in (aiohttp.WSMsgType.PING, aiohttp.WSMsgType.PONG): + msg = await self._connection.receive() + if msg.type in ( + aiohttp.WSMsgType.CLOSE, + aiohttp.WSMsgType.CLOSING, + aiohttp.WSMsgType.CLOSED, + ): + raise ConnectionResetError("The realtime connection was closed.") + if msg.type == aiohttp.WSMsgType.ERROR: + raise ConnectionResetError( + "The realtime connection encountered an error." + ) from self._connection.exception() + raw = msg.data.decode("utf-8") if msg.type == aiohttp.WSMsgType.BINARY else msg.data + payload: Dict[str, Any] = json.loads(raw) + event_type = payload.get("type") + if not isinstance(event_type, str): + return payload + event_cls = _SERVER_EVENT_TYPES.get(event_type) + if event_cls is None: + return payload + return event_cls(payload) + + async def send(self, event: ClientEvent) -> None: + """Send a client event over the connection. + + :param event: A strongly-typed client event, a ready-made mapping, or a raw JSON string. + :type event: ~azure.ai.projects.aio.ClientEvent or str + :raises ValueError: If ``event`` is a ``str`` that is not valid JSON. + """ + if isinstance(event, str): + try: + json.loads(event) + except ValueError as exc: + raise ValueError(f"'event' is not valid JSON: {exc}") from exc + payload = event + else: + payload = json.dumps(event, cls=SdkJSONEncoder) + await self._connection.send_str(payload) + + async def close(self, *, code: int = 1000, reason: str = "") -> None: + """Close the connection and release the underlying HTTP session. + + :keyword int code: The WebSocket close code. + :keyword str reason: The close reason. + """ + try: + await self._connection.close(code=code, message=reason.encode("utf-8")) + finally: + await self._session.close() + + +class AsyncRealtimeConnectionManager: # pylint: disable=too-many-instance-attributes + """Async context manager that opens an :class:`AsyncRealtimeConnection`. + + Returned by :meth:`AsyncRealtime.connect`; you normally use it as + ``async with client.beta.realtime.connect(...) as conn:``. + """ + + def __init__( # pylint: disable=too-many-arguments + self, + *, + endpoint: str, + credential: "AsyncTokenCredential", + credential_scopes: List[str], + api_version: str, + agent_name: str, + foundry_features: str, + agent_session_id: Optional[str] = None, + agent_version_override: Optional[str] = None, + structured_inputs: Optional[str] = None, + connection_url: Optional[str] = None, + extra_query: Optional[Mapping[str, str]] = None, + extra_headers: Optional[Mapping[str, str]] = None, + **kwargs: Any, + ) -> None: + self._endpoint = endpoint + self._credential = credential + self._credential_scopes = credential_scopes + self._api_version = api_version + self._agent_name = agent_name + self._foundry_features = foundry_features + self._agent_session_id = agent_session_id + self._agent_version_override = agent_version_override + self._structured_inputs = structured_inputs + self._connection_url = connection_url + self._extra_query = dict(extra_query or {}) + self._extra_headers = dict(extra_headers or {}) + self._kwargs = kwargs + self._connection: Optional[AsyncRealtimeConnection] = None + + async def __aenter__(self) -> AsyncRealtimeConnection: + return await self.enter() + + async def enter(self) -> AsyncRealtimeConnection: # pylint: disable=too-many-locals + """Open the connection. + + :return: The live realtime connection. + :rtype: ~azure.ai.projects.aio.AsyncRealtimeConnection + :raises RuntimeError: If ``aiohttp`` is not installed. + :raises ValueError: If the computed or supplied WebSocket URL does not use ``wss://``. + :raises ConnectionError: If the WebSocket upgrade handshake fails (for example, a + network error, DNS failure, or a non-101 response from the service). + """ + try: + import aiohttp # pylint: disable=import-outside-toplevel + except ImportError as exc: # pragma: no cover - dependency guard + raise RuntimeError( + "The realtime client requires `aiohttp`. Install it with `pip install aiohttp`." + ) from exc + + # ``connection_url`` fully overrides the computed route (scheme/host/path). This is the + # escape hatch used to reach a specific data-plane host/path directly. + if self._connection_url is not None: + _assert_trusted_connection_url(self._connection_url, self._endpoint) + url = self._connection_url or _to_ws_url(self._endpoint, self._agent_name) + if not url.startswith("wss://"): + raise ValueError("The realtime WebSocket URL must use wss:// to protect credentials in transit.") + + params: Dict[str, str] = {"api-version": self._api_version, "x-ms-client-sdk": _USER_AGENT} + if self._agent_session_id is not None: + params["agent_session_id"] = self._agent_session_id + if self._agent_version_override is not None: + params["x-agent-version-override"] = self._agent_version_override + params.update(self._extra_query) + + token = await self._credential.get_token(*self._credential_scopes) + headers: Dict[str, str] = { + "Authorization": "Bearer " + token.token, + _FOUNDRY_FEATURES_HEADER_NAME: self._foundry_features, + } + if self._structured_inputs is not None: + headers["x-ms-voice-structured-inputs"] = self._structured_inputs + headers.update(self._extra_headers) + if not _has_header_case_insensitive(headers, "User-Agent"): + # Only set our default if the caller didn't supply their own (in any casing) -- + # a plain dict merge would otherwise leave both as separate keys (HTTP header names + # are case-insensitive, but Python dict keys are not), sending two User-Agent-like + # headers instead of cleanly honoring the caller's override. + headers["User-Agent"] = _USER_AGENT + + session = aiohttp.ClientSession() + try: + # Force the "realtime" WebSocket subprotocol regardless of any caller-supplied + # override in ``self._kwargs``: the service requires this exact subprotocol, so + # silently accepting a different one here would just move the failure to a less + # clear error inside aiohttp's handshake. + ws_connect_kwargs = dict(self._kwargs) + ws_connect_kwargs.pop("protocols", None) + connection = await session.ws_connect( + url, headers=headers, params=params, protocols=("realtime",), **ws_connect_kwargs + ) + except BaseException as exc: + await session.close() + if not isinstance(exc, Exception) or isinstance(exc, (ValueError, RuntimeError)): + raise + raise ConnectionError( + f"Failed to open the realtime WebSocket connection to voice agent " + f"'{self._agent_name}' at '{url}': {exc}" + ) from exc + self._connection = AsyncRealtimeConnection(cast("ClientWebSocketResponse", connection), session) + return self._connection + + async def __aexit__(self, *exc_details: Any) -> None: + if self._connection is not None: + await self._connection.close() + self._connection = None + + +class AsyncRealtime: # pylint: disable=too-few-public-methods + """Realtime streaming entry point, exposed as ``client.beta.realtime``. + + Follows the OpenAI Python realtime surface: obtain it from the HTTP client and open a + connection with :meth:`connect`:: + + from azure.ai.projects.aio import AIProjectClient + from azure.identity.aio import DefaultAzureCredential + + client = AIProjectClient(endpoint, DefaultAzureCredential()) + async with client.beta.realtime.connect(agent_name="my-agent") as conn: + await conn.input_audio_buffer.append(audio=chunk) + await conn.input_audio_buffer.commit() + await conn.response.create() + async for event in conn: + if event.type == RealtimeServerEventType.RESPONSE_DONE: + break + + :param client: The object whose endpoint and credential are reused for the realtime + handshake -- either the top-level client or its ``.beta`` sub-client, since both share the + same underlying configuration. + :type client: ~azure.ai.projects.aio.AIProjectClient or ~azure.ai.projects.aio.operations.BetaOperations + """ + + def __init__(self, client: "_ConfigProvider") -> None: + self._config = client._config # pylint: disable=protected-access + + def connect( # pylint: disable=too-many-arguments + self, + *, + agent_name: str, + foundry_features: str = _VOICE_AGENT_FEATURE_HEADER, + agent_session_id: Optional[str] = None, + agent_version_override: Optional[str] = None, + structured_inputs: Optional[str] = None, + connection_url: Optional[str] = None, + api_version: Optional[str] = None, + credential_scopes: Optional[List[str]] = None, + extra_query: Optional[Mapping[str, str]] = None, + extra_headers: Optional[Mapping[str, str]] = None, + **kwargs: Any, + ) -> AsyncRealtimeConnectionManager: + """Open a realtime WebSocket connection to a voice agent. + + :keyword str agent_name: The name of the voice agent to connect to. + :keyword foundry_features: Preview opt-in value(s) for the ``Foundry-Features`` header. + Defaults to ``VoiceAgents=V1Preview``. Pass a comma-separated value to opt in to + additional preview features on the same request. + :paramtype foundry_features: str + :keyword agent_session_id: An optional identifier used to correlate the voice session. + Default value is None. + :paramtype agent_session_id: str or None + :keyword agent_version_override: Selects a specific version of the voice agent for this + session. Default value is None. + :paramtype agent_version_override: str or None + :keyword structured_inputs: A JSON object that maps structured-input names to their + values for this session. Default value is None. + :paramtype structured_inputs: str or None + :keyword connection_url: Full ``wss://`` URL that overrides the route computed + from the client endpoint. Query parameters are still appended. Default value is None. + :paramtype connection_url: str or None + :keyword api_version: Overrides the client's API version for the handshake. Default + value is None. + :paramtype api_version: str or None + :keyword credential_scopes: Overrides the client's token scopes for the handshake. + Default value is None. + :paramtype credential_scopes: list[str] or None + :keyword extra_query: Additional query-string parameters for the handshake. + :paramtype extra_query: Mapping[str, str] or None + :keyword extra_headers: Additional headers for the handshake. + :paramtype extra_headers: Mapping[str, str] or None + :return: An async context manager yielding an :class:`AsyncRealtimeConnection`. + :rtype: ~azure.ai.projects.aio.AsyncRealtimeConnectionManager + """ + return AsyncRealtimeConnectionManager( + endpoint=self._config.endpoint, + credential=self._config.credential, + credential_scopes=credential_scopes or self._config.credential_scopes, + api_version=api_version or self._config.api_version, + agent_name=agent_name, + foundry_features=foundry_features, + agent_session_id=agent_session_id, + agent_version_override=agent_version_override, + structured_inputs=structured_inputs, + connection_url=connection_url, + extra_query=extra_query, + extra_headers=extra_headers, + **kwargs, + ) diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/aio/operations/_patch.py b/sdk/ai/azure-ai-projects/azure/ai/projects/aio/operations/_patch.py index 9cfdb2d19823..408a29a98d68 100644 --- a/sdk/ai/azure-ai-projects/azure/ai/projects/aio/operations/_patch.py +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/aio/operations/_patch.py @@ -8,7 +8,7 @@ Follow our quickstart for examples: https://aka.ms/azsdk/python/dpcodegen/python/customize """ -from typing import Any, List +from typing import Any, List, Optional from ._patch_agents_async import AgentsOperations, BetaAgentsOperations from ._patch_agent_insights_async import BetaAgentInsightMonitorsOperations from ._patch_datasets_async import BetaDatasetsOperations, DatasetsOperations @@ -19,6 +19,14 @@ from ._patch_memories_async import BetaMemoryStoresOperations from ._patch_models_async import BetaModelsOperations from ...operations._patch import _BETA_OPERATION_FEATURE_HEADERS, _OperationMethodHeaderProxy +from .._realtime import ( + AsyncRealtime, + AsyncRealtimeConnection, + AsyncRealtimeConnectionManager, + ClientEvent, + ConversationItem, + ServerEvent, +) from ._operations import ( BetaAgentEndpointConversationsOperations, BetaAgentTelephonyOperations, @@ -90,6 +98,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: self.agent_insight_monitors = BetaAgentInsightMonitorsOperations( self._client, self._config, self._serialize, self._deserialize ) + self._realtime: Optional[AsyncRealtime] = None for property_name, foundry_features_value in _BETA_OPERATION_FEATURE_HEADERS.items(): setattr( @@ -98,9 +107,23 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: _OperationMethodHeaderProxy(getattr(self, property_name), foundry_features_value), ) + @property + def realtime(self) -> AsyncRealtime: + """Realtime streaming entry point for voice agents. + + :return: The realtime namespace, exposing ``connect(...)``. + :rtype: ~azure.ai.projects.aio.operations.AsyncRealtime + """ + if self._realtime is None: + self._realtime = AsyncRealtime(self) + return self._realtime + __all__: List[str] = [ "AgentsOperations", + "AsyncRealtime", + "AsyncRealtimeConnection", + "AsyncRealtimeConnectionManager", "BetaAgentEndpointConversationsOperations", "BetaAgentInsightMonitorsOperations", "BetaAgentTelephonyOperations", @@ -117,9 +140,12 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: "BetaSchedulesOperations", "BetaSkillsOperations", "BetaVoiceAgentWebSocketOperations", + "ClientEvent", "ConnectionsOperations", + "ConversationItem", "DatasetsOperations", "EvaluationRulesOperations", + "ServerEvent", "TelemetryOperations", ] # Add all objects you want publicly available to users at this package level diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/operations/_patch.py b/sdk/ai/azure-ai-projects/azure/ai/projects/operations/_patch.py index 405d40c98f95..cc72b141e846 100644 --- a/sdk/ai/azure-ai-projects/azure/ai/projects/operations/_patch.py +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/operations/_patch.py @@ -10,8 +10,16 @@ from functools import wraps import inspect -from typing import Any, Callable, List +from typing import Any, Callable, List, Optional from ..models._patch import _FOUNDRY_FEATURES_HEADER_NAME, _BETA_OPERATION_FEATURE_HEADERS, _has_header_case_insensitive +from .._realtime import ( + Realtime, + RealtimeConnection, + RealtimeConnectionManager, + ClientEvent, + ConversationItem, + ServerEvent, +) from ._patch_agents import AgentsOperations, BetaAgentsOperations from ._patch_agent_insights import BetaAgentInsightMonitorsOperations from ._patch_datasets import BetaDatasetsOperations, DatasetsOperations @@ -145,6 +153,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: self.agent_insight_monitors = BetaAgentInsightMonitorsOperations( self._client, self._config, self._serialize, self._deserialize ) + self._realtime: Optional[Realtime] = None for property_name, foundry_features_value in _BETA_OPERATION_FEATURE_HEADERS.items(): setattr( @@ -153,6 +162,17 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: _OperationMethodHeaderProxy(getattr(self, property_name), foundry_features_value), ) + @property + def realtime(self) -> Realtime: + """Realtime streaming entry point for voice agents. + + :return: The realtime namespace, exposing ``connect(...)``. + :rtype: ~azure.ai.projects.operations.Realtime + """ + if self._realtime is None: + self._realtime = Realtime(self) + return self._realtime + __all__: List[str] = [ "AgentsOperations", @@ -172,9 +192,15 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: "BetaSchedulesOperations", "BetaSkillsOperations", "BetaVoiceAgentWebSocketOperations", + "ClientEvent", "ConnectionsOperations", + "ConversationItem", "DatasetsOperations", "EvaluationRulesOperations", + "Realtime", + "RealtimeConnection", + "RealtimeConnectionManager", + "ServerEvent", "TelemetryOperations", ] # Add all objects you want publicly available to users at this package level diff --git a/sdk/ai/azure-ai-projects/dev_requirements.txt b/sdk/ai/azure-ai-projects/dev_requirements.txt index 6641c1e8f14a..a8928e8a7c9b 100644 --- a/sdk/ai/azure-ai-projects/dev_requirements.txt +++ b/sdk/ai/azure-ai-projects/dev_requirements.txt @@ -14,6 +14,7 @@ azure-monitor-query jsonref opentelemetry-sdk python-dotenv +websockets>=13.0 black # Can't include those, because they are not supported in Python 3.9. Samples that use these package # cannot be run as pytest, because the pipeline will fail on Python 3.9 jobs. diff --git a/sdk/ai/azure-ai-projects/pyproject.toml b/sdk/ai/azure-ai-projects/pyproject.toml index dea352a19763..494cef1bf4ba 100644 --- a/sdk/ai/azure-ai-projects/pyproject.toml +++ b/sdk/ai/azure-ai-projects/pyproject.toml @@ -42,6 +42,12 @@ dynamic = [ "version", "readme" ] +[project.optional-dependencies] +realtime = [ + "websockets>=13.0", + "aiohttp>=3.9.0,<4.0.0", +] + [project.urls] repository = "https://aka.ms/azsdk/azure-ai-projects-v2/python/code" diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_basic.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_basic.py new file mode 100644 index 000000000000..f451b8a57287 --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_basic.py @@ -0,0 +1,107 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + This sample demonstrates the voice-agent management lifecycle using the + unified Agents API in the Microsoft Foundry Python SDK (azure-ai-projects): + creating a voice agent (with an audio/voice configuration and conversation + storage enabled), retrieving it, listing the voice agents in the project, + creating a new version, disabling/enabling it, and deleting it. + + Voice agents are exposed through `project_client.agents` with + `kind="voice"`, the same surface used for prompt, workflow, hosted, and + external agents. + +USAGE: + python sample_voice_agent_basic.py + + Before running the sample: + + pip install "azure-ai-projects>=2.7.0b1" python-dotenv --pre + + Set these environment variables with your own values: + 1) FOUNDRY_PROJECT_ENDPOINT - The Azure AI Project endpoint, as found in the Overview + page of your Microsoft Foundry portal. + 2) FOUNDRY_VOICE_MODEL - Optional. The realtime model deployment name. + Defaults to "gpt-realtime". + 3) FOUNDRY_VOICE_AGENT_NAME - Optional. The name of the voice agent. If not + set, defaults to "MyVoiceAgent". +""" + +import os +from dotenv import load_dotenv +from azure.identity import DefaultAzureCredential +from azure.ai.projects import AIProjectClient +from azure.ai.projects.models import ( + AgentKind, + VoiceAgentDefinition, + VoiceAgentAudioConfig, + VoiceAgentAudioOutputConfig, + VoiceModelType, + VoiceOutputModality, + VoiceType, +) + +load_dotenv() + +endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] +model = os.environ.get("FOUNDRY_VOICE_MODEL") or "gpt-realtime" +agent_name = os.environ.get("FOUNDRY_VOICE_AGENT_NAME") or "MyVoiceAgent" + +with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as project_client, +): + try: + definition = VoiceAgentDefinition( + # `managed` uses a service-hosted model; use `self_deployed` with a Foundry + # deployment name to bring your own model. + model_type=VoiceModelType.MANAGED, + model=model, + instructions="You are a friendly voice assistant. Keep replies short and natural.", + audio=VoiceAgentAudioConfig( + output=VoiceAgentAudioOutputConfig(voice="en-US-AvaNeural", voice_type=VoiceType.AZURE_STANDARD), + ), + output_modalities=[VoiceOutputModality.AUDIO], + # Persist conversations so the transcript and audio can be read back later + # (see sample_voice_agent_read_conversation.py). Defaults to False, which stores nothing. + store=True, + ) + + created_version = project_client.agents.create_version(agent_name=agent_name, definition=definition) + print(f"Created voice agent '{agent_name}', version: {created_version.version}") + + agent = project_client.agents.get(agent_name=agent_name) + print(f"Retrieved voice agent: {agent.name} (state={agent.state})") + + print("Voice agents in this project:") + for item in project_client.agents.list(kind=AgentKind.VOICE): + print(f" - {item.name}") + + # Each update produces a new immutable version. + updated_version = project_client.agents.create_version( + agent_name=agent_name, + definition=VoiceAgentDefinition( + model_type=VoiceModelType.MANAGED, + model=model, + instructions="You are a friendly voice assistant. Always greet the caller warmly.", + audio=definition.audio, + output_modalities=definition.output_modalities, + store=definition.store, + ), + description="Updated instructions.", + ) + print(f"Updated voice agent to version: {updated_version.version}") + + # Disable the agent so its endpoint rejects new requests, then re-enable it. + project_client.agents.disable(agent_name=agent_name) + print("Disabled voice agent") + project_client.agents.enable(agent_name=agent_name) + print("Enabled voice agent") + finally: + project_client.agents.delete(agent_name=agent_name) + print(f"Deleted voice agent: {agent_name}") diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_basic_async.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_basic_async.py new file mode 100644 index 000000000000..a8b0e692cb0b --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_basic_async.py @@ -0,0 +1,73 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + This sample demonstrates the voice-agent management lifecycle using the + asynchronous AIProjectClient: creating a voice agent, retrieving it, + listing the voice agents in the project, and deleting it. + +USAGE: + python sample_voice_agent_basic_async.py + + Before running the sample: + + pip install "azure-ai-projects>=2.7.0b1" aiohttp python-dotenv --pre + + Set these environment variables with your own values: + 1) FOUNDRY_PROJECT_ENDPOINT - The Azure AI Project endpoint, as found in the Overview + page of your Microsoft Foundry portal. + 2) FOUNDRY_VOICE_MODEL - Optional. The realtime model deployment name. + Defaults to "gpt-realtime". + 3) FOUNDRY_VOICE_AGENT_NAME - Optional. The name of the voice agent. If not + set, defaults to "MyVoiceAgentAsync". +""" + +import asyncio +import os +from dotenv import load_dotenv +from azure.identity.aio import DefaultAzureCredential +from azure.ai.projects.aio import AIProjectClient +from azure.ai.projects.models import AgentKind, VoiceAgentDefinition, VoiceModelType + +load_dotenv() + + +async def main() -> None: + endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] + model = os.environ.get("FOUNDRY_VOICE_MODEL") or "gpt-realtime" + agent_name = os.environ.get("FOUNDRY_VOICE_AGENT_NAME") or "MyVoiceAgentAsync" + + async with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as project_client, + ): + try: + created_version = await project_client.agents.create_version( + agent_name=agent_name, + definition=VoiceAgentDefinition( + model_type=VoiceModelType.MANAGED, + model=model, + instructions="You are a friendly voice assistant. Keep replies short and natural.", + # Persist conversations so they can be read back later. Defaults to False. + store=True, + ), + ) + print(f"Created voice agent '{agent_name}', version: {created_version.version}") + + agent = await project_client.agents.get(agent_name=agent_name) + print(f"Retrieved voice agent: {agent.name}") + + print("Voice agents in this project:") + async for item in project_client.agents.list(kind=AgentKind.VOICE): + print(f" - {item.name}") + finally: + await project_client.agents.delete(agent_name=agent_name) + print(f"Deleted voice agent: {agent_name}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_generate.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_generate.py new file mode 100644 index 000000000000..79a55722aefb --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_generate.py @@ -0,0 +1,66 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + This sample demonstrates guided authoring: generating and creating a voice + agent through `POST /agents:generate` (`project_client.beta.agents.generate`) + with `kind="voice"`. The service creates a voice agent with a + service-selected starter definition, which is fully editable afterward + through the standard create_version/update flow. + +USAGE: + python sample_voice_agent_generate.py + + Before running the sample: + + pip install "azure-ai-projects>=2.7.0b1" python-dotenv --pre + + Set these environment variables with your own values: + 1) FOUNDRY_PROJECT_ENDPOINT - The Azure AI Project endpoint. + 2) FOUNDRY_VOICE_AGENT_NAME - Optional. The name of the voice agent. If not + set, defaults to "MyGeneratedVoiceAgent". +""" + +import os +import sys +from dotenv import load_dotenv +from azure.identity import DefaultAzureCredential +from azure.ai.projects import AIProjectClient +from azure.ai.projects.models import AgentKind, GenerateVoiceAgentRequest + +load_dotenv() + + +def _safe_print(text: str) -> None: + """Print text that may contain characters the current console can't display. + + The instructions below are model-generated and can contain characters (curly + quotes, em-dashes, etc.) outside some legacy, non-Unicode console encodings + (for example when stdout is piped/redirected on Windows). Rather than crashing + with UnicodeEncodeError, fall back to replacing just the unsupported characters; + a real interactive UTF-8 console prints unaffected. + """ + try: + print(text) + except UnicodeEncodeError: + encoding = sys.stdout.encoding or "ascii" + print(text.encode(encoding, errors="replace").decode(encoding)) + + +endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] +agent_name = os.environ.get("FOUNDRY_VOICE_AGENT_NAME") or "MyGeneratedVoiceAgent" + +with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as project_client, +): + agent = project_client.beta.agents.generate(GenerateVoiceAgentRequest(kind=AgentKind.VOICE, name=agent_name)) + print(f"Generated voice agent: {agent.name}") + _safe_print(f"Instructions:\n{agent.versions.latest.definition.instructions}") # type: ignore[attr-defined] + + project_client.agents.delete(agent_name=agent.name) + print(f"Deleted voice agent: {agent.name}") diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py new file mode 100644 index 000000000000..d47d43a482a8 --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py @@ -0,0 +1,469 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + End-to-end hands-free, bidirectional voice conversation using the + ``client.beta.realtime`` namespace added on top of the generated + azure-ai-projects client (see ``azure.ai.projects.aio.operations.AsyncRealtime``). + This mirrors the ergonomics of the OpenAI Python realtime client. + + 1. Generate a starter voice agent (see sample_voice_agent_generate.py), + then publish a version with `store=True` so the conversation can be + read back afterward. + 2. Stream live mic audio and let the agent's server-side VAD detect your + turns: your speech is transcribed, the agent replies through the + speakers, and talking over it barges in. + 3. Fetch the persisted conversation back by id. + 4. Delete the agent created for this sample. + + Capture and playback use non-blocking pyaudio callbacks; reply audio is + sequence-numbered so a barge-in can skip whatever is still queued. The + agent owns turn detection and noise suppression server-side. Use a headset + to avoid echo. + + Mic audio is sent as base64 PCM16; the reply arrives as typed + ``response.output_audio.*`` events, decoded to PCM16, mono, 24 kHz. + Requires ``aiohttp`` and ``pyaudio``. + + pip install "azure-ai-projects>=2.7.0b1" azure-identity aiohttp pyaudio --pre + +USAGE: + python sample_voice_agent_live_audio_conversation_async.py + + Environment variables: + 1) FOUNDRY_PROJECT_ENDPOINT (required) - Foundry project endpoint: + https://.services.ai.azure.com/api/projects/ + 2) FOUNDRY_VOICE_AGENT_NAME - Optional. Name for the agent created by this + sample. Defaults to "sample-live-audio-conversation-agent-async". + + Runs until you press Ctrl-C. Authenticates with DefaultAzureCredential, so + sign in first (e.g. `az login`). +""" + +import asyncio +import concurrent.futures +import os +import queue +import sys +from typing import Any, Final, Optional + +from dotenv import load_dotenv +from azure.core.exceptions import HttpResponseError +from azure.identity.aio import DefaultAzureCredential + +# AsyncRealtimeConnection is re-exported dynamically via aio/operations/_patch.py's `__all__`; +# pylint's static import resolution cannot trace that, but the symbol is valid (verified by +# Pyright/mypy). +from azure.ai.projects.aio.operations import AsyncRealtimeConnection # pylint: disable=no-name-in-module +from azure.ai.projects.aio import AIProjectClient +from azure.ai.projects.models import ( + AgentKind, + GenerateVoiceAgentRequest, + RealtimeServerEventConversationItemInputAudioTranscriptionCompleted, + RealtimeServerEventInputAudioBufferSpeechStarted, + RealtimeServerEventResponseAudioDelta, + RealtimeServerEventResponseAudioTranscriptDone, + RealtimeServerEventResponseCreated, + RealtimeServerEventResponseDone, + RealtimeServerEventSessionCreated, + RealtimeServerEventError, +) + +load_dotenv() + + +def _safe_print(text: str) -> None: + """Print text that may contain characters the current console can't display. + + The agent's replies below are model-generated and can contain characters (curly + quotes, em-dashes, etc.) outside some legacy, non-Unicode console encodings + (for example when stdout is piped/redirected on Windows). Rather than crashing + with UnicodeEncodeError, fall back to replacing just the unsupported characters; + a real interactive UTF-8 console prints unaffected. + """ + try: + print(text) + except UnicodeEncodeError: + encoding = sys.stdout.encoding or "ascii" + print(text.encode(encoding, errors="replace").decode(encoding)) + + +# Audio is streamed both ways as PCM16, mono, 24 kHz. +_SAMPLE_RATE: Final = 24000 + +# pyaudio callback buffer size (~50 ms of PCM16 audio per callback). +_CHUNK_SAMPLES: Final = 1200 + +try: + import pyaudio # type: ignore[import-not-found] +except ImportError: # pragma: no cover - required audio dependency + pyaudio: Any = None # type: ignore[no-redef] + + +def _format_size(num_bytes: int) -> str: + """Format a byte count as a human-readable string. + + :param num_bytes: The size in bytes. + :type num_bytes: int + :return: A string like "12345 bytes (12.1 KB)" or "2097152 bytes (2.00 MB)". + :rtype: str + """ + if num_bytes < 1024: + return f"{num_bytes} bytes" + if num_bytes < 1024 * 1024: + return f"{num_bytes} bytes ({num_bytes / 1024:.1f} KB)" + return f"{num_bytes} bytes ({num_bytes / (1024 * 1024):.2f} MB)" + + +class _AudioProcessor: # pylint: disable=too-many-instance-attributes + """Real-time mic capture and speaker playback via non-blocking pyaudio callbacks. + + * Capture appends each raw PCM16 frame to the input buffer (the realtime + client base64-encodes it). + * Playback pulls sequence-numbered PCM16 from a queue, always returning the + exact sample count pyaudio asked for (a wrong size corrupts audio). + * ``skip_pending_audio`` bumps a base sequence number so audio queued before + a barge-in is dropped, stopping playback the instant the user speaks. + """ + + def __init__(self, connection: "AsyncRealtimeConnection") -> None: + self._conn = connection + self._loop: Optional[asyncio.AbstractEventLoop] = None + self._audio = pyaudio.PyAudio() + + # Playback with sequence numbers for interrupt handling. + self._playback_queue: "queue.Queue[tuple[int, Optional[bytes]]]" = queue.Queue() + self._playback_base = 0 + self._next_seq = 0 + self._output_bytes = 0 + + # Bounds capture backpressure to a single in-flight send (see start_capture). + self._pending_send: "Optional[concurrent.futures.Future[None]]" = None + self._dropped_frames = 0 + # Only counts bytes actually handed to input_audio_buffer.append() -- frames dropped + # above due to backpressure are never sent, so they must not be counted here. + self._input_bytes = 0 + + self._input_stream = None + self._output_stream = None + + # -- capture ----------------------------------------------------------- + + def start_capture(self) -> None: + """Start streaming microphone audio to the service via a callback.""" + if self._input_stream is not None: + return + self._loop = asyncio.get_running_loop() + + def _capture_callback(in_data, _frame_count, _time_info, _status): + # Runs on a pyaudio thread: hand the frame to the event loop to append. Each call + # schedules a coroutine on the loop via a thread-safe handoff; if sending falls + # behind real-time capture (for example, network backpressure on the WebSocket), + # unconditionally scheduling a new one every callback would let pending sends + # accumulate without bound. Instead, only keep at most one in flight and drop + # (skip sending) this frame if the previous send hasn't completed yet. + assert self._loop is not None + if self._pending_send is not None and not self._pending_send.done(): + self._dropped_frames += 1 + return (None, pyaudio.paContinue) + self._input_bytes += len(in_data) + self._pending_send = asyncio.run_coroutine_threadsafe( + self._conn.input_audio_buffer.append(audio=in_data), self._loop + ) + return (None, pyaudio.paContinue) + + self._input_stream = self._audio.open( + format=pyaudio.paInt16, + channels=1, + rate=_SAMPLE_RATE, + input=True, + frames_per_buffer=_CHUNK_SAMPLES, + stream_callback=_capture_callback, + ) + + # -- playback ------------------------------------------------------------ + + def start_playback(self) -> None: + """Initialize the speaker playback callback.""" + if self._output_stream is not None: + return + remaining = b"" + # The sequence number the currently-buffered `remaining` bytes were dequeued from, so a + # barge-in that lands *between* callback invocations can still discard them below. + remaining_seq = -1 + + def _playback_callback(_in_data, frame_count, _time_info, _status): + nonlocal remaining, remaining_seq + if remaining and remaining_seq < self._playback_base: + remaining = b"" # a barge-in advanced the base since this chunk was dequeued + + wanted = frame_count * pyaudio.get_sample_size(pyaudio.paInt16) + out = remaining[:wanted] + remaining = remaining[wanted:] + + while len(out) < wanted: + try: + seq, data = self._playback_queue.get_nowait() + except queue.Empty: + out = out + bytes(wanted - len(out)) # pad with silence + continue + if not data: + # end-of-stream marker: pad up to the exact frame size pyaudio asked for + # instead of returning a short buffer, which would corrupt playback on close. + out = out + bytes(wanted - len(out)) + break + if seq < self._playback_base: + remaining = b"" # skipped by a barge-in + continue + take = wanted - len(out) + out = out + data[:take] + remaining = data[take:] + remaining_seq = seq + + return (out, pyaudio.paContinue) + + self._output_stream = self._audio.open( + format=pyaudio.paInt16, + channels=1, + rate=_SAMPLE_RATE, + output=True, + frames_per_buffer=_CHUNK_SAMPLES, + stream_callback=_playback_callback, + ) + + def _next_seq_num(self) -> int: + seq = self._next_seq + self._next_seq += 1 + return seq + + def queue_audio(self, pcm: bytes) -> None: + """Queue one decoded PCM16 chunk of the agent's reply for playback. + + :param pcm: Decoded PCM16 audio bytes. + :type pcm: bytes + """ + self._output_bytes += len(pcm) + self._playback_queue.put((self._next_seq_num(), pcm)) + + def skip_pending_audio(self) -> None: + """Drop audio still queued for playback (used on barge-in).""" + self._playback_base = self._next_seq_num() + + def shutdown(self) -> None: + """Stop capture and playback and release the audio device.""" + if self._input_stream is not None: + self._input_stream.stop_stream() + self._input_stream.close() + self._input_stream = None + if self._dropped_frames: + print(f"(dropped {self._dropped_frames} mic frame(s) while a send was still in flight)") + if self._output_stream is not None: + self.skip_pending_audio() + self._playback_queue.put((self._next_seq_num(), None)) + self._output_stream.stop_stream() + self._output_stream.close() + self._output_stream = None + self._audio.terminate() + + @property + def input_bytes_sent(self) -> int: + """Total raw PCM16 mic-audio bytes actually sent to the service (excludes dropped frames). + + :rtype: int + """ + return self._input_bytes + + @property + def output_bytes_received(self) -> int: + """Total decoded PCM16 reply-audio bytes received from the service. + + :rtype: int + """ + return self._output_bytes + + @property + def input_seconds(self) -> float: + """Total mic audio sent, in seconds (PCM16 = 2 bytes/sample). + + :rtype: float + """ + return self._input_bytes / 2 / _SAMPLE_RATE + + @property + def seconds(self) -> float: + """Total reply audio received, in seconds (PCM16 = 2 bytes/sample). + + :rtype: float + """ + return self._output_bytes / 2 / _SAMPLE_RATE + + +async def _run_audio_conversation(client: AIProjectClient, agent_name: str) -> Optional[str]: + """Hold a live, hands-free conversation with barge-in. + + :param client: The Foundry project client. + :param agent_name: The existing voice agent name. + :type client: ~azure.ai.projects.aio.AIProjectClient + :type agent_name: str + :return: The persisted conversation id, if one is created. + :rtype: str or None + """ + if pyaudio is None: + print("This sample needs pyaudio for audio: pip install pyaudio") + return None + + conversation_id: Optional[str] = None + response_active = False + + # Open the realtime session on the voice agent's dedicated route. + async with client.beta.realtime.connect(agent_name=agent_name) as conn: + # A voice agent owns its model, instructions, voice, turn detection, and + # noise suppression server-side, so this client sends no ``session.update``. + ap = _AudioProcessor(conn) + ap.start_playback() + ap.start_capture() + + print("Speak now -- the agent replies after you pause.") + print("(talk over the agent to interrupt it; press Ctrl-C to end the session)") + + try: + async for event in conn: + if isinstance(event, RealtimeServerEventSessionCreated): + # The persisted conversation id (only present when conversation + # persistence is enabled) is set here, not on response.done. + conversation_id = event.conversation_id or conversation_id + elif isinstance(event, RealtimeServerEventInputAudioBufferSpeechStarted): + # speech_started fires for every user turn, including the very first one, + # when no response is active yet. Only cancel (barge-in) if a response is + # actually in flight; canceling with none active is a service error. + if response_active: + await conn.response.cancel() + ap.skip_pending_audio() + print("(listening...)") + elif isinstance(event, RealtimeServerEventConversationItemInputAudioTranscriptionCompleted): + print(f"You: {event.transcript.strip()}") + elif isinstance(event, RealtimeServerEventError): + # Non-fatal errors are reported; a fatal one closes the socket. + print(f"Session error: {event.error.message}") + elif isinstance(event, RealtimeServerEventResponseCreated): + response_active = True + elif isinstance(event, RealtimeServerEventResponseAudioDelta): + # Each delta is a decoded PCM16 chunk; queue it. + ap.queue_audio(event.delta) + elif isinstance(event, RealtimeServerEventResponseAudioTranscriptDone): + _safe_print(f"Agent: {event.transcript}") + elif isinstance(event, RealtimeServerEventResponseDone): + response_active = False + except (KeyboardInterrupt, asyncio.CancelledError): + # Ctrl-C ends the session; read back whatever was persisted so far. + print("\n(ending session...)") + finally: + input_bytes = ap.input_bytes_sent + output_bytes = ap.output_bytes_received + print(f"(received {ap.seconds:.2f}s of reply audio this session)") + print( + f"Input audio (mic -> service): format=PCM16, sample_rate={_SAMPLE_RATE} Hz, channels=1, " + f"duration={ap.input_seconds:.2f}s, size={_format_size(input_bytes)}" + ) + print( + f"Output audio (service -> speakers): format=PCM16, sample_rate={_SAMPLE_RATE} Hz, channels=1, " + f"duration={ap.seconds:.2f}s, size={_format_size(output_bytes)}" + ) + print(f"Total audio transferred: size={_format_size(input_bytes + output_bytes)}") + ap.shutdown() + + return conversation_id + + +async def _read_conversation(client: AIProjectClient, agent_name: str, conversation_id: str) -> None: + """Read the persisted conversation back over the read-only conversation API. + + :param client: The Foundry project client. + :param agent_name: The voice agent name. + :param conversation_id: The persisted conversation id. + :type client: ~azure.ai.projects.aio.AIProjectClient + :type agent_name: str + :type conversation_id: str + """ + conversations = client.beta.agent_endpoint_conversations + + conversation = await conversations.get(agent_name, conversation_id) + print(f"Conversation {conversation.id}: status={conversation.status}, created_at={conversation.created_at}") + + print("Items (transcript):") + async for item in conversations.list_items(agent_name, conversation_id): + role = item.get("role") or item.get("type") + # Audio turns expose ``transcript``; text turns expose ``text``. + parts = [(part.get("transcript") or part.get("text") or "").strip() for part in (item.get("content") or [])] + transcript = " ".join(p for p in parts if p) + print(f" - {role} id={item.get('id')}") + if transcript: + _safe_print(f" {transcript}") + + +async def audio_conversation() -> None: + endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] + agent_name = os.environ.get("FOUNDRY_VOICE_AGENT_NAME") or "sample-live-audio-conversation-agent-async" + + async with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as project_client, + ): + try: + # 1) Generate a starter voice agent (see sample_voice_agent_generate.py). + generated = await project_client.beta.agents.generate( + GenerateVoiceAgentRequest(kind=AgentKind.VOICE, name=agent_name) + ) + definition = generated.versions.latest.definition # type: ignore[attr-defined] + + # 2) Publish a new version with conversation persistence enabled (`store=True`) so the + # session's conversation can be fetched back by id afterward. Reuse the generated + # definition as-is (instead of reconstructing a new one from a few fields) so audio, + # greeting, tools, and any other service-selected settings are preserved. + definition.store = True # type: ignore[attr-defined] + await project_client.agents.create_version( + agent_name=agent_name, + definition=definition, + ) + + # 3) Hold a live microphone conversation with the freshly created agent. + print(f"Starting realtime session with agent: {agent_name}") + conversation_id = await _run_audio_conversation(project_client, agent_name) + + # 4) Fetch the persisted conversation back by id. + if conversation_id: + print(f"Reading persisted conversation {conversation_id!r}...") + try: + await _read_conversation(project_client, agent_name, conversation_id) + except HttpResponseError as e: + print(f"Could not read conversation: {e.status_code} {e.reason}") + # To fetch this session's audio afterward, use + # `project_client.beta.agent_endpoint_conversations`: + # - get_audio(agent_name, conversation_id) for the merged + # whole-call stereo recording's metadata, then + # download_audio(agent_name, conversation_id) to stream + # the WAV bytes. + # - get_item_audio(agent_name, conversation_id, item_id) for a + # single turn's audio metadata, then + # download_item_audio(agent_name, conversation_id, item_id) + # to stream that turn's bytes. + # See sample_voice_agent_read_conversation_audio.py for a full example. + else: + print("No conversation id was returned; nothing to read.") + except HttpResponseError as e: + print(f"Service responded with an error: {e.status_code} {e.reason}") + finally: + # 5) Clean up the agent created for this sample. + await project_client.agents.delete(agent_name=agent_name) + print(f"Deleted voice agent: {agent_name}") + + +if __name__ == "__main__": + try: + asyncio.run(audio_conversation()) + except KeyboardInterrupt: + print("\nInterrupted.") diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_function_tool.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_function_tool.py new file mode 100644 index 000000000000..8c289a8eef69 --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_function_tool.py @@ -0,0 +1,204 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + This sample demonstrates handling a client-executed `function` tool during + a live voice-agent session: + + 1) Create a voice agent configured with a `get_weather` function tool. + 2) Open a realtime session and send a text turn that should trigger the tool. + 3) Listen for `response.function_call_arguments.done`, execute the function + locally, and send the result back with `conversation.item.create` + + `response.create` so the agent can finish its reply using the tool output. + +USAGE: + python sample_voice_agent_live_function_tool.py + + Before running the sample: + + pip install "azure-ai-projects[realtime]>=2.7.0b1" azure-identity python-dotenv --pre + + Set these environment variables with your own values: + 1) FOUNDRY_PROJECT_ENDPOINT - The Azure AI Project endpoint. + 2) FOUNDRY_VOICE_MODEL - Optional. The realtime model deployment name. + Defaults to "gpt-realtime". + 3) FOUNDRY_VOICE_AGENT_NAME - Optional. Name for the sample voice agent + created and deleted by this script. Defaults to + "sample-voice-agent-function-tool". +""" + +import json +import os +import sys +from typing import Any, Final, List, Tuple, cast + +from dotenv import load_dotenv +from azure.identity import DefaultAzureCredential +from azure.ai.projects import AIProjectClient +from azure.ai.projects.models import ( + RealtimeConversationItemFunctionCallOutput, + RealtimeConversationItemMessageUser, + RealtimeConversationItemMessageUserContent, + RealtimeConversationItemType, + RealtimeServerEventError, + VoiceAgentDefinition, + VoiceAgentFunctionTool, + RealtimeServerEventResponseDone, + RealtimeServerEventResponseFunctionCallArgumentsDone, + RealtimeServerEventResponseTextDone, + VoiceModelType, + VoiceOutputModality, +) + +load_dotenv() + +# Seconds to wait for the agent to finish a response. +_RESPONSE_TIMEOUT: Final = 45 + + +def get_weather(city: str) -> str: + """A trivial local "tool" implementation the agent can call. + + :param city: The city to look up. + :type city: str + :return: A canned weather report for the city. + :rtype: str + """ + return json.dumps({"city": city, "condition": "sunny", "temperature_f": 72}) + + +def _safe_print(text: str) -> None: + """Print text that may contain characters the current console can't display. + + The agent's reply below is model-generated and can contain characters (curly + quotes, em-dashes, etc.) outside some legacy, non-Unicode console encodings + (for example when stdout is piped/redirected on Windows). Rather than crashing + with UnicodeEncodeError, fall back to replacing just the unsupported characters; + a real interactive UTF-8 console prints unaffected. + """ + try: + print(text) + except UnicodeEncodeError: + encoding = sys.stdout.encoding or "ascii" + print(text.encode(encoding, errors="replace").decode(encoding)) + + +def _run_turn_with_tool_support(client: AIProjectClient, agent_name: str, prompt: str) -> None: + """Send one turn and resolve any function-call the agent makes before printing its reply. + + :param client: The Foundry project client. + :param agent_name: The voice agent name. + :param prompt: The user's message for this turn. + :type client: ~azure.ai.projects.AIProjectClient + :type agent_name: str + :type prompt: str + """ + with client.beta.realtime.connect(agent_name=agent_name) as conn: + conn.conversation.item.create( + item=RealtimeConversationItemMessageUser( + type=RealtimeConversationItemType.MESSAGE, + content=[RealtimeConversationItemMessageUserContent(type="input_text", text=prompt)], + ) + ) + conn.response.create() + + # Tool outputs collected from the current turn's function-call(s). These are held back + # and only sent once this turn's own response.done arrives (below) -- calling + # response.create() while the function-call response is still finishing can otherwise + # race with the service and produce a concurrent-response error. + pending_tool_outputs: List[Tuple[str, str]] = [] + while True: + try: + event = conn.recv(timeout=_RESPONSE_TIMEOUT) + except TimeoutError: + print("Timed out waiting for the agent's reply.") + conn.response.cancel() + return + if isinstance(event, RealtimeServerEventResponseFunctionCallArgumentsDone): + # The service forwards the call to us; execute it locally now, but defer sending + # the result until this response's own response.done arrives. + args = json.loads(event.arguments) + print(f"Tool call: {event.name}({args})") + if event.name == "get_weather": + result = get_weather(**args) + else: + result = json.dumps({"error": f"Unknown tool: {event.name}"}) + pending_tool_outputs.append((event.call_id, result)) + elif isinstance(event, RealtimeServerEventResponseTextDone): + # The sample agent uses a text-only output modality, so the + # reply arrives as output text rather than an audio transcript. + _safe_print(f"Agent: {event.text}") + elif isinstance(event, RealtimeServerEventResponseDone): + # A response.done that isn't a function call is the final answer for this turn. + # Output items are typed models in the tested scenarios here, but the underlying + # union is open (forward-compatible with item kinds this SDK doesn't map yet), so + # an unrecognized kind could still surface as a plain mapping; check both. + if pending_tool_outputs: + # The function-call response has now fully completed, so it's safe to submit + # its tool output(s) and ask for a new response. + for call_id, result in pending_tool_outputs: + conn.conversation.item.create( + item=RealtimeConversationItemFunctionCallOutput(call_id=call_id, output=result) + ) + pending_tool_outputs = [] + conn.response.create() + elif not any( + (item.get("type") if isinstance(item, dict) else getattr(item, "type", None)) == "function_call" + for item in (event.response.output or []) + ): + return + elif isinstance(event, RealtimeServerEventError): + print(f"Session error: {event.error.message}") + return + + +def main() -> None: + endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] + model = os.environ.get("FOUNDRY_VOICE_MODEL") or "gpt-realtime" + agent_name = os.environ.get("FOUNDRY_VOICE_AGENT_NAME") or "sample-voice-agent-function-tool" + + get_weather_tool = VoiceAgentFunctionTool( + name="get_weather", + description="Get the current weather for a city.", + parameters=cast( + Any, + { + "type": "object", + "properties": {"city": {"type": "string", "description": "City name, e.g. Seattle."}}, + "required": ["city"], + }, + ), + ) + + with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as project_client, + ): + try: + project_client.agents.create_version( + agent_name=agent_name, + definition=VoiceAgentDefinition( + model_type=VoiceModelType.MANAGED, + model=model, + instructions=( + "You are a helpful voice assistant. Use the get_weather tool when the " + "caller asks about the weather, then answer using its result." + ), + output_modalities=[VoiceOutputModality.TEXT], + tools=[get_weather_tool], + ), + ) + print(f"Created voice agent: {agent_name}") + + _run_turn_with_tool_support(project_client, agent_name, "What's the weather like in Seattle right now?") + finally: + project_client.agents.delete(agent_name=agent_name) + print(f"Deleted voice agent: {agent_name}") + + +if __name__ == "__main__": + main() diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation.py new file mode 100644 index 000000000000..26b85cfdf024 --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation.py @@ -0,0 +1,354 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + End-to-end typed conversation using the ``client.beta.realtime`` namespace added + on top of the generated azure-ai-projects client (see + ``azure.ai.projects.operations.Realtime``). + + 1. Generate a starter voice agent (see sample_voice_agent_generate.py), + then publish a version with `store=True` so the conversation can be + read back afterward. + 2. Hold a typed, multi-turn conversation: each prompt is sent as a + ``RealtimeConversationItemMessageUser`` and the reply streams back as + typed audio and transcript events. Blank line (or ``exit`` / ``quit``) + ends it. + 3. Fetch the persisted conversation back by id. + 4. Delete the agent created for this sample. + + Reply audio is PCM16, mono, 24 kHz and plays through the speakers when + ``pyaudio`` is installed; runs headless otherwise. For a hands-free mic + conversation with barge-in, see sample_voice_agent_live_audio_conversation_async.py + (that sample needs concurrent send/receive so it stays async-only; see + sample_voice_agent_live_text_conversation_async.py for the async version of + this one). + + pip install "azure-ai-projects[realtime]>=2.7.0b1" azure-identity pyaudio --pre + +USAGE: + python sample_voice_agent_live_text_conversation.py + + Environment variables: + 1) FOUNDRY_PROJECT_ENDPOINT (required) - Foundry project endpoint: + https://.services.ai.azure.com/api/projects/ + 2) FOUNDRY_VOICE_AGENT_NAME - Optional. Name for the agent created by this + sample. Defaults to "sample-live-text-conversation-agent". + + Authenticates with DefaultAzureCredential, so sign in first (e.g. `az login`). +""" + +import os +import sys +from typing import Final, Optional + +from dotenv import load_dotenv +from azure.core.exceptions import HttpResponseError +from azure.identity import DefaultAzureCredential +from azure.ai.projects import AIProjectClient +from azure.ai.projects.models import ( + AgentKind, + GenerateVoiceAgentRequest, + RealtimeConversationItemMessageUser, + RealtimeConversationItemMessageUserContent, + RealtimeConversationItemType, + RealtimeServerEventResponseAudioDelta, + RealtimeServerEventResponseAudioTranscriptDone, + RealtimeServerEventResponseDone, + RealtimeServerEventSessionCreated, + RealtimeServerEventError, +) + +load_dotenv() + + +def _safe_print(text: str) -> None: + """Print text that may contain characters the current console can't display. + + The agent's replies below are model-generated and can contain characters (curly + quotes, em-dashes, etc.) outside some legacy, non-Unicode console encodings + (for example when stdout is piped/redirected on Windows). Rather than crashing + with UnicodeEncodeError, fall back to replacing just the unsupported characters; + a real interactive UTF-8 console prints unaffected. + """ + try: + print(text) + except UnicodeEncodeError: + encoding = sys.stdout.encoding or "ascii" + print(text.encode(encoding, errors="replace").decode(encoding)) + + +# Seconds to wait for the agent to finish its reply. +_RESPONSE_TIMEOUT: Final = 45 + +# Reply audio format: PCM16, mono, 24 kHz. +_SAMPLE_RATE: Final = 24000 + + +def _format_size(num_bytes: int) -> str: + """Format a byte count as a human-readable string. + + :param num_bytes: The size in bytes. + :type num_bytes: int + :return: A string like "12345 bytes (12.1 KB)" or "2097152 bytes (2.00 MB)". + :rtype: str + """ + if num_bytes < 1024: + return f"{num_bytes} bytes" + if num_bytes < 1024 * 1024: + return f"{num_bytes} bytes ({num_bytes / 1024:.1f} KB)" + return f"{num_bytes} bytes ({num_bytes / (1024 * 1024):.2f} MB)" + + +try: + import pyaudio # type: ignore[import-not-found] +except ImportError: # pragma: no cover - optional playback dependency + pyaudio = None # type: ignore[assignment] + + +class _SpeakerPlayer: + """Play streamed PCM16 audio through the speakers with pyaudio. + + Optional: without pyaudio the player is a no-op and the sample still runs + headless, reporting how much audio it received. + """ + + def __init__(self) -> None: + self._audio = None + self._stream = None + self._bytes = 0 + if pyaudio is not None: + self._audio = pyaudio.PyAudio() + self._stream = self._audio.open( + format=pyaudio.paInt16, + channels=1, + rate=_SAMPLE_RATE, + output=True, + ) + + @property + def enabled(self) -> bool: + return self._stream is not None + + def play(self, pcm: bytes) -> None: + """Write one decoded PCM16 chunk to the speaker. + + :param pcm: Decoded PCM16 audio bytes. + :type pcm: bytes + """ + self._bytes += len(pcm) + if self._stream is not None: + self._stream.write(pcm) + + def close(self) -> None: + """Drain and release the audio device.""" + if self._stream is not None: + self._stream.stop_stream() + self._stream.close() + self._stream = None + if self._audio is not None: + self._audio.terminate() + self._audio = None + + @property + def bytes_received(self) -> int: + """Total decoded PCM16 output-audio bytes received from the service. + + :rtype: int + """ + return self._bytes + + @property + def seconds(self) -> float: + """Total audio received, in seconds (PCM16 = 2 bytes/sample). + + :rtype: float + """ + return self._bytes / 2 / _SAMPLE_RATE + + +def _run_text_conversation(client: AIProjectClient, agent_name: str, has_greeting: bool) -> Optional[str]: + """Hold a typed, multi-turn conversation. + + :param client: The Foundry project client. + :param agent_name: The existing voice agent name. + :param has_greeting: Whether the agent has a configured greeting, which the service plays + automatically as soon as the session opens (before any user turn). When True, that greeting + is drained and displayed before the interactive loop starts. + :type client: ~azure.ai.projects.AIProjectClient + :type agent_name: str + :type has_greeting: bool + :return: The persisted conversation id, if one is created. + :rtype: str or None + """ + conversation_id: Optional[str] = None + audio_delta_count = 0 + player = _SpeakerPlayer() + played = False + + try: + # Open the realtime session on the voice agent's dedicated route. + with client.beta.realtime.connect(agent_name=agent_name) as conn: + + def pump() -> None: + nonlocal conversation_id, audio_delta_count + while True: + try: + event = conn.recv(timeout=_RESPONSE_TIMEOUT) + except TimeoutError: + print("Timed out waiting for the agent's reply.") + conn.response.cancel() + return + if isinstance(event, RealtimeServerEventSessionCreated): + # The persisted conversation id (only present when conversation + # persistence is enabled) is set here, not on response.done. + conversation_id = event.conversation_id or conversation_id + if isinstance(event, RealtimeServerEventResponseDone): + return + if isinstance(event, RealtimeServerEventError): + print(f"Session error: {event.error.message}") + return + if isinstance(event, RealtimeServerEventResponseAudioDelta): + # Each delta is a decoded PCM16 chunk; play it. + audio_delta_count += 1 + player.play(event.delta) + elif isinstance(event, RealtimeServerEventResponseAudioTranscriptDone): + _safe_print(f"Agent: {event.transcript}") + + if has_greeting: + # The service sends the configured greeting as its own response cycle the + # instant the session opens, entirely independent of any user turn. Drain and + # display it here, before the interactive loop starts: otherwise the first + # pump() call below (triggered by the user's own first message) could instead + # observe this unrelated, already in-flight response.done and return early, + # silently dropping the real reply to what the user actually typed. + print("(agent is greeting...)") + pump() + + print("Type a message and press Enter. Blank line (or 'exit') ends the session.") + + while True: + prompt = input("You: ").strip() + if not prompt or prompt.lower() in ("exit", "quit"): + break + + # Send the turn and ask the agent to respond. + conn.conversation.item.create( + item=RealtimeConversationItemMessageUser( + type=RealtimeConversationItemType.MESSAGE, + content=[RealtimeConversationItemMessageUserContent(type="input_text", text=prompt)], + ) + ) + conn.response.create() + pump() + except KeyboardInterrupt: + print("\n(ending session...)") + finally: + played = player.enabled + player.close() + + detail = "played" if played else "received" + output_bytes = player.bytes_received + print(f"(streamed {audio_delta_count} audio chunks, {detail} {player.seconds:.2f}s of audio)") + print( + f"Output audio: format=PCM16, sample_rate={_SAMPLE_RATE} Hz, channels=1, " + f"duration={player.seconds:.2f}s, size={_format_size(output_bytes)}" + ) + if not played: + print("(install pyaudio to hear the reply: pip install pyaudio)") + return conversation_id + + +def _read_conversation(client: AIProjectClient, agent_name: str, conversation_id: str) -> None: + """Read the persisted conversation back over the read-only conversation API. + + :param client: The Foundry project client. + :param agent_name: The voice agent name. + :param conversation_id: The persisted conversation id. + :type client: ~azure.ai.projects.AIProjectClient + :type agent_name: str + :type conversation_id: str + """ + conversations = client.beta.agent_endpoint_conversations + + conversation = conversations.get(agent_name, conversation_id) + print(f"Conversation {conversation.id}: status={conversation.status}, created_at={conversation.created_at}") + + print("Items (transcript):") + for item in conversations.list_items(agent_name, conversation_id): + role = item.get("role") or item.get("type") + # Audio turns expose ``transcript``; text turns expose ``text``. + parts = [(part.get("transcript") or part.get("text") or "").strip() for part in (item.get("content") or [])] + transcript = " ".join(p for p in parts if p) + print(f" - {role} id={item.get('id')}") + if transcript: + _safe_print(f" {transcript}") + + +def text_conversation() -> None: + endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] + agent_name = os.environ.get("FOUNDRY_VOICE_AGENT_NAME") or "sample-live-text-conversation-agent" + + with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as project_client, + ): + try: + # 1) Generate a starter voice agent (see sample_voice_agent_generate.py). + generated = project_client.beta.agents.generate( + GenerateVoiceAgentRequest(kind=AgentKind.VOICE, name=agent_name) + ) + definition = generated.versions.latest.definition # type: ignore[attr-defined] + + # 2) Publish a new version with conversation persistence enabled (`store=True`) so the + # session's conversation can be fetched back by id afterward. Reuse the generated + # definition as-is (instead of reconstructing a new one from a few fields) so audio, + # greeting, tools, and any other service-selected settings are preserved. + definition.store = True # type: ignore[attr-defined] + project_client.agents.create_version( + agent_name=agent_name, + definition=definition, + ) + + # 3) Hold the realtime conversation against the freshly created agent. + print(f"Starting realtime session with agent: {agent_name}") + conversation_id = _run_text_conversation( + project_client, agent_name, has_greeting=definition.greeting is not None # type: ignore[attr-defined] + ) + + # 4) Fetch the persisted conversation back by id. + if conversation_id: + print(f"Reading persisted conversation {conversation_id}...") + try: + _read_conversation(project_client, agent_name, conversation_id) + except HttpResponseError as e: + print(f"Could not read conversation: {e.status_code} {e.reason}") + # To fetch this session's audio afterward, use + # `project_client.beta.agent_endpoint_conversations`: + # - get_audio(agent_name, conversation_id) for the merged + # whole-call stereo recording's metadata, then + # download_audio(agent_name, conversation_id) to stream + # the WAV bytes. + # - get_item_audio(agent_name, conversation_id, item_id) for a + # single turn's audio metadata, then + # download_item_audio(agent_name, conversation_id, item_id) + # to stream that turn's bytes. + # See sample_voice_agent_read_conversation_audio.py for a full example. + else: + print("No conversation id was returned; nothing to read.") + except HttpResponseError as e: + print(f"Service responded with an error: {e.status_code} {e.reason}") + finally: + # 5) Clean up the agent created for this sample. + project_client.agents.delete(agent_name=agent_name) + print(f"Deleted voice agent: {agent_name}") + + +if __name__ == "__main__": + try: + text_conversation() + except KeyboardInterrupt: + print("\nInterrupted.") diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation_async.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation_async.py new file mode 100644 index 000000000000..0a9c0486a023 --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation_async.py @@ -0,0 +1,357 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + End-to-end typed conversation using the ``client.beta.realtime`` namespace added + on top of the generated azure-ai-projects client (see + ``azure.ai.projects.aio.operations.AsyncRealtime``). + + 1. Generate a starter voice agent (see sample_voice_agent_generate.py), + then publish a version with `store=True` so the conversation can be + read back afterward. + 2. Hold a typed, multi-turn conversation: each prompt is sent as a + ``RealtimeConversationItemMessageUser`` and the reply streams back as + typed audio and transcript events. Blank line (or ``exit`` / ``quit``) + ends it. + 3. Fetch the persisted conversation back by id. + 4. Delete the agent created for this sample. + + Reply audio is PCM16, mono, 24 kHz and plays through the speakers when + ``pyaudio`` is installed; runs headless otherwise. For a hands-free mic + conversation with barge-in, see sample_voice_agent_live_audio_conversation_async.py. + + pip install "azure-ai-projects>=2.7.0b1" azure-identity aiohttp pyaudio --pre + +USAGE: + python sample_voice_agent_live_text_conversation_async.py + + Environment variables: + 1) FOUNDRY_PROJECT_ENDPOINT (required) - Foundry project endpoint: + https://.services.ai.azure.com/api/projects/ + 2) FOUNDRY_VOICE_AGENT_NAME - Optional. Name for the agent created by this + sample. Defaults to "sample-live-text-conversation-agent-async". + + Authenticates with DefaultAzureCredential, so sign in first (e.g. `az login`). +""" + +import asyncio +import os +import sys +from typing import Final, Optional + +from dotenv import load_dotenv +from azure.core.exceptions import HttpResponseError +from azure.identity.aio import DefaultAzureCredential +from azure.ai.projects.aio import AIProjectClient +from azure.ai.projects.models import ( + AgentKind, + GenerateVoiceAgentRequest, + RealtimeConversationItemMessageUser, + RealtimeConversationItemMessageUserContent, + RealtimeConversationItemType, + RealtimeServerEventResponseAudioDelta, + RealtimeServerEventResponseAudioTranscriptDone, + RealtimeServerEventResponseDone, + RealtimeServerEventSessionCreated, + RealtimeServerEventError, +) + +load_dotenv() + + +def _safe_print(text: str) -> None: + """Print text that may contain characters the current console can't display. + + The agent's replies below are model-generated and can contain characters (curly + quotes, em-dashes, etc.) outside some legacy, non-Unicode console encodings + (for example when stdout is piped/redirected on Windows). Rather than crashing + with UnicodeEncodeError, fall back to replacing just the unsupported characters; + a real interactive UTF-8 console prints unaffected. + """ + try: + print(text) + except UnicodeEncodeError: + encoding = sys.stdout.encoding or "ascii" + print(text.encode(encoding, errors="replace").decode(encoding)) + + +# Seconds to wait for the agent to finish its reply. +_RESPONSE_TIMEOUT: Final = 45 + +# Reply audio format: PCM16, mono, 24 kHz. +_SAMPLE_RATE: Final = 24000 + + +def _format_size(num_bytes: int) -> str: + """Format a byte count as a human-readable string. + + :param num_bytes: The size in bytes. + :type num_bytes: int + :return: A string like "12345 bytes (12.1 KB)" or "2097152 bytes (2.00 MB)". + :rtype: str + """ + if num_bytes < 1024: + return f"{num_bytes} bytes" + if num_bytes < 1024 * 1024: + return f"{num_bytes} bytes ({num_bytes / 1024:.1f} KB)" + return f"{num_bytes} bytes ({num_bytes / (1024 * 1024):.2f} MB)" + + +try: + import pyaudio # type: ignore[import-not-found] +except ImportError: # pragma: no cover - optional playback dependency + pyaudio = None # type: ignore[assignment] + + +class _SpeakerPlayer: + """Play streamed PCM16 audio through the speakers with pyaudio. + + Optional: without pyaudio the player is a no-op and the sample still runs + headless, reporting how much audio it received. + """ + + def __init__(self) -> None: + self._audio = None + self._stream = None + self._bytes = 0 + if pyaudio is not None: + self._audio = pyaudio.PyAudio() + self._stream = self._audio.open( + format=pyaudio.paInt16, + channels=1, + rate=_SAMPLE_RATE, + output=True, + ) + + @property + def enabled(self) -> bool: + return self._stream is not None + + def play(self, pcm: bytes) -> None: + """Write one decoded PCM16 chunk to the speaker. + + :param pcm: Decoded PCM16 audio bytes. + :type pcm: bytes + """ + self._bytes += len(pcm) + if self._stream is not None: + self._stream.write(pcm) + + def close(self) -> None: + """Drain and release the audio device.""" + if self._stream is not None: + self._stream.stop_stream() + self._stream.close() + self._stream = None + if self._audio is not None: + self._audio.terminate() + self._audio = None + + @property + def bytes_received(self) -> int: + """Total decoded PCM16 output-audio bytes received from the service. + + :rtype: int + """ + return self._bytes + + @property + def seconds(self) -> float: + """Total audio received, in seconds (PCM16 = 2 bytes/sample). + + :rtype: float + """ + return self._bytes / 2 / _SAMPLE_RATE + + +async def _run_text_conversation(client: AIProjectClient, agent_name: str, has_greeting: bool) -> Optional[str]: + """Hold a typed, multi-turn conversation. + + :param client: The Foundry project client. + :param agent_name: The existing voice agent name. + :param has_greeting: Whether the agent has a configured greeting, which the service plays + automatically as soon as the session opens (before any user turn). When True, that greeting + is drained and displayed before the interactive loop starts. + :type client: ~azure.ai.projects.aio.AIProjectClient + :type agent_name: str + :type has_greeting: bool + :return: The persisted conversation id, if one is created. + :rtype: str or None + """ + conversation_id: Optional[str] = None + audio_delta_count = 0 + player = _SpeakerPlayer() + + try: + # Open the realtime session on the voice agent's dedicated route. + async with client.beta.realtime.connect(agent_name=agent_name) as conn: + + async def pump() -> None: + nonlocal conversation_id, audio_delta_count + async for event in conn: + if isinstance(event, RealtimeServerEventSessionCreated): + # The persisted conversation id (only present when conversation + # persistence is enabled) is set here, not on response.done. + conversation_id = event.conversation_id or conversation_id + if isinstance(event, RealtimeServerEventResponseDone): + return + if isinstance(event, RealtimeServerEventError): + print(f"Session error: {event.error.message}") + return + if isinstance(event, RealtimeServerEventResponseAudioDelta): + # Each delta is a decoded PCM16 chunk; play it. + audio_delta_count += 1 + player.play(event.delta) + elif isinstance(event, RealtimeServerEventResponseAudioTranscriptDone): + _safe_print(f"Agent: {event.transcript}") + + if has_greeting: + # The service sends the configured greeting as its own response cycle the + # instant the session opens, entirely independent of any user turn. Drain and + # display it here, before the interactive loop starts: otherwise the first + # pump() call below (triggered by the user's own first message) could instead + # observe this unrelated, already in-flight response.done and return early, + # silently dropping the real reply to what the user actually typed. + print("(agent is greeting...)") + try: + await asyncio.wait_for(pump(), timeout=_RESPONSE_TIMEOUT) + except asyncio.TimeoutError: + print("Timed out waiting for the agent's greeting.") + await conn.response.cancel() + + print("Type a message and press Enter. Blank line (or 'exit') ends the session.") + + while True: + # input() blocks, so read it off the loop in a worker thread. + prompt = (await asyncio.to_thread(input, "You: ")).strip() + if not prompt or prompt.lower() in ("exit", "quit"): + break + + # Send the turn and ask the agent to respond. + await conn.conversation.item.create( + item=RealtimeConversationItemMessageUser( + type=RealtimeConversationItemType.MESSAGE, + content=[RealtimeConversationItemMessageUserContent(type="input_text", text=prompt)], + ) + ) + await conn.response.create() + + try: + await asyncio.wait_for(pump(), timeout=_RESPONSE_TIMEOUT) + except asyncio.TimeoutError: + print("Timed out waiting for the agent's reply.") + # The server-side response is still active even though we stopped waiting + # locally; cancel it so the next turn's response.create() isn't rejected. + await conn.response.cancel() + except (KeyboardInterrupt, asyncio.CancelledError): + print("\n(ending session...)") + finally: + played = player.enabled + player.close() + + detail = "played" if played else "received" + output_bytes = player.bytes_received + print(f"(streamed {audio_delta_count} audio chunks, {detail} {player.seconds:.2f}s of audio)") + print( + f"Output audio: format=PCM16, sample_rate={_SAMPLE_RATE} Hz, channels=1, " + f"duration={player.seconds:.2f}s, size={_format_size(output_bytes)}" + ) + if not played: + print("(install pyaudio to hear the reply: pip install pyaudio)") + return conversation_id + + +async def _read_conversation(client: AIProjectClient, agent_name: str, conversation_id: str) -> None: + """Read the persisted conversation back over the read-only conversation API. + + :param client: The Foundry project client. + :param agent_name: The voice agent name. + :param conversation_id: The persisted conversation id. + :type client: ~azure.ai.projects.aio.AIProjectClient + :type agent_name: str + :type conversation_id: str + """ + conversations = client.beta.agent_endpoint_conversations + + conversation = await conversations.get(agent_name, conversation_id) + print(f"Conversation {conversation.id}: status={conversation.status}, created_at={conversation.created_at}") + + print("Items (transcript):") + async for item in conversations.list_items(agent_name, conversation_id): + role = item.get("role") or item.get("type") + # Audio turns expose ``transcript``; text turns expose ``text``. + parts = [(part.get("transcript") or part.get("text") or "").strip() for part in (item.get("content") or [])] + transcript = " ".join(p for p in parts if p) + print(f" - {role} id={item.get('id')}") + if transcript: + _safe_print(f" {transcript}") + + +async def text_conversation() -> None: + endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] + agent_name = os.environ.get("FOUNDRY_VOICE_AGENT_NAME") or "sample-live-text-conversation-agent-async" + + async with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as project_client, + ): + try: + # 1) Generate a starter voice agent (see sample_voice_agent_generate.py). + generated = await project_client.beta.agents.generate( + GenerateVoiceAgentRequest(kind=AgentKind.VOICE, name=agent_name) + ) + definition = generated.versions.latest.definition # type: ignore[attr-defined] + + # 2) Publish a new version with conversation persistence enabled (`store=True`) so the + # session's conversation can be fetched back by id afterward. Reuse the generated + # definition as-is (instead of reconstructing a new one from a few fields) so audio, + # greeting, tools, and any other service-selected settings are preserved. + definition.store = True # type: ignore[attr-defined] + await project_client.agents.create_version( + agent_name=agent_name, + definition=definition, + ) + + # 3) Hold the realtime conversation against the freshly created agent. + print(f"Starting realtime session with agent: {agent_name}") + conversation_id = await _run_text_conversation( + project_client, agent_name, has_greeting=definition.greeting is not None # type: ignore[attr-defined] + ) + + # 4) Fetch the persisted conversation back by id. + if conversation_id: + print(f"Reading persisted conversation {conversation_id}...") + try: + await _read_conversation(project_client, agent_name, conversation_id) + except HttpResponseError as e: + print(f"Could not read conversation: {e.status_code} {e.reason}") + # To fetch this session's audio afterward, use + # `project_client.beta.agent_endpoint_conversations`: + # - get_audio(agent_name, conversation_id) for the merged + # whole-call stereo recording's metadata, then + # download_audio(agent_name, conversation_id) to stream + # the WAV bytes. + # - get_item_audio(agent_name, conversation_id, item_id) for a + # single turn's audio metadata, then + # download_item_audio(agent_name, conversation_id, item_id) + # to stream that turn's bytes. + # See sample_voice_agent_read_conversation_audio.py for a full example. + else: + print("No conversation id was returned; nothing to read.") + except HttpResponseError as e: + print(f"Service responded with an error: {e.status_code} {e.reason}") + finally: + # 5) Clean up the agent created for this sample. + await project_client.agents.delete(agent_name=agent_name) + print(f"Deleted voice agent: {agent_name}") + + +if __name__ == "__main__": + try: + asyncio.run(text_conversation()) + except KeyboardInterrupt: + print("\nInterrupted.") diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_read_conversation.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_read_conversation.py new file mode 100644 index 000000000000..b04f0d928aec --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_read_conversation.py @@ -0,0 +1,86 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + This sample demonstrates reading a persisted voice conversation back over + the read-only conversation API exposed by `project_client.beta.agent_endpoint_conversations`: + the conversation envelope, its responses (model inference turns), and its + ordered items (the transcript). Conversations are created and written by + the voice orchestrator during a live session; this client can only read + them, and only when the agent was configured with `store=True` (see + sample_voice_agent_basic.py). + +USAGE: + python sample_voice_agent_read_conversation.py + + Before running the sample: + + pip install "azure-ai-projects>=2.7.0b1" python-dotenv --pre + + Set these environment variables with your own values: + 1) FOUNDRY_PROJECT_ENDPOINT - The Azure AI Project endpoint. + 2) FOUNDRY_VOICE_AGENT_NAME - The name of the voice agent. + 3) FOUNDRY_VOICE_CONVERSATION_ID - The id of a persisted conversation + (captured from the `session.created` event during a live session, + see sample_voice_agent_live_audio_conversation_async.py). +""" + +import os +from dotenv import load_dotenv +from azure.core.exceptions import HttpResponseError +from azure.identity import DefaultAzureCredential +from azure.ai.projects import AIProjectClient + +load_dotenv() + +endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] +agent_name = os.environ["FOUNDRY_VOICE_AGENT_NAME"] +conversation_id = os.environ["FOUNDRY_VOICE_CONVERSATION_ID"] + +with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as project_client, +): + conversations = project_client.beta.agent_endpoint_conversations + try: + # The conversation envelope: status, timestamps, aggregate usage. + conversation = conversations.get(agent_name, conversation_id) + print(f"Conversation {conversation.id}: status={conversation.status}, created_at={conversation.created_at}") + + # The responses (model inference turns) in the conversation. + print("Responses:") + for response in conversations.list_responses(agent_name, conversation_id): + print(f" - {response.id}: status={response.status}") + + # Read a single response back, with its output and token usage. + detail = conversations.get_response(agent_name, conversation_id, response.id) + print(f" usage={detail.usage}") + + # The items produced by this specific response. Conversation items + # belong to an open union, so on read they surface as mappings + # keyed by their wire fields (``type``, ``id``, ...). + for response_item in conversations.list_response_items(agent_name, conversation_id, response.id): + print(f" item {response_item.get('type')} id={response_item.get('id')}") + + # The ordered conversation items -- the full transcript (user + assistant + tool events). + print("Items (transcript):") + for item in conversations.list_items(agent_name, conversation_id): + item_id = item.get("id") + print(f" - {item.get('type')} id={item_id}") + + # Read a single item back by id. + if item_id: + single = conversations.get_item(agent_name, conversation_id, item_id) + print(f" fetched item id={single.get('id')}") + + # Deleting a conversation removes it and all of its responses, items, and audio. + # This is destructive, so it is shown but not run by default. Uncomment to enable. + # deleted = conversations.delete(agent_name, conversation_id) + # print(f"Deleted conversation {deleted.id}: deleted={deleted.deleted}") + except HttpResponseError as e: + # 404 typically means the conversation was not persisted (agent ran with `store=False`). + print(f"Service responded with an error: {e.status_code} {e.reason}") diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_read_conversation_audio.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_read_conversation_audio.py new file mode 100644 index 000000000000..08be8991a3d6 --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_read_conversation_audio.py @@ -0,0 +1,136 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + This sample demonstrates reading the persisted audio of a voice + conversation via `project_client.beta.agent_endpoint_conversations`, both the + merged whole-call recording and a single turn's audio segment. For each it + reads the metadata first, then streams the WAV bytes to a local file. The + merged recording is stereo: the caller on the left channel and the agent + on the right. + + Audio is available only after the session has ended and only when the + agent was configured with `store=True`. For bring-your-own-storage (BYOS) + accounts the metadata carries a `blob_uri` instead, and the bytes are read + from your own storage rather than streamed here. + +USAGE: + python sample_voice_agent_read_conversation_audio.py + + Before running the sample: + + pip install "azure-ai-projects>=2.7.0b1" python-dotenv --pre + + Set these environment variables with your own values: + 1) FOUNDRY_PROJECT_ENDPOINT - The Azure AI Project endpoint. + 2) FOUNDRY_VOICE_AGENT_NAME - The name of the voice agent. + 3) FOUNDRY_VOICE_CONVERSATION_ID - The id of a persisted conversation. +""" + +import os +from dotenv import load_dotenv +from azure.core.exceptions import HttpResponseError +from azure.identity import DefaultAzureCredential +from azure.ai.projects import AIProjectClient + +load_dotenv() + + +def stream_to_wav(stream, output_path) -> None: + """Write a streamed audio-content response to a local WAV file. + + :param stream: An iterable of audio byte chunks. + :param output_path: The local output path. + :type stream: collections.abc.Iterable[bytes] + :type output_path: str + """ + with open(output_path, "wb") as f: + for chunk in stream: + f.write(chunk) + print(f"Wrote {output_path}") + + +def read_merged_recording(conversations, agent_name, conversation_id) -> None: + """Read the merged whole-call stereo recording (left=user, right=agent). + + :param conversations: The conversation operations client. + :param agent_name: The voice agent name. + :param conversation_id: The persisted conversation id. + :type conversations: azure.ai.projects.operations.BetaAgentEndpointConversationsOperations + :type agent_name: str + :type conversation_id: str + """ + recording = conversations.get_audio(agent_name, conversation_id) + print( + f"Recording: format={recording.format}, sample_rate={recording.sample_rate}, " + f"channels={recording.channels}, duration_ms={recording.duration_ms}" + ) + + if recording.blob_uri: + # Bring-your-own-storage: download from your own storage using the returned URI. + print(f"Recording is stored in your own storage at: {recording.blob_uri}") + return + + # Foundry-managed storage: stream the bytes and write them to a local WAV file. + stream = conversations.download_audio(agent_name, conversation_id) + stream_to_wav(stream, f"{conversation_id}.wav") + + +def read_first_item_audio(conversations, agent_name, conversation_id) -> None: + """Read the audio segment of the first conversation item that has one. + + :param conversations: The conversation operations client. + :param agent_name: The voice agent name. + :param conversation_id: The persisted conversation id. + :type conversations: azure.ai.projects.operations.BetaAgentEndpointConversationsOperations + :type agent_name: str + :type conversation_id: str + """ + for item in conversations.list_items(agent_name, conversation_id): + item_id = item.get("id") + if not item_id: + continue + try: + metadata = conversations.get_item_audio(agent_name, conversation_id, item_id) + except HttpResponseError as e: + # A 404 means this item has no persisted audio (for example, a text-only turn). + if e.status_code == 404: + continue + raise + + print(f"Item {item_id}: role={metadata.role}, duration_ms={metadata.duration_ms}") + if metadata.blob_uri: + print(f"Item audio is stored in your own storage at: {metadata.blob_uri}") + return + + stream = conversations.download_item_audio(agent_name, conversation_id, item_id) + stream_to_wav(stream, f"{conversation_id}_{item_id}.wav") + return + + print("No conversation item with audio was found.") + + +def main() -> None: + endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] + agent_name = os.environ["FOUNDRY_VOICE_AGENT_NAME"] + conversation_id = os.environ["FOUNDRY_VOICE_CONVERSATION_ID"] + + with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as project_client, + ): + conversations = project_client.beta.agent_endpoint_conversations + try: + read_merged_recording(conversations, agent_name, conversation_id) + read_first_item_audio(conversations, agent_name, conversation_id) + except HttpResponseError as e: + # 404: not persisted / not ready. 409: session still in progress. + print(f"Service responded with an error: {e.status_code} {e.reason}") + + +if __name__ == "__main__": + main() diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_versions.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_versions.py new file mode 100644 index 000000000000..441d67a2f02f --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_versions.py @@ -0,0 +1,92 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + This sample demonstrates working with voice-agent versions. Agents are + immutable: every `create_version` call produces a new version. This sample + creates an agent, adds a new version to it, adds a draft version, lists the + versions, and reads a single version back. + +USAGE: + python sample_voice_agent_versions.py + + Before running the sample: + + pip install "azure-ai-projects>=2.7.0b1" python-dotenv --pre + + Set these environment variables with your own values: + 1) FOUNDRY_PROJECT_ENDPOINT - The Azure AI Project endpoint. + 2) FOUNDRY_VOICE_MODEL - Optional. The realtime model deployment name. + Defaults to "gpt-realtime". + 3) FOUNDRY_VOICE_AGENT_NAME - Optional. The name of the voice agent. If not + set, defaults to "sample-versioned-voice-agent". +""" + +import os +from dotenv import load_dotenv +from azure.identity import DefaultAzureCredential +from azure.ai.projects import AIProjectClient +from azure.ai.projects.models import VoiceAgentDefinition, VoiceModelType + +load_dotenv() + +endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] +model = os.environ.get("FOUNDRY_VOICE_MODEL") or "gpt-realtime" +agent_name = os.environ.get("FOUNDRY_VOICE_AGENT_NAME") or "sample-versioned-voice-agent" + + +def make_definition(instructions: str) -> VoiceAgentDefinition: + # Each version differs only by its instructions; the rest is identical. + return VoiceAgentDefinition(model_type=VoiceModelType.MANAGED, model=model, instructions=instructions) + + +with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as project_client, +): + try: + # Create the initial agent (this is version 1). + created = project_client.agents.create_version( + agent_name=agent_name, + definition=make_definition("You are a helpful voice assistant."), + ) + print(f"Created agent '{agent_name}', version: {created.version}") + + # Create a new version with updated instructions. + new_version = project_client.agents.create_version( + agent_name=agent_name, + definition=make_definition("You are a helpful voice assistant. Always greet the caller by name."), + description="Added a personalized greeting.", + ) + print(f"Created new version: {new_version.version}") + + # Create a draft version. Drafts are recorded but excluded from the default + # 'latest' resolution and from version listings unless include_drafts=True. + draft_version = project_client.agents.create_version( + agent_name=agent_name, + definition=make_definition("You are a helpful voice assistant. Experimental draft persona."), + description="Candidate persona under review.", + draft=True, + ) + print(f"Created draft version: {draft_version.version}") + + # List released versions (drafts excluded by default). + print(f"Released versions of '{agent_name}':") + for version in project_client.agents.list_versions(agent_name=agent_name): + print(f" - version {version.version} (created_at={version.created_at})") + + # List including drafts. + print(f"All versions of '{agent_name}' (including drafts):") + for version in project_client.agents.list_versions(agent_name=agent_name, include_drafts=True): + print(f" - version {version.version} (draft={version.draft})") + + # Read a single version back. + fetched = project_client.agents.get_version(agent_name=agent_name, agent_version=new_version.version) + print(f"Fetched version {fetched.version}: {fetched.definition.instructions}") # type: ignore[attr-defined] + finally: + project_client.agents.delete(agent_name=agent_name) + print(f"Deleted agent: {agent_name}") diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_with_tools.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_with_tools.py new file mode 100644 index 000000000000..d643466f0bc4 --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_with_tools.py @@ -0,0 +1,146 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + This sample demonstrates the richer parts of a voice agent definition: + + * Input (microphone) audio configuration: audio format, server-side turn + detection (VAD), input-audio transcription. + * Tools the agent may use during a live session: a client-executed + `function` tool and a service-managed `system` control tool (`mcp` and + `toolbox` tools are shown as constructed objects for illustration). + * Bring-your-own-model (BYOM): set `model_type="self_deployed"` to point + the agent at your own Foundry model deployment instead of a + service-managed model. + +USAGE: + python sample_voice_agent_with_tools.py + + Before running the sample: + + pip install "azure-ai-projects>=2.7.0b1" python-dotenv --pre + + Set these environment variables with your own values: + 1) FOUNDRY_PROJECT_ENDPOINT - The Azure AI Project endpoint. + 2) FOUNDRY_VOICE_MODEL - Optional. The realtime model (managed) or the + Foundry deployment name (BYOM). Defaults to "gpt-realtime". + 3) FOUNDRY_VOICE_MODEL_TYPE - Optional. "managed" (default) for a + service-hosted model, or "self_deployed" to bring your own deployment. + 4) FOUNDRY_VOICE_AGENT_NAME - Optional. The name of the voice agent. If not + set, defaults to "sample-voice-agent-with-tools". +""" + +import os +from typing import Any, cast + +from dotenv import load_dotenv +from azure.identity import DefaultAzureCredential +from azure.ai.projects import AIProjectClient +from azure.ai.projects.models import ( + RealtimeAudioFormatsAudioPcm, + VoiceAgentDefinition, + VoiceAgentFunctionTool, + VoiceAgentMcpTool, + VoiceAgentAudioConfig, + VoiceAgentAudioInputConfig, + VoiceAgentAudioOutputConfig, + VoiceAgentEndConversationSystemTool, + VoiceAgentInputTranscription, + VoiceAgentInputTranscriptionModel, + VoiceModelType, + VoiceOutputModality, + VoiceAgentServerVadTurnDetection, + VoiceAgentToolboxTool, + VoiceType, +) + +load_dotenv() + +endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] +model = os.environ.get("FOUNDRY_VOICE_MODEL") or "gpt-realtime" +# "managed" runs a service-hosted model; "self_deployed" (BYOM) uses your own +# Foundry deployment named by `model`. The service derives whether the model is +# realtime or cascaded; you don't set that here. +model_type = os.environ.get("FOUNDRY_VOICE_MODEL_TYPE") or VoiceModelType.MANAGED +agent_name = os.environ.get("FOUNDRY_VOICE_AGENT_NAME") or "sample-voice-agent-with-tools" + +# A client-executed tool: the service forwards the function call to your app, +# and your app returns the result over the live session. +get_weather = VoiceAgentFunctionTool( + name="get_weather", + description="Get the current weather for a city.", + parameters=cast( + Any, + { + "type": "object", + "properties": {"city": {"type": "string", "description": "City name, e.g. Seattle."}}, + "required": ["city"], + }, + ), +) + +# A service-managed control tool: the platform can end the call on the agent's behalf. +end_call = VoiceAgentEndConversationSystemTool() + +# An MCP tool is executed by the service against a remote MCP server you own. +# It references an external server, so it is constructed here for illustration +# and not attached below. Provide one of server_url, connector_id, or tunnel_id. +_example_mcp_tool = VoiceAgentMcpTool( + server_label="my-mcp-server", + server_url="https://example.com/mcp", + require_approval="never", +) + +# A toolbox tool references a versioned Foundry toolbox you have created. It is +# constructed here for illustration; attach it only if the toolbox exists. +_example_toolbox_tool = VoiceAgentToolboxTool(toolbox_name="my-toolbox", toolbox_version="1") + +definition = VoiceAgentDefinition( + model_type=model_type, + model=model, + instructions="You are a helpful voice assistant. Use tools when they help answer the caller.", + audio=VoiceAgentAudioConfig( + # Input (microphone) side: 24 kHz PCM, server-side VAD so the agent + # auto-responds when the caller stops speaking, plus input-audio + # transcription so user speech is transcribed. + input=VoiceAgentAudioInputConfig( + format=RealtimeAudioFormatsAudioPcm(rate=24000), + turn_detection=VoiceAgentServerVadTurnDetection( + threshold=0.5, + prefix_padding_ms=300, + silence_duration_ms=500, + ), + transcription=VoiceAgentInputTranscription(model=VoiceAgentInputTranscriptionModel.WHISPER1), + ), + # Output (agent speech) side: the voice the agent speaks with. + output=VoiceAgentAudioOutputConfig(voice="en-US-AvaNeural", voice_type=VoiceType.AZURE_STANDARD), + ), + output_modalities=[VoiceOutputModality.AUDIO], + # Attach the self-contained tools. `_example_mcp_tool` and `_example_toolbox_tool` + # reference external resources you must own, so they are left out here. + tools=[get_weather, end_call], + store=True, +) + +with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as project_client, +): + try: + created_version = project_client.agents.create_version(agent_name=agent_name, definition=definition) + print(f"Created voice agent '{agent_name}' (model_type={model_type}, model={model})") + + agent_version = project_client.agents.get_version(agent_name=agent_name, agent_version=created_version.version) + tools = agent_version.definition.tools or [] # type: ignore[attr-defined] + print(f"Configured {len(tools)} tool(s):") + for tool in tools: + # `name` isn't declared on every tool kind (e.g. MCP tools have no `name`), + # so fall back to a placeholder for kinds that don't define it. + print(f" - {tool.type}: {getattr(tool, 'name', '(unnamed)')}") + finally: + project_client.agents.delete(agent_name=agent_name) + print(f"Deleted voice agent: {agent_name}") diff --git a/sdk/ai/azure-ai-projects/tests/agents/test_realtime_client.py b/sdk/ai/azure-ai-projects/tests/agents/test_realtime_client.py new file mode 100644 index 000000000000..9d216a9ceed0 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/test_realtime_client.py @@ -0,0 +1,395 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression,protected-access +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable +"""Transport-mocked unit tests for the hand-written sync realtime (WebSocket) client. + +Unlike ``test_voice_agent_crud.py``, these tests never make an HTTP/WS call: the underlying +``websockets.sync.client.connect`` is replaced with a fake so URL construction, header/auth +handling, event serialization/deserialization, connection cleanup, and dependency/error paths +can all be verified without a live service or a recorded transport. +""" + +import json +import inspect +from unittest.mock import MagicMock, patch +from urllib.parse import parse_qs, urlparse + +import pytest +from azure.core.credentials import AccessToken +from websockets.typing import Subprotocol + +from azure.ai.projects._realtime import ( + RealtimeConnectionManager, + _assert_trusted_connection_url, + _to_ws_url, + _USER_AGENT, +) +from azure.ai.projects._version import VERSION +from azure.ai.projects.models import ( + RealtimeClientEventResponseCreate, + RealtimeServerEventSessionCreated, +) + +_ENDPOINT = "https://my-account.services.ai.azure.com/api/projects/my-project" + + +class _FakeCredential: + """Sync stub credential that returns a never-expiring token.""" + + def __init__(self, token: str = "fake-token") -> None: + self._token = token + + def get_token(self, *args, **kwargs) -> AccessToken: # pylint: disable=unused-argument + return AccessToken(self._token, 9_999_999_999) + + +def _make_manager(**overrides) -> RealtimeConnectionManager: + kwargs = { + "endpoint": _ENDPOINT, + "credential": _FakeCredential(), + "credential_scopes": ["https://ai.azure.com/.default"], + "api_version": "v1", + "agent_name": "my-agent", + "foundry_features": "VoiceAgents=V1Preview", + } + kwargs.update(overrides) + return RealtimeConnectionManager(**kwargs) + + +class TestToWsUrl: + """Unit tests for the pure ``_to_ws_url`` URL-construction helper.""" + + def test_https_endpoint_becomes_wss(self): + url = _to_ws_url(_ENDPOINT, "my-agent") + assert ( + url + == "wss://my-account.services.ai.azure.com/api/projects/my-project/agents/my-agent/endpoint/protocols/voice" + ) + + def test_non_https_endpoint_scheme_is_left_unchanged(self): + # Regression test: _to_ws_url used to translate "http://" to "ws://", but + # RealtimeConnectionManager.enter() unconditionally rejects any non-"wss://" URL to + # protect the live Authorization token in transit, so that translated "ws://" URL could + # never actually be used to connect. Leaving the scheme untouched here means the + # downstream "wss://" check surfaces a clear error instead of an unreachable "ws://" path. + url = _to_ws_url("http://localhost:8080", "my-agent") + assert url == "http://localhost:8080/agents/my-agent/endpoint/protocols/voice" + + def test_trailing_slash_is_stripped(self): + url = _to_ws_url(_ENDPOINT + "/", "my-agent") + assert ( + url + == "wss://my-account.services.ai.azure.com/api/projects/my-project/agents/my-agent/endpoint/protocols/voice" + ) + + +class TestAssertTrustedConnectionUrl: + """Unit tests for the connection_url host allow-list guard (security fix).""" + + def test_matching_host_does_not_raise(self): + _assert_trusted_connection_url(f"wss://{'my-account.services.ai.azure.com'}/custom/path", _ENDPOINT) + + def test_mismatched_host_raises_value_error(self): + with pytest.raises(ValueError): + _assert_trusted_connection_url("wss://evil.example.com/steal-token", _ENDPOINT) + + def test_empty_host_raises_value_error(self): + with pytest.raises(ValueError): + _assert_trusted_connection_url("not-a-url", _ENDPOINT) + + def test_matching_host_explicit_default_port_does_not_raise(self): + # An explicit ":443" is the wss/https default, so this is the same origin as _ENDPOINT + # (which omits the port) and must be accepted. + _assert_trusted_connection_url("wss://my-account.services.ai.azure.com:443/custom/path", _ENDPOINT) + + def test_mismatched_port_raises_value_error(self): + # Regression test (security fix): comparing hostname alone let an override targeting the + # same host on a different, non-default port (a different origin) slip through and + # receive the live bearer token. + with pytest.raises(ValueError): + _assert_trusted_connection_url("wss://my-account.services.ai.azure.com:8443/steal-token", _ENDPOINT) + + +class TestRealtimeConnectionManagerEnter: + """Unit tests for ``RealtimeConnectionManager.enter()``: URL/header construction and errors.""" + + def test_enter_builds_bearer_auth_and_query(self): + fake_connection = MagicMock() + with patch("websockets.sync.client.connect", return_value=fake_connection) as mock_connect: + manager = _make_manager() + conn = manager.enter() + try: + assert conn is not None + finally: + manager.__exit__() + + assert mock_connect.call_count == 1 + _args, kwargs = mock_connect.call_args + called_url = _args[0] + assert called_url.startswith("wss://my-account.services.ai.azure.com") + assert "api-version=v1" in called_url + assert kwargs["additional_headers"]["Authorization"] == "Bearer fake-token" + assert kwargs["additional_headers"]["Foundry-Features"] == "VoiceAgents=V1Preview" + + def test_enter_identifies_sdk_via_user_agent_and_query(self): + # The generated HTTP surface gets SDK identification for free from the core pipeline's + # UserAgentPolicy; this hand-written client builds its own request and must opt in + # explicitly, both as a User-Agent header and (since some proxies/paths don't forward + # WebSocket upgrade headers) as an x-ms-client-sdk query parameter. + fake_connection = MagicMock() + with patch("websockets.sync.client.connect", return_value=fake_connection) as mock_connect: + manager = _make_manager() + manager.enter() + manager.__exit__() + + _args, kwargs = mock_connect.call_args + assert kwargs["additional_headers"]["User-Agent"] == _USER_AGENT + assert "azsdk-python-ai-projects" in _USER_AGENT + assert VERSION in _USER_AGENT + + query = parse_qs(urlparse(_args[0]).query) + assert query["x-ms-client-sdk"] == [_USER_AGENT] + + def test_enter_caller_user_agent_overrides_default(self): + fake_connection = MagicMock() + with patch("websockets.sync.client.connect", return_value=fake_connection) as mock_connect: + manager = _make_manager(extra_headers={"User-Agent": "custom-user-agent"}) + manager.enter() + manager.__exit__() + + _args, kwargs = mock_connect.call_args + assert kwargs["additional_headers"]["User-Agent"] == "custom-user-agent" + + def test_enter_caller_user_agent_overrides_default_case_insensitive(self): + # Regression test: a plain dict merge of extra_headers would leave a differently-cased + # caller override (e.g. "user-agent") as a *separate* key alongside our own "User-Agent" + # default, since Python dict keys are case-sensitive but HTTP header names are not -- + # sending two User-Agent-like headers instead of cleanly honoring the caller's override. + fake_connection = MagicMock() + with patch("websockets.sync.client.connect", return_value=fake_connection) as mock_connect: + manager = _make_manager(extra_headers={"user-agent": "custom-user-agent"}) + manager.enter() + manager.__exit__() + + _args, kwargs = mock_connect.call_args + headers = kwargs["additional_headers"] + assert "User-Agent" not in headers + assert headers["user-agent"] == "custom-user-agent" + + def test_enter_source_retains_client_identification_wiring(self): + # Regression guard for the SDK client-identification fix (ported from azure-ai-voicelive + # PR #48848) surviving a future TypeSpec regeneration. `_realtime.py` is a hand-written + # file that is NOT `_patch.py`-named, so it isn't covered by the code generator's own + # "never touch _patch.py" guarantee -- nothing in the TypeSpec emitter is aware this file + # exists. The tests above already fail on a *behavioral* regression (wrong header/query + # value), but they exercise the code through mocks and could, in principle, still pass + # against a rewritten implementation that happens to produce the same observable values by + # a different (less safe) path. This inspects the actual source of `enter()` so a partial + # revert -- one that drops the case-insensitive guard, say, while keeping the header value + # correct for the common case -- is caught directly, independent of the tests above. + source = inspect.getsource(RealtimeConnectionManager.enter) + assert "_USER_AGENT" in source + assert "_has_header_case_insensitive" in source + assert "x-ms-client-sdk" in source + + def test_enter_disables_library_default_user_agent_header(self): + # Regression test: unlike aiohttp (where an explicit "User-Agent" in `headers` already + # takes precedence over its own default), `websockets.sync.client.connect`'s + # `user_agent_header` is a wholly separate mechanism from `additional_headers` -- passing + # our own "User-Agent" there does not suppress it. Without explicitly disabling it, the + # connection would carry two distinct User-Agent-like values. + fake_connection = MagicMock() + with patch("websockets.sync.client.connect", return_value=fake_connection) as mock_connect: + manager = _make_manager() + manager.enter() + manager.__exit__() + + _args, kwargs = mock_connect.call_args + assert kwargs["user_agent_header"] is None + + def test_enter_overrides_caller_supplied_subprotocols_kwarg(self): + # Regression test: subprotocols=[Subprotocol("realtime")] is passed explicitly to + # _ws_connect, so a caller-supplied subprotocols override forwarded through **kwargs would + # otherwise collide ("got multiple values for keyword argument 'subprotocols'"). The + # service requires the "realtime" subprotocol, so the override is dropped rather than + # honored -- matching the async implementation's handling of its equivalent `protocols` + # kwarg. + fake_connection = MagicMock() + with patch("websockets.sync.client.connect", return_value=fake_connection) as mock_connect: + manager = _make_manager(subprotocols=["other"]) + manager.enter() + manager.__exit__() + + _args, kwargs = mock_connect.call_args + assert kwargs["subprotocols"] == [Subprotocol("realtime")] + + def test_enter_appends_extra_query_and_headers(self): + fake_connection = MagicMock() + with patch("websockets.sync.client.connect", return_value=fake_connection) as mock_connect: + manager = _make_manager(extra_query={"foo": "bar"}, extra_headers={"X-Custom": "1"}) + manager.enter() + manager.__exit__() + + _args, kwargs = mock_connect.call_args + assert "foo=bar" in _args[0] + assert kwargs["additional_headers"]["X-Custom"] == "1" + + def test_enter_preserves_existing_query_on_connection_url_override(self): + # Regression test: the URL builder used to unconditionally append "?", corrupting an + # override URL that already has a query string (e.g. a SAS-style "?sig=..."). + fake_connection = MagicMock() + override = f"wss://{'my-account.services.ai.azure.com'}/custom?sig=abc" + with patch("websockets.sync.client.connect", return_value=fake_connection) as mock_connect: + manager = _make_manager(connection_url=override) + manager.enter() + manager.__exit__() + + called_url = mock_connect.call_args[0][0] + assert called_url.count("?") == 1 + assert "sig=abc&api-version=v1" in called_url + + def test_enter_rejects_untrusted_connection_url_host(self): + manager = _make_manager(connection_url="wss://evil.example.com/steal-token") + with pytest.raises(ValueError): + manager.enter() + + def test_enter_rejects_non_wss_url(self): + # A plain http(s) endpoint that somehow produced a non-ws(s) URL should never proceed. + manager = _make_manager(endpoint="ftp://not-http-or-https") + with pytest.raises(ValueError): + manager.enter() + + def test_enter_raises_runtime_error_when_websockets_missing(self): + manager = _make_manager() + with patch.dict("sys.modules", {"websockets.sync.client": None, "websockets.typing": None}): + with pytest.raises(RuntimeError, match="websockets"): + manager.enter() + + def test_context_manager_closes_connection_on_exit(self): + fake_connection = MagicMock() + with patch("websockets.sync.client.connect", return_value=fake_connection): + with _make_manager() as conn: + pass + fake_connection.close.assert_called_once() + + +class TestRealtimeConnectionRecv: + """Unit tests for ``RealtimeConnection.recv()``: event dispatch and error/timeout handling.""" + + def test_recv_dispatches_known_event_type(self, request): + fake_connection = MagicMock() + with patch("websockets.sync.client.connect", return_value=fake_connection): + manager = _make_manager() + conn = manager.enter() + request.addfinalizer(manager.__exit__) + + fake_connection.recv.return_value = json.dumps({"type": "session.created", "session": {}}) + event = conn.recv() + assert isinstance(event, RealtimeServerEventSessionCreated) + + def test_recv_unknown_event_type_returns_dict(self, request): + fake_connection = MagicMock() + with patch("websockets.sync.client.connect", return_value=fake_connection): + manager = _make_manager() + conn = manager.enter() + request.addfinalizer(manager.__exit__) + + fake_connection.recv.return_value = json.dumps({"type": "some.new.event", "foo": "bar"}) + event = conn.recv() + assert isinstance(event, dict) + assert event["foo"] == "bar" + + def test_recv_forwards_timeout_to_underlying_connection(self, request): + fake_connection = MagicMock() + with patch("websockets.sync.client.connect", return_value=fake_connection): + manager = _make_manager() + conn = manager.enter() + request.addfinalizer(manager.__exit__) + + fake_connection.recv.return_value = json.dumps({"type": "error", "error": {"message": "boom"}}) + conn.recv(timeout=5.0) + fake_connection.recv.assert_called_once_with(timeout=5.0) + + def test_recv_timeout_error_propagates(self, request): + fake_connection = MagicMock() + with patch("websockets.sync.client.connect", return_value=fake_connection): + manager = _make_manager() + conn = manager.enter() + request.addfinalizer(manager.__exit__) + + fake_connection.recv.side_effect = TimeoutError() + with pytest.raises(TimeoutError): + conn.recv(timeout=0.1) + + def test_recv_connection_closed_raises_connection_reset_error(self, request): + from websockets.exceptions import ConnectionClosedOK + + fake_connection = MagicMock() + with patch("websockets.sync.client.connect", return_value=fake_connection): + manager = _make_manager() + conn = manager.enter() + request.addfinalizer(manager.__exit__) + + fake_connection.recv.side_effect = ConnectionClosedOK(None, None) + with pytest.raises(ConnectionResetError): + conn.recv() + + def test_iteration_stops_cleanly_on_connection_reset(self, request): + fake_connection = MagicMock() + with patch("websockets.sync.client.connect", return_value=fake_connection): + manager = _make_manager() + conn = manager.enter() + request.addfinalizer(manager.__exit__) + + fake_connection.recv.side_effect = ConnectionResetError() + assert list(conn) == [] + + +class TestRealtimeConnectionSend: + """Unit tests for ``RealtimeConnection.send()``: model/str/mapping serialization.""" + + def test_send_serializes_typed_model(self, request): + fake_connection = MagicMock() + with patch("websockets.sync.client.connect", return_value=fake_connection): + manager = _make_manager() + conn = manager.enter() + request.addfinalizer(manager.__exit__) + + conn.send(RealtimeClientEventResponseCreate()) + sent_raw = fake_connection.send.call_args[0][0] + payload = json.loads(sent_raw) + assert payload["type"] == "response.create" + + def test_send_passes_through_valid_json_string(self, request): + fake_connection = MagicMock() + with patch("websockets.sync.client.connect", return_value=fake_connection): + manager = _make_manager() + conn = manager.enter() + request.addfinalizer(manager.__exit__) + + conn.send('{"type": "response.create"}') + fake_connection.send.assert_called_once_with('{"type": "response.create"}') + + def test_send_rejects_invalid_json_string(self, request): + fake_connection = MagicMock() + with patch("websockets.sync.client.connect", return_value=fake_connection): + manager = _make_manager() + conn = manager.enter() + request.addfinalizer(manager.__exit__) + + with pytest.raises(ValueError): + conn.send("not valid json") + + def test_send_serializes_mapping(self, request): + fake_connection = MagicMock() + with patch("websockets.sync.client.connect", return_value=fake_connection): + manager = _make_manager() + conn = manager.enter() + request.addfinalizer(manager.__exit__) + + conn.send({"type": "response.cancel"}) + sent_raw = fake_connection.send.call_args[0][0] + assert json.loads(sent_raw) == {"type": "response.cancel"} diff --git a/sdk/ai/azure-ai-projects/tests/agents/test_realtime_client_async.py b/sdk/ai/azure-ai-projects/tests/agents/test_realtime_client_async.py new file mode 100644 index 000000000000..aa599ba87de8 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/test_realtime_client_async.py @@ -0,0 +1,349 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression,protected-access +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable +"""Transport-mocked unit tests for the hand-written async realtime (WebSocket) client. + +Async counterpart of ``test_realtime_client.py``. The underlying ``aiohttp.ClientSession`` is +replaced with a fake so URL construction, header/auth handling, event serialization/ +deserialization, connection cleanup, and dependency/error paths can all be verified without a +live service or a recorded transport. +""" + +import json +import inspect +from unittest.mock import AsyncMock, MagicMock, patch +from urllib.parse import parse_qs, urlparse + +import pytest +from azure.core.credentials import AccessToken + +from azure.ai.projects.aio._realtime import AsyncRealtimeConnectionManager, _USER_AGENT +from azure.ai.projects._version import VERSION +from azure.ai.projects.models import ( + RealtimeClientEventResponseCreate, + RealtimeServerEventSessionCreated, +) + +_ENDPOINT = "https://my-account.services.ai.azure.com/api/projects/my-project" + +pytestmark = pytest.mark.asyncio + + +class _AsyncFakeCredential: + """Async stub credential that returns a never-expiring token.""" + + def __init__(self, token: str = "fake-token") -> None: + self._token = token + + async def get_token(self, *args, **kwargs) -> AccessToken: # pylint: disable=unused-argument + return AccessToken(self._token, 9_999_999_999) + + +def _make_manager(**overrides) -> AsyncRealtimeConnectionManager: + kwargs = { + "endpoint": _ENDPOINT, + "credential": _AsyncFakeCredential(), + "credential_scopes": ["https://ai.azure.com/.default"], + "api_version": "v1", + "agent_name": "my-agent", + "foundry_features": "VoiceAgents=V1Preview", + } + kwargs.update(overrides) + return AsyncRealtimeConnectionManager(**kwargs) + + +def _make_fake_msg(msg_type, data=None): + msg = MagicMock() + msg.type = msg_type + msg.data = data + return msg + + +def _make_fake_ws(): + """A fake aiohttp ClientWebSocketResponse with async close() (always awaited by __aexit__).""" + fake_ws = MagicMock() + fake_ws.close = AsyncMock() + return fake_ws + + +def _patch_client_session(fake_ws_connection): + """Patch aiohttp.ClientSession() to return a fake session whose ws_connect/close are async.""" + fake_session = MagicMock() + fake_session.ws_connect = AsyncMock(return_value=fake_ws_connection) + fake_session.close = AsyncMock() + return patch("aiohttp.ClientSession", return_value=fake_session), fake_session + + +class TestAsyncRealtimeConnectionManagerEnter: + """Unit tests for ``AsyncRealtimeConnectionManager.enter()``: URL/header construction and errors.""" + + async def test_enter_builds_bearer_auth_and_query(self): + fake_ws = _make_fake_ws() + patcher, fake_session = _patch_client_session(fake_ws) + with patcher: + manager = _make_manager() + await manager.enter() + await manager.__aexit__() + + assert fake_session.ws_connect.call_count == 1 + _args, kwargs = fake_session.ws_connect.call_args + assert _args[0].startswith("wss://my-account.services.ai.azure.com") + assert kwargs["params"]["api-version"] == "v1" + assert kwargs["headers"]["Authorization"] == "Bearer fake-token" + assert kwargs["headers"]["Foundry-Features"] == "VoiceAgents=V1Preview" + assert "Sec-WebSocket-Protocol" not in kwargs["headers"] + assert kwargs["protocols"] == ("realtime",) + + async def test_enter_identifies_sdk_via_user_agent_and_query(self): + # The generated HTTP surface gets SDK identification for free from the core pipeline's + # UserAgentPolicy; this hand-written client builds its own request and must opt in + # explicitly, both as a User-Agent header and (since some proxies/paths don't forward + # WebSocket upgrade headers) as an x-ms-client-sdk query parameter. + fake_ws = _make_fake_ws() + patcher, fake_session = _patch_client_session(fake_ws) + with patcher: + manager = _make_manager() + await manager.enter() + await manager.__aexit__() + + _args, kwargs = fake_session.ws_connect.call_args + assert kwargs["headers"]["User-Agent"] == _USER_AGENT + assert "azsdk-python-ai-projects" in _USER_AGENT + assert VERSION in _USER_AGENT + assert kwargs["params"]["x-ms-client-sdk"] == _USER_AGENT + + async def test_enter_caller_user_agent_overrides_default(self): + fake_ws = _make_fake_ws() + patcher, fake_session = _patch_client_session(fake_ws) + with patcher: + manager = _make_manager(extra_headers={"User-Agent": "custom-user-agent"}) + await manager.enter() + await manager.__aexit__() + + _args, kwargs = fake_session.ws_connect.call_args + assert kwargs["headers"]["User-Agent"] == "custom-user-agent" + + async def test_enter_caller_user_agent_overrides_default_case_insensitive(self): + # Regression test: a plain dict merge of extra_headers would leave a differently-cased + # caller override (e.g. "user-agent") as a *separate* key alongside our own "User-Agent" + # default, since Python dict keys are case-sensitive but HTTP header names are not -- + # sending two User-Agent-like headers instead of cleanly honoring the caller's override. + fake_ws = _make_fake_ws() + patcher, fake_session = _patch_client_session(fake_ws) + with patcher: + manager = _make_manager(extra_headers={"user-agent": "custom-user-agent"}) + await manager.enter() + await manager.__aexit__() + + _args, kwargs = fake_session.ws_connect.call_args + headers = kwargs["headers"] + assert "User-Agent" not in headers + assert headers["user-agent"] == "custom-user-agent" + + async def test_enter_source_retains_client_identification_wiring(self): + # Regression guard for the SDK client-identification fix (ported from azure-ai-voicelive + # PR #48848) surviving a future TypeSpec regeneration. `aio/_realtime.py` is a hand-written + # file that is NOT `_patch.py`-named, so it isn't covered by the code generator's own + # "never touch _patch.py" guarantee -- nothing in the TypeSpec emitter is aware this file + # exists. The tests above already fail on a *behavioral* regression (wrong header/query + # value), but they exercise the code through mocks and could, in principle, still pass + # against a rewritten implementation that happens to produce the same observable values by + # a different (less safe) path. This inspects the actual source of `enter()` so a partial + # revert -- one that drops the case-insensitive guard, say, while keeping the header value + # correct for the common case -- is caught directly, independent of the tests above. + source = inspect.getsource(AsyncRealtimeConnectionManager.enter) + assert "_USER_AGENT" in source + assert "_has_header_case_insensitive" in source + assert "x-ms-client-sdk" in source + + async def test_enter_rejects_untrusted_connection_url_host(self): + manager = _make_manager(connection_url="wss://evil.example.com/steal-token") + with pytest.raises(ValueError): + await manager.enter() + + async def test_enter_overrides_caller_supplied_protocols_kwarg(self): + # Regression test: protocols=("realtime",) is now passed explicitly to ws_connect, so a + # caller-supplied protocols override forwarded through **kwargs would otherwise collide + # ("got multiple values for keyword argument 'protocols'"). The service requires the + # "realtime" subprotocol, so the override is dropped rather than honored. + fake_ws = _make_fake_ws() + patcher, fake_session = _patch_client_session(fake_ws) + with patcher: + manager = _make_manager(protocols=("other",)) + await manager.enter() + await manager.__aexit__() + + _args, kwargs = fake_session.ws_connect.call_args + assert kwargs["protocols"] == ("realtime",) + + async def test_enter_rejects_non_wss_url(self): + manager = _make_manager(endpoint="ftp://not-http-or-https") + with pytest.raises(ValueError): + await manager.enter() + + async def test_enter_raises_runtime_error_when_aiohttp_missing(self): + manager = _make_manager() + with patch.dict("sys.modules", {"aiohttp": None}): + with pytest.raises(RuntimeError, match="aiohttp"): + await manager.enter() + + async def test_enter_closes_session_on_connect_failure(self): + fake_session = MagicMock() + fake_session.ws_connect = AsyncMock(side_effect=OSError("connection refused")) + fake_session.close = AsyncMock() + with patch("aiohttp.ClientSession", return_value=fake_session): + manager = _make_manager() + with pytest.raises(ConnectionError): + await manager.enter() + fake_session.close.assert_awaited_once() + + async def test_context_manager_closes_connection_on_exit(self): + fake_ws = _make_fake_ws() + patcher, fake_session = _patch_client_session(fake_ws) + with patcher: + async with _make_manager(): + pass + fake_ws.close.assert_awaited_once() + fake_session.close.assert_awaited_once() + + +class TestAsyncRealtimeConnectionRecv: + """Unit tests for ``AsyncRealtimeConnection.recv()``: event dispatch and non-text frames.""" + + async def test_recv_dispatches_known_event_type(self): + import aiohttp + + fake_ws = _make_fake_ws() + fake_ws.receive = AsyncMock( + return_value=_make_fake_msg(aiohttp.WSMsgType.TEXT, json.dumps({"type": "session.created", "session": {}})) + ) + patcher, _ = _patch_client_session(fake_ws) + with patcher: + manager = _make_manager() + conn = await manager.enter() + try: + event = await conn.recv() + assert isinstance(event, RealtimeServerEventSessionCreated) + finally: + await manager.__aexit__() + + async def test_recv_skips_ping_pong_frames(self): + # Regression test locking in the existing PING/PONG handling. + import aiohttp + + fake_ws = _make_fake_ws() + fake_ws.receive = AsyncMock( + side_effect=[ + _make_fake_msg(aiohttp.WSMsgType.PING, b""), + _make_fake_msg(aiohttp.WSMsgType.PONG, b""), + _make_fake_msg(aiohttp.WSMsgType.TEXT, json.dumps({"type": "session.created", "session": {}})), + ] + ) + patcher, _ = _patch_client_session(fake_ws) + with patcher: + manager = _make_manager() + conn = await manager.enter() + try: + event = await conn.recv() + assert isinstance(event, RealtimeServerEventSessionCreated) + assert fake_ws.receive.await_count == 3 + finally: + await manager.__aexit__() + + async def test_recv_unknown_event_type_returns_dict(self): + import aiohttp + + fake_ws = _make_fake_ws() + fake_ws.receive = AsyncMock( + return_value=_make_fake_msg(aiohttp.WSMsgType.TEXT, json.dumps({"type": "some.new.event", "foo": "bar"})) + ) + patcher, _ = _patch_client_session(fake_ws) + with patcher: + manager = _make_manager() + conn = await manager.enter() + try: + event = await conn.recv() + assert isinstance(event, dict) + assert event["foo"] == "bar" + finally: + await manager.__aexit__() + + async def test_recv_close_frame_raises_connection_reset_error(self): + import aiohttp + + fake_ws = _make_fake_ws() + fake_ws.receive = AsyncMock(return_value=_make_fake_msg(aiohttp.WSMsgType.CLOSE)) + patcher, _ = _patch_client_session(fake_ws) + with patcher: + manager = _make_manager() + conn = await manager.enter() + try: + with pytest.raises(ConnectionResetError): + await conn.recv() + finally: + await manager.__aexit__() + + async def test_recv_error_frame_raises_connection_reset_error(self): + import aiohttp + + fake_ws = _make_fake_ws() + fake_ws.exception = MagicMock(return_value=RuntimeError("boom")) + fake_ws.receive = AsyncMock(return_value=_make_fake_msg(aiohttp.WSMsgType.ERROR)) + patcher, _ = _patch_client_session(fake_ws) + with patcher: + manager = _make_manager() + conn = await manager.enter() + try: + with pytest.raises(ConnectionResetError): + await conn.recv() + finally: + await manager.__aexit__() + + +class TestAsyncRealtimeConnectionSend: + """Unit tests for ``AsyncRealtimeConnection.send()``: model/str/mapping serialization.""" + + async def test_send_serializes_typed_model(self): + fake_ws = _make_fake_ws() + fake_ws.send_str = AsyncMock() + patcher, _ = _patch_client_session(fake_ws) + with patcher: + manager = _make_manager() + conn = await manager.enter() + try: + await conn.send(RealtimeClientEventResponseCreate()) + sent_raw = fake_ws.send_str.call_args[0][0] + payload = json.loads(sent_raw) + assert payload["type"] == "response.create" + finally: + await manager.__aexit__() + + async def test_send_rejects_invalid_json_string(self): + fake_ws = _make_fake_ws() + fake_ws.send_str = AsyncMock() + patcher, _ = _patch_client_session(fake_ws) + with patcher: + manager = _make_manager() + conn = await manager.enter() + try: + with pytest.raises(ValueError): + await conn.send("not valid json") + finally: + await manager.__aexit__() + + async def test_send_serializes_mapping(self): + fake_ws = _make_fake_ws() + fake_ws.send_str = AsyncMock() + patcher, _ = _patch_client_session(fake_ws) + with patcher: + manager = _make_manager() + conn = await manager.enter() + try: + await conn.send({"type": "response.cancel"}) + sent_raw = fake_ws.send_str.call_args[0][0] + assert json.loads(sent_raw) == {"type": "response.cancel"} + finally: + await manager.__aexit__() diff --git a/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_conversations.py b/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_conversations.py new file mode 100644 index 000000000000..d7827a82529a --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_conversations.py @@ -0,0 +1,239 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression,too-many-statements,broad-exception-caught +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +""" +Recorded tests covering the read-only voice-agent conversation REST API surface exposed through +``project_client.beta.agent_endpoint_conversations``. + +Conversations, their responses/items, and audio are written by the realtime WebSocket subsystem +during a live session (see ``test_voice_agent_realtime_live.py``) and can only be *read* here -- +there is no REST way to create one. A real ``conversation_id`` can therefore only be obtained by +actually running a live session, which is not itself something the test proxy can capture or +replay (it is a raw WebSocket connection, not an HTTP call through the SDK pipeline). + +To get real recorded/replayable coverage of the REST read-back surface anyway, this test: + * When run live (``AZURE_TEST_RUN_LIVE=true``): creates a `store=True` voice agent, opens a + short-lived realtime session directly (bypassing the recorded pipeline, same as any other + live network call), sends one turn, and waits for the resulting conversation to finalize. + The dynamic conversation id is then sanitized to a fixed placeholder before any of the + REST calls below are made, so what gets written to the recording cassette is stable. + * When replayed from the recording (the normal case in CI): skips the live session entirely + and uses the same fixed placeholder conversation id the cassette already expects. +Either way, the REST calls themselves (list/get conversation, responses, items, audio) go +through ``recorded_by_proxy`` exactly like any other recorded test in this package. +""" + +import re +import time +from typing import Final, Optional + +from test_base import TestBase, servicePreparer +from devtools_testutils import recorded_by_proxy, is_live, add_general_regex_sanitizer +from azure.core.exceptions import HttpResponseError +from azure.ai.projects.models import ( + RealtimeConversationItemMessageUser, + RealtimeConversationItemMessageUserContent, + RealtimeConversationItemType, + RealtimeServerEventResponseDone, + RealtimeServerEventSessionCreated, + VoiceAgentAudioConfig, + VoiceAgentAudioOutputConfig, + VoiceAgentDefinition, + VoiceModelType, + VoiceOutputModality, +) + +# Fixed test-owned agent name: unlike conversation_id (server-generated, truly dynamic), this is +# our own choice and does not need is_live()/sanitizer handling -- it is identical in both modes. +_AGENT_NAME: Final = "test-conversations-read-agent" + +# Best-effort fixed wait (live only, seconds) after the realtime session ends, before reading the +# conversation back, so persistence finalization (items/audio) is more likely to have completed. +# This must be a single, fixed wait rather than a poll loop through the recorded client: repeated +# polling would record multiple cassette entries for the same "get conversation" request, but +# playback only ever issues that request once (polling itself is live-only), so a replay would +# incorrectly consume the *first* (possibly still "in_progress") recorded entry instead of the +# settled one. A single wait keeps exactly one logical call -- and therefore one cassette entry +# -- for both the live recording and the replay to agree on. +_FINALIZATION_WAIT_SECONDS: Final = 30 + + +def _create_live_conversation(project_client, model: str) -> str: + """Create a `store=True` voice agent, hold one turn over a live realtime session, and + return the resulting conversation id. Only ever called when ``is_live()``. + + :param project_client: The Foundry project client. + :param model: The realtime model deployment name. + :type project_client: ~azure.ai.projects.AIProjectClient + :type model: str + :return: The persisted conversation id. + :rtype: str + """ + try: + project_client.agents.delete(agent_name=_AGENT_NAME) + except Exception: # pylint: disable=broad-except + pass + + project_client.agents.create_version( + agent_name=_AGENT_NAME, + definition=VoiceAgentDefinition( + model_type=VoiceModelType.MANAGED, + model=model, + instructions="You are a helpful voice assistant. Keep replies short.", + audio=VoiceAgentAudioConfig( + output=VoiceAgentAudioOutputConfig(voice="en-US-AvaNeural", voice_type="azure-standard") + ), + output_modalities=[VoiceOutputModality.AUDIO], + store=True, + ), + ) + + conversation_id: Optional[str] = None + with project_client.beta.realtime.connect(agent_name=_AGENT_NAME) as conn: + session_created = conn.recv(timeout=30) + assert isinstance(session_created, RealtimeServerEventSessionCreated) + conversation_id = session_created.conversation_id + + conn.conversation.item.create( + item=RealtimeConversationItemMessageUser( + type=RealtimeConversationItemType.MESSAGE, + content=[RealtimeConversationItemMessageUserContent(type="input_text", text="Say hello.")], + ) + ) + conn.response.create() + + deadline = time.monotonic() + 45 + while time.monotonic() < deadline: + event = conn.recv(timeout=30) + if isinstance(event, RealtimeServerEventResponseDone): + break + + assert conversation_id is not None, "Expected session.created to carry a conversation_id (store=True)" + time.sleep(_FINALIZATION_WAIT_SECONDS) + return conversation_id + + +class TestVoiceAgentConversations(TestBase): + """ + Recorded tests covering the read-only voice-agent conversation REST API surface exposed + through ``project_client.beta.agent_endpoint_conversations`` (conversation envelope, + responses, items, and audio). + + NOTE: The ``beta.agent_endpoint_conversations.get_item_generated_audio*`` + methods are intentionally NOT covered here: they return the played-back-interrupted + subordinate "generated" audio, which requires deliberately barging in mid-reply during a + live session to produce -- not exercised by the simple single-turn conversation created + here. See this package's engineering notes. + """ + + # To run only this test: + # pytest tests\agents\test_voice_agent_conversations.py::TestVoiceAgentConversations::test_read_conversation -s + @servicePreparer() + @recorded_by_proxy() + def test_read_conversation(self, **kwargs): # pylint: disable=too-many-locals + """ + Test reading back a persisted voice-agent conversation: the envelope, its responses + (with per-response output items), its ordered items (the transcript), the merged + whole-call audio recording, a single item's audio, and finally deleting the conversation. + + Routes used in this test: + + Action REST API Route Client Method + ------+-------------------------------------------------------------------------------+----------------------------------------------------------- + GET /agents/{agent_name}/endpoint/protocols/voice/conversations beta.agent_endpoint_conversations.list() + GET /agents/{agent_name}/endpoint/protocols/voice/conversations/{id} beta.agent_endpoint_conversations.get() + GET .../conversations/{id}/responses beta.agent_endpoint_conversations.list_responses() + GET .../conversations/{id}/responses/{response_id} beta.agent_endpoint_conversations.get_response() + GET .../conversations/{id}/responses/{response_id}/items beta.agent_endpoint_conversations.list_response_items() + GET .../conversations/{id}/items beta.agent_endpoint_conversations.list_items() + GET .../conversations/{id}/items/{item_id} beta.agent_endpoint_conversations.get_item() + GET .../conversations/{id}/audio beta.agent_endpoint_conversations.get_audio() + GET .../conversations/{id}/audio/content beta.agent_endpoint_conversations.download_audio() + GET .../conversations/{id}/items/{item_id}/audio beta.agent_endpoint_conversations.get_item_audio() + GET .../conversations/{id}/items/{item_id}/audio/content beta.agent_endpoint_conversations.download_item_audio() + DELETE .../conversations/{id} beta.agent_endpoint_conversations.delete() + """ + print("\n") + project_client = self.create_client(operation_group="agents", allow_preview=True, **kwargs) + conversations = project_client.beta.agent_endpoint_conversations + + if is_live(): + model = kwargs.get("foundry_voice_model_name") + assert model is not None + conversation_id = _create_live_conversation(project_client, model) + add_general_regex_sanitizer( + regex=re.escape(conversation_id), value="sanitized-conversation-id", function_scoped=True + ) + else: + conversation_id = "sanitized-conversation-id" + + try: + # The conversation should appear in the agent's conversation list. + found = any(c.id == conversation_id for c in conversations.list(_AGENT_NAME)) + assert found, "Expected the new conversation to appear in list" + + # The conversation envelope. + conversation = conversations.get(_AGENT_NAME, conversation_id) + assert conversation.id == conversation_id + assert conversation.status in ("in_progress", "completed", "failed") + assert conversation.created_at is not None + + # The responses (model inference turns) in the conversation. + responses = list(conversations.list_responses(_AGENT_NAME, conversation_id)) + assert len(responses) >= 1 + first_response = responses[0] + response_detail = conversations.get_response(_AGENT_NAME, conversation_id, first_response.id) + assert response_detail.id == first_response.id + + # The items produced by that response (does not raise; count may be 0 or more). + list(conversations.list_response_items(_AGENT_NAME, conversation_id, first_response.id)) + + # The ordered conversation items -- the full transcript (user + assistant + tool events). + items = list(conversations.list_items(_AGENT_NAME, conversation_id)) + assert len(items) >= 1 + first_item_id = items[0].get("id") + assert first_item_id + fetched_item = conversations.get_item(_AGENT_NAME, conversation_id, first_item_id) + assert fetched_item.get("id") == first_item_id + + # The merged whole-call recording and per-item audio. Completion is a hard requirement + # here (not a soft skip): a cassette recorded before the conversation finalized would + # otherwise let this test pass while silently never exercising any of the four audio + # methods below, hiding a regression in all of them (including permanently, if such a + # response were ever re-recorded). + assert ( + conversation.status == "completed" + ), f"Expected a completed conversation to exercise audio assertions, got {conversation.status!r}" + recording = conversations.get_audio(_AGENT_NAME, conversation_id) + assert recording.format is not None + if not recording.blob_uri: + audio_bytes = b"".join(conversations.download_audio(_AGENT_NAME, conversation_id)) + assert len(audio_bytes) > 0 + + # A single item's audio, if any item has one. + for item in items: + item_id = item.get("id") + if not item_id: + continue + try: + item_audio = conversations.get_item_audio(_AGENT_NAME, conversation_id, item_id) + except HttpResponseError as e: + if e.status_code == 404: + continue + raise + assert item_audio.role is not None + if not item_audio.blob_uri: + item_audio_bytes = b"".join( + conversations.download_item_audio(_AGENT_NAME, conversation_id, item_id) + ) + assert len(item_audio_bytes) > 0 + break + finally: + # Deleting a conversation removes it and all of its responses, items, and audio. + conversations.delete(_AGENT_NAME, conversation_id) + if is_live(): + project_client.agents.delete(agent_name=_AGENT_NAME) diff --git a/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_conversations_async.py b/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_conversations_async.py new file mode 100644 index 000000000000..2abde67e8c81 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_conversations_async.py @@ -0,0 +1,229 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression,too-many-statements,broad-exception-caught +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +""" +Recorded tests covering the read-only voice-agent conversation REST API surface exposed through +``project_client.beta.agent_endpoint_conversations`` (async client). + +Async counterpart of ``test_voice_agent_conversations.py``. See that module's docstring for the +overall rationale (live-only setup to obtain a real conversation id, sanitized to a fixed +placeholder so the recorded REST calls that follow can be replayed). +""" + +import re +import asyncio +import time +from typing import Final, Optional + +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live, add_general_regex_sanitizer +from devtools_testutils.aio import recorded_by_proxy_async +from azure.core.exceptions import HttpResponseError +from azure.ai.projects.models import ( + RealtimeConversationItemMessageUser, + RealtimeConversationItemMessageUserContent, + RealtimeConversationItemType, + RealtimeServerEventResponseDone, + RealtimeServerEventSessionCreated, + VoiceAgentAudioConfig, + VoiceAgentAudioOutputConfig, + VoiceAgentDefinition, + VoiceModelType, + VoiceOutputModality, +) + +# Fixed test-owned agent name: unlike conversation_id (server-generated, truly dynamic), this is +# our own choice and does not need is_live()/sanitizer handling -- it is identical in both modes. +_AGENT_NAME: Final = "test-conversations-read-agent-async" + +# Best-effort fixed wait (live only, seconds) after the realtime session ends, before reading the +# conversation back, so persistence finalization (items/audio) is more likely to have completed. +# This must be a single, fixed wait rather than a poll loop through the recorded client: repeated +# polling would record multiple cassette entries for the same "get conversation" request, but +# playback only ever issues that request once (polling itself is live-only), so a replay would +# incorrectly consume the *first* (possibly still "in_progress") recorded entry instead of the +# settled one. A single wait keeps exactly one logical call -- and therefore one cassette entry +# -- for both the live recording and the replay to agree on. +_FINALIZATION_WAIT_SECONDS: Final = 30 + + +async def _create_live_conversation(project_client, model: str) -> str: + """Create a `store=True` voice agent, hold one turn over a live realtime session, and + return the resulting conversation id. Only ever called when ``is_live()``. + + :param project_client: The Foundry project client. + :param model: The realtime model deployment name. + :type project_client: ~azure.ai.projects.aio.AIProjectClient + :type model: str + :return: The persisted conversation id. + :rtype: str + """ + try: + await project_client.agents.delete(agent_name=_AGENT_NAME) + except Exception: # pylint: disable=broad-except + pass + + await project_client.agents.create_version( + agent_name=_AGENT_NAME, + definition=VoiceAgentDefinition( + model_type=VoiceModelType.MANAGED, + model=model, + instructions="You are a helpful voice assistant. Keep replies short.", + audio=VoiceAgentAudioConfig( + output=VoiceAgentAudioOutputConfig(voice="en-US-AvaNeural", voice_type="azure-standard") + ), + output_modalities=[VoiceOutputModality.AUDIO], + store=True, + ), + ) + + conversation_id: Optional[str] = None + async with project_client.beta.realtime.connect(agent_name=_AGENT_NAME) as conn: + session_created = await asyncio.wait_for(conn.recv(), timeout=30) + assert isinstance(session_created, RealtimeServerEventSessionCreated) + conversation_id = session_created.conversation_id + + await conn.conversation.item.create( + item=RealtimeConversationItemMessageUser( + type=RealtimeConversationItemType.MESSAGE, + content=[RealtimeConversationItemMessageUserContent(type="input_text", text="Say hello.")], + ) + ) + await conn.response.create() + + got_response_done = False + deadline = time.monotonic() + 45 + while time.monotonic() < deadline and not got_response_done: + remaining = max(deadline - time.monotonic(), 0.1) + event = await asyncio.wait_for(conn.recv(), timeout=min(30, remaining)) + if isinstance(event, RealtimeServerEventResponseDone): + got_response_done = True + + assert conversation_id is not None, "Expected session.created to carry a conversation_id (store=True)" + await asyncio.sleep(_FINALIZATION_WAIT_SECONDS) + return conversation_id + + +class TestVoiceAgentConversationsAsync(TestBase): + """ + Recorded tests covering the read-only voice-agent conversation REST API surface exposed + through ``project_client.beta.agent_endpoint_conversations`` (conversation envelope, + responses, items, and audio), using the async client. + + NOTE: The ``beta.agent_endpoint_conversations.get_item_generated_audio*`` + methods are intentionally NOT covered here: they return the played-back-interrupted + subordinate "generated" audio, which requires deliberately barging in mid-reply during a + live session to produce -- not exercised by the simple single-turn conversation created + here. See this package's engineering notes. + """ + + # To run only this test: + # pytest tests\agents\test_voice_agent_conversations_async.py::TestVoiceAgentConversationsAsync::test_read_conversation_async -s + @servicePreparer() + @recorded_by_proxy_async() + async def test_read_conversation_async(self, **kwargs): # pylint: disable=too-many-locals + """ + Test reading back a persisted voice-agent conversation: the envelope, its responses + (with per-response output items), its ordered items (the transcript), the merged + whole-call audio recording, a single item's audio, and finally deleting the conversation. + + Routes used in this test: see the sync counterpart's docstring in + ``test_voice_agent_conversations.py`` for the full route table (identical here). + """ + print("\n") + project_client = self.create_async_client(operation_group="agents", allow_preview=True, **kwargs) + conversations = project_client.beta.agent_endpoint_conversations + + async with project_client: + if is_live(): + model = kwargs.get("foundry_voice_model_name") + assert model is not None + conversation_id = await _create_live_conversation(project_client, model) + add_general_regex_sanitizer( + regex=re.escape(conversation_id), value="sanitized-conversation-id", function_scoped=True + ) + else: + conversation_id = "sanitized-conversation-id" + + try: + # The conversation should appear in the agent's conversation list. + found = False + async for c in conversations.list(_AGENT_NAME): + if c.id == conversation_id: + found = True + break + assert found, "Expected the new conversation to appear in list" + + # The conversation envelope. + conversation = await conversations.get(_AGENT_NAME, conversation_id) + assert conversation.id == conversation_id + assert conversation.status in ("in_progress", "completed", "failed") + assert conversation.created_at is not None + + # The responses (model inference turns) in the conversation. + responses = [r async for r in conversations.list_responses(_AGENT_NAME, conversation_id)] + assert len(responses) >= 1 + first_response = responses[0] + response_detail = await conversations.get_response(_AGENT_NAME, conversation_id, first_response.id) + assert response_detail.id == first_response.id + + # The items produced by that response (does not raise; count may be 0 or more). + _ = [ + item + async for item in conversations.list_response_items(_AGENT_NAME, conversation_id, first_response.id) + ] + + # The ordered conversation items -- the full transcript (user + assistant + tool events). + items = [item async for item in conversations.list_items(_AGENT_NAME, conversation_id)] + assert len(items) >= 1 + first_item_id = items[0].get("id") + assert first_item_id + fetched_item = await conversations.get_item(_AGENT_NAME, conversation_id, first_item_id) + assert fetched_item.get("id") == first_item_id + + # The merged whole-call recording and per-item audio. Completion is a hard + # requirement here (not a soft skip): a cassette recorded before the conversation + # finalized would otherwise let this test pass while silently never exercising any + # of the four audio methods below, hiding a regression in all of them (including + # permanently, if such a response were ever re-recorded). + assert ( + conversation.status == "completed" + ), f"Expected a completed conversation to exercise audio assertions, got {conversation.status!r}" + recording = await conversations.get_audio(_AGENT_NAME, conversation_id) + assert recording.format is not None + if not recording.blob_uri: + audio_chunks = [ + chunk async for chunk in await conversations.download_audio(_AGENT_NAME, conversation_id) + ] + assert len(b"".join(audio_chunks)) > 0 + + # A single item's audio, if any item has one. + for item in items: + item_id = item.get("id") + if not item_id: + continue + try: + item_audio = await conversations.get_item_audio(_AGENT_NAME, conversation_id, item_id) + except HttpResponseError as e: + if e.status_code == 404: + continue + raise + assert item_audio.role is not None + if not item_audio.blob_uri: + item_audio_chunks = [ + chunk + async for chunk in await conversations.download_item_audio( + _AGENT_NAME, conversation_id, item_id + ) + ] + assert len(b"".join(item_audio_chunks)) > 0 + break + finally: + # Deleting a conversation removes it and all of its responses, items, and audio. + await conversations.delete(_AGENT_NAME, conversation_id) + if is_live(): + await project_client.agents.delete(agent_name=_AGENT_NAME) diff --git a/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_crud.py b/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_crud.py new file mode 100644 index 000000000000..500068db8b8d --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_crud.py @@ -0,0 +1,199 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +from test_base import TestBase, servicePreparer +from devtools_testutils import recorded_by_proxy, RecordedTransport +from azure.ai.projects.models import ( + AgentDetails, + AgentKind, + AgentVersionDetails, + GenerateVoiceAgentRequest, + VoiceAgentDefinition, + VoiceAgentAudioConfig, + VoiceAgentAudioOutputConfig, + VoiceOutputModality, +) + + +class TestVoiceAgentCrud(TestBase): + """ + Recorded tests covering the voice-agent (`kind="voice"`) REST API surface exposed through + `project_client.agents.*`. + + NOTE: Some voice-agent REST APIs are intentionally NOT covered here because they are + currently blocked by known service-side bugs (see this package's engineering notes): + - Reading back a conversation (`project_client.beta.agent_endpoint_conversations.*`) using a + `conversation_id` produced by a live realtime WebSocket session - the service's REST + conversation-ID validator rejects the ID format generated by the realtime WS subsystem. + This is also not practical to cover with HTTP-only recorded tests since it requires an + actual WebSocket session. + Once these are fixed service-side, tests can be added for them. + """ + + # To run only this test: + # pytest tests\agents\test_voice_agent_crud.py::TestVoiceAgentCrud::test_voice_agent_crud -s + @servicePreparer() + @recorded_by_proxy() + def test_voice_agent_crud(self, **kwargs): + """ + Test CRUD operations for voice Agents (`kind="voice"`). + + This test creates a voice agent, creates a new version of it, gets it, gets a specific + version, lists its versions, and deletes it. + + Routes used in this test: + + Action REST API Route Client Method + ------+---------------------------------------------+----------------------------------- + POST /agents/{agent_name}/versions project_client.agents.create_version() + GET /agents/{agent_name} project_client.agents.get() + GET /agents/{agent_name}/versions/{agent_version} project_client.agents.get_version() + GET /agents/{agent_name}/versions project_client.agents.list_versions() + DELETE /agents/{agent_name} project_client.agents.delete() + """ + print("\n") + model = kwargs.get("foundry_voice_model_name") + assert model is not None + # Voice-agent operations require the preview opt-in. + project_client = self.create_client(operation_group="agents", allow_preview=True, **kwargs) + agent_name = "MyVoiceAgentCrudTest" + + def make_definition(instructions: str) -> VoiceAgentDefinition: + return VoiceAgentDefinition( + model_type="managed", + model=model, + instructions=instructions, + audio=VoiceAgentAudioConfig( + output=VoiceAgentAudioOutputConfig(voice="en-US-AvaNeural", voice_type="azure-standard") + ), + output_modalities=[VoiceOutputModality.AUDIO], + ) + + # Create the initial voice agent (version 1). + agent_version1: AgentVersionDetails = project_client.agents.create_version( + agent_name=agent_name, + definition=make_definition("You are a helpful voice assistant."), + ) + self._validate_agent_version(agent_version1, expected_name=agent_name) + assert agent_version1.definition.kind == "voice" # type: ignore[attr-defined] + + # Create a new version with updated instructions. + agent_version2: AgentVersionDetails = project_client.agents.create_version( + agent_name=agent_name, + definition=make_definition("You are a helpful voice assistant. Always greet the caller by name."), + ) + self._validate_agent_version(agent_version2, expected_name=agent_name) + + # Get the voice agent. + retrieved_agent: AgentDetails = project_client.agents.get(agent_name=agent_name) + self._validate_agent(retrieved_agent, expected_name=agent_name, expected_latest_version=agent_version2.version) + + # Retrieve a specific version. + retrieved_agent_version: AgentVersionDetails = project_client.agents.get_version( + agent_name=agent_name, agent_version=agent_version1.version + ) + self._validate_agent_version( + retrieved_agent_version, expected_name=agent_name, expected_version=agent_version1.version + ) + + # List all versions. + item_count = 0 + for listed_agent_version in project_client.agents.list_versions(agent_name=agent_name): + item_count += 1 + self._validate_agent_version(listed_agent_version, expected_name=agent_name) + assert item_count >= 2 + + # Delete the voice agent. + result = project_client.agents.delete(agent_name=agent_name) + assert result.deleted + + # To run only this test: + # pytest tests\agents\test_voice_agent_crud.py::TestVoiceAgentCrud::test_voice_agent_disable_enable -s + @servicePreparer() + @recorded_by_proxy(RecordedTransport.AZURE_CORE, RecordedTransport.HTTPX2) + def test_voice_agent_disable_enable(self, **kwargs): + """ + Test disable and enable operations for a voice Agent. + + Routes used in this test: + + Action REST API Route Client Method + ------+---------------------------------------------+----------------------------------- + POST /agents/{agent_name}/versions project_client.agents.create_version() + POST /agents/{agent_name}:disable project_client.agents.disable() + POST /agents/{agent_name}:enable project_client.agents.enable() + DELETE /agents/{agent_name} project_client.agents.delete() + """ + print("\n") + model = kwargs.get("foundry_voice_model_name") + assert model is not None + project_client = self.create_client(operation_group="agents", allow_preview=True, **kwargs) + agent_name = "VoiceAgentDisableEnableTest" + + # Delete any existing agent from previous test runs (ignore failures) + try: + project_client.agents.delete(agent_name=agent_name) + except Exception: # pylint: disable=broad-except + pass + + agent_version: AgentVersionDetails = project_client.agents.create_version( + agent_name=agent_name, + definition=VoiceAgentDefinition( + model_type="managed", + model=model, + instructions="You are a helpful voice assistant.", + audio=VoiceAgentAudioConfig( + output=VoiceAgentAudioOutputConfig(voice="en-US-AvaNeural", voice_type="azure-standard") + ), + output_modalities=[VoiceOutputModality.AUDIO], + ), + ) + self._validate_agent_version(agent_version, expected_name=agent_name) + + # Disable the agent. + project_client.agents.disable(agent_name=agent_name) + disabled_agent: AgentDetails = project_client.agents.get(agent_name=agent_name) + assert str(disabled_agent.state) == "AgentState.DISABLED" or disabled_agent.state == "disabled" + + # Enable the agent. + project_client.agents.enable(agent_name=agent_name) + enabled_agent: AgentDetails = project_client.agents.get(agent_name=agent_name) + assert str(enabled_agent.state) == "AgentState.ENABLED" or enabled_agent.state == "enabled" + + # Delete the voice agent. + result = project_client.agents.delete(agent_name=agent_name) + assert result.deleted + + # To run only this test: + # pytest tests\agents\test_voice_agent_crud.py::TestVoiceAgentCrud::test_generate_agent -s + @servicePreparer() + @recorded_by_proxy() + def test_generate_agent(self, **kwargs): + """ + Test guided authoring for a voice Agent via `beta.agents.generate()`. + + Routes used in this test: + + Action REST API Route Client Method + ------+----------------------------+----------------------------------- + POST /agents:generate project_client.beta.agents.generate() + DELETE /agents/{agent_name} project_client.agents.delete() + """ + print("\n") + project_client = self.create_client(operation_group="agents", allow_preview=True, **kwargs) + agent_name = "VoiceAgentGenerateTest" + + agent: AgentDetails = project_client.beta.agents.generate( + GenerateVoiceAgentRequest(kind=AgentKind.VOICE, name=agent_name) + ) + self._validate_agent(agent, expected_name=agent_name) + assert agent.versions.latest.definition.kind == "voice" # type: ignore[attr-defined] + assert agent.versions.latest.definition.instructions # type: ignore[attr-defined] + + # Delete the voice agent. + result = project_client.agents.delete(agent_name=agent_name) + assert result.deleted diff --git a/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_crud_async.py b/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_crud_async.py new file mode 100644 index 000000000000..cb5213413b06 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_crud_async.py @@ -0,0 +1,205 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +from test_base import TestBase, servicePreparer +from devtools_testutils.aio import recorded_by_proxy_async +from devtools_testutils import RecordedTransport +from azure.ai.projects.models import ( + AgentDetails, + AgentKind, + AgentVersionDetails, + GenerateVoiceAgentRequest, + VoiceAgentDefinition, + VoiceAgentAudioConfig, + VoiceAgentAudioOutputConfig, + VoiceOutputModality, +) + + +class TestVoiceAgentCrudAsync(TestBase): + """ + Recorded tests covering the voice-agent (`kind="voice"`) REST API surface exposed through + `project_client.agents.*`. + + NOTE: Some voice-agent REST APIs are intentionally NOT covered here because they are + currently blocked by known service-side bugs (see this package's engineering notes): + - Reading back a conversation (`project_client.beta.agent_endpoint_conversations.*`) using a + `conversation_id` produced by a live realtime WebSocket session - the service's REST + conversation-ID validator rejects the ID format generated by the realtime WS subsystem. + This is also not practical to cover with HTTP-only recorded tests since it requires an + actual WebSocket session. + Once these are fixed service-side, tests can be added for them. + """ + + # To run only this test: + # pytest tests\agents\test_voice_agent_crud_async.py::TestVoiceAgentCrudAsync::test_voice_agent_crud_async -s + @servicePreparer() + @recorded_by_proxy_async() + async def test_voice_agent_crud_async(self, **kwargs): + """ + Test CRUD operations for voice Agents (`kind="voice"`). + + This test creates a voice agent, creates a new version of it, gets it, gets a specific + version, lists its versions, and deletes it. + + Routes used in this test: + + Action REST API Route Client Method + ------+---------------------------------------------+----------------------------------- + POST /agents/{agent_name}/versions project_client.agents.create_version() + GET /agents/{agent_name} project_client.agents.get() + GET /agents/{agent_name}/versions/{agent_version} project_client.agents.get_version() + GET /agents/{agent_name}/versions project_client.agents.list_versions() + DELETE /agents/{agent_name} project_client.agents.delete() + """ + print("\n") + model = kwargs.get("foundry_voice_model_name") + assert model is not None + # Voice-agent operations require the preview opt-in. + project_client = self.create_async_client(operation_group="agents", allow_preview=True, **kwargs) + agent_name = "MyVoiceAgentCrudTestAsync" + + def make_definition(instructions: str) -> VoiceAgentDefinition: + return VoiceAgentDefinition( + model_type="managed", + model=model, + instructions=instructions, + audio=VoiceAgentAudioConfig( + output=VoiceAgentAudioOutputConfig(voice="en-US-AvaNeural", voice_type="azure-standard") + ), + output_modalities=[VoiceOutputModality.AUDIO], + ) + + async with project_client: + # Create the initial voice agent (version 1). + agent_version1: AgentVersionDetails = await project_client.agents.create_version( + agent_name=agent_name, + definition=make_definition("You are a helpful voice assistant."), + ) + self._validate_agent_version(agent_version1, expected_name=agent_name) + assert agent_version1.definition.kind == "voice" # type: ignore[attr-defined] + + # Create a new version with updated instructions. + agent_version2: AgentVersionDetails = await project_client.agents.create_version( + agent_name=agent_name, + definition=make_definition("You are a helpful voice assistant. Always greet the caller by name."), + ) + self._validate_agent_version(agent_version2, expected_name=agent_name) + + # Get the voice agent. + retrieved_agent: AgentDetails = await project_client.agents.get(agent_name=agent_name) + self._validate_agent( + retrieved_agent, expected_name=agent_name, expected_latest_version=agent_version2.version + ) + + # Retrieve a specific version. + retrieved_agent_version: AgentVersionDetails = await project_client.agents.get_version( + agent_name=agent_name, agent_version=agent_version1.version + ) + self._validate_agent_version( + retrieved_agent_version, expected_name=agent_name, expected_version=agent_version1.version + ) + + # List all versions. + item_count = 0 + async for listed_agent_version in project_client.agents.list_versions(agent_name=agent_name): + item_count += 1 + self._validate_agent_version(listed_agent_version, expected_name=agent_name) + assert item_count >= 2 + + # Delete the voice agent. + result = await project_client.agents.delete(agent_name=agent_name) + assert result.deleted + + # To run only this test: + # pytest tests\agents\test_voice_agent_crud_async.py::TestVoiceAgentCrudAsync::test_voice_agent_disable_enable_async -s + @servicePreparer() + @recorded_by_proxy_async(RecordedTransport.AZURE_CORE, RecordedTransport.HTTPX2) + async def test_voice_agent_disable_enable_async(self, **kwargs): + """ + Test disable and enable operations for a voice Agent. + + Routes used in this test: + + Action REST API Route Client Method + ------+---------------------------------------------+----------------------------------- + POST /agents/{agent_name}/versions project_client.agents.create_version() + POST /agents/{agent_name}:disable project_client.agents.disable() + POST /agents/{agent_name}:enable project_client.agents.enable() + DELETE /agents/{agent_name} project_client.agents.delete() + """ + print("\n") + model = kwargs.get("foundry_voice_model_name") + assert model is not None + project_client = self.create_async_client(operation_group="agents", allow_preview=True, **kwargs) + agent_name = "VoiceAgentDisableEnableTestAsync" + + async with project_client: + # Delete any existing agent from previous test runs (ignore failures) + try: + await project_client.agents.delete(agent_name=agent_name) + except Exception: # pylint: disable=broad-except + pass + + agent_version: AgentVersionDetails = await project_client.agents.create_version( + agent_name=agent_name, + definition=VoiceAgentDefinition( + model_type="managed", + model=model, + instructions="You are a helpful voice assistant.", + audio=VoiceAgentAudioConfig( + output=VoiceAgentAudioOutputConfig(voice="en-US-AvaNeural", voice_type="azure-standard") + ), + output_modalities=[VoiceOutputModality.AUDIO], + ), + ) + self._validate_agent_version(agent_version, expected_name=agent_name) + + # Disable the agent. + await project_client.agents.disable(agent_name=agent_name) + disabled_agent: AgentDetails = await project_client.agents.get(agent_name=agent_name) + assert str(disabled_agent.state) == "AgentState.DISABLED" or disabled_agent.state == "disabled" + + # Enable the agent. + await project_client.agents.enable(agent_name=agent_name) + enabled_agent: AgentDetails = await project_client.agents.get(agent_name=agent_name) + assert str(enabled_agent.state) == "AgentState.ENABLED" or enabled_agent.state == "enabled" + + # Delete the voice agent. + result = await project_client.agents.delete(agent_name=agent_name) + assert result.deleted + + # To run only this test: + # pytest tests\agents\test_voice_agent_crud_async.py::TestVoiceAgentCrudAsync::test_generate_agent_async -s + @servicePreparer() + @recorded_by_proxy_async() + async def test_generate_agent_async(self, **kwargs): + """ + Test guided authoring for a voice Agent via `beta.agents.generate()`. + + Routes used in this test: + + Action REST API Route Client Method + ------+----------------------------+----------------------------------- + POST /agents:generate project_client.beta.agents.generate() + DELETE /agents/{agent_name} project_client.agents.delete() + """ + print("\n") + project_client = self.create_async_client(operation_group="agents", allow_preview=True, **kwargs) + agent_name = "VoiceAgentGenerateTestAsync" + + async with project_client: + agent: AgentDetails = await project_client.beta.agents.generate( + GenerateVoiceAgentRequest(kind=AgentKind.VOICE, name=agent_name) + ) + self._validate_agent(agent, expected_name=agent_name) + assert agent.versions.latest.definition.kind == "voice" # type: ignore[attr-defined] + assert agent.versions.latest.definition.instructions # type: ignore[attr-defined] + + # Delete the voice agent. + result = await project_client.agents.delete(agent_name=agent_name) + assert result.deleted diff --git a/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_realtime_live.py b/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_realtime_live.py new file mode 100644 index 000000000000..94b8b596c6f7 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_realtime_live.py @@ -0,0 +1,299 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +""" +Live-only tests for the hand-written sync ``client.beta.realtime`` WebSocket streaming client. + +Unlike ``tests/agents/test_realtime_client.py`` (which mocks the transport to unit-test URL +construction, auth, and error paths without a live service), these tests open a REAL WebSocket +connection to a live voice agent and assert on the actual streamed server events. They are +modeled on the live realtime test pattern used by the ``azure-ai-voicelive`` package +(``sdk/voicelive/azure-ai-voicelive/tests/live/``): skip entirely unless running live, use +generous per-event timeouts, and assert on event *types* and content presence/length rather than +exact audio bytes (the model's actual audio/text output is not deterministic). + +These tests do not use ``store=True`` / read back a persisted conversation -- that surface +(``project_client.beta.agent_endpoint_conversations.*``) is covered by the separate recorded +tests in ``test_voice_agent_conversations.py``, which need a real conversation id but replay +against a recorded cassette rather than opening a live WebSocket connection on every run. +""" + +import json +import time +from typing import Any, cast, Final, List, Tuple + +import pytest +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live +from azure.ai.projects.models import ( + RealtimeConversationItemFunctionCallOutput, + RealtimeConversationItemMessageUser, + RealtimeConversationItemMessageUserContent, + RealtimeConversationItemType, + RealtimeServerEventError, + RealtimeServerEventResponseAudioDelta, + RealtimeServerEventResponseAudioTranscriptDone, + RealtimeServerEventResponseDone, + RealtimeServerEventResponseFunctionCallArgumentsDone, + RealtimeServerEventResponseTextDone, + RealtimeServerEventSessionCreated, + VoiceAgentAudioConfig, + VoiceAgentAudioOutputConfig, + VoiceAgentDefinition, + VoiceAgentFunctionTool, + VoiceModelType, + VoiceOutputModality, +) + +# Seconds to wait for a single server event (session handshake, an audio delta, ...). +_EVENT_TIMEOUT: Final = 30 +# Seconds to wait for a full response turn to finish (may include a tool round-trip). +_RESPONSE_TIMEOUT: Final = 45 + + +def _get_weather(city: str) -> str: + """A trivial local "tool" implementation the agent can call. + + :param city: The city to look up. + :type city: str + :return: A canned weather report for the city. + :rtype: str + """ + return json.dumps({"city": city, "condition": "sunny", "temperature_f": 72}) + + +@pytest.mark.live_test_only +@pytest.mark.skipif( + not is_live(), + reason="Live-only: opens a real WebSocket connection to the realtime service, which cannot " + "be captured/replayed by the test proxy.", +) +class TestVoiceAgentRealtimeLive(TestBase): + """ + Live tests covering ``client.beta.realtime.connect()`` (the hand-written sync WebSocket streaming + client) against a real voice agent and a real service connection. + """ + + def _make_agent_name(self, suffix: str) -> str: + return f"test-realtime-live-{suffix}" + + def _create_basic_agent(self, project_client, agent_name: str, model: str) -> None: + project_client.agents.create_version( + agent_name=agent_name, + definition=VoiceAgentDefinition( + model_type=VoiceModelType.MANAGED, + model=model, + instructions="You are a helpful voice assistant. Keep replies short.", + audio=VoiceAgentAudioConfig( + output=VoiceAgentAudioOutputConfig(voice="en-US-AvaNeural", voice_type="azure-standard") + ), + output_modalities=[VoiceOutputModality.AUDIO], + ), + ) + + # To run only this test: + # pytest tests\agents\test_voice_agent_realtime_live.py::TestVoiceAgentRealtimeLive::test_realtime_session_lifecycle -s + @servicePreparer() + def test_realtime_session_lifecycle(self, **kwargs): + """ + Test opening and cleanly closing a realtime WebSocket session, and receiving the initial + ``session.created`` handshake event. + """ + print("\n") + model = kwargs.get("foundry_voice_model_name") + assert model is not None + project_client = self.create_client(operation_group="agents", allow_preview=True, **kwargs) + agent_name = self._make_agent_name("lifecycle") + + try: + self._create_basic_agent(project_client, agent_name, model) + + with project_client.beta.realtime.connect(agent_name=agent_name) as conn: + event = conn.recv(timeout=_EVENT_TIMEOUT) + assert isinstance(event, RealtimeServerEventSessionCreated) + assert event.type == "session.created" + # The `with` block above closes the connection; a second `recv()` after close + # would raise, so we don't attempt one -- clean exit from the block is the assertion. + finally: + project_client.agents.delete(agent_name=agent_name) + + # To run only this test: + # pytest tests\agents\test_voice_agent_realtime_live.py::TestVoiceAgentRealtimeLive::test_realtime_text_turn_produces_audio_and_transcript -s + @servicePreparer() + def test_realtime_text_turn_produces_audio_and_transcript(self, **kwargs): + """ + Test sending one typed user turn and receiving a streamed audio + transcript reply. + + Sends a ``RealtimeConversationItemMessageUser`` text turn and asserts that the service + streams back at least one non-empty audio delta, a transcript-done event with non-empty + text, and a final ``response.done``. Content is not asserted verbatim (the model's actual + wording is not deterministic); only event types, ordering-independent presence, and basic + size/non-emptiness are checked, matching the ``azure-ai-voicelive`` live test convention. + """ + print("\n") + model = kwargs.get("foundry_voice_model_name") + assert model is not None + project_client = self.create_client(operation_group="agents", allow_preview=True, **kwargs) + agent_name = self._make_agent_name("text-turn") + + try: + self._create_basic_agent(project_client, agent_name, model) + + with project_client.beta.realtime.connect(agent_name=agent_name) as conn: + session_created = conn.recv(timeout=_EVENT_TIMEOUT) + assert isinstance(session_created, RealtimeServerEventSessionCreated) + + conn.conversation.item.create( + item=RealtimeConversationItemMessageUser( + type=RealtimeConversationItemType.MESSAGE, + content=[ + RealtimeConversationItemMessageUserContent( + type="input_text", text="Say the word 'hello' and nothing else." + ) + ], + ) + ) + conn.response.create() + + audio_delta_count = 0 + audio_bytes = 0 + transcript_done_count = 0 + got_response_done = False + deadline = time.monotonic() + _RESPONSE_TIMEOUT + + while time.monotonic() < deadline and not got_response_done: + event = conn.recv(timeout=_EVENT_TIMEOUT) + if isinstance(event, RealtimeServerEventResponseAudioDelta): + audio_delta_count += 1 + audio_bytes += len(event.delta) + elif isinstance(event, RealtimeServerEventResponseAudioTranscriptDone): + transcript_done_count += 1 + assert event.transcript is not None and len(event.transcript.strip()) > 0 + elif isinstance(event, RealtimeServerEventResponseDone): + got_response_done = True + elif isinstance(event, RealtimeServerEventError): + pytest.fail(f"Session error: {event.error.message}") + + assert got_response_done, "Did not receive response.done within the timeout" + assert audio_delta_count > 0, "Expected at least one response.audio.delta event" + assert audio_bytes > 0, "Expected non-empty streamed audio" + assert transcript_done_count == 1, "Expected exactly one audio-transcript-done event" + finally: + project_client.agents.delete(agent_name=agent_name) + + # To run only this test: + # pytest tests\agents\test_voice_agent_realtime_live.py::TestVoiceAgentRealtimeLive::test_realtime_function_tool_call -s + @servicePreparer() + def test_realtime_function_tool_call(self, **kwargs): + """ + Test a client-executed function-tool round trip during a live realtime session. + + Configures the agent with a ``get_weather`` function tool, sends a prompt that should + trigger it, executes the tool call locally when the service asks for it, and sends the + result back so the agent can finish its reply -- mirroring + ``samples/agents/voice/sample_voice_agent_live_function_tool.py``, which this test + adapts into an automated assertion-based form. + """ + print("\n") + model = kwargs.get("foundry_voice_model_name") + assert model is not None + project_client = self.create_client(operation_group="agents", allow_preview=True, **kwargs) + agent_name = self._make_agent_name("tool-call") + + get_weather_tool = VoiceAgentFunctionTool( + name="get_weather", + description="Get the current weather for a city.", + parameters=cast( + Any, + { + "type": "object", + "properties": {"city": {"type": "string", "description": "City name, e.g. Seattle."}}, + "required": ["city"], + }, + ), + ) + + try: + project_client.agents.create_version( + agent_name=agent_name, + definition=VoiceAgentDefinition( + model_type=VoiceModelType.MANAGED, + model=model, + instructions=( + "You are a helpful voice assistant. Use the get_weather tool when the " + "caller asks about the weather, then answer using its result." + ), + output_modalities=[VoiceOutputModality.TEXT], + tools=[get_weather_tool], + ), + ) + + with project_client.beta.realtime.connect(agent_name=agent_name) as conn: + session_created = conn.recv(timeout=_EVENT_TIMEOUT) + assert isinstance(session_created, RealtimeServerEventSessionCreated) + + conn.conversation.item.create( + item=RealtimeConversationItemMessageUser( + type=RealtimeConversationItemType.MESSAGE, + content=[ + RealtimeConversationItemMessageUserContent( + type="input_text", text="What's the weather like in Seattle right now?" + ) + ], + ) + ) + conn.response.create() + + tool_call_count = 0 + final_text = "" + deadline = time.monotonic() + _RESPONSE_TIMEOUT + done = False + # Tool outputs collected from the current turn's function-call(s). These are held + # back and only sent once this turn's own response.done arrives (below) -- calling + # response.create() while the function-call response is still finishing can + # otherwise race with the service and produce a concurrent-response error. + pending_tool_outputs: List[Tuple[str, str]] = [] + + while time.monotonic() < deadline and not done: + event = conn.recv(timeout=_EVENT_TIMEOUT) + if isinstance(event, RealtimeServerEventResponseFunctionCallArgumentsDone): + tool_call_count += 1 + assert event.name == "get_weather" + args = json.loads(event.arguments) + assert "city" in args + result = _get_weather(**args) + pending_tool_outputs.append((event.call_id, result)) + elif isinstance(event, RealtimeServerEventResponseTextDone): + final_text = event.text + elif isinstance(event, RealtimeServerEventResponseDone): + # A response.done that isn't itself a function call is the final answer. + # Output items surface as plain mappings (open union) or typed models. + output = event.response.output or [] + is_function_call = any( + (item.get("type") if isinstance(item, dict) else getattr(item, "type", None)) + == "function_call" + for item in output + ) + if pending_tool_outputs: + # The function-call response has now fully completed, so it's safe to + # submit its tool output(s) and ask for a new response. + for call_id, result in pending_tool_outputs: + conn.conversation.item.create( + item=RealtimeConversationItemFunctionCallOutput(call_id=call_id, output=result) + ) + pending_tool_outputs = [] + conn.response.create() + elif not is_function_call: + done = True + elif isinstance(event, RealtimeServerEventError): + pytest.fail(f"Session error: {event.error.message}") + + assert done, "Did not receive a final (non-tool-call) response.done within the timeout" + assert tool_call_count >= 1, "Expected the agent to invoke the get_weather tool at least once" + assert final_text is not None and len(final_text.strip()) > 0 + finally: + project_client.agents.delete(agent_name=agent_name) diff --git a/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_realtime_live_async.py b/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_realtime_live_async.py new file mode 100644 index 000000000000..6c55a3d1cd3f --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_realtime_live_async.py @@ -0,0 +1,297 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +""" +Live-only tests for the hand-written async ``async_client.beta.realtime`` WebSocket streaming client. + +Async counterpart of ``test_voice_agent_realtime_live.py``. See that module's docstring for the +overall rationale (modeled on the ``azure-ai-voicelive`` package's live realtime test pattern: +skip entirely unless running live, generous per-event timeouts, assert on event types and +content presence/length rather than exact audio bytes). +""" + +import asyncio +import json +import time +from typing import Any, cast, Final, List, Tuple + +import pytest +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live +from azure.ai.projects.models import ( + RealtimeConversationItemFunctionCallOutput, + RealtimeConversationItemMessageUser, + RealtimeConversationItemMessageUserContent, + RealtimeConversationItemType, + RealtimeServerEventError, + RealtimeServerEventResponseAudioDelta, + RealtimeServerEventResponseAudioTranscriptDone, + RealtimeServerEventResponseDone, + RealtimeServerEventResponseFunctionCallArgumentsDone, + RealtimeServerEventResponseTextDone, + RealtimeServerEventSessionCreated, + VoiceAgentAudioConfig, + VoiceAgentAudioOutputConfig, + VoiceAgentDefinition, + VoiceAgentFunctionTool, + VoiceModelType, + VoiceOutputModality, +) + +# Seconds to wait for a single server event (session handshake, an audio delta, ...). +_EVENT_TIMEOUT: Final = 30 +# Seconds to wait for a full response turn to finish (may include a tool round-trip). +_RESPONSE_TIMEOUT: Final = 45 + + +def _get_weather(city: str) -> str: + """A trivial local "tool" implementation the agent can call. + + :param city: The city to look up. + :type city: str + :return: A canned weather report for the city. + :rtype: str + """ + return json.dumps({"city": city, "condition": "sunny", "temperature_f": 72}) + + +@pytest.mark.live_test_only +@pytest.mark.skipif( + not is_live(), + reason="Live-only: opens a real WebSocket connection to the realtime service, which cannot " + "be captured/replayed by the test proxy.", +) +class TestVoiceAgentRealtimeLiveAsync(TestBase): + """ + Live tests covering ``async_client.beta.realtime.connect()`` (the hand-written async WebSocket + streaming client) against a real voice agent and a real service connection. + """ + + def _make_agent_name(self, suffix: str) -> str: + return f"test-realtime-live-async-{suffix}" + + async def _create_basic_agent(self, project_client, agent_name: str, model: str) -> None: + await project_client.agents.create_version( + agent_name=agent_name, + definition=VoiceAgentDefinition( + model_type=VoiceModelType.MANAGED, + model=model, + instructions="You are a helpful voice assistant. Keep replies short.", + audio=VoiceAgentAudioConfig( + output=VoiceAgentAudioOutputConfig(voice="en-US-AvaNeural", voice_type="azure-standard") + ), + output_modalities=[VoiceOutputModality.AUDIO], + ), + ) + + # To run only this test: + # pytest tests\agents\test_voice_agent_realtime_live_async.py::TestVoiceAgentRealtimeLiveAsync::test_realtime_session_lifecycle_async -s + @servicePreparer() + async def test_realtime_session_lifecycle_async(self, **kwargs): + """ + Test opening and cleanly closing a realtime WebSocket session, and receiving the initial + ``session.created`` handshake event. + """ + print("\n") + model = kwargs.get("foundry_voice_model_name") + assert model is not None + project_client = self.create_async_client(operation_group="agents", allow_preview=True, **kwargs) + agent_name = self._make_agent_name("lifecycle") + + try: + await self._create_basic_agent(project_client, agent_name, model) + + async with project_client.beta.realtime.connect(agent_name=agent_name) as conn: + event = await asyncio.wait_for(conn.recv(), timeout=_EVENT_TIMEOUT) + assert isinstance(event, RealtimeServerEventSessionCreated) + assert event.type == "session.created" + # The `async with` block above closes the connection; a second `recv()` after close + # would raise, so we don't attempt one -- clean exit from the block is the assertion. + finally: + await project_client.agents.delete(agent_name=agent_name) + await project_client.close() + + # To run only this test: + # pytest tests\agents\test_voice_agent_realtime_live_async.py::TestVoiceAgentRealtimeLiveAsync::test_realtime_text_turn_produces_audio_and_transcript_async -s + @servicePreparer() + async def test_realtime_text_turn_produces_audio_and_transcript_async(self, **kwargs): + """ + Test sending one typed user turn and receiving a streamed audio + transcript reply. + + Sends a ``RealtimeConversationItemMessageUser`` text turn and asserts that the service + streams back at least one non-empty audio delta, a transcript-done event with non-empty + text, and a final ``response.done``. Content is not asserted verbatim (the model's actual + wording is not deterministic); only event types, ordering-independent presence, and basic + size/non-emptiness are checked, matching the ``azure-ai-voicelive`` live test convention. + """ + print("\n") + model = kwargs.get("foundry_voice_model_name") + assert model is not None + project_client = self.create_async_client(operation_group="agents", allow_preview=True, **kwargs) + agent_name = self._make_agent_name("text-turn") + + try: + await self._create_basic_agent(project_client, agent_name, model) + + async with project_client.beta.realtime.connect(agent_name=agent_name) as conn: + session_created = await asyncio.wait_for(conn.recv(), timeout=_EVENT_TIMEOUT) + assert isinstance(session_created, RealtimeServerEventSessionCreated) + + await conn.conversation.item.create( + item=RealtimeConversationItemMessageUser( + type=RealtimeConversationItemType.MESSAGE, + content=[ + RealtimeConversationItemMessageUserContent( + type="input_text", text="Say the word 'hello' and nothing else." + ) + ], + ) + ) + await conn.response.create() + + audio_delta_count = 0 + audio_bytes = 0 + transcript_done_count = 0 + got_response_done = False + deadline = time.monotonic() + _RESPONSE_TIMEOUT + + while time.monotonic() < deadline and not got_response_done: + remaining = max(deadline - time.monotonic(), 0.1) + event = await asyncio.wait_for(conn.recv(), timeout=min(_EVENT_TIMEOUT, remaining)) + if isinstance(event, RealtimeServerEventResponseAudioDelta): + audio_delta_count += 1 + audio_bytes += len(event.delta) + elif isinstance(event, RealtimeServerEventResponseAudioTranscriptDone): + transcript_done_count += 1 + assert event.transcript is not None and len(event.transcript.strip()) > 0 + elif isinstance(event, RealtimeServerEventResponseDone): + got_response_done = True + elif isinstance(event, RealtimeServerEventError): + pytest.fail(f"Session error: {event.error.message}") + + assert got_response_done, "Did not receive response.done within the timeout" + assert audio_delta_count > 0, "Expected at least one response.audio.delta event" + assert audio_bytes > 0, "Expected non-empty streamed audio" + assert transcript_done_count == 1, "Expected exactly one audio-transcript-done event" + finally: + await project_client.agents.delete(agent_name=agent_name) + await project_client.close() + + # To run only this test: + # pytest tests\agents\test_voice_agent_realtime_live_async.py::TestVoiceAgentRealtimeLiveAsync::test_realtime_function_tool_call_async -s + @servicePreparer() + async def test_realtime_function_tool_call_async(self, **kwargs): + """ + Test a client-executed function-tool round trip during a live realtime session. + + Configures the agent with a ``get_weather`` function tool, sends a prompt that should + trigger it, executes the tool call locally when the service asks for it, and sends the + result back so the agent can finish its reply -- the async counterpart of + ``sample_voice_agent_live_function_tool.py``'s pattern, adapted into an automated + assertion-based test. + """ + print("\n") + model = kwargs.get("foundry_voice_model_name") + assert model is not None + project_client = self.create_async_client(operation_group="agents", allow_preview=True, **kwargs) + agent_name = self._make_agent_name("tool-call") + + get_weather_tool = VoiceAgentFunctionTool( + name="get_weather", + description="Get the current weather for a city.", + parameters=cast( + Any, + { + "type": "object", + "properties": {"city": {"type": "string", "description": "City name, e.g. Seattle."}}, + "required": ["city"], + }, + ), + ) + + try: + await project_client.agents.create_version( + agent_name=agent_name, + definition=VoiceAgentDefinition( + model_type=VoiceModelType.MANAGED, + model=model, + instructions=( + "You are a helpful voice assistant. Use the get_weather tool when the " + "caller asks about the weather, then answer using its result." + ), + output_modalities=[VoiceOutputModality.TEXT], + tools=[get_weather_tool], + ), + ) + + async with project_client.beta.realtime.connect(agent_name=agent_name) as conn: + session_created = await asyncio.wait_for(conn.recv(), timeout=_EVENT_TIMEOUT) + assert isinstance(session_created, RealtimeServerEventSessionCreated) + + await conn.conversation.item.create( + item=RealtimeConversationItemMessageUser( + type=RealtimeConversationItemType.MESSAGE, + content=[ + RealtimeConversationItemMessageUserContent( + type="input_text", text="What's the weather like in Seattle right now?" + ) + ], + ) + ) + await conn.response.create() + + tool_call_count = 0 + final_text = "" + deadline = time.monotonic() + _RESPONSE_TIMEOUT + done = False + # Tool outputs collected from the current turn's function-call(s). These are held + # back and only sent once this turn's own response.done arrives (below) -- calling + # response.create() while the function-call response is still finishing can + # otherwise race with the service and produce a concurrent-response error. + pending_tool_outputs: List[Tuple[str, str]] = [] + + while time.monotonic() < deadline and not done: + remaining = max(deadline - time.monotonic(), 0.1) + event = await asyncio.wait_for(conn.recv(), timeout=min(_EVENT_TIMEOUT, remaining)) + if isinstance(event, RealtimeServerEventResponseFunctionCallArgumentsDone): + tool_call_count += 1 + assert event.name == "get_weather" + args = json.loads(event.arguments) + assert "city" in args + result = _get_weather(**args) + pending_tool_outputs.append((event.call_id, result)) + elif isinstance(event, RealtimeServerEventResponseTextDone): + final_text = event.text + elif isinstance(event, RealtimeServerEventResponseDone): + # A response.done that isn't itself a function call is the final answer. + # Output items surface as plain mappings (open union) or typed models. + output = event.response.output or [] + is_function_call = any( + (item.get("type") if isinstance(item, dict) else getattr(item, "type", None)) + == "function_call" + for item in output + ) + if pending_tool_outputs: + # The function-call response has now fully completed, so it's safe to + # submit its tool output(s) and ask for a new response. + for call_id, result in pending_tool_outputs: + await conn.conversation.item.create( + item=RealtimeConversationItemFunctionCallOutput(call_id=call_id, output=result) + ) + pending_tool_outputs = [] + await conn.response.create() + elif not is_function_call: + done = True + elif isinstance(event, RealtimeServerEventError): + pytest.fail(f"Session error: {event.error.message}") + + assert done, "Did not receive a final (non-tool-call) response.done within the timeout" + assert tool_call_count >= 1, "Expected the agent to invoke the get_weather tool at least once" + assert final_text is not None and len(final_text.strip()) > 0 + finally: + await project_client.agents.delete(agent_name=agent_name) + await project_client.close() diff --git a/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_telephony.py b/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_telephony.py new file mode 100644 index 000000000000..4b1f6a78e234 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_telephony.py @@ -0,0 +1,310 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +from test_base import TestBase, servicePreparer +from devtools_testutils import recorded_by_proxy +import pytest +from azure.core import MatchConditions +from azure.core.exceptions import HttpResponseError, ResourceNotFoundError +from azure.ai.projects.models import ( + AgentVersionDetails, + PSTNTelephonyTransferDestination, + TelephonyBindingStatus, + TelephonyTransferTarget, + TelephonyTransferTargets, + UpdateTelephonyBindingRequest, + VoiceAgentAudioConfig, + VoiceAgentAudioOutputConfig, + VoiceAgentDefinition, + VoiceOutputModality, +) + + +class TestVoiceAgentTelephony(TestBase): + """ + Recorded tests covering the voice-agent telephony REST API surface exposed through + `project_client.agents.*` (telephony bindings, calls, and transfer targets), and the + top-level `project_client.beta.agent_endpoint_conversations.*` generated-audio reads. + + NOTE: All tests in this file are currently marked `skip`: + - The telephony routes (`/agents/{agent_name}/telephony_bindings`, `/telephony_calls`, + `/telephony_transfer_targets`) are defined in the TypeSpec/SDK but not yet deployed to + the live test resource: every call returns an empty-body 404 (a routing-layer "no such + route" response from the service mesh, not an application-level not-found error - + confirmed by comparing against a known-working route's fully-populated JSON error body). + Un-skip `test_telephony_bindings_and_transfer_targets`/`test_telephony_calls_not_found` + once the service deploys these routes. + - `beta.agent_endpoint_conversations.get_item_generated_audio*` with a + made-up conversation/item ID hits the service's conversation-ID format validator and + returns an unhandled `500 server_error` instead of a clean `404` - the exact same + pre-existing behavior as the already-documented `beta.agent_endpoint_conversations` + limitation below. Testing the success path needs a live realtime session whose playback + was interrupted; testing the not-found path needs a validly-formatted but nonexistent ID + (the format isn't publicly documented). `test_generated_audio_not_found` is left in as a + placeholder and currently skipped. + + Further NOTE: the following are intentionally NOT covered here at all since they require real + infrastructure this test environment does not have: + - `create_telephony_binding` with a real Teams Phone Extension or Twilio provider account + (needs real provider credentials/connections). Its request/response wiring is still + exercised indirectly through the header-injection unit tests in + `tests/foundry_features_header/`. + - `list_telephony_calls`/`get_telephony_call`/`transfer_telephony_call`/`end_telephony_call` + against an actual in-progress or historical call (needs a real inbound telephony call). + - Reading back a conversation (`project_client.beta.agent_endpoint_conversations.*`) using a + `conversation_id` produced by a live realtime WebSocket session - the service's REST + conversation-ID validator rejects the ID format generated by the realtime WS subsystem. + This is also not practical to cover with HTTP-only recorded tests since it requires an + actual WebSocket session. + Once these are fixed/deployed service-side, tests can be added/enabled for them. + """ + + def _make_definition(self, model: str) -> VoiceAgentDefinition: + return VoiceAgentDefinition( + model_type="managed", + model=model, + instructions="You are a helpful voice assistant.", + audio=VoiceAgentAudioConfig( + output=VoiceAgentAudioOutputConfig(voice="en-US-AvaNeural", voice_type="azure-standard") + ), + output_modalities=[VoiceOutputModality.AUDIO], + ) + + # To run only this test: + # pytest tests\agents\test_voice_agent_telephony.py::TestVoiceAgentTelephony::test_telephony_bindings_and_transfer_targets -s + @pytest.mark.skip( + reason="Telephony routes are defined in the TypeSpec/SDK but not yet deployed on the live " + "test service (empty-body 404s at the routing layer). Un-skip once the service deploys them." + ) + @servicePreparer() + @recorded_by_proxy() + def test_telephony_bindings_and_transfer_targets(self, **kwargs): + """ + Test telephony bindings (list/get/update/delete against a nonexistent binding) and a + round-trip of the telephony transfer targets configured for a voice agent. + + Routes used in this test: + + Action REST API Route Client Method + ------+-------------------------------------------------------------+----------------------------------------------- + POST /agents/{agent_name}/versions project_client.agents.create_version() + GET /agents/{agent_name}/telephony_bindings project_client.beta.agents.list_telephony_bindings() + GET /agents/{agent_name}/telephony_transfer_targets project_client.beta.agents.get_telephony_transfer_targets() + PUT /agents/{agent_name}/telephony_transfer_targets project_client.beta.agents.replace_telephony_transfer_targets() + GET /agents/{agent_name}/telephony_bindings/{binding_id} project_client.beta.agents.get_telephony_binding() + PATCH /agents/{agent_name}/telephony_bindings/{binding_id} project_client.beta.agents.update_telephony_binding() + DELETE /agents/{agent_name}/telephony_bindings/{binding_id} project_client.beta.agents.delete_telephony_binding() + DELETE /agents/{agent_name} project_client.agents.delete() + """ + print("\n") + model = kwargs.get("foundry_voice_model_name") + assert model is not None + # Voice-agent operations require the preview opt-in. + project_client = self.create_client(allow_preview=True, **kwargs) + agent_name = "VoiceAgentTelephonyBindingsTest" + + # Delete any existing agent from previous test runs (ignore failures) + try: + project_client.agents.delete(agent_name=agent_name) + except Exception: # pylint: disable=broad-except + pass + + agent_version: AgentVersionDetails = project_client.agents.create_version( + agent_name=agent_name, + definition=self._make_definition(model), + ) + self._validate_agent_version(agent_version, expected_name=agent_name) + + # A freshly created agent has no telephony bindings. + bindings = list(project_client.beta.agents.list_telephony_bindings(agent_name=agent_name)) + assert len(bindings) == 0 + + # A freshly created agent has no telephony transfer targets configured. + targets: TelephonyTransferTargets = project_client.beta.agents.get_telephony_transfer_targets( + agent_name=agent_name + ) + assert targets is not None + assert len(targets.transfer_targets) == 0 + + # Configure one PSTN transfer target. + new_target = TelephonyTransferTarget( + name="sales_desk", + description="Transfers to the sales desk for pricing questions.", + destination=PSTNTelephonyTransferDestination(value="+14255550123"), + ) + replaced_targets: TelephonyTransferTargets = project_client.beta.agents.replace_telephony_transfer_targets( + agent_name=agent_name, + transfer_targets=[new_target], + etag=None, + match_condition=MatchConditions.Unconditionally, + ) + assert len(replaced_targets.transfer_targets) == 1 + assert replaced_targets.transfer_targets[0].name == "sales_desk" + assert replaced_targets.transfer_targets[0].destination.kind == "pstn" + + # Confirm the change persisted. + confirmed_targets: TelephonyTransferTargets = project_client.beta.agents.get_telephony_transfer_targets( + agent_name=agent_name + ) + assert len(confirmed_targets.transfer_targets) == 1 + assert confirmed_targets.transfer_targets[0].name == "sales_desk" + + # Clear the transfer targets (empty array clears all targets). + cleared_targets: TelephonyTransferTargets = project_client.beta.agents.replace_telephony_transfer_targets( + agent_name=agent_name, + transfer_targets=[], + etag=None, + match_condition=MatchConditions.Unconditionally, + ) + assert len(cleared_targets.transfer_targets) == 0 + + # A nonexistent telephony binding returns 404 on get/update/delete. + fake_binding_id = "nonexistent-binding-id" + with pytest.raises(ResourceNotFoundError): + project_client.beta.agents.get_telephony_binding(agent_name=agent_name, binding_id=fake_binding_id) + with pytest.raises(ResourceNotFoundError): + project_client.beta.agents.update_telephony_binding( + agent_name=agent_name, + binding_id=fake_binding_id, + body=UpdateTelephonyBindingRequest(status=TelephonyBindingStatus.SUSPENDED), + etag=None, + match_condition=MatchConditions.Unconditionally, + ) + with pytest.raises(ResourceNotFoundError): + project_client.beta.agents.delete_telephony_binding( + agent_name=agent_name, + binding_id=fake_binding_id, + etag=None, + match_condition=MatchConditions.Unconditionally, + ) + + # Delete the voice agent. + result = project_client.agents.delete(agent_name=agent_name) + assert result.deleted + + # To run only this test: + # pytest tests\agents\test_voice_agent_telephony.py::TestVoiceAgentTelephony::test_telephony_calls_not_found -s + @pytest.mark.skip( + reason="Telephony routes are defined in the TypeSpec/SDK but not yet deployed on the live " + "test service (empty-body 404s at the routing layer). Un-skip once the service deploys them." + ) + @servicePreparer() + @recorded_by_proxy() + def test_telephony_calls_not_found(self, **kwargs): + """ + Test telephony calls: listing (empty on a fresh agent) and get/transfer/end against a + nonexistent call, which return 404. + + Routes used in this test: + + Action REST API Route Client Method + ------+-------------------------------------------------------------+----------------------------------------------- + POST /agents/{agent_name}/versions project_client.agents.create_version() + GET /agents/{agent_name}/telephony_calls project_client.beta.agents.list_telephony_calls() + GET /agents/{agent_name}/telephony_calls/{call_id} project_client.beta.agents.get_telephony_call() + POST /agents/{agent_name}/telephony_calls/{call_id}:transfer project_client.beta.agents.transfer_telephony_call() + POST /agents/{agent_name}/telephony_calls/{call_id}:end project_client.beta.agents.end_telephony_call() + DELETE /agents/{agent_name} project_client.agents.delete() + """ + print("\n") + model = kwargs.get("foundry_voice_model_name") + assert model is not None + project_client = self.create_client(allow_preview=True, **kwargs) + agent_name = "VoiceAgentTelephonyCallsTest" + + # Delete any existing agent from previous test runs (ignore failures) + try: + project_client.agents.delete(agent_name=agent_name) + except Exception: # pylint: disable=broad-except + pass + + agent_version: AgentVersionDetails = project_client.agents.create_version( + agent_name=agent_name, + definition=self._make_definition(model), + ) + self._validate_agent_version(agent_version, expected_name=agent_name) + + # A freshly created agent has no telephony call history. + calls = list(project_client.beta.agents.list_telephony_calls(agent_name=agent_name)) + assert len(calls) == 0 + + fake_call_id = "nonexistent-call-id" + with pytest.raises(ResourceNotFoundError): + project_client.beta.agents.get_telephony_call(agent_name=agent_name, call_id=fake_call_id) + with pytest.raises(HttpResponseError) as transfer_exc_info: + project_client.beta.agents.transfer_telephony_call( + agent_name=agent_name, call_id=fake_call_id, target="nonexistent-target" + ) + assert transfer_exc_info.value.status_code == 404 + with pytest.raises(HttpResponseError) as end_exc_info: + project_client.beta.agents.end_telephony_call(agent_name=agent_name, call_id=fake_call_id) + assert end_exc_info.value.status_code == 404 + + # Delete the voice agent. + result = project_client.agents.delete(agent_name=agent_name) + assert result.deleted + + # To run only this test: + # pytest tests\agents\test_voice_agent_telephony.py::TestVoiceAgentTelephony::test_generated_audio_not_found -s + @pytest.mark.skip( + reason="A made-up conversation/item ID hits the service's conversation-ID format validator " + "and returns an unhandled 500 instead of a clean 404 (same pre-existing behavior as " + "beta.agent_endpoint_conversations). Needs a validly-formatted but nonexistent ID, or a real " + "realtime session, to test properly." + ) + @servicePreparer() + @recorded_by_proxy() + def test_generated_audio_not_found(self, **kwargs): + """ + Test the `beta.agent_endpoint_conversations.get_item_generated_audio`/ + `download_item_generated_audio` methods against a nonexistent + conversation item, which return 404. + + Routes used in this test: + + Action REST API Route Client Method + ------+-----------------------------------------------------------------------------------------+----------------------------------------------------------------------------- + POST /agents/{agent_name}/versions project_client.agents.create_version() + GET /agents/{agent_name}/endpoint/protocols/voice/conversations/{conversation_id}/items/{item_id}/audio/generated project_client.beta.agent_endpoint_conversations.get_item_generated_audio() + GET /agents/{agent_name}/endpoint/protocols/voice/conversations/{conversation_id}/items/{item_id}/audio/generated/content project_client.beta.agent_endpoint_conversations.download_item_generated_audio() + DELETE /agents/{agent_name} project_client.agents.delete() + """ + print("\n") + model = kwargs.get("foundry_voice_model_name") + assert model is not None + project_client = self.create_client(allow_preview=True, **kwargs) + agent_name = "VoiceAgentGeneratedAudioTest" + + # Delete any existing agent from previous test runs (ignore failures) + try: + project_client.agents.delete(agent_name=agent_name) + except Exception: # pylint: disable=broad-except + pass + + agent_version: AgentVersionDetails = project_client.agents.create_version( + agent_name=agent_name, + definition=self._make_definition(model), + ) + self._validate_agent_version(agent_version, expected_name=agent_name) + + fake_conversation_id = "nonexistent-conversation-id" + fake_item_id = "nonexistent-item-id" + with pytest.raises(ResourceNotFoundError): + project_client.beta.agent_endpoint_conversations.get_item_generated_audio( + agent_name=agent_name, conversation_id=fake_conversation_id, item_id=fake_item_id + ) + with pytest.raises(HttpResponseError) as content_exc_info: + list( + project_client.beta.agent_endpoint_conversations.download_item_generated_audio( + agent_name=agent_name, conversation_id=fake_conversation_id, item_id=fake_item_id + ) + ) + assert content_exc_info.value.status_code == 404 + + # Delete the voice agent. + result = project_client.agents.delete(agent_name=agent_name) + assert result.deleted diff --git a/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_telephony_async.py b/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_telephony_async.py new file mode 100644 index 000000000000..4bb559b3c0c7 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_telephony_async.py @@ -0,0 +1,313 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +from test_base import TestBase, servicePreparer +from devtools_testutils.aio import recorded_by_proxy_async +import pytest +from azure.core import MatchConditions +from azure.core.exceptions import HttpResponseError, ResourceNotFoundError +from azure.ai.projects.models import ( + AgentVersionDetails, + PSTNTelephonyTransferDestination, + TelephonyBindingStatus, + TelephonyTransferTarget, + TelephonyTransferTargets, + UpdateTelephonyBindingRequest, + VoiceAgentAudioConfig, + VoiceAgentAudioOutputConfig, + VoiceAgentDefinition, + VoiceOutputModality, +) + + +class TestVoiceAgentTelephonyAsync(TestBase): + """ + Recorded tests covering the voice-agent telephony REST API surface exposed through + `project_client.agents.*` (telephony bindings, calls, and transfer targets), and the + top-level `project_client.beta.agent_endpoint_conversations.*` generated-audio reads. + + NOTE: All tests in this file are currently marked `skip`: + - The telephony routes (`/agents/{agent_name}/telephony_bindings`, `/telephony_calls`, + `/telephony_transfer_targets`) are defined in the TypeSpec/SDK but not yet deployed to + the live test resource: every call returns an empty-body 404 (a routing-layer "no such + route" response from the service mesh, not an application-level not-found error - + confirmed by comparing against a known-working route's fully-populated JSON error body). + Un-skip `test_telephony_bindings_and_transfer_targets`/`test_telephony_calls_not_found` + once the service deploys these routes. + - `beta.agent_endpoint_conversations.get_item_generated_audio*` with a + made-up conversation/item ID hits the service's conversation-ID format validator and + returns an unhandled `500 server_error` instead of a clean `404` - the exact same + pre-existing behavior as the already-documented `beta.agent_endpoint_conversations` + limitation below. Testing the success path needs a live realtime session whose playback + was interrupted; testing the not-found path needs a validly-formatted but nonexistent ID + (the format isn't publicly documented). `test_generated_audio_not_found` is left in as a + placeholder and currently skipped. + + Further NOTE: the following are intentionally NOT covered here at all since they require real + infrastructure this test environment does not have: + - `create_telephony_binding` with a real Teams Phone Extension or Twilio provider account + (needs real provider credentials/connections). Its request/response wiring is still + exercised indirectly through the header-injection unit tests in + `tests/foundry_features_header/`. + - `list_telephony_calls`/`get_telephony_call`/`transfer_telephony_call`/`end_telephony_call` + against an actual in-progress or historical call (needs a real inbound telephony call). + - Reading back a conversation (`project_client.beta.agent_endpoint_conversations.*`) using a + `conversation_id` produced by a live realtime WebSocket session - the service's REST + conversation-ID validator rejects the ID format generated by the realtime WS subsystem. + This is also not practical to cover with HTTP-only recorded tests since it requires an + actual WebSocket session. + Once these are fixed/deployed service-side, tests can be added/enabled for them. + """ + + def _make_definition(self, model: str) -> VoiceAgentDefinition: + return VoiceAgentDefinition( + model_type="managed", + model=model, + instructions="You are a helpful voice assistant.", + audio=VoiceAgentAudioConfig( + output=VoiceAgentAudioOutputConfig(voice="en-US-AvaNeural", voice_type="azure-standard") + ), + output_modalities=[VoiceOutputModality.AUDIO], + ) + + # To run only this test: + # pytest tests\agents\test_voice_agent_telephony_async.py::TestVoiceAgentTelephonyAsync::test_telephony_bindings_and_transfer_targets -s + @pytest.mark.skip( + reason="Telephony routes are defined in the TypeSpec/SDK but not yet deployed on the live " + "test service (empty-body 404s at the routing layer). Un-skip once the service deploys them." + ) + @servicePreparer() + @recorded_by_proxy_async() + async def test_telephony_bindings_and_transfer_targets(self, **kwargs): + """ + Test telephony bindings (list/get/update/delete against a nonexistent binding) and a + round-trip of the telephony transfer targets configured for a voice agent. + + Routes used in this test: + + Action REST API Route Client Method + ------+-------------------------------------------------------------+----------------------------------------------- + POST /agents/{agent_name}/versions project_client.agents.create_version() + GET /agents/{agent_name}/telephony_bindings project_client.beta.agents.list_telephony_bindings() + GET /agents/{agent_name}/telephony_transfer_targets project_client.beta.agents.get_telephony_transfer_targets() + PUT /agents/{agent_name}/telephony_transfer_targets project_client.beta.agents.replace_telephony_transfer_targets() + GET /agents/{agent_name}/telephony_bindings/{binding_id} project_client.beta.agents.get_telephony_binding() + PATCH /agents/{agent_name}/telephony_bindings/{binding_id} project_client.beta.agents.update_telephony_binding() + DELETE /agents/{agent_name}/telephony_bindings/{binding_id} project_client.beta.agents.delete_telephony_binding() + DELETE /agents/{agent_name} project_client.agents.delete() + """ + print("\n") + model = kwargs.get("foundry_voice_model_name") + assert model is not None + # Voice-agent operations require the preview opt-in. + project_client = self.create_async_client(allow_preview=True, **kwargs) + agent_name = "VoiceAgentTelephonyBindingsTest" + + # Delete any existing agent from previous test runs (ignore failures) + try: + await project_client.agents.delete(agent_name=agent_name) + except Exception: # pylint: disable=broad-except + pass + + agent_version: AgentVersionDetails = await project_client.agents.create_version( + agent_name=agent_name, + definition=self._make_definition(model), + ) + self._validate_agent_version(agent_version, expected_name=agent_name) + + # A freshly created agent has no telephony bindings. + bindings = [b async for b in project_client.beta.agents.list_telephony_bindings(agent_name=agent_name)] + assert len(bindings) == 0 + + # A freshly created agent has no telephony transfer targets configured. + targets: TelephonyTransferTargets = await project_client.beta.agents.get_telephony_transfer_targets( + agent_name=agent_name + ) + assert targets is not None + assert len(targets.transfer_targets) == 0 + + # Configure one PSTN transfer target. + new_target = TelephonyTransferTarget( + name="sales_desk", + description="Transfers to the sales desk for pricing questions.", + destination=PSTNTelephonyTransferDestination(value="+14255550123"), + ) + replaced_targets: TelephonyTransferTargets = ( + await project_client.beta.agents.replace_telephony_transfer_targets( + agent_name=agent_name, + transfer_targets=[new_target], + etag=None, + match_condition=MatchConditions.Unconditionally, + ) + ) + assert len(replaced_targets.transfer_targets) == 1 + assert replaced_targets.transfer_targets[0].name == "sales_desk" + assert replaced_targets.transfer_targets[0].destination.kind == "pstn" + + # Confirm the change persisted. + confirmed_targets: TelephonyTransferTargets = await project_client.beta.agents.get_telephony_transfer_targets( + agent_name=agent_name + ) + assert len(confirmed_targets.transfer_targets) == 1 + assert confirmed_targets.transfer_targets[0].name == "sales_desk" + + # Clear the transfer targets (empty array clears all targets). + cleared_targets: TelephonyTransferTargets = await project_client.beta.agents.replace_telephony_transfer_targets( + agent_name=agent_name, + transfer_targets=[], + etag=None, + match_condition=MatchConditions.Unconditionally, + ) + assert len(cleared_targets.transfer_targets) == 0 + + # A nonexistent telephony binding returns 404 on get/update/delete. + fake_binding_id = "nonexistent-binding-id" + with pytest.raises(ResourceNotFoundError): + await project_client.beta.agents.get_telephony_binding(agent_name=agent_name, binding_id=fake_binding_id) + with pytest.raises(ResourceNotFoundError): + await project_client.beta.agents.update_telephony_binding( + agent_name=agent_name, + binding_id=fake_binding_id, + body=UpdateTelephonyBindingRequest(status=TelephonyBindingStatus.SUSPENDED), + etag=None, + match_condition=MatchConditions.Unconditionally, + ) + with pytest.raises(ResourceNotFoundError): + await project_client.beta.agents.delete_telephony_binding( + agent_name=agent_name, + binding_id=fake_binding_id, + etag=None, + match_condition=MatchConditions.Unconditionally, + ) + + # Delete the voice agent. + result = await project_client.agents.delete(agent_name=agent_name) + assert result.deleted + + # To run only this test: + # pytest tests\agents\test_voice_agent_telephony_async.py::TestVoiceAgentTelephonyAsync::test_telephony_calls_not_found -s + @pytest.mark.skip( + reason="Telephony routes are defined in the TypeSpec/SDK but not yet deployed on the live " + "test service (empty-body 404s at the routing layer). Un-skip once the service deploys them." + ) + @servicePreparer() + @recorded_by_proxy_async() + async def test_telephony_calls_not_found(self, **kwargs): + """ + Test telephony calls: listing (empty on a fresh agent) and get/transfer/end against a + nonexistent call, which return 404. + + Routes used in this test: + + Action REST API Route Client Method + ------+-------------------------------------------------------------+----------------------------------------------- + POST /agents/{agent_name}/versions project_client.agents.create_version() + GET /agents/{agent_name}/telephony_calls project_client.beta.agents.list_telephony_calls() + GET /agents/{agent_name}/telephony_calls/{call_id} project_client.beta.agents.get_telephony_call() + POST /agents/{agent_name}/telephony_calls/{call_id}:transfer project_client.beta.agents.transfer_telephony_call() + POST /agents/{agent_name}/telephony_calls/{call_id}:end project_client.beta.agents.end_telephony_call() + DELETE /agents/{agent_name} project_client.agents.delete() + """ + print("\n") + model = kwargs.get("foundry_voice_model_name") + assert model is not None + project_client = self.create_async_client(allow_preview=True, **kwargs) + agent_name = "VoiceAgentTelephonyCallsTest" + + # Delete any existing agent from previous test runs (ignore failures) + try: + await project_client.agents.delete(agent_name=agent_name) + except Exception: # pylint: disable=broad-except + pass + + agent_version: AgentVersionDetails = await project_client.agents.create_version( + agent_name=agent_name, + definition=self._make_definition(model), + ) + self._validate_agent_version(agent_version, expected_name=agent_name) + + # A freshly created agent has no telephony call history. + calls = [c async for c in project_client.beta.agents.list_telephony_calls(agent_name=agent_name)] + assert len(calls) == 0 + + fake_call_id = "nonexistent-call-id" + with pytest.raises(ResourceNotFoundError): + await project_client.beta.agents.get_telephony_call(agent_name=agent_name, call_id=fake_call_id) + with pytest.raises(HttpResponseError) as transfer_exc_info: + await project_client.beta.agents.transfer_telephony_call( + agent_name=agent_name, call_id=fake_call_id, target="nonexistent-target" + ) + assert transfer_exc_info.value.status_code == 404 + with pytest.raises(HttpResponseError) as end_exc_info: + await project_client.beta.agents.end_telephony_call(agent_name=agent_name, call_id=fake_call_id) + assert end_exc_info.value.status_code == 404 + + # Delete the voice agent. + result = await project_client.agents.delete(agent_name=agent_name) + assert result.deleted + + # To run only this test: + # pytest tests\agents\test_voice_agent_telephony_async.py::TestVoiceAgentTelephonyAsync::test_generated_audio_not_found -s + @pytest.mark.skip( + reason="A made-up conversation/item ID hits the service's conversation-ID format validator " + "and returns an unhandled 500 instead of a clean 404 (same pre-existing behavior as " + "beta.agent_endpoint_conversations). Needs a validly-formatted but nonexistent ID, or a real " + "realtime session, to test properly." + ) + @servicePreparer() + @recorded_by_proxy_async() + async def test_generated_audio_not_found(self, **kwargs): + """ + Test the `beta.agent_endpoint_conversations.get_item_generated_audio`/ + `download_item_generated_audio` methods against a nonexistent + conversation item, which return 404. + + Routes used in this test: + + Action REST API Route Client Method + ------+-----------------------------------------------------------------------------------------+----------------------------------------------------------------------------- + POST /agents/{agent_name}/versions project_client.agents.create_version() + GET /agents/{agent_name}/endpoint/protocols/voice/conversations/{conversation_id}/items/{item_id}/audio/generated project_client.beta.agent_endpoint_conversations.get_item_generated_audio() + GET /agents/{agent_name}/endpoint/protocols/voice/conversations/{conversation_id}/items/{item_id}/audio/generated/content project_client.beta.agent_endpoint_conversations.download_item_generated_audio() + DELETE /agents/{agent_name} project_client.agents.delete() + """ + print("\n") + model = kwargs.get("foundry_voice_model_name") + assert model is not None + project_client = self.create_async_client(allow_preview=True, **kwargs) + agent_name = "VoiceAgentGeneratedAudioTest" + + # Delete any existing agent from previous test runs (ignore failures) + try: + await project_client.agents.delete(agent_name=agent_name) + except Exception: # pylint: disable=broad-except + pass + + agent_version: AgentVersionDetails = await project_client.agents.create_version( + agent_name=agent_name, + definition=self._make_definition(model), + ) + self._validate_agent_version(agent_version, expected_name=agent_name) + + fake_conversation_id = "nonexistent-conversation-id" + fake_item_id = "nonexistent-item-id" + with pytest.raises(ResourceNotFoundError): + await project_client.beta.agent_endpoint_conversations.get_item_generated_audio( + agent_name=agent_name, conversation_id=fake_conversation_id, item_id=fake_item_id + ) + with pytest.raises(HttpResponseError) as content_exc_info: + [ + chunk + async for chunk in await project_client.beta.agent_endpoint_conversations.download_item_generated_audio( + agent_name=agent_name, conversation_id=fake_conversation_id, item_id=fake_item_id + ) + ] + assert content_exc_info.value.status_code == 404 + + # Delete the voice agent. + result = await project_client.agents.delete(agent_name=agent_name) + assert result.deleted diff --git a/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_telephony_campaign.py b/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_telephony_campaign.py new file mode 100644 index 000000000000..f4365743fd54 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_telephony_campaign.py @@ -0,0 +1,238 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +from test_base import TestBase, servicePreparer +from devtools_testutils import recorded_by_proxy +import pytest +from azure.core import MatchConditions +from azure.core.exceptions import ResourceNotFoundError +from azure.ai.projects.models import ( + AgentVersionDetails, + VoiceAgentAudioConfig, + VoiceAgentAudioOutputConfig, + VoiceAgentDefinition, + VoiceOutputModality, +) + + +class TestVoiceAgentTelephonyCampaign(TestBase): + """ + Recorded tests covering the outbound telephony call-job/campaign REST API surface exposed + through the top-level `project_client.beta.agent_telephony.*` operation group (added in the + "batch 2" Voice Agents TypeSpec, distinct from the existing `project_client.agents.*` + telephony binding/call methods). + + NOTE: All tests in this file are currently marked `skip`: + - Probing this environment's live Voice Agents test resource with + `agent_telephony.get_operation` (api-version "v1", the SDK's only known + version) returns `400 UnsupportedApiVersion` with a message identifying the resolved + route (".../agents/{agent_name}/telephony/operations/{operation_id}") but rejecting + "v1" for it - unlike the routing-layer empty-body 404s seen for the batch-1 + `agents.*` telephony bindings/calls routes (see `test_voice_agent_telephony.py`), this + route *is* registered, but the call-job/campaign feature isn't yet enabled for the API + version this SDK targets. Un-skip once the live test service accepts "v1" for these + routes. + + Further NOTE: the following are intentionally NOT covered here at all since they require real + infrastructure this test environment does not have: + - `create_call_job`/`create_campaign` need a real, working + `telephony_binding_id` from a provisioned Teams Phone/Twilio telephony binding (same + real-provider limitation documented for `create_telephony_binding` in + `test_voice_agent_telephony.py`). + - `begin_import_campaign_recipients`/`begin_publish_campaign`/ + `begin_validate_campaign` are long-running operations on a real campaign with + actual recipients, which in turn requires the real telephony binding above. + Once these are fixed/deployed service-side and real provider credentials are available, + tests can be added/enabled for them. + """ + + def _make_definition(self, model: str) -> VoiceAgentDefinition: + return VoiceAgentDefinition( + model_type="managed", + model=model, + instructions="You are a helpful voice assistant.", + audio=VoiceAgentAudioConfig( + output=VoiceAgentAudioOutputConfig(voice="en-US-AvaNeural", voice_type="azure-standard") + ), + output_modalities=[VoiceOutputModality.AUDIO], + ) + + # To run only this test: + # pytest tests\agents\test_voice_agent_telephony_campaign.py::TestVoiceAgentTelephonyCampaign::test_telephony_call_job_not_found -s + @pytest.mark.skip( + reason="agent_telephony routes are registered but return 400 UnsupportedApiVersion for " + "api-version 'v1' on the live test service. Un-skip once the service supports 'v1' for " + "this operation group." + ) + @servicePreparer() + @recorded_by_proxy() + def test_telephony_call_job_not_found(self, **kwargs): + """ + Test outbound telephony call jobs: get/cancel against a nonexistent call job, which + return 404. + + Routes used in this test: + + Action REST API Route Client Method + ------+-------------------------------------------------------------+----------------------------------------------- + POST /agents/{agent_name}/versions project_client.agents.create_version() + GET /agents/{agent_name}/telephony/call_jobs/{call_job_id} project_client.beta.agent_telephony.get_call_job() + POST /agents/{agent_name}/telephony/call_jobs/{call_job_id}:cancel project_client.beta.agent_telephony.cancel_call_job() + DELETE /agents/{agent_name} project_client.agents.delete() + """ + print("\n") + model = kwargs.get("foundry_voice_model_name") + assert model is not None + # Voice-agent operations require the preview opt-in. + project_client = self.create_client(allow_preview=True, **kwargs) + agent_name = "VoiceAgentTelephonyCallJobTest" + + # Delete any existing agent from previous test runs (ignore failures) + try: + project_client.agents.delete(agent_name=agent_name) + except Exception: # pylint: disable=broad-except + pass + + agent_version: AgentVersionDetails = project_client.agents.create_version( + agent_name=agent_name, + definition=self._make_definition(model), + ) + self._validate_agent_version(agent_version, expected_name=agent_name) + + fake_call_job_id = "nonexistent-call-job-id" + with pytest.raises(ResourceNotFoundError): + project_client.beta.agent_telephony.get_call_job(agent_name=agent_name, call_job_id=fake_call_job_id) + with pytest.raises(ResourceNotFoundError): + project_client.beta.agent_telephony.cancel_call_job( + agent_name=agent_name, + call_job_id=fake_call_job_id, + etag=None, + match_condition=MatchConditions.Unconditionally, + ) + + # Delete the voice agent. + result = project_client.agents.delete(agent_name=agent_name) + assert result.deleted + + # To run only this test: + # pytest tests\agents\test_voice_agent_telephony_campaign.py::TestVoiceAgentTelephonyCampaign::test_telephony_campaign_not_found -s + @pytest.mark.skip( + reason="agent_telephony routes are registered but return 400 UnsupportedApiVersion for " + "api-version 'v1' on the live test service. Un-skip once the service supports 'v1' for " + "this operation group." + ) + @servicePreparer() + @recorded_by_proxy() + def test_telephony_campaign_not_found(self, **kwargs): + """ + Test outbound telephony campaigns: get/cancel/pause/resume against a nonexistent + campaign, and get against a nonexistent recipient import, all of which return 404. + + Routes used in this test: + + Action REST API Route Client Method + ------+-------------------------------------------------------------------+----------------------------------------------- + POST /agents/{agent_name}/versions project_client.agents.create_version() + GET /agents/{agent_name}/telephony/campaigns/{campaign_id} project_client.beta.agent_telephony.get_campaign() + POST /agents/{agent_name}/telephony/campaigns/{campaign_id}:cancel project_client.beta.agent_telephony.cancel_campaign() + POST /agents/{agent_name}/telephony/campaigns/{campaign_id}:pause project_client.beta.agent_telephony.pause_campaign() + POST /agents/{agent_name}/telephony/campaigns/{campaign_id}:resume project_client.beta.agent_telephony.resume_campaign() + GET /agents/{agent_name}/telephony/campaigns/{campaign_id}/recipient_imports/{import_id} + project_client.beta.agent_telephony.get_campaign_recipient_import() + DELETE /agents/{agent_name} project_client.agents.delete() + """ + print("\n") + model = kwargs.get("foundry_voice_model_name") + assert model is not None + # Voice-agent operations require the preview opt-in. + project_client = self.create_client(allow_preview=True, **kwargs) + agent_name = "VoiceAgentTelephonyCampaignTest" + + # Delete any existing agent from previous test runs (ignore failures) + try: + project_client.agents.delete(agent_name=agent_name) + except Exception: # pylint: disable=broad-except + pass + + agent_version: AgentVersionDetails = project_client.agents.create_version( + agent_name=agent_name, + definition=self._make_definition(model), + ) + self._validate_agent_version(agent_version, expected_name=agent_name) + + fake_campaign_id = "nonexistent-campaign-id" + with pytest.raises(ResourceNotFoundError): + project_client.beta.agent_telephony.get_campaign(agent_name=agent_name, campaign_id=fake_campaign_id) + with pytest.raises(ResourceNotFoundError): + project_client.beta.agent_telephony.cancel_campaign(agent_name=agent_name, campaign_id=fake_campaign_id) + with pytest.raises(ResourceNotFoundError): + project_client.beta.agent_telephony.pause_campaign(agent_name=agent_name, campaign_id=fake_campaign_id) + with pytest.raises(ResourceNotFoundError): + project_client.beta.agent_telephony.resume_campaign(agent_name=agent_name, campaign_id=fake_campaign_id) + + fake_import_id = "nonexistent-import-id" + with pytest.raises(ResourceNotFoundError): + project_client.beta.agent_telephony.get_campaign_recipient_import( + agent_name=agent_name, + campaign_id=fake_campaign_id, + import_id=fake_import_id, + ) + + # Delete the voice agent. + result = project_client.agents.delete(agent_name=agent_name) + assert result.deleted + + # To run only this test: + # pytest tests\agents\test_voice_agent_telephony_campaign.py::TestVoiceAgentTelephonyCampaign::test_telephony_operation_not_found -s + @pytest.mark.skip( + reason="agent_telephony routes are registered but return 400 UnsupportedApiVersion for " + "api-version 'v1' on the live test service. Un-skip once the service supports 'v1' for " + "this operation group." + ) + @servicePreparer() + @recorded_by_proxy() + def test_telephony_operation_not_found(self, **kwargs): + """ + Test the generic long-running-operation status endpoint used to poll + `begin_import_campaign_recipients`/`begin_publish_campaign`/ + `begin_validate_campaign`, against a nonexistent operation id, which returns + 404. + + Routes used in this test: + + Action REST API Route Client Method + ------+-------------------------------------------------------------+----------------------------------------------- + POST /agents/{agent_name}/versions project_client.agents.create_version() + GET /agents/{agent_name}/telephony/operations/{operation_id} project_client.beta.agent_telephony.get_operation() + DELETE /agents/{agent_name} project_client.agents.delete() + """ + print("\n") + model = kwargs.get("foundry_voice_model_name") + assert model is not None + # Voice-agent operations require the preview opt-in. + project_client = self.create_client(allow_preview=True, **kwargs) + agent_name = "VoiceAgentTelephonyOperationTest" + + # Delete any existing agent from previous test runs (ignore failures) + try: + project_client.agents.delete(agent_name=agent_name) + except Exception: # pylint: disable=broad-except + pass + + agent_version: AgentVersionDetails = project_client.agents.create_version( + agent_name=agent_name, + definition=self._make_definition(model), + ) + self._validate_agent_version(agent_version, expected_name=agent_name) + + fake_operation_id = "nonexistent-operation-id" + with pytest.raises(ResourceNotFoundError): + project_client.beta.agent_telephony.get_operation(agent_name=agent_name, operation_id=fake_operation_id) + + # Delete the voice agent. + result = project_client.agents.delete(agent_name=agent_name) + assert result.deleted diff --git a/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_telephony_campaign_async.py b/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_telephony_campaign_async.py new file mode 100644 index 000000000000..2a228283232c --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_telephony_campaign_async.py @@ -0,0 +1,246 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +from test_base import TestBase, servicePreparer +from devtools_testutils.aio import recorded_by_proxy_async +import pytest +from azure.core import MatchConditions +from azure.core.exceptions import ResourceNotFoundError +from azure.ai.projects.models import ( + AgentVersionDetails, + VoiceAgentAudioConfig, + VoiceAgentAudioOutputConfig, + VoiceAgentDefinition, + VoiceOutputModality, +) + + +class TestVoiceAgentTelephonyCampaignAsync(TestBase): + """ + Recorded tests covering the outbound telephony call-job/campaign REST API surface exposed + through the top-level `project_client.beta.agent_telephony.*` operation group (added in the + "batch 2" Voice Agents TypeSpec, distinct from the existing `project_client.agents.*` + telephony binding/call methods). + + NOTE: All tests in this file are currently marked `skip`: + - Probing this environment's live Voice Agents test resource with + `agent_telephony.get_operation` (api-version "v1", the SDK's only known + version) returns `400 UnsupportedApiVersion` with a message identifying the resolved + route (".../agents/{agent_name}/telephony/operations/{operation_id}") but rejecting + "v1" for it - unlike the routing-layer empty-body 404s seen for the batch-1 + `agents.*` telephony bindings/calls routes (see `test_voice_agent_telephony_async.py`), + this route *is* registered, but the call-job/campaign feature isn't yet enabled for the + API version this SDK targets. Un-skip once the live test service accepts "v1" for these + routes. + + Further NOTE: the following are intentionally NOT covered here at all since they require real + infrastructure this test environment does not have: + - `create_call_job`/`create_campaign` need a real, working + `telephony_binding_id` from a provisioned Teams Phone/Twilio telephony binding (same + real-provider limitation documented for `create_telephony_binding` in + `test_voice_agent_telephony_async.py`). + - `begin_import_campaign_recipients`/`begin_publish_campaign`/ + `begin_validate_campaign` are long-running operations on a real campaign with + actual recipients, which in turn requires the real telephony binding above. + Once these are fixed/deployed service-side and real provider credentials are available, + tests can be added/enabled for them. + """ + + def _make_definition(self, model: str) -> VoiceAgentDefinition: + return VoiceAgentDefinition( + model_type="managed", + model=model, + instructions="You are a helpful voice assistant.", + audio=VoiceAgentAudioConfig( + output=VoiceAgentAudioOutputConfig(voice="en-US-AvaNeural", voice_type="azure-standard") + ), + output_modalities=[VoiceOutputModality.AUDIO], + ) + + # To run only this test: + # pytest tests\agents\test_voice_agent_telephony_campaign_async.py::TestVoiceAgentTelephonyCampaignAsync::test_telephony_call_job_not_found -s + @pytest.mark.skip( + reason="agent_telephony routes are registered but return 400 UnsupportedApiVersion for " + "api-version 'v1' on the live test service. Un-skip once the service supports 'v1' for " + "this operation group." + ) + @servicePreparer() + @recorded_by_proxy_async() + async def test_telephony_call_job_not_found(self, **kwargs): + """ + Test outbound telephony call jobs: get/cancel against a nonexistent call job, which + return 404. + + Routes used in this test: + + Action REST API Route Client Method + ------+-------------------------------------------------------------+----------------------------------------------- + POST /agents/{agent_name}/versions project_client.agents.create_version() + GET /agents/{agent_name}/telephony/call_jobs/{call_job_id} project_client.beta.agent_telephony.get_call_job() + POST /agents/{agent_name}/telephony/call_jobs/{call_job_id}:cancel project_client.beta.agent_telephony.cancel_call_job() + DELETE /agents/{agent_name} project_client.agents.delete() + """ + print("\n") + model = kwargs.get("foundry_voice_model_name") + assert model is not None + # Voice-agent operations require the preview opt-in. + project_client = self.create_async_client(allow_preview=True, **kwargs) + agent_name = "VoiceAgentTelephonyCallJobTest" + + # Delete any existing agent from previous test runs (ignore failures) + try: + await project_client.agents.delete(agent_name=agent_name) + except Exception: # pylint: disable=broad-except + pass + + agent_version: AgentVersionDetails = await project_client.agents.create_version( + agent_name=agent_name, + definition=self._make_definition(model), + ) + self._validate_agent_version(agent_version, expected_name=agent_name) + + fake_call_job_id = "nonexistent-call-job-id" + with pytest.raises(ResourceNotFoundError): + await project_client.beta.agent_telephony.get_call_job(agent_name=agent_name, call_job_id=fake_call_job_id) + with pytest.raises(ResourceNotFoundError): + await project_client.beta.agent_telephony.cancel_call_job( + agent_name=agent_name, + call_job_id=fake_call_job_id, + etag=None, + match_condition=MatchConditions.Unconditionally, + ) + + # Delete the voice agent. + result = await project_client.agents.delete(agent_name=agent_name) + assert result.deleted + + # To run only this test: + # pytest tests\agents\test_voice_agent_telephony_campaign_async.py::TestVoiceAgentTelephonyCampaignAsync::test_telephony_campaign_not_found -s + @pytest.mark.skip( + reason="agent_telephony routes are registered but return 400 UnsupportedApiVersion for " + "api-version 'v1' on the live test service. Un-skip once the service supports 'v1' for " + "this operation group." + ) + @servicePreparer() + @recorded_by_proxy_async() + async def test_telephony_campaign_not_found(self, **kwargs): + """ + Test outbound telephony campaigns: get/cancel/pause/resume against a nonexistent + campaign, and get against a nonexistent recipient import, all of which return 404. + + Routes used in this test: + + Action REST API Route Client Method + ------+-------------------------------------------------------------------+----------------------------------------------- + POST /agents/{agent_name}/versions project_client.agents.create_version() + GET /agents/{agent_name}/telephony/campaigns/{campaign_id} project_client.beta.agent_telephony.get_campaign() + POST /agents/{agent_name}/telephony/campaigns/{campaign_id}:cancel project_client.beta.agent_telephony.cancel_campaign() + POST /agents/{agent_name}/telephony/campaigns/{campaign_id}:pause project_client.beta.agent_telephony.pause_campaign() + POST /agents/{agent_name}/telephony/campaigns/{campaign_id}:resume project_client.beta.agent_telephony.resume_campaign() + GET /agents/{agent_name}/telephony/campaigns/{campaign_id}/recipient_imports/{import_id} + project_client.beta.agent_telephony.get_campaign_recipient_import() + DELETE /agents/{agent_name} project_client.agents.delete() + """ + print("\n") + model = kwargs.get("foundry_voice_model_name") + assert model is not None + # Voice-agent operations require the preview opt-in. + project_client = self.create_async_client(allow_preview=True, **kwargs) + agent_name = "VoiceAgentTelephonyCampaignTest" + + # Delete any existing agent from previous test runs (ignore failures) + try: + await project_client.agents.delete(agent_name=agent_name) + except Exception: # pylint: disable=broad-except + pass + + agent_version: AgentVersionDetails = await project_client.agents.create_version( + agent_name=agent_name, + definition=self._make_definition(model), + ) + self._validate_agent_version(agent_version, expected_name=agent_name) + + fake_campaign_id = "nonexistent-campaign-id" + with pytest.raises(ResourceNotFoundError): + await project_client.beta.agent_telephony.get_campaign(agent_name=agent_name, campaign_id=fake_campaign_id) + with pytest.raises(ResourceNotFoundError): + await project_client.beta.agent_telephony.cancel_campaign( + agent_name=agent_name, campaign_id=fake_campaign_id + ) + with pytest.raises(ResourceNotFoundError): + await project_client.beta.agent_telephony.pause_campaign( + agent_name=agent_name, campaign_id=fake_campaign_id + ) + with pytest.raises(ResourceNotFoundError): + await project_client.beta.agent_telephony.resume_campaign( + agent_name=agent_name, campaign_id=fake_campaign_id + ) + + fake_import_id = "nonexistent-import-id" + with pytest.raises(ResourceNotFoundError): + await project_client.beta.agent_telephony.get_campaign_recipient_import( + agent_name=agent_name, + campaign_id=fake_campaign_id, + import_id=fake_import_id, + ) + + # Delete the voice agent. + result = await project_client.agents.delete(agent_name=agent_name) + assert result.deleted + + # To run only this test: + # pytest tests\agents\test_voice_agent_telephony_campaign_async.py::TestVoiceAgentTelephonyCampaignAsync::test_telephony_operation_not_found -s + @pytest.mark.skip( + reason="agent_telephony routes are registered but return 400 UnsupportedApiVersion for " + "api-version 'v1' on the live test service. Un-skip once the service supports 'v1' for " + "this operation group." + ) + @servicePreparer() + @recorded_by_proxy_async() + async def test_telephony_operation_not_found(self, **kwargs): + """ + Test the generic long-running-operation status endpoint used to poll + `begin_import_campaign_recipients`/`begin_publish_campaign`/ + `begin_validate_campaign`, against a nonexistent operation id, which returns + 404. + + Routes used in this test: + + Action REST API Route Client Method + ------+-------------------------------------------------------------+----------------------------------------------- + POST /agents/{agent_name}/versions project_client.agents.create_version() + GET /agents/{agent_name}/telephony/operations/{operation_id} project_client.beta.agent_telephony.get_operation() + DELETE /agents/{agent_name} project_client.agents.delete() + """ + print("\n") + model = kwargs.get("foundry_voice_model_name") + assert model is not None + # Voice-agent operations require the preview opt-in. + project_client = self.create_async_client(allow_preview=True, **kwargs) + agent_name = "VoiceAgentTelephonyOperationTest" + + # Delete any existing agent from previous test runs (ignore failures) + try: + await project_client.agents.delete(agent_name=agent_name) + except Exception: # pylint: disable=broad-except + pass + + agent_version: AgentVersionDetails = await project_client.agents.create_version( + agent_name=agent_name, + definition=self._make_definition(model), + ) + self._validate_agent_version(agent_version, expected_name=agent_name) + + fake_operation_id = "nonexistent-operation-id" + with pytest.raises(ResourceNotFoundError): + await project_client.beta.agent_telephony.get_operation( + agent_name=agent_name, operation_id=fake_operation_id + ) + + # Delete the voice agent. + result = await project_client.agents.delete(agent_name=agent_name) + assert result.deleted diff --git a/sdk/ai/azure-ai-projects/tests/conftest.py b/sdk/ai/azure-ai-projects/tests/conftest.py index 5d5722183e64..5942ff85c6a1 100644 --- a/sdk/ai/azure-ai-projects/tests/conftest.py +++ b/sdk/ai/azure-ai-projects/tests/conftest.py @@ -369,6 +369,30 @@ def sanitize_url_paths(): # would otherwise fail to decode -> UnicodeDecodeError). add_remove_header_sanitizer(headers="Content-Encoding") + # Strip Foundry-Features from record/playback matching. Its value is a comma-joined list of + # preview opt-in flags that legitimately changes over time as new preview features are added + # (e.g. VoiceAgents=V1Preview was added later); exact-matching it against older cassettes + # would otherwise cause spurious playback failures unrelated to what a given test is actually + # validating. Some affected cassettes (test_ai_agents_instrumentor.py/_async.py) have been + # re-recorded and no longer need this, but others still rely on it pending re-recording (see + # test_responses_instrumentor_workflow.py, which currently fails to re-record live due to an + # unrelated pre-existing gap in its expected span-attribute list vs. actual gen_ai.usage.* + # token attributes now returned by the service). Tests that specifically need to assert on + # this header's value use a dedicated unit-test suite (tests/foundry_features_header) with a + # capturing transport instead of the test-proxy, so this does not reduce coverage of the + # header-injection behavior itself. + add_remove_header_sanitizer(headers="Foundry-Features") + + # Strip Accept from record/playback matching. It's a content-negotiation hint set by the + # HTTP client/transport layer, not something the tests are validating, and its value has been + # observed to drift across environments independent of any SDK code change here (e.g. the + # azure-storage-blob generated client hardcodes "application/xml" for blob uploads, but some + # environments send "*/*" instead depending on transport/dependency versions). Exact-matching + # it would otherwise cause spurious playback failures on samples like + # sample_models_create_and_poll.py and sample_datasets*.py that upload blobs via + # container_client.upload_blob(), unrelated to what those samples actually validate. + add_remove_header_sanitizer(headers="Accept") + # Remove the following sanitizers since certain fields are needed in tests and are non-sensitive: # - AZSDK3493: $..name # - AZSDK3430: $..id diff --git a/sdk/ai/azure-ai-projects/tests/foundry_features_header/foundry_features_header_test_base.py b/sdk/ai/azure-ai-projects/tests/foundry_features_header/foundry_features_header_test_base.py index ed46ae963413..55e5a869ec46 100644 --- a/sdk/ai/azure-ai-projects/tests/foundry_features_header/foundry_features_header_test_base.py +++ b/sdk/ai/azure-ai-projects/tests/foundry_features_header/foundry_features_header_test_base.py @@ -75,6 +75,18 @@ ), # multi-step helper: validate -> pending_upload -> azcopy -> pending_create_version -> poll get } +# Public `.beta` attributes that are NOT generated REST operations classes and therefore +# cannot be exercised by the generic header-injection test at all (unlike EXCLUDED_BETA_METHODS, +# which excludes specific methods on an otherwise-testable sub-client). +# +# `realtime` is a hand-written WebSocket entry point (azure/ai/projects/_realtime.py): +# `Realtime.connect(...)` synchronously builds and returns a RealtimeConnectionManager without +# performing any I/O -- the actual WebSocket handshake (which carries its own dedicated +# Foundry-Features header) only happens later, on `__enter__`/`__aenter__`. So it never triggers +# CapturingTransport, and doesn't have an EXPECTED_FOUNDRY_FEATURES entry. Its header behavior is +# verified independently in test_realtime_client.py / test_realtime_client_async.py. +NON_OPERATION_BETA_ATTRIBUTES: frozenset = frozenset({"realtime"}) + # Shared test cases for non-beta methods that optionally send the Foundry-Features header. # Used by both test_foundry_features_header_optional.py (sync) and # test_foundry_features_header_optional_async.py (async). diff --git a/sdk/ai/azure-ai-projects/tests/foundry_features_header/test_agent_telephony_protocol.py b/sdk/ai/azure-ai-projects/tests/foundry_features_header/test_agent_telephony_protocol.py new file mode 100644 index 000000000000..803ce909db91 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/foundry_features_header/test_agent_telephony_protocol.py @@ -0,0 +1,123 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +"""Mocked protocol-level tests for the `agent_telephony` (batch 2) call-job/campaign operation +group, covering request construction (HTTP method + URL path) without depending on the live test +service's currently-unavailable API-version support for these routes (see +`tests/agents/test_voice_agent_telephony_campaign.py`, whose recorded tests are all skipped for +that reason). + +Uses the same request-capturing-transport technique as +`tests/foundry_features_header/test_foundry_features_header_on_ga_operations.py`: a transport +that raises as soon as a request is about to be sent, so the generated request builder's URL/method +construction is exercised end-to-end (through the real client, real serialization, and the real +pipeline) without any network I/O or a live/recorded backend. +""" + +from typing import Any, Iterator, List, Tuple + +import pytest +from azure.core.pipeline.transport import HttpTransport +from azure.ai.projects import AIProjectClient + +from foundry_features_header_test_base import ( + FAKE_ENDPOINT, + FakeCredential, + FoundryFeaturesHeaderTestBase, + _RequestCaptured, +) + + +class CapturingTransport(HttpTransport): + """Sync transport that captures the outgoing request and raises _RequestCaptured.""" + + def send(self, request: Any, **kwargs: Any) -> Any: # type: ignore[override] + raise _RequestCaptured(request) + + def open(self) -> None: + pass + + def close(self) -> None: + pass + + def __enter__(self) -> "CapturingTransport": + return self + + def __exit__(self, *args: Any) -> None: + pass + + +@pytest.fixture(scope="module") +def client() -> Iterator[AIProjectClient]: + with AIProjectClient( + endpoint=FAKE_ENDPOINT, + credential=FakeCredential(), # type: ignore[arg-type] + allow_preview=True, + transport=CapturingTransport(), + ) as c: + yield c + + +# (method_name, expected HTTP method, expected static URL path -- fake string/required params are +# always rendered as the literal "fake-value" by FoundryFeaturesHeaderTestBase._fake_for_param). +_TELEPHONY_PROTOCOL_CASES: List[Tuple[str, str, str]] = [ + ("create_call_job", "POST", "/agents/fake-value/telephony/call_jobs"), + ("get_call_job", "GET", "/agents/fake-value/telephony/call_jobs/fake-value"), + ("cancel_call_job", "POST", "/agents/fake-value/telephony/call_jobs/fake-value:cancel"), + ("create_campaign", "POST", "/agents/fake-value/telephony/campaigns"), + ("get_campaign", "GET", "/agents/fake-value/telephony/campaigns/fake-value"), + ( + "begin_import_campaign_recipients", + "POST", + "/agents/fake-value/telephony/campaigns/fake-value/recipients:import", + ), + ( + "get_campaign_recipient_import", + "GET", + "/agents/fake-value/telephony/campaigns/fake-value/recipient_imports/fake-value", + ), + ("begin_validate_campaign", "POST", "/agents/fake-value/telephony/campaigns/fake-value:validate"), + ("begin_publish_campaign", "POST", "/agents/fake-value/telephony/campaigns/fake-value:publish"), + ("pause_campaign", "POST", "/agents/fake-value/telephony/campaigns/fake-value:pause"), + ("resume_campaign", "POST", "/agents/fake-value/telephony/campaigns/fake-value:resume"), + ("cancel_campaign", "POST", "/agents/fake-value/telephony/campaigns/fake-value:cancel"), + ("get_operation", "GET", "/agents/fake-value/telephony/operations/fake-value"), +] + + +class TestAgentTelephonyProtocol(FoundryFeaturesHeaderTestBase): + """Verify each `agent_telephony` method builds the correct HTTP method and URL path.""" + + @staticmethod + def _capture(call: Any) -> Any: + """Call *call()* and return the captured HttpRequest.""" + try: + result = call() + except _RequestCaptured as exc: + return exc.request + + try: + next(iter(result)) + except _RequestCaptured as exc: + return exc.request + except StopIteration: + raise AssertionError("Iterator exhausted without the transport being called") from None + + raise AssertionError("Transport was never called") + + @pytest.mark.parametrize("method_name,expected_http_method,expected_path", _TELEPHONY_PROTOCOL_CASES) + def test_agent_telephony_request_protocol( + self, + client: AIProjectClient, + method_name: str, + expected_http_method: str, + expected_path: str, + ) -> None: + method = getattr(client.beta.agent_telephony, method_name) + request = self._capture(self._make_fake_call(method)) + assert ( + request.method == expected_http_method + ), f"{method_name}: expected HTTP method {expected_http_method!r}, got {request.method!r}" + assert expected_path in request.url, f"{method_name}: expected path {expected_path!r} in URL {request.url!r}" diff --git a/sdk/ai/azure-ai-projects/tests/foundry_features_header/test_agent_telephony_protocol_async.py b/sdk/ai/azure-ai-projects/tests/foundry_features_header/test_agent_telephony_protocol_async.py new file mode 100644 index 000000000000..3526fd571297 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/foundry_features_header/test_agent_telephony_protocol_async.py @@ -0,0 +1,119 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +"""Async counterpart of test_agent_telephony_protocol.py -- see that module's docstring.""" + +import inspect +from typing import Any, Iterator, List, Tuple + +import pytest +from azure.core.pipeline.transport import AsyncHttpTransport +from azure.ai.projects.aio import AIProjectClient as AsyncAIProjectClient + +from foundry_features_header_test_base import ( + FAKE_ENDPOINT, + AsyncFakeCredential, + FoundryFeaturesHeaderTestBase, + _RequestCaptured, +) + +pytestmark = pytest.mark.asyncio + + +class CapturingAsyncTransport(AsyncHttpTransport): + """Async transport that captures the outgoing request and raises _RequestCaptured.""" + + async def send(self, request: Any, **kwargs: Any) -> Any: # type: ignore[override] + raise _RequestCaptured(request) + + async def open(self) -> None: + pass + + async def close(self) -> None: + pass + + async def __aenter__(self) -> "CapturingAsyncTransport": + return self + + async def __aexit__(self, *args: Any) -> None: + pass + + +@pytest.fixture(scope="module") +def async_client() -> Iterator[AsyncAIProjectClient]: + yield AsyncAIProjectClient( + endpoint=FAKE_ENDPOINT, + credential=AsyncFakeCredential(), # type: ignore[arg-type] + allow_preview=True, + transport=CapturingAsyncTransport(), + ) + + +# (method_name, expected HTTP method, expected static URL path -- fake string/required params are +# always rendered as the literal "fake-value" by FoundryFeaturesHeaderTestBase._fake_for_param). +_TELEPHONY_PROTOCOL_CASES: List[Tuple[str, str, str]] = [ + ("create_call_job", "POST", "/agents/fake-value/telephony/call_jobs"), + ("get_call_job", "GET", "/agents/fake-value/telephony/call_jobs/fake-value"), + ("cancel_call_job", "POST", "/agents/fake-value/telephony/call_jobs/fake-value:cancel"), + ("create_campaign", "POST", "/agents/fake-value/telephony/campaigns"), + ("get_campaign", "GET", "/agents/fake-value/telephony/campaigns/fake-value"), + ( + "begin_import_campaign_recipients", + "POST", + "/agents/fake-value/telephony/campaigns/fake-value/recipients:import", + ), + ( + "get_campaign_recipient_import", + "GET", + "/agents/fake-value/telephony/campaigns/fake-value/recipient_imports/fake-value", + ), + ("begin_validate_campaign", "POST", "/agents/fake-value/telephony/campaigns/fake-value:validate"), + ("begin_publish_campaign", "POST", "/agents/fake-value/telephony/campaigns/fake-value:publish"), + ("pause_campaign", "POST", "/agents/fake-value/telephony/campaigns/fake-value:pause"), + ("resume_campaign", "POST", "/agents/fake-value/telephony/campaigns/fake-value:resume"), + ("cancel_campaign", "POST", "/agents/fake-value/telephony/campaigns/fake-value:cancel"), + ("get_operation", "GET", "/agents/fake-value/telephony/operations/fake-value"), +] + + +class TestAgentTelephonyProtocolAsync(FoundryFeaturesHeaderTestBase): + """Verify each async `agent_telephony` method builds the correct HTTP method and URL path.""" + + @staticmethod + async def _capture(call: Any) -> Any: + """Invoke *call()* and return the captured HttpRequest.""" + result = call() + + if inspect.isawaitable(result): + try: + await result + except _RequestCaptured as exc: + return exc.request + raise AssertionError("Transport was never called (awaitable completed without raising)") + + ai = result.__aiter__() + try: + await ai.__anext__() + except _RequestCaptured as exc: + return exc.request + except StopAsyncIteration: + raise AssertionError("Iterator exhausted without the transport being called") from None + + raise AssertionError("Transport was never called") + + @pytest.mark.parametrize("method_name,expected_http_method,expected_path", _TELEPHONY_PROTOCOL_CASES) + async def test_agent_telephony_request_protocol_async( + self, + async_client: AsyncAIProjectClient, + method_name: str, + expected_http_method: str, + expected_path: str, + ) -> None: + method = getattr(async_client.beta.agent_telephony, method_name) + request = await self._capture(self._make_fake_call(method)) + assert ( + request.method == expected_http_method + ), f"{method_name}: expected HTTP method {expected_http_method!r}, got {request.method!r}" + assert expected_path in request.url, f"{method_name}: expected path {expected_path!r} in URL {request.url!r}" diff --git a/sdk/ai/azure-ai-projects/tests/foundry_features_header/test_foundry_features_header_on_beta_operations.py b/sdk/ai/azure-ai-projects/tests/foundry_features_header/test_foundry_features_header_on_beta_operations.py index 43db5c0811f6..b706a6fe8161 100644 --- a/sdk/ai/azure-ai-projects/tests/foundry_features_header/test_foundry_features_header_on_beta_operations.py +++ b/sdk/ai/azure-ai-projects/tests/foundry_features_header/test_foundry_features_header_on_beta_operations.py @@ -42,6 +42,7 @@ EXPECTED_FOUNDRY_FEATURES, FAKE_ENDPOINT, FOUNDRY_FEATURES_HEADER, + NON_OPERATION_BETA_ATTRIBUTES, FakeCredential, FoundryFeaturesHeaderTestBase, _RequestCaptured, @@ -95,6 +96,8 @@ def _discover_test_cases() -> list[pytest.param]: for sc_name in sorted(dir(temp.beta)): if sc_name.startswith("_"): continue + if sc_name in NON_OPERATION_BETA_ATTRIBUTES: + continue sc = getattr(temp.beta, sc_name) # Sub-clients are non-callable objects (instances of operations classes). # Skip anything callable (e.g. methods directly on BetaOperations itself). diff --git a/sdk/ai/azure-ai-projects/tests/foundry_features_header/test_foundry_features_header_on_beta_operations_async.py b/sdk/ai/azure-ai-projects/tests/foundry_features_header/test_foundry_features_header_on_beta_operations_async.py index afb065d6a155..51eb288510d8 100644 --- a/sdk/ai/azure-ai-projects/tests/foundry_features_header/test_foundry_features_header_on_beta_operations_async.py +++ b/sdk/ai/azure-ai-projects/tests/foundry_features_header/test_foundry_features_header_on_beta_operations_async.py @@ -44,6 +44,7 @@ EXPECTED_FOUNDRY_FEATURES, FAKE_ENDPOINT, FOUNDRY_FEATURES_HEADER, + NON_OPERATION_BETA_ATTRIBUTES, AsyncFakeCredential, FoundryFeaturesHeaderTestBase, _RequestCaptured, @@ -100,6 +101,8 @@ def _discover_async_test_cases() -> list[pytest.param]: for sc_name in sorted(dir(temp.beta)): if sc_name.startswith("_"): continue + if sc_name in NON_OPERATION_BETA_ATTRIBUTES: + continue sc = getattr(temp.beta, sc_name) # Sub-clients are non-callable objects (instances of operations classes). # Skip anything callable (e.g. methods directly on BetaOperations itself). diff --git a/sdk/ai/azure-ai-projects/tests/test_base.py b/sdk/ai/azure-ai-projects/tests/test_base.py index 5e6e48fef69f..08a28d5d7a40 100644 --- a/sdk/ai/azure-ai-projects/tests/test_base.py +++ b/sdk/ai/azure-ai-projects/tests/test_base.py @@ -43,6 +43,7 @@ foundry_project_api_key="sanitized-api-key", foundry_agent_name="sanitized-agent-name", foundry_model_name="sanitized-model-deployment-name", + foundry_voice_model_name="sanitized-model-deployment-name", llm_validation_project_endpoint="https://sanitized-account-name.services.ai.azure.com/api/projects/sanitized-project-name", image_generation_model_deployment_name="sanitized-gpt-image", bing_project_connection_id="/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/sanitized-resource-group/providers/Microsoft.CognitiveServices/accounts/sanitized-account/projects/sanitized-project/connections/sanitized-bing-connection", From 13136df18f7e96c6bb0c749613138b21cd31752c Mon Sep 17 00:00:00 2001 From: Xiting Zhang Date: Thu, 10 Sep 2026 16:35:27 -0700 Subject: [PATCH 2/4] Address PR review feedback and generate voice-agent test recordings Review feedback fixes: - CSpell: add "redef" to sdk/ai/cspell.yaml (fixes CI failure on the type: ignore[no-redef] comment in sample_voice_agent_live_audio_conversation_async.py). - PostEmitter.ps1: remove the _realtime.py regression guard -- verified empirically (ran a real tsp-client update against the pinned commit; the hand-written files came back byte-identical) that tsp-client update does not touch files it doesn't generate, so the guard was unnecessary. - pyproject.toml: rename the "realtime" optional-dependency extra to "voice" (matches how it'll be documented for voice agents); update the 2 samples that referenced [realtime] to [voice]. - Samples: pin azure-ai-projects>=2.7.0 (drop the b1/--pre prerelease pins -- this package no longer ships beta releases) across all 11 voice samples; use the [voice] extra instead of manually listing aiohttp in the 2 samples that need it. - _realtime.py / aio/_realtime.py: - Remove the foundry_features parameter from RealtimeConnectionManager/Realtime.connect/AsyncRealtime.connect -- the realtime route is voice-agent-specific, so the header value is always the same and callers should not need (or be able) to override it. The value is now a hardcoded module constant. - Fix structured_inputs' type: it was Optional[str], silently requiring callers to pre-serialize their own JSON string with no documentation of the expected shape. Retyped to Optional[Mapping[str, Any]], matching the analogous generated model field (CreateTelephonyCallJobRequest.structured_inputs: dict[str, any]), and now serialized internally via SdkJSONEncoder. - Updated test_realtime_client.py/_async.py's _make_manager() helpers to match (foundry_features removed; the header-value assertions still pass since it's now a module constant rather than a per-call override). Test recordings: - Generated live recordings for the 8 new voice-agent tests that had none yet (test_voice_agent_crud(_async).py x6, test_read_conversation(_async) x2), against a real Foundry project with a real gpt-realtime deployment. Verified sanitization (endpoint/auth stripped) and re-verified end-to-end in playback mode after a fresh assets restore from the new tag. - Pushed the new recordings to Azure/azure-sdk-assets (tag python/ai/azure-ai-projects_59c7584f68) via the GitHub Git Data REST API, since a direct git push to that repo hangs in this environment; updated assets.json accordingly. Validated: black/pylint/mypy clean; full existing test suite unaffected (740 passed, 49 skipped, 0 failed, up from 732 passed/8 failed). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk/ai/azure-ai-projects/PostEmitter.ps1 | 28 ---------------- sdk/ai/azure-ai-projects/assets.json | 2 +- .../azure/ai/projects/_realtime.py | 33 +++++++++---------- .../azure/ai/projects/aio/_realtime.py | 33 +++++++++---------- sdk/ai/azure-ai-projects/pyproject.toml | 2 +- .../agents/voice/sample_voice_agent_basic.py | 2 +- .../voice/sample_voice_agent_basic_async.py | 2 +- .../voice/sample_voice_agent_generate.py | 2 +- ...ice_agent_live_audio_conversation_async.py | 2 +- .../sample_voice_agent_live_function_tool.py | 2 +- ...mple_voice_agent_live_text_conversation.py | 2 +- ...oice_agent_live_text_conversation_async.py | 2 +- .../sample_voice_agent_read_conversation.py | 2 +- ...ple_voice_agent_read_conversation_audio.py | 2 +- .../voice/sample_voice_agent_versions.py | 2 +- .../voice/sample_voice_agent_with_tools.py | 2 +- .../tests/agents/test_realtime_client.py | 1 - .../agents/test_realtime_client_async.py | 1 - sdk/ai/cspell.yaml | 1 + 19 files changed, 44 insertions(+), 79 deletions(-) diff --git a/sdk/ai/azure-ai-projects/PostEmitter.ps1 b/sdk/ai/azure-ai-projects/PostEmitter.ps1 index 92bd83fd0910..2783c2cfdbea 100644 --- a/sdk/ai/azure-ai-projects/PostEmitter.ps1 +++ b/sdk/ai/azure-ai-projects/PostEmitter.ps1 @@ -120,34 +120,6 @@ $c = Get-Content $f -Raw $c = $c -replace ' if_match = prep_if_match\(etag, match_condition\)\r?\n if if_match is not None:\r?\n _headers\["If-Match"\] = _SERIALIZER\.header\("if_match", if_match, "str"\)', " if etag is not None:`r`n _headers[`"If-Match`"] = _SERIALIZER.header(`"if_match`", etag, `"str`")" Set-Content $f $c -NoNewline -# Regression guard: `_realtime.py` and `aio\_realtime.py` are hand-written files that are NOT -# `_patch.py`-named, so they aren't covered by the emitter's own "never touch _patch.py" guarantee -- -# nothing in the TypeSpec emitter is aware these files exist. They carry the SDK client-identification -# fix ported from the azure-ai-voicelive PR #48848 (a User-Agent header and x-ms-client-sdk query -# parameter, both derived from `_USER_AGENT = UserAgentPolicy(sdk_moniker=...)`, with a case-insensitive -# guard so a caller-supplied extra_headers User-Agent of any casing is honored instead of duplicated). -# If a future `tsp-client update` ever starts generating (and thus silently overwriting) a file at either -# of these paths, this fix would be lost with no other signal until someone happens to run the realtime -# test suite. Fail the emit step immediately instead, right after regeneration, rather than relying on -# that eventual test run. -$realtimeFiles = @('azure\ai\projects\_realtime.py', 'azure\ai\projects\aio\_realtime.py') -foreach ($f in $realtimeFiles) { - if (-not (Test-Path $f)) { - throw "PostEmitter safety check failed: '$f' is missing. This hand-written file (not tracked by the TypeSpec emitter) carries the SDK client-identification fix from PR #48848; if the emitter deleted or renamed it, restore it from git history before continuing." - } - $c = Get-Content $f -Raw - if ($c -notmatch 'UserAgentPolicy\(sdk_moniker=') { - throw "PostEmitter safety check failed: '$f' no longer defines _USER_AGENT via UserAgentPolicy(sdk_moniker=...). The SDK client-identification fix from PR #48848 appears to have been overwritten -- reinstate the User-Agent header + x-ms-client-sdk query param wiring." - } - if ($c -notmatch '_has_header_case_insensitive') { - throw "PostEmitter safety check failed: '$f' no longer guards the User-Agent header with _has_header_case_insensitive. A caller-supplied extra_headers User-Agent (in any casing) would be duplicated instead of honored -- reinstate the case-insensitive check." - } - if ($c -notmatch 'x-ms-client-sdk') { - throw "PostEmitter safety check failed: '$f' no longer sends the x-ms-client-sdk query parameter alongside the User-Agent header -- reinstate it so service telemetry can still attribute traffic on paths that don't forward the header." - } -} -Write-Host "PostEmitter safety check passed: SDK client-identification fix (PR #48848) is intact in both _realtime.py files." - # Finishing by running 'black' tool to format code. pip install black black --config ../../../eng/black-pyproject.toml . diff --git a/sdk/ai/azure-ai-projects/assets.json b/sdk/ai/azure-ai-projects/assets.json index f94aa7f2bd03..8ea4f92ef1a6 100644 --- a/sdk/ai/azure-ai-projects/assets.json +++ b/sdk/ai/azure-ai-projects/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "python", "TagPrefix": "python/ai/azure-ai-projects", - "Tag": "python/ai/azure-ai-projects_feead9fd04" + "Tag": "python/ai/azure-ai-projects_59c7584f68" } diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/_realtime.py b/sdk/ai/azure-ai-projects/azure/ai/projects/_realtime.py index 879bb801c116..89321b6a2cbe 100644 --- a/sdk/ai/azure-ai-projects/azure/ai/projects/_realtime.py +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/_realtime.py @@ -43,8 +43,9 @@ from ._utils.model_base import Model as _Model, SdkJSONEncoder from ._version import VERSION -# Scoped to just the voice-agent preview opt-in; callers connecting to other preview agent -# kinds through this same route can pass a broader value explicitly via ``foundry_features``. +# The realtime WebSocket route is voice-agent-specific (see `_to_ws_url`'s +# `/endpoint/protocols/voice` path), so this is always the correct opt-in value -- callers +# cannot and do not need to override it. _VOICE_AGENT_FEATURE_HEADER: str = _AgentDefinitionOptInKeys.VOICE_AGENTS_V1_PREVIEW.value # Identifies the SDK to the service on the WebSocket handshake, which otherwise falls back to @@ -674,10 +675,9 @@ def __init__( # pylint: disable=too-many-arguments credential_scopes: List[str], api_version: str, agent_name: str, - foundry_features: str, agent_session_id: Optional[str] = None, agent_version_override: Optional[str] = None, - structured_inputs: Optional[str] = None, + structured_inputs: Optional[Mapping[str, Any]] = None, connection_url: Optional[str] = None, extra_query: Optional[Mapping[str, str]] = None, extra_headers: Optional[Mapping[str, str]] = None, @@ -688,7 +688,6 @@ def __init__( # pylint: disable=too-many-arguments self._credential_scopes = credential_scopes self._api_version = api_version self._agent_name = agent_name - self._foundry_features = foundry_features self._agent_session_id = agent_session_id self._agent_version_override = agent_version_override self._structured_inputs = structured_inputs @@ -745,10 +744,10 @@ def enter(self) -> RealtimeConnection: # pylint: disable=too-many-locals token = self._credential.get_token(*self._credential_scopes) headers: Dict[str, str] = { "Authorization": "Bearer " + token.token, - _FOUNDRY_FEATURES_HEADER_NAME: self._foundry_features, + _FOUNDRY_FEATURES_HEADER_NAME: _VOICE_AGENT_FEATURE_HEADER, } if self._structured_inputs is not None: - headers["x-ms-voice-structured-inputs"] = self._structured_inputs + headers["x-ms-voice-structured-inputs"] = json.dumps(self._structured_inputs, cls=SdkJSONEncoder) headers.update(self._extra_headers) if not _has_header_case_insensitive(headers, "User-Agent"): # Only set our default if the caller didn't supply their own (in any casing) -- @@ -822,10 +821,9 @@ def connect( # pylint: disable=too-many-arguments self, *, agent_name: str, - foundry_features: str = _VOICE_AGENT_FEATURE_HEADER, agent_session_id: Optional[str] = None, agent_version_override: Optional[str] = None, - structured_inputs: Optional[str] = None, + structured_inputs: Optional[Mapping[str, Any]] = None, connection_url: Optional[str] = None, api_version: Optional[str] = None, credential_scopes: Optional[List[str]] = None, @@ -836,19 +834,17 @@ def connect( # pylint: disable=too-many-arguments """Open a realtime WebSocket connection to a voice agent. :keyword str agent_name: The name of the voice agent to connect to. - :keyword foundry_features: Preview opt-in value(s) for the ``Foundry-Features`` header. - Defaults to ``VoiceAgents=V1Preview``. Pass a comma-separated value to opt in to - additional preview features on the same request. - :paramtype foundry_features: str :keyword agent_session_id: An optional identifier used to correlate the voice session. Default value is None. :paramtype agent_session_id: str or None :keyword agent_version_override: Selects a specific version of the voice agent for this session. Default value is None. :paramtype agent_version_override: str or None - :keyword structured_inputs: A JSON object that maps structured-input names to their - values for this session. Default value is None. - :paramtype structured_inputs: str or None + :keyword structured_inputs: A mapping of structured-input names to their values for this + session (see :attr:`~azure.ai.projects.models.CreateTelephonyCallJobRequest.structured_inputs` + for the analogous shape used elsewhere). Serialized to JSON on the wire. Default value is + None. + :paramtype structured_inputs: Mapping[str, Any] or None :keyword connection_url: Full ``wss://`` URL that overrides the route computed from the client endpoint. Query parameters are still appended. Default value is None. :paramtype connection_url: str or None @@ -860,7 +856,9 @@ def connect( # pylint: disable=too-many-arguments :paramtype credential_scopes: list[str] or None :keyword extra_query: Additional query-string parameters for the handshake. :paramtype extra_query: Mapping[str, str] or None - :keyword extra_headers: Additional headers for the handshake. + :keyword extra_headers: Additional headers for the handshake. Pass + ``{"Foundry-Features": "..."}`` here to override the ``VoiceAgents=V1Preview`` value + this method always sends by default. :paramtype extra_headers: Mapping[str, str] or None :return: A context manager yielding a :class:`RealtimeConnection`. :rtype: ~azure.ai.projects.RealtimeConnectionManager @@ -871,7 +869,6 @@ def connect( # pylint: disable=too-many-arguments credential_scopes=credential_scopes or self._config.credential_scopes, api_version=api_version or self._config.api_version, agent_name=agent_name, - foundry_features=foundry_features, agent_session_id=agent_session_id, agent_version_override=agent_version_override, structured_inputs=structured_inputs, diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_realtime.py b/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_realtime.py index 9d8e26af0ff6..c5cb6658a76b 100644 --- a/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_realtime.py +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_realtime.py @@ -57,8 +57,9 @@ from .._utils.model_base import Model as _Model, SdkJSONEncoder from .._version import VERSION -# Scoped to just the voice-agent preview opt-in; callers connecting to other preview agent -# kinds through this same route can pass a broader value explicitly via ``foundry_features``. +# The realtime WebSocket route is voice-agent-specific (see `_to_ws_url`'s +# `/endpoint/protocols/voice` path), so this is always the correct opt-in value -- callers +# cannot and do not need to override it. _VOICE_AGENT_FEATURE_HEADER: str = _AgentDefinitionOptInKeys.VOICE_AGENTS_V1_PREVIEW.value # Identifies the SDK to the service on the WebSocket handshake, which otherwise falls back to @@ -694,10 +695,9 @@ def __init__( # pylint: disable=too-many-arguments credential_scopes: List[str], api_version: str, agent_name: str, - foundry_features: str, agent_session_id: Optional[str] = None, agent_version_override: Optional[str] = None, - structured_inputs: Optional[str] = None, + structured_inputs: Optional[Mapping[str, Any]] = None, connection_url: Optional[str] = None, extra_query: Optional[Mapping[str, str]] = None, extra_headers: Optional[Mapping[str, str]] = None, @@ -708,7 +708,6 @@ def __init__( # pylint: disable=too-many-arguments self._credential_scopes = credential_scopes self._api_version = api_version self._agent_name = agent_name - self._foundry_features = foundry_features self._agent_session_id = agent_session_id self._agent_version_override = agent_version_override self._structured_inputs = structured_inputs @@ -756,10 +755,10 @@ async def enter(self) -> AsyncRealtimeConnection: # pylint: disable=too-many-lo token = await self._credential.get_token(*self._credential_scopes) headers: Dict[str, str] = { "Authorization": "Bearer " + token.token, - _FOUNDRY_FEATURES_HEADER_NAME: self._foundry_features, + _FOUNDRY_FEATURES_HEADER_NAME: _VOICE_AGENT_FEATURE_HEADER, } if self._structured_inputs is not None: - headers["x-ms-voice-structured-inputs"] = self._structured_inputs + headers["x-ms-voice-structured-inputs"] = json.dumps(self._structured_inputs, cls=SdkJSONEncoder) headers.update(self._extra_headers) if not _has_header_case_insensitive(headers, "User-Agent"): # Only set our default if the caller didn't supply their own (in any casing) -- @@ -827,10 +826,9 @@ def connect( # pylint: disable=too-many-arguments self, *, agent_name: str, - foundry_features: str = _VOICE_AGENT_FEATURE_HEADER, agent_session_id: Optional[str] = None, agent_version_override: Optional[str] = None, - structured_inputs: Optional[str] = None, + structured_inputs: Optional[Mapping[str, Any]] = None, connection_url: Optional[str] = None, api_version: Optional[str] = None, credential_scopes: Optional[List[str]] = None, @@ -841,19 +839,17 @@ def connect( # pylint: disable=too-many-arguments """Open a realtime WebSocket connection to a voice agent. :keyword str agent_name: The name of the voice agent to connect to. - :keyword foundry_features: Preview opt-in value(s) for the ``Foundry-Features`` header. - Defaults to ``VoiceAgents=V1Preview``. Pass a comma-separated value to opt in to - additional preview features on the same request. - :paramtype foundry_features: str :keyword agent_session_id: An optional identifier used to correlate the voice session. Default value is None. :paramtype agent_session_id: str or None :keyword agent_version_override: Selects a specific version of the voice agent for this session. Default value is None. :paramtype agent_version_override: str or None - :keyword structured_inputs: A JSON object that maps structured-input names to their - values for this session. Default value is None. - :paramtype structured_inputs: str or None + :keyword structured_inputs: A mapping of structured-input names to their values for this + session (see :attr:`~azure.ai.projects.models.CreateTelephonyCallJobRequest.structured_inputs` + for the analogous shape used elsewhere). Serialized to JSON on the wire. Default value is + None. + :paramtype structured_inputs: Mapping[str, Any] or None :keyword connection_url: Full ``wss://`` URL that overrides the route computed from the client endpoint. Query parameters are still appended. Default value is None. :paramtype connection_url: str or None @@ -865,7 +861,9 @@ def connect( # pylint: disable=too-many-arguments :paramtype credential_scopes: list[str] or None :keyword extra_query: Additional query-string parameters for the handshake. :paramtype extra_query: Mapping[str, str] or None - :keyword extra_headers: Additional headers for the handshake. + :keyword extra_headers: Additional headers for the handshake. Pass + ``{"Foundry-Features": "..."}`` here to override the ``VoiceAgents=V1Preview`` value + this method always sends by default. :paramtype extra_headers: Mapping[str, str] or None :return: An async context manager yielding an :class:`AsyncRealtimeConnection`. :rtype: ~azure.ai.projects.aio.AsyncRealtimeConnectionManager @@ -876,7 +874,6 @@ def connect( # pylint: disable=too-many-arguments credential_scopes=credential_scopes or self._config.credential_scopes, api_version=api_version or self._config.api_version, agent_name=agent_name, - foundry_features=foundry_features, agent_session_id=agent_session_id, agent_version_override=agent_version_override, structured_inputs=structured_inputs, diff --git a/sdk/ai/azure-ai-projects/pyproject.toml b/sdk/ai/azure-ai-projects/pyproject.toml index 494cef1bf4ba..a59e849f31fb 100644 --- a/sdk/ai/azure-ai-projects/pyproject.toml +++ b/sdk/ai/azure-ai-projects/pyproject.toml @@ -43,7 +43,7 @@ dynamic = [ ] [project.optional-dependencies] -realtime = [ +voice = [ "websockets>=13.0", "aiohttp>=3.9.0,<4.0.0", ] diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_basic.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_basic.py index f451b8a57287..faea79fc878c 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_basic.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_basic.py @@ -21,7 +21,7 @@ Before running the sample: - pip install "azure-ai-projects>=2.7.0b1" python-dotenv --pre + pip install "azure-ai-projects>=2.7.0" python-dotenv Set these environment variables with your own values: 1) FOUNDRY_PROJECT_ENDPOINT - The Azure AI Project endpoint, as found in the Overview diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_basic_async.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_basic_async.py index a8b0e692cb0b..10f71323a1d4 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_basic_async.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_basic_async.py @@ -15,7 +15,7 @@ Before running the sample: - pip install "azure-ai-projects>=2.7.0b1" aiohttp python-dotenv --pre + pip install "azure-ai-projects>=2.7.0" aiohttp python-dotenv Set these environment variables with your own values: 1) FOUNDRY_PROJECT_ENDPOINT - The Azure AI Project endpoint, as found in the Overview diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_generate.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_generate.py index 79a55722aefb..421f54a7b6c3 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_generate.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_generate.py @@ -17,7 +17,7 @@ Before running the sample: - pip install "azure-ai-projects>=2.7.0b1" python-dotenv --pre + pip install "azure-ai-projects>=2.7.0" python-dotenv Set these environment variables with your own values: 1) FOUNDRY_PROJECT_ENDPOINT - The Azure AI Project endpoint. diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py index d47d43a482a8..8c7609e9b1d3 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py @@ -29,7 +29,7 @@ ``response.output_audio.*`` events, decoded to PCM16, mono, 24 kHz. Requires ``aiohttp`` and ``pyaudio``. - pip install "azure-ai-projects>=2.7.0b1" azure-identity aiohttp pyaudio --pre + pip install "azure-ai-projects[voice]>=2.7.0" azure-identity pyaudio USAGE: python sample_voice_agent_live_audio_conversation_async.py diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_function_tool.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_function_tool.py index 8c289a8eef69..8718c2820c37 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_function_tool.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_function_tool.py @@ -20,7 +20,7 @@ Before running the sample: - pip install "azure-ai-projects[realtime]>=2.7.0b1" azure-identity python-dotenv --pre + pip install "azure-ai-projects[voice]>=2.7.0" azure-identity python-dotenv Set these environment variables with your own values: 1) FOUNDRY_PROJECT_ENDPOINT - The Azure AI Project endpoint. diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation.py index 26b85cfdf024..4900e8532324 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation.py @@ -27,7 +27,7 @@ sample_voice_agent_live_text_conversation_async.py for the async version of this one). - pip install "azure-ai-projects[realtime]>=2.7.0b1" azure-identity pyaudio --pre + pip install "azure-ai-projects[voice]>=2.7.0" azure-identity pyaudio USAGE: python sample_voice_agent_live_text_conversation.py diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation_async.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation_async.py index 0a9c0486a023..b493666b5c04 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation_async.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation_async.py @@ -24,7 +24,7 @@ ``pyaudio`` is installed; runs headless otherwise. For a hands-free mic conversation with barge-in, see sample_voice_agent_live_audio_conversation_async.py. - pip install "azure-ai-projects>=2.7.0b1" azure-identity aiohttp pyaudio --pre + pip install "azure-ai-projects[voice]>=2.7.0" azure-identity pyaudio USAGE: python sample_voice_agent_live_text_conversation_async.py diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_read_conversation.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_read_conversation.py index b04f0d928aec..85f871f72679 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_read_conversation.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_read_conversation.py @@ -19,7 +19,7 @@ Before running the sample: - pip install "azure-ai-projects>=2.7.0b1" python-dotenv --pre + pip install "azure-ai-projects>=2.7.0" python-dotenv Set these environment variables with your own values: 1) FOUNDRY_PROJECT_ENDPOINT - The Azure AI Project endpoint. diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_read_conversation_audio.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_read_conversation_audio.py index 08be8991a3d6..0eb5a877805c 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_read_conversation_audio.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_read_conversation_audio.py @@ -23,7 +23,7 @@ Before running the sample: - pip install "azure-ai-projects>=2.7.0b1" python-dotenv --pre + pip install "azure-ai-projects>=2.7.0" python-dotenv Set these environment variables with your own values: 1) FOUNDRY_PROJECT_ENDPOINT - The Azure AI Project endpoint. diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_versions.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_versions.py index 441d67a2f02f..0681886e02d6 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_versions.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_versions.py @@ -16,7 +16,7 @@ Before running the sample: - pip install "azure-ai-projects>=2.7.0b1" python-dotenv --pre + pip install "azure-ai-projects>=2.7.0" python-dotenv Set these environment variables with your own values: 1) FOUNDRY_PROJECT_ENDPOINT - The Azure AI Project endpoint. diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_with_tools.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_with_tools.py index d643466f0bc4..eba1fbea2758 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_with_tools.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_with_tools.py @@ -22,7 +22,7 @@ Before running the sample: - pip install "azure-ai-projects>=2.7.0b1" python-dotenv --pre + pip install "azure-ai-projects>=2.7.0" python-dotenv Set these environment variables with your own values: 1) FOUNDRY_PROJECT_ENDPOINT - The Azure AI Project endpoint. diff --git a/sdk/ai/azure-ai-projects/tests/agents/test_realtime_client.py b/sdk/ai/azure-ai-projects/tests/agents/test_realtime_client.py index 9d216a9ceed0..14fa0c102791 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/test_realtime_client.py +++ b/sdk/ai/azure-ai-projects/tests/agents/test_realtime_client.py @@ -53,7 +53,6 @@ def _make_manager(**overrides) -> RealtimeConnectionManager: "credential_scopes": ["https://ai.azure.com/.default"], "api_version": "v1", "agent_name": "my-agent", - "foundry_features": "VoiceAgents=V1Preview", } kwargs.update(overrides) return RealtimeConnectionManager(**kwargs) diff --git a/sdk/ai/azure-ai-projects/tests/agents/test_realtime_client_async.py b/sdk/ai/azure-ai-projects/tests/agents/test_realtime_client_async.py index aa599ba87de8..cfbe4e4dd94b 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/test_realtime_client_async.py +++ b/sdk/ai/azure-ai-projects/tests/agents/test_realtime_client_async.py @@ -49,7 +49,6 @@ def _make_manager(**overrides) -> AsyncRealtimeConnectionManager: "credential_scopes": ["https://ai.azure.com/.default"], "api_version": "v1", "agent_name": "my-agent", - "foundry_features": "VoiceAgents=V1Preview", } kwargs.update(overrides) return AsyncRealtimeConnectionManager(**kwargs) diff --git a/sdk/ai/cspell.yaml b/sdk/ai/cspell.yaml index 18e70907235b..6aa78c08b840 100644 --- a/sdk/ai/cspell.yaml +++ b/sdk/ai/cspell.yaml @@ -93,6 +93,7 @@ words: - quantitive - rdel - recsmplmdl + - redef - reraises - roups - runid From 782642ae8e535ff294c9f3dc745641620bbad964 Mon Sep 17 00:00:00 2001 From: Xiting Zhang Date: Thu, 10 Sep 2026 19:40:22 -0700 Subject: [PATCH 3/4] Fix realtime client wire-contract and sample bugs; simplify voice-agent samples to use create_version directly - _realtime.py / aio/_realtime.py: send structured_inputs as the documented structured_input query parameter instead of a custom header; distinguish graceful WebSocket closure from abnormal closure/transport errors so `for event in conn:` no longer silently swallows real failures. - sample_voice_agent_live_audio_conversation_async.py: always clear buffered local playback audio on barge-in, independent of whether a server response is still active. - sample_voice_agent_live_text_conversation.py / _async.py: drain a cancelled response's terminal event (correlated by response id) before starting the next turn, so a late completion can't be mistaken for the next reply. - test_voice_agent_conversations.py / _async.py: require at least one successful per-item audio retrieval instead of tolerating an all-404 run. - test_realtime_client.py / _async.py: add regression tests for the structured_input query parameter and the graceful-vs-abnormal closure distinction. - sample_voice_agent_live_audio_conversation_async.py, sample_voice_agent_live_text_conversation.py, sample_voice_agent_live_text_conversation_async.py: replace the generate()-then-patch-store two-step pattern with a single direct VoiceAgentDefinition(..., store=True) + create_version() call, matching sample_voice_agent_basic.py. Live-tested all three end-to-end. - GeneratePublicMethods.ps1 / docs/public-methods.md / api.md / api.metadata.yml: fix the generator to recognize the hand-written beta.realtime sub-client (a plain property, not a generated *Operations group) and regenerate. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../GeneratePublicMethods.ps1 | 19 ++- sdk/ai/azure-ai-projects/api.md | 136 ++++++++++++++++++ sdk/ai/azure-ai-projects/api.metadata.yml | 4 +- .../azure/ai/projects/_realtime.py | 32 ++++- .../azure/ai/projects/aio/_realtime.py | 34 ++++- .../azure-ai-projects/docs/public-methods.md | 7 +- ...ice_agent_live_audio_conversation_async.py | 55 ++++--- ...mple_voice_agent_live_text_conversation.py | 110 ++++++++++---- ...oice_agent_live_text_conversation_async.py | 116 +++++++++++---- .../tests/agents/test_realtime_client.py | 48 +++++++ .../agents/test_realtime_client_async.py | 70 +++++++++ .../agents/test_voice_agent_conversations.py | 7 +- .../test_voice_agent_conversations_async.py | 7 +- 13 files changed, 553 insertions(+), 92 deletions(-) diff --git a/sdk/ai/azure-ai-projects/GeneratePublicMethods.ps1 b/sdk/ai/azure-ai-projects/GeneratePublicMethods.ps1 index 938e257ed608..609eef5a301a 100644 --- a/sdk/ai/azure-ai-projects/GeneratePublicMethods.ps1 +++ b/sdk/ai/azure-ai-projects/GeneratePublicMethods.ps1 @@ -54,6 +54,12 @@ def unwrap_operation(value: Any) -> Any: return getattr(value, "_operation", value) +# Hand-written sub-client properties that don't follow the generated *Operations naming +# convention (so they're invisible to the `vars(container)` scan below) but are still part +# of the public surface and should be counted, e.g. `beta.realtime`. +_EXTRA_SUBCLIENT_PROPERTIES = {"realtime"} + + def operation_instances(container: Any, *, exclude: set[str] | None = None) -> dict[str, Any]: excluded = exclude or set() operations: dict[str, Any] = {} @@ -63,15 +69,26 @@ def operation_instances(container: Any, *, exclude: set[str] | None = None) -> d operation = unwrap_operation(value) if type(operation).__name__.endswith("Operations"): operations[name] = operation + for name in _EXTRA_SUBCLIENT_PROPERTIES: + if name in excluded or name in operations: + continue + if isinstance(getattr(type(container), name, None), property): + operations[name] = getattr(container, name) return operations +# Filenames that are fully code-generated from TypeSpec; any other source file backing a +# method (including hand-written modules that aren't named `_patch*.py`, e.g. `_realtime.py`) +# counts as handwritten. +_GENERATED_SOURCE_FILENAMES = {"_operations.py", "_client.py"} + + def is_handwritten_method(cls: type[Any], name: str) -> bool: owner = next((base for base in cls.__mro__ if name in vars(base)), None) if owner is None: raise RuntimeError(f"Unable to find the class that defines {cls.__name__}.{name}") source_path = inspect.getsourcefile(owner) - return source_path is not None and "_patch" in Path(source_path).name + return source_path is not None and Path(source_path).name not in _GENERATED_SOURCE_FILENAMES def public_methods(instance: Any) -> dict[str, bool]: diff --git a/sdk/ai/azure-ai-projects/api.md b/sdk/ai/azure-ai-projects/api.md index c96502657ab5..26db1e62867a 100644 --- a/sdk/ai/azure-ai-projects/api.md +++ b/sdk/ai/azure-ai-projects/api.md @@ -517,6 +517,73 @@ namespace azure.ai.projects.aio.operations ) -> SessionFileWriteResult: ... + class azure.ai.projects.aio.operations.AsyncRealtime: + + def __init__(self, client: _ConfigProvider) -> None: ... + + def connect( + self, + *, + agent_name: str, + agent_session_id: Optional[str] = ..., + agent_version_override: Optional[str] = ..., + api_version: Optional[str] = ..., + connection_url: Optional[str] = ..., + credential_scopes: Optional[List[str]] = ..., + extra_headers: Optional[Mapping[str, str]] = ..., + extra_query: Optional[Mapping[str, str]] = ..., + structured_inputs: Optional[Mapping[str, Any]] = ..., + **kwargs: Any + ) -> AsyncRealtimeConnectionManager: ... + + + class azure.ai.projects.aio.operations.AsyncRealtimeConnection: implements AsyncContextManager + property closed: bool # Read-only + + def __aiter__(self) -> AsyncIterator[ServerEvent]: ... + + def __init__( + self, + connection: ClientWebSocketResponse, + session: ClientSession + ) -> None: ... + + def __repr__(self) -> str: ... + + async def close( + self, + *, + code: int = 1000, + reason: str = "" + ) -> None: ... + + async def recv(self) -> ServerEvent: ... + + async def send(self, event: ClientEvent) -> None: ... + + + class azure.ai.projects.aio.operations.AsyncRealtimeConnectionManager: implements AsyncContextManager + + def __init__( + self, + *, + agent_name: str, + agent_session_id: Optional[str] = ..., + agent_version_override: Optional[str] = ..., + api_version: str, + connection_url: Optional[str] = ..., + credential: AsyncTokenCredential, + credential_scopes: List[str], + endpoint: str, + extra_headers: Optional[Mapping[str, str]] = ..., + extra_query: Optional[Mapping[str, str]] = ..., + structured_inputs: Optional[Mapping[str, Any]] = ..., + **kwargs: Any + ) -> None: ... + + async def enter(self) -> AsyncRealtimeConnection: ... + + class azure.ai.projects.aio.operations.BetaAgentEndpointConversationsOperations: def __init__( @@ -2342,6 +2409,7 @@ namespace azure.ai.projects.aio.operations class azure.ai.projects.aio.operations.BetaOperations(GeneratedBetaOperations): + property realtime: AsyncRealtime # Read-only agent_endpoint_conversations: BetaAgentEndpointConversationsOperations agent_insight_monitors: BetaAgentInsightMonitorsOperations agent_telephony: BetaAgentTelephonyOperations @@ -19017,6 +19085,7 @@ namespace azure.ai.projects.operations class azure.ai.projects.operations.BetaOperations(GeneratedBetaOperations): + property realtime: Realtime # Read-only agent_endpoint_conversations: BetaAgentEndpointConversationsOperations agent_insight_monitors: BetaAgentInsightMonitorsOperations agent_telephony: BetaAgentTelephonyOperations @@ -19789,6 +19858,73 @@ namespace azure.ai.projects.operations ) -> ItemPaged[Index]: ... + class azure.ai.projects.operations.Realtime: + + def __init__(self, client: _ConfigProvider) -> None: ... + + def connect( + self, + *, + agent_name: str, + agent_session_id: Optional[str] = ..., + agent_version_override: Optional[str] = ..., + api_version: Optional[str] = ..., + connection_url: Optional[str] = ..., + credential_scopes: Optional[List[str]] = ..., + extra_headers: Optional[Mapping[str, str]] = ..., + extra_query: Optional[Mapping[str, str]] = ..., + structured_inputs: Optional[Mapping[str, Any]] = ..., + **kwargs: Any + ) -> RealtimeConnectionManager: ... + + + class azure.ai.projects.operations.RealtimeConnection: implements ContextManager + property closed: bool # Read-only + + def __init__(self, connection: ClientConnection) -> None: ... + + def __iter__(self) -> Iterator[ServerEvent]: ... + + def __repr__(self) -> str: ... + + def close( + self, + *, + code: int = 1000, + reason: str = "" + ) -> None: ... + + def recv( + self, + *, + timeout: Optional[float] = ... + ) -> ServerEvent: ... + + def send(self, event: ClientEvent) -> None: ... + + + class azure.ai.projects.operations.RealtimeConnectionManager: implements ContextManager + + def __init__( + self, + *, + agent_name: str, + agent_session_id: Optional[str] = ..., + agent_version_override: Optional[str] = ..., + api_version: str, + connection_url: Optional[str] = ..., + credential: TokenCredential, + credential_scopes: List[str], + endpoint: str, + extra_headers: Optional[Mapping[str, str]] = ..., + extra_query: Optional[Mapping[str, str]] = ..., + structured_inputs: Optional[Mapping[str, Any]] = ..., + **kwargs: Any + ) -> None: ... + + def enter(self) -> RealtimeConnection: ... + + class azure.ai.projects.operations.TelemetryOperations: def __init__(self, outer_instance: AIProjectClient) -> None: ... diff --git a/sdk/ai/azure-ai-projects/api.metadata.yml b/sdk/ai/azure-ai-projects/api.metadata.yml index ee3ce652a499..d1a175e7ef40 100644 --- a/sdk/ai/azure-ai-projects/api.metadata.yml +++ b/sdk/ai/azure-ai-projects/api.metadata.yml @@ -1,4 +1,4 @@ -apiMdSha256: 4a432b803472b497ba48f6170508904d0babfab022bab048f55af0aa7d0506bd +apiMdSha256: d7989a8351d8a1bd18aa587a497e4a8452c465b731848293adfab0c6b48a26c5 packageVersion: 2.7.0 parserVersion: 0.3.31 -pythonVersion: 3.12.10 +pythonVersion: 3.13.2 diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/_realtime.py b/sdk/ai/azure-ai-projects/azure/ai/projects/_realtime.py index 89321b6a2cbe..bcd02ddc9ac5 100644 --- a/sdk/ai/azure-ai-projects/azure/ai/projects/_realtime.py +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/_realtime.py @@ -593,8 +593,15 @@ def _iter(self) -> Iterator[ServerEvent]: while True: try: yield self.recv() - except ConnectionResetError: - return + except ConnectionResetError as exc: + # recv() below chains the *specific* websockets exception as the cause: a plain + # graceful closure has no cause (nothing went wrong), while an abnormal closure + # is chained from the ConnectionClosed that caused it. Only end iteration quietly + # for the former -- a `for event in conn:` caller must still see real failures + # (abnormal close codes, e.g. 1011) instead of silently observing end-of-stream. + if exc.__cause__ is None: + return + raise def recv(self, *, timeout: Optional[float] = None) -> ServerEvent: """Receive and parse the next server event. @@ -609,16 +616,24 @@ def recv(self, *, timeout: Optional[float] = None) -> ServerEvent: :paramtype timeout: float or None :return: The parsed server event. :rtype: ~azure.ai.projects.ServerEvent - :raises ConnectionResetError: If the connection was closed by the server. + :raises ConnectionResetError: If the connection was closed by the server, gracefully or + otherwise. Iterating over the connection (``for event in conn:``) treats only a graceful + closure as end-of-stream and re-raises this for an abnormal one. :raises TimeoutError: If ``timeout`` elapses before an event is received. """ - from websockets.exceptions import ConnectionClosed # pylint: disable=import-outside-toplevel + from websockets.exceptions import ( # pylint: disable=import-outside-toplevel + ConnectionClosed, + ConnectionClosedOK, + ) try: raw = self._connection.recv(timeout=timeout) + except ConnectionClosedOK: + self._closed = True + raise ConnectionResetError("The realtime connection was closed.") from None except ConnectionClosed as exc: self._closed = True - raise ConnectionResetError("The realtime connection was closed.") from exc + raise ConnectionResetError(f"The realtime connection was closed abnormally: {exc}") from exc data = raw.decode("utf-8") if isinstance(raw, (bytes, bytearray)) else raw payload: Dict[str, Any] = json.loads(data) event_type = payload.get("type") @@ -731,6 +746,11 @@ def enter(self) -> RealtimeConnection: # pylint: disable=too-many-locals params["agent_session_id"] = self._agent_session_id if self._agent_version_override is not None: params["x-agent-version-override"] = self._agent_version_override + if self._structured_inputs is not None: + # The service reads this from the `structured_input` query parameter (see the + # generated `build_beta_voice_agent_web_socket_connect_voice_agent_request`), not a + # header -- it must be serialized and appended to the URL below, not sent as one. + params["structured_input"] = json.dumps(self._structured_inputs, cls=SdkJSONEncoder) params.update(self._extra_query) if params: @@ -746,8 +766,6 @@ def enter(self) -> RealtimeConnection: # pylint: disable=too-many-locals "Authorization": "Bearer " + token.token, _FOUNDRY_FEATURES_HEADER_NAME: _VOICE_AGENT_FEATURE_HEADER, } - if self._structured_inputs is not None: - headers["x-ms-voice-structured-inputs"] = json.dumps(self._structured_inputs, cls=SdkJSONEncoder) headers.update(self._extra_headers) if not _has_header_case_insensitive(headers, "User-Agent"): # Only set our default if the caller didn't supply their own (in any casing) -- diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_realtime.py b/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_realtime.py index c5cb6658a76b..e7d8595b29e6 100644 --- a/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_realtime.py +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_realtime.py @@ -563,6 +563,12 @@ async def cancel(self, *, response_id: Optional[str] = None, event_id: Optional[ ) +class _AbnormalWebSocketClosure(Exception): + """Internal marker chained onto :exc:`ConnectionResetError` for a non-graceful WebSocket + closure (an abnormal close code), so :meth:`AsyncRealtimeConnection._iter` can tell it apart + from a normal end of stream, which chains no cause.""" + + class AsyncRealtimeConnection: # pylint: disable=too-many-instance-attributes """An open realtime WebSocket connection to a voice agent. @@ -612,8 +618,15 @@ async def _iter(self) -> AsyncIterator[ServerEvent]: while True: try: yield await self.recv() - except ConnectionResetError: - return + except ConnectionResetError as exc: + # recv() below only chains a cause for a non-graceful closure or transport error + # (an abnormal close code, or the real exception behind a WSMsgType.ERROR); a + # graceful closure chains none. A `for event in conn:` caller must still see real + # failures instead of silently observing end-of-stream, so only the former ends + # iteration quietly. + if exc.__cause__ is None: + return + raise async def recv(self) -> ServerEvent: """Receive and parse the next server event. @@ -624,7 +637,10 @@ async def recv(self) -> ServerEvent: :return: The parsed server event. :rtype: ~azure.ai.projects.aio.ServerEvent - :raises ConnectionResetError: If the connection was closed by the server. + :raises ConnectionResetError: If the connection was closed by the server, gracefully or + otherwise, or if the transport reported an error. Iterating over the connection + (``async for event in conn:``) treats only a graceful closure as end-of-stream and + re-raises this for an abnormal one. """ import aiohttp # pylint: disable=import-outside-toplevel @@ -636,6 +652,11 @@ async def recv(self) -> ServerEvent: aiohttp.WSMsgType.CLOSING, aiohttp.WSMsgType.CLOSED, ): + code = self._connection.close_code + if code not in (1000, 1001): + raise ConnectionResetError( + f"The realtime connection was closed abnormally (code {code!r})." + ) from _AbnormalWebSocketClosure(code) raise ConnectionResetError("The realtime connection was closed.") if msg.type == aiohttp.WSMsgType.ERROR: raise ConnectionResetError( @@ -750,6 +771,11 @@ async def enter(self) -> AsyncRealtimeConnection: # pylint: disable=too-many-lo params["agent_session_id"] = self._agent_session_id if self._agent_version_override is not None: params["x-agent-version-override"] = self._agent_version_override + if self._structured_inputs is not None: + # The service reads this from the `structured_input` query parameter (see the + # generated `build_beta_voice_agent_web_socket_connect_voice_agent_request`), not a + # header -- aiohttp appends `params` to the URL for us below. + params["structured_input"] = json.dumps(self._structured_inputs, cls=SdkJSONEncoder) params.update(self._extra_query) token = await self._credential.get_token(*self._credential_scopes) @@ -757,8 +783,6 @@ async def enter(self) -> AsyncRealtimeConnection: # pylint: disable=too-many-lo "Authorization": "Bearer " + token.token, _FOUNDRY_FEATURES_HEADER_NAME: _VOICE_AGENT_FEATURE_HEADER, } - if self._structured_inputs is not None: - headers["x-ms-voice-structured-inputs"] = json.dumps(self._structured_inputs, cls=SdkJSONEncoder) headers.update(self._extra_headers) if not _has_header_case_insensitive(headers, "User-Agent"): # Only set our default if the caller didn't supply their own (in any casing) -- diff --git a/sdk/ai/azure-ai-projects/docs/public-methods.md b/sdk/ai/azure-ai-projects/docs/public-methods.md index ace9de947616..fdf29c8d9f1b 100644 --- a/sdk/ai/azure-ai-projects/docs/public-methods.md +++ b/sdk/ai/azure-ai-projects/docs/public-methods.md @@ -6,11 +6,11 @@ This document lists all public methods available on `AIProjectClient` and its su ## Summary -There are a total of 198 unique public methods: +There are a total of 199 unique public methods: - 5 stable methods on the client - 59 stable methods on top-level sub-clients -- 134 beta methods on nested beta sub-clients +- 135 beta methods on nested beta sub-clients ### Top-level sub-clients (stable operations) @@ -39,6 +39,7 @@ There are a total of 198 unique public methods: | `beta.insights` | BetaInsightsOperations | 3 | | `beta.memory_stores` | BetaMemoryStoresOperations | 13 | | `beta.models` | BetaModelsOperations | 9 | +| `beta.realtime` | Realtime | 1 | | `beta.red_teams` | BetaRedTeamsOperations | 3 | | `beta.routines` | BetaRoutinesOperations | 8 | | `beta.schedules` | BetaSchedulesOperations | 6 | @@ -250,6 +251,8 @@ Alphabetically sorted. An asterisk at the end of the method name means it is a h .beta.models.pending_upload .beta.models.update +.beta.realtime.connect* + .beta.red_teams.create .beta.red_teams.get .beta.red_teams.list diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py index 8c7609e9b1d3..5fa464b5b57b 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py @@ -11,9 +11,8 @@ azure-ai-projects client (see ``azure.ai.projects.aio.operations.AsyncRealtime``). This mirrors the ergonomics of the OpenAI Python realtime client. - 1. Generate a starter voice agent (see sample_voice_agent_generate.py), - then publish a version with `store=True` so the conversation can be - read back afterward. + 1. Create a voice agent with conversation persistence enabled + (`store=True`) so the conversation can be read back afterward. 2. Stream live mic audio and let the agent's server-side VAD detect your turns: your speech is transcribed, the agent replies through the speakers, and talking over it barges in. @@ -37,7 +36,9 @@ Environment variables: 1) FOUNDRY_PROJECT_ENDPOINT (required) - Foundry project endpoint: https://.services.ai.azure.com/api/projects/ - 2) FOUNDRY_VOICE_AGENT_NAME - Optional. Name for the agent created by this + 2) FOUNDRY_VOICE_MODEL - Optional. The realtime model deployment name. + Defaults to "gpt-realtime". + 3) FOUNDRY_VOICE_AGENT_NAME - Optional. Name for the agent created by this sample. Defaults to "sample-live-audio-conversation-agent-async". Runs until you press Ctrl-C. Authenticates with DefaultAzureCredential, so @@ -61,8 +62,6 @@ from azure.ai.projects.aio.operations import AsyncRealtimeConnection # pylint: disable=no-name-in-module from azure.ai.projects.aio import AIProjectClient from azure.ai.projects.models import ( - AgentKind, - GenerateVoiceAgentRequest, RealtimeServerEventConversationItemInputAudioTranscriptionCompleted, RealtimeServerEventInputAudioBufferSpeechStarted, RealtimeServerEventResponseAudioDelta, @@ -71,6 +70,12 @@ RealtimeServerEventResponseDone, RealtimeServerEventSessionCreated, RealtimeServerEventError, + VoiceAgentAudioConfig, + VoiceAgentAudioOutputConfig, + VoiceAgentDefinition, + VoiceModelType, + VoiceOutputModality, + VoiceType, ) load_dotenv() @@ -338,11 +343,15 @@ async def _run_audio_conversation(client: AIProjectClient, agent_name: str) -> O conversation_id = event.conversation_id or conversation_id elif isinstance(event, RealtimeServerEventInputAudioBufferSpeechStarted): # speech_started fires for every user turn, including the very first one, - # when no response is active yet. Only cancel (barge-in) if a response is - # actually in flight; canceling with none active is a service error. + # when no response is active yet. Always drop whatever reply audio is still + # queued locally -- the speaker can lag well behind the server finishing + # generation, so buffered audio can outlive response_active going false and + # must still be cleared here. Only cancel the *server-side* response (a + # separate RPC) and announce the barge-in when a response is actually in + # flight; canceling with none active is a service error. + ap.skip_pending_audio() if response_active: await conn.response.cancel() - ap.skip_pending_audio() print("(listening...)") elif isinstance(event, RealtimeServerEventConversationItemInputAudioTranscriptionCompleted): print(f"You: {event.transcript.strip()}") @@ -407,6 +416,7 @@ async def _read_conversation(client: AIProjectClient, agent_name: str, conversat async def audio_conversation() -> None: endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] + model = os.environ.get("FOUNDRY_VOICE_MODEL") or "gpt-realtime" agent_name = os.environ.get("FOUNDRY_VOICE_AGENT_NAME") or "sample-live-audio-conversation-agent-async" async with ( @@ -414,27 +424,28 @@ async def audio_conversation() -> None: AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as project_client, ): try: - # 1) Generate a starter voice agent (see sample_voice_agent_generate.py). - generated = await project_client.beta.agents.generate( - GenerateVoiceAgentRequest(kind=AgentKind.VOICE, name=agent_name) + # 1) Create a voice agent with conversation persistence enabled (`store=True`) so the + # session's conversation can be fetched back by id afterward. + definition = VoiceAgentDefinition( + model_type=VoiceModelType.MANAGED, + model=model, + instructions="You are a friendly voice assistant. Keep replies short and natural.", + audio=VoiceAgentAudioConfig( + output=VoiceAgentAudioOutputConfig(voice="en-US-AvaNeural", voice_type=VoiceType.AZURE_STANDARD), + ), + output_modalities=[VoiceOutputModality.AUDIO], + store=True, ) - definition = generated.versions.latest.definition # type: ignore[attr-defined] - - # 2) Publish a new version with conversation persistence enabled (`store=True`) so the - # session's conversation can be fetched back by id afterward. Reuse the generated - # definition as-is (instead of reconstructing a new one from a few fields) so audio, - # greeting, tools, and any other service-selected settings are preserved. - definition.store = True # type: ignore[attr-defined] await project_client.agents.create_version( agent_name=agent_name, definition=definition, ) - # 3) Hold a live microphone conversation with the freshly created agent. + # 2) Hold a live microphone conversation with the freshly created agent. print(f"Starting realtime session with agent: {agent_name}") conversation_id = await _run_audio_conversation(project_client, agent_name) - # 4) Fetch the persisted conversation back by id. + # 3) Fetch the persisted conversation back by id. if conversation_id: print(f"Reading persisted conversation {conversation_id!r}...") try: @@ -457,7 +468,7 @@ async def audio_conversation() -> None: except HttpResponseError as e: print(f"Service responded with an error: {e.status_code} {e.reason}") finally: - # 5) Clean up the agent created for this sample. + # 4) Clean up the agent created for this sample. await project_client.agents.delete(agent_name=agent_name) print(f"Deleted voice agent: {agent_name}") diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation.py index 4900e8532324..0eac79a48a9d 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation.py @@ -10,9 +10,8 @@ on top of the generated azure-ai-projects client (see ``azure.ai.projects.operations.Realtime``). - 1. Generate a starter voice agent (see sample_voice_agent_generate.py), - then publish a version with `store=True` so the conversation can be - read back afterward. + 1. Create a voice agent with conversation persistence enabled + (`store=True`) so the conversation can be read back afterward. 2. Hold a typed, multi-turn conversation: each prompt is sent as a ``RealtimeConversationItemMessageUser`` and the reply streams back as typed audio and transcript events. Blank line (or ``exit`` / ``quit``) @@ -35,7 +34,9 @@ Environment variables: 1) FOUNDRY_PROJECT_ENDPOINT (required) - Foundry project endpoint: https://.services.ai.azure.com/api/projects/ - 2) FOUNDRY_VOICE_AGENT_NAME - Optional. Name for the agent created by this + 2) FOUNDRY_VOICE_MODEL - Optional. The realtime model deployment name. + Defaults to "gpt-realtime". + 3) FOUNDRY_VOICE_AGENT_NAME - Optional. Name for the agent created by this sample. Defaults to "sample-live-text-conversation-agent". Authenticates with DefaultAzureCredential, so sign in first (e.g. `az login`). @@ -43,25 +44,36 @@ import os import sys -from typing import Final, Optional +import time +from typing import Final, Optional, TYPE_CHECKING from dotenv import load_dotenv from azure.core.exceptions import HttpResponseError from azure.identity import DefaultAzureCredential from azure.ai.projects import AIProjectClient from azure.ai.projects.models import ( - AgentKind, - GenerateVoiceAgentRequest, RealtimeConversationItemMessageUser, RealtimeConversationItemMessageUserContent, RealtimeConversationItemType, RealtimeServerEventResponseAudioDelta, RealtimeServerEventResponseAudioTranscriptDone, + RealtimeServerEventResponseCreated, RealtimeServerEventResponseDone, RealtimeServerEventSessionCreated, RealtimeServerEventError, + VoiceAgentAudioConfig, + VoiceAgentAudioOutputConfig, + VoiceAgentDefinition, + VoiceAgentTemplateGreetingConfig, + VoiceModelType, + VoiceOutputModality, + VoiceType, ) +if TYPE_CHECKING: + from azure.ai.projects.operations import RealtimeConnection + + load_dotenv() @@ -170,7 +182,44 @@ def seconds(self) -> float: return self._bytes / 2 / _SAMPLE_RATE -def _run_text_conversation(client: AIProjectClient, agent_name: str, has_greeting: bool) -> Optional[str]: +class _CancellationNotConfirmed(Exception): + """Raised when a just-cancelled response's terminal event could not be confirmed within + ``_RESPONSE_TIMEOUT``, leaving the stream in an unknown state.""" + + +def _drain_cancelled_response(conn: "RealtimeConnection", response_id: Optional[str]) -> None: + """Wait (bounded) for a just-cancelled response's terminal event, discarding it and any of + its trailing content events, so the next turn's ``pump()`` doesn't mistake this stale + completion for its own. + + :param conn: The open realtime connection. + :param response_id: The id of the response that was just cancelled, if it was captured from + that response's ``response.created`` event. If None, the first terminal event seen is + accepted, since there is nothing more specific to correlate against. + :type conn: ~azure.ai.projects.RealtimeConnection + :type response_id: str or None + :raises _CancellationNotConfirmed: If no matching terminal event arrives in time. + """ + deadline = time.monotonic() + _RESPONSE_TIMEOUT + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise _CancellationNotConfirmed("Timed out waiting to confirm the cancelled response finished.") + try: + event = conn.recv(timeout=remaining) + except TimeoutError as exc: + raise _CancellationNotConfirmed("Timed out waiting to confirm the cancelled response finished.") from exc + if isinstance(event, RealtimeServerEventResponseDone): + if response_id is None or event.response.id == response_id: + return + # A stray completion for some other response id; keep draining. + elif isinstance(event, RealtimeServerEventError): + print(f"Session error while confirming cancellation: {event.error.message}") + + +def _run_text_conversation( # pylint: disable=too-many-statements + client: AIProjectClient, agent_name: str, has_greeting: bool +) -> Optional[str]: """Hold a typed, multi-turn conversation. :param client: The Foundry project client. @@ -195,17 +244,25 @@ def _run_text_conversation(client: AIProjectClient, agent_name: str, has_greetin def pump() -> None: nonlocal conversation_id, audio_delta_count + active_response_id: Optional[str] = None while True: try: event = conn.recv(timeout=_RESPONSE_TIMEOUT) except TimeoutError: print("Timed out waiting for the agent's reply.") - conn.response.cancel() + conn.response.cancel(response_id=active_response_id) + # Consume the cancellation's own terminal event now, before the next + # turn starts: otherwise a late response.done for *this* cancelled + # response could be mistaken by the next pump() call for its own, + # ending it early and silently dropping the real next reply. + _drain_cancelled_response(conn, active_response_id) return if isinstance(event, RealtimeServerEventSessionCreated): # The persisted conversation id (only present when conversation # persistence is enabled) is set here, not on response.done. conversation_id = event.conversation_id or conversation_id + if isinstance(event, RealtimeServerEventResponseCreated): + active_response_id = event.response.id if isinstance(event, RealtimeServerEventResponseDone): return if isinstance(event, RealtimeServerEventError): @@ -246,6 +303,8 @@ def pump() -> None: pump() except KeyboardInterrupt: print("\n(ending session...)") + except _CancellationNotConfirmed: + print("Could not confirm a cancelled response finished; ending the session.") finally: played = player.enabled player.close() @@ -290,6 +349,7 @@ def _read_conversation(client: AIProjectClient, agent_name: str, conversation_id def text_conversation() -> None: endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] + model = os.environ.get("FOUNDRY_VOICE_MODEL") or "gpt-realtime" agent_name = os.environ.get("FOUNDRY_VOICE_AGENT_NAME") or "sample-live-text-conversation-agent" with ( @@ -297,29 +357,29 @@ def text_conversation() -> None: AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as project_client, ): try: - # 1) Generate a starter voice agent (see sample_voice_agent_generate.py). - generated = project_client.beta.agents.generate( - GenerateVoiceAgentRequest(kind=AgentKind.VOICE, name=agent_name) + # 1) Create a voice agent with conversation persistence enabled (`store=True`) so the + # session's conversation can be fetched back by id afterward. + definition = VoiceAgentDefinition( + model_type=VoiceModelType.MANAGED, + model=model, + instructions="You are a friendly voice assistant. Keep replies short and natural.", + audio=VoiceAgentAudioConfig( + output=VoiceAgentAudioOutputConfig(voice="en-US-AvaNeural", voice_type=VoiceType.AZURE_STANDARD), + ), + output_modalities=[VoiceOutputModality.AUDIO], + greeting=VoiceAgentTemplateGreetingConfig(text="Hi, I'm here to help. What can I do for you?"), + store=True, ) - definition = generated.versions.latest.definition # type: ignore[attr-defined] - - # 2) Publish a new version with conversation persistence enabled (`store=True`) so the - # session's conversation can be fetched back by id afterward. Reuse the generated - # definition as-is (instead of reconstructing a new one from a few fields) so audio, - # greeting, tools, and any other service-selected settings are preserved. - definition.store = True # type: ignore[attr-defined] project_client.agents.create_version( agent_name=agent_name, definition=definition, ) - # 3) Hold the realtime conversation against the freshly created agent. + # 2) Hold the realtime conversation against the freshly created agent. print(f"Starting realtime session with agent: {agent_name}") - conversation_id = _run_text_conversation( - project_client, agent_name, has_greeting=definition.greeting is not None # type: ignore[attr-defined] - ) + conversation_id = _run_text_conversation(project_client, agent_name, has_greeting=True) - # 4) Fetch the persisted conversation back by id. + # 3) Fetch the persisted conversation back by id. if conversation_id: print(f"Reading persisted conversation {conversation_id}...") try: @@ -342,7 +402,7 @@ def text_conversation() -> None: except HttpResponseError as e: print(f"Service responded with an error: {e.status_code} {e.reason}") finally: - # 5) Clean up the agent created for this sample. + # 4) Clean up the agent created for this sample. project_client.agents.delete(agent_name=agent_name) print(f"Deleted voice agent: {agent_name}") diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation_async.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation_async.py index b493666b5c04..95e0e0b0782a 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation_async.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation_async.py @@ -10,9 +10,8 @@ on top of the generated azure-ai-projects client (see ``azure.ai.projects.aio.operations.AsyncRealtime``). - 1. Generate a starter voice agent (see sample_voice_agent_generate.py), - then publish a version with `store=True` so the conversation can be - read back afterward. + 1. Create a voice agent with conversation persistence enabled + (`store=True`) so the conversation can be read back afterward. 2. Hold a typed, multi-turn conversation: each prompt is sent as a ``RealtimeConversationItemMessageUser`` and the reply streams back as typed audio and transcript events. Blank line (or ``exit`` / ``quit``) @@ -32,7 +31,9 @@ Environment variables: 1) FOUNDRY_PROJECT_ENDPOINT (required) - Foundry project endpoint: https://.services.ai.azure.com/api/projects/ - 2) FOUNDRY_VOICE_AGENT_NAME - Optional. Name for the agent created by this + 2) FOUNDRY_VOICE_MODEL - Optional. The realtime model deployment name. + Defaults to "gpt-realtime". + 3) FOUNDRY_VOICE_AGENT_NAME - Optional. Name for the agent created by this sample. Defaults to "sample-live-text-conversation-agent-async". Authenticates with DefaultAzureCredential, so sign in first (e.g. `az login`). @@ -46,18 +47,29 @@ from dotenv import load_dotenv from azure.core.exceptions import HttpResponseError from azure.identity.aio import DefaultAzureCredential + +# AsyncRealtimeConnection is re-exported dynamically via aio/operations/_patch.py's `__all__`; +# pylint's static import resolution cannot trace that, but the symbol is valid (verified by +# Pyright/mypy). +from azure.ai.projects.aio.operations import AsyncRealtimeConnection # pylint: disable=no-name-in-module from azure.ai.projects.aio import AIProjectClient from azure.ai.projects.models import ( - AgentKind, - GenerateVoiceAgentRequest, RealtimeConversationItemMessageUser, RealtimeConversationItemMessageUserContent, RealtimeConversationItemType, RealtimeServerEventResponseAudioDelta, RealtimeServerEventResponseAudioTranscriptDone, + RealtimeServerEventResponseCreated, RealtimeServerEventResponseDone, RealtimeServerEventSessionCreated, RealtimeServerEventError, + VoiceAgentAudioConfig, + VoiceAgentAudioOutputConfig, + VoiceAgentDefinition, + VoiceAgentTemplateGreetingConfig, + VoiceModelType, + VoiceOutputModality, + VoiceType, ) load_dotenv() @@ -168,7 +180,43 @@ def seconds(self) -> float: return self._bytes / 2 / _SAMPLE_RATE -async def _run_text_conversation(client: AIProjectClient, agent_name: str, has_greeting: bool) -> Optional[str]: +class _CancellationNotConfirmed(Exception): + """Raised when a just-cancelled response's terminal event could not be confirmed within + ``_RESPONSE_TIMEOUT``, leaving the stream in an unknown state.""" + + +async def _drain_cancelled_response(conn: "AsyncRealtimeConnection", response_id: Optional[str]) -> None: + """Wait (bounded) for a just-cancelled response's terminal event, discarding it and any of + its trailing content events, so the next turn's ``pump()`` doesn't mistake this stale + completion for its own. + + :param conn: The open realtime connection. + :param response_id: The id of the response that was just cancelled, if it was captured from + that response's ``response.created`` event. If None, the first terminal event seen is + accepted, since there is nothing more specific to correlate against. + :type conn: ~azure.ai.projects.aio.AsyncRealtimeConnection + :type response_id: str or None + :raises _CancellationNotConfirmed: If no matching terminal event arrives in time. + """ + + async def _drain() -> None: + async for event in conn: + if isinstance(event, RealtimeServerEventResponseDone): + if response_id is None or event.response.id == response_id: + return + # A stray completion for some other response id; keep draining. + elif isinstance(event, RealtimeServerEventError): + print(f"Session error while confirming cancellation: {event.error.message}") + + try: + await asyncio.wait_for(_drain(), timeout=_RESPONSE_TIMEOUT) + except asyncio.TimeoutError as exc: + raise _CancellationNotConfirmed("Timed out waiting to confirm the cancelled response finished.") from exc + + +async def _run_text_conversation( # pylint: disable=too-many-statements + client: AIProjectClient, agent_name: str, has_greeting: bool +) -> Optional[str]: """Hold a typed, multi-turn conversation. :param client: The Foundry project client. @@ -184,6 +232,7 @@ async def _run_text_conversation(client: AIProjectClient, agent_name: str, has_g """ conversation_id: Optional[str] = None audio_delta_count = 0 + active_response_id: Optional[str] = None player = _SpeakerPlayer() try: @@ -191,12 +240,14 @@ async def _run_text_conversation(client: AIProjectClient, agent_name: str, has_g async with client.beta.realtime.connect(agent_name=agent_name) as conn: async def pump() -> None: - nonlocal conversation_id, audio_delta_count + nonlocal conversation_id, audio_delta_count, active_response_id async for event in conn: if isinstance(event, RealtimeServerEventSessionCreated): # The persisted conversation id (only present when conversation # persistence is enabled) is set here, not on response.done. conversation_id = event.conversation_id or conversation_id + if isinstance(event, RealtimeServerEventResponseCreated): + active_response_id = event.response.id if isinstance(event, RealtimeServerEventResponseDone): return if isinstance(event, RealtimeServerEventError): @@ -221,7 +272,12 @@ async def pump() -> None: await asyncio.wait_for(pump(), timeout=_RESPONSE_TIMEOUT) except asyncio.TimeoutError: print("Timed out waiting for the agent's greeting.") - await conn.response.cancel() + await conn.response.cancel(response_id=active_response_id) + # Consume the cancellation's own terminal event now, before the next turn + # starts: otherwise a late response.done for *this* cancelled response could + # be mistaken by the next pump() call for its own, ending it early and + # silently dropping the real next reply. + await _drain_cancelled_response(conn, active_response_id) print("Type a message and press Enter. Blank line (or 'exit') ends the session.") @@ -246,9 +302,16 @@ async def pump() -> None: print("Timed out waiting for the agent's reply.") # The server-side response is still active even though we stopped waiting # locally; cancel it so the next turn's response.create() isn't rejected. - await conn.response.cancel() + # Then consume its terminal event now, before the next turn starts: + # otherwise a late response.done for *this* cancelled response could be + # mistaken by the next pump() call for its own, ending it early and + # silently dropping the real next reply. + await conn.response.cancel(response_id=active_response_id) + await _drain_cancelled_response(conn, active_response_id) except (KeyboardInterrupt, asyncio.CancelledError): print("\n(ending session...)") + except _CancellationNotConfirmed: + print("Could not confirm a cancelled response finished; ending the session.") finally: played = player.enabled player.close() @@ -293,6 +356,7 @@ async def _read_conversation(client: AIProjectClient, agent_name: str, conversat async def text_conversation() -> None: endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] + model = os.environ.get("FOUNDRY_VOICE_MODEL") or "gpt-realtime" agent_name = os.environ.get("FOUNDRY_VOICE_AGENT_NAME") or "sample-live-text-conversation-agent-async" async with ( @@ -300,29 +364,29 @@ async def text_conversation() -> None: AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as project_client, ): try: - # 1) Generate a starter voice agent (see sample_voice_agent_generate.py). - generated = await project_client.beta.agents.generate( - GenerateVoiceAgentRequest(kind=AgentKind.VOICE, name=agent_name) + # 1) Create a voice agent with conversation persistence enabled (`store=True`) so the + # session's conversation can be fetched back by id afterward. + definition = VoiceAgentDefinition( + model_type=VoiceModelType.MANAGED, + model=model, + instructions="You are a friendly voice assistant. Keep replies short and natural.", + audio=VoiceAgentAudioConfig( + output=VoiceAgentAudioOutputConfig(voice="en-US-AvaNeural", voice_type=VoiceType.AZURE_STANDARD), + ), + output_modalities=[VoiceOutputModality.AUDIO], + greeting=VoiceAgentTemplateGreetingConfig(text="Hi, I'm here to help. What can I do for you?"), + store=True, ) - definition = generated.versions.latest.definition # type: ignore[attr-defined] - - # 2) Publish a new version with conversation persistence enabled (`store=True`) so the - # session's conversation can be fetched back by id afterward. Reuse the generated - # definition as-is (instead of reconstructing a new one from a few fields) so audio, - # greeting, tools, and any other service-selected settings are preserved. - definition.store = True # type: ignore[attr-defined] await project_client.agents.create_version( agent_name=agent_name, definition=definition, ) - # 3) Hold the realtime conversation against the freshly created agent. + # 2) Hold the realtime conversation against the freshly created agent. print(f"Starting realtime session with agent: {agent_name}") - conversation_id = await _run_text_conversation( - project_client, agent_name, has_greeting=definition.greeting is not None # type: ignore[attr-defined] - ) + conversation_id = await _run_text_conversation(project_client, agent_name, has_greeting=True) - # 4) Fetch the persisted conversation back by id. + # 3) Fetch the persisted conversation back by id. if conversation_id: print(f"Reading persisted conversation {conversation_id}...") try: @@ -345,7 +409,7 @@ async def text_conversation() -> None: except HttpResponseError as e: print(f"Service responded with an error: {e.status_code} {e.reason}") finally: - # 5) Clean up the agent created for this sample. + # 4) Clean up the agent created for this sample. await project_client.agents.delete(agent_name=agent_name) print(f"Deleted voice agent: {agent_name}") diff --git a/sdk/ai/azure-ai-projects/tests/agents/test_realtime_client.py b/sdk/ai/azure-ai-projects/tests/agents/test_realtime_client.py index 14fa0c102791..64ec6cef48ef 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/test_realtime_client.py +++ b/sdk/ai/azure-ai-projects/tests/agents/test_realtime_client.py @@ -236,6 +236,22 @@ def test_enter_appends_extra_query_and_headers(self): assert "foo=bar" in _args[0] assert kwargs["additional_headers"]["X-Custom"] == "1" + def test_enter_sends_structured_inputs_as_query_parameter(self): + # Regression test: structured_inputs used to be serialized into a custom + # "x-ms-voice-structured-inputs" header, but the generated request builder + # (build_beta_voice_agent_web_socket_connect_voice_agent_request) defines this as the + # "structured_input" query parameter -- the service never actually read the header. + fake_connection = MagicMock() + with patch("websockets.sync.client.connect", return_value=fake_connection) as mock_connect: + manager = _make_manager(structured_inputs={"greeting_name": "Alex"}) + manager.enter() + manager.__exit__() + + _args, kwargs = mock_connect.call_args + query = parse_qs(urlparse(_args[0]).query) + assert json.loads(query["structured_input"][0]) == {"greeting_name": "Alex"} + assert "x-ms-voice-structured-inputs" not in kwargs["additional_headers"] + def test_enter_preserves_existing_query_on_connection_url_override(self): # Regression test: the URL builder used to unconditionally append "?", corrupting an # override URL that already has a query string (e.g. a SAS-style "?sig=..."). @@ -346,6 +362,38 @@ def test_iteration_stops_cleanly_on_connection_reset(self, request): fake_connection.recv.side_effect = ConnectionResetError() assert list(conn) == [] + def test_iteration_stops_cleanly_on_graceful_close(self, request): + from websockets.exceptions import ConnectionClosedOK + from websockets.frames import Close + + fake_connection = MagicMock() + with patch("websockets.sync.client.connect", return_value=fake_connection): + manager = _make_manager() + conn = manager.enter() + request.addfinalizer(manager.__exit__) + + fake_connection.recv.side_effect = ConnectionClosedOK(Close(1000, "bye"), None) + assert list(conn) == [] + + def test_iteration_propagates_abnormal_closure(self, request): + # Regression test: recv() converts every websockets.exceptions.ConnectionClosed + # (graceful *and* abnormal) into ConnectionResetError, so a blanket except clause here + # made a real server-side failure (e.g. close code 1011) indistinguishable from a normal + # end of stream -- a `for event in conn:` caller could silently accept a truncated + # response. Only a graceful closure (ConnectionClosedOK) should end iteration quietly. + from websockets.exceptions import ConnectionClosedError + from websockets.frames import Close + + fake_connection = MagicMock() + with patch("websockets.sync.client.connect", return_value=fake_connection): + manager = _make_manager() + conn = manager.enter() + request.addfinalizer(manager.__exit__) + + fake_connection.recv.side_effect = ConnectionClosedError(Close(1011, "internal error"), None) + with pytest.raises(ConnectionResetError): + list(conn) + class TestRealtimeConnectionSend: """Unit tests for ``RealtimeConnection.send()``: model/str/mapping serialization.""" diff --git a/sdk/ai/azure-ai-projects/tests/agents/test_realtime_client_async.py b/sdk/ai/azure-ai-projects/tests/agents/test_realtime_client_async.py index cfbe4e4dd94b..85dc3de81b25 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/test_realtime_client_async.py +++ b/sdk/ai/azure-ai-projects/tests/agents/test_realtime_client_async.py @@ -125,6 +125,22 @@ async def test_enter_caller_user_agent_overrides_default(self): _args, kwargs = fake_session.ws_connect.call_args assert kwargs["headers"]["User-Agent"] == "custom-user-agent" + async def test_enter_sends_structured_inputs_as_query_parameter(self): + # Regression test: structured_inputs used to be serialized into a custom + # "x-ms-voice-structured-inputs" header, but the generated request builder + # (build_beta_voice_agent_web_socket_connect_voice_agent_request) defines this as the + # "structured_input" query parameter -- the service never actually read the header. + fake_ws = _make_fake_ws() + patcher, fake_session = _patch_client_session(fake_ws) + with patcher: + manager = _make_manager(structured_inputs={"greeting_name": "Alex"}) + await manager.enter() + await manager.__aexit__() + + _args, kwargs = fake_session.ws_connect.call_args + assert json.loads(kwargs["params"]["structured_input"]) == {"greeting_name": "Alex"} + assert "x-ms-voice-structured-inputs" not in kwargs["headers"] + async def test_enter_caller_user_agent_overrides_default_case_insensitive(self): # Regression test: a plain dict merge of extra_headers would leave a differently-cased # caller override (e.g. "user-agent") as a *separate* key alongside our own "User-Agent" @@ -301,6 +317,60 @@ async def test_recv_error_frame_raises_connection_reset_error(self): finally: await manager.__aexit__() + async def test_iteration_stops_cleanly_on_graceful_close(self): + import aiohttp + + fake_ws = _make_fake_ws() + fake_ws.close_code = 1000 # Normal Closure + fake_ws.receive = AsyncMock(return_value=_make_fake_msg(aiohttp.WSMsgType.CLOSE)) + patcher, _ = _patch_client_session(fake_ws) + with patcher: + manager = _make_manager() + conn = await manager.enter() + try: + assert [event async for event in conn] == [] + finally: + await manager.__aexit__() + + async def test_iteration_propagates_abnormal_closure(self): + # Regression test: recv() used to raise the same ConnectionResetError for every close + # frame regardless of code, and the async iterator caught all of them, so a real + # server-side failure (e.g. close code 1011) was indistinguishable from a normal end of + # stream -- an `async for event in conn:` caller could silently accept a truncated + # response. Only a graceful closure (code 1000/1001) should end iteration quietly. + import aiohttp + + fake_ws = _make_fake_ws() + fake_ws.close_code = 1011 # Internal Error + fake_ws.receive = AsyncMock(return_value=_make_fake_msg(aiohttp.WSMsgType.CLOSE)) + patcher, _ = _patch_client_session(fake_ws) + with patcher: + manager = _make_manager() + conn = await manager.enter() + try: + with pytest.raises(ConnectionResetError): + _ = [event async for event in conn] + finally: + await manager.__aexit__() + + async def test_iteration_propagates_transport_error(self): + # Regression test: same as above, but for the WSMsgType.ERROR path (an actual transport + # exception, not just an abnormal close code) -- this must never be swallowed either. + import aiohttp + + fake_ws = _make_fake_ws() + fake_ws.exception = MagicMock(return_value=RuntimeError("boom")) + fake_ws.receive = AsyncMock(return_value=_make_fake_msg(aiohttp.WSMsgType.ERROR)) + patcher, _ = _patch_client_session(fake_ws) + with patcher: + manager = _make_manager() + conn = await manager.enter() + try: + with pytest.raises(ConnectionResetError): + _ = [event async for event in conn] + finally: + await manager.__aexit__() + class TestAsyncRealtimeConnectionSend: """Unit tests for ``AsyncRealtimeConnection.send()``: model/str/mapping serialization.""" diff --git a/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_conversations.py b/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_conversations.py index d7827a82529a..e5d4794ad9c9 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_conversations.py +++ b/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_conversations.py @@ -214,7 +214,10 @@ def test_read_conversation(self, **kwargs): # pylint: disable=too-many-locals audio_bytes = b"".join(conversations.download_audio(_AGENT_NAME, conversation_id)) assert len(audio_bytes) > 0 - # A single item's audio, if any item has one. + # A single item's audio, if any item has one. Setup guarantees at least one audio + # item exists, so at least one retrieval must succeed -- otherwise a fully-broken + # get_item_audio/download_item_audio route would tolerate every 404 and still pass. + found_item_audio = False for item in items: item_id = item.get("id") if not item_id: @@ -225,6 +228,7 @@ def test_read_conversation(self, **kwargs): # pylint: disable=too-many-locals if e.status_code == 404: continue raise + found_item_audio = True assert item_audio.role is not None if not item_audio.blob_uri: item_audio_bytes = b"".join( @@ -232,6 +236,7 @@ def test_read_conversation(self, **kwargs): # pylint: disable=too-many-locals ) assert len(item_audio_bytes) > 0 break + assert found_item_audio, "Expected at least one conversation item to have retrievable audio" finally: # Deleting a conversation removes it and all of its responses, items, and audio. conversations.delete(_AGENT_NAME, conversation_id) diff --git a/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_conversations_async.py b/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_conversations_async.py index 2abde67e8c81..d7e99d13bb0d 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_conversations_async.py +++ b/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_conversations_async.py @@ -201,7 +201,10 @@ async def test_read_conversation_async(self, **kwargs): # pylint: disable=too-m ] assert len(b"".join(audio_chunks)) > 0 - # A single item's audio, if any item has one. + # A single item's audio, if any item has one. Setup guarantees at least one audio + # item exists, so at least one retrieval must succeed -- otherwise a fully-broken + # get_item_audio/download_item_audio route would tolerate every 404 and still pass. + found_item_audio = False for item in items: item_id = item.get("id") if not item_id: @@ -212,6 +215,7 @@ async def test_read_conversation_async(self, **kwargs): # pylint: disable=too-m if e.status_code == 404: continue raise + found_item_audio = True assert item_audio.role is not None if not item_audio.blob_uri: item_audio_chunks = [ @@ -222,6 +226,7 @@ async def test_read_conversation_async(self, **kwargs): # pylint: disable=too-m ] assert len(b"".join(item_audio_chunks)) > 0 break + assert found_item_audio, "Expected at least one conversation item to have retrievable audio" finally: # Deleting a conversation removes it and all of its responses, items, and audio. await conversations.delete(_AGENT_NAME, conversation_id) From 4a7ddc2a9632379dcd8503226ae9b45254856144 Mon Sep 17 00:00:00 2001 From: Xiting Zhang Date: Fri, 11 Sep 2026 09:13:42 -0700 Subject: [PATCH 4/4] Fix pylint line-too-long in VoiceConversationStatus docstring A parallel merge from feature/azure-ai-projects/vnext brought in a docstring bullet-list join fix (for a Sphinx warning) that pushed one line over pylint's 120-char limit. Re-wrap with correct RST continuation indentation; verified clean with docutils (no Sphinx warnings) and pylint (10.00/10). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk/ai/azure-ai-projects/azure/ai/projects/models/_enums.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/models/_enums.py b/sdk/ai/azure-ai-projects/azure/ai/projects/models/_enums.py index 281e80a53473..0eb5e5700c3e 100644 --- a/sdk/ai/azure-ai-projects/azure/ai/projects/models/_enums.py +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/models/_enums.py @@ -2368,7 +2368,8 @@ class VoiceConversationStatus(str, Enum, metaclass=CaseInsensitiveEnumMeta): """The lifecycle status of a persisted voice conversation: * `in_progress`: the live session is active, or post-session persistence finalization is pending. - * `completed`: finalization succeeded after normal or client close, `end_conversation`, a max-duration `1001` close, or a client or network disconnect that the service can still finalize. + * `completed`: finalization succeeded after normal or client close, `end_conversation`, a max-duration `1001` + close, or a client or network disconnect that the service can still finalize. * `failed`: a terminal service, bridge, storage, or unrecoverable transport failure prevented finalization. """