From 20edb7f8562fb17413bf927086209b80cdf2818d Mon Sep 17 00:00:00 2001 From: Josh Park <50765702+JoshParkSJ@users.noreply.github.com> Date: Fri, 9 Oct 2026 11:31:15 -0400 Subject: [PATCH] feat(voice): announce registered tools on voice_tools_ready [JAR-10305] URT now lists the tools it registered (name, resource name, and for MCP tools the server's tool name) in the voice_tools_ready payload. CAS names the tools it offers the voice model itself and dispatches each call by the name URT announced, so neither side has to predict the other's names. Also makes the drain-timeout warning report how many tool tasks were cancelled; it counted unfinished tasks after wait_for had already cancelled and awaited them, so it always reported 0. Co-Authored-By: Claude Opus 5.5 --- packages/uipath-core/pyproject.toml | 2 +- .../src/uipath/core/chat/__init__.py | 4 + .../uipath-core/src/uipath/core/chat/voice.py | 19 ++++ packages/uipath-core/uv.lock | 2 +- packages/uipath/pyproject.toml | 4 +- .../src/uipath/_cli/_chat/_voice_bridge.py | 38 ++++++-- .../tests/cli/chat/test_voice_bridge.py | 90 ++++++++++++++++++- packages/uipath/uv.lock | 4 +- 8 files changed, 147 insertions(+), 16 deletions(-) diff --git a/packages/uipath-core/pyproject.toml b/packages/uipath-core/pyproject.toml index 80e2327f8..1d328bd86 100644 --- a/packages/uipath-core/pyproject.toml +++ b/packages/uipath-core/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "uipath-core" -version = "0.5.33" +version = "0.5.34" description = "UiPath Core abstractions" readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" diff --git a/packages/uipath-core/src/uipath/core/chat/__init__.py b/packages/uipath-core/src/uipath/core/chat/__init__.py index ee4a4c674..45802f9bc 100644 --- a/packages/uipath-core/src/uipath/core/chat/__init__.py +++ b/packages/uipath-core/src/uipath/core/chat/__init__.py @@ -123,6 +123,8 @@ UiPathVoiceToolCallMessage, UiPathVoiceToolCallRequest, UiPathVoiceToolCallResult, + UiPathVoiceToolDescriptor, + UiPathVoiceToolsReady, ) __all__ = [ @@ -195,6 +197,8 @@ "UiPathVoiceToolCallRequest", "UiPathVoiceToolCallMessage", "UiPathVoiceToolCallResult", + "UiPathVoiceToolDescriptor", + "UiPathVoiceToolsReady", # Interrupt (compat shims — deprecated, see interrupt.py) "InterruptTypeEnum", "UiPathConversationInterruptStartEvent", diff --git a/packages/uipath-core/src/uipath/core/chat/voice.py b/packages/uipath-core/src/uipath/core/chat/voice.py index 7b1adf7e8..3f447a6c0 100644 --- a/packages/uipath-core/src/uipath/core/chat/voice.py +++ b/packages/uipath-core/src/uipath/core/chat/voice.py @@ -28,3 +28,22 @@ class UiPathVoiceToolCallResult(_VoiceWire): result: str is_error: bool = Field(..., alias="isError") + + +class UiPathVoiceToolDescriptor(_VoiceWire): + """A tool the runtime registered for a voice session. + + CAS names the tools it offers the voice model itself, then matches each one to + ``name`` by its agent-definition resource (and, for MCP, the server's tool name) + before dispatching a call, so ``name`` never has to be predicted on the CAS side. + """ + + name: str + resource_name: str = Field(..., alias="resourceName") + mcp_tool_name: str | None = Field(None, alias="mcpToolName") + + +class UiPathVoiceToolsReady(_VoiceWire): + """Payload of ``voice_tools_ready``: the tools this runtime can execute.""" + + tools: list[UiPathVoiceToolDescriptor] = Field(default_factory=list) diff --git a/packages/uipath-core/uv.lock b/packages/uipath-core/uv.lock index 8cc930b64..48fa850ef 100644 --- a/packages/uipath-core/uv.lock +++ b/packages/uipath-core/uv.lock @@ -1011,7 +1011,7 @@ wheels = [ [[package]] name = "uipath-core" -version = "0.5.33" +version = "0.5.34" source = { editable = "." } dependencies = [ { name = "opentelemetry-instrumentation" }, diff --git a/packages/uipath/pyproject.toml b/packages/uipath/pyproject.toml index dc3dc5795..0583be723 100644 --- a/packages/uipath/pyproject.toml +++ b/packages/uipath/pyproject.toml @@ -1,11 +1,11 @@ [project] name = "uipath" -version = "2.14.36" +version = "2.14.37" description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools." readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" dependencies = [ - "uipath-core>=0.5.33, <0.6.0", + "uipath-core>=0.5.34, <0.6.0", "uipath-runtime>=0.13.5, <0.14.0", "uipath-platform>=0.2.34, <0.3.0", "uipath-ipc>=2.5.1, <2.6.0", diff --git a/packages/uipath/src/uipath/_cli/_chat/_voice_bridge.py b/packages/uipath/src/uipath/_cli/_chat/_voice_bridge.py index 4240f28c0..bfb6e05e8 100644 --- a/packages/uipath/src/uipath/_cli/_chat/_voice_bridge.py +++ b/packages/uipath/src/uipath/_cli/_chat/_voice_bridge.py @@ -5,7 +5,7 @@ import asyncio import logging import os -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Sequence from copy import deepcopy from enum import Enum from typing import Any @@ -17,6 +17,8 @@ UiPathVoiceToolCallMessage, UiPathVoiceToolCallRequest, UiPathVoiceToolCallResult, + UiPathVoiceToolDescriptor, + UiPathVoiceToolsReady, ) from uipath.platform.constants import ( ENV_BASE_URL, @@ -65,7 +67,8 @@ class VoiceToolCallSession: Receives `voice_tool_call` batches, emits one `voice_tool_result` per `callId`, exits on `voice_session_ended` or disconnect. CAS pulls agent config from Orchestrator directly; this session carries only - tool calls. + tool calls, plus the registered tool names announced on + `voice_tools_ready` so CAS can dispatch to them. """ def __init__( @@ -74,11 +77,13 @@ def __init__( socketio_path: str, headers: dict[str, str], tool_handler: ToolHandler, + tools: Sequence[UiPathVoiceToolDescriptor] = (), ) -> None: self._url = url self._socketio_path = socketio_path self._headers = headers self._tool_handler = tool_handler + self._tools = list(tools) self._client: Any = None self._done = asyncio.Event() self._in_flight: set[asyncio.Task[None]] = set() @@ -141,31 +146,42 @@ def _end_session(self, reason: VoiceSessionEndReason) -> None: self._done.set() async def _drain_in_flight(self) -> None: - """Wait for in-flight tool tasks to finish, capped by the drain timeout.""" + """Wait for in-flight tool tasks to finish, capped by the drain timeout. + + On timeout the tasks still running are cancelled, and this returns only + once they have finished, so no tool call outlives the session. + """ if not self._in_flight: return + tasks = list(self._in_flight) logger.info( "[Voice] Session ended with %d in-flight tool task(s); draining (max %.0fs)", - len(self._in_flight), + len(tasks), _INFLIGHT_TOOL_DRAIN_AFTER_AGENT_END_TIMEOUT_SECONDS, ) try: + # On timeout wait_for cancels the gather, which cancels the tasks + # and waits for them to finish. await asyncio.wait_for( - asyncio.gather(*self._in_flight, return_exceptions=True), + asyncio.gather(*tasks, return_exceptions=True), timeout=_INFLIGHT_TOOL_DRAIN_AFTER_AGENT_END_TIMEOUT_SECONDS, ) except asyncio.TimeoutError: - unfinished = sum(1 for t in self._in_flight if not t.done()) logger.warning( - "[Voice] %d tool task(s) did not complete within %.0fs of session end", - unfinished, + "[Voice] Cancelled %d tool task(s) still running %.0fs after session end", + sum(1 for t in tasks if t.cancelled()), _INFLIGHT_TOOL_DRAIN_AFTER_AGENT_END_TIMEOUT_SECONDS, ) async def _handle_connect(self) -> None: logger.info("[Voice] Socket.io connected to CAS") try: - await self._client.emit(VoiceEvent.TOOLS_READY, {}) + await self._client.emit( + VoiceEvent.TOOLS_READY, + UiPathVoiceToolsReady(tools=self._tools).model_dump( + by_alias=True, exclude_none=True + ), + ) except Exception as exc: # CAS gates tool dispatch on this event; without it the session is dead. logger.warning("[Voice] emit voice_tools_ready failed: %s", exc) @@ -240,9 +256,12 @@ async def _handle_session_ended(self, data: Any = None, *_: Any) -> None: def get_voice_bridge( context: UiPathRuntimeContext, tool_handler: ToolHandler, + tools: Sequence[UiPathVoiceToolDescriptor] = (), ) -> VoiceToolCallSession: """Factory for a CAS voice tool-call session. + ``tools`` are announced to CAS on ``voice_tools_ready``. + Raises: RuntimeError: If UIPATH_URL is not set or invalid. """ @@ -289,4 +308,5 @@ def get_voice_bridge( socketio_path=socketio_path, headers=headers, tool_handler=tool_handler, + tools=tools, ) diff --git a/packages/uipath/tests/cli/chat/test_voice_bridge.py b/packages/uipath/tests/cli/chat/test_voice_bridge.py index c4e704922..e18016130 100644 --- a/packages/uipath/tests/cli/chat/test_voice_bridge.py +++ b/packages/uipath/tests/cli/chat/test_voice_bridge.py @@ -1,10 +1,12 @@ """Tests for VoiceToolCallSession and get_voice_bridge.""" +import asyncio from typing import Any from unittest.mock import AsyncMock, MagicMock import pytest +from uipath._cli._chat import _voice_bridge as voice_bridge from uipath._cli._chat._voice_bridge import ( VoiceSessionEndReason, VoiceToolCallSession, @@ -13,6 +15,7 @@ from uipath.core.chat import ( UiPathVoiceToolCallRequest, UiPathVoiceToolCallResult, + UiPathVoiceToolDescriptor, ) from uipath.platform.constants import ( HEADER_INTERNAL_ACCOUNT_ID, @@ -20,12 +23,15 @@ ) -def _make_session(tool_handler: Any = None) -> VoiceToolCallSession: +def _make_session( + tool_handler: Any = None, tools: list[UiPathVoiceToolDescriptor] | None = None +) -> VoiceToolCallSession: session = VoiceToolCallSession( url="wss://example/test", socketio_path="/socket.io", headers={}, tool_handler=tool_handler or AsyncMock(), + tools=tools or [], ) session._client = MagicMock() session._client.emit = AsyncMock() @@ -114,6 +120,53 @@ async def test_disconnect_sets_disconnected(self) -> None: assert session._end_reason == VoiceSessionEndReason.DISCONNECTED +class TestToolsReady: + async def test_announces_registered_tools(self) -> None: + session = _make_session( + tools=[ + UiPathVoiceToolDescriptor(name="Lookup", resource_name="Lookup"), + UiPathVoiceToolDescriptor( + name="mcp-crm-tool-get_account", + resource_name="CRM", + mcp_tool_name="get_account", + ), + ] + ) + + await session._handle_connect() + + session._client.emit.assert_awaited_once_with( + "voice_tools_ready", + { + "tools": [ + {"name": "Lookup", "resourceName": "Lookup"}, + { + "name": "mcp-crm-tool-get_account", + "resourceName": "CRM", + "mcpToolName": "get_account", + }, + ] + }, + ) + + async def test_announces_empty_list_without_tools(self) -> None: + session = _make_session() + + await session._handle_connect() + + session._client.emit.assert_awaited_once_with( + "voice_tools_ready", {"tools": []} + ) + + async def test_emit_failure_ends_session(self) -> None: + session = _make_session() + session._client.emit.side_effect = RuntimeError("socket closed") + + await session._handle_connect() + + assert session._end_reason == VoiceSessionEndReason.READY_EMIT_FAILED + + class TestHandleToolCall: async def test_dispatches_handler_and_emits_result(self) -> None: handler = AsyncMock( @@ -177,7 +230,42 @@ async def test_handler_exception_emits_error_result(self) -> None: ) +class TestDrainInFlight: + async def test_cancels_tasks_still_running_at_timeout( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr( + voice_bridge, "_INFLIGHT_TOOL_DRAIN_AFTER_AGENT_END_TIMEOUT_SECONDS", 0.01 + ) + never_finishes = asyncio.Event() + + async def handler(_: Any) -> UiPathVoiceToolCallResult: + await never_finishes.wait() + return UiPathVoiceToolCallResult(result="late", is_error=False) + + session = _make_session(handler) + await session._handle_tool_call( + {"calls": [{"callId": "c1", "toolName": "slow", "args": {}}]} + ) + tasks = list(session._in_flight) + + await session._drain_in_flight() + + # The call is over before run() returns, so callers can release what it used. + assert all(task.cancelled() for task in tasks) + session._client.emit.assert_not_awaited() + + class TestGetVoiceBridge: + def test_passes_tools_to_session(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("UIPATH_URL", "https://cloud.uipath.com") + ctx = MagicMock(conversation_id="conv-1", tenant_id="t", org_id="o") + tools = [UiPathVoiceToolDescriptor(name="Lookup", resource_name="Lookup")] + + bridge = get_voice_bridge(ctx, AsyncMock(), tools=tools) + + assert bridge._tools == tools + def test_raises_when_uipath_url_missing( self, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/packages/uipath/uv.lock b/packages/uipath/uv.lock index 9887fc03a..ff4dec6b9 100644 --- a/packages/uipath/uv.lock +++ b/packages/uipath/uv.lock @@ -2599,7 +2599,7 @@ wheels = [ [[package]] name = "uipath" -version = "2.14.36" +version = "2.14.37" source = { editable = "." } dependencies = [ { name = "applicationinsights" }, @@ -2721,7 +2721,7 @@ dev = [ [[package]] name = "uipath-core" -version = "0.5.33" +version = "0.5.34" source = { editable = "../uipath-core" } dependencies = [ { name = "opentelemetry-instrumentation" },