From 22e87ec42f0ae90399e257c1f8e82fa0a8f059e1 Mon Sep 17 00:00:00 2001 From: philipph-askui Date: Tue, 22 Sep 2026 12:12:41 +0200 Subject: [PATCH 1/2] fix(tools): coerce model-provided tool inputs to the declared schema types `ToolCollection` passed the model's tool arguments straight into the tool implementation without looking at the tool's `input_schema`. When a model emitted integers as JSON strings (observed with Claude via Bedrock, e.g. `{"x": "726", "y": "122", "repeat": "1", "repeat_delay_in_ms": "50"}` for the Android tap tool), tools failed with opaque errors such as Tool raised an unexpected error: '<' not supported between instances of 'str' and 'int' which made the agent fall back to drag-and-drop workarounds and break test runs. Add `coerce_tool_input()`, which walks the tool's JSON schema (refs resolved, `anyOf`/`oneOf`/type lists, nested objects and arrays) and performs only lossless conversions: numeric strings and integral floats to `integer`, numeric strings to `number`, `"true"`/`"false"`/`0`/`1` to `boolean`, and numbers to `string`. Anything that cannot be converted unambiguously is left untouched so the tool's own validation still applies. Apply it in `ToolCollection._run_regular_tool` so every regular tool benefits. Add unit tests for the coercion itself and a regression test that runs the real `AndroidTapTool` through `ToolCollection` with the string payload seen in the failing reports. Co-Authored-By: Claude Fable 5.1 --- .../models/shared/tool_input_coercion.py | 181 +++++++++++++++++ src/askui/models/shared/tools.py | 6 +- .../models/shared/test_tool_input_coercion.py | 182 ++++++++++++++++++ tests/unit/tools/android/test_tap_tool.py | 69 +++++++ 4 files changed, 437 insertions(+), 1 deletion(-) create mode 100644 src/askui/models/shared/tool_input_coercion.py create mode 100644 tests/unit/models/shared/test_tool_input_coercion.py create mode 100644 tests/unit/tools/android/test_tap_tool.py diff --git a/src/askui/models/shared/tool_input_coercion.py b/src/askui/models/shared/tool_input_coercion.py new file mode 100644 index 00000000..048031e6 --- /dev/null +++ b/src/askui/models/shared/tool_input_coercion.py @@ -0,0 +1,181 @@ +"""Coercion of model-provided tool inputs against a tool's JSON input schema. + +Models occasionally emit tool arguments with the wrong JSON type, most commonly +integers as strings (`{"x": "726"}` instead of `{"x": 726}`). Passing such values +straight into a tool implementation typically fails with an opaque `TypeError` +(e.g. `'<' not supported between instances of 'str' and 'int'`). The functions in +this module coerce scalar values to the type declared in the tool's +`input_schema` wherever that is possible without ambiguity, and leave everything +else untouched so the tool's own validation still applies. +""" + +import logging +from typing import Any + +import jsonref + +logger = logging.getLogger(__name__) + +_TRUE_STRINGS = frozenset({"true", "1", "yes"}) +_FALSE_STRINGS = frozenset({"false", "0", "no"}) + + +def coerce_tool_input( + tool_input: dict[str, Any], input_schema: dict[str, Any] +) -> dict[str, Any]: + """Coerce the values of `tool_input` to the types declared in `input_schema`. + + Only lossless, unambiguous conversions are performed: + + - `"42"` -> `42` for `integer` + - `3.0` -> `3` for `integer` + - `"3.5"` -> `3.5` for `number` + - `"true"` / `"false"` (and `1` / `0`) -> `bool` for `boolean` + - `42` -> `"42"` for `string` + + Values that cannot be converted, keys that are not described by the schema, + and schemas without a recognizable scalar `type` are returned unchanged. The + input dictionary is never mutated. + + Args: + tool_input (dict[str, Any]): The arguments the model passed to the tool. + input_schema (dict[str, Any]): The tool's JSON schema (`type: object`). + `$ref`s are resolved before coercion. + + Returns: + dict[str, Any]: A new dictionary with coerced values. + """ + try: + resolved_schema = jsonref.replace_refs( + input_schema, lazy_load=False, proxies=False + ) + except Exception: # noqa: BLE001 + logger.debug("Could not resolve refs in tool input schema", exc_info=True) + resolved_schema = input_schema + coerced = _coerce_value(tool_input, resolved_schema) + return coerced if isinstance(coerced, dict) else tool_input + + +def _coerce_value(value: Any, schema: Any) -> Any: + if not isinstance(schema, dict): + return value + declared_types = _declared_types(schema) + + if isinstance(value, dict) and "object" in declared_types: + return _coerce_object(value, schema) + if isinstance(value, list) and "array" in declared_types: + return _coerce_array(value, schema) + if isinstance(value, (dict, list)): + return value + return _coerce_scalar(value, declared_types) + + +def _coerce_object(value: dict[str, Any], schema: dict[str, Any]) -> dict[str, Any]: + properties = schema.get("properties") + if not isinstance(properties, dict): + return value + return { + key: _coerce_value(item, properties.get(key)) for key, item in value.items() + } + + +def _coerce_array(value: list[Any], schema: dict[str, Any]) -> list[Any]: + items_schema = schema.get("items") + if not isinstance(items_schema, dict): + return value + return [_coerce_value(item, items_schema) for item in value] + + +def _declared_types(schema: dict[str, Any]) -> set[str]: + """Collect the JSON types a schema accepts, including `anyOf`/`oneOf` variants.""" + declared: set[str] = set() + schema_type = schema.get("type") + if isinstance(schema_type, str): + declared.add(schema_type) + elif isinstance(schema_type, list): + declared.update(t for t in schema_type if isinstance(t, str)) + for combinator in ("anyOf", "oneOf"): + variants = schema.get(combinator) + if isinstance(variants, list): + for variant in variants: + if isinstance(variant, dict): + declared.update(_declared_types(variant)) + return declared + + +def _coerce_scalar(value: Any, declared_types: set[str]) -> Any: + if value is None or not declared_types: + return value + if _matches_declared_type(value, declared_types): + return value + if "integer" in declared_types: + as_int = _to_int(value) + if as_int is not None: + return as_int + if "number" in declared_types: + as_float = _to_float(value) + if as_float is not None: + return as_float + if "boolean" in declared_types: + as_bool = _to_bool(value) + if as_bool is not None: + return as_bool + if "string" in declared_types and isinstance(value, (int, float)): + return str(value) + return value + + +def _matches_declared_type(value: Any, declared_types: set[str]) -> bool: + if isinstance(value, bool): + return "boolean" in declared_types + if isinstance(value, int): + return "integer" in declared_types or "number" in declared_types + if isinstance(value, float): + return "number" in declared_types + if isinstance(value, str): + return "string" in declared_types + return True + + +def _to_int(value: Any) -> int | None: + if isinstance(value, bool): + return None + if isinstance(value, float): + return int(value) if value.is_integer() else None + if isinstance(value, str): + stripped = value.strip() + try: + return int(stripped) + except ValueError: + pass + try: + as_float = float(stripped) + except ValueError: + return None + return int(as_float) if as_float.is_integer() else None + return None + + +def _to_float(value: Any) -> float | None: + if isinstance(value, bool): + return None + if isinstance(value, int): + return float(value) + if isinstance(value, str): + try: + return float(value.strip()) + except ValueError: + return None + return None + + +def _to_bool(value: Any) -> bool | None: + if isinstance(value, int) and not isinstance(value, bool) and value in (0, 1): + return bool(value) + if isinstance(value, str): + lowered = value.strip().lower() + if lowered in _TRUE_STRINGS: + return True + if lowered in _FALSE_STRINGS: + return False + return None diff --git a/src/askui/models/shared/tools.py b/src/askui/models/shared/tools.py index fc461e2d..9e5da569 100644 --- a/src/askui/models/shared/tools.py +++ b/src/askui/models/shared/tools.py @@ -38,6 +38,7 @@ ToolUseBlockParam, ) from askui.models.shared.secrets import SecretVault +from askui.models.shared.tool_input_coercion import coerce_tool_input from askui.tools import ComputerAgentOS from askui.tools.android.agent_os import AndroidAgentOs from askui.utils.image_utils import ImageSource, base64_to_image @@ -744,7 +745,10 @@ def _run_regular_tool( tool: Tool, ) -> ToolResultBlockParam: try: - tool_input = self._secret_vault.substitute(tool_use_block_param.input) + tool_input = coerce_tool_input( + self._secret_vault.substitute(tool_use_block_param.input), + tool.input_schema, + ) tool_result: ToolCallResult = tool(**tool_input) # Redact secret values that a tool may echo back in its output, so they do # not leak into the conversation history / model. diff --git a/tests/unit/models/shared/test_tool_input_coercion.py b/tests/unit/models/shared/test_tool_input_coercion.py new file mode 100644 index 00000000..c8eda189 --- /dev/null +++ b/tests/unit/models/shared/test_tool_input_coercion.py @@ -0,0 +1,182 @@ +"""Tests for coercing model-provided tool inputs against the tool's input schema. + +Models occasionally emit tool arguments with the wrong JSON type, most commonly +integers as strings (e.g. `{"x": "726", "y": "122"}`). The tool-calling loop must +coerce such values to the declared schema type before invoking the tool so the +tool does not fail with `'<' not supported between instances of 'str' and 'int'`. +""" + +from typing import Any + +import pytest + +from askui.models.shared.agent_message_param import ( + ToolResultBlockParam, + ToolUseBlockParam, +) +from askui.models.shared.tool_input_coercion import coerce_tool_input +from askui.models.shared.tools import Tool, ToolCollection + +_TAP_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "x": {"type": "integer"}, + "y": {"type": "integer"}, + "repeat": {"type": "integer", "default": 1}, + "repeat_delay_in_ms": {"type": "integer", "default": 50}, + }, + "required": ["x", "y", "repeat", "repeat_delay_in_ms"], +} + + +class TestCoerceToolInput: + def test_string_integers_are_coerced(self) -> None: + tool_input = {"x": "726", "y": "122", "repeat": "1", "repeat_delay_in_ms": "50"} + + assert coerce_tool_input(tool_input, _TAP_SCHEMA) == { + "x": 726, + "y": 122, + "repeat": 1, + "repeat_delay_in_ms": 50, + } + + def test_input_is_not_mutated(self) -> None: + tool_input = {"x": "726", "y": "122"} + + coerce_tool_input(tool_input, _TAP_SCHEMA) + + assert tool_input == {"x": "726", "y": "122"} + + def test_matching_types_are_left_untouched(self) -> None: + tool_input = {"x": 726, "y": 122, "repeat": 1, "repeat_delay_in_ms": 50} + + assert coerce_tool_input(tool_input, _TAP_SCHEMA) == tool_input + + def test_integral_floats_are_coerced_to_integer(self) -> None: + assert coerce_tool_input({"x": 726.0, "y": "122.0"}, _TAP_SCHEMA) == { + "x": 726, + "y": 122, + } + + def test_non_integral_values_are_left_for_the_tool_to_reject(self) -> None: + tool_input = {"x": "abc", "y": 1.5} + + assert coerce_tool_input(tool_input, _TAP_SCHEMA) == tool_input + + def test_unknown_keys_are_left_untouched(self) -> None: + tool_input = {"x": "1", "unknown": "2"} + + assert coerce_tool_input(tool_input, _TAP_SCHEMA) == {"x": 1, "unknown": "2"} + + def test_none_is_left_untouched(self) -> None: + assert coerce_tool_input({"x": None}, _TAP_SCHEMA) == {"x": None} + + @pytest.mark.parametrize( + ("value", "expected"), + [ + ("true", True), + ("False", False), + ("1", True), + (0, False), + ("maybe", "maybe"), + (2, 2), + ], + ) + def test_boolean_coercion(self, value: Any, expected: Any) -> None: + schema = {"type": "object", "properties": {"flag": {"type": "boolean"}}} + + assert coerce_tool_input({"flag": value}, schema) == {"flag": expected} + + def test_number_coercion(self) -> None: + schema = {"type": "object", "properties": {"ratio": {"type": "number"}}} + + assert coerce_tool_input({"ratio": "0.25"}, schema) == {"ratio": 0.25} + assert coerce_tool_input({"ratio": 2}, schema) == {"ratio": 2} + + def test_numbers_are_coerced_to_string(self) -> None: + schema = {"type": "object", "properties": {"text": {"type": "string"}}} + + assert coerce_tool_input({"text": 42}, schema) == {"text": "42"} + + def test_union_types_are_respected(self) -> None: + schema = { + "type": "object", + "properties": { + "a": {"type": ["integer", "null"]}, + "b": {"anyOf": [{"type": "string"}, {"type": "integer"}]}, + }, + } + + assert coerce_tool_input({"a": "3", "b": "3"}, schema) == {"a": 3, "b": "3"} + + def test_nested_objects_and_arrays_are_coerced(self) -> None: + schema = { + "type": "object", + "properties": { + "point": { + "type": "object", + "properties": { + "x": {"type": "integer"}, + "y": {"type": "integer"}, + }, + }, + "ids": {"type": "array", "items": {"type": "integer"}}, + }, + } + + assert coerce_tool_input( + {"point": {"x": "1", "y": "2"}, "ids": ["3", 4, "x"]}, schema + ) == {"point": {"x": 1, "y": 2}, "ids": [3, 4, "x"]} + + def test_refs_are_resolved(self) -> None: + schema = { + "type": "object", + "properties": {"x": {"$ref": "#/$defs/Coordinate"}}, + "$defs": {"Coordinate": {"type": "integer"}}, + } + + assert coerce_tool_input({"x": "7"}, schema) == {"x": 7} + + def test_schema_without_properties_is_a_noop(self) -> None: + tool_input = {"x": "7"} + + assert coerce_tool_input(tool_input, {"type": "object"}) == tool_input + assert coerce_tool_input(tool_input, {}) == tool_input + + +class _TapLikeTool(Tool): + """Mimics a tool that compares integer arguments, as the Android tap tool does.""" + + def __init__(self) -> None: + super().__init__( + name="tap_like_tool", + description="Compares integer arguments.", + input_schema=_TAP_SCHEMA, + ) + + def __call__( + self, x: int, y: int, repeat: int = 1, repeat_delay_in_ms: int = 50 + ) -> str: + if repeat_delay_in_ms < 0 or repeat < 1: + error_msg = "invalid arguments" + raise ValueError(error_msg) + return f"Tapped at ({x}, {y}) {repeat}x" + + +class TestToolCollectionCoercesInput: + def test_string_integers_reach_the_tool_as_integers(self) -> None: + tool = _TapLikeTool() + collection = ToolCollection(tools=[tool]) + tool_use = ToolUseBlockParam( + id="tool_use_1", + input={"x": "726", "y": "122", "repeat": "2", "repeat_delay_in_ms": "50"}, + name=tool.name, + ) + + results = collection.run([tool_use]) + + assert len(results) == 1 + result = results[0] + assert isinstance(result, ToolResultBlockParam) + assert result.is_error is None or result.is_error is False + assert "Tapped at (726, 122) 2x" in str(result.content) diff --git a/tests/unit/tools/android/test_tap_tool.py b/tests/unit/tools/android/test_tap_tool.py new file mode 100644 index 00000000..22e0ea8b --- /dev/null +++ b/tests/unit/tools/android/test_tap_tool.py @@ -0,0 +1,69 @@ +"""Unit tests for `AndroidTapTool`. + +Includes a regression test for models emitting integer tool arguments as JSON +strings, e.g. `{"x": "726", "y": "122", "repeat": "1", "repeat_delay_in_ms": +"50"}`. Calling the tool through `ToolCollection` must coerce these instead of +failing with `'<' not supported between instances of 'str' and 'int'`. +""" + +from unittest.mock import MagicMock + +import pytest +from pytest_mock import MockerFixture + +from askui.models.shared.agent_message_param import ( + ToolResultBlockParam, + ToolUseBlockParam, +) +from askui.models.shared.tools import ToolCollection +from askui.tools.android.agent_os_facade import AndroidAgentOsFacade +from askui.tools.android.tools import AndroidTapTool + + +@pytest.fixture +def agent_os() -> MagicMock: + return MagicMock(spec=AndroidAgentOsFacade) + + +@pytest.fixture +def tap_tool(agent_os: MagicMock, mocker: MockerFixture) -> AndroidTapTool: + mocker.patch("askui.tools.android.tools.time.sleep") + return AndroidTapTool(agent_os=agent_os) + + +class TestAndroidTapTool: + def test_taps_with_integer_arguments( + self, tap_tool: AndroidTapTool, agent_os: MagicMock + ) -> None: + result = tap_tool(x=726, y=122, repeat=1, repeat_delay_in_ms=50) + + agent_os.tap.assert_called_once_with(726, 122) + assert result == "Tapped at (726, 122)" + + def test_rejects_negative_delay(self, tap_tool: AndroidTapTool) -> None: + with pytest.raises(ValueError, match="Delay between taps"): + tap_tool(x=1, y=1, repeat_delay_in_ms=-1) + + def test_rejects_zero_repeat(self, tap_tool: AndroidTapTool) -> None: + with pytest.raises(ValueError, match="Number of taps"): + tap_tool(x=1, y=1, repeat=0) + + def test_string_arguments_from_model_are_coerced( + self, tap_tool: AndroidTapTool, agent_os: MagicMock + ) -> None: + collection = ToolCollection(tools=[tap_tool]) + tool_use = ToolUseBlockParam( + id="tool_use_1", + input={"x": "726", "y": "122", "repeat": "2", "repeat_delay_in_ms": "50"}, + name=tap_tool.name, + ) + + results = collection.run([tool_use]) + + assert len(results) == 1 + result = results[0] + assert isinstance(result, ToolResultBlockParam) + assert not result.is_error + assert "Tapped at (726, 122)" in str(result.content) + assert agent_os.tap.call_count == 2 + agent_os.tap.assert_called_with(726, 122) From c75334999aa105ed68607e98eb81f4620af9b1be Mon Sep 17 00:00:00 2001 From: philipph-askui Date: Tue, 22 Sep 2026 14:48:33 +0200 Subject: [PATCH 2/2] fix(tools): keep tool input coercion strictly lossless Address review feedback on the tool input coercion: - Do not convert booleans to strings for `string` fields. Choosing between `"True"` and `"true"` would be a guess, so the value is left for the tool. - Drop `"yes"`/`"no"` from boolean coercion so the accepted spellings match the documented rules (`"true"`/`"false"`, `1`/`0`). - Add a regression test that a non-coercible value still surfaces as an `is_error` tool result through `ToolCollection.run()` instead of escaping the dispatcher. Co-Authored-By: Claude Fable 5.1 --- .../models/shared/tool_input_coercion.py | 13 +++++++--- .../models/shared/test_tool_input_coercion.py | 26 +++++++++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/src/askui/models/shared/tool_input_coercion.py b/src/askui/models/shared/tool_input_coercion.py index 048031e6..0629dfb1 100644 --- a/src/askui/models/shared/tool_input_coercion.py +++ b/src/askui/models/shared/tool_input_coercion.py @@ -16,8 +16,8 @@ logger = logging.getLogger(__name__) -_TRUE_STRINGS = frozenset({"true", "1", "yes"}) -_FALSE_STRINGS = frozenset({"false", "0", "no"}) +_TRUE_STRINGS = frozenset({"true", "1"}) +_FALSE_STRINGS = frozenset({"false", "0"}) def coerce_tool_input( @@ -31,7 +31,8 @@ def coerce_tool_input( - `3.0` -> `3` for `integer` - `"3.5"` -> `3.5` for `number` - `"true"` / `"false"` (and `1` / `0`) -> `bool` for `boolean` - - `42` -> `"42"` for `string` + - `42` -> `"42"` for `string` (booleans are not converted, since `"True"` + vs. `"true"` would be a guess) Values that cannot be converted, keys that are not described by the schema, and schemas without a recognizable scalar `type` are returned unchanged. The @@ -120,7 +121,11 @@ def _coerce_scalar(value: Any, declared_types: set[str]) -> Any: as_bool = _to_bool(value) if as_bool is not None: return as_bool - if "string" in declared_types and isinstance(value, (int, float)): + if ( + "string" in declared_types + and isinstance(value, (int, float)) + and not isinstance(value, bool) + ): return str(value) return value diff --git a/tests/unit/models/shared/test_tool_input_coercion.py b/tests/unit/models/shared/test_tool_input_coercion.py index c8eda189..3839b2c3 100644 --- a/tests/unit/models/shared/test_tool_input_coercion.py +++ b/tests/unit/models/shared/test_tool_input_coercion.py @@ -78,6 +78,7 @@ def test_none_is_left_untouched(self) -> None: ("False", False), ("1", True), (0, False), + ("yes", "yes"), ("maybe", "maybe"), (2, 2), ], @@ -97,6 +98,12 @@ def test_numbers_are_coerced_to_string(self) -> None: schema = {"type": "object", "properties": {"text": {"type": "string"}}} assert coerce_tool_input({"text": 42}, schema) == {"text": "42"} + assert coerce_tool_input({"text": 1.5}, schema) == {"text": "1.5"} + + def test_booleans_are_not_coerced_to_string(self) -> None: + schema = {"type": "object", "properties": {"text": {"type": "string"}}} + + assert coerce_tool_input({"text": True}, schema) == {"text": True} def test_union_types_are_respected(self) -> None: schema = { @@ -180,3 +187,22 @@ def test_string_integers_reach_the_tool_as_integers(self) -> None: assert isinstance(result, ToolResultBlockParam) assert result.is_error is None or result.is_error is False assert "Tapped at (726, 122) 2x" in str(result.content) + + def test_non_coercible_value_yields_error_result(self) -> None: + tool = _TapLikeTool() + collection = ToolCollection(tools=[tool]) + tool_use = ToolUseBlockParam( + id="tool_use_1", + input={"x": 726, "y": 122, "repeat": 1, "repeat_delay_in_ms": "abc"}, + name=tool.name, + ) + + results = collection.run([tool_use]) + + assert len(results) == 1 + result = results[0] + assert isinstance(result, ToolResultBlockParam) + assert result.is_error is True + assert "not supported between instances of 'str' and 'int'" in str( + result.content + )