diff --git a/homeassistant/components/conversation/chat_log.py b/homeassistant/components/conversation/chat_log.py index 8927ce6310b3..7cf36eed470e 100644 --- a/homeassistant/components/conversation/chat_log.py +++ b/homeassistant/components/conversation/chat_log.py @@ -20,7 +20,7 @@ from homeassistant.util.hass_dict import HassKey from homeassistant.util.json import JsonObjectType from . import trace -from .const import ChatLogEventType +from .const import DOMAIN, ChatLogEventType from .models import ConversationInput, ConversationResult DATA_CHAT_LOGS: HassKey[dict[str, ChatLog]] = HassKey("conversation_chat_logs") @@ -298,7 +298,20 @@ class ToolResultContent: @property def tool_result(self) -> JsonObjectType: - """Return the data of the result.""" + """Return the data of the result. + + Deprecated compatibility shim: the result is available as `result`. + It can be removed in HA Core 2027.11. + """ + frame.report_usage( + "accesses `ToolResultContent.tool_result`, which is deprecated; " + "use `ToolResultContent.result` instead", + breaks_in_ha_version="2027.11.0", + core_behavior=frame.ReportBehavior.ERROR, + core_integration_behavior=frame.ReportBehavior.ERROR, + custom_integration_behavior=frame.ReportBehavior.LOG, + exclude_integrations={DOMAIN}, + ) return self.result.data def as_dict(self) -> dict[str, Any]: @@ -309,6 +322,7 @@ class ToolResultContent: "tool_call_id": self.tool_call_id, "tool_name": self.tool_name, "result": asdict(self.result), + # Deprecated, can be removed in HA Core 2027.11. "tool_result": self.result.data, "created": self.created, } @@ -594,6 +608,15 @@ class ChatLog: self.delta_listener(self, filtered_delta) elif delta["role"] == "tool_result": if (result := delta.get("result")) is None: + frame.report_usage( + "sets `tool_result` on a tool result delta, which is " + "deprecated; set `result` to a ToolResult instead", + breaks_in_ha_version="2027.11.0", + core_behavior=frame.ReportBehavior.ERROR, + core_integration_behavior=frame.ReportBehavior.ERROR, + custom_integration_behavior=frame.ReportBehavior.LOG, + exclude_integrations={DOMAIN}, + ) result = llm.ToolResult(data=delta["tool_result"]) content = ToolResultContent( agent_id=agent_id, diff --git a/homeassistant/helpers/llm.py b/homeassistant/helpers/llm.py index ed96216781ce..77f9405ac74e 100644 --- a/homeassistant/helpers/llm.py +++ b/homeassistant/helpers/llm.py @@ -25,6 +25,7 @@ from . import ( config_validation as cv, device_registry as dr, floor_registry as fr, + frame, intent, selector, service, @@ -213,9 +214,30 @@ class APIInstance: result = await tool.async_call(self.api.hass, tool_input, self.llm_context) if isinstance(result, ToolResult): return result + frame.report_usage( + "returns a JSON object from a tool, which is deprecated; return a " + "ToolResult instead", + breaks_in_ha_version="2027.11.0", + core_behavior=frame.ReportBehavior.ERROR, + core_integration_behavior=frame.ReportBehavior.ERROR, + custom_integration_behavior=frame.ReportBehavior.LOG, + # The tool call has returned, so its frame is gone from the stack. + integration_domain=_tool_integration_domain(tool), + ) return ToolResult(data=result) +def _tool_integration_domain(tool: Tool) -> str | None: + """Return the domain of the integration that provides the tool.""" + while isinstance(tool, NamespacedTool): + tool = tool.tool + module = type(tool).__module__ + for prefix in ("custom_components.", "homeassistant.components."): + if module.startswith(prefix): + return module.removeprefix(prefix).partition(".")[0] + return None + + @dataclass(slots=True, kw_only=True) class API(ABC): """An API to expose to LLMs.""" diff --git a/tests/components/anthropic/test_conversation.py b/tests/components/anthropic/test_conversation.py index b1b409ab052a..78dd9828b4b9 100644 --- a/tests/components/anthropic/test_conversation.py +++ b/tests/components/anthropic/test_conversation.py @@ -433,7 +433,7 @@ async def test_function_call( mock_tool.parameters = probatio.Schema( {probatio.Optional("param1", description="Test parameters"): str} ) - mock_tool.async_call.return_value = "Test response" + mock_tool.async_call.return_value = llm.ToolResult(data="Test response") mock_get_tools.return_value = LLMTools(tools=[mock_tool]) @@ -958,7 +958,7 @@ async def test_extended_thinking_tool_call( mock_tool.parameters = probatio.Schema( {probatio.Optional("param1", description="Test parameters"): str} ) - mock_tool.async_call.return_value = "Test response" + mock_tool.async_call.return_value = llm.ToolResult(data="Test response") mock_get_tools.return_value = LLMTools(tools=[mock_tool]) diff --git a/tests/components/assist_pipeline/test_pipeline.py b/tests/components/assist_pipeline/test_pipeline.py index 99044dd9437f..c24cab61953a 100644 --- a/tests/components/assist_pipeline/test_pipeline.py +++ b/tests/components/assist_pipeline/test_pipeline.py @@ -1869,7 +1869,7 @@ async def test_chat_log_tts_streaming( mock_tool.name = "test_tool" mock_tool.description = "Test function" mock_tool.parameters = probatio.Schema({}) - mock_tool.async_call.return_value = "Test response" + mock_tool.async_call.return_value = llm.ToolResult(data="Test response") with ( patch( diff --git a/tests/components/conversation/test_chat_log.py b/tests/components/conversation/test_chat_log.py index 249388456b71..72b505daf6dc 100644 --- a/tests/components/conversation/test_chat_log.py +++ b/tests/components/conversation/test_chat_log.py @@ -439,7 +439,7 @@ async def test_tool_call( mock_tool.parameters = probatio.Schema( {probatio.Optional("param1", description="Test parameters"): str} ) - mock_tool.async_call.return_value = "Test response" + mock_tool.async_call.return_value = llm.ToolResult(data="Test response") with ( patch( @@ -704,9 +704,9 @@ async def test_add_delta_content_stream( async def tool_call( hass: HomeAssistant, tool_input: llm.ToolInput, llm_context: llm.LLMContext - ) -> str: + ) -> llm.ToolResult: """Call the tool.""" - return tool_input.tool_args["param1"] + return llm.ToolResult(data=tool_input.tool_args["param1"]) mock_tool.async_call.side_effect = tool_call expected_delta = [] @@ -1042,8 +1042,9 @@ async def test_chat_log_subscription( assert len(received_events) == events_before_unsubscribe +@pytest.mark.usefixtures("mock_integration_frame") async def test_tool_result_content_deprecated_property() -> None: - """Test the deprecated tool_result property returns the result data.""" + """Test reading the deprecated tool_result property is reported.""" content = ToolResultContent( agent_id="mock-agent-id", tool_call_id="mock-tool-call-id", @@ -1051,14 +1052,16 @@ async def test_tool_result_content_deprecated_property() -> None: result=llm.ToolResult(data={"answer": 42}), ) - assert content.tool_result == {"answer": 42} + with pytest.raises(RuntimeError, match="ToolResultContent.tool_result"): + _ = content.tool_result +@pytest.mark.usefixtures("mock_integration_frame") async def test_add_delta_content_stream_deprecated_tool_result( hass: HomeAssistant, mock_conversation_input: ConversationInput, ) -> None: - """Test a delta carrying the deprecated tool_result key is still accepted.""" + """Test setting the deprecated tool_result key on a delta is reported.""" async def stream(): """Yield a tool result delta using the deprecated key.""" @@ -1072,12 +1075,11 @@ async def test_add_delta_content_stream_deprecated_tool_result( with ( chat_session.async_get_chat_session(hass) as session, async_get_chat_log(hass, session, mock_conversation_input) as chat_log, + pytest.raises(RuntimeError, match="tool result delta"), ): - results = [ + _ = [ content async for content in chat_log.async_add_delta_content_stream( "mock-agent-id", stream() ) ] - - assert results[0].result == llm.ToolResult(data={"answer": 42}) diff --git a/tests/components/conversation/test_default_agent.py b/tests/components/conversation/test_default_agent.py index 54732b164959..5ff00c24323c 100644 --- a/tests/components/conversation/test_default_agent.py +++ b/tests/components/conversation/test_default_agent.py @@ -3520,7 +3520,7 @@ async def test_intent_tool_call_in_chat_log(hass: HomeAssistant) -> None: # Verify tool result was stored assert tool_result_content is not None assert tool_result_content.tool_name == "HassTurnOn" - assert tool_result_content.tool_result["response_type"] == "action_done" + assert tool_result_content.result.data["response_type"] == "action_done" # Verify final assistant content with speech assert assistant_content is not None @@ -3569,7 +3569,7 @@ async def test_trigger_tool_call_in_chat_log(hass: HomeAssistant) -> None: # Verify tool result was stored assert tool_result_content is not None assert tool_result_content.tool_name == "trigger_sentence" - assert tool_result_content.tool_result["response"] == trigger_response + assert tool_result_content.result.data["response"] == trigger_response @pytest.mark.usefixtures("init_components") diff --git a/tests/components/ollama/test_conversation.py b/tests/components/ollama/test_conversation.py index 6a365d9a17c1..90989e51aed4 100644 --- a/tests/components/ollama/test_conversation.py +++ b/tests/components/ollama/test_conversation.py @@ -314,7 +314,7 @@ async def test_function_call( {probatio.Optional("param1", description="Test parameters"): str}, extra=probatio.ALLOW_EXTRA, ) - mock_tool.async_call.return_value = "Test response" + mock_tool.async_call.return_value = llm.ToolResult(data="Test response") mock_get_tools.return_value = LLMTools(tools=[mock_tool]) diff --git a/tests/helpers/test_llm.py b/tests/helpers/test_llm.py index 1872a294ddb2..fb75012b45f3 100644 --- a/tests/helpers/test_llm.py +++ b/tests/helpers/test_llm.py @@ -25,7 +25,7 @@ from homeassistant.helpers import ( from homeassistant.setup import async_setup_component from homeassistant.util.json import JsonObjectType -from tests.common import MockConfigEntry +from tests.common import MockConfigEntry, MockModule, mock_integration @pytest.fixture(autouse=True) @@ -150,7 +150,9 @@ async def test_call_non_intent_tool_preserves_blank_arguments( tool_args = {"name": "", "response": " ", "other": None} tool = MagicMock(spec=llm.Tool) tool.name = "test_tool" - tool.async_call = AsyncMock(return_value={"tool_args": tool_args}) + tool.async_call = AsyncMock( + return_value=llm.ToolResult(data={"tool_args": tool_args}) + ) instance = llm.APIInstance( MyAPI(hass=hass, id="test", name="Test"), "", llm_context, [tool] ) @@ -161,31 +163,14 @@ async def test_call_non_intent_tool_preserves_blank_arguments( assert tool.async_call.await_args.args[1].tool_args is tool_args -@pytest.mark.parametrize( - ("tool_return_value", "expected"), - [ - pytest.param( - {"answer": 42}, - llm.ToolResult(data={"answer": 42}), - id="plain-json-object", - ), - pytest.param( - llm.ToolResult(data={"answer": 42}, error=True), - llm.ToolResult(data={"answer": 42}, error=True), - id="tool-result", - ), - ], -) async def test_call_tool_result( - hass: HomeAssistant, - llm_context: llm.LLMContext, - tool_return_value: llm.ToolResult | JsonObjectType, - expected: llm.ToolResult, + hass: HomeAssistant, llm_context: llm.LLMContext ) -> None: - """Test a tool result is returned as is and a JSON object is wrapped.""" + """Test a tool result is returned as is.""" + expected = llm.ToolResult(data={"answer": 42}, error=True) tool = MagicMock(spec=llm.Tool) tool.name = "test_tool" - tool.async_call = AsyncMock(return_value=tool_return_value) + tool.async_call = AsyncMock(return_value=expected) instance = llm.APIInstance( MyAPI(hass=hass, id="test", name="Test"), "", llm_context, [tool] ) @@ -193,6 +178,58 @@ async def test_call_tool_result( assert await instance.async_call_tool(llm.ToolInput(tool.name, {})) == expected +@pytest.mark.usefixtures("mock_integration_frame") +async def test_call_tool_deprecated_json_object( + hass: HomeAssistant, llm_context: llm.LLMContext +) -> None: + """Test returning a JSON object from a tool is reported.""" + tool = MagicMock(spec=llm.Tool) + tool.name = "test_tool" + tool.async_call = AsyncMock(return_value={"answer": 42}) + instance = llm.APIInstance( + MyAPI(hass=hass, id="test", name="Test"), "", llm_context, [tool] + ) + + with pytest.raises(RuntimeError, match="returns a JSON object from a tool"): + await instance.async_call_tool(llm.ToolInput(tool.name, {})) + + +async def test_call_tool_deprecated_json_object_custom_integration( + hass: HomeAssistant, + llm_context: llm.LLMContext, + caplog: pytest.LogCaptureFixture, +) -> None: + """Test a custom integration tool returning a JSON object is logged, not raised.""" + mock_integration(hass, MockModule("my_custom"), built_in=False) + + class CustomTool(llm.Tool): + """Tool provided by a custom integration.""" + + name = "test_tool" + + async def async_call( + self, + hass: HomeAssistant, + tool_input: llm.ToolInput, + llm_context: llm.LLMContext, + ) -> JsonObjectType: + """Return a plain JSON object.""" + return {"answer": 42} + + # The tool call has returned by the time it is reported, so the domain is + # taken from the tool rather than the stack. + CustomTool.__module__ = "custom_components.my_custom.llm" + tool = CustomTool() + instance = llm.APIInstance( + MyAPI(hass=hass, id="test", name="Test"), "", llm_context, [tool] + ) + + assert await instance.async_call_tool( + llm.ToolInput(tool.name, {}) + ) == llm.ToolResult(data={"answer": 42}) + assert "returns a JSON object from a tool" in caplog.text + + @pytest.mark.parametrize("namespaced", [False, True]) async def test_intent_tool_omits_blank_arguments( hass: HomeAssistant, llm_context: llm.LLMContext, namespaced: bool @@ -1381,8 +1418,10 @@ async def test_merged_api(hass: HomeAssistant, llm_context: llm.LLMContext) -> N async def async_call( self, hass: HomeAssistant, tool_input: llm.ToolInput, _: llm.LLMContext - ) -> JsonObjectType: - return {"result": {tool_input.tool_name: tool_input.tool_args}} + ) -> llm.ToolResult: + return llm.ToolResult( + data={"result": {tool_input.tool_name: tool_input.tool_args}} + ) api1 = MyAPI(hass=hass, id="api-1", name="API 1") api1.prompt = "This is prompt 1"