Deprecate ToolResultContent.tool_result (#182551)

Co-authored-by: Claude <noreply@anthropic.com>

homeassistant/components/conversation/chat_log.py

@@ -20,7 +20,7 @@ from homeassistant.util.hass_dict import HassKey
 from homeassistant.util.json import JsonObjectType
 
 from . import trace
-from .const import ChatLogEventType
+from .const import DOMAIN, ChatLogEventType
 from .models import ConversationInput, ConversationResult
 
 DATA_CHAT_LOGS: HassKey[dict[str, ChatLog]] = HassKey("conversation_chat_logs")
@@ -298,7 +298,20 @@ class ToolResultContent:
 
     @property
     def tool_result(self) -> JsonObjectType:
-        """Return the data of the result."""
+        """Return the data of the result.
+
+        Deprecated compatibility shim: the result is available as `result`.
+        It can be removed in HA Core 2027.11.
+        """
+        frame.report_usage(
+            "accesses `ToolResultContent.tool_result`, which is deprecated; "
+            "use `ToolResultContent.result` instead",
+            breaks_in_ha_version="2027.11.0",
+            core_behavior=frame.ReportBehavior.ERROR,
+            core_integration_behavior=frame.ReportBehavior.ERROR,
+            custom_integration_behavior=frame.ReportBehavior.LOG,
+            exclude_integrations={DOMAIN},
+        )
         return self.result.data
 
     def as_dict(self) -> dict[str, Any]:
@@ -309,6 +322,7 @@ class ToolResultContent:
             "tool_call_id": self.tool_call_id,
             "tool_name": self.tool_name,
             "result": asdict(self.result),
+            # Deprecated, can be removed in HA Core 2027.11.
             "tool_result": self.result.data,
             "created": self.created,
         }
@@ -594,6 +608,15 @@ class ChatLog:
                         self.delta_listener(self, filtered_delta)
             elif delta["role"] == "tool_result":
                 if (result := delta.get("result")) is None:
+                    frame.report_usage(
+                        "sets `tool_result` on a tool result delta, which is "
+                        "deprecated; set `result` to a ToolResult instead",
+                        breaks_in_ha_version="2027.11.0",
+                        core_behavior=frame.ReportBehavior.ERROR,
+                        core_integration_behavior=frame.ReportBehavior.ERROR,
+                        custom_integration_behavior=frame.ReportBehavior.LOG,
+                        exclude_integrations={DOMAIN},
+                    )
                     result = llm.ToolResult(data=delta["tool_result"])
                 content = ToolResultContent(
                     agent_id=agent_id,

homeassistant/helpers/llm.py

@@ -25,6 +25,7 @@ from . import (
     config_validation as cv,
     device_registry as dr,
     floor_registry as fr,
+    frame,
     intent,
     selector,
     service,
@@ -213,9 +214,30 @@ class APIInstance:
         result = await tool.async_call(self.api.hass, tool_input, self.llm_context)
         if isinstance(result, ToolResult):
             return result
+        frame.report_usage(
+            "returns a JSON object from a tool, which is deprecated; return a "
+            "ToolResult instead",
+            breaks_in_ha_version="2027.11.0",
+            core_behavior=frame.ReportBehavior.ERROR,
+            core_integration_behavior=frame.ReportBehavior.ERROR,
+            custom_integration_behavior=frame.ReportBehavior.LOG,
+            # The tool call has returned, so its frame is gone from the stack.
+            integration_domain=_tool_integration_domain(tool),
+        )
         return ToolResult(data=result)
 
 
+def _tool_integration_domain(tool: Tool) -> str | None:
+    """Return the domain of the integration that provides the tool."""
+    while isinstance(tool, NamespacedTool):
+        tool = tool.tool
+    module = type(tool).__module__
+    for prefix in ("custom_components.", "homeassistant.components."):
+        if module.startswith(prefix):
+            return module.removeprefix(prefix).partition(".")[0]
+    return None
+
+
 @dataclass(slots=True, kw_only=True)
 class API(ABC):
     """An API to expose to LLMs."""

tests/components/anthropic/test_conversation.py

@@ -433,7 +433,7 @@ async def test_function_call(
     mock_tool.parameters = probatio.Schema(
         {probatio.Optional("param1", description="Test parameters"): str}
     )
-    mock_tool.async_call.return_value = "Test response"
+    mock_tool.async_call.return_value = llm.ToolResult(data="Test response")
 
     mock_get_tools.return_value = LLMTools(tools=[mock_tool])
 
@@ -958,7 +958,7 @@ async def test_extended_thinking_tool_call(
     mock_tool.parameters = probatio.Schema(
         {probatio.Optional("param1", description="Test parameters"): str}
     )
-    mock_tool.async_call.return_value = "Test response"
+    mock_tool.async_call.return_value = llm.ToolResult(data="Test response")
 
     mock_get_tools.return_value = LLMTools(tools=[mock_tool])
 

tests/components/assist_pipeline/test_pipeline.py

@@ -1869,7 +1869,7 @@ async def test_chat_log_tts_streaming(
     mock_tool.name = "test_tool"
     mock_tool.description = "Test function"
     mock_tool.parameters = probatio.Schema({})
-    mock_tool.async_call.return_value = "Test response"
+    mock_tool.async_call.return_value = llm.ToolResult(data="Test response")
 
     with (
         patch(

tests/components/conversation/test_chat_log.py

@@ -439,7 +439,7 @@ async def test_tool_call(
     mock_tool.parameters = probatio.Schema(
         {probatio.Optional("param1", description="Test parameters"): str}
     )
-    mock_tool.async_call.return_value = "Test response"
+    mock_tool.async_call.return_value = llm.ToolResult(data="Test response")
 
     with (
         patch(
@@ -704,9 +704,9 @@ async def test_add_delta_content_stream(
 
     async def tool_call(
         hass: HomeAssistant, tool_input: llm.ToolInput, llm_context: llm.LLMContext
-    ) -> str:
+    ) -> llm.ToolResult:
         """Call the tool."""
-        return tool_input.tool_args["param1"]
+        return llm.ToolResult(data=tool_input.tool_args["param1"])
 
     mock_tool.async_call.side_effect = tool_call
     expected_delta = []
@@ -1042,8 +1042,9 @@ async def test_chat_log_subscription(
     assert len(received_events) == events_before_unsubscribe
 
 
+@pytest.mark.usefixtures("mock_integration_frame")
 async def test_tool_result_content_deprecated_property() -> None:
-    """Test the deprecated tool_result property returns the result data."""
+    """Test reading the deprecated tool_result property is reported."""
     content = ToolResultContent(
         agent_id="mock-agent-id",
         tool_call_id="mock-tool-call-id",
@@ -1051,14 +1052,16 @@ async def test_tool_result_content_deprecated_property() -> None:
         result=llm.ToolResult(data={"answer": 42}),
     )
 
-    assert content.tool_result == {"answer": 42}
+    with pytest.raises(RuntimeError, match="ToolResultContent.tool_result"):
+        _ = content.tool_result
 
 
+@pytest.mark.usefixtures("mock_integration_frame")
 async def test_add_delta_content_stream_deprecated_tool_result(
     hass: HomeAssistant,
     mock_conversation_input: ConversationInput,
 ) -> None:
-    """Test a delta carrying the deprecated tool_result key is still accepted."""
+    """Test setting the deprecated tool_result key on a delta is reported."""
 
     async def stream():
         """Yield a tool result delta using the deprecated key."""
@@ -1072,12 +1075,11 @@ async def test_add_delta_content_stream_deprecated_tool_result(
     with (
         chat_session.async_get_chat_session(hass) as session,
         async_get_chat_log(hass, session, mock_conversation_input) as chat_log,
+        pytest.raises(RuntimeError, match="tool result delta"),
     ):
-        results = [
+        _ = [
             content
             async for content in chat_log.async_add_delta_content_stream(
                 "mock-agent-id", stream()
             )
         ]
-
-    assert results[0].result == llm.ToolResult(data={"answer": 42})

tests/components/conversation/test_default_agent.py

@@ -3520,7 +3520,7 @@ async def test_intent_tool_call_in_chat_log(hass: HomeAssistant) -> None:
     # Verify tool result was stored
     assert tool_result_content is not None
     assert tool_result_content.tool_name == "HassTurnOn"
-    assert tool_result_content.tool_result["response_type"] == "action_done"
+    assert tool_result_content.result.data["response_type"] == "action_done"
 
     # Verify final assistant content with speech
     assert assistant_content is not None
@@ -3569,7 +3569,7 @@ async def test_trigger_tool_call_in_chat_log(hass: HomeAssistant) -> None:
     # Verify tool result was stored
     assert tool_result_content is not None
     assert tool_result_content.tool_name == "trigger_sentence"
-    assert tool_result_content.tool_result["response"] == trigger_response
+    assert tool_result_content.result.data["response"] == trigger_response
 
 
 @pytest.mark.usefixtures("init_components")

tests/components/ollama/test_conversation.py

@@ -314,7 +314,7 @@ async def test_function_call(
         {probatio.Optional("param1", description="Test parameters"): str},
         extra=probatio.ALLOW_EXTRA,
     )
-    mock_tool.async_call.return_value = "Test response"
+    mock_tool.async_call.return_value = llm.ToolResult(data="Test response")
 
     mock_get_tools.return_value = LLMTools(tools=[mock_tool])
 

tests/helpers/test_llm.py

@@ -25,7 +25,7 @@ from homeassistant.helpers import (
 from homeassistant.setup import async_setup_component
 from homeassistant.util.json import JsonObjectType
 
-from tests.common import MockConfigEntry
+from tests.common import MockConfigEntry, MockModule, mock_integration
 
 
 @pytest.fixture(autouse=True)
@@ -150,7 +150,9 @@ async def test_call_non_intent_tool_preserves_blank_arguments(
     tool_args = {"name": "", "response": " ", "other": None}
     tool = MagicMock(spec=llm.Tool)
     tool.name = "test_tool"
-    tool.async_call = AsyncMock(return_value={"tool_args": tool_args})
+    tool.async_call = AsyncMock(
+        return_value=llm.ToolResult(data={"tool_args": tool_args})
+    )
     instance = llm.APIInstance(
         MyAPI(hass=hass, id="test", name="Test"), "", llm_context, [tool]
     )
@@ -161,31 +163,14 @@ async def test_call_non_intent_tool_preserves_blank_arguments(
     assert tool.async_call.await_args.args[1].tool_args is tool_args
 
 
-@pytest.mark.parametrize(
-    ("tool_return_value", "expected"),
-    [
-        pytest.param(
-            {"answer": 42},
-            llm.ToolResult(data={"answer": 42}),
-            id="plain-json-object",
-        ),
-        pytest.param(
-            llm.ToolResult(data={"answer": 42}, error=True),
-            llm.ToolResult(data={"answer": 42}, error=True),
-            id="tool-result",
-        ),
-    ],
-)
 async def test_call_tool_result(
-    hass: HomeAssistant,
-    llm_context: llm.LLMContext,
-    tool_return_value: llm.ToolResult | JsonObjectType,
-    expected: llm.ToolResult,
+    hass: HomeAssistant, llm_context: llm.LLMContext
 ) -> None:
-    """Test a tool result is returned as is and a JSON object is wrapped."""
+    """Test a tool result is returned as is."""
+    expected = llm.ToolResult(data={"answer": 42}, error=True)
     tool = MagicMock(spec=llm.Tool)
     tool.name = "test_tool"
-    tool.async_call = AsyncMock(return_value=tool_return_value)
+    tool.async_call = AsyncMock(return_value=expected)
     instance = llm.APIInstance(
         MyAPI(hass=hass, id="test", name="Test"), "", llm_context, [tool]
     )
@@ -193,6 +178,58 @@ async def test_call_tool_result(
     assert await instance.async_call_tool(llm.ToolInput(tool.name, {})) == expected
 
 
+@pytest.mark.usefixtures("mock_integration_frame")
+async def test_call_tool_deprecated_json_object(
+    hass: HomeAssistant, llm_context: llm.LLMContext
+) -> None:
+    """Test returning a JSON object from a tool is reported."""
+    tool = MagicMock(spec=llm.Tool)
+    tool.name = "test_tool"
+    tool.async_call = AsyncMock(return_value={"answer": 42})
+    instance = llm.APIInstance(
+        MyAPI(hass=hass, id="test", name="Test"), "", llm_context, [tool]
+    )
+
+    with pytest.raises(RuntimeError, match="returns a JSON object from a tool"):
+        await instance.async_call_tool(llm.ToolInput(tool.name, {}))
+
+
+async def test_call_tool_deprecated_json_object_custom_integration(
+    hass: HomeAssistant,
+    llm_context: llm.LLMContext,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """Test a custom integration tool returning a JSON object is logged, not raised."""
+    mock_integration(hass, MockModule("my_custom"), built_in=False)
+
+    class CustomTool(llm.Tool):
+        """Tool provided by a custom integration."""
+
+        name = "test_tool"
+
+        async def async_call(
+            self,
+            hass: HomeAssistant,
+            tool_input: llm.ToolInput,
+            llm_context: llm.LLMContext,
+        ) -> JsonObjectType:
+            """Return a plain JSON object."""
+            return {"answer": 42}
+
+    # The tool call has returned by the time it is reported, so the domain is
+    # taken from the tool rather than the stack.
+    CustomTool.__module__ = "custom_components.my_custom.llm"
+    tool = CustomTool()
+    instance = llm.APIInstance(
+        MyAPI(hass=hass, id="test", name="Test"), "", llm_context, [tool]
+    )
+
+    assert await instance.async_call_tool(
+        llm.ToolInput(tool.name, {})
+    ) == llm.ToolResult(data={"answer": 42})
+    assert "returns a JSON object from a tool" in caplog.text
+
+
 @pytest.mark.parametrize("namespaced", [False, True])
 async def test_intent_tool_omits_blank_arguments(
     hass: HomeAssistant, llm_context: llm.LLMContext, namespaced: bool
@@ -1381,8 +1418,10 @@ async def test_merged_api(hass: HomeAssistant, llm_context: llm.LLMContext) -> N
 
         async def async_call(
             self, hass: HomeAssistant, tool_input: llm.ToolInput, _: llm.LLMContext
-        ) -> JsonObjectType:
-            return {"result": {tool_input.tool_name: tool_input.tool_args}}
+        ) -> llm.ToolResult:
+            return llm.ToolResult(
+                data={"result": {tool_input.tool_name: tool_input.tool_args}}
+            )
 
     api1 = MyAPI(hass=hass, id="api-1", name="API 1")
     api1.prompt = "This is prompt 1"