diff --git a/homeassistant/components/llm/__init__.py b/homeassistant/components/llm/__init__.py index e64fd7a02a68..a12240f40750 100644 --- a/homeassistant/components/llm/__init__.py +++ b/homeassistant/components/llm/__init__.py @@ -15,6 +15,7 @@ from homeassistant.helpers.llm import ( LLMContext, Tool, async_register_api, + report_untagged_tool, selector_serializer, ) from homeassistant.helpers.typing import ConfigType @@ -91,7 +92,7 @@ async def async_get_tools( continue if result is None: continue - _async_report_unprefixed_tools(hass, domain, result.tools) + _async_report_tool_issues(hass, domain, result.tools) tools.extend(result.tools) if result.prompt: prompts.append(result.prompt) @@ -99,12 +100,18 @@ async def async_get_tools( @callback -def _async_report_unprefixed_tools( +def _async_report_tool_issues( hass: HomeAssistant, domain: str, tools: list[Tool] ) -> None: - """Report tools that are not prefixed with the domain offering them.""" + """Report tools that do not follow the current requirements.""" prefix = f"{domain}__" - unprefixed = [tool.name for tool in tools if not tool.name.startswith(prefix)] + unprefixed: list[str] = [] + for tool in tools: + if not tool.name.startswith(prefix): + unprefixed.append(tool.name) + if tool.integration is None: + report_untagged_tool(tool, domain) + if not unprefixed: return diff --git a/homeassistant/helpers/llm.py b/homeassistant/helpers/llm.py index 128c1eee3fc0..b4659dc5b2ec 100644 --- a/homeassistant/helpers/llm.py +++ b/homeassistant/helpers/llm.py @@ -42,6 +42,8 @@ APIS_CACHE: HassKey[dict[str, API]] = HassKey("llm_apis") LLM_API_ASSIST = "assist" +TOOL_INTEGRATION_BREAKS_IN_HA_VERSION = "2027.10" + DATE_TIME_PROMPT = ( 'Current time is {{ now().strftime("%H:%M:%S") }}. ' 'Today\'s date is {{ now().strftime("%Y-%m-%d") }}.\n' @@ -210,6 +212,19 @@ class APIInstance: tools: list[Tool] custom_serializer: Callable[[Any], Any] | None = None + def __post_init__(self) -> None: + """Report a tool that does not record the integration providing it.""" + for tool in self.tools: + if tool.integration is not None: + continue + # A tool class outside an integration, such as a shared helper tool, + # belongs to whichever integration provides the API. + domain = _tool_integration_domain(tool) or _integration_domain( + type(self.api).__module__ + ) + if domain is not None: + report_untagged_tool(tool, domain) + async def async_call_tool(self, tool_input: ToolInput) -> ToolResult: """Call a LLM tool, validate args and return the response.""" from homeassistant.components.conversation import ( # noqa: PLC0415 @@ -244,11 +259,35 @@ class APIInstance: return ToolResult(data=result) +@callback +def report_untagged_tool(tool: Tool, domain: str) -> None: + """Report a tool that does not record the integration providing it.""" + wrapped = [tool] + while isinstance(wrapped[-1], NamespacedTool): + wrapped.append(wrapped[-1].tool) + frame.report_usage( + f"provides the LLM tool {wrapped[-1].name} without an integration", + breaks_in_ha_version=TOOL_INTEGRATION_BREAKS_IN_HA_VERSION, + core_behavior=frame.ReportBehavior.ERROR, + core_integration_behavior=frame.ReportBehavior.ERROR, + custom_integration_behavior=frame.ReportBehavior.LOG, + integration_domain=domain, + ) + # Record the domain on the tool and every wrapper around it, so it carries + # the integration until the requirement is enforced. + for entry in wrapped: + entry.integration = domain + + def _tool_integration_domain(tool: Tool) -> str | None: """Return the domain of the integration that provides the tool.""" while isinstance(tool, NamespacedTool): tool = tool.tool - module = type(tool).__module__ + return _integration_domain(type(tool).__module__) + + +def _integration_domain(module: str) -> str | None: + """Return the domain of the integration that defines the module.""" for prefix in ("custom_components.", "homeassistant.components."): if module.startswith(prefix): return module.removeprefix(prefix).partition(".")[0] diff --git a/tests/components/conversation/test_chat_log.py b/tests/components/conversation/test_chat_log.py index 72b505daf6dc..669479b4fa28 100644 --- a/tests/components/conversation/test_chat_log.py +++ b/tests/components/conversation/test_chat_log.py @@ -141,6 +141,7 @@ async def test_multiple_llm_apis( name = "test_tool" description = "Test function" + integration = "test" parameters = probatio.Schema( {probatio.Optional("param1", description="Test parameters"): str} ) diff --git a/tests/components/llm/test_init.py b/tests/components/llm/test_init.py index 94be815f2952..49a63c05c62a 100644 --- a/tests/components/llm/test_init.py +++ b/tests/components/llm/test_init.py @@ -17,10 +17,11 @@ from tests.common import mock_platform class _StubTool(llm.Tool): """Minimal tool for registry tests.""" - def __init__(self, name: str) -> None: + def __init__(self, name: str, integration: str | None = "test") -> None: """Initialize the stub tool.""" self.name = name self.description = f"{name} description" + self.integration = integration async def async_call( self, @@ -188,6 +189,71 @@ async def test_get_tools_reports_unprefixed_tool_names( assert record.levelno == expected_level +async def test_get_tools_untagged_tool_raises_for_core( + hass: HomeAssistant, + llm_context: llm.LLMContext, +) -> None: + """Test a core integration must record the integration on its tools.""" + tools = [_StubTool("test__untagged", integration=None)] + _mock_tools_platform(hass, "test", LLMTools(tools=tools)) + + assert await async_setup_component(hass, "llm", {}) + + with ( + patch.object(frame, "_REPORTED_INTEGRATIONS", set()), + pytest.raises( + RuntimeError, + match="provides the LLM tool test__untagged without an integration", + ), + ): + await async_get_tools(hass, llm_context, "assist") + + +async def test_get_tools_untagged_tool_reported_for_custom( + hass: HomeAssistant, + llm_context: llm.LLMContext, + caplog: pytest.LogCaptureFixture, +) -> None: + """Test a custom integration is warned about tools without an integration.""" + tools = [_StubTool("test__untagged", integration=None)] + _mock_tools_platform(hass, "test", LLMTools(tools=tools), built_in=False) + + assert await async_setup_component(hass, "llm", {}) + + with patch.object(frame, "_REPORTED_INTEGRATIONS", set()): + result = await async_get_tools(hass, llm_context, "assist") + + # The tool is still returned until the requirement starts to fail. + assert "test__untagged" in [tool.name for tool in result.tools] + assert ( + "Detected that custom integration 'test' provides the LLM tool test__untagged " + "without an integration. This will stop working in Home Assistant 2027.10" + in caplog.text + ) + # The platform domain is recorded on the tool, so it is not reported again. + assert tools[0].integration == "test" + + +async def test_get_tools_untagged_wrapped_tool_reported_once( + hass: HomeAssistant, + llm_context: llm.LLMContext, + caplog: pytest.LogCaptureFixture, +) -> None: + """Test a tool wrapper is tagged along with the tool it wraps.""" + wrapper = llm.NamespacedTool("test", _StubTool("untagged", integration=None)) + _mock_tools_platform(hass, "test", LLMTools(tools=[wrapper]), built_in=False) + + assert await async_setup_component(hass, "llm", {}) + + with patch.object(frame, "_REPORTED_INTEGRATIONS", set()): + await async_get_tools(hass, llm_context, "assist") + + # The wrapper is tagged too, so the API instance does not report it again. + assert wrapper.integration == "test" + assert wrapper.tool.integration == "test" + assert caplog.text.count("without an integration") == 1 + + async def test_get_tools_prefixed_tool_names_not_reported( hass: HomeAssistant, llm_context: llm.LLMContext, diff --git a/tests/components/mcp_server/test_http.py b/tests/components/mcp_server/test_http.py index d343dc6ebcc7..d831f3eae374 100644 --- a/tests/components/mcp_server/test_http.py +++ b/tests/components/mcp_server/test_http.py @@ -86,6 +86,7 @@ class _StubTool(llm.Tool): """Minimal tool with a configurable parameter schema.""" name = "test_tool" + integration = "test" def __init__(self, parameters: probatio.Schema) -> None: """Initialize the stub tool.""" diff --git a/tests/helpers/test_llm.py b/tests/helpers/test_llm.py index 9be07fe4322f..81db738b6b74 100644 --- a/tests/helpers/test_llm.py +++ b/tests/helpers/test_llm.py @@ -230,6 +230,100 @@ async def test_call_tool_deprecated_json_object_custom_integration( assert "returns a JSON object from a tool" in caplog.text +def _untagged_tool(module: str) -> llm.Tool: + """Return a tool that does not record the integration providing it.""" + + class UntaggedTool(llm.Tool): + """Tool that declares no integration.""" + + name = "test_tool" + + async def async_call( + self, hass: HomeAssistant, tool_input: llm.ToolInput, _: llm.LLMContext + ) -> llm.ToolResult: + return llm.ToolResult(data={}) + + # The tool is reported against the integration its class comes from. + UntaggedTool.__module__ = module + return UntaggedTool() + + +async def test_api_instance_reports_untagged_tool_for_custom_integration( + hass: HomeAssistant, + llm_context: llm.LLMContext, + caplog: pytest.LogCaptureFixture, +) -> None: + """Test a custom integration is warned about a tool without an integration.""" + mock_integration(hass, MockModule("my_custom"), built_in=False) + tool = _untagged_tool("custom_components.my_custom.llm") + + llm.APIInstance(MyAPI(hass=hass, id="test", name="Test"), "", llm_context, [tool]) + + assert "provides the LLM tool test_tool without an integration" in caplog.text + + +async def test_api_instance_raises_untagged_tool_for_core_integration( + hass: HomeAssistant, + llm_context: llm.LLMContext, +) -> None: + """Test a core integration must record the integration on its tools.""" + mock_integration(hass, MockModule("my_core")) + tool = _untagged_tool("homeassistant.components.my_core.llm") + + with pytest.raises( + RuntimeError, match="provides the LLM tool test_tool without an integration" + ): + llm.APIInstance( + MyAPI(hass=hass, id="test", name="Test"), "", llm_context, [tool] + ) + + +async def test_api_instance_reports_untagged_tool_from_the_api( + hass: HomeAssistant, + llm_context: llm.LLMContext, + caplog: pytest.LogCaptureFixture, +) -> None: + """Test a tool defined outside an integration is reported against its API.""" + mock_integration(hass, MockModule("my_custom"), built_in=False) + tool = _untagged_tool("homeassistant.helpers.llm") + + class CustomAPI(MyAPI): + """API provided by a custom integration.""" + + CustomAPI.__module__ = "custom_components.my_custom.llm_api" + llm.APIInstance( + CustomAPI(hass=hass, id="test", name="Test"), "", llm_context, [tool] + ) + + assert ( + "custom integration 'my_custom' provides the LLM tool test_tool without an " + "integration" in caplog.text + ) + # The tool carries the domain until the requirement is enforced. + assert tool.integration == "my_custom" + + +async def test_merged_api_reports_untagged_tool_once( + hass: HomeAssistant, + llm_context: llm.LLMContext, + caplog: pytest.LogCaptureFixture, +) -> None: + """Test a merged API reports the wrapped tool under its own name.""" + mock_integration(hass, MockModule("my_custom"), built_in=False) + + api = MyAPI(hass=hass, id="api-1", name="API 1") + api.tools = [_untagged_tool("custom_components.my_custom.llm")] + llm.async_register_api(hass, api) + other = MyAPI(hass=hass, id="api-2", name="API 2") + llm.async_register_api(hass, other) + + await llm.async_get_api(hass, ["api-1", "api-2"], llm_context) + + assert "provides the LLM tool test_tool without an integration" in caplog.text + # The wrapper reports the tool it wraps, so the report is not repeated. + assert caplog.text.count("without an integration") == 1 + + def test_tool_metadata_defaults() -> None: """Test a tool that declares no metadata is taken to be unsafe.""" @@ -313,7 +407,7 @@ async def test_intent_tool_omits_blank_arguments( probatio.Optional("enabled"): cv.boolean, } - intent_tool = llm.IntentTool("test_intent", MyIntentHandler()) + intent_tool = llm.IntentTool("test_intent", MyIntentHandler(), integration="test") tool: llm.Tool = ( llm.NamespacedTool("test_api", intent_tool) if namespaced else intent_tool ) @@ -379,7 +473,7 @@ async def test_assist_api( intent_handler = MyIntentHandler() - tool = llm.IntentTool("test_intent", intent_handler) + tool = llm.IntentTool("test_intent", intent_handler, integration="test") assert tool.name == "test_intent" assert tool.description == "Execute Home Assistant test_intent intent" assert tool.parameters == probatio.Schema( @@ -541,7 +635,7 @@ async def test_assist_api_description( intent_type = "test_intent" description = "my intent handler" - tool = llm.IntentTool("test_intent", MyIntentHandler()) + tool = llm.IntentTool("test_intent", MyIntentHandler(), integration="test") assert tool.name == "test_intent" assert tool.description == "my intent handler" @@ -1483,6 +1577,7 @@ async def test_merged_api(hass: HomeAssistant, llm_context: llm.LLMContext) -> N def __init__(self, name: str, description: str) -> None: self.name = name self.description = description + self.integration = "test" async def async_call( self, hass: HomeAssistant, tool_input: llm.ToolInput, _: llm.LLMContext