Require LLM tool names to be prefixed with the offering integration's domain (#179938)

Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Paulus Schoutsen
2026-08-24 07:09:04 +02:00
committed by GitHub
co-authored by Claude
parent 24d099832a
commit 207c4f049b
39 changed files with 252 additions and 137 deletions
+3 -3
View File
@@ -933,9 +933,9 @@ class AnthropicBaseLLMEntity(CoordinatorEntity[AnthropicCoordinator]):
options: dict[str, Any] = DEFAULT | self.subentry.data
preloaded_tools = [
"HassTurnOn",
"HassTurnOff",
"GetLiveContext",
"intent__HassTurnOn",
"intent__HassTurnOff",
"homeassistant__GetLiveContext",
"code_execution",
"web_search",
"web_fetch",
@@ -5,6 +5,8 @@ from homeassistant.core import HomeAssistant, callback
from homeassistant.helpers import intent
from homeassistant.helpers.llm import LLM_API_ASSIST, IntentTool, LLMContext, Tool
from .const import DOMAIN
@callback
def async_get_tools(
@@ -17,7 +19,7 @@ def async_get_tools(
# assist_satellite registers the broadcast intent when it is set up, and
# this platform is only queried once that has happened.
tools: list[Tool] = [
IntentTool(handler.intent_type, handler)
IntentTool(f"{DOMAIN}__{handler.intent_type}", handler)
for handler in intent.async_get(hass)
if handler.intent_type == intent.INTENT_BROADCAST
]
+1 -1
View File
@@ -21,7 +21,7 @@ from .const import DOMAIN
class CalendarGetEventsTool(Tool):
"""LLM Tool allowing querying a calendar."""
name = "calendar_get_events"
name = "calendar__get_events"
description = (
"Get events from a calendar. "
"When asked if something happens, search the whole week. "
+1 -1
View File
@@ -30,7 +30,7 @@ def async_get_tools(
return None
tools: list[Tool] = [
IntentTool(handler.intent_type, handler)
IntentTool(f"{DOMAIN}__{handler.intent_type}", handler)
for handler in intent.async_get(hass)
if handler.intent_type in LLM_INTENTS
]
+1 -1
View File
@@ -31,7 +31,7 @@ def async_get_tools(
return None
tools: list[Tool] = [
IntentTool(handler.intent_type, handler)
IntentTool(f"{DOMAIN}__{handler.intent_type}", handler)
for handler in intent.async_get(hass)
if handler.intent_type in LLM_INTENTS
]
@@ -36,7 +36,7 @@ NO_ENTITIES_PROMPT = (
DYNAMIC_CONTEXT_PROMPT = (
"You ARE equipped to answer questions about the"
" current state of\n"
"the home using the `GetLiveContext` tool."
"the home using the `homeassistant__GetLiveContext` tool."
" This is a primary function."
" Do not state you lack the\n"
"functionality if the question requires live data.\n"
@@ -50,7 +50,7 @@ DYNAMIC_CONTEXT_PROMPT = (
' "What mode is the thermostat in?",'
' "What is the temperature outside?"):\n'
" 1. Recognize this requires live data.\n"
" 2. You MUST call `GetLiveContext`."
" 2. You MUST call `homeassistant__GetLiveContext`."
" This tool will provide the needed real-time"
" information (like temperature from the local"
" weather, lock status, etc.).\n"
@@ -198,7 +198,7 @@ class GetLiveContextTool(Tool):
returns state for entities based on intent parameters.
"""
name = "GetLiveContext"
name = "homeassistant__GetLiveContext"
description = (
"Provides real-time information about the"
" CURRENT state, value, or mode of devices,"
+1 -1
View File
@@ -28,7 +28,7 @@ def async_get_tools(
return None
tools: list[Tool] = [
IntentTool(handler.intent_type, handler)
IntentTool(f"{DOMAIN}__{handler.intent_type}", handler)
for handler in intent.async_get(hass)
if handler.intent_type in LLM_INTENTS
]
+3 -2
View File
@@ -16,6 +16,7 @@ from homeassistant.helpers import (
)
from homeassistant.helpers.llm import LLM_API_ASSIST, IntentTool, LLMContext, Tool
from .const import DOMAIN
from .timers import async_device_supports_timers
# Generic intents exposed as LLM tools regardless of a timer-capable device.
@@ -40,7 +41,7 @@ TIMER_INTENTS = (
DEVICE_CONTROL_TOOL_USAGE_PROMPT = (
"When controlling Home Assistant always call the intent tools. "
"Use HassTurnOn to lock and HassTurnOff to unlock a lock. "
"Use intent__HassTurnOn to lock and intent__HassTurnOff to unlock a lock. "
"When controlling a device, prefer passing just name and domain. "
"When controlling an area, prefer passing just area name and domain."
)
@@ -75,7 +76,7 @@ def async_get_tools(
]
tools: list[Tool] = [
IntentTool(handler.intent_type, handler) for handler in handlers
IntentTool(f"{DOMAIN}__{handler.intent_type}", handler) for handler in handlers
]
if not tools:
return None
@@ -8,7 +8,7 @@ from homeassistant.core import HomeAssistant, callback
from homeassistant.helpers import intent
from homeassistant.helpers.llm import LLM_API_ASSIST, IntentTool, LLMContext, Tool
from . import ScriptIntentHandler
from . import DOMAIN, ScriptIntentHandler
@callback
@@ -43,7 +43,8 @@ def async_get_tools(
# valid tool names.
tools: list[Tool] = [
IntentTool(
unicode_slug.slugify(handler.intent_type, separator="_", lowercase=False),
f"{DOMAIN}__"
+ unicode_slug.slugify(handler.intent_type, separator="_", lowercase=False),
handler,
)
for handler in handlers
+1 -1
View File
@@ -31,7 +31,7 @@ def async_get_tools(
return None
tools: list[Tool] = [
IntentTool(handler.intent_type, handler)
IntentTool(f"{DOMAIN}__{handler.intent_type}", handler)
for handler in intent.async_get(hass)
if handler.intent_type in LLM_INTENTS
]
+1 -1
View File
@@ -31,7 +31,7 @@ def async_get_tools(
return None
tools: list[Tool] = [
IntentTool(handler.intent_type, handler)
IntentTool(f"{DOMAIN}__{handler.intent_type}", handler)
for handler in intent.async_get(hass)
if handler.intent_type in LLM_INTENTS
]
+30
View File
@@ -6,6 +6,7 @@ from typing import Protocol, override
from homeassistant.core import HomeAssistant, callback
from homeassistant.helpers import config_validation as cv
from homeassistant.helpers.frame import ReportBehavior, report_usage
from homeassistant.helpers.integration_platform import LazyIntegrationPlatforms
from homeassistant.helpers.llm import (
API,
@@ -17,6 +18,7 @@ from homeassistant.helpers.llm import (
selector_serializer,
)
from homeassistant.helpers.typing import ConfigType
from homeassistant.loader import async_get_issue_integration
from homeassistant.util.hass_dict import HassKey
from .const import DOMAIN
@@ -26,6 +28,8 @@ _LOGGER = logging.getLogger(__name__)
CONFIG_SCHEMA = cv.empty_config_schema(DOMAIN)
TOOL_PREFIX_BREAKS_IN_HA_VERSION = "2027.3"
DATA_PLATFORMS: HassKey[LazyIntegrationPlatforms[LLMToolsPlatformProtocol]] = HassKey(
"llm_platforms"
)
@@ -87,12 +91,38 @@ async def async_get_tools(
continue
if result is None:
continue
_async_report_unprefixed_tools(hass, domain, result.tools)
tools.extend(result.tools)
if result.prompt:
prompts.append(result.prompt)
return LLMTools(tools=tools, prompt="\n".join(prompts) if prompts else None)
@callback
def _async_report_unprefixed_tools(
hass: HomeAssistant, domain: str, tools: list[Tool]
) -> None:
"""Report tools that are not prefixed with the domain offering them."""
prefix = f"{domain}__"
unprefixed = [tool.name for tool in tools if not tool.name.startswith(prefix)]
if not unprefixed:
return
integration = async_get_issue_integration(hass, domain)
report_usage(
f"provides LLM tools that are not prefixed with '{prefix}': "
f"{', '.join(sorted(unprefixed))}",
breaks_in_ha_version=TOOL_PREFIX_BREAKS_IN_HA_VERSION,
core_behavior=ReportBehavior.LOG,
core_integration_behavior=ReportBehavior.LOG,
custom_integration_behavior=ReportBehavior.LOG,
integration_domain=domain,
level=logging.WARNING
if integration and not integration.is_built_in
else logging.ERROR,
)
class AssistAPI(API):
"""API exposing Assist API to LLMs."""
+1 -1
View File
@@ -13,7 +13,7 @@ from . import LLMTools
class GetDateTimeTool(Tool):
"""Tool for getting the current date and time."""
name = "GetDateTime"
name = "llm__GetDateTime"
description = "Provides the current date and time."
@override
@@ -30,7 +30,7 @@ _LOGGER = logging.getLogger(__name__)
SNAPSHOT_RESOURCE_URI = "homeassistant://assist/context-snapshot"
SNAPSHOT_RESOURCE_URL = AnyUrl(SNAPSHOT_RESOURCE_URI)
SNAPSHOT_RESOURCE_MIME_TYPE = "text/plain"
LIVE_CONTEXT_TOOL_NAME = "GetLiveContext"
LIVE_CONTEXT_TOOL_NAME = "homeassistant__GetLiveContext"
def _has_live_context_tool(llm_api: llm.APIInstance) -> bool:
@@ -115,7 +115,7 @@ async def create_server(
title="Assist context snapshot",
description=(
"A snapshot of the current Assist context, matching the"
" existing GetLiveContext tool output."
" existing homeassistant__GetLiveContext tool output."
),
mimeType=SNAPSHOT_RESOURCE_MIME_TYPE,
)
+1 -1
View File
@@ -51,7 +51,7 @@ def async_get_tools(
return None
tools: list[Tool] = [
IntentTool(handler.intent_type, handler)
IntentTool(f"{DOMAIN}__{handler.intent_type}", handler)
for handler in intent.async_get(hass)
if handler.intent_type in LLM_INTENTS
]
+1 -3
View File
@@ -30,9 +30,7 @@ class ScriptTool(ActionTool):
super().__init__(hass, DOMAIN, action)
self.name = script_name
if self.name[0].isdigit():
self.name = "_" + self.name
self.name = f"{DOMAIN}__{script_name}"
if entity_entry and (
aliases := er.async_get_entity_aliases(hass, entity_entry)
+2 -2
View File
@@ -32,7 +32,7 @@ LLM_INTENTS = (INTENT_LIST_ADD_ITEM, INTENT_LIST_COMPLETE_ITEM, INTENT_LIST_REMO
class TodoGetItemsTool(Tool):
"""LLM Tool allowing querying a to-do list."""
name = "todo_get_items"
name = "todo__get_items"
description = (
"Query a to-do list to find out what items are on it. "
"Use this to answer questions like "
@@ -115,7 +115,7 @@ def async_get_tools(
tools: list[Tool] = [TodoGetItemsTool(names)]
tools.extend(
IntentTool(handler.intent_type, handler)
IntentTool(f"{DOMAIN}__{handler.intent_type}", handler)
for handler in intent.async_get(hass)
if handler.intent_type in LLM_INTENTS
)
+1 -1
View File
@@ -39,7 +39,7 @@ def async_get_tools(
return None
tools: list[Tool] = [
IntentTool(handler.intent_type, handler)
IntentTool(f"{DOMAIN}__{handler.intent_type}", handler)
for handler in intent.async_get(hass)
if handler.intent_type in LLM_INTENTS
]
+4 -2
View File
@@ -230,8 +230,10 @@ class IntentTool(Tool):
) -> None:
"""Init the class."""
self.name = name
self.intent_type = intent_handler.intent_type
self.description = (
intent_handler.description or f"Execute Home Assistant {self.name} intent"
intent_handler.description
or f"Execute Home Assistant {self.intent_type} intent"
)
self.extra_slots = None
if not (slot_schema := intent_handler.slot_schema):
@@ -281,7 +283,7 @@ class IntentTool(Tool):
intent_response = await intent.async_handle(
hass=hass,
platform=llm_context.platform,
intent_type=self.name,
intent_type=self.intent_type,
slots=slots,
text_input=None,
context=llm_context.context,
@@ -1785,9 +1785,9 @@ async def test_tool_search(
} in tools
for tool in tools:
if tool["name"] in (
"HassTurnOn",
"HassTurnOff",
"GetLiveContext",
"intent__HassTurnOn",
"intent__HassTurnOff",
"homeassistant__GetLiveContext",
"tool_search_tool_bm25",
):
assert "defer_loading" not in tool
@@ -31,4 +31,4 @@ def _llm_context() -> llm.LLMContext:
async def test_broadcast_tool_offered(hass: HomeAssistant) -> None:
"""Test the broadcast intent is exposed as an LLM tool."""
result = await llm_component.async_get_tools(hass, _llm_context(), "assist")
assert "HassBroadcast" in [tool.name for tool in result.tools]
assert "assist_satellite__HassBroadcast" in [tool.name for tool in result.tools]
+6 -6
View File
@@ -44,7 +44,7 @@ async def test_get_tools_no_exposed_calendar(hass: HomeAssistant) -> None:
"""Test no calendar tool is offered when no calendar is exposed."""
async_expose_entity(hass, "conversation", ENTITY_ID, False)
result = await llm_component.async_get_tools(hass, _llm_context(), "assist")
assert "calendar_get_events" not in [tool.name for tool in result.tools]
assert "calendar__get_events" not in [tool.name for tool in result.tools]
assert calendar_llm.async_get_tools(hass, _llm_context(), "assist") is None
@@ -58,7 +58,7 @@ async def test_calendar_get_events_tool(hass: HomeAssistant) -> None:
llm_context = _llm_context()
result = await llm_component.async_get_tools(hass, llm_context, "assist")
tool = next(
(tool for tool in result.tools if tool.name == "calendar_get_events"), None
(tool for tool in result.tools if tool.name == "calendar__get_events"), None
)
assert tool is not None
assert tool.parameters.schema["calendar"].container == ["Mock Calendar Name"]
@@ -90,7 +90,7 @@ async def test_calendar_get_events_tool(hass: HomeAssistant) -> None:
)
tool_input = llm.ToolInput(
tool_name="calendar_get_events",
tool_name="calendar__get_events",
tool_args={"calendar": "Mock Calendar Name", "range": "today"},
)
now = dt_util.now()
@@ -141,7 +141,7 @@ async def test_calendar_get_events_tool_not_found(hass: HomeAssistant) -> None:
"""Test the tool reports when the requested calendar no longer matches."""
llm_context = _llm_context()
result = await llm_component.async_get_tools(hass, llm_context, "assist")
tool = next(tool for tool in result.tools if tool.name == "calendar_get_events")
tool = next(tool for tool in result.tools if tool.name == "calendar__get_events")
# Unexpose after the tool (and its calendar enum) was built, so the call-time
# match no longer finds the calendar.
@@ -149,7 +149,7 @@ async def test_calendar_get_events_tool_not_found(hass: HomeAssistant) -> None:
response = await tool.async_call(
hass,
llm.ToolInput(
"calendar_get_events", {"calendar": "Mock Calendar Name", "range": "today"}
"calendar__get_events", {"calendar": "Mock Calendar Name", "range": "today"}
),
llm_context,
)
@@ -168,5 +168,5 @@ async def test_calendar_get_events_tool_uses_aliases(
async_expose_entity(hass, "conversation", entry.entity_id, True)
result = await llm_component.async_get_tools(hass, _llm_context(), "assist")
tool = next(tool for tool in result.tools if tool.name == "calendar_get_events")
tool = next(tool for tool in result.tools if tool.name == "calendar__get_events")
assert "Family Calendar" in tool.parameters.schema["calendar"].container
+2 -2
View File
@@ -43,13 +43,13 @@ async def _tool_names(hass: HomeAssistant) -> set[str]:
async def test_intent_tool_exposed(hass: HomeAssistant) -> None:
"""Test the intent tool is offered for an exposed climate entity."""
assert "HassClimateSetTemperature" in await _tool_names(hass)
assert "climate__HassClimateSetTemperature" in await _tool_names(hass)
async def test_intent_tool_not_exposed(hass: HomeAssistant) -> None:
"""Test the intent tool is hidden when no climate entity is exposed."""
async_expose_entity(hass, "conversation", ENTITY_ID, False)
assert "HassClimateSetTemperature" not in await _tool_names(hass)
assert "climate__HassClimateSetTemperature" not in await _tool_names(hass)
assert climate_llm.async_get_tools(hass, _llm_context(), "assist") is None
+2 -2
View File
@@ -43,13 +43,13 @@ async def _tool_names(hass: HomeAssistant) -> set[str]:
async def test_intent_tool_exposed(hass: HomeAssistant) -> None:
"""Test the intent tool is offered for an exposed fan entity."""
assert "HassFanSetSpeed" in await _tool_names(hass)
assert "fan__HassFanSetSpeed" in await _tool_names(hass)
async def test_intent_tool_not_exposed(hass: HomeAssistant) -> None:
"""Test the intent tool is hidden when no fan entity is exposed."""
async_expose_entity(hass, "conversation", ENTITY_ID, False)
assert "HassFanSetSpeed" not in await _tool_names(hass)
assert "fan__HassFanSetSpeed" not in await _tool_names(hass)
assert fan_llm.async_get_tools(hass, _llm_context(), "assist") is None
+18 -13
View File
@@ -44,10 +44,10 @@ def _llm_context() -> llm.LLMContext:
async def test_live_context_always_offered(hass: HomeAssistant) -> None:
"""Test GetLiveContext is offered even when nothing is exposed."""
"""Test homeassistant__GetLiveContext is offered even when nothing is exposed."""
async_expose_entity(hass, "conversation", ENTITY_ID, False)
result = await llm_component.async_get_tools(hass, _llm_context(), "assist")
assert "GetLiveContext" in [tool.name for tool in result.tools]
assert "homeassistant__GetLiveContext" in [tool.name for tool in result.tools]
async def test_no_tools_for_other_api(hass: HomeAssistant) -> None:
@@ -72,27 +72,32 @@ async def test_prompt_no_entities(hass: HomeAssistant) -> None:
async def test_get_live_context_no_exposed_entities(hass: HomeAssistant) -> None:
"""Test GetLiveContext reports an error when nothing is exposed."""
"""Test homeassistant__GetLiveContext reports an error when nothing is exposed."""
async_expose_entity(hass, "conversation", ENTITY_ID, False)
llm_context = _llm_context()
result = await llm_component.async_get_tools(hass, llm_context, "assist")
tool = next(tool for tool in result.tools if tool.name == "GetLiveContext")
tool = next(
tool for tool in result.tools if tool.name == "homeassistant__GetLiveContext"
)
response = await tool.async_call(
hass, llm.ToolInput("GetLiveContext", {}), llm_context
hass, llm.ToolInput("homeassistant__GetLiveContext", {}), llm_context
)
assert response == {"success": False, "error": ha_llm.NO_ENTITIES_PROMPT}
async def test_get_live_context_tool(hass: HomeAssistant) -> None:
"""Test GetLiveContext returns exposed entity state."""
"""Test homeassistant__GetLiveContext returns exposed entity state."""
llm_context = _llm_context()
result = await llm_component.async_get_tools(hass, llm_context, "assist")
tool = next((tool for tool in result.tools if tool.name == "GetLiveContext"), None)
tool = next(
(tool for tool in result.tools if tool.name == "homeassistant__GetLiveContext"),
None,
)
assert tool is not None
response = await tool.async_call(
hass, llm.ToolInput("GetLiveContext", {}), llm_context
hass, llm.ToolInput("homeassistant__GetLiveContext", {}), llm_context
)
assert response["success"] is True
assert "Kitchen Light" in response["result"]
@@ -158,7 +163,7 @@ async def test_get_live_context_tool_filter(
entity_registry: er.EntityRegistry,
area_registry: ar.AreaRegistry,
) -> None:
"""Test the filter parameters of the GetLiveContext tool."""
"""Test the filter parameters of the homeassistant__GetLiveContext tool."""
# The autouse fixture exposes light.kitchen; drop it for a clean entity set.
async_expose_entity(hass, "conversation", ENTITY_ID, False)
assert await async_setup_component(hass, "intent", {})
@@ -254,11 +259,11 @@ async def test_get_live_context_tool_filter(
await hass.async_block_till_done()
tools = await llm_component.async_get_tools(hass, llm_context, "assist")
tool = next(t for t in tools.tools if t.name == "GetLiveContext")
tool = next(t for t in tools.tools if t.name == "homeassistant__GetLiveContext")
async def _get_live_context(tool_args: dict) -> dict:
return await tool.async_call(
hass, llm.ToolInput("GetLiveContext", tool_args), llm_context
hass, llm.ToolInput("homeassistant__GetLiveContext", tool_args), llm_context
)
# Filter by area and domain (example 1)
@@ -400,9 +405,9 @@ async def test_get_live_context_tool_filter(
async def test_get_live_context_schema(
hass: HomeAssistant, snapshot: SnapshotAssertion
) -> None:
"""Test that GetLiveContext tool parameters convert to a sane OpenAPI schema."""
"""Test that homeassistant__GetLiveContext tool parameters convert to a sane OpenAPI schema."""
result = await llm_component.async_get_tools(hass, _llm_context(), "assist")
tool = next(t for t in result.tools if t.name == "GetLiveContext")
tool = next(t for t in result.tools if t.name == "homeassistant__GetLiveContext")
api = await llm.async_get_api(hass, "assist", _llm_context())
schema = convert(tool.parameters, custom_serializer=api.custom_serializer)
+3 -3
View File
@@ -10,7 +10,7 @@ from homeassistant.helpers import llm
from homeassistant.setup import async_setup_component
ENTITY_ID = "humidifier.test"
INTENTS = {"HassHumidifierMode", "HassHumidifierSetpoint"}
TOOL_NAMES = {"humidifier__HassHumidifierMode", "humidifier__HassHumidifierSetpoint"}
@pytest.fixture(autouse=True)
@@ -44,13 +44,13 @@ async def _tool_names(hass: HomeAssistant) -> set[str]:
async def test_intent_tool_exposed(hass: HomeAssistant) -> None:
"""Test the intent tool is offered for an exposed humidifier entity."""
assert await _tool_names(hass) >= INTENTS
assert await _tool_names(hass) >= TOOL_NAMES
async def test_intent_tool_not_exposed(hass: HomeAssistant) -> None:
"""Test the intent tool is hidden when no humidifier entity is exposed."""
async_expose_entity(hass, "conversation", ENTITY_ID, False)
assert not INTENTS & await _tool_names(hass)
assert not TOOL_NAMES & await _tool_names(hass)
assert humidifier_llm.async_get_tools(hass, _llm_context(), "assist") is None
+8 -8
View File
@@ -44,13 +44,13 @@ async def _tool_names(hass: HomeAssistant) -> set[str]:
async def test_generic_intents_exposed(hass: HomeAssistant) -> None:
"""Test the always-on generic intents are exposed."""
names = await _tool_names(hass)
assert "HassTurnOn" in names
assert "HassTurnOff" in names
assert "intent__HassTurnOn" in names
assert "intent__HassTurnOff" in names
async def test_timer_intents_require_timer_device(hass: HomeAssistant) -> None:
"""Test timer intents are not exposed without a timer-capable device."""
assert "HassStartTimer" not in await _tool_names(hass)
assert "intent__HassStartTimer" not in await _tool_names(hass)
async def test_timer_intents_offered_for_timer_device(hass: HomeAssistant) -> None:
@@ -66,16 +66,16 @@ async def test_timer_intents_offered_for_timer_device(hass: HomeAssistant) -> No
hass, _llm_context(device_id="test_device"), "assist"
)
names = {tool.name for tool in result.tools}
assert "HassStartTimer" in names
assert "HassTimerStatus" in names
assert "intent__HassStartTimer" in names
assert "intent__HassTimerStatus" in names
async def test_set_position_requires_exposed_cover(hass: HomeAssistant) -> None:
"""Test HassSetPosition is only exposed when a cover/valve is exposed."""
assert "HassSetPosition" in await _tool_names(hass)
"""Test intent__HassSetPosition is only exposed when a cover/valve is exposed."""
assert "intent__HassSetPosition" in await _tool_names(hass)
async_expose_entity(hass, "conversation", COVER_ENTITY_ID, False)
assert "HassSetPosition" not in await _tool_names(hass)
assert "intent__HassSetPosition" not in await _tool_names(hass)
async def test_prompt_includes_device_control(hass: HomeAssistant) -> None:
+4 -4
View File
@@ -66,17 +66,17 @@ async def test_intent_scripts_exposed(hass: HomeAssistant) -> None:
"""Test intent scripts are exposed as LLM tools with slugified names."""
names = await _tool_names(hass)
# The user-provided "Tell a joke" name is slugified into a valid tool name.
assert "Tell_a_joke" in names
assert "LightAction" in names
assert "intent_script__Tell_a_joke" in names
assert "intent_script__LightAction" in names
async def test_intent_script_platform_filtered(hass: HomeAssistant) -> None:
"""Test a platform-restricted intent script requires an exposed entity."""
async_expose_entity(hass, "conversation", LIGHT_ENTITY_ID, False)
names = await _tool_names(hass)
assert "LightAction" not in names
assert "intent_script__LightAction" not in names
# Unrestricted intent scripts stay exposed.
assert "Tell_a_joke" in names
assert "intent_script__Tell_a_joke" in names
async def test_no_tools_for_other_api(hass: HomeAssistant) -> None:
+3 -3
View File
@@ -10,7 +10,7 @@ from homeassistant.helpers import llm
from homeassistant.setup import async_setup_component
ENTITY_ID = "lawn_mower.test"
INTENTS = {"HassLawnMowerDock", "HassLawnMowerStartMowing"}
TOOL_NAMES = {"lawn_mower__HassLawnMowerDock", "lawn_mower__HassLawnMowerStartMowing"}
@pytest.fixture(autouse=True)
@@ -44,13 +44,13 @@ async def _tool_names(hass: HomeAssistant) -> set[str]:
async def test_intent_tool_exposed(hass: HomeAssistant) -> None:
"""Test the intent tool is offered for an exposed lawn_mower entity."""
assert await _tool_names(hass) >= INTENTS
assert await _tool_names(hass) >= TOOL_NAMES
async def test_intent_tool_not_exposed(hass: HomeAssistant) -> None:
"""Test the intent tool is hidden when no lawn_mower entity is exposed."""
async_expose_entity(hass, "conversation", ENTITY_ID, False)
assert not INTENTS & await _tool_names(hass)
assert not TOOL_NAMES & await _tool_names(hass)
assert lawn_mower_llm.async_get_tools(hass, _llm_context(), "assist") is None
+2 -2
View File
@@ -43,13 +43,13 @@ async def _tool_names(hass: HomeAssistant) -> set[str]:
async def test_intent_tool_exposed(hass: HomeAssistant) -> None:
"""Test the intent tool is offered for an exposed light entity."""
assert "HassLightSet" in await _tool_names(hass)
assert "light__HassLightSet" in await _tool_names(hass)
async def test_intent_tool_not_exposed(hass: HomeAssistant) -> None:
"""Test the intent tool is hidden when no light entity is exposed."""
async_expose_entity(hass, "conversation", ENTITY_ID, False)
assert "HassLightSet" not in await _tool_names(hass)
assert "light__HassLightSet" not in await _tool_names(hass)
assert light_llm.async_get_tools(hass, _llm_context(), "assist") is None
+75 -10
View File
@@ -1,12 +1,13 @@
"""Tests for the LLM integration."""
from unittest.mock import Mock
import logging
from unittest.mock import Mock, patch
import pytest
from homeassistant.components.llm import DATA_PLATFORMS, LLMTools, async_get_tools
from homeassistant.core import HomeAssistant
from homeassistant.helpers import llm
from homeassistant.helpers import frame, llm
from homeassistant.setup import async_setup_component
from homeassistant.util.json import JsonObjectType
@@ -44,7 +45,10 @@ def llm_context() -> llm.LLMContext:
def _mock_tools_platform(
hass: HomeAssistant, domain: str, tools: LLMTools | Exception | None
hass: HomeAssistant,
domain: str,
tools: LLMTools | Exception | None,
built_in: bool = True,
) -> Mock:
"""Register a mock <integration>/llm.py platform returning the given tools."""
if isinstance(tools, Exception):
@@ -52,7 +56,9 @@ def _mock_tools_platform(
else:
async_get_tools = Mock(return_value=tools)
hass.config.components.add(domain)
mock_platform(hass, f"{domain}.llm", Mock(async_get_tools=async_get_tools))
mock_platform(
hass, f"{domain}.llm", Mock(async_get_tools=async_get_tools), built_in=built_in
)
return async_get_tools
@@ -72,8 +78,8 @@ async def test_get_tools(hass: HomeAssistant, llm_context: llm.LLMContext) -> No
assert await async_setup_component(hass, "llm", {})
result = await async_get_tools(hass, llm_context, "assist")
# The llm integration also exposes its own GetDateTime tool (domain "llm").
assert [tool.name for tool in result.tools] == ["GetDateTime", "my_tool"]
# The llm integration also exposes its own llm__GetDateTime tool (domain "llm").
assert [tool.name for tool in result.tools] == ["llm__GetDateTime", "my_tool"]
assert result.prompt == "use my_tool wisely"
platform_get_tools.assert_called_once_with(hass, llm_context, "assist")
@@ -85,7 +91,7 @@ async def test_get_tools_empty(
assert await async_setup_component(hass, "llm", {})
result = await async_get_tools(hass, llm_context, "assist")
assert [tool.name for tool in result.tools] == ["GetDateTime"]
assert [tool.name for tool in result.tools] == ["llm__GetDateTime"]
assert result.prompt is None
@@ -102,7 +108,11 @@ async def test_get_tools_merges_sorted(
assert await async_setup_component(hass, "llm", {})
result = await async_get_tools(hass, llm_context, "assist")
assert [tool.name for tool in result.tools] == ["GetDateTime", "tool_a", "tool_b"]
assert [tool.name for tool in result.tools] == [
"llm__GetDateTime",
"tool_a",
"tool_b",
]
assert result.prompt == "prompt a\nprompt b"
@@ -117,7 +127,7 @@ async def test_get_tools_skips_none_platform(
assert await async_setup_component(hass, "llm", {})
result = await async_get_tools(hass, llm_context, "assist")
assert [tool.name for tool in result.tools] == ["GetDateTime", "good_tool"]
assert [tool.name for tool in result.tools] == ["llm__GetDateTime", "good_tool"]
assert result.prompt is None
@@ -134,6 +144,61 @@ async def test_get_tools_isolates_failing_platform(
assert await async_setup_component(hass, "llm", {})
result = await async_get_tools(hass, llm_context, "assist")
assert [tool.name for tool in result.tools] == ["GetDateTime", "good_tool"]
assert [tool.name for tool in result.tools] == ["llm__GetDateTime", "good_tool"]
assert result.prompt == "prompt"
assert "Error getting tools from LLM platform test_bad" in caplog.text
@pytest.mark.parametrize(
("built_in", "expected_level", "expected_type"),
[(True, logging.ERROR, ""), (False, logging.WARNING, "custom ")],
ids=["core", "custom"],
)
async def test_get_tools_reports_unprefixed_tool_names(
hass: HomeAssistant,
llm_context: llm.LLMContext,
caplog: pytest.LogCaptureFixture,
built_in: bool,
expected_level: int,
expected_type: str,
) -> None:
"""Test tools not prefixed with the offering domain are reported."""
tools = [_StubTool("test__prefixed"), _StubTool("unprefixed")]
_mock_tools_platform(hass, "test", LLMTools(tools=tools), built_in=built_in)
assert await async_setup_component(hass, "llm", {})
with patch.object(frame, "_REPORTED_INTEGRATIONS", set()):
result = await async_get_tools(hass, llm_context, "assist")
# The tools are still returned until the requirement starts to fail.
assert [tool.name for tool in result.tools] == [
"llm__GetDateTime",
"test__prefixed",
"unprefixed",
]
expected_message = (
f"Detected that {expected_type}integration 'test' provides LLM tools that are "
"not prefixed with 'test__': unprefixed. This will stop working in Home "
"Assistant 2027.3"
)
record = next(
record for record in caplog.records if expected_message in record.getMessage()
)
assert record.levelno == expected_level
async def test_get_tools_prefixed_tool_names_not_reported(
hass: HomeAssistant,
llm_context: llm.LLMContext,
caplog: pytest.LogCaptureFixture,
) -> None:
"""Test a platform prefixing all its tools is not reported."""
_mock_tools_platform(hass, "test", LLMTools(tools=[_StubTool("test__tool")]))
assert await async_setup_component(hass, "llm", {})
with patch.object(frame, "_REPORTED_INTEGRATIONS", set()):
await async_get_tools(hass, llm_context, "assist")
assert "not prefixed with 'test__'" not in caplog.text
+5 -3
View File
@@ -30,15 +30,17 @@ def _llm_context() -> llm.LLMContext:
async def test_get_datetime_tool(hass: HomeAssistant) -> None:
"""Test the GetDateTime tool is always offered and returns the current time."""
"""Test the llm__GetDateTime tool is always offered and returns the current time."""
llm_context = _llm_context()
result = await llm_component.async_get_tools(hass, llm_context, "assist")
tool = next((tool for tool in result.tools if tool.name == "GetDateTime"), None)
tool = next(
(tool for tool in result.tools if tool.name == "llm__GetDateTime"), None
)
assert tool is not None
with freeze_time("2025-09-17 13:00:00"):
response = await tool.async_call(
hass, llm.ToolInput("GetDateTime", {}), llm_context
hass, llm.ToolInput("llm__GetDateTime", {}), llm_context
)
assert response == {
+6 -4
View File
@@ -385,8 +385,10 @@ async def test_mcp_tools_list(
# Pick a single arbitrary tool and test that description and parameters
# are converted correctly.
tool = next(iter(tool for tool in result.tools if tool.name == "HassTurnOn"))
assert tool.name == "HassTurnOn"
tool = next(
iter(tool for tool in result.tools if tool.name == "intent__HassTurnOn")
)
assert tool.name == "intent__HassTurnOn"
assert tool.description is not None
assert tool.inputSchema
assert tool.inputSchema.get("type") == "object"
@@ -410,7 +412,7 @@ async def test_mcp_tool_call(
async with mcp_client(hass, mcp_url, hass_supervisor_access_token) as session:
result = await session.call_tool(
name="HassTurnOn",
name="intent__HassTurnOn",
arguments={"name": "kitchen light"},
)
@@ -439,7 +441,7 @@ async def test_mcp_tool_call_failed(
async with mcp_client(hass, mcp_url, hass_supervisor_access_token) as session:
result = await session.call_tool(
name="HassTurnOn",
name="intent__HassTurnOn",
arguments={"name": "backyard"},
)
+12 -12
View File
@@ -10,16 +10,16 @@ from homeassistant.helpers import llm
from homeassistant.setup import async_setup_component
ENTITY_ID = "media_player.test"
INTENTS = {
"HassMediaNext",
"HassMediaPause",
"HassMediaPlayerMute",
"HassMediaPlayerUnmute",
"HassMediaPrevious",
"HassMediaSearchAndPlay",
"HassMediaUnpause",
"HassSetVolume",
"HassSetVolumeRelative",
TOOL_NAMES = {
"media_player__HassMediaNext",
"media_player__HassMediaPause",
"media_player__HassMediaPlayerMute",
"media_player__HassMediaPlayerUnmute",
"media_player__HassMediaPrevious",
"media_player__HassMediaSearchAndPlay",
"media_player__HassMediaUnpause",
"media_player__HassSetVolume",
"media_player__HassSetVolumeRelative",
}
@@ -54,13 +54,13 @@ async def _tool_names(hass: HomeAssistant) -> set[str]:
async def test_intent_tool_exposed(hass: HomeAssistant) -> None:
"""Test the intent tool is offered for an exposed media_player entity."""
assert await _tool_names(hass) >= INTENTS
assert await _tool_names(hass) >= TOOL_NAMES
async def test_intent_tool_not_exposed(hass: HomeAssistant) -> None:
"""Test the intent tool is hidden when no media_player entity is exposed."""
async_expose_entity(hass, "conversation", ENTITY_ID, False)
assert not INTENTS & await _tool_names(hass)
assert not TOOL_NAMES & await _tool_names(hass)
assert media_player_llm.async_get_tools(hass, _llm_context(), "assist") is None
@@ -145,9 +145,9 @@ async def test_web_search_with_assist(
assert {"type": "openrouter:web_search", "parameters": {"engine": "auto"}} in call[
"extra_body"
]["tools"]
# Ensure GetDateTime is in the tools list
# Ensure llm__GetDateTime is in the tools list
assert any(
tool.get("function", {}).get("name") == "GetDateTime"
tool.get("function", {}).get("name") == "llm__GetDateTime"
for tool in call["extra_body"]["tools"]
)
+6 -6
View File
@@ -58,7 +58,7 @@ async def test_script_tool_only_exposed(hass: HomeAssistant) -> None:
"""Test only exposed scripts get a tool."""
result = await llm_component.async_get_tools(hass, _llm_context(), "assist")
names = [tool.name for tool in result.tools]
assert "test_script" in names
assert "script__test_script" in names
assert "unexposed_script" not in names
@@ -66,7 +66,7 @@ async def test_script_tool_not_exposed(hass: HomeAssistant) -> None:
"""Test no script tool is offered when the script is not exposed."""
async_expose_entity(hass, "conversation", ENTITY_ID, False)
result = await llm_component.async_get_tools(hass, _llm_context(), "assist")
assert "test_script" not in [tool.name for tool in result.tools]
assert "script__test_script" not in [tool.name for tool in result.tools]
assert script_llm.async_get_tools(hass, _llm_context(), "assist") is None
@@ -79,10 +79,10 @@ async def test_script_tool_call(hass: HomeAssistant) -> None:
"""Test calling the exposed script through its tool."""
llm_context = _llm_context()
result = await llm_component.async_get_tools(hass, llm_context, "assist")
tool = next(tool for tool in result.tools if tool.name == "test_script")
tool = next(tool for tool in result.tools if tool.name == "script__test_script")
response = await tool.async_call(
hass, llm.ToolInput("test_script", {"beer": 1}), llm_context
hass, llm.ToolInput("script__test_script", {"beer": 1}), llm_context
)
assert response == {"success": True, "result": {"drinks": 2}}
@@ -91,7 +91,7 @@ async def test_script_tool_name_not_started_with_digit(hass: HomeAssistant) -> N
"""Test a script whose id starts with a digit gets a valid tool name."""
async_expose_entity(hass, "conversation", "script.123456", True)
result = await llm_component.async_get_tools(hass, _llm_context(), "assist")
assert "_123456" in [tool.name for tool in result.tools]
assert "script__123456" in [tool.name for tool in result.tools]
async def test_script_tool_description_includes_aliases(
@@ -100,7 +100,7 @@ async def test_script_tool_description_includes_aliases(
"""Test the script tool description is extended with the entity aliases."""
entity_registry.async_update_entity(ENTITY_ID, aliases=["barkeep", "pour a drink"])
result = await llm_component.async_get_tools(hass, _llm_context(), "assist")
tool = next(tool for tool in result.tools if tool.name == "test_script")
tool = next(tool for tool in result.tools if tool.name == "script__test_script")
assert tool.description == (
"This is a test script. Aliases: ['barkeep', 'pour a drink']"
)
+8 -8
View File
@@ -41,7 +41,7 @@ async def test_get_tools_no_exposed_todo(hass: HomeAssistant) -> None:
"""Test no todo tool is offered when no to-do list is exposed."""
async_expose_entity(hass, "conversation", ENTITY_ID, False)
result = await llm_component.async_get_tools(hass, _llm_context(), "assist")
assert "todo_get_items" not in [tool.name for tool in result.tools]
assert "todo__get_items" not in [tool.name for tool in result.tools]
assert todo_llm.async_get_tools(hass, _llm_context(), "assist") is None
@@ -54,7 +54,7 @@ async def test_todo_get_items_tool(hass: HomeAssistant) -> None:
"""Test the todo get items tool is exposed and works via the platform."""
llm_context = _llm_context()
result = await llm_component.async_get_tools(hass, llm_context, "assist")
tool = next((tool for tool in result.tools if tool.name == "todo_get_items"), None)
tool = next((tool for tool in result.tools if tool.name == "todo__get_items"), None)
assert tool is not None
assert tool.parameters.schema["todo_list"].container == ["Mock Todo List Name"]
@@ -74,7 +74,7 @@ async def test_todo_get_items_tool(hass: HomeAssistant) -> None:
result = await tool.async_call(
hass,
llm.ToolInput("todo_get_items", {"todo_list": "Mock Todo List Name"}),
llm.ToolInput("todo__get_items", {"todo_list": "Mock Todo List Name"}),
llm_context,
)
@@ -101,7 +101,7 @@ async def test_todo_get_items_status_filter(
"""Test the status filter is translated into the service call."""
llm_context = _llm_context()
result = await llm_component.async_get_tools(hass, llm_context, "assist")
tool = next(tool for tool in result.tools if tool.name == "todo_get_items")
tool = next(tool for tool in result.tools if tool.name == "todo__get_items")
calls = async_mock_service(
hass,
@@ -113,7 +113,7 @@ async def test_todo_get_items_status_filter(
await tool.async_call(
hass,
llm.ToolInput(
"todo_get_items", {"todo_list": "Mock Todo List Name", "status": status}
"todo__get_items", {"todo_list": "Mock Todo List Name", "status": status}
),
llm_context,
)
@@ -124,6 +124,6 @@ async def test_todo_list_intents_exposed(hass: HomeAssistant) -> None:
"""Test the todo list intents are exposed as tools when a list is exposed."""
result = await llm_component.async_get_tools(hass, _llm_context(), "assist")
names = {tool.name for tool in result.tools}
assert "HassListAddItem" in names
assert "HassListCompleteItem" in names
assert "HassListRemoveItem" in names
assert "todo__HassListAddItem" in names
assert "todo__HassListCompleteItem" in names
assert "todo__HassListRemoveItem" in names
+7 -3
View File
@@ -10,7 +10,11 @@ from homeassistant.helpers import llm
from homeassistant.setup import async_setup_component
ENTITY_ID = "vacuum.test"
INTENTS = {"HassVacuumCleanArea", "HassVacuumReturnToBase", "HassVacuumStart"}
TOOL_NAMES = {
"vacuum__HassVacuumCleanArea",
"vacuum__HassVacuumReturnToBase",
"vacuum__HassVacuumStart",
}
@pytest.fixture(autouse=True)
@@ -44,13 +48,13 @@ async def _tool_names(hass: HomeAssistant) -> set[str]:
async def test_intent_tool_exposed(hass: HomeAssistant) -> None:
"""Test the intent tool is offered for an exposed vacuum entity."""
assert await _tool_names(hass) >= INTENTS
assert await _tool_names(hass) >= TOOL_NAMES
async def test_intent_tool_not_exposed(hass: HomeAssistant) -> None:
"""Test the intent tool is hidden when no vacuum entity is exposed."""
async_expose_entity(hass, "conversation", ENTITY_ID, False)
assert not INTENTS & await _tool_names(hass)
assert not TOOL_NAMES & await _tool_names(hass)
assert vacuum_llm.async_get_tools(hass, _llm_context(), "assist") is None
+16 -13
View File
@@ -316,14 +316,14 @@ async def test_assist_api_get_timer_tools(
assert await async_setup_component(hass, "intent", {})
api = await llm.async_get_api(hass, "assist", llm_context)
assert "HassStartTimer" not in [tool.name for tool in api.tools]
assert "intent__HassStartTimer" not in [tool.name for tool in api.tools]
llm_context.device_id = "test_device"
async_register_timer_handler(hass, "test_device", lambda *args: None)
api = await llm.async_get_api(hass, "assist", llm_context)
assert "HassStartTimer" in [tool.name for tool in api.tools]
assert "intent__HassStartTimer" in [tool.name for tool in api.tools]
async def test_assist_api_description(
@@ -670,7 +670,7 @@ Static Context: An overview of the areas and the devices in this smart home:
"""
first_part_prompt = (
"When controlling Home Assistant always call the intent tools. "
"Use HassTurnOn to lock and HassTurnOff to unlock a lock. "
"Use intent__HassTurnOn to lock and intent__HassTurnOff to unlock a lock. "
"When controlling a device, prefer passing just name and domain. "
"When controlling an area, prefer passing just area name and domain."
)
@@ -683,7 +683,7 @@ Static Context: An overview of the areas and the devices in this smart home:
dynamic_context_prompt = (
"You ARE equipped to answer questions about the"
" current state of\nthe home using the"
" `GetLiveContext` tool. This is a primary"
" `homeassistant__GetLiveContext` tool. This is a primary"
" function. Do not state you lack the\n"
"functionality if the question requires live"
" data.\nIf the user asks about device"
@@ -695,7 +695,7 @@ Static Context: An overview of the areas and the devices in this smart home:
' "What mode is the thermostat in?", "What'
' is the temperature outside?"):\n'
" 1. Recognize this requires live data.\n"
" 2. You MUST call `GetLiveContext`. This"
" 2. You MUST call `homeassistant__GetLiveContext`. This"
" tool will provide the needed real-time"
" information (like temperature from the local"
" weather, lock status, etc.).\n"
@@ -715,10 +715,10 @@ Static Context: An overview of the areas and the devices in this smart home:
{no_timer_prompt}"""
)
# Verify that the GetLiveContext tool returns the same results
# Verify that the homeassistant__GetLiveContext tool returns the same results
# as the exposed_entities_prompt
result = await api.async_call_tool(
llm.ToolInput(tool_name="GetLiveContext", tool_args={})
llm.ToolInput(tool_name="homeassistant__GetLiveContext", tool_args={})
)
assert result == {
"success": True,
@@ -845,7 +845,7 @@ async def test_action_tool(
assert len(tools) == 2
tool = tools[0]
assert tool.name == "test_script"
assert tool.name == "script__test_script"
assert (
tool.description
== "This is a test script. Aliases: ['script alias', 'script name']"
@@ -869,7 +869,7 @@ async def test_action_tool(
# Test script with response
tool_input = llm.ToolInput(
tool_name="test_script",
tool_name="script__test_script",
tool_args={
"beer": "3",
"wine": 0,
@@ -908,7 +908,7 @@ async def test_action_tool(
# Test script with no response
tool_input = llm.ToolInput(
tool_name="script_with_no_fields",
tool_name="script__script_with_no_fields",
tool_args={},
)
@@ -964,7 +964,7 @@ async def test_action_tool(
assert len(tools) == 2
tool = tools[0]
assert tool.name == "test_script"
assert tool.name == "script__test_script"
assert (
tool.description
== "This is a new test script. Aliases: ['script alias', 'script name']"
@@ -1269,8 +1269,11 @@ async def test_no_tools_exposed(hass: HomeAssistant) -> None:
device_id=None,
)
api = await llm.async_get_api(hass, "assist", llm_context)
# GetLiveContext is always offered; it reports when nothing is exposed.
assert [tool.name for tool in api.tools] == ["GetLiveContext", "GetDateTime"]
# homeassistant__GetLiveContext is always offered; it reports when nothing is exposed.
assert [tool.name for tool in api.tools] == [
"homeassistant__GetLiveContext",
"llm__GetDateTime",
]
async def test_merged_api(hass: HomeAssistant, llm_context: llm.LLMContext) -> None: