Add gazetter fallback to default conversation agent (#180197)

Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Paulus Schoutsen <balloob@gmail.com>
This commit is contained in:
Michael Hansen
2026-08-26 17:20:29 +02:00
committed by GitHub
co-authored by Claude Opus 5 Copilot Autofix powered by AI Paulus Schoutsen
parent 2ee39eeacb
commit a3fe696713
7 changed files with 1394 additions and 30 deletions
@@ -10,6 +10,7 @@ from pathlib import Path
import time
from typing import IO, Any, cast, override
from gazetteer_matcher import FrameCandidate, GazetteerMatcher
from hassil.expression import Expression, Group, ListReference, TextChunk
from hassil.intents import (
Intents,
@@ -75,6 +76,12 @@ from .const import (
IntentSource,
)
from .entity import ConversationEntity
from .gazetteer import (
GazetteerFallback,
async_refusal,
async_targets_from_intent,
join_speech,
)
from .models import ConversationInput, ConversationResult
from .trace import ConversationTraceEventType, async_conversation_trace_append
@@ -82,8 +89,20 @@ _LOGGER = logging.getLogger(__name__)
_DEFAULT_ERROR_TEXT = "Sorry, I couldn't understand that"
_ENTITY_REGISTRY_UPDATE_FIELDS = ["aliases", "device_id", "name", "original_name"]
_DEVICE_REGISTRY_UPDATE_FIELDS = ["name", "name_by_user"]
# `area_id` is for the gazetteer, which binds each entity to an area
_ENTITY_REGISTRY_UPDATE_FIELDS = [
"aliases",
"area_id",
"device_id",
"name",
"original_name",
]
_DEVICE_REGISTRY_UPDATE_FIELDS = [
"area_id",
"name",
"name_by_user",
"parent_device_id",
]
_DEFAULT_EXPOSED_ATTRIBUTES = {"device_class"}
@@ -245,6 +264,9 @@ class DefaultAgent(ConversationEntity):
# LRU cache to avoid unnecessary intent matching
self._intent_cache = IntentCache(capacity=128)
# Second recognizer, tried only when hassil does not match
self._gazetteer = GazetteerFallback(hass)
@override
async def async_added_to_hass(self) -> None:
"""Subscribe to intents updates when added to hass."""
@@ -471,13 +493,23 @@ class DefaultAgent(ConversationEntity):
)
response.async_set_speech(response_text)
# An automation ran, and nothing in it is a target
self._gazetteer.async_forget(chat_log.conversation_id)
if response is None:
# Match intents
intent_result = await self.async_recognize_intent(user_input)
response = await self._async_process_intent_result(
intent_result, user_input, chat_log
)
if intent_result is None or intent_result.unmatched_entities:
# Nothing is configured behind this agent, so try the gazetteer
response = await self._async_gazetteer_fallback(
intent_result, user_input, chat_log
)
if response is None:
response = await self._async_process_intent_result(
intent_result, user_input, chat_log
)
speech: str = response.speech.get("plain", {}).get("speech", "")
chat_log.async_add_assistant_content_without_tools(
@@ -546,26 +578,56 @@ class DefaultAgent(ConversationEntity):
for entity in result.entities_list
}
satellite_id = user_input.satellite_id
device_id = user_input.device_id
satellite_area, device_id = self._get_satellite_area_and_device(
satellite_id, device_id
intent_response = await self._async_execute_intent(
result.intent.name,
slots,
{
entity_name: entity_value.text or entity_value.value
for entity_name, entity_value in result.entities.items()
},
result.response,
user_input,
chat_log,
lang_intents,
)
if satellite_area is not None:
slots["preferred_area_id"] = {"value": satellite_area.id}
if intent_response.response_type is not intent.IntentResponseType.ERROR:
self._gazetteer.async_remember(
chat_log.conversation_id,
async_targets_from_intent(slots, intent_response),
)
return intent_response
async def _async_execute_intent(
self,
intent_name: str,
slots: dict[str, Any],
speech_slots: dict[str, Any],
response_key: str | None,
user_input: ConversationInput,
chat_log: ChatLog,
lang_intents: LanguageIntents,
) -> intent.IntentResponse:
"""Handle one recognized intent and give it something to say."""
language = user_input.language or self.hass.config.language
satellite_area, device_id = self._get_satellite_area_and_device(
user_input.satellite_id, user_input.device_id
)
tool_args = {name: value["value"] for name, value in slots.items()}
async_conversation_trace_append(
ConversationTraceEventType.TOOL_CALL,
{
"intent_name": result.intent.name,
"slots": {entity.name: entity.value for entity in result.entities_list},
},
{"intent_name": intent_name, "slots": tool_args},
)
tool_input = llm.ToolInput(
tool_name=result.intent.name,
tool_args={entity.name: entity.value for entity in result.entities_list},
tool_name=intent_name,
tool_args=tool_args,
external=True,
)
if satellite_area is not None:
slots = slots | {"preferred_area_id": {"value": satellite_area.id}}
chat_log.async_add_assistant_content_without_tools(
AssistantContent(
agent_id=user_input.agent_id,
@@ -578,14 +640,14 @@ class DefaultAgent(ConversationEntity):
intent_response = await intent.async_handle(
self.hass,
DOMAIN,
result.intent.name,
intent_name,
slots,
user_input.text,
user_input.context,
language,
assistant=DOMAIN,
device_id=device_id,
satellite_id=satellite_id,
satellite_id=user_input.satellite_id,
conversation_agent_id=user_input.agent_id,
)
except intent.MatchFailedError as match_error:
@@ -622,16 +684,16 @@ class DefaultAgent(ConversationEntity):
if (
(not intent_response.speech)
and (intent_response.intent is not None)
and (response_key := result.response)
and response_key
):
# Use response template, if available
response_template_str = lang_intents.intent_responses.get(
result.intent.name, {}
intent_name, {}
).get(response_key)
if response_template_str:
response_template = template.Template(response_template_str, self.hass)
speech = await self._build_speech(
language, response_template, intent_response, result
response_template, intent_response, speech_slots
)
intent_response.async_set_speech(speech)
@@ -647,6 +709,117 @@ class DefaultAgent(ConversationEntity):
return intent_response
async def _async_gazetteer_fallback(
self,
hassil_result: RecognizeResult | None,
user_input: ConversationInput,
chat_log: ChatLog,
) -> intent.IntentResponse | None:
"""Try to recognize a sentence hassil could not, or return None to give up."""
if hassil_result is not None and (hassil_result.intent_metadata or {}).get(
METADATA_CUSTOM_SENTENCE
):
# A sentence somebody wrote themselves, matched to the intent they wrote
# it for, with only the target left unresolved. Its error says so; the
# gazetteer would answer a different intent it happens to recognize.
return None
language = user_input.language or self.hass.config.language
if not self._gazetteer.supports(language):
return None
lang_intents = await self.async_get_or_load_intents(language)
if lang_intents is None:
return None
satellite_area, _ = self._get_satellite_area_and_device(
user_input.satellite_id, user_input.device_id
)
matcher, interpretation = await self._gazetteer.async_interpret(
user_input.text, chat_log.conversation_id, satellite_area
)
if not interpretation.accepted or not interpretation.frames:
_LOGGER.debug(
"Gazetteer rejected '%s': %s (%s)",
user_input.text,
interpretation.rejection_code,
interpretation.reason,
)
if hassil_result is not None:
# hassil got far enough to say something specific about what was wrong.
return None
refusal = async_refusal(interpretation)
if refusal is None:
return None
return _make_error_result(
language,
intent.IntentResponseErrorCode.NO_INTENT_MATCH,
refusal,
)
# Only take over a sentence we can also answer, since acting on one
# mutely is worse than leaving it to hassil's own error
for frame in interpretation.frames:
if frame.response_key is None:
_LOGGER.debug(
"Gazetteer matched '%s' as %s/%s, which has no single response",
user_input.text,
frame.intent,
frame.combination,
)
return None
intent_response = await self._async_process_frames(
matcher, interpretation.frames, user_input, chat_log, lang_intents, language
)
if intent_response.response_type is not intent.IntentResponseType.ERROR:
self._gazetteer.async_remember(
chat_log.conversation_id, interpretation.targets
)
return intent_response
async def _async_process_frames(
self,
matcher: GazetteerMatcher,
frames: list[FrameCandidate],
user_input: ConversationInput,
chat_log: ChatLog,
lang_intents: LanguageIntents,
language: str,
) -> intent.IntentResponse:
"""Handle every frame the matcher recognized, stopping at the first error."""
responses: list[intent.IntentResponse] = []
for frame in frames:
intent_response = await self._async_execute_intent(
frame.intent,
{
slot: {"value": value, "text": matcher.display_name(slot, value)}
for slot, value in frame.slots.items()
},
{
slot: matcher.display_name(slot, value)
for slot, value in frame.slots.items()
},
frame.response_key,
user_input,
chat_log,
lang_intents,
)
if intent_response.response_type is intent.IntentResponseType.ERROR:
return intent_response
responses.append(intent_response)
if len(responses) == 1:
return responses[0]
return _merge_intent_responses(responses, language)
def _recognize(
self,
user_input: ConversationInput,
@@ -942,10 +1115,9 @@ class DefaultAgent(ConversationEntity):
async def _build_speech(
self,
language: str,
response_template: template.Template,
intent_response: intent.IntentResponse,
recognize_result: RecognizeResult,
speech_slots: dict[str, Any],
) -> str:
# Get first matched or unmatched state.
# This is available in the response template as "state".
@@ -956,11 +1128,7 @@ class DefaultAgent(ConversationEntity):
state1 = intent_response.unmatched_states[0]
# Render response template
speech_slots = {
entity_name: entity_value.text or entity_value.value
for entity_name, entity_value in recognize_result.entities.items()
}
speech_slots.update(intent_response.speech_slots)
speech_slots = speech_slots | intent_response.speech_slots
speech = response_template.async_render(
{
@@ -1160,6 +1328,7 @@ class DefaultAgent(ConversationEntity):
"""Clear slot lists when a registry has changed."""
# Two subscribers can be scheduled at same time
_LOGGER.debug("Clearing slot lists")
self._gazetteer.async_invalidate()
if self._unsub_clear_slot_list is None:
return
self._slot_lists = None
@@ -1492,6 +1661,43 @@ class DefaultAgent(ConversationEntity):
return await self.hass.async_add_executor_job(get_language_scores)
def _merge_intent_responses(
responses: list[intent.IntentResponse], language: str
) -> intent.IntentResponse:
"""Combine what each frame of one coordinated command did into one response.
Every frame ran, so every frame's targets belong in the result: a pipeline reads
them to decide whether it acted only within the satellite's own area.
"""
merged = intent.IntentResponse(language=language, intent=responses[0].intent)
merged.response_type = (
intent.IntentResponseType.QUERY_ANSWER
if any(
response.response_type is intent.IntentResponseType.QUERY_ANSWER
for response in responses
)
else responses[0].response_type
)
for response in responses:
merged.success_results.extend(response.success_results)
merged.failed_results.extend(response.failed_results)
merged.matched_states.extend(response.matched_states)
merged.unmatched_states.extend(response.unmatched_states)
merged.speech_slots.update(response.speech_slots)
merged.async_set_speech(
join_speech(
[
speech
for response in responses
if (speech := response.speech.get("plain", {}).get("speech"))
]
)
)
return merged
def _make_error_result(
language: str,
error_code: intent.IntentResponseErrorCode,
@@ -0,0 +1,280 @@
"""Gazetteer intent matching for the default conversation agent.
hassil recognizes a sentence by template, so a phrasing it has no template for does
not match, and neither does one whose entity name was misheard. gazetteer-matcher
reaches the same intents by tagging spans against a gazetteer of the home, which
covers wordings and near-miss names hassil cannot.
It runs behind hassil, and only when the default agent is answering on its own.
With "prefer local intents" set, hassil is a fast path in front of an LLM and a
sentence it declines is one the LLM is meant to get, so that path
(`DefaultAgent.async_handle_intents`) does not come through here.
"""
import asyncio
from collections.abc import Sequence
from functools import partial
from typing import Any
from gazetteer_matcher import (
AreaSpec,
EntitySpec,
FloorSpec,
GazetteerMatcher,
Home,
Interpretation,
TargetReference,
)
from homeassistant.components.homeassistant.exposed_entities import async_should_expose
from homeassistant.const import ATTR_DEVICE_CLASS
from homeassistant.core import HomeAssistant, State, callback
from homeassistant.helpers import (
area_registry as ar,
chat_session,
entity_registry as er,
floor_registry as fr,
intent,
)
from homeassistant.util import language as language_util
from .const import DOMAIN
LANGUAGE = "en"
_SENTENCE_END = (".", "!", "?")
# Widest selector first: "turn them off" after "turn on the kitchen lights" means
# the kitchen lights, not the one entity that happened to match.
_TARGET_SCOPES = (
(intent.IntentResponseTargetType.FLOOR, TargetReference.for_floor),
(intent.IntentResponseTargetType.AREA, TargetReference.for_area),
(intent.IntentResponseTargetType.ENTITY, TargetReference.for_entity),
)
@callback
def async_refusal(interpretation: Interpretation) -> str | None:
"""Return the matcher's wording for a refusal that named a target."""
if not interpretation.refusal_target:
return None
return interpretation.response
def join_speech(parts: Sequence[str]) -> str:
"""Join the answers of one coordinated command into one thing to speak."""
# Each response stands alone and starts capitalized, and a coordinated command
# can pair an acknowledgement with a whole sentence, so they are run together
# as sentences rather than joined into one.
sentences = [
part if part.endswith(_SENTENCE_END) else f"{part}."
for part in (part.strip() for part in parts)
if part
]
return " ".join(sentences)
@callback
def async_build_home(hass: HomeAssistant) -> Home:
"""Build the matcher's gazetteer of the home from the registries."""
entity_registry = er.async_get(hass)
floors: dict[str, FloorSpec] = {
floor.floor_id: {"name": floor.name, "aliases": list(floor.aliases)}
for floor in fr.async_get(hass).async_list_floors()
}
areas: dict[str, AreaSpec] = {
area.id: {
"name": area.name,
"aliases": list(area.aliases),
"floor": area.floor_id,
}
for area in ar.async_get(hass).async_list_areas()
}
entities: dict[str, EntitySpec] = {}
for state in hass.states.async_all():
if not async_should_expose(hass, DOMAIN, state.entity_id):
continue
entry = entity_registry.async_get(state.entity_id)
if not (names := _names(hass, state, entry)):
continue
spec: EntitySpec = {
"name": names[0],
"aliases": names[1:],
"domain": state.domain,
"area": (
er.async_get_effective_area_id(hass, entry)
if entry is not None
else None
),
}
if device_class := state.attributes.get(ATTR_DEVICE_CLASS):
spec["device_class"] = device_class
entities[state.entity_id] = spec
return {"areas": areas, "floors": floors, "entities": entities}
@callback
def _names(
hass: HomeAssistant, state: State, entry: er.RegistryEntry | None
) -> list[str]:
"""Return the names an entity answers to, without duplicates."""
seen: set[str] = set()
names: list[str] = []
for name in intent.async_get_entity_aliases(hass, entry, state=state):
name = " ".join(name.split())
if name and name.casefold() not in seen:
seen.add(name.casefold())
names.append(name)
return names
@callback
def async_targets_from_intent(
slots: dict[str, Any], intent_response: intent.IntentResponse
) -> tuple[TargetReference, ...]:
"""Return what a sentence hassil recognized selected, for a later "it"/"them".
The matcher resolves a follow-up pronoun only against targets it is handed, and
hassil answers most sentences without reaching it, so the turn that named the
thing has to be recorded from there too.
"""
resolved: dict[str, list[str]] = {}
for target in intent_response.success_results:
if target.id:
resolved.setdefault(target.type, []).append(target.id)
# Carried alongside the selector: "turn them off" should mean the lights it was
# just told about, not everything in the area.
values = {
slot: slots[slot]["value"]
for slot in ("domain", "device_class")
if slot in slots
}
for target_type, build in _TARGET_SCOPES:
ids = resolved.get(target_type)
if not ids:
continue
if len(ids) > 1:
# Nothing a pronoun picks out, and the matcher reuses one selector.
return ()
return (build(ids[0], **values),)
return ()
class GazetteerFallback:
"""The matcher, kept in step with the home it resolves names against."""
def __init__(self, hass: HomeAssistant) -> None:
"""Initialize the fallback without loading anything yet."""
self.hass = hass
self._matcher: GazetteerMatcher | None = None
self._build_lock = asyncio.Lock()
self._wanted_home = 0
self._built_home = 0
self._previous_targets: dict[str, tuple[TargetReference, ...]] = {}
@callback
def async_invalidate(self) -> None:
"""Note that a registry or exposure change has outdated the home."""
self._wanted_home += 1
def supports(self, language: str) -> bool:
"""Return whether the matcher has a vocabulary for this language."""
return language_util.Dialect.parse(language).language == LANGUAGE
async def async_interpret(
self,
text: str,
conversation_id: str,
area: ar.AreaEntry | None,
) -> tuple[GazetteerMatcher, Interpretation]:
"""Interpret text, returning it with the matcher that read it.
The matcher comes back because answering needs it too, to name what the
frames resolved. A rebuild replaces the matcher rather than changing it, so
the one handed back keeps describing what was acted on.
"""
matcher = await self._async_get_matcher()
interpret = partial(
matcher.interpret,
text,
previous_targets=self._previous_targets.get(conversation_id, ()),
)
try:
result = await self.hass.async_add_executor_job(
partial(interpret, context_area=area.id if area else None)
)
except ValueError:
# The registries moved on from the snapshot mid-request. The sentence is
# still worth trying unplaced; shapes needing a room refuse anyway.
result = await self.hass.async_add_executor_job(interpret)
return matcher, result
@callback
def async_remember(
self, conversation_id: str, targets: Sequence[TargetReference] = ()
) -> None:
"""Make this turn the one a pronoun refers back to (it/them).
The entry is dropped when its chat session is cleaned up, so what a
conversation was about lasts exactly as long as the conversation.
"""
if conversation_id not in self._previous_targets:
session = chat_session.current_session.get()
if session is None:
# Nothing would ever clean this up, so do not start it.
return
session.async_on_cleanup(partial(self.async_forget, conversation_id))
self._previous_targets[conversation_id] = tuple(targets)
@callback
def async_forget(self, conversation_id: str) -> None:
"""Drop what a conversation was about, once it is over."""
self._previous_targets.pop(conversation_id, None)
async def _async_get_matcher(self) -> GazetteerMatcher:
"""Return the matcher, rebuilt from the registries if the home has changed."""
if self._matcher is not None and self._built_home == self._wanted_home:
return self._matcher
async with self._build_lock:
if self._matcher is not None and self._built_home == self._wanted_home:
return self._matcher
# Noted before the home is read, so a change arriving during the build
# leaves the result behind rather than counting as current.
building = self._wanted_home
self._matcher = await self.hass.async_add_executor_job(
_build_matcher, self._matcher, async_build_home(self.hass)
)
self._built_home = building
return self._matcher
def _build_matcher(previous: GazetteerMatcher | None, home: Home) -> GazetteerMatcher:
"""Build a matcher over a home, reusing the data files of the last one.
A new matcher rather than a change to the old one, so a request already reading
from that one is unaffected and no locking is needed around either.
"""
if previous is None:
return GazetteerMatcher(home=home)
config = previous.config
return GazetteerMatcher(
home=home,
vocabulary=config.vocabulary,
intents=config.intents,
responses=config.responses,
)
@@ -6,5 +6,9 @@
"documentation": "https://www.home-assistant.io/integrations/conversation",
"integration_type": "entity",
"quality_scale": "internal",
"requirements": ["hassil==3.12.0", "home-assistant-intents==2026.8.25"]
"requirements": [
"gazetteer-matcher==1.0.0",
"hassil==3.12.0",
"home-assistant-intents==2026.8.25"
]
}
+1
View File
@@ -33,6 +33,7 @@ cryptography==48.0.1
dbus-fast==5.0.22
file-read-backwards==2.0.0
fnv-hash-fast==2.0.3
gazetteer-matcher==1.0.0
go2rtc-client==0.4.0
ha-ffmpeg==3.2.2
habluetooth==6.26.7
+1
View File
@@ -23,6 +23,7 @@ ciso8601==2.3.3
cronsim==2.7
cryptography==48.0.1
fnv-hash-fast==2.0.3
gazetteer-matcher==1.0.0
ha-ffmpeg==3.2.2
hass-nabucasa==2.6.0
hassil==3.12.0
+3
View File
@@ -1099,6 +1099,9 @@ gassist-text==0.0.14
# homeassistant.components.gatus
gatus-api==1.2.0
# homeassistant.components.conversation
gazetteer-matcher==1.0.0
# homeassistant.components.google
gcal-sync==9.1.0
@@ -0,0 +1,869 @@
"""Test the gazetteer fallback in the default agent."""
import asyncio
from datetime import timedelta
import threading
import time
from typing import Any
from unittest.mock import patch
from gazetteer_matcher import GazetteerMatcher, TargetReference
import pytest
from homeassistant.components import conversation
from homeassistant.components.conversation import default_agent, gazetteer
from homeassistant.components.conversation.chat_log import async_get_chat_log
from homeassistant.components.conversation.gazetteer import (
GazetteerFallback,
async_build_home,
join_speech,
)
from homeassistant.components.conversation.models import ConversationInput
from homeassistant.const import (
ATTR_DEVICE_CLASS,
ATTR_FRIENDLY_NAME,
STATE_CLOSED,
STATE_OFF,
STATE_ON,
)
from homeassistant.core import Context, HomeAssistant
from homeassistant.helpers import (
area_registry as ar,
chat_session,
config_validation as cv,
device_registry as dr,
entity_registry as er,
intent,
)
from homeassistant.setup import async_setup_component
from homeassistant.util import dt as dt_util
from . import expose_entity
from tests.common import MockConfigEntry, async_fire_time_changed, async_mock_service
KITCHEN_LIGHT = "light.kitchen_ceiling"
BEDROOM_BLINDS = "cover.bedroom_blinds"
GARAGE_DOOR = "cover.garage_door"
GARAGE_SHUTTERS = "cover.garage_shutters"
def _outdated(agent: default_agent.DefaultAgent) -> bool:
"""Return whether the gazetteer wants a newer home than it has built."""
fallback = agent._gazetteer
return fallback._built_home != fallback._wanted_home
@pytest.fixture
def home(
hass: HomeAssistant,
area_registry: ar.AreaRegistry,
entity_registry: er.EntityRegistry,
) -> ar.AreaEntry:
"""Set up a kitchen with a light, a bedroom with blinds and a garage with covers."""
kitchen = area_registry.async_update(
area_registry.async_get_or_create("kitchen_id").id, name="Kitchen"
)
bedroom = area_registry.async_update(
area_registry.async_get_or_create("bedroom_id").id, name="Bedroom"
)
garage = area_registry.async_update(
area_registry.async_get_or_create("garage_id").id, name="Garage"
)
# The garage covers carry a device class so an area can be addressed by it
# ("the garage shutters") without colliding with any one entity's name.
for entity_id, name, area, state, device_class in (
(KITCHEN_LIGHT, "Kitchen Ceiling Lights", kitchen, STATE_OFF, None),
(BEDROOM_BLINDS, "Bedroom Blinds", bedroom, STATE_OFF, None),
(GARAGE_DOOR, "Garage Door", garage, STATE_CLOSED, "garage"),
(GARAGE_SHUTTERS, "Side Window", garage, STATE_CLOSED, "shutter"),
):
domain, object_id = entity_id.split(".")
entry = entity_registry.async_get_or_create(
domain, "demo", object_id, suggested_object_id=object_id
)
assert entry.entity_id == entity_id
entity_registry.async_update_entity(
entity_id, name=name, area_id=area.id, aliases=[er.COMPUTED_NAME]
)
attributes: dict[str, str] = {ATTR_FRIENDLY_NAME: name}
if device_class:
attributes[ATTR_DEVICE_CLASS] = device_class
hass.states.async_set(entity_id, state, attributes=attributes)
return kitchen
@pytest.mark.usefixtures("init_components", "home")
async def test_recognizes_a_misheard_name(hass: HomeAssistant) -> None:
"""Test a name hassil cannot resolve is matched by the gazetteer."""
calls = async_mock_service(hass, "light", "turn_on")
agent = conversation.async_get_agent(hass)
assert isinstance(agent, default_agent.DefaultAgent)
user_input = ConversationInput(
text="turn on the kichen lights",
context=Context(),
conversation_id=None,
device_id=None,
satellite_id=None,
language="en",
agent_id=conversation.HOME_ASSISTANT_AGENT,
)
hassil_result = await agent.async_recognize_intent(user_input)
assert hassil_result is None or hassil_result.unmatched_entities
result = await conversation.async_converse(
hass, "turn on the kichen lights", None, Context(), None
)
assert result.response.response_type is intent.IntentResponseType.ACTION_DONE
assert len(calls) == 1
assert calls[0].data["entity_id"] == [KITCHEN_LIGHT]
@pytest.mark.usefixtures("init_components", "home")
async def test_response_uses_display_names(hass: HomeAssistant) -> None:
"""Test the spoken response names the target instead of reading out its id."""
result = await conversation.async_converse(
hass, "what is the state of the bedrom blinds", None, Context(), None
)
assert result.response.response_type is intent.IntentResponseType.QUERY_ANSWER
# The template's own `| capitalize` lowercases the rest, as it does for hassil.
assert result.response.speech["plain"]["speech"] == "Bedroom blinds is off"
@pytest.mark.usefixtures("init_components", "home")
async def test_handles_every_frame_of_a_coordinated_command(
hass: HomeAssistant,
) -> None:
"""Test a sentence holding two commands runs both, and answers for both."""
turn_off = async_mock_service(hass, "light", "turn_off")
open_cover = async_mock_service(hass, "cover", "open_cover")
result = await conversation.async_converse(
hass,
"switch off the kichen lights and open the bedrom blinds",
None,
Context(),
None,
)
assert result.response.response_type is intent.IntentResponseType.ACTION_DONE
assert len(turn_off) == 1
assert turn_off[0].data["entity_id"] == [KITCHEN_LIGHT]
assert len(open_cover) == 1
assert open_cover[0].data["entity_id"] == BEDROOM_BLINDS
assert (
result.response.speech["plain"]["speech"] == "Turned off the lights. Opening."
)
@pytest.mark.usefixtures("init_components", "home")
async def test_refusal_names_the_target(hass: HomeAssistant) -> None:
"""Test a refusal that resolved a target explains itself."""
result = await conversation.async_converse(
hass, "write a poem about my kitchen lights", None, Context(), None
)
assert result.response.response_type is intent.IntentResponseType.ERROR
assert result.response.error_code is intent.IntentResponseErrorCode.NO_INTENT_MATCH
assert "Kitchen" in result.response.speech["plain"]["speech"]
@pytest.mark.usefixtures("init_components", "home")
@pytest.mark.parametrize("text", ["asdfgh", "do something"])
async def test_refusal_that_explains_nothing_keeps_the_default_error(
hass: HomeAssistant, text: str
) -> None:
"""Test noise still gets Home Assistant's own translated error."""
result = await conversation.async_converse(hass, text, None, Context(), None)
assert result.response.response_type is intent.IntentResponseType.ERROR
assert result.response.error_code is intent.IntentResponseErrorCode.NO_INTENT_MATCH
assert (
result.response.speech["plain"]["speech"] == "Sorry, I couldn't understand that"
)
@pytest.mark.usefixtures("init_components", "home")
async def test_not_used_when_prefer_local_intents(hass: HomeAssistant) -> None:
"""Test the gazetteer does not answer on the prefer-local-intents path."""
calls = async_mock_service(hass, "light", "turn_on")
agent = conversation.async_get_agent(hass)
assert isinstance(agent, default_agent.DefaultAgent)
user_input = ConversationInput(
text="turn on the kichen lights",
context=Context(),
conversation_id=None,
device_id=None,
satellite_id=None,
language="en",
agent_id=conversation.HOME_ASSISTANT_AGENT,
)
with (
chat_session.async_get_chat_session(hass) as session,
async_get_chat_log(hass, session, user_input) as chat_log,
):
assert await agent.async_handle_intents(user_input, chat_log) is None
assert not calls
@pytest.mark.usefixtures("init_components", "home")
async def test_not_used_for_other_languages(hass: HomeAssistant) -> None:
"""Test the matcher's English vocabulary is not applied to other languages."""
calls = async_mock_service(hass, "light", "turn_on")
result = await conversation.async_converse(
hass, "turn on the kichen lights", None, Context(), "de"
)
assert result.response.response_type is intent.IntentResponseType.ERROR
assert not calls
@pytest.mark.usefixtures("init_components", "home")
async def test_follow_up_pronoun_reuses_the_previous_target(
hass: HomeAssistant,
) -> None:
"""Test "them" refers to what the last successful turn targeted."""
async_mock_service(hass, "cover", "open_cover")
close_cover = async_mock_service(hass, "cover", "close_cover")
result = await conversation.async_converse(
hass, "open the bedrom blinds", None, Context(), None
)
assert result.response.response_type is intent.IntentResponseType.ACTION_DONE
result = await conversation.async_converse(
hass, "close them", result.conversation_id, Context(), None
)
assert result.response.response_type is intent.IntentResponseType.ACTION_DONE
assert len(close_cover) == 1
assert close_cover[0].data["entity_id"] == BEDROOM_BLINDS
@pytest.mark.usefixtures("init_components", "home")
async def test_home_follows_the_registries(
hass: HomeAssistant, area_registry: ar.AreaRegistry
) -> None:
"""Test a renamed area is resolvable without restarting."""
calls = async_mock_service(hass, "light", "turn_on")
area_registry.async_update("kitchen_id", name="Scullery")
await hass.async_block_till_done()
result = await conversation.async_converse(
hass, "turn on the scullary lights", None, Context(), None
)
assert result.response.response_type is intent.IntentResponseType.ACTION_DONE
assert len(calls) == 1
assert calls[0].data["entity_id"] == [KITCHEN_LIGHT]
@pytest.mark.usefixtures("init_components", "home")
async def test_hassil_matches_are_left_alone(hass: HomeAssistant) -> None:
"""Test a sentence hassil recognizes never reaches the gazetteer."""
calls = async_mock_service(hass, "light", "turn_on")
agent = conversation.async_get_agent(hass)
assert isinstance(agent, default_agent.DefaultAgent)
agent._gazetteer.async_invalidate()
result = await conversation.async_converse(
hass, "turn on the kitchen lights", None, Context(), None
)
assert result.response.response_type is intent.IntentResponseType.ACTION_DONE
assert len(calls) == 1
# The matcher was never built, so nothing asked it to interpret anything.
assert agent._gazetteer._matcher is None
@pytest.mark.usefixtures("init_components", "home")
async def test_unexposed_entities_are_not_targets(
hass: HomeAssistant, entity_registry: er.EntityRegistry
) -> None:
"""Test the gazetteer only resolves names exposed to conversation."""
entry = entity_registry.async_get_or_create("light", "demo", "hidden")
entity_registry.async_update_entity(
entry.entity_id, name="Wine Cellar Lamp", aliases=[er.COMPUTED_NAME]
)
hass.states.async_set(
entry.entity_id, STATE_ON, attributes={ATTR_FRIENDLY_NAME: "Wine Cellar Lamp"}
)
expose_entity(hass, entry.entity_id, False)
await hass.async_block_till_done()
calls = async_mock_service(hass, "light", "turn_off")
result = await conversation.async_converse(
hass, "switch off the wine cellar lamp", None, Context(), None
)
assert result.response.response_type is intent.IntentResponseType.ERROR
assert not calls
@pytest.mark.usefixtures("init_components", "home")
async def test_declines_a_frame_it_cannot_answer(hass: HomeAssistant) -> None:
"""Test a frame the matcher cannot answer is left to hassil."""
calls = async_mock_service(hass, "light", "turn_on")
real_interpret = GazetteerMatcher.interpret
def unanswerable(self, text, **kwargs):
result = real_interpret(self, text, **kwargs)
for frame in result.frames:
frame.response_key = None
return result
with patch.object(GazetteerMatcher, "interpret", unanswerable):
result = await conversation.async_converse(
hass, "turn on the kichen lights", None, Context(), None
)
assert result.response.response_type is intent.IntentResponseType.ERROR
assert not calls
@pytest.mark.usefixtures("init_components", "home")
@pytest.mark.parametrize(
("text", "speech"),
[
("how many kichen lights are on", "0"),
("are the kichen lights on", "No"),
],
ids=["how_many", "any"],
)
async def test_wording_picks_between_identical_frames(
hass: HomeAssistant, text: str, speech: str
) -> None:
"""Test the frame's own response key tells identical frames apart."""
result = await conversation.async_converse(hass, text, None, Context(), None)
assert result.response.response_type is intent.IntentResponseType.QUERY_ANSWER
assert result.response.speech["plain"]["speech"] == speech
@pytest.mark.usefixtures("init_components", "home")
async def test_pronoun_follows_a_turn_hassil_answered(hass: HomeAssistant) -> None:
"""Test "open it" refers to what the previous hassil turn was about."""
calls = async_mock_service(hass, "cover", "open_cover")
agent = conversation.async_get_agent(hass)
assert isinstance(agent, default_agent.DefaultAgent)
result = await conversation.async_converse(
hass, "is the garage door closed", None, Context(), None
)
assert result.response.response_type is intent.IntentResponseType.QUERY_ANSWER
# Nothing built the matcher, so that turn was hassil's alone.
assert agent._gazetteer._matcher is None
result = await conversation.async_converse(
hass, "open it", result.conversation_id, Context(), None
)
assert result.response.response_type is intent.IntentResponseType.ACTION_DONE
assert len(calls) == 1
assert calls[0].data["entity_id"] == GARAGE_DOOR
@pytest.mark.usefixtures("init_components", "home")
async def test_pronoun_follows_the_area_a_turn_named(
hass: HomeAssistant, entity_registry: er.EntityRegistry
) -> None:
"""Test "them" reuses the area a command scoped to, not what it resolved to."""
second = "light.kitchen_lamp"
entry = entity_registry.async_get_or_create(
"light", "demo", "kitchen_lamp", suggested_object_id="kitchen_lamp"
)
assert entry.entity_id == second
entity_registry.async_update_entity(
second, name="Kitchen Lamp", area_id="kitchen_id", aliases=[er.COMPUTED_NAME]
)
hass.states.async_set(
second, STATE_OFF, attributes={ATTR_FRIENDLY_NAME: "Kitchen Lamp"}
)
await hass.async_block_till_done()
agent = conversation.async_get_agent(hass)
assert isinstance(agent, default_agent.DefaultAgent)
async_mock_service(hass, "light", "turn_on")
turn_off = async_mock_service(hass, "light", "turn_off")
result = await conversation.async_converse(
hass, "turn on the kitchen lights", None, Context(), None
)
assert result.response.response_type is intent.IntentResponseType.ACTION_DONE
# What the turn left behind is the selector the sentence used, not its result.
assert agent._gazetteer._previous_targets[result.conversation_id] == (
TargetReference.for_area("kitchen_id", domain="light"),
)
result = await conversation.async_converse(
hass, "turn them off", result.conversation_id, Context(), None
)
assert result.response.response_type is intent.IntentResponseType.ACTION_DONE
# One call per entity, so both lights in the area went off rather than just the
# one the first sentence resolved to.
targeted = [
entity_id
for call in turn_off
for entity_id in cv.ensure_list(call.data["entity_id"])
]
assert sorted(targeted) == sorted([KITCHEN_LIGHT, second])
@pytest.mark.usefixtures("init_components", "home")
async def test_a_failed_turn_leaves_the_antecedent_alone(hass: HomeAssistant) -> None:
"""Test a sentence nobody recognized does not strand a following pronoun."""
calls = async_mock_service(hass, "cover", "open_cover")
result = await conversation.async_converse(
hass, "is the garage door closed", None, Context(), None
)
conversation_id = result.conversation_id
result = await conversation.async_converse(
hass, "asdfgh", conversation_id, Context(), None
)
assert result.response.response_type is intent.IntentResponseType.ERROR
result = await conversation.async_converse(
hass, "open it", conversation_id, Context(), None
)
assert result.response.response_type is intent.IntentResponseType.ACTION_DONE
assert len(calls) == 1
@pytest.mark.usefixtures("init_components", "home")
async def test_a_turn_with_no_target_clears_the_antecedent(
hass: HomeAssistant,
) -> None:
"""Test a successful turn that targeted nothing clears the antecedent."""
calls = async_mock_service(hass, "cover", "open_cover")
result = await conversation.async_converse(
hass, "is the garage door closed", None, Context(), None
)
conversation_id = result.conversation_id
result = await conversation.async_converse(
hass, "what time is it", conversation_id, Context(), None
)
assert result.response.response_type is intent.IntentResponseType.ACTION_DONE
result = await conversation.async_converse(
hass, "open it", conversation_id, Context(), None
)
assert result.response.response_type is intent.IntentResponseType.ERROR
assert not calls
@pytest.mark.usefixtures("init_components", "home")
async def test_them_reopens_the_area_a_cover_command_named(
hass: HomeAssistant,
) -> None:
"""Test "open them" after an area command acts on that area again."""
async_mock_service(hass, "cover", "close_cover")
calls = async_mock_service(hass, "cover", "open_cover")
result = await conversation.async_converse(
hass, "close the garage shutters", None, Context(), None
)
assert result.response.response_type is intent.IntentResponseType.ACTION_DONE
result = await conversation.async_converse(
hass, "open them", result.conversation_id, Context(), None
)
assert result.response.response_type is intent.IntentResponseType.ACTION_DONE
assert [call.data["entity_id"] for call in calls] == [GARAGE_SHUTTERS]
@pytest.mark.usefixtures("home")
async def test_a_custom_sentence_still_leaves_an_antecedent(
hass: HomeAssistant,
) -> None:
"""Test a phrasing only hassil knows is still something "it" can refer to."""
assert await async_setup_component(hass, "homeassistant", {})
assert await async_setup_component(
hass,
conversation.DOMAIN,
{"conversation": {"intents": {"HassTurnOn": ["give the {name} some juice"]}}},
)
assert await async_setup_component(hass, "intent", {})
async_mock_service(hass, "light", "turn_on")
turn_off = async_mock_service(hass, "light", "turn_off")
result = await conversation.async_converse(
hass, "give the kitchen ceiling lights some juice", None, Context(), None
)
assert result.response.response_type is intent.IntentResponseType.ACTION_DONE
result = await conversation.async_converse(
hass, "turn it off", result.conversation_id, Context(), None
)
assert result.response.response_type is intent.IntentResponseType.ACTION_DONE
assert len(turn_off) == 1
assert turn_off[0].data["entity_id"] == [KITCHEN_LIGHT]
@pytest.mark.parametrize(
("parts", "expected"),
[
(["Unlocking", "Opening"], "Unlocking. Opening."),
(
["Turned off the lights", "Garage door is closed"],
"Turned off the lights. Garage door is closed.",
),
(
["Hello from Home Assistant.", "Opening"],
"Hello from Home Assistant. Opening.",
),
(["Opening", "", " "], "Opening."),
],
ids=["two_acknowledgements", "mixed_with_a_query", "already_punctuated", "empties"],
)
def test_join_speech(parts: list[str], expected: str) -> None:
"""Test the frames of one command are spoken as sentences, not one clause."""
assert join_speech(parts) == expected
@pytest.mark.usefixtures("init_components", "home")
async def test_an_antecedent_lasts_as_long_as_its_conversation(
hass: HomeAssistant,
) -> None:
"""Test what a conversation was about is dropped when the conversation ends."""
agent = conversation.async_get_agent(hass)
assert isinstance(agent, default_agent.DefaultAgent)
async_mock_service(hass, "light", "turn_on")
result = await conversation.async_converse(
hass, "turn on the kichen lights", None, Context(), None
)
conversation_id = result.conversation_id
assert conversation_id in agent._gazetteer._previous_targets
# Nothing evicts it while other conversations come and go.
for _ in range(12):
await conversation.async_converse(
hass, "turn on the kichen lights", None, Context(), None
)
assert conversation_id in agent._gazetteer._previous_targets
# Its session expiring is what ends it.
async_fire_time_changed(
hass, dt_util.utcnow() + chat_session.CONVERSATION_TIMEOUT * 2 + timedelta(1)
)
await hass.async_block_till_done()
assert conversation_id not in agent._gazetteer._previous_targets
@pytest.mark.usefixtures("init_components", "home")
async def test_an_area_that_vanished_mid_request_is_retried_unplaced(
hass: HomeAssistant,
area_registry: ar.AreaRegistry,
device_registry: dr.DeviceRegistry,
) -> None:
"""Test a sentence is retried unplaced when the speaker's area has gone."""
calls = async_mock_service(hass, "light", "turn_on")
agent = conversation.async_get_agent(hass)
assert isinstance(agent, default_agent.DefaultAgent)
entry = MockConfigEntry(domain="test")
entry.add_to_hass(hass)
device = device_registry.async_get_or_create(
config_entry_id=entry.entry_id, identifiers={("test", "satellite")}
)
device_registry.async_update_device(
device.id, area_id=area_registry.async_get_or_create("ghost_id").id
)
real_interpret = GazetteerMatcher.interpret
placed: list[str | None] = []
def interpret(self, text, *, context_area=None, **kwargs):
"""Refuse the context once, as the matcher does for an unknown area."""
placed.append(context_area)
if context_area is not None:
raise ValueError(f"unknown context area {context_area!r}")
return real_interpret(self, text, **kwargs)
with patch.object(GazetteerMatcher, "interpret", interpret):
result = await conversation.async_converse(
hass,
"turn on the kichen lights",
None,
Context(),
None,
device_id=device.id,
)
assert placed == ["ghost_id", None]
assert result.response.response_type is intent.IntentResponseType.ACTION_DONE
assert len(calls) == 1
assert calls[0].data["entity_id"] == [KITCHEN_LIGHT]
@pytest.mark.usefixtures("init_components", "home")
async def test_the_home_is_built_off_the_event_loop(
hass: HomeAssistant, area_registry: ar.AreaRegistry
) -> None:
"""Test building the matcher does not block the event loop."""
off_loop: list[bool] = []
def record() -> None:
"""Note whether this ran somewhere with a running event loop."""
try:
asyncio.get_running_loop()
except RuntimeError:
off_loop.append(True)
else:
off_loop.append(False)
real_init = GazetteerMatcher.__init__
def init(self, **kwargs):
record()
real_init(self, **kwargs)
with patch.object(GazetteerMatcher, "__init__", init):
# The first sentence builds the matcher.
await conversation.async_converse(
hass, "turn on the kichen lights", None, Context(), None
)
assert off_loop == [True]
# A registry change makes the home stale, so the next one refreshes it.
area_registry.async_update("kitchen_id", name="Scullery")
await hass.async_block_till_done()
await conversation.async_converse(
hass, "turn on the scullary lights", None, Context(), None
)
assert off_loop == [True, True]
@pytest.mark.usefixtures("init_components", "home")
async def test_a_change_during_a_rebuild_is_not_lost(
hass: HomeAssistant, area_registry: ar.AreaRegistry
) -> None:
"""Test a registry change arriving mid-rebuild still outdates the home.
The home is read before the rebuild is handed to the executor, so a change
landing during it is not in the snapshot and has to survive the flag reset.
"""
agent = conversation.async_get_agent(hass)
assert isinstance(agent, default_agent.DefaultAgent)
async_mock_service(hass, "light", "turn_on")
await conversation.async_converse(
hass, "turn on the kichen lights", None, Context(), None
)
rebuilding = threading.Event()
release = threading.Event()
real_build = gazetteer._build_matcher
def build(previous, home):
rebuilding.set()
release.wait(timeout=5)
return real_build(previous, home)
area_registry.async_update("kitchen_id", name="Scullery")
assert _outdated(agent)
with patch.object(gazetteer, "_build_matcher", build):
pending = hass.async_create_task(
conversation.async_converse(
hass, "turn on the scullary lights", None, Context(), None
)
)
assert await hass.async_add_executor_job(rebuilding.wait, 5)
# This arrives after the home was read, so the rebuild in flight misses it.
area_registry.async_update("bedroom_id", name="Nursery")
release.set()
async with asyncio.timeout(5):
await pending
assert _outdated(agent), "the change made during the rebuild was counted as built"
async def test_an_antecedent_needs_a_session_to_belong_to(
hass: HomeAssistant,
) -> None:
"""Test nothing is kept for a turn with no session to end it."""
fallback = GazetteerFallback(hass)
fallback.async_remember("orphan", (TargetReference.for_entity(KITCHEN_LIGHT),))
assert not fallback._previous_targets
@pytest.mark.usefixtures("init_components", "home")
async def test_a_rebuild_that_fails_leaves_the_home_out_of_date(
hass: HomeAssistant, area_registry: ar.AreaRegistry
) -> None:
"""Test a rebuild that did not finish is tried again rather than given up on."""
agent = conversation.async_get_agent(hass)
assert isinstance(agent, default_agent.DefaultAgent)
async_mock_service(hass, "light", "turn_on")
await conversation.async_converse(
hass, "turn on the kichen lights", None, Context(), None
)
area_registry.async_update("kitchen_id", name="Scullery")
assert _outdated(agent)
with (
patch(
"homeassistant.components.conversation.gazetteer._build_matcher",
side_effect=RuntimeError("no can do"),
),
pytest.raises(RuntimeError),
):
await conversation.async_converse(
hass, "turn on the scullary lights", None, Context(), None
)
# The matcher still holds the old home, so it has to still count as stale.
assert _outdated(agent)
result = await conversation.async_converse(
hass, "turn on the scullary lights", None, Context(), None
)
assert result.response.response_type is intent.IntentResponseType.ACTION_DONE
@pytest.mark.usefixtures("init_components", "home")
@pytest.mark.parametrize(
("registry", "event", "outdated"),
[
("entity", {"action": "update", "changes": {"area_id": None}}, True),
("entity", {"action": "update", "changes": {"name": None}}, True),
("entity", {"action": "update", "changes": {"icon": None}}, False),
("entity", {"action": "remove"}, False),
("entity", {"action": "create"}, False),
("device", {"action": "update", "changes": {"area_id": None}}, True),
("device", {"action": "update", "changes": {"parent_device_id": None}}, True),
("device", {"action": "update", "changes": {"sw_version": None}}, False),
("device", {"action": "remove"}, False),
],
ids=[
"entity_moved",
"entity_renamed",
"entity_restyled",
"entity_removed",
"entity_created",
"device_moved",
"device_became_a_child",
"device_upgraded",
"device_removed",
],
)
async def test_what_outdates_the_home(
hass: HomeAssistant, registry: str, event: dict[str, Any], outdated: bool
) -> None:
"""Test the registry events that change what an entity is called or where."""
agent = conversation.async_get_agent(hass)
assert isinstance(agent, default_agent.DefaultAgent)
matches = {
"entity": agent._filter_entity_registry_changes,
"device": agent._filter_device_registry_changes,
}[registry]
assert matches(event) is outdated
@pytest.mark.usefixtures("init_components", "home")
async def test_an_entity_with_no_aliases_is_not_a_target(
hass: HomeAssistant, entity_registry: er.EntityRegistry
) -> None:
"""Test an entity nobody has given a name to is not something to act on."""
entry = entity_registry.async_get_or_create(
"light", "demo", "nameless", suggested_object_id="nameless"
)
entity_registry.async_update_entity(entry.entity_id, aliases=[])
hass.states.async_set(entry.entity_id, STATE_OFF)
await hass.async_block_till_done()
assert entry.entity_id not in async_build_home(hass)["entities"]
@pytest.mark.usefixtures("init_components", "home")
async def test_the_home_is_built_once_for_concurrent_sentences(
hass: HomeAssistant,
) -> None:
"""Test sentences arriving together do not each build a matcher."""
async_mock_service(hass, "light", "turn_on")
builds = 0
real_build = gazetteer._build_matcher
def build(previous, home):
nonlocal builds
builds += 1
# Slow enough that the other sentences reach the lock while this holds it.
time.sleep(0.1)
return real_build(previous, home)
with patch.object(gazetteer, "_build_matcher", build):
await asyncio.gather(
*(
conversation.async_converse(
hass, "turn on the kichen lights", None, Context(), None
)
for _ in range(4)
)
)
assert builds == 1
@pytest.mark.usefixtures("home")
async def test_a_custom_sentence_keeps_its_own_error(hass: HomeAssistant) -> None:
"""Test a sentence somebody wrote themselves is not answered by the gazetteer.
hassil matched it to the intent it was written for and only the target failed,
so its error is the answer. The gazetteer would recognize the same words as a
built-in command and run that instead.
"""
assert await async_setup_component(hass, "homeassistant", {})
assert await async_setup_component(
hass,
conversation.DOMAIN,
{"conversation": {"intents": {"MoodLight": ["please activate {name} now"]}}},
)
assert await async_setup_component(hass, "intent", {})
assert await async_setup_component(
hass,
"intent_script",
{"intent_script": {"MoodLight": {"speech": {"text": "Mood set"}}}},
)
calls = async_mock_service(hass, "light", "turn_on")
result = await conversation.async_converse(
hass, "please activate kichen lights now", None, Context(), None
)
assert result.response.response_type is intent.IntentResponseType.ERROR
assert (
result.response.speech["plain"]["speech"]
== "Sorry, I am not aware of any device called kichen lights"
)
assert not calls