mirror of
https://github.com/home-assistant/core.git
synced 2026-09-26 09:23:17 -04:00
Add Assist API behavior-parity snapshot test
Capture the assembled Assist prompt and the full serialized tool set (name, description, parameters) for a scenario exercising every built-in tool type — intents, timer tools, calendar/todo/script tools, GetDateTime and GetLiveContext. This is the parity net for the upcoming LLM tool platform refactor: the snapshot must stay identical as tools and intents move out of AssistAPI into per-integration platforms. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Paulus Schoutsen
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
f0c0e937d1
commit
184025cfb4
@@ -0,0 +1,529 @@
|
||||
# serializer version: 1
|
||||
# name: test_assist_api_snapshot[prompt]
|
||||
'''
|
||||
When controlling Home Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff to unlock a lock. When controlling a device, prefer passing just name and domain. When controlling an area, prefer passing just area name and domain.
|
||||
You ARE equipped to answer questions about the current state of
|
||||
the home using the `GetLiveContext` tool. This is a primary function. Do not state you lack the
|
||||
functionality if the question requires live data.
|
||||
If the user asks about device existence/type (e.g., "Do I have lights in the bedroom?"): Answer
|
||||
from the static context below.
|
||||
If the user asks about the CURRENT state, value, or mode (e.g., "Is the lock locked?",
|
||||
"Is the fan on?", "What mode is the thermostat in?", "What is the temperature outside?"):
|
||||
1. Recognize this requires live data.
|
||||
2. You MUST call `GetLiveContext`. This tool will provide the needed real-time information (like temperature from the local weather, lock status, etc.).
|
||||
3. Use the tool's response** to answer the user accurately (e.g., "The temperature outside is [value from tool].").
|
||||
For general knowledge questions not about the home: Answer truthfully from internal knowledge.
|
||||
|
||||
Static Context: An overview of the areas and the devices in this smart home:
|
||||
- names: Kitchen
|
||||
domain: light
|
||||
- names: Shopping
|
||||
domain: todo
|
||||
|
||||
You are in area Test Area and all generic commands like 'turn on the lights' should target this area.
|
||||
'''
|
||||
# ---
|
||||
# name: test_assist_api_snapshot[tools]
|
||||
list([
|
||||
dict({
|
||||
'description': "Turns on/opens/presses a device or entity. For locks, this performs a 'lock' action. Use for requests like 'turn on', 'activate', 'enable', or 'lock'.",
|
||||
'name': 'HassTurnOn',
|
||||
'parameters': dict({
|
||||
'properties': dict({
|
||||
'area': dict({
|
||||
'type': 'string',
|
||||
}),
|
||||
'device_class': dict({
|
||||
'items': dict({
|
||||
'enum': list([
|
||||
'awning',
|
||||
'blind',
|
||||
'curtain',
|
||||
'damper',
|
||||
'door',
|
||||
'garage',
|
||||
'gas',
|
||||
'gate',
|
||||
'identify',
|
||||
'outlet',
|
||||
'projector',
|
||||
'receiver',
|
||||
'restart',
|
||||
'shade',
|
||||
'shutter',
|
||||
'speaker',
|
||||
'switch',
|
||||
'tv',
|
||||
'update',
|
||||
'water',
|
||||
'window',
|
||||
]),
|
||||
'type': 'string',
|
||||
}),
|
||||
'type': 'array',
|
||||
}),
|
||||
'domain': dict({
|
||||
'items': dict({
|
||||
'type': 'string',
|
||||
}),
|
||||
'type': 'array',
|
||||
}),
|
||||
'floor': dict({
|
||||
'type': 'string',
|
||||
}),
|
||||
'name': dict({
|
||||
'type': 'string',
|
||||
}),
|
||||
}),
|
||||
'required': list([
|
||||
]),
|
||||
'type': 'object',
|
||||
}),
|
||||
}),
|
||||
dict({
|
||||
'description': "Turns off/closes a device or entity. For locks, this performs an 'unlock' action. Use for requests like 'turn off', 'deactivate', 'disable', or 'unlock'.",
|
||||
'name': 'HassTurnOff',
|
||||
'parameters': dict({
|
||||
'properties': dict({
|
||||
'area': dict({
|
||||
'type': 'string',
|
||||
}),
|
||||
'device_class': dict({
|
||||
'items': dict({
|
||||
'enum': list([
|
||||
'awning',
|
||||
'blind',
|
||||
'curtain',
|
||||
'damper',
|
||||
'door',
|
||||
'garage',
|
||||
'gas',
|
||||
'gate',
|
||||
'identify',
|
||||
'outlet',
|
||||
'projector',
|
||||
'receiver',
|
||||
'restart',
|
||||
'shade',
|
||||
'shutter',
|
||||
'speaker',
|
||||
'switch',
|
||||
'tv',
|
||||
'update',
|
||||
'water',
|
||||
'window',
|
||||
]),
|
||||
'type': 'string',
|
||||
}),
|
||||
'type': 'array',
|
||||
}),
|
||||
'domain': dict({
|
||||
'items': dict({
|
||||
'type': 'string',
|
||||
}),
|
||||
'type': 'array',
|
||||
}),
|
||||
'floor': dict({
|
||||
'type': 'string',
|
||||
}),
|
||||
'name': dict({
|
||||
'type': 'string',
|
||||
}),
|
||||
}),
|
||||
'required': list([
|
||||
]),
|
||||
'type': 'object',
|
||||
}),
|
||||
}),
|
||||
dict({
|
||||
'description': 'Starts a new timer',
|
||||
'name': 'HassStartTimer',
|
||||
'parameters': dict({
|
||||
'anyOf': list([
|
||||
dict({
|
||||
'required': list([
|
||||
'hours',
|
||||
]),
|
||||
}),
|
||||
dict({
|
||||
'required': list([
|
||||
'minutes',
|
||||
]),
|
||||
}),
|
||||
dict({
|
||||
'required': list([
|
||||
'seconds',
|
||||
]),
|
||||
}),
|
||||
]),
|
||||
'properties': dict({
|
||||
'conversation_command': dict({
|
||||
'type': 'string',
|
||||
}),
|
||||
'hours': dict({
|
||||
'minimum': 0,
|
||||
'type': 'integer',
|
||||
}),
|
||||
'minutes': dict({
|
||||
'minimum': 0,
|
||||
'type': 'integer',
|
||||
}),
|
||||
'name': dict({
|
||||
'type': 'string',
|
||||
}),
|
||||
'seconds': dict({
|
||||
'minimum': 0,
|
||||
'type': 'integer',
|
||||
}),
|
||||
}),
|
||||
'required': list([
|
||||
]),
|
||||
'type': 'object',
|
||||
}),
|
||||
}),
|
||||
dict({
|
||||
'description': 'Cancels a timer',
|
||||
'name': 'HassCancelTimer',
|
||||
'parameters': dict({
|
||||
'properties': dict({
|
||||
'area': dict({
|
||||
'type': 'string',
|
||||
}),
|
||||
'name': dict({
|
||||
'type': 'string',
|
||||
}),
|
||||
'start_hours': dict({
|
||||
'minimum': 0,
|
||||
'type': 'integer',
|
||||
}),
|
||||
'start_minutes': dict({
|
||||
'minimum': 0,
|
||||
'type': 'integer',
|
||||
}),
|
||||
'start_seconds': dict({
|
||||
'minimum': 0,
|
||||
'type': 'integer',
|
||||
}),
|
||||
}),
|
||||
'required': list([
|
||||
]),
|
||||
'type': 'object',
|
||||
}),
|
||||
}),
|
||||
dict({
|
||||
'description': 'Cancels all timers',
|
||||
'name': 'HassCancelAllTimers',
|
||||
'parameters': dict({
|
||||
'properties': dict({
|
||||
'area': dict({
|
||||
'type': 'string',
|
||||
}),
|
||||
}),
|
||||
'required': list([
|
||||
]),
|
||||
'type': 'object',
|
||||
}),
|
||||
}),
|
||||
dict({
|
||||
'description': 'Adds more time to a timer',
|
||||
'name': 'HassIncreaseTimer',
|
||||
'parameters': dict({
|
||||
'properties': dict({
|
||||
'area': dict({
|
||||
'type': 'string',
|
||||
}),
|
||||
'hours': dict({
|
||||
'minimum': 0,
|
||||
'type': 'integer',
|
||||
}),
|
||||
'minutes': dict({
|
||||
'minimum': 0,
|
||||
'type': 'integer',
|
||||
}),
|
||||
'name': dict({
|
||||
'type': 'string',
|
||||
}),
|
||||
'seconds': dict({
|
||||
'minimum': 0,
|
||||
'type': 'integer',
|
||||
}),
|
||||
'start_hours': dict({
|
||||
'minimum': 0,
|
||||
'type': 'integer',
|
||||
}),
|
||||
'start_minutes': dict({
|
||||
'minimum': 0,
|
||||
'type': 'integer',
|
||||
}),
|
||||
'start_seconds': dict({
|
||||
'minimum': 0,
|
||||
'type': 'integer',
|
||||
}),
|
||||
}),
|
||||
'required': list([
|
||||
]),
|
||||
'type': 'object',
|
||||
}),
|
||||
}),
|
||||
dict({
|
||||
'description': 'Removes time from a timer',
|
||||
'name': 'HassDecreaseTimer',
|
||||
'parameters': dict({
|
||||
'anyOf': list([
|
||||
dict({
|
||||
'required': list([
|
||||
'hours',
|
||||
]),
|
||||
}),
|
||||
dict({
|
||||
'required': list([
|
||||
'minutes',
|
||||
]),
|
||||
}),
|
||||
dict({
|
||||
'required': list([
|
||||
'seconds',
|
||||
]),
|
||||
}),
|
||||
]),
|
||||
'properties': dict({
|
||||
'area': dict({
|
||||
'type': 'string',
|
||||
}),
|
||||
'hours': dict({
|
||||
'minimum': 0,
|
||||
'type': 'integer',
|
||||
}),
|
||||
'minutes': dict({
|
||||
'minimum': 0,
|
||||
'type': 'integer',
|
||||
}),
|
||||
'name': dict({
|
||||
'type': 'string',
|
||||
}),
|
||||
'seconds': dict({
|
||||
'minimum': 0,
|
||||
'type': 'integer',
|
||||
}),
|
||||
'start_hours': dict({
|
||||
'minimum': 0,
|
||||
'type': 'integer',
|
||||
}),
|
||||
'start_minutes': dict({
|
||||
'minimum': 0,
|
||||
'type': 'integer',
|
||||
}),
|
||||
'start_seconds': dict({
|
||||
'minimum': 0,
|
||||
'type': 'integer',
|
||||
}),
|
||||
}),
|
||||
'required': list([
|
||||
]),
|
||||
'type': 'object',
|
||||
}),
|
||||
}),
|
||||
dict({
|
||||
'description': 'Pauses a running timer',
|
||||
'name': 'HassPauseTimer',
|
||||
'parameters': dict({
|
||||
'properties': dict({
|
||||
'area': dict({
|
||||
'type': 'string',
|
||||
}),
|
||||
'name': dict({
|
||||
'type': 'string',
|
||||
}),
|
||||
'start_hours': dict({
|
||||
'minimum': 0,
|
||||
'type': 'integer',
|
||||
}),
|
||||
'start_minutes': dict({
|
||||
'minimum': 0,
|
||||
'type': 'integer',
|
||||
}),
|
||||
'start_seconds': dict({
|
||||
'minimum': 0,
|
||||
'type': 'integer',
|
||||
}),
|
||||
}),
|
||||
'required': list([
|
||||
]),
|
||||
'type': 'object',
|
||||
}),
|
||||
}),
|
||||
dict({
|
||||
'description': 'Resumes a paused timer',
|
||||
'name': 'HassUnpauseTimer',
|
||||
'parameters': dict({
|
||||
'properties': dict({
|
||||
'area': dict({
|
||||
'type': 'string',
|
||||
}),
|
||||
'name': dict({
|
||||
'type': 'string',
|
||||
}),
|
||||
'start_hours': dict({
|
||||
'minimum': 0,
|
||||
'type': 'integer',
|
||||
}),
|
||||
'start_minutes': dict({
|
||||
'minimum': 0,
|
||||
'type': 'integer',
|
||||
}),
|
||||
'start_seconds': dict({
|
||||
'minimum': 0,
|
||||
'type': 'integer',
|
||||
}),
|
||||
}),
|
||||
'required': list([
|
||||
]),
|
||||
'type': 'object',
|
||||
}),
|
||||
}),
|
||||
dict({
|
||||
'description': 'Reports the current status of timers',
|
||||
'name': 'HassTimerStatus',
|
||||
'parameters': dict({
|
||||
'properties': dict({
|
||||
'area': dict({
|
||||
'type': 'string',
|
||||
}),
|
||||
'name': dict({
|
||||
'type': 'string',
|
||||
}),
|
||||
'start_hours': dict({
|
||||
'minimum': 0,
|
||||
'type': 'integer',
|
||||
}),
|
||||
'start_minutes': dict({
|
||||
'minimum': 0,
|
||||
'type': 'integer',
|
||||
}),
|
||||
'start_seconds': dict({
|
||||
'minimum': 0,
|
||||
'type': 'integer',
|
||||
}),
|
||||
}),
|
||||
'required': list([
|
||||
]),
|
||||
'type': 'object',
|
||||
}),
|
||||
}),
|
||||
dict({
|
||||
'description': 'Provides the current date and time.',
|
||||
'name': 'GetDateTime',
|
||||
'parameters': dict({
|
||||
'properties': dict({
|
||||
}),
|
||||
'required': list([
|
||||
]),
|
||||
'type': 'object',
|
||||
}),
|
||||
}),
|
||||
dict({
|
||||
'description': "Get events from a calendar. When asked if something happens, search the whole week. Results are RFC 5545 which means 'end' is exclusive.",
|
||||
'name': 'calendar_get_events',
|
||||
'parameters': dict({
|
||||
'properties': dict({
|
||||
'calendar': dict({
|
||||
'enum': list([
|
||||
'Personal',
|
||||
]),
|
||||
'type': 'string',
|
||||
}),
|
||||
'range': dict({
|
||||
'enum': list([
|
||||
'today',
|
||||
'week',
|
||||
]),
|
||||
'type': 'string',
|
||||
}),
|
||||
}),
|
||||
'required': list([
|
||||
'calendar',
|
||||
'range',
|
||||
]),
|
||||
'type': 'object',
|
||||
}),
|
||||
}),
|
||||
dict({
|
||||
'description': "Query a to-do list to find out what items are on it. Use this to answer questions like 'What's on my task list?' or 'Read my grocery list'. Filters items by status (needs_action, completed, all).",
|
||||
'name': 'todo_get_items',
|
||||
'parameters': dict({
|
||||
'properties': dict({
|
||||
'status': dict({
|
||||
'default': 'needs_action',
|
||||
'description': 'Filter returned items by status, by default returns incomplete items',
|
||||
'enum': list([
|
||||
'all',
|
||||
'completed',
|
||||
'needs_action',
|
||||
]),
|
||||
'type': 'string',
|
||||
}),
|
||||
'todo_list': dict({
|
||||
'enum': list([
|
||||
'Shopping',
|
||||
]),
|
||||
'type': 'string',
|
||||
}),
|
||||
}),
|
||||
'required': list([
|
||||
'todo_list',
|
||||
]),
|
||||
'type': 'object',
|
||||
}),
|
||||
}),
|
||||
dict({
|
||||
'description': "This is a test script. Aliases: ['test_script']",
|
||||
'name': 'test_script',
|
||||
'parameters': dict({
|
||||
'properties': dict({
|
||||
'beer': dict({
|
||||
'description': 'Number of beers',
|
||||
'type': 'string',
|
||||
}),
|
||||
'wine': dict({
|
||||
'type': 'string',
|
||||
}),
|
||||
}),
|
||||
'required': list([
|
||||
]),
|
||||
'type': 'object',
|
||||
}),
|
||||
}),
|
||||
dict({
|
||||
'description': "Provides real-time information about the CURRENT state, value, or mode of devices, sensors, entities, or areas. Use this tool for: 1. Answering questions about current conditions (e.g., 'Is the light on?'). 2. As the first step in conditional actions (e.g., 'If the weather is rainy, turn off sprinklers' requires checking the weather first). You may filter for devices by name, domain, and area, including combining those filters. Prefer filtering by domain when searching for multiple devices of the same type.",
|
||||
'name': 'GetLiveContext',
|
||||
'parameters': dict({
|
||||
'properties': dict({
|
||||
'area': dict({
|
||||
'description': 'Filter entities by area name or alias (case-insensitive).',
|
||||
'type': 'string',
|
||||
}),
|
||||
'domain': dict({
|
||||
'anyOf': list([
|
||||
dict({
|
||||
}),
|
||||
dict({
|
||||
'items': dict({
|
||||
'type': 'string',
|
||||
}),
|
||||
'type': 'array',
|
||||
}),
|
||||
]),
|
||||
'description': "Filter entities by domain (e.g. 'light', 'sensor'). Accepts a single domain or a list.",
|
||||
}),
|
||||
'name': dict({
|
||||
'description': 'Filter entities by name or alias (case-insensitive).',
|
||||
'type': 'string',
|
||||
}),
|
||||
}),
|
||||
'required': list([
|
||||
]),
|
||||
'type': 'object',
|
||||
}),
|
||||
}),
|
||||
])
|
||||
# ---
|
||||
@@ -2,10 +2,13 @@
|
||||
|
||||
from datetime import timedelta
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from syrupy.assertion import SnapshotAssertion
|
||||
import voluptuous as vol
|
||||
from voluptuous_openapi import convert
|
||||
|
||||
from homeassistant.components import calendar, todo
|
||||
from homeassistant.components.homeassistant.exposed_entities import async_expose_entity
|
||||
@@ -327,6 +330,103 @@ async def test_assist_api(
|
||||
}
|
||||
|
||||
|
||||
def _normalize_schema(value: Any) -> Any:
|
||||
"""Recursively sort scalar lists (e.g. enum values) for a stable snapshot.
|
||||
|
||||
Some tool parameter schemas build enum options from Python sets, so their
|
||||
order varies per process. Order is semantically irrelevant here.
|
||||
"""
|
||||
if isinstance(value, dict):
|
||||
return {key: _normalize_schema(val) for key, val in value.items()}
|
||||
if isinstance(value, list):
|
||||
items = [_normalize_schema(item) for item in value]
|
||||
if all(isinstance(item, (str, int, float, bool)) for item in items):
|
||||
return sorted(items, key=repr)
|
||||
return items
|
||||
return value
|
||||
|
||||
|
||||
async def test_assist_api_snapshot(
|
||||
hass: HomeAssistant,
|
||||
entity_registry: er.EntityRegistry,
|
||||
device_registry: dr.DeviceRegistry,
|
||||
snapshot: SnapshotAssertion,
|
||||
) -> None:
|
||||
"""Golden snapshot of the Assist API prompt + tools.
|
||||
|
||||
Behavior-parity net for the v1 tool-platform refactor: the assembled prompt
|
||||
and the full serialized tool set (name, description, parameters) must stay
|
||||
identical as built-in tools and intents move out of AssistAPI into per-
|
||||
integration platforms.
|
||||
"""
|
||||
assert await async_setup_component(hass, "homeassistant", {})
|
||||
assert await async_setup_component(hass, "intent", {})
|
||||
assert await async_setup_component(
|
||||
hass,
|
||||
"script",
|
||||
{
|
||||
"script": {
|
||||
"test_script": {
|
||||
"description": "This is a test script",
|
||||
"sequence": [],
|
||||
"fields": {
|
||||
"beer": {"description": "Number of beers"},
|
||||
"wine": {},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
entry = MockConfigEntry(title=None)
|
||||
entry.add_to_hass(hass)
|
||||
device = device_registry.async_get_or_create(
|
||||
config_entry_id=entry.entry_id,
|
||||
connections={("test", "1234")},
|
||||
suggested_area="Test Area",
|
||||
)
|
||||
|
||||
# Expose one entity per tool-bearing domain so every built-in tool appears.
|
||||
for domain, object_id, name in (
|
||||
("light", "kitchen", "Kitchen"),
|
||||
("calendar", "personal", "Personal"),
|
||||
("todo", "shopping", "Shopping"),
|
||||
):
|
||||
created = entity_registry.async_get_or_create(
|
||||
domain,
|
||||
"test",
|
||||
f"mock-{object_id}",
|
||||
original_name=name,
|
||||
suggested_object_id=object_id,
|
||||
)
|
||||
hass.states.async_set(created.entity_id, "on", {"friendly_name": name})
|
||||
async_expose_entity(hass, "conversation", created.entity_id, True)
|
||||
|
||||
async_expose_entity(hass, "conversation", "script.test_script", True)
|
||||
async_register_timer_handler(hass, device.id, lambda *args: None)
|
||||
|
||||
llm_context = llm.LLMContext(
|
||||
platform="test_platform",
|
||||
context=Context(),
|
||||
language="*",
|
||||
assistant="conversation",
|
||||
device_id=device.id,
|
||||
)
|
||||
api = await llm.async_get_api(hass, "assist", llm_context)
|
||||
|
||||
assert api.api_prompt == snapshot(name="prompt")
|
||||
assert [
|
||||
{
|
||||
"name": tool.name,
|
||||
"description": tool.description,
|
||||
"parameters": _normalize_schema(
|
||||
convert(tool.parameters, custom_serializer=api.custom_serializer)
|
||||
),
|
||||
}
|
||||
for tool in api.tools
|
||||
] == snapshot(name="tools")
|
||||
|
||||
|
||||
async def test_assist_api_get_timer_tools(
|
||||
hass: HomeAssistant, llm_context: llm.LLMContext
|
||||
) -> None:
|
||||
|
||||
Reference in New Issue
Block a user