diff --git a/.core_files.yaml b/.core_files.yaml index 72bbba323336..87c6a5c6e765 100644 --- a/.core_files.yaml +++ b/.core_files.yaml @@ -11,6 +11,12 @@ core: &core - requirements.txt - setup.cfg +# Performance benchmark suite (CodSpeed). Only gates the benchmark job; kept out +# of the `any` aggregate below so it does not pull in the full test suite. +benchmarks: &benchmarks + - benchmarks/** + - requirements_test.txt + # Our base platforms, that are used by other integrations base_platforms: &base_platforms - homeassistant/components/ai_task/** diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d17c64fb493a..6b1267f19b1f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -741,6 +741,47 @@ jobs: python --version mypy --num-workers=4 $(printf "homeassistant/components/%s " ${INTEGRATIONS_GLOB}) + benchmarks: + name: Run benchmarks + runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write # OIDC token CodSpeed mints (no CODSPEED_TOKEN secret) + needs: + - info + - base + # Run only when core code or the benchmark suite itself changed. Skipped on + # forks, where the OIDC token CodSpeed needs is unavailable. Pushes to dev + # that touch core refresh the CodSpeed baseline. + if: >- + needs.info.outputs.lint_only != 'true' + && (github.event_name != 'pull_request' + || !github.event.pull_request.head.repo.fork) + && (contains(fromJSON(needs.info.outputs.core), 'core') + || contains(fromJSON(needs.info.outputs.core), 'benchmarks')) + steps: + - name: Check out code from GitHub + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - name: Set up Python and restore venv + id: python + uses: $/.github/actions/restore-or-build-venv + with: + uv-version: ${{ needs.info.outputs.uv_version }} + python-version: ${{ needs.info.outputs.default_python }} + python-cache-key: ${{ needs.info.outputs.python_cache_key }} + uv-cache-dir: ${{ env.UV_CACHE_DIR }} + apt-cache-version: ${{ env.APT_CACHE_VERSION }} + - name: Run benchmarks + uses: CodSpeedHQ/action@a4a36bb07c0638b0b4ca52bf1f3dad1b4289e52f # v4.18.1 + with: + mode: simulation,memory + # No token: auth uses the OIDC id-token minted by the job permissions. + run: | + . venv/bin/activate + pytest benchmarks --codspeed --no-cov -o addopts="" + prepare-pytest-full: name: Split tests for full run runs-on: ubuntu-24.04 diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 000000000000..df0fb56caaae --- /dev/null +++ b/benchmarks/__init__.py @@ -0,0 +1 @@ +"""CodSpeed performance benchmarks for Home Assistant core hot paths.""" diff --git a/benchmarks/conftest.py b/benchmarks/conftest.py new file mode 100644 index 000000000000..c83e7ed57239 --- /dev/null +++ b/benchmarks/conftest.py @@ -0,0 +1,45 @@ +"""Shared fixtures for the CodSpeed benchmark suite. + +These benchmarks live outside ``tests`` on purpose: ``testpaths`` only points at +``tests``, so the regular suite never collects them. CodSpeed runs them with +``pytest benchmarks --codspeed`` and tracks the results per pull request. +""" + +from collections.abc import AsyncGenerator, Callable + +import pytest + +from homeassistant.core import HomeAssistant +from tests.common import async_test_home_assistant + + +@pytest.fixture +async def hass() -> AsyncGenerator[HomeAssistant]: + """Return a running Home Assistant instance for benchmarking. + + Most hot paths under test (``async_fire``, ``async_set``, ``async_render``) + are ``@callback`` methods, so the benchmark fixture can drive them + synchronously from within the running loop. + """ + async with async_test_home_assistant() as hass: + yield hass + + +@pytest.fixture +def populate_states(hass: HomeAssistant) -> Callable[[int], None]: + """Return a helper that fills the state machine with ``count`` sensors. + + Used by the scaling benchmarks to measure a path at several sizes, so an + algorithmic regression shows up as the curve bending instead of hiding + behind a single constant-factor number. + """ + + def _populate(count: int) -> None: + for index in range(count): + hass.states.async_set( + f"sensor.bench_{index}", + str(index), + {"friendly_name": f"Bench {index}", "unit_of_measurement": "W"}, + ) + + return _populate diff --git a/benchmarks/test_core_events.py b/benchmarks/test_core_events.py new file mode 100644 index 000000000000..a12dddb36935 --- /dev/null +++ b/benchmarks/test_core_events.py @@ -0,0 +1,186 @@ +"""CodSpeed benchmarks for the event bus and event helpers. + +The event bus carries every state change, and ``async_track_state_change_event`` +is the routing layer almost every automation, template and trigger sits on. A +regression in either is felt across the whole system. + +Run locally with: ``pytest benchmarks --codspeed``. +""" + +from collections.abc import Callable + +import pytest +from pytest_codspeed import BenchmarkFixture + +from homeassistant.core import Event, HomeAssistant, callback +from homeassistant.helpers.dispatcher import ( + async_dispatcher_connect, + async_dispatcher_send, +) +from homeassistant.helpers.event import async_call_later, async_track_state_change_event + + +@callback +def _noop(event: Event) -> None: + """Do nothing, cheaply.""" + + +def test_event_fire_no_listeners( + benchmark: BenchmarkFixture, hass: HomeAssistant +) -> None: + """Fire an event nobody listens to (the bare dispatch cost).""" + benchmark(lambda: hass.bus.async_fire("benchmark_event", {"value": 1})) + + +@pytest.mark.parametrize("listeners", [1, 10]) +def test_event_fire_callbacks( + benchmark: BenchmarkFixture, hass: HomeAssistant, listeners: int +) -> None: + """Fire an event with N callback listeners that run inline.""" + fired = 0 + + @callback + def listener(event: Event) -> None: + nonlocal fired + fired += 1 + + for _ in range(listeners): + hass.bus.async_listen("benchmark_event", listener) + + benchmark(lambda: hass.bus.async_fire("benchmark_event", {"value": 1})) + + # Each fire increments `fired` once per listener, but the benchmark may + # invoke the target multiple times (e.g. in walltime mode), so only the + # per-fire multiple is a stable invariant. + assert fired > 0 + assert fired % listeners == 0 + + +def test_event_fire_filtered_reject( + benchmark: BenchmarkFixture, hass: HomeAssistant +) -> None: + """Fire an event whose listener is gated out by an event_filter. + + The filter runs but the listener does not, so this isolates the filter + short-circuit cost from the listener body. + """ + fired = 0 + + @callback + def listener(event: Event) -> None: + nonlocal fired + fired += 1 + + @callback + def event_filter(event_data: dict) -> bool: + return False + + hass.bus.async_listen("benchmark_event", listener, event_filter=event_filter) + + benchmark(lambda: hass.bus.async_fire("benchmark_event", {"value": 1})) + + assert fired == 0 + + +def test_state_change_tracked(benchmark: BenchmarkFixture, hass: HomeAssistant) -> None: + """Fire a state change routed to a tracked entity's listener. + + This is the real automation hot path: ``async_set`` fires + EVENT_STATE_CHANGED, the dispatcher does a dict lookup on the entity_id and + runs the inline callback. + """ + fired = 0 + + @callback + def listener(event: Event) -> None: + nonlocal fired + fired += 1 + + async_track_state_change_event(hass, "sensor.tracked", listener) + counter = 0 + + def _set() -> None: + nonlocal counter + counter += 1 + hass.states.async_set("sensor.tracked", str(counter)) + + benchmark(_set) + + assert fired + + +def test_state_change_untracked( + benchmark: BenchmarkFixture, hass: HomeAssistant +) -> None: + """Fire a state change for an entity nobody tracks (the dict-miss path). + + Tracking is installed for a different entity, so the dispatcher's lookup + misses and returns fast. This is the common case on a busy bus. + """ + async_track_state_change_event(hass, "sensor.tracked", _noop) + counter = 0 + + def _set() -> None: + nonlocal counter + counter += 1 + hass.states.async_set("sensor.untracked", str(counter)) + + benchmark(_set) + + +def test_dispatcher_send_no_receivers( + benchmark: BenchmarkFixture, hass: HomeAssistant +) -> None: + """Send a dispatcher signal with nobody connected (the bare dispatch cost).""" + benchmark(lambda: async_dispatcher_send(hass, "benchmark_signal", 1)) + + +@pytest.mark.parametrize("receivers", [1, 10]) +def test_dispatcher_send( + benchmark: BenchmarkFixture, hass: HomeAssistant, receivers: int +) -> None: + """Send a dispatcher signal to N connected receivers.""" + fired = 0 + + def _make_receiver() -> Callable[..., None]: + @callback + def receiver(*args: object) -> None: + nonlocal fired + fired += 1 + + return receiver + + # async_dispatcher_connect keys receivers by the callable itself, so each + # connection needs its own object to actually register as a receiver. + for _ in range(receivers): + async_dispatcher_connect(hass, "benchmark_signal", _make_receiver()) + + benchmark(lambda: async_dispatcher_send(hass, "benchmark_signal", 1)) + + # Each send increments `fired` once per receiver, but the benchmark may + # invoke the target multiple times (e.g. in walltime mode), so only the + # per-send multiple is a stable invariant. + assert fired > 0 + assert fired % receivers == 0 + + +def test_call_later_schedule(benchmark: BenchmarkFixture, hass: HomeAssistant) -> None: + """Schedule a delayed callback and cancel it (the timer-tracking cost). + + Cancelling inside the measured call keeps timers from piling up on the loop + across iterations. + """ + called = 0 + + @callback + def listener() -> None: + nonlocal called + called += 1 + + def _schedule() -> None: + cancel: Callable[[], None] = async_call_later(hass, 60, listener) + cancel() + + benchmark(_schedule) + + assert called == 0 diff --git a/benchmarks/test_core_states.py b/benchmarks/test_core_states.py new file mode 100644 index 000000000000..f6e390808f77 --- /dev/null +++ b/benchmarks/test_core_states.py @@ -0,0 +1,139 @@ +"""CodSpeed benchmarks for the state machine and entity write path. + +Every state update in Home Assistant flows through ``StateMachine.async_set``, +and every entity that pushes an update lands in ``Entity._async_write_ha_state``. +These are among the busiest call sites in the whole process. + +Run locally with: ``pytest benchmarks --codspeed``. +""" + +from collections.abc import Callable +from typing import Any + +import pytest +from pytest_codspeed import BenchmarkFixture + +from homeassistant.core import HomeAssistant, State +from homeassistant.helpers.entity import Entity + +_ATTRS = { + "friendly_name": "Benchmark light", + "brightness": 255, + "color_temp_kelvin": 4000, + "supported_color_modes": ["color_temp", "rgb"], +} + + +def test_state_set_create(benchmark: BenchmarkFixture, hass: HomeAssistant) -> None: + """Write a brand new entity into the state machine (the create branch). + + ``pedantic`` with a teardown that removes the entity keeps every measured + call on the create path; without it only the first call creates and the rest + measure the update path. + """ + benchmark.pedantic( + lambda: hass.states.async_set("light.benchmark", "on", _ATTRS), + teardown=lambda: hass.states.async_remove("light.benchmark"), + rounds=1000, + ) + + +def test_state_set_update(benchmark: BenchmarkFixture, hass: HomeAssistant) -> None: + """Write a changed state for an existing entity (the update branch).""" + counter = 0 + + def _set() -> None: + nonlocal counter + counter += 1 + hass.states.async_set("light.benchmark", str(counter), _ATTRS) + + benchmark(_set) + + assert hass.states.get("light.benchmark") is not None + + +def test_state_set_report(benchmark: BenchmarkFixture, hass: HomeAssistant) -> None: + """Re-set an unchanged state (the EVENT_STATE_REPORTED fast path). + + Polling integrations hammer this branch: same value, same attributes, over + and over. It fires a lightweight reported event instead of a state change. + """ + hass.states.async_set("sensor.benchmark", "21.5", _ATTRS) + + benchmark(lambda: hass.states.async_set("sensor.benchmark", "21.5", _ATTRS)) + + +def test_state_get(benchmark: BenchmarkFixture, hass: HomeAssistant) -> None: + """Look a single state up by entity_id.""" + hass.states.async_set("sensor.benchmark", "21.5", _ATTRS) + + state: State | None = benchmark(lambda: hass.states.get("sensor.benchmark")) + + assert state is not None + assert state.state == "21.5" + + +@pytest.mark.parametrize("count", [10, 100, 1000]) +def test_state_all( + benchmark: BenchmarkFixture, + populate_states: Callable[[int], None], + hass: HomeAssistant, + count: int, +) -> None: + """Read the full state list out of a populated machine, at several sizes.""" + populate_states(count) + + states: list[State] = benchmark(hass.states.async_all) + + assert len(states) == count + + +@pytest.mark.parametrize("count", [10, 100, 1000]) +def test_state_entity_ids( + benchmark: BenchmarkFixture, + populate_states: Callable[[int], None], + hass: HomeAssistant, + count: int, +) -> None: + """List entity ids out of a populated machine, at several sizes.""" + populate_states(count) + + entity_ids = benchmark(hass.states.async_entity_ids) + + assert len(entity_ids) == count + + +class _BenchmarkEntity(Entity): + """A minimal entity carrying capability and extra state attributes.""" + + _attr_should_poll = False + _attr_name = "Benchmark" + _attr_supported_features = 3 + + def __init__(self, state: str) -> None: + """Initialize the benchmark entity.""" + self._attr_state = state + self._attr_extra_state_attributes: dict[str, Any] = { + "brightness": 255, + "color_temp_kelvin": 4000, + } + + @property + def capability_attributes(self) -> dict[str, Any]: + """Return capability attributes, assembled on every write.""" + return {"supported_color_modes": ["color_temp", "rgb"]} + + +def test_entity_write(benchmark: BenchmarkFixture, hass: HomeAssistant) -> None: + """Write an entity's state through the entity layer. + + This measures ``__async_calculate_state`` (assembling state and attributes + from the entity's properties) plus the ``async_set`` it lands in. + """ + entity = _BenchmarkEntity("on") + entity.hass = hass + entity.entity_id = "light.benchmark" + + benchmark(entity._async_write_ha_state) # noqa: SLF001 + + assert hass.states.get("light.benchmark") is not None diff --git a/benchmarks/test_core_template.py b/benchmarks/test_core_template.py new file mode 100644 index 000000000000..08bba389f761 --- /dev/null +++ b/benchmarks/test_core_template.py @@ -0,0 +1,112 @@ +"""CodSpeed benchmarks for the template engine. + +Templates render on dashboards, in automations and in many entity attributes. +Both the compile step and the warm render matter, and templates that walk the +state machine scale with the number of entities. + +Run locally with: ``pytest benchmarks --codspeed``. +""" + +from collections.abc import Callable + +import pytest +from pytest_codspeed import BenchmarkFixture + +from homeassistant.core import HomeAssistant +from homeassistant.helpers.template import Template + +_STATES_TEMPLATE = ( + "{{ (states('sensor.power') | float / 1000) | round(2) }} kW " + "{{ is_state('binary_sensor.motion', 'on') }} " + "{{ state_attr('sensor.power', 'unit_of_measurement') }}" +) + + +def test_template_compile(benchmark: BenchmarkFixture, hass: HomeAssistant) -> None: + """Compile a template from source (parse plus codegen). + + A unique source on every call (a varying Jinja comment) keeps the + environment's template cache missing, so each run actually compiles instead + of returning cached bytecode. The comment is stripped during compilation, so + the cost matches the real template. + """ + counter = 0 + + def _compile() -> Template: + nonlocal counter + counter += 1 + template = Template(f"{{# {counter} #}}{_STATES_TEMPLATE}", hass) + template.ensure_valid() + return template + + template = benchmark(_compile) + + assert template.is_static is False + + +def test_template_render_simple( + benchmark: BenchmarkFixture, hass: HomeAssistant +) -> None: + """Render a pure-math template (warm), the engine's baseline overhead.""" + template = Template("{{ 1 + 1 }}", hass) + template.ensure_valid() + + result = benchmark(template.async_render) + + assert result == 2 + + +def test_template_render_states( + benchmark: BenchmarkFixture, hass: HomeAssistant +) -> None: + """Render a template that reads states, attributes and a filter (warm).""" + hass.states.async_set("sensor.power", "1200", {"unit_of_measurement": "W"}) + hass.states.async_set("binary_sensor.motion", "on") + template = Template(_STATES_TEMPLATE, hass) + template.ensure_valid() + + result = benchmark(template.async_render) + + assert result.startswith("1.2 kW") + + +def test_template_render_to_info( + benchmark: BenchmarkFixture, hass: HomeAssistant +) -> None: + """Render and collect the entity dependency filter (the tracking path). + + ``async_render_to_info`` is what template triggers and template entities use + to learn which entities to subscribe to. + """ + hass.states.async_set("sensor.power", "1200", {"unit_of_measurement": "W"}) + hass.states.async_set("binary_sensor.motion", "on") + template = Template(_STATES_TEMPLATE, hass) + template.ensure_valid() + + info = benchmark(template.async_render_to_info) + + assert info.entities or info.all_states + + +@pytest.mark.parametrize("count", [10, 100, 1000]) +def test_template_iterate_states( + benchmark: BenchmarkFixture, + populate_states: Callable[[int], None], + hass: HomeAssistant, + count: int, +) -> None: + """Render a template that walks every sensor state, at several sizes. + + This is where an O(n) template touches an O(n) state machine; the cost + should grow linearly and a worse-than-linear regression should stand out. + """ + populate_states(count) + template = Template( + "{{ states.sensor | selectattr('state', 'eq', '1') | list | count }}", + hass, + ) + template.ensure_valid() + + result = benchmark(template.async_render) + + assert result == 1 diff --git a/pyproject.toml b/pyproject.toml index 932ea28644fc..d97ebf1101ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -438,6 +438,13 @@ runtime-typing = false [tool.pylint.CODE_STYLE] max-line-length-suggestions = 72 +[tool.uv] +# surepy (a surepetcare dependency) carries a stale upper pin on rich (<11) that +# blocks pytest-codspeed (needs rich>=13.8.1) from resolving in the same +# environment. rich keeps the Console API surepy uses backwards compatible, so +# override the pin rather than holding the benchmark tooling back. +override-dependencies = ["rich>=13.8.1"] + [tool.pytest.ini_options] pythonpath = ["pylint/plugins"] testpaths = ["tests"] @@ -876,6 +883,9 @@ split-on-trailing-comma = false "homeassistant/components/*/*/*" = ["TID252"] "tests/components/*/*/*" = ["TID252"] +# Benchmarks reuse the test helpers to spin up a Home Assistant instance +"benchmarks/**" = ["TID251"] + # Temporary "homeassistant/**" = ["PTH"] "tests/**" = ["PTH"] diff --git a/requirements_test.txt b/requirements_test.txt index 831b19fda925..d444d946292e 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -25,6 +25,7 @@ pylint-per-file-ignores==3.2.1 pipdeptree==4.2.2 pytest-asyncio==1.4.0 pytest-aiohttp==1.1.1 +pytest-codspeed==5.0.3 pytest-cov==7.1.0 pytest-freezer==0.4.9 pytest-github-actions-annotate-failures==0.4.2