From c9b0e6b230772010441c0abfa8fee02cace23fe0 Mon Sep 17 00:00:00 2001 From: epenet <6771947+epenet@users.noreply.github.com> Date: Mon, 28 Sep 2026 20:07:14 +0200 Subject: [PATCH] Adjust CI to always include dependent integrations (#182182) Co-authored-by: Claude Opus 5 --- .core_files.yaml | 4 - .github/workflows/ci.yaml | 7 +- .gitignore | 3 + script/gen_integration_paths.py | 91 +++++++++++++++ tests/scripts/test_gen_integration_paths.py | 122 ++++++++++++++++++++ 5 files changed, 217 insertions(+), 10 deletions(-) create mode 100644 script/gen_integration_paths.py create mode 100644 tests/scripts/test_gen_integration_paths.py diff --git a/.core_files.yaml b/.core_files.yaml index 87c6a5c6e765..5380f7bcf526 100644 --- a/.core_files.yaml +++ b/.core_files.yaml @@ -124,13 +124,9 @@ components: &components - homeassistant/components/sun/** - homeassistant/components/system_health/** - homeassistant/components/tag/** - # telegram_bot is used by telegram - - homeassistant/components/telegram_bot/** - homeassistant/components/template/** - homeassistant/components/timer/** - homeassistant/components/trace/** - # twilio is used by twilio_call and twilio_sms - - homeassistant/components/twilio/** - homeassistant/components/usb/** - homeassistant/components/webhook/** - homeassistant/components/websocket_api/** diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 70d7a41300e1..d5071073bbf3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -132,12 +132,7 @@ jobs: filters: .core_files.yaml - name: Create a list of integrations to filter for changes run: | - integrations=$(ls -Ad ./homeassistant/components/[!_]* | xargs -n 1 basename) - touch .integration_paths.yaml - for integration in $integrations; do - echo "${integration}: [homeassistant/components/${integration}/**, tests/components/${integration}/**]" \ - >> .integration_paths.yaml; - done + python3 -m script.gen_integration_paths echo "Result:" cat .integration_paths.yaml - name: Filter for integration changes diff --git a/.gitignore b/.gitignore index 5fb2ad904d14..8cd02d4456d0 100644 --- a/.gitignore +++ b/.gitignore @@ -140,6 +140,9 @@ tmp_cache # Will be created from script/split_tests.py pytest_buckets.txt +# Will be created from script/gen_integration_paths.py +.integration_paths.yaml + # AI tooling .claude/settings.local.json .claude/worktrees/ diff --git a/script/gen_integration_paths.py b/script/gen_integration_paths.py new file mode 100644 index 000000000000..81080c4b541e --- /dev/null +++ b/script/gen_integration_paths.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Generate the paths-filter config used by CI to detect changed integrations. + +Each integration maps to its own source and test paths, plus the source paths of +its (transitive) dependencies. That way a change to a dependency, such as +`twilio`, also selects its dependents, `twilio_call` and `twilio_sms`, for +testing. + +Dependencies listed in .core_files.yaml are skipped, as any change to those +already triggers the full test suite. +""" + +import json +from pathlib import Path +import re + +COMPONENTS_DIR = Path("homeassistant/components") +CORE_FILES = Path(".core_files.yaml") +OUTPUT_FILE = Path(".integration_paths.yaml") + +CORE_COMPONENT_PATTERN = re.compile( + r"^\s*-\s*homeassistant/components/([a-z0-9_]+)/\*\*\s*$", re.MULTILINE +) + + +def get_core_integrations() -> set[str]: + """Return the integrations that trigger the full test suite.""" + return set(CORE_COMPONENT_PATTERN.findall(CORE_FILES.read_text())) + + +def get_dependencies() -> dict[str, list[str]]: + """Return the dependencies of every integration.""" + return { + manifest_path.parent.name: json.loads(manifest_path.read_text()).get( + "dependencies", [] + ) + for manifest_path in sorted(COMPONENTS_DIR.glob("*/manifest.json")) + } + + +def get_transitive_dependencies( + dependencies: dict[str, list[str]], +) -> dict[str, set[str]]: + """Return the transitive dependencies of every integration.""" + transitive: dict[str, set[str]] = {} + for integration, integration_dependencies in dependencies.items(): + found: set[str] = set() + queue = list(integration_dependencies) + while queue: + dependency = queue.pop() + if dependency in found or dependency == integration: + continue + if dependency not in dependencies: + # hassfest rejects unknown dependencies: this only guards + # against a KeyError on a branch that hasn't been validated + continue + found.add(dependency) + queue.extend(dependencies[dependency]) + transitive[integration] = found + return transitive + + +def generate() -> str: + """Generate the paths-filter config.""" + core_integrations = get_core_integrations() + dependencies = get_dependencies() + transitive = get_transitive_dependencies(dependencies) + + lines: list[str] = [] + for integration, integration_dependencies in transitive.items(): + paths = [ + f"homeassistant/components/{integration}/**", + f"tests/components/{integration}/**", + ] + # Only source changes of a dependency can affect this integration, its + # tests cannot. + paths.extend( + f"homeassistant/components/{dependency}/**" + for dependency in sorted(integration_dependencies - core_integrations) + ) + lines.append(f"{integration}: [{', '.join(paths)}]") + return "\n".join(lines) + "\n" + + +def main() -> None: + """Write the paths-filter config.""" + OUTPUT_FILE.write_text(generate()) + + +if __name__ == "__main__": + main() diff --git a/tests/scripts/test_gen_integration_paths.py b/tests/scripts/test_gen_integration_paths.py new file mode 100644 index 000000000000..d09c24917e00 --- /dev/null +++ b/tests/scripts/test_gen_integration_paths.py @@ -0,0 +1,122 @@ +"""Test the gen_integration_paths script.""" + +from collections.abc import Generator +import json +from pathlib import Path +from unittest.mock import patch + +import pytest + +from script import gen_integration_paths +from script.gen_integration_paths import ( + generate, + get_core_integrations, + get_transitive_dependencies, + main, +) + +CORE_FILES = """ +base_platforms: &base_platforms + - homeassistant/components/sensor/** + +components: &components + # http is used by a lot of integrations + - homeassistant/components/http/** +""" + + +@pytest.fixture +def components_dir(tmp_path: Path) -> Generator[Path]: + """Create integrations with a dependency on each other.""" + dependencies = { + "http": [], + "twilio": ["http"], + "twilio_call": ["twilio"], + "twilio_sms": ["twilio"], + } + components_dir = tmp_path / "homeassistant" / "components" + for integration, integration_dependencies in dependencies.items(): + manifest_path = components_dir / integration / "manifest.json" + manifest_path.parent.mkdir(parents=True) + manifest_path.write_text( + json.dumps( + {"domain": integration, "dependencies": integration_dependencies} + ) + ) + with patch.object(gen_integration_paths, "COMPONENTS_DIR", components_dir): + yield components_dir + + +@pytest.fixture +def core_files(tmp_path: Path) -> Generator[Path]: + """Create a core files config.""" + core_files = tmp_path / ".core_files.yaml" + core_files.write_text(CORE_FILES) + with patch.object(gen_integration_paths, "CORE_FILES", core_files): + yield core_files + + +@pytest.mark.usefixtures("core_files") +def test_get_core_integrations() -> None: + """Test that both base platforms and components are picked up.""" + assert get_core_integrations() == {"http", "sensor"} + + +@pytest.mark.parametrize( + ("dependencies", "expected"), + [ + pytest.param( + {"a": ["b"], "b": [], "c": []}, + {"a": {"b"}, "b": set(), "c": set()}, + id="direct", + ), + pytest.param( + {"a": ["b"], "b": ["c"], "c": []}, + {"a": {"b", "c"}, "b": {"c"}, "c": set()}, + id="transitive", + ), + pytest.param( + {"a": ["b"], "b": ["a"]}, + {"a": {"b"}, "b": {"a"}}, + id="cycle", + ), + pytest.param( + {"a": ["a"]}, + {"a": set()}, + id="self", + ), + pytest.param( + {"a": ["missing"]}, + {"a": set()}, + id="unknown-dependency", + ), + ], +) +def test_get_transitive_dependencies( + dependencies: dict[str, list[str]], expected: dict[str, set[str]] +) -> None: + """Test that dependencies are resolved recursively.""" + assert get_transitive_dependencies(dependencies) == expected + + +@pytest.mark.usefixtures("components_dir", "core_files") +def test_generate() -> None: + """Test that dependencies are added, except the ones triggering a full run.""" + assert generate() == ( + "http: [homeassistant/components/http/**, tests/components/http/**]\n" + "twilio: [homeassistant/components/twilio/**, tests/components/twilio/**]\n" + "twilio_call: [homeassistant/components/twilio_call/**, " + "tests/components/twilio_call/**, homeassistant/components/twilio/**]\n" + "twilio_sms: [homeassistant/components/twilio_sms/**, " + "tests/components/twilio_sms/**, homeassistant/components/twilio/**]\n" + ) + + +@pytest.mark.usefixtures("components_dir", "core_files") +def test_main(tmp_path: Path) -> None: + """Test that the config is written to the output file.""" + output_file = tmp_path / ".integration_paths.yaml" + with patch.object(gen_integration_paths, "OUTPUT_FILE", output_file): + main() + + assert output_file.read_text() == generate()