Adjust CI to always include dependent integrations (#182182)

Co-authored-by: Claude Opus 5 <noreply@anthropic.com>

.core_files.yaml

@@ -124,13 +124,9 @@ components: &components
   - homeassistant/components/sun/**
   - homeassistant/components/system_health/**
   - homeassistant/components/tag/**
-  # telegram_bot is used by telegram
-  - homeassistant/components/telegram_bot/**
   - homeassistant/components/template/**
   - homeassistant/components/timer/**
   - homeassistant/components/trace/**
-  # twilio is used by twilio_call and twilio_sms
-  - homeassistant/components/twilio/**
   - homeassistant/components/usb/**
   - homeassistant/components/webhook/**
   - homeassistant/components/websocket_api/**

.github/workflows/ci.yaml

@@ -132,12 +132,7 @@ jobs:
           filters: .core_files.yaml
       - name: Create a list of integrations to filter for changes
         run: |
-          integrations=$(ls -Ad ./homeassistant/components/[!_]*  | xargs -n 1 basename)
-          touch .integration_paths.yaml
-          for integration in $integrations; do
-            echo "${integration}: [homeassistant/components/${integration}/**, tests/components/${integration}/**]" \
-              >> .integration_paths.yaml;
-          done
+          python3 -m script.gen_integration_paths
           echo "Result:"
           cat .integration_paths.yaml
       - name: Filter for integration changes

.gitignore

@@ -140,6 +140,9 @@ tmp_cache
 # Will be created from script/split_tests.py
 pytest_buckets.txt
 
+# Will be created from script/gen_integration_paths.py
+.integration_paths.yaml
+
 # AI tooling
 .claude/settings.local.json
 .claude/worktrees/

script/gen_integration_paths.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+"""Generate the paths-filter config used by CI to detect changed integrations.
+
+Each integration maps to its own source and test paths, plus the source paths of
+its (transitive) dependencies. That way a change to a dependency, such as
+`twilio`, also selects its dependents, `twilio_call` and `twilio_sms`, for
+testing.
+
+Dependencies listed in .core_files.yaml are skipped, as any change to those
+already triggers the full test suite.
+"""
+
+import json
+from pathlib import Path
+import re
+
+COMPONENTS_DIR = Path("homeassistant/components")
+CORE_FILES = Path(".core_files.yaml")
+OUTPUT_FILE = Path(".integration_paths.yaml")
+
+CORE_COMPONENT_PATTERN = re.compile(
+    r"^\s*-\s*homeassistant/components/([a-z0-9_]+)/\*\*\s*$", re.MULTILINE
+)
+
+
+def get_core_integrations() -> set[str]:
+    """Return the integrations that trigger the full test suite."""
+    return set(CORE_COMPONENT_PATTERN.findall(CORE_FILES.read_text()))
+
+
+def get_dependencies() -> dict[str, list[str]]:
+    """Return the dependencies of every integration."""
+    return {
+        manifest_path.parent.name: json.loads(manifest_path.read_text()).get(
+            "dependencies", []
+        )
+        for manifest_path in sorted(COMPONENTS_DIR.glob("*/manifest.json"))
+    }
+
+
+def get_transitive_dependencies(
+    dependencies: dict[str, list[str]],
+) -> dict[str, set[str]]:
+    """Return the transitive dependencies of every integration."""
+    transitive: dict[str, set[str]] = {}
+    for integration, integration_dependencies in dependencies.items():
+        found: set[str] = set()
+        queue = list(integration_dependencies)
+        while queue:
+            dependency = queue.pop()
+            if dependency in found or dependency == integration:
+                continue
+            if dependency not in dependencies:
+                # hassfest rejects unknown dependencies: this only guards
+                # against a KeyError on a branch that hasn't been validated
+                continue
+            found.add(dependency)
+            queue.extend(dependencies[dependency])
+        transitive[integration] = found
+    return transitive
+
+
+def generate() -> str:
+    """Generate the paths-filter config."""
+    core_integrations = get_core_integrations()
+    dependencies = get_dependencies()
+    transitive = get_transitive_dependencies(dependencies)
+
+    lines: list[str] = []
+    for integration, integration_dependencies in transitive.items():
+        paths = [
+            f"homeassistant/components/{integration}/**",
+            f"tests/components/{integration}/**",
+        ]
+        # Only source changes of a dependency can affect this integration, its
+        # tests cannot.
+        paths.extend(
+            f"homeassistant/components/{dependency}/**"
+            for dependency in sorted(integration_dependencies - core_integrations)
+        )
+        lines.append(f"{integration}: [{', '.join(paths)}]")
+    return "\n".join(lines) + "\n"
+
+
+def main() -> None:
+    """Write the paths-filter config."""
+    OUTPUT_FILE.write_text(generate())
+
+
+if __name__ == "__main__":
+    main()

tests/scripts/test_gen_integration_paths.py

@@ -0,0 +1,122 @@
+"""Test the gen_integration_paths script."""
+
+from collections.abc import Generator
+import json
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+from script import gen_integration_paths
+from script.gen_integration_paths import (
+    generate,
+    get_core_integrations,
+    get_transitive_dependencies,
+    main,
+)
+
+CORE_FILES = """
+base_platforms: &base_platforms
+  - homeassistant/components/sensor/**
+
+components: &components
+  # http is used by a lot of integrations
+  - homeassistant/components/http/**
+"""
+
+
+@pytest.fixture
+def components_dir(tmp_path: Path) -> Generator[Path]:
+    """Create integrations with a dependency on each other."""
+    dependencies = {
+        "http": [],
+        "twilio": ["http"],
+        "twilio_call": ["twilio"],
+        "twilio_sms": ["twilio"],
+    }
+    components_dir = tmp_path / "homeassistant" / "components"
+    for integration, integration_dependencies in dependencies.items():
+        manifest_path = components_dir / integration / "manifest.json"
+        manifest_path.parent.mkdir(parents=True)
+        manifest_path.write_text(
+            json.dumps(
+                {"domain": integration, "dependencies": integration_dependencies}
+            )
+        )
+    with patch.object(gen_integration_paths, "COMPONENTS_DIR", components_dir):
+        yield components_dir
+
+
+@pytest.fixture
+def core_files(tmp_path: Path) -> Generator[Path]:
+    """Create a core files config."""
+    core_files = tmp_path / ".core_files.yaml"
+    core_files.write_text(CORE_FILES)
+    with patch.object(gen_integration_paths, "CORE_FILES", core_files):
+        yield core_files
+
+
+@pytest.mark.usefixtures("core_files")
+def test_get_core_integrations() -> None:
+    """Test that both base platforms and components are picked up."""
+    assert get_core_integrations() == {"http", "sensor"}
+
+
+@pytest.mark.parametrize(
+    ("dependencies", "expected"),
+    [
+        pytest.param(
+            {"a": ["b"], "b": [], "c": []},
+            {"a": {"b"}, "b": set(), "c": set()},
+            id="direct",
+        ),
+        pytest.param(
+            {"a": ["b"], "b": ["c"], "c": []},
+            {"a": {"b", "c"}, "b": {"c"}, "c": set()},
+            id="transitive",
+        ),
+        pytest.param(
+            {"a": ["b"], "b": ["a"]},
+            {"a": {"b"}, "b": {"a"}},
+            id="cycle",
+        ),
+        pytest.param(
+            {"a": ["a"]},
+            {"a": set()},
+            id="self",
+        ),
+        pytest.param(
+            {"a": ["missing"]},
+            {"a": set()},
+            id="unknown-dependency",
+        ),
+    ],
+)
+def test_get_transitive_dependencies(
+    dependencies: dict[str, list[str]], expected: dict[str, set[str]]
+) -> None:
+    """Test that dependencies are resolved recursively."""
+    assert get_transitive_dependencies(dependencies) == expected
+
+
+@pytest.mark.usefixtures("components_dir", "core_files")
+def test_generate() -> None:
+    """Test that dependencies are added, except the ones triggering a full run."""
+    assert generate() == (
+        "http: [homeassistant/components/http/**, tests/components/http/**]\n"
+        "twilio: [homeassistant/components/twilio/**, tests/components/twilio/**]\n"
+        "twilio_call: [homeassistant/components/twilio_call/**, "
+        "tests/components/twilio_call/**, homeassistant/components/twilio/**]\n"
+        "twilio_sms: [homeassistant/components/twilio_sms/**, "
+        "tests/components/twilio_sms/**, homeassistant/components/twilio/**]\n"
+    )
+
+
+@pytest.mark.usefixtures("components_dir", "core_files")
+def test_main(tmp_path: Path) -> None:
+    """Test that the config is written to the output file."""
+    output_file = tmp_path / ".integration_paths.yaml"
+    with patch.object(gen_integration_paths, "OUTPUT_FILE", output_file):
+        main()
+
+    assert output_file.read_text() == generate()