From 48185743a4d4948a985a5aa88aad4d4986678ec2 Mon Sep 17 00:00:00 2001 From: Franck Nijhof Date: Mon, 14 Sep 2026 21:59:12 +0200 Subject: [PATCH] Check custom integration requirements against Home Assistant in hassfest (#181913) Co-authored-by: Paulus Schoutsen Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- script/hassfest/docker.py | 2 +- script/hassfest/docker/Dockerfile | 2 +- script/hassfest/requirements.py | 191 +++++++++++++++++++++++++++- tests/hassfest/test_requirements.py | 164 ++++++++++++++++++++++++ 4 files changed, 354 insertions(+), 5 deletions(-) diff --git a/script/hassfest/docker.py b/script/hassfest/docker.py index cd3360541769..b0c1e1594de7 100644 --- a/script/hassfest/docker.py +++ b/script/hassfest/docker.py @@ -97,7 +97,7 @@ SHELL ["/bin/sh", "-o", "pipefail", "-c"] ENTRYPOINT ["/usr/src/homeassistant/script/hassfest/docker/entrypoint.sh"] WORKDIR "/github/workspace" -COPY --parents requirements.txt homeassistant/ script /usr/src/homeassistant/ +COPY --parents requirements.txt requirements_all.txt homeassistant/ script /usr/src/homeassistant/ # Uv creates a lock file in /tmp RUN --mount=type=tmpfs,target=/tmp \ diff --git a/script/hassfest/docker/Dockerfile b/script/hassfest/docker/Dockerfile index ccdd9446b90e..0d808eb8ae60 100644 --- a/script/hassfest/docker/Dockerfile +++ b/script/hassfest/docker/Dockerfile @@ -12,7 +12,7 @@ SHELL ["/bin/sh", "-o", "pipefail", "-c"] ENTRYPOINT ["/usr/src/homeassistant/script/hassfest/docker/entrypoint.sh"] WORKDIR "/github/workspace" -COPY --parents requirements.txt homeassistant/ script /usr/src/homeassistant/ +COPY --parents requirements.txt requirements_all.txt homeassistant/ script /usr/src/homeassistant/ # Uv creates a lock file in /tmp RUN --mount=type=tmpfs,target=/tmp \ diff --git a/script/hassfest/requirements.py b/script/hassfest/requirements.py index 5e38fd322ed1..057175bd16f7 100644 --- a/script/hassfest/requirements.py +++ b/script/hassfest/requirements.py @@ -2,16 +2,22 @@ from collections import deque from collections.abc import Collection +from contextlib import suppress from functools import cache from importlib.metadata import PackageMetadata, files, metadata import json import os +from pathlib import Path import re import subprocess import sys from typing import Any, TypedDict from awesomeversion import AwesomeVersion, AwesomeVersionStrategy +from packaging.requirements import InvalidRequirement, Requirement +from packaging.specifiers import SpecifierSet +from packaging.utils import canonicalize_name +from packaging.version import InvalidVersion, Version from tqdm import tqdm import homeassistant.util.package as pkg_util @@ -83,6 +89,27 @@ PACKAGE_CHECK_VERSION_RANGE_EXCEPTIONS: dict[str, dict[str, set[str]]] = { }, } +# Constraints use an impossible version to prohibit a package altogether. +PROHIBITED_VERSION = "1000000000.0.0" + +# Hassfest runs the Python version Home Assistant requires, but on a single +# platform. Markers are evaluated against every platform a requirement could +# land on, so one is only skipped when it can never be installed at all. +MARKER_ENVIRONMENTS = tuple( + { + "os_name": os_name, + "platform_machine": platform_machine, + "platform_system": platform_system, + "sys_platform": sys_platform, + } + for os_name, platform_system, sys_platform in ( + ("posix", "Linux", "linux"), + ("posix", "Darwin", "darwin"), + ("nt", "Windows", "win32"), + ) + for platform_machine in ("aarch64", "armv7l", "i686", "x86_64") +) + PACKAGE_REGEX = re.compile( r"^(?:--.+\s)?([-_,\.\w\d\[\]]+)(==|>=|<=|~=|!=|<|>|===)*(.*)$" ) @@ -370,7 +397,8 @@ def validate(integrations: dict[str, Integration], config: Config) -> None: # Check if we are doing format-only validation. if not config.requirements: for integration in integrations.values(): - validate_requirements_format(integration) + if validate_requirements_format(integration) and not integration.core: + validate_custom_requirements(integration, config) return # check for incompatible requirements @@ -378,7 +406,7 @@ def validate(integrations: dict[str, Integration], config: Config) -> None: disable_tqdm = bool(config.specific_integrations or os.environ.get("CI")) for integration in tqdm(integrations.values(), disable=disable_tqdm): - validate_requirements(integration) + validate_requirements(integration, config) def validate_requirements_format(integration: Integration) -> bool: @@ -431,11 +459,168 @@ def validate_requirements_format(integration: Integration) -> bool: return len(integration.errors) == start_errors -def validate_requirements(integration: Integration) -> None: +@cache +def _load_requirement_file(path: Path) -> dict[str, SpecifierSet]: + """Read a pip requirements file into a map of package name to version specifier.""" + requirements: dict[str, SpecifierSet] = {} + if not path.is_file(): + return requirements + + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + + # Skip comments and pip options such as "-r requirements.txt" + if not line or line.startswith(("#", "-")): + continue + + try: + requirement = Requirement(line) + except InvalidRequirement: + continue + + # These files can name a package more than once, each line narrows it. + package = canonicalize_name(requirement.name) + requirements[package] = ( + requirements.get(package, SpecifierSet()) & requirement.specifier + ) + + return requirements + + +def _probe_versions(*specifier_sets: SpecifierSet) -> set[Version]: + """Return the versions worth probing to compare specifier sets. + + A specifier set describes a union of version intervals, so a non-empty + intersection of two of them always contains a version that sits on, just + below, or just above one of the boundaries either of them mentions. The + release suffix covers boundaries that exclude their own pre and post + releases, such as ">1.0" not allowing "1.0.post0". + """ + versions = {Version("0")} + + for specifiers in specifier_sets: + for specifier in specifiers: + boundary = specifier.version.removesuffix(".*") + for suffix in ("", ".dev0", ".post0", ".0.0.1"): + with suppress(InvalidVersion): + versions.add(Version(f"{boundary}{suffix}")) + + return versions + + +def _specifiers_conflict(left: SpecifierSet, right: SpecifierSet) -> bool: + """Return if no single version can satisfy both specifier sets.""" + return not any( + left.contains(version, prereleases=True) + and right.contains(version, prereleases=True) + for version in _probe_versions(left, right) + ) + + +def validate_custom_requirements(integration: Integration, config: Config) -> bool: + """Validate a custom integration against the requirements of Home Assistant. + + Custom integrations are installed into the same Python environment as Home + Assistant itself. A requirement that rules out the version Home Assistant + needs takes the whole installation down with it. + + Returns if valid. + """ + if integration.core: + return True + + start_errors = len(integration.errors) + + core_requirements = _load_requirement_file(config.root / "requirements.txt") + all_requirements = _load_requirement_file(config.root / "requirements_all.txt") + constraints = _load_requirement_file( + config.root / "homeassistant/package_constraints.txt" + ) + + for req in integration.requirements: + try: + requirement = Requirement(req) + except InvalidRequirement: + continue + + if requirement.marker and not any( + requirement.marker.evaluate(environment) + for environment in MARKER_ENVIRONMENTS + ): + continue + + package = canonicalize_name(requirement.name) + + if package in core_requirements: + integration.add_error( + "requirements", + f"Requirement {req} is a dependency of Home Assistant itself and " + "must not be listed in the manifest of a custom integration.", + ) + continue + + pinned = all_requirements.get(package) + constraint = constraints.get(package) + + if constraint is not None and any( + specifier.version == PROHIBITED_VERSION for specifier in constraint + ): + integration.add_error( + "requirements", + f"Requirement {req} is prohibited by Home Assistant, " + f"{package} must not be installed.", + ) + continue + + if pinned is not None and _specifiers_conflict(requirement.specifier, pinned): + integration.add_error( + "requirements", + f"Requirement {req} is incompatible with {package}{pinned}, which " + "Home Assistant depends on.", + ) + continue + + if constraint is not None and _specifiers_conflict( + requirement.specifier, constraint + ): + integration.add_error( + "requirements", + f"Requirement {req} is incompatible with {package}{constraint}, " + "which Home Assistant's package constraints require.", + ) + continue + + # Pinning a package Home Assistant ships breaks the moment we bump it. + # Constrained packages are left alone, we only bound those. + if pinned is not None and ( + exact := sorted( + specifier.version + for specifier in requirement.specifier + if specifier.operator in ("==", "===") + and not specifier.version.endswith(".*") + ) + ): + suggestion = f"{package}>={exact[0]}" + integration.add_error( + "requirements", + f"Requirement {req} pins a package Home Assistant depends on " + f'({package}{pinned}). Use a minimum version ("{suggestion}") ' + "instead, so it can follow along when Home Assistant updates it.", + ) + + return len(integration.errors) == start_errors + + +def validate_requirements(integration: Integration, config: Config) -> None: """Validate requirements.""" if not validate_requirements_format(integration): return + # Installing a requirement we already rejected would downgrade the + # environment hassfest itself runs in. + if not validate_custom_requirements(integration, config): + return + integration_requirements = set() integration_packages = set() for req in integration.requirements: diff --git a/tests/hassfest/test_requirements.py b/tests/hassfest/test_requirements.py index 7f938ef666dd..d76d17f77ff0 100644 --- a/tests/hassfest/test_requirements.py +++ b/tests/hassfest/test_requirements.py @@ -12,9 +12,11 @@ from script.hassfest.requirements import ( FORBIDDEN_PACKAGE_NAMES, PACKAGE_CHECK_PREPARE_UPDATE, PACKAGE_CHECK_VERSION_RANGE, + _load_requirement_file, _packages_checked_files_cache, check_dependency_files, check_dependency_version_range, + validate_custom_requirements, validate_requirements_format, ) @@ -332,3 +334,165 @@ def test_check_dependency_file_names(integration: Integration) -> None: assert check_dependency_files(integration, package, pkg, ()) is True assert mock_files.call_count == 1 assert len(integration.errors) == 0 + + +@pytest.fixture +def core_config(tmp_path: Path) -> Generator[Config]: + """Fixture for a Config pointing at a stubbed Home Assistant checkout.""" + (tmp_path / "homeassistant").mkdir() + (tmp_path / "requirements.txt").write_text( + "# Home Assistant Core\n" + "-c homeassistant/package_constraints.txt\n" + "aiohttp==3.14.3\n" + ) + (tmp_path / "requirements_all.txt").write_text( + "-r requirements.txt\n\n# homeassistant.components.modbus\npymodbus==3.13.1\n" + ) + (tmp_path / "homeassistant" / "package_constraints.txt").write_text( + "pymodbus==3.13.1\n" + "aiofiles>=24.1.0\n" + "poetry==1000000000.0.0\n" + "tenacity!=8.4.0\n" + "auth0-python<5.0\n" + # Listed twice, as package_constraints.txt does for some packages + "dupe-package<2.0\n" + "dupe-package>=1.5\n" + ) + + _load_requirement_file.cache_clear() + yield Config( + root=tmp_path, + specific_integrations=None, + action="validate", + requirements=False, + ) + _load_requirement_file.cache_clear() + + +@pytest.fixture +def custom_integration(core_config: Config) -> Integration: + """Fixture for a custom integration validated against a stubbed core.""" + return Integration( + path=Path("custom_components/test").absolute(), + _config=core_config, + _manifest={ + "domain": "test", + "documentation": "https://example.com", + "name": "test", + "codeowners": ["@awesome"], + "requirements": [], + }, + ) + + +@pytest.mark.parametrize( + ("requirement", "error"), + [ + pytest.param( + "aiohttp==3.14.3", + "Requirement aiohttp==3.14.3 is a dependency of Home Assistant itself " + "and must not be listed in the manifest of a custom integration.", + id="core_dependency", + ), + pytest.param( + "pymodbus==3.6.2", + "Requirement pymodbus==3.6.2 is incompatible with pymodbus==3.13.1, " + "which Home Assistant depends on.", + id="pinned_below_core", + ), + pytest.param( + "pymodbus>=3.20.0", + "Requirement pymodbus>=3.20.0 is incompatible with pymodbus==3.13.1, " + "which Home Assistant depends on.", + id="minimum_above_core", + ), + pytest.param( + "pymodbus==3.13.1", + "Requirement pymodbus==3.13.1 pins a package Home Assistant depends on " + '(pymodbus==3.13.1). Use a minimum version ("pymodbus>=3.13.1") instead, ' + "so it can follow along when Home Assistant updates it.", + id="pinned_to_core_version", + ), + pytest.param( + "aiofiles<24.0.0", + "Requirement aiofiles<24.0.0 is incompatible with aiofiles>=24.1.0, " + "which Home Assistant's package constraints require.", + id="violates_package_constraint", + ), + pytest.param( + "poetry>=1", + "Requirement poetry>=1 is prohibited by Home Assistant, poetry must " + "not be installed.", + id="prohibited_package", + ), + pytest.param( + "pymodbus==3.6.2;platform_machine=='aarch64'", + "Requirement pymodbus==3.6.2;platform_machine=='aarch64' is " + "incompatible with pymodbus==3.13.1, which Home Assistant depends on.", + id="marker_applying_on_another_platform", + ), + pytest.param( + "tenacity==8.4.0", + "Requirement tenacity==8.4.0 is incompatible with tenacity!=8.4.0, " + "which Home Assistant's package constraints require.", + id="violates_excluded_version", + ), + pytest.param( + "dupe-package==3.0", + "Requirement dupe-package==3.0 is incompatible with " + "dupe-package<2.0,>=1.5, which Home Assistant's package constraints " + "require.", + id="violates_merged_constraints", + ), + ], +) +def test_validate_custom_requirements_invalid( + custom_integration: Integration, + core_config: Config, + requirement: str, + error: str, +) -> None: + """Test custom integration requirements that clash with Home Assistant.""" + custom_integration.manifest["requirements"] = [requirement] + + assert not validate_custom_requirements(custom_integration, core_config) + assert [x.error for x in custom_integration.errors] == [error] + + +@pytest.mark.parametrize( + "requirement", + [ + pytest.param("pymodbus>=3.10.0", id="minimum_below_core"), + pytest.param("pymodbus>=3.13.1", id="minimum_equal_to_core"), + pytest.param("aiofiles>=25.0.0", id="within_package_constraint"), + pytest.param("unknown-package==1.2.3", id="unknown_package"), + pytest.param("pymodbus==3.6.2;python_version<'3.0'", id="marker_not_applying"), + pytest.param("aiofiles>25,<25.0.1", id="range_excluding_own_boundaries"), + pytest.param("pymodbus>3.13.0,<4", id="range_around_core_version"), + pytest.param("pymodbus==3.13.*", id="wildcard_matching_core_version"), + pytest.param("pymodbus~=3.13.1", id="compatible_release"), + pytest.param("tenacity>8.4.0,<9", id="range_around_excluded_version"), + pytest.param("auth0-python==4.9.0", id="pinned_package_we_only_constrain"), + pytest.param("dupe-package==1.7", id="within_merged_constraints"), + pytest.param("git+https://github.com/user/project.git@1.2.3", id="git_url"), + ], +) +def test_validate_custom_requirements_valid( + custom_integration: Integration, core_config: Config, requirement: str +) -> None: + """Test custom integration requirements that Home Assistant is fine with.""" + custom_integration.manifest["requirements"] = [requirement] + + assert validate_custom_requirements(custom_integration, core_config) + assert not custom_integration.errors + + +def test_validate_custom_requirements_skips_core( + custom_integration: Integration, core_config: Config +) -> None: + """Test core integrations are exempt, they are what we validate against.""" + custom_integration.path = core_config.root / "homeassistant/components/modbus" + custom_integration.manifest["requirements"] = ["pymodbus==3.13.1"] + + assert validate_custom_requirements(custom_integration, core_config) + assert not custom_integration.errors