Trigger recovery mode on registry major version downgrade (#164340)

homeassistant/bootstrap.py

@@ -70,7 +70,7 @@ from .const import (
     SIGNAL_BOOTSTRAP_INTEGRATIONS,
 )
 from .core_config import async_process_ha_core_config
-from .exceptions import HomeAssistantError
+from .exceptions import HomeAssistantError, UnsupportedStorageVersionError
 from .helpers import (
     area_registry,
     category_registry,
@@ -433,32 +433,56 @@ def _init_blocking_io_modules_in_executor() -> None:
     is_docker_env()
 
 
-async def async_load_base_functionality(hass: core.HomeAssistant) -> None:
-    """Load the registries and modules that will do blocking I/O."""
+async def async_load_base_functionality(hass: core.HomeAssistant) -> bool:
+    """Load the registries and modules that will do blocking I/O.
+
+    Return whether loading succeeded.
+    """
     if DATA_REGISTRIES_LOADED in hass.data:
-        return
+        return True
+
     hass.data[DATA_REGISTRIES_LOADED] = None
     entity.async_setup(hass)
     frame.async_setup(hass)
     template.async_setup(hass)
     translation.async_setup(hass)
-    await asyncio.gather(
-        create_eager_task(get_internal_store_manager(hass).async_initialize()),
-        create_eager_task(area_registry.async_load(hass)),
-        create_eager_task(category_registry.async_load(hass)),
-        create_eager_task(device_registry.async_load(hass)),
-        create_eager_task(entity_registry.async_load(hass)),
-        create_eager_task(floor_registry.async_load(hass)),
-        create_eager_task(issue_registry.async_load(hass)),
-        create_eager_task(label_registry.async_load(hass)),
-        hass.async_add_executor_job(_init_blocking_io_modules_in_executor),
-        create_eager_task(template.async_load_custom_templates(hass)),
-        create_eager_task(restore_state.async_load(hass)),
-        create_eager_task(hass.config_entries.async_initialize()),
-        create_eager_task(async_get_system_info(hass)),
-        create_eager_task(condition.async_setup(hass)),
-        create_eager_task(trigger.async_setup(hass)),
-    )
+
+    recovery = hass.config.recovery_mode
+    try:
+        await asyncio.gather(
+            create_eager_task(get_internal_store_manager(hass).async_initialize()),
+            create_eager_task(area_registry.async_load(hass, load_empty=recovery)),
+            create_eager_task(category_registry.async_load(hass, load_empty=recovery)),
+            create_eager_task(device_registry.async_load(hass, load_empty=recovery)),
+            create_eager_task(entity_registry.async_load(hass, load_empty=recovery)),
+            create_eager_task(floor_registry.async_load(hass, load_empty=recovery)),
+            create_eager_task(issue_registry.async_load(hass, load_empty=recovery)),
+            create_eager_task(label_registry.async_load(hass, load_empty=recovery)),
+            hass.async_add_executor_job(_init_blocking_io_modules_in_executor),
+            create_eager_task(template.async_load_custom_templates(hass)),
+            create_eager_task(restore_state.async_load(hass, load_empty=recovery)),
+            create_eager_task(hass.config_entries.async_initialize()),
+            create_eager_task(async_get_system_info(hass)),
+            create_eager_task(condition.async_setup(hass)),
+            create_eager_task(trigger.async_setup(hass)),
+        )
+    except UnsupportedStorageVersionError as err:
+        # If we're already in recovery mode, we don't want to handle the exception
+        # and activate recovery mode again, as that would lead to an infinite loop.
+        if recovery:
+            raise
+
+        _LOGGER.error(
+            "Storage file %s was created by a newer version of Home Assistant"
+            " (storage version %s > %s); activating recovery mode; on-disk data"
+            " is preserved; upgrade Home Assistant or restore from a backup",
+            err.storage_key,
+            err.found_version,
+            err.max_supported_version,
+        )
+        return False
+
+    return True
 
 
 async def async_from_config_dict(
@@ -475,7 +499,9 @@ async def async_from_config_dict(
     # Prime custom component cache early so we know if registry entries are tied
     # to a custom integration
     await loader.async_get_custom_components(hass)
-    await async_load_base_functionality(hass)
+
+    if not await async_load_base_functionality(hass):
+        return None
 
     # Set up core.
     _LOGGER.debug("Setting up %s", CORE_INTEGRATIONS)

homeassistant/components/backup/store.py

@@ -29,12 +29,17 @@ class StoredBackupData(TypedDict):
 class _BackupStore(Store[StoredBackupData]):
     """Class to help storing backup data."""
 
+    # Maximum version we support reading for forward compatibility.
+    # This allows reading data written by a newer HA version after downgrade.
+    _MAX_READABLE_VERSION = 2
+
     def __init__(self, hass: HomeAssistant) -> None:
         """Initialize storage class."""
         super().__init__(
             hass,
             STORAGE_VERSION,
             STORAGE_KEY,
+            max_readable_version=self._MAX_READABLE_VERSION,
             minor_version=STORAGE_VERSION_MINOR,
         )
 
@@ -86,8 +91,8 @@ class _BackupStore(Store[StoredBackupData]):
         # data["config"]["schedule"]["state"] will be removed. The bump to 2 is
         # planned to happen after a 6 month quiet period with no minor version
         # changes.
-        # Reject if major version is higher than 2.
-        if old_major_version > 2:
+        # Reject if major version is higher than _MAX_READABLE_VERSION.
+        if old_major_version > self._MAX_READABLE_VERSION:
             raise NotImplementedError
         return data
 

homeassistant/exceptions.py

@@ -381,3 +381,20 @@ class DependencyError(HomeAssistantError):
             f"Could not setup dependencies: {', '.join(failed_dependencies)}",
         )
         self.failed_dependencies = failed_dependencies
+
+
+class UnsupportedStorageVersionError(HomeAssistantError):
+    """Raised when a storage file has a newer major version than expected."""
+
+    def __init__(
+        self, storage_key: str, found_version: int, max_supported_version: int
+    ) -> None:
+        """Initialize error."""
+        super().__init__(
+            f"Storage file {storage_key} has version {found_version}"
+            f" which is newer than the max supported version {max_supported_version};"
+            " upgrade Home Assistant or restore from a backup",
+        )
+        self.storage_key = storage_key
+        self.found_version = found_version
+        self.max_supported_version = max_supported_version

homeassistant/helpers/area_registry.py

@@ -447,7 +447,7 @@ class AreaRegistry(BaseRegistry[AreasRegistryStoreData]):
             EventAreaRegistryUpdatedData(action="reorder", area_id=None),
         )
 
-    async def async_load(self) -> None:
+    async def _async_load(self) -> None:
         """Load the area registry."""
         self._async_setup_cleanup()
 
@@ -549,10 +549,10 @@ def async_get(hass: HomeAssistant) -> AreaRegistry:
     return AreaRegistry(hass)
 
 
-async def async_load(hass: HomeAssistant) -> None:
+async def async_load(hass: HomeAssistant, *, load_empty: bool = False) -> None:
     """Load area registry."""
     assert DATA_REGISTRY not in hass.data
-    await async_get(hass).async_load()
+    await async_get(hass).async_load(load_empty=load_empty)
 
 
 @callback

homeassistant/helpers/category_registry.py

@@ -77,7 +77,7 @@ class CategoryRegistryStore(Store[CategoryRegistryStoreData]):
     ) -> CategoryRegistryStoreData:
         """Migrate to the new version."""
         if old_major_version > STORAGE_VERSION_MAJOR:
-            raise ValueError("Can't migrate to future version")
+            raise NotImplementedError
 
         if old_major_version == 1:
             if old_minor_version < 2:
@@ -204,7 +204,7 @@ class CategoryRegistry(BaseRegistry[CategoryRegistryStoreData]):
 
         return new
 
-    async def async_load(self) -> None:
+    async def _async_load(self) -> None:
         """Load the category registry."""
         data = await self._store.async_load()
         category_entries: dict[str, dict[str, CategoryEntry]] = {}
@@ -265,7 +265,7 @@ def async_get(hass: HomeAssistant) -> CategoryRegistry:
     return CategoryRegistry(hass)
 
 
-async def async_load(hass: HomeAssistant) -> None:
+async def async_load(hass: HomeAssistant, *, load_empty: bool = False) -> None:
     """Load category registry."""
     assert DATA_REGISTRY not in hass.data
-    await async_get(hass).async_load()
+    await async_get(hass).async_load(load_empty=load_empty)

homeassistant/helpers/device_registry.py

@@ -1461,7 +1461,7 @@ class DeviceRegistry(BaseRegistry[dict[str, list[dict[str, Any]]]]):
         )
         self.async_schedule_save()
 
-    async def async_load(self) -> None:
+    async def _async_load(self) -> None:
         """Load the device registry."""
         async_setup_cleanup(self.hass, self)
 
@@ -1706,10 +1706,10 @@ def async_get(hass: HomeAssistant) -> DeviceRegistry:
     return DeviceRegistry(hass)
 
 
-async def async_load(hass: HomeAssistant) -> None:
+async def async_load(hass: HomeAssistant, *, load_empty: bool = False) -> None:
     """Load device registry."""
     assert DATA_REGISTRY not in hass.data
-    await async_get(hass).async_load()
+    await async_get(hass).async_load(load_empty=load_empty)
 
 
 @callback

homeassistant/helpers/entity_registry.py

@@ -1678,7 +1678,7 @@ class EntityRegistry(BaseRegistry):
             new_options[domain] = options
         return self._async_update_entity(entity_id, options=new_options)
 
-    async def async_load(self) -> None:
+    async def _async_load(self) -> None:
         """Load the entity registry."""
         _async_setup_cleanup(self.hass, self)
         _async_setup_entity_restore(self.hass, self)
@@ -1945,10 +1945,10 @@ def async_get(hass: HomeAssistant) -> EntityRegistry:
     return EntityRegistry(hass)
 
 
-async def async_load(hass: HomeAssistant) -> None:
+async def async_load(hass: HomeAssistant, *, load_empty: bool = False) -> None:
     """Load entity registry."""
     assert DATA_REGISTRY not in hass.data
-    await async_get(hass).async_load()
+    await async_get(hass).async_load(load_empty=load_empty)
 
 
 @callback

homeassistant/helpers/floor_registry.py

@@ -94,7 +94,7 @@ class FloorRegistryStore(Store[FloorRegistryStoreData]):
     ) -> FloorRegistryStoreData:
         """Migrate to the new version."""
         if old_major_version > STORAGE_VERSION_MAJOR:
-            raise ValueError("Can't migrate to future version")
+            raise NotImplementedError
 
         if old_major_version == 1:
             if old_minor_version < 2:
@@ -307,7 +307,7 @@ class FloorRegistry(BaseRegistry[FloorRegistryStoreData]):
             _EventFloorRegistryUpdatedData_Reorder(action="reorder"),
         )
 
-    async def async_load(self) -> None:
+    async def _async_load(self) -> None:
         """Load the floor registry."""
         data = await self._store.async_load()
         floors = FloorRegistryItems()
@@ -353,7 +353,7 @@ def async_get(hass: HomeAssistant) -> FloorRegistry:
     return FloorRegistry(hass)
 
 
-async def async_load(hass: HomeAssistant) -> None:
+async def async_load(hass: HomeAssistant, *, load_empty: bool = False) -> None:
     """Load floor registry."""
     assert DATA_REGISTRY not in hass.data
-    await async_get(hass).async_load()
+    await async_get(hass).async_load(load_empty=load_empty)

homeassistant/helpers/issue_registry.py

@@ -251,7 +251,7 @@ class IssueRegistry(BaseRegistry):
         """
         self._store.make_read_only()
 
-    async def async_load(self) -> None:
+    async def _async_load(self) -> None:
         """Load the issue registry."""
         data = await self._store.async_load()
 
@@ -314,12 +314,17 @@ def async_get(hass: HomeAssistant) -> IssueRegistry:
     return IssueRegistry(hass)
 
 
-async def async_load(hass: HomeAssistant, *, read_only: bool = False) -> None:
+async def async_load(
+    hass: HomeAssistant,
+    *,
+    read_only: bool = False,
+    load_empty: bool = False,
+) -> None:
     """Load issue registry."""
     ir = async_get(hass)
     if read_only:  # only used in for check config script
         ir.make_read_only()
-    return await ir.async_load()
+    await ir.async_load(load_empty=load_empty)
 
 
 @callback

homeassistant/helpers/label_registry.py

@@ -80,7 +80,7 @@ class LabelRegistryStore(Store[LabelRegistryStoreData]):
     ) -> LabelRegistryStoreData:
         """Migrate to the new version."""
         if old_major_version > STORAGE_VERSION_MAJOR:
-            raise ValueError("Can't migrate to future version")
+            raise NotImplementedError
 
         if old_major_version == 1:
             if old_minor_version < 2:
@@ -224,7 +224,7 @@ class LabelRegistry(BaseRegistry[LabelRegistryStoreData]):
 
         return new
 
-    async def async_load(self) -> None:
+    async def _async_load(self) -> None:
         """Load the label registry."""
         data = await self._store.async_load()
         labels = NormalizedNameBaseRegistryItems[LabelEntry]()
@@ -270,7 +270,7 @@ def async_get(hass: HomeAssistant) -> LabelRegistry:
     return LabelRegistry(hass)
 
 
-async def async_load(hass: HomeAssistant) -> None:
+async def async_load(hass: HomeAssistant, *, load_empty: bool = False) -> None:
     """Load label registry."""
     assert DATA_REGISTRY not in hass.data
-    await async_get(hass).async_load()
+    await async_get(hass).async_load(load_empty=load_empty)

homeassistant/helpers/registry.py

@@ -77,6 +77,19 @@ class BaseRegistry[_StoreDataT: Mapping[str, Any] | Sequence[Any]](ABC):
         delay = SAVE_DELAY if self.hass.state is CoreState.running else SAVE_DELAY_LONG
         self._store.async_delay_save(self._data_to_save, delay)
 
+    async def async_load(self, *, load_empty: bool = False) -> None:
+        """Load the registry.
+
+        Optionally set the store to load empty and become read-only.
+        """
+        if load_empty:
+            self._store.set_load_empty()
+        await self._async_load()
+
+    @abstractmethod
+    async def _async_load(self) -> None:
+        """Load the registry."""
+
     @abstractmethod
     def _data_to_save(self) -> _StoreDataT:
         """Return data of registry to store in a file."""

homeassistant/helpers/restore_state.py

@@ -9,7 +9,7 @@ from typing import Any, Self, cast
 
 from homeassistant.const import ATTR_RESTORED, EVENT_HOMEASSISTANT_STOP
 from homeassistant.core import HomeAssistant, State, callback, valid_entity_id
-from homeassistant.exceptions import HomeAssistantError
+from homeassistant.exceptions import HomeAssistantError, UnsupportedStorageVersionError
 from homeassistant.util import dt as dt_util
 from homeassistant.util.hass_dict import HassKey
 from homeassistant.util.json import json_loads
@@ -95,9 +95,12 @@ class StoredState:
         )
 
 
-async def async_load(hass: HomeAssistant) -> None:
+async def async_load(hass: HomeAssistant, *, load_empty: bool = False) -> None:
     """Load the restore state task."""
-    await async_get(hass).async_setup()
+    data = async_get(hass)
+    if load_empty:
+        data.set_load_empty()
+    await data.async_setup()
 
 
 @callback
@@ -124,6 +127,10 @@ class RestoreStateData:
         self.last_states: dict[str, StoredState] = {}
         self.entities: dict[str, RestoreEntity] = {}
 
+    def set_load_empty(self) -> None:
+        """Set the store to load empty and become read-only."""
+        self.store.set_load_empty()
+
     async def async_setup(self) -> None:
         """Set up up the instance of this data helper."""
         await self.async_load()
@@ -139,6 +146,8 @@ class RestoreStateData:
         """Load the instance of this data helper."""
         try:
             stored_states = await self.store.async_load()
+        except UnsupportedStorageVersionError:
+            raise
         except HomeAssistantError as exc:
             _LOGGER.error("Error loading last states", exc_info=exc)
             stored_states = None

homeassistant/helpers/storage.py

@@ -28,7 +28,7 @@ from homeassistant.core import (
     HomeAssistant,
     callback,
 )
-from homeassistant.exceptions import HomeAssistantError
+from homeassistant.exceptions import HomeAssistantError, UnsupportedStorageVersionError
 from homeassistant.loader import bind_hass
 from homeassistant.util import dt as dt_util, json as json_util
 from homeassistant.util.file import WriteError, write_utf8_file, write_utf8_file_atomic
@@ -239,6 +239,7 @@ class Store[_T: Mapping[str, Any] | Sequence[Any]]:
         *,
         atomic_writes: bool = False,
         encoder: type[JSONEncoder] | None = None,
+        max_readable_version: int | None = None,
         minor_version: int = 1,
         read_only: bool = False,
         serialize_in_event_loop: bool = True,
@@ -246,6 +247,10 @@ class Store[_T: Mapping[str, Any] | Sequence[Any]]:
         """Initialize storage class.
 
         Args:
+            max_readable_version: Maximum major version that can be read. Defaults
+            to version. Set higher than version to support forward compatibility,
+            allowing reading data written by newer versions (e.g., after downgrade).
+
             serialize_in_event_loop: Whether to serialize data in the event loop.
             Set to True (default) if data passed to async_save and data produced by
             data_func passed to async_delay_save needs to be serialized in the event
@@ -273,6 +278,10 @@ class Store[_T: Mapping[str, Any] | Sequence[Any]]:
         self._encoder = encoder
         self._atomic_writes = atomic_writes
         self._read_only = read_only
+        self._load_empty = False
+        self._max_readable_version = (
+            max_readable_version if max_readable_version is not None else version
+        )
         self._next_write_time = 0.0
         self._manager = get_internal_store_manager(hass)
         self._serialize_in_event_loop = serialize_in_event_loop
@@ -289,6 +298,14 @@ class Store[_T: Mapping[str, Any] | Sequence[Any]]:
         """
         self._read_only = True
 
+    def set_load_empty(self) -> None:
+        """Set the store to load empty data and become read-only.
+
+        When set, the store will skip loading data from disk and return None,
+        while also becoming read-only to preserve on-disk data untouched.
+        """
+        self._load_empty = True
+
     async def async_load(self) -> _T | None:
         """Load data.
 
@@ -328,6 +345,12 @@ class Store[_T: Mapping[str, Any] | Sequence[Any]]:
 
     async def _async_load_data(self):
         """Load the data."""
+        # When load_empty is set, skip loading storage files and use empty
+        # data while preserving the on-disk files untouched.
+        if self._load_empty:
+            self.make_read_only()
+            return None
+
         # Check if we have a pending write
         if self._data is not None:
             data = self._data
@@ -415,6 +438,10 @@ class Store[_T: Mapping[str, Any] | Sequence[Any]]:
         ):
             stored = data["data"]
         else:
+            if data["version"] > self._max_readable_version:
+                raise UnsupportedStorageVersionError(
+                    self.key, data["version"], self._max_readable_version
+                )
             _LOGGER.info(
                 "Migrating %s storage from %s.%s to %s.%s",
                 self.key,

tests/helpers/test_registry.py

@@ -21,6 +21,9 @@ class SampleRegistry(BaseRegistry):
         self._store = storage.Store(hass, 1, "test")
         self.save_calls = 0
 
+    async def _async_load(self) -> None:
+        """Load the registry."""
+
     def _data_to_save(self) -> dict[str, Any]:
         """Return data of registry to save."""
         self.save_calls += 1

tests/helpers/test_storage.py

@@ -25,7 +25,7 @@ from homeassistant.core import (
     HomeAssistant,
     callback,
 )
-from homeassistant.exceptions import HomeAssistantError
+from homeassistant.exceptions import HomeAssistantError, UnsupportedStorageVersionError
 from homeassistant.helpers import issue_registry as ir, storage
 from homeassistant.helpers.json import json_bytes, prepare_save_json
 from homeassistant.util import dt as dt_util
@@ -657,14 +657,23 @@ async def test_minor_version(
     assert hass_storage[store_v_1_2.key]["minor_version"] == MOCK_MINOR_VERSION_2
 
 
-async def test_migrate_major_not_implemented_raises(
-    hass: HomeAssistant, store: storage.Store, store_v_2_1: storage.Store
+async def test_loading_newer_major_version_raises(
+    hass: HomeAssistant,
+    hass_storage: dict[str, Any],
+    store: storage.Store,
+    store_v_2_1: storage.Store,
 ) -> None:
-    """Test migrating between major versions fails if not implemented."""
-
+    """Test loading storage with a newer major version raises and preserves data."""
     await store_v_2_1.async_save(MOCK_DATA)
-    with pytest.raises(NotImplementedError):
+    with pytest.raises(UnsupportedStorageVersionError) as exc_info:
         await store.async_load()
+    assert exc_info.value.storage_key == MOCK_KEY
+    assert exc_info.value.found_version == MOCK_VERSION_2
+    assert exc_info.value.max_supported_version == MOCK_VERSION
+    # Verify on-disk data is not modified
+    assert hass_storage[MOCK_KEY]["version"] == MOCK_VERSION_2
+    assert hass_storage[MOCK_KEY]["minor_version"] == MOCK_MINOR_VERSION_1
+    assert hass_storage[MOCK_KEY]["data"] == MOCK_DATA
 
 
 async def test_migrate_minor_not_implemented(
@@ -1349,3 +1358,27 @@ async def test_storage_concurrent_load(hass: HomeAssistant) -> None:
         )
         for load in loads:
             assert load == "data"
+
+
+async def test_load_empty_returns_none_and_read_only(
+    hass: HomeAssistant, hass_storage: dict[str, Any]
+) -> None:
+    """Test store with load_empty returns None, becomes read-only, and skips version checks."""
+    # Use a future version to also verify no version error is raised
+    hass_storage[MOCK_KEY] = {
+        "version": 99,
+        "minor_version": 1,
+        "key": MOCK_KEY,
+        "data": MOCK_DATA,
+    }
+
+    store = storage.Store(hass, MOCK_VERSION, MOCK_KEY)
+    store.set_load_empty()
+
+    data = await store.async_load()
+    assert data is None
+    assert store._read_only is True
+
+    await store.async_save({"new": "data"})
+    assert hass_storage[MOCK_KEY]["data"] == MOCK_DATA
+    assert hass_storage[MOCK_KEY]["version"] == 99

tests/test_bootstrap.py

@@ -965,6 +965,46 @@ async def test_setup_hass_recovery_mode_and_safe_mode(
     assert "Starting in safe mode" not in caplog.text
 
 
+@pytest.mark.parametrize("hass_config", [{"frontend": {}}])
+@pytest.mark.usefixtures("mock_hass_config")
+async def test_storage_version_too_new_triggers_recovery_mode(
+    hass_storage: dict[str, Any],
+    mock_enable_logging: AsyncMock,
+    mock_is_virtual_env: Mock,
+    mock_mount_local_lib_path: AsyncMock,
+    mock_ensure_config_exists: AsyncMock,
+    mock_process_ha_config_upgrade: Mock,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """Test that a storage file with a newer major version triggers recovery mode."""
+    hass_storage["core.entity_registry"] = {
+        "version": 99,
+        "minor_version": 1,
+        "key": "core.entity_registry",
+        "data": {},
+    }
+
+    hass = await bootstrap.async_setup_hass(
+        runner.RuntimeConfig(
+            config_dir=get_test_config_dir(),
+            verbose=False,
+            log_rotate_days=10,
+            log_file="",
+            log_no_color=False,
+            skip_pip=True,
+            recovery_mode=False,
+        ),
+    )
+
+    assert hass is not None
+    assert hass.config.recovery_mode is True
+    assert "recovery_mode" in hass.config.components
+    assert (
+        "Storage file core.entity_registry was created"
+        " by a newer version of Home Assistant" in caplog.text
+    )
+
+
 @pytest.mark.parametrize("hass_config", [{"homeassistant": {"non-existing": 1}}])
 @pytest.mark.usefixtures("mock_hass_config")
 async def test_setup_hass_invalid_core_config(