Add Cloud STT v2 as labs feature (#180259)

homeassistant/components/cloud/const.py

@@ -26,6 +26,8 @@ EVENT_CLOUD_EVENT = "cloud_event"
 
 REQUEST_TIMEOUT = 10
 
+PREVIEW_FEATURE_STT_V2 = "stt_v2"
+
 PREF_ENABLE_ALEXA = "alexa_enabled"
 PREF_ENABLE_GOOGLE = "google_enabled"
 PREF_ENABLE_REMOTE = "remote_enabled"

homeassistant/components/cloud/manifest.json

@@ -13,6 +13,12 @@
   "integration_type": "system",
   "iot_class": "cloud_push",
   "loggers": ["acme", "hass_nabucasa", "snitun"],
+  "preview_features": {
+    "stt_v2": {
+      "feedback_url": "https://forms.gle/Juf2vsxJHUX5y6Ev5",
+      "learn_more_url": "https://support.nabucasa.com/hc/en-us/articles/29718084245149-Speech-to-text-STT"
+    }
+  },
   "requirements": ["hass-nabucasa==2.6.0", "openai==2.45.0"],
   "single_config_entry": true
 }

homeassistant/components/cloud/strings.json

@@ -84,6 +84,14 @@
       "title": "Detected wrong custom domain configuration"
     }
   },
+  "preview_features": {
+    "stt_v2": {
+      "description": "We're testing a new speech-to-text engine for Home Assistant Cloud, the part of voice control that turns what you say into words. In our testing it copes better with accents, background noise, and languages other than English.\n\nWhile this is enabled, any audio from your voice commands is processed by [Soniox](https://soniox.com/) rather than our current provider. This is relevant only to Assistants configured to use Home Assistant Cloud as an STT engine. It travels through a Nabu Casa proxy, which handles authentication and forwards it to the processing region closest to you. Nabu Casa doesn't process, store or log your audio at any point. Everything else about your voice pipeline stays the same, and you can turn this off again at any time.",
+      "disable_confirmation": "Any Assistant currently using the new engine will go back to being transcribed by our current speech-to-text provider.",
+      "enable_confirmation": "This feature is still in development and may change. While it's enabled, audio from your voice commands in supported languages is transcribed by Soniox instead of our current provider.",
+      "name": "Home Assistant Cloud: Speech-to-text"
+    }
+  },
   "services": {
     "remote_connect": {
       "description": "Makes the instance UI accessible from outside of the local network by enabling your Home Assistant Cloud connection.",

homeassistant/components/cloud/stt.py

@@ -4,15 +4,18 @@ from collections.abc import AsyncIterable
 import logging
 from typing import override
 
-from hass_nabucasa import Cloud
-from hass_nabucasa.voice import STT_LANGUAGES, VoiceError
+from hass_nabucasa import Cloud, SpeechToTextV2Error
+from hass_nabucasa.voice import STT_LANGUAGES, STTResponse, VoiceError
 
+from homeassistant.components import labs
 from homeassistant.components.stt import (
+    DEFAULT_AUDIO_PROCESSING,
     AudioBitRates,
     AudioChannels,
     AudioCodecs,
     AudioFormats,
     AudioSampleRates,
+    SpeechAudioProcessing,
     SpeechMetadata,
     SpeechResult,
     SpeechResultState,
@@ -26,10 +29,23 @@ from homeassistant.setup import async_when_setup
 
 from .assist_pipeline import async_migrate_cloud_pipeline_engine
 from .client import CloudClient
-from .const import DATA_CLOUD, DATA_PLATFORMS_SETUP, STT_ENTITY_UNIQUE_ID
+from .const import (
+    DATA_CLOUD,
+    DATA_PLATFORMS_SETUP,
+    DOMAIN,
+    PREVIEW_FEATURE_STT_V2,
+    STT_ENTITY_UNIQUE_ID,
+)
 
 _LOGGER = logging.getLogger(__name__)
 
+# STT v2 detects the end of speech itself and works best on untouched audio.
+STT_V2_AUDIO_PROCESSING = SpeechAudioProcessing(
+    requires_external_vad=True,
+    prefers_auto_gain_enabled=False,
+    prefers_noise_reduction_enabled=False,
+)
+
 
 async def async_setup_entry(
     hass: HomeAssistant,
@@ -53,6 +69,13 @@ class CloudProviderEntity(SpeechToTextEntity):
         """Initialize cloud Speech to text entity."""
         self.cloud = cloud
 
+    @property
+    def _stt_v2_enabled(self) -> bool:
+        """Return if the v2 speech to text service is enabled."""
+        return labs.async_is_preview_feature_enabled(
+            self.hass, DOMAIN, PREVIEW_FEATURE_STT_V2
+        )
+
     @property
     @override
     def supported_languages(self) -> list[str]:
@@ -89,6 +112,14 @@ class CloudProviderEntity(SpeechToTextEntity):
         """Return a list of supported channels."""
         return [AudioChannels.CHANNEL_MONO]
 
+    @property
+    @override
+    def audio_processing(self) -> SpeechAudioProcessing:
+        """Return required/preferred input audio processing settings."""
+        if self._stt_v2_enabled:
+            return STT_V2_AUDIO_PROCESSING
+        return DEFAULT_AUDIO_PROCESSING
+
     @override
     async def async_added_to_hass(self) -> None:
         """Run when entity is about to be added to hass."""
@@ -105,29 +136,75 @@ class CloudProviderEntity(SpeechToTextEntity):
 
         async_when_setup(self.hass, "assist_pipeline", pipeline_setup)
 
+        self.async_on_remove(
+            labs.async_subscribe_preview_feature(
+                self.hass,
+                DOMAIN,
+                PREVIEW_FEATURE_STT_V2,
+                self._async_handle_labs_update,
+            )
+        )
+
+    @override
+    async def async_will_remove_from_hass(self) -> None:
+        """Close the connection when the entity is removed."""
+        await self.cloud.stt_v2.disconnect()
+
+    async def _async_handle_labs_update(
+        self, event_data: labs.EventLabsUpdatedData
+    ) -> None:
+        """Close the connection to the v2 service when it is turned off."""
+        if not event_data["enabled"]:
+            await self.cloud.stt_v2.disconnect()
+
     @override
     async def async_process_audio_stream(
         self, metadata: SpeechMetadata, stream: AsyncIterable[bytes]
     ) -> SpeechResult:
         """Process an audio stream to STT service."""
-        content_type = (
-            f"audio/{metadata.format!s}; codecs=audio/{metadata.codec!s};"
-            " samplerate=16000"
+        # STT v2 covers fewer languages, so fall back for the rest.
+        use_stt_v2 = self._stt_v2_enabled and bool(
+            self.cloud.stt_v2.resolve_language(metadata.language)
         )
 
-        # Process STT
         try:
-            result = await self.cloud.voice.process_stt(
-                stream=stream,
-                content_type=content_type,
-                language=metadata.language,
-            )
-        except VoiceError as err:
+            if use_stt_v2:
+                result = await self._async_process_stt_v2(metadata, stream)
+            else:
+                result = await self._async_process_azure_stt(metadata, stream)
+        except (SpeechToTextV2Error, VoiceError) as err:
             _LOGGER.error("Voice error: %s", err)
             return SpeechResult(None, SpeechResultState.ERROR)
 
-        # Return Speech as Text
         return SpeechResult(
             result.text,
             SpeechResultState.SUCCESS if result.success else SpeechResultState.ERROR,
         )
+
+    async def _async_process_stt_v2(
+        self, metadata: SpeechMetadata, stream: AsyncIterable[bytes]
+    ) -> STTResponse:
+        """Process an audio stream with the v2 speech to text service."""
+        return await self.cloud.stt_v2.process_stt(
+            stream=stream,
+            language=metadata.language,
+            audio_format=metadata.format.value,
+            codec=metadata.codec.value,
+            bit_rate=metadata.bit_rate.value,
+            sample_rate=metadata.sample_rate.value,
+            channel=metadata.channel.value,
+        )
+
+    async def _async_process_azure_stt(
+        self, metadata: SpeechMetadata, stream: AsyncIterable[bytes]
+    ) -> STTResponse:
+        """Process an audio stream with the Azure speech to text service."""
+        content_type = (
+            f"audio/{metadata.format!s}; codecs=audio/{metadata.codec!s};"
+            " samplerate=16000"
+        )
+        return await self.cloud.voice.process_stt(
+            stream=stream,
+            content_type=content_type,
+            language=metadata.language,
+        )

homeassistant/generated/labs.py

@@ -11,6 +11,13 @@ LABS_PREVIEW_FEATURES = {
             "report_issue_url": "https://github.com/OHF-Device-Database/device-database/issues/new",
         },
     },
+    "cloud": {
+        "stt_v2": {
+            "feedback_url": "https://forms.gle/Juf2vsxJHUX5y6Ev5",
+            "learn_more_url": "https://support.nabucasa.com/hc/en-us/articles/29718084245149-Speech-to-text-STT",
+            "report_issue_url": "",
+        },
+    },
     "frontend": {
         "winter_mode": {
             "feedback_url": "",

tests/components/cloud/conftest.py

@@ -21,6 +21,7 @@ from hass_nabucasa.google_report_state import GoogleReportState
 from hass_nabucasa.ice_servers import IceServers
 from hass_nabucasa.iot import CloudIoT
 from hass_nabucasa.remote import RemoteUI
+from hass_nabucasa.stt_v2 import SpeechToTextV2
 from hass_nabucasa.voice import Voice
 import jwt
 import pytest
@@ -82,6 +83,10 @@ async def cloud_fixture() -> AsyncGenerator[MagicMock]:
         mock_cloud.iot = MagicMock(
             spec=CloudIoT, last_disconnect_reason=None, state=STATE_CONNECTED, tries=0
         )
+        mock_cloud.stt_v2 = MagicMock(
+            spec=SpeechToTextV2,
+            resolve_language=SpeechToTextV2.resolve_language,
+        )
         mock_cloud.voice = MagicMock(spec=Voice)
         mock_cloud.files = MagicMock(spec=Files)
         mock_cloud.started = None

tests/components/cloud/test_stt.py

@@ -4,15 +4,22 @@ from collections.abc import AsyncGenerator
 from copy import deepcopy
 from http import HTTPStatus
 from typing import Any
-from unittest.mock import AsyncMock, MagicMock, patch
+from unittest.mock import ANY, AsyncMock, MagicMock, patch
 
+from hass_nabucasa import SpeechToTextV2Error
 from hass_nabucasa.voice import STTResponse, VoiceError
 import pytest
 
 from homeassistant.components.assist_pipeline.pipeline import (  # pylint: disable=home-assistant-component-root-import
     STORAGE_KEY,
 )
-from homeassistant.components.cloud.const import DOMAIN
+from homeassistant.components.cloud.const import DOMAIN, PREVIEW_FEATURE_STT_V2
+from homeassistant.components.labs import async_update_preview_feature
+from homeassistant.components.stt import (
+    DEFAULT_AUDIO_PROCESSING,
+    SpeechAudioProcessing,
+    async_get_speech_to_text_entity,
+)
 from homeassistant.const import STATE_UNAVAILABLE, STATE_UNKNOWN
 from homeassistant.core import HomeAssistant
 from homeassistant.setup import async_setup_component
@@ -21,6 +28,8 @@ from . import PIPELINE_DATA
 
 from tests.typing import ClientSessionGenerator
 
+ENTITY_ID = "stt.home_assistant_cloud"
+
 
 @pytest.fixture(autouse=True)
 async def delay_save_fixture() -> AsyncGenerator[None]:
@@ -150,3 +159,154 @@ async def test_migrating_pipelines(
     assert hass_storage[STORAGE_KEY]["data"]["items"][0]["wake_word_id"] is None
     assert hass_storage[STORAGE_KEY]["data"]["items"][1] == PIPELINE_DATA["items"][1]
     assert hass_storage[STORAGE_KEY]["data"]["items"][2] == PIPELINE_DATA["items"][2]
+
+
+@pytest.fixture(name="setup_stt")
+async def setup_stt_fixture(hass: HomeAssistant, cloud: MagicMock) -> None:
+    """Set up the cloud speech-to-text entity with labs available."""
+    assert await async_setup_component(hass, "labs", {})
+    assert await async_setup_component(hass, DOMAIN, {"cloud": {}})
+    await hass.async_block_till_done()
+
+    on_start_callback = cloud.register_on_start.call_args[0][0]
+    await on_start_callback()
+
+
+async def _process_audio(
+    hass_client: ClientSessionGenerator, language: str = "de-DE"
+) -> dict[str, Any]:
+    """Post an audio stream to the cloud speech-to-text entity."""
+    client = await hass_client()
+    response = await client.post(
+        f"/api/stt/{ENTITY_ID}",
+        headers={
+            "X-Speech-Content": (
+                "format=wav; codec=pcm; sample_rate=16000; bit_rate=16; channel=1;"
+                f" language={language}"
+            )
+        },
+        data=b"Test",
+    )
+
+    assert response.status == HTTPStatus.OK
+    return await response.json()
+
+
+@pytest.mark.usefixtures("setup_stt")
+async def test_stt_v2_disabled_by_default(
+    hass: HomeAssistant,
+    cloud: MagicMock,
+    hass_client: ClientSessionGenerator,
+) -> None:
+    """Test that audio goes to the legacy service while the lab is off."""
+    cloud.voice.process_stt = AsyncMock(return_value=STTResponse(True, "Lights on"))
+    cloud.stt_v2.process_stt = AsyncMock()
+
+    assert await _process_audio(hass_client) == {
+        "text": "Lights on",
+        "result": "success",
+    }
+    assert cloud.voice.process_stt.call_count == 1
+    cloud.stt_v2.process_stt.assert_not_called()
+
+    entity = async_get_speech_to_text_entity(hass, ENTITY_ID)
+    assert entity
+    assert entity.audio_processing == DEFAULT_AUDIO_PROCESSING
+
+
+@pytest.mark.usefixtures("setup_stt")
+async def test_stt_v2_enabled(
+    hass: HomeAssistant,
+    cloud: MagicMock,
+    hass_client: ClientSessionGenerator,
+) -> None:
+    """Test that audio goes to the v2 service while the lab is on."""
+    cloud.voice.process_stt = AsyncMock()
+    cloud.stt_v2.process_stt = AsyncMock(return_value=STTResponse(True, "Lights on"))
+
+    await async_update_preview_feature(hass, DOMAIN, PREVIEW_FEATURE_STT_V2, True)
+
+    assert await _process_audio(hass_client) == {
+        "text": "Lights on",
+        "result": "success",
+    }
+    cloud.voice.process_stt.assert_not_called()
+    assert cloud.stt_v2.process_stt.call_count == 1
+    assert cloud.stt_v2.process_stt.call_args.kwargs == {
+        "stream": ANY,
+        "language": "de-DE",
+        "audio_format": "wav",
+        "codec": "pcm",
+        "bit_rate": 16,
+        "sample_rate": 16000,
+        "channel": 1,
+    }
+
+    entity = async_get_speech_to_text_entity(hass, ENTITY_ID)
+    assert entity
+    assert entity.audio_processing == SpeechAudioProcessing(
+        requires_external_vad=True,
+        prefers_auto_gain_enabled=False,
+        prefers_noise_reduction_enabled=False,
+    )
+
+
+@pytest.mark.usefixtures("setup_stt")
+async def test_stt_v2_falls_back_on_unsupported_language(
+    hass: HomeAssistant,
+    cloud: MagicMock,
+    hass_client: ClientSessionGenerator,
+) -> None:
+    """Test that a language v2 lacks still uses the legacy service."""
+    cloud.voice.process_stt = AsyncMock(return_value=STTResponse(True, "Lights on"))
+    cloud.stt_v2.process_stt = AsyncMock()
+
+    await async_update_preview_feature(hass, DOMAIN, PREVIEW_FEATURE_STT_V2, True)
+
+    assert await _process_audio(hass_client, language="hy-AM") == {
+        "text": "Lights on",
+        "result": "success",
+    }
+    assert cloud.voice.process_stt.call_count == 1
+    cloud.stt_v2.process_stt.assert_not_called()
+
+
+@pytest.mark.parametrize(
+    "mock_process_stt",
+    [
+        AsyncMock(side_effect=SpeechToTextV2Error("Boom!")),
+        AsyncMock(return_value=STTResponse(False, None)),
+    ],
+    ids=["error", "no-speech"],
+)
+@pytest.mark.usefixtures("setup_stt")
+async def test_stt_v2_failure(
+    hass: HomeAssistant,
+    cloud: MagicMock,
+    hass_client: ClientSessionGenerator,
+    mock_process_stt: AsyncMock,
+) -> None:
+    """Test that a failure of the v2 service is reported as an error."""
+    cloud.stt_v2.process_stt = mock_process_stt
+
+    await async_update_preview_feature(hass, DOMAIN, PREVIEW_FEATURE_STT_V2, True)
+
+    assert await _process_audio(hass_client) == {"text": None, "result": "error"}
+
+
+@pytest.mark.usefixtures("setup_stt")
+async def test_stt_v2_disconnects_when_turned_off(
+    hass: HomeAssistant, cloud: MagicMock
+) -> None:
+    """Test that turning the lab off closes the connection to the v2 service."""
+    cloud.stt_v2.disconnect = AsyncMock()
+
+    await async_update_preview_feature(hass, DOMAIN, PREVIEW_FEATURE_STT_V2, True)
+    await hass.async_block_till_done()
+
+    cloud.stt_v2.disconnect.assert_not_called()
+
+    await async_update_preview_feature(hass, DOMAIN, PREVIEW_FEATURE_STT_V2, False)
+    await hass.async_block_till_done()
+
+    cloud.stt_v2.disconnect.assert_awaited_once()