From 51493b1929cb0502a1919bd2de456248b602beda Mon Sep 17 00:00:00 2001 From: TechArtDev Date: Sat, 26 Sep 2026 22:34:40 +0300 Subject: [PATCH 1/3] feat: route chat-card TTS through media players --- CHANGELOG.md | 10 ++ IMPLEMENTATION_PLAN.md | 7 +- README.md | 32 ++++- custom_components/openclaw/__init__.py | 2 +- custom_components/openclaw/manifest.json | 2 +- .../openclaw/www/openclaw-chat-card.js | 111 +++++++++++++++++- tests/test_chat_card_media_player.mjs | 63 ++++++++++ www/openclaw-chat-card.js | 2 +- 8 files changed, 218 insertions(+), 11 deletions(-) create mode 100644 tests/test_chat_card_media_player.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index dd5726a..c888bcd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ All notable changes to the OpenClaw Home Assistant Integration will be documented in this file. +## [0.1.65-rc.2] - 2026-09-26 + +### Added +- Added optional `voice_output_mode: media_player` chat-card output. Replies can + now be sent through Home Assistant `tts.speak` using configured + `ha_tts_engine` and `voice_output_media_player` entities, bypassing unreliable + Android/iOS WebView audio playback (issues #1 and #29). +- Added matching visual card-editor fields and playback-state waiting so + continuous voice recognition does not immediately capture the spoken reply. + ## [0.1.64] - 2026-09-26 ### Fixed diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md index db218fe..8640515 100644 --- a/IMPLEMENTATION_PLAN.md +++ b/IMPLEMENTATION_PLAN.md @@ -14,7 +14,7 @@ This plan reflects the actual implementation status as of release `0.1.34`. | 4 | Lovelace chat card (text + voice) | P0 | βœ… Done | | 5 | Voice provider choice (`browser` / `assist_stt`) | P0 | βœ… Done | | 6 | Production polish and broad compatibility hardening | P1 | 🚧 In progress | -| 7 | Optional native media player/TTS routing entity | P2 | ⏳ Not started | +| 7 | Optional Home Assistant media-player TTS routing | P2 | βœ… Done | --- @@ -144,9 +144,8 @@ OpenClawHomeAssistantIntegration/ ### R3 β€” Optional enhancements (after stabilization) -1. Evaluate media-player based TTS routing entity. -2. Explore optional continuous flow for HA STT/TTS pipeline mode. -3. Add automated tests around settings websocket payload and card voice state transitions. +1. Explore optional continuous flow for HA STT/TTS pipeline mode. +2. Add broader automated tests around settings websocket payload and card voice state transitions. --- diff --git a/README.md b/README.md index e5bc630..34f21f2 100644 --- a/README.md +++ b/README.md @@ -148,6 +148,9 @@ show_timestamps: true show_voice_button: true show_clear_button: true session_id: default +voice_output_mode: media_player +ha_tts_engine: tts.openai_tts +voice_output_media_player: media_player.bedroom_speaker ``` Minimal config: @@ -156,6 +159,23 @@ Minimal config: type: custom:openclaw-chat-card ``` +### Voice output + +By default, the card speaks on the device displaying Home Assistant. To avoid +mobile WebView audio restrictions, set `voice_output_mode: media_player` and +select a Home Assistant TTS entity plus a target media player: + +```yaml +type: custom:openclaw-chat-card +voice_output_mode: media_player +ha_tts_engine: tts.openai_tts +voice_output_media_player: media_player.bedroom_speaker +``` + +In media-player mode the card calls `tts.speak` and pauses continuous voice +recognition until playback finishes (with a timeout fallback for players that +do not report playback state). + --- ## Assist entity exposure context @@ -188,6 +208,17 @@ Open **Settings β†’ Devices & Services β†’ OpenClaw β†’ Configure**. When enabled, OpenClaw tool-call responses can execute Home Assistant services. +These options only control data and actions handled by this Home Assistant +integration: + +- Disabling exposed-entity context stops the integration from adding HA entity + state to the prompt. It does not clear OpenClaw's conversation history or + change OpenClaw/MCP context. +- The maximum context size applies only to the exposed HA entity block. +- Disabling tool calls prevents this integration from executing returned Home + Assistant service calls. It does not disable tools or MCP servers configured + inside OpenClaw itself. + ### Voice options - **Wake word enabled** @@ -410,4 +441,3 @@ MIT. See [LICENSE](LICENSE). If you find this useful and you want to bring me a coffee to make more nice stuff, or support the project, use the link below: - https://revolut.me/vanyo6dhw - diff --git a/custom_components/openclaw/__init__.py b/custom_components/openclaw/__init__.py index 44e5dc4..4f80436 100644 --- a/custom_components/openclaw/__init__.py +++ b/custom_components/openclaw/__init__.py @@ -107,7 +107,7 @@ # URL at which the card JS is served (registered via register_static_path) _CARD_STATIC_URL = f"/openclaw/{_CARD_FILENAME}" # Versioned URL used for Lovelace resource registration to avoid stale browser cache -_CARD_URL = f"{_CARD_STATIC_URL}?v=0.1.63" +_CARD_URL = f"{_CARD_STATIC_URL}?v=0.1.65-rc.2" OpenClawConfigEntry = ConfigEntry diff --git a/custom_components/openclaw/manifest.json b/custom_components/openclaw/manifest.json index d7981e4..d843c08 100644 --- a/custom_components/openclaw/manifest.json +++ b/custom_components/openclaw/manifest.json @@ -8,7 +8,7 @@ "iot_class": "local_polling", "issue_tracker": "https://github.com/techartdev/OpenClawHomeAssistantIntegration/issues", "requirements": [], - "version": "0.1.64", + "version": "0.1.65-rc.2", "dependencies": ["conversation"], "after_dependencies": ["hassio", "lovelace"] } diff --git a/custom_components/openclaw/www/openclaw-chat-card.js b/custom_components/openclaw/www/openclaw-chat-card.js index a9ab1c4..362bdcf 100644 --- a/custom_components/openclaw/www/openclaw-chat-card.js +++ b/custom_components/openclaw/www/openclaw-chat-card.js @@ -13,7 +13,7 @@ * + subscribes to openclaw_message_received events. */ -const CARD_VERSION = "0.3.13"; +const CARD_VERSION = "0.3.14"; // Max time (ms) to show the thinking indicator before falling back to an error (default; overridable via card config `thinking_timeout` in seconds) const THINKING_TIMEOUT_MS = 120_000; @@ -117,6 +117,7 @@ class OpenClawChatCard extends HTMLElement { show_clear_button: config.show_clear_button !== false, allow_brave_webspeech: config.allow_brave_webspeech === true, voice_provider: config.voice_provider || null, + voice_output_mode: config.voice_output_mode || "local", session_id: config.session_id || null, thinking_timeout: config.thinking_timeout ?? 120, ...config, @@ -1402,6 +1403,22 @@ class OpenClawChatCard extends HTMLElement { this._hass?.locale?.language || this._hass?.selectedLanguage || this._hass?.language || navigator.language || "en-US" ); + if (this._config?.voice_output_mode === "media_player") { + this._voiceStatus = "Speaking through Home Assistant media player…"; + this._render(); + this._speakViaMediaPlayer(plain).then((ok) => { + if (ok) { + this._voiceStatus = ""; + } else { + const reason = this._lastHaTtsAttempt ? ` (${this._lastHaTtsAttempt})` : ""; + this._voiceStatus = `Media-player TTS unavailable${reason}`; + } + this._render(); + this._resumeVoiceInputAfterTts(); + }); + return; + } + if (!hasBrowserTts) { this._voiceStatus = "Browser TTS unavailable, trying Home Assistant TTS…"; this._render(); @@ -1766,6 +1783,72 @@ class OpenClawChatCard extends HTMLElement { } } + async _speakViaMediaPlayer(text) { + if (!this._hass?.callService) return false; + + this._lastHaTtsAttempt = null; + const engine = + typeof this._config?.ha_tts_engine === "string" + ? this._config.ha_tts_engine.trim() + : ""; + const mediaPlayer = + typeof this._config?.voice_output_media_player === "string" + ? this._config.voice_output_media_player.trim() + : ""; + + if (!engine || !engine.startsWith("tts.")) { + this._lastHaTtsAttempt = "ha_tts_engine_must_be_a_tts_entity"; + return false; + } + if (!mediaPlayer || !mediaPlayer.startsWith("media_player.")) { + this._lastHaTtsAttempt = "voice_output_media_player_must_be_a_media_player_entity"; + return false; + } + + try { + await this._hass.callService( + "tts", + "speak", + { + media_player_entity_id: mediaPlayer, + message: text, + cache: true, + }, + { entity_id: engine } + ); + await this._waitForMediaPlayerPlayback(mediaPlayer); + this._lastHaTtsAttempt = `engine=${engine}, media_player=${mediaPlayer}, ok`; + return true; + } catch (err) { + console.debug("OpenClaw: media-player TTS failed", err); + this._lastHaTtsAttempt = + `engine=${engine}, media_player=${mediaPlayer}, service_call_failed`; + return false; + } + } + + async _waitForMediaPlayerPlayback(entityId) { + const activeStates = new Set(["buffering", "playing"]); + const startedAt = Date.now(); + const activityDeadline = startedAt + 10_000; + const timeoutAt = startedAt + 120_000; + let sawPlayback = activeStates.has(this._hass?.states?.[entityId]?.state); + + while (Date.now() < timeoutAt) { + const state = this._hass?.states?.[entityId]?.state; + if (activeStates.has(state)) { + sawPlayback = true; + } else if (sawPlayback) { + return; + } else if (Date.now() >= activityDeadline) { + // Some players complete short clips without exposing a playing state to + // the frontend. Avoid blocking voice input for the full timeout. + return; + } + await new Promise((resolve) => setTimeout(resolve, 500)); + } + } + // ── File attachments ──────────────────────────────────────────────── _handleFileAttachment() { @@ -2232,7 +2315,7 @@ class OpenClawChatCardEditor extends HTMLElement { :host { display: block; } .row { display: flex; align-items: center; padding: 8px 0; } .row label { flex: 1; font-size: 14px; color: var(--primary-text-color); } - .row input[type="text"] { + .row input[type="text"], .row select { flex: 1; padding: 6px 10px; border: 1px solid var(--divider-color, #ccc); @@ -2258,6 +2341,21 @@ class OpenClawChatCardEditor extends HTMLElement { +
+ + +
+
+ + +
+
+ + +
@@ -2274,7 +2372,14 @@ class OpenClawChatCardEditor extends HTMLElement { `; // Bind events - for (const id of ["title", "height", "session_id"]) { + for (const id of [ + "title", + "height", + "session_id", + "voice_output_mode", + "ha_tts_engine", + "voice_output_media_player", + ]) { const el = this.shadowRoot.getElementById(id); if (el) { el.addEventListener("change", (e) => this._fireChanged(id, e.target.value)); diff --git a/tests/test_chat_card_media_player.mjs b/tests/test_chat_card_media_player.mjs new file mode 100644 index 0000000..05c85db --- /dev/null +++ b/tests/test_chat_card_media_player.mjs @@ -0,0 +1,63 @@ +import assert from "node:assert/strict"; + +const registry = new Map(); + +globalThis.HTMLElement = class { + attachShadow() { + this.shadowRoot = { + innerHTML: "", + getElementById: () => null, + }; + } +}; +globalThis.customElements = { + define: (name, value) => registry.set(name, value), + get: (name) => registry.get(name), +}; +globalThis.window = { customCards: [] }; +globalThis.document = { createElement: () => ({}) }; +Object.defineProperty(globalThis, "navigator", { + configurable: true, + value: { language: "en-US" }, +}); + +await import("../custom_components/openclaw/www/openclaw-chat-card.js"); + +const Card = registry.get("openclaw-chat-card"); +assert.ok(Card, "chat card should register itself"); + +const calls = []; +const card = new Card(); +card._config = { + ha_tts_engine: "tts.openai_tts", + voice_output_media_player: "media_player.bedroom_speaker", +}; +card._hass = { + states: {}, + callService: async (...args) => calls.push(args), +}; +card._waitForMediaPlayerPlayback = async () => {}; + +assert.equal(await card._speakViaMediaPlayer("Hello from OpenClaw"), true); +assert.deepEqual(calls, [ + [ + "tts", + "speak", + { + media_player_entity_id: "media_player.bedroom_speaker", + message: "Hello from OpenClaw", + cache: true, + }, + { entity_id: "tts.openai_tts" }, + ], +]); + +const invalidCard = new Card(); +invalidCard._config = { + ha_tts_engine: "openai_tts", + voice_output_media_player: "bedroom_speaker", +}; +invalidCard._hass = { callService: async () => assert.fail("must not call service") }; + +assert.equal(await invalidCard._speakViaMediaPlayer("Hello"), false); +assert.equal(invalidCard._lastHaTtsAttempt, "ha_tts_engine_must_be_a_tts_entity"); diff --git a/www/openclaw-chat-card.js b/www/openclaw-chat-card.js index eb7489d..3442647 100644 --- a/www/openclaw-chat-card.js +++ b/www/openclaw-chat-card.js @@ -1,7 +1,7 @@ (async () => { try { if (!customElements.get("openclaw-chat-card")) { - const src = "/openclaw/openclaw-chat-card.js?v=0.1.63"; + const src = "/openclaw/openclaw-chat-card.js?v=0.1.65-rc.2"; console.info("OpenClaw loader importing", src); await import(src); } From 5a0cda3efe8a79c2673c086e4b8c574ce409a482 Mon Sep 17 00:00:00 2001 From: TechArtDev Date: Sat, 26 Sep 2026 22:41:55 +0300 Subject: [PATCH 2/3] fix: handle already-playing media targets --- .../openclaw/www/openclaw-chat-card.js | 14 ++++++++++---- tests/test_chat_card_media_player.mjs | 6 ++++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/custom_components/openclaw/www/openclaw-chat-card.js b/custom_components/openclaw/www/openclaw-chat-card.js index 362bdcf..08f4564 100644 --- a/custom_components/openclaw/www/openclaw-chat-card.js +++ b/custom_components/openclaw/www/openclaw-chat-card.js @@ -1806,6 +1806,7 @@ class OpenClawChatCard extends HTMLElement { } try { + const initialPlayerState = this._hass?.states?.[mediaPlayer]?.state; await this._hass.callService( "tts", "speak", @@ -1816,7 +1817,7 @@ class OpenClawChatCard extends HTMLElement { }, { entity_id: engine } ); - await this._waitForMediaPlayerPlayback(mediaPlayer); + await this._waitForMediaPlayerPlayback(mediaPlayer, initialPlayerState); this._lastHaTtsAttempt = `engine=${engine}, media_player=${mediaPlayer}, ok`; return true; } catch (err) { @@ -1827,16 +1828,21 @@ class OpenClawChatCard extends HTMLElement { } } - async _waitForMediaPlayerPlayback(entityId) { + async _waitForMediaPlayerPlayback(entityId, initialState) { const activeStates = new Set(["buffering", "playing"]); const startedAt = Date.now(); const activityDeadline = startedAt + 10_000; const timeoutAt = startedAt + 120_000; - let sawPlayback = activeStates.has(this._hass?.states?.[entityId]?.state); + const initiallyActive = activeStates.has(initialState); + let sawPlayback = false; + let sawStateChange = false; while (Date.now() < timeoutAt) { const state = this._hass?.states?.[entityId]?.state; - if (activeStates.has(state)) { + if (state !== initialState) { + sawStateChange = true; + } + if (activeStates.has(state) && (!initiallyActive || sawStateChange)) { sawPlayback = true; } else if (sawPlayback) { return; diff --git a/tests/test_chat_card_media_player.mjs b/tests/test_chat_card_media_player.mjs index 05c85db..c7a90bc 100644 --- a/tests/test_chat_card_media_player.mjs +++ b/tests/test_chat_card_media_player.mjs @@ -33,10 +33,11 @@ card._config = { voice_output_media_player: "media_player.bedroom_speaker", }; card._hass = { - states: {}, + states: { "media_player.bedroom_speaker": { state: "off" } }, callService: async (...args) => calls.push(args), }; -card._waitForMediaPlayerPlayback = async () => {}; +let waitArgs; +card._waitForMediaPlayerPlayback = async (...args) => { waitArgs = args; }; assert.equal(await card._speakViaMediaPlayer("Hello from OpenClaw"), true); assert.deepEqual(calls, [ @@ -51,6 +52,7 @@ assert.deepEqual(calls, [ { entity_id: "tts.openai_tts" }, ], ]); +assert.deepEqual(waitArgs, ["media_player.bedroom_speaker", "off"]); const invalidCard = new Card(); invalidCard._config = { From eefd51cbbc19344426ea299734d6ae81f4697f7f Mon Sep 17 00:00:00 2001 From: TechArtDev Date: Sat, 26 Sep 2026 22:42:17 +0300 Subject: [PATCH 3/3] chore: refresh media-player TTS release candidate --- CHANGELOG.md | 2 +- custom_components/openclaw/__init__.py | 2 +- custom_components/openclaw/manifest.json | 2 +- custom_components/openclaw/www/openclaw-chat-card.js | 2 +- www/openclaw-chat-card.js | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c888bcd..a75f1f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ All notable changes to the OpenClaw Home Assistant Integration will be documented in this file. -## [0.1.65-rc.2] - 2026-09-26 +## [0.1.65-rc.3] - 2026-09-26 ### Added - Added optional `voice_output_mode: media_player` chat-card output. Replies can diff --git a/custom_components/openclaw/__init__.py b/custom_components/openclaw/__init__.py index 4f80436..1a009c3 100644 --- a/custom_components/openclaw/__init__.py +++ b/custom_components/openclaw/__init__.py @@ -107,7 +107,7 @@ # URL at which the card JS is served (registered via register_static_path) _CARD_STATIC_URL = f"/openclaw/{_CARD_FILENAME}" # Versioned URL used for Lovelace resource registration to avoid stale browser cache -_CARD_URL = f"{_CARD_STATIC_URL}?v=0.1.65-rc.2" +_CARD_URL = f"{_CARD_STATIC_URL}?v=0.1.65-rc.3" OpenClawConfigEntry = ConfigEntry diff --git a/custom_components/openclaw/manifest.json b/custom_components/openclaw/manifest.json index d843c08..5125e95 100644 --- a/custom_components/openclaw/manifest.json +++ b/custom_components/openclaw/manifest.json @@ -8,7 +8,7 @@ "iot_class": "local_polling", "issue_tracker": "https://github.com/techartdev/OpenClawHomeAssistantIntegration/issues", "requirements": [], - "version": "0.1.65-rc.2", + "version": "0.1.65-rc.3", "dependencies": ["conversation"], "after_dependencies": ["hassio", "lovelace"] } diff --git a/custom_components/openclaw/www/openclaw-chat-card.js b/custom_components/openclaw/www/openclaw-chat-card.js index 08f4564..763b04a 100644 --- a/custom_components/openclaw/www/openclaw-chat-card.js +++ b/custom_components/openclaw/www/openclaw-chat-card.js @@ -13,7 +13,7 @@ * + subscribes to openclaw_message_received events. */ -const CARD_VERSION = "0.3.14"; +const CARD_VERSION = "0.3.15"; // Max time (ms) to show the thinking indicator before falling back to an error (default; overridable via card config `thinking_timeout` in seconds) const THINKING_TIMEOUT_MS = 120_000; diff --git a/www/openclaw-chat-card.js b/www/openclaw-chat-card.js index 3442647..40078bc 100644 --- a/www/openclaw-chat-card.js +++ b/www/openclaw-chat-card.js @@ -1,7 +1,7 @@ (async () => { try { if (!customElements.get("openclaw-chat-card")) { - const src = "/openclaw/openclaw-chat-card.js?v=0.1.65-rc.2"; + const src = "/openclaw/openclaw-chat-card.js?v=0.1.65-rc.3"; console.info("OpenClaw loader importing", src); await import(src); }