From 659c470eba542383987aad227b7f918100f131fa Mon Sep 17 00:00:00 2001
From: wangzhibo <wangzhibo@shsening.com>
Date: 星期五, 11 九月 2026 15:09:05 +0800
Subject: [PATCH] 小智终端 5种 场景模式 OK
---
xiaozhi-esp32-server/main/xiaozhi-server/core/handle/helloHandle.py | 8
xiaozhi-esp32-2.2.6/main/boards/common/wifi_board.cc | 13
xiaozhi-esp32-2.2.6/main/audio/audio_codec.h | 1
xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dto/TaskInstanceListItemDTO.java | 24
phase-2-voice-test.md | 175
xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dto/TaskEventDTO.java | 20
xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/security/config/ShiroConfig.java | 2
xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/app_task.h | 15
xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/mqtt.c | 97
xiaozhi-esp32-server/main/xiaozhi-server/tests/conftest.py | 66
xiaozhi-esp32-server/main/xiaozhi-server/core/providers/asr/base.py | 39
xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/service/TaskInstanceService.java | 259 +
xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/mqtt_interface.md | 4
xiaozhi-esp32-server/main/xiaozhi-server/core/providers/tools/device_mcp/mcp_handler.py | 9
xiaozhi-esp32-2.2.6/sdkconfig | 3
phase-3-test.md | 216 +
xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/controller/TaskInstanceController.java | 53
xiaozhi-esp32-server/main/xiaozhi-server/core/app_runtime/playback.py | 312 +
phase-3-review.md | 345 +
xiaozhi-esp32-2.2.6/main/audio/audio_codec.cc | 6
xiaozhi-esp32-server/main/xiaozhi-server/core/utils/audioRateController.py | 24
xiaozhi-esp32-2.2.6/main/audio/audio_service.h | 5
xiaozhi-esp32-2.2.6/main/audio/audio_service.cc | 49
roadmap.md | 603 ++
xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/entity/TaskEventEntity.java | 29
xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dto/TaskTurnDTO.java | 37
xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/entity/TaskStepEntity.java | 49
xiaozhi-esp32-server/main/xiaozhi-server/core/providers/tts/base.py | 152
xiaozhi-esp32-server/main/xiaozhi-server/core/websocket_handshake.py | 56
xiaozhi-esp32-2.2.6/main/application.cc | 609 ++
xiaozhi-esp32-server/main/xiaozhi-server/core/websocket_server.py | 37
phase-4-plan.md | 349 +
xiaozhi-esp32-server/main/manager-api/src/main/resources/db/changelog/202609011100.sql | 6
phase-1-review.md | 475 ++
xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dao/TaskStepDao.java | 17
xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/mp3_player.cc | 529 ++
xiaozhi-esp32-server/main/xiaozhi-server/tests/test_audio_archive.py | 59
xiaozhi-esp32-server/main/xiaozhi-server/core/connection.py | 90
xiaozhi-esp32-server/main/xiaozhi-server/core/handle/sendAudioHandle.py | 37
xiaozhi-esp32-2.2.6/scripts/patch_wifi_connect.py | 110
xiaozhi-esp32-server/main/xiaozhi-server/core/audio_archive.py | 103
xiaozhi-esp32-2.2.6/main/protocols/websocket_protocol.cc | 23
xiaozhi-esp32-2.2.6/main/CMakeLists.txt | 18
xiaozhi-esp32-server/main/xiaozhi-server/core/app_runtime/assessment.py | 387 +
xiaozhi-esp32-server/main/xiaozhi-server/core/app_runtime/apps.json | 113
xiaozhi-esp32-server/main/xiaozhi-server/tests/test_assessment.py | 544 ++
.cursor/rules/xiaozhi-locked-modes.mdc | 32
phase-2-plan.md | 425 ++
xiaozhi-esp32-server/main/xiaozhi-server/core/handle/textHandler/listenMessageHandler.py | 75
xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dto/TaskInstanceUpsertDTO.java | 53
xiaozhi-esp32-2.2.6/main/audio/wake_words/afe_wake_word.cc | 32
xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/system.h | 54
xiaozhi-esp32-server/main/xiaozhi-server/core/utils/modules_initialize.py | 11
xiaozhi-esp32-server/main/xiaozhi-server/tests/test_playback.py | 384 +
xiaozhi-esp32-2.2.6/main/application.h | 19
xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/system.c | 89
xiaozhi-esp32-server/main/xiaozhi-server/core/handle/reportHandle.py | 18
xiaozhi-esp32-server/main/xiaozhi-server/config/assets/wakeup_words/ed76d459636c2481aec828516c1b4f54.wav | 0
xiaozhi-esp32-server/main/xiaozhi-server/core/handle/receiveAudioHandle.py | 84
xiaozhi-esp32-server/main/manager-api/src/main/resources/db/changelog/202504151206.sql | 2
xiaozhi-esp32-server/main/xiaozhi-server/config/manage_api_client.py | 46
xiaozhi-esp32-2.2.6/main/protocols/protocol.h | 6
xiaozhi-esp32-server/main/xiaozhi-server/core/app_runtime/runtime.py | 1477 ++++++
Xiaozhi Device Capability Specification v1.0.md | 622 +++
xiaozhi-esp32-2.2.6/main/Kconfig.projbuild | 2
xiaozhi-esp32-server/main/xiaozhi-server/tests/test_app_runtime.py | 474 ++
xiaozhi-esp32-2.2.6/main/boards/waveshare/esp32-s3-touch-lcd-4.3c/sdkconfig.4_3c | 3
xiaozhi-esp32-2.2.6/main/protocols/websocket_protocol.h | 1
xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/agent/service/biz/impl/AgentChatHistoryBizServiceImpl.java | 18
xiaozhi-esp32-server/main/xiaozhi-server/core/app_runtime/registry.py | 101
xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentChatHistoryReportDTO.java | 4
xiaozhi-esp32-server/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml | 14
phase-3-plan.md | 380 +
xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dao/TaskInstanceDao.java | 11
phase-4-test.md | 149
xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dto/TaskMetricDTO.java | 12
xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dto/TaskInstanceListDTO.java | 12
xiaozhi-esp32-server/main/xiaozhi-server/core/app_runtime/__init__.py | 5
xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/sening_xiaozhi_s3.cc | 141
xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dao/TaskEventDao.java | 11
xiaozhi-esp32-2.2.6/main/audio/wake_words/custom_wake_word.cc | 24
xiaozhi-esp32-server/main/xiaozhi-server/config.yaml | 16
xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/entity/TaskInstanceEntity.java | 54
xiaozhi-esp32-server/main/xiaozhi-server/plugins_func/functions/switch_app.py | 19
xiaozhi-esp32-2.2.6/main/protocols/protocol.cc | 28
phase-2-review.md | 388 +
xiaozhi-esp32-server/main/xiaozhi-server/core/app_runtime/session.py | 275 +
xiaozhi-esp32-server/main/xiaozhi-server/plugins_func/functions/start_shadowing.py | 42
xiaozhi-esp32-server/main/manager-api/src/main/resources/db/changelog/202608311800.sql | 52
xiaozhi-esp32-server/main/xiaozhi-server/plugins_func/functions/start_repeater.py | 29
xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/app_task.cc | 513 ++
91 files changed, 11,822 insertions(+), 562 deletions(-)
diff --git a/.cursor/rules/xiaozhi-locked-modes.mdc b/.cursor/rules/xiaozhi-locked-modes.mdc
new file mode 100644
index 0000000..7dfb907
--- /dev/null
+++ b/.cursor/rules/xiaozhi-locked-modes.mdc
@@ -0,0 +1,32 @@
+---
+description: 宸查獙鏀舵ā寮忎笉鍙洖閫�锛涘彧鏀瑰璇�/鍚娓呭崟涓庢挱鏀鹃�昏緫
+alwaysApply: true
+---
+
+# 宸查攣瀹氾細涓嶈鏀硅繖浜涘凡姝ｇ‘璺緞
+
+寮�鏈烘杩庤瘝銆佸璇濇ā寮忋�佺炕璇戞ā寮忋�佷腑鑻卞璇濆凡缁忛獙鏀堕�氳繃銆傚悗缁彧璁告敼澶嶈鏈� / 鍚锛堣窡璇伙級銆備笉瑕佷负淇璇�/鍚鍘绘敼 chat銆乼ranslation銆乥ilingual銆乭ello/TTS 娆㈣繋璇嶆椂搴忋��
+
+涓嶈鍔細
+
+- 寮�鏈猴細寮�閫氶亾鍚庝繚鎸� idle锛孲erver 鎾杩庤瘝锛岃瀹屽啀 listening銆備笉瑕佹棤鏉′欢 `ResumeListening` / `WakeWordInvoke`銆�
+- `interaction_mode == CHAT | TRANSLATION | BILINGUAL` 鐨� ASR鈫扡LM/缈昏瘧 璺緞銆�
+- 娆㈣繋璇� TTS锛氬厛鏈� opus 鍐嶅彂 start/瀛楀箷銆�
+
+# 澶嶈鏈� PLAY_AUDIO
+
+JSON task 涓嬪彂鎾斁鍒楄〃锛屾瘡椤� `{url|name, repeat, speed}`锛宻peed 榛樿 1.0銆傛祴璇曟竻鍗曪細
+
+- `/english_learning/shadowing/lessons/unit01/01.mp3` 脳 10
+- `/english_learning/shadowing/lessons/unit01/02.mp3` 脳 5
+
+鎾畬鑷姩鍥炲璇濓紝鎾�屽璇荤粨鏉熴�嶃�傛挱鏀句腑蹇界暐闂茶亰锛屽彧鍝嶅簲绂荤嚎鍙ｄ护锛氶��鍑哄璇汇�佹垜浠潵鑱婂ぉ鍚с��
+
+# 鍚 SHADOWING
+
+JSON task 涓�娆℃�т笅鍙戞暣浠藉垪琛紝姣忛」 `{url|name, repeat, speed}`锛宻peed 榛樿 1.0銆備箣鍚庡湪璁惧涓婃帓搴忔墽琛岋紝涓嶈姣忔挱涓�鍙ュ啀 MCP 涓�娆°�傛祴璇曟竻鍗曪細
+
+- `/english_learning/shadowing/lessons/unit01/01.mp3` 脳 5锛堟瘡閬嶅悗褰曢煶锛�
+- `/english_learning/shadowing/lessons/unit01/02.mp3` 脳 10
+
+璁惧鏈湴寰幆锛氭挱涓�鍙� 鈫� 鍚獥褰曢煶锛堝閽熸挱鏀炬椂闀� 脳 1.2锛夆啋 褰曢煶涓婁紶 鈫� 鍐嶆挱涓嬩竴鍙ワ紝鐩村埌娓呭崟缁撴潫銆侫SR 鍙殏瀛�/璇勪及锛屼笉杩� chat/LLM銆傛竻鍗曠粨鏉熸挱銆岃窡璇荤粨鏉熴�嶅啀鍥炲璇濄��
diff --git a/Xiaozhi Device Capability Specification v1.0.md b/Xiaozhi Device Capability Specification v1.0.md
new file mode 100644
index 0000000..dd5d5ce
--- /dev/null
+++ b/Xiaozhi Device Capability Specification v1.0.md
@@ -0,0 +1,622 @@
+# Xiaozhi Device Capability Specification v1.0
+
+> Status: **Freeze Analysis锛堥拤姝昏涔夛級**
+> Date: 2026-08-31
+> Based on: Phase 1鈥�2 鐪熷疄浠ｇ爜 路 Master Plan 路 鏁欒偛 / 鐩婃櫤 / 闄即 / 鍔╂墜鍥涚被 App 鍙嶅悜褰掔撼
+> 鏈枃涓嶆槸鍙堜竴浠� Master Plan锛屼篃涓嶆槸鏌愪竴 Phase 鐨� Coding Plan銆�
+> 鏈枃鍥炵瓟锛�**浠ュ悗鏂板 App锛屽浐浠跺師鍒欎笂涓嶅啀鏀逛粈涔堛��**
+
+**Phase 2 宸插畬鎴愶紙瑙� `phase-2-review.md`锛夛細** D1 瀹屾垚浜嬩欢銆丏6/D7 v1 璺熻閲囬泦銆丏8 v1 鏂囨湰褰掓。锛汼D 鎸� App 鍞竴鍚嶅缓鏍戙��
+**涓嬩竴缂栫爜闃舵鏄� `phase-3-plan.md`锛�** TaskInstance 钀� manager-api锛�**鏃犳柊 Device 鑳藉姏銆佷笉鏀瑰浐浠躲��**
+
+瑕嗙洊鐭╅樀鐨勫彲瑙嗗寲瑙� Canvas锛歔鑳藉姏瑕嗙洊鐭╅樀](C:\Users\shi\.cursor\projects\d-xiaozhiAI\canvases\xiaozhi-device-capability-spec.canvas.tsx)銆�
+
+---
+
+## 0. 鍐荤粨鍘熷垯
+
+```text
+鏂板 App
+ 鈫� 鍙敼 Server锛欰pp / Task / Step / Assessment / Resource
+ 鈫� Device锛氳鍙栭厤缃垨鎵ц宸叉湁 Capability
+ 鈫� 绂佹锛氫负鍗曚釜 App 鏀� C++銆佹敼鍗忚瀛楁鍚箟銆佸己鍒� OTA
+```
+
+鍏佽 OTA 鐨勫敮涓�鐞嗙敱锛氳ˉ榻� **鏈� Spec 宸插０鏄庛�佸綋鍓嶅浐浠跺皻鏈叿澶�** 鐨勫師瀛愯兘鍔涳紙渚嬪缁熶竴 `play_audio` 瀹屾垚浜嬩欢銆乴isten 瀹屾垚浜嬩欢锛夈�備笉鍏佽涓恒�屾垚璇帴榫欍�嶅崟鐙� OTA銆�
+
+涓夊眰妯″瀷锛�
+
+```text
+Application 鏁欒偛 / 鐩婃櫤 / 闄即 / 鍔╂墜 鈫� 鏃犻檺鎵╁睍锛屽叏鍦� Server + 璧勬簮
+ 鈫�
+Task Runtime Task / Step / Flow / 璇勪及 鈫� 灏藉揩绋冲畾锛屾潈濞佸湪 Server
+ 鈫�
+Device Capability 鎾� / 鍚� / VAD / 绛夊緟 / 鎸囩ず 鈫� 鐜板湪鍐荤粨璇箟
+```
+
+鍐嶆媶涓�灞傦紝閬垮厤鎶娿�岄害鍏嬮銆嶇洿鎺ュ綋 API锛�
+
+```text
+Device Primitive I2S / 鎸夐敭 GPIO / LED / SD / 缃戠粶
+ 鈫�
+Interaction Capability play_audio / listen / vad / speak / wait / indicate
+ 鈫�
+Task Capability 璺熻涓�杞�佸彛绠椾竴棰樸�佹帴榫欎竴鍥炲悎 鈫� 绂佹鍐欒繘鍥轰欢
+```
+
+**鍒ゅ畾瑙勫垯锛�** 鑻ュ畬鎴愭潯浠朵緷璧栥�岀瓟妗堟槸鍚︽纭� / 璇箟鏄惁鍚堟牸 / 璇ヤ笉璇ヤ笅涓�棰樸�嶏紝杩欐槸 Server Task Capability锛屼笉鏄� Device Capability銆�
+
+---
+
+## 1. Device 鍘熷瓙鑳藉姏娓呭崟
+
+### 1.1 蹇呴』鍐荤粨锛圖evice Interaction Capability锛�
+
+| ID | 鑳藉姏 | 涓�鍙ヨ瘽 |
+|----|------|--------|
+| D1 | `play_audio` | 鎾斁涓�椤归煶棰戣祫婧愶紝鍙惊鐜紝蹇呮湁瀹屾垚/鍋滄浜嬩欢 |
+| D2 | `audio.pause` / `resume` / `stop` | 鎵撴柇鎴栨寕璧峰綋鍓嶆挱鏀炬簮锛堜笌 TTS 浜掓枼锛� |
+| D3 | `speak` | 鏈楄鏂囨湰锛涘疄鐜板彲浠ユ槸浜� TTS锛岃澶囧彧璐熻矗鍑哄０涓庢墦鏂� |
+| D4 | `barge_in` | 鎾斁涓娴嬪埌鐢ㄦ埛璇磋瘽 鈫� 鍋滄挱 鈫� Listening |
+| D5 | `play_system_sound` | 缁熶竴鎻愮ず闊筹紝App 涓嶅緱鑷甫 ding 瀹炵幇 |
+| D6 | `listen` | 涓婅璇煶娴侊紙鐜版湁 Opus锛夛紝鍙寜 Mode 涓嶈繘 Chat |
+| D7 | `vad` | 涓婃姤鐢ㄦ埛寮�濮嬭 / 鍋滄璇达紙纭欢鍙娴嬶級 |
+| D8 | `record_clip` | v1 鍙敤 listen 娴� + Server 褰掓。锛泇1.1 鏈湴鏂囦欢 + 涓婁紶 |
+| D9 | `button` | 鐭寜 / 闀挎寜 / 鍙屽嚮 鈫� 寮�濮嬨�佸仠姝€�侀��鍑� Task |
+| D10 | `indicate` | 鎶婇�昏緫鎬佹槧灏勫埌 LED/灞忥紝App 涓嶅啓 GPIO |
+| D11 | `wait` | 鍙彇娑堢殑鏈湴寤舵椂 |
+| D12 | `volume` | 鎾斁闊抽噺 |
+| D13 | `resource.fetch` | 鎶婅祫婧愯惤鍒� SD/缂撳瓨锛堢幇鏈� DOWNLOAD 鐨勮涔夊崌绾э級 |
+| D14 | `task_control` | start / pause / resume / stop / restart **褰撳墠 Task 鎵ц鍣�** |
+
+### 1.2 鍏佽鍦� Device 鎵ц銆佷絾鍙傛暟鏉ヨ嚜閰嶇疆鐨勬帶鍒剁粨鏋�
+
+| ID | 鑳藉姏 | 鑼冨洿 |
+|----|------|------|
+| D15 | `loop`锛堟挱鏀剧骇锛� | `play_audio(loop=N)`锛屽畬鎴愭潯浠� = 鎾畬 N 娆°�侾hase 1 宸叉湁 `repeat` |
+| D16 | `loop`锛堟楠ょ骇锛� | **Server Task Runtime** 閲嶅涓嬪彂 Step锛涜澶囦笉瑙ｉ噴銆屽啀鍋氫竴棰樸�� |
+
+### 1.3 绂佹杩涘叆 Device 鐨勮兘鍔涳紙Server 鍘熷瓙鑳藉姏锛�
+
+| ID | 鑳藉姏 | 鍘熷洜 |
+|----|------|------|
+| S1 | ASR | 璁惧鍙湁 PCM/Opus |
+| S2 | LLM | 鈥� |
+| S3 | Translation | Phase 1 宸茶瘉鏄庤蛋 Server |
+| S4 | Assessment / Parser / 鍒ら | 鍙ｇ畻銆佸�掕儗銆佹帴榫欏閿欓兘涓嶅湪鍥轰欢 |
+| S5 | Memory / Knowledge / RAG | 鈥� |
+| S6 | Task Generation / 鍔ㄦ�佸嚭棰� | 鈥� |
+| S7 | Speech feature 鈫� 涓村簥璇婃柇 | 鐗瑰緛鍙悗鎶斤紝璇婃柇姘镐笉涓婅澶� |
+| S8 | `branch(if correct)` | 璁惧涓嶇煡閬� correct锛涜嫢灏嗘潵鍋氱簿缁冩ā寮忥紝鍙墽琛屻�屽啀鎾� / 涓嬩竴棰樸�嶅懡浠ゃ��**榛樿鏁欒偛 App 鐑矾寰勪笉鐢� S8**锛氬畾鏃剁粨鏉熷嵆涓嬩竴姝ワ紙瑙� phase-4-plan锛� |
+
+---
+
+## 2. 姣忎釜鑳藉姏鐨勮緭鍏� / 杈撳嚭
+
+涓嬪垪涓� **璇箟濂戠害**銆侻CP 鍚嶃�丮QTT action銆佹湭鏉� `type=command` 閮藉彧鏄繍杈撱�傛敼杩愯緭涓嶇畻寮曡繘鏂拌兘鍔涖��
+
+### D1 `play_audio`
+
+```text
+in:
+ resource: { kind: sd_path | http | asset_id, uri }
+ volume?: 0-100
+ loop?: int >= 1 # 榛樿 1锛涘湪璁惧涓婂惊鐜�
+ speed?: 0.5-2.0 float # 榛樿 1.0锛涜秴鍑哄す绱с�傞噸閲囨牱瀹炵幇锛屼换鎰忔诞鐐癸紝闈炵鏁ｆ。锛涢潪 1.0 浼氬彉璋�
+ power_off_after?: bool # 宸叉湁杞叧鏈烘寕閽╋紝鍙繚鐣欎负鎾斁绛栫暐鑰岄潪 App 閫昏緫
+
+out (events):
+ audio.started
+ audio.completed # 鑷劧缁撴潫锛堝惈 loop 鍏ㄩ儴瀹屾垚锛�
+ audio.stopped # 琚� stop / 鎶㈠崰 / 鐢ㄦ埛閫�鍑�
+ audio.paused
+ audio.resumed
+ audio.failed # 缂烘枃浠躲�佽В鐮佸け璐�
+```
+
+**褰撳墠浠ｇ爜锛�** Sening `self.mp3.play(name, speed, repeat, power_off_after)`锛汸hase 2 宸蹭笂鎶� `audio.completed` / `audio.stopped`锛堣繍杈擄細`[skill_event]...|duration_ms`锛夈�傛棤缁熶竴 WAV/HTTP play銆�
+
+**鍐荤粨缂哄彛锛堜粛寮�鏀撅級锛�** 璧勬簮 ID 鑰屼笉鏄笟鍔¤矾寰勬暎钀斤紱闈� Sening 鏉挎棤姝� Skill锛沗audio.paused/resumed/failed` 鏈綔涓虹嫭绔嬩簨浠躲��
+
+### D2 鎾斁鎺у埗
+
+```text
+audio.pause | audio.resume | audio.stop
+```
+
+褰撳墠锛歚self.mp3.stop`銆乀TS 鏃� `PauseSdCardMp3`銆俻ause/resume 瀵� SD MP3 宸叉湁銆孴TS 鏆傚仠銆嶈涔夛紝闇�鎻愬崌涓哄彲琚� Task 璋冪敤锛岃�屼笉浠呮槸 Chat 鍓綔鐢ㄣ��
+
+### D3 `speak`
+
+```text
+in: text, optional interruptible: bool
+out: speak.started, speak.completed, speak.interrupted
+```
+
+褰撳墠锛歋erver TTS 鈫� `type=tts` + Opus銆傝澶囨病鏈夌嫭绔嬨�屾湰鍦� TTS Skill銆嶃��**鍐荤粨锛氳澶囪礋璐ｅ嚭澹颁笌 barge_in锛屼笉璐熻矗鍚堟垚銆�** App/Task 鍙啓 `speak("璇峰洖绛�")`銆�
+
+### D4 `barge_in`
+
+```text
+policy (鐢� InteractionMode 閰嶇疆锛屼笉鏄� App 鍚�):
+ CHAT / BILINGUAL 鍏佽鎵撴柇 TTS
+ PLAY_AUDIO 榛樿涓嶅厑璁告妸鍠囧彮褰撶敤鎴锋彃璇濓紙澶嶈锛�
+ FOLLOW_READ/SHADOWING 鎾斁鑼冭鏃跺拷鐣ユ嬀闊筹紱鑼冭缁撴潫鍚庢墠 listen
+ TRANSLATION 鎾瘧鏂囨椂蹇界暐鍥炲０
+```
+
+褰撳墠锛歋erver 宸叉寜 mode 闄愬埗 AEC 鎵撴柇銆傝繖鏄� **Mode 绛栫暐**锛屽繀椤诲喕缁撹繘 Runtime锛岀姝㈡瘡涓� App 鑷繁 if銆�
+
+### D5 `play_system_sound`
+
+```text
+in: type = ding | beep | success | error | ...
+out: sound.completed
+```
+
+褰撳墠锛歚AudioService::PlaySound` OGG銆傞渶缂栨垚绋冲畾 type锛岃�屼笉鏄悇鏉胯矾寰勫瓧绗︿覆銆�
+
+### D6 `listen` / D7 `vad`
+
+```text
+listen.start(mode)
+listen.stop
+vad 鈫� user.speech.started | user.speech.stopped
+```
+
+褰撳墠锛氫笂琛� Opus + Server VAD锛涜澶囩涔熸湁 AFE/VAD 鐢ㄤ簬鍞ら啋銆�
+**鍐荤粨 v1锛�** 璺熻/鍙ｇ畻缁х画鐢ㄧ幇鏈夊惉娴� + Server VAD 浜嬩欢鍗冲彲锛屼笉寮哄埗璁惧鍐嶅仛涓�濂楁枃浠跺綍闊炽��
+**蹇呴』琛ョ殑浜嬩欢锛�** Server锛堟垨璁惧锛夊湪 Mode鈮燙HAT 鏃朵篃瑕佽兘鍖哄垎銆岀敤鎴疯瀹屼竴鍙ャ�嶏紝鍚﹀垯灏侀棴寮忚瘎浼版棤娉曞垏 Step銆�
+
+### D8 `record_clip`
+
+鍒嗕袱妗ｏ紝閬垮厤鎶婂喕缁撳崱姝诲湪銆岃澶囧啓 WAV銆嶏細
+
+| 妗� | 琛屼负 | 瑕嗙洊 |
+|----|------|------|
+| v1 | listen 娴� + Server `report` 褰掓。锛堝凡鏈� chat audio BLOB锛� | 璺熻銆佸彛绠椼�佸�掕儗銆佹帴榫� |
+| v1.1 | `save_local` + `upload` + `record.completed(audio_ref)` | 寮辩綉銆佷細璁邯瑕併�佸緥甯堣闂暱褰曢煶 |
+
+v1.1 **涓嶉樆濉�** 鐢� 5 涓唬琛ㄦ�� Task 鍐荤粨纭欢銆備細璁邯瑕佺被鍙鍔犮�岄暱褰曢煶涓婁紶銆嶏紝涓嶅鍔犳柊浼犳劅鍣ㄣ��
+
+### D9 `button`
+
+```text
+short 鈫� toggle listen / 纭
+long 鈫� 閫�鍑哄綋鍓� Task 鍥� chat锛堝彲閰嶏級
+double 鈫� 棰勭暀 pause
+```
+
+褰撳墠锛欱OOT 閿粦 `ToggleChatState`銆傚喕缁撴椂瑕佸厑璁搞�岄��鍑� Task銆嶈�屼笉鎷嗘帀 Chat 閿��
+
+### D10 `indicate`
+
+```text
+in: IDLE | LISTENING | PROCESSING | PLAYING | ERROR | TASK_RUNNING | TASK_COMPLETE
+```
+
+鏄犲皠鍒� LED/灞忔槸鏉跨骇瀹炵幇銆侫pp 鍙彂閫昏緫鎬併�傚綋鍓� DeviceState 鏄‖浠跺崰鐢ㄦ�侊紝**涓嶈**鎶� TASK_RUNNING 濉炶繘 `Listening/Speaking` 杩佺Щ琛ㄣ�俙indicate` 涓� `DeviceState` 姝ｄ氦銆�
+
+### D11 `wait`
+
+```text
+in: ms, cancellable
+out: wait.elapsed | wait.cancelled
+```
+
+褰撳墠锛�**鏃�**涓�绛� Skill銆傚璇诲惊鐜妸绛夊緟钘忓湪鎾斁鍣ㄥ唴閮ㄣ�傚喕缁撳悗銆屾挱鏀� 鈫� 绛夊緟 800ms 鈫� listen銆嶄笉寰楅潬 Server 杩炲彂涓ゆ鍐嶈嚜宸� sleep 鍏呮暟锛堝彲鍏� Server 鐫★紝浣嗗崗璁鐣� `wait`锛岄伩鍏嶆瘡涓� App 鍚勫啓鍚勭殑锛夈��
+
+### D12 `volume` / D13 `resource.fetch`
+
+褰撳墠锛歁CP 闊抽噺銆乻nlp `DOWNLOAD`銆傚喕缁撲负璧勬簮鍒拌揪 SD 鍚� `play_audio` 鍙鏈湴 `resource_id`/`sd_path`銆�
+
+### D14 `task_control`
+
+```text
+TASK_START / PAUSE / RESUME / STOP / RESTART
+```
+
+褰撳墠锛歚START_APP_TASK`銆乣STOP_MEDIA`銆佷紭鍏堢骇鎶㈠崰銆傛病鏈夌粺涓� PAUSE/RESUME **Task**锛堝彧鏈� MP3 琚� TTS 鏆傚仠锛夈��
+PAUSE 璇箟锛氬仠鎾斁涓庢嬀闊筹紝淇濈暀 pending 涓� loop 鍓╀綑娆℃暟銆係TOP锛氭竻鎵ц鍣紝pending 鍥� `chat.conversation`銆�
+
+### D15 vs D16 Loop
+
+```text
+鍚屼竴闊抽鍚� 10 閬� 鈫� Device play_audio.loop=10 锛堝凡楠岃瘉锛屼繚鎸侊級
+10 閬撲笉鍚屽彛绠� 鈫� Server 寰幆 Step锛屾瘡姝� speak+listen
+璺熻 10 鍙ヤ笉鍚岄煶棰� 鈫� Server 寰幆 Step锛堟垨 Task JSON 鍒楄〃锛�
+```
+
+绂佹 Server 瀵瑰悓涓�鏂囦欢杩炲彂 10 娆� `play`锛圥hase 1 宸插惁鍐筹級銆�
+
+---
+
+## 3. Client Runtime API锛堥�昏緫锛屼笉鏄珛鍒绘敼鍑芥暟鍚嶏級
+
+璁惧鎵ц鍣ㄥ彧鏆撮湶锛�
+
+```text
+Capability.execute(name, args) -> accepted
+Capability.cancel(name)
+on Event(name, payload)
+```
+
+瀵圭収鐜扮姸锛�
+
+| 閫昏緫 API | 浠婃棩钀界偣 |
+|----------|----------|
+| `play_audio` | `self.mp3.play` / `PLAY_MEDIA` |
+| `audio.stop` | `self.mp3.stop` / `STOP_MEDIA` |
+| `speak` | 琚姩鏀� `type=tts` |
+| `listen` | `SendStartListening` / 鐜版湁鎷鹃煶 |
+| `wait` | 鏃� |
+| `indicate` | Display/LED 鏁ｈ惤 |
+| `task_control` | `Board_StartAppTask` + preempt |
+| `resource.fetch` | MQTT DOWNLOAD |
+
+**鍐荤粨绛栫暐锛�** 鏂� App 鍙厑璁歌皟鐢ㄤ笂琛ㄣ�傛澘绾� `self.mp3.*` 浣滀负 `play_audio` 鐨� Sening 瀹炵幇锛岄�愭鏀瑰悕涓洪�昏緫鍚嶆垨淇濈暀鍒悕銆傜姝㈠啀鏂板 `self.english.*`銆�
+
+---
+
+## 4. Server Command锛堥�昏緫锛�
+
+Server 鈫� Device锛屾渶缁堣嚦灏戣繖浜� **鍛戒护璇箟**锛堣繍杈撳彲鍙橈級锛�
+
+```text
+TASK_START {app_id, task_id, params, source, instance_id?}
+TASK_PAUSE
+TASK_RESUME
+TASK_STOP
+TASK_RESTART
+PLAY_AUDIO {resource, loop, volume, speed}
+AUDIO_PAUSE | AUDIO_RESUME | AUDIO_STOP
+SPEAK {text} # 涔熷彲缁х画璧扮幇鏈� tts 甯э紝绠楀悓涓�鑳藉姏
+LISTEN_START | LISTEN_STOP
+WAIT {ms}
+INDICATE {state}
+SOUND {type}
+FETCH {url, dest}
+```
+
+浠婃棩绛変环鐗╋細`type=mcp tools/call`銆乣type=tts`銆丮QTT `START_APP_TASK`銆�
+**v1 杩愯緭鍐荤粨锛氬彲浠ョ户缁� MCP + MQTT锛屼笉寮哄埗鏂� WS type銆�**
+**v1 璇箟鍐荤粨锛氫笂琛ㄩ泦鍚堜笉鍐嶄负鏂� App 杩藉姞鍛戒护绉嶇被銆�**
+
+---
+
+## 5. Device Event锛堥�昏緫锛�
+
+Device 鈫� Server锛�
+
+```text
+TASK_STARTED | TASK_COMPLETED | TASK_FAILED | TASK_CANCELLED
+STEP_STARTED | STEP_COMPLETED | STEP_FAILED # Step 鏉冨▉浠嶅湪 Server锛涜澶囧彲鍙姤 skill 瀹屾垚
+AUDIO_STARTED | AUDIO_COMPLETED | AUDIO_STOPPED | AUDIO_FAILED
+SPEAK_COMPLETED | SPEAK_INTERRUPTED
+LISTEN_STARTED
+USER_SPEECH_STARTED | USER_SPEECH_STOPPED
+RECORD_COMPLETED {audio_ref?} # v1.1
+WAIT_ELAPSED
+BUTTON {kind: short|long|double}
+FETCH_PROGRESS | FETCH_COMPLETED | FETCH_FAILED
+```
+
+浠婃棩绛変环鐗╋細MCP result銆乣[app_task]` / 瑙勫垝涓殑 `[skill_event]`銆丮QTT `rpt/ack`銆�
+璺熻 v1 鍏佽 USER_SPEECH_* 鐢� **Server VAD** 浜х敓锛堣澶囨湭鎶ヤ篃涓嶇畻缂轰紶鎰熷櫒锛夈��
+
+---
+
+## 6. 鐘舵�佹満锛堜袱濂楋紝绂佹鍚堝苟锛�
+
+### 6.1 DeviceState锛堝凡鏈夛紝鍐荤粨杩佺Щ琛級
+
+```text
+Idle 鈫� Connecting 鈫� Listening 鈫� Speaking
++ Upgrading / WifiConfiguring / ...
+```
+
+鍙洖绛旓細楹﹀拰鍠囧彮鐜板湪褰掕皝銆俆ask 涓嶅緱鏂板 `kDeviceStateEnglish`銆�
+
+### 6.2 InteractionMode锛圫erver 宸叉湁瀛楃涓诧紝鍥轰欢鍙笉鏋氫妇锛�
+
+```text
+CHAT
+PLAY_AUDIO
+LISTEN # 鍙惉锛屼笉鑷姩 chat锛堣窡璇荤殑鍚崐鎷嶏紱鍙ｇ畻鍚瓟妗堬級
+FOLLOW_READ # 鍏堟挱鍚庡惉锛涗笌浠婃棩 SHADOWING 瀵归綈锛屽缓璁渶缁堢粺涓�鍛藉悕
+TRANSLATION
+BILINGUAL # 浠嶈蛋 LLM锛屼絾鏄� Mode 绛栫暐锛堟墦鏂�/鎻愮ず璇嶏級涓嶅悓
+```
+
+QUIZ/GAME **涓嶆槸**鏂� Mode銆傚彛绠�/鍊掕儗/鎺ラ緳 = `speak + listen + Server 璇勪及`锛孧ode 鐢� LISTEN 鎴� CHAT 绛栫暐寮�鍏� `skip_llm=true`銆�
+
+### 6.3 Task Executor锛堣澶囩紦瀛橈級
+
+```text
+IDLE 鈫� EXECUTING 鈫� WAITING_USER 鈫� COMPLETED | FAILED | CANCELLED
+PAUSE 鍙粠 EXECUTING/WAITING_USER 杩涘叆锛孯ESUME 杩斿洖
+```
+
+鏉冨▉鐘舵�佸湪 Server锛涜澶囩紦瀛樹粎鐢ㄤ簬鏂嚎 UI 涓庣绾垮璇汇��
+
+---
+
+## 7. 涓柇 / 鏆傚仠 / 鎭㈠
+
+| 鏉ユ簮 | 琛屼负 |
+|------|------|
+| 鐢ㄦ埛 barge_in锛圡ode 鍏佽锛� | 鍋� TTS/闊抽 鈫� listen |
+| 鐢ㄦ埛鎸夐挳闀挎寜 / 銆岄��鍑恒�� | TASK_STOP 鈫� chat.conversation |
+| 瀹堕暱 MQTT 鍗虫椂 | 浼樺厛绾� 3 鎶㈠崰锛孲TOP 褰撳墠 Skill 鍐� START |
+| 璁″垝浠诲姟 | 浼樺厛绾� 2锛屼笉鍙姠 MQTT 鍗虫椂 |
+| 璇煶鍒� App | 浼樺厛绾� 1锛屼笉鍙姠 2/3 |
+| Chat TTS vs 璇剧▼ MP3 | 宸叉湁 PauseSdCardMp3锛屼繚鎸佷负闊抽浜掓枼锛屼笉鏄� Task Pause |
+
+`TASK_PAUSE`锛氫繚鐣� loop 鍓╀綑銆佸綋鍓� step 绱㈠紩锛堢储寮曞湪 Server锛夈�傝澶囧彧鏆傚仠鎾斁/鎷鹃煶銆�
+
+---
+
+## 8. 绂荤嚎鎵ц
+
+鍒嗙被鍐荤粨锛圥hase 1 宸茬敤浜嬪疄璇佹槑锛夛細
+
+| 绫诲瀷 | 娌＄綉 | 渚嬪瓙 |
+|------|------|------|
+| 鍙绾� Skill | 璁惧鐩存帴鎵ц | 澶嶈 `play_audio` + loop |
+| 浠呭湪绾� Mode | 绂佹 WakeWordInvoke 骞茬瓑 | 缈昏瘧銆佽窡璇昏瘎浼般�佹帴榫欍�佸彛绠� |
+| 鍏ュ彛鍙ｄ护 | MultiNet 鍙繚鐣欐�诲叆鍙� + 鍙绾� Skill | 绯荤粺璇嶈〃鎸� `device.locale` 閰嶇疆锛圥hase 6b锛夛紱榛樿涓枃銆屽垏鎹㈡ā寮忋�嶃�屽璇绘満銆嶃��**绂佹**鎶婅窡璇�/缈昏瘧/璇█瀵瑰彛浠ゅ杩涘浐浠� |
+
+**绂佹**鎶婃墍鏈� App 鍙ｄ护濉炶繘 MultiNet銆�
+**绂佹**鎶� `apps.json` 褰撲綔绂荤嚎鏉冨▉涓氬姟寮曟搸锛汼D 涓婄殑 JSON 鑻ュ瓨鍦紝鍙槸缂撳瓨鐨勫０鏄庡紡閰嶇疆 + 璧勬簮锛屽閿欎粛涓嶈兘绂荤嚎鍒わ紙闄ら潪浠ュ悗鍋氭湰鍦� parser鈥斺�旈粯璁や笉鍋氾級銆�
+
+---
+
+## 9. SD Card Resource
+
+Phase 1 鍐宠浠嶇劧鏈夋晥锛�**瀹氫箟鏉冨▉鍦� Server锛屼笉涓嬪彂 App 鍟嗗簵銆�**
+璧勬簮灞備笌瀹氫箟灞傚垎寮�銆侾hase 2 宸茶惤鍦扮殑涓氬姟璺緞锛堢粡 `SDCard_Map_Business_Path` 鎺ュ埌 `MOUNT_POINT`锛夛細
+
+```text
+/{app_id}/{task_id}/lessons/{lesson_id}/file.mp3
+渚嬶細
+ /english_learning/repeater/lessons/unit01/full.mp3
+ /english_learning/shadowing/lessons/unit01/01.mp3
+```
+
+`app_id` 鏄� App 鍞竴鍚嶃�傜敤鎴疯闃呮柊 App 鍚庣敱绋嬪簭 mkdir锛圡CP `self.sd.ensure_app` / MQTT `ENSURE_APP_DIRS`锛夛紝**涓嶉噸鐑у浐浠�**銆傝繖鏄繍杈撴毚闇插凡鏈� `mkdir`锛屼笉鏄柊浼犳劅鍣ㄣ��
+
+闀挎湡浠嶅彲澧炲姞缂撳瓨鐩綍锛圥hase 6锛夛紝涓嶅繀鍐嶆惉瀹朵笂闈㈣繖妫垫爲锛�
+
+```text
+/sdcard/
+ {app_id}/{task_id}/lessons/ # 闊抽锛圥hase 2 浜嬪疄锛�
+ cache/
+ recordings/pending/ # 浠� v1.1
+ apps/{app_id}/ # 鍙�夛細manifest 缂撳瓨锛圥hase 6锛夛紱璁惧浠嶅彧鎵ц skills[]
+```
+
+璁惧 **涓嶇煡閬�**銆岃繖鏄嫳璇� App銆嶃�傚畠鍙煡閬擄細
+
+```text
+resource_id 鈫� 鏂囦欢
+play_audio(resource_id)
+```
+
+Sening 鐜版湁 `/data/courses/...` 鍏佽鏄犲皠锛屼笉寮鸿揩涓�娆℃惉瀹躲��
+
+---
+
+## 10. 闊抽褰曞埗涓庝笂浼�
+
+```text
+v1 鐑矾寰勶細Mic 鈫� Opus 娴� 鈫� Server ASR 鈫� 鍙�� WAV BLOB锛堢幇鏈� report锛�
+v1 璺熻锛氬悓涓�鏉″惉娴侊紱绂佹涓鸿窡璇诲啀寮�绗簩濂楃紪瑙ｇ爜
+v1.1 鏂囦欢锛歋D pending 鈫� 鑱旂綉 upload 鈫� audio_ref
+```
+
+鐑矾寰勭姝㈡敼锛�16k / 60ms Opus銆俙record_clip` 涓嶅緱鎹竴濂� AAC 寮曟搸鏉ャ�屼紭鍖栦細璁�嶃��
+
+---
+
+## 11. VAD
+
+- 鍞ら啋 / 鍛戒护璇嶏細璁惧 AFE锛堝凡鏈夛級
+- 鍙ヨ竟鐣岋紙璺熻銆佸彛绠楋級锛歋erver VAD 鍙敤锛涜澶囧彲鍚庤ˉ `user.speech.stopped`
+- Mode 绛栫暐锛氳寖璇�/澶嶈鎾斁涓� **涓嶅緱** 鎶婂枃鍙綋 VAD 鍛戒腑
+
+杩欐槸鎵�鏈夊皝闂紡璇煶璇勪及鐨勫叡鍚屽墠鎻愩��
+
+---
+
+## 12. TTS
+
+Task JSON 鍙嚭鐜� `speak(text)` 鎴� `play_audio(resource)`銆�
+鍚堟垚钀界偣锛堜簯 / 鏈嶅姟绔� / 鏈潵鏈湴锛夌敱 Runtime 閫夈�傚浐浠剁姝㈠啓姝汇�岃窡璇诲繀椤讳簯 TTS銆嶃��
+
+---
+
+## 13. Timer / Loop / Branch
+
+```text
+Timer Device wait(ms) 鎾斁涓庡惉涔嬮棿鐨勯棿闅�
+Loop Device锛氬悓涓� play_audio N 娆�
+ Server锛氫笉鍚� Step 閲嶅
+Branch 浠� Server锛歩f assessment.pass then next else retry
+ Device 鍙敹鍒� PLAY 鎴� SPEAK 鎴� STOP
+```
+
+**Branch 杩涘叆鍥轰欢 = 鍙ｇ畻鍒ら杩� C++銆傜粷瀵圭姝€��**
+
+---
+
+## 14. App Manifest锛堝０鏄庯紝涓嶆槸鍥轰欢锛�
+
+鐩爣褰㈡�侊紙Phase 6 钀藉湴锛岀幇鍦ㄥ彧鍐荤粨瀛楁鏂瑰悜锛夛細
+
+```text
+app_id
+name
+default_task
+tasks[]:
+ task_id
+ interaction_mode
+ skills[] / steps[] # steps 鍦� Server 鎵ц
+ resources[] # 瑕佷笅杞界殑闊抽
+```
+
+鍥轰欢 **涓嶈В鏋愪笟鍔¤鍒�**銆傛渶澶氾細绂荤嚎鏃惰 cached task 鐨� `skills: [play_audio]` 浠ヤ究澶嶈銆�
+
+---
+
+## 15. Task DSL锛圫erver锛岃澶囦笉瑙ｉ噴瀹屾暣 DSL锛�
+
+鍐荤粨鏂瑰悜锛�
+
+```text
+Task
+ Step: speak | play_audio | wait | listen | stop
+ on listen.completed 鈫� Server ASR 鈫� (鍙��) Assessment 鈫� next Step
+```
+
+璁惧鐪嬪埌鐨勬案杩滄槸 **鍗曟 Capability**锛屼笉鏄�屽鏋滃瀛愮瓟閿欏氨闄嶉毦搴︺�嶃��
+
+鎴愯鎺ラ緳 / 鍙ｇ畻 / 鍊掕儗 鐨勫樊寮傚彧鍦� Server 鐨� Parser 涓� Prompt锛屼笉鍦ㄥ浐浠躲��
+
+---
+
+## 16. 鍝簺鑳藉姏蹇呴』 Firmware 鍥哄寲
+
+涓�娆★紙鎴栧皯鏁板嚑娆★級OTA 琛ラ綈鍚庡簲闀挎湡绋冲畾锛�
+
+```text
+Codec / AEC / 鍞ら啋 / MultiNet 瀹夸富锛堣瘝琛ㄥ彲閰嶏紝璇嶄箟涓嶅啓涓氬姟锛�
+WS + 鑹插畞 MQTT 鏍�
+MCP / Skill 瀹夸富
+SD 鏂囦欢绯荤粺 + 涓嬭浇
+闊抽浜掓枼锛圱TS vs play_audio锛�
+play_audio + loop + stop + 瀹屾垚浜嬩欢
+listen 娴�
+barge_in 涓� Mode 绛栫暐琛�
+button 浜嬩欢
+indicate 鏄犲皠
+wait
+task_control 鎶㈠崰
+杞叧鏈� / RTC 闂归挓鎵ц pending锛堝凡鏈夛級
+```
+
+---
+
+## 17. 鍝簺鑳藉姏缁濆涓嶈兘杩涘叆 Firmware
+
+```text
+ASR / LLM / 缈昏瘧绠楁硶 / 璇勫垎 / 鍙ｇ畻瑙ｆ瀽 / 鎴愯璇嶅簱
+鎯呯华涓庝复搴婂垽鏂�
+if (app_id == "english") { ... } 涓氬姟鏍�
+涓哄崟涓� App 澧炲姞 GPIO 鍗忚
+鎶� QUIZ 鍋氭垚鏂� DeviceState
+鎶� apps.json 缂栬瘧杩涘浐浠跺綋鍞竴璇嶈〃
+```
+
+鍥轰欢閲屽嚭鐜� `english_learning` 瀛楃涓蹭粎鍏佽浣滀负 **Skill 鏌ユ壘琛ㄩ敭**锛圥hase 1 宸叉湁锛夛紝涓嶅厑璁稿湪鏃佽竟鍐欓煶棰戦�昏緫銆傞暱鏈熷簲鐢� `task.skills[]` 缂撳瓨鍙栦唬纭紪鐮佽〃銆�
+
+---
+
+## 18. 鐢� 5 涓唬琛ㄦ�� App/Task 鍋氳兘鍔涜鐩栭獙璇�
+
+涓嶆槸鍥犱负杩� 5 涓渶璧氶挶锛岃�屾槸 **瑕嗙洊鐜囨渶楂�**銆備骇鍝佸寲椤哄簭浠嶅彲鍏堝璇�/璺熻锛涘�掕儗/鍙ｇ畻/鎺ラ緳鐢ㄤ簬 **楠岃瘉 Runtime 鏄惁鐪熷喕缁�**锛屼笉蹇呭厛鍋氭紓浜� UI銆�
+
+| 浠ｈ〃 Task | 楠岃瘉鐨� Device 鑳藉姏 | 楠岃瘉鐨� Server 鑳藉姏 | 鐜扮姸 |
+|-----------|-------------------|-------------------|------|
+| 鑻辫澶嶈鏈� | play_audio, loop, stop, 瀹屾垚浜嬩欢, resource, 绂荤嚎 | Task 鍚姩銆佷紭鍏堢骇 | Phase 2锛氭挱鏀�+loop+瀹屾垚浜嬩欢宸叉湁 |
+| 鑻辫璺熻 | play/speak, wait, listen, VAD, barge_in 绛栫暐, 闊抽褰掓。 | ASR锛堟梺璺級, 鍚庤瘎浼板畾閲�, Session | Phase 2 閲囬泦宸叉湁锛汸hase 4锛氭椂閽熸杩� + txt 濂戠害锛屾瘮瀵逛笉鎸℃挱鏀� |
+| 鏁板瓧鍊掕儗 | speak, listen, VAD, wait | ASR, Sequence parser, **娴佸唴** score + 鍙婃椂 speak | **鏈仛**锛堝彛浠ゆЫ宸查鐣欙紱Phase 4b inline锛� |
+| 10 浠ュ唴鍙ｇ畻 | speak, listen, wait, timer | 鍑洪, parser, 榛樿娴佸唴鍙嶉 | **鏈仛** |
+| 鎴愯鎺ラ緳 | speak, listen, barge_in(CHAT 绛栫暐) | ASR, LLM/璇嶅簱, **娴佸唴** 鍚堟硶鎬т笌涓嬩竴鎻愮ず | **鏈仛** |
+
+杩� 5 涓仛瀹岋紙鍝�曞悗涓変釜鍙仛 Server 缂栨帓 + 鐜版湁 listen/speak锛夛紝Device 渚т笉搴斿啀鍑虹幇鏂颁紶鎰熷櫒绫昏兘鍔涖��
+
+鍏跺悗锛�
+
+```text
+鏁呬簨澶嶈堪 / AI 涓绘寔浜� / 鍥炲繂褰� / 蹇冪悊闄即 / 浼氳绾 / 寰嬪笀璇㈤棶
+```
+
+鍙鍔� Task Definition銆丳rompt銆丮emory銆丄ssessment銆侊紙v1.1锛夐暱褰曢煶涓婁紶銆�
+蹇冪悊闄即锛�**闀挎湡琛屼负淇″彿**锛堝搷搴旀椂寤躲�佸畬鎴愮巼锛夋潵鑷� Event 鏃堕棿鎴筹紝涓嶆槸鏂扮‖浠讹紱**绂佹**褰撲复搴婅瘖鏂��
+
+---
+
+## 19. 鑳藉姏瑕嗙洊鐭╅樀锛堝綊绾崇粨鏋滐級
+
+鍥句緥锛歚D` = 闇�瑕佸凡鍐荤粨鐨� Device 鑳藉姏锛沗S` = Server锛涚┖ = 涓嶉渶瑕併��
+
+| App/Task | 鎾� | TTS | VAD | 鍚�/褰� | ASR | LLM | 璇勪及 | wait/loop | 鍔ㄦ�佷笅涓�姝� |
+|----------|----|-----|-----|-------|-----|-----|------|-----------|------------|
+| 绾 / 澶嶈 | D | D | | | | | | D | |
+| 鑻辫璺熻 | D | D | D | D | S | | S | D | S |
+| 涓嫳缈昏瘧 | | D | D | D | S | S* | | | |
+| 涓嫳瀵硅瘽 | | D | D | D | S | S | | | S |
+| 鎴愯鎺ラ緳 | | D | D | D | S | S | S | | S |
+| 鍙ｇ畻 | | D | D | D | S | 鍙�� | S | D | S |
+| 鏁板瓧鍊掕儗 | | D | D | D | S | | S | D | S |
+| 蹇棶蹇瓟 | | D | D | D | S | S | S | D | S |
+| 鏁呬簨澶嶈堪 | D | D | D | D | S | S | S | | S |
+| AI 涓绘寔浜� | | D | D | D | S | S | S | | S |
+| 鍥炲繂褰� | | D | D | D | S | S | S | | S |
+| 浼氳绾 | | | | D鈥� | S | S | S | | |
+| 蹇冪悊闄即 | | D | D | D | S | S | S鈥� | | S |
+
+\* 缈昏瘧鍙敤涓撶敤妯″瀷锛屼粛鏄� Server銆�
+鈥� 闀垮綍闊冲亸 v1.1 `record_clip`銆�
+鈥� 鍙仛鍙備笌搴�/琛屼负淇″彿锛屼笉鍋氳瘖鏂��
+
+**纭欢鍒楀彧鏈夛細鎾�乀TS 鍑哄０銆乂AD銆佸惉/褰曘�亀ait/loop銆佹寜閽�/鎸囩ず銆�** 鍏朵綑鍏ㄦ槸 Server銆�
+
+---
+
+## 20. 涓� Phase 1 浠ｇ爜鐨勫樊璺濓紙鍐荤粨娓呭崟锛屼笉鏄幇鍦ㄥ氨鍐欏畬锛�
+
+蹇呴』鍦ㄣ�屽绉� Device 宸插喕缁撱�嶄箣鍓嶈ˉ榻愮殑鍥轰欢/鍗忚璇箟锛�
+
+1. `audio.completed` / `audio.stopped`锛堥�昏緫浜嬩欢锛涜繍杈撳彲鐢� `[skill_event]` 鎴� MCP锛� 鉁� Phase 2
+2. Mode鈮燙HAT 鏃跺彲闈犵殑銆屼竴鍙ヨ瀹屻�嶏紙VAD/listen.completed锛� 鉁� Phase 2 v1锛圫erver VAD + ASR + 鍚獥涓婇檺锛�
+3. 璺熻/璇勪及璺緞鐨勯煶棰戝綊妗ｏ紙鐜版湁 report 鍗冲彲锛� 鉁� Phase 2 鏂囨湰 + report锛汸hase 3 鍙�� `audio_id` 鎸� step
+4. `wait` 鏄惁涓�绛夎兘鍔涳細鍙厛 Server 渚у欢鏃讹紝浣嗕笉瑕佸啀涓烘瘡涓� App 澶嶅埗
+5. `task pause/resume` 璇箟锛堝彲鍚庝簬 1鈥�3锛�
+6. `play_audio` 璧勬簮 ID 缁熶竴锛堝彲涓� Phase 6 璧勬簮绠＄悊涓�璧凤紱闊抽璺緞缁х画鐢� `/{app}/{task}/lessons`锛�
+
+**涓嶉渶瑕佷负鍐荤粨鑰屽仛锛�** Assessment 绠楁硶銆佹垚璇瘝搴撱�佸彛绠楀嚭棰樺櫒銆乼ype=task 鍏ㄥ銆佸悎骞跺弻 MQTT銆�
+**Java 琛ㄦ槸 Task Runtime锛圥hase 3锛夛紝涓嶆槸 Device 鍐荤粨鏉′欢銆�**
+
+---
+
+## 21. 瀵� Roadmap 鐨勬彃鍏ョ偣
+
+```text
+Phase 1 钖� App Runtime 鉁�
+ 鈫�
+鏈� Spec Device Capability Freeze 鉁� 鏂囨。
+ 鈫�
+Phase 2 鐢ㄨ窡璇�/澶嶈楠岃瘉 listen/VAD/浜嬩欢 鉁� D1 / D6 / D7 v1 / D8 v1锛涜 phase-2-review.md
+ 鈫�
+Phase 3 TaskInstance 钀藉簱 鉁� 鏃犳柊 Device 鑳藉姏锛涜 phase-3-review.md
+ 鈫�
+Phase 4 鏃堕挓姝ヨ繘 + 鍚庤瘎浼� 鈫� 涓嬩竴缂栫爜锛涜 phase-4-plan.md
+ 鈫�
+鍊掕儗 / 鍙ｇ畻 / 鎺ラ緳 锛圥hase 4b锛氭祦鍐呰瘎浼� + 鍙婃椂 speak锛涘弽璇佸浐浠跺喕缁擄級
+```
+
+Phase 2 鐨勫伐绋嬩环鍊兼槸锛�**鎶婇噰闆� + 瀹屾垚浜嬩欢閽夎繘 Runtime銆�** 宸插畬鎴愩��
+Phase 3 鐨勫伐绋嬩环鍊兼槸锛�**Instance 鍙煡璇€��** 宸插畬鎴愩��
+璺熻鍚庤瘎浼版槸 Phase 4锛涘�掕儗/鎺ラ緳鐨勫強鏃跺弽棣堟槸 Phase 4b锛屼笉鑳芥贩鎴愬悗璇勪及銆傛柦宸ュ崟锛歚phase-4-plan.md`銆乣phase-4-test.md`銆乣roadmap.md`銆�
+
+---
+
+## 22. 鎺ュ彈鏈� Spec 鐨勬爣鍑�
+
+褰撲笖浠呭綋鍥㈤槦鍚屾剰涓嬮潰鍥涘彞锛屾墠鍏佽杩涘叆鍚庣画缂栫爜锛圥hase 3 璧峰悓鏍烽�傜敤锛夛細
+
+1. 鏂� App 涓嶅緱瑕佹眰鏂扮殑浼犳劅鍣ㄦ垨鏂扮殑 WS 娑堟伅绉嶇被锛涘彧璁哥敤鏈枃浠跺懡浠�/浜嬩欢琛ㄣ��
+2. 瀵归敊銆佸嚭棰樸�佺炕璇戙�佹帴榫欒瘝搴撳彧鍦� Server銆�
+3. 绂荤嚎鍙繚璇併�屽彲绂荤嚎 Skill銆嶏紙鎾斁锛夛紝涓嶄繚璇佹櫤鑳� Task銆�
+4. SD 鏀捐祫婧愶紱澹版槑寮� Task JSON 鑻ョ紦瀛樺埌 SD锛屼篃涓嶇瓑浜庡浐浠舵噦鑻辫銆�
diff --git a/phase-1-review.md b/phase-1-review.md
new file mode 100644
index 0000000..1e8ecdf
--- /dev/null
+++ b/phase-1-review.md
@@ -0,0 +1,475 @@
+# Phase 1 Completion Review
+
+> Date: 2026-08-31
+> Basis: **current code** (not the original Phase 1 wish-list)
+> Compared against: `phase-1.md` 路 Master Plan 路 Architecture Audit 路 Client Plan V1.1 路 Server Plan V1.1
+> Repositories: `xiaozhi-esp32-2.2.6` (Client) 路 `xiaozhi-esp32-server` (Server)
+
+鏈枃浠跺彧璁板綍浜嬪疄銆佸樊寮傘�佸�哄姟涓庡 Master Plan 鐨勪慨姝ｅ缓璁�備笉鍖呭惈涓嬩竴闃舵鐨勮缁嗙紪鐮佷换鍔★紙瑙� `phase-2-plan.md`锛夈��
+
+---
+
+## 1. 褰撳墠浠ｇ爜鐩樼偣锛堜互浠撳簱涓哄噯锛�
+
+### 1.1 Phase 1 瀹為檯鏀瑰姩鐨勬枃浠�
+
+#### Client锛坄xiaozhi-esp32-2.2.6`锛�
+
+| 鏂囦欢 | 瑙掕壊 |
+|------|------|
+| `main/boards/sening-xiaozhi-s3/app_task.h` | **鏂板銆�** 璁惧渚� Task 鍚姩 / pending / 绂荤嚎鍙ｄ护 / 澶嶈璁板繂 |
+| `main/boards/sening-xiaozhi-s3/app_task.cc` | **鏂板銆�** `Board_StartAppTask`銆佷紭鍏堢骇銆丯VS pending銆乣last_rep`銆佹妧鑳藉垎鍙� |
+| `main/boards/sening-xiaozhi-s3/mqtt.c` | `START_APP_TASK`锛沗SET_ALARM` 鍙檮甯� `app_id/task_id/params` 鍐欏叆 pending |
+| `main/boards/sening-xiaozhi-s3/sening_xiaozhi_s3.cc` | MCP `self.mp3.play` 澧炲姞 `repeat` / `power_off_after` / 绌� name 鍥為��锛涘紑鏈� `Board_ResolveBootOrRestore` |
+| `main/boards/sening-xiaozhi-s3/mp3_player.cc` | 鏁撮寰幆 N 娆★紱鑷劧缁撴潫鎵� `power_off`锛沗repeat_left` |
+| `main/boards/sening-xiaozhi-s3/system.h` / `system.c` | 绂荤嚎鍙ｄ护瀹忥紱`SDCard_Play_MP3_With_Speed_Repeat` |
+| `main/audio/wake_words/afe_wake_word.cc` | MultiNet 娉ㄥ唽 4 鏉＄绾垮彛浠� |
+| `main/audio/wake_words/custom_wake_word.cc` | 鍚屼笂 |
+| `main/application.cc` | `SendListenDetect`锛氭病缃戜笖閫氶亾鏈紑鏃朵笉 `WakeWordInvoke` |
+
+鏈敼锛歄pus 鐑矾寰勩�乣DeviceState` 杩佺Щ琛ㄣ�乣hello/listen/tts/stt` 瀛楁鍚箟銆傞潪 Sening 鏉挎棤 App Task 鎵ц鍣ㄣ��
+
+#### Server锛坄xiaozhi-esp32-server`锛屼粎 Python `xiaozhi-server`锛�
+
+| 鏂囦欢 | 瑙掕壊 |
+|------|------|
+| `core/app_runtime/__init__.py` | **鏂板妯″潡銆�** |
+| `core/app_runtime/registry.py` | 鍔犺浇 `apps.json`锛坆undled锛涘彲閫� `data/apps.json`锛� |
+| `core/app_runtime/apps.json` | App / Task 娉ㄥ唽琛� |
+| `core/app_runtime/runtime.py` | `start_app_task` / `switch_app` / mode 鍒嗘祦 / 鍙ｄ护 / 璺熻杞 / 缈昏瘧 |
+| `plugins_func/functions/switch_app.py` | LLM 鍙皟鐨勫垏 App |
+| `plugins_func/functions/start_repeater.py` | 鎵撳紑澶嶈鏈� |
+| `plugins_func/functions/start_shadowing.py` | 鎵撳紑璺熻 |
+| `plugins_func/functions/translate.py` | 杩炴帴 LLM 缈昏瘧锛堟櫤鎺у彴 Key锛� |
+| `mcp_servers/translate/server.py` | 鐙珛 translate MCP锛堝洖閫�锛� |
+| `mcp_server_settings.json` | 娉ㄥ唽 `english-translate` |
+| `core/connection.py` | 杩炴帴鍐呭瓨锛歚active_app_id` / `interaction_mode` / `tool_policy` 绛� |
+| `core/handle/receiveAudioHandle.py` | `startToChat` 鍓嶆寜 mode 鍒嗘祦锛涘彛浠や紭鍏堜簬 LLM |
+| `core/handle/textHandler/listenMessageHandler.py` | `[app_task]` / `[voice_entry]` |
+| `core/providers/tools/unified_tool_manager.py` | `filter_tools` 鐧藉悕鍗� |
+| `config.yaml` | 鍔犺浇 `switch_app` / `start_repeater` / `start_shadowing` |
+
+#### manager-api / manager-web
+
+**鏃� App Runtime 鐩稿叧鏀瑰姩銆�** 鏃犳柊琛ㄣ�佹棤鏂� REST銆傛櫤鎺у彴浠嶅彧绠＄悊浜鸿 / 鎻掍欢缁戝畾銆�
+锛堝巻鍙插璇濋噷 manager-web 鐨� `vue.config.js` / teleport 淇涓� Phase 1 涓氬姟鏃犲叧锛屼笉璁″叆鏈樁娈垫灦鏋勩�傦級
+
+### 1.2 瀹為檯澧炲姞鐨勬ā鍧�
+
+| 妯″潡 | 浣嶇疆 | 鑱岃矗 |
+|------|------|------|
+| App Runtime | Server `core/app_runtime/` | 娉ㄥ唽琛� + 缁熶竴 `start_app_task` + mode 琛屼负 |
+| Device Task Gateway | Client `app_task.cc` | 涓夐�氶亾姹囧悎銆佹姠鍗犮�乸ending銆佹湰鍦� Skill 鎵ц |
+| Translate plugin + MCP | Server plugins + `mcp_servers/translate` | 缈昏瘧鏃佽矾 `chat()` |
+| Repeater / Shadowing plugins | Server plugins | 杩涘叆瀵瑰簲 Task |
+
+娌℃湁锛歍ask Runtime 鐘舵�佹満绫汇�丼tep 寮曟搸銆丄ssessment 鏈嶅姟銆丼cheduler 鏈嶅姟銆丄pp Store銆�
+
+### 1.3 瀹為檯澧炲姞鐨� API
+
+| 绫诲瀷 | 鍚嶇О | 璇存槑 |
+|------|------|------|
+| REST | **鏃�** | manager-api 鏈鎺ュ彛 |
+| Python 鎻掍欢 | `switch_app` | `app_id` + 鍙�� `task_id` |
+| Python 鎻掍欢 | `start_repeater` | 鍙�� `name/repeat/power_off_after` |
+| Python 鎻掍欢 | `start_shadowing` | 鏃犲弬鏁� |
+| Python 鎻掍欢 | `translate` | `text` + 鍙�夎瑷� |
+| Device MCP | `self.mp3.play` 鎵╁睍 | `repeat`銆乣power_off_after`锛沗name` 鍙┖ |
+| Device MCP | `self.mp3.get_status` | 宸叉湁锛涘鍔� `repeat_left` |
+
+### 1.4 瀹為檯澧炲姞鐨勫崗璁�
+
+**娌℃湁**鏂扮殑 Xiaozhi WS `type`銆傜儹璺緞璇箟鍐荤粨寰楀埌閬靛畧銆傚疄闄呴噰鐢ㄧ殑鏄�屽鐢� listen detect 鏂囨湰銆嶅拰銆岃壊瀹� MQTT 鏂� action銆嶃��
+
+| 閫氶亾 | 杞借嵎 | 鐢ㄩ�� |
+|------|------|------|
+| WS `listen` `state=detect` | `[app_task]{app}\|{task}\|{source}` | 璁惧閫氱煡 Server 杩涘叆鏌� Task锛坄execute_skills=false`锛岄伩鍏嶅璇诲弻鎾級 |
+| WS `listen` `state=detect` | `[voice_entry]` | 绂荤嚎銆屽垏鎹㈡ā寮忋�嶄笖**宸茶仈缃�**鏃惰鑿滃崟 |
+| MQTT `snlp/v1/.../cmd` | `action=START_APP_TASK` | 閫氶亾鈶犲嵆鏃跺惎鍔� |
+| MQTT `SET_ALARM` | 鍙�� `app_id/task_id/params` | 鍐欏叆璁惧 pending锛岄�氶亾鈶� |
+
+鏈鍔狅細`type=task`銆乣mode.set`銆乣play.completed` 鐙珛娑堟伅銆乣speech.stopped`銆�
+
+### 1.5 瀹為檯澧炲姞鐨勬暟鎹簱缁撴瀯
+
+**鏃犮��** App 鐘舵�佸湪 `ConnectionHandler` 鍐呭瓨銆傝澶� pending / `last_rep` 鍦� NVS銆俉S 鏂嚎鍚� Server 鍥炲埌 default chat锛涜澶� pending 浠嶅湪鏈湴銆�
+
+### 1.6 Client 鏋舵瀯鍙樺寲
+
+Phase 1 涔嬪墠锛圓udit锛夛細
+
+```text
+Application 鍗曚緥 + DeviceState + Opus 鍙屽伐 + MCP
+Sening 鏃佽矾锛歴elf.mp3.* / snlp MQTT PLAY_MEDIA / RTC 闂归挓
+绂荤嚎銆屽垏鎹㈡ā寮忋�嶅彧鎵撴棩蹇�
+```
+
+Phase 1 涔嬪悗锛�
+
+```text
+Application锛圕hat 鐑矾寰勪笉鍙橈級
+ 鈹斺攢鈹� Board_StartAppTask锛圫ening 鏉跨骇 Task Gateway锛�
+ 鈹溾攢鈹� source 浼樺厛绾� mqtt_immediate > schedule > voice > default_chat
+ 鈹溾攢鈹� NVS pending + last_rep
+ 鈹溾攢鈹� 鎶�鑳芥煡鎵捐〃 k_task_skills[]锛堝浐浠跺唴纭紪鐮� id锛屼笉鏄笟鍔� if 鏍戯級
+ 鈹斺攢鈹� 鏈湴鎵ц self.mp3.play / 鎴栧彧 notify Server
+MultiNet锛氬垏鎹㈡ā寮� / 澶嶈鏈� / 寮�濮嬪璇� / 閫�鍑哄璇�
+娌＄綉锛氬垏鎹㈡ā寮忎笉杩炰簯锛涘璇绘満鍙挱涓婃闊抽鎴� SD 绗竴棣�
+```
+
+InteractionMode **娌℃湁**鎴愪负鍥轰欢鏋氫妇銆傝澶囦笉鎸佷箙鍖� App 鍘嗗彶銆侱eviceState 浠嶅彧绠″惉/璇�/鍗囩骇銆�
+
+### 1.7 Server 鏋舵瀯鍙樺寲
+
+Phase 1 涔嬪墠锛�
+
+```text
+ConnectionHandler.chat() 鏄敮涓�瀵硅瘽缂栨帓
+Agent + Plugin + MCP
+鏃� App / 鏃� InteractionMode
+```
+
+Phase 1 涔嬪悗锛�
+
+```text
+receiveAudioHandle.startToChat
+ 鈫� 鍙ｄ护 match_voice_switch锛堝厛浜� LLM锛�
+ 鈫� mode 鍒嗘祦锛�
+ PLAY_AUDIO 蹇界暐涓婅
+ SHADOWING handle_shadowing_turn锛堜笉 chat锛�
+ TRANSLATION run_translation锛堜笉 chat锛�
+ BILINGUAL wrap 鍚庝粛 chat()
+ CHAT 鐜扮綉 intent/LLM
+App Runtime锛歛pps.json + start_app_task
+宸ュ叿锛歩nherit_agent | allowlist
+```
+
+鏉冨▉鐘舵�佷粛鍦� **褰撳墠 WS 杩炴帴鍐呭瓨**锛屼笉鏄� TaskInstance 琛ㄣ��
+
+### 1.8 Client 鈫� Server 瀹為檯閫氫俊娴佺▼
+
+```text
+鈶� 灏忕▼搴� / MQTT 璋冭瘯瀹㈡埛绔�
+ START_APP_TASK
+ 鈫� snlp MQTT
+ Board_StartAppTask
+ 鈹溾攢 鏈湴 Skill锛堝璇伙細SD MP3锛�
+ 鈹斺攢 鑻� Xiaozhi 閫氶亾宸插紑锛歭isten detect [app_task]
+ 鈫�
+ Server start_app_task(execute_skills=false)
+ 鏇存柊 active_app_id / interaction_mode锛孴TS 鎾姤
+
+鈶� SET_ALARM + pending
+ 鍒扮偣 RTC 鍞ら啋 鈫� Board_ResolveBootOrRestore
+ 闈� chat 璁″垝 鈫� 鐩存帴 StartAppTask(schedule)
+ 鍚﹀垯 鈫� chat.conversation
+
+鈶� 绂荤嚎璇煶
+ MultiNet 澶嶈鏈� 鈫� 鏈湴鎾斁锛屼笉渚濊禆 Server
+ MultiNet 鍒囨崲妯″紡 鈫� 浠� IsNetworkReady 鏃� [voice_entry]
+ 鍦ㄧ嚎 ASR 鐭彞 鈫� match_voice_switch 鈫� start_app_task
+
+鈶� Chat 鐑矾寰勶紙鏈寮� CHAT锛�
+ Mic 鈫� Opus 鈫� ASR 鈫� LLM 鈫� TTS 鈫� Speaker
+ 涓� Phase 1 鍓嶇浉鍚�
+```
+
+### 1.9 Task Runtime 褰撳墠瀹炵幇鐘舵��
+
+| 鑳藉姏 | 鐘舵�� |
+|------|------|
+| 缁熶竴鍏ュ彛 `start_app_task` | **鏈�**锛圫erver + 璁惧鍚勪竴浠斤級 |
+| App / Task 娉ㄥ唽琛� | **鏈�**锛圫erver JSON锛涘浐浠跺彟鏈夋妧鑳借〃锛� |
+| InteractionMode 鍒嗘祦 | **鏈�**锛堣繛鎺ュ瓧绗︿覆锛�5 绉� mode锛� |
+| Step / StepInstance | **鏃�** |
+| TaskInstance 鎸佷箙鍖� | **鏃�** |
+| Event锛坧lay.completed 绛夛級 | **鏃犳寮忓崗璁�**锛涙挱鏀剧粨鏉熻澶囦細 `[app_task]chat` 閫氱煡鍥炶亰 |
+| 璺熻 | **鏈� Demo**锛歍TS 鑼冭 + ASR 杞 + 鍥炲０鎶戝埗锛涙棤褰掓。銆佹棤璇勫垎 |
+| Assessment / Metric | **鏃�** |
+
+缁撹锛氳繖鏄� **钖� App Runtime + 鍗� Task 鍔ㄤ綔**锛屼笉鏄� Master Plan 鐨� Task Runtime銆�
+
+### 1.10 娴嬭瘯鎯呭喌
+
+| 椤� | 鐜扮姸 |
+|----|------|
+| `phase-1.md` 搂10 楠屾敹娓呭崟 | 鏂囨。瀛樺湪锛�**浠撳簱鍐呮棤瀵瑰簲鑷姩鍖栫敤渚�** |
+| xiaozhi-server pytest / unittest锛圓pp Runtime锛� | **鏃�** |
+| 鍥轰欢鍗曞厓娴嬭瘯锛坅pp_task / MultiNet锛� | **鏃�** |
+| 鐜版湁娴嬭瘯璧勪骇 | 浠� ASR/TTS/LLM performance_tester |
+| 瀹為檯楠岃瘉鏂瑰紡 | 浜哄伐鐑у綍 + 閲嶅惎 Server + MQTT/鍙ｄ护鑱旇皟 |
+
+**Phase 1 鍦ㄥ伐绋嬩笂銆屽仛瀹屼簡鍔熻兘銆嶏紝娌℃湁銆屾祴瀹屼簡鍥炲綊銆嶃��** 杩欐槸杩涘叆 Phase 2 鍓嶆渶澶х殑杩囩▼椋庨櫓锛屼笉鏄姛鑳界己鍙ｃ��
+
+---
+
+## 2. Phase 1 Implementation Review锛堣鍒� vs 浠ｇ爜锛�
+
+### 2.1 瀵圭収琛�
+
+| 椤圭洰 | 鍘熻鍒掞紙`phase-1.md`锛� | 瀹為檯瀹炵幇 | 宸紓鍘熷洜 | 鍚庣画澶勭悊 |
+|------|------------------------|----------|----------|----------|
+| 钖� App 妗嗘灦 | Server JSON + `active_app_id`锛涜澶囨棤 App 鐩綍 | 宸插疄鐜帮紱娉ㄥ唽琛ㄥ湪 `core/app_runtime/apps.json` | 鈥� | 淇濇寔锛涗笉瑕佷笅鍙戝埌 SD |
+| Default `chat.conversation` | 閰嶇綉/寮�鏈鸿繘鍏ュ苟涓诲姩寮�鍙� | pending 榛樿 chat锛沗agent_proactive` + 杩涘叆 CHAT 鏃� TTS 鎷涘懠 | 鍐峰満鍐嶅偓淇冩湭鍋氭垚鍙厤寰幆 | Phase 2 涓嶄紭鍏堬紱鍙厤娆℃暟鐣欏�� |
+| 涓夐�氶亾 + 浼樺厛绾� | MQTT > Schedule > Voice | 璁惧 `Board_AppTask_SourcePriority`锛汼erver `SOURCE_PRIORITY` | 涓ょ鍚勭畻涓�娆★紝鍙兘鐭殏涓嶄竴鑷� | Phase 2 鐢� skill_event 瀵归綈缁撴潫鎬佸嵆鍙紝涓嶅悎骞� MQTT |
+| `START_APP_TASK` | 鏂� MQTT action | 宸插疄鐜� | 鈥� | 淇濇寔 |
+| pending + 闂归挓鍞ら啋 | NVS锛涢潪 chat 璁″垝涓嶅厛闂茶亰 | `Board_ResolveBootOrRestore` + `SET_ALARM` 鍐� pending | Server **鏃�** Schedule 鏉冨▉琛� | 鎺ュ彈锛涙潈濞佽〃鏀惧埌 Scheduler Phase |
+| 绂荤嚎銆屽垏鎹㈡ā寮忋�� | 涓嶅啀绌� log锛涜繘鑿滃崟 | 鏈夌綉鎵� `[voice_entry]`锛涙病缃戣烦杩� | 瀹炴柦涓彂鐜� WakeWordInvoke 浼氬共绛� Server | **姝ｇ‘淇**锛屽啓鍏ユ灦鏋� V2 |
+| 澶嶈 N 娆″湪璁惧 | `repeat` 鍙傛暟 | mp3_player 寰幆 | 鈥� | 淇濇寔 |
+| `power_off_after` | 鑷劧缁撴潫鎵嶈蒋鍏虫満 | 宸插疄鐜� | 鈥� | 淇濇寔 |
+| 缈昏瘧鏃佽矾 LLM chat | 鐙珛 MCP | **涓昏矾寰勬敼涓� `conn.llm` + translate 鎻掍欢**锛汳CP 浠呭洖閫� | 鐙珛 MCP 渚濊禆鍗犱綅 Key锛屾櫤鎺у彴 Key 鎵嶆槸鐜扮綉 | **姝ｇ‘淇**锛汳CP 鍙檷绾т负鍙�� |
+| 宸ュ叿鐧藉悕鍗� | allowlist | `filter_tools` | 鈥� | 淇濇寔 |
+| Xiaozhi `type=task` | 鏄庣‘涓嶅仛 | 鏈仛 | 鈥� | Phase 2 浠嶄笉鍋氬畬鏁� type=task |
+| Java 鏂拌〃 | 鏄庣‘涓嶅仛 | 鏈仛 | 鈥� | Phase 3 鍐嶅仛 Instance 琛� |
+| 涓変釜 App | chat / 澶嶈 / 缈昏瘧 | **鍥涗釜 App + 璺熻 Task**锛氬彟澧� `english_conversation`銆乣shadowing` | 浜у搧瑕佹妸璺熻/涓嫳瀵硅瘽灏芥棭鍙紨绀� | 璺熻浠嶆槸 Demo锛涙寮忛噰闆嗘槸 Phase 2 |
+| 鍥轰欢鏃犱笟鍔″悕 | 绂佹 `if (app == english)` | 鍥轰欢 `k_task_skills[]` **鍚� app_id 瀛楃涓�** | MQTT 绂荤嚎鍚姩蹇呴』鑳芥煡 Skill锛屾棤娉曞彧闂� Server | 鎺ュ彈涓恒�岄厤缃〃銆嶏紱绂佹鍐嶅姞鑻辫璇勫垎/闊抽閫昏緫 |
+| 鍙ｄ护琛ㄥ湪 Assets JSON | 鏄犲皠琛ㄥ彲鎵� | 鍥轰欢纭紪鐮� 4 鏉℃嫾闊筹紱鍦ㄧ嚎鍙ｄ护鍦� `runtime.py` | MultiNet 瀹归噺涓庤瑙﹀彂 | **姝ｇ‘鏀剁獎**锛涗笉瑕佹妸缈昏瘧/璺熻濉炶繘 MultiNet |
+| apps.json 璺緞 | 寤鸿 `data/apps.json` | bundled `core/app_runtime/apps.json`锛屽彲閫� data 瑕嗙洊 | 閮ㄧ讲鏇寸畝鍗� | 淇濇寔 |
+| PLAY_AUDIO 蹇界暐 ASR | 鏄� | 鏄� | 鈥� | 淇濇寔 |
+| 鐢ㄦ埛鍞ら啋澶嶄綅 chat | 鏄紱闂归挓涓嶅浣� | `on_user_wakeup`锛沗[app_task]` 涓嶈蛋鍞ら啋 | 鈥� | 淇濇寔 |
+| 鏃� play.completed | 鎺ュ彈 | 鎾畬璁惧鐩存帴鍒囧洖 chat 骞� detect | Server 浠嶄笉鐭ラ亾绗嚑閬� | Phase 2 琛� skill_event |
+| 灏忕▼搴� UI | 鍚庣疆 | 鏈仛 | 鈥� | Parent UI 寰堝悗 |
+| 娴嬭瘯娓呭崟 | 璇︾粏 checkbox | 鏃犺嚜鍔ㄥ寲銆佹棤涔﹂潰璺戞祴璁板綍 | 璧跺姛鑳� | **Phase 2 蹇呴』甯� pytest + 鎵嬪伐鍥炲綊琛�** |
+
+### 2.2 鍝簺璁捐琚慨鏀癸紙鐩稿鍘熸枃 Plan锛�
+
+1. **绂荤嚎鍏ュ彛鏀剁獎**锛氬彧淇濈暀銆屽垏鎹㈡ā寮� + 澶嶈鏈�/寮�濮嬪璇�/閫�鍑哄璇汇�嶃�傜炕璇�/璺熻/涓嫳瀵硅瘽鍙蛋鍦ㄧ嚎 ASR銆�
+2. **娌＄綉涓嶈繛浜�**锛歚SendListenDetect` 鍦ㄦ棤缃戜笖閫氶亾鏈紑鏃剁洿鎺� return銆�
+3. **澶嶈鍙畬鍏ㄧ绾�**锛歂VS `last_rep` + SD 绗竴棣栵紱娉ㄥ唽琛ㄤ粛鍙湪 Server銆�
+4. **缈昏瘧瀹炵幇**锛氳繛鎺� LLM 浼樺厛浜庣嫭绔� MCP 杩涚▼銆�
+5. **Mode 闆嗗悎鎵╁ぇ**锛歚SHADOWING`銆乣BILINGUAL` 鎻愬墠钀藉湴锛圕lient Plan 鎶� TRANSLATION/GAME 鏀惧湪 Phase 3锛夈��
+6. **璺熻鎻愬墠**锛氫笉鏄� SD 鑼冭 + 褰曢煶瀵硅薄锛岃�屾槸 TTS 鍙ュ瓙鍒楄〃 + 鏂囨湰鍥炲０杩囨护銆�
+7. **鍗忚鎸傞挬**锛氱敤 `[app_task]` / `[voice_entry]` 瀵勭敓鍦� `listen/detect`锛岃�屼笉鏄柊 type銆�
+
+杩欎簺淇敼搴旇涓� **Phase 1 鐨勭湡瀹炴灦鏋�**锛屽悗缁� Phase 涓嶅緱鍋囪銆岃澶囦粛鍙湁 CHAT+PLAY_AUDIO銆嶃��
+
+### 2.3 鍝簺璁捐娌℃湁瀹炵幇
+
+- 鍐峰満澶氭涓诲姩寮�鍙ｏ紙鍙厤闂撮殧锛�
+- 鍙ｄ护鈫抜d 鐨勮澶囩 JSON/Assets 琛�
+- Server 渚� Schedule 鏉冨▉瀛樺偍
+- `play.completed` / 閬嶆暟涓婃姤
+- 璺熻闊抽瀵硅薄涓� step 鍏宠仈
+- Assessment
+- 闈炴硶 `app_id` 鐨勭郴缁熷寲閿欒鐮侊紙MQTT 鏈� ack锛涜闊冲涓� TTS 鍙ュ瓙锛�
+- Phase 1 娴嬭瘯璁″垝鐨勮嚜鍔ㄥ寲
+- 闈� Sening 鏉跨殑澶嶈
+
+### 2.4 瀹炴柦涓彂鐜扮殑闂
+
+1. `self.mp3.play` 绔嬪嵆 `accepted`锛孲erver 鏃犳硶缂栨帓 N 娆″惊鐜� 鈫� 寰幆蹇呴』鍦ㄨ澶囷紙Plan 宸查瑙侊紝浠ｇ爜璇佸疄锛夈��
+2. 鐙珛缈昏瘧 MCP 鐨� yaml Key 涓庢櫤鎺у彴 Key 鍒嗚 鈫� 鏀硅蛋 `conn.llm`銆�
+3. ASR 甯告妸缁撴灉鍖呮垚 `{content,language,emotion}` 鈫� 蹇呴』鎷� `content`锛屽惁鍒欑炕璇�/鍙ｄ护澶辫触銆�
+4. 璺熻/缈昏瘧鏃跺枃鍙洖澹颁細褰撶敤鎴锋彃璇� 鈫� AEC 鎵撴柇浠呴檺 `CHAT`/`BILINGUAL`锛涜窡璇诲啀鍔犳枃鏈浉浼煎害杩囨护銆�
+5. 銆屽垏鎹㈡ā寮忋�嶅湪娌＄綉鏃� `WakeWordInvoke` 浼氭寕璧� 鈫� 蹇呴』鐭矾銆�
+6. 璁惧鏈湴鍚姩澶嶈鍚庡啀閫氱煡 Server锛岃嫢 `execute_skills=true` 浼� **鎾袱閬�** 鈫� `[app_task]` 蹇呴』 `execute_skills=false`銆�
+7. MultiNet 璇嶈〃涓嶈兘濉炴弧涓氬姟鍙ｄ护锛屽惁鍒欒瑙﹀彂涓旀病缃戜篃鏃犳硶璺戠炕璇戙��
+
+### 2.5 鎶�鏈�哄姟锛堝疄鏂戒骇鐢燂級
+
+瑙佺 5 鑺傚垎绾с�傛牳蹇冿細鍙屾妧鑳借〃銆佸瘎鐢熷崗璁�佹棤娴嬭瘯銆佽窡璇讳笉鏄煶棰戜换鍔°�丄pp 鐘舵�佷笉钀藉簱銆�
+
+### 2.6 鍘熷厛娌¤�冭檻鍒般�佷絾宸插嚭鐜扮殑闂
+
+- 鐢ㄦ埛鍙ｈ銆屾ā寮忋�嶄笌鍐呴儴 `InteractionMode` / App 鍛藉悕鍐茬獊锛堝凡瑙勫畾锛氬鐢ㄦ埛浠嶈銆屾ā寮忋�嶏紝瀵逛唬鐮佷粛鐢� App/Task锛夈��
+- 绂荤嚎澶嶈涓庡湪绾挎潈濞佺姸鎬佷細鍒嗗弶锛氭病缃戞挱瀹屽悗 Server 浠嶄互涓哄湪 CHAT銆�
+- 鍥轰欢蹇呴』璁よ瘑 `english_learning` 鎵嶈兘绂荤嚎鎵ц锛屼笌銆屼笟鍔″悕涓嶄笂鍥轰欢銆嶇悊鎯充笉瀹屽叏涓�鑷淬��
+- manager-web 寮�鍙戞瀯寤洪棶棰橈紙teleport/`usedExports`锛変笌 App 鏃犲叧锛屼絾璇存槑鎺у埗鍙版爤涔熸湁鍊猴紝涓嶈鍦� Phase 2 椤烘墜澶ф敼銆�
+
+---
+
+## 3. 涓� Master Plan 鏍稿績鎶借薄瀵圭収
+
+| 鎶借薄 | 褰撳墠鏄惁瀛樺湪 | 瀹炵幇浣嶇疆 | 瀹屾垚绋嬪害 | 鏄惁闇�瑕佽皟鏁� | 鏈潵鏄惁鎵╁睍 |
+|------|--------------|----------|----------|--------------|--------------|
+| **App** | 鏄紙钖勶級 | Server `apps.json` + `active_app_id`锛涜澶囨妧鑳借〃鏈� app_id | 杩炴帴绾у鍣紝鏃犲畨瑁�/璁㈤槄/璧勬簮鍖� | 淇濇寔銆孲erver 鏉冨▉銆佷笉涓嬪彂 SD銆� | Phase 6 鍐嶅仛璧勬簮涓庢竻鍗� |
+| **Task** | 鏄紙钖勶級 | `apps.json` tasks + `start_app_task` | 涓�涓� Task 鈮� 涓�绉嶅姩浣滐紝鏃� Step 鍥� | 涓嶈鎶婅窡璇� Demo 褰撴垚瀹屾暣 Task Definition | Phase 3 璧� JSON Step |
+| **Step** | 鍚� | 鈥� | 0% | Phase 2 **涓嶈**鍏堜笂瀹屾暣 Step 寮曟搸 | Phase 3+ 璺熻涓�姝ヤ竴 Step |
+| **Skill** | 鏄紙MCP + 鏉跨骇锛� | 璁惧 MCP锛汼erver 璋� DeviceMCP锛涘浐浠� `k_task_skills` | 鑳借窇 play/stop/power_off/translate | 鍙岃〃蹇呴』褰撳绾︾淮鎶� | 鏍稿績 `play_audio` 骞冲彴鍖栧悗绉� |
+| **InteractionMode** | 鏄紙Server 瀛楃涓诧級 | `conn.interaction_mode` | `CHAT/PLAY_AUDIO/TRANSLATION/SHADOWING/BILINGUAL` | 鍥轰欢浠嶆棤 Mode 鏋氫妇锛屽彲淇濇寔姝ｄ氦 | 姝ｅ紡 FOLLOW_READ 鍙笌 SHADOWING 鍚堝苟鍛藉悕 |
+| **TaskInstance** | 鍚� | 鈥� | 0% | Phase 2 鍙敤鍐呭瓨/鏂囦欢 Session锛屼笉寤� Java 琛� | Phase 3 钀� manager-api |
+| **StepInstance** | 鍚� | 鈥� | 0% | 鍚屼笂 | 璺熻閲囬泦鍚庢墠鏈夋剰涔� |
+| **Event** | 閮ㄥ垎 | `[app_task]` / `[voice_entry]`锛汳QTT ack | 鏃犵粺涓� Event 妯″瀷 | Phase 2 澧炲姞 `[skill_event]` 鍗冲彲锛屼笉涓婄鍥涘鎬荤嚎 | 浠ュ悗鍐嶈�冭檻 `type=task` |
+| **Audio** | 鑱婂ぉ闄勪欢绾� | `ai_agent_chat_audio` LONGBLOB锛涜窡璇绘湭寮哄埗褰掓。 | 璺熻杞鏈繀璧� report | Phase 2 蹇呴』鎶婅窡璇� ASR/闊抽鎸備笂 Session | 鍐嶈縼瀵硅薄瀛樺偍 |
+| **Assessment** | 鍚� | 鈥� | 0% | 涓嶈鍦� Phase 2 鍋氳瘎鍒� | Phase 4 鏂囨湰姣斿 v0 |
+| **Metric** | 鍚� | 鈥� | 0% | 鈥� | Assessment 涔嬪悗 |
+| **Schedule** | 鍗婃埅 | 璁惧 RTC + pending锛涙棤 Server 鏃ュ巻 | 鑳� 7 鐐瑰敜閱掑璇� | 涓嶈鐢� RTC 鍐掑厖瀹堕暱璁″垝鏉冨▉ | Phase 5 Server Scheduler |
+| **Plan** | 鍚� | 鈥� | 0% | 鈥� | 瀹堕暱浠诲姟绠＄悊 |
+| **Subscription** | 鍚� | 鈥� | 0% | 鈥� | 骞冲彴鍚庢湡 |
+
+**绗﹀悎 Master Plan 鐨勬柟鍚戯細** 鏉冨▉鍦� Server銆佽澶囨墽琛屻�丆hat 鐑矾寰勪繚鐣欍�丄pp 涓嶆槸鍥轰欢銆佸璇讳笉璧� `play_music`銆�
+
+**涓嶇鍚堟垨鎻愬墠/婊炲悗锛�**
+
+- Server Plan 璇� Phase 1 鍙互娌℃湁 App 瀹炰綋 鈫� 瀹為檯鍋氫簡钖� App锛岃繖鏄� **鏈夌泭鎻愬墠**锛屽簲鍐欒繘 Master Plan銆�
+- Client Plan 璇� Phase 1 鐢氳嚦涓嶉渶瑕� `activeAppId` 鈫� 瀹為檯璁惧鏈� pending app_id锛孲erver 鏈� active_app_id銆�
+- 璺熻/涓嫳瀵硅瘽/缈昏瘧 Mode 鎻愬墠 鈫� Master Plan 鐨� Phase 杈圭晫瑕佹寜浠ｇ爜鏀癸紝鑰屼笉鏄妸宸茶惤鍦板姛鑳藉亣瑁呫�屾湭寮�濮嬨�嶃��
+- TaskInstance / Assessment / Scheduler 浠嶆湭寮�濮� 鈫� 涓� Master Plan 涓�鑷达紝涓嶈鍦ㄤ笅涓�闃舵涓�娆″仛瀹屻��
+
+---
+
+## 4. Current Architecture V2锛堝熀浜庝唬鐮侊級
+
+### 4.1 Current Client Architecture V2
+
+```text
+ESP32-S3 (sening-xiaozhi-s3)
+鈹溾攢鈹� Application::Run DeviceState: Idle/Listening/Speaking/...
+鈹� 鈹溾攢鈹� AudioService Opus 16k 鐑矾寰勶紙鏈敼璇箟锛�
+鈹� 鈹溾攢鈹� Protocol (Xiaozhi WS/MQTT) hello/listen/abort/tts/stt/mcp
+鈹� 鈹斺攢鈹� MCP Server self.mp3.* / self.power_off / alarm / ...
+鈹�
+鈹溾攢鈹� WakeNet + MultiNet
+鈹� 鍙ｄ护: 鍒囨崲妯″紡 | 澶嶈鏈� | 寮�濮嬪璇� | 閫�鍑哄璇�
+鈹� 鈫� Board_Handle_Voice_Command
+鈹�
+鈹斺攢鈹� Task Gateway (app_task.cc) 鈫� Phase 1 鏂板
+ Board_StartAppTask(app, task, params, source)
+ 鈹溾攢鈹� 浼樺厛绾ф姠鍗� / preempt skills
+ 鈹溾攢鈹� NVS pending + last_rep
+ 鈹溾攢鈹� k_task_skills[] 鏌� Skill
+ 鈹斺攢鈹� 鎵ц: SD MP3 寰幆 | notify Server | 绌� Skill 鍙垏鎬�
+
+鏃佽矾锛堜粛骞跺瓨锛�:
+ snlp MQTT START_APP_TASK / SET_ALARM / PLAY_MEDIA / DOWNLOAD
+ DS3231 RTC 闂归挓鍞ら啋 鈫� OnRestoredFromPowerOff 鈫� ResolveBootOrRestore
+```
+
+闈� Sening 鏉匡細鏃� Task Gateway锛涚炕璇戠瓑绾� Server Mode 浠嶅彲璧� Xiaozhi WS銆�
+
+### 4.2 Current Server Architecture V2
+
+```text
+xiaozhi-server (Python)
+鈹溾攢鈹� ConnectionHandler
+鈹� session_id (WS 杩炴帴绾э紝鈮� TaskInstance)
+鈹� active_app_id / current_task_id / interaction_mode / tool_policy
+鈹� shadowing_index / shadowing_prompt
+鈹�
+鈹溾攢鈹� receiveAudioHandle.startToChat
+鈹� 1. match_voice_switch
+鈹� 2. mode 鍒嗘祦锛堜笉杩� chat 鐨�: PLAY_AUDIO / SHADOWING / TRANSLATION锛�
+鈹� 3. CHAT / BILINGUAL 鈫� intent + LLM
+鈹�
+鈹溾攢鈹� app_runtime
+鈹� registry(apps.json)
+鈹� start_app_task / switch_app / filter_tools
+鈹� handle_shadowing_turn / run_translation
+鈹�
+鈹溾攢鈹� Plugins: switch_app, start_repeater, start_shadowing, translate
+鈹溾攢鈹� Device MCP executor 鈫� self.mp3.play/stop
+鈹斺攢鈹� 鍙�� Server MCP translate 杩涚▼
+
+manager-api (Java) 鐢ㄦ埛/璁惧/Agent/鑱婂ぉ WAV 鈫� 鏈弬涓� App Runtime
+manager-web 鏅烘帶鍙� 鈫� 鏈弬涓�
+```
+
+### 4.3 Client 鈫� Server Architecture V2
+
+```text
+ 鈹屸攢鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹� 灏忕▼搴� / MQTT 瀹㈡埛绔� 鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�
+ 鈹� START_APP_TASK / SET_ALARM 鈹�
+ 鈹斺攢鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹攢鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�
+ 鈹� snlp MQTT
+ 鈻�
+鈹屸攢鈹�鈹�鈹�鈹�鈹�鈹�鈹� Client 鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹� Xiaozhi WS 鈹屸攢鈹�鈹�鈹�鈹�鈹�鈹�鈹� Server 鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�
+鈹� Task Gateway 鈹傗梽鈹�鈹� listen detect 鈹�鈹�鈻衡攤 App Runtime 鈹�
+鈹� [app_task] [voice_entry] 鈹� start_app_task 鈹�
+鈹� SD MP3 Skill 鈹傗梽鈹�鈹� type=mcp 鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈻衡攤 DeviceMCP play/stop 鈹�
+鈹� Chat AudioService 鈹傗梽鈹�鈹� opus/tts/stt 鈹�鈹�鈹�鈻衡攤 ASR / LLM / TTS 鈹�
+鈹� pending NVS 鈹� 鈹� connection memory 鈹�
+鈹斺攢鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹� 鈹斺攢鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹攢鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�
+ 鈹� 鐜版湁 report
+ 鈻�
+ manager-api
+ chat_history + audio BLOB
+```
+
+鍚屼竴鏃跺埢锛氫竴涓澶囦竴涓� active App/Task銆侰hat 涓� Task 闊抽浜掓枼娌跨敤 `PauseSdCardMp3`銆�
+
+---
+
+## 5. 鎶�鏈�哄姟
+
+### P0 鈥� 涓嶈В鍐冲氨鏃犳硶瀹夊叏缁х画寮�鍙�
+
+| ID | 鍊哄姟 | 绔嬪嵆淇紵 |
+|----|------|----------|
+| P0-1 | **鏃� App Runtime 鑷姩鍖栨祴璇�**锛孭hase 2 鏋佹槗鍥炲綊 Chat / 鍙ｄ护 / 澶嶈鍙屾挱 | **鏄��** Phase 2 绗竴鍒�灏辨槸 pytest + 鎵嬪伐鍥炲綊琛紝鍚﹀垯涓嶅噯鍔犺〃/鍔犲崗璁� |
+| P0-2 | 璺熻 Demo 琚褰撴垚銆屽凡鏈� FOLLOW_READ 閲囬泦銆嶁�斺�旀病鏈� prompt/asr/audio 缁戝畾锛孉ssessment 鏃犲钀界偣 | **鏄紙浣滀负 Phase 2 鐩爣锛夈��** 鍏佽鏆傛椂瀛樺湪璇勫垎锛屼笉鍏佽缁х画鍋囪宸查噰闆� |
+
+P0 涓嶆槸銆岀郴缁熶笉鑳借窇銆嶏紝鑰屾槸銆屼笅涓�闃舵浼氬缓鍦ㄦ祦娌欎笂銆嶃��
+
+### P1 鈥� 杩戞湡搴旇瑙ｅ喅
+
+| ID | 鍊哄姟 | 绔嬪嵆淇紵 |
+|----|------|----------|
+| P1-1 | 鏃� `play.completed`锛歋erver 闈犺澶囦富鍔ㄥ垏鍥� chat | Phase 2 鐢� `[skill_event]`锛屼笉瑕佹柊 WS type |
+| P1-2 | 鍙屾妧鑳借〃锛歚apps.json` vs `k_task_skills[]` 浼氭紓 | Phase 2 鍐欏绾︽鏌ユ竻鍗曪紱涓嶅悓姝ヤ笅杞� JSON |
+| P1-3 | 瀵勭敓鍗忚 `[app_task]` 涓庡敜閱掕瘝/ASR 鏂囨湰绌洪棿閲嶅彔 | 淇濇寔鍓嶇紑绾﹀畾锛涙枃妗ｅ寲锛汸hase 2 鍔� skill_event 鍓嶇紑 |
+| P1-4 | 绂荤嚎澶嶈涓� Server 杩炴帴鎬佸垎鍙� | 鍏佽鏆傛椂瀛樺湪锛涢噸杩炲悗浠ヨ澶� notify 鎴栦笅娆� detect 瀵归綈 |
+| P1-5 | 璺熻鑼冭璧� TTS 鑰岄潪闊抽闊抽锛屽拰澶嶈璧勬簮妯″瀷涓嶄竴鑷� | Phase 2 鍙厛閲囬泦 TTS 璺熻锛汼D 鑼冭鏀惧埌璧勬簮 Phase |
+| P1-6 | 缈昏瘧鍙屽疄鐜帮紙plugin + MCP锛� | 鍏佽鏆傛椂瀛樺湪锛汸hase 2 涓嶅垹 MCP锛屾祴璇曚互 plugin 涓哄噯 |
+| P1-7 | `BILINGUAL` 浠嶈蛋 `chat()`锛岄潬 prompt 绾︽潫锛屾槗澶规潅璇█ | 鍏佽鏆傛椂瀛樺湪锛涗笉鏄� Phase 2 涓昏矾寰� |
+
+### P2 鈥� 骞冲彴鍖栨椂鍐嶈В鍐�
+
+| ID | 鍊哄姟 | 绔嬪嵆淇紵 |
+|----|------|----------|
+| P2-1 | 鍙� MQTT锛圶iaozhi vs snlp锛� | 鍚� |
+| P2-2 | App 鐘舵�佷笉钀藉簱 | 鍚︼紙Phase 3 Instance锛� |
+| P2-3 | 澶嶈浠� Sening | 鍚� |
+| P2-4 | 鑱婂ぉ WAV LONGBLOB | 鍚� |
+| P2-5 | smartlearning 璇剧▼浜戞湭鍚告敹 | 鍚︼紙Phase 6锛� |
+| P2-6 | 鍥轰欢璁よ瘑涓氬姟 app_id 瀛楃涓� | 鍚︼紱淇濇寔鏌ユ壘琛紝绂佹涓氬姟鍒嗘敮鑶ㄨ儉 |
+| P2-7 | 鏃� `type=task` / `mode.set` | 鍚︼紱绛� Instance 鐪熸闇�瑕佹杩涘啀鍔� |
+| P2-8 | 灏忕▼搴� / 瀹堕暱鏃ュ巻 UI | 鍚� |
+
+---
+
+## 6. Master Plan 鍐嶅瑙�
+
+### 宸插畬鎴愶紙鐩稿闀挎湡鎰挎櫙鐨勩�岀涓�鍒�銆嶏級
+
+- Chat 浣滀负 default App/Task 娲讳笅鏉�
+- 璁惧鍙墽琛岄潪 Chat 鐨勬挱鏀� Skill
+- Server 鍙寜 Mode 璺宠繃 LLM
+- 涓夐�氶亾杩涘叆 Task 鐨勯鏋�
+- App 涓� Agent 鍒嗙锛圓gent 浠嶆槸浜鸿锛汚pp 鏄� `apps.json`锛�
+
+### 姝ｅ湪杩涜
+
+- 鑻辫瀛︿範锛氬璇诲彲鐢紱璺熻浠� Demo
+- 缈昏瘧 / 涓嫳瀵硅瘽锛氬彲鐢紝鏈骇鍝佸寲璇勪及
+
+### 闇�瑕佽皟鏁寸殑璁捐
+
+1. **銆孭hase 1 涓嶅仛 App銆嶄綔搴熴��** 钖� App Runtime 宸叉槸 Current Architecture V2 鐨勬牳蹇冿紝鍚庣画鍙姞娣憋紝涓嶆帹鍊掗噸鏉ャ��
+2. **銆岃窡璇� = Client Phase 2 鎵嶅嚭鐜般�嶄綔搴熴��** Mode 鍜屽彛浠ゅ凡鍦紱缂虹殑鏄噰闆嗕笌 Instance銆�
+3. **銆岀敤 type=task 鎵嶈兘鍋� Task銆嶆殏缂撱��** 瀵勭敓 detect + MQTT 宸插鍚姩锛涘厛琛� Event 涓� Session锛屽啀鑰冭檻鏂� WS type銆�
+4. **銆宎pps 涓嬪埌 SD銆嶆槑纭嫆缁濄��** 涓� Phase 1 鍚庢湡缁撹涓�鑷达紝Master Plan 鐨� App Sync 鏀逛负銆岃祫婧愪笅鍙戙�佸畾涔夌暀浜戙�嶃��
+5. **绂荤嚎绛栫暐鍐欏叆 Master Plan锛�** MultiNet 鍙繚鐣欐�诲叆鍙� + 鍙绾挎墽琛岀殑 Skill锛堝璇伙級锛涗簯鑳藉姏鍙ｄ护璧� ASR銆�
+
+### 宸蹭笉鍚堢悊鎴栧簲闄嶄紭鍏堢骇
+
+- 绗竴闃舵灏变笂 QUIZ/GAME 瀹屾暣 Mode
+- Phase 2 涓�娆″仛瀹� Task 琛� + 璇勫垎 + 瀵硅薄瀛樺偍 + 鏂板崗璁�
+- 鍚堝苟鍙� MQTT 浣滀负杩戞湡鐩爣
+- 鎶婅澶囧缓鎴愬畬鏁� Task Runtime
+
+### 闇�瑕佹柊鍔犲叆鐨勮兘鍔涳紙浠ｇ爜宸查�煎嚭鏉ワ級
+
+- 绂荤嚎鍙墽琛� Skill 涓庛�屼粎鍦ㄧ嚎 Mode銆嶇殑鏄庣‘鍒嗙被
+- ASR JSON `content` 瑙勮寖鍖栵紙鎵�鏈夐潪 Chat Mode 閮借锛�
+- 鍠囧彮鍥炲０绛栫暐鎸� Mode 閰嶇疆锛堟墦鏂� / 蹇界暐 / 鏂囨湰杩囨护锛�
+
+---
+
+## 7. 缁撹锛歅hase 1 鏄惁 Done
+
+**鍔熻兘 Done锛堝甫鏉′欢锛夛細** 钖勬鏋� + chat 榛樿 + 澶嶈 + 缈昏瘧 + 涓夐�氶亾鎸傞挬 + 绂荤嚎澶嶈锛屼笖鏈牬鍧忓崗璁涔夈�佹湭鏀� Java DDL銆�
+
+**宸ョ▼鏈� Done锛�** 鏃犺嚜鍔ㄥ寲娴嬭瘯銆佹棤璺熻閲囬泦銆佹棤鎾斁瀹屾垚浜嬩欢銆佸弻琛ㄦ紓绉婚闄┿��
+
+**鍥犳锛�** 涓嶅紑鍚� Java Task 琛ㄣ�佷笉寮�鍚� Assessment銆佷笉寮�鍚� App Store銆�
+涓嬩竴闃舵鍙仛涓�浠朵簨锛氭妸銆岄煶棰戜换鍔°�嶄粠 Demo 鍙樻垚 **鍙綊妗ｇ殑璺熻浼氳瘽**锛堣 `roadmap.md` / `phase-2-plan.md`锛夈��
diff --git a/phase-2-plan.md b/phase-2-plan.md
new file mode 100644
index 0000000..7d2a1bc
--- /dev/null
+++ b/phase-2-plan.md
@@ -0,0 +1,425 @@
+# Phase 2 Coding Plan 鈥� 鍐荤粨闈㈤獙璇侊紙閲囬泦 + 瀹屾垚浜嬩欢锛�
+
+> File: `phase-2-plan.md`
+> Date: 2026-08-31
+> **鐘舵�侊細宸插疄鏂姐��** 瀵圭収浜嬪疄瑙� `phase-2-review.md`銆備笅涓�闃舵 `phase-3-plan.md`銆�
+> 绾︽潫鏂囨。锛歚Xiaozhi Device Capability Specification v1.0.md`锛堣涔夊喕缁擄級
+> 璺嚎鍥撅細`roadmap.md`
+> 浠ｇ爜浜嬪疄锛堝紑宸ユ椂锛夛細`phase-1-review.md`
+> **涓庡師鏂� V1.1 Phase 2锛坱ype=task + Java 琛� + 璇勫垎锛夊啿绐佹椂锛屼互鏈枃 + Spec 涓哄噯銆�**
+
+鏈樁娈� **涓嶆槸** 鍐嶅仛涓�涓� App锛屼篃 **涓嶆槸** 鍋氬�掕儗/鍙ｇ畻/鎺ラ緳銆�
+鏈樁娈电敤 **宸叉湁** 澶嶈鏈� + 璺熻 Demo锛屾妸 Spec 宸插０鏄庛�佸浐浠跺皻鏈叿澶囩殑鍘熷瓙鑳藉姏琛ラ綈銆�
+
+缁� Coding Agent锛氬厛璇� Spec 搂1銆伮�4鈥�5銆伮�16鈥�17銆伮�20锛屽啀璇� `runtime.py`銆乣receiveAudioHandle.py`銆乣listenMessageHandler.py`銆乣reportHandle.py`銆乣app_task.cc`銆乣Board_OnSkillPlaybackFinished`銆�
+
+鍏佽 OTA 鐨勫敮涓�鐞嗙敱锛圫pec锛夛細琛ラ綈宸插０鏄庤兘鍔涖�傛湰闃舵鍏佽鐨勫浐浠舵敼鍔ㄤ粎闄� **D1 瀹屾垚/鍋滄浜嬩欢**銆傜姝负璺熻闊抽鎴栬瘎鍒嗘敼 C++銆�
+
+---
+
+## 0. 涓� Spec 鐨勫鐓э紙鏈樁娈靛彧閽夎繖浜涳級
+
+| Spec ID | 鑳藉姏 | Phase 2 鍋氫粈涔� | 涓嶅仛浠�涔� |
+|---------|------|----------------|----------|
+| D1 | `play_audio` 瀹屾垚浜嬩欢 | 涓婃姤 `AUDIO_COMPLETED` / `AUDIO_STOPPED` | 涓嶆敼 `repeat` loop锛圖15 宸叉湁锛夛紱涓嶇粺涓� resource_id锛涗笉涓� HTTP/WAV play |
+| D2 | pause/resume/stop | 浠呯敤鐜版湁 stop锛泂topped 浜嬩欢 | 涓嶅仛 TASK 绾� pause锛涗笉鎶� TTS-Pause-MP3 鏀规垚鏂板崗璁� |
+| D3 | `speak` | 缁х画鐜版湁 `type=tts` | 涓嶅湪璁惧鍋� TTS Skill |
+| D4 | barge_in 绛栫暐 | **淇濇寔** Phase 1锛欳HAT/BILINGUAL 鍙墦鏂紱PLAY_AUDIO/SHADOWING/TRANSLATION 闃插洖澹� | 涓嶆寜 App 鍚嶅啓 if |
+| D6 | `listen` 涓嶈繘 Chat | 璺熻缁х画璧扮幇鏈夊惉娴� + `handle_shadowing_turn` | 涓嶆敼 Opus 鐑矾寰� |
+| D7 | VAD / 涓�鍙ヨ瀹� | v1 鐢� **Server VAD + ASR 鍑哄彞** 浣滀负鍙ヨ竟鐣� | 涓嶅己鍒惰澶囨柊鎶� `user.speech.stopped` |
+| D8 | `record_clip` v1 | 鏈夋晥璺熻璧扮幇鏈� `enqueue_asr_report` | 涓嶅仛鏈湴 WAV 鏂囦欢 + upload锛坴1.1锛� |
+| D11 | `wait` | **涓嶅仛**锛涜窡璇诲彞闂寸敤 TTS 鑷劧闂撮殧 | 涓嶄负姣忎釜 App 澶嶅埗 sleep |
+| D14 | task_control | 鍙敤宸叉湁 START + 缁撴潫鍥� chat | 涓嶅仛 PAUSE/RESUME/RESTART 鏂板懡浠� |
+| D15 | 鎾斁绾� loop | **绂佹鏀�** mp3 `repeat` 璇箟 | 鈥� |
+| D16 | 姝ラ绾� loop | 璺熻涓嬩竴鍙ョ敱 Server `shadowing_index++` | 璁惧涓嶅緱銆岃嚜宸卞惊鐜崄鍙ヨ嫳璇�� |
+| S4/S8 | 璇勪及 / branch | **绂佹** | 瀵归敊涓嬩竴闃舵锛涜澶囦笉鍒ら |
+
+浠ｇ爜閲� Mode 鍚嶅彲浠嶄负 `SHADOWING`銆傛枃妗ｄ笌 Event payload 瑙嗗悓 Spec `FOLLOW_READ`銆備笉瑕佹湰闃舵鍏ㄥ眬閲嶅懡鍚嶏紝浠ュ厤鏃犳剰涔� diff銆�
+
+---
+
+## 1. 鐩爣涓庨潪鐩爣
+
+### 1.1 Objective
+
+1. 澶嶈锛圖1锛夛細鑷劧缁撴潫 / 鐢ㄦ埛鍋滄蹇呴』鍙樻垚 **閫昏緫浜嬩欢**锛孲erver 涓嶅啀闈� `[app_task]chat` 鐚滄祴銆�
+2. 璺熻锛圖6/D7/D8 v1锛夛細涓�娆′細璇濆彲褰掓。涓� prompt + asr_text + 鍙�夐煶棰戯紝涓� **涓嶈繘 LLM chat()**銆�
+3. 杩愯緭鍙彉銆佽涔変笉鍙橈細鏂板墠缂�鍙壙杞� Spec 宸叉湁 Event 鍚嶏紝**涓嶅鍔犲懡浠ょ绫�**銆�
+
+```text
+task_session_id
+ turns[]: index, prompt_text, asr_text, echo_rejected, recorded_at
+ events[]: audio.completed | audio.stopped | ...
+```
+
+### 1.2 Why Now
+
+Spec 搂20 鍐荤粨缂哄彛鍓嶄笁鏉★細瀹屾垚浜嬩欢銆侀潪 Chat 鍙ヨ竟鐣屻�佽窡璇诲綊妗ｃ�備笉琛ヨ繖涓夐」锛屼笉鑳藉绉� Device Capability 宸插喕缁擄紝Phase 3 寤鸿〃涔熸病鏈夊唴瀹广��
+
+### 1.3 Acceptance Criteria锛堝繀椤诲叏閮ㄦ弧瓒筹級
+
+**鑳藉姏锛�**
+
+1. 澶嶈鑷劧缁撴潫锛歋erver 鏀跺埌閫昏緫 `audio.completed`锛堣繍杈撹 搂4锛夛紝鍥� CHAT **鍙嫑鍛间竴娆�**銆�
+2. 澶嶈琚� stop/鎶㈠崰锛歚audio.stopped`锛�**涓�**涓诲姩鎷涘懠銆�
+3. 绂荤嚎澶嶈 / 绂荤嚎銆屽垏鎹㈡ā寮忋�嶏細浠嶄笉 `WakeWordInvoke`锛圥hase 1 绾緥锛夈��
+4. 璺熻鍏ㄧ▼涓嶈蛋 `chat()`锛涜绗戣瘽涓嶄細褰撻棽鑱婂洖绛斻��
+5. 瀹屾暣璺熻鎴栥�岄��鍑鸿窡璇汇�嶅悗锛宍data/task_sessions/*.json` 杞涓庡疄闄呬竴鑷达紱**鏈�鍚庝竴杞� asr 蹇呴』鍐欏叆鍚庡啀鍥� chat**銆�
+6. `report_asr_enable` 涓� `chat_history_conf != 0` 鏃讹紝鏈夋晥璺熻鍏ラ槦 report锛涘洖澹版嫆缁濅笌 PLAY_AUDIO 蹇界暐涓婅 **涓嶅緱** 褰撲綔鐢ㄦ埛璇濊涓婃姤銆�
+
+**鍐荤粨绾緥锛圫pec 搂22锛夛細**
+
+7. 涓嶆柊澧� WS `type`銆佷笉鏂板 MQTT action 绉嶇被銆佷笉鏂板 MCP 宸ュ叿绉嶇被锛堝埆鍚嶆槧灏勯櫎澶栵紝鏈樁娈典篃涓嶆敼 MCP 鍚嶏級銆�
+8. 涓嶅閿欍�佸嚭棰樸�佺炕璇戠畻娉曘�佹帴榫欒瘝搴撹繘鍥轰欢銆�
+9. 涓嶆妸 `apps.json` 涓嬪彂鍒� SD銆�
+10. 涓嶆敼 `hello/listen/tts/stt` 瀛楁鍚箟锛屼笉鏀� Opus 甯с��
+11. 鏃� Java DDL銆佹棤姝ｅ紡鍒嗘暟瀛楁銆�
+
+### 1.4 Out of Scope
+
+- Phase 4b锛氬�掕儗 / 鍙ｇ畻 / 鎺ラ緳
+- `task_instance` 琛ㄣ�乣mode.set`銆乣type=task` / `type=command`
+- Assessment銆丏evice `if correct`
+- 璺熻蹇呮挱 SD MP3锛圱TS 鑼冭鍚堟硶锛涙爣鍑嗛煶鏄� Phase 6锛�
+- MultiNet 澧炲姞璺熻/缈昏瘧鍙ｄ护
+- `wait` Skill銆乣TASK_PAUSE`銆乣indicate` 鏂伴�昏緫鎬併�佹寜閽暱鎸夐��鍑� Task
+- `record_clip` 鏈湴鏂囦欢
+- 鍚堝苟鍙� MQTT銆乵anager-web / 灏忕▼搴�
+- 鍐峰満澶氭涓诲姩鑱婂ぉ銆侀潪 Sening 鏉� play_audio 骞冲彴鍖�
+
+---
+
+## 2. 鏋舵瀯锛堟湰闃舵锛�
+
+```text
+Device Capability锛堝喕缁撹涔夛級 杩愯緭锛堟湰闃舵锛� Server
+play_audio loop=N (宸叉湁 D15) 鈫� MCP self.mp3.play 鈫� 涓嶈疆璇㈤亶鏁�
+audio.completed|stopped 鈫� [skill_event]... 鈫� Session.events + 鍥� chat
+listen 娴� (D6) 鈫� 鐜版湁 Opus 鈫� ASR锛汳ode鈮燙HAT 涓� chat()
+鍙ヨ竟鐣� (D7 v1) 鈫� ASR 鍑哄彞 鈫� handle_shadowing_turn
+record_clip v1 (D8) 鈫� 鐜版湁 report HTTP 鈫� chat_history WAV
+speak (D3) 鈫� type=tts 鈫� 璺熻鑼冭 / 鍥炶亰鎷涘懠
+```
+
+TaskSession **涓嶆槸** TaskInstance锛堥偅鏄� Phase 3锛夈�俙task_session_id` = uuid4锛屼笉瑕佺敤 WS `session_id` 褰撻暱鏈熶富閿��
+
+璺熻涓嬩竴鍙� = Spec **D16 姝ラ绾� loop锛圫erver锛�**锛岀姝㈠仛鎴愯澶囧唴鍙ュ瓙鍒楄〃銆�
+
+---
+
+## 3. TaskSession schema
+
+璺緞锛歚xiaozhi-esp32-server/main/xiaozhi-server/data/task_sessions/{task_session_id}.json`
+
+```json
+{
+ "task_session_id": "uuid",
+ "ws_session_id": "conn.session_id",
+ "device_id": "mac",
+ "app_id": "english_learning",
+ "task_id": "shadowing",
+ "source": "voice",
+ "interaction_mode": "SHADOWING",
+ "started_at": 0,
+ "ended_at": 0,
+ "status": "running",
+ "end_reason": "",
+ "turns": [
+ {
+ "index": 0,
+ "prompt_text": "Hello, how are you today?",
+ "asr_text": "hello how are you today",
+ "echo_rejected": false,
+ "recorded_at": 0
+ }
+ ],
+ "events": [
+ {
+ "name": "audio.completed",
+ "app_id": "english_learning",
+ "task_id": "repeater",
+ "at": 0
+ }
+ ]
+}
+```
+
+`status`: `running | completed | cancelled`
+`end_reason`: `finished | user_exit | preempted | disconnected | play_completed | play_stopped`
+
+浜嬩欢 `name` 鐢� Spec 閫昏緫鍚嶏細`audio.completed` / `audio.stopped`锛堜笉瑕佸啀鐢ㄥ惈绯婄殑 `play.xxx` 褰撴。妗堝瓧娈碉紱杩愯緭瀛楃涓茶 搂4锛夈��
+
+鏂嚎锛歚ConnectionHandler` 娓呯悊璺緞 flush锛宍cancelled` + `disconnected`锛屽凡鏈� turns 涓嶅緱涓€��
+
+---
+
+## 4. 鍗忚锛氳繍杈撴槧灏勫埌 Spec Event
+
+**绂佹**鏂板 Xiaozhi `type`銆傛湰闃舵杩愯緭 = `listen` `state=detect` 鏂囨湰鍓嶇紑锛圥hase 1 宸茬敤锛夈��
+鏀硅繍杈撲互鍚庯紙`type=event`锛変笉绠楀紩杩涙柊鑳藉姏锛涙湰闃舵涓嶈鍋氶偅涓�姝ャ��
+
+### 4.1 `[skill_event]` 鈫� Spec Device Event
+
+| detect 鏂囨湰 | Spec Event | 浣曟椂 |
+|-------------|------------|------|
+| `[skill_event]audio.completed|{app_id}|{task_id}` | `AUDIO_COMPLETED` | `Board_OnSkillPlaybackFinished(stopped=false)` |
+| `[skill_event]audio.stopped|{app_id}|{task_id}` | `AUDIO_STOPPED` | `stopped=true`锛堢敤鎴�/鎶㈠崰/stop锛� |
+
+瑙勫垯锛�
+
+- 涓夋锛宍|` 鍒嗛殧銆俙app_id`/`task_id` 鏉ヨ嚜褰撳墠 pending锛坄s_current_app` / `s_current_task`锛夛紝**涓嶆槸**鏂扮殑 `if (english)` 鍒嗘敮銆�
+- Server **绂佹**鍥� completed 鍐嶈皟涓�娆� `self.mp3.play`銆�
+- **鍙暀涓�鏉″洖鑱婅矾寰�**锛岀姝� completed 鍚庡啀鍙� `[app_task]chat` 瀵艰嚧涓ゆ鎷涘懠銆�
+
+**鎺ㄨ崘锛�**
+
+```text
+鑷劧缁撴潫锛氬彧鍙� [skill_event]audio.completed|english_learning|repeater
+ Server start_app_task(chat, execute_skills=false) 骞� TTS 鎷涘懠
+鐢ㄦ埛鍋滄锛氬彧鍙� [skill_event]audio.stopped|...
+ 涓嶄富鍔ㄦ嫑鍛硷紱鏈湴 pending 浠嶅啓鍥� chat.conversation
+```
+
+娌＄綉锛歚SendListenDetect` 宸茬煭璺紝涓嶄负鏈簨浠跺己琛岃繛浜戙�傜绾挎挱瀹屽彧鏀� NVS pending銆�
+
+鍏煎锛氭棫鍥轰欢鑻ヤ粛鍙� `[app_task]chat`锛孲erver 鍙綋鍥炶亰澶勭悊銆�**鏂板浐浠朵笉瑕佷袱绉嶉兘鍙戙��**
+
+### 4.2 淇濇寔涓嶅彉锛堜笉澧炲姞鍛戒护绉嶇被锛�
+
+```text
+[app_task]app|task|source 鎬佸悓姝ワ紝execute_skills=false
+[voice_entry] 鑿滃崟
+hello / listen start|stop / tts / mcp / START_APP_TASK / SET_ALARM
+```
+
+`listenMessageHandler` 瑙ｆ瀽椤哄簭锛�
+
+```text
+[skill_event] 鈫� [app_task] 鈫� [voice_entry] 鈫� 鍞ら啋璇�
+```
+
+鏈煡鍓嶇紑涓嶅緱閫佽繘 LLM銆�
+
+---
+
+## 5. Server 鏀瑰姩娓呭崟
+
+浠撳簱锛歚xiaozhi-esp32-server/main/xiaozhi-server`
+鍙姩 Python銆俶anager-api **绂佹鏀硅〃**銆�
+
+| ID | 浠诲姟 | 鏂囦欢 | 璇存槑 |
+|----|------|------|------|
+| S1 | `TaskSession` | `core/app_runtime/session.py` | dataclass + load/save锛沗data/task_sessions/`锛沵kdir 澶辫触鍙墦鏃ュ織 |
+| S2 | 杩炴帴鎸佹湁 session | `core/connection.py` | `self.task_session = None` |
+| S3 | begin/end | `runtime.py` `start_app_task` | 绂诲紑 chat 鏃� `begin_session`锛涜繘鍏� chat 鏃� `end_session` |
+| S4 | 璺熻璁� turn | `handle_shadowing_turn` | 鏈夋晥杞� report锛沞cho **涓嶄笂鎶�**锛�**鍏� add_turn 鍐�**鍦ㄦ渶鍚庝竴杞洖 chat |
+| S5 | 缁撴潫鍘熷洜 | 鍙ュ瓙鑰楀敖 `finished`锛涢��鍑� `user_exit` | flush |
+| S6 | 鍚� skill_event | `listenMessageHandler.py` | completed 鈫� 鍥� chat + `end_reason=play_completed`锛泂topped 涓嶆嫑鍛� |
+| S7 | 鏂嚎 flush | `connection.py` 娓呯悊璺緞 | cancelled |
+| S8 | 瀵煎嚭 | `__init__.py` | 鏃犲惊鐜鍏� |
+| S9 | pytest | `tests/test_app_runtime.py` | 搂8 |
+| S10 | ASR JSON | `_plain_user_text` 鍗曟祴 | 涓嶆敼 FunASR |
+
+璺熻 turn 绛栫暐锛堢畝鍗曪級锛氱敤鎴� **鏈夋晥**寮�鍙ｆ墠 `add_turn(index, current_prompt, asr)`锛屽啀 `index+1` 骞� `speak` 涓嬩竴鍙ワ紙D3锛夈�傚綋鍓嶄唬鐮佹渶鍚庝竴杞洿鎺ュ洖 chat銆佷笉璁� asr 鈥斺�� 蹇呴』淇��
+
+`enqueue_asr_report` 鍘婚噸锛氬厛 grep `startToChat` / ASR stop 璋冪敤閾俱�傚凡涓婃姤鍒欒窡璇绘湁鏁堣疆涓嶈鍐嶅叆闃熴�侾LAY_AUDIO 蹇界暐涓婅涓� echo 杞姝� report銆�
+
+---
+
+## 6. Client 鏀瑰姩娓呭崟
+
+浠撳簱锛歚xiaozhi-esp32-2.2.6`
+浠� Sening Task Gateway銆傜姝㈡柊 DeviceState銆佺姝㈣嫳璇瘎鍒嗐�佺姝㈣窡璇诲彞瀛愯〃杩涘浐浠躲��
+
+| ID | 浠诲姟 | 鏂囦欢 | 璇存槑 |
+|----|------|------|------|
+| C1 | D1 completed | `app_task.cc` `Board_OnSkillPlaybackFinished` | 鍙� detect `audio.completed`锛�**鍒犻櫎** `notify_server(chat)` |
+| C2 | D1 stopped | 鍚屼笂 | `open_if_idle=false` |
+| C3 | 绂荤嚎 | 宸叉湁 `SendListenDetect` 鐭矾 | 涓嶈涓轰簨浠跺己杩炰簯 |
+| C4 | D15 | `mp3_player.cc` | **绂佹鏀�** repeat / power_off_after 璇箟 |
+| C5 | 绂佸尯 | 鈥� | 涓嶆柊澧� MultiNet 璇嶏紱涓嶆敼 Opus锛涗笉鍔� `wait`/PAUSE |
+
+浠婃棩锛�
+
+```text
+pending 鈫� chat
+was_repeater && !stopped 鈫� notify_server(chat, true)
+```
+
+鏀逛负锛�
+
+```text
+pending 鈫� chat.conversation // 绂荤嚎榛樿涓嶅彉
+鑻� was_repeater 涓旓紙閫氶亾宸插紑鎴栫綉缁滃氨缁級锛�
+ completed 鈫� SendListenDetect("[skill_event]audio.completed|...|...", open_if_idle=true)
+ stopped 鈫� SendListenDetect("[skill_event]audio.stopped|...|...", open_if_idle=false)
+```
+
+`app_id|task_id` 鐢ㄧ粨鏉熷墠鐨� `s_current_app` / `s_current_task`锛堝嚱鏁板紑澶村厛鎷疯礉鍐嶆敼 pending锛夈��
+
+---
+
+## 7. 璺熻閲囬泦锛圖6 / D7 / D8 v1锛�
+
+浠婃棩椤哄簭锛�
+
+```text
+exit? 鈫� speaking? drop 鈫� echo? drop 鈫� send_stt 鈫� index+1 鈫� 鏃犲彞鍒欏洖 chat
+```
+
+鐩爣锛�
+
+```text
+exit? 鈫� end_session(user_exit) 鈫� start_app_task(chat)
+speaking? drop锛堜笉璁� turn锛孌4 鑼冭涓拷鐣ユ嬀闊筹級
+echo? 鍙 events.echo_rejected锛涗笉 report锛涗笉鎺ㄨ繘
+鏈夋晥锛�
+ add_turn
+ enqueue_asr_report锛堣嫢 ASR 灞傚皻鏈姤锛�
+ send_stt_message 鍙繚鐣�
+ 鏃犱笅涓�鍙ワ細end_session(finished) 鈫� chat
+ 鍚﹀垯锛歩ndex+1锛宻peak 涓嬩竴鍙�
+```
+
+鑼冭浠嶇敤 Server TTS锛圖3锛夈�備笉瑕佹湰闃舵鏀规垚 SD 鏍囧噯闊炽��
+
+---
+
+## 8. 娴嬭瘯
+
+### 8.1 pytest锛圥0锛�
+
+`xiaozhi-esp32-server/main/xiaozhi-server/tests/test_app_runtime.py`
+鍋� `conn`锛屼笉鍚� WS锛屼笉鏀� Opus銆�
+
+| 鐢ㄤ緥 | 鏂█ |
+|------|------|
+| `match_voice_switch("寮�濮嬭窡璇�")` | `english_learning, shadowing` |
+| `match_voice_switch("鍒囨崲妯″紡")` | `__menu__` |
+| `match_voice_switch("鎵撳紑缈昏瘧")` | translation |
+| `is_echo_of_prompt` 鐩稿悓鍙� | True |
+| `TaskSession` save/load | 杞涓�鑷� |
+| `_plain_user_text` JSON | 鎶藉嚭 content |
+| 瑙ｆ瀽 `[skill_event]audio.completed|english_learning|repeater` | 寰楀埌閫昏緫浜嬩欢鍚� |
+
+mode 鍒嗘祦鎶� `route_user_text` **浠呭綋鏀瑰姩寰堝皬**銆傚惁鍒欏彛浠� + session 鍗曟祴 + 鎵嬪伐琛ㄣ��
+
+### 8.2 鎵嬪伐鍥炲綊锛圥R 鍕鹃�夛級
+
+Chat锛�
+
+- [ ] 鍞ら啋瀵硅瘽銆佹墦鏂紙D4 CHAT锛夈�佸ぉ姘旀彃浠�
+- [ ] 銆屾垜浠潵鑱婂ぉ鍚с�嶄粠缈昏瘧/璺熻鍥炴潵
+
+璺熻锛堝喕缁撻獙璇佷欢锛夛細
+
+- [ ] 涓嶈繘闂茶亰 LLM
+- [ ] 閫�鍑鸿窡璇诲洖 chat
+- [ ] JSON 鍚叏閮ㄦ湁鏁堣疆锛堝惈鏈�鍚庝竴杞級
+- [ ] 鍠囧彮鍥炲０涓嶆帹杩涘彞瀛�
+
+澶嶈锛圖1锛夛細
+
+- [ ] MQTT START_APP_TASK 涓嶅弻鎾�
+- [ ] 鑷劧缁撴潫鍙嫑鍛间竴娆�
+- [ ] 绂荤嚎澶嶈銆佺绾垮垏鎹㈡ā寮忎笉杩炰簯
+- [ ] `repeat` N 娆¤涓轰笌 Phase 1 鐩稿悓锛圖15 鏈牬鍧忥級
+
+缈昏瘧锛�
+
+- [ ] 涓啍鑻憋紱閫�鍑哄悗绗戣瘽鎭㈠锛堣瘉鏄庢湭鎶婄炕璇戝杩涘浐浠讹級
+
+**涓嶈**鏈樁娈甸獙鏀跺�掕儗/鍙ｇ畻/鎺ラ緳銆�
+
+---
+
+## 9. 瀹炴柦椤哄簭
+
+```text
+P2-A session.py + begin/end + 鏂嚎 flush
+ 杩涜窡璇�/澶嶈鏈� running JSON锛涘洖 chat 鍙� completed
+
+P2-B handle_shadowing_turn 璁� turn + report 鍘婚噸
+ 瀹屾暣璺熻杞姝ｇ‘锛堝惈鏈�鍚庝竴杞級
+
+P2-C [skill_event] audio.completed|stopped + Handler
+ 澶嶈缁撴潫鍗曟鎷涘懠锛泂topped 涓嶆嫑鍛�
+
+P2-D pytest + 鎵嬪伐鍥炲綊锛堝惈绂荤嚎涓� D15 repeat锛�
+```
+
+鏈畬鎴� A 涓嶈鏀规挱鏀惧櫒銆傛湭瀹屾垚 C 涓嶅緱瀹ｇО D1 宸插喕缁撱�傛湭瀹屾垚 D 涓嶅緱瀹ｇО Phase 2 Done銆�
+
+---
+
+## 10. 缂栫爜绾︽潫锛圫pec 搂16鈥�17 钀藉埌鏈� PR锛�
+
+1. 鍥轰欢涓嶅緱鏂板 `if (app == english)` 涓氬姟鏍戯紱浜嬩欢閲岀殑 id 鏉ヨ嚜 pending 鏌ユ壘琛ㄣ��
+2. 涓嶅緱鎶� ASR/LLM/璇勫垎/鍙ｇ畻瑙ｆ瀽/鎴愯璇嶅簱鍐欏叆 Client銆�
+3. 涓嶅緱鎶� QUIZ/FOLLOW_READ 鍋氭垚鏂� `DeviceState`銆�
+4. 涓嶅緱鏂板 WS type / MQTT action / MCP 宸ュ叿銆�
+5. 涓嶅緱鏀� `repeat` 璇箟銆佷笉寰椾笅鍙� `apps.json`銆佷笉寰楀啓 Redis/MySQL Session銆�
+6. manager-api / manager-web 闆舵彁浜わ紙report 鐜版湁 HTTP 涓嶅鐢ㄦ椂鍏堝湪 PR 璇存槑锛屼粛浼樺厛 `manage_report`锛夈��
+7. 姣忎釜 PR 鍏� Chat 鍥炲綊鍐嶅悎璺熻/澶嶈浜嬩欢銆�
+
+---
+
+## 11. 椋庨櫓涓庡洖褰�
+
+| 椋庨櫓 | 缂撹В |
+|------|------|
+| 鍥炲０褰撹窡璇绘垚鍔� | 淇濇寔 speaking 蹇界暐 + echo 鏂囨湰杩囨护锛沞cho 涓� report |
+| completed + `[app_task]chat` 鍙屾嫑鍛� | 搂4.1 鍙暀涓�鏉¤矾寰� |
+| report 鍙屼唤 | grep 璋冪敤閾撅紝鍙暀涓�澶� |
+| 鏃у浐浠舵棤 skill_event | Server 浠嶆帴鍙楁棫 `[app_task]chat`锛涙柊鍥轰欢鍙彂浜嬩欢 |
+| 鎶� Phase 2 鍋氭垚鏂� App | 楠屾敹琛ㄤ笉鍚�掕儗/鍙ｇ畻/鎺ラ緳 |
+| 鎶婃楠ゅ惊鐜笅娌夎澶� | 鍙ュ瓙鍒楄〃鍙湪 Server `apps.json` / config |
+
+---
+
+## 12. Data Flow
+
+```text
+澶嶈锛堥獙璇� D1 + D15锛�
+ START_APP_TASK / 绂荤嚎鍙ｄ护
+ 鈫� play_audio loop=N锛堣澶囧唴寰幆锛�
+ 鈫� audio.completed|stopped
+ 鈫� Server Session.events
+ 鈫� completed锛歋erver 鍥� chat锛泂topped锛氫笉鎷涘懠
+
+璺熻锛堥獙璇� D3/D4/D6/D7/D8v1 + D16锛�
+ 銆屽紑濮嬭窡璇汇�嶁啋 start_app_task(shadowing) 鈫� begin_session
+ 鈫� speak 绗竴鍙ワ紙Server TTS锛�
+ 鈫� listen 娴� 鈫� ASR 鍑哄彞 鈫� handle_shadowing_turn
+ 鈫� add_turn + report 鈫� Server 鍐冲畾涓嬩竴鍙ユ垨缁撴潫
+ 鈫� 缁撴潫 鈫� chat 鎷涘懠
+```
+
+---
+
+## 13. State Machine
+
+```text
+DeviceState: 浠嶅彧鏈� Idle / Listening / Speaking / ... 锛堢姝㈠姞 SHADOWING 杈癸級
+
+InteractionMode:
+ CHAT 鈹�鈹�start repeater鈹�鈹�鈻� PLAY_AUDIO 鈹�鈹�audio.completed鈹�鈹�鈻� CHAT
+ 鈹斺攢鈹�audio.stopped鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈻� CHAT锛堟棤鎷涘懠锛�
+ CHAT 鈹�鈹�start shadowing鈹�鈻� SHADOWING(=FOLLOW_READ)
+ prompting(speak) 鈫� listening 鈫� turn_recorded 鈫� 鈥�
+ user_exit | finished 鈹�鈹�鈻� CHAT
+```
+
+---
+
+## 14. 瀹屾垚鍚庯紙浜虹被娴佺▼锛屼笉鏄湰 PR 浠ｇ爜锛�
+
+```text
+Coding
+ 鈫� Completion Review锛堝鐓ф湰鏂� 搂1.3 + Spec 搂20 鍓嶄笁鏉★級 鉁� phase-2-review.md
+ 鈫� Architecture V3锛堜簨浠惰繍杈撱�丼ession 鏂囦欢銆丼D 鎸� App 鍚嶅缓鏍戯級
+ 鈫� Roadmap Review
+ 鈫� phase-3-plan.md锛圛nstance 琛紱浠嶆棤鏂� Device 鑳藉姏锛�
+```
+
+绂佹鏈樁娈点�岄『渚裤�岼ava 琛ㄣ�丄ssessment銆佸�掕儗/鍙ｇ畻/鎺ラ緳銆乣wait` Skill銆佹湰鍦板綍闊充笂浼犮��
diff --git a/phase-2-review.md b/phase-2-review.md
new file mode 100644
index 0000000..e48a754
--- /dev/null
+++ b/phase-2-review.md
@@ -0,0 +1,388 @@
+# Phase 2 Completion Review
+
+> Date: 2026-08-31
+> Basis: **current code**锛堜笉鏄� `phase-2-plan.md` 鐨勫師鏂囨効鏈涳級
+> Compared against: `phase-2-plan.md` 路 `phase-2-voice-test.md` 路 `roadmap.md` 路 `Xiaozhi Device Capability Specification v1.0.md` 路 `phase-1-review.md`
+> Repositories: `xiaozhi-esp32-2.2.6`锛圕lient锛壜� `xiaozhi-esp32-server`锛圫erver锛�
+
+鏈枃浠跺彧璁板綍浜嬪疄銆佸樊寮傘�佸�哄姟涓庡 Spec / Roadmap 鐨勪慨姝ｃ�備笅涓�闃舵鏂藉伐瑙� `phase-3-plan.md`锛岄獙鏀惰 `phase-3-test.md`銆�
+
+鐩稿叧鏂囨。绱㈠紩锛�
+
+| 鏂囨。 | 瑙掕壊 |
+|------|------|
+| `Xiaozhi Device Capability Specification v1.0.md` | Device 璇箟鍐荤粨 |
+| `roadmap.md` | 闃舵鍒囧垎 |
+| `phase-1.md` / `phase-1-review.md` | 钖� App Runtime 鐨勮鍒掍笌浜嬪疄 |
+| `phase-2-plan.md` | 鏈樁娈靛師鏂囨柦宸ュ崟锛堥噰闆� + 瀹屾垚浜嬩欢锛� |
+| `phase-2-voice-test.md` | 鐑у綍鍚庡彛浠ゅ洖褰� |
+| Client / Server Plan V1.1 | 闀挎湡鎰挎櫙锛涗笌 Spec 鍐茬獊鏃朵互 Spec + 鏈� Review 涓哄噯 |
+
+---
+
+## 1. 褰撳墠浠ｇ爜鐩樼偣锛堜互浠撳簱涓哄噯锛�
+
+### 1.1 Phase 2 瀹為檯鏀瑰姩鐨勬枃浠�
+
+#### Client锛坄xiaozhi-esp32-2.2.6`锛屼粎 Sening锛�
+
+| 鏂囦欢 | 瑙掕壊 |
+|------|------|
+| `main/boards/sening-xiaozhi-s3/app_task.cc` | `[skill_event]audio.{completed\|stopped}\|app\|task\|duration_ms`锛沗Board_EnsureAppLayout` |
+| `main/boards/sening-xiaozhi-s3/app_task.h` | 澹版槑 `Board_EnsureAppLayout` |
+| `main/boards/sening-xiaozhi-s3/mp3_player.cc` | 鎾斁鏃堕暱 `duration_ms`锛坄xTaskGetTickCount`锛夛紱**鏈敼** `repeat` / `power_off_after` |
+| `main/boards/sening-xiaozhi-s3/system.c` / `system.h` | `SDCard_Ensure_Business_Dir`锛堝寘瑁呭凡鏈� `mkdir_recursive`锛� |
+| `main/boards/sening-xiaozhi-s3/sening_xiaozhi_s3.cc` | MCP `self.sd.ensure_app` |
+| `main/boards/sening-xiaozhi-s3/mqtt.c` | `ENSURE_APP_DIRS`锛沗START_APP_TASK` 琛ヨ繘 ctl 鐧藉悕鍗� |
+
+鏈敼锛歄pus 鐑矾寰勩�乣DeviceState` 杩佺Щ琛ㄣ�乣hello/listen/tts/stt` 瀛楁鍚箟銆丮ultiNet 璇嶈〃銆乣repeat` 寰幆璇箟銆傞潪 Sening 鏉夸粛鏃� Task Gateway銆�
+
+#### Server锛堜粎 Python `xiaozhi-server`锛�
+
+| 鏂囦欢 | 瑙掕壊 |
+|------|------|
+| `core/app_runtime/session.py` | **鏂板銆�** `TaskSession` 鏂囦欢褰掓。锛沗parse_skill_event`锛汚SR report 鎺ㄨ繜绛栫暐 |
+| `core/app_runtime/runtime.py` | begin/end session锛涘悆 skill_event锛涜窡璇� SD 閫愬彞 play锛涘惉绐� `1.2 脳 duration_ms`锛涜闃� App 寤虹洰褰� |
+| `core/app_runtime/apps.json` | 璺熻 enter_prompt 鏀逛负 SD 璺緞锛沘llowlist 澧炲姞 `self_sd_ensure_app` |
+| `core/connection.py` | `task_session`锛涙柇绾� `end_session(disconnected)` |
+| `core/handle/textHandler/listenMessageHandler.py` | `[skill_event]` 浼樺厛浜� `[app_task]` |
+| `core/providers/asr/base.py` | PLAY_AUDIO / SHADOWING 涓嶇珛鍗� `enqueue_asr_report` |
+| `core/providers/tools/device_mcp/mcp_handler.py` | tools/list 灏辩华鍚庤皟搴﹀缓鐩綍 |
+| `tests/test_app_runtime.py` | pytest锛堝彛浠� / session / skill_event / SD 璺緞锛� |
+
+#### manager-api / manager-web
+
+**鏃� App Runtime 鐩稿叧鏀瑰姩銆�** 鏃犳柊琛ㄣ�佹棤鏂� REST銆傝窡璇婚煶棰戜粛璧扮幇鏈� `POST /agent/chat-history/report`銆�
+
+### 1.2 瀹為檯澧炲姞鐨勬ā鍧�
+
+| 妯″潡 | 浣嶇疆 | 鑱岃矗 |
+|------|------|------|
+| TaskSession | Server `session.py` | 杩炴帴绾у綊妗ｏ細turns + events 鈫� `data/task_sessions/{uuid}.json` |
+| Skill Event 杩愯緭 | detect 鏂囨湰 `[skill_event]` | Spec `AUDIO_COMPLETED` / `AUDIO_STOPPED` |
+| SD App Layout | 鍥轰欢 mkdir + Server ensure | `/{app_id}/{task_id}/lessons` |
+| 璺熻姝ヨ繘 | Server `runtime.py` | D16锛氫竴鍙ヤ竴 MP3 鈫� 鍚獥 鈫� 涓嬩竴鍙� |
+
+娌℃湁锛歍askInstance 琛ㄣ�丼tep 寮曟搸銆丄ssessment銆丼cheduler銆丄pp Store銆乣type=task`銆�
+
+### 1.3 瀹為檯澧炲姞鐨� API / 鍗忚
+
+| 绫诲瀷 | 鍚嶇О | 璇存槑 |
+|------|------|------|
+| REST | **鏃�** | 鈥� |
+| Device MCP | `self.sd.ensure_app` | **璁″垝澶栥��** `app_id` + 鍙�� `task_id` 鈫� mkdir |
+| MQTT | `ENSURE_APP_DIRS` | **璁″垝澶栥��** payload `{app_id, tasks:[]}` |
+| WS type | **鏃犳柊 type** | 浠嶅瘎鐢� `listen/detect` |
+| detect 鍓嶇紑 | `[skill_event]audio.completed\|app\|task\|duration_ms` | Spec D1锛涚 4 娈垫椂闀垮彲閫� |
+| detect 鍓嶇紑 | `[skill_event]audio.stopped\|...` | 鐢ㄦ埛/鎶㈠崰鍋滄 |
+
+`duration_ms` 鏄師鏂囪鍒掓病鏈夈�佸疄鏂戒腑琛ヤ笂鐨勫瓧娈碉紱璇箟浠嶆槸 D1 瀹屾垚浜嬩欢锛屼笉鏄柊鍛戒护绉嶇被銆�
+
+### 1.4 鏁版嵁搴�
+
+**鏃犳柊琛ㄣ��** Session 鍦ㄦ湰鍦� JSON銆侰hat WAV 浠嶅湪 `ai_agent_chat_audio`銆俉S 鏂嚎浼� flush JSON锛坄disconnected`锛夛紝涓嶈惤 manager-api銆�
+
+### 1.5 Client 鏋舵瀯鍙樺寲锛堢浉瀵� Phase 1 V2锛�
+
+```text
+Task Gateway
+ Board_OnSkillPlaybackFinished(stopped, power_off, duration_ms)
+ 鈹溾攢鈹� 浠呭綋褰撳墠 task 鐨� skill 鍚� self.mp3.play锛堝璇伙級鏃讹細pending 鍐欏洖 chat
+ 鈹溾攢鈹� 璺熻锛氭挱瀹屽彧涓婃姤 completed锛屼笉鍋滀换鍔�
+ 鈹斺攢鈹� SendListenDetect([skill_event]...) 涓嶅啀鍙� [app_task]chat
+
+ Board_EnsureAppLayout(app_id, task_id)
+ 鈹斺攢鈹� SDCard_Ensure_Business_Dir("/{app}" 鎴� "/{app}/{task}/lessons")
+
+MCP self.sd.ensure_app
+MQTT ENSURE_APP_DIRS锛坈tl 涓婚锛�
+```
+
+DeviceState 浠嶅彧绠″惉/璇淬�傚浐浠朵粛鏃� InteractionMode 鏋氫妇銆傝窡璇诲彞瀛愯〃 **涓嶅湪** 鍥轰欢銆�
+
+### 1.6 Server 鏋舵瀯鍙樺寲
+
+```text
+listen detect
+ [skill_event] 鈫� handle_skill_event
+ PLAY_AUDIO completed 鈫� end_session(play_completed) 鈫� 鍥� chat 鎷涘懠涓�娆�
+ PLAY_AUDIO stopped 鈫� end_session(play_stopped) 鈫� 鍥� chat 涓嶆嫑鍛�
+ SHADOWING completed 鈫� 寮�鍚獥 1.2脳duration_ms锛堟渶灏� 800ms锛�
+ SHADOWING stopped 鈫� 缁撴潫璺熻鍥� chat
+
+start_app_task(shadowing)
+ ensure_app_sd_dirs
+ self_mp3_get_list 鈫� lesson_mp3_clips(/{app}/{task}/lessons/)
+ 閫愬彞 self_mp3_play(repeat=1)
+ 鏈夋晥 ASR 鈫� add_turn + report 鈫� 涓嬩竴鍙�
+ 鍚獥瓒呮椂 鈫� listen_timeout 浜嬩欢 鈫� 涓嬩竴鍙ワ紙涓嶈鎴愬姛 turn锛�
+
+MCP tools/list 灏辩华 鈫� 鍚庡彴涓烘敞鍐岃〃閲岄渶 SD 鐨� App mkdir
+```
+
+鏉冨▉鐘舵�佷粛鍦� **褰撳墠 WS 杩炴帴鍐呭瓨 + JSON 鏂囦欢**锛屼笉鏄� TaskInstance 琛ㄣ��
+
+### 1.7 Client 鈫� Server 瀹為檯閫氫俊娴佺▼
+
+```text
+澶嶈
+ 鍙ｄ护 / MQTT START_APP_TASK
+ 鈫� 璁惧 play_audio loop=N锛圖15锛�
+ 鈫� [skill_event]audio.completed|english_learning|repeater|{ms}
+ 鈫� Server Session.events + 鍥� chat锛堝彧鎷涘懠涓�娆★級
+
+璺熻
+ 銆屽紑濮嬭窡璇汇��
+ 鈫� Server ensure dirs + list MP3
+ 鈫� MCP play 01.mp3
+ 鈫� completed + duration_ms 鈫� 鍚獥
+ 鈫� 鐢ㄦ埛璺熻 / 瓒呮椂 鈫� Server D16 涓嬩竴鍙�
+ 鈫� 鍙ュ瓙鑰楀敖 鈫� JSON flush 鈫� chat
+
+璁㈤槄鏂� App锛堜笉鐑у綍锛�
+ apps.json 澧炲姞 app_id
+ 鈫� 涓嬫 MCP 灏辩华鎴� start_app_task
+ 鈫� self.sd.ensure_app 鍒涘缓 /{app_id}/{task}/lessons
+```
+
+### 1.8 Task Runtime 褰撳墠瀹炵幇鐘舵��
+
+| 鑳藉姏 | 鐘舵�� |
+|------|------|
+| 缁熶竴鍏ュ彛 `start_app_task` | 鏈夛紙Phase 1锛� |
+| App / Task 娉ㄥ唽琛� | 鏈夛紙Server JSON锛� |
+| InteractionMode 鍒嗘祦 | 鏈� |
+| D1 `audio.completed/stopped` | **鏈�**锛坉etect 杩愯緭 + duration_ms锛� |
+| D6 璺熻涓嶈繘 `chat()` | **鏈�** |
+| D7 鍙ヨ竟鐣� | **鏈夛紙v1锛�**锛歋erver VAD + ASR 鍑哄彞锛涘彟鏈夋椂闀垮惉绐椾笂闄� |
+| D8 v1 褰掓。 | **閮ㄥ垎**锛氭湁鏁堣窡璇� `enqueue_asr_report`锛汼ession JSON 鏈夋枃鏈紱**鏈�**鎶� `audio_id` 鍐欏洖 turn |
+| D16 姝ラ绾� loop | **鏈�**锛氬彞瀛愬垪琛ㄥ彧鍦� Server |
+| D13 璧勬簮鐩綍 | **鎻愬墠钀藉湴**锛氭寜 App 鍞竴鍚嶅缓鏍戯紱浠嶄笉鏄� `resource_id` 鎶借薄 |
+| Step / TaskInstance | **鏃�** |
+| Assessment / Metric | **鏃�** |
+
+缁撹锛氳繖鏄� **鍙綊妗ｇ殑钖� Runtime锛堟枃浠� Session锛�**锛屼粛涓嶆槸 Master Plan / Server Plan V1.1 鍘熸枃閲岀殑銆孭hase 2 = Java 琛� + type=task + 璇勫垎銆嶃�傚悗鑰呭凡鐢� Spec 鎷嗗埌 Phase 3 / 4銆�
+
+### 1.9 娴嬭瘯鎯呭喌
+
+| 椤� | 鐜扮姸 |
+|----|------|
+| pytest `tests/test_app_runtime.py` | **鏈�**锛�13 passed锛氬彛浠ゃ�乪cho銆乻ession save/load銆乻kill_event銆乴esson 璺緞銆乴ayout 璺緞锛� |
+| 鍥轰欢鍗曞厓娴嬭瘯 | **鏃�** |
+| `phase-2-voice-test.md` | **鏈変功闈㈠彛浠よ〃**锛涗粨搴撳唴鏃犺窇娴嬭褰� |
+| Java / REST | 涓嶉�傜敤锛堟湰闃舵鏃犺〃锛� |
+
+**Phase 2 鍦ㄥ伐绋嬩笂琛ヤ笂浜� Phase 1 鐨� P0-1锛坧ytest锛夈��** 璇煶琛ㄤ粛闈犱汉宸ョ儳褰曞嬀閫夈��
+
+---
+
+## 2. Phase 2 Implementation Review锛堣鍒� vs 浠ｇ爜锛�
+
+### 2.1 瀵圭収 `phase-2-plan.md` 搂1.3 楠屾敹
+
+| # | 璁″垝楠屾敹 | 浠ｇ爜浜嬪疄 | 缁撹 |
+|---|---------|----------|------|
+| 1 | 澶嶈鑷劧缁撴潫 鈫� `audio.completed`锛屽洖 CHAT 鍙嫑鍛间竴娆� | `Board_OnSkillPlaybackFinished` 鍙彂 skill_event锛屽垹闄� `[app_task]chat`锛汼erver `handle_skill_event` 鎷涘懠 | **婊¤冻**锛堥渶鎸� `phase-2-voice-test.md` 搂5 浜哄伐纭鍗曟鎷涘懠锛� |
+| 2 | stop/鎶㈠崰 鈫� `audio.stopped`锛屼笉涓诲姩鎷涘懠 | stopped 璺緞 `speak_switch=False` | **婊¤冻** |
+| 3 | 绂荤嚎澶嶈 / 绂荤嚎鍒囨崲妯″紡涓� WakeWordInvoke | Phase 1 鐭矾鏈敼 | **婊¤冻** |
+| 4 | 璺熻鍏ㄧ▼涓� `chat()` | `handle_shadowing_turn` 涓嶈皟鐢� chat | **婊¤冻** |
+| 5 | JSON 鍚叏閮ㄦ湁鏁堣疆锛堝惈鏈�鍚庝竴杞級 | 鍏� `add_turn` 鍐嶅洖 chat | **婊¤冻** |
+| 6 | 鏈夋晥璺熻 report锛沞cho / PLAY_AUDIO 涓嶄笂鎶� | ASR 灞傛帹杩燂紱璺熻鏈夋晥杞墠 enqueue | **婊¤冻** |
+| 7 | 涓嶆柊澧� WS type / MQTT action / MCP 宸ュ叿 | **鏂板浜�** `self.sd.ensure_app` 涓� `ENSURE_APP_DIRS` | **绾緥鐮翠緥**锛岃 2.2 |
+| 8鈥�11 | 涓嶅閿欒繘鍥轰欢銆佷笉涓嬪彂 apps.json銆佷笉鏀圭儹璺緞銆佹棤 Java DDL | 閬靛畧 | **婊¤冻** |
+
+鍘熸枃璁″垝閲岃窡璇昏寖璇绘槸 **Server TTS**銆傚疄鏂芥敼涓� **涓�鍙ヤ竴涓� SD MP3**锛堣 `phase-2-voice-test.md`锛夈�傝繖姣斿師鏂囨洿鎺ヨ繎 Spec 鐨� `play_audio` + D16锛屼篃鎻愬墠鍔ㄤ簡 Phase 6 鐨勮祫婧愬舰鎬併��
+
+### 2.2 鍝簺璁捐琚慨鏀癸紙鐩稿 `phase-2-plan.md`锛�
+
+1. **璺熻濯掍綋锛歍TS 鈫� SD 鍙ユ枃浠躲��** 鍘熷洜锛氭暣娈靛甫闈欓粯鐨勯煶棰戝湪鎾斁鎬佷細涓㈡帀璺熻鎷鹃煶锛涗竴鍙ヤ竴 MP3 鎵嶈兘銆屾挱瀹� 鈫� 鍐嶅惉銆嶃�係pec D3 `speak` 浠嶅彲鐢ㄤ簬鏃犻煶棰戞椂鐨勬彁绀猴紝涓嶅啀褰撹寖璇讳富璺緞銆�
+2. **瀹屾垚浜嬩欢甯� `duration_ms`銆�** Server 鐢ㄥ畠鍋氬惉绐椾笂闄愶紙`1.2 脳`锛屾渶灏� 800ms锛夈�俈AD 璇村畬鍙彁鍓嶇粨鏉熴�傝繖鏄� D7 v1 鐨勫寮猴紝涓嶆槸璁惧 `wait` Skill锛圖11 浠嶆湭鍋氾級銆�
+3. **鏂板 MCP / MQTT 寤虹洰褰曘��** 浜у搧瑕佹眰锛氭瘡涓� App 鍞竴鍚嶏紱鐢ㄦ埛璁㈤槄鍚� **涓嶉噸鐑� bin**锛岀敱绋嬪簭鍒涘缓 `/{app}/{task}/lessons`銆傝繍杈撳彉浜嗭紝璇箟鏄凡鏈� `mkdir` + Spec 搂9 璧勬簮鏍戯紝涓嶆槸鏂颁紶鎰熷櫒銆�
+4. **`START_APP_TASK` 琛ヨ繘 MQTT ctl 鐧藉悕鍗曘��** 瀹炴柦涓彂鐜板畠鍘熷厛涓嶅湪 `BoardMqtt_ActionIsCtl`锛宑tl 涓婚浼氭嫆鏀躲�傚睘浜� Phase 1 婕忕綉锛屾湰闃舵淇ソ銆�
+5. **璺熻鎾畬涓嶅仠浠诲姟銆�** 鍙湁 skill 琛ㄥ惈 `self.mp3.play` 鐨� Task锛堝璇伙級鎵嶅湪鎾斁缁撴潫鏃舵妸 pending 鍐欏洖 chat銆傜鍚� D16锛氭楠ゅ惊鐜湪 Server銆�
+
+杩欎簺淇敼搴旇涓� **Phase 2 鐨勭湡瀹炴灦鏋勶紙Current Architecture V3锛�**銆傚悗缁笉寰楀亣瑁呫�岃窡璇讳粛鏄� TTS Demo銆嶆垨銆孲D 鏍圭洰褰曢殢渚垮爢鏂囦欢銆嶃��
+
+### 2.3 鍝簺璁捐娌℃湁瀹炵幇锛堝師鏂囨湁銆佷唬鐮佹棤锛�
+
+- 璺熻 TTS 鑼冭涓昏矾寰勶紙鏈夋剰鏇挎崲锛屼笉鏄紡鍋氾級
+- `wait` 涓�绛� Device Skill锛圖11锛夛細鐢� Server 鍚獥浠ｆ浛
+- 鎶� `audio_id` 鍐欒繘 Session turn
+- 璁惧 `user.speech.stopped` 浜嬩欢锛堢户缁敤 Server VAD锛�
+- Java 琛� / `type=task` / `mode.set`锛堟寜 Spec 鏈氨涓嶈鍦� Phase 2锛�
+- 鍊掕儗 / 鍙ｇ畻 / 鎺ラ緳
+
+### 2.4 瀹炴柦涓彂鐜扮殑闂
+
+1. 鏁磋 MP3 鍚彞闂撮潤榛樻椂锛屾挱鏀炬�佷細蹇界暐璺熻鎷鹃煶 鈫� 蹇呴』涓�鍙ヤ竴鏂囦欢銆�
+2. 瀹屾垚浜嬩欢鑻ヤ笉甯︽椂闀匡紝鍚獥鍙兘鎷嶈剳琚� 鈫� 琛� `duration_ms`銆�
+3. 鏂� App 鑻ユ妸鐩綍鍐欐鍦� C++ 鎴栬姹傞噸鐑э紝鍐荤粨灏辨槸绌鸿瘽 鈫� 鏆撮湶宸叉湁 mkdir銆�
+4. MQTT action 蹇呴』鍚屾椂杩� **Dispatch 涓� ctl/task 鐧藉悕鍗�**锛屽惁鍒� ack 涓婚涓嶅尮閰嶃��
+5. 鍦� `tools/list` 澶勭悊鍑芥暟閲屽悓姝� `call_mcp_tool` 浼氬拰 WS 鏀跺寘姝婚攣 鈫� 寤虹洰褰曞繀椤� `create_task` 鍚庡彴璺戙��
+6. 鏃� SD / mkdir 澶辫触涓嶅緱鏂紑 WS銆�
+
+### 2.5 涓� Spec v1.0 鐨勫鐓�
+
+| Spec ID | Phase 2 璁″垝閽夋锛� | 浠ｇ爜 | 瀹ｇО鍐荤粨锛� |
+|---------|-------------------|------|------------|
+| D1 瀹屾垚/鍋滄浜嬩欢 | 鏄� | 鏈夛紱鍙︽湁 duration_ms | **鍙绉拌涔夊凡钀藉湴**锛堣繍杈撲粛鏄� detect 鍓嶇紑锛� |
+| D2 pause/resume | 鍚︼紙鍙敤 stop锛� | 浠嶆槸 TTS-Pause-MP3 | 鏈喕缁撲负 Task 璋冪敤 |
+| D3 speak | 淇濇寔 type=tts | 鎻愮ず/鍥炶亰浠� TTS锛涜窡璇昏寖璇绘敼 play_audio | 璁惧浠嶄笉鍚堟垚 |
+| D4 barge_in 鎸� Mode | 淇濇寔 | 鏈敼鍧� | 淇濇寔 |
+| D6 listen 涓嶈繘 Chat | 鏄� | 鏈� | **鍙绉�** |
+| D7 涓�鍙ヨ瀹� | v1 = Server VAD+ASR | 鏈夛紱鍔犲惉绐椾笂闄� | **v1 鍙绉�**锛涜澶� USER_SPEECH_* 浠嶇己 |
+| D8 record_clip v1 | 鏈夋晥璺熻 report | 鏈夋枃鏈�+鍙�� WAV锛泃urn 鏃� audio_id | **閲囬泦鏈�**锛涘璞″叧鑱旂暀缁� Phase 3 |
+| D11 wait | 鏄庣‘涓嶅仛 | Server sleep 鍚獥 | 浠嶆槸缂哄彛锛屼笉鎸� Phase 3 |
+| D13 fetch / SD | 璁″垝澶� | **鎻愬墠**锛歟nsure dir + 璺緞绾﹀畾 | 甯冨眬鍙敤锛況esource_id / DOWNLOAD 鏉冨▉浠嶆槸 Phase 6 |
+| D14 PAUSE/RESUME Task | 涓嶅仛 | 鏃� | 鈥� |
+| D15 鎾斁绾� loop | 绂佹鏀� | 鏈敼 | **淇濇寔** |
+| D16 姝ラ绾� loop | Server 姝ヨ繘 | 鏈� | **鍙绉�** |
+| S4/S8 璇勪及/branch | 绂佹 | 鏃� | 姝ｇ‘ |
+
+Spec 搂9 鍘熸枃绀轰緥鏄� `/sdcard/resources/<app_id>/`銆備唬鐮侀噰鐢細
+
+```text
+/{app_id}/{task_id}/lessons/...
+渚嬶細/english_learning/shadowing/lessons/unit01/01.mp3
+```
+
+缁� `SDCard_Map_Business_Path` 鏄犲皠鍒� `MOUNT_POINT + business_path`銆傝繖搴斿啓杩� Spec / Architecture V3锛屼笉瑕佸湪 Phase 3 鍐嶆惉涓�娆＄洰褰曘��
+
+Spec 搂22 鍥涙潯绾緥锛氭柊 App 涓嶈姹傛柊浼犳劅鍣紱瀵归敊鍙湪 Server锛涚绾垮彧淇濊瘉鎾斁锛汼D 鏀捐祫婧愩��**浠嶆垚绔嬨��** 鏈樁娈垫柊澧炵殑 MCP/MQTT 鏄繍杈擄紝mkdir 涓嶆槸鏂拌兘鍔涚绫汇��
+
+### 2.6 涓庡師鏂� V1.1 Client/Server Plan 鐨勫樊寮�
+
+V1.1 鎶� Phase 2 鍐欐垚 `type=task` + Java 琛� + 璇勫垎銆係pec 宸叉妸瀹冩媶寮�銆傛湰闃舵 **娌℃湁** 鎸� V1.1 鍋氳〃鍜岃瘎鍒嗭紝杩欐槸姝ｇ‘鐨勩��
+V1.1 Client Phase 3 鎵嶅仛銆孉pp 璧勬簮鐩綍銆嶁�斺�斿疄闄呭湪 Phase 2 鍥犺闃呬笉鐑у綍鎻愬墠鍋氫簡甯冨眬銆侾hase 6 搴旂缉灏忎负銆屾竻鍗�/DOWNLOAD/resource_id銆嶏紝涓嶈閲嶅仛 mkdir銆�
+
+---
+
+## 3. 涓� Master Plan 鏍稿績鎶借薄瀵圭収锛堟洿鏂帮級
+
+| 鎶借薄 | 褰撳墠鏄惁瀛樺湪 | 瀹炵幇浣嶇疆 | 瀹屾垚绋嬪害 | Phase 3 鏄惁鍔� |
+|------|--------------|----------|----------|----------------|
+| **App** | 鏄� | `apps.json` + 鍞竴 `app_id` 鐩綍 | 杩炴帴绾у鍣� + SD 鏍戯紱鏃犺闃呰〃 | 涓嶅崌绾ф垚鍟嗗簵 |
+| **Task** | 鏄紙钖勶級 | `apps.json` tasks | 涓�涓� Task 鈮� 涓�绉嶅姩浣� | 浠嶆棤 JSON Step 鍥� |
+| **Step** | 浠呰窡璇昏疆娆� | `TaskSession.turns[]` | 鏂囦欢閲岀殑 index锛屼笉鏄紩鎿� | **鏄犲皠涓� `step_instance` 琛�** |
+| **Skill** | 鏄� | MCP + `k_task_skills[]` | 鍙岃〃浠嶅湪 | 涓嶅悎骞讹紱鏂� App 杩愯璧� Server MCP play |
+| **InteractionMode** | 鏄� | `conn.interaction_mode` | 浠ｇ爜鍚嶄粛 `SHADOWING` = Spec `FOLLOW_READ` | 涓嶅叏灞�閲嶅懡鍚� |
+| **TaskInstance** | 鍚� | JSON `task_session_id` | 鏈� uuid锛屾棤搴� | **鏈樁娈电洰鏍�** |
+| **Event** | 閮ㄥ垎 | `[skill_event]` + session.events | 鏃犵鍥涘鎬荤嚎 | 鍏ュ簱锛屼笉鍔犳柊 WS type |
+| **Audio** | 鑱婂ぉ闄勪欢绾� | report 鈫� LONGBLOB | 鏈寕 step | **灏介噺鎸� audio_id锛涘厑璁哥┖** |
+| **Assessment** | 鍚� | 鈥� | 0% | **绂佹** |
+| **Schedule** | 鍗婃埅 | RTC + pending | 鈥� | **绂佹**锛圥hase 5锛� |
+
+---
+
+## 4. Current Architecture V3锛堝熀浜庝唬鐮侊級
+
+### 4.1 Client V3
+
+```text
+ESP32-S3 (sening-xiaozhi-s3)
+鈹溾攢鈹� Application 鐑矾寰� 鏈敼璇箟
+鈹溾攢鈹� MCP
+鈹� self.mp3.* / self.power_off / self.sd.ensure_app
+鈹斺攢鈹� Task Gateway
+ Board_StartAppTask
+ Board_OnSkillPlaybackFinished 鈫� [skill_event] + duration_ms
+ Board_EnsureAppLayout 鈫� mkdir /{app}/.../lessons
+ k_task_skills[] 浠呯绾�/MQTT 鏌� Skill锛涙柊璁㈤槄 App 涓嶉潬姝よ〃杩愯
+```
+
+### 4.2 Server V3
+
+```text
+xiaozhi-server
+鈹溾攢鈹� App Runtime
+鈹� apps.json锛堟潈濞佸畾涔夛級
+鈹� start_app_task / filter_tools / ensure_app_sd_dirs
+鈹� SHADOWING锛歋D 鍙ユ枃浠� + 鍚獥 + D16
+鈹� TaskSession 鈫� data/task_sessions/*.json
+鈹溾攢鈹� detect锛歔skill_event] 鈫� [app_task] 鈫� [voice_entry]
+鈹斺攢鈹� report锛氱幇鏈� manage_api chat-history锛堜笌 Task 琛ㄤ粛鑴辫妭锛�
+
+manager-api 浠嶆湭鍙備笌 Task Runtime
+```
+
+### 4.3 SD 甯冨眬锛圴3 浜嬪疄锛�
+
+```text
+/{app_id}/{task_id}/lessons/{lesson_id}/file.mp3
+```
+
+- `app_id` / `task_id`锛歚[A-Za-z0-9_-]`锛岄暱搴� < 32
+- 璺熻锛氳 task 涓嬪瓧鍏稿簭绗竴涓� lesson 鏂囦欢澶瑰唴鍏ㄩ儴 MP3
+- 澶嶈绌� `name`锛氫紭鍏� `/{app}/{task}/lessons/`锛屽啀 NVS `last_rep`锛屽啀鍏ㄥ崱绗竴棣�
+- 璁㈤槄鍚� mkdir锛屽け璐ュ彧鎵撴棩蹇�
+
+### 4.4 閫氫俊 V3
+
+```text
+灏忕▼搴�/MQTT
+ START_APP_TASK | SET_ALARM | ENSURE_APP_DIRS
+ 鈫� snlp
+Device Task Gateway
+ 鈫� Xiaozhi WS listen detect [skill_event] / [app_task]
+Server App Runtime
+ 鈫� 鐜版湁 report HTTP
+manager-api chat_history + audio BLOB 鈫� 灏氭湭鎸� instance 鏌ヨ
+ 鈫� 鏈湴鏂囦欢
+data/task_sessions/{uuid}.json 鈫� Phase 2 鍞竴鍙煡璇㈠綊妗�
+```
+
+---
+
+## 5. 鎶�鏈�哄姟锛圥hase 2 涔嬪悗锛�
+
+### P0 鈥� 涓嶈В鍐冲氨鏃犳硶瀹夊叏鍋� Phase 3
+
+鏃犮�俻ytest 涓� Session schema 宸茶冻澶熷缓琛ㄣ��
+**杩涘叆 Phase 3 鐨勫墠鎻愭槸锛氫笉瑕佸湪寤鸿〃鍚屾椂鍋氳瘎鍒� / 鏂� Device 鑳藉姏銆�**
+
+### P1 鈥� 杩戞湡搴旇瑙ｅ喅锛圥hase 3 鑼冨洿鎴栫揣鎸ㄧ潃锛�
+
+| ID | 鍊哄姟 | 鏀惧摢 |
+|----|------|------|
+| P1-1 | Session 鍙湪璁惧鏃� JSON锛屾柇绾挎崲鏈�/鎹� Python 杩涚▼鏃犳硶鏌ヨ | **Phase 3 Instance 琛�** |
+| P1-2 | 璺熻 WAV 鍦� `chat_audio`锛宼urn 鏃� `audio_id` | Phase 3 灏介噺鍏宠仈锛涜瘎鍒嗕负 Phase 4 |
+| P1-3 | 鍙屾妧鑳借〃 `apps.json` vs `k_task_skills[]` | 淇濇寔锛涙柊 App 涓嶉潬鍥轰欢琛ㄨ繍琛岋紱绂佹鍐嶅姞涓氬姟鍒嗘敮 |
+| P1-4 | 瀵勭敓 `[skill_event]` 涓� ASR 鏂囨湰绌洪棿閲嶅彔 | 淇濇寔鍓嶇紑锛汸hase 3 **浠嶄笉涓�** `type=event` |
+| P1-5 | 寤虹洰褰� MCP/MQTT 涓� Spec 杩愯緭琛ㄥ皻鏈啓杩� Spec 姝ｆ枃 | 鏇存柊 Spec 搂9锛汸hase 3 涓嶆敼杩欎簺鎺ュ彛 |
+
+### P2 鈥� 浠ュ悗鍐嶈
+
+| ID | 鍊哄姟 |
+|----|------|
+| P2-1 | D11 `wait` 涓�绛� Skill |
+| P2-2 | D14 TASK_PAUSE/RESUME |
+| P2-3 | D8 v1.1 鏈湴 WAV 涓婁紶 |
+| P2-4 | 鍙� MQTT 鍚堝苟 |
+| P2-5 | `SHADOWING` 閲嶅懡鍚嶄负 `FOLLOW_READ` |
+| P2-6 | 闈� Sening 鏉� `play_audio` |
+| P2-7 | 瀵硅薄瀛樺偍銆佸闀� UI銆丄pp Store |
+| P2-8 | 鍥轰欢鏌ユ壘琛ㄤ粛鍚� `english_learning` 瀛楃涓� |
+
+---
+
+## 6. 缁撹锛歅hase 2 鏄惁 Done
+
+**鍔熻兘 Done锛堝甫鏉′欢锛夛細**
+
+- D1 瀹屾垚/鍋滄浜嬩欢宸蹭粠銆岀寽 `[app_task]chat`銆嶅彉鎴愰�昏緫浜嬩欢
+- 璺熻鏄彲褰掓。浼氳瘽锛圝SON turns锛屽惈鏈�鍚庝竴杞級锛屼笉杩� LLM
+- 姝ラ寰幆鍦� Server锛圖16锛�
+- 鏂� App 鐩綍鍙湪涓嶇儳褰曠殑鍓嶆彁涓嬪垱寤�
+- Chat / 缈昏瘧 / 绂荤嚎澶嶈绾緥鏈晠鎰忕牬鍧忥紱pytest 瑕嗙洊鏍稿績绾嚱鏁�
+
+**宸ョ▼鏈� Done锛�**
+
+- 璇煶鍥炲綊渚濊禆浜哄伐鍕鹃�� `phase-2-voice-test.md`
+- `audio_id` 鏈寕鍒� turn
+- 娌℃湁鍙法杩涚▼鏌ヨ鐨� Instance
+
+**鍥犳锛�** 鍙互瀹ｇО Spec 搂20 鍓嶄笁鏉★紙瀹屾垚浜嬩欢銆侀潪 Chat 鍙ヨ竟鐣屻�佽窡璇诲綊妗� v1锛�**鍦� Runtime 灞傚凡閽変綇**銆�
+**涓嶅彲浠�**瀹ｇО Device Capability 宸� 100% 瀹炵幇锛堢己 wait / pause Task / 缁熶竴 resource_id锛夈��
+**涓嶅彲浠�**寮�鍚� Assessment銆佸�掕儗/鍙ｇ畻/鎺ラ緳銆乣type=task`銆�
+
+涓嬩竴闃舵鍙仛涓�浠朵簨锛氭妸宸茬粡绋冲畾鐨� `TaskSession` **鎸佷箙鍖栨垚鍙煡璇㈢殑 TaskInstance**锛堣 `phase-3-plan.md`锛夈��
diff --git a/phase-2-voice-test.md b/phase-2-voice-test.md
new file mode 100644
index 0000000..cc910a2
--- /dev/null
+++ b/phase-2-voice-test.md
@@ -0,0 +1,175 @@
+# Phase 2 璇煶娴嬭瘯锛堢儳褰曞悗鎬庝箞璇达級
+
+鐑у綍鏂板浐浠跺苟杩炰笂 Wi鈥慒i / Server 鍚庯紝鎸変笅闈㈤『搴忓璁惧璇磋瘽銆�
+姣忎竴姝ュ厛鐪嬨�屼綘搴旇鍚埌銆嶏紝鍐嶆墦鍕俱�傛湰闃舵**涓嶈**娴嬫暟瀛楀�掕儗銆佸彛绠椼�佹垚璇帴榫欍��
+
+鍞ら啋璇嶇敤锛�**浣犲ソ灏忔櫤**锛堢瓑瀹冨紑濮嬪惉鍐嶈涓嬩竴鍙ワ級銆�
+
+---
+
+## 0. 鐑у綍鍚庡厛纭
+
+1. 璁惧寮�鏈哄悗杩涘叆**瀵硅瘽妯″紡**锛坈hat锛夛紝涓嶈鑷繁鎾煶棰戙��
+2. 璇淬�屼綘濂藉皬鏅恒�嶏紝鑳借繘瀵硅瘽銆�
+3. 璇淬�屼粖澶╁ぉ姘旀�庝箞鏍枫�嶁�斺�斾細褰撻棽鑱�/鏌ュぉ姘斿洖绛旓紙璇佹槑鐜板湪鏄� Chat锛屼笉鏄炕璇戞満锛夈��
+
+涓嶈繃杩欎竴鍏筹紝鍚庨潰鐨勬ā寮忓垏鎹㈤兘娌℃湁鎰忎箟銆�
+
+---
+
+## 1. 鑿滃崟鍒囨崲锛堟帹鑽愪富璺緞锛�
+
+瀵硅澶囪锛�
+
+| 姝� | 浣犺 | 浣犲簲璇ュ惉鍒� | 閫氳繃锛� |
+|----|------|------------|--------|
+| 1 | **浣犲ソ灏忔櫤** | 鍞ら啋锛岃繘鍏ヨ亞鍚� | |
+| 2 | **鍒囨崲妯″紡** | 銆屾兂鑱婂ぉ銆佸惉澶嶈鏈恒�佽窡璇汇�佺炕璇戯紝杩樻槸涓嫳瀵硅瘽锛熶篃鍙互璇存垜浠潵鑱婂ぉ鍚с�傘�� | |
+| 3 | **澶嶈鏈�** | 銆岀幇鍦ㄥ凡缁忓垏鎹㈣嚦澶嶈鏈烘ā寮忊�︺�嶇劧鍚� **SD 闊抽寮�濮嬪惊鐜挱鏀�**锛屼笉鍐嶈窡浣犻棽鑱� | |
+| 4 | 闊抽鎾斁涓殢渚胯涓�鍙ャ�岃涓瑧璇濄�� | **缁х画鎾煶棰戯紝涓嶈璁茬瑧璇�** | |
+| 5 | **浣犲ソ灏忔櫤**锛屽啀璇� **閫�鍑哄璇�** | 鍋滄挱锛屽苟鎷涘懠涓�娆″洖鍒板璇濓紙绫讳技銆屽凡缁忛��鍑哄璇绘満妯″紡锛岀幇鍦ㄦ槸瀵硅瘽妯″紡鈥︺�嶏級 | |
+| 6 | **璁蹭釜绗戣瘽** | 浼氳绗戣瘽锛堣瘉鏄庡凡鍥� Chat锛� | |
+
+鍚屼竴濂楄彍鍗曪紝鍐嶆祴鍙﹀涓変釜鍏ュ彛锛堟瘡娆″厛銆屼綘濂藉皬鏅恒�嶁啋銆屽垏鎹㈡ā寮忋�嶏級锛�
+
+| 浣犺 | 浣犲簲璇ュ惉鍒� | 閫�鍑烘椂璇� |
+|------|------------|----------|
+| **璺熻** 鎴� **寮�濮嬭窡璇�** | 銆岀幇鍦ㄥ凡缁忓垏鎹㈣嚦璺熻妯″紡銆傝璺熸垜璇伙細Hello, how are you today?銆� | **閫�鍑鸿窡璇�** |
+| **鎵撳紑缈昏瘧** | 銆岀幇鍦ㄥ凡缁忓垏鎹㈣嚦缈昏瘧妯″紡鈥︺�� | **閫�鍑虹炕璇�** |
+| **涓嫳瀵硅瘽** | 銆岀幇鍦ㄥ凡缁忓垏鎹㈣嚦涓嫳瀵硅瘽妯″紡鈥︺�� | **閫�鍑轰腑鑻卞璇�** |
+
+涔熷彲浠ヤ笉缁忚繃鑿滃崟锛屽敜閱掑悗鐩存帴璇村彛浠わ紙鏁堟灉鐩稿悓锛夛細
+
+- 澶嶈锛歚澶嶈鏈篳 / `寮�濮嬪璇籤
+- 璺熻锛歚寮�濮嬭窡璇籤 / `鎴戣璺熻`
+- 缈昏瘧锛歚鎵撳紑缈昏瘧` / `缈昏瘧妯″紡`
+- 涓嫳锛歚涓嫳瀵硅瘽`
+- 鍥炶亰锛歚鎴戜滑鏉ヨ亰澶╁惂` / `鍥炲埌鑱婂ぉ`
+
+---
+
+## 2. 璺熻锛堝繀椤绘暣娈佃蛋瀹岋級
+
+SD 鍗″厛鏀惧ソ闊抽锛�**涓�鍙ヤ竴涓� MP3**锛屼笉瑕佹暣璇惧甫闈欓粯锛�
+
+```text
+/english_learning/shadowing/lessons/unit01/01.mp3
+/english_learning/shadowing/lessons/unit01/02.mp3
+/english_learning/shadowing/lessons/unit01/03.mp3
+```
+
+澶嶈鏈洪煶棰戞斁鍦ㄥ彟涓�妫电洰褰曪紝涓嶈娣疯繘璺熻锛�
+
+```text
+/english_learning/repeater/lessons/unit01/full.mp3
+```
+
+浠庡璇濇ā寮忓紑濮嬨��
+
+| 姝� | 浣犺 / 浣犲仛 | 浣犲簲璇ュ惉鍒版垨鐪嬪埌 | 閫氳繃锛� |
+|----|-------------|------------------|--------|
+| 1 | **寮�濮嬭窡璇�** | 寮�濮嬫挱 `01.mp3`锛堜笉瑕佺敤闂茶亰 TTS 蹇佃嫳鏂囷級 | |
+| 2 | **绛夊畠鎾畬**锛屽啀鐢ㄨ嚜宸辩殑澹伴煶璺熻锛堜笉瑕佸拰鍠囧彮鍙犲湪涓�璧凤級 | 璺熻琚敹涓嬪悗锛屽啀鎾笅涓�鍙� MP3 | |
+| 3 | 姣忎竴鍙ラ兘銆屾挱瀹� 鈫� 浣犺窡璇汇�� | 涓嶄細璁茬瑧璇濄�佷笉浼氬綋鑱婂ぉ | |
+| 4 | 鏈�鍚庝竴鍙ヨ窡璇诲畬 | **鍥炲埌瀵硅瘽妯″紡骞舵嫑鍛间竴娆�** | |
+| 5 | **璁蹭釜绗戣瘽** | 浼氳绗戣瘽 | |
+
+鍚獥澶х害鏄鍙ユ椂闀跨殑 **1.2 鍊�**锛氳瀹屼細鎻愬墠杩涘叆涓嬩竴鍙ワ紱涓�鐩翠笉璇翠細瓒呮椂鍚庤繘涓嬩竴鍙ャ��
+
+鍙︽祴涓�娆°�屼腑閫旈��鍑恒�嶏細绗簩鍙ユ挱瀹岃窡璇绘椂璇� **閫�鍑鸿窡璇�** 鈫� 绔嬪埢鍥� Chat銆�
+
+璺熻鏃�**涓嶈**娴嬭繖浜涳細
+
+- 鍠囧彮杩樺湪鎾椂鎶㈢潃璇磋瘽 鈫� 搴旇蹇界暐
+- 璇淬�岀粰鎴戣涓瑧璇濄�嶁啋 **绂佹**寮�濮嬭绗戣瘽
+
+鏌ユ。妗堬細`data/task_sessions/` 鏈�鏂� JSON 閲� `turns` 鏉℃暟搴旂瓑浜庝綘瀹為檯璺熻鎴愬姛鐨勫彞鏁帮紙鍚渶鍚庝竴杞級銆�
+
+---
+
+## 3. 缈昏瘧
+
+| 姝� | 浣犺 | 浣犲簲璇ュ惉鍒� | 閫氳繃锛� |
+|----|------|------------|--------|
+| 1 | **鎵撳紑缈昏瘧** | 鍒囨崲鑷崇炕璇戞ā寮� | |
+| 2 | **浠婂ぉ澶╂皵鐪熷ソ** | 鍙粰鑻辨枃璇戞枃锛屼笉瑕侀棽鑱娿�佷笉瑕佽В閲� | |
+| 3 | **How are you?** | 鍙粰涓枃璇戞枃 | |
+| 4 | **閫�鍑虹炕璇�** | 鎷涘懠鍥炲埌瀵硅瘽 | |
+| 5 | **璁蹭釜绗戣瘽** | 浼氳绗戣瘽锛堣瘉鏄庣炕璇戞病杩涘浐浠躲�丆hat 宸叉仮澶嶏級 | |
+
+---
+
+## 4. 涓嫳瀵硅瘽锛堜笉鏄炕璇戞満锛�
+
+| 姝� | 浣犺 | 浣犲簲璇ュ惉鍒� | 閫氳繃锛� |
+|----|------|------------|--------|
+| 1 | **涓嫳瀵硅瘽** | 鍒囨崲璇存槑 | |
+| 2 | **鎴戝枩娆㈣涔�** | **鐢ㄨ嫳鏂囧洖绛斾綘**锛屼笉鏄妸鍘熷彞璇戝洖鍘讳氦宸� | |
+| 3 | **I like music** | **鐢ㄤ腑鏂囧洖绛斾綘** | |
+| 4 | **閫�鍑轰腑鑻卞璇�** | 鍥炲埌瀵硅瘽 | |
+
+---
+
+## 5. 澶嶈缁撴潫浜嬩欢锛圖1锛岀儳褰曞悗蹇呮祴锛�
+
+SD 鍗′笂瑕佹湁 MP3銆傞煶棰戝敖閲忕煭锛屾垨鎺ュ彈寰幆鍑犻亶銆�
+
+**A. 鑷劧鎾畬锛堝彧鎷涘懠涓�娆★級**
+
+1. 璇� **澶嶈鏈�**锛岃瀹冩挱銆�
+2. **涓嶈璇磋瘽**锛岀瓑闊抽鎸� `repeat` 閬嶆暟鍏ㄩ儴缁撴潫銆�
+3. 缁撴潫鏃跺簲 **鍙嫑鍛间竴娆�** 鍥炲埌瀵硅瘽锛堢被浼笺�屽凡缁忛��鍑哄璇绘満妯″紡鈥︺�嶏級銆�
+4. 鑻ヨ繛缁涓ら亶銆岀幇鍦ㄦ槸瀵硅瘽妯″紡銆嶏紝鍗冲け璐ワ紙鏃у浐浠朵細 `[app_task]chat` 鍐嶆嫑鍛间竴娆★級銆�
+
+**B. 涓�旈��鍑猴紙涓嶄富鍔ㄦ嫑鍛硷級**
+
+1. 鍐嶈 **澶嶈鏈�**锛屾挱璧锋潵銆�
+2. 璇� **閫�鍑哄璇�**銆�
+3. 搴旂珛鍒诲仠鎾苟鍥� Chat銆�
+ 鑻ヤ綘鏄敤銆岄��鍑哄璇汇�嶅彛浠ゅ仠鐨勶紝浼氭湁鍥炶亰鎷涘懠锛涜嫢鏄澶� stop 浜嬩欢鍗曠嫭涓婃潵銆佹病鏈夎繖鍙ュ彛浠わ紝鍒�**涓嶅簲鍐嶉澶栨嫑鍛间竴閬�**銆�
+
+**C. 鎾斁涓笉鑳藉綋闂茶亰**
+
+鎾斁鏃惰銆屼綘濂藉悧銆嶁啋 缁х画鎾紝涓嶅洖绛斻��
+
+鏌ユ。妗堬細瀵瑰簲 session 鐨� `events` 閲屽簲鏈� `audio.completed`锛堣嚜鐒剁粨鏉燂級鎴� `audio.stopped`锛堣鍋滐級銆�
+
+---
+
+## 6. 绂荤嚎锛堟嫈缃戞垨鍏宠矾鐢卞櫒鍚庡啀娴嬶級
+
+MultiNet 鍙杩欏嚑鍙ワ紝**涓嶈**鎸囨湜绂荤嚎璺熻/缈昏瘧銆�
+
+| 浣犺 | 鏈熸湜 |
+|------|------|
+| **澶嶈鏈�** | 鎾� NVS 閲屼笂娆￠煶棰戯紝鎴� SD 绗竴棣栵紱**涓嶈**鍘昏繛浜� |
+| **鍒囨崲妯″紡** | **涓嶈**杩炰簯銆佷笉瑕佹挱鑿滃崟锛涙棩蹇楅噷搴旇烦杩� voice_entry |
+| **閫�鍑哄璇�** / **鎴戜滑鏉ヨ亰澶╁惂** | 鍋滄挱锛屾湰鍦板洖鍒� chat |
+
+鎻掑洖缃戝悗锛屼粠绗� 1 鑺傚啀璧颁竴閬嶅湪绾胯彍鍗曘��
+
+---
+
+## 7. 澶辫触鏃跺厛鐪嬪摢
+
+| 鐜拌薄 | 澶氬崐鏄� |
+|------|--------|
+| 銆屽垏鎹㈡ā寮忋�嶆病鑿滃崟 | 娌＄綉锛屾垨杩樺湪鎾煶棰戜笖浼樺厛绾ф洿楂� |
+| 璺熻寮�濮嬭绗戣瘽 | 娌¤繘 SHADOWING锛屾垨鏈�鍚庝竴杞涪浜嗙洿鎺� chat |
+| 澶嶈缁撴潫鎷涘懠涓ゆ | 鏂板浐浠朵粛鍦ㄥ彂 `[app_task]chat`锛屾垨 Server 鏀朵簡涓ゆ鍒囨崲 |
+| 璺熻 JSON 灏戞渶鍚庝竴杞� | `handle_shadowing_turn` 鏈寜 Phase 2 椤哄簭璁� turn |
+| 绂荤嚎銆屽垏鎹㈡ā寮忋�嶄粛杩炰簯 | `SendListenDetect` 娌＄綉鐭矾琚牬鍧� |
+
+---
+
+## 鍙ｄ护閫熸煡
+
+| 鎰忓浘 | 璇村叾涓竴鍙ュ嵆鍙� |
+|------|----------------|
+| 鍞ら啋 | 浣犲ソ灏忔櫤 |
+| 鑿滃崟 | 鍒囨崲妯″紡 |
+| 澶嶈 | 澶嶈鏈恒�佸紑濮嬪璇� |
+| 璺熻 | 寮�濮嬭窡璇汇�佹垜瑕佽窡璇� |
+| 缈昏瘧 | 鎵撳紑缈昏瘧銆佺炕璇戞ā寮� |
+| 涓嫳 | 涓嫳瀵硅瘽 |
+| 鍥炶亰 | 鎴戜滑鏉ヨ亰澶╁惂銆佸洖鍒拌亰澶┿�侀��鍑哄璇汇�侀��鍑鸿窡璇汇�侀��鍑虹炕璇戙�侀��鍑轰腑鑻卞璇� |
diff --git a/phase-3-plan.md b/phase-3-plan.md
new file mode 100644
index 0000000..113d44e
--- /dev/null
+++ b/phase-3-plan.md
@@ -0,0 +1,380 @@
+# Phase 3 Coding Plan 鈥� TaskInstance 鎸佷箙鍖�
+
+> File: `phase-3-plan.md`
+> Date: 2026-08-31
+> **鐘舵�侊細宸插疄鏂姐��** 瀵圭収浜嬪疄瑙� `phase-3-review.md`銆備笅涓�闃舵 `phase-4-plan.md`銆�
+> 绾︽潫鏂囨。锛歚Xiaozhi Device Capability Specification v1.0.md`
+> 浠ｇ爜浜嬪疄锛堝紑宸ユ椂锛夛細`phase-2-review.md`锛圕urrent Architecture V3锛�
+> 璺嚎鍥撅細`roadmap.md`
+> 楠屾敹锛歚phase-3-test.md`
+> **涓庡師鏂� V1.1銆孭hase 2 = type=task + 琛� + 璇勫垎銆嶅啿绐佹椂锛屼互 Spec + 鏈枃涓哄噯锛氭湰闃舵鍙湁琛ㄥ拰鏌ヨ锛屾病鏈夋柊 Device 鑳藉姏锛屾病鏈夎瘎鍒嗐��**
+
+鏈樁娈� **涓嶆槸** 鍐嶅仛涓�涓� App锛屼篃 **涓嶆槸** 鍋氬�掕儗/鍙ｇ畻/鎺ラ緳锛屼篃 **涓嶆槸** 鏀瑰浐浠躲��
+鏈樁娈垫妸 Phase 2 宸茬粡鍐欒繘 `data/task_sessions/*.json` 鐨勫瓧娈碉紝钀藉埌 manager-api锛岃涓�娆¤窡璇�/澶嶈鍙互鎸� id 鏌ュ埌銆�
+
+缁� Coding Agent锛氬厛璇� Spec 搂0銆伮�4鈥�5銆伮�17銆伮�20锛涘啀璇� `phase-2-review.md` 搂4锛涘啀璇� `session.py`銆乣runtime.py`銆乣manage_api_client.py`銆乣AgentChatHistoryController.java`銆乣ShiroConfig.java`銆佺幇鏈� Liquibase 椋庢牸銆�
+
+---
+
+## 0. 涓� Spec 鐨勫鐓э紙鏈樁娈靛彧閽夎繖浜涳級
+
+| Spec ID | 鑳藉姏 | Phase 3 鍋氫粈涔� | 涓嶅仛浠�涔� |
+|---------|------|----------------|----------|
+| 鈥� | Task Runtime 涓眰 | Session 鈫� `task_instance` / `task_step` / `task_event` | 涓嶆妸 Instance 鏉冨▉鏀惧埌璁惧 |
+| D1 / D6 / D16 | 宸茶惤鍦� | **鍙叆搴�** events / turns | 涓嶆敼瀹屾垚浜嬩欢杩愯緭銆佷笉鏀硅窡璇绘杩� |
+| D8 v1 | 闊抽褰掓。 | 灏介噺鎶婄幇鏈� report 鐨� `audio_id` 鍐欏埌 step | 涓嶅仛 v1.1 鏈湴鏂囦欢涓婁紶锛涗笉鏀� Opus |
+| D13 | SD 甯冨眬 | **娌跨敤** Phase 2 `/{app}/{task}/lessons` | 涓嶆敼 mkdir MCP/MQTT锛涗笉鍋� resource_id |
+| D11 / D14 | wait / TASK_PAUSE | **涓嶅仛** | 鈥� |
+| S4 / S8 | 璇勪及 / branch | **绂佹** | 琛ㄤ笂鍙互鐣欑┖ `score` 鍒楋紝浣嗘湰闃舵涓嶅啓鍊笺�佷笉璺戞瘮瀵� |
+| 搂22 | 鏂� App 闆舵柊浼犳劅鍣� | 閬靛畧 | 涓嶆柊澧� WS type銆佷笉涓� Instance 鍔� Device 鍛戒护 |
+
+杩愯緭鍙彉銆佽涔変笉鍙橈細Instance 璧� **HTTP锛堝凡鏈� manage_api 閫氶亾锛�**銆傜姝负浜� `instance_id` 鏂板 `type=task`銆傝澶� **涓嶅繀** 缂撳瓨 `instance_id`銆�
+
+---
+
+## 1. 鐩爣涓庨潪鐩爣
+
+### 1.1 Objective
+
+1. 涓�娆¤窡璇绘垨澶嶈缁撴潫鍚庯紝鐢� HTTP GET 鑳芥煡鍒颁笌 JSON 鏂囦欢涓�鑷寸殑锛歛pp/task銆乻tatus銆乼urns锛坧rompt + asr锛夈�乪vents锛堝惈 `audio.completed` / `listen_timeout`锛夈��
+2. 涓婚敭 = 鐜版湁 `task_session_id`锛坲uid锛夛紝涓嶈鍐嶇敤 WS `session_id` 褰撻暱鏈熶富閿��
+3. Agent 鈮� App锛歍ask 琛� **涓嶅緱** 濉炶繘 `ai_agent_chat_history` 鍐掑厖姝ｅ紡妯″瀷锛沜hat report 绠￠亾淇濈暀銆�
+4. manager-api 鎸備簡鎴栦笂鎶ュけ璐ワ細**涓嶅緱**褰卞搷璇煶锛堜笌鐜版湁 chat report 鐩稿悓锛氬彧鎵撴棩蹇楋級銆�
+
+### 1.2 Why Now
+
+Phase 2 瀛楁宸茬ǔ瀹氥�傛病鏈� Instance锛孭hase 4 鍒嗘暟鏃犲鎸傦紝Phase 5 璋冨害鏃犳硶瀵硅处銆傜幇鍦ㄥ缓琛ㄤ笉浼氱┖銆�
+
+### 1.3 Acceptance Criteria锛堝繀椤诲叏閮ㄦ弧瓒筹級
+
+**鑳藉姏锛�**
+
+1. 瀹屾暣璺熻涓�娆� 鈫� `GET /task/instance/{id}` 鐨� `steps` 鏉℃暟 = 鏈夋晥璺熻鍙ユ暟锛堝惈鏈�鍚庝竴杞級锛宍prompt_text` / `asr_text` 涓� `data/task_sessions/{id}.json` 涓�鑷淬��
+2. 澶嶈鑷劧缁撴潫 鈫� instance `end_reason=play_completed`锛宍events` 鍚� `audio.completed`銆�
+3. 閫�鍑鸿窡璇� / 鏂� WS 鈫� `cancelled` + 瀵瑰簲 `end_reason`锛坄user_exit` / `disconnected`锛夛紝宸叉湁 turns 涓嶄涪銆�
+4. `GET` 鎸� `macAddress` 鍙垪鍑烘渶杩� instance锛堟渶灏� 20 鏉★級銆�
+5. 鏃ц矾寰勪粛鍦細JSON 鏂囦欢缁х画鍐欙紙鍙屽啓锛夛紱chat 澶╂皵/绗戣瘽浠嶈蛋 `ai_agent_chat_history`銆�
+6. 鍏抽棴鎴栭厤閿� manage_api锛�**璺熻/澶嶈/Chat 浠嶈兘璺戝畬**銆�
+
+**鍐荤粨绾緥锛�**
+
+7. **鍥轰欢闆舵彁浜�**锛堝惈 `app_task.cc` / MCP / MQTT锛夈�侾hase 2 宸茬儳褰曠殑 bin 蹇呴』鑳藉 Phase 3 Server 宸ヤ綔銆�
+8. 涓嶆柊澧� Xiaozhi WS `type`銆佷笉鏂板 MQTT action銆佷笉鏂板 MCP 宸ュ叿銆�
+9. 涓嶈瘎鍒嗐�佷笉鍑洪銆佷笉鏀� `repeat`銆佷笉鏀� SD 璺緞绾﹀畾銆佷笉鎶� `apps.json` 涓嬪彂鍒� SD銆�
+10. 涓嶆敼 `hello/listen/tts/stt` 涓� Opus銆�
+
+### 1.4 Out of Scope
+
+- Phase 4锛氭枃鏈瘮瀵广�乻core銆乺etry JSON
+- Phase 4b锛氬�掕儗 / 鍙ｇ畻 / 鎺ラ緳
+- Phase 5锛歋erver 鏃ュ巻
+- Phase 6锛欴OWNLOAD / resource_id / 鎼 SD
+- manager-web / 灏忕▼搴� UI锛堟湰闃舵 curl / 鏅烘帶鍙� Swagger 鍗冲彲锛�
+- 瀵硅薄瀛樺偍銆丷edis Session銆佸井鏈嶅姟鎷嗗垎
+- `type=task` / `mode.set` / 璁惧缂撳瓨 instance_id
+- `wait` Skill銆乀ASK_PAUSE銆乮ndicate銆侀暱鎸夐��鍑�
+
+---
+
+## 2. 鏋舵瀯锛堟湰闃舵锛�
+
+```text
+Device锛圥hase 2 bin锛屼笉鍔級
+ [skill_event] / Opus listen / MCP play
+ 鈫�
+xiaozhi-server App Runtime
+ TaskSession.save() 鈫� data/task_sessions/{id}.json 锛堜繚鐣欙級
+ 鈫� HTTP upsert锛坰erver secret锛�
+ manager-api
+ ai_task_instance
+ ai_task_step
+ ai_task_event
+ 锛堝彲閫夛級step.audio_id 鈫� ai_agent_chat_audio.id
+
+鏌ヨ锛歰auth2 GET锛堝闀�/鎺у埗鍙颁互鍚庣敤锛涙湰闃舵鐢� token 鎴栧厛 server 鍐呯綉娴嬶級
+```
+
+`task_session_id` 鍦� `begin_session` 宸茬粡鐢熸垚銆侾hase 3 **涓嶈**鍐嶅彂涓�涓� id 缁欒澶囥��
+
+璺熻涓嬩竴鍙ヤ粛鐒舵槸 Spec **D16 Server loop**銆傝〃鍙槸鎶曞奖锛屼笉鏄柊鐨� Step 寮曟搸銆�
+
+---
+
+## 3. 琛ㄧ粨鏋勶紙鏈�灏忥級
+
+Liquibase锛歚manager-api/src/main/resources/db/changelog/202608311800.sql`
+蹇呴』鐧昏 `db.changelog-master.yaml`銆�**绂佹鏀瑰凡缁忔墽琛岃繃鐨� changeset**锛堝惈 `202504151206.sql` 鐨� URL 鍗犱綅锛夈��
+
+鍛藉悕涓庣幇缃戜竴鑷达細`utf8mb4`銆乣DATETIME(3)`銆両nnoDB銆�
+**涓嶈** `DROP TABLE IF EXISTS` 涓氬姟琛ㄣ��
+
+### 3.1 `ai_task_instance`
+
+| 鍒� | 绫诲瀷 | 璇存槑 |
+|----|------|------|
+| `id` | VARCHAR(32) PK | = `task_session_id` |
+| `mac_address` | VARCHAR(50) | 璁惧 |
+| `agent_id` | VARCHAR(32) NULL | 鑻ヨ繛鎺ヨ兘鍙栧埌鍒欏～锛涙病鏈夊氨绌� |
+| `ws_session_id` | VARCHAR(64) | Phase 2 `ws_session_id` |
+| `app_id` | VARCHAR(32) | |
+| `task_id` | VARCHAR(32) | |
+| `source` | VARCHAR(24) | voice / mqtt_immediate / schedule / 鈥� |
+| `interaction_mode` | VARCHAR(32) | 瀛樹唬鐮佸悕 `SHADOWING` 鍗冲彲 |
+| `status` | VARCHAR(16) | `running` / `completed` / `cancelled` |
+| `end_reason` | VARCHAR(32) | 涓� JSON 鐩稿悓 |
+| `started_at` | BIGINT | epoch ms |
+| `ended_at` | BIGINT NULL | |
+| `created_at` / `updated_at` | DATETIME(3) | |
+
+绱㈠紩锛歚(mac_address, started_at)`銆乣(app_id, task_id, started_at)`銆乣(status)`銆�
+
+### 3.2 `ai_task_step`
+
+瀵瑰簲 `turns[]`銆備竴鏉℃湁鏁堣窡璇讳竴杞竴琛屻��
+
+| 鍒� | 绫诲瀷 | 璇存槑 |
+|----|------|------|
+| `id` | BIGINT PK AI | |
+| `instance_id` | VARCHAR(32) | FK 閫昏緫鍏宠仈鍗冲彲锛圛nnoDB FK 鍙�夛紱寤鸿鏈夛級 |
+| `step_index` | INT | `turns[].index` |
+| `prompt_text` | TEXT | 璺熻鍙兘鏄� SD 璺緞 |
+| `asr_text` | TEXT | |
+| `echo_rejected` | TINYINT(1) | 榛樿 0 |
+| `audio_id` | VARCHAR(32) NULL | 鎸囧悜 `ai_agent_chat_audio.id` |
+| `recorded_at` | BIGINT | |
+
+鍞竴锛歚(instance_id, step_index)`銆�
+
+澶嶈娌℃湁 turns 鏄悎娉曠殑锛堝彧鏈� events锛夈��
+
+### 3.3 `ai_task_event`
+
+瀵瑰簲 `events[]`銆�
+
+| 鍒� | 绫诲瀷 | 璇存槑 |
+|----|------|------|
+| `id` | BIGINT PK AI | |
+| `instance_id` | VARCHAR(32) | |
+| `name` | VARCHAR(64) | `audio.completed` / `audio.stopped` / `listen_timeout` / `echo_rejected` |
+| `app_id` / `task_id` | VARCHAR(32) | 鍙┖鍒欑户鎵� instance |
+| `at_ms` | BIGINT | JSON `at` |
+
+绂佹鎶� score銆乸arser銆乨ifficulty 鍋氭垚 NOT NULL銆傝嫢棰勭暀 `score` 鍒楋紝蹇呴』鍙┖涓旀湰闃舵鎭掍负 NULL銆�
+
+---
+
+## 4. HTTP API
+
+閴存潈瀵归綈鐜扮綉锛�
+
+| 璺緞 | 杩囨护鍣� | 璋冪敤鏂� |
+|------|--------|--------|
+| `PUT /task/instance/{id}` | `server`锛坄ServerSecretFilter`锛屼笌 chat report 鐩稿悓锛� | xiaozhi-server |
+| `GET /task/instance/{id}` | `oauth2`锛堟帶鍒跺彴鐢ㄦ埛锛� | 娴嬭瘯 / 浠ュ悗瀹堕暱 |
+| `GET /task/instance` | `oauth2` | 鍒楄〃锛泀uery: `macAddress` 蹇呭～锛屽彲閫� `appId`銆乣limit` |
+
+**涓嶈**鎶� PUT 璁炬垚 `anon`銆�
+**涓嶈**鏂板缓绗簩濂楀瘑閽ャ��
+
+### 4.1 Upsert 杞借嵎
+
+涓� `TaskSession.to_dict()` 瀵归綈锛宑amelCase 鎴� snake 浜岄�変竴锛�**Python / Java 蹇呴』鍚屼竴濂�**銆傛帹鑽� **snake_case 涓� JSON 鏂囦欢涓�鑷�**锛屽噺灏戞槧灏� bug锛�
+
+```json
+{
+ "task_session_id": "uuid",
+ "ws_session_id": "",
+ "device_id": "mac",
+ "agent_id": "",
+ "app_id": "english_learning",
+ "task_id": "shadowing",
+ "source": "voice",
+ "interaction_mode": "SHADOWING",
+ "started_at": 0,
+ "ended_at": 0,
+ "status": "running",
+ "end_reason": "",
+ "turns": [
+ {
+ "index": 0,
+ "prompt_text": "/english_learning/shadowing/lessons/unit01/01.mp3",
+ "asr_text": "...",
+ "echo_rejected": false,
+ "recorded_at": 0,
+ "audio_id": null
+ }
+ ],
+ "events": [
+ { "name": "audio.completed", "app_id": "", "task_id": "", "at": 0 }
+ ]
+}
+```
+
+璇箟锛氭寜 `id` **鏁村崟瑕嗙洊** steps/events锛堝厛鍒犲悗鎻掓垨鎸� unique key upsert锛夈�傝皟鐢ㄦ柟姣忔 `session.save()` 閮藉彲 PUT 鍏ㄦ枃銆備笉瑕佸仛澶嶆潅 patch 鍗忚銆�
+
+骞傜瓑锛氬悓涓� id 閲嶅 PUT 鎴愬姛锛屾渶缁堜笌鏈�鍚庝竴娆� body 涓�鑷淬��
+
+### 4.2 GET 鍝嶅簲
+
+杩斿洖涓婅〃鍚屼竴 JSON锛屽鍔� `mac_address` 鑻ヤ笌 `device_id` 闇�瑕佸苟瀛樺垯鍙繚鐣欎竴涓瓧娈靛苟鍦ㄦ枃妗ｅ啓姝汇��
+
+鍒楄〃锛歚{ "list": [ { id, app_id, task_id, status, end_reason, started_at } ], "total": n }`
+涓嶈鍦ㄥ垪琛ㄩ噷杩斿洖鍏ㄩ儴 turns銆�
+
+### 4.3 `audio_id`锛堝敖鍔涜�屼负锛�
+
+鐜版湁 `POST /agent/chat-history/report` 杩斿洖 `Result<Boolean>`锛孭ython 鎷夸笉鍒� id銆�
+
+鏈樁娈靛厑璁镐袱鏉¤矾寰勶紝**閫夊伐浣滈噺灏忕殑涓�鏉★紝涓嶈涓ゆ潯閮藉仛**锛�
+
+**A锛堟帹鑽愶級銆�** report DTO 澧炲姞鍙�� `taskInstanceId` + `stepIndex`锛汮ava 鍐欏叆 chat_audio 鍚庢洿鏂板搴� `ai_task_step.audio_id`銆傝窡璇� `enqueue_asr_report` 甯︿笂杩欎袱涓彲閫夊瓧娈点��
+
+**B銆�** 鏈樁娈� step.audio_id 鎭掔┖锛汫ET 浠嶈兘鏌ユ枃鏈�傚湪 PR 璇存槑鐣欑粰 Phase 4銆傞獙鏀� #1 涓嶅己鍒堕煶棰戙��
+
+绂佹涓轰簡鎸傞煶棰戝啀涓婁紶涓�浠� WAV銆�
+
+---
+
+## 5. Server 鏀瑰姩娓呭崟
+
+### 5.1 manager-api锛圝ava锛�
+
+| ID | 浠诲姟 | 璇存槑 |
+|----|------|------|
+| J1 | Liquibase SQL + master yaml | 鍙鏂版枃浠� |
+| J2 | Entity / Dao / Service | 妯″潡寤鸿 `xiaozhi.modules.task`锛屼笉瑕佸杩� agent chat 鍖呭綋銆屼竴绉嶈亰澶┿�� |
+| J3 | `TaskInstanceController` | PUT upsert銆丟ET one銆丟ET list |
+| J4 | `ShiroConfig` | `PUT /task/instance/**` 鈫� `server`锛汫ET 璧伴粯璁� oauth2 |
+| J5 | 鏍￠獙 | `app_id`/`task_id` 闀垮害涓庡瓧绗︿笌鍥轰欢绾﹀畾涓�鑷达紱闈炴硶 400锛涚己 id 400 |
+| J6 | 鍙�� A | report DTO 鎵╁睍 + 鍥炲啓 audio_id |
+
+manager-web锛�**闆舵彁浜�**锛堥櫎闈� Swagger 鑷姩鐢熸垚锛屼笉瑕佹墜鏀� Vue锛夈��
+
+### 5.2 xiaozhi-server锛圥ython锛�
+
+| ID | 浠诲姟 | 鏂囦欢 | 璇存槑 |
+|----|------|------|------|
+| S1 | HTTP 瀹㈡埛绔� | `config/manage_api_client.py` | `upsert_task_instance(payload)`锛涙棤 client 鎴栧紓甯� 鈫� log锛宺eturn None |
+| S2 | 鎶曞奖 | `core/app_runtime/session.py` | `TaskSession.save()` 鎴愬姛鍐欑洏鍚� **寮傛/鍚屾鍧囧彲** 璋冪敤 upsert锛涘け璐ヤ笉褰卞搷 save |
+| S3 | 杩炴帴瀛楁 | 鑻� `conn.agent_id` 宸叉湁鍒欏啓鍏� payload | 娌℃湁灏辩┖ |
+| S4 | 鍙�� A | `enqueue_asr_report` / `report()` | 闄勫甫 instance/step |
+| S5 | pytest | `tests/test_app_runtime.py` | `to_dict` 瀛楁锛泆psert 琚皟鐢紱upsert 鎶涢敊鏃� save 浠嶆垚鍔燂紙mock锛� |
+
+**涓嶈**鍦� `runtime.py` 鍐嶅鍒朵竴濂� schema銆倁psert 鍙粠 `TaskSession.to_dict()` 鍑哄彂銆�
+
+璺熻/澶嶈/skill_event 鐨� begin/end/add_turn **涓嶈鏀硅涓�**锛屽彧鍊熺幇鏈� `save()` 鎸傞挬銆�
+
+---
+
+## 6. Client 鏀瑰姩娓呭崟
+
+**绌恒��** 绂佹鏀� `xiaozhi-esp32-2.2.6`銆�
+
+鏃у浐浠舵棤 `duration_ms`銆佹棤 `ensure_app` 鏃讹細Server 浠嶆寜 Phase 2 鍏煎锛堟椂闀跨己鐪併�佸缓鐩綍澶辫触蹇界暐锛夈�侾hase 3 涓嶅緱渚濊禆鏂板浐浠跺瓧娈点��
+
+---
+
+## 7. 娴嬭瘯
+
+璇﹁ `phase-3-test.md`銆傛湰 PR 鏈�浣庨檺搴︼細
+
+1. pytest锛歴chema 鎶曞奖 + upsert 澶辫触闅旂銆�
+2. 鎵嬪伐锛氳窡璇绘暣娈� 鈫� GET instance 瀵瑰緱涓� JSON锛汣hat 绗戣瘽浠嶅湪 chat_history銆�
+3. 鍥炲綊锛歚phase-2-voice-test.md` 绗� 0銆�2銆�5 鑺傦紙璺熻 + 澶嶈缁撴潫浜嬩欢锛変笉寰楀洖閫�銆�
+
+涓嶈鏈樁娈甸獙鏀跺�掕儗/鍙ｇ畻/鎺ラ緳銆�
+
+---
+
+## 8. 瀹炴柦椤哄簭
+
+```text
+P3-A Liquibase + Entity + PUT upsert + GET
+ 鐢� curl + server secret 鍐欏叆涓�鏉℃墜宸� JSON锛岃兘璇诲洖
+
+P3-B Python manage_api_client + TaskSession.save() 鎸傞挬
+ 璺熻涓�娆★紝搴撻噷鍑虹幇琛岋紱鍏虫帀 manager-api 鍐嶈窡璇伙紝璁惧涓嶅仠
+
+P3-C 锛堝彲閫夛級report 鍥炲啓 audio_id
+ 鍋氫笉瀹屽氨鐣欑┖鍒楋紝涓嶈鍗′綇 A/B
+
+P3-D pytest + phase-3-test.md 鍕鹃��
+```
+
+鏈畬鎴� A 涓嶈鏀� Python 鐑矾寰勩�傛湭瀹屾垚 B 涓嶅緱瀹ｇО Phase 3 Done銆傛湭瀹屾垚 D 涓嶅緱寮� Phase 4 璇勫垎銆�
+
+---
+
+## 9. 缂栫爜绾︽潫锛圫pec 搂16鈥�17 钀藉埌鏈� PR锛�
+
+1. 鍥轰欢闆� diff銆�
+2. 涓嶅緱鎶� ASR/LLM/璇勫垎/鍙ｇ畻/璇嶅簱鍐欏叆 Client 鎴� Java銆岃澶囬�昏緫銆嶃��
+3. 涓嶅緱鏂板 WS type / MQTT / MCP銆�
+4. 涓嶅緱鎶� Task 琛屽啓鍏� `ai_agent_chat_history` 褰撴潈濞併��
+5. 涓嶅緱淇敼宸叉墽琛� Liquibase 鐨� checksum 鍐呭銆�
+6. Chat 鐑矾寰勪笌 Mode 鍒嗘祦閫昏緫鍘熷垯涓婁笉鏀癸紱鍙湪 `save()` / report 鏃佽矾鍔犱笂鎶ャ��
+7. 姣忎釜 PR 鍏� Chat 鍥炲綊锛屽啀鏌� GET銆�
+
+---
+
+## 10. 椋庨櫓涓庡洖褰�
+
+| 椋庨櫓 | 缂撹В |
+|------|------|
+| 姣忔 save 鍏ㄩ噺鍒犳彃 steps 骞跺彂涓㈣ | 鍗曡繛鎺ュ崟 session锛涗笉瑕佸苟琛� save 鍚屼竴 id |
+| manage_api 瓒呮椂鎷栨璇煶 | upsert 蹇呴』 catch锛涘繀瑕佹椂 `create_task` 鍚庡彴鍙戯紝浣嗚淇濊瘉 end 鏃舵渶缁堜竴鑷达紙鍙湪 `end_session` 鍐� PUT 涓�娆★級 |
+| 涓� chat_history 鍙屽啓璁╀骇鍝佷互涓恒�岃亰澶╄褰曞氨鏄窡璇汇�� | API 璺緞鍒嗗紑锛涙枃妗ｅ啓娓� Agent 鈮� App |
+| 鎶� Phase 3 鍋氭垚璇勫垎 | 楠屾敹琛ㄤ笉鍚� score锛涘垪鍙┖ |
+| 涓� instance_id 鏀� detect 鏂囨湰 | 绂佹 |
+| 鏂� App 鍙堝幓鏀� C++ 寤虹洰褰� | 宸叉湁 `ensure_app`锛涘彧鏀� `apps.json` |
+
+---
+
+## 11. Data Flow
+
+```text
+寮�濮嬭窡璇�
+ begin_session 鈫� 鍐� JSON + PUT running锛坱urns=[]锛�
+
+姣忓彞鏈夋晥 ASR
+ add_turn 鈫� save 鈫� PUT 鍏ㄦ枃
+ enqueue_asr_report锛堝彲閫夊甫 instance/step锛�
+
+audio.completed锛堣窡璇伙級
+ add_event 鈫� save 鈫� PUT锛涘紑鍚獥锛堥�昏緫涓嶅彉锛�
+
+缁撴潫
+ finish(reason) 鈫� save 鈫� PUT completed/cancelled
+ GET /task/instance/{id} 鍙煡
+```
+
+澶嶈锛氬彲鑳介浂 turn銆佽嫢骞� `audio.*` event銆傚悓鏍� upsert銆�
+
+---
+
+## 12. State Machine
+
+```text
+DeviceState / InteractionMode锛氫笌 Phase 2 鐩稿悓锛岀姝㈠姞杈广��
+
+Instance锛�
+ 锛堟棤锛夆攢鈹�begin鈹�鈹�鈻� running 鈹�鈹�finish(finished|play_completed)鈹�鈹�鈻� completed
+ 鈹斺攢鈹�finish(user_exit|disconnected|play_stopped|preempted)鈹�鈹�鈻� cancelled
+```
+
+璁惧鎵ц鍣ㄧ姸鎬佷笉鏄湰闃舵宸ヤ綔銆�
+
+---
+
+## 13. 瀹屾垚鍚庯紙浜虹被娴佺▼锛�
+
+```text
+Coding
+ 鈫� 鍕鹃�� phase-3-test.md
+ 鈫� Completion Review锛堝鐓ф湰鏂� 搂1.3锛� 鉁� phase-3-review.md
+ 鈫� Architecture V4锛堣〃 + HTTP锛�
+ 鈫� Roadmap Review
+ 鈫� phase-4-plan.md锛堟椂閽熸杩� + 鍚庤瘎浼帮紱鏃� JUDGE銆佹棤褰撳満鍐嶆挱锛�
+```
+
+绂佹鏈樁娈点�岄『渚裤�嶈瘎鍒嗐�佸�掕儗銆丼cheduler銆丼D 鎼銆乣type=task`銆乵anager-web 璺熻椤点��
diff --git a/phase-3-review.md b/phase-3-review.md
new file mode 100644
index 0000000..9fa7213
--- /dev/null
+++ b/phase-3-review.md
@@ -0,0 +1,345 @@
+# Phase 3 Completion Review
+
+> Date: 2026-08-31
+> Basis: **current code**锛堜笉鏄� `phase-3-plan.md` 鐨勫師鏂囨効鏈涳級
+> Compared against: `phase-3-plan.md` 路 `phase-3-test.md` 路 `phase-2-review.md` 路 `roadmap.md` 路 `Xiaozhi Device Capability Specification v1.0.md`
+> Repositories: `xiaozhi-esp32-2.2.6`锛圕lient锛屾湰闃舵闆� diff锛壜� `xiaozhi-esp32-server`锛圥ython + manager-api锛�
+
+鏈枃浠跺彧璁板綍浜嬪疄銆佸樊寮傘�佸�哄姟涓庡 Spec / Roadmap 鐨勪慨姝ｃ�備笅涓�闃舵鏂藉伐瑙� `phase-4-plan.md`锛岄獙鏀惰 `phase-4-test.md`銆�
+
+鐩稿叧鏂囨。绱㈠紩锛�
+
+| 鏂囨。 | 瑙掕壊 |
+|------|------|
+| `Xiaozhi Device Capability Specification v1.0.md` | Device 璇箟鍐荤粨 |
+| `roadmap.md` | 闃舵鍒囧垎 |
+| `phase-2-review.md` | Current Architecture V3锛堟枃浠� Session锛� |
+| `phase-3-plan.md` | 鏈樁娈靛師鏂囨柦宸ュ崟锛圛nstance 钀藉簱锛� |
+| `phase-3-test.md` | GET 瀵硅处 + 璇煶鍥炲綊 + 澶辫触闅旂 |
+| `phase-2-voice-test.md` | 鍙ｄ护搴曡〃锛堟湰闃舵涓嶉噸鐑э級 |
+
+---
+
+## 1. 褰撳墠浠ｇ爜鐩樼偣锛堜互浠撳簱涓哄噯锛�
+
+### 1.1 Phase 3 瀹為檯鏀瑰姩鐨勬枃浠�
+
+#### Client锛坄xiaozhi-esp32-2.2.6`锛�
+
+**鏃犱笟鍔℃彁浜ゃ��** 璺熻/澶嶈浠嶇敤 Phase 2 bin锛歚[skill_event]`銆佷竴鍙ヤ竴 MP3銆乣ensure_app` mkdir銆�
+
+#### Server Python锛坄xiaozhi-server`锛�
+
+| 鏂囦欢 | 瑙掕壊 |
+|------|------|
+| `core/app_runtime/session.py` | `agent_id`锛沗save()` 鍐欑洏鍚� `_schedule_upsert`锛坄create_task`锛屾棤 running loop 鍒欒烦杩囷級 |
+| `config/manage_api_client.py` | `upsert_task_instance`锛氬崟娆� `_async_request` PUT锛沗report()` 鍙�� `taskInstanceId`/`stepIndex` |
+| `core/handle/reportHandle.py` | `enqueue_asr_report` 鍏厓缁勯�忎紶 instance/step |
+| `core/connection.py` | `_process_report` 榛樿鍙傛暟锛岃浆鍙戝埌 manage_api |
+| `core/app_runtime/runtime.py` | 璺熻鏈夋晥杞� `enqueue_asr_report(..., task_session_id, step_index)` |
+| `tests/test_app_runtime.py` | `to_dict` 鍚� Phase 3 瀛楁锛泆psert 璋冨害鎶涢敊鏃� `save` 浠嶆垚鍔� |
+
+璺熻姝ヨ繘銆佸惉绐椼�乣begin_session`/`add_turn` 璇箟 **鏈敼**銆倁psert 鍙寕鍦ㄥ凡鏈� `save()` 涓娿��
+
+#### manager-api锛圝ava锛�
+
+| 鏂囦欢 | 瑙掕壊 |
+|------|------|
+| `db/changelog/202608311800.sql` | 鏂拌〃锛涘凡鐧昏 `db.changelog-master.yaml`銆�**鏈敼**宸叉墽琛� changeset |
+| `modules/task/**` | Entity / Dao / DTO / `TaskInstanceService` / `TaskInstanceController` |
+| `security/config/ShiroConfig.java` | `/task/instance` 涓� `/task/instance/**` 鈫� `server` |
+| `agent/dto/AgentChatHistoryReportDTO.java` | 鍙�� `taskInstanceId` / `stepIndex` |
+| `agent/.../AgentChatHistoryBizServiceImpl.java` | report 鎴愬姛鍚� `attachTaskStepAudio` |
+
+#### manager-web / manager-mobile
+
+**闆舵彁浜ゃ��** 鏌ヨ闈� curl / Swagger / server secret銆�
+
+### 1.2 瀹為檯澧炲姞鐨勬ā鍧�
+
+| 妯″潡 | 浣嶇疆 | 鑱岃矗 |
+|------|------|------|
+| Task Instance 鎶曞奖 | manager-api `xiaozhi.modules.task` | Session JSON 鈫� 涓夎〃锛涙潈濞佷粛鏄� Server Runtime 鍐呭瓨 + 鏂囦欢锛屽簱鏄煡璇㈠壇鏈� |
+| HTTP upsert | `PUT /xiaozhi/task/instance/{id}` | 鍏ㄦ枃瑕嗙洊 turns/events |
+| HTTP GET | `GET .../{id}`銆乣GET ...?macAddress=` | 璇︽儏鍚� turns锛涘垪琛ㄤ笉鍚鏂� |
+| audio_id 鎸傞挬 | 璁″垝 搂4.3 **璺緞 A** | chat report 鍥炲啓 `ai_task_step.audio_id` |
+
+娌℃湁锛欰ssessment銆乣type=task`銆佽澶囩紦瀛� `instance_id`銆佹柊 WS/MQTT/MCP銆佺鎴峰垪銆乵anager-web 璺熻椤点��
+
+### 1.3 瀹為檯澧炲姞鐨� API / 鍗忚
+
+| 绫诲瀷 | 鍚嶇О | 璇存槑 |
+|------|------|------|
+| REST PUT | `/task/instance/{id}` | server secret锛泂nake_case 涓� JSON 鏂囦欢瀵归綈 |
+| REST GET | `/task/instance/{id}` | **璁″垝鍐� oauth2锛屽疄鏂戒负 server**锛圫hiro 鎸夎矾寰勪笉鑳芥寜鏂规硶鎷嗭級 |
+| REST GET | `/task/instance?macAddress=&appId=&limit=` | 鍚屼笂 |
+| REST | 鐜版湁 `POST /agent/chat-history/report` | 澧炲彲閫夊瓧娈碉紝鏃犳柊璺緞 |
+| WS / MQTT / MCP | **鏃�** | 璁惧浠嶄笉鐭� instance_id |
+
+鍝嶅簲鍖呭湪鐜扮綉 `Result<T>` 閲岋紙`code` + `data`锛夛紝涓嶆槸瑁� DTO銆俢ontext-path锛歚/xiaozhi`銆�
+
+### 1.4 鏁版嵁搴�
+
+| 琛� | 涓婚敭 | 鍐呭 |
+|----|------|------|
+| `ai_task_instance` | `id` = `task_session_id`锛�32 hex锛� | mac / agent_id / app_id / task_id / status / end_reason / 鏃堕棿鎴� |
+| `ai_task_step` | BIGINT + UNIQUE `(instance_id, step_index)` | prompt / asr / echo / audio_id / recorded_at |
+| `ai_task_event` | BIGINT | name / app / task / at_ms锛汧K CASCADE |
+
+**娌℃湁** `score` / `expected_text` / `tenant_id` 鍒楋紙璁″垝鍏佽棰勭暀 score 涓旀亽 NULL锛涘疄鏂介�夋嫨 **涓嶅缓璇ュ垪**锛岀暀缁� Phase 4 鏂� changeset锛夈��
+
+瑕嗙洊 steps 鏃讹細鏂� `audio_id` 涓虹┖鍒欎繚鐣欐棫琛岋紙閬垮厤涓� report 绔炴�侊級銆�
+
+### 1.5 Client 鏋舵瀯鍙樺寲
+
+鐩稿 V3锛�**鏃犮��** 璁惧涓嶆寔鏈� instance_id锛屼笉璇诲簱銆�
+
+### 1.6 Server 鏋舵瀯鍙樺寲
+
+```text
+TaskSession.save()
+ 鈹溾攢鈹� data/task_sessions/{id}.json 锛堜繚鐣欙紝澶辫触鍙� log锛�
+ 鈹斺攢鈹� create_task 鈫� PUT /task/instance/{id}
+ 澶辫触 print/log锛屼笉鎶涘洖鐑矾寰�
+ 鏃� ManageApiClient / 鏃� running loop 鈫� 璺宠繃
+
+璺熻鏈夋晥 ASR
+ add_turn 鈫� save 鈫� PUT
+ enqueue_asr_report(instance, step) 鈫� 鐜版湁 WAV BLOB 鈫� Java attach audio_id
+```
+
+鏉冨▉鐘舵�侊細**褰撳墠杩炴帴鍐呭瓨 + JSON 鏂囦欢**锛沵anager-api 鏄渶缁堜竴鑷存姇褰便�俶anage_api 鎸備簡锛岃窡璇讳粛鎸� Phase 2 璺戙��
+
+### 1.7 Client 鈫� Server 瀹為檯閫氫俊娴佺▼
+
+涓� Phase 2 鐩稿悓銆傛柊澧炵殑鍙槸 Server鈫抦anager-api 鐨� HTTP锛岃澶囩湅涓嶈銆�
+
+```text
+寮�濮嬭窡璇�
+ begin_session 鈫� JSON + PUT running锛坱urns=[]锛�
+
+姣忓彞鏈夋晥 ASR
+ add_turn 鈫� save 鈫� PUT 鍏ㄦ枃
+ report锛堝彲閫� audio_id 鍥炲啓锛�
+
+audio.completed
+ add_event 鈫� save 鈫� PUT锛涘紑鍚獥锛堥�昏緫涓嶅彉锛�
+
+缁撴潫
+ finish 鈫� save 鈫� PUT completed/cancelled
+ GET /xiaozhi/task/instance/{id} 鍙煡锛堥渶 server secret锛�
+```
+
+澶嶈锛氶浂 turn銆佽嫢骞� `audio.*` event锛屽悓鏍� upsert銆侰hat **涓�** `begin_session`銆�
+
+### 1.8 Task Runtime 褰撳墠瀹炵幇鐘舵��
+
+| 鑳藉姏 | 鐘舵�� |
+|------|------|
+| TaskSession JSON | **鏈�**锛圥hase 2锛� |
+| TaskInstance 鍙煡璇� | **鏈�**锛圚TTP GET锛涘弻鍐欙級 |
+| Step 琛� = 璺熻涓�杞� | **鏈�**锛堟姇褰憋紝涓嶆槸 Step 寮曟搸锛� |
+| Event 鍏ュ簱 | **鏈�** |
+| D8 audio 鎸� step | **灏藉姏**锛氳矾寰� A 宸叉帴锛涘彇鍐充簬 report 閰嶇疆涓庣珵鎬� |
+| Assessment / score | **鏃�** |
+| expected_text锛堝彲璇勫垎鍘熸枃锛� | **鏃�**锛沗prompt_text` 璺熻鏃舵槸 **SD 璺緞** |
+| Scheduler / 璧勬簮鍟嗗簵 / 绉熸埛 | **鏃�** |
+
+缁撹锛氳繖鏄� **鍙煡璇㈢殑钖� Runtime锛圝SON + 涓夎〃锛�**锛屼粛涓嶆槸 V1.1 鐨勩�孭hase 2 = 璇勫垎銆嶃�傝瘎鍒嗗凡姝ｇ‘鐣欏埌 Phase 4銆�
+
+### 1.9 娴嬭瘯鎯呭喌
+
+| 椤� | 鐜扮姸 |
+|----|------|
+| pytest `tests/test_app_runtime.py` | **鏈�**锛�15 鏉￠噺绾э細鍙ｄ护銆乻ession銆乻kill_event銆丼D 璺緞銆丳hase 3 schema銆乽psert 澶辫触闅旂锛� |
+| Java 鍗曟祴 / Maven 鍦ㄦ湰浠撳簱瀵硅瘽涓殑缂栬瘧璁板綍 | **鏃�** |
+| `phase-3-test.md` 璇煶 + GET 瀵硅处 | **鏈変功闈㈡竻鍗�**锛涗粨搴撳唴鏃犺窇娴嬪嬀閫夎褰� |
+| 鍥轰欢娴嬭瘯 | 涓嶉�傜敤锛堥浂 diff锛� |
+
+---
+
+## 2. Phase 3 Implementation Review锛堣鍒� vs 浠ｇ爜锛�
+
+### 2.1 瀵圭収 `phase-3-plan.md` 搂1.3 楠屾敹
+
+| # | 璁″垝楠屾敹 | 浠ｇ爜浜嬪疄 | 缁撹 |
+|---|---------|----------|------|
+| 1 | 璺熻 GET steps = 鏈夋晥鍙ユ暟锛宲rompt/asr 涓� JSON 涓�鑷� | upsert 鍏ㄦ枃瑕嗙洊锛沺rompt 涓� SD 璺緞 | **浠ｇ爜婊¤冻**锛堥渶鎸� test.md 浜哄伐 GET锛� |
+| 2 | 澶嶈 `end_reason=play_completed` + `audio.completed` | 鏈敼缁撴潫璺緞锛屽彧澶� PUT | **浠ｇ爜婊¤冻** |
+| 3 | 閫�鍑�/鏂嚎 cancelled + 宸叉湁 turns 涓嶄涪 | `finish` + save锛涙柇绾� `end_session(disconnected)` | **浠ｇ爜婊¤冻** |
+| 4 | GET 鎸� mac 鍒楀嚭鏈�杩� 鈮�20 | `limit` 榛樿 20銆佷笂闄� 100 | **婊¤冻** |
+| 5 | JSON 鍙屽啓锛汣hat 浠嶈蛋 chat_history | chat 涓� begin_session | **婊¤冻** |
+| 6 | manage_api 鎸備簡璇煶浠嶈窇 | upsert catch + 鍚庡彴 task | **浠ｇ爜婊¤冻** |
+| 7 | 鍥轰欢闆舵彁浜� | 鏃� Client 涓氬姟 diff | **婊¤冻** |
+| 8 | 鏃犳柊 WS/MQTT/MCP | 鏃� | **婊¤冻** |
+| 9 | 涓嶈瘎鍒嗐�佷笉鏀� repeat/SD銆佷笉涓嬪彂 apps.json | 鏃� score 鍒椼�佹棤璇勫垎璋冪敤 | **婊¤冻** |
+| 10 | 涓嶆敼 hello/listen/tts/stt 涓� Opus | 鏈敼 | **婊¤冻** |
+
+璁″垝 搂4 GET 璧� oauth2锛�**鏈寜鍘熸枃鍋�**銆傝 2.2銆�
+
+璁″垝 搂4.3 閫� A 鎴� B锛氬疄鏂戒簡 **A**锛坮eport 鍥炲啓 audio_id锛夈��
+
+### 2.2 鍝簺璁捐琚慨鏀癸紙鐩稿 `phase-3-plan.md`锛�
+
+1. **GET 閴存潈 = server锛屼笉鏄� oauth2銆�** Shiro 鎸夎矾寰勬寕杩囨护鍣紝涓嶈兘鎸� HTTP 鏂规硶鎷嗐�備笌 chat report 鍚屼竴瀵嗛挜鍗冲彲娴嬮�氥�傚闀�/鎺у埗鍙� oauth2 鐣欑粰 Phase 7锛屼笉瑕佷负鏈樁娈靛啀鍋氱浜屽杩囨护鍣ㄣ��
+2. **鏈鐣� `score` 鍒椼��** 璁″垝鍏佽鍙┖涓旀亽 NULL銆傚疄鏂介�夋嫨涓嶅缓锛岄伩鍏嶇┖鍒楄褰撴垚銆屽凡缁忔湁璇勪及銆嶃�侾hase 4 鐢� **鏂� changeset** 鍔犲垪銆�
+3. **PUT 鍗曟璇锋眰锛屼笉鍋� 6脳10s 閲嶈瘯銆�** 閬垮厤鍒风垎 manage_api锛涘け璐ュ彧 log銆�
+4. **瑕嗙洊 steps 鍚堝苟鏃� `audio_id`銆�** 璁″垝鍐欏厛鍒犲悗鎻掞紱瀹炴柦鍒犳彃浣嗙┖ audio 淇濈暀鏃у�硷紝鍑忓皯涓� report 绔炴�併��
+
+杩欎簺搴旇涓� **Phase 3 鐨勭湡瀹炴灦鏋勶紙Current Architecture V4锛�**銆�
+
+### 2.3 鍝簺璁捐娌℃湁瀹炵幇锛堝師鏂囨湁銆佷唬鐮佹棤鎴栨湭楠岃瘉锛�
+
+- GET oauth2锛堟湁鎰忔敼涓� server锛�
+- 琛ㄤ笂鐨勭┖ `score` 鍒楋紙鏈夋剰鎺ㄨ繜锛�
+- 浠撳簱鍐� Java 缂栬瘧 / 鐪熷疄 HTTP 鑱旇皟璁板綍
+- `phase-3-test.md` 浜哄伐鍕鹃�夛紙璇煶 + GET锛�
+- manager-web 椤甸潰锛堣鍒掓湰灏� Out of Scope锛�
+
+### 2.4 瀹炴柦涓彂鐜扮殑闂锛堝 Phase 4 鏈夌害鏉燂級
+
+1. **`prompt_text` 鍦� SD 璺熻涓嬫槸鏂囦欢璺緞**锛堝 `/english_learning/shadowing/lessons/unit01/01.mp3`锛夛紝涓嶆槸鑻辫鍘熸枃銆侾hase 4 **绂佹**鎷胯矾寰勫仛鏂囨湰姣斿銆傚繀椤诲彟鏈� `expected_text`銆�
+2. **鍥炲０妫�娴嬪璺緞鏄叧鎺夌殑**锛坄_is_clip_path`锛夈�備笉瑕佹妸銆岃瘎鍒嗙敤 expected_text銆嶅拰銆�0.82 鍥炲０鎷掔粷銆嶇粦鎴愬悓涓�鏉★細鍚獥鍐呯敤鎴疯窡瀵逛簡涔熶細鍍忚寖璇汇�傝瘎鍒嗛槇鍊煎繀椤荤嫭绔嬶紝涓斿彧鍦ㄥ凡缁忚繃鍚獥闂ㄩ棭鐨勬湁鏁堣疆涓婄畻銆�
+3. **`uk_task_step_instance_index` 鍞竴銆�** 鍚屼竴鍚獥澶氭 ASR 蹇呴』 **鎸� index 鏇存柊鍚屼竴 turn**锛屼笉鑳借拷鍔犮��
+4. 鏃� running asyncio loop 鏃� upsert 闈欓粯璺宠繃锛堜緥濡傛煇浜涘悓姝ユ祴璇曡矾寰勶級锛涚儹璺緞鏈� loop锛屾甯搞��
+5. 鍏ㄩ噺鍒犳彃鍚屼竴 id 涓嶈骞惰 `save`锛堝崟杩炴帴鍗� session锛岀幇缃戝姝わ級銆�
+
+### 2.5 涓� Spec v1.0 鐨勫鐓�
+
+| Spec ID | Phase 3 璁″垝 | 浠ｇ爜 | 瀹ｇО |
+|---------|--------------|------|------|
+| 涓眰 Task Runtime | Session 鈫� 琛� | 鏈� | **鍙煡璇㈠壇鏈凡钀藉湴**锛涙潈濞佷粛鍦� Server 杩涚▼ |
+| D1 / D6 / D16 | 鍙叆搴擄紝涓嶆敼杩愯緭/姝ヨ繘 | 閬靛畧 | 淇濇寔 |
+| D8 v1 | 灏介噺鎸� audio_id | 璺緞 A | **鍏宠仈绠￠亾鏈�**锛涢潪姣忔潯蹇呮湁 id |
+| D13 | 娌跨敤 `/{app}/{task}/lessons` | 鏈惉鐩綍 | 淇濇寔 |
+| D11 / D14 | 涓嶅仛 | 鏃� | 鈥� |
+| S4 / S8 | 绂佹 | 鏃犺瘎鍒嗐�佹棤 JUDGE | **姝ｇ‘** |
+| 搂22 | 鏃犳柊浼犳劅鍣� | 鍥轰欢闆� diff | **婊¤冻** |
+
+### 2.6 涓庡師鏂� V1.1 / 绉熸埛璁ㄨ鐨勫樊寮�
+
+V1.1 鎶婅瘎鍒嗗拰琛ㄧ粦鍦ㄥ悓涓� Phase銆係pec 宸叉媶寮�銆傛湰闃舵 **娌℃湁** 璇勫垎锛屾纭��
+
+鍦烘櫙鍒囨崲璇嶃�佺鎴凤細**鏈樁娈垫湭鍋氥�佷篃涓嶈鍋氥��** 璇嶈〃浠嶅湪 `runtime.py`锛沗tenant_id` 鏈繘琛ㄣ�佹湭杩涘浐浠躲�傝 搂5 绾緥銆�
+
+---
+
+## 3. 涓� Master Plan 鏍稿績鎶借薄瀵圭収锛堟洿鏂帮級
+
+| 鎶借薄 | 褰撳墠鏄惁瀛樺湪 | 瀹炵幇浣嶇疆 | 瀹屾垚绋嬪害 | Phase 4 鏄惁鍔� |
+|------|--------------|----------|----------|----------------|
+| **App** | 鏄� | `apps.json` | 杩炴帴绾э紱鏃犺闃呰〃 | 鍙姞 `assessment` 鍧楋紱涓嶅崌绾у晢搴� |
+| **Task** | 鏄紙钖勶級 | `apps.json` tasks | 涓�绉嶅姩浣� | 璺熻鍔犺瘎浼扮瓥鐣� JSON |
+| **Step** | 鏄紙琛岋級 | `ai_task_step` / `turns[]` | 鎶曞奖 | **鍐欏叆 score / expected_text** |
+| **Skill** | 鏄� | MCP + 鍥轰欢鏌ユ壘琛� | 鍙岃〃浠嶅湪 | 涓嶅悎骞讹紱涓嶆柊鍛戒护 |
+| **InteractionMode** | 鏄� | `SHADOWING` | = Spec FOLLOW_READ | 涓嶉噸鍛藉悕 |
+| **TaskInstance** | **鏄�** | 涓夎〃 + JSON | 鍙� GET | 涓嶆敼涓婚敭锛涗笉鎶婂垎鏁版敼鎸� chat_history |
+| **Event** | 鏄� | 琛� + JSON | 鏃犵鍥涘鎬荤嚎 | 鍙姞 `assessment.*` 浜嬩欢鍚嶏紙浠嶈蛋鐜版湁 events[]锛� |
+| **Audio** | 鑱婂ぉ闄勪欢 + 鍙�夋寕 step | report | 灏藉姏 | 涓嶆柊涓婁紶閫氶亾 |
+| **Assessment** | 鍚� | 鈥� | 0% | **鍚庤瘎浼板畾閲� metrics锛涗笉鎸囨尌姝ヨ繘** |
+| **Schedule / Tenant** | 鍚� | 鈥� | 鈥� | **绂佹** |
+
+---
+
+## 4. Current Architecture V4锛堝熀浜庝唬鐮侊級
+
+### 4.1 Client V4
+
+涓� V3 鐩稿悓銆傚浐浠跺喕缁撻潰涓嶅洜钀藉簱鑰岀Щ鍔ㄣ��
+
+### 4.2 Server V4
+
+```text
+xiaozhi-server
+鈹溾攢鈹� App Runtime
+鈹� apps.json
+鈹� SHADOWING锛歋D 鍙ユ枃浠� + 鍚獥 + D16锛堟棤瀵归敊锛�
+鈹� TaskSession 鈫� JSON + 寮傛 PUT
+鈹溾攢鈹� detect锛歔skill_event] 鈫� [app_task] 鈫� [voice_entry]
+鈹斺攢鈹� report锛歝hat-history锛涘彲閫夊洖鍐� step.audio_id
+
+manager-api
+鈹溾攢鈹� PUT/GET /task/instance 锛坰erver secret锛�
+鈹溾攢鈹� ai_task_instance / step / event
+鈹斺攢鈹� 涓嶈绠楀垎鏁般�佷笉鎵ц Skill
+```
+
+### 4.3 閫氫俊 V4
+
+```text
+Device 鈹�鈹�WS鈹�鈹�鈻� xiaozhi-server Runtime
+ 鈹溾攢鈹� JSON 鏂囦欢
+ 鈹斺攢鈹� HTTP PUT 鈹�鈹�鈻� manager-api Instance
+ 鈹斺攢鈹� HTTP report 鈹�鈹�鈻� chat_audio 鈹�鈹�鈻� step.audio_id
+
+鏌ヨ锛歴erver secret GET /xiaozhi/task/instance/{id}
+璁惧浠庝笉璋冪敤涓婅堪 HTTP銆�
+```
+
+### 4.4 SD 甯冨眬
+
+浠嶄负 V3锛�
+
+```text
+/{app_id}/{task_id}/lessons/{lesson_id}/file.mp3
+```
+
+---
+
+## 5. 鎶�鏈�哄姟涓庢灦鏋勭邯寰嬶紙Phase 3 涔嬪悗锛�
+
+### P0 鈥� 涓嶈В鍐冲氨鏃犳硶瀹夊叏鍋� Phase 4
+
+| ID | 鍊哄姟 | 鏀惧摢 |
+|----|------|------|
+| P0-1 | 璺熻 `prompt_text` = SD 璺緞锛屼笉鑳藉綋璇勫垎鍘熸枃 | **Phase 4 鐢ㄥ悓 stem 鐨� `01.txt` / transcripts 浣� `expected_text`锛涜瘎浼颁笉鎸℃挱鏀�** |
+
+### P1 鈥� 杩戞湡
+
+| ID | 鍊哄姟 | 鏀惧摢 |
+|----|------|------|
+| P1-1 | GET 浠� server secret | Phase 7 瀹堕暱 UI 鍐嶆媶 oauth2 |
+| P1-2 | 浜哄伐鏈嬀閫� `phase-3-test.md` | 寮� Phase 4 缂栫爜鍓嶅缓璁厛瀵逛竴娆� GET锛涗笉鎸″啓璁″垝 |
+| P1-3 | 鏃� Java 缂栬瘧璁板綍 | 鍔犲垪 changeset 鏃剁敤鐜扮綉椋庢牸锛岀姝㈡敼 `202608311800.sql` |
+| P1-4 | 璇嶈〃纭紪鐮佸湪 `runtime.py` | 淇濇寔鍒� Phase 6 catalog锛�**绂佹**鎶娿�屽紑濮嬭窡璇汇�嶅啓杩涘浐浠� |
+| P1-5 | 鍙屾妧鑳借〃 | 淇濇寔 |
+
+### P2 鈥� 浠ュ悗鍐嶈
+
+| ID | 鍊哄姟 |
+|----|------|
+| P2-1 | D11 `wait` / D14 PAUSE |
+| P2-2 | 绉熸埛 / 鏈烘瀯鏁欐潗闅旂锛堟渶鏃╃紳鍦� Phase 6锛屼骇鍝佸寲 Phase 7锛� |
+| P2-3 | 闊崇礌/鍙戦煶/鎯呯华鍒嗘瀽 |
+| P2-4 | 鍊掕儗 / 鍙ｇ畻 / 鎺ラ緳锛圥hase 4b锛� |
+| P2-5 | `type=task`銆佸璞″瓨鍌ㄣ�乵anager-web 璺熻椤� |
+
+**鐜板湪灏遍伒瀹堛�佷笉蹇呯紪鐮侊細**
+
+- 鍥轰欢涓嶅啀澧炲姞涓氬姟 App 鍙ｄ护锛涚郴缁熺骇浠� Chat + 绂荤嚎澶嶈鏈� + 銆屽垏鎹㈡ā寮忋�嶆�诲叆鍙�
+- 璁惧姘镐笉鎸佷箙鍖� `tenant_id`
+- 鎵撳崱/鍒嗘瀽鍙秷璐� Instance + audio锛屼笉鍔犳柊 Device 鑳藉姏
+
+---
+
+## 6. 缁撹锛歅hase 3 鏄惁 Done
+
+**鍔熻兘 Done锛堝甫鏉′欢锛夛細**
+
+- `task_session_id` 鍙湪 manager-api 鎸� id / mac 鏌ヨ
+- JSON 鍙屽啓淇濈暀锛汸UT 澶辫触涓嶆嫋姝昏闊筹紙pytest 瑕嗙洊璋冨害澶辫触锛�
+- Agent 鈮� App锛欳hat 涓嶈繘 `ai_task_instance`
+- 鍥轰欢闆� diff锛涙棤鏂拌繍杈撶绫伙紱鏃犺瘎鍒�
+- audio_id 鎸傞挬锛堣矾寰� A锛夊凡鎺�
+
+**宸ョ▼鏈� Done锛�**
+
+- `phase-3-test.md` 渚濊禆浜哄伐 GET / 璇煶鍕鹃��
+- Java 渚ф棤浠撳簱鍐呯紪璇戣瘉鏄�
+- 璺熻鍘熸枃涓庤矾寰勬湭鍒嗙锛圥0-1锛屼氦缁� Phase 4锛�
+
+**鍥犳锛�** 鍙互瀹ｇО Task Runtime **鍙綊妗ｄ笖鍙煡璇�**銆�
+**涓嶅彲浠�**瀹ｇО宸茶瘎浼般�佸凡鎵撳崱浜у搧鍖栥�佸凡澶氱鎴枫��
+**涓嶅彲浠�**涓哄垎鏁版柊澧� `JUDGE` 鍛戒护鎴栨敼 C++銆�
+
+涓嬩竴闃舵鍙仛涓�浠朵簨锛氭妸璺熻鐑矾寰勬敹鎴� **MP3 搴忓垪 + 瀹氭椂褰曢煶**锛屽垎鏋愬叏閮ㄥ悗缃负瀹氶噺 metrics銆傝 `phase-4-plan.md`銆�
diff --git a/phase-3-test.md b/phase-3-test.md
new file mode 100644
index 0000000..e36eb02
--- /dev/null
+++ b/phase-3-test.md
@@ -0,0 +1,216 @@
+# Phase 3 娴嬭瘯
+
+> 瀵瑰簲鏂藉伐鍗曪細`phase-3-plan.md`
+> 浠ｇ爜浜嬪疄锛歚phase-2-review.md`
+> 璇煶鍥炲綊搴曡〃锛歚phase-2-voice-test.md`锛堟湰闃舵鍥轰欢 **涓嶅繀閲嶇儳**锛岄櫎闈炰綘杩樺仠鐣欏湪 Phase 1 bin锛�
+
+鏈樁娈甸獙鏀朵袱浠朵簨锛�**搴撻噷鑳芥煡鍒拌窡璇�/澶嶈瀹炰緥**锛屼互鍙� **Phase 2 璇煶琛屼负娌℃湁鍥為��**銆�
+涓嶈娴嬫暟瀛楀�掕儗銆佸彛绠椼�佹垚璇帴榫欙紝涓嶈娴嬪垎鏁般��
+
+---
+
+## 0. 寮�濮嬪墠
+
+1. manager-api 宸茶窇 Liquibase锛堝瓨鍦� `ai_task_instance` / `ai_task_step` / `ai_task_event`锛夈��
+2. xiaozhi-server 鑳借繛涓� manage_api锛堜笌鐜扮綉 chat report 鍚屼竴濂楀瘑閽ワ級銆�
+3. 璁惧浠嶆槸 Phase 2 鍥轰欢锛氬璇荤粨鏉熷彧鎷涘懠涓�娆★紱璺熻涓�鍙ヤ竴 MP3銆�
+4. SD 闊抽浠嶆寜锛�
+
+```text
+/english_learning/shadowing/lessons/unit01/01.mp3
+/english_learning/shadowing/lessons/unit01/02.mp3
+/english_learning/repeater/lessons/unit01/full.mp3
+```
+
+5. pytest锛�
+
+```text
+cd xiaozhi-esp32-server/main/xiaozhi-server
+python -m pytest tests/test_app_runtime.py -q
+```
+
+搴旈�氳繃锛堝惈 upsert 澶辫触涓嶉樆鏂� `save` 鐨勬柊鐢ㄤ緥锛夈�備笉杩囪繖涓�鍏筹紝涓嶈涓婅澶囥��
+
+鍑嗗锛�
+
+- 鎺у埗鍙扮櫥褰� token锛圙ET 鐢級
+- server secret锛圥UT 璋冭瘯鐢紝涓� `/agent/chat-history/report` 鐩稿悓锛�
+- 璁惧 `mac` / `device_id`
+
+---
+
+## 1. 璇煶鍥炲綊锛堝繀椤诲厛杩囷紝鍐嶆煡搴擄級
+
+鍙窇 `phase-2-voice-test.md` 鐨勮繖浜涜妭锛屽嬀閫夋爣鍑嗕笉鍙橈細
+
+| 鑺� | 鍐呭 | 閫氳繃锛� |
+|----|------|--------|
+| 搂0 | 寮�鏈哄璇濇ā寮忥紱鍞ら啋锛涖�屼粖澶╁ぉ姘旀�庝箞鏍枫�嶅綋闂茶亰 | |
+| 搂2 | 璺熻鏁存锛氭挱瀹屽啀璺燂紱涓嶈绗戣瘽锛涙渶鍚庝竴杞洖 Chat锛汮SON 鍚叏閮ㄦ湁鏁堣疆 | |
+| 搂5-A | 澶嶈鑷劧缁撴潫 **鍙嫑鍛间竴娆�** | |
+| 搂5-C | 鎾斁涓銆屼綘濂藉悧銆嶇户缁挱锛屼笉鍥炵瓟 | |
+
+鏈樁娈� **鏂板** 鐨勮闊冲け璐ユ爣鍑嗭細
+
+- 涓�璺熻灏卞崱姝� / 涓�鐩翠笉鎾笅涓�鍙� 鈫� 澶氬崐鏄� upsert 鍚屾鍫典綇浜� WS 绾跨▼锛堣璁″垝 搂10锛夈��
+- 璺熻姝ｅ父浣� GET 404 鈫� 鍙槸涓婃姤澶辫触锛屽厛璁� 搂3锛屼笉瑕佹敼鍙ｄ护閫昏緫銆�
+
+缈昏瘧 / 涓嫳 / 绂荤嚎锛氳嫢鏃堕棿绱у彲鎶芥祴銆屾墦寮�缈昏瘧 鈫� 閫�鍑虹炕璇� 鈫� 璁蹭釜绗戣瘽銆嶃�傜绾垮璇讳粛涓嶅緱杩炰簯銆�
+
+---
+
+## 2. 璺熻 Instance锛堟牳蹇冿級
+
+浠庡璇濇ā寮忚 **寮�濮嬭窡璇�**锛屾寜闊抽璺熷畬锛堟垨鑷冲皯涓ゅ彞鍚庤 **閫�鍑鸿窡璇�**锛夈��
+
+### 2.1 鏂囦欢浠嶅湪
+
+鎵撳紑 `xiaozhi-esp32-server/main/xiaozhi-server/data/task_sessions/` 鏈�鏂� JSON锛�
+
+| 妫�鏌� | 閫氳繃锛� |
+|------|--------|
+| `task_session_id` 涓� 32 浣� hex | |
+| `app_id=english_learning`锛宍task_id=shadowing` | |
+| `turns` 鏉℃暟 = 浣犺窡璇绘垚鍔熺殑鍙ユ暟锛堥��鍑哄垯灏戜簬闊抽鎬绘暟锛� | |
+| 鏈�鍚庝竴杞� `asr_text` 涓嶆槸绌猴紙鑻ヤ綘璇翠簡鏈�鍚庝竴鍙ワ級 | |
+| `status` / `end_reason` 涓� `completed`+`finished` 鎴� `cancelled`+`user_exit` | |
+| `events` 閲屾湁 `audio.completed`锛堟瘡鍙ユ挱瀹岋級 | |
+
+璁颁笅 `task_session_id` = `{id}`銆�
+
+### 2.2 GET 涓庢枃浠朵竴鑷�
+
+```text
+ GET /task/instance/{id}
+ Authorization: Bearer {oauth token 鎴栦笌 chat report 鐩稿悓鐨� server secret}
+ 锛堟湰闃舵 Shiro 灏� /task/instance/** 閰嶆垚 server 杩囨护鍣紝鐢� server secret 鍗冲彲楠岄�氾級
+```
+
+| 妫�鏌� | 閫氳繃锛� |
+|------|--------|
+| HTTP 200 | |
+| `status` / `end_reason` / `app_id` / `task_id` 涓� JSON 鐩稿悓 | |
+| `turns`锛堟垨鍝嶅簲閲岀殑 `steps`锛夋潯鏁颁笌 JSON `turns` 鐩稿悓 | |
+| 绗� 0 鍙� `prompt_text` 浠嶆槸 SD 璺緞锛屼笉鏄� TTS 鑻辨枃鑼冭 | |
+| 瀵瑰簲 `asr_text` 瀵瑰緱涓婁綘璇寸殑璇濓紙鍏佽 ASR 璇樊锛� | |
+| `events` 鍚� `audio.completed` | |
+
+鑻ュ疄鐜伴�変簡璁″垝 搂4.3-A锛氭湁鏁堣疆鐨� `audio_id` 闈炵┖锛屼笖鑳藉湪 `ai_agent_chat_audio` 鏌ュ埌銆傞�� B 鍒欏厑璁稿叏绌恒��
+
+### 2.3 鍒楄〃
+
+```text
+GET /task/instance?macAddress={mac}&limit=20
+```
+
+| 妫�鏌� | 閫氳繃锛� |
+|------|--------|
+| 鍒氶偅鏉� `{id}` 鍑虹幇鍦� list | |
+| list 椤� **娌℃湁** 鏁翠唤 turns 姝ｆ枃锛堥伩鍏嶆妸璺熻褰撹亰澶╄褰� dump锛� | |
+
+---
+
+## 3. 澶嶈 Instance
+
+璇� **澶嶈鏈�**锛岀瓑鑷劧缁撴潫锛堟垨闊抽寰堢煭锛夈��
+
+| 妫�鏌� | 閫氳繃锛� |
+|------|--------|
+| 鏂� JSON锛歚task_id=repeater`锛宍end_reason=play_completed` | |
+| GET锛歚turns` 鍙互涓虹┖ | |
+| GET锛歚events` 鍚� `audio.completed` | |
+| 璁惧鍙嫑鍛煎洖 Chat **涓�娆�**锛堝洖褰掞級 | |
+
+涓�� **閫�鍑哄璇�**锛歚end_reason` 涓� `user_exit` 鎴� `play_stopped`锛堜笌 Phase 2 璋佸厛鍒版湁鍏筹級锛孏ET 鑳界湅鍒帮紝涓嶄涪銆�
+
+---
+
+## 4. 澶辫触闅旂锛堝繀椤绘祴锛�
+
+**A. manage_api 涓嶅彲鐢�**
+
+鍋滄帀 manager-api锛堟垨鏀归敊瀵嗛挜锛夛紝鍐嶈 **寮�濮嬭窡璇�** 璺熶竴鍙ュ啀 **閫�鍑鸿窡璇�**銆�
+
+| 妫�鏌� | 閫氳繃锛� |
+|------|--------|
+| 璁惧浠嶆挱闊抽銆佷粛鑳介��鍑哄洖 Chat | |
+| 鏈湴 JSON **浠嶇劧**鍐欏嚭 | |
+| xiaozhi-server 鏃ュ織鏈� upsert 澶辫触锛�**娌℃湁**鎶� WS 鎵撴寕 | |
+| GET 鍙互 404 / 鏃ф暟鎹紱杩欎笉绠楄闊冲け璐� | |
+
+娴嬪畬鎭㈠ manager-api銆�
+
+**B. 闈炴硶 PUT锛堝彲閫夛級**
+
+鐢� server secret PUT 涓�涓� `app_id` 鍚� `../` 鎴栬秴闀� id 鈫� 400锛屼笉鍐欏簱銆�
+
+---
+
+## 5. Agent 鈮� App
+
+鍦� **瀵硅瘽妯″紡** 璇淬�屼粖澶╁ぉ姘旀�庝箞鏍枫�嶆垨銆岃涓瑧璇濄�嶃��
+
+| 妫�鏌� | 閫氳繃锛� |
+|------|--------|
+| `ai_agent_chat_history` 浠嶅鍔犺亰澶╄锛堢幇缃� report锛� | |
+| **娌℃湁**鍥犳鏂板涓�鏉� `app_id=chat` 鐨� task_instance锛坈hat 涓� begin_session锛涜嫢瀹炵幇璇姤浜� chat instance 鍒欏け璐ワ級 | |
+| 璺熻 instance 浠嶇劧鍙湪 `/task/instance` 鏌ワ紝涓嶅嚭鐜板湪銆岃亰澶╄褰曞氨鏄煶棰戙�嶇殑閿欒閲� | |
+
+---
+
+## 6. 鏂嚎
+
+璺熻璇村埌绗簩鍙ヨ繃绋嬩腑锛屾嫈鎺夎澶� Wi鈥慒i 鎴栨潃 xiaozhi-server 杩炴帴銆�
+
+| 妫�鏌� | 閫氳繃锛� |
+|------|--------|
+| 鏈湴 JSON `end_reason=disconnected` 鎴栫瓑浠� cancelled | |
+| 鑻ユ柇绾垮墠鑷冲皯鎴愬姛 save/PUT 杩囦竴娆★細GET 鑳界湅鍒板凡鏈� turns锛屼笉鏄┖瀹炰緥鎶婃垚鍔熻疆鎶规帀 | |
+
+锛堣嫢鏂嚎鍙戠敓鍦ㄧ涓�娆� PUT 涔嬪墠锛屽簱閲屽彲鑳芥病鏈夎锛氫互 JSON 涓哄噯锛屽苟鍦� PR 璇存槑銆傦級
+
+---
+
+## 7. 鍥轰欢鍐荤粨鎶芥
+
+| 妫�鏌� | 閫氳繃锛� |
+|------|--------|
+| `xiaozhi-esp32-2.2.6` 鐩稿 Phase 2 **鏃犱笟鍔� diff**锛堟湰闃舵 PR 涓嶅簲鍚鐩綍锛� | |
+| 鏈姹傞噸鐑� bin 鍗冲彲瀹屾垚 搂1鈥撀�3 | |
+| 鏈嚭鐜版柊鐨� WS `type`銆佹柊 MQTT action銆佹柊 MCP 鍚� | |
+
+鏂拌闃� App 寤虹洰褰曞睘浜� Phase 2 宸蹭氦浠樿兘鍔涳紝鏈樁娈� **涓嶈**褰撴柊鍔熻兘鍐嶆祴涓�閬嶏紱鑻ュ洖褰掓椂鍙湅杩炰笂鍚� SD 鏄惁浠嶆湁 `/english_learning/.../lessons`銆�
+
+---
+
+## 8. 涓嶈娴嬩粈涔�
+
+- 鍒嗘暟銆佸閿欍�佸啀璇讳竴閬嶇殑璇勪及绛栫暐
+- 鍊掕儗 / 鍙ｇ畻 / 鎺ラ緳
+- 瀹堕暱灏忕▼搴� UI銆乵anager-web 鏂伴〉闈�
+- `TASK_PAUSE`銆乣wait` Skill銆佹湰鍦板綍闊充笂浼�
+- 鎶� SD 浠� `/{app}/{task}/lessons` 鎼埌 `/resources/`
+
+---
+
+## 9. 澶辫触鏃跺厛鐪嬪摢
+
+| 鐜拌薄 | 澶氬崐鏄� |
+|------|--------|
+| 璺熻鍗℃ | upsert 鍚屾闃诲锛涙敼涓� end 鏃跺啀 PUT 鎴栧悗鍙颁换鍔� |
+| JSON 鏈夈�丟ET 404 | Python 鏈寕閽� / Shiro 鎶� PUT 鎷︿簡 / 瀵嗛挜涓嶄竴鑷� |
+| GET 200 浣� turns 灏戞渶鍚庝竴杞� | `end_session` 娌″湪鏈�鍚庝竴杞� `add_turn` 涔嬪悗 save锛圥hase 2 鍥炲綊锛� |
+| 澶嶈缁撴潫鎷涘懠涓ゆ | 涓嶆槸 Phase 3 闂锛涘浐浠跺張鍙戜簡 `[app_task]chat` |
+| chat 绗戣瘽杩涗簡 task_instance | begin_session 璇粦 chat |
+| Liquibase checksum 澶辫触 | 鏀逛簡鏃� SQL锛涘簲鍙姞鏂� changeset |
+
+---
+
+## 鍙ｄ护閫熸煡锛堜笌 Phase 2 鐩稿悓锛�
+
+| 鎰忓浘 | 璇村叾涓竴鍙ュ嵆鍙� |
+|------|----------------|
+| 鍞ら啋 | 浣犲ソ灏忔櫤 |
+| 璺熻 | 寮�濮嬭窡璇汇�佹垜瑕佽窡璇� |
+| 澶嶈 | 澶嶈鏈恒�佸紑濮嬪璇� |
+| 鍥炶亰 | 閫�鍑鸿窡璇汇�侀��鍑哄璇汇�佹垜浠潵鑱婂ぉ鍚� |
diff --git a/phase-4-plan.md b/phase-4-plan.md
new file mode 100644
index 0000000..65a5400
--- /dev/null
+++ b/phase-4-plan.md
@@ -0,0 +1,349 @@
+# Phase 4 Coding Plan 鈥� 鏃堕挓姝ヨ繘 + 鍚庤瘎浼帮紙瀹氶噺锛�
+
+> File: `phase-4-plan.md`
+> Date: 2026-08-31
+> 绾︽潫鏂囨。锛歚Xiaozhi Device Capability Specification v1.0.md`
+> 浠ｇ爜浜嬪疄锛歚phase-3-review.md`锛圕urrent Architecture V4锛�
+> 璺嚎鍥撅細`roadmap.md`
+> 楠屾敹锛歚phase-4-test.md`
+> **瑕嗙洊**姝ゅ墠銆孲8 褰撳満鍒ゅ閿欏啀鎾悓涓�鍙ャ�嶇殑鑽夋銆傜儹璺緞鏄� **MP3 搴忓垪 + 瀹氭椂褰曢煶**锛涘垎鏋愬叏閮ㄥ悗缃�佸畾閲忋�佸厑璁稿け璐ャ��
+
+鏈樁娈� **涓嶆槸** 鍐嶅仛涓�涓� App锛屼篃 **涓嶆槸** 鍋氬�掕儗/鍙ｇ畻/鎺ラ緳锛堥偅鏄� Phase 4b锛屼笖閭ｄ簺鏄� **娴佸唴鍙婃椂璇勪及**锛屼笉鑳藉鏈樁娈靛悗璇勪及锛夛紝涔� **涓嶆槸** 鏀瑰浐浠躲��
+
+缁� Coding Agent锛氬厛璇� Spec 搂1.3 S4銆伮�15銆伮�17銆伮�22锛涘啀璇� `phase-3-review.md` 搂2.4锛坄prompt_text` 鏄矾寰勶級锛涘啀璇� `runtime.py` 鍚獥 / `_schedule_shadowing_listen_timeout` / `handle_shadowing_turn`銆乣session.py`銆乣TaskTurnDTO.java`銆乣202608311800.sql`锛�**绂佹鏀瑰凡鎵ц changeset**锛夈��
+
+---
+
+## 0. 浜у搧妯″瀷锛堟湰闃舵閽夋锛屽悗闈㈠緢澶� App 鍏辩敤锛�
+
+```text
+宸茬煡锛氭瘡涓寖璇� 01.mp3 鏈夊搴� 01.txt锛堝悓 stem锛岃祫婧愬寘濂戠害锛�
+ 鎾斁鏃堕暱 duration_ms 鏉ヨ嚜宸叉湁 D1 瀹屾垚浜嬩欢锛堣澶囧凡鎶ワ級
+
+鐑矾寰勶紙蹇呴』娴佺晠锛屼换浣曞垎鏋愬け璐ラ兘涓嶅緱鎵撴柇锛夛細
+ play 01.mp3
+ audio.completed
+ 绔嬪嵆寮�褰曢煶/鍚獥 = listen_ratio 脳 duration_ms 锛坴0锛�1.2锝�1.5锛岄粯 1.2锛�
+ 瀹氭椂缁撴潫 鈫� 涓婁紶灏藉姏鑰屼负 鈫� play 02.mp3
+ 鈥� 鐩村埌搴忓垪缁撴潫 鈫� 鍥� Chat
+
+绂佹锛�
+ 绛� ASR 鎴愬姛鎵嶆挱涓嬩竴鍙�
+ 绛変笂浼犳垚鍔熸墠鎾笅涓�鍙�
+ 鐢ㄦ枃鏈瘮瀵瑰喅瀹氬啀鎾悓涓�鍙ユ垨涓嬩竴鍙�
+ 鍦ㄨ澶囦笂璇诲嚭銆屾纭�/閿欒銆嶃�岃秴杩� 82% 绠楀ソ銆�
+```
+
+璺熻銆佹晠浜嬪杩般�佹墦鍗″綍闊筹細**鍚庤瘎浼�**鈥斺�旂粰瀹氬惉绐楋紝鏃堕棿鍒板氨涓嬩竴姝ワ紱瀹堕暱绋嶅悗鐪� metrics銆�
+
+鍊掕儗鏁板瓧銆佹垚璇帴榫欍�侀粯璁ゅ彛绠楋細**涓嶆槸鏈樁娈垫ā鍨�**銆傚惉绐楃粨鏉熷悗蹇呴』椹笂璇勪及骞� `speak` 鍙嶉锛屽睘浜� Phase 4b 鐨� `assessment.timing=inline`锛堣 `roadmap.md`銆岃瘎浼版椂鏈恒�嶏級銆備笉瑕佸湪鏈� PR 閲屾妸瀹冧滑鍋氭垚鍚庤瘎浼般��
+
+Spec **S8 `branch(if correct)` 涓嶆槸鏈骇鍝侀粯璁ょ儹璺緞銆�** 鐣欑粰浠ュ悗鍙�夌殑銆岀簿缁冩ā寮忋�嶏紝鏈樁娈典笉鍋氥�係4 璇勪及浠嶅湪 Server锛屼絾鏄� **寮傛/鍙欢杩�**锛屽啓鍦� Instance 涓婏紝涓嶆寚鎸ュ枃鍙��
+
+---
+
+## 1. 涓� Spec 鐨勫鐓�
+
+| Spec ID | 鑳藉姏 | Phase 4 鍋氫粈涔� | 涓嶅仛浠�涔� |
+|---------|------|----------------|----------|
+| S4 | Assessment | 鍚庤瘎浼帮細瀵瑰凡褰掓。 ASR/闊抽鐩稿 `expected_text` 绠� **瀹氶噺鎸囨爣** | 鍙婃牸绾裤�佸ソ鍧忕粨璁恒�佷复搴婅瘖鏂� |
+| S8 | branch if correct | **涓嶅仛** | 鍐嶆挱鍚屼竴鍙ャ�丣UDGE銆乺etry |
+| D1 | duration_ms | 鐢ㄥ畬鎴愪簨浠舵椂闀跨畻鍚獥 | 涓嶆敼鍥轰欢鎶ユ椂 |
+| D16 | 姝ラ绾� loop | Server 浠嶆寜 clip 搴忓垪涓嬪彂涓嬩竴鍙� play锛�**瑙﹀彂鍣ㄥ彧鏈夋椂閽�** | 璁惧鑷繁鎾暣璇撅紱璇勪及椹卞姩姝ヨ繘 |
+| D6/D7/D8 | 鍚�/褰掓。 | 鍚獥鍐呭敖鍔� ASR + report锛涘け璐ュ拷鐣� | 涓嶆敼 Opus |
+| D11 wait | | 浠嶇敤 Server 瀹氭椂锛屼笉绛夊緟 Skill | |
+| D13 | 璧勬簮 | `01.mp3` 鈫� `01.txt` 涓哄凡鐭ュ绾︼紱鐑矾寰勪笉 MCP 璇诲崱涓� txt | 涓嶆惉鐩綍 |
+| 琛� | | 鍙┖ `expected_text` + `metrics` JSON | 涓嶅缓 `passed` 褰撲骇鍝佺粨璁� |
+
+---
+
+## 2. 鐩爣涓庨潪鐩爣
+
+### 2.1 Objective
+
+1. 璺熻 = **棰勫畾 MP3 搴忓垪**銆傚彞闂村彧绛� `listen_ratio 脳 duration_ms`锛堜笅闄愪繚鎸佺幇缃戞渶灏� 800ms锛岄櫎闈為厤缃樉寮忔敼锛夈��
+2. **鏈夋晥 ASR 涓嶅啀鎻愬墠鍒囦笅涓�鍙�**锛堢浉瀵� Phase 2 鐨勮涓哄彉鏇达細璇村畬涔熻绛夊埌鏃堕挓锛涙椂閽熷埌浜嗘病璇村畬涔熷垏锛夈��
+3. 褰曢煶/ASR/PUT/姣斿 **鍏ㄩ儴鏃佽矾**锛氬け璐ュ彧 log锛屽簭鍒楃収璧般��
+4. `expected_text` 鏉ヨ嚜涓� basename 瀵瑰簲鐨� txt 濂戠害锛堣 搂4锛夛紝渚涘悗璇勪及浣跨敤锛�**涓�**鎷� SD 璺緞褰撳師鏂囥��
+5. 鍚庤瘎浼板啓鍏� **瀹氶噺** `metrics[]`锛堟柟娉曞悕 + 鏁板�硷級锛屼笉鍐欍�屽ソ/鍧�/鍙婃牸銆嶃�傝瘎浼板彲鍦ㄦ湰鍙ユ椂閽熷凡缁忚繘鍏ヤ笅涓�鍙ヤ箣鍚庢墠瀹屾垚銆�
+6. `listen_ratio` 鍙厤缃紱棰勭暀鎸夌敤鎴锋按骞冲彇鍊肩殑鍑芥暟鍙ｏ紙v0 鐢� task/JSON 甯告暟鍗冲彲锛夈��
+7. 澶嶈 / Chat / 缈昏瘧涓嶈瘎浼般�佷笉鏀规椂閽熻涔夈��
+
+### 2.2 Why Now
+
+Phase 3 宸叉湁 Instance 鍙寕缁撴灉銆傝嫢涓嶆妸銆屽垎鏋愩�嶄粠鐑矾寰勬媶寮�锛屽悗闈㈡瘡涓� App 閮戒細鎶婂枃鍙崱鍦ㄤ簯绔帹鐞嗕笂銆傝窡璇绘槸鏈�渚垮疁鐨勯拤閽夈��
+
+### 2.3 Acceptance Criteria
+
+**鐑矾寰勶細**
+
+1. 璇撮敊銆佷笉璇淬�丄SR 澶辫触銆佸叧鎺� manage_api锛氬惉绐楃粨鏉熷悗 **浠嶆挱涓嬩竴鍙�**锛屼笉閲嶆挱鍚屼竴鍙ャ�佷笉鍗℃銆�
+2. 璺熷涔熶笉鎻愬墠鍒囧彞锛堢瓑鍒版椂閽燂級銆傚厑璁� 卤200ms 閲忕骇瀹氭椂璇樊銆�
+3. 鍏ㄧ▼涓嶈绗戣瘽锛涢��鍑鸿窡璇讳粛绔嬪埢鍥� Chat銆�
+4. 澶嶈缁撴潫浠嶅彧鎷涘懠涓�娆°��
+
+**鍚庤瘎浼帮細**
+
+5. 鏈� txt 鍘熸枃涓旀湰鍙ユ湁 ASR 鏃讹細鍦ㄣ�屼竴瀹氭椂闂村唴銆嶏紙鏈樁娈靛厑璁镐細璇濈粨鏉熷悗鏁扮鍐呭悗鍙板畬鎴愬嵆鍙級GET 鍒拌 step 鐨� `expected_text` + `metrics`锛堣嚦灏戜竴椤� `text_similarity_v0` 鐨� 0鈥�1 鏁板�硷級銆�
+6. 鏃� ASR / 鏃� txt / 璇勪及鎶涢敊锛歚metrics` 绌猴紝**涓嶅奖鍝�**宸插彂鐢熺殑鎾斁銆�
+7. 浠讳綍 API/椤甸潰 **涓嶅緱**鎶婂崟涓�闃堝�兼覆鏌撴垚蹇呴』鐨勫閿欑粨璁猴紙鏈樁娈垫棤 UI锛涘瓧娈甸噷涓嶈 `passed`/`grade`锛夈��
+
+**绾緥锛�**
+
+8. 鍥轰欢闆舵彁浜ゃ�傛棤鏂� WS/MQTT/MCP銆傛棤 JUDGE銆備笉鏀� Opus銆備笉鍔犵鎴枫�備笟鍔″彛浠や笉杩� C++銆�
+
+### 2.4 Out of Scope
+
+- 褰撳満绾犻敊鍐嶆挱銆乵ax_retry銆佸強鏍肩嚎浜у搧鍖�
+- 闊崇礌/娴佸埄搴�/鎯呯华/鏅哄晢缁撹/蹇冪悊璇婃柇锛堜互鍚庝篃鏄柊 method 鐨勫畾閲忓�硷紝涓嶆槸鏈樁娈碉級
+- 4b 涓変釜 Task 鐨勯搴�
+- 瀹屾暣鐢ㄦ埛绛夌骇寮曟搸锛堝彧鐣� `resolve_listen_ratio` 鍙ｏ級
+- 鐑矾寰� MCP 璇昏澶囦笂鐨� `01.txt`
+- manager-web 鎵撳崱椤点�佺鎴�
+- 璁惧榛樿璇█ / 缈昏瘧璇█瀵癸紙Roadmap Phase 6 / 6b / 7锛�
+
+---
+
+## 3. 鏋舵瀯
+
+```text
+Device锛圥hase 2 bin锛�
+ play clip i 鈫� [skill_event]completed|duration_ms 鈫� 鍚祦
+ 鈫� 鈫�
+xiaozhi-server 鏃堕挓 鏃佽矾锛堝け璐ュ拷鐣ワ級
+ sleep(listen_ratio * duration_ms) ASR 灏藉姏
+ play clip i+1 锛堜笉绛夋梺璺級 report 闊抽灏藉姏
+ 鍚庡彴 assess 鈫� metrics
+ PUT Instance 灏藉姏
+```
+
+**姝ヨ繘鏉冨▉鏄椂閽燂紝涓嶆槸璇勪及鍣ㄣ��**
+璇勪及鍙互鍦� xiaozhi-server 鐢� `create_task` 鍋氾紙鏈樁娈靛厑璁革級锛屼篃鍙互浠ュ悗鏀规垚 manager-api Worker锛�**涓嶅緱** `await assess()` 鍐� play銆�
+
+---
+
+## 4. expected_text = 涓� MP3 鍚屽悕鐨� txt锛堝凡鐭ユ潯浠讹級
+
+璧勬簮鍖呯害瀹氾紙SD 涓庝簯绔竻鍗曞悓涓�濂� stem锛孭hase 6 涓嬭浇涓よ�咃紱鏈樁娈典笉鏂板仛 DOWNLOAD锛夛細
+
+```text
+.../lessons/unit01/01.mp3
+.../lessons/unit01/01.txt # UTF-8锛岃寖璇诲師鏂�
+```
+
+Server **瑙ｆ瀽椤哄簭**锛堢儹璺緞涓嶈璁惧 SD锛夛細
+
+1. `task.transcripts["01.mp3"]` 鎴� `["01"]`锛坢anifest 宸插甫姝ｆ枃锛岀瓑浜� txt 宸插悓姝ュ埌浜戯級銆�
+2. 杩涚▼鍐呴煶棰戠紦瀛樼洰褰曡嫢瀛樺湪鍚� stem 鐨� `.txt` 鍒欒鏂囦欢锛堜粎 Server 鏈湴/宸蹭笅杞藉寘锛�**绂佹**涓烘鍔� MCP锛夈��
+3. 鍏煎鐜扮綉锛歚sentences[i]` 涓庢帓搴忓悗 clips 涓嬫爣瀵归綈銆�
+4. 閮芥病鏈� 鈫� `expected_text=null`锛屾湰鍙ヤ笉璇勪及锛屾挱鏀句笉鍙樸��
+
+绂佹瀵� `prompt_text`锛堣矾寰勶級鍋氱浉浼煎害銆�
+
+`apps.json` 璺熻 task 绀轰緥锛�
+
+```json
+"listen_ratio": 1.2,
+"assessment": {
+ "enabled": true,
+ "methods": ["text_similarity_v0"],
+ "defer": true
+}
+```
+
+涓嶈 `threshold`銆乣max_retry`銆乣passed`銆�
+`listen_ratio` 鍏佽 1.2鈥�1.5锛涜秴鍑哄垯澶圭揣銆備互鍚� `resolve_listen_ratio(conn, task)` 鍙寜鐢ㄦ埛姘村钩杩斿洖涓嶅悓鍊硷紱v0 璇� JSON銆�
+
+---
+
+## 5. 瀹氶噺鎸囨爣锛堢姝㈠仛鎴愮粨璁猴級
+
+`core/app_runtime/assessment.py` 绾嚱鏁帮細
+
+```text
+normalize(s): 灏忓啓銆佸幓鏍囩偣绌虹櫧锛汚SR 鍏� _plain_user_text
+
+metrics = [
+ { "method": "text_similarity_v0", "value": <0-1 SequenceMatcher ratio> }
+]
+```
+
+- **鍙骇鍑� method + value**锛堝彲閫� `unit` 鐪佺暐涓� ratio锛夈��
+- **绂佹**鍑芥暟杩斿洖 `passed` / `grade` / `level=good`銆�
+- 瀹堕暱绔互鍚庤嫢瑕佸睍绀恒�屽缓璁啀缁冦�嶏紝閭ｆ槸 **鍙︿竴涓骇鍝佸眰** 鐢ㄥ悓涓�鏁板瓧鍋氳В閲婏紝涓嶆槸鏈樁娈靛瓧娈点��
+- 鐩婃櫤/蹇冪悊锛氫互鍚庡姞 `method=...` 鏂拌锛屼粛鐒跺彧瀹氶噺銆�
+
+璇勪及鎶涢敊鎴栫己灏� asr/expected锛氳繑鍥� `[]`锛屼笉鎶涘埌鐑矾寰勩��
+
+涓� `is_echo_of_prompt`锛�0.82锛夊畬鍏ㄥ垎寮�銆傚惉绐楅棬闂╀繚鎸併�備笉瑕佺敤 echo 闃堝�煎綋浠讳綍浜у搧鍚箟銆�
+
+---
+
+## 6. 鐑矾寰勭浉瀵� Phase 2/3 鐨勮涓哄彉鏇�
+
+| 鐜扮綉 Phase 2/3 | Phase 4 |
+|----------------|---------|
+| 鍚獥鍐呮湁鏁� ASR 鈫� **绔嬪埢**涓嬩竴鍙� | ASR 鍙綊妗ｏ紱**鏃堕挓鍒颁簡**鎵嶄笅涓�鍙� |
+| 瓒呮椂鎵嶄笅涓�鍙� | 瓒呮椂 = 鏃堕挓姝ｅ父寮�鐏紙涓昏矾寰勶級 |
+| 鏃犲悗璇勪及 | 鏃堕挓鍚�/浼氳瘽鍚庡啓 metrics |
+
+瀹炴柦锛歚handle_shadowing_turn` **涓嶈**鍐嶈皟鐢� `_play_shadowing_clip` / `index++`銆傚彧 `upsert_turn` + `enqueue_asr_report`銆�
+涓嬩竴鍙ュ彧鏀惧湪鍚獥 timeout 鍥炶皟閲岋紙宸叉湁 `_schedule_shadowing_listen_timeout`锛夈�傞��鍑哄彛浠や粛绔嬪嵆鍥� Chat銆�
+
+鎾斁涓� ignore銆佹湭鍒板惉绐� ignore锛氫繚鎸併��
+
+---
+
+## 7. Turn / 琛�
+
+鍚屼竴 `step_index` 涓�琛屻�傜獥鍙ｅ唴澶氭 ASR锛歷0 **淇濈暀鏈�鍚庝竴娆�** 鏂囨湰涓庨煶棰戞剰鍥撅紙鏇存柊鍚屼竴 turn锛夈��
+
+### 7.1 JSON / PUT锛坰nake_case锛�
+
+| 瀛楁 | 璇存槑 |
+|------|------|
+| `prompt_text` | 浠嶄负 SD 璺緞 |
+| `expected_text` | txt 姝ｆ枃锛屽彲绌� |
+| `asr_text` | 鍙┖锛堟病璇�/娌¤鍑猴級 |
+| `metrics` | 鏁扮粍锛屽彲 `[]`锛涘厓绱� `{method, value}` |
+| `assessed_at` | epoch ms锛屽彲绌鸿〃绀哄皻鏈瘎浼� |
+
+**涓嶈** `passed` / `retry_count` / `grade`銆�
+
+鍙�� event锛歚listen_window_closed`锛堟椂閽熷紑鐏級銆備笉鏄柊 WS銆�
+
+### 7.2 Liquibase
+
+鏂版枃浠讹紝渚嬪 `db/changelog/202609011100.sql`锛岀櫥璁� master銆�**绂佹鏀�** `202608311800.sql`銆�
+
+```text
+ai_task_step
+ expected_text TEXT NULL
+ metrics JSON NULL
+ assessed_at BIGINT NULL
+```
+
+MySQL JSON 绫诲瀷涓庣幇缃戝叾瀹冭〃涓�鑷村嵆鍙紱璇诲洖涓� list銆�
+缂� metrics 鐨勬棫 PUT 蹇呴』 200銆�
+瑕嗙洊鎻掑叆锛氳嫢鏂� payload `metrics` 涓虹┖涓旀棫琛屽凡鏈夛紝**淇濈暀鏃� metrics**锛堝悗璇勪及鏅氫簬涓�娆＄┖ PUT 鏃朵笉琚啿鎺夛級銆傛樉寮忛潪绌烘暟缁勪互鏂颁负鍑嗐��
+
+Java DTO `@JsonIgnoreProperties(ignoreUnknown=true)` 缁х画銆�
+
+---
+
+## 8. HTTP
+
+涓嶆柊澧炶矾寰勩�侾UT/GET 浠� `/task/instance/{id}`锛宻erver secret銆�
+鍚庤瘎浼板畬鎴愬悗鍐� PUT 涓�娆″叏鏂囷紙鎴栧彧闈犻偅娆″甫 metrics 鐨� save锛夊嵆鍙�備笉瑕佷负 metrics 鍗曠嫭鍋氱浜屽閴存潈銆�
+
+---
+
+## 9. 鏀瑰姩娓呭崟
+
+### 9.1 manager-api
+
+| ID | 浠诲姟 |
+|----|------|
+| J1 | 鏂板垪 + DTO |
+| J2 | upsert 鍚堝苟绌� metrics锛堣 搂7.2锛� |
+| J3 | 涓嶈鏍￠獙銆屽繀椤绘湁 score銆嶏紱涓嶈鍑虹幇 passed 鍒� |
+
+manager-web 闆舵彁浜ゃ��
+
+### 9.2 xiaozhi-server
+
+| ID | 浠诲姟 |
+|----|------|
+| S1 | `listen_ratio` 澶圭揣 1.2鈥�1.5锛泃imeout = max(800, ratio * duration_ms) |
+| S2 | ASR 鏈夋晥 鈫� 鍙� upsert_turn + report锛�**涓�** play next |
+| S3 | 浠� timeout 鍥炶皟锛歩ndex++ 骞� play next / 缁撴潫 |
+| S4 | `resolve_expected_text`锛埪�4锛� |
+| S5 | `assessment.py` 鍙繑鍥� metrics 鍒楄〃锛沗create_task` 閲岃窇锛�**await 涓嶆尅 play** |
+| S6 | pytest |
+| S7 | 澶嶈/Chat/缈昏瘧涓嶇璇勪及 |
+
+---
+
+## 10. 娴嬭瘯
+
+瑙� `phase-4-test.md`銆傛渶浣庨檺搴︼細
+
+1. pytest锛氭椂閽熸帹杩涗笌 ASR 瑙ｈ�︼紱metrics 鏃� passed锛涜矾寰勪笉鑳藉綋 expected銆�
+2. 璇煶锛氳閿欎粛杩涗笅涓�鍙ワ紱璇村涔熶笉绉掑垏锛堢瓑鍒扮獥锛夈��
+3. manage_api 鍋滄帀锛氬簭鍒椾粛璧板畬銆�
+4. Phase 3 GET 瀛楁浠嶅湪锛涘浜� expected_text/metrics銆�
+
+---
+
+## 11. 瀹炴柦椤哄簭
+
+```text
+P4-A Liquibase + Java 鏂板垪锛堟棤 passed锛�
+P4-B 鏃堕挓涓� ASR 瑙ｈ�� + listen_ratio + pytest
+P4-C 鍚庡彴 metrics锛坱xt 濂戠害锛�+ 绌� metrics 涓嶅啿鏃у��
+P4-D phase-4-test.md
+```
+
+瀹ｇО Done = B+D锛堣В鑰︽槸浜у搧鏍稿績锛夈�侰 鍋氫笉瀹屽垯 metrics 鍏ㄧ┖锛屼絾 **涓嶅緱**鍥為��鍒般�屾瘮瀵瑰け璐ュ氨閲嶆挱銆嶃�傛湭瀹屾垚 D 涓嶅緱寮� 4b銆�
+
+---
+
+## 12. 缂栫爜绾︽潫
+
+1. 鍥轰欢闆� diff銆�
+2. 鐑矾寰勪笉寰� `await` 璇勪及/姣斿/HTTP銆�
+3. 涓嶅緱鎶婂閿欍�佸強鏍笺�乺etry 鍐欏叆璁惧鎴� `speak`銆�
+4. 涓嶅緱鏂板 WS/MQTT/MCP銆�
+5. 涓嶅緱鏀瑰凡鎵ц Liquibase銆�
+6. 涓嶅緱瀵� SD 璺緞鍋� SequenceMatcher銆�
+7. 鎸囨爣瀛楁绂佹 `passed`/`good`/`bad`/`grade`銆�
+8. 鍏堝洖褰掑璇�/Chat锛屽啀娴嬨�岄棴鍢寸瓑鍒颁笅涓�鍙ャ�嶃��
+
+---
+
+## 13. Data Flow
+
+```text
+completed(duration_ms)
+ 寮�绐� T = clamp(ratio)*duration_ms
+ 绐楀唴 ASR? 鈫� upsert asr + 寮傛 report + 寮傛 assess
+ T 鍒� 鈫� 涓嬩竴 clip锛堟棤璁轰笂闈㈡垚鍔熶笌鍚︼級
+
+assess锛堝彲鏅氫簬涓嬩竴 clip锛�
+ expected_text from 01.txt 濂戠害
+ metrics=[{method, value}] 鈫� save/PUT
+```
+
+---
+
+## 14. State Machine
+
+```text
+璺熻鍙ワ紙Server锛夛細
+ play 鈫� (completed) 鈫� recording_window --timer--> next play | finish
+ 鈹斺攢鈹� ASR/report/assess 鏃佽矾
+
+Instance锛歳unning / completed / cancelled锛堜笉鍔� graded锛�
+```
+
+---
+
+## 15. 椋庨櫓
+
+| 椋庨櫓 | 缂撹В |
+|------|------|
+| 鎶婅崏绋裤�岃閿欓噸鎾�嶅仛鍥炲幓 | 楠屾敹鏄庣‘绂佹锛沺ytest 璇撮敊涔� index++ |
+| 绌� PUT 鍐叉帀鏅氬埌鐨� metrics | 搂7.2 鍚堝苟 |
+| 鍚獥杩囬暱浣撻獙宸� | ratio 1.2鈥�1.5锛涗互鍚庢寜姘村钩璋冿紝涓嶉潬鍒ら缂╃煭 |
+| 鐢ㄦ埛浠ヤ负璁惧鍦ㄥ垽瀵归敊 | 涓嶈 TTS 鍒嗘暟 |
+
+---
+
+## 16. 瀹屾垚鍚�
+
+```text
+Review 鈫� Architecture V5锛堣窡璇伙細鏃堕挓 + 鍚庤瘎浼帮級
+ 鈫� 4b锛氬�掕儗/鎺ラ緳/鍙ｇ畻 = inline 璇勪及 + 鍙婃椂 speak锛涙柦宸ュ崟 phase-4b-plan.md
+```
+
+绂佹鏈樁娈点�岄『渚裤�嶅彛绠楀嚭棰樸�佸�掕儗鍙婃椂鍙嶉銆丼cheduler銆佺鎴枫�丼D 鎼銆乣type=task`銆�
diff --git a/phase-4-test.md b/phase-4-test.md
new file mode 100644
index 0000000..da462ae
--- /dev/null
+++ b/phase-4-test.md
@@ -0,0 +1,149 @@
+# Phase 4 娴嬭瘯
+
+> 瀵瑰簲鏂藉伐鍗曪細`phase-4-plan.md`
+> 浠ｇ爜浜嬪疄锛歚phase-3-review.md`
+> 璇煶鍥炲綊锛歚phase-3-test.md` 搂1 + `phase-2-voice-test.md` 搂0 / 搂5
+> 鍥轰欢 **涓嶅繀閲嶇儳**
+
+鏈樁娈甸獙鏀讹細**璺熻鏄� MP3 搴忓垪 + 瀹氭椂褰曢煶**锛涘垎鏋愬け璐ヤ笉褰卞搷涓嬩竴鍙ワ紱鍚庤瘎浼板彧鏈夊畾閲� metrics锛屾病鏈夊強鏍肩粨璁恒��
+涓嶈娴嬪綋鍦恒�岃閿欓噸鎾�嶃�佸�掕儗/鍙ｇ畻浜у搧銆侀煶绱犮�佺鎴枫��
+
+鐩稿 Phase 2 鍙ｄ护琛ㄧ殑鍒绘剰鍙樺寲锛氬惉绐楀唴璇村畬 **涓嶄細椹笂** 涓嬩竴鍙ワ紝瑕佺瓑鍒� `listen_ratio 脳 璇ュ彞 duration_ms`銆�
+
+---
+
+## 0. 寮�濮嬪墠
+
+1. Liquibase 鏂� changeset锛歚expected_text` / `metrics` / `assessed_at`銆�**娌℃湁** `passed` 鍒椼�傛湭鏀� `202608311800.sql`銆�
+2. 闊抽濂戠害锛堣嚦灏� unit01 涓ゅ彞锛夛細
+
+```text
+/english_learning/shadowing/lessons/unit01/01.mp3
+/english_learning/shadowing/lessons/unit01/01.txt
+/english_learning/shadowing/lessons/unit01/02.mp3
+/english_learning/shadowing/lessons/unit01/02.txt
+```
+
+浜戠 `transcripts` 鎴� `sentences[]` 涓庝笂杩� txt 姝ｆ枃涓�鑷达紙鐑矾寰勪笉璇诲崱涓� txt 涔熷彲浠ユ祴璇勪及锛屼絾 **stem 蹇呴』鑳藉涓�**锛夈��
+3. `listen_ratio` 鍦� 1.2鈥�1.5銆�
+4. pytest锛�
+
+```text
+cd xiaozhi-esp32-server/main/xiaozhi-server
+python -m pytest tests/test_app_runtime.py tests/test_assessment.py -q
+```
+
+鍑嗗 server secret銆乵ac銆侴ET锛歚/xiaozhi/task/instance/{id}`銆�
+
+---
+
+## 1. pytest锛堝繀椤诲厛杩囷級
+
+| 鐢ㄤ緥 | 閫氳繃锛� |
+|------|--------|
+| `resolve_expected_text`锛歜asename `01.mp3` 鈫� txt 姝ｆ枃锛�**璺緞瀛楃涓蹭笉寰楀綋鍘熸枃** | |
+| `text_similarity_v0` 杩斿洖 `{method, value}`锛�**瀵硅薄涓嶅惈** `passed`/`grade` | |
+| 璇勪及缂� asr 鎴栫己 expected 鈫� `metrics=[]`锛屼笉鎶� | |
+| 妯℃嫙鍚獥锛氭敞鍏� ASR **涓�**瑙﹀彂 play next锛涘彧鏈� timer 鍥炶皟鎵� index+1 | |
+| timer 鍥炶皟鍦� ASR 澶辫触銆乤ssess 鎶涢敊銆乽psert 鎶涢敊鍚庝粛 index+1 | |
+| `listen_ratio` 0.5 鈫� 澶圭揣鍒� 1.2锛�3.0 鈫� 澶圭揣鍒� 1.5 | |
+| Phase 3锛歶psert 璋冨害澶辫触浠嶈兘 save | |
+
+---
+
+## 2. 璇煶锛氬簭鍒楁椂閽燂紙鏍稿績锛�
+
+浠� Chat 璇� **寮�濮嬭窡璇�**銆傝涓� 01 鎾畬鍒� 02 鍝嶈捣鐨勯棿闅旓紝搴斿ぇ绾︽槸 `ratio 脳 01 鐨勬椂闀縛锛堜笉瑕佺煭鍒般�屼竴璇村畬灏卞垏銆嶏級銆�
+
+| 妫�鏌� | 閫氳繃锛� |
+|------|--------|
+| 01 鎾畬鍚庣珛鍒昏繘鍏ュ彲璺熻锛堜笉瑕佸厛璁茬瑧璇濓級 | |
+| 鍚獥鍐呰鐪熻窡璇伙細02 **涓嶄細绔嬪埢**鍝嶏紝瑕佺瓑鍒版椂閽� | |
+| 鍚獥鍐呰銆屼粖澶╁ぉ姘旀�庝箞鏍枫�嶆垨涓嶈璇濓細鏃堕挓鍒颁簡浠嶆挱 **02**锛�**涓�**鍐嶆挱 01 | |
+| 璇� **閫�鍑鸿窡璇�** 鍙珛鍒诲洖 Chat | |
+| 鏁存缁撴潫鍚庢嫑鍛煎洖 Chat锛�**璁蹭釜绗戣瘽**鍙敤 | |
+
+---
+
+## 3. 澶辫触涓嶅緱鎸℃挱鏀撅紙蹇呴』锛�
+
+| 鍦烘櫙 | 閫氳繃锛� |
+|------|--------|
+| 鍋� manage_api锛岃窡璇昏嚦灏戜袱鍙ワ細02 浠嶄細鍦ㄦ椂閽熷悗鍑虹幇 | |
+| 鏃� 01.txt / 鏃� transcripts锛氭挱鏀句袱鍙ヤ粛瀹屾垚锛沵etrics 鍙┖ | |
+| 鍏ㄧ▼闂槾锛氫袱鍙ヤ粛鎸夋椂閽熸挱瀹岋紙turns 鐨� asr 鍙┖锛� | |
+
+鎭㈠ manage_api銆�
+
+---
+
+## 4. 鍚庤瘎浼板璐︼紙鍏佽鏅氬嚑绉掞級
+
+璺熻鑷冲皯涓�鍙ワ紙鏈夎璇濓級锛岀瓑浼氳瘽缁撴潫鎴栧啀绛夊嚑绉掞紝GET `{id}`銆�
+
+| 妫�鏌� | 閫氳繃锛� |
+|------|--------|
+| `prompt_text` 浠嶄互 `.mp3` 缁撳熬 | |
+| `expected_text` 绛変簬瀵瑰簲 txt 姝ｆ枃锛堜笉鏄矾寰勶級 | |
+| 鑻ユ湁 asr锛歚metrics` 鍚� `method=text_similarity_v0` 涓� `value` 鍦� 0鈥�1 | |
+| JSON 涓� GET **娌℃湁** `passed` / `grade` / 銆屾纭�嶅瓧娈� | |
+| 鍒楄〃鎺ュ彛浠嶄笉杩斿洖鏁翠唤 turns | |
+
+璇撮敊閭ｅ彞锛歚value` 鍙互寰堜綆鈥斺�旇繖鏄畾閲忕粨鏋滐紝**涓嶈**鍥犳鏈熸湜璁惧閲嶆挱銆�
+
+---
+
+## 5. Agent 鈮� App锛涘璇�
+
+| 妫�鏌� | 閫氳繃锛� |
+|------|--------|
+| Chat 绗戣瘽鍙繘 chat_history | |
+| 澶嶈锛氭棤 metrics 瑕佹眰锛涚粨鏉熶粛鍙嫑鍛间竴娆� | |
+| 澶嶈榛樿浼氬甫 `speed`锛堢渷鐣ュ垯涓烘寜澶╂暟绠楀嚭鐨勫�硷紝绗竴澶╃害 0.8锛� | |
+| `repeat=5` 鏃舵瘡涓�閬嶅�嶉�熺害 0.8鈫�0.9鈫�1.0鈫�1.1鈫�1.2锛�**涓嶄細**鍗″湪 0.75/1.0 妗� | |
+| 鏄惧紡鎸囧畾 `speed=1.05` 鏃朵簲閬嶉兘鐢� 1.05 | |
+
+---
+
+## 6. 鍥轰欢
+
+鏈樁娈佃窡璇昏瘎浼颁粛涓嶆敼鍥轰欢鍗忚銆�**鍏佽鐨勮ˉ鍏咃細** `play_audio.speed` 浠庣鏁ｆ。鏀逛负 `[0.5, 2.0]` 澶圭揣锛堥噸閲囨牱鏈潵灏辨敮鎸佷换鎰忓�嶇巼锛夈��
+
+| 妫�鏌� | 閫氳繃锛� |
+|------|--------|
+| 鏃� JUDGE / 鏂� MQTT action / 鏂� MCP 宸ュ叿鍚� | |
+| `self.mp3.play` 浠嶆槸 `name/speed/repeat/power_off_after`锛泂peed 鍙紶 0.8 / 1.05 | |
+| 璇撮敊鍚庢棩蹇楅噷 **娌℃湁** 瀵瑰悓涓�璺緞鐨勭珛鍗崇浜屾 play锛堝彧鏈夋椂閽熷悗鐨� 02锛� | |
+
+---
+
+## 7. 涓嶈娴嬩粈涔�
+
+- 璇撮敊閲嶆挱鍚屼竴鍙ワ紙閭ｆ槸宸插簾寮冭崏妗堬級
+- 銆�>82% 绠楀ソ銆嶇殑浜у搧缁撹
+- 鍊掕儗 / 鍙ｇ畻 / 鎺ラ緳 / 蹇冪悊璇婃柇
+- 瀹堕暱 UI銆佺鎴�
+- 鎶婂垏鎹㈣瘝鍐欏叆鍥轰欢
+
+---
+
+## 8. 澶辫触鏃跺厛鐪嬪摢
+
+| 鐜拌薄 | 澶氬崐鏄� |
+|------|--------|
+| 涓�璇村畬灏卞垏涓嬩竴鍙� | ASR 浠嶅湪 `handle_shadowing_turn` 閲� play next锛堟湭瑙ｈ�︼級 |
+| 璇撮敊閲嶆挱 01 | 鎶� S8 retry 鍋氬洖鍘讳簡 |
+| 鍋� API 鍚庡崱鍦ㄧ涓�鍙� | upsert/assess 鍙堝彉鎴愬悓姝� await |
+| metrics 鏈� passed | DTO 娌℃寜璁″垝 |
+| expected 鏄矾寰� | 鐢� prompt_text 褰撲簡鍘熸枃 |
+| 鏃堕挓鏄庢樉鐭簬 1.2脳 | 浠嶇敤鍥哄畾 800ms 鑰屽拷鐣� duration_ms |
+
+---
+
+## 鍙ｄ护锛堜笌 Phase 2 鐩稿悓锛�
+
+| 鎰忓浘 | 璇� |
+|------|----|
+| 璺熻 | 寮�濮嬭窡璇� |
+| 澶嶈 | 澶嶈鏈� |
+| 鍥炶亰 | 閫�鍑鸿窡璇� / 鎴戜滑鏉ヨ亰澶╁惂 |
diff --git a/roadmap.md b/roadmap.md
new file mode 100644
index 0000000..df9698d
--- /dev/null
+++ b/roadmap.md
@@ -0,0 +1,603 @@
+# Roadmap锛圕urrent Architecture V4 + Capability Spec v1.0锛�
+
+> Date: 2026-09-02
+> 杈撳叆锛歚phase-1-review.md` 路 `phase-2-review.md` 路 `phase-3-review.md` 路 `Xiaozhi Device Capability Specification v1.0.md` 路 璁惧榛樿璇█ / 璇█瀵硅瘎浼� 路 **浜у搧浠ｉ檯 / 绔簯鐗堟湰鍖归厤**
+> 绮掑害锛氭灦鏋勪笌椤圭洰绠＄悊銆備笉鏄� Coding Plan銆�
+> Phase 3 宸插疄鏂斤細`phase-3-review.md`銆備笅涓�缂栫爜闃舵锛歚phase-4-plan.md`锛堥獙鏀� `phase-4-test.md`锛夈��
+> 璇█锛氳鏈枃 **Phase 6 / 6b / 7**锛�**涓嶈**濉炶繘 Phase 4銆�
+> 鐗堟湰涓庡吋瀹癸細瑙佹湰鏂� **Phase V**锛涙憚鍍忓ご SKU锛堝皬鏅� AI 2锛変箣鍓嶅繀椤昏惤鍦版彙鎵嬪瓧娈典笌鍏煎鐭╅樀锛�**涓嶈**绛� Phase 8銆�
+
+鍘熷垯锛氭瘡瀹屾垚涓�涓� Phase 鈫� Completion Review 鈫� Architecture 鏇存柊 鈫� Roadmap Review 鈫� 涓嬩竴浠� Phase Plan銆�
+**绂佹**鎶婃湰鏂囧綋鎴愰攣姝诲埌 Phase 9 鐨勬柦宸ュ浘銆傚悗闈� Phase 鍙繚璇佺洰鏍囥�佽竟鐣屼笌 **瀵瑰簲鐨� Spec 鑳藉姏 ID**銆�
+
+---
+
+## 0. 鍐荤粨涔嬪悗鎬庝箞鍒� Phase
+
+Capability Spec 鎶婄郴缁熷垎鎴愪笁灞傦細
+
+```text
+Application 浠ュ悗鎵嶆棤闄愭墿灞曪紙鏁欒偛 / 鐩婃櫤 / 闄即 / 鍔╂墜锛�
+Task Runtime 灏藉揩绋冲畾锛屾潈濞佸湪 Server
+Device Capability 璇箟宸插喕缁擄細鎾� / 鍚� / VAD / speak 鍑哄０ / wait / loop / 鎸囩ず / 鎶㈠崰
+```
+
+鍥犳璺嚎鍥句笉鍐嶆寜銆屽啀鍋� 10 涓� App銆嶆帓鍒楋紝鑰屾寜锛�
+
+```text
+1. 鎶婂凡澹版槑銆佸浐浠跺皻鏈叿澶囩殑鍘熷瓙鑳藉姏琛ラ綈锛堝彧鍏佽鍥犳 OTA锛�
+2. 鎶� Task Runtime 鍋氭垚鍙綊妗ｃ�佸彲鏌ヨ
+3. 鎶婅瘎浼� / 璋冨害 / 璧勬簮鏀惧埌 Server
+4. 鐢ㄥ�掕儗 / 鍙ｇ畻 / 鎺ラ緳鍙嶈瘉锛氭柊 App 闆跺浐浠跺閲�
+```
+
+鍘� V1.1銆孭hase 2 = type=task + Java 琛� + 璺熻 + 璇勫垎銆嶄粛鐒舵媶寮�銆係pec 棰濆瑕佹眰锛�
+
+- 杩愯緭鍙彉锛圡CP / MQTT / detect 鍓嶇紑锛夛紝**璇箟闆嗗悎涓嶅啀涓烘柊 App 鍔犲懡浠ょ绫�**
+- `loop`锛堝悓涓�闊抽 N 娆★級鍦ㄨ澶囷紱`branch(绛斿浜嗗悧)` 鍙湪 Server锛屼笖 **浠� Task 澹版槑涓烘祦鍐呰瘎浼版椂**鎵嶈蛋杩涗笟鍔★紙鍊掕儗/鎺ラ緳锛夛紱璺熻榛樿涓嶈蛋
+- 绂荤嚎鍙繚璇佸彲绂荤嚎 Skill锛堟挱鏀撅級锛涚炕璇�/璇勪及/鎺ラ緳蹇呴』鍦ㄧ嚎
+- 璇勪及鏃舵満鏄� Task 灞炴�э紙鍚庤瘎浼� vs 娴佸唴锛夛紝涓嶆槸鏂颁紶鎰熷櫒
+- 绂荤嚎鍙繚璇佸彲绂荤嚎 Skill锛堟挱鏀撅級锛涚炕璇�/璇勪及/鎺ラ緳蹇呴』鍦ㄧ嚎
+- SD 鏀捐祫婧愶紱`apps.json` 涓嶆槸绂荤嚎涓氬姟寮曟搸
+- `record_clip` v1 = 鍚祦 + Server 褰掓。锛涙湰鍦� WAV 涓婁紶鏄� v1.1
+
+| 缂栧彿 | 涓�鍙ヨ瘽 | Spec 涓昏楠岃瘉 / 浜や粯 |
+|------|--------|----------------------|
+| Phase 1 | 钖� App Runtime | 宸插畬鎴愶細D15 璁惧 loop銆丏4 Mode 绛栫暐闆忓舰銆佺炕璇戣蛋 Server |
+| Freeze | Capability Spec v1.0 | 鏂囨。鍐荤粨锛涗笉缂栫爜 |
+| Phase 2 | 鐢ㄥ璇�+璺熻閽夋浜嬩欢涓庨噰闆� | **宸插畬鎴愶紙瑙� phase-2-review.md锛�**锛欴1銆丏6銆丏7 v1銆丏8 v1 鏂囨湰锛涘彟鎻愬墠 D13 鐩綍 `/{app}/{task}/lessons` |
+| Phase 3 | TaskInstance 鎸佷箙鍖� | **宸插畬鎴愶紙瑙� phase-3-review.md锛�**锛氫笁琛� + PUT/GET锛涘浐浠堕浂 diff |
+| Phase 4 | 璺熻锛氭椂閽熸杩� + **鍚庤瘎浼�** | 鍚獥缁撴潫鍗充笅涓�鍙ワ紱metrics 涓嶆寚鎸ュ枃鍙� |
+| Phase V | 浜у搧浠ｉ檯 + 绔簯鍏煎 | 灏忔櫤 AI 1 / 1-1 / 2锛沨ello 鎶ヤ唬闄呬笌 cap_set锛汼erver 鎸夌煩闃甸檷绾ф垨 OTA |
+| Phase 4b | 鍊掕儗 / 鍙ｇ畻 / 鎺ラ緳 | **娴佸唴璇勪及 + 鍙婃椂 speak 鍙嶉**锛涘弽璇佸浐浠跺喕缁� |
+| Phase 5 | Server Scheduler | D14 TASK_START 鐨勪簯绔潈濞� |
+| Phase 6 | App / 璧勬簮 + **鍦ㄧ嚎 Locale** | D13 fetch锛沺hrase pack锛涚炕璇�/鍙岃鐨� language_pair |
+| Phase 6b | 绂荤嚎 Locale锛堝彲閫� OTA锛� | MultiNet/鍞ら啋璇嶈〃鎸� locale锛�**浠� ESP-SR 宸叉湁妯″瀷鐨勮绉嶏紙鍏� zh/en锛�** |
+| Phase 7 | 瀹堕暱浠诲姟绠＄悊 | 鍚細璁惧榛樿璇█銆佽瑷�瀵归�夋嫨锛涘彧娑堣垂宸叉湁 Command |
+| Phase 8 | 寮�鍙戣�� App 骞冲彴 | Manifest 鍙兘璋冨凡鍐荤粨 Skill |
+| Phase 9 | Subscription / Marketplace | 璐﹀彿锛屼笉鏀归煶棰� |
+
+---
+
+## 1. Phase 渚濊禆鍥�
+
+```text
+Phase 1 钖� App Runtime 鉁�
+ 鈫�
+Capability Spec v1.0 鉁� 鏂囨。
+ 鈫�
+Phase 2 閽夋 D1 浜嬩欢 + D6/D7/D8v1 閲囬泦 鉁� 瑙� phase-2-review.md
+ 鈫�
+Phase 3 TaskInstance 鎸佷箙鍖� 鉁� 瑙� phase-3-review.md
+ 鈹溾攢鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹攢鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�
+ 鈫� 鈫� 鈫�
+Phase 4 璺熻鍚庤瘎浼� Phase 5 璋冨害 Phase V 浠ｉ檯/鍏煎
+涓嬩竴缂栫爜 鍙悗鍋� 鍙笌 4 骞惰锛涙憚鍍忓ご SKU 鍓嶅繀椤诲畬鎴�
+ 鈹�
+ 鈫�
+Phase 4b 鍊掕儗/鍙ｇ畻/鎺ラ緳锛堟祦鍐呭強鏃跺弽棣堬紱鍙笌 5 鍚庢湡閲嶅彔锛�
+ 鈹斺攢鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹攢鈹�鈹�鈹�鈹�鈹�鈹�鈹�
+ 鈫�
+ Phase 6 璧勬簮 + 鍦ㄧ嚎 Locale锛堝彲涓� 4b/5 鍚庢湡閲嶅彔锛�
+ 鈹�
+ 鈹溾攢 Phase 6b 绂荤嚎 Locale锛坺h/en 璇嶈〃 OTA锛屼笉鎸� 7锛�
+ 鈫�
+ Phase 7 瀹堕暱 UI锛堣 device.locale / 璇█瀵癸級
+ 鈫�
+ Phase 8 寮�鍙戣�呭钩鍙�
+ 鈫�
+ Phase 9 璁㈤槄
+```
+
+**蹇呴』涓茶锛�** 1 鈫� Freeze 鈫� 2 鈫� 3 鈫� 4銆傝窡璇诲悗璇勪及鎸� Instance銆侾hase 4b **鍙﹀**瑕佹祦鍐呰瘎浼帮細鍚獥缁撴潫鍚庡厛 Server 绠楀畬鍐� speak锛屼笉鑳藉璺熻鐨勩�屽垎鏋愭案涓嶆尅鍠囧彮銆嶃��
+**Phase V 蹇呴』鍦ㄦ憚鍍忓ご SKU / 鐮村潖鎬� API 涔嬪墠瀹屾垚**锛涗笉鎸� Phase 4 璺熻鏂藉伐銆�
+
+**鍙苟琛岋細** Phase 5銆丳hase V 涓� Phase 4 鏃犳暟鎹緷璧栥�備骇鍝侀『搴忎粛鍏堝仛 4锛堣窡璇绘椂閽燂級銆�
+**Phase 4b** 渚濊禆 Phase 4 鐨勫惉绐�/Instance 鍘熻锛屼笉渚濊禆 Phase 6銆傜姝负姝ゆ敼 C++銆傝瘎浼版椂鏈鸿涓嬫枃銆岃瘎浼版椂鏈恒�嶈〃銆�
+
+**Phase 6锛�** 鐩綍鏍戜笌 `ensure_app` 宸插湪 Phase 2 鎻愬墠钀藉湴銆傛湰闃舵鍋氭竻鍗� / DOWNLOAD / `resource_id`锛屽苟鍋� **鍦ㄧ嚎 Locale**锛坧hrase pack + 璇█瀵癸級銆備笉瑕侀噸鍋� mkdir锛屼篃涓嶈鎶婅矾寰勫啀鎼竴娆°��
+
+**Phase 6b锛�** 浠呭綋闇�瑕併�屾病缃戜篃鐢ㄥ痉璇�/鑻辫鍠婂垏鎹€�嶆椂鎵� OTA 绂荤嚎璇嶈〃銆傚痉璇瓑 **娌℃湁** 瀹樻柟 MultiNet 鐨勮绉嶏紝绂荤嚎鍏ュ彛淇濇寔 zh 鎴� en锛屽湪绾胯彍鍗曚粛鍙敤璇� locale銆�
+
+**鏄庣‘鍚庣疆銆佷笉鎸″喕缁撳绉帮細**
+
+| Spec 鑳藉姏 | 鏈�鏃� Phase |
+|-----------|------------|
+| D11 `wait` 涓�绛� Skill | 4b 鍓嶅彲浠嶇敤 Server 寤舵椂锛涢渶瑕併�屾挱瀹屽仠 800ms 鍐嶅惉銆嶄笖涓嶆兂浜戠鐫℃椂鍐嶄笅鍥轰欢 |
+| D14 TASK_PAUSE/RESUME | 5 鎴� 7 瀹堕暱鎺у埗闇�瑕佹椂 |
+| D8 v1.1 鏈湴鏂囦欢涓婁紶 | 浼氳绾 / 寰嬪笀璇㈤棶浜у搧鍖栨椂 |
+| D9 闀挎寜閫�鍑� Task | 鍙笌 2 鐨勬寜閽幇鐘跺苟瀛橈紱姝ｅ紡缁戝畾鏀� 7 鍓� |
+| D10 `indicate` 閫昏緫鎬� | 鏈夌┖鍐嶅仛锛涚姝㈠杩� DeviceState 杩佺Щ琛� |
+| 鏂� WS `type=command` | 浠呭綋 detect 鍓嶇紑涓嶅琛ㄨ揪姝ヨ繘锛�**涓嶅姞鏂拌涔�** |
+| 鍦ㄧ嚎 Locale / 璇█瀵� | Phase 6锛坧hrase pack + pair锛夛紱鎺у埗鍙� Phase 7 |
+| 鎽勫儚澶� / 鏂颁紶鎰熷櫒 Skill | **Phase V 涔嬪悗**锛涙柊 SKU = 灏忔櫤 AI 2锛岀姝㈡贩杩� 1 鐨勫浐浠跺寘 |
+
+---
+
+## 2. 鍚� Phase锛圧oadmap 绾э級
+
+### Phase 2 鈥� 鍐荤粨闈㈤獙璇侊細閲囬泦 + 瀹屾垚浜嬩欢 鉁�
+
+1. **Objective**
+ 涓嶆槸鍋氫竴涓柊 App銆傜敤 **宸叉湁** 澶嶈鏈� + 璺熻 Demo锛屾妸 Spec 閲屽皻鏈惤鍦扮殑涓夊潡閽夎繘 Runtime锛�
+ `audio.completed/stopped`锛圖1锛夈�侀潪 Chat 鐨� listen/鍙ヨ竟鐣岋紙D6/D7锛夈�佽窡璇诲綊妗� v1锛圖8锛夈��
+ **瀹屾垚浜嬪疄锛�** 瑙� `phase-2-review.md`銆傝窡璇绘敼涓� SD 涓�鍙ヤ竴 MP3锛涜闃� App 鍙� mkdir 涓嶇儳褰曘��
+
+2. **Why Now**
+ Spec 搂20锛氳繖涓夐」涓嶈ˉ榻愬氨涓嶈兘瀹ｇО Device 宸插喕缁撱�傝窡璇� Mode 宸插湪绾匡紝缁х画鍫嗗�掕儗/鍙ｇ畻鍙細鍦ㄦ病鏈変簨浠剁殑搴曞骇涓婂鍒� Demo銆�
+
+3. **Dependencies**
+ Phase 1 鎵ц鍣ㄣ�乣SHADOWING`/`PLAY_AUDIO` 鍒嗘祦銆丱pus 鍚祦銆乣report`銆佽澶� `repeat` loop锛圖15锛屾湰闃舵 **涓嶆敼**锛夈��
+
+4. **Scope**
+ 閫昏緫浜嬩欢锛堣繍杈擄細`[skill_event]`锛屽搴� Spec `AUDIO_COMPLETED/STOPPED`锛夛紱TaskSession 鏂囦欢锛涜窡璇� turn = prompt + asr + 鍙�� WAV锛沺ytest銆侻ode 鍚嶄唬鐮侀噷鍙粛鍙� `SHADOWING`锛屾枃妗ｅ榻� Spec `FOLLOW_READ`銆�
+
+5. **Out of Scope**
+ 璇勫垎锛圫4锛夈�丣ava 琛ㄣ�乣type=task`銆佸�掕儗/鍙ｇ畻/鎺ラ緳浜у搧銆丼D 鑼冭蹇呬慨銆乣wait` Skill銆乣TASK_PAUSE`銆乣record_clip` 鏈湴鏂囦欢銆乤pps.json 涓嬪彂銆佹柊 DeviceState銆丮ultiNet 鍔犱笟鍔″彛浠ゃ��
+
+6. **Client**
+ 澶嶈缁撴潫涓婃姤瀹屾垚/鍋滄浜嬩欢锛涗笉鏀� Opus锛涗笉鎶婅窡璇诲彞瀛愯〃鍐欒繘鍥轰欢銆�
+
+7. **Server**
+ Session + 璺熻璁拌疆娆� + 鍚冧簨浠跺悗鍥� chat锛汚SR/LLM/缈昏瘧浠嶅湪 Server銆�
+
+8. **Protocol**
+ **涓嶅鍔犲懡浠ょ绫汇��** 鍙妸宸插０鏄庣殑 Event 鐢ㄧ幇鏈� `listen/detect` 鏂囨湰杩愯繃鍘汇�傝涔夎 Spec 搂4鈥�5銆�
+
+9. **Database**
+ 鏃犳柊琛ㄣ�傚綊妗ｈ蛋鐜版湁 `ai_agent_chat_audio`锛圖8 v1锛夈��
+
+10. **API**
+ 鏃犳柊 REST銆�
+
+11. **Data Flow**
+ 瑙� `phase-2-plan.md`銆傝窡璇讳笅涓�姝ョ敱 **Server 姝ヨ繘**锛圖16锛夛紝涓嶆槸璁惧 loop銆�
+
+12. **State Machine**
+ DeviceState 涓嶅姩銆侷nteractionMode锛歚PLAY_AUDIO` / `SHADOWING(FOLLOW_READ)` / 鍥� `CHAT`銆�
+
+13. **Acceptance Criteria**
+ Spec 搂22 鍥涙潯绾緥 + 鏈枃 Phase 2 楠屾敹锛氬畬鎴愪簨浠跺崟娆″洖鑱娿�佽窡璇� JSON 杞瀹屾暣銆丆hat/绂荤嚎澶嶈/缈昏瘧涓嶅洖褰掋��
+
+14. **Risks**
+ 鍥炲０褰撹窡璇伙紱浜嬩欢涓� `[app_task]chat` 鍙屾嫑鍛笺��
+
+15. **Regression Risks**
+ 澶嶈鍙屾挱銆佺绾裤�屽垏鎹㈡ā寮忋�嶈繛浜戙�丆hat barge_in銆�
+
+璇︾粏鏂藉伐锛歚phase-2-plan.md`銆�
+
+---
+
+### Phase 3 鈥� TaskInstance Persistence 鉁�
+
+1. **Objective**
+ 涓眰 Task Runtime 鍙煡璇細Session JSON 鈫� `ai_task_instance` / `ai_task_step` / `ai_task_event`銆備笉鏂板 Device Capability銆�
+ **瀹屾垚浜嬪疄锛�** 瑙� `phase-3-review.md`銆侴ET 璧� server secret锛堥潪璁″垝涓殑 oauth2锛夈�傛湭寤虹┖ score 鍒椼�傝窡璇� `prompt_text` 浠嶆槸 SD 璺緞銆�
+
+2. **Why Now**
+ Phase 2 瀛楁宸茬ǔ瀹氾紝寤鸿〃涓嶄細绌恒�傝瘎浼板拰璋冨害闇�瑕佷富閿��
+
+3. **Dependencies**
+ Phase 2 `TaskSession` schema锛汥8 v1 report 绠￠亾銆傚浐浠朵繚鎸� Phase 2 bin銆�
+
+4. **Scope**
+ manager-api 鏈�灏� DDL锛沗TaskSession.save()` upsert锛涘彧璇� GET銆侫gent 鈮� App銆侸SON 鏂囦欢鍙屽啓淇濈暀銆�
+
+5. **Out of Scope**
+ 璇勫垎绠楁硶銆佸璞″瓨鍌ㄣ�佸闀� UI銆佹柊 WS type銆佽澶囩紦瀛� instance_id銆佹敼 SD 甯冨眬銆�
+
+6鈥�10.
+ Client **闆舵敼鍔�**銆係erver Python 涓婃姤 + Java CRUD銆傚崗璁笉鍔犳柊鍛戒护绉嶇被銆傞煶棰戞寕 step 灏藉姏鑰屼负銆�
+
+13. **Acceptance**
+ 涓�娆¤窡璇诲彲鍦� API 鏌ュ埌 steps + 鏂囨湰 + 鍙�� audio_id锛沵anage_api 鎸備簡璇煶浠嶅彲鐢ㄣ��
+
+14鈥�15.
+ 涓� `chat_history` 鍙屽啓浣嗚〃鍒嗗紑锛涙棫鍥轰欢鏃� instance_id 蹇呴』鍏煎锛坕d 鍙湪浜戠锛夈��
+
+璇︾粏鏂藉伐锛歚phase-3-plan.md`銆傞獙鏀讹細`phase-3-test.md`銆�
+
+---
+
+### Phase 4 鈥� 璺熻锛氭椂閽熸杩� + 鍚庤瘎浼帮紙瀹氶噺锛�
+
+1. **Objective**
+ **浠呰窡璇昏繖涓�绫�**锛氶瀹� MP3 搴忓垪 + 瀹氭椂褰曢煶銆傚垎鏋愬叏閮ㄥ悗缃�佸畾閲忋�佸厑璁稿け璐ャ�傝澶囦笉瀹ｈ瀵归敊銆�
+ **涓嶈**鎶婂�掕儗銆佹帴榫欏杩涙湰闃舵鐨勩�屽垎鏋愭案涓嶆尅鍠囧彮銆嶃��
+
+2. **Why Now**
+ 璺熻/鎵撳崱褰曢煶鐨勪环鍊煎湪褰掓。涓庝簨鍚庣湅鎸囨爣銆傝嫢鎶婁簯绔瘮瀵圭粦鍦ㄥ彞闂达紝鍠囧彮浼氬崱銆�
+
+3. **Dependencies**
+ Phase 3 Instance锛汸hase 2 `duration_ms` 鍚獥銆傚師鏂囨潵鑷� txt 濂戠害锛�**涓嶈兘**鐢� `prompt_text` 璺緞銆�
+
+4. **Scope**
+ ASR 涓� play next 瑙ｈ�︼紱`listen_ratio` 1.2鈥�1.5锛涘悗鍙� `metrics[{method,value}]`銆傛棤 `passed`銆佹棤 retry銆佹棤褰撳満 TTS 鍒嗘暟銆�
+
+5. **Out of Scope**
+ 鍊掕儗/鎺ラ緳鐨勫強鏃跺弽棣堬紙4b锛夛紱Spec S8 鍐嶆挱鍚屼竴鍙ワ紱鍙婃牸绾夸骇鍝佸寲锛涢煶绱狅紱绉熸埛锛涘彛浠よ繘鍥轰欢銆�
+
+6. **Client**
+ 闆舵敼鍔ㄣ��
+
+8. **Protocol**
+ 涓嬩竴鍙ュ彧鐢辨椂閽熻Е鍙戙�備笉鍔� `JUDGE`銆�
+
+13. **Acceptance**
+ 璇撮敊/涓嶈/API 鎸備簡浠嶆挱涓嬩竴鍙ャ�傝 `phase-4-test.md`銆�
+
+璇︾粏鏂藉伐锛歚phase-4-plan.md`銆�
+
+---
+
+### Phase V 鈥� 浜у搧浠ｉ檯 + 绔簯鐗堟湰鍖归厤
+
+> 鐜扮姸缁撹锛�2026-09-02 鏍稿浠ｇ爜锛夛細**鐜板湪娌℃湁銆屽皬鏅� AI 1 / 1-1 / 2銆嶈繖濂椾骇鍝佷唬闄咃紝涔熸病鏈夊鎴风涓� Server 鐨勫吋瀹圭煩闃点��**
+> 宸叉湁鐨勫彧鏄�岃兘涓嶈兘鍗� bin銆嶅拰銆屾彙鎵嬮噷寮�娌″紑 mcp/aec銆嶃�傛憚鍍忓ご SKU 鎴栫牬鍧忔�� API 涔嬪墠蹇呴』琛ラ綈銆�
+> 涓嶆尅 Phase 4 璺熻銆傛柦宸ュ崟浠ュ悗鍐� `phase-v-plan.md`銆�
+
+1. **Objective**
+ 鎶婁笁濂楃嫭绔嬬殑缂栧彿閽夋锛屽苟鍦ㄨ繛鎺ユ椂瀵逛笂锛�
+ **浜у搧浠ｉ檯**锛堝皬鏅� AI 1 / 1-1 / 2锛夆墵 **鍥轰欢 semver**锛圤TA 鍖咃級鈮� **鍗忚/鑳藉姏闆�**锛圫erver 鍐冲畾寮�鍝簺涓氬姟锛夈��
+ Server 瀵硅繃鏃ц澶囬檷绾ф垨寮曞 OTA锛涘杩囨柊璁惧鎷掔粷鏈煡鑳藉姏锛岃�屼笉鏄� silently 褰撹亰澶┿��
+
+2. **Why Now**
+ 涓嶅姞鎽勫儚澶翠篃鍙互鍑� 1-1锛堣窡璇�/Task Runtime 宸叉槸澶у姛鑳介潰锛夈�備笉鍔犱唬闄呭瓧娈碉紝OTA 鍙兘鎸夋澘鍨嬫瘮 `2.2.6` vs `1.0.0` 涓や釜瀵逛笉涓婄殑鏁板瓧锛孲erver 鏃犳硶鍒ゆ柇銆岃繖鍙版湁娌℃湁鐩告満 MCP銆佽兘涓嶈兘璺� cap.v1.1 鐨勮窡璇诲惉绐椼�嶃��
+
+3. **Dependencies**
+ 鐜扮綉宸叉湁锛歄TA 涓婃姤 `application.version` + `board.type`锛沇S hello `version`锛堝抚鏍煎紡 1/2/3锛�+ `features.mcp|aec`锛汳QTT `snlp/v1` + header `"1.0"`锛沵anager-api Liquibase 鏃ユ湡 changeset銆�
+ **涓嶄緷璧�** Phase 4 瀹屾垚銆傛憚鍍忓ご纭欢涓嶅湪鏈樁娈点��
+
+4. **缂栧彿妯″瀷锛堝喕缁撹涔夛紝瀹炵幇鍙悗鍐欙級**
+
+```text
+product.generation 1 / 2 / 3 纭欢鏃忋��1=鐜扮綉闊崇锛�2=鍔犳憚鍍忓ご锛堟垨鍏跺畠鏂颁紶鎰熷櫒锛�
+product.revision 0 / 1 / 2 鍚屼唬纭欢涓婄殑杞欢澶ф敼銆傚澶栵細灏忔櫤 AI 1銆佸皬鏅� AI 1-1銆佸皬鏅� AI 2
+firmware.semver 1.4.2 姣忎竴娆″彲 OTA 鐨� bin锛涘彧鍗囪繖涓暟瀛楀幓鍗囧寘
+capability.set cap.v1 / v1.1 涓� Spec 淇瀵归綈锛堟挱/鍚�/璺熻鍚獥/鈥︼級锛涙柊 App 涓嶅姞 cap 鍙�
+protocol.ws 1 宸叉湁 hello.version锛屽彧涓轰簩杩涘埗甯э紱绂佹鎷垮畠褰撲骇鍝佷唬闄�
+protocol.mqtt snlp/v1 topic 澶ф敼鎵嶅崌 v2锛沨eader.version 鍙崌鐮村潖鎬у瓧娈�
+server.release 2026.09.x 鎴� semver xiaozhi-server + manager-api 鍚屽彂
+api.rest /xiaozhi/v1/ 鐮村潖鎬� REST 鎵� /v2/锛涙棫璺緞淇濈暀鍒板吋瀹圭獥缁撴潫
+```
+
+瀵瑰绉板懠锛�
+
+| 瀵瑰 | generation | revision | 鍚箟 |
+|------|------------|----------|------|
+| 灏忔櫤 AI 1 | 1 | 0 | 鐜扮綉锛氭棤鎽勫儚澶达紝褰撳墠鑳藉姏闈� |
+| 灏忔櫤 AI 1-1 | 1 | 1 | **鍚屼竴鍧楃‖浠�**锛岃兘鍔涢泦鎴栦笟鍔￠潰澶ф敼锛堜緥濡� Task Runtime 鎴愪负榛樿锛� |
+| 灏忔櫤 AI 2 | 2 | 0 | **鏂扮‖浠�**锛堟憚鍍忓ご绛夛級锛涘浐浠跺寘涓� 1.x **鍒嗚建**锛岀姝㈢儳鍒板鏂规澘涓� |
+
+5. **鐜扮姸缂哄彛锛堝鐓т唬鐮侊紝涓嶈鍋囪宸叉湁锛�**
+
+| 灞� | 鐜板湪鏈変粈涔� | 缂轰粈涔� |
+|----|------------|--------|
+| 缁堢浜у搧鍙� | `system.h` 閲� `APP_FIRMWARE_VERSION "1.0.0"` 鍑犱箮涓� bump锛汣Make `PROJECT_VER "2.2.6"` 鏄笂娓镐粨搴撳彿锛沀ser-Agent 鏄� `BOARD_NAME/esp_app.version` | 娌℃湁 `product.generation/revision`锛�1 鍜� 1-1 鏃犳硶鍖哄垎 |
+| WS 鎻℃墜 | hello锛歚version=1` + `features.{mcp,aec}` + 闊抽鍙傛暟 | 涓嶆姤鍥轰欢銆佷笉鎶ヤ唬闄呫�佷笉鎶� cap_set锛汼erver 涓嶆嫆涓嶅尮閰� |
+| OTA | 鎸� **鏉垮瀷** 姣� semver锛屾柊鍒欑粰涓嬭浇 URL | 涓嶆寜浜у搧浠ｉ檯鍒嗚建锛涗笉妫�鏌� Server 鏈�浣� cap锛�1.0.0 vs 2.2.6 涓ゅ鏁板瓧骞跺瓨 |
+| MQTT snlp | topic `v1`銆乭eader `"1.0"` 鍐欐 | 鏃犲崗鍟嗭紱鏃犺兘鍔涗綅 |
+| Server | `config.yaml` `xiaozhi.version: 1`锛沵anager-api 闈犳棩鏈� SQL | 鏃� `server.release`锛汻EST 鏃犲吋瀹圭獥锛涙棤銆屾 Server 鏀寔鐨� generation 鑼冨洿銆� |
+| 涓氬姟鍖归厤 | MCP 宸ュ叿鍒楄〃杩愯鏃跺彂鐜� | 娌℃湁銆岃繖鍙� gen1 姘歌繙涓嶈 vision銆嶇殑鐭╅樀锛涙柊 API 榛樿鎵�鏈夋棫璁惧閮借兘鎵� |
+
+6. **Scope**
+ - 璁惧 OTA JSON + WS hello **澧炶ˉ**锛堜笉鏀规棫瀛楁鍚箟锛夛細`product_generation`銆乣product_revision`銆乣firmware`銆乣capability_set`銆�
+ - Server 缁存姢涓�寮� **compat 鐭╅樀**锛堝摢涓�妗� Server 鎺ュ彈鍝竴妗ｈ澶囷紱杩囨棫 鈫� 鍙紑鏃т笟鍔℃垨涓嬪彂 OTA锛涜繃鏂�/鏈煡 cap 鈫� 鎷掔粷鎴栧拷鐣ユ湭鐭� MCP锛夈��
+ - manager-api锛氬浐浠跺寘缁戝畾 `(board_type, generation)`锛岀姝� gen2 鍖呮帹缁� gen1銆�
+ - REST锛氱幇鏈� `/task/instance` 瑙嗕负 v1锛涚牬鍧忔�у彉鏇磋蛋鏂拌矾寰勬垨甯� `Accept-Version`锛屾棫瀹㈡埛绔湁鍏煎绐椼��
+ - 鏂囨。锛氫竴寮犺〃鍐欐竻銆屽皬鏅� AI 1 鏈�浣� Server銆嶃�孲erver x 鏈�浣庡浐浠躲�嶃��
+
+7. **Out of Scope**
+ 鎽勫儚澶撮┍鍔ㄣ�乿ision MCP 瀹炵幇銆佹妸 cap_set 褰� App 鍟嗗簵銆佷负姣忎釜 App 鍗� generation銆佹妸 `hello.version` 鏀规垚 2 鏉ヨ〃绀� 1-1銆�
+
+8. **Protocol**
+ **鍙姞瀛楁锛屼笉鍔犳柊 WS type銆�** 鏃у浐浠朵笉甯﹁繖浜涘瓧娈垫椂锛歋erver 瑙嗕负 **灏忔櫤 AI 1 / revision 0 / cap.v1**锛屼繚鎸佺幇缃戣兘杩炪��
+ 鍖归厤鍙戠敓鍦� hello 涓� OTA check锛屼笉鍦ㄦ瘡鍙� ASR銆�
+
+9. **Client / Server 鍖归厤瑙勫垯锛堜骇鍝佽涔夛級**
+
+```text
+杩炴帴鏃�
+ Device 鈫� generation + revision + firmware + cap_set
+ Server 鈫� 鏌ョ煩闃碉細
+ 鏈煡 generation 鈫� 鎷掔粷鎴栧彧寮� CHAT
+ cap_set < server.min 鈫� 寮曞 OTA锛汷TA 鍓嶅彧璺� cap 浜ら泦锛堜緥濡傛病鏈夎窡璇诲惉绐楀氨褰� PLAY_AUDIO锛�
+ cap_set > server.max 鈫� Server 蹇界暐鏈煡鑳藉姏锛屼笉宕╂簝
+ generation=1 鏃� camera 鈫� 姘镐笉涓嬪彂 vision 宸ュ叿
+ generation=2 鈫� 鍏佽 vision锛涘浐浠跺寘涓� 1 鍒嗚建
+```
+
+鍚屼竴纭欢鍑� 1-1锛氬彧鍗� `product.revision` + `capability_set` + `firmware.semver`锛�**涓�**鍗� generation銆�
+鏂颁紶鎰熷櫒锛氬崌 generation锛屾柊 board_type锛屾柊 OTA 鐩綍銆�
+
+13. **Acceptance**
+ - 鐜扮綉鏈敼瀛楁鐨勫浐浠朵粛鑳借繛褰撳墠 Server锛堥粯璁� 1.0 / cap.v1锛夈��
+ - 鎺у埗鍙拌兘鐪嬪埌璁惧鏄� 1 杩樻槸 1-1锛孫TA 涓嶄細鎶� gen2 bin 鎺ㄧ粰 gen1銆�
+ - hello 鏃ュ織鑳芥墦鍗颁袱绔殑 generation / cap_set / 閲囩敤鐨勪氦闆嗐��
+ - 鏂囨。鏈夊吋瀹圭煩闃碉紱鐮村潖鎬� REST 鏈夋棫璺緞鎴栨槑纭埅姝㈡棩鏈熴��
+
+14. **Risks**
+ 鎶婁笟鍔＄増鏈拰鍗忚鐗堟湰娣锋垚涓�涓暟锛�1-1 璇綋鎴愬繀椤绘崲鏈猴紱Server 闈欓粯鍚炴帀鏈煡瀛楁瀵艰嚧銆屼互涓烘敮鎸佹憚鍍忓ご銆嶃��
+
+15. **Regression Risks**
+ hello 鍔犲瓧娈靛鑷存棫 Server 瑙ｆ瀽澶辫触锛堝繀椤诲拷鐣ユ湭鐭ラ敭锛夛紱OTA 鏉垮瀷瀛楃涓蹭笌 generation 涓嶄竴鑷村鑷村崌閿欏寘銆�
+
+---
+
+### 璇勪及鏃舵満锛圱ask 澹版槑锛屽悗缁� 4b 鏂藉伐鍗曞啀缁嗗寲锛�
+
+涓嶆槸鏂� Device 鑳藉姏銆侻anifest / Task JSON 鏍囨槑 `assessment.timing`锛�
+
+| timing | 鍚獥缁撴潫鍚� | 浠ｈ〃 | 瀹炵幇闃舵 |
+|--------|------------|------|----------|
+| **`deferred`** | 绔嬪埢涓嬩竴姝ワ紱metrics 鍚庡彴鍙櫄鍒� | 璺熻銆佹晠浜嬪杩般�佹墦鍗″綍闊� | **Phase 4** |
+| **`inline`** | **鍏�** Server 璇勪及锛�**鍐�** `speak` 鍙嶉锛岀劧鍚庝笅涓�棰� | 鍊掕儗鏁板瓧銆佹垚璇帴榫欍�佸彛绠� | **Phase 4b** |
+
+鍏卞悓鐐癸細鍒ら鍙湪 Server锛涜澶囧彧鏀跺埌宸叉湁 `speak` / `play_audio`锛涘浐浠舵棤 `if correct`銆佹棤 `JUDGE`銆�
+宸埆锛氳窡璇诲彲浠ヨ瘎浼板け璐ヤ粛寰�涓嬫挱锛涘�掕儗鑻ヨ瘎浼板け璐ュ簲 speak銆屾病鍚竻銆嶄箣绫诲啀杩涘叆涓嬩竴棰橈紝**涓嶈兘**鎶婂弽棣堟嫋鍒板闀夸簨鍚庢墦寮� App 鎵嶇畻瀹屾垚杩欎竴鍥炲悎銆�
+
+鍊掕儗绀轰緥锛圫erver 缂栨帓锛屽彲鍚庤ˉ杩� 4b-plan锛夛細
+
+```text
+speak 鏁板瓧锛岄棿闅� ~1s锛圫erver 鐫℃垨浠ュ悗 D11 wait锛�
+speak銆屽紑濮嬨��
+listen 鍥哄畾鏃堕暱
+ASR + parser 鈫� metrics锛堝浜嗗嚑涓�佸噯纭巼锛�
+speak 鍙嶉锛堝強鏃躲�佸彲鍚畾閲忥紝濡傘�屽�掑浜� 4 涓�嶏級
+涓嬩竴棰樻垨缁撴潫
+```
+
+鎴愯鎺ラ緳锛氬惉瀹岀敤鎴蜂竴鍙� 鈫� Server 鍒ゆ槸鍚﹀悎娉曟帴榫� 鈫� 椹笂 speak 涓嬩竴鎴愯鎴栫籂閿欐彁绀恒�傚悓鏍锋槸 `inline`銆�
+
+鍙ｇ畻锛氶粯璁ゅ悓 `inline`锛堝/閿欐垨鎶ュ嚭姝ｇ‘绛旀灞炰簬涓氬姟娴侊級銆傝嫢鏌愮粌涔犳ā寮忓彧瑕佷氦鍗峰悗鐪嬫姤琛紝鍐嶅湪 Task JSON 鏀规垚 `deferred`銆�
+
+---
+
+### Phase 4b 鈥� 鍐荤粨楠屾敹闆嗭紙鍊掕儗 / 鍙ｇ畻 / 鎺ラ緳锛壜� 娴佸唴璇勪及
+
+1. **Objective**
+ 涓変釜浠ｈ〃 Task **鍙姞 Server Task / Parser / Prompt**銆傚叾涓�掕儗銆佹帴榫欙紙鍙婇粯璁ゅ彛绠楋級蹇呴』鏄� **`inline`**锛氬惉绐楃粨鏉熷氨瑕佽瘎浼板苟鍑哄０鍙嶉锛�**绂佹**鍋氭垚璺熻閭ｆ牱鐨勫悗璇勪及銆�
+
+2. **Why Now**
+ 璺熻鍙鐩栥�屽綍瀹岀户缁�嶃�傝鍒欏瀷/娓告垙鍨嬪洖鍚堣嫢涓嶅強鏃跺弽棣堬紝浜у搧灏变笉鏄偅涓� App銆�
+
+3. **Dependencies**
+ Phase 4 鐨勫惉绐椼�両nstance銆乵etrics 瀛楁銆傚彟澧烇細鍚獥缁撴潫 鈫� await 鐭瘎浼� 鈫� speak锛堜粛涓嶆敼鍥轰欢锛夈��
+
+4. **Scope**
+ 鍊掕儗锛氶棿闅斿嚭棰� 鈫� 銆屽紑濮嬨�嶁啋 褰曢煶绐� 鈫� 绔嬪埢鎶ュ浜嗗嚑涓�/鍑嗙‘鐜囥��
+ 鎺ラ緳锛氱敤鎴疯瀹� 鈫� 绔嬪埢鍚堟硶鎬т笌涓嬩竴鎻愮ず銆�
+ 鍙ｇ畻锛氶粯璁ゅ悓 inline銆�
+ 鍑洪 TTS 璇█ = `device.locale`锛堟棤鍒� zh锛夈�侾arser 浠嶄骇鍑哄畾閲� `metrics[]`锛涘弽棣堟枃妗堢敱 Server 鐢熸垚鍚� `speak`锛屼笉鏄浐浠舵ā鏉裤��
+
+5. **Out of Scope**
+ 鏂� Mode銆佹柊 DeviceState銆丮ultiNet 涓氬姟璇嶃�佹柊 WS銆乣JUDGE` 鍛戒护銆侀毦搴﹀紩鎿庝骇鍝佸寲銆佸皬绋嬪簭銆乸hrase pack銆佹妸璺熻鏀瑰洖娴佸唴绾犻敊銆�
+
+6. **Client**
+ **绂佹鏀� C++銆�**
+
+13. **Acceptance**
+ 鍊掕儗涓�杞細褰曢煶缁撴潫鍚� **鏁扮鍐�** 璁惧璇村嚭璇勪及缁撴灉锛屽啀鍑轰笅涓�棰樻垨缁撴潫銆傛柇璇勪及鏈嶅姟鏃舵湁瓒呮椂鍏滃簳 speak锛屼笉寰楁绛夈�傚浐浠� git diff 涓虹┖銆�
+
+鏂藉伐鍗曪細浠ュ悗鍐� `phase-4b-plan.md`锛堟湰鏂囦欢鍙拤鏃舵満宸紓锛夈��
+
+### Phase 5 鈥� Server Scheduler
+
+1. **Objective**
+ 瀹堕暱銆屾瘡澶� 07:00 澶嶈銆嶆潈濞佸湪 Server銆傝澶� RTC + pending 鏄� D14 鎵ц鍣紝涓嶆槸鏃ュ巻鐪熺浉銆�
+
+2. **Why Now**
+ 璁惧闂归挓鑳� Demo锛屼笉鑳藉璐� Instance銆�
+
+3. **Dependencies**
+ Phase 3锛汸hase 1 `SET_ALARM` / `START_APP_TASK`銆�
+
+4. **Scope**
+ Schedule 琛紱鍒扮偣 `TASK_START`锛圡QTT锛夛紱璺宠繃 mqtt_immediate銆�
+
+5. **Out of Scope**
+ 瀹屾暣瀹堕暱 UI锛汥evice 涓婄殑澶嶆潅 Plan DSL銆�
+
+8. **Protocol**
+ 澶嶇敤宸叉湁 MQTT 鍛戒护锛屼笉鏂板绉嶇被銆侾AUSE/RESUME 鑻ュ仛锛屽繀椤绘槸 Spec D14锛屼笉鏄浜斿鍗忚銆�
+
+---
+
+### Phase 6 鈥� App / Resource Management锛圖13锛�+ 鍦ㄧ嚎 Locale
+
+1. **Objective**
+ 璧勬簮鏉冨▉鍦� Server锛涜澶囧彧缂撳瓨鏂囦欢銆傚悓鏃惰惤鍦� **璁惧榛樿璇█锛堝湪绾垮崐锛�** 涓� **缈昏瘧/鍙岃鐨勫彲鏇挎崲璇█瀵�**銆�
+
+2. **Why Now**
+ 璺熻鎹㈡爣鍑嗛煶銆佸 App 闊抽锛屼笉鑳介潬鎵嬪伐鎷� SD銆傝瘝琛ㄨ嫢缁х画鍐欐涓枃锛屽痉璇満鑱旂綉鍚庝粛鍙兘鍠娿�屽紑濮嬭窡璇汇�嶃��
+
+3. **Dependencies**
+ Phase 3銆備笉渚濊禆 4b銆佷笉渚濊禆 6b 鍥轰欢銆傜幇缃� `ai_agent.language` / `lang_code` 鍙厛褰� Chat 璇鏉ユ簮銆�
+
+4. **Scope 鈥� 璧勬簮**
+ 娓呭崟 / DOWNLOAD / `resource_id`銆傚彲閫夌紦瀛� manifest銆傝澶囦粛鍙墽琛� skills[]銆�
+
+5. **Scope 鈥� Locale锛堝湪绾匡紝鏈樁娈靛繀椤诲仛锛�**
+ 瑙佷笅鏂� **銆岃瑷�妯″瀷銆�**銆俻hrase pack銆乣device.locale`銆佺炕璇�/鍙岃 `language_pair`銆傜儹璺緞锛欰SR 鏂囨湰鐢ㄨ locale 鐨� enter_phrases 鍖归厤锛汿TS 杩涘満璇敤璇� locale锛汣hat 绯荤粺鎻愮ず涓庨煶鑹茶窡 locale銆�
+
+6. **Out of Scope**
+ App Store锛涙妸鍒ら JSON 褰撳浐浠堕�昏緫锛涘痉璇绾垮敜閱掞紙6b / 鍙兘姘镐箙鍋氫笉鍒帮級锛汸hase 4 鏃堕挓閲嶅仛銆�
+
+---
+
+### Phase 6b 鈥� 绂荤嚎 Locale锛堝彲閫� OTA锛�
+
+1. **Objective**
+ 娌＄綉鏃讹紝鍞ら啋璇嶄笌绯荤粺绾у彛浠わ紙鍒囨崲鍏ュ彛 / 澶嶈鏈� / 鍥� Chat锛変笌 `device.locale` 涓�鑷淬��**涓嶆槸鏂� Device 鑳藉姏绉嶇被**锛圫pec锛歁ultiNet 璇嶈〃鍙厤锛夈��
+
+2. **Why Now / 涓轰綍鍗曠嫭**
+ ESP-SR 鐨� WakeNet / MultiNet **鎸夎绉嶅嚭妯″瀷**銆備腑鏂囧彛浠ゆ槸鎷奸煶闊崇礌锛涜嫳璇槸鍙︿竴濂� MN锛涘畼鏂� **娌℃湁** 寰疯 MultiNet 鍙鏈熴�傛妸銆岀儳涓�濂楀痉璇彞琛ㄣ�嶅杩� Phase 6 浼氬亣瑁呯绾垮痉璇凡缁忓瓨鍦ㄣ��
+
+3. **Scope**
+ NVS `locale`锛涗粠 assets 鍔犺浇璇ヨ绉嶇殑 wake + 绯荤粺鍛戒护琛ㄣ�倂1 **鍙壙璇� zh 涓� en**锛堟湁妯″瀷鎵嶅仛锛夈�傚垏鎹粛鍙厑璁哥郴缁熺骇璇嶏紝绂佹鎶娿�屽紑濮嬭窡璇汇�嶅啓杩涘浐浠躲��
+
+4. **Out of Scope**
+ 涓烘嘲/娉�/寰疯缁冨敜閱掓ā鍨嬶紙浜у搧/鏁版嵁椤圭洰锛屼笉鏄紪鐮� Phase锛夛紱杩愯鏃朵粠浜戜笅杞藉０瀛︽ā鍨嬭繘 PSRAM锛堥櫎闈炰互鍚庡崟鐙珛椤癸級銆�
+
+5. **鎺ュ彈鐜板疄**
+ `locale=de-DE` 鐨勮澶囷細鍦ㄧ嚎鑿滃崟/Chat 鍙互鏄痉璇紱绂荤嚎鍞ら啋鍙互浠嶆槸鑻辨枃鎴栦腑鏂� SKU锛岀洿鍒版湁妯″瀷銆傛枃妗ｅ繀椤诲啓娓咃紝涓嶈褰� bug銆�
+
+---
+
+### Phase 7 鈥� Parent Task Management
+
+1. **Objective**
+ 灏忕▼搴忚皟鐢ㄥ凡鏈� Command锛氬嵆鏃� START銆佺湅 Instance/瀹氶噺缁撴灉銆佹敼 Schedule锛�**璁剧疆璁惧榛樿璇█涓庣炕璇�/鍙岃璇█瀵�**銆�
+
+2. **Dependencies**
+ Phase 5 + Phase 6锛汸hase 4 metrics 鍙�夈��6b 涓嶆槸鍓嶆彁锛氭帶鍒跺彴鏀� locale 鍚庯紝鍦ㄧ嚎绔嬪嵆鐢熸晥銆�
+
+4. **Scope**
+ UI銆侱9 闀挎寜閫�鍑鸿嫢鏈仛鍙竴骞讹紙浠嶆槸宸插垪鑳藉姏锛夈�傝瑷�瀵瑰彧鑳介�� App Manifest 澹版槑鐨� supported_pairs銆�
+
+5. **Out of Scope**
+ 寮�鍙戣�呭钩鍙般�佹敮浠樸�佸湪灏忕▼搴忛噷璁粌鍞ら啋璇嶃��
+
+---
+
+### Phase 8 鈥� Developer App Platform
+
+1. **Objective**
+ 绗笁鏂� Task JSON **鍙兘寮曠敤 Spec 宸插喕缁� Skill**銆傛牎楠屽け璐ュ嵆鎷掋��
+
+2. **Dependencies**
+ Phase 6锛汭nteractionMode 闆嗗悎绋冲畾銆�
+
+5. **Out of Scope**
+ GPIO銆佽澶囪剼鏈紩鎿庛�佹柊浼犳劅鍣ㄣ��
+
+---
+
+### Phase 9 鈥� Subscription / Marketplace
+
+璐﹀彿涓庢潈鐩娿�備笉鏀� Opus锛屼笉涓婃儏缁瘖鏂��
+
+---
+
+### 璇█妯″瀷锛堣瘎浼扮粨璁猴紱瀹炵幇钀藉湪 6 / 6b / 7锛�
+
+鍒嗕笁灞傦紝涓嶈娣锋垚銆屼竴涓� language 瀛楁鎼炲畾鍞ら啋鍜岀炕璇戙�嶃��
+
+| 灞� | 鏄粈涔� | 渚嬪瓙 | 闅惧害 |
+|----|--------|------|------|
+| **A 璁惧榛樿璇█ `device.locale`** | 寮�鏈� Chat銆佽繘鍦� TTS銆佸湪绾垮満鏅彛浠ゃ�佹棤璇█瀵圭殑 App锛堝�掕儗銆佸彛绠椼�佹晠浜嬪杩帮級 | `zh-CN` / `en-US` / `de-DE` | 鍦ㄧ嚎锛�**瀹规槗**锛圫erver phrase pack + 宸叉湁 `ai_agent.language`锛夈�傜绾垮敜閱�/鍙ｄ护锛�**涓嶅鏄�**锛岃 6b |
+| **B App 璇█瀵� `language_pair`** | 缈昏瘧銆佸弻璇璇濈殑鍙浛鎹㈤�夐」 | 缈昏瘧 `th鈫攆r`锛涘璇� `en鈫攄e` | **瀹规槗**锛岀函 Server锛涗竴涓� App + 鍙傛暟锛屼笉瑕佹瘡绉嶈瑷�瀵逛竴涓� `app_id` |
+| **C 璧勬簮鍐呭璇█** | 闊抽 MP3/txt 鏈韩 | 鑻辫璺熻璇炬案杩滄槸鑻辨枃闊抽 | 璺� lesson 鍖呰蛋锛屼笉璺� locale銆傚痉璇満缁冭嫳璇窡璇伙細鑿滃崟寰疯銆佸枃鍙煶棰戣嫳璇� |
+
+**鍔熻兘 1 濂戒笉濂藉仛锛�**
+
+- 鑱旂綉鍚庣殑涓枃鏀瑰痉璇� Chat / 銆屽垏鎹㈡ā寮忋�嶈彍鍗� / 銆屽紑濮嬭窡璇汇�嶇殑寰疯鍚屼箟鍙ワ細鎶� `runtime.py` 纭紪鐮佽瘝琛ㄦ崲鎴� `locales/{locale}/phrases.json`锛宧ello 鏃跺甫涓� `device.locale`銆�**宸ヤ綔閲忎腑绛夈�侀闄╀綆锛孭hase 6銆�**
+- 寮�鏈虹绾垮氨鐢ㄥ痉璇枈鍞ら啋銆佸枈銆屽璇绘満銆嶏細蹇呴』鎹� WakeNet + MultiNet 妯″瀷涓庨煶绱犳牸寮忋�備腑鏂囩幇缃戞槸鎷奸煶缂栬瘧杩� `system.h`銆傝嫳璇畼鏂� MN 鍙仛绗簩濂� assets銆�**寰疯娌℃湁鐜版垚 ESP-SR 妯″瀷鏃讹紝绂荤嚎鍏ュ彛鍋氫笉鍒�**锛屽彧鑳藉湪绾裤�備笉瑕佹壙璇恒�屾敼涓�涓厤缃叏涓栫晫绂荤嚎寰疯銆嶃��
+- 宸ュ巶 SKU锛堝嚭鍘傜儳涓枃鏈� / 鑻辨枃鏈猴級姣旇繍琛屾椂鐑垏鎹㈢绾挎ā鍨嬪鏄撳緱澶氥�傝繍琛屾椂鍒囨崲 locale锛氬湪绾跨珛鍒荤敓鏁堬紱绂荤嚎瑕� 6b 涓斾粎 zh/en銆�
+
+**鍔熻兘 2 鎬庝箞閰嶏細**
+
+涓嶈鍐嶅鍒� `english_translation` / `english_conversation` 涓や釜 App 鍚嶃�侻anifest锛�
+
+```text
+app_id: translation
+ default_pair: ["zh","en"] # 璁惧 locale 涓� zh 鏃剁殑缂虹渷
+ supported_pairs: [["zh","en"],["th","fr"],["en","de"], ...]
+ instance.options.language_pair # 瀹堕暱/鎺у埗鍙拌鐩�
+
+app_id: bilingual_chat
+ default_pair: ["zh","en"]
+ supported_pairs: [["zh","en"],["en","de"], ...]
+ 璇箟锛氱敤鎴疯 lang_a 鈫� 鍙瓟 lang_b锛屽弽涔�
+```
+
+缈昏瘧鏈猴細鏈寚瀹氭椂浠嶅彲 auto 妫�娴嬶紝鐩爣璇彇 pair 鐨勫彟涓�渚ф垨 `target_lang`銆�
+缂虹渷 pair 鐨勬帹瀵硷細`[device.locale 鐨勪富璇█, 鏈烘瀯閰嶇疆鐨� L2]`锛屾病鏈� L2 鍒� `zh-en`銆�
+
+鍊掕儗 / 鏁呬簨澶嶈堪 / 鍙ｇ畻锛氬彧鏈� A锛孴TS 涓庡惉绐楁彁绀虹敤 `device.locale`銆備笉瑕佺粰瀹冧滑閰� language_pair銆�
+
+**鏉冨▉锛�** `device.locale` 鍦� manager-api锛堢粦瀹�/鍑哄巶锛夛紱Agent.language 涓庡叾瀵归綈锛岄伩鍏嶄袱濂楁墦鏋躲�倄iaozhi-server 杩炴帴鏃惰鍙栵紝鍥轰欢 NVS 鍙湇鍔� 6b 绂荤嚎銆傝澶� **浠嶇劧娌℃湁 tenant_id**銆�
+
+**绂佹锛�** 涓烘瘡绉嶈瑷�瀵规敼 C++锛涙妸銆屽紑濮嬭窡璇汇�嶇殑寰疯鍐欒繘鍥轰欢锛汸hase 4 鍋� Locale銆�
+
+---
+
+## 3. 璺ㄧ渚濊禆锛圕ommand / Event / State锛�
+
+瀵归綈 Spec 搂4鈥�5銆傛瘡涓� Phase 蹇呴』鍚屾椂鏈� Server鈫扗evice 涓� Device鈫扴erver锛岀姝㈠崟绔〃銆�
+
+```text
+Phase 2
+ Device AUDIO_COMPLETED|STOPPED 鈹�鈹�detect鈹�鈹�鈻� Server
+ Device listen 娴� + VAD 鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈹�鈻� Server ASR / D8 v1 report
+ Server SPEAK / PLAY_AUDIO / 鍥� CHAT 鈹�鈹�tts/mcp鈹�鈹�鈻� Device
+ Server ensure_app dirs 鈹�鈹�MCP/MQTT鈹�鈹�鈻� Device mkdir锛堣繍杈擄紝闈炴柊鑳藉姏锛�
+
+Phase 3 鉁�
+ Server TaskSession JSON 鈹�鈹�HTTP upsert鈹�鈹�鈻� manager-api Instance
+ 鏃犳柊 Device 鑳藉姏锛涜澶囦笉鎸佹湁 instance_id
+
+Phase 4
+ Server 浠呮椂閽熷埌鐐� PLAY_AUDIO 涓嬩竴鍙� 鈹�鈹�鈻� Device
+ Device 鍚祦锛堟梺璺� ASR/report锛� 鈹�鈹�鈻� Server 鍚庤瘎浼� metrics
+ 璺熻锛氬垎鏋愬け璐ヤ笉鎸′笅涓�鍙ワ紱鏃� JUDGE
+
+Phase V
+ Device hello/OTA 甯� generation+revision+cap_set 鈹�鈹�鈻� Server 鐭╅樀
+ Server 浜ら泦涓氬姟 / 寮曞 OTA / 鎷掓湭鐭� SKU 鈹�鈹�鈻� Device
+ 鏃у浐浠剁己瀛楁 = 灏忔櫤 AI 1 / cap.v1锛涗笉鍔犳柊 WS type
+
+Phase 4b
+ Device 鍚祦 鈹�鈹�鈻� Server ASR + parser锛坕nline锛岀煭瓒呮椂锛�
+ Server SPEAK 鍙嶉 / 涓嬩竴棰� 鈹�鈹�鈻� Device
+ 浠嶆棤 JUDGE銆佹棤鍥轰欢 if correct锛涜瘎浼板湪涓氬姟娴佸唴锛屼笉鑳芥嫋鍒板悗璇勪及
+
+Phase 5
+ Server TASK_START 鈹�鈹�MQTT鈹�鈹�鈻� Device
+ Device TASK_STARTED/COMPLETED 鈹�鈹�ack/event鈹�鈹�鈻� Server
+
+Phase 6
+ Server FETCH 鈹�鈹�DOWNLOAD鈹�鈹�鈻� Device SD
+ Device PLAY_AUDIO(resource) 鈹�鈹�event鈹�鈹�鈻� Server
+```
+
+---
+
+## 4. Master Plan / Spec 瀵圭収
+
+| 闀挎湡鐩爣 | Roadmap |
+|----------|---------|
+| Device 鑳藉姏鍐荤粨 | Freeze 鏂囨。 + Phase 2 琛ヤ簨浠�/閲囬泦 |
+| Task Runtime 鍦� Server | P2 Session 鈫� P3 Instance 鉁� 鈫� P4 璺熻鍚庤瘎浼� 鈫� P4b 娴佸唴璇勪及 |
+| 鏂� App 涓嶆敼 C++ | P4b 涓変釜浠ｈ〃 Task 楠屾敹 |
+| 璺熻璇煶瀵硅薄 | P2 D8 v1 鈫� P3 鎸� step 鈫� P6 鏍囧噯闊� |
+| Scheduler | P5 / P7 |
+| App 鈮� Firmware | 璐┛锛汸6 璧勬簮銆丳8 娌欑 |
+| 璁惧榛樿璇█ / 璇█瀵� | P6 鍦ㄧ嚎 pack + pair锛汸6b 绂荤嚎 zh/en锛汸7 鎺у埗鍙帮紱P4b 缁ф壙 locale |
+| 浜у搧浠ｉ檯 / 绔簯鍖归厤 | **Phase V**锛涙憚鍍忓ご SKU 涔嬪墠锛涚幇缃戠己瀛楁瑙嗕负 灏忔櫤 AI 1 |
+| 蹇冪悊闄即 / 涓绘寔浜� | 鏃犳柊纭欢锛涘鐢� listen/speak + Server Memory |
+| Subscription | P9 |
+
+---
+
+## 5. 鍒绘剰涓嶅仛杩涜繎鏈熺殑浜嬮」
+
+鏉ヨ嚜 Spec 搂17 涓� 搂21锛�
+
+- 涓烘垚璇�/鍙ｇ畻/鎺ラ緳鏀� C++ 鎴栧姞 MultiNet 涓氬姟鍙ｄ护
+- 鎶� `branch(if correct)` 鏀捐繘鍥轰欢
+- 鍚堝苟 Xiaozhi MQTT 涓� snlp锛堟敼杩愯緭 鈮� 鍔犺兘鍔涳紝浣嗕笉浣滀负杩戞湡椤圭洰锛�
+- 鎶� apps.json 褰撶绾挎潈濞佸紩鎿�
+- Phase 2 鍋� `wait` Skill銆乣TASK_PAUSE`銆佹湰鍦� WAV 涓婁紶銆乣indicate` 澶ф敼
+- Phase 2 鎷嗗井鏈嶅姟 / 瀵硅薄瀛樺偍 / 鎯呯华鍒嗘瀽 / 涓村簥璇婃柇
+- 涓�娆℃�у啓姝� Phase 5鈥�9 鐨勭被鍚嶄笌 REST 璺緞
+- 鎶婁笟鍔� App 鍒囨崲璇嶅啓鍏ュ浐浠讹紙绯荤粺绾т粎 Chat + 绂荤嚎澶嶈 + 銆屽垏鎹㈡ā寮忋�嶆�诲叆鍙ｏ級
+- 鍦� Phase 4 寮曞叆绉熸埛 / `tenant_id`锛堟渶鏃╃紳鍦� Phase 6锛屼骇鍝佸寲 Phase 7锛�
+- 鍦� Phase 4 鍋氬璇█ / 鎶婂痉璇敜閱掑啓杩涘浐浠�
+- 涓烘瘡绉嶇炕璇戣瑷�瀵规柊澧炰竴涓� `app_id` 鎴栨柊 WS type
+- 鐢ㄤ竴涓暟瀛楀悓鏃惰〃绀轰骇鍝佷唬闄呫�佸浐浠� semver 鍜屽崗璁増鏈�
+- 鎶婃憚鍍忓ご鍥轰欢鍖呮寜 semver 鎺ㄧ粰鏃犳憚鍍忓ご鐨勫皬鏅� AI 1
+- 鍦� Phase 4 鍋氬吋瀹圭煩闃� / 鏀� hello 瀛楁锛堟斁鍒� Phase V锛�
+
+**涓嬩竴缂栫爜闃舵锛�** 鍙墦寮� `phase-4-plan.md`锛岄獙鏀跺鐓� `phase-4-test.md`銆傚浐浠跺喕缁撱�傝窡璇绘寜 MP3 搴忓垪瀹氭椂褰曢煶锛涘悗璇勪及鍙畾閲忋�佷笉鎸囨尌鍠囧彮銆侺ocale 绛夊埌 Phase 6銆傜浜戠増鏈尮閰嶇瓑鍒� **Phase V**锛堝彲涓� 4 骞惰璁捐锛屾憚鍍忓ご SKU 鍓嶅繀椤昏惤鍦帮級銆�
diff --git a/xiaozhi-esp32-2.2.6/main/CMakeLists.txt b/xiaozhi-esp32-2.2.6/main/CMakeLists.txt
index c27455e..ac0d6bb 100644
--- a/xiaozhi-esp32-2.2.6/main/CMakeLists.txt
+++ b/xiaozhi-esp32-2.2.6/main/CMakeLists.txt
@@ -1201,3 +1201,21 @@
else()
message(STATUS "Assets partition not found, using v1 partition table")
endif()
+
+# Patch 78/esp-wifi-connect so the captive portal accepts zh_CN and reclaims
+# httpd sockets stolen by WeChat /mmtls captive-DNS probes.
+find_component_by_pattern("esp-wifi-connect" WIFI_CONNECT_COMPONENT WIFI_CONNECT_PATH)
+if(WIFI_CONNECT_PATH)
+ execute_process(
+ COMMAND python "${PROJECT_DIR}/scripts/patch_wifi_connect.py" "${WIFI_CONNECT_PATH}"
+ RESULT_VARIABLE WIFI_CONNECT_PATCH_RESULT
+ OUTPUT_VARIABLE WIFI_CONNECT_PATCH_OUTPUT
+ ERROR_VARIABLE WIFI_CONNECT_PATCH_OUTPUT
+ )
+ if(WIFI_CONNECT_PATCH_OUTPUT)
+ message(STATUS "${WIFI_CONNECT_PATCH_OUTPUT}")
+ endif()
+ if(NOT WIFI_CONNECT_PATCH_RESULT EQUAL 0)
+ message(WARNING "Failed to patch esp-wifi-connect at ${WIFI_CONNECT_PATH}")
+ endif()
+endif()
diff --git a/xiaozhi-esp32-2.2.6/main/Kconfig.projbuild b/xiaozhi-esp32-2.2.6/main/Kconfig.projbuild
index b2c86a7..3ddedfc 100644
--- a/xiaozhi-esp32-2.2.6/main/Kconfig.projbuild
+++ b/xiaozhi-esp32-2.2.6/main/Kconfig.projbuild
@@ -2,7 +2,7 @@
config OTA_URL
string "Default OTA URL"
- default "https://api.tenclass.net/xiaozhi/ota/"
+ default "http://192.168.0.107:8002/xiaozhi/ota/"
help
The application will access this URL to check for new firmwares and server address.
diff --git a/xiaozhi-esp32-2.2.6/main/application.cc b/xiaozhi-esp32-2.2.6/main/application.cc
index 643d8db..f4cb306 100644
--- a/xiaozhi-esp32-2.2.6/main/application.cc
+++ b/xiaozhi-esp32-2.2.6/main/application.cc
@@ -6,6 +6,7 @@
#include "mqtt_protocol.h"
#include "websocket_protocol.h"
#include "boards/sening-xiaozhi-s3/mqtt.h"
+#include "boards/sening-xiaozhi-s3/app_task.h"
#include "assets/lang_config.h"
#include "mcp_server.h"
#include "assets.h"
@@ -97,7 +98,7 @@
.arg = this,
.dispatch_method = ESP_TIMER_TASK,
.name = "clock_timer",
- .skip_unhandled_events = true
+ .skip_unhandled_events = false
};
esp_timer_create(&clock_timer_args, &clock_timer_handle_);
}
@@ -143,9 +144,14 @@
}
};
callbacks.on_offline_command_detected = [this](const std::string& command) {
- if (!power_off_standby_active_.load()) {
- Board_Handle_Offline_Command(command.c_str());
+ if (power_off_standby_active_.load()) {
+ return;
}
+ // AFE audio_detection only has a 4KB stack. Starting repeater (NVS, cJSON,
+ // SD list, MP3, JPG) on that task overflows and reboots.
+ Schedule([command]() {
+ Board_Handle_Offline_Command(command.c_str());
+ });
};
callbacks.on_vad_change = [this](bool speaking) {
xEventGroupSetBits(event_group_, MAIN_EVENT_VAD_CHANGE);
@@ -314,13 +320,66 @@
for (auto& task : tasks) {
task();
}
+ if (GetDeviceState() == kDeviceStateSpeaking) {
+ MaybeFinishTtsAfterPlayback();
+ }
}
if (bits & MAIN_EVENT_CLOCK_TICK) {
clock_ticks_++;
auto display = Board::GetInstance().GetDisplay();
display->UpdateStatusBar();
-
+ auto state = GetDeviceState();
+ if (state == kDeviceStateConnecting || state == kDeviceStateSpeaking ||
+ (state == kDeviceStateIdle && suppress_wake_until_listen_)) {
+ audio_service_.KeepCodecReadyForPlayback();
+ }
+ MaybeFinishTtsAfterPlayback();
+ if (suppress_wake_until_listen_ && state == kDeviceStateIdle) {
+ if (protocol_ && protocol_->IsAudioChannelConnected()) {
+ boot_wake_restore_ticks_ = 0;
+ if (greeting_listen_wait_ticks_ > 0) {
+ greeting_listen_wait_ticks_--;
+ if (greeting_listen_wait_ticks_ == 0) {
+ ESP_LOGW(TAG, "greeting TTS missing, start listening");
+ SetListeningMode(GetDefaultListeningMode());
+ }
+ }
+ } else if (++boot_wake_restore_ticks_ >= 20) {
+ ESP_LOGW(TAG, "boot websocket not opened, restore wake word");
+ suppress_wake_until_listen_ = false;
+ boot_wake_restore_ticks_ = 0;
+ audio_service_.EnableWakeWordDetection(true);
+ }
+ } else {
+ boot_wake_restore_ticks_ = 0;
+ }
+ if (state == kDeviceStateSpeaking && tts_audio_expected_ && !tts_audio_received_ &&
+ speaking_started_us_ > 0) {
+ const int64_t wait_ms = (esp_timer_get_time() - speaking_started_us_) / 1000;
+ if (wait_ms >= 1000 && (wait_ms / 1000) <= 8) {
+ ESP_LOGW(TAG, "TTS audio still missing after %lldms", (long long)wait_ms);
+ }
+ }
+ if (protocol_ && IsSdCardMp3Playing() && protocol_->IsAudioChannelConnected()) {
+ protocol_->MarkIncomingActivity();
+ }
+ if (!pending_detect_text_.empty() &&
+ GetDeviceState() == kDeviceStateIdle &&
+ protocol_ && !protocol_->IsAudioChannelConnected()) {
+ if (pending_detect_retry_ticks_ > 0) {
+ pending_detect_retry_ticks_--;
+ } else if (pending_detect_retries_left_ > 0) {
+ pending_detect_retries_left_--;
+ pending_detect_retry_ticks_ = 25;
+ ESP_LOGW(TAG, "retry listen detect after handshake fail: %s left=%d",
+ pending_detect_text_.c_str(), pending_detect_retries_left_);
+ SendListenDetect(pending_detect_text_, true);
+ } else {
+ pending_detect_text_.clear();
+ }
+ }
+
// Print debug info every 10 seconds
if (clock_ticks_ % 10 == 0) {
SystemInfo::PrintHeapStats();
@@ -376,6 +435,10 @@
ESP_LOGI(TAG, "Activation done");
SystemInfo::PrintHeapStats();
+ // 寮�鏈洪┈涓婅杩炶闊� WebSocket銆傚厛涓嶈鎷夎捣 AFE锛氬唴閮� SRAM 琚� WakeNet 鍗犳弧鏃�
+ // 绗竴娆℃彙鎵嬩細 10 绉掕秴鏃讹紝閲嶈瘯铏借兘杩炰笂锛屾杩庤瘝 opus 涔熷鏄撲涪銆�
+ suppress_wake_until_listen_ = true;
+ boot_wake_restore_ticks_ = 0;
SetDeviceState(kDeviceStateIdle);
has_server_time_ = ota_->HasServerTime();
@@ -388,7 +451,9 @@
// Release OTA object after activation is complete
ota_.reset();
auto& board = Board::GetInstance();
- board.SetPowerSaveLevel(PowerSaveLevel::LOW_POWER);
+ // 寮�鏈洪┈涓婂氨瑕佽繛璇煶 WebSocket 鎾杩庤瘝銆傝繖閲屽垏 LOW_POWER 浼氳闅忓悗鐨勬彙鎵�
+ // 鍗″湪 handshake timeout锛屾杩庤瘝鏍规湰鍙戜笉鍑烘潵銆備繚鎸侀珮鎬ц兘鐩村埌閫氶亾鎵撳紑銆�
+ board.SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE);
Schedule([this]() {
// Play the success sound to indicate the device is ready
@@ -582,6 +647,10 @@
protocol_->OnNetworkError([this](const std::string& message) {
last_error_message_ = message;
+ if (network_error_suppressed_) {
+ ESP_LOGW(TAG, "network error suppressed during retry: %s", message.c_str());
+ return;
+ }
xEventGroupSetBits(event_group_, MAIN_EVENT_ERROR);
});
@@ -589,14 +658,25 @@
auto shared_packet = std::shared_ptr<AudioStreamPacket>(packet.release());
Schedule([this, shared_packet]() {
auto state = GetDeviceState();
- if (state != kDeviceStateSpeaking && !tts_audio_expected_) {
+ // speak_txt 涓嶅彂鐙珛 tts start锛岄鍙� opus 鍙兘鏃╀簬 sentence_start銆�
+ // 寰呭懡/鑱嗗惉鏃朵笉瑕佷涪鍖咃紝浜ょ粰 ScheduleTtsAudio 绛夊埌 speaking 灏辩华鍐嶆挱銆�
+ if (state != kDeviceStateSpeaking && !tts_audio_expected_ &&
+ state != kDeviceStateListening && state != kDeviceStateIdle) {
return;
+ }
+ if (state == kDeviceStateListening || state == kDeviceStateIdle) {
+ tts_audio_expected_ = true;
}
tts_response_started_ = true;
+ ESP_LOGI(TAG, "TTS audio packet bytes=%u expected=%d ready=%d state=%d",
+ (unsigned)shared_packet->payload.size(),
+ tts_audio_expected_ ? 1 : 0,
+ tts_speaking_state_ready_ ? 1 : 0,
+ static_cast<int>(state));
ScheduleTtsAudio(shared_packet);
- if (state == kDeviceStateListening) {
+ if (state == kDeviceStateListening || state == kDeviceStateIdle) {
SetDeviceState(kDeviceStateSpeaking);
}
});
@@ -611,10 +691,25 @@
});
protocol_->OnAudioChannelClosed([this, &board]() {
- board.SetPowerSaveLevel(PowerSaveLevel::LOW_POWER);
+ if (opening_audio_channel_) {
+ ESP_LOGI(TAG, "Skip channel-closed teardown while opening/reusing websocket");
+ return;
+ }
+ // 璺熻/澶嶈杩樺湪鎾椂涓嶈鍒� LOW_POWER锛氬悗闈㈣ˉ鍙� playlist_completed 浼� handshake timeout銆�
+ if ((Board_IsShadowingTask() || Board_IsRepeaterTask()) && IsSdCardMp3Playing()) {
+ board.SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE);
+ } else {
+ board.SetPowerSaveLevel(PowerSaveLevel::LOW_POWER);
+ }
Schedule([this]() {
auto display = Board::GetInstance().GetDisplay();
opening_audio_channel_ = false;
+ if ((Board_IsShadowingTask() || Board_IsRepeaterTask()) && IsSdCardMp3Playing()) {
+ if (GetDeviceState() == kDeviceStateListening) {
+ SetDeviceState(kDeviceStateIdle);
+ }
+ return;
+ }
tts_audio_expected_ = false;
tts_response_started_ = false;
tts_audio_received_ = false;
@@ -656,14 +751,24 @@
tts_audio_expected_ = true;
tts_response_started_ = false;
tts_audio_received_ = false;
+ tts_stop_pending_ = false;
+ tts_stop_waiting_ = false;
+ tts_playback_empty_ticks_ = 0;
+ tts_stop_wait_ticks_ = 0;
tts_activity_generation_++;
+ audio_service_.SetAudioPowerInhibit(true);
+ audio_service_.KeepCodecReadyForPlayback();
+ auto state = GetDeviceState();
+ if (state == kDeviceStateListening || state == kDeviceStateIdle) {
+ SetDeviceState(kDeviceStateSpeaking);
+ }
+ ESP_LOGI(TAG, "TTS start output=%d",
+ Board::GetInstance().GetAudioCodec()->output_enabled() ? 1 : 0);
});
} else if (strcmp(state->valuestring, "stop") == 0) {
Schedule([this]() {
- if (!tts_response_started_ && !standby_requested_ && !power_off_requested_) {
- tts_stop_pending_ = true;
- return;
- }
+ ESP_LOGI(TAG, "TTS stop started=%d received=%d",
+ tts_response_started_, tts_audio_received_);
tts_stop_pending_ = true;
FinishTtsResponseAfterPlayback();
});
@@ -683,14 +788,17 @@
tts_audio_expected_ = true;
tts_response_started_ = true;
tts_activity_generation_++;
+ tts_playback_empty_ticks_ = 0;
+ audio_service_.SetAudioPowerInhibit(true);
+ audio_service_.KeepCodecReadyForPlayback();
if (IsSdCardMp3Playing() && !IsSdCardMp3Paused()) {
PauseSdCardMp3();
resume_sdcard_mp3_after_tts_ = true;
}
- if (GetDeviceState() == kDeviceStateListening) {
+ auto state = GetDeviceState();
+ if (state == kDeviceStateListening || state == kDeviceStateIdle) {
SetDeviceState(kDeviceStateSpeaking);
}
- ScheduleTtsIdleFallback();
display->SetChatMessage("assistant", message.c_str());
});
}
@@ -844,18 +952,58 @@
xEventGroupSetBits(event_group_, MAIN_EVENT_STOP_LISTENING);
}
+bool Application::SendListenDetectNow(const std::string& text) {
+ if (!protocol_ || text.empty()) {
+ return false;
+ }
+ ESP_LOGI(TAG, "Send listen detect now: %s connected=%d",
+ text.c_str(), protocol_->IsAudioChannelConnected() ? 1 : 0);
+ if (!protocol_->IsAudioChannelConnected()) {
+ return false;
+ }
+ Board::GetInstance().SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE);
+ protocol_->MarkIncomingActivity();
+ return protocol_->SendWakeWordDetected(text);
+}
+
void Application::SendListenDetect(const std::string& text, bool open_if_idle) {
Schedule([this, text, open_if_idle]() {
if (!protocol_ || text.empty()) {
return;
}
- if (protocol_->IsAudioChannelOpened()) {
- protocol_->SendWakeWordDetected(text);
+ ESP_LOGI(TAG, "Send listen detect: %s connected=%d",
+ text.c_str(), protocol_->IsAudioChannelConnected() ? 1 : 0);
+ if (protocol_->IsAudioChannelConnected()) {
+ Board::GetInstance().SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE);
+ protocol_->MarkIncomingActivity();
+ if (protocol_->SendWakeWordDetected(text)) {
+ return;
+ }
+ ESP_LOGW(TAG, "detect send failed, reopen audio channel");
+ }
+ if (!open_if_idle) {
return;
}
- if (open_if_idle) {
- WakeWordInvoke(text);
+ if (!Board::GetInstance().IsNetworkReady()) {
+ ESP_LOGW(TAG, "skip listen detect, network not ready: %s", text.c_str());
+ return;
}
+ auto state = GetDeviceState();
+ // 寮�鏈�/TTS/閰嶇綉鏃朵笉瑕� WakeWordInvoke锛氶偅浼氳繘 listening銆佸紑 AFE銆丷esetDecoder锛�
+ // 娆㈣繋璇嶈娓呮帀锛孎EED 鐜紦鍐蹭篃浼氱珛鍒绘拺婊°��
+ if (state == kDeviceStateSpeaking || state == kDeviceStateWifiConfiguring ||
+ state == kDeviceStateAudioTesting || state == kDeviceStateConnecting ||
+ state == kDeviceStateStarting || state == kDeviceStateActivating) {
+ ESP_LOGI(TAG, "defer listen detect until idle: state=%d", static_cast<int>(state));
+ return;
+ }
+ if (state != kDeviceStateIdle) {
+ return;
+ }
+ SetDeviceState(kDeviceStateConnecting);
+ Schedule([this, text]() {
+ ContinueOpenAudioChannelForDetect(text);
+ });
});
}
@@ -913,26 +1061,103 @@
if (GetDeviceState() != kDeviceStateConnecting) {
return;
}
- if (opening_audio_channel_) {
+ if (!OpenAudioChannelWithRetry(2)) {
+ SetDeviceState(kDeviceStateIdle);
return;
}
+ SetListeningMode(mode);
+}
+
+bool Application::OpenAudioChannelWithRetry(int attempts) {
+ if (!protocol_) {
+ return false;
+ }
+ if (protocol_->IsAudioChannelConnected()) {
+ protocol_->MarkIncomingActivity();
+ return true;
+ }
+ if (opening_audio_channel_) {
+ return false;
+ }
+ if (attempts < 1) {
+ attempts = 1;
+ }
+
opening_audio_channel_ = true;
- if (!protocol_->IsAudioChannelOpened()) {
- if (!Board::GetInstance().EnsureNetworkReadyForWake(30000)) {
- opening_audio_channel_ = false;
- SetDeviceState(kDeviceStateIdle);
- return;
+ // 鎻℃墜鏈熼棿鍋� AFE锛岄伩鍏嶅拰 Wi-Fi 鎶㈠唴閮� SRAM/CPU銆備笉瑕佸彟寤轰换鍔★細
+ // AFE 璧锋潵鍚庡唴閮ㄥ爢鍙墿鍑� KB锛寈TaskCreate(8KB) 浼氱洿鎺ュけ璐ャ��
+ audio_service_.EnableVoiceProcessing(false);
+ audio_service_.EnableWakeWordDetection(false);
+ Board::GetInstance().SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE);
+ audio_service_.SetAudioPowerInhibit(true);
+
+ bool ok = false;
+ for (int i = 0; i < attempts; ++i) {
+ if (protocol_->IsAudioChannelConnected()) {
+ ok = true;
+ break;
}
- if (!protocol_->OpenAudioChannel()) {
- opening_audio_channel_ = false;
- SetDeviceState(kDeviceStateIdle);
- return;
+ if (!Board::GetInstance().EnsureNetworkReadyForWake(30000)) {
+ ESP_LOGW(TAG, "OpenAudioChannel aborted, network not ready");
+ break;
+ }
+ Board::GetInstance().SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE);
+ const bool last = (i == attempts - 1);
+ network_error_suppressed_ = !last;
+ ok = protocol_->OpenAudioChannel();
+ network_error_suppressed_ = false;
+ if (ok) {
+ break;
+ }
+ ESP_LOGW(TAG, "OpenAudioChannel failed (%d/%d)", i + 1, attempts);
+ protocol_->CloseAudioChannel(false);
+ if (!last) {
+ ESP_LOGI(TAG, "Retry websocket handshake in 500ms");
+ vTaskDelay(pdMS_TO_TICKS(500));
+ audio_service_.EnableWakeWordDetection(false);
+ Board::GetInstance().SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE);
+ audio_service_.KeepCodecReadyForPlayback();
}
}
- opening_audio_channel_ = false;
- SetListeningMode(mode);
+ opening_audio_channel_ = false;
+ audio_service_.SetAudioPowerInhibit(false);
+ return ok;
+}
+
+void Application::ContinueOpenAudioChannelForDetect(const std::string& detect_text) {
+ if (power_off_standby_active_.load()) {
+ return;
+ }
+ if (GetDeviceState() != kDeviceStateConnecting) {
+ return;
+ }
+ listening_mode_ = GetDefaultListeningMode();
+ suppress_wake_until_listen_ = true;
+ if (!OpenAudioChannelWithRetry(2)) {
+ pending_detect_text_ = detect_text;
+ if (pending_detect_retries_left_ <= 0) {
+ pending_detect_retries_left_ = 3;
+ }
+ pending_detect_retry_ticks_ = 3;
+ suppress_wake_until_listen_ = false;
+ audio_service_.EnableWakeWordDetection(true);
+ SetDeviceState(kDeviceStateIdle);
+ ESP_LOGW(TAG, "boot websocket failed, will retry detect in 3s");
+ return;
+ }
+
+ pending_detect_text_.clear();
+ pending_detect_retries_left_ = 0;
+ pending_detect_retry_ticks_ = 0;
+ greeting_listen_wait_ticks_ = 8;
+ if (!detect_text.empty()) {
+ protocol_->MarkIncomingActivity();
+ protocol_->SendWakeWordDetected(detect_text);
+ }
+ // 淇濇寔寰呭懡涓斾笉寮� AFE锛氭杩庤瘝浠� idle 杩� speaking銆傝瀹屽啀 listening 寮�楹︺��
+ SetDeviceState(kDeviceStateIdle);
}
void Application::PauseSdCardMp3ForVoice() {
@@ -946,8 +1171,87 @@
if (resume_sdcard_mp3_after_voice_ && !sdcard_mp3_stop_requested_) {
resume_sdcard_mp3_after_voice_ = false;
ResumeSdCardMp3();
+ ApplyMp3CommandOnlyMic();
} else {
resume_sdcard_mp3_after_voice_ = false;
+ }
+}
+
+void Application::ApplyMp3CommandOnlyMic() {
+ if (GetDeviceState() == kDeviceStateWifiConfiguring ||
+ GetDeviceState() == kDeviceStateSpeaking) {
+ return;
+ }
+ if (!IsSdCardMp3Playing() || IsSdCardMp3Paused()) {
+ return;
+ }
+ // 闊抽鍦ㄦ挱锛氫笉涓婅褰曢煶/闂茶亰锛屽彧鐣欑绾垮彛浠わ紙閫�鍑哄璇�/璺熻銆佹垜浠亰澶╁惂锛夈��
+ audio_service_.EnableVoiceProcessing(false);
+ audio_service_.EnableWakeWordDetection(audio_service_.IsAfeWakeWord());
+ if (GetDeviceState() != kDeviceStateListening) {
+ return;
+ }
+ if (protocol_ && protocol_->IsAudioChannelOpened()) {
+ protocol_->SendStopListening();
+ }
+ // 鎾鏂囨椂鍒囧緟鍛斤細鍏虫帀 AFE/涓婅锛屽彧鐣欑绾垮彛浠ゃ�傝窡璇诲彞闂寸敱鎾畬鍚庡啀 ResumeListening銆�
+ Schedule([this]() {
+ if (GetDeviceState() == kDeviceStateListening &&
+ IsSdCardMp3Playing() && !IsSdCardMp3Paused()) {
+ SetDeviceState(kDeviceStateIdle);
+ }
+ });
+}
+
+void Application::ResumeListeningAfterMp3Clip() {
+ if (power_off_standby_active_.load()) {
+ return;
+ }
+ if (IsSdCardMp3Playing() && !IsSdCardMp3Paused() && !Board_IsShadowingTask()) {
+ return;
+ }
+ auto state = GetDeviceState();
+ // 寮�鏈烘杩庤瘝 / 杩涘満 TTS 鏈熼棿绂佹寮�楹︼細EnableVoiceProcessing 浼� ResetDecoder 骞跺杺 AFE銆�
+ if (state == kDeviceStateSpeaking || state == kDeviceStateWifiConfiguring ||
+ state == kDeviceStateAudioTesting || state == kDeviceStateConnecting ||
+ state == kDeviceStateStarting || state == kDeviceStateActivating) {
+ return;
+ }
+ if (!protocol_) {
+ return;
+ }
+ ESP_LOGI(TAG, "Resume listening after MP3 clip");
+ if (state == kDeviceStateListening) {
+ protocol_->SendStartListening(listening_mode_);
+ audio_service_.EnableVoiceProcessing(true);
+#ifdef CONFIG_WAKE_WORD_DETECTION_IN_LISTENING
+ audio_service_.EnableWakeWordDetection(audio_service_.IsAfeWakeWord());
+#else
+ audio_service_.EnableWakeWordDetection(false);
+#endif
+ return;
+ }
+ if (state == kDeviceStateIdle && !protocol_->IsAudioChannelConnected()) {
+ return;
+ }
+ skip_playback_wait_on_listen_ = true;
+ SetListeningMode(GetDefaultListeningMode());
+}
+
+void Application::CloseShadowingListenWindow()
+{
+ audio_service_.EnableVoiceProcessing(false);
+ audio_service_.EnableWakeWordDetection(audio_service_.IsAfeWakeWord());
+ if (protocol_ && protocol_->IsAudioChannelOpened()) {
+ while (auto packet = audio_service_.PopPacketFromSendQueue()) {
+ if (!protocol_->SendAudio(std::move(packet))) {
+ break;
+ }
+ }
+ protocol_->SendStopListening();
+ }
+ if (GetDeviceState() == kDeviceStateListening) {
+ SetDeviceState(kDeviceStateIdle);
}
}
@@ -1074,37 +1378,23 @@
return;
}
- opening_audio_channel_ = true;
- if (!Board::GetInstance().EnsureNetworkReadyForWake(30000)) {
- opening_audio_channel_ = false;
- SetDeviceState(kDeviceStateIdle);
+ if (!OpenAudioChannelWithRetry(2)) {
+ ResumeSdCardMp3AfterVoice();
audio_service_.EnableWakeWordDetection(true);
+ SetDeviceState(kDeviceStateIdle);
return;
}
- if (!protocol_->IsAudioChannelOpened()) {
- if (!protocol_->OpenAudioChannel()) {
- opening_audio_channel_ = false;
- ResumeSdCardMp3AfterVoice();
- audio_service_.EnableWakeWordDetection(true);
- return;
- }
- }
- opening_audio_channel_ = false;
ESP_LOGI(TAG, "Wake word detected: %s", wake_word.c_str());
#if CONFIG_SEND_WAKE_WORD_DATA
- // Encode and send the wake word data to the server
while (auto packet = audio_service_.PopWakeWordPacket()) {
if (!protocol_->SendAudio(std::move(packet))) {
break;
}
}
- // Set the chat state to wake word detected
protocol_->SendWakeWordDetected(wake_word);
SetListeningMode(GetDefaultListeningMode());
#else
- // Set flag to play popup sound after state changes to listening
- // (PlaySound here would be cleared by ResetDecoder in EnableVoiceProcessing)
play_popup_on_listening_ = true;
SetListeningMode(GetDefaultListeningMode());
#endif
@@ -1132,20 +1422,35 @@
case kDeviceStateUnknown:
case kDeviceStateIdle:
opening_audio_channel_ = false;
- tts_audio_expected_ = false;
- tts_response_started_ = false;
- tts_audio_received_ = false;
- tts_stop_pending_ = false;
- tts_stop_waiting_ = false;
- tts_idle_waiting_ = false;
- tts_speaking_state_ready_ = false;
- tts_activity_generation_++;
+ if (!suppress_wake_until_listen_) {
+ tts_audio_expected_ = false;
+ tts_response_started_ = false;
+ tts_audio_received_ = false;
+ tts_stop_pending_ = false;
+ tts_stop_waiting_ = false;
+ tts_idle_waiting_ = false;
+ tts_speaking_state_ready_ = false;
+ tts_activity_generation_++;
+ }
ResumeSdCardMp3AfterVoice();
display->SetStatus(Lang::Strings::STANDBY);
display->ClearChatMessages(); // Clear messages first
display->SetEmotion("neutral"); // Then set emotion (wechat mode checks child count)
audio_service_.EnableVoiceProcessing(false);
- audio_service_.EnableWakeWordDetection(true);
+ if (suppress_wake_until_listen_) {
+ audio_service_.EnableWakeWordDetection(false);
+ audio_service_.SetAudioPowerInhibit(true);
+ audio_service_.KeepCodecReadyForPlayback();
+ } else if (Board_IsRepeaterTask()) {
+ audio_service_.EnableVoiceProcessing(false);
+ audio_service_.EnableWakeWordDetection(audio_service_.IsAfeWakeWord());
+ audio_service_.SetAudioPowerInhibit(false);
+ } else if (IsSdCardMp3Playing() && !IsSdCardMp3Paused()) {
+ ApplyMp3CommandOnlyMic();
+ } else {
+ audio_service_.EnableWakeWordDetection(true);
+ audio_service_.SetAudioPowerInhibit(false);
+ }
break;
case kDeviceStateConnecting:
display->SetStatus(Lang::Strings::CONNECTING);
@@ -1157,28 +1462,25 @@
display->SetStatus(Lang::Strings::LISTENING);
display->SetEmotion("neutral");
- // Make sure the audio processor is running
- if (play_popup_on_listening_ || !audio_service_.IsAudioProcessorRunning()) {
- // For auto mode, wait for playback queue to be empty before enabling voice processing
- // This prevents audio truncation when STOP arrives late due to network jitter
- if (listening_mode_ == kListeningModeAutoStop) {
- audio_service_.WaitForPlaybackQueueEmpty();
+ if (IsSdCardMp3Playing() && !IsSdCardMp3Paused() && !Board_IsShadowingTask()) {
+ ApplyMp3CommandOnlyMic();
+ } else {
+ if (play_popup_on_listening_ || !audio_service_.IsAudioProcessorRunning()) {
+ if (listening_mode_ == kListeningModeAutoStop && !skip_playback_wait_on_listen_) {
+ audio_service_.WaitForPlaybackQueueEmpty();
+ }
+ protocol_->SendStartListening(listening_mode_);
}
-
- // Send the start listening command
- protocol_->SendStartListening(listening_mode_);
audio_service_.EnableVoiceProcessing(true);
- }
-
#ifdef CONFIG_WAKE_WORD_DETECTION_IN_LISTENING
- // Enable wake word detection in listening mode (configured via Kconfig)
- audio_service_.EnableWakeWordDetection(audio_service_.IsAfeWakeWord());
+ audio_service_.EnableWakeWordDetection(audio_service_.IsAfeWakeWord());
#else
- // Disable wake word detection in listening mode
- audio_service_.EnableWakeWordDetection(false);
+ audio_service_.EnableWakeWordDetection(false);
#endif
-
- // Play popup sound after ResetDecoder (in EnableVoiceProcessing) has been called
+ }
+ audio_service_.SetAudioPowerInhibit(false);
+ suppress_wake_until_listen_ = false;
+ skip_playback_wait_on_listen_ = false;
if (play_popup_on_listening_) {
play_popup_on_listening_ = false;
audio_service_.PlaySound(Lang::Sounds::OGG_POPUP);
@@ -1186,19 +1488,35 @@
break;
case kDeviceStateSpeaking:
tts_speaking_state_ready_ = false;
+ tts_playback_empty_ticks_ = 0;
+ tts_stop_wait_ticks_ = 0;
+ greeting_listen_wait_ticks_ = 0;
+ speaking_started_us_ = esp_timer_get_time();
display->SetStatus(Lang::Strings::SPEAKING);
if (listening_mode_ != kListeningModeRealtime) {
audio_service_.EnableVoiceProcessing(false);
- // Only AFE wake word can be detected in speaking mode
- audio_service_.EnableWakeWordDetection(audio_service_.IsAfeWakeWord());
+ // TTS 鍥炲０浼氭妸鑿滃崟閲岀殑銆岃窡璇�/鎴戜滑鑱婂ぉ鍚с�嶅惉鎴愬彛浠わ紝鑰屼笖 AFE 浼氭尋鐖嗗唴閮� SRAM銆�
+ audio_service_.EnableWakeWordDetection(false);
}
- audio_service_.ResetDecoder();
+ // 娆㈣繋璇嶇殑 opus 鍙兘宸茬粡鍦ㄥ緟鍛芥椂鍏ラ槦銆傝繖閲� ResetDecoder 浼氭竻鎺夊垰鍒扮殑 TTS锛�
+ // 鍚捣鏉ュ儚娌℃挱锛岄殢鍚庡張绛変笉鍒� tts stop锛屼竴鐩村仠鍦ㄨ璇濅腑銆�
+ if (!tts_audio_expected_ && !tts_audio_received_ && audio_service_.IsPlaybackQueueEmpty()) {
+ audio_service_.ResetDecoder();
+ }
+ audio_service_.SetAudioPowerInhibit(true);
+ audio_service_.KeepCodecReadyForPlayback();
+ ESP_LOGI(TAG, "Speaking: codec ready output=%d",
+ Board::GetInstance().GetAudioCodec()->output_enabled() ? 1 : 0);
tts_speaking_state_ready_ = true;
break;
case kDeviceStateWifiConfiguring:
audio_service_.EnableVoiceProcessing(false);
- audio_service_.EnableWakeWordDetection(true);
+ // MultiNet/AFE wake word consumes internal SRAM. The captive portal HTML
+ // and /scan both fail when free SRAM drops to hundreds of bytes
+ // (httpd send EAGAIN, blank page, empty AP list). Keep it off until
+ // WiFi is configured; Idle re-enables detection afterwards.
+ audio_service_.EnableWakeWordDetection(false);
break;
default:
// Do nothing
@@ -1459,19 +1777,31 @@
tts_stop_waiting_ = false;
tts_idle_waiting_ = false;
tts_speaking_state_ready_ = false;
- if (GetDeviceState() == kDeviceStateSpeaking) {
- if (listening_mode_ == kListeningModeManualStop) {
- SetDeviceState(kDeviceStateIdle);
- } else {
- SetDeviceState(kDeviceStateListening);
- }
- }
+ tts_playback_empty_ticks_ = 0;
+ tts_stop_wait_ticks_ = 0;
+ speaking_started_us_ = 0;
+ audio_service_.SetAudioPowerInhibit(false);
+ suppress_wake_until_listen_ = false;
+ pending_detect_text_.clear();
+ pending_detect_retries_left_ = 0;
ResumeSdCardMp3AfterVoice();
if (resume_sdcard_mp3_after_tts_ && !sdcard_mp3_stop_requested_) {
resume_sdcard_mp3_after_tts_ = false;
ResumeSdCardMp3();
} else {
resume_sdcard_mp3_after_tts_ = false;
+ }
+ if (GetDeviceState() == kDeviceStateSpeaking) {
+ if (listening_mode_ == kListeningModeManualStop ||
+ (IsSdCardMp3Playing() && !IsSdCardMp3Paused())) {
+ SetDeviceState(kDeviceStateIdle);
+ } else {
+ listening_mode_ = GetDefaultListeningMode();
+ SetListeningMode(listening_mode_);
+ }
+ }
+ if (IsSdCardMp3Playing() && !IsSdCardMp3Paused()) {
+ ApplyMp3CommandOnlyMic();
}
if (power_off_requested_) {
power_off_requested_ = false;
@@ -1486,18 +1816,22 @@
void Application::ScheduleTtsAudio(std::shared_ptr<AudioStreamPacket> packet) {
Schedule([this, packet]() {
- if (GetDeviceState() != kDeviceStateSpeaking || !tts_audio_expected_) {
+ auto state = GetDeviceState();
+ if (state != kDeviceStateSpeaking && state != kDeviceStateIdle &&
+ state != kDeviceStateListening) {
+ ESP_LOGW(TAG, "TTS audio ignored state=%d expected=%d",
+ static_cast<int>(state), tts_audio_expected_ ? 1 : 0);
return;
}
- if (!tts_speaking_state_ready_) {
- ScheduleTtsAudio(packet);
- return;
- }
+ tts_audio_expected_ = true;
tts_audio_received_ = true;
tts_activity_generation_++;
+ tts_playback_empty_ticks_ = 0;
+ audio_service_.KeepCodecReadyForPlayback();
auto audio_packet = std::make_unique<AudioStreamPacket>(std::move(*packet));
- audio_service_.PushPacketToDecodeQueue(std::move(audio_packet));
- ScheduleTtsIdleFallback();
+ if (!audio_service_.PushPacketToDecodeQueue(std::move(audio_packet), true)) {
+ ESP_LOGW(TAG, "TTS audio dropped, decode queue full");
+ }
if (tts_stop_pending_) {
FinishTtsResponseAfterPlayback();
}
@@ -1505,61 +1839,54 @@
}
void Application::FinishTtsResponseAfterPlayback() {
- if (tts_stop_waiting_) {
- return;
- }
- if (tts_audio_expected_ && !tts_audio_received_) {
- ScheduleTtsIdleFallback();
- return;
- }
+ tts_stop_pending_ = true;
tts_stop_waiting_ = true;
- xTaskCreate([](void* arg) {
- auto* app = static_cast<Application*>(arg);
- uint32_t generation = app->tts_activity_generation_;
- app->audio_service_.WaitForPlaybackQueueEmpty();
- vTaskDelay(pdMS_TO_TICKS(800));
- app->Schedule([app, generation]() {
- if (app->tts_activity_generation_ == generation) {
- app->FinishTtsResponse();
- } else {
- app->tts_stop_waiting_ = false;
- if (app->GetDeviceState() == kDeviceStateSpeaking) {
- if (app->tts_stop_pending_) {
- app->FinishTtsResponseAfterPlayback();
- } else {
- app->ScheduleTtsIdleFallback();
- }
- }
- }
- });
- vTaskDelete(NULL);
- }, "tts_finish", 4096, this, 2, NULL);
+ tts_stop_wait_ticks_ = 0;
}
-void Application::ScheduleTtsIdleFallback() {
- if (tts_idle_waiting_ || tts_stop_waiting_) {
+void Application::MaybeFinishTtsAfterPlayback() {
+ if (GetDeviceState() != kDeviceStateSpeaking) {
+ tts_playback_empty_ticks_ = 0;
+ tts_stop_wait_ticks_ = 0;
return;
}
- tts_idle_waiting_ = true;
- xTaskCreate([](void* arg) {
- auto* app = static_cast<Application*>(arg);
- uint32_t generation = app->tts_activity_generation_;
- vTaskDelay(pdMS_TO_TICKS(1500));
- app->Schedule([app, generation]() {
- app->tts_idle_waiting_ = false;
- if (app->tts_activity_generation_ != generation || app->GetDeviceState() != kDeviceStateSpeaking) {
- return;
- }
- if (app->tts_audio_received_) {
- app->FinishTtsResponseAfterPlayback();
- return;
- }
- if (app->tts_stop_pending_ || app->tts_audio_expected_) {
- app->FinishTtsResponse();
- }
- });
- vTaskDelete(NULL);
- }, "tts_idle", 4096, this, 2, NULL);
+
+ const bool playback_empty = audio_service_.IsPlaybackQueueEmpty();
+ if (playback_empty) {
+ tts_playback_empty_ticks_++;
+ } else {
+ tts_playback_empty_ticks_ = 0;
+ }
+
+ const int64_t elapsed_ms = speaking_started_us_ > 0
+ ? (esp_timer_get_time() - speaking_started_us_) / 1000
+ : 0;
+
+ if (tts_stop_waiting_ || tts_stop_pending_) {
+ tts_stop_wait_ticks_++;
+ const int empty_needed = tts_audio_received_ ? 1 : 2;
+ if (playback_empty && tts_playback_empty_ticks_ >= empty_needed) {
+ ESP_LOGI(TAG, "TTS done, return to listen empty=%d wait=%d received=%d",
+ tts_playback_empty_ticks_, tts_stop_wait_ticks_, tts_audio_received_);
+ FinishTtsResponse();
+ } else if (!playback_empty && tts_stop_wait_ticks_ >= 30) {
+ ESP_LOGW(TAG, "TTS playback stuck after stop, return to listen");
+ FinishTtsResponse();
+ }
+ return;
+ }
+
+ // 娆㈣繋璇�/闀垮洖绛旓細stop 娌″埌鏃讹紝鍙湪銆屼竴鐩存病鏀跺埌 opus銆嶆垨銆屾挱瀹屽悗闀挎椂闂寸┖闃熷垪銆嶆墠鍥炲惉銆�
+ // 涓嶈鐢ㄨ璇濆閽� 15s/闃熷垪绌� 2s锛氫腑鑻卞璇濋暱鍙ヤ細鍦ㄥ彞闂寸┖闅欐垨 15 绉掑琚帎鏂��
+ if (!tts_audio_received_ && elapsed_ms >= 5000) {
+ ESP_LOGW(TAG, "TTS stop missing, return to listen elapsed=%lldms received=0 empty=%d",
+ (long long)elapsed_ms, tts_playback_empty_ticks_);
+ FinishTtsResponse();
+ } else if (playback_empty && tts_audio_received_ && tts_playback_empty_ticks_ >= 8) {
+ ESP_LOGW(TAG, "TTS stop missing after idle playback elapsed=%lldms empty=%d",
+ (long long)elapsed_ms, tts_playback_empty_ticks_);
+ FinishTtsResponse();
+ }
}
bool Application::CanEnterSleepMode() {
diff --git a/xiaozhi-esp32-2.2.6/main/application.h b/xiaozhi-esp32-2.2.6/main/application.h
index 8842bf0..4713b56 100644
--- a/xiaozhi-esp32-2.2.6/main/application.h
+++ b/xiaozhi-esp32-2.2.6/main/application.h
@@ -105,6 +105,7 @@
*/
void StopListening();
void SendListenDetect(const std::string& text, bool open_if_idle = true);
+ bool SendListenDetectNow(const std::string& text);
void Reboot();
void WakeWordInvoke(const std::string& wake_word);
@@ -125,6 +126,9 @@
AecMode GetAecMode() const { return aec_mode_; }
void PlaySound(const std::string_view& sound);
AudioService& GetAudioService() { return audio_service_; }
+ void ApplyMp3CommandOnlyMic();
+ void ResumeListeningAfterMp3Clip();
+ void CloseShadowingListenWindow();
/**
* Reset protocol resources (thread-safe)
@@ -154,10 +158,18 @@
bool assets_version_checked_ = false;
bool local_assets_applied_ = false;
bool play_popup_on_listening_ = false; // Flag to play popup sound after state changes to listening
+ bool skip_playback_wait_on_listen_ = false;
bool resume_sdcard_mp3_after_tts_ = false;
bool resume_sdcard_mp3_after_voice_ = false;
bool sdcard_mp3_stop_requested_ = false;
bool opening_audio_channel_ = false;
+ bool suppress_wake_until_listen_ = false;
+ bool network_error_suppressed_ = false;
+ std::string pending_detect_text_;
+ int pending_detect_retry_ticks_ = 0;
+ int pending_detect_retries_left_ = 0;
+ int greeting_listen_wait_ticks_ = 0;
+ int boot_wake_restore_ticks_ = 0;
bool tts_audio_expected_ = false;
bool tts_response_started_ = false;
bool tts_audio_received_ = false;
@@ -166,6 +178,9 @@
bool tts_idle_waiting_ = false;
bool tts_speaking_state_ready_ = false;
uint32_t tts_activity_generation_ = 0;
+ int tts_playback_empty_ticks_ = 0;
+ int tts_stop_wait_ticks_ = 0;
+ int64_t speaking_started_us_ = 0;
bool standby_requested_ = false;
bool power_off_requested_ = false;
std::atomic_bool power_off_standby_active_{false};
@@ -184,7 +199,9 @@
void HandleActivationDoneEvent();
void HandleWakeWordDetectedEvent();
void ContinueOpenAudioChannel(ListeningMode mode);
+ void ContinueOpenAudioChannelForDetect(const std::string& detect_text);
void ContinueWakeWordInvoke(const std::string& wake_word);
+ bool OpenAudioChannelWithRetry(int attempts);
// Activation task (runs in background)
void ActivationTask();
@@ -197,8 +214,8 @@
void ShowActivationCode(const std::string& code, const std::string& message);
void FinishTtsResponse();
void FinishTtsResponseAfterPlayback();
+ void MaybeFinishTtsAfterPlayback();
void ScheduleTtsAudio(std::shared_ptr<AudioStreamPacket> packet);
- void ScheduleTtsIdleFallback();
void PauseSdCardMp3ForVoice();
void ResumeSdCardMp3AfterVoice();
void SetListeningMode(ListeningMode mode);
diff --git a/xiaozhi-esp32-2.2.6/main/audio/audio_codec.cc b/xiaozhi-esp32-2.2.6/main/audio/audio_codec.cc
index 03e2baa..6b39cc6 100644
--- a/xiaozhi-esp32-2.2.6/main/audio/audio_codec.cc
+++ b/xiaozhi-esp32-2.2.6/main/audio/audio_codec.cc
@@ -18,6 +18,12 @@
Write(data.data(), data.size());
}
+void AudioCodec::OutputPcm(const int16_t* data, int samples) {
+ if (data && samples > 0) {
+ Write(data, samples);
+ }
+}
+
bool AudioCodec::InputData(std::vector<int16_t>& data) {
int samples = Read(data.data(), data.size());
if (samples > 0) {
diff --git a/xiaozhi-esp32-2.2.6/main/audio/audio_codec.h b/xiaozhi-esp32-2.2.6/main/audio/audio_codec.h
index 819e646..f6155b0 100644
--- a/xiaozhi-esp32-2.2.6/main/audio/audio_codec.h
+++ b/xiaozhi-esp32-2.2.6/main/audio/audio_codec.h
@@ -25,6 +25,7 @@
virtual void EnableOutput(bool enable);
virtual void OutputData(std::vector<int16_t>& data);
+ void OutputPcm(const int16_t* data, int samples);
virtual bool InputData(std::vector<int16_t>& data);
virtual void Start();
diff --git a/xiaozhi-esp32-2.2.6/main/audio/audio_service.cc b/xiaozhi-esp32-2.2.6/main/audio/audio_service.cc
index 4489cae..209f7fe 100644
--- a/xiaozhi-esp32-2.2.6/main/audio/audio_service.cc
+++ b/xiaozhi-esp32-2.2.6/main/audio/audio_service.cc
@@ -1,6 +1,7 @@
#include "audio_service.h"
#include <esp_log.h>
#include <cstring>
+#include <chrono>
#define RATE_CVT_CFG(_src_rate, _dest_rate, _channel) \
(esp_ae_rate_cvt_cfg_t) \
@@ -554,6 +555,11 @@
return;
}
+ const bool running = IsWakeWordRunning();
+ if (enable == running && (!enable || wake_word_initialized_)) {
+ return;
+ }
+
ESP_LOGD(TAG, "%s wake word detection", enable ? "Enabling" : "Disabling");
if (enable) {
if (!wake_word_initialized_) {
@@ -577,6 +583,15 @@
wake_word_->Stop();
xEventGroupClearBits(event_group_, AS_EVENT_WAKE_WORD_RUNNING);
}
+}
+
+void AudioService::EnsureVoiceProcessorReady() {
+ if (audio_processor_initialized_ || audio_processor_ == nullptr) {
+ return;
+ }
+ ESP_LOGI(TAG, "Preload voice processor");
+ audio_processor_->Initialize(codec_, OPUS_FRAME_DURATION_MS, models_list_);
+ audio_processor_initialized_ = true;
}
void AudioService::EnableVoiceProcessing(bool enable) {
@@ -663,9 +678,14 @@
void AudioService::WaitForPlaybackQueueEmpty() {
std::unique_lock<std::mutex> lock(audio_queue_mutex_);
- audio_queue_cv_.wait(lock, [this]() {
- return service_stopped_ || (audio_decode_queue_.empty() && audio_playback_queue_.empty());
+ audio_queue_cv_.wait_for(lock, std::chrono::milliseconds(5000), [this]() {
+ return service_stopped_ || (audio_decode_queue_.empty() && audio_playback_queue_.empty());
});
+}
+
+bool AudioService::IsPlaybackQueueEmpty() {
+ std::lock_guard<std::mutex> lock(audio_queue_mutex_);
+ return audio_decode_queue_.empty() && audio_playback_queue_.empty();
}
void AudioService::ResetDecoder() {
@@ -682,7 +702,32 @@
audio_queue_cv_.notify_all();
}
+void AudioService::KeepCodecReadyForPlayback() {
+ last_output_time_ = std::chrono::steady_clock::now();
+ last_input_time_ = std::chrono::steady_clock::now();
+ if (codec_ == nullptr) {
+ return;
+ }
+ if (!codec_->output_enabled()) {
+ esp_timer_stop(audio_power_timer_);
+ esp_timer_start_periodic(audio_power_timer_, AUDIO_POWER_CHECK_INTERVAL_MS * 1000);
+ codec_->EnableOutput(true);
+ }
+}
+
+void AudioService::SetAudioPowerInhibit(bool inhibit) {
+ audio_power_inhibit_ = inhibit;
+ if (inhibit) {
+ KeepCodecReadyForPlayback();
+ }
+}
+
void AudioService::CheckAndUpdateAudioPowerState() {
+ if (audio_power_inhibit_) {
+ last_input_time_ = std::chrono::steady_clock::now();
+ last_output_time_ = last_input_time_;
+ return;
+ }
auto now = std::chrono::steady_clock::now();
auto input_elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(now - last_input_time_).count();
auto output_elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(now - last_output_time_).count();
diff --git a/xiaozhi-esp32-2.2.6/main/audio/audio_service.h b/xiaozhi-esp32-2.2.6/main/audio/audio_service.h
index 3732ee4..3699578 100644
--- a/xiaozhi-esp32-2.2.6/main/audio/audio_service.h
+++ b/xiaozhi-esp32-2.2.6/main/audio/audio_service.h
@@ -117,12 +117,14 @@
bool IsVoiceDetected() const { return voice_detected_; }
bool IsIdle();
void WaitForPlaybackQueueEmpty();
+ bool IsPlaybackQueueEmpty();
bool IsWakeWordRunning() const { return xEventGroupGetBits(event_group_) & AS_EVENT_WAKE_WORD_RUNNING; }
bool IsAudioProcessorRunning() const { return xEventGroupGetBits(event_group_) & AS_EVENT_AUDIO_PROCESSOR_RUNNING; }
bool IsAfeWakeWord();
void EnableWakeWordDetection(bool enable);
void EnableVoiceProcessing(bool enable);
+ void EnsureVoiceProcessorReady();
void EnableAudioTesting(bool enable);
void EnableDeviceAec(bool enable);
@@ -133,6 +135,8 @@
void PlaySound(const std::string_view& sound);
bool ReadAudioData(std::vector<int16_t>& data, int sample_rate, int samples);
void ResetDecoder();
+ void KeepCodecReadyForPlayback();
+ void SetAudioPowerInhibit(bool inhibit);
void SetModelsList(srmodel_list_t* models_list);
private:
@@ -180,6 +184,7 @@
bool voice_detected_ = false;
bool service_stopped_ = true;
bool audio_input_need_warmup_ = false;
+ bool audio_power_inhibit_ = false;
esp_timer_handle_t audio_power_timer_ = nullptr;
std::chrono::steady_clock::time_point last_input_time_;
diff --git a/xiaozhi-esp32-2.2.6/main/audio/wake_words/afe_wake_word.cc b/xiaozhi-esp32-2.2.6/main/audio/wake_words/afe_wake_word.cc
index bfae615..3436fac 100644
--- a/xiaozhi-esp32-2.2.6/main/audio/wake_words/afe_wake_word.cc
+++ b/xiaozhi-esp32-2.2.6/main/audio/wake_words/afe_wake_word.cc
@@ -79,6 +79,13 @@
multinet_->set_det_threshold(multinet_data_, 0.2f);
esp_mn_commands_clear();
esp_mn_commands_add(1, OFFLINE_COMMAND_WORD_PINYIN);
+ esp_mn_commands_add(2, OFFLINE_REPEATER_WORD_PINYIN);
+ esp_mn_commands_add(3, OFFLINE_START_REPEATER_WORD_PINYIN);
+ esp_mn_commands_add(4, OFFLINE_EXIT_REPEATER_WORD_PINYIN);
+ esp_mn_commands_add(5, OFFLINE_COMMAND_WORD_ALT_PINYIN);
+ esp_mn_commands_add(6, OFFLINE_CHAT_WORD_PINYIN);
+ esp_mn_commands_add(7, OFFLINE_EXIT_SHADOWING_WORD_PINYIN);
+ esp_mn_commands_add(8, OFFLINE_CHAT_WORD_FULL_PINYIN);
esp_mn_commands_update();
multinet_->print_active_speech_commands(multinet_data_);
} else {
@@ -106,7 +113,7 @@
auto this_ = (AfeWakeWord*)arg;
this_->AudioDetectionTask();
vTaskDelete(NULL);
- }, "audio_detection", 4096, this, 3, nullptr);
+ }, "audio_detection", 8192, this, 3, nullptr);
return true;
}
@@ -187,10 +194,27 @@
if (mn_state == ESP_MN_STATE_DETECTED) {
esp_mn_results_t *mn_result = multinet_->get_results(multinet_data_);
for (int i = 0; i < mn_result->num; i++) {
+ int command_id = mn_result->command_id[i];
+ const char *command_text = nullptr;
+ if (command_id == 1 || command_id == 5) {
+ command_text = OFFLINE_COMMAND_WORD;
+ } else if (command_id == 2) {
+ command_text = OFFLINE_REPEATER_WORD;
+ } else if (command_id == 3) {
+ command_text = OFFLINE_START_REPEATER_WORD;
+ } else if (command_id == 4) {
+ command_text = OFFLINE_EXIT_REPEATER_WORD;
+ } else if (command_id == 6) {
+ command_text = OFFLINE_CHAT_WORD;
+ } else if (command_id == 7) {
+ command_text = OFFLINE_EXIT_SHADOWING_WORD;
+ } else if (command_id == 8) {
+ command_text = OFFLINE_CHAT_WORD_FULL;
+ }
ESP_LOGI(TAG, "Offline command detected: command_id=%d, string=%s, prob=%f",
- mn_result->command_id[i], mn_result->string, mn_result->prob[i]);
- if (mn_result->command_id[i] == 1 && offline_command_detected_callback_) {
- offline_command_detected_callback_(OFFLINE_COMMAND_WORD);
+ command_id, mn_result->string, mn_result->prob[i]);
+ if (command_text && offline_command_detected_callback_) {
+ offline_command_detected_callback_(command_text);
}
}
multinet_->clean(multinet_data_);
diff --git a/xiaozhi-esp32-2.2.6/main/audio/wake_words/custom_wake_word.cc b/xiaozhi-esp32-2.2.6/main/audio/wake_words/custom_wake_word.cc
index 129b63d..7fb298f 100644
--- a/xiaozhi-esp32-2.2.6/main/audio/wake_words/custom_wake_word.cc
+++ b/xiaozhi-esp32-2.2.6/main/audio/wake_words/custom_wake_word.cc
@@ -98,6 +98,13 @@
commands_.push_back({CONFIG_CUSTOM_WAKE_WORD, CONFIG_CUSTOM_WAKE_WORD_DISPLAY, "wake"});
#endif
commands_.push_back({OFFLINE_COMMAND_WORD_PINYIN, OFFLINE_COMMAND_WORD, "offline_command"});
+ commands_.push_back({OFFLINE_COMMAND_WORD_ALT_PINYIN, OFFLINE_COMMAND_WORD_ALT, "offline_command"});
+ commands_.push_back({OFFLINE_REPEATER_WORD_PINYIN, OFFLINE_REPEATER_WORD, "offline_command"});
+ commands_.push_back({OFFLINE_START_REPEATER_WORD_PINYIN, OFFLINE_START_REPEATER_WORD, "offline_command"});
+ commands_.push_back({OFFLINE_EXIT_REPEATER_WORD_PINYIN, OFFLINE_EXIT_REPEATER_WORD, "offline_command"});
+ commands_.push_back({OFFLINE_EXIT_SHADOWING_WORD_PINYIN, OFFLINE_EXIT_SHADOWING_WORD, "offline_command"});
+ commands_.push_back({OFFLINE_CHAT_WORD_PINYIN, OFFLINE_CHAT_WORD, "offline_command"});
+ commands_.push_back({OFFLINE_CHAT_WORD_FULL_PINYIN, OFFLINE_CHAT_WORD_FULL, "offline_command"});
} else {
models_ = models_list;
ParseWakenetModelConfig();
@@ -113,6 +120,13 @@
commands_.push_back({BOARD_CUSTOM_WAKE_WORD_PINYIN, BOARD_CUSTOM_WAKE_WORD_DISPLAY, "wake"});
#endif
commands_.push_back({OFFLINE_COMMAND_WORD_PINYIN, OFFLINE_COMMAND_WORD, "offline_command"});
+ commands_.push_back({OFFLINE_COMMAND_WORD_ALT_PINYIN, OFFLINE_COMMAND_WORD_ALT, "offline_command"});
+ commands_.push_back({OFFLINE_REPEATER_WORD_PINYIN, OFFLINE_REPEATER_WORD, "offline_command"});
+ commands_.push_back({OFFLINE_START_REPEATER_WORD_PINYIN, OFFLINE_START_REPEATER_WORD, "offline_command"});
+ commands_.push_back({OFFLINE_EXIT_REPEATER_WORD_PINYIN, OFFLINE_EXIT_REPEATER_WORD, "offline_command"});
+ commands_.push_back({OFFLINE_EXIT_SHADOWING_WORD_PINYIN, OFFLINE_EXIT_SHADOWING_WORD, "offline_command"});
+ commands_.push_back({OFFLINE_CHAT_WORD_PINYIN, OFFLINE_CHAT_WORD, "offline_command"});
+ commands_.push_back({OFFLINE_CHAT_WORD_FULL_PINYIN, OFFLINE_CHAT_WORD_FULL, "offline_command"});
}
if (models_ == nullptr || models_->num == -1) {
@@ -194,9 +208,13 @@
if (mn_state == ESP_MN_STATE_DETECTED) {
esp_mn_results_t *mn_result = multinet_->get_results(multinet_model_data_);
for (int i = 0; i < mn_result->num && running_; i++) {
- ESP_LOGI(TAG, "Custom wake word detected: command_id=%d, string=%s, prob=%f",
- mn_result->command_id[i], mn_result->string, mn_result->prob[i]);
- auto& command = commands_[mn_result->command_id[i] - 1];
+ int command_id = mn_result->command_id[i];
+ ESP_LOGI(TAG, "Custom wake word detected: command_id=%d, string=%s, prob=%f",
+ command_id, mn_result->string, mn_result->prob[i]);
+ if (command_id < 1 || command_id > static_cast<int>(commands_.size())) {
+ continue;
+ }
+ auto& command = commands_[command_id - 1];
if (command.action == "wake") {
last_detected_wake_word_ = command.text;
running_ = false;
diff --git a/xiaozhi-esp32-2.2.6/main/boards/common/wifi_board.cc b/xiaozhi-esp32-2.2.6/main/boards/common/wifi_board.cc
index 7a70d39..118224b 100644
--- a/xiaozhi-esp32-2.2.6/main/boards/common/wifi_board.cc
+++ b/xiaozhi-esp32-2.2.6/main/boards/common/wifi_board.cc
@@ -10,6 +10,7 @@
#include <freertos/task.h>
#include <esp_network.h>
#include <esp_log.h>
+#include <string>
#include <utility>
#include <font_awesome.h>
@@ -55,7 +56,16 @@
// Initialize WiFi manager
WifiManagerConfig config;
config.ssid_prefix = "Xiaozhi";
- config.language = Lang::CODE;
+ // Captive portal languageMap keys use BCP-47 hyphens (zh-CN). Android/WeChat
+ // often report zh_CN; passing the underscore form leaves the <select> on the
+ // first option and can crash loadAPList() via translations[lang].scanning.
+ std::string language = Lang::CODE ? Lang::CODE : "zh-CN";
+ for (char& ch : language) {
+ if (ch == '_') {
+ ch = '-';
+ }
+ }
+ config.language = language;
wifi_manager.Initialize(config);
// Set unified event callback - forward to NetworkEvent with SSID data
@@ -186,6 +196,7 @@
auto& wifi_manager = WifiManager::GetInstance();
wifi_manager.StartConfigAp();
+ ESP_LOGI(TAG, "WiFi config portal at %s", wifi_manager.GetApWebUrl().c_str());
// Show config prompt after a short delay
Application::GetInstance().Schedule([&wifi_manager]() {
diff --git a/xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/app_task.cc b/xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/app_task.cc
index b9dcc3d..df9e80e 100644
--- a/xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/app_task.cc
+++ b/xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/app_task.cc
@@ -1,6 +1,7 @@
#include "app_task.h"
#include "application.h"
+#include "board.h"
#include "mqtt.h"
#include "system.h"
@@ -15,6 +16,7 @@
#define TAG "app_task"
#define APP_TASK_NVS_NS "app_task"
#define APP_TASK_NVS_KEY "pending"
+#define APP_TASK_NVS_LAST_REP "last_rep"
#define APP_TASK_JSON_MAX 768
#define APP_TASK_ID_MAX 32
#define APP_TASK_SOURCE_MAX 24
@@ -23,6 +25,8 @@
static char s_current_task[APP_TASK_ID_MAX] = "conversation";
static char s_current_source[APP_TASK_SOURCE_MAX] = "default_chat";
static int s_current_priority = 0;
+static bool s_finish_task_after_mp3 = true;
+static float s_shadowing_listen_ratio = 1.2f;
typedef struct {
const char *app_id;
@@ -34,7 +38,9 @@
static const app_task_skill_map_t k_task_skills[] = {
{"chat", "conversation", {NULL, NULL, NULL, NULL}},
{"english_learning", "repeater", {"self.mp3.play", NULL, NULL, NULL}},
+ {"english_learning", "shadowing", {NULL, NULL, NULL, NULL}},
{"english_translation", "live_translate", {"translate", NULL, NULL, NULL}},
+ {"english_conversation", "bilingual_chat", {NULL, NULL, NULL, NULL}},
};
int Board_AppTask_SourcePriority(const char *source)
@@ -82,10 +88,292 @@
(!task_id || task_id[0] == '\0' || strcmp(task_id, "conversation") == 0);
}
+bool Board_IsShadowingTask(void)
+{
+ return strcmp(s_current_task, "shadowing") == 0;
+}
+
+bool Board_IsRepeaterTask(void)
+{
+ return strcmp(s_current_task, "repeater") == 0;
+}
+
+void Board_SetShadowingListenRatio(float ratio)
+{
+ if (ratio < 1.2f) {
+ ratio = 1.2f;
+ }
+ if (ratio > 1.5f) {
+ ratio = 1.5f;
+ }
+ s_shadowing_listen_ratio = ratio;
+}
+
+unsigned Board_ShadowingListenWindowMs(unsigned duration_ms)
+{
+ if (duration_ms == 0) {
+ return 800;
+ }
+ unsigned wait_ms = static_cast<unsigned>(static_cast<float>(duration_ms) * s_shadowing_listen_ratio);
+ if (wait_ms < 800) {
+ wait_ms = 800;
+ }
+ return wait_ms;
+}
+
static void copy_id(char *dest, size_t dest_size, const char *src, const char *fallback)
{
const char *value = (src && src[0]) ? src : fallback;
strlcpy(dest, value, dest_size);
+}
+
+void Board_SetActiveTask(const char *app_id, const char *task_id)
+{
+ if (!app_id || app_id[0] == '\0' || !task_id || task_id[0] == '\0') {
+ return;
+ }
+ if (strcmp(s_current_app, app_id) == 0 && strcmp(s_current_task, task_id) == 0) {
+ return;
+ }
+ copy_id(s_current_app, sizeof(s_current_app), app_id, "chat");
+ copy_id(s_current_task, sizeof(s_current_task), task_id, "conversation");
+ ESP_LOGI(TAG, "set active task %s/%s", s_current_app, s_current_task);
+}
+
+static void normalize_media_path(char *path)
+{
+ if (!path) {
+ return;
+ }
+ for (char *p = path; *p; ++p) {
+ if (*p == '\\') {
+ *p = '/';
+ }
+ }
+}
+
+static const char *path_without_leading_slash(const char *path)
+{
+ if (!path) {
+ return "";
+ }
+ return path[0] == '/' ? path + 1 : path;
+}
+
+static bool path_has_folder_prefix(const char *path, const char *prefix)
+{
+ if (!path || !prefix || prefix[0] == '\0') {
+ return false;
+ }
+ char normalized[SDCARD_PATH_MAX_LEN];
+ char needle[96];
+ strlcpy(normalized, path, sizeof(normalized));
+ strlcpy(needle, prefix, sizeof(needle));
+ normalize_media_path(normalized);
+ normalize_media_path(needle);
+ const char *hay = path_without_leading_slash(normalized);
+ const char *pre = path_without_leading_slash(needle);
+ size_t n = strlen(pre);
+ if (n == 0 || strncmp(hay, pre, n) != 0) {
+ return false;
+ }
+ return hay[n] == '\0' || hay[n] == '/';
+}
+
+static bool path_belongs_to_current_task(const char *path)
+{
+ if (is_chat_task(s_current_app, s_current_task)) {
+ return true;
+ }
+ char prefix[96];
+ snprintf(prefix, sizeof(prefix), "/%s/%s/", s_current_app, s_current_task);
+ return path_has_folder_prefix(path, prefix);
+}
+
+static bool basename_is_full_mp3(const char *name)
+{
+ if (!name || name[0] == '\0') {
+ return false;
+ }
+ const char *base = strrchr(name, '/');
+ base = base ? base + 1 : name;
+ if (strcasecmp(base, "full.mp3") == 0) {
+ return true;
+ }
+ size_t n = strlen(base);
+ return n > 9 && strcasecmp(base + n - 9, "_full.mp3") == 0;
+}
+
+void Board_BindTaskFromMediaPath(const char *path)
+{
+ // 璇炬枃鏂囦欢鍙兘閮藉湪 /english_learning/shadowing/ 涓嬨�備换鍔″彧鐢� MCP app_id/task_id 鍐冲畾銆�
+ (void)path;
+}
+
+static bool task_owns_play_audio(const char *app_id, const char *task_id)
+{
+ const app_task_skill_map_t *spec = find_task_spec(app_id, task_id);
+ if (!spec) {
+ return false;
+ }
+ for (int i = 0; i < 4 && spec->skills[i]; ++i) {
+ if (strcmp(spec->skills[i], "self.mp3.play") == 0) {
+ return true;
+ }
+ }
+ return false;
+}
+
+static bool first_sd_mp3_name(char *name, size_t name_size, const char *prefix, bool prefer_full)
+{
+ if (!name || name_size == 0) {
+ return false;
+ }
+ name[0] = '\0';
+ const char *list = SDCard_Get_MP3_List();
+ if (!list || list[0] == '\0') {
+ return false;
+ }
+ cJSON *root = cJSON_Parse(list);
+ if (!root) {
+ return false;
+ }
+ const cJSON *files = cJSON_GetObjectItem(root, "files");
+ if (!cJSON_IsArray(files)) {
+ cJSON_Delete(root);
+ return false;
+ }
+ const cJSON *best = NULL;
+ bool best_full = false;
+ const cJSON *file = NULL;
+ cJSON_ArrayForEach(file, files) {
+ const cJSON *item = cJSON_GetObjectItem(file, "name");
+ if (!cJSON_IsString(item) || !item->valuestring || !item->valuestring[0]) {
+ continue;
+ }
+ if (prefix && prefix[0] && !path_has_folder_prefix(item->valuestring, prefix)) {
+ continue;
+ }
+ bool is_full = basename_is_full_mp3(item->valuestring);
+ if (!best) {
+ best = item;
+ best_full = is_full;
+ continue;
+ }
+ if (prefer_full && is_full && !best_full) {
+ best = item;
+ best_full = true;
+ continue;
+ }
+ if ((!prefer_full || is_full == best_full) &&
+ strcmp(item->valuestring, best->valuestring) < 0) {
+ best = item;
+ best_full = is_full;
+ }
+ }
+ bool ok = best != NULL;
+ if (ok) {
+ strlcpy(name, best->valuestring, name_size);
+ }
+ cJSON_Delete(root);
+ return ok;
+}
+
+void Board_SetFinishTaskAfterMp3(bool finish)
+{
+ s_finish_task_after_mp3 = finish;
+}
+
+void Board_RememberRepeaterPlayback(const char *name, float speed, int repeat, bool power_off_after)
+{
+ if (!name || name[0] == '\0') {
+ return;
+ }
+ if (strcmp(s_current_task, "repeater") != 0) {
+ return;
+ }
+ cJSON *root = cJSON_CreateObject();
+ if (!root) {
+ return;
+ }
+ cJSON_AddStringToObject(root, "name", name);
+ cJSON_AddNumberToObject(root, "speed", speed);
+ cJSON_AddNumberToObject(root, "repeat", repeat < 1 ? 1 : repeat);
+ cJSON_AddBoolToObject(root, "power_off_after", power_off_after);
+ char *printed = cJSON_PrintUnformatted(root);
+ cJSON_Delete(root);
+ if (!printed) {
+ return;
+ }
+ nvs_handle_t handle;
+ if (nvs_open(APP_TASK_NVS_NS, NVS_READWRITE, &handle) == ESP_OK) {
+ if (nvs_set_str(handle, APP_TASK_NVS_LAST_REP, printed) == ESP_OK) {
+ nvs_commit(handle);
+ }
+ nvs_close(handle);
+ }
+ free(printed);
+}
+
+bool Board_ResolveRepeaterPlayback(char *name, size_t name_size, float *speed, int *repeat, bool *power_off_after)
+{
+ if (!name || name_size == 0) {
+ return false;
+ }
+ if (name[0] != '\0') {
+ if (speed) {
+ *speed = SDCard_Clamp_MP3_Speed(*speed);
+ }
+ return true;
+ }
+
+ nvs_handle_t handle;
+ if (nvs_open(APP_TASK_NVS_NS, NVS_READONLY, &handle) == ESP_OK) {
+ char buf[APP_TASK_JSON_MAX];
+ size_t length = sizeof(buf);
+ if (nvs_get_str(handle, APP_TASK_NVS_LAST_REP, buf, &length) == ESP_OK) {
+ cJSON *root = cJSON_Parse(buf);
+ if (root) {
+ const cJSON *name_item = cJSON_GetObjectItem(root, "name");
+ const cJSON *repeat_item = cJSON_GetObjectItem(root, "repeat");
+ const cJSON *power_item = cJSON_GetObjectItem(root, "power_off_after");
+ if (cJSON_IsString(name_item) && name_item->valuestring && name_item->valuestring[0]) {
+ strlcpy(name, name_item->valuestring, name_size);
+ if (repeat && cJSON_IsNumber(repeat_item)) {
+ *repeat = repeat_item->valueint;
+ }
+ if (power_off_after) {
+ if (cJSON_IsBool(power_item)) {
+ *power_off_after = cJSON_IsTrue(power_item);
+ } else if (cJSON_IsNumber(power_item)) {
+ *power_off_after = power_item->valueint != 0;
+ }
+ }
+ }
+ cJSON_Delete(root);
+ }
+ }
+ nvs_close(handle);
+ }
+ if (name[0] != '\0' && !path_belongs_to_current_task(name)) {
+ ESP_LOGW(TAG, "discard last_rep from other task: %s", name);
+ name[0] = '\0';
+ }
+
+ if (name[0] == '\0') {
+ char prefix[96];
+ snprintf(prefix, sizeof(prefix), "/%s/%s/lessons", s_current_app, s_current_task);
+ bool prefer_full = strcmp(s_current_task, "repeater") == 0;
+ if (!first_sd_mp3_name(name, name_size, prefix, prefer_full)) {
+ if (is_chat_task(s_current_app, s_current_task)) {
+ first_sd_mp3_name(name, name_size, NULL, false);
+ }
+ }
+ }
+ if (speed) {
+ *speed = SDCard_Clamp_MP3_Speed(*speed);
+ }
+ return name[0] != '\0';
}
esp_err_t Board_SavePendingTask(const char *app_id, const char *task_id, const char *params_json,
@@ -195,6 +483,7 @@
const char *params_json, const char *source)
{
if (!skill || skill[0] == '\0') {
+ // 涓婃姤 [app_task]銆係endListenDetect 浼氬紑閫氶亾浣嗕繚鎸� idle锛屼笉寮� listening/AFE銆�
notify_server(app_id, task_id, source, true);
return;
}
@@ -214,7 +503,7 @@
strlcpy(name, name_item->valuestring, sizeof(name));
}
if (cJSON_IsNumber(speed_item)) {
- speed = static_cast<float>(speed_item->valuedouble);
+ speed = SDCard_Clamp_MP3_Speed(static_cast<float>(speed_item->valuedouble));
}
if (cJSON_IsNumber(repeat_item)) {
repeat = repeat_item->valueint;
@@ -227,19 +516,29 @@
cJSON_Delete(params);
}
}
- if (name[0] == '\0') {
- ESP_LOGW(TAG, "self.mp3.play missing name");
+ if (!Board_ResolveRepeaterPlayback(name, sizeof(name), &speed, &repeat, &power_off_after)) {
+ ESP_LOGW(TAG, "self.mp3.play has no last lesson and no SD MP3");
+ notify_server(app_id, task_id, source, true);
return;
}
if (repeat < 1) {
repeat = 1;
}
+ speed = SDCard_Clamp_MP3_Speed(speed);
+ Board_SetFinishTaskAfterMp3(true);
+ // 鏈夌綉鐨勮闊冲叆鍙ｏ細鍏堣 Server 鎾姤杩涘満璇嶏紝闊抽鐢� MCP 鍦� TTS 缁撴潫鍚庡啀鎾��
+ if (source && strcmp(source, "voice") == 0 && Board::GetInstance().IsNetworkReady()) {
+ Board_RememberRepeaterPlayback(name, speed, repeat, power_off_after);
+ notify_server(app_id, task_id, source, true);
+ return;
+ }
esp_err_t err = SDCard_Play_MP3_With_Speed_Repeat(name, speed, repeat, power_off_after);
if (err != ESP_OK) {
ESP_LOGW(TAG, "self.mp3.play failed: %s", esp_err_to_name(err));
- Board_OnSkillPlaybackFinished(true, false);
+ Board_OnSkillPlaybackFinished(true, false, 0);
return;
}
+ Board_RememberRepeaterPlayback(name, speed, repeat, power_off_after);
notify_server(app_id, task_id, source, false);
return;
}
@@ -256,18 +555,31 @@
return ESP_ERR_INVALID_ARG;
}
const char *src = (source && source[0]) ? source : "mqtt_immediate";
- const char *tid = (task_id && task_id[0]) ? task_id : "conversation";
+ const char *tid = (task_id && task_id[0]) ? task_id : NULL;
const app_task_skill_map_t *spec = find_task_spec(app_id, tid);
if (!spec) {
- ESP_LOGW(TAG, "unknown app/task: %s/%s", app_id, tid);
+ ESP_LOGW(TAG, "unknown app/task: %s/%s", app_id, tid ? tid : "");
return ESP_ERR_NOT_FOUND;
}
+ tid = spec->task_id;
int new_priority = Board_AppTask_SourcePriority(src);
if (new_priority < s_current_priority) {
ESP_LOGW(TAG, "reject %s/%s source=%s prio %d < current %d",
app_id, tid, src, new_priority, s_current_priority);
return ESP_ERR_INVALID_STATE;
+ }
+
+ if (strcmp(s_current_app, app_id) == 0 && strcmp(s_current_task, tid) == 0 &&
+ strcmp(app_id, "chat") == 0 && strcmp(src, "voice") == 0) {
+ if (SDCard_Is_Playing()) {
+ ESP_LOGI(TAG, "stop leftover MP3 while already in chat");
+ preempt_current_skills();
+ }
+ // 璺熻鐢� MCP 璧锋挱鏃舵湰鍦颁粛鍙兘鏄� chat銆傘�岄��鍑鸿窡璇汇�嶅繀椤婚�氱煡 Server 鍙栨秷鍚獥銆�
+ // 涓嶈鍦ㄨ繖閲� ResumeListening锛氫細 ResetDecoder銆佹彁鍓嶅紑 AFE锛屾杩庤瘝/缁撴潫鎾姤浼氳娓呮帀銆�
+ notify_server("chat", "conversation", "voice", true);
+ return ESP_OK;
}
preempt_current_skills();
@@ -288,15 +600,107 @@
return ESP_OK;
}
-void Board_OnSkillPlaybackFinished(bool stopped, bool power_off_requested)
+void Board_OnSkillPlaybackFinished(bool stopped, bool power_off_requested, unsigned duration_ms)
{
(void)power_off_requested;
- ESP_LOGI(TAG, "skill playback %s", stopped ? "stopped" : "finished");
- copy_id(s_current_app, sizeof(s_current_app), "chat", "chat");
- copy_id(s_current_task, sizeof(s_current_task), "conversation", "conversation");
- copy_id(s_current_source, sizeof(s_current_source), "default_chat", "default_chat");
- s_current_priority = 0;
- Board_SavePendingTask("chat", "conversation", "{}", "default_chat", NULL);
+ // MP3 浠诲姟鏍堝湪 PSRAM锛屼笉鑳藉湪杩欓噷 nvs_set_str / 鏀圭姸鎬佹満銆�
+ Application::GetInstance().Schedule([stopped, duration_ms]() {
+ ESP_LOGI(TAG, "skill playback %s duration_ms=%u task=%s/%s",
+ stopped ? "stopped" : "finished", duration_ms, s_current_app, s_current_task);
+
+ char app_id[APP_TASK_ID_MAX];
+ char task_id[APP_TASK_ID_MAX];
+ copy_id(app_id, sizeof(app_id), s_current_app, "chat");
+ copy_id(task_id, sizeof(task_id), s_current_task, "conversation");
+ const bool shadowing = strcmp(task_id, "shadowing") == 0;
+ const bool repeater = strcmp(task_id, "repeater") == 0;
+ // 澶嶈娓呭崟鏄惁缁撴潫鐢� Server 鍐冲畾銆傝澶囦笉瑕佸湪姝ゅ啓 NVS 鍒囧洖 chat銆�
+ const bool end_task = !shadowing && !repeater &&
+ task_owns_play_audio(app_id, task_id) && s_finish_task_after_mp3;
+
+ if (end_task) {
+ copy_id(s_current_app, sizeof(s_current_app), "chat", "chat");
+ copy_id(s_current_task, sizeof(s_current_task), "conversation", "conversation");
+ copy_id(s_current_source, sizeof(s_current_source), "default_chat", "default_chat");
+ s_current_priority = 0;
+ Board_SavePendingTask("chat", "conversation", "{}", "default_chat", NULL);
+ }
+
+ char detect[208];
+ snprintf(detect, sizeof(detect), "[skill_event]audio.%s|%s|%s|%u",
+ stopped ? "stopped" : "completed", app_id, task_id, duration_ms);
+ const bool open_mic = !stopped && shadowing;
+ // 鍚獥浠庤鏂囦竴鍋滃氨寮�濮嬨�備笉瑕佺瓑寮�楹�/AFE 鍒濆鍖栧悗鍐嶄笂鎶� completed銆�
+ if (!Application::GetInstance().SendListenDetectNow(detect)) {
+ Application::GetInstance().SendListenDetect(detect, false);
+ }
+ if (open_mic) {
+ Application::GetInstance().GetAudioService().ResetDecoder();
+ Application::GetInstance().ResumeListeningAfterMp3Clip();
+ } else if (repeater && !stopped) {
+ // 娓呭崟宸叉挱瀹岋紝椹笂瑕佹挱銆屽璇荤粨鏉熴�嶏細淇濇寔鍔熸斁锛屼笉瑕佸啀 ResetDecoder銆�
+ Application::GetInstance().GetAudioService().KeepCodecReadyForPlayback();
+ Application::GetInstance().GetAudioService().SetAudioPowerInhibit(true);
+ } else if (!repeater) {
+ Application::GetInstance().GetAudioService().ResetDecoder();
+ }
+ });
+}
+
+void Board_OnShadowingClipFinished(unsigned duration_ms)
+{
+ Application::GetInstance().Schedule([duration_ms]() {
+ ESP_LOGI(TAG, "shadowing clip finished duration_ms=%u task=%s/%s",
+ duration_ms, s_current_app, s_current_task);
+
+ char app_id[APP_TASK_ID_MAX];
+ char task_id[APP_TASK_ID_MAX];
+ copy_id(app_id, sizeof(app_id), s_current_app, "chat");
+ copy_id(task_id, sizeof(task_id), s_current_task, "shadowing");
+
+ char detect[208];
+ snprintf(detect, sizeof(detect), "[skill_event]audio.completed|%s|%s|%u",
+ app_id, task_id, duration_ms);
+ if (!Application::GetInstance().SendListenDetectNow(detect)) {
+ Application::GetInstance().SendListenDetect(detect, false);
+ }
+ Application::GetInstance().ResumeListeningAfterMp3Clip();
+ });
+}
+
+void Board_OnShadowingPlaylistFinished(void)
+{
+ Application::GetInstance().Schedule([]() {
+ ESP_LOGI(TAG, "shadowing playlist finished task=%s/%s", s_current_app, s_current_task);
+
+ char app_id[APP_TASK_ID_MAX];
+ char task_id[APP_TASK_ID_MAX];
+ copy_id(app_id, sizeof(app_id), s_current_app, "chat");
+ copy_id(task_id, sizeof(task_id), s_current_task, "shadowing");
+
+ char detect[208];
+ snprintf(detect, sizeof(detect), "[skill_event]audio.playlist_completed|%s|%s",
+ app_id, task_id);
+ if (!Application::GetInstance().SendListenDetectNow(detect)) {
+ ESP_LOGW(TAG, "playlist_completed not sent, return to chat locally");
+ copy_id(s_current_app, sizeof(s_current_app), "chat", "chat");
+ copy_id(s_current_task, sizeof(s_current_task), "conversation", "conversation");
+ copy_id(s_current_source, sizeof(s_current_source), "default_chat", "default_chat");
+ s_current_priority = 0;
+ Board_SavePendingTask("chat", "conversation", "{}", "default_chat", NULL);
+ auto &app = Application::GetInstance();
+ app.GetAudioService().SetAudioPowerInhibit(false);
+ app.GetAudioService().EnableVoiceProcessing(false);
+ app.GetAudioService().EnableWakeWordDetection(true);
+ if (app.GetDeviceState() == kDeviceStateListening ||
+ app.GetDeviceState() == kDeviceStateConnecting) {
+ app.SetDeviceState(kDeviceStateIdle);
+ }
+ return;
+ }
+ Application::GetInstance().GetAudioService().KeepCodecReadyForPlayback();
+ Application::GetInstance().GetAudioService().SetAudioPowerInhibit(true);
+ });
}
void Board_ResolveBootOrRestore(bool from_alarm_wake)
@@ -313,9 +717,12 @@
return;
}
- if (from_alarm_wake && strcmp(source, "schedule") == 0 && !is_chat_task(app_id, task_id)) {
- ESP_LOGI(TAG, "alarm restore runs pending %s/%s", app_id, task_id);
- Board_StartAppTask(app_id, task_id, params, "schedule");
+ const bool scheduled = strcmp(source, "schedule") == 0 || strcmp(source, "mqtt_immediate") == 0;
+ const bool offline = !Board::GetInstance().IsNetworkReady();
+ if (!is_chat_task(app_id, task_id) && (from_alarm_wake || scheduled || offline)) {
+ ESP_LOGI(TAG, "restore pending %s/%s source=%s offline=%d",
+ app_id, task_id, source, offline ? 1 : 0);
+ Board_StartAppTask(app_id, task_id, params, source);
return;
}
@@ -328,6 +735,10 @@
ESP_LOGI(TAG, "ignore voice_entry, current priority=%d", s_current_priority);
return;
}
+ if (!Board::GetInstance().IsNetworkReady()) {
+ ESP_LOGW(TAG, "voice_entry skipped, network not ready; say 澶嶈鏈� to play locally");
+ return;
+ }
Application::GetInstance().SendListenDetect("[voice_entry]", true);
}
@@ -336,36 +747,66 @@
if (!command || command[0] == '\0') {
return false;
}
- if (strcmp(command, "鍒囨崲妯″紡") == 0 || strcmp(command, "qie huan mo shi") == 0) {
+ const bool speaking = Application::GetInstance().GetDeviceState() == kDeviceStateSpeaking;
+ if (Board_Is_Mode_Switch_Command(command)) {
Board_EnterVoiceEntry();
return true;
}
- if (strstr(command, "鎴戜滑鏉ヨ亰澶╁惂") || strstr(command, "鑱婁細澶╁惂")) {
- if (s_current_priority >= 2) {
- ESP_LOGI(TAG, "ignore voice chat switch, priority=%d", s_current_priority);
- return true;
- }
+ if (Board_Is_Chat_Return_Command(command)) {
+ s_current_priority = 0;
Board_StartAppTask("chat", "conversation", "{}", "voice");
return true;
}
- if (strstr(command, "鎵撳紑缈昏瘧") || strstr(command, "寮�濮嬬炕璇�")) {
+ if (speaking) {
+ if (Board_Is_App_Launch_Command(command)) {
+ ESP_LOGI(TAG, "ignore app launch while TTS is speaking: %s", command);
+ return true;
+ }
+ return false;
+ }
+ if (strcmp(command, OFFLINE_REPEATER_WORD) == 0 ||
+ strcmp(command, OFFLINE_START_REPEATER_WORD) == 0 ||
+ strstr(command, "寮�濮嬪璇�") || strstr(command, "澶嶈鏈�")) {
if (s_current_priority >= 2) {
return true;
}
- Board_StartAppTask("english_translation", "live_translate", "{}", "voice");
- return true;
- }
- if (strstr(command, "寮�濮嬪璇�") || strstr(command, "鎵撳紑澶嶈")) {
- if (s_current_priority >= 2) {
- return true;
- }
- Board_EnterVoiceEntry();
- return true;
- }
- if (strstr(command, "鏁板瓧鑳岃") || strstr(command, "鎴愯鎺ラ緳") || strstr(command, "鍙ｇ畻")) {
- ESP_LOGI(TAG, "reserved voice slot: %s", command);
- Board_EnterVoiceEntry();
+ Board_StartAppTask("english_learning", "repeater", "{}", "voice");
return true;
}
return false;
}
+
+static bool layout_id_is_safe(const char *id)
+{
+ if (!id || id[0] == '\0' || strlen(id) >= APP_TASK_ID_MAX) {
+ return false;
+ }
+ for (const char *p = id; *p; ++p) {
+ if (!((*p >= 'a' && *p <= 'z') || (*p >= 'A' && *p <= 'Z') || (*p >= '0' && *p <= '9') ||
+ *p == '_' || *p == '-')) {
+ return false;
+ }
+ }
+ return true;
+}
+
+esp_err_t Board_EnsureAppLayout(const char *app_id, const char *task_id)
+{
+ if (!layout_id_is_safe(app_id)) {
+ return ESP_ERR_INVALID_ARG;
+ }
+ char path[SDCARD_PATH_MAX_LEN];
+ int written;
+ if (task_id && task_id[0]) {
+ if (!layout_id_is_safe(task_id)) {
+ return ESP_ERR_INVALID_ARG;
+ }
+ written = snprintf(path, sizeof(path), "/%s/%s/lessons", app_id, task_id);
+ } else {
+ written = snprintf(path, sizeof(path), "/%s", app_id);
+ }
+ if (written <= 0 || (size_t)written >= sizeof(path)) {
+ return ESP_ERR_INVALID_SIZE;
+ }
+ return SDCard_Ensure_Business_Dir(path);
+}
diff --git a/xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/app_task.h b/xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/app_task.h
index 21deddf..991d6ed 100644
--- a/xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/app_task.h
+++ b/xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/app_task.h
@@ -3,6 +3,7 @@
#include "esp_err.h"
#include <stdbool.h>
+#include <stddef.h>
#ifdef __cplusplus
extern "C" {
@@ -14,8 +15,20 @@
void Board_ResolveBootOrRestore(bool from_alarm_wake);
void Board_EnterVoiceEntry(void);
bool Board_Handle_Voice_Command(const char *command);
-void Board_OnSkillPlaybackFinished(bool stopped, bool power_off_requested);
+void Board_OnSkillPlaybackFinished(bool stopped, bool power_off_requested, unsigned duration_ms);
+void Board_OnShadowingClipFinished(unsigned duration_ms);
+void Board_OnShadowingPlaylistFinished(void);
+void Board_SetShadowingListenRatio(float ratio);
+unsigned Board_ShadowingListenWindowMs(unsigned duration_ms);
+bool Board_IsShadowingTask(void);
+bool Board_IsRepeaterTask(void);
+void Board_BindTaskFromMediaPath(const char *path);
+void Board_SetActiveTask(const char *app_id, const char *task_id);
int Board_AppTask_CurrentPriority(void);
+bool Board_ResolveRepeaterPlayback(char *name, size_t name_size, float *speed, int *repeat, bool *power_off_after);
+void Board_RememberRepeaterPlayback(const char *name, float speed, int repeat, bool power_off_after);
+void Board_SetFinishTaskAfterMp3(bool finish);
+esp_err_t Board_EnsureAppLayout(const char *app_id, const char *task_id);
#ifdef __cplusplus
}
diff --git a/xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/mp3_player.cc b/xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/mp3_player.cc
index cc87034..ca97f2b 100644
--- a/xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/mp3_player.cc
+++ b/xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/mp3_player.cc
@@ -6,7 +6,9 @@
#include "application.h"
#include <esp_log.h>
+#include <esp_heap_caps.h>
#include <freertos/FreeRTOS.h>
+#include <freertos/semphr.h>
#include <freertos/task.h>
#include <esp_audio_simple_dec.h>
@@ -18,32 +20,60 @@
#include <cstdio>
#include <cstdlib>
#include <cstring>
-#include <vector>
#define TAG "sdcard_mp3"
#define MP3_READ_BUFFER_SIZE 512
#define MP3_OUTPUT_BUFFER_SIZE 4096
#define MP3_PLAYER_STACK_SIZE 8192
-#define MP3_PLAYER_PRIORITY 5
+#define MP3_PLAYER_PRIORITY 2
+#define MP3_PSRAM_CACHE_MAX (768 * 1024)
typedef struct {
- char path[SDCARD_PATH_MAX_LEN];
- float speed;
- int repeat;
- bool power_off_after;
+ int count;
+ sdcard_mp3_playlist_item_t items[SDCARD_MP3_PLAYLIST_MAX];
} mp3_player_task_arg_t;
static TaskHandle_t mp3_player_task_handle;
+static StackType_t *mp3_player_stack;
+static StaticTask_t *mp3_player_tcb;
static volatile bool mp3_stop_requested;
static volatile bool mp3_pause_requested;
static volatile int mp3_repeat_left;
+static TickType_t s_mp3_audible_started;
+static SemaphoreHandle_t s_shadowing_listen_closed;
-static std::vector<int16_t> convert_channels(const int16_t *input, size_t frame_count, uint8_t src_channels, uint8_t dst_channels)
+static int16_t *s_pcm_a = nullptr;
+static int16_t *s_pcm_b = nullptr;
+static size_t s_pcm_a_cap = 0;
+static size_t s_pcm_b_cap = 0;
+
+static int16_t *ensure_psram_samples(int16_t **buf, size_t *cap, size_t samples)
{
- std::vector<int16_t> output(frame_count * dst_channels);
+ if (samples == 0) {
+ return *buf;
+ }
+ if (*buf && *cap >= samples) {
+ return *buf;
+ }
+ int16_t *p = static_cast<int16_t *>(
+ heap_caps_realloc(*buf, samples * sizeof(int16_t), MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT));
+ if (!p) {
+ return nullptr;
+ }
+ *buf = p;
+ *cap = samples;
+ return p;
+}
+
+static bool convert_channels(const int16_t *input, size_t frame_count, uint8_t src_channels, uint8_t dst_channels,
+ int16_t *output)
+{
+ if (!input || !output || frame_count == 0 || src_channels == 0 || dst_channels == 0) {
+ return false;
+ }
if (src_channels == dst_channels) {
- std::memcpy(output.data(), input, output.size() * sizeof(int16_t));
- return output;
+ std::memcpy(output, input, frame_count * dst_channels * sizeof(int16_t));
+ return true;
}
for (size_t frame = 0; frame < frame_count; ++frame) {
@@ -63,24 +93,47 @@
}
}
}
-
- return output;
+ return true;
}
-static bool is_supported_speed(float speed)
+static size_t stretch_for_speed(const int16_t *input, size_t in_frames, uint8_t channels, float speed,
+ int16_t *output, size_t out_cap_frames)
{
- const float supported_speeds[] = {0.5f, 0.75f, 1.0f, 1.25f, 1.5f, 2.0f};
- for (size_t i = 0; i < sizeof(supported_speeds) / sizeof(supported_speeds[0]); ++i) {
- float diff = speed - supported_speeds[i];
- if (diff < 0) {
- diff = -diff;
+ if (!input || !output || channels == 0 || in_frames == 0) {
+ return 0;
+ }
+ if (speed >= 0.999f && speed <= 1.001f) {
+ size_t n = in_frames * channels;
+ std::memcpy(output, input, n * sizeof(int16_t));
+ return in_frames;
+ }
+ size_t out_frames = static_cast<size_t>(static_cast<double>(in_frames) / static_cast<double>(speed) + 0.5);
+ if (out_frames < 1) {
+ out_frames = 1;
+ }
+ if (out_frames > out_cap_frames) {
+ out_frames = out_cap_frames;
+ }
+ for (size_t i = 0; i < out_frames; ++i) {
+ const double src = static_cast<double>(i) * static_cast<double>(speed);
+ size_t i0 = static_cast<size_t>(src);
+ size_t i1 = i0 + 1;
+ if (i0 >= in_frames) {
+ i0 = in_frames - 1;
}
- if (diff < 0.001f) {
- return true;
+ if (i1 >= in_frames) {
+ i1 = in_frames - 1;
+ }
+ const float t = static_cast<float>(src - static_cast<double>(i0));
+ for (uint8_t c = 0; c < channels; ++c) {
+ const float a = input[i0 * channels + c];
+ const float b = input[i1 * channels + c];
+ output[i * channels + c] = static_cast<int16_t>(a + (b - a) * t);
}
}
- return false;
+ return out_frames;
}
+
static bool wait_if_paused()
{
@@ -90,8 +143,68 @@
return !mp3_stop_requested;
}
+static void wait_for_sd_dma(void)
+{
+ for (int i = 0; i < 20; ++i) {
+ size_t largest = heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL | MALLOC_CAP_DMA);
+ if (largest >= 4096) {
+ return;
+ }
+ ESP_LOGW(TAG, "wait SD DMA largest=%u try=%d", (unsigned)largest, i);
+ vTaskDelay(pdMS_TO_TICKS(40));
+ }
+}
+
+static uint8_t *load_mp3_to_psram(const char *path, size_t *out_len)
+{
+ *out_len = 0;
+ wait_for_sd_dma();
+ FILE *fp = fopen(path, "rb");
+ if (!fp) {
+ return nullptr;
+ }
+ size_t cap = 32 * 1024;
+ uint8_t *buf = static_cast<uint8_t *>(
+ heap_caps_malloc(cap, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT));
+ if (!buf) {
+ fclose(fp);
+ return nullptr;
+ }
+ size_t n = 0;
+ while (n < MP3_PSRAM_CACHE_MAX) {
+ if (n == cap) {
+ size_t ncap = cap * 2;
+ if (ncap > MP3_PSRAM_CACHE_MAX) {
+ ncap = MP3_PSRAM_CACHE_MAX;
+ }
+ uint8_t *grown = static_cast<uint8_t *>(
+ heap_caps_realloc(buf, ncap, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT));
+ if (!grown) {
+ break;
+ }
+ buf = grown;
+ cap = ncap;
+ }
+ size_t got = fread(buf + n, 1, cap - n, fp);
+ if (got == 0) {
+ break;
+ }
+ n += got;
+ }
+ const bool at_eof = feof(fp) != 0;
+ fclose(fp);
+ if (n == 0 || !at_eof) {
+ ESP_LOGW(TAG, "MP3 PSRAM cache miss n=%u eof=%d: %s", (unsigned)n, at_eof ? 1 : 0, path);
+ heap_caps_free(buf);
+ return nullptr;
+ }
+ *out_len = n;
+ ESP_LOGI(TAG, "cached MP3 in PSRAM %u bytes: %s", (unsigned)n, path);
+ return buf;
+}
+
static bool output_pcm(AudioCodec *codec, esp_ae_rate_cvt_handle_t rate_cvt_handle, const uint8_t *buffer,
- uint32_t size, uint8_t src_channels, uint32_t src_rate)
+ uint32_t size, uint8_t src_channels, uint32_t src_rate, float speed)
{
uint8_t dst_channels = codec->output_channels();
uint32_t dst_rate = codec->output_sample_rate();
@@ -100,65 +213,189 @@
return true;
}
- auto channel_converted = convert_channels(reinterpret_cast<const int16_t *>(buffer), frame_count, src_channels, dst_channels);
+ int16_t *converted = ensure_psram_samples(&s_pcm_a, &s_pcm_a_cap, frame_count * dst_channels);
+ if (!converted) {
+ ESP_LOGE(TAG, "mp3 pcm alloc failed");
+ return false;
+ }
+ if (!convert_channels(reinterpret_cast<const int16_t *>(buffer), frame_count, src_channels, dst_channels, converted)) {
+ return false;
+ }
+
+ const int16_t *play = converted;
+ size_t play_frames = frame_count;
+ if (speed < 0.999f || speed > 1.001f) {
+ size_t out_frames = static_cast<size_t>(static_cast<double>(frame_count) / static_cast<double>(speed) + 0.5);
+ if (out_frames < 1) {
+ out_frames = 1;
+ }
+ int16_t *stretched = ensure_psram_samples(&s_pcm_b, &s_pcm_b_cap, out_frames * dst_channels);
+ if (!stretched) {
+ ESP_LOGE(TAG, "mp3 stretch alloc failed");
+ return false;
+ }
+ play_frames = stretch_for_speed(converted, frame_count, dst_channels, speed, stretched, out_frames);
+ play = stretched;
+ }
+ if (play_frames == 0) {
+ return true;
+ }
+
if (src_rate == dst_rate || !rate_cvt_handle) {
- codec->OutputData(channel_converted);
+ codec->OutputPcm(play, static_cast<int>(play_frames * dst_channels));
+ if (s_mp3_audible_started == 0) {
+ s_mp3_audible_started = xTaskGetTickCount();
+ }
return true;
}
uint32_t max_out_frames = 0;
- if (esp_ae_rate_cvt_get_max_out_sample_num(rate_cvt_handle, frame_count, &max_out_frames) != ESP_AE_ERR_OK) {
+ if (esp_ae_rate_cvt_get_max_out_sample_num(rate_cvt_handle, play_frames, &max_out_frames) != ESP_AE_ERR_OK) {
ESP_LOGE(TAG, "get resample output size failed");
return false;
}
-
- std::vector<int16_t> resampled(max_out_frames * dst_channels);
+ int16_t **out_slot = (play == s_pcm_b) ? &s_pcm_a : &s_pcm_b;
+ size_t *out_cap = (play == s_pcm_b) ? &s_pcm_a_cap : &s_pcm_b_cap;
+ int16_t *resampled = ensure_psram_samples(out_slot, out_cap, max_out_frames * dst_channels);
+ if (!resampled) {
+ ESP_LOGE(TAG, "mp3 resample alloc failed");
+ return false;
+ }
uint32_t actual_out_frames = max_out_frames;
- if (esp_ae_rate_cvt_process(rate_cvt_handle, reinterpret_cast<esp_ae_sample_t>(channel_converted.data()), frame_count,
- reinterpret_cast<esp_ae_sample_t>(resampled.data()), &actual_out_frames) != ESP_AE_ERR_OK) {
+ if (esp_ae_rate_cvt_process(rate_cvt_handle, reinterpret_cast<esp_ae_sample_t>(const_cast<int16_t *>(play)), play_frames,
+ reinterpret_cast<esp_ae_sample_t>(resampled), &actual_out_frames) != ESP_AE_ERR_OK) {
ESP_LOGE(TAG, "resample MP3 PCM failed");
return false;
}
-
- resampled.resize(actual_out_frames * dst_channels);
- codec->OutputData(resampled);
+ codec->OutputPcm(resampled, static_cast<int>(actual_out_frames * dst_channels));
+ if (s_mp3_audible_started == 0) {
+ s_mp3_audible_started = xTaskGetTickCount();
+ }
return true;
+}
+
+static void pause_wake_for_sd(void)
+{
+ Application::GetInstance().Schedule([]() {
+ auto &audio = Application::GetInstance().GetAudioService();
+ audio.EnableVoiceProcessing(false);
+ audio.EnableWakeWordDetection(false);
+ });
+ vTaskDelay(pdMS_TO_TICKS(40));
+}
+
+static void wait_shadowing_listen(unsigned duration_ms)
+{
+ unsigned wait_ms = Board_ShadowingListenWindowMs(duration_ms);
+ // Server 鍚獥涓庤澶囧悓涓� 1.2脳銆傚绛� 200ms锛岄伩鍏嶄笅涓�鍙ヨ鏂囧紑澶磋涓婁竴鍙� ASR 鍚冩帀銆�
+ wait_ms += 200;
+ ESP_LOGI(TAG, "shadowing listen window %ums after clip %ums", wait_ms, duration_ms);
+ unsigned waited = 0;
+ while (waited < wait_ms && !mp3_stop_requested) {
+ unsigned slice = wait_ms - waited;
+ if (slice > 50) {
+ slice = 50;
+ }
+ vTaskDelay(pdMS_TO_TICKS(slice));
+ waited += slice;
+ }
+}
+
+static void ensure_shadowing_listen_sem(void)
+{
+ if (s_shadowing_listen_closed == nullptr) {
+ s_shadowing_listen_closed = xSemaphoreCreateBinary();
+ }
+}
+
+static void wait_shadowing_mic_off(void)
+{
+ ensure_shadowing_listen_sem();
+ xSemaphoreTake(s_shadowing_listen_closed, 0);
+ Application::GetInstance().Schedule([]() {
+ Application::GetInstance().CloseShadowingListenWindow();
+ if (s_shadowing_listen_closed != nullptr) {
+ xSemaphoreGive(s_shadowing_listen_closed);
+ }
+ });
+ // 蹇呴』绛� AFE 鐪熸鍏虫帀鍐嶈В鐮佷笅涓�鍙ャ�傚彧 delay 80ms 浼氬拰褰曢煶閲嶅彔锛屽唴閮� SRAM 浼氳鎵撴弧銆�
+ if (xSemaphoreTake(s_shadowing_listen_closed, pdMS_TO_TICKS(2500)) != pdTRUE) {
+ ESP_LOGW(TAG, "shadowing wait mic off timeout, force AFE stop");
+ Application::GetInstance().GetAudioService().EnableVoiceProcessing(false);
+ }
}
static void mp3_player_task(void *arg)
{
mp3_player_task_handle = xTaskGetCurrentTaskHandle();
mp3_player_task_arg_t *task_arg = static_cast<mp3_player_task_arg_t *>(arg);
- char path[SDCARD_PATH_MAX_LEN];
- strlcpy(path, task_arg->path, sizeof(path));
- float speed = task_arg->speed;
- int repeat = task_arg->repeat < 1 ? 1 : task_arg->repeat;
- bool power_off_after = task_arg->power_off_after;
- free(task_arg);
-
- FILE *fp = fopen(path, "rb");
- if (!fp) {
- ESP_LOGE(TAG, "open MP3 failed: %s", path);
- mp3_player_task_handle = nullptr;
- mp3_stop_requested = false;
- mp3_repeat_left = 0;
- vTaskDelete(nullptr);
- return;
+ sdcard_mp3_playlist_item_t items[SDCARD_MP3_PLAYLIST_MAX];
+ int count = task_arg->count;
+ if (count < 0) {
+ count = 0;
}
+ if (count > SDCARD_MP3_PLAYLIST_MAX) {
+ count = SDCARD_MP3_PLAYLIST_MAX;
+ }
+ if (count > 0) {
+ memcpy(items, task_arg->items, sizeof(items[0]) * static_cast<size_t>(count));
+ }
+ bool power_off_after = count > 0 ? items[count - 1].power_off_after : false;
+ heap_caps_free(task_arg);
auto codec = Board::GetInstance().GetAudioCodec();
- if (!codec) {
- ESP_LOGE(TAG, "audio codec unavailable");
- fclose(fp);
+ if (!codec || count < 1) {
+ ESP_LOGE(TAG, "audio codec unavailable or empty playlist");
mp3_player_task_handle = nullptr;
mp3_stop_requested = false;
mp3_repeat_left = 0;
+ Board_OnSkillPlaybackFinished(true, false, 0);
vTaskDelete(nullptr);
return;
}
esp_audio_dec_register_default();
esp_audio_simple_dec_register_default();
+ uint8_t *in_buf = static_cast<uint8_t *>(
+ heap_caps_malloc(MP3_READ_BUFFER_SIZE, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT));
+ uint8_t *out_buf = static_cast<uint8_t *>(
+ heap_caps_malloc(MP3_OUTPUT_BUFFER_SIZE, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT));
+ uint32_t out_buf_size = MP3_OUTPUT_BUFFER_SIZE;
+ bool output_enabled_before = codec->output_enabled();
+ TickType_t play_started = xTaskGetTickCount();
+ s_mp3_audible_started = 0;
+ const bool shadowing = Board_IsShadowingTask();
+ char path[SDCARD_PATH_MAX_LEN] = {0};
+
+ if (!in_buf || !out_buf) {
+ ESP_LOGE(TAG, "no memory for MP3 playback");
+ }
+
+ for (int item_i = 0; in_buf && out_buf && item_i < count && !mp3_stop_requested; ++item_i) {
+ strlcpy(path, items[item_i].path, sizeof(path));
+ float speed = SDCard_Clamp_MP3_Speed(items[item_i].speed);
+ int repeat = items[item_i].repeat < 1 ? 1 : items[item_i].repeat;
+ if (item_i > 0) {
+ pause_wake_for_sd();
+ }
+
+ size_t file_len = 0;
+ uint8_t *file_buf = load_mp3_to_psram(path, &file_len);
+ FILE *fp = nullptr;
+ if (!file_buf) {
+ wait_for_sd_dma();
+ fp = fopen(path, "rb");
+ }
+ if (!file_buf && !fp) {
+ ESP_LOGE(TAG, "open MP3 failed: %s", path);
+ continue;
+ }
+
+ if (!mp3_pause_requested) {
+ Application::GetInstance().Schedule([]() {
+ Application::GetInstance().ApplyMp3CommandOnlyMic();
+ });
+ }
esp_audio_simple_dec_handle_t decoder = nullptr;
esp_audio_simple_dec_cfg_t dec_cfg = {
@@ -171,30 +408,24 @@
esp_audio_err_t ret = esp_audio_simple_dec_open(&dec_cfg, &decoder);
if (ret != ESP_AUDIO_ERR_OK) {
ESP_LOGE(TAG, "open MP3 decoder failed: %d", ret);
- esp_audio_simple_dec_unregister_default();
- esp_audio_dec_unregister_default();
- fclose(fp);
- mp3_player_task_handle = nullptr;
- mp3_stop_requested = false;
- mp3_repeat_left = 0;
- vTaskDelete(nullptr);
- return;
+ if (fp) {
+ fclose(fp);
+ fp = nullptr;
+ }
+ if (file_buf) {
+ heap_caps_free(file_buf);
+ file_buf = nullptr;
+ }
+ continue;
}
- uint8_t *in_buf = static_cast<uint8_t *>(malloc(MP3_READ_BUFFER_SIZE));
- uint8_t *out_buf = static_cast<uint8_t *>(malloc(MP3_OUTPUT_BUFFER_SIZE));
- uint32_t out_buf_size = MP3_OUTPUT_BUFFER_SIZE;
- bool output_enabled_before = codec->output_enabled();
bool audio_info_checked = false;
uint32_t mp3_sample_rate = 0;
uint8_t mp3_channels = 0;
esp_ae_rate_cvt_handle_t rate_cvt_handle = nullptr;
- if (!in_buf || !out_buf) {
- ESP_LOGE(TAG, "no memory for MP3 playback");
- ret = ESP_AUDIO_ERR_MEM_LACK;
- } else {
- ESP_LOGI(TAG, "play MP3: %s, speed: %.2fx, repeat: %d, power_off_after: %d", path, speed, repeat, power_off_after ? 1 : 0);
+ ESP_LOGI(TAG, "play MP3: %s, speed: %.2fx, repeat: %d, item=%d/%d",
+ path, speed, repeat, item_i + 1, count);
if (wait_if_paused()) {
codec->EnableOutput(true);
@@ -202,13 +433,26 @@
for (int pass = 0; pass < repeat && ret == ESP_AUDIO_ERR_OK && !mp3_stop_requested; ++pass) {
mp3_repeat_left = repeat - pass;
+ s_mp3_audible_started = 0;
+ size_t mem_off = 0;
if (pass > 0) {
- if (fseek(fp, 0, SEEK_SET) != 0) {
- ESP_LOGE(TAG, "rewind MP3 failed: %s", path);
- break;
+ if (!file_buf) {
+ if (fp) {
+ fclose(fp);
+ fp = nullptr;
+ }
+ vTaskDelay(pdMS_TO_TICKS(30));
+ wait_for_sd_dma();
+ fp = fopen(path, "rb");
+ if (!fp) {
+ ESP_LOGE(TAG, "reopen MP3 failed: %s", path);
+ break;
+ }
}
- esp_audio_simple_dec_close(decoder);
- decoder = nullptr;
+ if (decoder) {
+ esp_audio_simple_dec_close(decoder);
+ decoder = nullptr;
+ }
ret = esp_audio_simple_dec_open(&dec_cfg, &decoder);
if (ret != ESP_AUDIO_ERR_OK || !decoder) {
ESP_LOGE(TAG, "reopen MP3 decoder failed: %d", ret);
@@ -221,7 +465,21 @@
break;
}
- int read_size = fread(in_buf, 1, MP3_READ_BUFFER_SIZE, fp);
+ int read_size = 0;
+ if (file_buf) {
+ if (mem_off >= file_len) {
+ break;
+ }
+ size_t chunk = file_len - mem_off;
+ if (chunk > MP3_READ_BUFFER_SIZE) {
+ chunk = MP3_READ_BUFFER_SIZE;
+ }
+ std::memcpy(in_buf, file_buf + mem_off, chunk);
+ mem_off += chunk;
+ read_size = static_cast<int>(chunk);
+ } else {
+ read_size = fread(in_buf, 1, MP3_READ_BUFFER_SIZE, fp);
+ }
if (read_size <= 0) {
break;
}
@@ -229,7 +487,7 @@
esp_audio_simple_dec_raw_t raw = {
.buffer = in_buf,
.len = static_cast<uint32_t>(read_size),
- .eos = read_size < MP3_READ_BUFFER_SIZE,
+ .eos = file_buf ? (mem_off >= file_len) : (read_size < MP3_READ_BUFFER_SIZE),
.consumed = 0,
.frame_recover = ESP_AUDIO_SIMPLE_DEC_RECOVERY_NONE,
};
@@ -248,7 +506,8 @@
ret = esp_audio_simple_dec_process(decoder, &raw, &out_frame);
if (ret == ESP_AUDIO_ERR_BUFF_NOT_ENOUGH) {
- uint8_t *new_buf = static_cast<uint8_t *>(realloc(out_buf, out_frame.needed_size));
+ uint8_t *new_buf = static_cast<uint8_t *>(
+ heap_caps_realloc(out_buf, out_frame.needed_size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT));
if (!new_buf) {
ESP_LOGE(TAG, "resize MP3 output buffer failed");
ret = ESP_AUDIO_ERR_MEM_LACK;
@@ -274,7 +533,7 @@
ret = ESP_AUDIO_ERR_NOT_SUPPORT;
break;
}
- mp3_sample_rate = static_cast<uint32_t>(info.sample_rate * speed);
+ mp3_sample_rate = info.sample_rate;
mp3_channels = info.channel;
if (mp3_sample_rate != static_cast<uint32_t>(codec->output_sample_rate())) {
esp_ae_rate_cvt_cfg_t rate_cvt_cfg = {
@@ -295,10 +554,11 @@
audio_info_checked = true;
}
- if (!output_pcm(codec, rate_cvt_handle, out_frame.buffer, out_frame.decoded_size, mp3_channels, mp3_sample_rate)) {
+ if (!output_pcm(codec, rate_cvt_handle, out_frame.buffer, out_frame.decoded_size, mp3_channels, mp3_sample_rate, speed)) {
ret = ESP_AUDIO_ERR_FAIL;
break;
}
+ taskYIELD();
}
if (raw.consumed == 0) {
@@ -313,27 +573,55 @@
break;
}
}
+
+ if (shadowing && !mp3_stop_requested && ret == ESP_AUDIO_ERR_OK) {
+ unsigned duration_ms = s_mp3_audible_started
+ ? pdTICKS_TO_MS(xTaskGetTickCount() - s_mp3_audible_started)
+ : 0;
+ Board_OnShadowingClipFinished(duration_ms);
+ wait_shadowing_listen(duration_ms);
+ wait_shadowing_mic_off();
+ s_mp3_audible_started = 0;
+ }
}
- }
if (rate_cvt_handle) {
esp_ae_rate_cvt_close(rate_cvt_handle);
+ rate_cvt_handle = nullptr;
}
+ if (decoder) {
+ esp_audio_simple_dec_close(decoder);
+ decoder = nullptr;
+ }
+ if (fp) {
+ fclose(fp);
+ fp = nullptr;
+ }
+ if (file_buf) {
+ heap_caps_free(file_buf);
+ file_buf = nullptr;
+ }
+ ESP_LOGI(TAG, "MP3 item %s: %s", mp3_stop_requested ? "stopped" : "finished", path);
+ }
+
if (!output_enabled_before) {
codec->EnableOutput(false);
}
- free(in_buf);
- free(out_buf);
- if (decoder) {
- esp_audio_simple_dec_close(decoder);
+ if (in_buf) {
+ heap_caps_free(in_buf);
+ }
+ if (out_buf) {
+ heap_caps_free(out_buf);
}
esp_audio_simple_dec_unregister_default();
esp_audio_dec_unregister_default();
- fclose(fp);
- ESP_LOGI(TAG, "MP3 playback %s: %s", mp3_stop_requested ? "stopped" : "finished", path);
+ ESP_LOGI(TAG, "MP3 playlist %s items=%d", mp3_stop_requested ? "stopped" : "finished", count);
bool stopped = mp3_stop_requested;
bool should_power_off = power_off_after && !stopped;
+ unsigned duration_ms = s_mp3_audible_started
+ ? pdTICKS_TO_MS(xTaskGetTickCount() - s_mp3_audible_started)
+ : pdTICKS_TO_MS(xTaskGetTickCount() - play_started);
mp3_player_task_handle = nullptr;
mp3_stop_requested = false;
mp3_pause_requested = false;
@@ -341,7 +629,11 @@
if (should_power_off) {
Application::GetInstance().RequestPowerOffStandby();
}
- Board_OnSkillPlaybackFinished(stopped, power_off_after);
+ if (shadowing && !stopped) {
+ Board_OnShadowingPlaylistFinished();
+ } else {
+ Board_OnSkillPlaybackFinished(stopped, power_off_after, duration_ms);
+ }
vTaskDelete(nullptr);
}
@@ -357,11 +649,24 @@
extern "C" esp_err_t SDCard_Play_MP3_File_With_Speed_Repeat(const char *path, float speed, int repeat, bool power_off_after)
{
- if (!path || path[0] == '\0' || !is_supported_speed(speed)) {
+ if (!path || path[0] == '\0') {
return ESP_ERR_INVALID_ARG;
}
- if (repeat < 1) {
- repeat = 1;
+ sdcard_mp3_playlist_item_t item = {};
+ strlcpy(item.path, path, sizeof(item.path));
+ item.speed = speed;
+ item.repeat = repeat;
+ item.power_off_after = power_off_after;
+ return SDCard_Play_MP3_File_Playlist(&item, 1);
+}
+
+extern "C" esp_err_t SDCard_Play_MP3_File_Playlist(const sdcard_mp3_playlist_item_t *items, int count)
+{
+ if (!items || count < 1) {
+ return ESP_ERR_INVALID_ARG;
+ }
+ if (count > SDCARD_MP3_PLAYLIST_MAX) {
+ count = SDCARD_MP3_PLAYLIST_MAX;
}
bool start_paused = mp3_pause_requested;
@@ -373,22 +678,46 @@
}
}
- mp3_player_task_arg_t *arg = static_cast<mp3_player_task_arg_t *>(calloc(1, sizeof(mp3_player_task_arg_t)));
+ mp3_player_task_arg_t *arg = static_cast<mp3_player_task_arg_t *>(
+ heap_caps_calloc(1, sizeof(mp3_player_task_arg_t), MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT));
if (!arg) {
return ESP_ERR_NO_MEM;
}
- strlcpy(arg->path, path, sizeof(arg->path));
- arg->speed = speed;
- arg->repeat = repeat;
- arg->power_off_after = power_off_after;
+ arg->count = count;
+ for (int i = 0; i < count; ++i) {
+ strlcpy(arg->items[i].path, items[i].path, sizeof(arg->items[i].path));
+ arg->items[i].speed = SDCard_Clamp_MP3_Speed(items[i].speed);
+ arg->items[i].repeat = items[i].repeat < 1 ? 1 : items[i].repeat;
+ arg->items[i].power_off_after = items[i].power_off_after;
+ }
mp3_stop_requested = false;
mp3_pause_requested = start_paused;
- mp3_repeat_left = repeat;
+ mp3_repeat_left = arg->items[0].repeat;
- BaseType_t ret = xTaskCreate(mp3_player_task, "mp3_player", MP3_PLAYER_STACK_SIZE, arg, MP3_PLAYER_PRIORITY, &mp3_player_task_handle);
- if (ret != pdPASS) {
- free(arg);
- mp3_player_task_handle = nullptr;
+ if (mp3_player_stack == nullptr) {
+ mp3_player_stack = static_cast<StackType_t *>(
+ heap_caps_malloc(MP3_PLAYER_STACK_SIZE, MALLOC_CAP_SPIRAM));
+ }
+ if (mp3_player_tcb == nullptr) {
+ mp3_player_tcb = static_cast<StaticTask_t *>(
+ heap_caps_malloc(sizeof(StaticTask_t), MALLOC_CAP_INTERNAL));
+ }
+ if (mp3_player_stack == nullptr || mp3_player_tcb == nullptr) {
+ ESP_LOGE(TAG, "mp3 task alloc failed internal=%u psram=%u",
+ (unsigned)heap_caps_get_free_size(MALLOC_CAP_INTERNAL),
+ (unsigned)heap_caps_get_free_size(MALLOC_CAP_SPIRAM));
+ heap_caps_free(arg);
+ mp3_repeat_left = 0;
+ return ESP_ERR_NO_MEM;
+ }
+
+ mp3_player_task_handle = xTaskCreateStatic(
+ mp3_player_task, "mp3_player", MP3_PLAYER_STACK_SIZE, arg,
+ MP3_PLAYER_PRIORITY, mp3_player_stack, mp3_player_tcb);
+ if (mp3_player_task_handle == nullptr) {
+ ESP_LOGE(TAG, "mp3 xTaskCreateStatic failed internal=%u",
+ (unsigned)heap_caps_get_free_size(MALLOC_CAP_INTERNAL));
+ heap_caps_free(arg);
mp3_repeat_left = 0;
return ESP_FAIL;
}
@@ -400,7 +729,6 @@
{
mp3_pause_requested = false;
if (!mp3_player_task_handle) {
- ESP_LOGW(TAG, "MP3 player is not running");
return;
}
@@ -415,6 +743,9 @@
extern "C" void SDCard_Resume_Play(void)
{
mp3_pause_requested = false;
+ Application::GetInstance().Schedule([]() {
+ Application::GetInstance().ApplyMp3CommandOnlyMic();
+ });
}
extern "C" bool SDCard_Is_Playing(void)
diff --git a/xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/mqtt.c b/xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/mqtt.c
index 219255b..f0a4cb4 100644
--- a/xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/mqtt.c
+++ b/xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/mqtt.c
@@ -18,7 +18,7 @@
#include "app_task.h"
#define BOARD_MQTT_KEEPALIVE_SEC 120
-#define BOARD_MQTT_STATE_REPORT_INTERVAL_MS 5000
+#define BOARD_MQTT_STATE_REPORT_INTERVAL_MS 30000
#define BOARD_MQTT_RECONNECT_DELAY_MS 30000
#define BOARD_MQTT_NETWORK_TIMEOUT_MS 15000
#define BOARD_MQTT_FINAL_REPORT_WAIT_MS 10000
@@ -80,7 +80,6 @@
static void BoardMqtt_WaitConnectedForFinalReport(void);
static bool BoardMqtt_BeginTask(board_mqtt_task_type_t type);
static void BoardMqtt_FinishTask(board_mqtt_task_type_t type);
-static bool BoardMqtt_PlaybackSpeedIsSupported(float speed);
static void BoardMqtt_StopPlaylistTask(void);
static esp_err_t BoardMqtt_StartPlaylistTask(void);
@@ -145,6 +144,7 @@
int volume;
bool has_volume;
float speed;
+ int repeat;
int index;
int count;
} board_mqtt_playlist_item_t;
@@ -616,7 +616,8 @@
strcmp(action, "SET_CLOCK") == 0 || strcmp(action, "PLAY_MEDIA") == 0 ||
strcmp(action, "STOP_MEDIA") == 0 || strcmp(action, "DELETE_FILE") == 0 ||
strcmp(action, "SET_ALARM") == 0 || strcmp(action, "UNSET_ALARM") == 0 ||
- strcmp(action, "REBOOT") == 0;
+ strcmp(action, "REBOOT") == 0 || strcmp(action, "START_APP_TASK") == 0 ||
+ strcmp(action, "ENSURE_APP_DIRS") == 0;
}
static bool BoardMqtt_ActionIsTask(const char *action)
@@ -675,21 +676,6 @@
esp_err_t err = DS2312_Set_Unix_Time_Saved((time_t)ts->valuedouble);
return err == ESP_OK ? BoardMqtt_ResultOk("clock updated") :
BoardMqtt_ResultFail(BOARD_MQTT_ERR_EXEC_FAILED, esp_err_to_name(err));
-}
-
-static bool BoardMqtt_PlaybackSpeedIsSupported(float speed)
-{
- const float supported_speeds[] = {0.5f, 0.75f, 1.0f, 1.25f, 1.5f, 2.0f};
- for (size_t i = 0; i < sizeof(supported_speeds) / sizeof(supported_speeds[0]); ++i) {
- float diff = speed - supported_speeds[i];
- if (diff < 0) {
- diff = -diff;
- }
- if (diff < 0.001f) {
- return true;
- }
- }
- return false;
}
static char *BoardMqtt_ReadPlaylistFile(void)
@@ -850,13 +836,22 @@
cJSON_Delete(root);
return ESP_ERR_INVALID_ARG;
}
- speed = (float)speed_json->valuedouble;
+ speed = SDCard_Clamp_MP3_Speed((float)speed_json->valuedouble);
}
- if (!BoardMqtt_PlaybackSpeedIsSupported(speed)) {
- cJSON_Delete(root);
- return ESP_ERR_INVALID_ARG;
+ int repeat = 1;
+ cJSON *repeat_json = cJSON_GetObjectItem(item, "repeat");
+ if (!repeat_json) {
+ repeat_json = cJSON_GetObjectItem(item, "count");
}
-
+ if (!repeat_json) {
+ repeat_json = cJSON_GetObjectItem(item, "play_count");
+ }
+ if (repeat_json) {
+ if (!BoardMqtt_NumberToInt(repeat_json, &repeat) || repeat < 1 || repeat > 99) {
+ cJSON_Delete(root);
+ return ESP_ERR_INVALID_ARG;
+ }
+ }
cJSON *stored_item = cJSON_CreateObject();
if (!stored_item) {
cJSON_Delete(root);
@@ -868,6 +863,7 @@
cJSON_AddNumberToObject(stored_item, "volume", volume_value);
}
cJSON_AddNumberToObject(stored_item, "speed", speed);
+ cJSON_AddNumberToObject(stored_item, "repeat", repeat);
cJSON_AddBoolToObject(stored_item, "completed", false);
cJSON_AddItemToArray(playlist_items, stored_item);
}
@@ -901,6 +897,7 @@
cJSON *sd_path = cJSON_GetObjectItem(item, "sd_path");
cJSON *volume = cJSON_GetObjectItem(item, "volume");
cJSON *speed_json = cJSON_GetObjectItem(item, "speed");
+ cJSON *repeat_json = cJSON_GetObjectItem(item, "repeat");
if (!cJSON_IsString(path) || !cJSON_IsString(sd_path)) {
cJSON_Delete(root);
return false;
@@ -925,10 +922,14 @@
}
next_item->has_volume = true;
}
- next_item->speed = cJSON_IsNumber(speed_json) ? (float)speed_json->valuedouble : 1.0f;
- if (!BoardMqtt_PlaybackSpeedIsSupported(next_item->speed)) {
- cJSON_Delete(root);
- return false;
+ next_item->speed = SDCard_Clamp_MP3_Speed(
+ cJSON_IsNumber(speed_json) ? (float)speed_json->valuedouble : 1.0f);
+ next_item->repeat = 1;
+ if (repeat_json) {
+ int repeat = 1;
+ if (BoardMqtt_NumberToInt(repeat_json, &repeat) && repeat >= 1) {
+ next_item->repeat = repeat > 99 ? 99 : repeat;
+ }
}
next_item->index = index;
next_item->count = count;
@@ -992,8 +993,10 @@
BoardMqtt_ShowPlaylistAlert(alert_message, "music");
SDCard_Show_Matching_JPG_For_MP3_Path(item.sd_path);
- ESP_LOGI(TAG, "playlist play item %d/%d: %s speed=%.2fx", item.index + 1, item.count, item.sd_path, item.speed);
- esp_err_t err = SDCard_Play_MP3_File_With_Speed(item.sd_path, item.speed);
+ ESP_LOGI(TAG, "playlist play item %d/%d: %s speed=%.2fx repeat=%d",
+ item.index + 1, item.count, item.sd_path, item.speed, item.repeat);
+ esp_err_t err = SDCard_Play_MP3_File_With_Speed_Repeat(
+ item.sd_path, item.speed, item.repeat > 0 ? item.repeat : 1, false);
if (err != ESP_OK) {
ESP_LOGE(TAG, "playlist item playback failed: %s", esp_err_to_name(err));
snprintf(alert_message, sizeof(alert_message), "鎾斁澶辫触 %d/%d: %s", item.index + 1, item.count, esp_err_to_name(err));
@@ -1894,6 +1897,38 @@
return BoardMqtt_ResultOk("app task started");
}
+static board_mqtt_action_result_t BoardMqtt_HandleEnsureAppDirs(cJSON *payload)
+{
+ if (!payload) {
+ return BoardMqtt_ResultFail(BOARD_MQTT_ERR_INVALID_PARAM, "missing payload");
+ }
+ cJSON *app_id = cJSON_GetObjectItem(payload, "app_id");
+ if (!cJSON_IsString(app_id) || !app_id->valuestring || app_id->valuestring[0] == '\0') {
+ return BoardMqtt_ResultFail(BOARD_MQTT_ERR_INVALID_PARAM, "missing app_id");
+ }
+ esp_err_t err = Board_EnsureAppLayout(app_id->valuestring, NULL);
+ if (err == ESP_ERR_INVALID_ARG || err == ESP_ERR_INVALID_SIZE) {
+ return BoardMqtt_ResultFail(BOARD_MQTT_ERR_INVALID_PARAM, "invalid app_id");
+ }
+ if (err != ESP_OK) {
+ return BoardMqtt_ResultFail(BOARD_MQTT_ERR_EXEC_FAILED, esp_err_to_name(err));
+ }
+ cJSON *tasks = cJSON_GetObjectItem(payload, "tasks");
+ if (cJSON_IsArray(tasks)) {
+ cJSON *task = NULL;
+ cJSON_ArrayForEach(task, tasks) {
+ if (!cJSON_IsString(task) || !task->valuestring) {
+ return BoardMqtt_ResultFail(BOARD_MQTT_ERR_INVALID_PARAM, "invalid tasks");
+ }
+ err = Board_EnsureAppLayout(app_id->valuestring, task->valuestring);
+ if (err != ESP_OK) {
+ return BoardMqtt_ResultFail(BOARD_MQTT_ERR_EXEC_FAILED, esp_err_to_name(err));
+ }
+ }
+ }
+ return BoardMqtt_ResultOk("app dirs ensured");
+}
+
static board_mqtt_action_result_t BoardMqtt_DispatchAction(const char *req_id, const char *action, cJSON *payload)
{
if (strcmp(action, "SET_VOLUME") == 0) return BoardMqtt_HandleSetVolume(payload);
@@ -1909,6 +1944,7 @@
if (strcmp(action, "DOWNLOAD") == 0) return BoardMqtt_HandleDownload(req_id, payload);
if (strcmp(action, "OTA_UPDATE") == 0) return BoardMqtt_HandleOtaUpdate(req_id, payload);
if (strcmp(action, "START_APP_TASK") == 0) return BoardMqtt_HandleStartAppTask(payload);
+ if (strcmp(action, "ENSURE_APP_DIRS") == 0) return BoardMqtt_HandleEnsureAppDirs(payload);
return BoardMqtt_ResultFail(BOARD_MQTT_ERR_UNSUPPORTED, "unsupported action");
}
@@ -2098,6 +2134,9 @@
(void)arg;
while (s_mqtt_client) {
vTaskDelay(pdMS_TO_TICKS(BOARD_MQTT_STATE_REPORT_INTERVAL_MS));
+ if (Board_Is_Audio_Busy()) {
+ continue;
+ }
BoardMqtt_PublishState();
}
s_state_task_handle = NULL;
diff --git a/xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/mqtt_interface.md b/xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/mqtt_interface.md
index ecf7655..c64e77a 100644
--- a/xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/mqtt_interface.md
+++ b/xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/mqtt_interface.md
@@ -242,7 +242,7 @@
| SET_VOLUME | `{"level": 50}` | 璁剧疆闊抽噺 0鈥�100 |
| SET_POWER | `{"state": "wake"}` | 鍞ら啋 / `sleep` 浼戠湢 |
| SET_CLOCK | `{"ts": 1718365200}` | 鍚屾 Unix 鏃堕棿鎴� |
-| PLAY_MEDIA | `{"items":[{"path":"courses/course01/chapter01/resources/001.mp3","volume":50,"speed":1.25,"loop":false}]}` | 淇濆瓨骞舵挱鏀炬湰鍦� MP3 娓呭崟锛涘彧淇濈暀涓�浠芥竻鍗曪紝鍚庣画瑕嗙洊锛泂peed 鍙�夛紝浠呮敮鎸� 0.5/0.75/1.0/1.25/1.5/2.0锛岄潪 1.0 浼氬彉璋冿紱ttl 鍜� loop 蹇界暐 |
+| PLAY_MEDIA | `{"items":[{"path":"courses/course01/chapter01/resources/001.mp3","volume":50,"speed":1.05,"loop":false}]}` | 淇濆瓨骞舵挱鏀炬湰鍦� MP3 娓呭崟锛涘彧淇濈暀涓�浠芥竻鍗曪紝鍚庣画瑕嗙洊锛泂peed 鍙�夛紝鑼冨洿 0.5锝�2.0 浠绘剰娴偣锛堥粯璁� 1.0锛岃秴鍑哄す绱э級锛岄潪 1.0 浼氬彉璋冿紱ttl 鍜� loop 蹇界暐 |
| STOP_MEDIA | `{}` | 鍋滄鎾斁 |
| SET_ALARM | `{"id": "alarm_01", "time": "07:00", "repeat": [1,2,3,4,5], "action": "add"}` | 璁剧疆闂归挓 |
| UNSET_ALARM | `{"id": "alarm_01"}` | 鍒犻櫎闂归挓 |
@@ -250,7 +250,7 @@
**PLAY_MEDIA 娓呭崟琛屼负锛�**
-- `payload.items` 涓烘挱鏀炬竻鍗曟暟缁勶紝姣忛」 `path` 蹇呭～锛宍volume` 鍜� `speed` 鍙�夈��
+- `payload.items` 涓烘挱鏀炬竻鍗曟暟缁勶紝姣忛」 `path` 蹇呭～锛宍volume` 鍜� `speed` 鍙�夈�俙speed` 鑼冨洿 0.5锝�2.0锛堥粯璁� 1.0锛岃秴鍑哄す绱э級锛屼笉鏄鏁ｆ。銆�
- 鏀跺埌娓呭崟鍚庣粓绔繚瀛樺埌 SD 鍗℃湰鍦帮紝浠呬繚鐣欎竴浠斤紱鍚庣画鏂版竻鍗曚細瑕嗙洊鏃ф竻鍗曘��
- 姣忔挱鏀惧畬涓�棣栵紝缁堢浼氬湪鏈湴娓呭崟涓爣璁拌椤瑰畬鎴愶紝骞堕�氳繃 `rpt/progress` 涓婃姤杩涘害銆�
- `ttl` 鍜屾瘡椤� `loop` 褰撳墠蹇界暐銆�
diff --git a/xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/sening_xiaozhi_s3.cc b/xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/sening_xiaozhi_s3.cc
index 7d087c6..7561ee8 100644
--- a/xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/sening_xiaozhi_s3.cc
+++ b/xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/sening_xiaozhi_s3.cc
@@ -121,6 +121,15 @@
return state == kDeviceStateConnecting || state == kDeviceStateListening || state == kDeviceStateSpeaking;
}
+extern "C" bool Board_Is_Audio_Busy(void)
+{
+ if (Board_IsShadowingTask() || Board_IsRepeaterTask()) {
+ return true;
+ }
+ DeviceState state = Application::GetInstance().GetDeviceState();
+ return state == kDeviceStateConnecting || state == kDeviceStateSpeaking;
+}
+
extern "C" void Board_Show_JPG(const char *jpg_name)
{
if (!jpg_name || jpg_name[0] == '\0') {
@@ -719,31 +728,138 @@
return std::string(SDCard_Get_MP3_List());
});
+ mcp_server.AddTool("self.sd.ensure_app",
+ "Create SD directories for a subscribed app without flashing firmware. Unique app_id creates /{app_id}. Optional task_id also creates /{app_id}/{task_id}/lessons.",
+ PropertyList({
+ Property("app_id", kPropertyTypeString),
+ Property("task_id", kPropertyTypeString, std::string(""))
+ }),
+ [](const PropertyList& properties) -> ReturnValue {
+ std::string app_id = properties["app_id"].value<std::string>();
+ std::string task_id = properties["task_id"].value<std::string>();
+ esp_err_t err = Board_EnsureAppLayout(app_id.c_str(), task_id.c_str());
+ if (err != ESP_OK) {
+ return std::string("{\"ok\":false,\"error\":\"") + esp_err_to_name(err) + "\"}";
+ }
+ return std::string("{\"ok\":true,\"app_id\":\"") + app_id +
+ "\",\"task_id\":\"" + task_id + "\"}";
+ });
+
static std::string last_mp3_play_name;
static float last_mp3_play_speed = 1.0f;
static int64_t last_mp3_play_time = 0;
mcp_server.AddTool("self.mp3.play",
- "Play one MP3 file from the SD card. The name argument must exactly match a files[].name from self.mp3.get_list. Optional speed supports only 0.5, 0.75, 1.0, 1.25, 1.5, or 2.0; non-1.0 speeds change pitch and affect only SD card MP3 playback. Optional repeat is the number of full-file loops (default 1). Optional power_off_after requests existing power-off standby after a natural finish, not after stop. For play, random play, switch song, or named-song requests, call this tool before telling the user playback has started. Do not call this tool repeatedly for the same user request after it has returned accepted=true.",
+ "Play one MP3 file from the SD card. If name is omitted or empty, play the last lesson stored on device, or the first MP3 on the SD card. Otherwise name must exactly match a files[].name from self.mp3.get_list. Optional speed is a playback rate in [0.5, 2.0], default 1.0; values outside the range are clamped. Non-1.0 speeds change pitch and affect only SD card MP3 playback. Optional repeat is the number of full-file loops (default 1). Optional power_off_after requests existing power-off standby after a natural finish, not after stop. For play, random play, switch song, or named-song requests, call this tool before telling the user playback has started. Do not call this tool repeatedly for the same user request after it has returned accepted=true.",
PropertyList({
- Property("name", kPropertyTypeString),
+ Property("name", kPropertyTypeString, std::string("")),
+ Property("items", kPropertyTypeString, std::string("")),
Property("speed", kPropertyTypeNumber, 1.0),
Property("repeat", kPropertyTypeInteger, 1, 1, 99),
- Property("power_off_after", kPropertyTypeBoolean, false)
+ Property("power_off_after", kPropertyTypeBoolean, false),
+ Property("finish_task", kPropertyTypeBoolean, true),
+ Property("app_id", kPropertyTypeString, std::string("")),
+ Property("task_id", kPropertyTypeString, std::string("")),
+ Property("force", kPropertyTypeBoolean, false),
+ Property("listen_ratio", kPropertyTypeNumber, 1.2)
}),
[](const PropertyList& properties) -> ReturnValue {
std::string name = properties["name"].value<std::string>();
- float speed = static_cast<float>(properties["speed"].value<double>());
+ std::string items_json = properties["items"].value<std::string>();
+ float speed = SDCard_Clamp_MP3_Speed(static_cast<float>(properties["speed"].value<double>()));
int repeat = properties["repeat"].value<int>();
bool power_off_after = properties["power_off_after"].value<bool>();
- if (repeat < 1) {
- repeat = 1;
+ std::string task_id = properties["task_id"].value<std::string>();
+ bool force = properties["force"].value<bool>();
+ Board_SetFinishTaskAfterMp3(properties["finish_task"].value<bool>());
+ Board_SetActiveTask(
+ properties["app_id"].value<std::string>().c_str(),
+ task_id.c_str());
+ Board_SetShadowingListenRatio(
+ static_cast<float>(properties["listen_ratio"].value<double>()));
+ sdcard_mp3_playlist_item_t playlist[SDCARD_MP3_PLAYLIST_MAX];
+ int playlist_n = 0;
+ if (!items_json.empty()) {
+ cJSON *root = cJSON_Parse(items_json.c_str());
+ cJSON *arr = root;
+ if (root && cJSON_IsObject(root)) {
+ arr = cJSON_GetObjectItem(root, "items");
+ }
+ if (cJSON_IsArray(arr)) {
+ cJSON *it = nullptr;
+ cJSON_ArrayForEach(it, arr) {
+ if (playlist_n >= SDCARD_MP3_PLAYLIST_MAX) {
+ break;
+ }
+ const cJSON *name_item = cJSON_GetObjectItem(it, "name");
+ if (!cJSON_IsString(name_item) || !name_item->valuestring ||
+ name_item->valuestring[0] == '\0') {
+ name_item = cJSON_GetObjectItem(it, "url");
+ }
+ if (!cJSON_IsString(name_item) || !name_item->valuestring ||
+ name_item->valuestring[0] == '\0') {
+ continue;
+ }
+ if (!SDCard_Resolve_MP3_Path(name_item->valuestring,
+ playlist[playlist_n].path,
+ sizeof(playlist[playlist_n].path))) {
+ ESP_LOGW(TAG, "playlist skip missing %s", name_item->valuestring);
+ continue;
+ }
+ const cJSON *rep = cJSON_GetObjectItem(it, "repeat");
+ playlist[playlist_n].repeat = cJSON_IsNumber(rep) ? rep->valueint : 1;
+ if (playlist[playlist_n].repeat < 1) {
+ playlist[playlist_n].repeat = 1;
+ }
+ const cJSON *spd = cJSON_GetObjectItem(it, "speed");
+ playlist[playlist_n].speed = cJSON_IsNumber(spd)
+ ? SDCard_Clamp_MP3_Speed(static_cast<float>(spd->valuedouble))
+ : 1.0f;
+ const cJSON *po = cJSON_GetObjectItem(it, "power_off_after");
+ playlist[playlist_n].power_off_after = cJSON_IsTrue(po);
+ playlist_n++;
+ }
+ }
+ cJSON_Delete(root);
+ }
+ if (playlist_n < 1) {
+ char resolved[SDCARD_PATH_MAX_LEN] = {0};
+ strlcpy(resolved, name.c_str(), sizeof(resolved));
+ if (!Board_ResolveRepeaterPlayback(resolved, sizeof(resolved), &speed, &repeat, &power_off_after)) {
+ return std::string("{\"accepted\":false,\"error\":\"no_mp3\"}");
+ }
+ name = resolved;
+ if (repeat < 1) {
+ repeat = 1;
+ }
+ } else {
+ name = playlist[0].path;
+ speed = playlist[0].speed;
+ repeat = playlist[0].repeat;
+ power_off_after = playlist[playlist_n - 1].power_off_after;
}
int64_t now = esp_timer_get_time();
- if (name == last_mp3_play_name && std::fabs(speed - last_mp3_play_speed) < 0.001f && now - last_mp3_play_time < 15000000) {
+ // 璺熻/澶嶈浼氳繛缁挱鍚屼竴鏂囦欢锛涗笉瑕佹寜璺緞鍘婚噸銆俧orce 缁欐棫鍥轰欢閲嶈瘯鐢ㄣ��
+ bool lesson = (task_id == "shadowing" || task_id == "repeater");
+ if (!force && !lesson && playlist_n < 2 && name == last_mp3_play_name &&
+ std::fabs(speed - last_mp3_play_speed) < 0.001f &&
+ now - last_mp3_play_time < 400000) {
return std::string("{\"accepted\":true,\"duplicate_ignored\":true}");
}
-
- esp_err_t err = SDCard_Play_MP3_With_Speed_Repeat(name.data(), speed, repeat, power_off_after);
+ auto &audio = Application::GetInstance().GetAudioService();
+ if (task_id == "shadowing") {
+ audio.EnsureVoiceProcessorReady();
+ }
+ audio.EnableVoiceProcessing(false);
+ audio.EnableWakeWordDetection(false);
+ esp_err_t err = ESP_FAIL;
+ if (playlist_n > 0) {
+ err = SDCard_Play_MP3_File_Playlist(playlist, playlist_n);
+ if (err == ESP_OK) {
+ SDCard_Show_Matching_JPG_For_MP3_Path(playlist[0].path);
+ }
+ } else {
+ err = SDCard_Play_MP3_With_Speed_Repeat(name.data(), speed, repeat, power_off_after);
+ }
if (err != ESP_OK) {
return std::string("{\"accepted\":false,\"error\":\"") + esp_err_to_name(err) +
"\",\"retry\":\"use exact files[].name from self.mp3.get_list\"}";
@@ -751,7 +867,12 @@
last_mp3_play_name = name;
last_mp3_play_speed = speed;
last_mp3_play_time = now;
+ std::string remembered = name;
+ Application::GetInstance().Schedule([remembered, speed, repeat, power_off_after]() {
+ Board_RememberRepeaterPlayback(remembered.c_str(), speed, repeat, power_off_after);
+ });
return std::string("{\"accepted\":true,\"repeat\":") + std::to_string(repeat) +
+ ",\"items\":" + std::to_string(playlist_n > 0 ? playlist_n : 1) +
",\"power_off_after\":" + (power_off_after ? "true" : "false") + "}";
});
@@ -868,7 +989,7 @@
void OnDeviceReady() override {
xTaskCreate([](void *arg) {
(void)arg;
- vTaskDelay(pdMS_TO_TICKS(1500));
+ vTaskDelay(pdMS_TO_TICKS(2500));
Board_ResolveBootOrRestore(false);
vTaskDelete(nullptr);
}, "boot_app_task", 4096, nullptr, 3, nullptr);
diff --git a/xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/system.c b/xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/system.c
index a9c94be..d1c2a1f 100644
--- a/xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/system.c
+++ b/xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/system.c
@@ -126,6 +126,45 @@
ESP_LOGI(TAG, "offline command triggered: %s", OFFLINE_COMMAND_WORD);
}
+bool Board_Is_Mode_Switch_Command(const char *command)
+{
+ return command &&
+ (strcmp(command, OFFLINE_COMMAND_WORD) == 0 ||
+ strcmp(command, OFFLINE_COMMAND_WORD_PINYIN) == 0 ||
+ strcmp(command, OFFLINE_COMMAND_WORD_ALT) == 0 ||
+ strcmp(command, OFFLINE_COMMAND_WORD_ALT_PINYIN) == 0);
+}
+
+bool Board_Is_Chat_Return_Command(const char *command)
+{
+ if (!command || command[0] == '\0') {
+ return false;
+ }
+ return strcmp(command, OFFLINE_CHAT_WORD) == 0 ||
+ strcmp(command, OFFLINE_CHAT_WORD_PINYIN) == 0 ||
+ strcmp(command, OFFLINE_CHAT_WORD_FULL) == 0 ||
+ strcmp(command, OFFLINE_CHAT_WORD_FULL_PINYIN) == 0 ||
+ strcmp(command, OFFLINE_EXIT_REPEATER_WORD) == 0 ||
+ strcmp(command, OFFLINE_EXIT_REPEATER_WORD_PINYIN) == 0 ||
+ strcmp(command, OFFLINE_EXIT_SHADOWING_WORD) == 0 ||
+ strcmp(command, OFFLINE_EXIT_SHADOWING_WORD_PINYIN) == 0 ||
+ strstr(command, OFFLINE_CHAT_WORD_FULL) ||
+ strstr(command, OFFLINE_CHAT_WORD) ||
+ strstr(command, OFFLINE_EXIT_REPEATER_WORD) ||
+ strstr(command, OFFLINE_EXIT_SHADOWING_WORD);
+}
+
+bool Board_Is_App_Launch_Command(const char *command)
+{
+ if (!command || command[0] == '\0') {
+ return false;
+ }
+ return strcmp(command, OFFLINE_REPEATER_WORD) == 0 ||
+ strcmp(command, OFFLINE_START_REPEATER_WORD) == 0 ||
+ strstr(command, OFFLINE_REPEATER_WORD) ||
+ strstr(command, OFFLINE_START_REPEATER_WORD);
+}
+
bool Board_Handle_Offline_Command(const char *command)
{
if (!command || Board_Is_Power_Off()) {
@@ -134,7 +173,7 @@
if (Board_Handle_Voice_Command(command)) {
return true;
}
- if (strcmp(command, OFFLINE_COMMAND_WORD) != 0 && strcmp(command, OFFLINE_COMMAND_WORD_PINYIN) != 0) {
+ if (!Board_Is_Mode_Switch_Command(command)) {
return false;
}
@@ -530,19 +569,23 @@
return build_media_list_json(mp3_list_buffer, sizeof(mp3_list_buffer), mp3_name_has_extension);
}
-static bool is_supported_mp3_speed(float speed)
+float SDCard_Clamp_MP3_Speed(float speed)
{
- const float supported_speeds[] = {0.5f, 0.75f, 1.0f, 1.25f, 1.5f, 2.0f};
- for (size_t i = 0; i < sizeof(supported_speeds) / sizeof(supported_speeds[0]); ++i) {
- float diff = speed - supported_speeds[i];
- if (diff < 0) {
- diff = -diff;
- }
- if (diff < 0.001f) {
- return true;
- }
+ if (speed != speed) {
+ return SDCARD_MP3_SPEED_DEFAULT;
}
- return false;
+ if (speed < SDCARD_MP3_SPEED_MIN) {
+ return SDCARD_MP3_SPEED_MIN;
+ }
+ if (speed > SDCARD_MP3_SPEED_MAX) {
+ return SDCARD_MP3_SPEED_MAX;
+ }
+ return speed;
+}
+
+bool SDCard_Resolve_MP3_Path(const char *mp3_name, char *path, size_t path_size)
+{
+ return resolve_mp3_path(mp3_name, path, path_size);
}
esp_err_t SDCard_Play_MP3(char *mp3_name)
@@ -557,11 +600,7 @@
esp_err_t SDCard_Play_MP3_With_Speed_Repeat(char *mp3_name, float speed, int repeat, bool power_off_after)
{
- if (!is_supported_mp3_speed(speed)) {
- ESP_LOGW(TAG, "invalid MP3 playback speed: %.2f", speed);
- SDCard_Resume_Play();
- return ESP_ERR_INVALID_ARG;
- }
+ speed = SDCard_Clamp_MP3_Speed(speed);
char path[SDCARD_PATH_MAX_LEN];
if (!resolve_mp3_path(mp3_name, path, sizeof(path))) {
@@ -610,6 +649,22 @@
return ESP_OK;
}
+esp_err_t SDCard_Ensure_Business_Dir(const char *business_path)
+{
+ char sd_path[SDCARD_PATH_MAX_LEN];
+ esp_err_t err = SDCard_Map_Business_Path(business_path, sd_path, sizeof(sd_path));
+ if (err != ESP_OK) {
+ return err;
+ }
+ err = mkdir_recursive(sd_path);
+ if (err != ESP_OK) {
+ ESP_LOGE(TAG, "ensure dir failed: %s", sd_path);
+ return err;
+ }
+ ESP_LOGI(TAG, "ensure dir: %s", sd_path);
+ return ESP_OK;
+}
+
esp_err_t SDCard_Prepare_Business_File(const char *business_path,
char *final_path,
size_t final_path_size,
diff --git a/xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/system.h b/xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/system.h
index cced675..5500bc2 100644
--- a/xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/system.h
+++ b/xiaozhi-esp32-2.2.6/main/boards/sening-xiaozhi-s3/system.h
@@ -36,6 +36,14 @@
#define MOUNT_POINT "/sdcard"
#define SDCARD_PATH_MAX_LEN 256
+#define SDCARD_MP3_PLAYLIST_MAX 16
+
+typedef struct {
+ char path[SDCARD_PATH_MAX_LEN];
+ float speed;
+ int repeat;
+ bool power_off_after;
+} sdcard_mp3_playlist_item_t;
#define SCREEN_MIRROR_X false
#define SCREEN_MIRROR_Y false
@@ -75,14 +83,32 @@
#define BOARD_CUSTOM_WAKE_WORD_THRESHOLD_PERCENT 20
/**
- * @brief 鏉跨骇 MultiNet 绂荤嚎鍛戒护璇嶉厤缃��
- *
+ * @brief 鏉跨骇 MultiNet 绂荤嚎鍛戒护璇嶃�傚彧鐣欐病缃戜篃蹇呴』鑳界敤鐨勫叆鍙ｏ細
+ * 銆屽垏鎹㈡ā寮忋�嶃�屾ā寮忓垏鎹€�嶃�屽璇绘満銆嶃�屽紑濮嬪璇汇�嶃�岄��鍑哄璇汇�嶃�岄��鍑鸿窡璇汇�嶃�屾垜浠亰澶╁惂銆嶃�屾垜浠潵鑱婂ぉ鍚с�嶃��
+ * 澶嶈/璺熻鎾煶棰戞椂涓嶄笂琛� ASR锛岄��鍑哄彛浠ゅ繀椤昏兘绂荤嚎璇嗗埆銆傚叾瀹� App 鐨勮繘鍏ヨ瘝浠嶅彧鍦� Server apps.json銆�
*/
#define OFFLINE_COMMAND_WORD "鍒囨崲妯″紡"
#define OFFLINE_COMMAND_WORD_PINYIN "qie huan mo shi"
+#define OFFLINE_COMMAND_WORD_ALT "妯″紡鍒囨崲"
+#define OFFLINE_COMMAND_WORD_ALT_PINYIN "mo shi qie huan"
+#define OFFLINE_REPEATER_WORD "澶嶈鏈�"
+#define OFFLINE_REPEATER_WORD_PINYIN "fu du ji"
+#define OFFLINE_START_REPEATER_WORD "寮�濮嬪璇�"
+#define OFFLINE_START_REPEATER_WORD_PINYIN "kai shi fu du"
+#define OFFLINE_EXIT_REPEATER_WORD "閫�鍑哄璇�"
+#define OFFLINE_EXIT_REPEATER_WORD_PINYIN "tui chu fu du"
+#define OFFLINE_EXIT_SHADOWING_WORD "閫�鍑鸿窡璇�"
+#define OFFLINE_EXIT_SHADOWING_WORD_PINYIN "tui chu gen du"
+#define OFFLINE_CHAT_WORD "鎴戜滑鑱婂ぉ鍚�"
+#define OFFLINE_CHAT_WORD_PINYIN "wo men liao tian ba"
+#define OFFLINE_CHAT_WORD_FULL "鎴戜滑鏉ヨ亰澶╁惂"
+#define OFFLINE_CHAT_WORD_FULL_PINYIN "wo men lai liao tian ba"
void Board_Offline_Command_Callback(void);
bool Board_Handle_Offline_Command(const char *command);
+bool Board_Is_Mode_Switch_Command(const char *command);
+bool Board_Is_Chat_Return_Command(const char *command);
+bool Board_Is_App_Launch_Command(const char *command);
/**
* @brief 浣跨敤 SNTP 缃戠粶鏃堕棿鍚屾 DS3231 RTC銆�
@@ -192,14 +218,26 @@
*/
esp_err_t SDCard_Play_MP3(char *mp3_name);
+#define SDCARD_MP3_SPEED_MIN 0.5f
+#define SDCARD_MP3_SPEED_MAX 2.0f
+#define SDCARD_MP3_SPEED_DEFAULT 1.0f
+
+/**
+ * @brief 灏嗘挱鏀惧�嶇巼澶圭揣鍒� [0.5, 2.0]锛汵aN/闈炴硶鍒欏洖閫� 1.0銆�
+ *
+ * 瀹炵幇鏄厛鎸� speed 鍋氱嚎鎬ф椂鍩熸媺浼革紝鍐嶇敤鏂囦欢鍘熷閲囨牱鐜囪浆鍒板枃鍙噰鏍风巼銆�
+ * 0.5锝�2.0 鍐呬换鎰忔诞鐐瑰潎鍙紱闈� 1.0 浼氭敼鍙橀煶璋冦�備笉瑕佹妸 sample_rate*speed 浜ょ粰 AE 鍙橀噰鏍枫��
+ */
+float SDCard_Clamp_MP3_Speed(float speed);
+
/**
* @brief 鎸夋枃浠跺悕鍜岄�熷害鍚姩 SD 鍗� MP3 鎾斁銆�
*
- * speed 浠呮敮鎸� 0.5銆�0.75銆�1.0銆�1.25銆�1.5銆�2.0銆傞潪 1.0 閫熷害閫氳繃閲嶉噰鏍峰疄鐜帮紝浼氭敼鍙橀煶璋冦��
+ * speed 鍙栧�� [0.5, 2.0]锛岀己鐪� 1.0锛涜秴鍑鸿寖鍥翠細琚す绱с�傞潪 1.0 閫氳繃閲嶉噰鏍峰疄鐜帮紝浼氭敼鍙橀煶璋冦��
*
* @param mp3_name MP3 鏂囦欢鍚嶆垨鍖归厤鍏抽敭瀛楋紝涓嶈兘涓虹┖銆�
* @param speed 鎾斁閫熷害鍊嶇巼銆�
- * @return ESP_OK 琛ㄧず鎾斁浠诲姟宸叉垚鍔熷惎鍔紱ESP_ERR_INVALID_ARG 琛ㄧず閫熷害鎴栨枃浠跺悕闈炴硶銆�
+ * @return ESP_OK 琛ㄧず鎾斁浠诲姟宸叉垚鍔熷惎鍔紱ESP_ERR_INVALID_ARG 琛ㄧず鏂囦欢鍚嶉潪娉曘��
*/
esp_err_t SDCard_Play_MP3_With_Speed(char *mp3_name, float speed);
esp_err_t SDCard_Play_MP3_With_Speed_Repeat(char *mp3_name, float speed, int repeat, bool power_off_after);
@@ -266,16 +304,19 @@
/**
* @brief 鎸夊畬鏁磋矾寰勫拰閫熷害鍚姩 MP3 鏂囦欢鎾斁銆�
*
- * speed 浠呮敮鎸� 0.5銆�0.75銆�1.0銆�1.25銆�1.5銆�2.0銆傞潪 1.0 閫熷害閫氳繃閲嶉噰鏍峰疄鐜帮紝浼氭敼鍙橀煶璋冦��
+ * speed 鍙栧�� [0.5, 2.0]锛岀己鐪� 1.0锛涜秴鍑鸿寖鍥翠細琚す绱с�傞潪 1.0 閫氳繃閲嶉噰鏍峰疄鐜帮紝浼氭敼鍙橀煶璋冦��
*
* @param path MP3 鏂囦欢瀹屾暣璺緞锛屼笉鑳戒负绌恒��
* @param speed 鎾斁閫熷害鍊嶇巼銆�
- * @return ESP_OK 琛ㄧず鎾斁浠诲姟宸叉垚鍔熷惎鍔紱ESP_ERR_INVALID_ARG 琛ㄧず璺緞鎴栭�熷害闈炴硶銆�
+ * @return ESP_OK 琛ㄧず鎾斁浠诲姟宸叉垚鍔熷惎鍔紱ESP_ERR_INVALID_ARG 琛ㄧず璺緞闈炴硶銆�
*/
esp_err_t SDCard_Play_MP3_File_With_Speed(const char *path, float speed);
esp_err_t SDCard_Play_MP3_File_With_Speed_Repeat(const char *path, float speed, int repeat, bool power_off_after);
+bool SDCard_Resolve_MP3_Path(const char *mp3_name, char *path, size_t path_size);
+esp_err_t SDCard_Play_MP3_File_Playlist(const sdcard_mp3_playlist_item_t *items, int count);
int SDCard_Get_Repeat_Left(void);
esp_err_t SDCard_Map_Business_Path(const char *business_path, char *sd_path, size_t sd_path_size);
+esp_err_t SDCard_Ensure_Business_Dir(const char *business_path);
esp_err_t SDCard_Prepare_Business_File(const char *business_path,
char *final_path,
size_t final_path_size,
@@ -394,6 +435,7 @@
void Board_Acquire_Sleep_Inhibit(void);
void Board_Release_Sleep_Inhibit(void);
bool Board_Is_Voice_Session_Active(void);
+bool Board_Is_Audio_Busy(void);
void Board_Show_JPG(const char *jpg_name);
void Board_Show_Black_Screen(void);
void Board_Show_Alert(const char *status, const char *message, const char *emotion);
diff --git a/xiaozhi-esp32-2.2.6/main/boards/waveshare/esp32-s3-touch-lcd-4.3c/sdkconfig.4_3c b/xiaozhi-esp32-2.2.6/main/boards/waveshare/esp32-s3-touch-lcd-4.3c/sdkconfig.4_3c
index 2f73cb6..0e9ffb0 100755
--- a/xiaozhi-esp32-2.2.6/main/boards/waveshare/esp32-s3-touch-lcd-4.3c/sdkconfig.4_3c
+++ b/xiaozhi-esp32-2.2.6/main/boards/waveshare/esp32-s3-touch-lcd-4.3c/sdkconfig.4_3c
@@ -597,7 +597,8 @@
#
# Xiaozhi Assistant
#
-CONFIG_OTA_URL="https://api.tenclass.net/xiaozhi/ota/"
+#CONFIG_OTA_URL="https://api.tenclass.net/xiaozhi/ota/"
+CONFIG_OTA_URL="http://192.168.0.107:8002/xiaozhi/ota/"
# CONFIG_FLASH_NONE_ASSETS is not set
CONFIG_FLASH_DEFAULT_ASSETS=y
# CONFIG_FLASH_CUSTOM_ASSETS is not set
diff --git a/xiaozhi-esp32-2.2.6/main/protocols/protocol.cc b/xiaozhi-esp32-2.2.6/main/protocols/protocol.cc
index 470cc91..360e277 100644
--- a/xiaozhi-esp32-2.2.6/main/protocols/protocol.cc
+++ b/xiaozhi-esp32-2.2.6/main/protocols/protocol.cc
@@ -48,10 +48,20 @@
SendText(message);
}
-void Protocol::SendWakeWordDetected(const std::string& wake_word) {
- std::string json = "{\"session_id\":\"" + session_id_ +
- "\",\"type\":\"listen\",\"state\":\"detect\",\"text\":\"" + wake_word + "\"}";
- SendText(json);
+bool Protocol::SendWakeWordDetected(const std::string& wake_word) {
+ cJSON* root = cJSON_CreateObject();
+ cJSON_AddStringToObject(root, "session_id", session_id_.c_str());
+ cJSON_AddStringToObject(root, "type", "listen");
+ cJSON_AddStringToObject(root, "state", "detect");
+ cJSON_AddStringToObject(root, "text", wake_word.c_str());
+ char* json = cJSON_PrintUnformatted(root);
+ cJSON_Delete(root);
+ if (json == nullptr) {
+ return false;
+ }
+ bool ok = SendText(json);
+ cJSON_free(json);
+ return ok;
}
void Protocol::SendStartListening(ListeningMode mode) {
@@ -78,13 +88,21 @@
SendText(message);
}
+bool Protocol::IsAudioChannelConnected() const {
+ return IsAudioChannelOpened();
+}
+
+void Protocol::MarkIncomingActivity() {
+ last_incoming_time_ = std::chrono::steady_clock::now();
+}
+
bool Protocol::IsTimeout() const {
const int kTimeoutSeconds = 120;
auto now = std::chrono::steady_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::seconds>(now - last_incoming_time_);
bool timeout = duration.count() > kTimeoutSeconds;
if (timeout) {
- ESP_LOGE(TAG, "Channel timeout %ld seconds", (long)duration.count());
+ ESP_LOGW(TAG, "Channel idle %ld seconds (socket may still be up)", (long)duration.count());
}
return timeout;
}
diff --git a/xiaozhi-esp32-2.2.6/main/protocols/protocol.h b/xiaozhi-esp32-2.2.6/main/protocols/protocol.h
index 0f927f6..0f793f0 100644
--- a/xiaozhi-esp32-2.2.6/main/protocols/protocol.h
+++ b/xiaozhi-esp32-2.2.6/main/protocols/protocol.h
@@ -67,8 +67,12 @@
virtual bool OpenAudioChannel() = 0;
virtual void CloseAudioChannel(bool send_goodbye = true) = 0;
virtual bool IsAudioChannelOpened() const = 0;
+ // Socket is up and can send. Unlike IsAudioChannelOpened(), this ignores idle timeout
+ // so long local MP3 playback does not force a reconnect.
+ virtual bool IsAudioChannelConnected() const;
+ void MarkIncomingActivity();
virtual bool SendAudio(std::unique_ptr<AudioStreamPacket> packet) = 0;
- virtual void SendWakeWordDetected(const std::string& wake_word);
+ virtual bool SendWakeWordDetected(const std::string& wake_word);
virtual void SendStartListening(ListeningMode mode);
virtual void SendStopListening();
virtual void SendAbortSpeaking(AbortReason reason);
diff --git a/xiaozhi-esp32-2.2.6/main/protocols/websocket_protocol.cc b/xiaozhi-esp32-2.2.6/main/protocols/websocket_protocol.cc
index 4b74a5e..d232739 100644
--- a/xiaozhi-esp32-2.2.6/main/protocols/websocket_protocol.cc
+++ b/xiaozhi-esp32-2.2.6/main/protocols/websocket_protocol.cc
@@ -8,6 +8,7 @@
#include <cJSON.h>
#include <esp_log.h>
#include <arpa/inet.h>
+#include <freertos/task.h>
#include "assets/lang_config.h"
#define TAG "WS"
@@ -71,8 +72,14 @@
return true;
}
+bool WebsocketProtocol::IsAudioChannelConnected() const {
+ return websocket_ != nullptr && websocket_->IsConnected() && !error_occurred_;
+}
+
bool WebsocketProtocol::IsAudioChannelOpened() const {
- return websocket_ != nullptr && websocket_->IsConnected() && !error_occurred_ && !IsTimeout();
+ // Local MP3 can idle the websocket for minutes. Treat a live socket as open
+ // so detect/TTS reuse it instead of tearing it down and handshake-timing-out.
+ return IsAudioChannelConnected();
}
void WebsocketProtocol::CloseAudioChannel(bool send_goodbye) {
@@ -81,6 +88,18 @@
}
bool WebsocketProtocol::OpenAudioChannel() {
+ Board::GetInstance().SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE);
+
+ if (IsAudioChannelConnected()) {
+ MarkIncomingActivity();
+ ESP_LOGI(TAG, "Reuse existing websocket channel");
+ return true;
+ }
+
+ // MODEM 浠� MAX_MODEM 鍒囧洖 NONE 涓嶆槸鐬椂瀹屾垚鐨勩�傜珛鍒绘彙鎵嬩細
+ // WebSocket handshake timeout锛屽紑鏈烘杩庤瘝鍜屽璇荤粨鏉熷悗鍥� chat 閮戒細澶辫触銆�
+ vTaskDelay(pdMS_TO_TICKS(400));
+
Settings settings("websocket", false);
std::string url = settings.GetString("url");
std::string token = settings.GetString("token");
@@ -186,7 +205,7 @@
}
// Wait for server hello
- EventBits_t bits = xEventGroupWaitBits(event_group_handle_, WEBSOCKET_PROTOCOL_SERVER_HELLO_EVENT, pdTRUE, pdFALSE, pdMS_TO_TICKS(10000));
+ EventBits_t bits = xEventGroupWaitBits(event_group_handle_, WEBSOCKET_PROTOCOL_SERVER_HELLO_EVENT, pdTRUE, pdFALSE, pdMS_TO_TICKS(20000));
if (!(bits & WEBSOCKET_PROTOCOL_SERVER_HELLO_EVENT)) {
ESP_LOGE(TAG, "Failed to receive server hello");
SetError(Lang::Strings::SERVER_TIMEOUT);
diff --git a/xiaozhi-esp32-2.2.6/main/protocols/websocket_protocol.h b/xiaozhi-esp32-2.2.6/main/protocols/websocket_protocol.h
index 48bbc00..ea4e094 100644
--- a/xiaozhi-esp32-2.2.6/main/protocols/websocket_protocol.h
+++ b/xiaozhi-esp32-2.2.6/main/protocols/websocket_protocol.h
@@ -20,6 +20,7 @@
bool OpenAudioChannel() override;
void CloseAudioChannel(bool send_goodbye = true) override;
bool IsAudioChannelOpened() const override;
+ bool IsAudioChannelConnected() const override;
private:
EventGroupHandle_t event_group_handle_;
diff --git a/xiaozhi-esp32-2.2.6/scripts/patch_wifi_connect.py b/xiaozhi-esp32-2.2.6/scripts/patch_wifi_connect.py
new file mode 100644
index 0000000..3424ede
--- /dev/null
+++ b/xiaozhi-esp32-2.2.6/scripts/patch_wifi_connect.py
@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+"""Idempotent patches for 78/esp-wifi-connect captive portal.
+
+The managed component is gitignored; this script is applied at CMake configure
+so Android/WeChat locale codes (zh_CN) and socket exhaustion from captive DNS
+hijack (WeChat /mmtls) do not break the config page.
+"""
+
+from __future__ import annotations
+
+import re
+import sys
+from pathlib import Path
+
+HTML_MARKER = "xiaozhi-portal-lang-patch"
+HTTPD_MARKER = "xiaozhi-portal-httpd-patch"
+
+
+def _normalize_ws(text: str) -> str:
+ return text.replace("\r\n", "\n")
+
+
+def patch_html(path: Path) -> bool:
+ raw = _normalize_ws(path.read_text(encoding="utf-8"))
+ original = raw
+
+ if HTML_MARKER not in raw:
+ raw, n = re.subn(
+ r"function getSupportedLanguage\(lang\) \{\s*"
+ r"// First try exact match",
+ "function getSupportedLanguage(lang) {\n"
+ f" // {HTML_MARKER}\n"
+ " if (!lang) { return 'en-US'; }\n"
+ " lang = String(lang).replace(/_/g, '-');\n"
+ " // First try exact match",
+ raw,
+ count=1,
+ )
+ if n == 0:
+ print(f"warn: could not patch getSupportedLanguage in {path}")
+
+ if "getSupportedLanguage(langParam ||" not in raw:
+ raw, n = re.subn(
+ r"const savedLang = langParam \|\|\s*"
+ r"localStorage\.getItem\('preferred_language'\) \|\|\s*"
+ r"getSupportedLanguage\(browserLang\);",
+ "const savedLang = getSupportedLanguage(langParam ||\n"
+ " localStorage.getItem('preferred_language') ||\n"
+ " browserLang);",
+ raw,
+ count=1,
+ )
+ if n == 0:
+ print(f"warn: could not patch savedLang in {path}")
+
+ if raw == original:
+ print(f"already patched {path}")
+ return False
+ path.write_text(raw, encoding="utf-8", newline="\n")
+ print(f"patched {path}")
+ return True
+
+
+def patch_httpd(path: Path) -> bool:
+ raw = _normalize_ws(path.read_text(encoding="utf-8"))
+ if HTTPD_MARKER in raw:
+ print(f"already patched {path}")
+ return False
+ raw2, n = re.subn(
+ r"httpd_config_t config = HTTPD_DEFAULT_CONFIG\(\);\s*"
+ r"config\.max_uri_handlers = 24;\s*"
+ r"config\.uri_match_fn = httpd_uri_match_wildcard;",
+ "httpd_config_t config = HTTPD_DEFAULT_CONFIG();\n"
+ " config.max_uri_handlers = 24;\n"
+ f" config.lru_purge_enable = true; // {HTTPD_MARKER}\n"
+ " config.uri_match_fn = httpd_uri_match_wildcard;",
+ raw,
+ count=1,
+ )
+ if n == 0:
+ print(f"warn: could not patch httpd config in {path}")
+ return False
+ path.write_text(raw2, encoding="utf-8", newline="\n")
+ print(f"patched {path}")
+ return True
+
+
+def main() -> int:
+ if len(sys.argv) != 2:
+ print("usage: patch_wifi_connect.py <esp-wifi-connect component dir>")
+ return 2
+ root = Path(sys.argv[1])
+ if not root.is_dir():
+ print(f"error: not a directory: {root}")
+ return 1
+ html = root / "assets" / "wifi_configuration.html"
+ cc = root / "wifi_configuration_ap.cc"
+ if html.is_file():
+ patch_html(html)
+ else:
+ print(f"warn: missing {html}")
+ if cc.is_file():
+ patch_httpd(cc)
+ else:
+ print(f"warn: missing {cc}")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/xiaozhi-esp32-2.2.6/sdkconfig b/xiaozhi-esp32-2.2.6/sdkconfig
index ea4ad40..51943e9 100644
--- a/xiaozhi-esp32-2.2.6/sdkconfig
+++ b/xiaozhi-esp32-2.2.6/sdkconfig
@@ -600,7 +600,8 @@
#
# Xiaozhi Assistant
#
-CONFIG_OTA_URL="https://api.tenclass.net/xiaozhi/ota/"
+#CONFIG_OTA_URL="https://api.tenclass.net/xiaozhi/ota/"
+CONFIG_OTA_URL="http://192.168.0.107:8002/xiaozhi/ota/"
# CONFIG_FLASH_NONE_ASSETS is not set
CONFIG_FLASH_DEFAULT_ASSETS=y
# CONFIG_FLASH_CUSTOM_ASSETS is not set
diff --git a/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentChatHistoryReportDTO.java b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentChatHistoryReportDTO.java
index ae58783..3362439 100644
--- a/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentChatHistoryReportDTO.java
+++ b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/agent/dto/AgentChatHistoryReportDTO.java
@@ -30,4 +30,8 @@
private String audioBase64;
@Schema(description = "涓婃姤鏃堕棿锛屽崄浣嶆椂闂存埑锛岀┖鏃堕粯璁や娇鐢ㄥ綋鍓嶆椂闂�", example = "1745657732")
private Long reportTime;
+ @Schema(description = "鍙�夛細鎸傚埌 TaskInstance 鐨� id")
+ private String taskInstanceId;
+ @Schema(description = "鍙�夛細鎸傚埌 Task step_index")
+ private Integer stepIndex;
}
diff --git a/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/agent/service/biz/impl/AgentChatHistoryBizServiceImpl.java b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/agent/service/biz/impl/AgentChatHistoryBizServiceImpl.java
index ac1b5d0..6a97891 100644
--- a/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/agent/service/biz/impl/AgentChatHistoryBizServiceImpl.java
+++ b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/agent/service/biz/impl/AgentChatHistoryBizServiceImpl.java
@@ -22,6 +22,7 @@
import xiaozhi.modules.agent.service.biz.AgentChatHistoryBizService;
import xiaozhi.modules.device.entity.DeviceEntity;
import xiaozhi.modules.device.service.DeviceService;
+import xiaozhi.modules.task.service.TaskInstanceService;
/**
* {@link AgentChatHistoryBizService} impl
@@ -40,6 +41,7 @@
private final AgentChatSummaryService agentChatSummaryService;
private final RedisUtils redisUtils;
private final DeviceService deviceService;
+ private final TaskInstanceService taskInstanceService;
/**
* 澶勭悊鑱婂ぉ璁板綍涓婃姤锛屽寘鎷枃浠朵笂浼犲拰鐩稿叧淇℃伅璁板綍
@@ -65,12 +67,14 @@
Integer chatHistoryConf = agentEntity.getChatHistoryConf();
String agentId = agentEntity.getId();
+ String audioId = null;
if (Objects.equals(chatHistoryConf, Constant.ChatHistoryConfEnum.RECORD_TEXT.getCode())) {
saveChatText(report, agentId, macAddress, null, reportTimeMillis);
} else if (Objects.equals(chatHistoryConf, Constant.ChatHistoryConfEnum.RECORD_TEXT_AUDIO.getCode())) {
- String audioId = saveChatAudio(report);
+ audioId = saveChatAudio(report);
saveChatText(report, agentId, macAddress, audioId, reportTimeMillis);
}
+ attachTaskStepAudio(report, audioId);
// 鏇存柊璁惧鏈�鍚庡璇濇椂闂�
redisUtils.set(RedisKeys.getAgentDeviceLastConnectedAtById(agentId), new Date());
@@ -127,4 +131,16 @@
log.info("璁惧 {} 瀵瑰簲鏅鸿兘浣� {} 涓婃姤鎴愬姛", macAddress, agentId);
}
+
+ private void attachTaskStepAudio(AgentChatHistoryReportDTO report, String audioId) {
+ if (report == null || audioId == null || report.getTaskInstanceId() == null || report.getStepIndex() == null) {
+ return;
+ }
+ try {
+ taskInstanceService.attachAudio(report.getTaskInstanceId(), report.getStepIndex(), audioId);
+ } catch (Exception e) {
+ log.warn("鍥炲啓 Task step audio_id 澶辫触 instance={} step={}", report.getTaskInstanceId(),
+ report.getStepIndex(), e);
+ }
+ }
}
diff --git a/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/security/config/ShiroConfig.java b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/security/config/ShiroConfig.java
index 9e7c638..0960f16 100644
--- a/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/security/config/ShiroConfig.java
+++ b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/security/config/ShiroConfig.java
@@ -93,6 +93,8 @@
filterMap.put("/config/**", "server");
filterMap.put("/device/address-book/call", "server");
filterMap.put("/agent/chat-history/report", "server");
+ filterMap.put("/task/instance", "server");
+ filterMap.put("/task/instance/**", "server");
filterMap.put("/agent/chat-history/download/**", "anon");
filterMap.put("/agent/chat-summary/**", "server");
filterMap.put("/agent/chat-title/**", "server");
diff --git a/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/controller/TaskInstanceController.java b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/controller/TaskInstanceController.java
new file mode 100644
index 0000000..888ec7b
--- /dev/null
+++ b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/controller/TaskInstanceController.java
@@ -0,0 +1,53 @@
+package xiaozhi.modules.task.controller;
+
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PutMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.tags.Tag;
+import lombok.RequiredArgsConstructor;
+import xiaozhi.common.exception.ErrorCode;
+import xiaozhi.common.exception.RenException;
+import xiaozhi.common.utils.Result;
+import xiaozhi.modules.task.dto.TaskInstanceListDTO;
+import xiaozhi.modules.task.dto.TaskInstanceUpsertDTO;
+import xiaozhi.modules.task.service.TaskInstanceService;
+
+@Tag(name = "Task Instance")
+@RestController
+@RequestMapping("/task/instance")
+@RequiredArgsConstructor
+public class TaskInstanceController {
+ private final TaskInstanceService taskInstanceService;
+
+ @Operation(summary = "xiaozhi-server 涓婃姤 TaskSession 鍏ㄦ枃锛堝箓绛夎鐩栵級")
+ @PutMapping("/{id}")
+ public Result<Boolean> upsert(@PathVariable("id") String id, @RequestBody TaskInstanceUpsertDTO body) {
+ taskInstanceService.upsert(id, body);
+ return new Result<Boolean>().ok(Boolean.TRUE);
+ }
+
+ @Operation(summary = "鏌ヨ涓�鏉� TaskInstance锛堝惈 turns/events锛�")
+ @GetMapping("/{id}")
+ public Result<TaskInstanceUpsertDTO> get(@PathVariable("id") String id) {
+ TaskInstanceUpsertDTO data = taskInstanceService.getDetail(id);
+ if (data == null) {
+ throw new RenException(ErrorCode.RESOURCE_NOT_FOUND);
+ }
+ return new Result<TaskInstanceUpsertDTO>().ok(data);
+ }
+
+ @Operation(summary = "鎸夎澶囧垪鍑烘渶杩� TaskInstance锛堜笉鍚� turns 姝ｆ枃锛�")
+ @GetMapping
+ public Result<TaskInstanceListDTO> list(
+ @RequestParam("macAddress") String macAddress,
+ @RequestParam(value = "appId", required = false) String appId,
+ @RequestParam(value = "limit", required = false) Integer limit) {
+ return new Result<TaskInstanceListDTO>().ok(taskInstanceService.listByMac(macAddress, appId, limit));
+ }
+}
diff --git a/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dao/TaskEventDao.java b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dao/TaskEventDao.java
new file mode 100644
index 0000000..9df6a7c
--- /dev/null
+++ b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dao/TaskEventDao.java
@@ -0,0 +1,11 @@
+package xiaozhi.modules.task.dao;
+
+import org.apache.ibatis.annotations.Mapper;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+import xiaozhi.modules.task.entity.TaskEventEntity;
+
+@Mapper
+public interface TaskEventDao extends BaseMapper<TaskEventEntity> {
+}
diff --git a/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dao/TaskInstanceDao.java b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dao/TaskInstanceDao.java
new file mode 100644
index 0000000..6b74a4b
--- /dev/null
+++ b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dao/TaskInstanceDao.java
@@ -0,0 +1,11 @@
+package xiaozhi.modules.task.dao;
+
+import org.apache.ibatis.annotations.Mapper;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+import xiaozhi.modules.task.entity.TaskInstanceEntity;
+
+@Mapper
+public interface TaskInstanceDao extends BaseMapper<TaskInstanceEntity> {
+}
diff --git a/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dao/TaskStepDao.java b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dao/TaskStepDao.java
new file mode 100644
index 0000000..7e56ec4
--- /dev/null
+++ b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dao/TaskStepDao.java
@@ -0,0 +1,17 @@
+package xiaozhi.modules.task.dao;
+
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+import org.apache.ibatis.annotations.Update;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+import xiaozhi.modules.task.entity.TaskStepEntity;
+
+@Mapper
+public interface TaskStepDao extends BaseMapper<TaskStepEntity> {
+
+ @Update("UPDATE ai_task_step SET audio_id = #{audioId} WHERE instance_id = #{instanceId} AND step_index = #{stepIndex}")
+ int updateAudioId(@Param("instanceId") String instanceId, @Param("stepIndex") Integer stepIndex,
+ @Param("audioId") String audioId);
+}
diff --git a/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dto/TaskEventDTO.java b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dto/TaskEventDTO.java
new file mode 100644
index 0000000..dd5ddc4
--- /dev/null
+++ b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dto/TaskEventDTO.java
@@ -0,0 +1,20 @@
+package xiaozhi.modules.task.dto;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import lombok.Data;
+
+@Data
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class TaskEventDTO {
+ private String name;
+
+ @JsonProperty("app_id")
+ private String appId;
+
+ @JsonProperty("task_id")
+ private String taskId;
+
+ private Long at;
+}
diff --git a/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dto/TaskInstanceListDTO.java b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dto/TaskInstanceListDTO.java
new file mode 100644
index 0000000..29456fa
--- /dev/null
+++ b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dto/TaskInstanceListDTO.java
@@ -0,0 +1,12 @@
+package xiaozhi.modules.task.dto;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import lombok.Data;
+
+@Data
+public class TaskInstanceListDTO {
+ private List<TaskInstanceListItemDTO> list = new ArrayList<>();
+ private long total;
+}
diff --git a/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dto/TaskInstanceListItemDTO.java b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dto/TaskInstanceListItemDTO.java
new file mode 100644
index 0000000..2cafa02
--- /dev/null
+++ b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dto/TaskInstanceListItemDTO.java
@@ -0,0 +1,24 @@
+package xiaozhi.modules.task.dto;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import lombok.Data;
+
+@Data
+public class TaskInstanceListItemDTO {
+ private String id;
+
+ @JsonProperty("app_id")
+ private String appId;
+
+ @JsonProperty("task_id")
+ private String taskId;
+
+ private String status;
+
+ @JsonProperty("end_reason")
+ private String endReason;
+
+ @JsonProperty("started_at")
+ private Long startedAt;
+}
diff --git a/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dto/TaskInstanceUpsertDTO.java b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dto/TaskInstanceUpsertDTO.java
new file mode 100644
index 0000000..ba2cee8
--- /dev/null
+++ b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dto/TaskInstanceUpsertDTO.java
@@ -0,0 +1,53 @@
+package xiaozhi.modules.task.dto;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.Data;
+
+@Data
+@JsonIgnoreProperties(ignoreUnknown = true)
+@Schema(description = "TaskInstance upsert / GET 璇︽儏锛屽瓧娈典笌 TaskSession JSON 瀵归綈")
+public class TaskInstanceUpsertDTO {
+ @JsonProperty("task_session_id")
+ private String taskSessionId;
+
+ @JsonProperty("ws_session_id")
+ private String wsSessionId;
+
+ @JsonProperty("device_id")
+ private String deviceId;
+
+ @JsonProperty("agent_id")
+ private String agentId;
+
+ @JsonProperty("app_id")
+ private String appId;
+
+ @JsonProperty("task_id")
+ private String taskId;
+
+ private String source;
+
+ @JsonProperty("interaction_mode")
+ private String interactionMode;
+
+ @JsonProperty("started_at")
+ private Long startedAt;
+
+ @JsonProperty("ended_at")
+ private Long endedAt;
+
+ private String status;
+
+ @JsonProperty("end_reason")
+ private String endReason;
+
+ private List<TaskTurnDTO> turns = new ArrayList<>();
+
+ private List<TaskEventDTO> events = new ArrayList<>();
+}
diff --git a/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dto/TaskMetricDTO.java b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dto/TaskMetricDTO.java
new file mode 100644
index 0000000..6fb91a4
--- /dev/null
+++ b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dto/TaskMetricDTO.java
@@ -0,0 +1,12 @@
+package xiaozhi.modules.task.dto;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+import lombok.Data;
+
+@Data
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class TaskMetricDTO {
+ private String method;
+ private Double value;
+}
diff --git a/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dto/TaskTurnDTO.java b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dto/TaskTurnDTO.java
new file mode 100644
index 0000000..0a84f92
--- /dev/null
+++ b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/dto/TaskTurnDTO.java
@@ -0,0 +1,37 @@
+package xiaozhi.modules.task.dto;
+
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import lombok.Data;
+
+@Data
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class TaskTurnDTO {
+ private Integer index;
+
+ @JsonProperty("prompt_text")
+ private String promptText;
+
+ @JsonProperty("expected_text")
+ private String expectedText;
+
+ @JsonProperty("asr_text")
+ private String asrText;
+
+ @JsonProperty("echo_rejected")
+ private Boolean echoRejected;
+
+ @JsonProperty("recorded_at")
+ private Long recordedAt;
+
+ @JsonProperty("audio_id")
+ private String audioId;
+
+ private List<TaskMetricDTO> metrics;
+
+ @JsonProperty("assessed_at")
+ private Long assessedAt;
+}
diff --git a/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/entity/TaskEventEntity.java b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/entity/TaskEventEntity.java
new file mode 100644
index 0000000..afbad1b
--- /dev/null
+++ b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/entity/TaskEventEntity.java
@@ -0,0 +1,29 @@
+package xiaozhi.modules.task.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+
+import lombok.Data;
+
+@Data
+@TableName("ai_task_event")
+public class TaskEventEntity {
+ @TableId(type = IdType.AUTO)
+ private Long id;
+
+ @TableField("instance_id")
+ private String instanceId;
+
+ private String name;
+
+ @TableField("app_id")
+ private String appId;
+
+ @TableField("task_id")
+ private String taskId;
+
+ @TableField("at_ms")
+ private Long atMs;
+}
diff --git a/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/entity/TaskInstanceEntity.java b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/entity/TaskInstanceEntity.java
new file mode 100644
index 0000000..63c11af
--- /dev/null
+++ b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/entity/TaskInstanceEntity.java
@@ -0,0 +1,54 @@
+package xiaozhi.modules.task.entity;
+
+import java.util.Date;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+
+import lombok.Data;
+
+@Data
+@TableName("ai_task_instance")
+public class TaskInstanceEntity {
+ @TableId(type = IdType.INPUT)
+ private String id;
+
+ @TableField("mac_address")
+ private String macAddress;
+
+ @TableField("agent_id")
+ private String agentId;
+
+ @TableField("ws_session_id")
+ private String wsSessionId;
+
+ @TableField("app_id")
+ private String appId;
+
+ @TableField("task_id")
+ private String taskId;
+
+ private String source;
+
+ @TableField("interaction_mode")
+ private String interactionMode;
+
+ private String status;
+
+ @TableField("end_reason")
+ private String endReason;
+
+ @TableField("started_at")
+ private Long startedAt;
+
+ @TableField("ended_at")
+ private Long endedAt;
+
+ @TableField("created_at")
+ private Date createdAt;
+
+ @TableField("updated_at")
+ private Date updatedAt;
+}
diff --git a/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/entity/TaskStepEntity.java b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/entity/TaskStepEntity.java
new file mode 100644
index 0000000..dc351b1
--- /dev/null
+++ b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/entity/TaskStepEntity.java
@@ -0,0 +1,49 @@
+package xiaozhi.modules.task.entity;
+
+import java.util.List;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import com.baomidou.mybatisplus.extension.handlers.JacksonTypeHandler;
+
+import lombok.Data;
+import xiaozhi.modules.task.dto.TaskMetricDTO;
+
+@Data
+@TableName(value = "ai_task_step", autoResultMap = true)
+public class TaskStepEntity {
+ @TableId(type = IdType.AUTO)
+ private Long id;
+
+ @TableField("instance_id")
+ private String instanceId;
+
+ @TableField("step_index")
+ private Integer stepIndex;
+
+ @TableField("prompt_text")
+ private String promptText;
+
+ @TableField("expected_text")
+ private String expectedText;
+
+ @TableField("asr_text")
+ private String asrText;
+
+ @TableField("echo_rejected")
+ private Integer echoRejected;
+
+ @TableField("audio_id")
+ private String audioId;
+
+ @TableField(value = "metrics", typeHandler = JacksonTypeHandler.class)
+ private List<TaskMetricDTO> metrics;
+
+ @TableField("assessed_at")
+ private Long assessedAt;
+
+ @TableField("recorded_at")
+ private Long recordedAt;
+}
diff --git a/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/service/TaskInstanceService.java b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/service/TaskInstanceService.java
new file mode 100644
index 0000000..aa2f419
--- /dev/null
+++ b/xiaozhi-esp32-server/main/manager-api/src/main/java/xiaozhi/modules/task/service/TaskInstanceService.java
@@ -0,0 +1,259 @@
+package xiaozhi.modules.task.service;
+
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
+
+import lombok.RequiredArgsConstructor;
+import xiaozhi.common.exception.ErrorCode;
+import xiaozhi.common.exception.RenException;
+import xiaozhi.modules.task.dao.TaskEventDao;
+import xiaozhi.modules.task.dao.TaskInstanceDao;
+import xiaozhi.modules.task.dao.TaskStepDao;
+import xiaozhi.modules.task.dto.TaskEventDTO;
+import xiaozhi.modules.task.dto.TaskInstanceListDTO;
+import xiaozhi.modules.task.dto.TaskInstanceListItemDTO;
+import xiaozhi.modules.task.dto.TaskInstanceUpsertDTO;
+import xiaozhi.modules.task.dto.TaskTurnDTO;
+import xiaozhi.modules.task.entity.TaskEventEntity;
+import xiaozhi.modules.task.entity.TaskInstanceEntity;
+import xiaozhi.modules.task.entity.TaskStepEntity;
+
+@Service
+@RequiredArgsConstructor
+public class TaskInstanceService {
+ private static final Pattern LAYOUT_ID = Pattern.compile("^[A-Za-z0-9_-]{1,31}$");
+ private static final int LIST_MAX = 100;
+
+ private final TaskInstanceDao instanceDao;
+ private final TaskStepDao stepDao;
+ private final TaskEventDao eventDao;
+
+ @Transactional(rollbackFor = Exception.class)
+ public void upsert(String id, TaskInstanceUpsertDTO body) {
+ if (StringUtils.isBlank(id) || body == null) {
+ throw new RenException(ErrorCode.PARAM_VALUE_NULL);
+ }
+ if (StringUtils.isBlank(body.getTaskSessionId())) {
+ body.setTaskSessionId(id);
+ }
+ if (!id.equals(body.getTaskSessionId())) {
+ throw new RenException("path id must match task_session_id");
+ }
+ validateLayoutId(body.getAppId(), "app_id");
+ validateLayoutId(body.getTaskId(), "task_id");
+
+ String status = StringUtils.defaultIfBlank(body.getStatus(), "running");
+ if (!status.equals("running") && !status.equals("completed") && !status.equals("cancelled")) {
+ throw new RenException("invalid status");
+ }
+
+ try {
+ persistInstance(id, body, status);
+ } catch (RenException ex) {
+ throw ex;
+ } catch (Exception ex) {
+ throw new RenException("TaskInstance 鍐欏叆澶辫触: " + ex.getMessage(), ex);
+ }
+ }
+
+ private void persistInstance(String id, TaskInstanceUpsertDTO body, String status) {
+ Map<Integer, TaskStepEntity> existingSteps = loadExistingSteps(id);
+
+ TaskInstanceEntity existing = instanceDao.selectById(id);
+ Date now = new Date();
+ TaskInstanceEntity entity = new TaskInstanceEntity();
+ entity.setId(id);
+ entity.setMacAddress(StringUtils.left(StringUtils.trimToNull(body.getDeviceId()), 50));
+ entity.setAgentId(StringUtils.left(StringUtils.trimToNull(body.getAgentId()), 32));
+ entity.setWsSessionId(StringUtils.left(StringUtils.trimToEmpty(body.getWsSessionId()), 64));
+ entity.setAppId(body.getAppId());
+ entity.setTaskId(body.getTaskId());
+ entity.setSource(StringUtils.left(StringUtils.defaultIfBlank(body.getSource(), "voice"), 24));
+ entity.setInteractionMode(StringUtils.left(StringUtils.trimToEmpty(body.getInteractionMode()), 32));
+ entity.setStatus(status);
+ entity.setEndReason(StringUtils.left(StringUtils.trimToEmpty(body.getEndReason()), 32));
+ entity.setStartedAt(body.getStartedAt() == null ? 0L : body.getStartedAt());
+ entity.setEndedAt(body.getEndedAt());
+ entity.setUpdatedAt(now);
+ if (existing == null) {
+ entity.setCreatedAt(now);
+ instanceDao.insert(entity);
+ } else {
+ entity.setCreatedAt(existing.getCreatedAt());
+ instanceDao.updateById(entity);
+ }
+
+ stepDao.delete(new LambdaQueryWrapper<TaskStepEntity>().eq(TaskStepEntity::getInstanceId, id));
+ eventDao.delete(new LambdaQueryWrapper<TaskEventEntity>().eq(TaskEventEntity::getInstanceId, id));
+
+ List<TaskTurnDTO> turns = body.getTurns() == null ? List.of() : body.getTurns();
+ for (TaskTurnDTO turn : turns) {
+ if (turn == null || turn.getIndex() == null) {
+ continue;
+ }
+ TaskStepEntity step = new TaskStepEntity();
+ step.setInstanceId(id);
+ step.setStepIndex(turn.getIndex());
+ step.setPromptText(turn.getPromptText());
+ step.setAsrText(turn.getAsrText());
+ step.setEchoRejected(Boolean.TRUE.equals(turn.getEchoRejected()) ? 1 : 0);
+ step.setRecordedAt(turn.getRecordedAt());
+ TaskStepEntity previous = existingSteps.get(turn.getIndex());
+ String audioId = StringUtils.trimToNull(turn.getAudioId());
+ if (audioId == null && previous != null) {
+ audioId = previous.getAudioId();
+ }
+ step.setAudioId(audioId);
+ String expected = StringUtils.trimToNull(turn.getExpectedText());
+ if (expected == null && previous != null) {
+ expected = previous.getExpectedText();
+ }
+ step.setExpectedText(expected);
+ if (turn.getMetrics() != null && !turn.getMetrics().isEmpty()) {
+ step.setMetrics(turn.getMetrics());
+ step.setAssessedAt(turn.getAssessedAt());
+ } else if (previous != null) {
+ step.setMetrics(previous.getMetrics());
+ step.setAssessedAt(
+ turn.getAssessedAt() != null ? turn.getAssessedAt() : previous.getAssessedAt());
+ } else {
+ step.setMetrics(null);
+ step.setAssessedAt(turn.getAssessedAt());
+ }
+ stepDao.insert(step);
+ }
+
+ List<TaskEventDTO> events = body.getEvents() == null ? List.of() : body.getEvents();
+ for (TaskEventDTO event : events) {
+ if (event == null || StringUtils.isBlank(event.getName())) {
+ continue;
+ }
+ TaskEventEntity row = new TaskEventEntity();
+ row.setInstanceId(id);
+ row.setName(event.getName());
+ row.setAppId(StringUtils.defaultIfBlank(event.getAppId(), body.getAppId()));
+ row.setTaskId(StringUtils.defaultIfBlank(event.getTaskId(), body.getTaskId()));
+ row.setAtMs(event.getAt());
+ eventDao.insert(row);
+ }
+ }
+
+ public TaskInstanceUpsertDTO getDetail(String id) {
+ if (StringUtils.isBlank(id)) {
+ return null;
+ }
+ TaskInstanceEntity entity = instanceDao.selectById(id);
+ if (entity == null) {
+ return null;
+ }
+ TaskInstanceUpsertDTO dto = new TaskInstanceUpsertDTO();
+ dto.setTaskSessionId(entity.getId());
+ dto.setWsSessionId(entity.getWsSessionId());
+ dto.setDeviceId(entity.getMacAddress());
+ dto.setAgentId(entity.getAgentId());
+ dto.setAppId(entity.getAppId());
+ dto.setTaskId(entity.getTaskId());
+ dto.setSource(entity.getSource());
+ dto.setInteractionMode(entity.getInteractionMode());
+ dto.setStartedAt(entity.getStartedAt());
+ dto.setEndedAt(entity.getEndedAt());
+ dto.setStatus(entity.getStatus());
+ dto.setEndReason(entity.getEndReason());
+
+ List<TaskStepEntity> steps = stepDao.selectList(
+ new LambdaQueryWrapper<TaskStepEntity>().eq(TaskStepEntity::getInstanceId, id)
+ .orderByAsc(TaskStepEntity::getStepIndex));
+ for (TaskStepEntity step : steps) {
+ TaskTurnDTO turn = new TaskTurnDTO();
+ turn.setIndex(step.getStepIndex());
+ turn.setPromptText(step.getPromptText());
+ turn.setExpectedText(step.getExpectedText());
+ turn.setAsrText(step.getAsrText());
+ turn.setEchoRejected(step.getEchoRejected() != null && step.getEchoRejected() == 1);
+ turn.setRecordedAt(step.getRecordedAt());
+ turn.setAudioId(step.getAudioId());
+ turn.setMetrics(step.getMetrics());
+ turn.setAssessedAt(step.getAssessedAt());
+ dto.getTurns().add(turn);
+ }
+
+ List<TaskEventEntity> events = eventDao.selectList(
+ new LambdaQueryWrapper<TaskEventEntity>().eq(TaskEventEntity::getInstanceId, id)
+ .orderByAsc(TaskEventEntity::getId));
+ for (TaskEventEntity event : events) {
+ TaskEventDTO item = new TaskEventDTO();
+ item.setName(event.getName());
+ item.setAppId(event.getAppId());
+ item.setTaskId(event.getTaskId());
+ item.setAt(event.getAtMs());
+ dto.getEvents().add(item);
+ }
+ return dto;
+ }
+
+ public TaskInstanceListDTO listByMac(String macAddress, String appId, Integer limit) {
+ if (StringUtils.isBlank(macAddress)) {
+ throw new RenException(ErrorCode.PARAM_VALUE_NULL);
+ }
+ int size = limit == null ? 20 : Math.min(Math.max(limit, 1), LIST_MAX);
+ LambdaQueryWrapper<TaskInstanceEntity> wrapper = new LambdaQueryWrapper<TaskInstanceEntity>()
+ .eq(TaskInstanceEntity::getMacAddress, macAddress)
+ .eq(StringUtils.isNotBlank(appId), TaskInstanceEntity::getAppId, appId)
+ .orderByDesc(TaskInstanceEntity::getStartedAt)
+ .last("LIMIT " + size);
+
+ List<TaskInstanceEntity> rows = instanceDao.selectList(wrapper);
+ Long total = instanceDao.selectCount(new LambdaQueryWrapper<TaskInstanceEntity>()
+ .eq(TaskInstanceEntity::getMacAddress, macAddress)
+ .eq(StringUtils.isNotBlank(appId), TaskInstanceEntity::getAppId, appId));
+
+ TaskInstanceListDTO result = new TaskInstanceListDTO();
+ result.setTotal(total == null ? 0 : total);
+ for (TaskInstanceEntity row : rows) {
+ TaskInstanceListItemDTO item = new TaskInstanceListItemDTO();
+ item.setId(row.getId());
+ item.setAppId(row.getAppId());
+ item.setTaskId(row.getTaskId());
+ item.setStatus(row.getStatus());
+ item.setEndReason(row.getEndReason());
+ item.setStartedAt(row.getStartedAt());
+ result.getList().add(item);
+ }
+ return result;
+ }
+
+ @Transactional(rollbackFor = Exception.class)
+ public void attachAudio(String instanceId, Integer stepIndex, String audioId) {
+ if (StringUtils.isBlank(instanceId) || stepIndex == null || StringUtils.isBlank(audioId)) {
+ return;
+ }
+ stepDao.updateAudioId(instanceId, stepIndex, audioId);
+ }
+
+ private Map<Integer, TaskStepEntity> loadExistingSteps(String instanceId) {
+ Map<Integer, TaskStepEntity> map = new HashMap<>();
+ List<TaskStepEntity> steps = stepDao.selectList(
+ new LambdaQueryWrapper<TaskStepEntity>().eq(TaskStepEntity::getInstanceId, instanceId));
+ for (TaskStepEntity step : steps) {
+ if (step.getStepIndex() != null) {
+ map.put(step.getStepIndex(), step);
+ }
+ }
+ return map;
+ }
+
+ private void validateLayoutId(String id, String field) {
+ if (StringUtils.isBlank(id) || !LAYOUT_ID.matcher(id).matches()) {
+ throw new RenException("invalid " + field);
+ }
+ }
+}
diff --git a/xiaozhi-esp32-server/main/manager-api/src/main/resources/db/changelog/202504151206.sql b/xiaozhi-esp32-server/main/manager-api/src/main/resources/db/changelog/202504151206.sql
index 4113e24..cb51cef 100644
--- a/xiaozhi-esp32-server/main/manager-api/src/main/resources/db/changelog/202504151206.sql
+++ b/xiaozhi-esp32-server/main/manager-api/src/main/resources/db/changelog/202504151206.sql
@@ -20,7 +20,7 @@
-- 澧炲姞鏄惁鍏佽鐢ㄦ埛娉ㄥ唽鍙傛暟
delete from `sys_params` where id in (103,104);
INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (103, 'server.allow_user_register', 'false', 'boolean', 1, '鏄惁杩愯绠＄悊鍛樹互澶栫殑浜烘敞鍐�');
-INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (104, 'server.fronted_url', 'http://192.168.0.107', 'string', 1, '涓嬪彂鍏綅楠岃瘉鐮佹椂鏄剧ず鐨勬帶鍒堕潰鏉垮湴鍧�');
+INSERT INTO `sys_params` (id, param_code, param_value, value_type, param_type, remark) VALUES (104, 'server.fronted_url', 'http://xiaozhi.server.com', 'string', 1, '涓嬪彂鍏綅楠岃瘉鐮佹椂鏄剧ず鐨勬帶鍒堕潰鏉垮湴鍧�');
-- 淇CosyVoiceSiliconflow闊宠壊
delete from `ai_tts_voice` where tts_model_id = 'TTS_CosyVoiceSiliconflow';
diff --git a/xiaozhi-esp32-server/main/manager-api/src/main/resources/db/changelog/202608311800.sql b/xiaozhi-esp32-server/main/manager-api/src/main/resources/db/changelog/202608311800.sql
new file mode 100644
index 0000000..688ff86
--- /dev/null
+++ b/xiaozhi-esp32-server/main/manager-api/src/main/resources/db/changelog/202608311800.sql
@@ -0,0 +1,52 @@
+-- Phase 3 TaskInstance persistence (Session JSON projection). Do not DROP.
+
+CREATE TABLE `ai_task_instance`
+(
+ `id` VARCHAR(32) NOT NULL COMMENT '涓婚敭=task_session_id',
+ `mac_address` VARCHAR(50) DEFAULT NULL COMMENT '璁惧MAC',
+ `agent_id` VARCHAR(32) DEFAULT NULL COMMENT '鏅鸿兘浣搃d锛屽彲绌�',
+ `ws_session_id` VARCHAR(64) DEFAULT NULL COMMENT 'WS杩炴帴绾ession_id',
+ `app_id` VARCHAR(32) NOT NULL COMMENT 'App鍞竴鍚�',
+ `task_id` VARCHAR(32) NOT NULL COMMENT 'Task id',
+ `source` VARCHAR(24) DEFAULT NULL COMMENT 'voice/mqtt_immediate/schedule',
+ `interaction_mode` VARCHAR(32) DEFAULT NULL COMMENT 'SHADOWING/PLAY_AUDIO/...',
+ `status` VARCHAR(16) NOT NULL DEFAULT 'running' COMMENT 'running/completed/cancelled',
+ `end_reason` VARCHAR(32) DEFAULT NULL COMMENT 'finished/user_exit/disconnected/...',
+ `started_at` BIGINT DEFAULT NULL COMMENT 'epoch ms',
+ `ended_at` BIGINT DEFAULT NULL COMMENT 'epoch ms',
+ `created_at` DATETIME(3) DEFAULT CURRENT_TIMESTAMP(3) NOT NULL COMMENT '鍒涘缓鏃堕棿',
+ `updated_at` DATETIME(3) DEFAULT CURRENT_TIMESTAMP(3) NOT NULL ON UPDATE CURRENT_TIMESTAMP(3) COMMENT '鏇存柊鏃堕棿',
+ PRIMARY KEY (`id`),
+ KEY `idx_task_instance_mac_started` (`mac_address`, `started_at`),
+ KEY `idx_task_instance_app_task_started` (`app_id`, `task_id`, `started_at`),
+ KEY `idx_task_instance_status` (`status`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='Task瀹炰緥锛堣窡璇�/澶嶈浼氳瘽锛�';
+
+CREATE TABLE `ai_task_step`
+(
+ `id` BIGINT NOT NULL AUTO_INCREMENT COMMENT '涓婚敭',
+ `instance_id` VARCHAR(32) NOT NULL COMMENT 'ai_task_instance.id',
+ `step_index` INT NOT NULL COMMENT 'turns[].index',
+ `prompt_text` TEXT COMMENT '鑼冭鏂囨湰鎴朣D璺緞',
+ `asr_text` TEXT COMMENT '鐢ㄦ埛ASR',
+ `echo_rejected` TINYINT(1) NOT NULL DEFAULT 0 COMMENT '鏄惁鍥炲０鎷掔粷',
+ `audio_id` VARCHAR(32) DEFAULT NULL COMMENT 'ai_agent_chat_audio.id',
+ `recorded_at` BIGINT DEFAULT NULL COMMENT 'epoch ms',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `uk_task_step_instance_index` (`instance_id`, `step_index`),
+ KEY `idx_task_step_audio` (`audio_id`),
+ CONSTRAINT `fk_task_step_instance` FOREIGN KEY (`instance_id`) REFERENCES `ai_task_instance` (`id`) ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='Task姝ラ锛堣窡璇讳竴杞級';
+
+CREATE TABLE `ai_task_event`
+(
+ `id` BIGINT NOT NULL AUTO_INCREMENT COMMENT '涓婚敭',
+ `instance_id` VARCHAR(32) NOT NULL COMMENT 'ai_task_instance.id',
+ `name` VARCHAR(64) NOT NULL COMMENT 'audio.completed/audio.stopped/listen_timeout/...',
+ `app_id` VARCHAR(32) DEFAULT NULL,
+ `task_id` VARCHAR(32) DEFAULT NULL,
+ `at_ms` BIGINT DEFAULT NULL COMMENT 'JSON at锛宔poch ms',
+ PRIMARY KEY (`id`),
+ KEY `idx_task_event_instance` (`instance_id`),
+ CONSTRAINT `fk_task_event_instance` FOREIGN KEY (`instance_id`) REFERENCES `ai_task_instance` (`id`) ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='Task浜嬩欢';
diff --git a/xiaozhi-esp32-server/main/manager-api/src/main/resources/db/changelog/202609011100.sql b/xiaozhi-esp32-server/main/manager-api/src/main/resources/db/changelog/202609011100.sql
new file mode 100644
index 0000000..d59144d
--- /dev/null
+++ b/xiaozhi-esp32-server/main/manager-api/src/main/resources/db/changelog/202609011100.sql
@@ -0,0 +1,6 @@
+-- Phase 4 deferred assessment columns. Do not DROP. Do not add passed/grade.
+
+ALTER TABLE `ai_task_step`
+ ADD COLUMN `expected_text` TEXT NULL COMMENT '鑼冭鍘熸枃锛坱xt 濂戠害锛岄潪 SD 璺緞锛�' AFTER `prompt_text`,
+ ADD COLUMN `metrics` JSON NULL COMMENT '瀹氶噺鎸囨爣 [{method,value}]' AFTER `audio_id`,
+ ADD COLUMN `assessed_at` BIGINT NULL COMMENT '鍚庤瘎浼板畬鎴� epoch ms' AFTER `metrics`;
diff --git a/xiaozhi-esp32-server/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/xiaozhi-esp32-server/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml
index 17d4fb7..02a035d 100644
--- a/xiaozhi-esp32-server/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml
+++ b/xiaozhi-esp32-server/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml
@@ -739,3 +739,17 @@
- sqlFile:
encoding: utf8
path: classpath:db/changelog/202608131030.sql
+ - changeSet:
+ id: 202608311800
+ author: xiaozhi
+ changes:
+ - sqlFile:
+ encoding: utf8
+ path: classpath:db/changelog/202608311800.sql
+ - changeSet:
+ id: 202609011100
+ author: xiaozhi
+ changes:
+ - sqlFile:
+ encoding: utf8
+ path: classpath:db/changelog/202609011100.sql
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/config.yaml b/xiaozhi-esp32-server/main/xiaozhi-server/config.yaml
index 7ff25d3..6487bee 100644
--- a/xiaozhi-esp32-server/main/xiaozhi-server/config.yaml
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/config.yaml
@@ -56,7 +56,13 @@
data_dir: data
# 浣跨敤瀹屽０闊虫枃浠跺悗鍒犻櫎鏂囦欢(Delete the sound file when you are done using it)
+# 杩欏彧褰卞搷 ASR/TTS 涓存椂鏂囦欢锛屼笉褰卞搷涓嬮潰鐨� audio_archive 褰掓。锛屼篃涓嶅奖鍝嶆櫤鎺у彴鑱婂ぉ璁板綍
delete_audio: true
+# 鎸夎澶囩紪鍙� / 鍦烘櫙妯″紡 / 鏃ユ湡褰掓。鐢ㄦ埛闊抽锛屼究浜庡洖鏀惧拰鍒嗘瀽
+# 璺緞锛歞ata/recordings/{璁惧MAC}/{chat|shadowing|repeater|translation|bilingual}/{鏃ユ湡}/{鏃跺垎绉掓绉拀_asr.wav
+audio_archive:
+ enabled: true
+ dir: data/recordings
# 娌℃湁璇煶杈撳叆澶氫箙鍚庢柇寮�杩炴帴(绉�)锛岄粯璁�2鍒嗛挓锛屽嵆120绉�
close_connection_no_voice_time: 120
# TTS璇锋眰瓒呮椂鏃堕棿(绉�)
@@ -70,6 +76,14 @@
# Phase 1锛氳繘鍏� chat.conversation 鍚庝富鍔ㄥ紑鍙�
enable_proactive_greeting: true
proactive_chat_greeting: 鎴戜滑鏉ヨ亰澶╁惂銆�
+# 璺熻妯″紡榛樿鍙ュ瓙锛屽彲鎸夎绋嬫浛鎹�
+app_runtime:
+ shadowing_sentences:
+ - "Hello, how are you today?"
+ - "I like to read English books."
+ - "The weather is nice this morning."
+ - "Can you say that again, please?"
+ - "Practice a little every day."
# 璇村畬璇濇槸鍚﹀紑鍚彁绀洪煶
enable_stop_tts_notify: false
# 璇村畬璇濇槸鍚﹀紑鍚彁绀洪煶锛岄煶鏁堝湴鍧�
@@ -275,6 +289,7 @@
- play_music
- switch_app
- start_repeater
+ - start_shadowing
function_call:
# 涓嶉渶瑕佸姩type
type: function_call
@@ -293,6 +308,7 @@
- play_music
- switch_app
- start_repeater
+ - start_shadowing
#- hass_state
#- hass_play_music
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/config/assets/wakeup_words/ed76d459636c2481aec828516c1b4f54.wav b/xiaozhi-esp32-server/main/xiaozhi-server/config/assets/wakeup_words/ed76d459636c2481aec828516c1b4f54.wav
new file mode 100644
index 0000000..66d820c
--- /dev/null
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/config/assets/wakeup_words/ed76d459636c2481aec828516c1b4f54.wav
Binary files differ
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/config/manage_api_client.py b/xiaozhi-esp32-server/main/xiaozhi-server/config/manage_api_client.py
index 8eb2da2..002c5cb 100644
--- a/xiaozhi-esp32-server/main/xiaozhi-server/config/manage_api_client.py
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/config/manage_api_client.py
@@ -221,31 +221,55 @@
async def report(
- mac_address: str, session_id: str, chat_type: int, content: str, audio, report_time
+ mac_address: str, session_id: str, chat_type: int, content: str, audio, report_time,
+ task_instance_id: str = None, step_index: int = None,
) -> Optional[Dict]:
"""寮傛鑱婂ぉ璁板綍涓婃姤"""
if not content or not ManageApiClient._instance:
return None
try:
+ payload = {
+ "macAddress": mac_address,
+ "sessionId": session_id,
+ "chatType": chat_type,
+ "content": content,
+ "reportTime": report_time,
+ "audioBase64": (
+ base64.b64encode(audio).decode("utf-8") if audio else None
+ ),
+ }
+ if task_instance_id:
+ payload["taskInstanceId"] = task_instance_id
+ if step_index is not None:
+ payload["stepIndex"] = step_index
return await ManageApiClient._instance._execute_async_request(
"POST",
f"/agent/chat-history/report",
- json={
- "macAddress": mac_address,
- "sessionId": session_id,
- "chatType": chat_type,
- "content": content,
- "reportTime": report_time,
- "audioBase64": (
- base64.b64encode(audio).decode("utf-8") if audio else None
- ),
- },
+ json=payload,
)
except Exception as e:
print(f"TTS涓婃姤澶辫触: {e}")
return None
+async def upsert_task_instance(payload: Dict) -> Optional[Dict]:
+ """涓婃姤 TaskSession 鍏ㄦ枃銆傚け璐ヨ繑鍥� None锛屼笉鎶涚粰璇煶鐑矾寰勩�傛湭閰嶇疆绠＄悊绔涓烘垚鍔熴��"""
+ if not payload or not ManageApiClient._instance:
+ return {"skipped": True}
+ instance_id = payload.get("task_session_id") or ""
+ if not instance_id:
+ return {"skipped": True}
+ try:
+ return await ManageApiClient._instance._async_request(
+ "PUT",
+ f"/task/instance/{instance_id}",
+ json=payload,
+ )
+ except Exception as e:
+ print(f"TaskInstance 涓婃姤澶辫触: {e}")
+ return None
+
+
async def lookup_address_book(caller_mac: str, nickname: str) -> Optional[Dict]:
"""鏍规嵁鏄电О鏌ユ壘鐩爣璁惧"""
if not ManageApiClient._instance:
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/core/app_runtime/__init__.py b/xiaozhi-esp32-server/main/xiaozhi-server/core/app_runtime/__init__.py
index 8146ddb..cad45d9 100644
--- a/xiaozhi-esp32-server/main/xiaozhi-server/core/app_runtime/__init__.py
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/core/app_runtime/__init__.py
@@ -1,7 +1,8 @@
-"""Phase 1 App Runtime: chat / english_learning / english_translation."""
+"""App Runtime: chat / english_learning / english_translation / english_conversation."""
from .registry import get_app, get_task, get_registry, default_app_id
from .runtime import start_app_task, switch_app, filter_tools, on_user_wakeup
+from .session import TaskSession, parse_skill_event
__all__ = [
"get_app",
@@ -12,4 +13,6 @@
"switch_app",
"filter_tools",
"on_user_wakeup",
+ "TaskSession",
+ "parse_skill_event",
]
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/core/app_runtime/apps.json b/xiaozhi-esp32-server/main/xiaozhi-server/core/app_runtime/apps.json
index 2ddb63d..f69ad89 100644
--- a/xiaozhi-esp32-server/main/xiaozhi-server/core/app_runtime/apps.json
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/core/app_runtime/apps.json
@@ -1,47 +1,131 @@
{
"default_app_id": "chat",
+ "voice_menu_phrases": ["鍒囨崲妯″紡", "妯″紡鍒囨崲", "qie huan mo shi", "mo shi qie huan"],
+ "reserved_voice_enter": ["鏁板瓧鑳岃", "鎴愯鎺ラ緳", "鍙ｇ畻", "鍔犲噺娉�"],
"apps": {
"chat": {
"name": "鑱婂ぉ",
+ "display_name": "瀵硅瘽妯″紡",
"mode": "CHAT",
"default_task": "conversation",
"tool_policy": "inherit_agent",
"system_prompt_mode": "agent_proactive",
"tasks": {
"conversation": {
+ "display_name": "瀵硅瘽妯″紡",
+ "menu_name": "鑱婂ぉ",
"interaction_mode": "CHAT",
+ "enter_prompt": "",
+ "voice_enter": [
+ "鎴戜滑鏉ヨ亰澶╁惂",
+ "鎴戜滑鑱婂ぉ鍚�",
+ "鑱婁細澶╁惂",
+ "鑱婅亰澶╁惂",
+ "鍥炲埌鑱婂ぉ",
+ "杩斿洖鑱婂ぉ",
+ "鍥炲幓鑱婂ぉ",
+ "鍥炲埌榛樿",
+ "閫�鍑烘ā寮�",
+ "缁撴潫妯″紡",
+ "閫�鍑哄璇�",
+ "缁撴潫瀵硅瘽"
+ ],
+ "voice_exit": [],
"skills": []
}
}
},
"english_learning": {
- "name": "鑻辫瀛︿範",
+ "name": "鑻辫鍚姏",
"mode": "PLAY_AUDIO",
"default_task": "repeater",
"tool_policy": "allowlist",
"tools": [
"switch_app",
"start_repeater",
+ "start_shadowing",
"self_mp3_play",
"self_mp3_stop",
"self_mp3_get_list",
+ "self_sd_ensure_app",
"self_power_off"
],
"system_prompt_mode": "none",
"tasks": {
"repeater": {
+ "display_name": "澶嶈鏈烘ā寮�",
+ "menu_name": "鍚璇绘満",
+ "enter_prompt": "鐜板湪宸茬粡鍒囨崲鑷冲璇绘満妯″紡锛屽彧寰幆鎾斁闊抽銆�",
+ "voice_enter": ["寮�濮嬪璇�", "鎵撳紑澶嶈", "鑻辫瀛︿範", "鍚璇�", "澶嶈鏈�"],
+ "voice_exit": ["閫�鍑哄璇�", "缁撴潫澶嶈", "鍋滄澶嶈", "鍏虫帀澶嶈", "閫�鍑哄璇绘満"],
"interaction_mode": "PLAY_AUDIO",
"skills": [
{
"name": "self.mp3.play",
- "args_from": ["name", "repeat", "power_off_after"]
+ "args_from": ["name", "repeat", "power_off_after", "speed"]
}
+ ],
+ "playback": {
+ "speed": 1.0,
+ "daily_speeds": [0.8, 0.9, 1.0, 1.05, 1.1, 1.15, 1.2],
+ "repeat_speeds": [0.8, 0.9, 1.0, 1.1, 1.2]
+ },
+ "default_playlist": [
+ {
+ "name": "/english_learning/shadowing/lessons/unit01/01.mp3",
+ "repeat": 10,
+ "speed": 1.0
+ },
+ {
+ "name": "/english_learning/shadowing/lessons/unit01/02.mp3",
+ "repeat": 5,
+ "speed": 1.0
+ }
+ ]
+ },
+ "shadowing": {
+ "display_name": "璺熻妯″紡",
+ "menu_name": "璺熻",
+ "enter_prompt": "鐜板湪宸茬粡鍒囨崲鑷宠窡璇绘ā寮忥紝璇疯窡鐫�闊抽璇汇��",
+ "voice_enter": ["寮�濮嬭窡璇�", "鎵撳紑璺熻", "璺熻妯″紡", "鎴戣璺熻", "璺熻", "寮�濮嬪惉璇�", "鎵撳紑鍚", "鍚妯″紡", "鍚"],
+ "voice_exit": ["閫�鍑鸿窡璇�", "缁撴潫璺熻", "鍋滄璺熻", "鍏虫帀璺熻", "涓嶆兂璺熻", "閫�鍑哄惉璇�", "缁撴潫鍚"],
+ "interaction_mode": "SHADOWING",
+ "listen_ratio": 1.2,
+ "playback": {
+ "speed": 1.0
+ },
+ "assessment": {
+ "enabled": true,
+ "methods": ["text_similarity_v0"],
+ "defer": true,
+ "endpoint": ""
+ },
+ "skills": [],
+ "default_playlist": [
+ {
+ "name": "/english_learning/shadowing/lessons/unit01/01.mp3",
+ "repeat": 5,
+ "speed": 1.0
+ },
+ {
+ "name": "/english_learning/shadowing/lessons/unit01/02.mp3",
+ "repeat": 10,
+ "speed": 1.0
+ }
+ ],
+ "sentences": [
+ "Hello, how are you today?",
+ "I like to read English books.",
+ "The weather is nice this morning.",
+ "Can you say that again, please?",
+ "Practice a little every day."
]
}
}
},
"english_translation": {
"name": "鑻辫缈昏瘧",
+ "display_name": "缈昏瘧妯″紡",
"mode": "TRANSLATION",
"default_task": "live_translate",
"tool_policy": "allowlist",
@@ -49,6 +133,11 @@
"system_prompt_mode": "translator",
"tasks": {
"live_translate": {
+ "display_name": "缈昏瘧妯″紡",
+ "menu_name": "缈昏瘧",
+ "enter_prompt": "鐜板湪宸茬粡鍒囨崲鑷崇炕璇戞ā寮忥紝鑷姩杩涜涓嫳浜掕瘧銆�",
+ "voice_enter": ["鎵撳紑缈昏瘧", "寮�濮嬬炕璇�", "鑻辫缈昏瘧", "缈昏瘧妯″紡", "缈昏瘧"],
+ "voice_exit": ["閫�鍑虹炕璇�", "缁撴潫缈昏瘧", "鍋滄缈昏瘧", "鍏虫帀缈昏瘧", "涓嶆兂缈昏瘧"],
"interaction_mode": "TRANSLATION",
"skills": [
{
@@ -58,6 +147,26 @@
]
}
}
+ },
+ "english_conversation": {
+ "name": "涓嫳瀵硅瘽",
+ "display_name": "涓嫳瀵硅瘽妯″紡",
+ "mode": "BILINGUAL",
+ "default_task": "bilingual_chat",
+ "tool_policy": "inherit_agent",
+ "tools": ["switch_app"],
+ "system_prompt_mode": "bilingual",
+ "tasks": {
+ "bilingual_chat": {
+ "display_name": "涓嫳瀵硅瘽妯″紡",
+ "menu_name": "涓嫳瀵硅瘽",
+ "enter_prompt": "鐜板湪宸茬粡鍒囨崲鑷充腑鑻卞璇濇ā寮忋�備綘璇翠腑鏂囷紝鎴戠敤鑻辨枃鍥炵瓟锛涗綘璇磋嫳鏂囷紝鎴戠敤涓枃鍥炵瓟銆�",
+ "voice_enter": ["鎵撳紑涓嫳瀵硅瘽", "涓嫳瀵硅瘽妯″紡", "涓嫳瀵硅瘽"],
+ "voice_exit": ["閫�鍑轰腑鑻卞璇�", "缁撴潫涓嫳瀵硅瘽"],
+ "interaction_mode": "BILINGUAL",
+ "skills": []
+ }
+ }
}
}
}
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/core/app_runtime/assessment.py b/xiaozhi-esp32-server/main/xiaozhi-server/core/app_runtime/assessment.py
new file mode 100644
index 0000000..6eea494
--- /dev/null
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/core/app_runtime/assessment.py
@@ -0,0 +1,387 @@
+"""Deferred quantitative assessment. Never drives playback or speak()."""
+
+from __future__ import annotations
+
+import difflib
+import os
+import re
+from typing import Any, Dict, List, Optional
+
+from .playback import PLAY_SPEED_DEFAULT, clamp_play_speed
+
+LISTEN_RATIO_DEFAULT = 1.2
+LISTEN_RATIO_MIN = 1.2
+LISTEN_RATIO_MAX = 1.5
+LISTEN_WINDOW_MIN_MS = 800
+METHOD_TEXT_SIMILARITY_V0 = "text_similarity_v0"
+
+
+def looks_like_path(text: str) -> bool:
+ """True if text is an SD / file path, not a transcript line."""
+ raw = (text or "").strip().replace("\\", "/")
+ if not raw:
+ return False
+ lower = raw.lower()
+ if lower.endswith((".mp3", ".txt", ".wav", ".opus", ".p3")):
+ return True
+ if "/lessons/" in lower:
+ return True
+ if raw.startswith("/") and "/" in raw[1:]:
+ return True
+ return False
+
+
+def clip_stem(clip: str) -> str:
+ base = os.path.basename((clip or "").replace("\\", "/"))
+ return os.path.splitext(base)[0]
+
+
+def clip_basename(clip: str) -> str:
+ return os.path.basename((clip or "").replace("\\", "/"))
+
+
+def normalize(text: str) -> str:
+ from core.utils.util import remove_punctuation_and_length
+
+ _, filtered = remove_punctuation_and_length(text or "")
+ return re.sub(r"\s+", "", (filtered or "").lower())
+
+
+def text_similarity_v0(asr_text: str, expected_text: str) -> float:
+ a = normalize(asr_text)
+ b = normalize(expected_text)
+ if not a or not b:
+ return 0.0
+ value = float(difflib.SequenceMatcher(None, a, b).ratio())
+ if value < 0.0:
+ return 0.0
+ if value > 1.0:
+ return 1.0
+ return value
+
+
+def build_metrics(asr_text: str, expected_text: str) -> List[Dict[str, Any]]:
+ """Return [{method, value}] or []. Never includes passed/grade."""
+ expected = (expected_text or "").strip()
+ asr = (asr_text or "").strip()
+ if not asr or not expected or looks_like_path(expected):
+ return []
+ return [
+ {
+ "method": METHOD_TEXT_SIMILARITY_V0,
+ "value": text_similarity_v0(asr, expected),
+ }
+ ]
+
+
+def silent_pcm_snapshot(duration_ms: int = 200, sample_rate: int = 16000) -> list:
+ """Placeholder PCM so a silent follow-read still writes a wav."""
+ samples = max(1, int(sample_rate * max(duration_ms, 1) / 1000.0))
+ return [b"\x00\x00" * samples]
+
+
+def pcm_snapshot_to_wav(snapshot: Any, sample_rate: int = 16000) -> bytes:
+ """UDP/ASR PCM frames (list[bytes] or bytes) 鈫� WAV bytes. Empty input 鈫� b''."""
+ if not snapshot:
+ return b""
+ if isinstance(snapshot, (bytes, bytearray, memoryview)):
+ pcm = bytes(snapshot)
+ elif isinstance(snapshot, list):
+ parts = []
+ for item in snapshot:
+ if isinstance(item, (bytes, bytearray, memoryview)):
+ parts.append(bytes(item))
+ elif isinstance(item, str):
+ continue
+ else:
+ try:
+ parts.append(bytes(item))
+ except Exception:
+ continue
+ pcm = b"".join(parts)
+ else:
+ return b""
+ if len(pcm) % 2 == 1:
+ pcm = pcm[:-1]
+ if not pcm:
+ return b""
+ header = bytearray()
+ header.extend(b"RIFF")
+ header.extend((36 + len(pcm)).to_bytes(4, "little"))
+ header.extend(b"WAVE")
+ header.extend(b"fmt ")
+ header.extend((16).to_bytes(4, "little"))
+ header.extend((1).to_bytes(2, "little"))
+ header.extend((1).to_bytes(2, "little"))
+ header.extend(int(sample_rate).to_bytes(4, "little"))
+ header.extend((int(sample_rate) * 2).to_bytes(4, "little"))
+ header.extend((2).to_bytes(2, "little"))
+ header.extend((16).to_bytes(2, "little"))
+ header.extend(b"data")
+ header.extend(len(pcm).to_bytes(4, "little"))
+ return bytes(header) + pcm
+
+
+def save_turn_wav(
+ session_id: str,
+ step_index: int,
+ wav_bytes: bytes,
+ conn: Any = None,
+) -> Optional[str]:
+ """Persist the follow-read utterance. Returns relative audio_id or None."""
+ if not wav_bytes:
+ return None
+ extra = f"{session_id}_{int(step_index)}" if session_id else str(int(step_index))
+ if conn is not None:
+ from core.audio_archive import archive_wav
+
+ archived = archive_wav(conn, wav_bytes, kind="asr", extra=extra)
+ if archived:
+ return archived
+ if not session_id:
+ return None
+ try:
+ from config.config_loader import get_project_dir
+
+ directory = os.path.join(get_project_dir(), "data", "task_sessions", "audio")
+ os.makedirs(directory, exist_ok=True)
+ filename = f"{session_id}_{int(step_index)}.wav"
+ path = os.path.join(directory, filename)
+ with open(path, "wb") as handle:
+ handle.write(wav_bytes)
+ return filename
+ except Exception:
+ return None
+
+
+def _metrics_from_api_payload(data: Any) -> List[Dict[str, Any]]:
+ if not isinstance(data, dict):
+ return []
+ raw = data.get("metrics")
+ if not isinstance(raw, list):
+ if data.get("method") and data.get("value") is not None:
+ raw = [data]
+ else:
+ raw = []
+ out: List[Dict[str, Any]] = []
+ for item in raw:
+ if not isinstance(item, dict):
+ continue
+ method = str(item.get("method") or "").strip()
+ if not method:
+ continue
+ try:
+ value = float(item.get("value"))
+ except (TypeError, ValueError):
+ continue
+ if value < 0.0:
+ value = 0.0
+ if value > 1.0:
+ value = 1.0
+ out.append({"method": method, "value": value})
+ return out
+
+
+def call_assessment_endpoint(
+ endpoint: str,
+ asr_text: str,
+ expected_text: str,
+ wav_bytes: bytes,
+ extra: Optional[Dict[str, Any]] = None,
+) -> List[Dict[str, Any]]:
+ """POST ASR text + WAV to the scoring API. Failures return [] and never raise."""
+ url = (endpoint or "").strip()
+ if not url:
+ return []
+ try:
+ import httpx
+ except Exception:
+ return []
+ files = None
+ data = {
+ "asr_text": asr_text or "",
+ "expected_text": expected_text or "",
+ }
+ if extra:
+ for key, value in extra.items():
+ if value is None:
+ continue
+ data[str(key)] = str(value)
+ if wav_bytes:
+ files = {"audio": ("utterance.wav", wav_bytes, "audio/wav")}
+ try:
+ with httpx.Client(timeout=8.0) as client:
+ response = client.post(url, data=data, files=files)
+ if response.status_code >= 400:
+ return []
+ try:
+ payload = response.json()
+ except Exception:
+ return []
+ return _metrics_from_api_payload(payload)
+ except Exception:
+ return []
+
+
+def evaluate_turn(
+ asr_text: str,
+ expected_text: str,
+ wav_bytes: bytes = b"",
+ cfg: Optional[Dict[str, Any]] = None,
+ extra: Optional[Dict[str, Any]] = None,
+) -> List[Dict[str, Any]]:
+ """Quantitative follow-read score. Text similarity always; HTTP endpoint optional.
+
+ Never speaks, never advances playback. Missing expected text still saves audio
+ and may still call the remote API.
+ """
+ cfg = cfg or {}
+ by_method = {item["method"]: item for item in build_metrics(asr_text, expected_text)}
+ endpoint = str(cfg.get("endpoint") or cfg.get("url") or "").strip()
+ if endpoint:
+ for item in call_assessment_endpoint(
+ endpoint, asr_text or "", expected_text or "", wav_bytes or b"", extra
+ ):
+ by_method[item["method"]] = item
+ return list(by_method.values())
+
+
+def clamp_listen_ratio(raw: Any) -> float:
+ try:
+ value = float(raw)
+ except (TypeError, ValueError):
+ return LISTEN_RATIO_DEFAULT
+ if value < LISTEN_RATIO_MIN:
+ return LISTEN_RATIO_MIN
+ if value > LISTEN_RATIO_MAX:
+ return LISTEN_RATIO_MAX
+ return value
+
+
+def resolve_listen_ratio(conn: Any = None, task: Optional[Dict[str, Any]] = None) -> float:
+ """v0: task JSON constant. Hook for later per-user ratios."""
+ if task and task.get("listen_ratio") is not None:
+ return clamp_listen_ratio(task.get("listen_ratio"))
+ if conn is not None:
+ cfg = (getattr(conn, "config", None) or {}).get("app_runtime") or {}
+ if cfg.get("listen_ratio") is not None:
+ return clamp_listen_ratio(cfg.get("listen_ratio"))
+ return LISTEN_RATIO_DEFAULT
+
+
+def listen_window_ms(
+ duration_ms: Any,
+ ratio: Any = None,
+ min_ms: int = LISTEN_WINDOW_MIN_MS,
+ speed: Any = None,
+ wall_clock: bool = True,
+) -> int:
+ """Pause after a shadowing clip. Not a fixed 2.5s.
+
+ Firmware reports wall-clock play time, which already includes speed
+ (slower rate 鈫� longer duration_ms). Window = that time 脳 listen_ratio (1.2).
+ If duration_ms is the original file length, pass wall_clock=False and speed.
+ """
+ clamped = clamp_listen_ratio(ratio if ratio is not None else LISTEN_RATIO_DEFAULT)
+ duration = 0
+ try:
+ duration = int(duration_ms or 0)
+ except (TypeError, ValueError):
+ duration = 0
+ if duration <= 0:
+ return int(min_ms)
+ played = duration
+ if not wall_clock:
+ rate = clamp_play_speed(speed) if speed is not None else PLAY_SPEED_DEFAULT
+ if rate <= 0:
+ rate = PLAY_SPEED_DEFAULT
+ played = int(round(duration / rate))
+ return max(int(min_ms), int(played * clamped))
+
+
+def transcript_cache_dir(app_id: str = "", task_id: str = "") -> Optional[str]:
+ try:
+ from config.config_loader import get_project_dir
+
+ path = os.path.join(
+ get_project_dir(),
+ "data",
+ "lesson_cache",
+ app_id or "",
+ task_id or "",
+ )
+ if os.path.isdir(path):
+ return path
+ except Exception:
+ return None
+ return None
+
+
+def _usable_text(value: Any) -> Optional[str]:
+ text = str(value).strip() if value is not None else ""
+ if not text or looks_like_path(text):
+ return None
+ return text
+
+
+def _read_txt_file(path: str) -> Optional[str]:
+ try:
+ if not path or not os.path.isfile(path):
+ return None
+ with open(path, "r", encoding="utf-8") as handle:
+ return _usable_text(handle.read())
+ except Exception:
+ return None
+
+
+def _read_cached_transcript(clip: str, cache_dir: Optional[str]) -> Optional[str]:
+ if not cache_dir or not os.path.isdir(cache_dir) or not clip:
+ return None
+ stem = clip_stem(clip)
+ candidates = [
+ os.path.join(cache_dir, f"{stem}.txt"),
+ os.path.join(cache_dir, os.path.splitext(clip_basename(clip))[0] + ".txt"),
+ ]
+ name = (clip or "").replace("\\", "/")
+ marker = "/lessons/"
+ if marker in name:
+ rest = name.split(marker, 1)[1]
+ rest_txt = os.path.splitext(rest)[0] + ".txt"
+ candidates.append(os.path.join(cache_dir, rest_txt.replace("/", os.sep)))
+ for path in candidates:
+ text = _read_txt_file(path)
+ if text:
+ return text
+ return None
+
+
+def resolve_expected_text(
+ task: Optional[Dict[str, Any]],
+ clips: Optional[List[str]],
+ index: int,
+ cache_dir: Optional[str] = None,
+) -> Optional[str]:
+ """01.mp3 鈫� 01.txt contract. Never treat an SD path as expected_text."""
+ clips = list(clips or [])
+ task = task or {}
+ transcripts = task.get("transcripts") if isinstance(task.get("transcripts"), dict) else {}
+ sentences = task.get("sentences") or []
+
+ clip = ""
+ if 0 <= index < len(clips):
+ clip = str(clips[index] or "")
+ stem = clip_stem(clip)
+ basename = clip_basename(clip)
+ for key in (basename, stem, clip):
+ found = _usable_text(transcripts.get(key) if key else None)
+ if found:
+ return found
+ cached = _read_cached_transcript(clip, cache_dir)
+ if cached:
+ return cached
+
+ if 0 <= index < len(sentences):
+ found = _usable_text(sentences[index])
+ if found:
+ return found
+ return None
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/core/app_runtime/playback.py b/xiaozhi-esp32-server/main/xiaozhi-server/core/app_runtime/playback.py
new file mode 100644
index 0000000..e40dca9
--- /dev/null
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/core/app_runtime/playback.py
@@ -0,0 +1,312 @@
+"""SD MP3 playback speed: clamp [0.5, 2.0], default 1.0. Gradients live on Server."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import time
+from typing import Any, Dict, List, Optional
+
+from config.config_loader import get_project_dir
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+PLAY_SPEED_MIN = 0.5
+PLAY_SPEED_MAX = 2.0
+PLAY_SPEED_DEFAULT = 1.0
+
+# 灏忕▼搴�/瀹氭椂浠诲姟涓嬪彂娓呭崟涔嬪墠鐨勪复鏃惰鏂囥�傝矾寰勭浉瀵� SD 鍗℃牴锛�/english_learning/...锛夈��
+TEMP_REPEATER_PLAYLIST = (
+ {
+ "name": "/english_learning/shadowing/lessons/unit01/01.mp3",
+ "repeat": 10,
+ "speed": PLAY_SPEED_DEFAULT,
+ },
+ {
+ "name": "/english_learning/shadowing/lessons/unit01/02.mp3",
+ "repeat": 5,
+ "speed": PLAY_SPEED_DEFAULT,
+ },
+)
+
+TEMP_SHADOWING_PLAYLIST = (
+ {
+ "name": "/english_learning/shadowing/lessons/unit01/01.mp3",
+ "repeat": 5,
+ "speed": PLAY_SPEED_DEFAULT,
+ },
+ {
+ "name": "/english_learning/shadowing/lessons/unit01/02.mp3",
+ "repeat": 10,
+ "speed": PLAY_SPEED_DEFAULT,
+ },
+)
+
+
+def clamp_play_speed(raw: Any) -> float:
+ try:
+ value = float(raw)
+ except (TypeError, ValueError):
+ return PLAY_SPEED_DEFAULT
+ if value != value: # NaN
+ return PLAY_SPEED_DEFAULT
+ if value < PLAY_SPEED_MIN:
+ return PLAY_SPEED_MIN
+ if value > PLAY_SPEED_MAX:
+ return PLAY_SPEED_MAX
+ return value
+
+
+def playback_config(task: Optional[Dict[str, Any]]) -> Dict[str, Any]:
+ cfg = (task or {}).get("playback")
+ return cfg if isinstance(cfg, dict) else {}
+
+
+def _progress_path(device_id: str) -> str:
+ safe = re.sub(r"[^A-Za-z0-9_-]+", "_", device_id or "unknown") or "unknown"
+ return os.path.join(get_project_dir(), "data", "playback_progress", f"{safe}.json")
+
+
+def _load_progress(device_id: str) -> Dict[str, Any]:
+ path = _progress_path(device_id)
+ if not os.path.isfile(path):
+ return {}
+ try:
+ with open(path, "r", encoding="utf-8") as handle:
+ data = json.load(handle)
+ return data if isinstance(data, dict) else {}
+ except Exception as exc:
+ logger.bind(tag=TAG).warning(f"璇诲彇鎾斁杩涘害澶辫触: {exc}")
+ return {}
+
+
+def _save_progress(device_id: str, data: Dict[str, Any]) -> None:
+ path = _progress_path(device_id)
+ try:
+ os.makedirs(os.path.dirname(path), exist_ok=True)
+ with open(path, "w", encoding="utf-8") as handle:
+ json.dump(data, handle, ensure_ascii=False, indent=2)
+ except Exception as exc:
+ logger.bind(tag=TAG).warning(f"淇濆瓨鎾斁杩涘害澶辫触: {exc}")
+
+
+def _lesson_key(app_id: str, task_id: str) -> str:
+ return f"{app_id or ''}/{task_id or ''}"
+
+
+def lesson_day_index(
+ device_id: str,
+ app_id: str,
+ task_id: str,
+ lesson: str = "",
+) -> int:
+ """0-based count of distinct calendar days this lesson has already been played."""
+ data = _load_progress(device_id)
+ bucket = data.get(_lesson_key(app_id, task_id)) or {}
+ lessons = bucket.get("lessons") if isinstance(bucket, dict) else {}
+ key = lesson or "_default"
+ days = lessons.get(key) if isinstance(lessons, dict) else None
+ if not isinstance(days, list):
+ return 0
+ return len([d for d in days if d])
+
+
+def mark_lesson_played_today(
+ device_id: str,
+ app_id: str,
+ task_id: str,
+ lesson: str = "",
+) -> None:
+ today = time.strftime("%Y-%m-%d")
+ data = _load_progress(device_id)
+ key = _lesson_key(app_id, task_id)
+ bucket = data.get(key)
+ if not isinstance(bucket, dict):
+ bucket = {}
+ lessons = bucket.get("lessons")
+ if not isinstance(lessons, dict):
+ lessons = {}
+ lesson_key = lesson or "_default"
+ days = lessons.get(lesson_key)
+ if not isinstance(days, list):
+ days = []
+ if today not in days:
+ days.append(today)
+ lessons[lesson_key] = days
+ bucket["lessons"] = lessons
+ data[key] = bucket
+ _save_progress(device_id, data)
+
+
+def daily_base_speed(task: Optional[Dict[str, Any]], day_index: int = 0) -> float:
+ cfg = playback_config(task)
+ daily = cfg.get("daily_speeds")
+ if isinstance(daily, list) and daily:
+ idx = max(0, int(day_index or 0))
+ if idx >= len(daily):
+ idx = len(daily) - 1
+ return clamp_play_speed(daily[idx])
+ if cfg.get("speed") is not None:
+ return clamp_play_speed(cfg.get("speed"))
+ return PLAY_SPEED_DEFAULT
+
+
+def speeds_for_repeats(
+ task: Optional[Dict[str, Any]],
+ repeat_n: int = 1,
+ day_index: int = 0,
+) -> List[float]:
+ """One speed per pass. repeat=1 uses daily curve; repeat>1 samples repeat_speeds if set."""
+ n = 1
+ try:
+ n = int(repeat_n or 1)
+ except (TypeError, ValueError):
+ n = 1
+ if n < 1:
+ n = 1
+ base = daily_base_speed(task, day_index)
+ curve = playback_config(task).get("repeat_speeds")
+ if n == 1 or not isinstance(curve, list) or not curve:
+ return [base] * n
+ values = [clamp_play_speed(item) for item in curve]
+ if n == len(values):
+ return values
+ if n == 1:
+ return [values[0]]
+ out: List[float] = []
+ last = len(values) - 1
+ for i in range(n):
+ pos = (i / (n - 1)) * last
+ lo = int(pos)
+ hi = min(lo + 1, last)
+ frac = pos - lo
+ out.append(clamp_play_speed(values[lo] * (1.0 - frac) + values[hi] * frac))
+ return out
+
+
+def resolve_play_speed(
+ task: Optional[Dict[str, Any]],
+ day_index: int = 0,
+ explicit: Any = None,
+) -> float:
+ if explicit is not None:
+ return clamp_play_speed(explicit)
+ return daily_base_speed(task, day_index)
+
+
+def normalize_media_name(raw: Any) -> str:
+ """SD files[].name, MQTT path, or HTTP url 鈫� /app/task/lessons/... form."""
+ name = str(raw or "").strip().replace("\\", "/")
+ if not name:
+ return ""
+ for marker in ("/english_learning/", "english_learning/"):
+ idx = name.find(marker)
+ if idx >= 0:
+ rest = name[idx:]
+ return rest if rest.startswith("/") else "/" + rest
+ if "://" in name:
+ path = name.split("://", 1)[1]
+ _, _, rest = path.partition("/")
+ if rest:
+ return "/" + rest.lstrip("/")
+ return ""
+ if not name.startswith("/"):
+ name = "/" + name.lstrip("/")
+ return name
+
+
+def _item_repeat(raw: Dict[str, Any]) -> int:
+ for key in ("repeat", "count", "play_count"):
+ if raw.get(key) is None:
+ continue
+ try:
+ value = int(raw.get(key))
+ except (TypeError, ValueError):
+ continue
+ return max(1, value)
+ return 1
+
+
+def playlist_item_from_value(raw: Any) -> Optional[Dict[str, Any]]:
+ if isinstance(raw, str):
+ name = normalize_media_name(raw)
+ if not name:
+ return None
+ return {"name": name, "repeat": 1}
+ if not isinstance(raw, dict):
+ return None
+ name = normalize_media_name(
+ raw.get("name")
+ or raw.get("path")
+ or raw.get("target_path")
+ or raw.get("url")
+ or ""
+ )
+ if not name:
+ return None
+ item: Dict[str, Any] = {"name": name, "repeat": _item_repeat(raw)}
+ if raw.get("speed") is not None:
+ item["speed"] = clamp_play_speed(raw.get("speed"))
+ if raw.get("power_off_after") is not None:
+ item["power_off_after"] = bool(raw.get("power_off_after"))
+ return item
+
+
+def parse_playlist(params: Optional[Dict[str, Any]]) -> List[Dict[str, Any]]:
+ """Repeater/shadowing playlist. Each item is {name, repeat, optional speed}."""
+ params = params or {}
+ raw_items = params.get("items") or params.get("playlist") or params.get("clips")
+ if isinstance(raw_items, list) and raw_items:
+ items = []
+ for raw in raw_items:
+ item = playlist_item_from_value(raw)
+ if item:
+ items.append(item)
+ return items
+ item = playlist_item_from_value(
+ {
+ "name": params.get("name") or params.get("path") or params.get("url") or "",
+ "repeat": params.get("repeat") or params.get("count") or 1,
+ "speed": params.get("speed"),
+ "power_off_after": params.get("power_off_after"),
+ }
+ )
+ if item and item["name"]:
+ return [item]
+ return []
+
+
+def expand_playlist_plays(items: Optional[List[Dict[str, Any]]]) -> List[str]:
+ """Flatten {name, repeat} into one path per play. Shadowing uses this so each pass has its own listen window."""
+ clips: List[str] = []
+ for item in items or []:
+ if not isinstance(item, dict):
+ continue
+ name = str(item.get("name") or "")
+ if not name:
+ continue
+ clips.extend([name] * _item_repeat(item))
+ return clips
+
+
+def default_repeater_playlist(task: Optional[Dict[str, Any]] = None) -> List[Dict[str, Any]]:
+ """No downlink yet: task.default_playlist, else the unit01 01脳10 / 02脳5 stub."""
+ raw = (task or {}).get("default_playlist")
+ if isinstance(raw, list) and raw:
+ items = parse_playlist({"items": raw})
+ if items:
+ return items
+ return [dict(item) for item in TEMP_REPEATER_PLAYLIST]
+
+
+def default_shadowing_playlist(task: Optional[Dict[str, Any]] = None) -> List[Dict[str, Any]]:
+ """No downlink yet: task.default_playlist, else the unit01 01脳5 / 02脳10 stub."""
+ raw = (task or {}).get("default_playlist")
+ if isinstance(raw, list) and raw:
+ items = parse_playlist({"items": raw})
+ if items:
+ return items
+ return [dict(item) for item in TEMP_SHADOWING_PLAYLIST]
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/core/app_runtime/registry.py b/xiaozhi-esp32-server/main/xiaozhi-server/core/app_runtime/registry.py
index 42c4436..22dbcb3 100644
--- a/xiaozhi-esp32-server/main/xiaozhi-server/core/app_runtime/registry.py
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/core/app_runtime/registry.py
@@ -1,4 +1,10 @@
-"""Load the Phase 1 App / Task registry from JSON."""
+"""Load the Phase 1 App / Task registry from JSON.
+
+Registry lives on xiaozhi-server only (bundled apps.json, optional data/apps.json).
+Task voice_enter / voice_exit / enter_prompt live here so new Apps do not need a
+firmware flash. The device only keeps a small offline MultiNet set (mode switch,
+repeater, exit repeater, return to chat) plus NVS pending / last-repeater state.
+"""
import json
import os
@@ -11,8 +17,14 @@
logger = setup_logging()
_REGISTRY: Optional[Dict[str, Any]] = None
+_VOICE_INDEX: Optional[list] = None
_BUNDLED_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "apps.json")
+_MENU_PRIO = 0
+_EXIT_PRIO = 1
+_CHAT_ENTER_PRIO = 2
+_ENTER_PRIO = 3
+_RESERVED_PRIO = 4
def _load_registry() -> Dict[str, Any]:
@@ -54,6 +66,93 @@
return _REGISTRY
+def reset_registry_cache() -> None:
+ """Tests may swap the registry; drop derived indexes."""
+ global _VOICE_INDEX
+ _VOICE_INDEX = None
+
+
+def iter_tasks():
+ apps = get_registry().get("apps") or {}
+ for app_id, app in apps.items():
+ if not isinstance(app, dict):
+ continue
+ for task_id, task in (app.get("tasks") or {}).items():
+ if isinstance(task, dict):
+ yield app_id, app, task_id, task
+
+
+def _phrase_list(raw) -> list:
+ if raw is None:
+ return []
+ if isinstance(raw, str):
+ text = raw.strip()
+ return [text] if text else []
+ phrases = []
+ for item in raw:
+ text = str(item or "").strip()
+ if text:
+ phrases.append(text)
+ return phrases
+
+
+def _build_voice_index() -> list:
+ """Longer phrase first, then menu/exit/chat-enter/enter/reserved."""
+ entries = []
+ registry = get_registry()
+ for phrase in _phrase_list(registry.get("voice_menu_phrases")):
+ entries.append((phrase, ("__menu__", ""), _MENU_PRIO))
+ for app_id, app, task_id, task in iter_tasks():
+ default_task = app.get("default_task") or task_id
+ for phrase in _phrase_list(task.get("voice_exit")):
+ entries.append((phrase, ("chat", "conversation"), _EXIT_PRIO))
+ for phrase in _phrase_list(task.get("voice_enter")):
+ if app_id == "chat":
+ entries.append((phrase, ("chat", default_task or task_id), _CHAT_ENTER_PRIO))
+ else:
+ entries.append((phrase, (app_id, task_id), _ENTER_PRIO))
+ for phrase in _phrase_list(registry.get("reserved_voice_enter")):
+ entries.append((phrase, ("__reserved__", ""), _RESERVED_PRIO))
+ entries.sort(key=lambda item: (-len(item[0]), item[2]))
+ return entries
+
+
+def _voice_index() -> list:
+ global _VOICE_INDEX
+ if _VOICE_INDEX is None:
+ _VOICE_INDEX = _build_voice_index()
+ return _VOICE_INDEX
+
+
+def match_registry_voice(filtered: str) -> Optional[Tuple[str, str]]:
+ """Map stripped ASR text to (app_id, task_id) or ('__menu__'|'__reserved__', '')."""
+ if not filtered:
+ return None
+ for phrase, target, _prio in _voice_index():
+ if phrase and phrase in filtered:
+ return target
+ return None
+
+
+def voice_menu_text() -> str:
+ labels = []
+ chat_hint = "鎴戜滑鏉ヨ亰澶╁惂"
+ for app_id, _app, task_id, task in iter_tasks():
+ enters = _phrase_list(task.get("voice_enter"))
+ if app_id == "chat" and task_id == "conversation" and enters:
+ chat_hint = enters[0]
+ name = str(task.get("menu_name") or "").strip()
+ if name:
+ labels.append(name)
+ if not labels:
+ return f"鎯宠亰澶╋紵涔熷彲浠ヨ{chat_hint}銆�"
+ if len(labels) == 1:
+ body = labels[0]
+ else:
+ body = "銆�".join(labels[:-1]) + f"锛岃繕鏄瘂labels[-1]}"
+ return f"鎯硔body}锛熶篃鍙互璇磠chat_hint}銆�"
+
+
def default_app_id() -> str:
return get_registry().get("default_app_id") or "chat"
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/core/app_runtime/runtime.py b/xiaozhi-esp32-server/main/xiaozhi-server/core/app_runtime/runtime.py
index e62fdcf..71dc2af 100644
--- a/xiaozhi-esp32-server/main/xiaozhi-server/core/app_runtime/runtime.py
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/core/app_runtime/runtime.py
@@ -2,13 +2,48 @@
from __future__ import annotations
+import asyncio
+import json
+import difflib
+import queue
+import random
+import re
import uuid
-from typing import Any, Dict, Optional, TYPE_CHECKING
+from typing import Any, Dict, List, Optional, TYPE_CHECKING
from config.logger import setup_logging
from plugins_func.register import Action, ActionResponse
-from .registry import default_app_id, get_app, get_task
+from .assessment import (
+ evaluate_turn,
+ listen_window_ms,
+ pcm_snapshot_to_wav,
+ resolve_expected_text,
+ resolve_listen_ratio,
+ save_turn_wav,
+ silent_pcm_snapshot,
+ transcript_cache_dir,
+)
+from .playback import (
+ clamp_play_speed,
+ default_repeater_playlist,
+ default_shadowing_playlist,
+ expand_playlist_plays,
+ lesson_day_index,
+ mark_lesson_played_today,
+ parse_playlist,
+ resolve_play_speed,
+ speeds_for_repeats,
+)
+from .registry import (
+ default_app_id,
+ get_app,
+ get_registry,
+ get_task,
+ match_registry_voice,
+ voice_menu_text,
+)
+from .session import begin_session, current_session, end_session, parse_skill_event
if TYPE_CHECKING:
from core.connection import ConnectionHandler
@@ -28,19 +63,37 @@
"鏈寚瀹氳瑷�鏃讹細涓枃璇戞垚鑻辨枃锛岃嫳鏂囪瘧鎴愪腑鏂囷紱鍏跺畠璇█榛樿璇戞垚涓枃銆�"
)
+BILINGUAL_PROMPT = (
+ "浣犳鍦ㄨ繘琛屼腑鑻卞鐓у璇濈粌涔狅紝涓嶆槸缈昏瘧鏈猴紝涓嶈鎶婄敤鎴风殑璇濆師鏍疯瘧鍥炲幓浜ゅ樊銆�"
+ "涓ユ牸鎵ц锛氱敤鎴疯涓枃鏃朵綘鍙敤鑻辨枃瀹屾暣鍥炵瓟锛涚敤鎴疯鑻辨枃鏃朵綘鍙敤涓枃瀹屾暣鍥炵瓟銆�"
+ "涓嶈涓嫳澶规潅锛屼笉瑕佸悓鏃剁粰鍑轰袱绉嶈瑷�锛屼笉瑕侀棶瑕佷笉瑕佺炕璇戙��"
+ "鐢ㄦ埛璇撮��鍑轰腑鑻卞璇濄�佸洖鍒拌亰澶╂椂璋冪敤 switch_app锛宎pp_id=chat銆�"
+)
+
CHAT_PROACTIVE_HINT = (
"浣犳槸鐢ㄦ埛鐨勮闊充紮浼淬�傝繘鍏ュ璇濆悗鍙互鍏堜富鍔ㄧ煭鍙ラ個璇疯亰澶╋紝渚嬪銆屾垜浠潵鑱婂ぉ鍚с�嶃��"
"鐢ㄦ埛寮�鍙ｅ悗鎸夋甯稿璇濆洖绛旓紝涓嶈鍙嶅鍌績銆�"
)
-VOICE_MENU = "鎯宠亰澶┿�佸惉澶嶈锛岃繕鏄墦寮�缈昏瘧锛熶篃鍙互璇存垜浠潵鑱婂ぉ鍚с��"
+CHAT_RETURN_TOPICS = ("浠婂ぉ杩囧緱鎬庝箞鏍�", "浣犳渶杩戝湪蹇欎粈涔�", "鏈変粈涔堟柊椴滀簨")
-CHAT_PHRASES = ("鎴戜滑鏉ヨ亰澶╁惂", "鑱婁細澶╁惂", "鍥炲埌鑱婂ぉ", "閫�鍑虹炕璇�", "缁撴潫缈昏瘧", "閫�鍑哄璇�")
-TRANSLATE_PHRASES = ("鎵撳紑缈昏瘧", "寮�濮嬬炕璇�", "鑻辫缈昏瘧", "缈昏瘧妯″紡")
-REPEATER_PHRASES = ("寮�濮嬪璇�", "鎵撳紑澶嶈", "鑻辫瀛︿範", "鍚璇�")
+DEFAULT_SHADOWING_SENTENCES = (
+ "Hello, how are you today?",
+ "I like to read English books.",
+ "The weather is nice this morning.",
+ "Can you say that again, please?",
+ "Practice a little every day.",
+)
DEVICE_PLAY_TOOL = "self_mp3_play"
DEVICE_STOP_TOOL = "self_mp3_stop"
+DEVICE_LIST_TOOL = "self_mp3_get_list"
+DEVICE_ENSURE_APP_TOOL = "self_sd_ensure_app"
+SD_LAYOUT_MODES = ("PLAY_AUDIO", "SHADOWING")
+SHADOWING_LISTEN_RATIO = 1.2
+SHADOWING_LISTEN_MIN_MS = 800
+REPEATER_END_PHRASE = "澶嶈缁撴潫"
+SHADOWING_END_PHRASE = "璺熻缁撴潫"
def source_priority(source: str) -> int:
@@ -62,6 +115,150 @@
conn.tool_allowlist = []
if not hasattr(conn, "original_system_prompt"):
conn.original_system_prompt = conn.prompt
+ if not hasattr(conn, "shadowing_index"):
+ conn.shadowing_index = 0
+ if not hasattr(conn, "shadowing_prompt"):
+ conn.shadowing_prompt = ""
+ if not hasattr(conn, "shadowing_clips"):
+ conn.shadowing_clips = []
+ if not hasattr(conn, "shadowing_awaiting_user"):
+ conn.shadowing_awaiting_user = False
+ if not hasattr(conn, "shadowing_clip_gen"):
+ conn.shadowing_clip_gen = 0
+ if not hasattr(conn, "shadowing_timeout_task"):
+ conn.shadowing_timeout_task = None
+ if not hasattr(conn, "last_asr_language"):
+ conn.last_asr_language = None
+ if not hasattr(conn, "task_session"):
+ conn.task_session = None
+ if not hasattr(conn, "last_asr_audio_snapshot"):
+ conn.last_asr_audio_snapshot = None
+ if not hasattr(conn, "shadowing_listen_pcm"):
+ conn.shadowing_listen_pcm = []
+ if not hasattr(conn, "shadowing_audio_saved_index"):
+ conn.shadowing_audio_saved_index = -1
+ if not hasattr(conn, "ensured_app_dirs"):
+ conn.ensured_app_dirs = set()
+ if not hasattr(conn, "repeater_speed_queue"):
+ conn.repeater_speed_queue = []
+ if not hasattr(conn, "repeater_play_name"):
+ conn.repeater_play_name = ""
+ if not hasattr(conn, "repeater_power_off_after"):
+ conn.repeater_power_off_after = False
+ if not hasattr(conn, "repeater_playlist"):
+ conn.repeater_playlist = []
+ if not hasattr(conn, "shadowing_last_speed"):
+ conn.shadowing_last_speed = 1.0
+ if not hasattr(conn, "shadowing_assess_index"):
+ conn.shadowing_assess_index = 0
+ if not hasattr(conn, "chat_greeted"):
+ conn.chat_greeted = False
+ if not hasattr(conn, "pending_play_after_tts"):
+ conn.pending_play_after_tts = None
+
+
+def _task_needs_sd_layout(task: Optional[Dict[str, Any]]) -> bool:
+ if not task:
+ return False
+ mode = task.get("interaction_mode") or ""
+ if mode in SD_LAYOUT_MODES:
+ return True
+ for skill in task.get("skills") or []:
+ name = (skill or {}).get("name") or ""
+ if name in ("self.mp3.play", DEVICE_PLAY_TOOL):
+ return True
+ return False
+
+
+def app_sd_layout_paths(app_id: str, app: Optional[Dict[str, Any]] = None) -> list:
+ """Return SD lesson dirs for an App: /{app_id}/{task_id}/lessons."""
+ if not app_id:
+ return []
+ if app is None:
+ app = get_app(app_id)
+ if not app:
+ return []
+ paths = []
+ for task_id, task in (app.get("tasks") or {}).items():
+ if _task_needs_sd_layout(task):
+ paths.append(f"/{app_id}/{task_id}/lessons")
+ return paths
+
+
+def _mcp_layout_ok(result) -> bool:
+ if result is None:
+ return False
+ if result.action in (Action.ERROR, Action.NOTFOUND):
+ return False
+ blob = str(getattr(result, "response", None) or getattr(result, "result", None) or "")
+ compact = blob.replace(" ", "").lower()
+ if '"ok":false' in compact:
+ return False
+ return True
+
+
+async def ensure_app_sd_dirs(conn: "ConnectionHandler", app_id: str) -> bool:
+ """Create /{app} and /{app}/{task}/lessons on device SD. Never raises; missing SD is ignored."""
+ _ensure_runtime_fields(conn)
+ app = get_app(app_id)
+ paths = app_sd_layout_paths(app_id, app)
+ if not paths:
+ return True
+ already = getattr(conn, "ensured_app_dirs", None)
+ if already is None:
+ conn.ensured_app_dirs = set()
+ already = conn.ensured_app_dirs
+ if app_id in already:
+ return True
+ try:
+ root = await _execute_named_tool(conn, DEVICE_ENSURE_APP_TOOL, {"app_id": app_id, "task_id": ""})
+ if not _mcp_layout_ok(root):
+ logger.bind(tag=TAG).warning(f"鍒涘缓璁惧 App 鐩綍澶辫触 {app_id}: {getattr(root, 'response', root)}")
+ return False
+ for task_id, task in (app.get("tasks") or {}).items():
+ if not _task_needs_sd_layout(task):
+ continue
+ result = await _execute_named_tool(
+ conn, DEVICE_ENSURE_APP_TOOL, {"app_id": app_id, "task_id": task_id}
+ )
+ if not _mcp_layout_ok(result):
+ logger.bind(tag=TAG).warning(
+ f"鍒涘缓璁惧 Task 鐩綍澶辫触 {app_id}/{task_id}: {getattr(result, 'response', result)}"
+ )
+ return False
+ already.add(app_id)
+ logger.bind(tag=TAG).info(f"宸茬‘淇� SD 鐩綍 {paths}")
+ return True
+ except Exception as e:
+ logger.bind(tag=TAG).warning(f"鍒涘缓璁惧 App 鐩綍寮傚父 {app_id}: {e}")
+ return False
+
+
+async def ensure_all_subscribed_app_sd_dirs(conn: "ConnectionHandler") -> None:
+ """After MCP tools/list, mkdir for every registry App that needs SD. Failures do not drop WS."""
+ try:
+ apps = (get_registry().get("apps") or {})
+ for app_id, app in apps.items():
+ if app_sd_layout_paths(app_id, app):
+ await ensure_app_sd_dirs(conn, app_id)
+ except Exception as e:
+ logger.bind(tag=TAG).warning(f"璁㈤槄 App 寤虹洰褰曞け璐ワ紙蹇界暐锛�: {e}")
+
+
+def schedule_ensure_subscribed_app_sd_dirs(conn: "ConnectionHandler") -> None:
+ """Fire-and-forget so tools/list handling is not blocked waiting for MCP mkdir."""
+
+ async def _run():
+ await ensure_all_subscribed_app_sd_dirs(conn)
+
+ try:
+ loop = getattr(conn, "loop", None)
+ if loop:
+ loop.create_task(_run())
+ else:
+ asyncio.create_task(_run())
+ except Exception as e:
+ logger.bind(tag=TAG).warning(f"鏃犳硶璋冨害 App 寤虹洰褰�: {e}")
def filter_tools(conn: "ConnectionHandler", tools: Dict[str, Any]) -> Dict[str, Any]:
@@ -94,6 +291,8 @@
"浣犵幇鍦ㄦ槸鑻辫缈昏瘧鏈恒�傜敤鎴疯璇濆悗搴旇皟鐢� translate锛涚敤鎴疯閫�鍑虹炕璇戙�佸洖鍒拌亰澶╂椂璋冪敤 switch_app锛宎pp_id=chat銆�"
"涓嶈璁茬瑧璇濄�佷笉瑕佹煡澶╂皵銆佷笉瑕侀棽鑱娿��"
)
+ elif mode == "bilingual":
+ conn.change_system_prompt(BILINGUAL_PROMPT)
elif mode == "agent_proactive":
base = conn.original_system_prompt or conn.prompt or ""
if CHAT_PROACTIVE_HINT not in (base or ""):
@@ -118,13 +317,58 @@
async def _stop_current_skills(conn: "ConnectionHandler") -> None:
mode = getattr(conn, "interaction_mode", "CHAT")
- if mode == "PLAY_AUDIO":
+ _cancel_shadowing_timeout(conn)
+ if mode in ("PLAY_AUDIO", "SHADOWING"):
await _execute_named_tool(conn, DEVICE_STOP_TOOL, {})
conn.awaiting_voice_app_choice = False
+ conn.pending_play_after_tts = None
+ conn.shadowing_index = 0
+ conn.shadowing_prompt = ""
+ conn.shadowing_clips = []
+ conn.shadowing_awaiting_user = False
+ conn.shadowing_clip_gen = int(getattr(conn, "shadowing_clip_gen", 0) or 0) + 1
+ conn.shadowing_assess_index = 0
+ conn.shadowing_listen_pcm = []
+ conn.shadowing_audio_saved_index = -1
+ conn.repeater_speed_queue = []
+ conn.repeater_play_name = ""
+ conn.repeater_power_off_after = False
+ conn.repeater_playlist = []
+
+
+def tts_channels_ready(conn: "ConnectionHandler") -> bool:
+ tts = getattr(conn, "tts", None)
+ if tts is None:
+ return False
+ play = getattr(tts, "audio_play_priority_thread", None)
+ text_th = getattr(tts, "tts_priority_thread", None)
+ return bool(
+ play is not None
+ and play.is_alive()
+ and text_th is not None
+ and text_th.is_alive()
+ )
+
+
+async def wait_for_tts_channels(conn: "ConnectionHandler", timeout: float = 8.0) -> bool:
+ loop = asyncio.get_running_loop()
+ start = loop.time()
+ while (loop.time() - start) < timeout:
+ if tts_channels_ready(conn):
+ return True
+ await asyncio.sleep(0.1)
+ return tts_channels_ready(conn)
def _speak(conn: "ConnectionHandler", text: str) -> None:
if not text:
+ return
+ tts = getattr(conn, "tts", None)
+ if tts is None:
+ logger.bind(tag=TAG).warning("TTS鏈垵濮嬪寲锛岃烦杩囨挱鎶�")
+ return
+ if not tts_channels_ready(conn):
+ logger.bind(tag=TAG).warning("TTS閫氶亾鏈氨缁紝璺宠繃鎾姤")
return
from core.handle.intentHandler import speak_txt
@@ -140,6 +384,877 @@
return (conn.config or {}).get("proactive_chat_greeting") or "鎴戜滑鏉ヨ亰澶╁惂銆�"
+def _display_name(app: Optional[Dict[str, Any]], task: Optional[Dict[str, Any]] = None) -> str:
+ if task and task.get("display_name"):
+ return str(task["display_name"])
+ if app and app.get("display_name"):
+ return str(app["display_name"])
+ if app and app.get("name"):
+ return str(app["name"])
+ return "褰撳墠妯″紡"
+
+
+def _enter_prompt(app: Dict[str, Any], task: Dict[str, Any]) -> str:
+ return (task.get("enter_prompt") or app.get("enter_prompt") or "").strip()
+
+
+def _switch_announcement(
+ prev_app: Optional[Dict[str, Any]],
+ prev_task: Optional[Dict[str, Any]],
+ app: Dict[str, Any],
+ task: Dict[str, Any],
+ app_id: str,
+) -> str:
+ if app_id == "chat":
+ prev_name = _display_name(prev_app, prev_task)
+ topic = random.choice(CHAT_RETURN_TOPICS)
+ return f"宸茬粡閫�鍑簕prev_name}锛岀幇鍦ㄦ槸瀵硅瘽妯″紡锛屾垜浠潵鑱婄偣{topic}鍚с��"
+ return _enter_prompt(app, task) or f"鐜板湪宸茬粡鍒囨崲鑷硔_display_name(app, task)}銆�"
+
+
+def _shadowing_sentences(conn: "ConnectionHandler") -> list:
+ cfg = (conn.config or {}).get("app_runtime") or {}
+ sentences = cfg.get("shadowing_sentences")
+ if not sentences:
+ _, task = get_task("english_learning", "shadowing")
+ sentences = (task or {}).get("sentences")
+ if not sentences:
+ sentences = DEFAULT_SHADOWING_SENTENCES
+ return [str(s).strip() for s in sentences if str(s).strip()]
+
+
+def lesson_mp3_clips(names, app_id: str, task_id: str, lesson_id: str = "") -> list:
+ """Pick sentence MP3s under /{app}/{task}/lessons/{lesson}/..."""
+ prefix = f"/{app_id}/{task_id}/lessons/"
+ clips = []
+ for raw in names or []:
+ name = str(raw or "").replace("\\", "/")
+ if not name.lower().endswith(".mp3"):
+ continue
+ if not name.startswith("/"):
+ name = "/" + name.lstrip("/")
+ if not name.startswith(prefix):
+ continue
+ rest = name[len(prefix) :]
+ if not rest:
+ continue
+ base = rest.rsplit("/", 1)[-1].lower()
+ if base == "full.mp3" or base.endswith("_full.mp3"):
+ continue
+ if lesson_id:
+ if rest != lesson_id and not rest.startswith(str(lesson_id) + "/"):
+ continue
+ clips.append(name)
+ clips.sort()
+ if lesson_id or not clips:
+ return clips
+ first_lesson = clips[0][len(prefix) :].split("/", 1)[0]
+ return [
+ name
+ for name in clips
+ if name[len(prefix) :] == first_lesson
+ or name[len(prefix) :].startswith(first_lesson + "/")
+ ]
+
+
+def repeater_lesson_mp3(names, app_id: str, task_id: str) -> str:
+ """Pick the full-lesson MP3 under /{app}/{task}/lessons/, else the first file there."""
+ prefix = f"/{app_id}/{task_id}/lessons/"
+ fulls = []
+ others = []
+ for raw in names or []:
+ name = str(raw or "").replace("\\", "/")
+ if not name.lower().endswith(".mp3"):
+ continue
+ if not name.startswith("/"):
+ name = "/" + name.lstrip("/")
+ if not name.startswith(prefix):
+ continue
+ base = name.rsplit("/", 1)[-1].lower()
+ if base == "full.mp3" or base.endswith("_full.mp3"):
+ fulls.append(name)
+ else:
+ others.append(name)
+ fulls.sort()
+ others.sort()
+ if fulls:
+ return fulls[0]
+ return others[0] if others else ""
+
+
+def _is_clip_path(text: str) -> bool:
+ t = (text or "").replace("\\", "/").lower()
+ return t.endswith(".mp3") or t.startswith("/")
+
+
+def _shadowing_items(conn: "ConnectionHandler") -> list:
+ clips = getattr(conn, "shadowing_clips", None) or []
+ if clips:
+ return list(clips)
+ return _shadowing_sentences(conn)
+
+
+def _begin_shadowing(conn: "ConnectionHandler", enter_text: str) -> str:
+ sentences = _shadowing_sentences(conn)
+ conn.shadowing_index = 0
+ if not sentences:
+ conn.shadowing_prompt = ""
+ return enter_text or "鐜板湪宸茬粡鍒囨崲鑷宠窡璇绘ā寮忥紝浣嗚繕娌℃湁璺熻鍙ュ瓙銆�"
+ conn.shadowing_prompt = sentences[0]
+ prefix = enter_text or "鐜板湪宸茬粡鍒囨崲鑷宠窡璇绘ā寮忋��"
+ return f"{prefix}璇疯窡鎴戣锛歿sentences[0]}"
+
+
+def _cancel_shadowing_timeout(conn: "ConnectionHandler") -> None:
+ task = getattr(conn, "shadowing_timeout_task", None)
+ if task is not None:
+ try:
+ if not task.done():
+ task.cancel()
+ except Exception:
+ pass
+ conn.shadowing_timeout_task = None
+
+
+def _task_for(conn: "ConnectionHandler") -> Optional[Dict[str, Any]]:
+ session = current_session(conn) or getattr(conn, "task_session", None)
+ app_id = getattr(conn, "active_app_id", None) or getattr(session, "app_id", None) or "english_learning"
+ task_id = getattr(conn, "current_task_id", None) or getattr(session, "task_id", None) or "shadowing"
+ _, task = get_task(app_id, task_id)
+ return task
+
+
+def _expected_for_index(conn: "ConnectionHandler", index: int) -> Optional[str]:
+ session = current_session(conn) or getattr(conn, "task_session", None)
+ app_id = getattr(conn, "active_app_id", None) or getattr(session, "app_id", "") or ""
+ task_id = getattr(conn, "current_task_id", None) or getattr(session, "task_id", "") or ""
+ clips = list(getattr(conn, "shadowing_clips", None) or [])
+ return resolve_expected_text(
+ _task_for(conn),
+ clips,
+ index,
+ transcript_cache_dir(app_id, task_id),
+ )
+
+
+def apply_deferred_assessment(
+ conn: "ConnectionHandler",
+ step_index: int,
+ session: Any = None,
+ clips: Optional[list] = None,
+ app_id: str = "",
+ task_id: str = "",
+ audio_snapshot: Any = None,
+ persist_audio: bool = True,
+) -> None:
+ """Write expected_text + metrics onto the captured session. Never raises into play."""
+ import time as _time
+
+ session = session or getattr(conn, "task_session", None)
+ if not session:
+ return
+ app_id = app_id or getattr(conn, "active_app_id", None) or getattr(session, "app_id", "") or ""
+ task_id = task_id or getattr(conn, "current_task_id", None) or getattr(session, "task_id", "") or ""
+ _, task = get_task(app_id or "english_learning", task_id or "shadowing")
+ cfg = (task or {}).get("assessment") or {}
+ clip_list = list(clips if clips is not None else (getattr(conn, "shadowing_clips", None) or []))
+ turn = session.find_turn(step_index)
+ asr = _plain_user_text((turn or {}).get("asr_text") or "")
+ expected = (turn or {}).get("expected_text") or resolve_expected_text(
+ task, clip_list, step_index, transcript_cache_dir(app_id, task_id)
+ )
+ snapshot = audio_snapshot
+ if snapshot is None:
+ snapshot = getattr(conn, "last_asr_audio_snapshot", None)
+ wav_bytes = pcm_snapshot_to_wav(snapshot)
+ existing_audio = (turn or {}).get("audio_id")
+ audio_id = existing_audio
+ if persist_audio and not existing_audio:
+ audio_id = save_turn_wav(
+ getattr(session, "task_session_id", "") or "",
+ step_index,
+ wav_bytes,
+ conn=conn,
+ )
+ try:
+ from core.handle.reportHandle import enqueue_asr_report
+
+ enqueue_asr_report(
+ conn,
+ asr or "",
+ snapshot,
+ getattr(session, "task_session_id", None),
+ int(step_index),
+ )
+ except Exception:
+ pass
+ clip = ""
+ if 0 <= int(step_index) < len(clip_list):
+ clip = str(clip_list[step_index] or "")
+ metrics = []
+ if not (cfg and cfg.get("enabled") is False):
+ metrics = evaluate_turn(
+ asr,
+ expected or "",
+ wav_bytes,
+ cfg,
+ extra={"clip": clip, "step_index": step_index},
+ )
+ session.upsert_turn(
+ index=step_index,
+ prompt_text=(turn or {}).get("prompt_text") or getattr(conn, "shadowing_prompt", "") or "",
+ expected_text=expected,
+ metrics=metrics or None,
+ assessed_at=int(_time.time() * 1000) if metrics else None,
+ audio_id=audio_id,
+ )
+ session.save()
+
+
+def _schedule_deferred_assessment(conn: "ConnectionHandler", step_index: int) -> None:
+ session = getattr(conn, "task_session", None)
+ clips = list(getattr(conn, "shadowing_clips", None) or [])
+ app_id = getattr(conn, "active_app_id", None) or getattr(session, "app_id", "") or ""
+ task_id = getattr(conn, "current_task_id", None) or getattr(session, "task_id", "") or ""
+ audio_snapshot = getattr(conn, "last_asr_audio_snapshot", None)
+
+ async def _run():
+ try:
+ apply_deferred_assessment(
+ conn,
+ step_index,
+ session=session,
+ clips=clips,
+ app_id=app_id,
+ task_id=task_id,
+ audio_snapshot=audio_snapshot,
+ persist_audio=False,
+ )
+ except Exception as exc:
+ logger.bind(tag=TAG).warning(f"璺熻鍚庤瘎浼板け璐�: {exc}")
+
+ loop = getattr(conn, "loop", None)
+ try:
+ loop = loop or asyncio.get_running_loop()
+ loop.create_task(_run())
+ except RuntimeError:
+ try:
+ apply_deferred_assessment(
+ conn,
+ step_index,
+ session=session,
+ clips=clips,
+ app_id=app_id,
+ task_id=task_id,
+ audio_snapshot=audio_snapshot,
+ persist_audio=False,
+ )
+ except Exception as exc:
+ logger.bind(tag=TAG).warning(f"璺熻鍚庤瘎浼板け璐�: {exc}")
+
+
+def _schedule_shadowing_listen_timeout(conn: "ConnectionHandler", duration_ms: int) -> None:
+ _cancel_shadowing_timeout(conn)
+ ratio = resolve_listen_ratio(conn, _task_for(conn))
+ wait_ms = listen_window_ms(
+ duration_ms or 0,
+ ratio,
+ SHADOWING_LISTEN_MIN_MS,
+ speed=getattr(conn, "shadowing_last_speed", None),
+ wall_clock=True,
+ )
+ conn.shadowing_awaiting_user = True
+ conn.shadowing_assess_index = int(getattr(conn, "shadowing_index", 0) or 0)
+ conn.shadowing_audio_saved_index = -1
+ conn.shadowing_listen_pcm = []
+ gen = int(getattr(conn, "shadowing_clip_gen", 0) or 0)
+ logger.bind(tag=TAG).info(
+ f"璺熻鍚獥 {wait_ms}ms = 瀹為檯鎾斁 {duration_ms}ms 脳 {ratio}"
+ f"锛堝�嶉�熷凡鍚湪瀹為檯鎾斁鏃堕暱閲岋級 gen={gen}"
+ )
+
+ async def _fire():
+ try:
+ await asyncio.sleep(wait_ms / 1000.0)
+ except asyncio.CancelledError:
+ return
+ if int(getattr(conn, "shadowing_clip_gen", 0) or 0) != gen:
+ return
+ if getattr(conn, "interaction_mode", "") != "SHADOWING":
+ return
+ if not getattr(conn, "shadowing_awaiting_user", False):
+ return
+ await _on_shadowing_listen_window_closed(conn)
+
+ loop = getattr(conn, "loop", None)
+ if loop:
+ conn.shadowing_timeout_task = loop.create_task(_fire())
+ else:
+ conn.shadowing_timeout_task = asyncio.create_task(_fire())
+
+
+async def close_shadowing_listen_window(conn: "ConnectionHandler") -> None:
+ """Device listen stop: archive this step once, cancel the clock, advance index."""
+ _cancel_shadowing_timeout(conn)
+ await _on_shadowing_listen_window_closed(conn)
+
+
+def _drain_asr_queue_into_shadowing_pcm(conn: "ConnectionHandler") -> int:
+ """Binary opus is queued; listen stop/timeout run on the WS loop first. Take pending PCM now."""
+ q = getattr(conn, "asr_audio_queue", None)
+ if q is None or not getattr(conn, "shadowing_awaiting_user", False):
+ return 0
+ drained = 0
+ while True:
+ try:
+ frame = q.get_nowait()
+ except (queue.Empty, EOFError, OSError):
+ break
+ if frame:
+ append_shadowing_listen_pcm(conn, frame)
+ drained += 1
+ if drained:
+ logger.bind(tag=TAG).info(f"璺熻鍏崇獥鍓嶆娊骞� ASR 闃熷垪 frames={drained}")
+ return drained
+
+
+def append_shadowing_listen_pcm(conn: "ConnectionHandler", pcm_frame: Any) -> None:
+ if getattr(conn, "interaction_mode", "") != "SHADOWING":
+ return
+ if not getattr(conn, "shadowing_awaiting_user", False):
+ return
+ if not pcm_frame:
+ return
+ buf = getattr(conn, "shadowing_listen_pcm", None)
+ if buf is None:
+ conn.shadowing_listen_pcm = []
+ buf = conn.shadowing_listen_pcm
+ buf.append(pcm_frame)
+
+
+def _take_shadowing_listen_pcm(conn: "ConnectionHandler") -> list:
+ frames = list(getattr(conn, "shadowing_listen_pcm", None) or [])
+ conn.shadowing_listen_pcm = []
+ return frames
+
+
+def save_shadowing_listen_audio(conn: "ConnectionHandler", step_index: Optional[int] = None) -> None:
+ """Persist exactly one wav per follow-read step. Never merge neighboring windows."""
+ if getattr(conn, "interaction_mode", "") != "SHADOWING":
+ return
+ index = int(
+ step_index
+ if step_index is not None
+ else (getattr(conn, "shadowing_assess_index", getattr(conn, "shadowing_index", 0)) or 0)
+ )
+ if int(getattr(conn, "shadowing_audio_saved_index", -1)) == index:
+ conn.shadowing_listen_pcm = []
+ return
+ snapshot = _take_shadowing_listen_pcm(conn)
+ if not snapshot:
+ logger.bind(tag=TAG).warning(f"璺熻鍚獥鏃� PCM锛屽啓闈欓煶鍗犱綅 step={index}")
+ snapshot = silent_pcm_snapshot()
+ conn.shadowing_audio_saved_index = index
+ from core.app_runtime.session import stash_asr_audio
+
+ stash_asr_audio(conn, snapshot)
+ try:
+ apply_deferred_assessment(conn, index, audio_snapshot=snapshot)
+ except Exception as exc:
+ logger.bind(tag=TAG).warning(f"璺熻鍚獥褰曢煶钀界洏澶辫触 step={index}: {exc}")
+
+
+async def _on_shadowing_listen_window_closed(conn: "ConnectionHandler") -> None:
+ """Clock or listen stop: archive this listen. Device already sequences play 鈫� record 鈫� play."""
+ if not getattr(conn, "shadowing_awaiting_user", False):
+ return
+ _drain_asr_queue_into_shadowing_pcm(conn)
+ conn.shadowing_awaiting_user = False
+ listened_idx = int(getattr(conn, "shadowing_index", 0) or 0)
+ listened_prompt = getattr(conn, "shadowing_prompt", "") or ""
+ save_shadowing_listen_audio(conn, listened_idx)
+ items = _shadowing_items(conn)
+ nxt = listened_idx + 1
+ conn.shadowing_index = nxt
+ if nxt < len(items):
+ conn.shadowing_prompt = items[nxt]
+ logger.bind(tag=TAG).info(f"璺熻锛氬惉绐楃粨鏉� step={listened_idx}")
+ try:
+ session = current_session(conn) or getattr(conn, "task_session", None)
+ if session:
+ if session.find_turn(listened_idx) is None:
+ session.upsert_turn(
+ index=listened_idx,
+ prompt_text=listened_prompt,
+ asr_text="",
+ expected_text=_expected_for_index(conn, listened_idx),
+ )
+ session.add_event("listen_window_closed")
+ session.add_event("listen_timeout")
+ session.save()
+ except Exception as exc:
+ logger.bind(tag=TAG).warning(f"璺熻鍚獥褰掓。澶辫触: {exc}")
+
+
+def _parse_mp3_list_names(raw: Any) -> list:
+ text = ""
+ if raw is None:
+ return []
+ if hasattr(raw, "response") or hasattr(raw, "result"):
+ text = str(getattr(raw, "response", None) or getattr(raw, "result", None) or "")
+ else:
+ text = str(raw)
+ text = text.strip()
+ if not text:
+ return []
+ try:
+ import json
+
+ data = json.loads(text)
+ except Exception:
+ return []
+ files = data.get("files") if isinstance(data, dict) else None
+ names = []
+ if isinstance(files, list):
+ for item in files:
+ if isinstance(item, dict) and item.get("name"):
+ names.append(str(item["name"]))
+ elif isinstance(item, str):
+ names.append(item)
+ return names
+
+
+async def _list_sd_mp3_names(conn: "ConnectionHandler") -> list:
+ result = await _execute_named_tool(conn, DEVICE_LIST_TOOL, {})
+ return _parse_mp3_list_names(result)
+
+
+def _is_full_lesson_mp3(name: str) -> bool:
+ base = str(name or "").replace("\\", "/").rsplit("/", 1)[-1].lower()
+ return base == "full.mp3" or base.endswith("_full.mp3")
+
+
+def _compact_shadowing_playlist(
+ conn: "ConnectionHandler",
+ params: Optional[Dict[str, Any]],
+) -> List[Dict[str, Any]]:
+ """Keep {name, repeat, speed}. Device expands repeats locally with a listen window each pass."""
+ items = parse_playlist(params)
+ if not items:
+ items = default_shadowing_playlist(_task_for(conn))
+ compact: List[Dict[str, Any]] = []
+ for raw in items:
+ if not isinstance(raw, dict):
+ continue
+ name = str(raw.get("name") or "")
+ if not name or _is_full_lesson_mp3(name):
+ continue
+ try:
+ repeat_n = int(raw.get("repeat") or 1)
+ except (TypeError, ValueError):
+ repeat_n = 1
+ if repeat_n < 1:
+ repeat_n = 1
+ compact.append(
+ {
+ "name": name,
+ "repeat": repeat_n,
+ "speed": clamp_play_speed(
+ raw["speed"] if raw.get("speed") is not None else 1.0
+ ),
+ "power_off_after": False,
+ "finish_task": False,
+ }
+ )
+ return compact
+
+
+def _begin_shadowing_playlist(
+ conn: "ConnectionHandler",
+ params: Optional[Dict[str, Any]],
+) -> Dict[str, Any]:
+ """One shot: send the whole shadowing playlist. Device plays, records, plays."""
+ items = _compact_shadowing_playlist(conn, params)
+ clips = expand_playlist_plays(items)
+ conn.shadowing_clips = clips
+ conn.shadowing_index = 0
+ conn.shadowing_prompt = clips[0] if clips else ""
+ conn.shadowing_awaiting_user = False
+ first = items[0] if items else dict(params or {})
+ play = dict(first)
+ play["items"] = items
+ play["finish_task"] = False
+ play["listen_ratio"] = resolve_listen_ratio(conn, _task_for(conn))
+ if items:
+ conn.shadowing_last_speed = items[0].get("speed")
+ return play
+
+
+async def _play_shadowing_clip(conn: "ConnectionHandler", name: str) -> bool:
+ if getattr(conn, "interaction_mode", "") != "SHADOWING":
+ logger.bind(tag=TAG).info(f"璺宠繃璺熻鎾斁锛屽綋鍓嶅凡绂诲紑璺熻: {name}")
+ return False
+ conn.shadowing_awaiting_user = False
+ conn.shadowing_clip_gen = int(getattr(conn, "shadowing_clip_gen", 0) or 0) + 1
+ speed = resolve_play_speed(_task_for(conn))
+ conn.shadowing_last_speed = speed
+ notice = await _execute_named_tool_play(
+ conn,
+ {
+ "name": name,
+ "repeat": 1,
+ "power_off_after": False,
+ "speed": speed,
+ "finish_task": False,
+ "listen_ratio": resolve_listen_ratio(conn, _task_for(conn)),
+ },
+ )
+ if notice and notice.action in (Action.ERROR, Action.NOTFOUND):
+ return False
+ return True
+
+
+async def _prepare_shadowing_sd(conn: "ConnectionHandler", params: Optional[Dict[str, Any]]) -> bool:
+ params = params or {}
+ items = _compact_shadowing_playlist(conn, params)
+ clips = expand_playlist_plays(items)
+ if not clips:
+ names = await _list_sd_mp3_names(conn)
+ lesson = str(params.get("lesson") or "").strip()
+ single = str(params.get("name") or params.get("path") or params.get("url") or "")
+ if not lesson and single and not single.lower().endswith(".mp3"):
+ lesson = single
+ if lesson.lower().endswith(".mp3"):
+ lesson = ""
+ clips = lesson_mp3_clips(
+ names,
+ getattr(conn, "active_app_id", "english_learning"),
+ getattr(conn, "current_task_id", "shadowing"),
+ lesson,
+ )
+ clips = [name for name in clips if not _is_full_lesson_mp3(name)]
+ if not clips:
+ conn.shadowing_clips = []
+ conn.shadowing_prompt = ""
+ return False
+ conn.shadowing_clips = clips
+ conn.shadowing_index = 0
+ conn.shadowing_prompt = clips[0]
+ conn.shadowing_awaiting_user = False
+ return True
+
+
+async def _start_shadowing_sd(conn: "ConnectionHandler", params: Optional[Dict[str, Any]]) -> bool:
+ if not await _prepare_shadowing_sd(conn, params):
+ return False
+ play = _begin_shadowing_playlist(conn, params)
+ conn.shadowing_clip_gen = int(getattr(conn, "shadowing_clip_gen", 0) or 0) + 1
+ notice = await _execute_named_tool_play(conn, play)
+ if notice and notice.action in (Action.ERROR, Action.NOTFOUND):
+ return False
+ return True
+
+
+async def _fill_repeater_play_name(
+ conn: "ConnectionHandler", params: Optional[Dict[str, Any]]
+) -> Dict[str, Any]:
+ play = dict(params or {})
+ if play.get("items"):
+ return play
+ if str(play.get("name") or "").strip():
+ return play
+ stub = default_repeater_playlist(_task_for(conn))
+ if stub:
+ first = stub[0]
+ play["name"] = first.get("name") or ""
+ if play.get("repeat") is None:
+ play["repeat"] = first.get("repeat") or 1
+ if play.get("speed") is None and first.get("speed") is not None:
+ play["speed"] = first.get("speed")
+ conn.repeater_play_name = play["name"]
+ return play
+ try:
+ names = await _list_sd_mp3_names(conn)
+ except Exception as exc:
+ logger.bind(tag=TAG).warning(f"鍒椾妇澶嶈璇炬枃澶辫触: {exc}")
+ return play
+ picked = repeater_lesson_mp3(
+ names,
+ getattr(conn, "active_app_id", "") or "english_learning",
+ getattr(conn, "current_task_id", "") or "repeater",
+ )
+ if picked:
+ play["name"] = picked
+ conn.repeater_play_name = picked
+ return play
+
+
+async def maybe_flush_pending_play_after_tts(conn: "ConnectionHandler") -> None:
+ """Start queued lesson audio only after enter-prompt TTS has finished on device."""
+ _ensure_runtime_fields(conn)
+ pending = getattr(conn, "pending_play_after_tts", None)
+ if not pending:
+ return
+ conn.pending_play_after_tts = None
+ kind = pending.get("kind") if isinstance(pending, dict) else None
+ try:
+ if kind == "repeater":
+ play_params = await _fill_repeater_play_name(conn, pending.get("params") or {})
+ await _execute_named_tool_play(conn, play_params)
+ elif kind == "shadowing":
+ play_params = pending.get("params") if isinstance(pending, dict) else None
+ if not play_params:
+ play_params = _begin_shadowing_playlist(conn, {})
+ conn.shadowing_awaiting_user = False
+ conn.shadowing_clip_gen = int(getattr(conn, "shadowing_clip_gen", 0) or 0) + 1
+ await _execute_named_tool_play(conn, play_params)
+ except Exception as exc:
+ logger.bind(tag=TAG).warning(f"寤跺悗鎾斁璇炬枃澶辫触: {exc}")
+
+
+def _lesson_end_speech(conn: "ConnectionHandler", phrase: str) -> str:
+ """Short lesson-end cue plus the same default chat greeting used after 缈昏瘧/涓嫳瀵硅瘽."""
+ greeting = (_greeting_text(conn) or "").strip()
+ phrase = (phrase or "").strip()
+ if phrase and greeting:
+ if phrase.endswith("銆�") or phrase.endswith("."):
+ return f"{phrase}{greeting}"
+ return f"{phrase}銆倇greeting}"
+ return phrase or greeting
+
+
+async def _return_to_chat_after_lesson(
+ conn: "ConnectionHandler",
+ phrase: str,
+ reason: str = "play_completed",
+) -> None:
+ """Playlist already finished on device: speak the end cue now, then chat. Do not MCP-stop."""
+ conn.repeater_speed_queue = []
+ conn.repeater_playlist = []
+ end_session(conn, reason)
+ if getattr(conn, "tts", None) is not None:
+ await wait_for_tts_channels(conn, timeout=3)
+ conn.sentence_id = uuid.uuid4().hex
+ _speak(conn, _lesson_end_speech(conn, phrase))
+ await start_app_task(
+ conn,
+ "chat",
+ "conversation",
+ {},
+ source="voice",
+ execute_skills=False,
+ speak_switch=False,
+ stop_skills=False,
+ )
+
+
+async def _shadowing_play_next_or_finish(conn: "ConnectionHandler") -> None:
+ """Legacy helper: device now sequences clips. Only used if a caller still asks to finish."""
+ conn.shadowing_index = 0
+ conn.shadowing_prompt = ""
+ conn.shadowing_clips = []
+ await _return_to_chat_after_lesson(conn, SHADOWING_END_PHRASE, "finished")
+
+
+def _mark_repeater_day(conn: "ConnectionHandler") -> None:
+ try:
+ mark_lesson_played_today(
+ str(getattr(conn, "device_id", "") or ""),
+ getattr(conn, "active_app_id", "") or "",
+ getattr(conn, "current_task_id", "") or "",
+ getattr(conn, "repeater_play_name", "") or "",
+ )
+ except Exception as exc:
+ logger.bind(tag=TAG).warning(f"璁板綍澶嶈澶╂暟澶辫触: {exc}")
+
+
+def _apply_playback_speed(
+ conn: "ConnectionHandler",
+ task: Optional[Dict[str, Any]],
+ params: Optional[Dict[str, Any]],
+) -> Dict[str, Any]:
+ """Attach clamped speed. Multi-speed repeats are Server-driven (one pass per play)."""
+ play = dict(params or {})
+ try:
+ repeat_n = int(play.get("repeat") or 1)
+ except (TypeError, ValueError):
+ repeat_n = 1
+ if repeat_n < 1:
+ repeat_n = 1
+ device_id = str(getattr(conn, "device_id", "") or "")
+ app_id = getattr(conn, "active_app_id", "") or ""
+ task_id = getattr(conn, "current_task_id", "") or ""
+ lesson = str(play.get("name") or "")
+ day_index = lesson_day_index(device_id, app_id, task_id, lesson)
+ if play.get("speed") is not None:
+ speeds = [clamp_play_speed(play.get("speed"))] * repeat_n
+ else:
+ speeds = speeds_for_repeats(task, repeat_n, day_index)
+ conn.repeater_play_name = lesson
+ conn.repeater_power_off_after = bool(play.get("power_off_after") or False)
+ varied = len(speeds) > 1 and any(abs(item - speeds[0]) > 0.001 for item in speeds[1:])
+ if varied:
+ conn.repeater_speed_queue = speeds[1:]
+ play["speed"] = speeds[0]
+ play["repeat"] = 1
+ play["power_off_after"] = False
+ else:
+ conn.repeater_speed_queue = []
+ play["speed"] = speeds[0]
+ play["repeat"] = repeat_n
+ # 澶嶈娓呭崟缁撴潫鐢� Server 鏀� audio.completed 鍚庡垏鍥炲璇濄�傝澶囦笉瑕� finish_task 鍐� NVS銆�
+ play["finish_task"] = False
+ return play
+
+
+def _begin_repeater_playlist(
+ conn: "ConnectionHandler",
+ task: Optional[Dict[str, Any]],
+ params: Optional[Dict[str, Any]],
+) -> Dict[str, Any]:
+ """One shot: send the whole playlist. Repeat is just the same clip again, no mode switch."""
+ items = parse_playlist(params)
+ if not items:
+ items = default_repeater_playlist(task)
+ normalized = []
+ for raw in items:
+ if not isinstance(raw, dict):
+ continue
+ name = str(raw.get("name") or "")
+ if not name:
+ continue
+ try:
+ repeat_n = int(raw.get("repeat") or 1)
+ except (TypeError, ValueError):
+ repeat_n = 1
+ if repeat_n < 1:
+ repeat_n = 1
+ item = {
+ "name": name,
+ "repeat": repeat_n,
+ "speed": clamp_play_speed(
+ raw["speed"] if raw.get("speed") is not None else 1.0
+ ),
+ "power_off_after": bool(raw.get("power_off_after") or False),
+ "finish_task": False,
+ }
+ normalized.append(item)
+ conn.repeater_playlist = []
+ conn.repeater_speed_queue = []
+ first = normalized[0] if normalized else dict(params or {})
+ conn.repeater_play_name = str(first.get("name") or "")
+ conn.repeater_power_off_after = bool(first.get("power_off_after") or False)
+ play = dict(first)
+ play["items"] = normalized
+ play["finish_task"] = False
+ return play
+
+
+async def _continue_repeater_if_needed(conn: "ConnectionHandler") -> bool:
+ queue = list(getattr(conn, "repeater_speed_queue", None) or [])
+ if queue:
+ nxt = queue.pop(0)
+ conn.repeater_speed_queue = queue
+ last_speed = len(queue) == 0 and not list(getattr(conn, "repeater_playlist", None) or [])
+ try:
+ play_params = await _fill_repeater_play_name(
+ conn,
+ {
+ "name": getattr(conn, "repeater_play_name", "") or "",
+ "speed": nxt,
+ "repeat": 1,
+ "power_off_after": last_speed
+ and bool(getattr(conn, "repeater_power_off_after", False)),
+ "finish_task": False,
+ },
+ )
+ await _execute_named_tool_play(conn, play_params)
+ except Exception as exc:
+ logger.bind(tag=TAG).warning(f"澶嶈涓嬩竴閬嶅け璐�: {exc}")
+ conn.repeater_speed_queue = []
+ return False
+ return True
+
+ rest = list(getattr(conn, "repeater_playlist", None) or [])
+ if not rest:
+ return False
+ nxt, conn.repeater_playlist = rest[0], rest[1:]
+ play_params = _apply_playback_speed(conn, _task_for(conn), nxt)
+ play_params["finish_task"] = False
+ try:
+ play_params = await _fill_repeater_play_name(conn, play_params)
+ await _execute_named_tool_play(conn, play_params)
+ except Exception as exc:
+ logger.bind(tag=TAG).warning(f"澶嶈涓嬩竴棣栧け璐�: {exc}")
+ conn.repeater_playlist = []
+ return False
+ return True
+
+
+async def _execute_named_tool_play(conn, params):
+ items = params.get("items") if isinstance(params.get("items"), list) else []
+ first = items[0] if items else params
+ play_args = {
+ "name": (first.get("name") if isinstance(first, dict) else None) or params.get("name") or "",
+ "repeat": int((first.get("repeat") if isinstance(first, dict) else None) or params.get("repeat") or 1),
+ "power_off_after": bool(params.get("power_off_after") or False),
+ "speed": clamp_play_speed(
+ (first.get("speed") if isinstance(first, dict) and first.get("speed") is not None else None)
+ if items
+ else (params["speed"] if params.get("speed") is not None else 1.0)
+ ),
+ "finish_task": bool(params.get("finish_task", True)),
+ "app_id": str(getattr(conn, "active_app_id", "") or ""),
+ "task_id": str(getattr(conn, "current_task_id", "") or ""),
+ "force": True,
+ }
+ if items:
+ play_args["items"] = json.dumps(items, ensure_ascii=False)
+ play_args["speed"] = clamp_play_speed(
+ first["speed"] if first.get("speed") is not None else 1.0
+ )
+ play_args["power_off_after"] = bool(
+ items[-1].get("power_off_after") if items else False
+ )
+ if params.get("listen_ratio") is not None:
+ try:
+ play_args["listen_ratio"] = float(params["listen_ratio"])
+ except (TypeError, ValueError):
+ pass
+ if play_args["task_id"] in ("repeater", "shadowing"):
+ play_args["finish_task"] = False
+ result = await _execute_named_tool(conn, DEVICE_PLAY_TOOL, play_args)
+ blob = ""
+ if result:
+ blob = str(result.response or result.result or "")
+ compact = blob.replace(" ", "")
+ if '"duplicate_ignored":true' in compact:
+ logger.bind(tag=TAG).warning("璁惧蹇界暐浜嗗悓涓�璇炬枃鐨勪笅涓�閬嶏紝寮哄埗鍐嶆挱涓�娆�")
+ play_args["speed"] = clamp_play_speed(float(play_args["speed"]) + 0.02)
+ play_args["force"] = True
+ result = await _execute_named_tool(conn, DEVICE_PLAY_TOOL, play_args)
+ blob = ""
+ if result:
+ blob = str(result.response or result.result or "")
+ compact = blob.replace(" ", "")
+ rejected = (result and result.action in (Action.ERROR, Action.NOTFOUND)) or (
+ "no_mp3" in blob
+ or '"accepted":false' in compact
+ or '"duplicate_ignored":true' in compact
+ )
+ if rejected:
+ if not play_args["name"]:
+ return ActionResponse(
+ action=Action.RESPONSE,
+ response="澶嶈鏈哄凡鎵撳紑锛屼絾 SD 鍗′笂杩樻病鏈夊彲鎾斁鐨勮鏂囥��",
+ )
+ return ActionResponse(
+ action=Action.RESPONSE,
+ response=(result.response if result else None) or "杩欏彴璁惧娌℃湁鏈湴澶嶈鎾斁鑳藉姏",
+ )
+ return None
+
+
async def start_app_task(
conn: "ConnectionHandler",
app_id: str,
@@ -147,6 +1262,8 @@
params: Optional[Dict[str, Any]] = None,
source: str = "voice",
execute_skills: bool = True,
+ speak_switch: bool = True,
+ stop_skills: bool = True,
) -> ActionResponse:
"""Single entry for MQTT / schedule / voice / default chat."""
_ensure_runtime_fields(conn)
@@ -176,8 +1293,16 @@
response="褰撳墠鏈夋洿楂樹紭鍏堢骇鐨勪换鍔★紝鍏堝畬鎴愬悗鍐嶅垏鎹�",
)
- if not same_task or new_p >= cur_p:
- await _stop_current_skills(conn)
+ prev_app_id = getattr(conn, "active_app_id", "chat")
+ prev_task_id = getattr(conn, "current_task_id", "conversation")
+ prev_app, prev_task = get_task(prev_app_id, prev_task_id)
+ switching = (prev_app_id != app_id) or (prev_task_id != resolved_task_id)
+
+ if not same_task:
+ if stop_skills:
+ await _stop_current_skills(conn)
+ if app_id != "chat":
+ conn.chat_greeted = False
conn.active_app_id = app_id
conn.current_task_id = resolved_task_id
@@ -192,51 +1317,92 @@
f"start_app_task app={app_id} task={resolved_task_id} mode={conn.interaction_mode} source={source}"
)
+ if app_id == "chat":
+ end_session(conn, "user_exit")
+ elif switching:
+ begin_session(conn, app_id, resolved_task_id, source, conn.interaction_mode)
+
+ pending_play = None
if execute_skills:
+ if app_sd_layout_paths(app_id, app):
+ await ensure_app_sd_dirs(conn, app_id)
for skill in task.get("skills") or []:
skill_name = (skill or {}).get("name") or ""
if skill_name in ("self.mp3.play", DEVICE_PLAY_TOOL):
- play_args = {
- "name": params.get("name") or "",
- "repeat": int(params.get("repeat") or 1),
- "power_off_after": bool(params.get("power_off_after") or False),
- }
- if params.get("speed") is not None:
- play_args["speed"] = params.get("speed")
- if not play_args["name"]:
- return ActionResponse(
- action=Action.RESPONSE,
- response="娌℃湁鎸囧畾瑕佸璇荤殑鏂囦欢鍚嶏紝璇峰厛鐢� self.mp3.get_list 纭 files[].name",
- )
- result = await _execute_named_tool(conn, DEVICE_PLAY_TOOL, play_args)
- if result and result.action in (Action.ERROR, Action.NOTFOUND):
- return ActionResponse(
- action=Action.RESPONSE,
- response=result.response or "杩欏彴璁惧娌℃湁鏈湴澶嶈鎾斁鑳藉姏",
- )
+ play_params = _begin_repeater_playlist(conn, task, params)
+ pending_play = {"kind": "repeater", "params": play_params}
elif skill_name == "self.power_off":
await _execute_named_tool(conn, "self_power_off", {})
- if conn.interaction_mode == "CHAT":
+ announce = ""
+ boot_chat = app_id == "chat" and source == "default_chat"
+ if boot_chat:
+ # 寮�鏈鸿繘 chat锛氬繀椤绘挱娆㈣繋璇嶃�備笉瑕佽 same_task / just_woken_up 璺宠繃銆�
+ enable = (conn.config or {}).get("enable_proactive_greeting", True)
+ if enable and not getattr(conn, "chat_greeted", False):
+ if not await wait_for_tts_channels(conn, timeout=10):
+ logger.bind(tag=TAG).error("寮�鏈烘杩庤瘝璺宠繃锛歍TS閫氶亾鏈氨缁�")
+ else:
+ conn.chat_greeted = True
+ conn.sentence_id = uuid.uuid4().hex
+ _speak(conn, _greeting_text(conn))
+ elif switching:
+ announce = _switch_announcement(prev_app, prev_task, app, task, app_id)
+ if conn.interaction_mode == "SHADOWING":
+ started = await _prepare_shadowing_sd(conn, params)
+ if not started:
+ prefix = announce or "鐜板湪宸茬粡鍒囨崲鑷宠窡璇绘ā寮忋��"
+ announce = f"{prefix}璺熻璇炬枃杩樻病鏈夋斁鍒癝D鍗°��"
+ else:
+ announce = announce or "鐜板湪宸茬粡鍒囨崲鑷宠窡璇绘ā寮忥紝璇疯窡鐫�璇炬枃璇汇��"
+ pending_play = {
+ "kind": "shadowing",
+ "params": _begin_shadowing_playlist(conn, params),
+ }
+ if speak_switch and announce:
+ conn.sentence_id = uuid.uuid4().hex
+ conn.pending_play_after_tts = pending_play
+ _speak(conn, announce)
+ elif pending_play:
+ conn.pending_play_after_tts = pending_play
+ await maybe_flush_pending_play_after_tts(conn)
+ if app_id == "chat":
+ conn.chat_greeted = True
+ elif conn.interaction_mode == "CHAT":
enable = (conn.config or {}).get("enable_proactive_greeting", True)
woken = bool(getattr(conn, "just_woken_up", False))
- if enable and not woken:
+ if enable and not woken and not getattr(conn, "chat_greeted", False):
+ conn.chat_greeted = True
+ conn.sentence_id = uuid.uuid4().hex
_speak(conn, _greeting_text(conn))
+ elif conn.interaction_mode == "SHADOWING" and not getattr(conn, "shadowing_clips", None):
+ started = await _start_shadowing_sd(conn, params)
+ if not started and speak_switch:
+ conn.sentence_id = uuid.uuid4().hex
+ _speak(conn, "璺熻璇炬枃杩樻病鏈夋斁鍒癝D鍗°��")
+ elif pending_play:
+ conn.pending_play_after_tts = pending_play
+ await maybe_flush_pending_play_after_tts(conn)
- app_name = app.get("name") or app_id
+ app_name = _display_name(app, task)
return ActionResponse(
action=Action.NONE,
result=f"{app_id}/{resolved_task_id}",
- response=f"宸叉墦寮�{app_name}",
+ response=announce or f"宸叉墦寮�{app_name}",
)
-async def switch_app(conn: "ConnectionHandler", app_id: str, source: str = "voice") -> ActionResponse:
+async def switch_app(
+ conn: "ConnectionHandler",
+ app_id: str,
+ source: str = "voice",
+ task_id: str = "",
+) -> ActionResponse:
app = get_app(app_id)
if not app:
return ActionResponse(action=Action.ERROR, response=f"涓嶆敮鎸佺殑 App: {app_id}")
- task_id = app.get("default_task") or ""
- return await start_app_task(conn, app_id, task_id, {}, source)
+ resolved = task_id or app.get("default_task") or ""
+ return await start_app_task(conn, app_id, resolved, {}, source)
async def on_user_wakeup(conn: "ConnectionHandler") -> None:
@@ -248,24 +1414,11 @@
def match_voice_switch(text: str) -> Optional[tuple]:
- """Map a short phrase to (app_id, task_id) or ('__menu__', '')."""
+ """Map a short phrase to (app_id, task_id) using apps.json voice_enter / voice_exit."""
if not text:
return None
_, filtered = _strip(text)
- if filtered in ("鍒囨崲妯″紡", "qie huan mo shi"):
- return ("__menu__", "")
- for p in CHAT_PHRASES:
- if p in filtered:
- return ("chat", "conversation")
- for p in TRANSLATE_PHRASES:
- if p in filtered:
- return ("english_translation", "live_translate")
- for p in REPEATER_PHRASES:
- if p in filtered:
- return ("english_learning", "repeater")
- if any(k in filtered for k in ("鏁板瓧鑳岃", "鎴愯鎺ラ緳", "鍙ｇ畻", "鍔犲噺娉�")):
- return ("__reserved__", "")
- return None
+ return match_registry_voice(filtered)
def _strip(text: str) -> tuple:
@@ -274,9 +1427,23 @@
return remove_punctuation_and_length(text)
+def is_exit_to_chat(text: str) -> bool:
+ mapped = match_voice_switch(text)
+ return bool(mapped and mapped[0] == "chat")
+
+
+def should_skip_listen_stop_asr(conn: Any) -> bool:
+ """True when listen stop is closing the mic for lesson playback, not a follow-read."""
+ mode = getattr(conn, "interaction_mode", "CHAT")
+ if mode == "PLAY_AUDIO":
+ return True
+ if mode == "SHADOWING" and not getattr(conn, "shadowing_awaiting_user", False):
+ return True
+ return False
+
+
def is_exit_translation(text: str) -> bool:
- _, filtered = _strip(text)
- return any(p in filtered for p in ("閫�鍑虹炕璇�", "缁撴潫缈昏瘧", "鍥炲埌鑱婂ぉ", "鎴戜滑鏉ヨ亰澶╁惂", "鑱婁細澶╁惂"))
+ return is_exit_to_chat(text)
def _plain_user_text(text: str) -> str:
@@ -296,6 +1463,40 @@
return raw
+def _norm_for_echo(text: str) -> str:
+ _, filtered = _strip(text or "")
+ return re.sub(r"\s+", "", (filtered or "").lower())
+
+
+def is_echo_of_prompt(user_text: str, prompt: str) -> bool:
+ a = _norm_for_echo(user_text)
+ b = _norm_for_echo(prompt)
+ if not a or not b:
+ return False
+ if a == b or a in b or b in a:
+ return True
+ return difflib.SequenceMatcher(None, a, b).ratio() >= 0.82
+
+
+def detect_user_language(text: str, hint: Optional[str] = None) -> str:
+ if hint:
+ h = str(hint).lower()
+ if h.startswith("zh") or h in ("cn", "yue", "chinese"):
+ return "zh"
+ if h.startswith("en") or h in ("english",):
+ return "en"
+ body = _plain_user_text(text)
+ cjk = len(re.findall(r"[\u4e00-\u9fff]", body))
+ latin = len(re.findall(r"[A-Za-z]", body))
+ return "zh" if cjk >= max(latin, 1) else "en"
+
+
+def wrap_bilingual_user_text(text: str, user_lang: str) -> str:
+ if user_lang == "zh":
+ return "Respond in English only. Do not use Chinese characters.\n\n" + text
+ return "璇峰彧鐢ㄤ腑鏂囧洖绛旓紝涓嶈鍑虹幇鑻辨枃鍗曡瘝鎴栧彞瀛愩�俓n\n" + text
+
+
def _looks_like_tool_error(text: str) -> bool:
if not text:
return True
@@ -307,6 +1508,180 @@
return False
+def _record_shadowing_turn(
+ conn: "ConnectionHandler",
+ text: str,
+ echo_rejected: bool = False,
+ index: Optional[int] = None,
+) -> None:
+ session = current_session(conn)
+ if not session:
+ return
+ if index is None:
+ index = int(getattr(conn, "shadowing_assess_index", getattr(conn, "shadowing_index", 0)) or 0)
+ session.upsert_turn(
+ index=index,
+ prompt_text=getattr(conn, "shadowing_prompt", "") or "",
+ asr_text=_plain_user_text(text),
+ echo_rejected=echo_rejected,
+ expected_text=_expected_for_index(conn, index),
+ )
+ session.save()
+
+
+def apply_shadowing_user_text(conn: "ConnectionHandler", text: str) -> str:
+ """Apply one ASR utterance. Returns exit | ignore | echo | recorded.
+
+ ASR never advances the clip index; the listen-window clock does.
+ Late ASR after the window still scores the sentence that was just listened to.
+ Device plays the next clip itself; Server does not MCP the next sentence.
+ """
+ if is_exit_to_chat(text):
+ end_session(conn, "user_exit")
+ return "exit"
+ if getattr(conn, "client_is_speaking", False) and not getattr(
+ conn, "shadowing_awaiting_user", False
+ ):
+ logger.bind(tag=TAG).info("璺熻锛氭挱鏀句腑蹇界暐鎷鹃煶锛岄伩鍏嶅枃鍙洖澹拌Е鍙戝璇�")
+ return "ignore"
+ prompt = getattr(conn, "shadowing_prompt", "") or ""
+ if (
+ getattr(conn, "shadowing_awaiting_user", False)
+ and prompt
+ and not _is_clip_path(prompt)
+ and is_echo_of_prompt(text, prompt)
+ ):
+ logger.bind(tag=TAG).info("璺熻锛氬拷鐣ヤ笌鑼冭杩囦簬鐩镐技鐨勮瘑鍒粨鏋�")
+ session = current_session(conn)
+ if session:
+ session.add_event("echo_rejected")
+ session.save()
+ return "echo"
+ _record_shadowing_turn(conn, text, echo_rejected=False)
+ return "recorded"
+
+
+async def handle_shadowing_turn(conn: "ConnectionHandler", text: str) -> None:
+ """Score this utterance off the chat path. Next clip is sequenced on device."""
+ outcome = apply_shadowing_user_text(conn, text)
+ if outcome == "exit":
+ _cancel_shadowing_timeout(conn)
+ await start_app_task(conn, "chat", "conversation", {}, source="voice")
+ return
+ if outcome in ("ignore", "echo"):
+ return
+
+ step_index = int(
+ getattr(conn, "shadowing_assess_index", getattr(conn, "shadowing_index", 0)) or 0
+ )
+ logger.bind(tag=TAG).info(f"璺熻 ASR 浠呰瘎浼颁笉涓婂璇� step={step_index}")
+ _schedule_deferred_assessment(conn, step_index)
+
+
+async def handle_skill_event(conn: "ConnectionHandler", text: str) -> bool:
+ """Handle `[skill_event]audio.completed|app|task|duration_ms`. Returns True if consumed."""
+ parsed = parse_skill_event(text)
+ if parsed is None:
+ return False
+ _ensure_runtime_fields(conn)
+ name = parsed.get("name") or ""
+ app_id = parsed.get("app_id") or ""
+ task_id = parsed.get("task_id") or ""
+ try:
+ duration_ms = int(parsed.get("duration_ms") or 0)
+ except (TypeError, ValueError):
+ duration_ms = 0
+ logger.bind(tag=TAG).info(f"skill_event {name} {app_id}/{task_id} duration_ms={duration_ms}")
+
+ mode = getattr(conn, "interaction_mode", "CHAT")
+ # 鍚獥鏃堕挓蹇呴』鍏堜簬 session.save()锛氱鐩�/HTTP 涓嶈兘鎸′綇 1.2脳duration 鐨勭儹璺緞銆�
+ shadowing_playlist_done = name in ("audio.playlist_completed", "playlist.completed") and (
+ mode == "SHADOWING" or task_id == "shadowing"
+ )
+ if shadowing_playlist_done:
+ _cancel_shadowing_timeout(conn)
+ if getattr(conn, "shadowing_awaiting_user", False):
+ await _on_shadowing_listen_window_closed(conn)
+ else:
+ save_shadowing_listen_audio(conn)
+ session = getattr(conn, "task_session", None)
+ if session:
+ session.add_event(name, app_id, task_id)
+ await _return_to_chat_after_lesson(conn, SHADOWING_END_PHRASE, "finished")
+ return True
+ if mode == "SHADOWING" and name == "audio.completed":
+ if getattr(conn, "shadowing_awaiting_user", False):
+ await _on_shadowing_listen_window_closed(conn)
+ conn.shadowing_clip_gen = int(getattr(conn, "shadowing_clip_gen", 0) or 0) + 1
+ _schedule_shadowing_listen_timeout(conn, duration_ms)
+ session = getattr(conn, "task_session", None)
+ if session:
+ session.add_event(name, app_id, task_id)
+ return True
+ if mode == "SHADOWING" and name == "audio.stopped":
+ session = getattr(conn, "task_session", None)
+ if session:
+ session.add_event(name, app_id, task_id)
+ end_session(conn, "play_stopped")
+ await start_app_task(
+ conn,
+ "chat",
+ "conversation",
+ {},
+ source="voice",
+ execute_skills=False,
+ speak_switch=False,
+ )
+ return True
+
+ if mode == "PLAY_AUDIO" and name == "audio.completed":
+ continued = await _continue_repeater_if_needed(conn)
+ session = getattr(conn, "task_session", None)
+ if session:
+ session.add_event(name, app_id, task_id)
+ if continued:
+ if session:
+ session.save()
+ return True
+ _mark_repeater_day(conn)
+ await _return_to_chat_after_lesson(conn, REPEATER_END_PHRASE, "play_completed")
+ return True
+ if mode == "PLAY_AUDIO" and name == "audio.stopped":
+ conn.repeater_speed_queue = []
+
+ session = getattr(conn, "task_session", None)
+ if session:
+ session.add_event(name, app_id, task_id)
+ session.save()
+
+ if name == "audio.completed":
+ end_session(conn, "play_completed")
+ await start_app_task(
+ conn,
+ "chat",
+ "conversation",
+ {},
+ source="voice",
+ execute_skills=False,
+ speak_switch=True,
+ )
+ elif name == "audio.stopped":
+ end_session(conn, "play_stopped")
+ if getattr(conn, "active_app_id", "chat") != "chat":
+ await start_app_task(
+ conn,
+ "chat",
+ "conversation",
+ {},
+ source="voice",
+ execute_skills=False,
+ speak_switch=False,
+ )
+ else:
+ logger.bind(tag=TAG).warning(f"鏈煡 skill_event: {name}")
+ return True
+
+
async def run_translation(conn: "ConnectionHandler", text: str) -> str:
"""Translate with the connection LLM first (鏅烘帶鍙板凡閰嶇殑妯″瀷). MCP is fallback only."""
text = _plain_user_text(text)
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/core/app_runtime/session.py b/xiaozhi-esp32-server/main/xiaozhi-server/core/app_runtime/session.py
new file mode 100644
index 0000000..33fb816
--- /dev/null
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/core/app_runtime/session.py
@@ -0,0 +1,275 @@
+"""Phase 2 TaskSession: in-memory + JSON file. Not a TaskInstance."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import time
+import uuid
+from dataclasses import asdict, dataclass, field
+from typing import Any, Dict, List, Optional, TYPE_CHECKING
+
+from config.config_loader import get_project_dir
+from config.logger import setup_logging
+
+if TYPE_CHECKING:
+ from core.connection import ConnectionHandler
+
+TAG = __name__
+logger = setup_logging()
+
+SKILL_EVENT_PREFIX = "[skill_event]"
+DEFER_ASR_REPORT_MODES = ("PLAY_AUDIO", "SHADOWING")
+
+
+def _now_ms() -> int:
+ return int(time.time() * 1000)
+
+
+def session_dir() -> str:
+ return os.path.join(get_project_dir(), "data", "task_sessions")
+
+
+def parse_skill_event(text: str) -> Optional[Dict[str, str]]:
+ """Parse `[skill_event]audio.completed|app_id|task_id` into a dict."""
+ if not text:
+ return None
+ raw = str(text).strip()
+ if not raw.startswith(SKILL_EVENT_PREFIX):
+ return None
+ payload = raw[len(SKILL_EVENT_PREFIX) :].strip()
+ parts = [p.strip() for p in payload.split("|")]
+ name = parts[0] if parts else ""
+ if not name:
+ return None
+ return {
+ "name": name,
+ "app_id": parts[1] if len(parts) > 1 else "",
+ "task_id": parts[2] if len(parts) > 2 else "",
+ "duration_ms": parts[3] if len(parts) > 3 else "",
+ }
+
+
+def should_report_asr_now(conn: Any) -> bool:
+ """PLAY_AUDIO / SHADOWING defer report so echo and ignored uplink are not archived."""
+ mode = getattr(conn, "interaction_mode", "CHAT")
+ return mode not in DEFER_ASR_REPORT_MODES
+
+
+def stash_asr_audio(conn: Any, audio_snapshot: Any) -> None:
+ conn.last_asr_audio_snapshot = audio_snapshot
+
+
+@dataclass
+class TaskSession:
+ task_session_id: str
+ ws_session_id: str = ""
+ device_id: str = ""
+ agent_id: str = ""
+ app_id: str = ""
+ task_id: str = ""
+ source: str = "voice"
+ interaction_mode: str = ""
+ started_at: int = 0
+ ended_at: int = 0
+ status: str = "running"
+ end_reason: str = ""
+ turns: List[Dict[str, Any]] = field(default_factory=list)
+ events: List[Dict[str, Any]] = field(default_factory=list)
+
+ def find_turn(self, index: int) -> Optional[Dict[str, Any]]:
+ for turn in self.turns:
+ if turn.get("index") == index:
+ return turn
+ return None
+
+ def upsert_turn(
+ self,
+ index: int,
+ prompt_text: str = "",
+ asr_text: Optional[str] = None,
+ echo_rejected: bool = False,
+ expected_text: Optional[str] = None,
+ metrics: Optional[List[Dict[str, Any]]] = None,
+ assessed_at: Optional[int] = None,
+ audio_id: Optional[str] = None,
+ ) -> Dict[str, Any]:
+ """Same step_index stays one row; last ASR wins. Empty metrics do not wipe old."""
+ existing = self.find_turn(index)
+ if existing is None:
+ turn: Dict[str, Any] = {
+ "index": index,
+ "prompt_text": prompt_text or "",
+ "asr_text": asr_text or "",
+ "echo_rejected": bool(echo_rejected),
+ "recorded_at": _now_ms(),
+ }
+ if expected_text:
+ turn["expected_text"] = expected_text
+ if metrics:
+ turn["metrics"] = metrics
+ if assessed_at is not None:
+ turn["assessed_at"] = assessed_at
+ if audio_id:
+ turn["audio_id"] = audio_id
+ self.turns.append(turn)
+ return turn
+ if prompt_text:
+ existing["prompt_text"] = prompt_text
+ if asr_text is not None:
+ existing["asr_text"] = asr_text
+ existing["echo_rejected"] = bool(echo_rejected)
+ existing["recorded_at"] = _now_ms()
+ if expected_text:
+ existing["expected_text"] = expected_text
+ if metrics:
+ existing["metrics"] = metrics
+ if assessed_at is not None:
+ existing["assessed_at"] = assessed_at
+ if audio_id:
+ existing["audio_id"] = audio_id
+ return existing
+
+ def add_turn(
+ self,
+ index: int,
+ prompt_text: str,
+ asr_text: str,
+ echo_rejected: bool = False,
+ ) -> None:
+ self.upsert_turn(
+ index=index,
+ prompt_text=prompt_text,
+ asr_text=asr_text,
+ echo_rejected=echo_rejected,
+ )
+
+ def add_event(self, name: str, app_id: str = "", task_id: str = "") -> None:
+ self.events.append(
+ {
+ "name": name,
+ "app_id": app_id or self.app_id,
+ "task_id": task_id or self.task_id,
+ "at": _now_ms(),
+ }
+ )
+
+ def finish(self, reason: str) -> None:
+ if self.status != "running":
+ return
+ self.status = "completed" if reason in ("finished", "play_completed") else "cancelled"
+ self.end_reason = reason
+ self.ended_at = _now_ms()
+
+ def to_dict(self) -> Dict[str, Any]:
+ return asdict(self)
+
+ def save(self) -> None:
+ try:
+ directory = session_dir()
+ os.makedirs(directory, exist_ok=True)
+ path = os.path.join(directory, f"{self.task_session_id}.json")
+ with open(path, "w", encoding="utf-8") as handle:
+ json.dump(self.to_dict(), handle, ensure_ascii=False, indent=2)
+ except Exception as exc:
+ logger.bind(tag=TAG).warning(f"淇濆瓨 TaskSession 澶辫触: {exc}")
+ return
+ if getattr(self, "_skip_upsert", False):
+ return
+ try:
+ _schedule_upsert(self.to_dict(), self)
+ except Exception as exc:
+ logger.bind(tag=TAG).warning(f"TaskInstance 涓婃姤璋冨害澶辫触: {exc}")
+
+ @classmethod
+ def load(cls, task_session_id: str) -> Optional["TaskSession"]:
+ path = os.path.join(session_dir(), f"{task_session_id}.json")
+ if not os.path.isfile(path):
+ return None
+ with open(path, "r", encoding="utf-8") as handle:
+ data = json.load(handle)
+ return cls(
+ task_session_id=data.get("task_session_id") or task_session_id,
+ ws_session_id=data.get("ws_session_id") or "",
+ device_id=data.get("device_id") or "",
+ agent_id=data.get("agent_id") or "",
+ app_id=data.get("app_id") or "",
+ task_id=data.get("task_id") or "",
+ source=data.get("source") or "voice",
+ interaction_mode=data.get("interaction_mode") or "",
+ started_at=int(data.get("started_at") or 0),
+ ended_at=int(data.get("ended_at") or 0),
+ status=data.get("status") or "running",
+ end_reason=data.get("end_reason") or "",
+ turns=list(data.get("turns") or []),
+ events=list(data.get("events") or []),
+ )
+
+
+def begin_session(conn: "ConnectionHandler", app_id: str, task_id: str, source: str, mode: str) -> TaskSession:
+ existing = getattr(conn, "task_session", None)
+ if existing and getattr(existing, "status", "") == "running":
+ end_session(conn, "preempted")
+ session = TaskSession(
+ task_session_id=uuid.uuid4().hex,
+ ws_session_id=str(getattr(conn, "session_id", "") or ""),
+ device_id=str(getattr(conn, "device_id", "") or ""),
+ agent_id=str(getattr(conn, "agent_id", "") or ""),
+ app_id=app_id,
+ task_id=task_id,
+ source=source or "voice",
+ interaction_mode=mode or "",
+ started_at=_now_ms(),
+ status="running",
+ )
+ conn.task_session = session
+ session.save()
+ logger.bind(tag=TAG).info(
+ f"TaskSession 寮�濮� {session.task_session_id} {app_id}/{task_id} mode={mode}"
+ )
+ return session
+
+
+def end_session(conn: "ConnectionHandler", reason: str) -> None:
+ session = getattr(conn, "task_session", None)
+ if not session or getattr(session, "status", "") != "running":
+ return
+ session.finish(reason)
+ session.save()
+ logger.bind(tag=TAG).info(
+ f"TaskSession 缁撴潫 {session.task_session_id} status={session.status} reason={reason}"
+ )
+
+
+def current_session(conn: "ConnectionHandler") -> Optional[TaskSession]:
+ session = getattr(conn, "task_session", None)
+ if session and getattr(session, "status", "") == "running":
+ return session
+ return None
+
+
+async def _upsert_task_instance(payload: Dict[str, Any]) -> bool:
+ from config.manage_api_client import upsert_task_instance
+
+ data = await upsert_task_instance(payload)
+ return data is not None
+
+
+def _schedule_upsert(payload: Dict[str, Any], session: Optional["TaskSession"] = None) -> None:
+ async def _run():
+ try:
+ ok = await _upsert_task_instance(payload)
+ if not ok and session is not None:
+ session._skip_upsert = True
+ logger.bind(tag=TAG).warning("TaskInstance 涓婃姤澶辫触锛屾湰浼氳瘽涓嶅啀閲嶅涓婃姤")
+ except Exception as exc:
+ if session is not None:
+ session._skip_upsert = True
+ logger.bind(tag=TAG).warning(f"TaskInstance 涓婃姤澶辫触: {exc}")
+
+ try:
+ loop = asyncio.get_running_loop()
+ except RuntimeError:
+ return
+ loop.create_task(_run())
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/core/audio_archive.py b/xiaozhi-esp32-server/main/xiaozhi-server/core/audio_archive.py
new file mode 100644
index 0000000..7e5e701
--- /dev/null
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/core/audio_archive.py
@@ -0,0 +1,103 @@
+"""Persist user/agent audio by device, scene mode, and time for later replay."""
+
+from __future__ import annotations
+
+import os
+import time
+from typing import Any, Optional
+
+from config.config_loader import get_project_dir
+from config.logger import setup_logging
+
+TAG = __name__
+logger = setup_logging()
+
+MODE_DIR = {
+ "CHAT": "chat",
+ "SHADOWING": "shadowing",
+ "PLAY_AUDIO": "repeater",
+ "TRANSLATION": "translation",
+ "BILINGUAL": "bilingual",
+}
+
+
+def _safe_segment(value: Any, fallback: str = "unknown") -> str:
+ raw = str(value or "").strip() or fallback
+ out = []
+ for ch in raw:
+ if ch.isalnum() or ch in "-_":
+ out.append(ch)
+ elif ch in (":", " ", "."):
+ out.append("-")
+ else:
+ out.append("_")
+ text = "".join(out).strip("-_") or fallback
+ return text[:80]
+
+
+def _mode_dir(conn: Any) -> str:
+ mode = str(getattr(conn, "interaction_mode", "") or "").strip().upper()
+ return MODE_DIR.get(mode, _safe_segment(mode.lower() or "unknown"))
+
+
+def _archive_cfg(conn: Any) -> dict:
+ return dict(((getattr(conn, "config", None) or {}).get("audio_archive") or {}))
+
+
+def archive_enabled(conn: Any = None) -> bool:
+ cfg = _archive_cfg(conn)
+ if cfg.get("enabled") is False:
+ return False
+ return True
+
+
+def archive_root(conn: Any = None) -> str:
+ cfg = _archive_cfg(conn)
+ rel = str(cfg.get("dir") or "data/recordings")
+ if os.path.isabs(rel):
+ return rel
+ return os.path.join(get_project_dir(), rel)
+
+
+def archive_wav(
+ conn: Any,
+ wav_bytes: bytes,
+ *,
+ kind: str = "asr",
+ extra: str = "",
+) -> Optional[str]:
+ """Write WAV under recordings/{device}/{mode}/{date}/{time}_{kind}_{extra}.wav."""
+ if not wav_bytes or not archive_enabled(conn):
+ return None
+ try:
+ now = time.time()
+ local = time.localtime(now)
+ day = time.strftime("%Y-%m-%d", local)
+ stamp = time.strftime("%H%M%S", local) + f"{int((now % 1) * 1000):03d}"
+ device = _safe_segment(getattr(conn, "device_id", "") or "", "unknown")
+ mode = _mode_dir(conn)
+ parts = [stamp, _safe_segment(kind, "asr")]
+ extra_part = _safe_segment(extra, "") if extra else ""
+ if extra_part:
+ parts.append(extra_part)
+ filename = "_".join(parts) + ".wav"
+ directory = os.path.join(archive_root(conn), device, mode, day)
+ os.makedirs(directory, exist_ok=True)
+ path = os.path.join(directory, filename)
+ with open(path, "wb") as handle:
+ handle.write(wav_bytes)
+ relative = os.path.join(device, mode, day, filename).replace("\\", "/")
+ logger.bind(tag=TAG).info(f"闊抽褰掓。 {relative} bytes={len(wav_bytes)}")
+ return relative
+ except Exception as exc:
+ logger.bind(tag=TAG).warning(f"闊抽褰掓。澶辫触: {exc}")
+ return None
+
+
+def archive_pcm_snapshot(conn: Any, snapshot: Any, *, extra: str = "") -> Optional[str]:
+ from core.app_runtime.assessment import pcm_snapshot_to_wav
+
+ wav_bytes = pcm_snapshot_to_wav(snapshot)
+ if not wav_bytes:
+ return None
+ return archive_wav(conn, wav_bytes, kind="asr", extra=extra)
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/core/connection.py b/xiaozhi-esp32-server/main/xiaozhi-server/core/connection.py
index 6e5f417..69a53f3 100644
--- a/xiaozhi-esp32-server/main/xiaozhi-server/core/connection.py
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/core/connection.py
@@ -45,6 +45,7 @@
from core.utils.voiceprint_provider import VoiceprintProvider
from core.utils.util import get_system_error_response
from core.utils import textUtils
+from core.websocket_handshake import is_client_hello, is_priority_listen_detect
TAG = __name__
@@ -122,6 +123,7 @@
self.loop = None # 鍦� handle_connection 涓幏鍙栬繍琛屼腑鐨勪簨浠跺惊鐜�
self.stop_event = threading.Event()
self.executor = ThreadPoolExecutor(max_workers=5)
+ self._tts_channel_lock = threading.Lock()
# 娣诲姞涓婃姤绾跨▼姹�
self.report_queue = queue.Queue()
@@ -209,6 +211,18 @@
self.tool_allowlist = []
self.original_system_prompt = None
self.awaiting_voice_app_choice = False
+ self.shadowing_index = 0
+ self.shadowing_prompt = ""
+ self.shadowing_clips = []
+ self.shadowing_awaiting_user = False
+ self.shadowing_clip_gen = 0
+ self.shadowing_timeout_task = None
+ self.shadowing_assess_index = 0
+ self.last_asr_language = None
+ self.task_session = None
+ self.last_asr_audio_snapshot = None
+ self.shadowing_listen_pcm = []
+ self.shadowing_audio_saved_index = -1
async def handle_connection(self, ws: websockets.ServerConnection):
try:
@@ -232,6 +246,8 @@
# 璁よ瘉閫氳繃,缁х画澶勭悊
self.websocket = ws
+ if self.server is not None:
+ await self.server.register_device_handler(self.device_id, self)
# 妫�鏌ユ槸鍚︽潵鑷狹QTT杩炴帴
request_path = ws.request.path
@@ -249,7 +265,7 @@
# 鍚姩AEC缂撳瓨娓呯悊浠诲姟
self._aec_cache_cleanup_task = asyncio.create_task(self._check_aec_cache_expiry())
- self.welcome_msg = self.config["xiaozhi"]
+ self.welcome_msg = copy.deepcopy(self.config.get("xiaozhi") or {})
self.welcome_msg["session_id"] = self.session_id
# 浠庨厤缃腑璇诲彇閲囨牱鐜�
@@ -354,6 +370,12 @@
async def _route_message(self, message):
"""娑堟伅璺敱"""
+ # hello 蹇呴』绔嬪埢搴旂瓟銆傜粦瀹氱姸鎬佽繕鍦ㄦ媺閰嶇疆鏃跺鏋滀涪鎺� hello锛�
+ # 璁惧浼氬崱鍦� handshake/hello 绛夊緟锛岃〃鐜颁负杩炰笉涓婅闊虫湇鍔°��
+ if is_client_hello(message) or is_priority_listen_detect(message):
+ await handleTextMessage(self, message)
+ return
+
# 妫�鏌ユ槸鍚﹀凡缁忚幏鍙栧埌鐪熷疄鐨勭粦瀹氱姸鎬�
if not self.bind_completed_event.is_set():
# 杩樻病鏈夎幏鍙栧埌鐪熷疄鐘舵�侊紝绛夊緟鐩村埌鑾峰彇鍒扮湡瀹炵姸鎬佹垨瓒呮椂
@@ -610,14 +632,45 @@
)
)
- def _initialize_components(self):
- try:
+ def _tts_config_ready(self) -> bool:
+ selected = (self.config.get("selected_module") or {}).get("TTS")
+ providers = self.config.get("TTS") or {}
+ return bool(selected) and selected in providers
+
+ async def _open_tts_channels(self):
+ """Open TTS worker threads. Must run on the connection event loop after TTS config exists."""
+ if self.tts is None and not self._tts_config_ready():
+ self.logger.bind(tag=TAG).info("TTS閰嶇疆灏氭湭浠庢櫤鎺у彴鍔犺浇锛屽欢鍚庢墦寮�閫氶亾")
+ return
+ with self._tts_channel_lock:
if self.tts is None:
self.tts = self._initialize_tts()
- # 鎵撳紑璇煶鍚堟垚閫氶亾
- asyncio.run_coroutine_threadsafe(
- self.tts.open_audio_channels(self), self.loop
- )
+ tts = self.tts
+ if tts is None:
+ self.logger.bind(tag=TAG).warning("TTS瀹炰緥涓嶅彲鐢紝鏃犳硶鎵撳紑閫氶亾")
+ return
+ await tts.open_audio_channels(self)
+ play = getattr(tts, "audio_play_priority_thread", None)
+ text_th = getattr(tts, "tts_priority_thread", None)
+ if play and play.is_alive() and text_th and text_th.is_alive():
+ self.logger.bind(tag=TAG).info("TTS閫氶亾宸叉墦寮�")
+ else:
+ self.logger.bind(tag=TAG).error("TTS閫氶亾鎵撳紑鍚庡伐浣滅嚎绋嬫湭杩愯")
+
+ def _ensure_tts_channels(self):
+ """Start TTS worker threads once; used after private-config init."""
+ try:
+ if self.loop is None:
+ return
+ fut = asyncio.run_coroutine_threadsafe(self._open_tts_channels(), self.loop)
+ fut.result(timeout=15)
+ except Exception as e:
+ self.logger.bind(tag=TAG).error(f"鎵撳紑TTS閫氶亾澶辫触: {e}")
+
+ def _initialize_components(self):
+ try:
+ if self.tts is not None:
+ self._ensure_tts_channels()
if self.need_bind:
self.bind_completed_event.set()
return
@@ -755,8 +808,11 @@
def _initialize_tts(self):
"""鍒濆鍖朤TS"""
tts = None
- if not self.need_bind:
+ if not self.need_bind and self._tts_config_ready():
tts = initialize_tts(self.config)
+ elif not self._tts_config_ready():
+ self.logger.bind(tag=TAG).info("璺宠繃TTS鍒濆鍖栵細閰嶇疆涓病鏈夊彲鐢ㄧ殑TTS妯″潡")
+ return None
if tts is None:
tts = DefaultTTS(self.config, delete_audio_file=True)
@@ -954,6 +1010,11 @@
modules = {}
if modules.get("tts", None) is not None:
self.tts = modules["tts"]
+ if self.tts is not None:
+ try:
+ await self._open_tts_channels()
+ except Exception as e:
+ self.logger.bind(tag=TAG).error(f"鎵撳紑TTS閫氶亾澶辫触: {e}")
if modules.get("vad", None) is not None:
self.vad = modules["vad"]
if modules.get("asr", None) is not None:
@@ -1534,11 +1595,11 @@
self.logger.bind(tag=TAG).info("鑱婂ぉ璁板綍涓婃姤绾跨▼宸查��鍑�")
- def _process_report(self, type, text, audio_data, report_time):
+ def _process_report(self, type, text, audio_data, report_time, task_instance_id=None, step_index=None):
"""澶勭悊涓婃姤浠诲姟"""
try:
# 鎵ц寮傛涓婃姤锛堝湪浜嬩欢寰幆涓繍琛岋級
- asyncio.run(report(self, type, text, audio_data, report_time))
+ asyncio.run(report(self, type, text, audio_data, report_time, task_instance_id, step_index))
except Exception as e:
self.logger.bind(tag=TAG).error(f"涓婃姤澶勭悊寮傚父: {e}")
finally:
@@ -1552,6 +1613,15 @@
async def close(self, ws=None):
"""璧勬簮娓呯悊鏂规硶"""
try:
+ if self.server is not None:
+ await self.server.unregister_device_handler(self.device_id, self)
+ try:
+ from core.app_runtime.session import end_session
+
+ end_session(self, "disconnected")
+ except Exception as session_error:
+ self.logger.bind(tag=TAG).warning(f"鏂嚎 flush TaskSession 澶辫触: {session_error}")
+
# 娓呯悊 VAD 杩炴帴璧勬簮
if (
hasattr(self, "vad")
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/helloHandle.py b/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/helloHandle.py
index 2f84a7b..60c39a4 100644
--- a/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/helloHandle.py
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/helloHandle.py
@@ -14,6 +14,7 @@
from core.handle.sendAudioHandle import sendAudioMessage, send_tts_message
from core.utils.util import remove_punctuation_and_length, opus_datas_to_wav_bytes
from core.providers.tools.device_mcp import MCPClient, send_mcp_initialize_message
+from core.websocket_handshake import merge_hello_audio_params
TAG = __name__
@@ -46,7 +47,12 @@
format = audio_params.get("format")
conn.logger.bind(tag=TAG).debug(f"瀹㈡埛绔煶棰戞牸寮�: {format}")
conn.audio_format = format
- conn.welcome_msg["audio_params"] = audio_params
+ conn.welcome_msg["audio_params"] = merge_hello_audio_params(
+ conn.welcome_msg.get("audio_params"), audio_params
+ )
+ output_rate = (conn.welcome_msg.get("audio_params") or {}).get("sample_rate")
+ if output_rate:
+ conn.sample_rate = output_rate
features = msg_json.get("features")
if features:
conn.logger.bind(tag=TAG).debug(f"瀹㈡埛绔壒鎬�: {features}")
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/receiveAudioHandle.py b/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/receiveAudioHandle.py
index 2985600..e26fea1 100644
--- a/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/receiveAudioHandle.py
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/receiveAudioHandle.py
@@ -17,16 +17,21 @@
async def handleAudioMessage(conn: "ConnectionHandler", pcm_frame):
# 褰撳墠鐗囨鏄惁鏈変汉璇磋瘽
have_voice = conn.vad.is_vad(conn, pcm_frame)
- # 濡傛灉璁惧鍒氬垰琚敜閱掞紝鐭殏蹇界暐VAD妫�娴�
- if hasattr(conn, "just_woken_up") and conn.just_woken_up:
+ mode = getattr(conn, "interaction_mode", "CHAT")
+ # 璺熻鍚獥蹇呴』鏀堕綈 PCM銆俲ust_woken_up 鐨� 2s 涓㈠抚鍙敤浜庡璇濓紝閬垮厤璺熻鍓嶅嚑鍙ヨ惤鎴愮┖ wav銆�
+ if mode != "SHADOWING" and hasattr(conn, "just_woken_up") and conn.just_woken_up:
have_voice = False
# 璁剧疆涓�涓煭鏆傚欢杩熷悗鎭㈠VAD妫�娴�
if not hasattr(conn, "vad_resume_task") or conn.vad_resume_task.done():
conn.vad_resume_task = asyncio.create_task(resume_vad_detection(conn))
return
- # 鏈嶅姟绔疉EC鍔熻兘闇�瑕佸疄鏃惰Е鍙戞墦鏂�
+ # 鏈嶅姟绔疉EC锛氫粎瀵硅瘽绫绘ā寮忓厑璁告墦鏂�傚璇�/璺熻/缈昏瘧鏃跺枃鍙洖澹颁笉鑳藉綋鎴愭彃璇濄��
if conn.client_aec and have_voice:
- if conn.client_is_speaking and conn.client_listen_mode != "manual":
+ if (
+ conn.client_is_speaking
+ and conn.client_listen_mode != "manual"
+ and mode in ("CHAT", "BILINGUAL")
+ ):
await handleAbortMessage(conn)
# 璁惧闀挎椂闂寸┖闂叉娴嬶紝鐢ㄤ簬say goodbye
await no_voice_close_connect(conn, have_voice)
@@ -44,6 +49,7 @@
# 妫�鏌ヨ緭鍏ユ槸鍚︽槸JSON鏍煎紡锛堝寘鍚璇濅汉淇℃伅锛�
speaker_name = None
actual_text = text
+ conn.last_asr_language = getattr(conn, "last_asr_language", None)
try:
# FunASR 绛変細鎶婄粨鏋滃寘鎴� {"content","language","emotion"}锛涘０绾瑰満鏅繕鏈� speaker
@@ -52,6 +58,8 @@
if isinstance(data, dict) and "content" in data:
speaker_name = data.get("speaker")
actual_content = data.get("content") or ""
+ if data.get("language"):
+ conn.last_asr_language = data.get("language")
if speaker_name:
conn.logger.bind(tag=TAG).info(f"瑙ｆ瀽鍒拌璇濅汉淇℃伅: {speaker_name}")
# 浠呭湪璇ヨ璇濅汉棣栨鍑虹幇鏃朵繚鐣� {"speaker":...} JSON锛岃妯″瀷鑷劧绉板懠涓�娆★紱
@@ -86,15 +94,28 @@
return
from core.app_runtime.runtime import (
- is_exit_translation,
+ detect_user_language,
+ handle_shadowing_turn,
+ is_exit_to_chat,
match_voice_switch,
run_translation,
start_app_task,
- VOICE_MENU,
+ wrap_bilingual_user_text,
+ voice_menu_text,
)
from core.handle.intentHandler import speak_txt
mode = getattr(conn, "interaction_mode", "CHAT")
+
+ # 鑿滃崟 TTS 閲岀殑銆岃窡璇�/澶嶈鏈�/缈昏瘧銆嶄笉鑳藉綋鎴愬垏 App锛涢��鍑鸿亰澶╁彛浠や粛瑕佺敓鏁�
+ if conn.client_is_speaking and getattr(conn, "awaiting_voice_app_choice", False):
+ mapped = match_voice_switch(actual_text)
+ if mapped and mapped[0] == "chat":
+ conn.awaiting_voice_app_choice = False
+ await start_app_task(conn, "chat", "conversation", {}, source="voice")
+ return
+ conn.logger.bind(tag=TAG).info("璇煶鑿滃崟鎾姤涓紝蹇界暐鍒� App")
+ return
if getattr(conn, "awaiting_voice_app_choice", False):
mapped = match_voice_switch(actual_text)
@@ -105,39 +126,50 @@
return
if mapped and mapped[0] == "__menu__":
conn.awaiting_voice_app_choice = True
- speak_txt(conn, VOICE_MENU)
+ speak_txt(conn, voice_menu_text())
return
- if mapped and mapped[0] != "__menu__":
- params = {}
- await start_app_task(conn, mapped[0], mapped[1], params, source="voice")
+ if mapped and mapped[0] == "chat":
+ await start_app_task(conn, "chat", "conversation", {}, source="voice")
return
- speak_txt(conn, VOICE_MENU)
- conn.awaiting_voice_app_choice = True
+ if mapped:
+ await start_app_task(conn, mapped[0], mapped[1], {}, source="voice")
+ return
+ conn.logger.bind(tag=TAG).info("璇煶鑿滃崟鏈尮閰嶏紝淇濇寔 chat")
return
mapped = match_voice_switch(actual_text)
if mapped and mapped[0] == "__menu__":
conn.awaiting_voice_app_choice = True
- speak_txt(conn, VOICE_MENU)
+ speak_txt(conn, voice_menu_text())
return
if mapped and mapped[0] == "__reserved__":
speak_txt(conn, "杩欎釜杩樻病寮�鏀撅紝鍏堢户缁綋鍓嶅唴瀹瑰惂銆�")
return
- if mapped and mapped[0] in ("english_translation", "english_learning", "chat"):
- if mapped[0] == "english_learning" and mapped[1] == "repeater":
- speak_txt(conn, "澶嶈闇�瑕佹寚瀹� SD 鍗′笂鐨勬枃浠跺悕锛屾垨鐢ㄥ皬绋嬪簭寮�濮嬨��")
- return
+ if conn.client_is_speaking and mapped and mapped[0] not in ("chat",):
+ conn.logger.bind(tag=TAG).info(f"TTS 鎾斁涓拷鐣ュ垏 App: {mapped}")
+ return
+ if mapped and mapped[0] in (
+ "english_translation",
+ "english_learning",
+ "english_conversation",
+ "chat",
+ ):
await start_app_task(conn, mapped[0], mapped[1], {}, source="voice")
return
+ if mode != "CHAT" and is_exit_to_chat(actual_text):
+ await start_app_task(conn, "chat", "conversation", {}, source="voice")
+ return
+
if mode == "PLAY_AUDIO":
- conn.logger.bind(tag=TAG).info("PLAY_AUDIO 妯″紡蹇界暐涓婅瀵硅瘽")
+ conn.logger.bind(tag=TAG).info("澶嶈鏈烘ā寮忓拷鐣ヤ笂琛屽璇�")
+ return
+
+ if mode == "SHADOWING":
+ await handle_shadowing_turn(conn, actual_text)
return
if mode == "TRANSLATION":
- if is_exit_translation(actual_text):
- await start_app_task(conn, "chat", "conversation", {}, source="voice")
- return
await send_stt_message(conn, actual_text)
conn.client_abort = False
translated = await run_translation(conn, actual_text)
@@ -149,6 +181,16 @@
speak_txt(conn, translated)
return
+ if mode == "BILINGUAL":
+ await send_stt_message(conn, actual_text)
+ conn.client_abort = False
+ user_lang = detect_user_language(
+ actual_text, getattr(conn, "last_asr_language", None)
+ )
+ wrapped = wrap_bilingual_user_text(actual_text, user_lang)
+ conn.executor.submit(conn.chat, wrapped)
+ return
+
# manual 妯″紡涓嬩笉鎵撴柇姝ｅ湪鎾斁鐨勫唴瀹�
if conn.client_is_speaking and conn.client_listen_mode != "manual":
await handleAbortMessage(conn)
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/reportHandle.py b/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/reportHandle.py
index 15beb47..100094d 100644
--- a/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/reportHandle.py
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/reportHandle.py
@@ -22,7 +22,8 @@
TAG = __name__
-async def report(conn: "ConnectionHandler", chat_type, text, audio_data, report_time):
+async def report(conn: "ConnectionHandler", chat_type, text, audio_data, report_time,
+ task_instance_id=None, step_index=None):
"""鎵ц鑱婂ぉ璁板綍涓婃姤鎿嶄綔
Args:
@@ -43,6 +44,13 @@
wav_data = None
else:
wav_data = None
+ if wav_data and chat_type == 2:
+ try:
+ from core.audio_archive import archive_wav
+
+ archive_wav(conn, wav_data, kind="tts")
+ except Exception:
+ pass
# 鎵ц寮傛涓婃姤
await manage_report(
mac_address=conn.device_id,
@@ -51,6 +59,8 @@
content=text,
audio=wav_data,
report_time=report_time,
+ task_instance_id=task_instance_id,
+ step_index=step_index,
)
except Exception as e:
conn.logger.bind(tag=TAG).error(f"鑱婂ぉ璁板綍涓婃姤澶辫触: {e}")
@@ -224,7 +234,7 @@
conn.logger.bind(tag=TAG).error(f"鍔犲叆宸ュ叿涓婃姤闃熷垪澶辫触: {e}")
-def enqueue_asr_report(conn: "ConnectionHandler", text, opus_data):
+def enqueue_asr_report(conn: "ConnectionHandler", text, opus_data, task_instance_id=None, step_index=None):
"""灏咥SR鏁版嵁鍔犲叆涓婃姤闃熷垪
Args:
@@ -239,12 +249,12 @@
try:
# 浣跨敤杩炴帴瀵硅薄鐨勯槦鍒楋紝浼犲叆鏂囨湰鍜屼簩杩涘埗鏁版嵁鑰岄潪鏂囦欢璺緞
if conn.chat_history_conf == 2:
- conn.report_queue.put((1, text, opus_data, int(time.time() * 1000)))
+ conn.report_queue.put((1, text, opus_data, int(time.time() * 1000), task_instance_id, step_index))
conn.logger.bind(tag=TAG).debug(
f"ASR鏁版嵁宸插姞鍏ヤ笂鎶ラ槦鍒�: {conn.device_id}, 闊抽澶у皬: {len(opus_data)} "
)
else:
- conn.report_queue.put((1, text, None, int(time.time() * 1000)))
+ conn.report_queue.put((1, text, None, int(time.time() * 1000), task_instance_id, step_index))
conn.logger.bind(tag=TAG).debug(
f"ASR鏁版嵁宸插姞鍏ヤ笂鎶ラ槦鍒�: {conn.device_id}, 涓嶄笂鎶ラ煶棰�"
)
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/sendAudioHandle.py b/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/sendAudioHandle.py
index 421554d..c2ecdb9 100644
--- a/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/sendAudioHandle.py
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/sendAudioHandle.py
@@ -9,7 +9,7 @@
from core.utils import textUtils
from core.utils.util import audio_to_data
from core.providers.tts.dto.dto import SentenceType
-from core.utils.audioRateController import AudioRateController
+from core.utils.audioRateController import AudioRateController, audio_completion_timeout_sec
TAG = __name__
# 闊抽甯ф椂闀匡紙姣锛�
@@ -21,11 +21,16 @@
async def sendAudioMessage(conn: "ConnectionHandler", sentenceType, audios, text, sentence_id=None):
# 璺宠繃鏃у彞瀛愭畫鐣欓煶棰�
if sentence_id is not None and sentence_id != conn.sentence_id:
+ conn.logger.bind(tag=TAG).warning(
+ f"涓㈠純杩囨湡TTS闊抽 sentence={sentence_id} current={conn.sentence_id} type={sentenceType}"
+ )
return
if conn.tts.tts_audio_first_sentence:
conn.logger.bind(tag=TAG).info(f"鍙戦�佺涓�娈佃闊�: {text}")
conn.tts.tts_audio_first_sentence = False
+ # speak_txt 涓嶈蛋 STT锛岃ˉ涓�鏉� tts start锛岃璁惧鍏堣繘鍏� speaking 鍐嶆敹 opus銆�
+ await send_tts_message(conn, "start")
if sentenceType == SentenceType.FIRST:
# 鍚屼竴鍙ュ瓙鐨勫悗缁秷鎭姞鍏ユ祦鎺ч槦鍒楋紝鍏朵粬鎯呭喌绔嬪嵆鍙戦��
@@ -45,7 +50,10 @@
await sendAudio(conn, audios)
# 鍙戦�佸彞瀛愬紑濮嬫秷鎭�
if sentenceType is not SentenceType.MIDDLE:
- conn.logger.bind(tag=TAG).info(f"鍙戦�侀煶棰戞秷鎭�: {sentenceType}, {text}")
+ sent = (getattr(conn, "audio_flow_control", None) or {}).get("packet_count", 0)
+ conn.logger.bind(tag=TAG).info(
+ f"鍙戦�侀煶棰戞秷鎭�: {sentenceType}, {text}, opus鍖�={sent}"
+ )
# 鍙戦�佺粨鏉熸秷鎭紙濡傛灉鏄渶鍚庝竴涓枃鏈級
# 閫氳瘽闇�瑕佺淮鎸乻peaking鐘舵��
@@ -58,19 +66,22 @@
async def _wait_for_audio_completion(conn: "ConnectionHandler"):
"""
绛夊緟闊抽闃熷垪娓呯┖骞剁瓑寰呴缂撳啿鍖呮挱鏀惧畬鎴�
-
- Args:
- conn: 杩炴帴瀵硅薄
"""
if hasattr(conn, "audio_rate_controller") and conn.audio_rate_controller:
rate_controller = conn.audio_rate_controller
- conn.logger.bind(tag=TAG).debug(
- f"绛夊緟闊抽鍙戦�佸畬鎴愶紝闃熷垪涓繕鏈� {len(rate_controller.queue)} 涓寘"
- )
- await rate_controller.queue_empty_event.wait()
+ try:
+ queued = len(rate_controller.queue)
+ timeout = audio_completion_timeout_sec(queued, rate_controller.frame_duration)
+ conn.logger.bind(tag=TAG).debug(
+ f"绛夊緟闊抽鍙戦�佸畬鎴愶紝闃熷垪涓繕鏈� {queued} 涓寘锛宼imeout={timeout:.1f}s"
+ )
+ await asyncio.wait_for(rate_controller.queue_empty_event.wait(), timeout=timeout)
+ except asyncio.TimeoutError:
+ leftover = len(rate_controller.queue) if rate_controller else -1
+ conn.logger.bind(tag=TAG).warning(
+ f"绛夊緟闊抽鍙戦�佸畬鎴愯秴鏃� leftover={leftover}锛岀户缁笅鍙� tts stop"
+ )
- # 绛夊緟棰勭紦鍐插寘鎾斁瀹屾垚
- # 鍓峃涓寘鐩存帴鍙戦�侊紝澧炲姞2涓綉缁滄姈鍔ㄥ寘锛岄渶瑕侀澶栫瓑寰呭畠浠湪瀹㈡埛绔挱鏀惧畬鎴�
frame_duration_ms = rate_controller.frame_duration
pre_buffer_playback_time = (PRE_BUFFER_COUNT + 2) * frame_duration_ms / 1000.0
await asyncio.sleep(pre_buffer_playback_time)
@@ -263,12 +274,12 @@
sequence = flow_control.get("sequence", 0)
if conn.conn_from_mqtt_gateway:
- # 璁＄畻鏃堕棿鎴筹紙鍩轰簬鎾斁浣嶇疆锛�
start_time = time.time()
timestamp = int(start_time * 1000) % (2**32)
await _send_to_mqtt_gateway(conn, opus_packet, timestamp, sequence)
else:
- # 鐩存帴鍙戦�乷pus鏁版嵁鍖�
+ if packet_index == 0:
+ conn.logger.bind(tag=TAG).info(f"寮�濮嬩笅鍙慣TS opus锛岄鍖卋ytes={len(opus_packet)}")
await conn.websocket.send(opus_packet)
# 鏇存柊娴佹帶鐘舵��
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/textHandler/listenMessageHandler.py b/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/textHandler/listenMessageHandler.py
index 5952f75..8d1ced9 100644
--- a/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/textHandler/listenMessageHandler.py
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/textHandler/listenMessageHandler.py
@@ -35,24 +35,46 @@
if msg_json["state"] == "start":
# 璁惧浠庢挱鏀炬ā寮忓垏鍥炲綍闊虫ā寮�,娓呴櫎鎵�鏈夐煶棰戠姸鎬佸拰缂撳啿鍖�
conn.reset_audio_states()
+ from core.app_runtime.runtime import maybe_flush_pending_play_after_tts
+
+ await maybe_flush_pending_play_after_tts(conn)
elif msg_json["state"] == "stop":
# 鏀跺埌stop浣哸sr鏈垵濮嬪寲锛岃烦杩囧鐞�
if conn.asr is None:
return
conn.client_voice_stop = True
- if conn.asr.interface_type == InterfaceType.STREAM:
- # 娴佸紡妯″紡涓嬶紝鍙戦�佺粨鏉熻姹�
- asyncio.create_task(conn.asr._send_stop_request())
- else:
- # 闈炴祦寮忔ā寮忥細鐩存帴瑙﹀彂ASR璇嗗埆
- if len(conn.asr_audio) > 0:
- asr_audio_task = conn.asr_audio.copy()
- conn.reset_audio_states()
+ from core.app_runtime.runtime import close_shadowing_listen_window, should_skip_listen_stop_asr
- if len(asr_audio_task) > 0:
- await conn.asr.handle_voice_stop(conn, asr_audio_task)
+ mode = getattr(conn, "interaction_mode", "CHAT")
+ if mode == "SHADOWING":
+ await close_shadowing_listen_window(conn)
+ if should_skip_listen_stop_asr(conn):
+ conn.reset_audio_states()
+ return
+ if conn.asr.interface_type == InterfaceType.STREAM:
+ asyncio.create_task(conn.asr._send_stop_request())
+ elif len(conn.asr_audio) > 0:
+ asr_audio_task = conn.asr_audio.copy()
+ conn.reset_audio_states()
+ if not asr_audio_task:
+ return
+ if mode == "SHADOWING":
+ asyncio.create_task(conn.asr.handle_voice_stop(conn, asr_audio_task))
+ else:
+ await conn.asr.handle_voice_stop(conn, asr_audio_task)
elif msg_json["state"] == "detect":
+ original_text = msg_json.get("text") or ""
+ # 璇炬枃鎾畬浜嬩欢鍏堣惤鐩樹笂涓�鍙ワ紝鍐嶆竻 ASR 缂撳啿锛岄伩鍏嶆妸涓婁竴鍙ュ惉绐楁姽鎺夋垨骞惰繘涓嬩竴鍙ャ��
+ if original_text.startswith("[skill_event]"):
+ conn.last_activity_time = time.time() * 1000
+ from core.app_runtime.runtime import handle_skill_event
+
+ await handle_skill_event(conn, original_text)
+ conn.client_have_voice = False
+ conn.reset_audio_states()
+ return
+
conn.client_have_voice = False
conn.reset_audio_states()
if "text" in msg_json:
@@ -72,23 +94,33 @@
conn.logger.bind(tag=TAG).info(
f"鏀跺埌 app_task: {app_id}/{task_id} source={source}"
)
- from core.app_runtime.runtime import start_app_task
+ from core.app_runtime.runtime import start_app_task, wait_for_tts_channels
- start_time = time.time()
- while time.time() - start_time < 3:
- if conn.tts:
- break
- await asyncio.sleep(0.1)
- await start_app_task(conn, app_id, task_id, {}, source=source, execute_skills=False)
+ if not await wait_for_tts_channels(conn, timeout=8):
+ conn.logger.bind(tag=TAG).warning(
+ "TTS閫氶亾鏈氨缁紝浠嶅皾璇曞惎鍔� app_task"
+ )
+ # 鍙ｄ护浠庤澶囦笂鏉ワ細鍥轰欢鏈夌綉鏃朵笉鍐嶅厛鎾� MP3锛岀敱 Server 鍏堟姤杩涘満璇嶅啀鏀鹃煶棰戙��
+ await start_app_task(
+ conn,
+ app_id,
+ task_id,
+ {},
+ source=source,
+ execute_skills=(source == "voice"),
+ )
return
if original_text.startswith("[voice_entry]"):
- from core.app_runtime.runtime import VOICE_MENU
+ from core.app_runtime.runtime import voice_menu_text, wait_for_tts_channels
from core.handle.intentHandler import speak_txt
conn.awaiting_voice_app_choice = True
conn.sentence_id = uuid.uuid4().hex
- speak_txt(conn, VOICE_MENU)
+ if await wait_for_tts_channels(conn, timeout=8):
+ speak_txt(conn, voice_menu_text())
+ else:
+ conn.logger.bind(tag=TAG).warning("TTS閫氶亾鏈氨缁紝璺宠繃璇煶鑿滃崟")
return
# 妫�鏌ユ槸鍚︽槸璁惧鍛煎彨鎸囦护 [device_call]
@@ -122,6 +154,11 @@
conn.dialogue.put(Message(role="assistant", content=call_text))
return
+ if original_text.startswith("[") and "]" in original_text:
+ prefix = original_text.split("]", 1)[0] + "]"
+ conn.logger.bind(tag=TAG).warning(f"蹇界暐鏈煡 detect 鍓嶇紑: {prefix}")
+ return
+
# 璇嗗埆鏄惁鏄敜閱掕瘝
is_wakeup_words = filtered_text in conn.config.get("wakeup_words")
# 鏄惁寮�鍚敜閱掕瘝鍥炲
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/core/providers/asr/base.py b/xiaozhi-esp32-server/main/xiaozhi-server/core/providers/asr/base.py
index dcfaf3a..32c79b4 100644
--- a/xiaozhi-esp32-server/main/xiaozhi-server/core/providers/asr/base.py
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/core/providers/asr/base.py
@@ -66,7 +66,16 @@
# 鑷姩/瀹炴椂妯″紡锛氫娇鐢╒AD妫�娴�
conn.asr_audio.append(pcm_frame)
- # 濡傛灉娌℃湁璇煶锛屼笖涔嬪墠涔熸病鏈夊０闊筹紝缂撳瓨閮ㄥ垎闊抽
+ shadowing_listen = (
+ getattr(conn, "interaction_mode", "") == "SHADOWING"
+ and getattr(conn, "shadowing_awaiting_user", False)
+ )
+ if shadowing_listen:
+ from core.app_runtime.runtime import append_shadowing_listen_pcm
+
+ append_shadowing_listen_pcm(conn, pcm_frame)
+
+ # 璺熻鏁村彞褰曢煶璧� shadowing_listen_pcm銆俛sr_audio 浠嶆寜 VAD 瑁佸壀锛岄伩鍏嶈法鍙ョ矘杩炪��
if not audio_have_voice and not conn.client_have_voice:
conn.asr_audio = conn.asr_audio[-10:]
return
@@ -77,7 +86,11 @@
pcm_bytes = b"".join(conn.asr_audio)
# 妫�鏌ユ槸鍚︽湁瓒冲鐨勯煶棰戞暟鎹紙姣忓抚1920瀛楄妭锛�15甯х害28800瀛楄妭锛�
if len(pcm_bytes) > 1920 * 15:
- await self.handle_voice_stop(conn, [pcm_bytes])
+ mode = getattr(conn, "interaction_mode", "CHAT")
+ if mode == "SHADOWING":
+ asyncio.create_task(self.handle_voice_stop(conn, [pcm_bytes]))
+ else:
+ await self.handle_voice_stop(conn, [pcm_bytes])
conn.reset_audio_states()
# 澶勭悊璇煶鍋滄
@@ -163,10 +176,26 @@
text_len, _ = remove_punctuation_and_length(content_for_length_check)
self.stop_ws_connection()
+ audio_snapshot = asr_audio_task.copy() if asr_audio_task else []
+ mode = getattr(conn, "interaction_mode", "")
+ if mode == "SHADOWING":
+ from core.app_runtime.session import stash_asr_audio
+
+ stash_asr_audio(conn, audio_snapshot)
+
if text_len > 0:
- audio_snapshot = asr_audio_task.copy()
- enqueue_asr_report(conn, enhanced_text, audio_snapshot)
- # 浣跨敤鑷畾涔夋ā鍧楄繘琛屼笂鎶�
+ from core.app_runtime.session import should_report_asr_now, stash_asr_audio
+
+ if mode != "SHADOWING":
+ stash_asr_audio(conn, audio_snapshot)
+ if should_report_asr_now(conn):
+ try:
+ from core.audio_archive import archive_pcm_snapshot
+
+ archive_pcm_snapshot(conn, audio_snapshot)
+ except Exception:
+ pass
+ enqueue_asr_report(conn, enhanced_text, audio_snapshot)
await startToChat(conn, enhanced_text)
except Exception as e:
logger.bind(tag=TAG).error(f"澶勭悊璇煶鍋滄澶辫触: {e}")
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/core/providers/tools/device_mcp/mcp_handler.py b/xiaozhi-esp32-server/main/xiaozhi-server/core/providers/tools/device_mcp/mcp_handler.py
index 76994ca..7e12b60 100644
--- a/xiaozhi-esp32-server/main/xiaozhi-server/core/providers/tools/device_mcp/mcp_handler.py
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/core/providers/tools/device_mcp/mcp_handler.py
@@ -216,6 +216,15 @@
if hasattr(conn, "func_handler") and conn.func_handler:
conn.func_handler.tool_manager.refresh_tools()
conn.func_handler.current_support_functions()
+
+ try:
+ from core.app_runtime.runtime import (
+ schedule_ensure_subscribed_app_sd_dirs,
+ )
+
+ schedule_ensure_subscribed_app_sd_dirs(conn)
+ except Exception as e:
+ logger.bind(tag=TAG).warning(f"璋冨害璁㈤槄 App 寤虹洰褰曞け璐�: {e}")
return
# Handle method calls (requests from the client)
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/core/providers/tts/base.py b/xiaozhi-esp32-server/main/xiaozhi-server/core/providers/tts/base.py
index 2ff779c..965b648 100644
--- a/xiaozhi-esp32-server/main/xiaozhi-server/core/providers/tts/base.py
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/core/providers/tts/base.py
@@ -117,8 +117,38 @@
)
def handle_opus(self, opus_data: bytes):
+ self.last_stream_had_audio = True
+ if not getattr(self, "_logged_first_opus", False):
+ self._logged_first_opus = True
+ logger.bind(tag=TAG).info(f"棣栧抚opus宸插叆闃� bytes={len(opus_data)}")
logger.bind(tag=TAG).debug(f"鎺ㄩ�佹暟鎹埌闃熷垪閲岄潰甯ф暟锝烇綖 {len(opus_data)}")
self.tts_audio_queue.put((SentenceType.MIDDLE, opus_data, None, getattr(self, 'current_sentence_id', None)))
+
+ def _run_text_to_speak(self, text, output_file=None):
+ """Generate TTS with a hard timeout so LAST/stop is not blocked forever.
+
+ Edge TTS (and some HTTP TTS APIs) can hang past asyncio.wait_for cancel.
+ Run synthesis in a daemon thread and abandon it if the join times out.
+ """
+ result = {"data": None, "error": None}
+
+ def worker():
+ try:
+ result["data"] = asyncio.run(self.text_to_speak(text, output_file))
+ except Exception as e:
+ result["error"] = e
+
+ worker_thread = threading.Thread(target=worker, name="tts-synth", daemon=True)
+ worker_thread.start()
+ worker_thread.join(timeout=self.tts_timeout)
+ if worker_thread.is_alive():
+ logger.bind(tag=TAG).error(
+ f"TTS timed out after {self.tts_timeout}s: {text}"
+ )
+ raise TimeoutError(f"TTS timed out after {self.tts_timeout}s")
+ if result["error"] is not None:
+ raise result["error"]
+ return result["data"]
def handle_audio_file(self, file_audio: bytes, text):
self.before_stop_play_files.append((file_audio, text))
@@ -130,21 +160,42 @@
# 浣跨敤姝ｅ垯涓�娆℃�ф浛鎹紝閬垮厤閲嶅閬嶅巻鍜岄儴鍒嗗尮閰嶉棶棰�
if self._correct_words_pattern:
text = self._correct_words_pattern.sub(lambda m: self.correct_words[m.group(0)], text)
- max_repeat_time = 5
+ # 鍏堝悎鎴� opus锛屽啀涓嬪彂 sentence_start銆傚厛鍙戝瓧骞曞啀鍚堟垚鏃讹紝璁惧浼氳繘鍏ヨ璇濅腑鍗存敹涓嶅埌闊抽銆�
+ max_repeat_time = 2
+ opus_frames = {"n": 0}
+ pending_opus = []
+ self.last_stream_had_audio = False
+
+ def _count_opus(data):
+ opus_frames["n"] += 1
+ pending_opus.append(bytes(data) if not isinstance(data, bytes) else data)
+
+ def _flush_ready_audio():
+ sid = getattr(self, "current_sentence_id", None)
+ self.tts_audio_queue.put((SentenceType.FIRST, None, original_text, sid))
+ for frame in pending_opus:
+ if opus_handler:
+ opus_handler(frame)
+ else:
+ self.tts_audio_queue.put((SentenceType.MIDDLE, frame, None, sid))
+ self.last_stream_had_audio = True
+ pending_opus.clear()
+
+ counted_handler = _count_opus
if self.delete_audio_file:
# 闇�瑕佸垹闄ゆ枃浠剁殑鐩存帴杞负闊抽鏁版嵁
+ audio_bytes = None
while max_repeat_time > 0:
try:
- audio_bytes = asyncio.run(self.text_to_speak(text, None))
+ audio_bytes = self._run_text_to_speak(text, None)
if audio_bytes:
- # 浣跨敤鍘熷鏂囨湰鐢ㄤ簬鏄剧ず/涓婃姤
- self.tts_audio_queue.put((SentenceType.FIRST, None, original_text, getattr(self, 'current_sentence_id', None)))
+ encode_rate = getattr(self.opus_encoder, "sample_rate", None) or self.conn.sample_rate
audio_bytes_to_data_stream(
audio_bytes,
file_type=self.audio_file_type,
is_opus=True,
- callback=opus_handler,
- sample_rate=self.conn.sample_rate,
+ callback=counted_handler,
+ sample_rate=encode_rate,
opus_encoder=self.opus_encoder,
)
break
@@ -155,9 +206,14 @@
f"璇煶鐢熸垚澶辫触{5 - max_repeat_time + 1}娆�: {original_text}锛岄敊璇�: {e}"
)
max_repeat_time -= 1
- if max_repeat_time > 0:
+ if max_repeat_time > 0 and opus_frames["n"] > 0:
logger.bind(tag=TAG).info(
- f"璇煶鐢熸垚鎴愬姛: {original_text}锛岄噸璇晎5 - max_repeat_time}娆�"
+ f"璇煶鐢熸垚鎴愬姛: {original_text}锛宱pus甯�={opus_frames['n']}锛岄噸璇晎5 - max_repeat_time}娆�"
+ )
+ _flush_ready_audio()
+ elif max_repeat_time > 0:
+ logger.bind(tag=TAG).error(
+ f"璇煶鐢熸垚鏃爋pus甯�: {original_text}锛宐ytes={0 if not audio_bytes else len(audio_bytes)}"
)
else:
logger.bind(tag=TAG).error(
@@ -169,7 +225,7 @@
try:
while not os.path.exists(tmp_file) and max_repeat_time > 0:
try:
- asyncio.run(self.text_to_speak(text, tmp_file))
+ self._run_text_to_speak(text, tmp_file)
except Exception as e:
logger.bind(tag=TAG).warning(
f"璇煶鐢熸垚澶辫触{5 - max_repeat_time + 1}娆�: {original_text}锛岄敊璇�: {e}"
@@ -183,12 +239,17 @@
logger.bind(tag=TAG).info(
f"璇煶鐢熸垚鎴愬姛: {original_text}:{tmp_file}锛岄噸璇晎5 - max_repeat_time}娆�"
)
+ self._process_audio_file_stream(tmp_file, callback=counted_handler)
+ if opus_frames["n"] > 0:
+ _flush_ready_audio()
+ else:
+ logger.bind(tag=TAG).error(
+ f"璇煶鐢熸垚鏃爋pus甯�: {original_text}:{tmp_file}"
+ )
else:
logger.bind(tag=TAG).error(
f"璇煶鐢熸垚澶辫触: {original_text}锛岃妫�鏌ョ綉缁滄垨鏈嶅姟鏄惁姝ｅ父"
)
- self.tts_audio_queue.put((SentenceType.FIRST, None, original_text, getattr(self, 'current_sentence_id', None)))
- self._process_audio_file_stream(tmp_file, callback=opus_handler)
except Exception as e:
logger.bind(tag=TAG).error(f"Failed to generate TTS file: {e}")
return None
@@ -204,7 +265,7 @@
# 闇�瑕佸垹闄ゆ枃浠剁殑鐩存帴杞负闊抽鏁版嵁
while max_repeat_time > 0:
try:
- audio_bytes = asyncio.run(self.text_to_speak(text, None))
+ audio_bytes = self._run_text_to_speak(text, None)
if audio_bytes:
audio_datas = []
audio_bytes_to_data_stream(
@@ -236,7 +297,7 @@
try:
while not os.path.exists(tmp_file) and max_repeat_time > 0:
try:
- asyncio.run(self.text_to_speak(text, tmp_file))
+ self._run_text_to_speak(text, tmp_file)
except Exception as e:
logger.bind(tag=TAG).warning(
f"璇煶鐢熸垚澶辫触{5 - max_repeat_time + 1}娆�: {original_text}锛岄敊璇�: {e}"
@@ -306,24 +367,32 @@
async def open_audio_channels(self, conn):
self.conn = conn
+ if not hasattr(self, "_open_lock"):
+ self._open_lock = threading.Lock()
- # 鏍规嵁conn鐨剆ample_rate鍒涘缓缂栫爜鍣紝濡傛灉瀛愮被宸茬粡鍒涘缓鍒欎笉瑕嗙洊锛圛ndexTTS鎺ュ彛杩斿洖涓�24kHZ-寰呴噸閲囨牱澶勭悊锛�
- if not hasattr(self, 'opus_encoder') or self.opus_encoder is None:
- self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
- sample_rate=conn.sample_rate, channels=1, frame_size_ms=60
+ with self._open_lock:
+ encoder = getattr(self, "opus_encoder", None)
+ encoder_rate = getattr(encoder, "sample_rate", None) if encoder is not None else None
+ if encoder is None or encoder_rate != conn.sample_rate:
+ self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
+ sample_rate=conn.sample_rate, channels=1, frame_size_ms=60
+ )
+
+ if getattr(self, "_channels_opened", False):
+ play = getattr(self, "audio_play_priority_thread", None)
+ text_th = getattr(self, "tts_priority_thread", None)
+ if play and play.is_alive() and text_th and text_th.is_alive():
+ return
+
+ self.tts_priority_thread = threading.Thread(
+ target=self.tts_text_priority_thread, daemon=True
)
-
- # tts 娑堝寲绾跨▼
- self.tts_priority_thread = threading.Thread(
- target=self.tts_text_priority_thread, daemon=True
- )
- self.tts_priority_thread.start()
-
- # 闊抽鎾斁 娑堝寲绾跨▼
- self.audio_play_priority_thread = threading.Thread(
- target=self._audio_play_priority_thread, daemon=True
- )
- self.audio_play_priority_thread.start()
+ self.tts_priority_thread.start()
+ self.audio_play_priority_thread = threading.Thread(
+ target=self._audio_play_priority_thread, daemon=True
+ )
+ self.audio_play_priority_thread.start()
+ self._channels_opened = True
def store_tts_text(self, sentence_id, text):
"""瀛樺偍鎸囧畾 sentence_id 瀵瑰簲鐨勬枃鏈紝鐢ㄤ簬娴佸紡TTS鑾峰彇姝ｇ‘鐨勫瓧骞曟枃鏈�
@@ -385,11 +454,16 @@
self.tts_text_buff = []
self.is_first_sentence = True
self.tts_audio_first_sentence = True
+ self._logged_first_opus = False
+ self.last_stream_had_audio = False
elif ContentType.TEXT == message.content_type:
self.tts_text_buff.append(message.content_detail)
segment_text = self._get_segment_text()
if segment_text:
- self.to_tts_stream(segment_text, opus_handler=self.handle_opus)
+ try:
+ self.to_tts_stream(segment_text, opus_handler=self.handle_opus)
+ except Exception as e:
+ logger.bind(tag=TAG).error(f"TTS鍒嗘鍚堟垚澶辫触: {e}")
elif ContentType.FILE == message.content_type:
self._process_remaining_text_stream(opus_handler=self.handle_opus)
tts_file = message.content_file
@@ -398,10 +472,18 @@
tts_file, callback=self.handle_opus
)
if message.sentence_type == SentenceType.LAST:
- self._process_remaining_text_stream(opus_handler=self.handle_opus)
- self.tts_audio_queue.put(
- (message.sentence_type, [], message.content_detail, message.sentence_id)
- )
+ try:
+ self._process_remaining_text_stream(opus_handler=self.handle_opus)
+ except Exception as e:
+ logger.bind(tag=TAG).error(
+ f"澶勭悊鍓╀綑TTS鏂囨湰澶辫触: {e}",
+ )
+ if getattr(self, "last_stream_had_audio", False):
+ self.tts_audio_queue.put(
+ (message.sentence_type, [], message.content_detail, message.sentence_id)
+ )
+ else:
+ logger.bind(tag=TAG).error("璺宠繃TTS stop锛氭湰鍙ユ病鏈塷pus闊抽")
except queue.Empty:
continue
@@ -462,7 +544,7 @@
sendAudioMessage(self.conn, sentence_type, audio_datas, text, sentence_id),
self.conn.loop,
)
- future.result()
+ future.result(timeout=20)
# 璁板綍杈撳嚭鍜屾姤鍛�
if self.conn.max_output_size > 0 and text:
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/core/utils/audioRateController.py b/xiaozhi-esp32-server/main/xiaozhi-server/core/utils/audioRateController.py
index ef59870..769f983 100644
--- a/xiaozhi-esp32-server/main/xiaozhi-server/core/utils/audioRateController.py
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/core/utils/audioRateController.py
@@ -6,6 +6,30 @@
TAG = __name__
logger = setup_logging()
+AUDIO_FRAME_DURATION = 60
+AUDIO_COMPLETION_TIMEOUT_MIN = 8.0
+AUDIO_COMPLETION_TIMEOUT_MAX = 180.0
+
+
+def audio_completion_timeout_sec(queue_len: int, frame_ms: int = AUDIO_FRAME_DURATION) -> float:
+ """Wait long enough for remaining opus frames, plus a small drain margin."""
+ try:
+ remaining = int(queue_len or 0)
+ except (TypeError, ValueError):
+ remaining = 0
+ if remaining < 0:
+ remaining = 0
+ try:
+ frame = int(frame_ms or AUDIO_FRAME_DURATION)
+ except (TypeError, ValueError):
+ frame = AUDIO_FRAME_DURATION
+ if frame <= 0:
+ frame = AUDIO_FRAME_DURATION
+ return min(
+ AUDIO_COMPLETION_TIMEOUT_MAX,
+ max(AUDIO_COMPLETION_TIMEOUT_MIN, remaining * frame / 1000.0 + AUDIO_COMPLETION_TIMEOUT_MIN),
+ )
+
class AudioRateController:
"""
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/core/utils/modules_initialize.py b/xiaozhi-esp32-server/main/xiaozhi-server/core/utils/modules_initialize.py
index 7c43dfe..b89704d 100644
--- a/xiaozhi-esp32-server/main/xiaozhi-server/core/utils/modules_initialize.py
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/core/utils/modules_initialize.py
@@ -99,8 +99,15 @@
def initialize_tts(config):
- select_tts_module = config["selected_module"]["TTS"]
- tts_config = config["TTS"][select_tts_module].copy()
+ selected = (config.get("selected_module") or {}).get("TTS")
+ providers = config.get("TTS") or {}
+ if not selected or selected not in providers:
+ logger.bind(tag=TAG).warning(
+ f"璺宠繃TTS鍒濆鍖栵細閰嶇疆涓病鏈夊彲鐢ㄧ殑TTS妯″潡 selected={selected}"
+ )
+ return None
+ select_tts_module = selected
+ tts_config = providers[select_tts_module].copy()
tts_config.setdefault("tts_timeout", config.get("tts_timeout", 15))
tts_type = (
select_tts_module
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/core/websocket_handshake.py b/xiaozhi-esp32-server/main/xiaozhi-server/core/websocket_handshake.py
new file mode 100644
index 0000000..23e9d17
--- /dev/null
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/core/websocket_handshake.py
@@ -0,0 +1,56 @@
+"""WebSocket handshake helpers. Keep this module free of ASR/TTS imports."""
+
+import json
+
+
+def is_websocket_upgrade(headers) -> bool:
+ """True when the HTTP request is a WebSocket handshake, not a health check."""
+ if headers is None:
+ return False
+ try:
+ connection_hdr = str(headers.get("connection", "") or "")
+ upgrade_hdr = str(headers.get("upgrade", "") or "")
+ except Exception:
+ return False
+ return "upgrade" in connection_hdr.lower() or "websocket" in upgrade_hdr.lower()
+
+
+def is_client_hello(message) -> bool:
+ """Device hello must be answered even while private config is still loading."""
+ if not isinstance(message, str):
+ return False
+ try:
+ data = json.loads(message)
+ except Exception:
+ return False
+ return isinstance(data, dict) and data.get("type") == "hello"
+
+
+def merge_hello_audio_params(welcome_params, client_params) -> dict:
+ """Keep server TTS output rate. Device hello sample_rate is capture-only."""
+ merged = dict(welcome_params or {})
+ if not isinstance(client_params, dict):
+ return merged
+ fmt = client_params.get("format")
+ if fmt:
+ merged["format"] = fmt
+ channels = client_params.get("channels")
+ if channels:
+ merged["channels"] = channels
+ return merged
+
+
+def is_priority_listen_detect(message) -> bool:
+ """Boot/app-task detect must not be dropped while bind config is still loading."""
+ if not isinstance(message, str):
+ return False
+ try:
+ data = json.loads(message)
+ except Exception:
+ return False
+ if not isinstance(data, dict) or data.get("type") != "listen":
+ return False
+ if data.get("state") != "detect":
+ return False
+ text = str(data.get("text") or "")
+ return text.startswith("[app_task]") or text.startswith("[voice_entry]") or text.startswith("[skill_event]")
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/core/websocket_server.py b/xiaozhi-esp32-server/main/xiaozhi-server/core/websocket_server.py
index 2ef7cda..bee13d1 100644
--- a/xiaozhi-esp32-server/main/xiaozhi-server/core/websocket_server.py
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/core/websocket_server.py
@@ -35,6 +35,7 @@
from core.auth import AuthManager, AuthenticationError
from core.utils.modules_initialize import initialize_modules
from core.utils.util import check_vad_update, check_asr_update
+from core.websocket_handshake import is_websocket_upgrade
TAG = __name__
@@ -67,6 +68,33 @@
secret_key = self.config["server"]["auth_key"]
expire_seconds = auth_config.get("expire_seconds", None)
self.auth = AuthManager(secret_key=secret_key, expire_seconds=expire_seconds)
+ self.device_handlers = {}
+ self.device_handlers_lock = asyncio.Lock()
+
+ async def register_device_handler(self, device_id, handler):
+ if not device_id:
+ return
+ async with self.device_handlers_lock:
+ old = self.device_handlers.get(device_id)
+ self.device_handlers[device_id] = handler
+ if old is not None and old is not handler:
+ self.logger.bind(tag=TAG).info(f"鍏抽棴鍚岃澶囨棫杩炴帴: {device_id}")
+ asyncio.create_task(self._close_stale_handler(old))
+
+ async def _close_stale_handler(self, handler):
+ try:
+ handler.stop_event.set()
+ if handler.websocket:
+ await handler.websocket.close()
+ except Exception as e:
+ self.logger.bind(tag=TAG).warning(f"鍏抽棴鏃ц繛鎺ュけ璐�: {e}")
+
+ async def unregister_device_handler(self, device_id, handler):
+ if not device_id:
+ return
+ async with self.device_handlers_lock:
+ if self.device_handlers.get(device_id) is handler:
+ self.device_handlers.pop(device_id, None)
async def start(self):
server_config = self.config["server"]
@@ -144,13 +172,10 @@
)
async def _http_response(self, websocket, request_headers):
- # 妫�鏌ユ槸鍚︿负 WebSocket 鍗囩骇璇锋眰
- if request_headers.headers.get("connection", "").lower() == "upgrade":
- # 濡傛灉鏄� WebSocket 璇锋眰锛岃繑鍥� None 鍏佽鎻℃墜缁х画
+ headers = getattr(request_headers, "headers", None) or request_headers
+ if is_websocket_upgrade(headers):
return None
- else:
- # 濡傛灉鏄櫘閫� HTTP 璇锋眰锛岃繑鍥� "server is running"
- return websocket.respond(200, "Server is running\n")
+ return websocket.respond(200, "Server is running\n")
async def update_config(self) -> bool:
"""鏇存柊鏈嶅姟鍣ㄩ厤缃苟閲嶆柊鍒濆鍖栫粍浠�
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/plugins_func/functions/start_repeater.py b/xiaozhi-esp32-server/main/xiaozhi-server/plugins_func/functions/start_repeater.py
index 58d2a1d..7a1e0e6 100644
--- a/xiaozhi-esp32-server/main/xiaozhi-server/plugins_func/functions/start_repeater.py
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/plugins_func/functions/start_repeater.py
@@ -9,24 +9,28 @@
"type": "function",
"function": {
"name": "start_repeater",
- "description": "鍦ㄨ澶� SD 鍗′笂澶嶈鎸囧畾 MP3銆俷ame 蹇呴』涓� self.mp3.get_list 杩斿洖鐨� files[].name 瀹屽叏涓�鑷淬�俽epeat 鏄暣棣栧惊鐜鏁般�備笉瑕佺敤 play_music銆�",
+ "description": "鎵撳紑澶嶈鏈猴紝鍦ㄨ澶� SD 鍗′笂寰幆鎾斁 MP3銆俷ame 涓� self.mp3.get_list 鐨� files[].name 涓�鑷达紱鐪佺暐 name 鏃惰澶囦細鎾斁涓婃闊抽鎴� SD 鍗＄涓�棣栥�俽epeat 鏄暣棣栧惊鐜鏁般�俿peed 鏄挱鏀惧�嶇巼 0.5锝�2.0锛岄粯璁ょ敱闊抽鐔熸倝澶╂暟鍐冲畾銆備笉瑕佺敤 play_music銆傛挱鏀炬湡闂翠笉浼氳亰澶┿��",
"parameters": {
"type": "object",
"properties": {
"name": {
"type": "string",
- "description": "SD 鍗� MP3 鏂囦欢鍚嶏紝蹇呴』涓� get_list 鐨� files[].name 涓�鑷�",
+ "description": "鍙�夈�係D 鍗� MP3 鏂囦欢鍚嶏紝蹇呴』涓� get_list 鐨� files[].name 涓�鑷达紱鐪佺暐鍒欐挱涓婃鎴栫涓�棣�",
},
"repeat": {
"type": "integer",
- "description": "鏁撮寰幆娆℃暟锛岄粯璁� 1",
+ "description": "鏁撮寰幆娆℃暟锛岄粯璁� 1銆傚ぇ浜� 1 涓旈厤缃簡 repeat_speeds 鏃讹紝姣忎竴閬嶅彲浠ョ敤涓嶅悓鍊嶉��",
+ },
+ "speed": {
+ "type": "number",
+ "description": "鍙�夈�傛挱鏀惧�嶇巼 0.5锝�2.0锛岀渷鐣ュ垯鎸夌啛鎮夊ぉ鏁拌嚜鍔ㄩ�夋嫨",
},
"power_off_after": {
"type": "boolean",
"description": "鎾畬鍚庢槸鍚﹁蒋鍏虫満锛岄粯璁� false",
},
},
- "required": ["name"],
+ "required": [],
},
},
}
@@ -35,9 +39,10 @@
@register_function("start_repeater", start_repeater_function_desc, ToolType.SYSTEM_CTL)
async def start_repeater(
conn: "ConnectionHandler",
- name: str,
+ name: str = "",
repeat: int = 1,
power_off_after: bool = False,
+ speed: float = None,
):
from core.app_runtime.runtime import start_app_task
@@ -48,15 +53,19 @@
if repeat_n < 1:
repeat_n = 1
+ params = {
+ "name": name,
+ "repeat": repeat_n,
+ "power_off_after": bool(power_off_after),
+ }
+ if speed is not None and speed != "":
+ params["speed"] = speed
+
result = await start_app_task(
conn,
"english_learning",
"repeater",
- {
- "name": name,
- "repeat": repeat_n,
- "power_off_after": bool(power_off_after),
- },
+ params,
source="voice",
)
if result.action in (Action.ERROR, Action.RESPONSE):
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/plugins_func/functions/start_shadowing.py b/xiaozhi-esp32-server/main/xiaozhi-server/plugins_func/functions/start_shadowing.py
new file mode 100644
index 0000000..5d97816
--- /dev/null
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/plugins_func/functions/start_shadowing.py
@@ -0,0 +1,42 @@
+from typing import TYPE_CHECKING
+
+from plugins_func.register import register_function, ToolType, ActionResponse, Action
+
+if TYPE_CHECKING:
+ from core.connection import ConnectionHandler
+
+start_shadowing_function_desc = {
+ "type": "function",
+ "function": {
+ "name": "start_shadowing",
+ "description": "寮�濮嬭窡璇伙細璁惧鐢ㄨ闊宠涓�鍙ヨ嫳璇紝鐢ㄦ埛璺熺潃璇伙紝褰曢煶浼氫笂鎶ワ紝浣嗕笉浼氳繘鍏ヨ亰澶┿�傜敤鎴疯閫�鍑鸿窡璇绘垨鍥炲埌鑱婂ぉ鏃剁粨鏉熴��",
+ "parameters": {
+ "type": "object",
+ "properties": {},
+ },
+ },
+}
+
+
+@register_function("start_shadowing", start_shadowing_function_desc, ToolType.SYSTEM_CTL)
+async def start_shadowing(conn: "ConnectionHandler"):
+ from core.app_runtime.runtime import start_app_task
+
+ result = await start_app_task(
+ conn,
+ "english_learning",
+ "shadowing",
+ {},
+ source="voice",
+ )
+ if result.action in (Action.ERROR, Action.RESPONSE):
+ return ActionResponse(
+ action=Action.RESPONSE,
+ result=result.result,
+ response=result.response or "鏃犳硶寮�濮嬭窡璇�",
+ )
+ return ActionResponse(
+ action=Action.NONE,
+ result=result.result,
+ response=result.response,
+ )
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/plugins_func/functions/switch_app.py b/xiaozhi-esp32-server/main/xiaozhi-server/plugins_func/functions/switch_app.py
index 7282138..63993f1 100644
--- a/xiaozhi-esp32-server/main/xiaozhi-server/plugins_func/functions/switch_app.py
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/plugins_func/functions/switch_app.py
@@ -9,14 +9,23 @@
"type": "function",
"function": {
"name": "switch_app",
- "description": "鍒囨崲璁惧褰撳墠 App銆傛墦寮�缈昏瘧銆佹墦寮�鑻辫瀛︿範/澶嶈銆佸洖鍒拌亰澶┿�侀��鍑虹炕璇戞椂璋冪敤銆俛pp_id 鍙兘鏄� chat銆乪nglish_learning銆乪nglish_translation銆�",
+ "description": (
+ "鍒囨崲璁惧褰撳墠 App銆�"
+ "chat=瀵硅瘽锛沞nglish_translation=缈昏瘧锛沞nglish_conversation=涓嫳瀵硅瘽锛�"
+ "english_learning 闇�閰嶅悎 task_id锛歳epeater=澶嶈鏈猴紝shadowing=璺熻銆�"
+ "閫�鍑哄綋鍓嶆ā寮忋�佸洖鍒拌亰澶╂椂 app_id=chat銆�"
+ ),
"parameters": {
"type": "object",
"properties": {
"app_id": {
"type": "string",
- "description": "鐩爣 App锛歝hat锛堣亰澶╋級銆乪nglish_learning锛堝璇伙級銆乪nglish_translation锛堢炕璇戯級",
- }
+ "description": "chat銆乪nglish_learning銆乪nglish_translation銆乪nglish_conversation",
+ },
+ "task_id": {
+ "type": "string",
+ "description": "鍙�夈�俥nglish_learning 涓� repeater 鎴� shadowing锛涘叾瀹� App 鍙┖",
+ },
},
"required": ["app_id"],
},
@@ -25,10 +34,10 @@
@register_function("switch_app", switch_app_function_desc, ToolType.SYSTEM_CTL)
-async def switch_app(conn: "ConnectionHandler", app_id: str):
+async def switch_app(conn: "ConnectionHandler", app_id: str, task_id: str = ""):
from core.app_runtime.runtime import switch_app as runtime_switch
- result = await runtime_switch(conn, app_id, source="voice")
+ result = await runtime_switch(conn, app_id, source="voice", task_id=task_id or "")
if result.action == Action.ERROR:
return ActionResponse(
action=Action.RESPONSE,
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/tests/conftest.py b/xiaozhi-esp32-server/main/xiaozhi-server/tests/conftest.py
new file mode 100644
index 0000000..82a1e76
--- /dev/null
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/tests/conftest.py
@@ -0,0 +1,66 @@
+"""Lightweight stubs so App Runtime tests run without the full server venv."""
+
+import os
+import sys
+from types import ModuleType
+
+ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+if ROOT not in sys.path:
+ sys.path.insert(0, ROOT)
+
+
+class _SilentLogger:
+ def bind(self, **kwargs):
+ return self
+
+ def info(self, *args, **kwargs):
+ return None
+
+ def warning(self, *args, **kwargs):
+ return None
+
+ def error(self, *args, **kwargs):
+ return None
+
+ def debug(self, *args, **kwargs):
+ return None
+
+
+_LOG = _SilentLogger()
+
+
+def _install(name, module):
+ sys.modules.setdefault(name, module)
+
+
+if "config.logger" not in sys.modules:
+ logger_mod = ModuleType("config.logger")
+ logger_mod.setup_logging = lambda *args, **kwargs: _LOG
+ logger_mod.build_module_string = lambda *args, **kwargs: "0" * 14
+ logger_mod.create_connection_logger = lambda *args, **kwargs: _LOG
+ logger_mod.logger = _LOG
+ _install("config.logger", logger_mod)
+
+if "config.config_loader" not in sys.modules:
+ loader_mod = ModuleType("config.config_loader")
+ loader_mod.get_project_dir = lambda: ROOT.replace("\\", "/") + "/"
+ _install("config.config_loader", loader_mod)
+
+
+def _remove_punctuation_and_length(text):
+ full_width = "锛侊紓锛冿紕锛咃紗锛囷紙锛夛紛锛嬶紝锛嶃�傦紡锛氾紱锛滐紳锛烇紵锛狅蓟锛硷冀锛撅伎锝�锝涳綔锝濓綖"
+ half_width = r'!"#$%&\'()*+,-./:;<=>?@[\]^_`{|}~'
+ result = "".join(
+ ch
+ for ch in (text or "")
+ if ch not in full_width and ch not in half_width and ch not in (" ", "銆�")
+ )
+ if result == "Yeah":
+ return 0, ""
+ return len(result), result
+
+
+if "core.utils.util" not in sys.modules:
+ util_mod = ModuleType("core.utils.util")
+ util_mod.remove_punctuation_and_length = _remove_punctuation_and_length
+ _install("core.utils.util", util_mod)
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/tests/test_app_runtime.py b/xiaozhi-esp32-server/main/xiaozhi-server/tests/test_app_runtime.py
new file mode 100644
index 0000000..6e4d490
--- /dev/null
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/tests/test_app_runtime.py
@@ -0,0 +1,474 @@
+"""Phase 2 App Runtime unit tests. No websocket, no Opus."""
+
+from types import SimpleNamespace
+
+from core.app_runtime.runtime import (
+ _plain_user_text,
+ app_sd_layout_paths,
+ is_echo_of_prompt,
+ lesson_mp3_clips,
+ match_voice_switch,
+ repeater_lesson_mp3,
+)
+from core.app_runtime.session import (
+ TaskSession,
+ parse_skill_event,
+ should_report_asr_now,
+)
+
+
+def test_match_voice_switch_shadowing():
+ assert match_voice_switch("寮�濮嬭窡璇�") == ("english_learning", "shadowing")
+
+
+def test_match_voice_switch_menu():
+ assert match_voice_switch("鍒囨崲妯″紡") == ("__menu__", "")
+ assert match_voice_switch("妯″紡鍒囨崲") == ("__menu__", "")
+
+
+def test_match_voice_switch_chat_return():
+ assert match_voice_switch("鎴戜滑鏉ヨ亰澶╁惂") == ("chat", "conversation")
+ assert match_voice_switch("鎴戜滑鑱婂ぉ鍚�") == ("chat", "conversation")
+ assert match_voice_switch("涓嶆兂璺熻浜�") == ("chat", "conversation")
+ assert match_voice_switch("閫�鍑虹炕璇�") == ("chat", "conversation")
+ assert match_voice_switch("杩斿洖鑱婂ぉ") == ("chat", "conversation")
+ assert match_voice_switch("閫�鍑鸿窡璇�") == ("chat", "conversation")
+ assert match_voice_switch("閫�鍑哄璇�") == ("chat", "conversation")
+
+
+def test_enter_prompts_announce_mode_without_exit_echo():
+ from core.app_runtime.registry import get_task
+
+ _, repeater = get_task("english_learning", "repeater")
+ _, shadowing = get_task("english_learning", "shadowing")
+ repeater_prompt = (repeater or {}).get("enter_prompt") or ""
+ shadowing_prompt = (shadowing or {}).get("enter_prompt") or ""
+ assert "鐜板湪宸茬粡鍒囨崲鑷冲璇绘満妯″紡" in repeater_prompt
+ assert "閫�鍑哄璇�" not in repeater_prompt
+ assert "鐜板湪宸茬粡鍒囨崲鑷宠窡璇绘ā寮�" in shadowing_prompt
+ assert "閫�鍑鸿窡璇�" not in shadowing_prompt
+
+
+def test_voice_menu_text_uses_task_menu_names():
+ from core.app_runtime.runtime import voice_menu_text
+
+ menu = voice_menu_text()
+ assert "鍚璇绘満" in menu
+ assert "璺熻" in menu
+ assert "缈昏瘧" in menu
+ assert "涓嫳瀵硅瘽" in menu
+ assert "鎴戜滑鏉ヨ亰澶╁惂" in menu
+
+
+def test_exit_phrases_beat_short_enter_phrases():
+ from core.app_runtime.runtime import is_exit_to_chat
+
+ assert match_voice_switch("璺熻") == ("english_learning", "shadowing")
+ assert match_voice_switch("鍚") == ("english_learning", "shadowing")
+ assert match_voice_switch("鎴戞兂閫�鍑鸿窡璇�") == ("chat", "conversation")
+ assert is_exit_to_chat("閫�鍑虹炕璇�") is True
+ assert is_exit_to_chat("寮�濮嬭窡璇�") is False
+ assert is_exit_to_chat("閫�鍑哄惉璇�") is True
+
+
+def test_new_task_phrases_come_from_registry_only(monkeypatch):
+ from copy import deepcopy
+ from core.app_runtime import registry as reg
+
+ data = deepcopy(reg.get_registry())
+ data["apps"]["story_retell"] = {
+ "name": "鏁呬簨澶嶈堪",
+ "default_task": "play",
+ "tasks": {
+ "play": {
+ "menu_name": "鏁呬簨澶嶈堪",
+ "voice_enter": ["寮�濮嬫晠浜嬪杩�", "鏁呬簨澶嶈堪"],
+ "voice_exit": ["閫�鍑哄杩�"],
+ "enter_prompt": "鐜板湪宸茬粡鍒囨崲鑷虫晠浜嬪杩般��",
+ }
+ },
+ }
+ monkeypatch.setattr(reg, "_REGISTRY", data)
+ monkeypatch.setattr(reg, "_VOICE_INDEX", None)
+ assert match_voice_switch("寮�濮嬫晠浜嬪杩�") == ("story_retell", "play")
+ assert match_voice_switch("閫�鍑哄杩�") == ("chat", "conversation")
+ menu = reg.voice_menu_text()
+ assert "鏁呬簨澶嶈堪" in menu
+
+
+def test_match_voice_switch_select_after_menu():
+ assert match_voice_switch("璺熻") == ("english_learning", "shadowing")
+ assert match_voice_switch("缈昏瘧") == ("english_translation", "live_translate")
+
+
+def test_match_voice_switch_translate():
+ assert match_voice_switch("鎵撳紑缈昏瘧") == ("english_translation", "live_translate")
+
+
+def test_reserved_voice_enter_stays_in_registry():
+ assert match_voice_switch("鎴愯鎺ラ緳") == ("__reserved__", "")
+ assert match_voice_switch("鏁板瓧鑳岃") == ("__reserved__", "")
+
+
+def test_is_echo_of_prompt_same_sentence():
+ prompt = "Hello, how are you today?"
+ assert is_echo_of_prompt(prompt, prompt) is True
+ assert is_echo_of_prompt("unrelated park visit", prompt) is False
+
+
+def test_plain_user_text_extracts_json_content():
+ assert _plain_user_text('{"content":"hello world","language":"en"}') == "hello world"
+ assert _plain_user_text("just text") == "just text"
+
+
+def test_parse_skill_event_completed():
+ event = parse_skill_event(
+ "[skill_event]audio.completed|english_learning|repeater|1234"
+ )
+ assert event is not None
+ assert event["name"] == "audio.completed"
+ assert event["app_id"] == "english_learning"
+ assert event["task_id"] == "repeater"
+ assert event["duration_ms"] == "1234"
+
+
+def test_parse_skill_event_stopped_and_reject_plain():
+ event = parse_skill_event("[skill_event]audio.stopped|english_learning|repeater")
+ assert event["name"] == "audio.stopped"
+ assert parse_skill_event("鎴戜滑鏉ヨ亰澶╁惂") is None
+ assert parse_skill_event("[app_task]chat|conversation|voice") is None
+
+
+def test_task_session_save_load(tmp_path, monkeypatch):
+ monkeypatch.setattr("core.app_runtime.session.session_dir", lambda: str(tmp_path))
+ session = TaskSession(
+ task_session_id="sess-demo",
+ ws_session_id="ws-1",
+ device_id="mac",
+ app_id="english_learning",
+ task_id="shadowing",
+ source="voice",
+ interaction_mode="SHADOWING",
+ started_at=1,
+ status="running",
+ )
+ session.add_turn(0, "Hello, how are you today?", "hello how are you today", False)
+ session.add_turn(1, "I like to read English books.", "i like to read", False)
+ session.finish("finished")
+ session.save()
+
+ loaded = TaskSession.load("sess-demo")
+ assert loaded is not None
+ assert loaded.status == "completed"
+ assert loaded.end_reason == "finished"
+ assert len(loaded.turns) == 2
+ assert loaded.turns[0]["prompt_text"] == "Hello, how are you today?"
+ assert loaded.turns[1]["asr_text"] == "i like to read"
+
+
+def test_should_report_asr_now_defers_play_and_shadowing():
+ assert should_report_asr_now(SimpleNamespace(interaction_mode="CHAT")) is True
+ assert should_report_asr_now(SimpleNamespace(interaction_mode="PLAY_AUDIO")) is False
+ assert should_report_asr_now(SimpleNamespace(interaction_mode="SHADOWING")) is False
+ assert should_report_asr_now(SimpleNamespace(interaction_mode="TRANSLATION")) is True
+
+
+def test_last_shadowing_turn_is_saved_before_chat(tmp_path, monkeypatch):
+ from core.app_runtime.runtime import apply_shadowing_user_text
+ from core.app_runtime.session import begin_session
+
+ monkeypatch.setattr("core.app_runtime.session.session_dir", lambda: str(tmp_path))
+
+ conn = SimpleNamespace(
+ session_id="ws-1",
+ device_id="mac",
+ interaction_mode="SHADOWING",
+ client_is_speaking=False,
+ shadowing_index=0,
+ shadowing_prompt="Hello, how are you today?",
+ shadowing_clips=[],
+ shadowing_awaiting_user=True,
+ config={"app_runtime": {"shadowing_sentences": ["Hello, how are you today?"]}},
+ last_asr_audio_snapshot=[],
+ sentence_id=None,
+ task_session=None,
+ active_app_id="english_learning",
+ current_task_id="shadowing",
+ )
+ begin_session(conn, "english_learning", "shadowing", "voice", "SHADOWING")
+ outcome = apply_shadowing_user_text(conn, "I went to the park yesterday")
+ assert outcome == "recorded"
+ assert conn.shadowing_index == 0
+ loaded = TaskSession.load(conn.task_session.task_session_id)
+ assert loaded.status == "running"
+ assert len(loaded.turns) == 1
+ assert loaded.turns[0]["asr_text"] == "I went to the park yesterday"
+
+
+def test_lesson_mp3_clips_picks_first_lesson_folder():
+ names = [
+ "/english_learning/repeater/lessons/unit01/full.mp3",
+ "/english_learning/shadowing/lessons/unit02/02.mp3",
+ "/english_learning/shadowing/lessons/unit01/02.mp3",
+ "/english_learning/shadowing/lessons/unit01/01.mp3",
+ "/english_learning/shadowing/lessons/unit01/full.mp3",
+ "/english_learning/shadowing/lessons/unit01/audio/03.mp3",
+ "loose.mp3",
+ ]
+ clips = lesson_mp3_clips(names, "english_learning", "shadowing")
+ assert clips == [
+ "/english_learning/shadowing/lessons/unit01/01.mp3",
+ "/english_learning/shadowing/lessons/unit01/02.mp3",
+ "/english_learning/shadowing/lessons/unit01/audio/03.mp3",
+ ]
+ only_unit2 = lesson_mp3_clips(names, "english_learning", "shadowing", "unit02")
+ assert only_unit2 == ["/english_learning/shadowing/lessons/unit02/02.mp3"]
+ assert repeater_lesson_mp3(names, "english_learning", "repeater") == (
+ "/english_learning/repeater/lessons/unit01/full.mp3"
+ )
+ assert repeater_lesson_mp3(names, "english_learning", "shadowing") == (
+ "/english_learning/shadowing/lessons/unit01/full.mp3"
+ )
+
+
+def test_task_session_to_dict_has_phase3_fields():
+ session = TaskSession(
+ task_session_id="abc123",
+ ws_session_id="ws-1",
+ device_id="mac",
+ agent_id="",
+ app_id="english_learning",
+ task_id="shadowing",
+ source="voice",
+ interaction_mode="SHADOWING",
+ started_at=1,
+ status="running",
+ )
+ session.add_turn(0, "/english_learning/shadowing/lessons/unit01/01.mp3", "hello", False)
+ session.add_event("audio.completed", "english_learning", "shadowing")
+ data = session.to_dict()
+ assert data["task_session_id"] == "abc123"
+ assert data["device_id"] == "mac"
+ assert "agent_id" in data
+ assert data["turns"][0]["prompt_text"].endswith("01.mp3")
+ assert data["events"][0]["name"] == "audio.completed"
+
+
+def test_save_survives_upsert_scheduler_error(tmp_path, monkeypatch):
+ monkeypatch.setattr("core.app_runtime.session.session_dir", lambda: str(tmp_path))
+
+ def boom(_payload):
+ raise RuntimeError("manage_api down")
+
+ monkeypatch.setattr("core.app_runtime.session._schedule_upsert", boom)
+ session = TaskSession(
+ task_session_id="sess-upsert-fail",
+ ws_session_id="ws-1",
+ device_id="mac",
+ app_id="english_learning",
+ task_id="shadowing",
+ source="voice",
+ interaction_mode="SHADOWING",
+ started_at=1,
+ status="running",
+ )
+ session.add_turn(0, "Hello", "hello", False)
+ session.finish("finished")
+ session.save()
+ loaded = TaskSession.load("sess-upsert-fail")
+ assert loaded is not None
+ assert loaded.status == "completed"
+ assert len(loaded.turns) == 1
+
+
+def test_repeater_queues_play_until_listen_start(monkeypatch):
+ import asyncio
+ from core.app_runtime.runtime import maybe_flush_pending_play_after_tts, start_app_task
+
+ plays = []
+
+ async def fake_play(conn, params):
+ plays.append(params)
+ return None
+
+ async def fake_dirs(conn, app_id):
+ return None
+
+ async def fake_list(conn):
+ return [
+ "/english_learning/shadowing/lessons/unit01/full.mp3",
+ "/english_learning/repeater/lessons/unit01/full.mp3",
+ "/english_learning/shadowing/lessons/unit01/01.mp3",
+ ]
+
+ monkeypatch.setattr("core.app_runtime.runtime._execute_named_tool_play", fake_play)
+ monkeypatch.setattr("core.app_runtime.runtime.ensure_app_sd_dirs", fake_dirs)
+ monkeypatch.setattr("core.app_runtime.runtime._list_sd_mp3_names", fake_list)
+ spoken = []
+ monkeypatch.setattr("core.app_runtime.runtime._speak", lambda conn, text: spoken.append(text))
+ conn = SimpleNamespace(config={}, prompt="", func_handler=None)
+ asyncio.run(start_app_task(conn, "english_learning", "repeater", {}, source="voice"))
+ assert spoken and "澶嶈鏈�" in spoken[0]
+ assert plays == []
+ assert conn.pending_play_after_tts and conn.pending_play_after_tts["kind"] == "repeater"
+ asyncio.run(maybe_flush_pending_play_after_tts(conn))
+ assert len(plays) == 1
+ assert plays[0]["name"] == "/english_learning/shadowing/lessons/unit01/01.mp3"
+ assert plays[0]["repeat"] == 10
+ assert plays[0].get("finish_task") is False
+ assert len(plays[0]["items"]) == 2
+ assert plays[0]["items"][1]["name"].endswith("/02.mp3")
+ assert plays[0]["items"][1]["repeat"] == 5
+ assert conn.repeater_playlist == []
+ assert conn.pending_play_after_tts is None
+
+
+def test_shadowing_queues_whole_playlist_until_listen_start(monkeypatch):
+ import asyncio
+ from core.app_runtime.runtime import maybe_flush_pending_play_after_tts, start_app_task
+
+ plays = []
+
+ async def fake_play(conn, params):
+ plays.append(params)
+ return None
+
+ async def fake_dirs(conn, app_id):
+ return None
+
+ monkeypatch.setattr("core.app_runtime.runtime._execute_named_tool_play", fake_play)
+ monkeypatch.setattr("core.app_runtime.runtime.ensure_app_sd_dirs", fake_dirs)
+ spoken = []
+ monkeypatch.setattr("core.app_runtime.runtime._speak", lambda conn, text: spoken.append(text))
+ conn = SimpleNamespace(config={}, prompt="", func_handler=None)
+ asyncio.run(start_app_task(conn, "english_learning", "shadowing", {}, source="voice"))
+ assert spoken and "璺熻" in spoken[0]
+ assert plays == []
+ assert conn.pending_play_after_tts and conn.pending_play_after_tts["kind"] == "shadowing"
+ queued = conn.pending_play_after_tts["params"]["items"]
+ assert len(queued) == 2
+ assert queued[0]["repeat"] == 5
+ assert queued[1]["name"].endswith("/02.mp3")
+ assert queued[1]["repeat"] == 10
+ asyncio.run(maybe_flush_pending_play_after_tts(conn))
+ assert len(plays) == 1
+ assert len(plays[0]["items"]) == 2
+ assert plays[0]["items"][0]["repeat"] == 5
+ assert plays[0]["items"][1]["repeat"] == 10
+ assert plays[0].get("finish_task") is False
+ assert plays[0].get("listen_ratio") == 1.2
+ assert conn.pending_play_after_tts is None
+
+
+def test_shadowing_expands_default_playlist_into_listen_turns():
+ import asyncio
+ from core.app_runtime.runtime import _prepare_shadowing_sd
+
+ conn = SimpleNamespace(
+ config={},
+ active_app_id="english_learning",
+ current_task_id="shadowing",
+ )
+ ok = asyncio.run(_prepare_shadowing_sd(conn, {}))
+ assert ok is True
+ unit01_01 = "/english_learning/shadowing/lessons/unit01/01.mp3"
+ unit01_02 = "/english_learning/shadowing/lessons/unit01/02.mp3"
+ assert conn.shadowing_clips.count(unit01_01) == 5
+ assert conn.shadowing_clips.count(unit01_02) == 10
+ assert len(conn.shadowing_clips) == 15
+ assert conn.shadowing_prompt == unit01_01
+
+
+def test_app_sd_layout_paths_from_registry():
+ from core.app_runtime.registry import get_app
+
+ paths = app_sd_layout_paths("english_learning", get_app("english_learning"))
+ assert set(paths) == {
+ "/english_learning/repeater/lessons",
+ "/english_learning/shadowing/lessons",
+ }
+ assert app_sd_layout_paths("chat", get_app("chat")) == []
+ assert app_sd_layout_paths("english_translation", get_app("english_translation")) == []
+
+
+def test_boot_default_chat_speaks_greeting(monkeypatch):
+ import asyncio
+ from core.app_runtime.runtime import start_app_task
+
+ spoken = []
+ monkeypatch.setattr("core.app_runtime.runtime._speak", lambda conn, text: spoken.append(text))
+
+ async def _ready(conn, timeout=8):
+ return True
+
+ monkeypatch.setattr("core.app_runtime.runtime.wait_for_tts_channels", _ready)
+ conn = SimpleNamespace(
+ config={
+ "enable_proactive_greeting": True,
+ "proactive_chat_greeting": "鎴戜滑鏉ヨ亰澶╁惂銆�",
+ },
+ prompt="",
+ func_handler=None,
+ just_woken_up=True,
+ chat_greeted=False,
+ active_app_id="chat",
+ current_task_id="conversation",
+ current_source="default_chat",
+ interaction_mode="CHAT",
+ )
+ conn.change_system_prompt = lambda text: setattr(conn, "prompt", text)
+ asyncio.run(start_app_task(conn, "chat", "conversation", {}, source="default_chat"))
+ assert spoken == ["鎴戜滑鏉ヨ亰澶╁惂銆�"]
+ assert conn.chat_greeted is True
+
+
+def test_client_hello_is_recognized_before_bind():
+ from core.websocket_handshake import is_client_hello
+
+ assert is_client_hello('{"type":"hello","version":1}') is True
+ assert is_client_hello('{"type":"listen","state":"detect"}') is False
+ assert is_client_hello(b"\x00\x01") is False
+
+
+def test_merge_hello_audio_params_keeps_server_output_rate():
+ from core.websocket_handshake import merge_hello_audio_params
+
+ merged = merge_hello_audio_params(
+ {"format": "opus", "sample_rate": 24000, "channels": 1, "frame_duration": 60},
+ {"format": "opus", "sample_rate": 16000, "channels": 1, "frame_duration": 60},
+ )
+ assert merged["sample_rate"] == 24000
+ assert merged["frame_duration"] == 60
+ assert merged["format"] == "opus"
+
+
+def test_priority_listen_detect_is_not_dropped_during_bind():
+ from core.websocket_handshake import is_priority_listen_detect
+
+ assert is_priority_listen_detect(
+ '{"type":"listen","state":"detect","text":"[app_task]chat|conversation|default_chat"}'
+ ) is True
+ assert is_priority_listen_detect(
+ '{"type":"listen","state":"detect","text":"浣犲ソ灏忔櫤"}'
+ ) is False
+
+
+def test_websocket_upgrade_accepts_keep_alive_connection_header():
+ from core.websocket_handshake import is_websocket_upgrade
+
+ assert is_websocket_upgrade({"connection": "Upgrade", "upgrade": "websocket"}) is True
+ assert is_websocket_upgrade({"connection": "keep-alive, Upgrade", "upgrade": "websocket"}) is True
+ assert is_websocket_upgrade({"connection": "keep-alive"}) is False
+
+
+def test_app_sd_layout_paths_custom_subscribed_app():
+ app = {
+ "tasks": {
+ "listen": {
+ "interaction_mode": "PLAY_AUDIO",
+ "skills": [{"name": "self.mp3.play"}],
+ },
+ "chatty": {"interaction_mode": "CHAT", "skills": []},
+ }
+ }
+ assert app_sd_layout_paths("story_time", app) == ["/story_time/listen/lessons"]
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/tests/test_assessment.py b/xiaozhi-esp32-server/main/xiaozhi-server/tests/test_assessment.py
new file mode 100644
index 0000000..51bed6e
--- /dev/null
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/tests/test_assessment.py
@@ -0,0 +1,544 @@
+"""Phase 4: listen-window clock + deferred quantitative assessment."""
+
+import asyncio
+from types import SimpleNamespace
+
+from core.app_runtime.assessment import (
+ build_metrics,
+ clamp_listen_ratio,
+ listen_window_ms,
+ resolve_expected_text,
+ text_similarity_v0,
+)
+from core.app_runtime.session import TaskSession, begin_session
+
+
+CLIPS = [
+ "/english_learning/shadowing/lessons/unit01/01.mp3",
+ "/english_learning/shadowing/lessons/unit01/02.mp3",
+]
+
+
+def test_resolve_expected_text_from_transcripts_not_path():
+ task = {
+ "transcripts": {"01.mp3": "Hello, how are you today?", "01": "Hello, how are you today?"},
+ "sentences": ["/english_learning/shadowing/lessons/unit01/01.mp3"],
+ }
+ expected = resolve_expected_text(task, CLIPS, 0)
+ assert expected == "Hello, how are you today?"
+ assert not expected.endswith(".mp3")
+ assert resolve_expected_text(task, CLIPS, 0) != CLIPS[0]
+
+
+def test_resolve_expected_text_rejects_path_as_original():
+ task = {
+ "transcripts": {"01.mp3": "/english_learning/shadowing/lessons/unit01/01.mp3"},
+ "sentences": [CLIPS[0]],
+ }
+ assert resolve_expected_text(task, CLIPS, 0) is None
+
+
+def test_resolve_expected_text_from_cache_txt(tmp_path):
+ (tmp_path / "01.txt").write_text("I like to read English books.", encoding="utf-8")
+ expected = resolve_expected_text({}, CLIPS, 0, cache_dir=str(tmp_path))
+ assert expected == "I like to read English books."
+
+
+def test_resolve_expected_text_sentences_fallback():
+ task = {"sentences": ["Hello, how are you today?", "I like to read English books."]}
+ assert resolve_expected_text(task, CLIPS, 1) == "I like to read English books."
+
+
+def test_text_similarity_metrics_have_no_passed_or_grade():
+ metrics = build_metrics("Hello, how are you today?", "Hello, how are you today?")
+ assert len(metrics) == 1
+ assert metrics[0]["method"] == "text_similarity_v0"
+ assert 0.0 <= metrics[0]["value"] <= 1.0
+ assert "passed" not in metrics[0]
+ assert "grade" not in metrics[0]
+ ratio = text_similarity_v0("hello how are you today", "Hello, how are you today?")
+ assert ratio > 0.9
+
+
+def test_build_metrics_empty_when_missing_asr_or_expected():
+ assert build_metrics("", "Hello") == []
+ assert build_metrics("Hello", "") == []
+ assert build_metrics("Hello", CLIPS[0]) == []
+
+
+def test_listen_ratio_is_clamped():
+ assert clamp_listen_ratio(0.5) == 1.2
+ assert clamp_listen_ratio(3.0) == 1.5
+ assert clamp_listen_ratio(1.3) == 1.3
+ assert listen_window_ms(1000, 0.5) == max(800, int(1000 * 1.2))
+ assert listen_window_ms(1000, 3.0) == int(1000 * 1.5)
+ assert listen_window_ms(100, 1.2) == 800
+ # 鍚獥闅忎笂涓�鍙ュ疄闄呮挱鏀炬椂闀垮彉鍖栵紝涓嶆槸鍥哄畾 2.5s锛涘�嶉�熷凡鍚湪 wall-clock duration 閲屻��
+ assert listen_window_ms(2050, 1.2) == int(2050 * 1.2)
+ assert listen_window_ms(4000, 1.2) == int(4000 * 1.2)
+ assert listen_window_ms(2050, 1.2, speed=0.9, wall_clock=True) == int(2050 * 1.2)
+ assert listen_window_ms(1800, 1.2, speed=0.9, wall_clock=False) == int(round(1800 / 0.9) * 1.2)
+
+
+def _shadowing_conn(tmp_path, monkeypatch):
+ monkeypatch.setattr("core.app_runtime.session.session_dir", lambda: str(tmp_path))
+ monkeypatch.setattr("core.app_runtime.session._schedule_upsert", lambda _payload: None)
+ monkeypatch.setattr("core.audio_archive.get_project_dir", lambda: str(tmp_path))
+ conn = SimpleNamespace(
+ session_id="ws-1",
+ device_id="mac",
+ interaction_mode="SHADOWING",
+ client_is_speaking=False,
+ shadowing_index=0,
+ shadowing_prompt=CLIPS[0],
+ shadowing_clips=list(CLIPS),
+ shadowing_awaiting_user=True,
+ shadowing_clip_gen=1,
+ shadowing_assess_index=0,
+ shadowing_listen_pcm=[],
+ shadowing_audio_saved_index=-1,
+ asr_audio=[],
+ config={"audio_archive": {"enabled": True, "dir": "data/recordings"}},
+ last_asr_audio_snapshot=[],
+ sentence_id=None,
+ task_session=None,
+ active_app_id="english_learning",
+ current_task_id="shadowing",
+ loop=None,
+ prompt="",
+ )
+ begin_session(conn, "english_learning", "shadowing", "voice", "SHADOWING")
+ return conn
+
+
+def test_asr_does_not_play_next_only_timer_advances(tmp_path, monkeypatch):
+ from core.app_runtime.runtime import (
+ _on_shadowing_listen_window_closed,
+ apply_shadowing_user_text,
+ )
+
+ played = []
+
+ async def fake_play(conn, name):
+ played.append(name)
+ return True
+
+ monkeypatch.setattr("core.app_runtime.runtime._play_shadowing_clip", fake_play)
+ conn = _shadowing_conn(tmp_path, monkeypatch)
+ outcome = apply_shadowing_user_text(conn, "today weather is bad")
+ assert outcome == "recorded"
+ assert conn.shadowing_index == 0
+ assert played == []
+ asyncio.run(_on_shadowing_listen_window_closed(conn))
+ assert conn.shadowing_index == 1
+ assert played == []
+ assert conn.task_session.find_turn(0)["asr_text"] == "today weather is bad"
+
+
+def test_skip_listen_stop_asr_while_shadowing_clip_plays():
+ from core.app_runtime.runtime import should_skip_listen_stop_asr
+
+ playing = SimpleNamespace(interaction_mode="SHADOWING", shadowing_awaiting_user=False)
+ listening = SimpleNamespace(interaction_mode="SHADOWING", shadowing_awaiting_user=True)
+ chat = SimpleNamespace(interaction_mode="CHAT", shadowing_awaiting_user=False)
+ repeater = SimpleNamespace(interaction_mode="PLAY_AUDIO", shadowing_awaiting_user=False)
+ assert should_skip_listen_stop_asr(playing) is True
+ assert should_skip_listen_stop_asr(listening) is False
+ assert should_skip_listen_stop_asr(chat) is False
+ assert should_skip_listen_stop_asr(repeater) is True
+
+
+def test_timer_advances_after_assess_and_upsert_errors(tmp_path, monkeypatch):
+ from core.app_runtime.runtime import _on_shadowing_listen_window_closed
+
+ played = []
+
+ async def fake_play(conn, name):
+ played.append(name)
+ return True
+
+ def boom_assess(*_args, **_kwargs):
+ raise RuntimeError("assess exploded")
+
+ monkeypatch.setattr("core.app_runtime.runtime._play_shadowing_clip", fake_play)
+ monkeypatch.setattr("core.app_runtime.runtime.apply_deferred_assessment", boom_assess)
+ conn = _shadowing_conn(tmp_path, monkeypatch)
+
+ def boom_save():
+ raise RuntimeError("upsert exploded")
+
+ conn.task_session.save = boom_save
+ asyncio.run(_on_shadowing_listen_window_closed(conn))
+ assert conn.shadowing_index == 1
+ assert played == []
+
+
+def test_timer_advances_without_asr(tmp_path, monkeypatch):
+ from core.app_runtime.runtime import _on_shadowing_listen_window_closed
+
+ played = []
+
+ async def fake_play(conn, name):
+ played.append(name)
+ return True
+
+ monkeypatch.setattr("core.app_runtime.runtime._play_shadowing_clip", fake_play)
+ conn = _shadowing_conn(tmp_path, monkeypatch)
+ asyncio.run(_on_shadowing_listen_window_closed(conn))
+ assert conn.shadowing_index == 1
+ turn = conn.task_session.find_turn(0)
+ assert turn is not None
+ assert turn["asr_text"] == ""
+ names = [event["name"] for event in conn.task_session.events]
+ assert "listen_window_closed" in names
+ assert played == []
+
+
+def test_upsert_turn_keeps_last_asr_same_index(tmp_path, monkeypatch):
+ monkeypatch.setattr("core.app_runtime.session.session_dir", lambda: str(tmp_path))
+ session = TaskSession(task_session_id="dup", app_id="english_learning", task_id="shadowing")
+ session.upsert_turn(0, CLIPS[0], "first try")
+ session.upsert_turn(0, CLIPS[0], "second try")
+ assert len(session.turns) == 1
+ assert session.turns[0]["asr_text"] == "second try"
+
+
+def test_deferred_assessment_writes_metrics_not_passed(tmp_path, monkeypatch):
+ from core.app_runtime.runtime import apply_deferred_assessment, apply_shadowing_user_text
+
+ conn = _shadowing_conn(tmp_path, monkeypatch)
+ apply_shadowing_user_text(conn, "Hello, how are you today?")
+ apply_deferred_assessment(conn, 0)
+ turn = conn.task_session.find_turn(0)
+ assert turn["expected_text"] == "Hello, how are you today?"
+ assert not str(turn["expected_text"]).endswith(".mp3")
+ assert turn["metrics"][0]["method"] == "text_similarity_v0"
+ assert 0.0 <= turn["metrics"][0]["value"] <= 1.0
+ assert "passed" not in turn
+ assert "grade" not in turn
+ assert "passed" not in turn["metrics"][0]
+
+
+def test_upsert_empty_metrics_do_not_wipe_existing(tmp_path, monkeypatch):
+ monkeypatch.setattr("core.app_runtime.session.session_dir", lambda: str(tmp_path))
+ session = TaskSession(task_session_id="keep-m", app_id="english_learning", task_id="shadowing")
+ session.upsert_turn(
+ 0,
+ CLIPS[0],
+ "hello",
+ expected_text="Hello, how are you today?",
+ metrics=[{"method": "text_similarity_v0", "value": 0.9}],
+ assessed_at=1,
+ )
+ session.upsert_turn(0, CLIPS[0], "hello")
+ assert session.turns[0]["metrics"][0]["value"] == 0.9
+ assert "passed" not in session.turns[0]
+ assert "passed" not in session.turns[0]["metrics"][0]
+
+
+def test_skill_event_starts_listen_window_before_save(tmp_path, monkeypatch):
+ from core.app_runtime.runtime import handle_skill_event, listen_window_ms
+
+ scheduled = []
+
+ def fake_schedule(conn, duration_ms):
+ scheduled.append(duration_ms)
+
+ monkeypatch.setattr(
+ "core.app_runtime.runtime._schedule_shadowing_listen_timeout", fake_schedule
+ )
+ conn = _shadowing_conn(tmp_path, monkeypatch)
+ conn.shadowing_awaiting_user = False
+ asyncio.run(
+ handle_skill_event(
+ conn, "[skill_event]audio.completed|english_learning|shadowing|2050"
+ )
+ )
+ assert scheduled == [2050]
+ assert listen_window_ms(2050, 1.2) == int(2050 * 1.2)
+ assert conn.task_session.events[-1]["name"] == "audio.completed"
+
+
+def test_repeater_completed_returns_to_chat_even_if_event_says_shadowing(monkeypatch):
+ from core.app_runtime.runtime import handle_skill_event
+
+ switched = []
+
+ async def fake_start(conn, app_id, task_id, params, source="voice", **kwargs):
+ switched.append(
+ (app_id, task_id, kwargs.get("speak_switch"), kwargs.get("stop_skills"))
+ )
+ return None
+
+ async def fake_continue(conn):
+ return False
+
+ monkeypatch.setattr("core.app_runtime.runtime.start_app_task", fake_start)
+ monkeypatch.setattr("core.app_runtime.runtime._continue_repeater_if_needed", fake_continue)
+ spoken = []
+ monkeypatch.setattr("core.app_runtime.runtime._speak", lambda conn, text: spoken.append(text))
+ conn = SimpleNamespace(
+ interaction_mode="PLAY_AUDIO",
+ active_app_id="english_learning",
+ current_task_id="repeater",
+ task_session=None,
+ repeater_speed_queue=[],
+ repeater_play_name="",
+ prompt="",
+ config={},
+ device_id="mac",
+ )
+ asyncio.run(
+ handle_skill_event(
+ conn, "[skill_event]audio.completed|english_learning|shadowing|6130"
+ )
+ )
+ assert switched == [("chat", "conversation", False, False)]
+ assert spoken == ["澶嶈缁撴潫銆傛垜浠潵鑱婂ぉ鍚с��"]
+
+
+def test_late_asr_after_listen_window_still_scores_previous_sentence(tmp_path, monkeypatch):
+ from core.app_runtime.runtime import apply_shadowing_user_text
+
+ conn = _shadowing_conn(tmp_path, monkeypatch)
+ conn.shadowing_awaiting_user = False
+ conn.shadowing_index = 1
+ conn.shadowing_assess_index = 0
+ conn.shadowing_prompt = CLIPS[1]
+ conn.client_is_speaking = False
+ outcome = apply_shadowing_user_text(conn, "hello how are you today")
+ assert outcome == "recorded"
+ assert conn.shadowing_index == 1
+ turn = conn.task_session.find_turn(0)
+ assert turn["asr_text"] == "hello how are you today"
+ assert conn.task_session.find_turn(1) is None
+
+
+def test_evaluate_turn_saves_wav_and_metrics(tmp_path, monkeypatch):
+ from core.app_runtime.assessment import pcm_snapshot_to_wav
+ from core.app_runtime.runtime import apply_deferred_assessment, apply_shadowing_user_text
+
+ captured = {}
+
+ def fake_save(session_id, step_index, wav_bytes, conn=None):
+ captured["wav"] = wav_bytes
+ captured["id"] = f"{session_id}_{step_index}.wav"
+ return captured["id"]
+
+ monkeypatch.setattr("core.app_runtime.runtime.save_turn_wav", fake_save)
+ conn = _shadowing_conn(tmp_path, monkeypatch)
+ # 16-bit silence ~ 20ms
+ conn.last_asr_audio_snapshot = [b"\x00\x00" * 160]
+ apply_shadowing_user_text(conn, "Hello, how are you today?")
+ apply_deferred_assessment(conn, 0, audio_snapshot=conn.last_asr_audio_snapshot)
+ turn = conn.task_session.find_turn(0)
+ assert turn["metrics"][0]["method"] == "text_similarity_v0"
+ assert turn["audio_id"] == captured["id"]
+ wav = pcm_snapshot_to_wav(conn.last_asr_audio_snapshot)
+ assert captured["wav"][:4] == b"RIFF"
+ assert wav[8:12] == b"WAVE"
+ assert captured["wav"] == wav
+
+
+def test_shadowing_playlist_end_announces_then_chat(tmp_path, monkeypatch):
+ from core.app_runtime.runtime import handle_skill_event
+
+ switched = []
+ spoken = []
+ played = []
+
+ async def fake_play(conn, name):
+ played.append(name)
+ return True
+
+ async def fake_start(conn, app_id, task_id, params, source="voice", **kwargs):
+ switched.append((app_id, task_id, kwargs.get("speak_switch")))
+ conn.interaction_mode = "CHAT"
+ conn.active_app_id = app_id
+ conn.current_task_id = task_id
+ return None
+
+ monkeypatch.setattr("core.app_runtime.runtime._play_shadowing_clip", fake_play)
+ monkeypatch.setattr("core.app_runtime.runtime.start_app_task", fake_start)
+ monkeypatch.setattr("core.app_runtime.runtime._speak", lambda conn, text: spoken.append(text))
+ conn = _shadowing_conn(tmp_path, monkeypatch)
+ conn.shadowing_index = 1
+ conn.shadowing_prompt = CLIPS[1]
+ conn.shadowing_awaiting_user = False
+ asyncio.run(
+ handle_skill_event(
+ conn, "[skill_event]audio.playlist_completed|english_learning|shadowing"
+ )
+ )
+ assert played == []
+ assert switched == [("chat", "conversation", False)]
+ assert spoken == ["璺熻缁撴潫銆傛垜浠潵鑱婂ぉ鍚с��"]
+
+
+def test_playlist_completed_returns_chat_after_reconnect(monkeypatch):
+ """WS 鎺夌嚎鍚庤澶囧彲鑳藉湪鏂扮殑 CHAT 杩炴帴涓婅ˉ鍙� playlist_completed銆�"""
+ from core.app_runtime.runtime import handle_skill_event
+
+ switched = []
+ spoken = []
+
+ async def fake_start(conn, app_id, task_id, params, source="voice", **kwargs):
+ switched.append((app_id, task_id, kwargs.get("speak_switch")))
+ conn.interaction_mode = "CHAT"
+ return None
+
+ monkeypatch.setattr("core.app_runtime.runtime.start_app_task", fake_start)
+ monkeypatch.setattr("core.app_runtime.runtime._speak", lambda conn, text: spoken.append(text))
+ conn = SimpleNamespace(
+ interaction_mode="CHAT",
+ active_app_id="chat",
+ current_task_id="conversation",
+ task_session=None,
+ repeater_speed_queue=[],
+ repeater_playlist=[],
+ shadowing_awaiting_user=False,
+ prompt="",
+ config={},
+ device_id="mac",
+ tts=None,
+ )
+ asyncio.run(
+ handle_skill_event(
+ conn, "[skill_event]audio.playlist_completed|english_learning|shadowing"
+ )
+ )
+ assert switched == [("chat", "conversation", False)]
+ assert spoken == ["璺熻缁撴潫銆傛垜浠潵鑱婂ぉ鍚с��"]
+
+
+def test_handle_shadowing_turn_does_not_enter_chat(tmp_path, monkeypatch):
+ from core.app_runtime.runtime import handle_shadowing_turn
+
+ conn = _shadowing_conn(tmp_path, monkeypatch)
+ conn.last_asr_audio_snapshot = [b"\x00\x00" * 80]
+ asyncio.run(handle_shadowing_turn(conn, "Hello, how are you today?"))
+ assert conn.interaction_mode == "SHADOWING"
+ assert conn.shadowing_index == 0
+ turn = conn.task_session.find_turn(0)
+ assert turn["asr_text"] == "Hello, how are you today?"
+
+
+def test_listen_window_saves_wav_even_if_user_silent(tmp_path, monkeypatch):
+ from pathlib import Path
+ from core.app_runtime.runtime import _on_shadowing_listen_window_closed
+
+ conn = _shadowing_conn(tmp_path, monkeypatch)
+ conn.shadowing_listen_pcm = []
+ conn.asr_audio = []
+ asyncio.run(_on_shadowing_listen_window_closed(conn))
+ turn = conn.task_session.find_turn(0)
+ assert turn is not None
+ assert turn.get("audio_id")
+ assert "mac/shadowing/" in turn["audio_id"]
+ wav_path = tmp_path / "data" / "recordings" / Path(turn["audio_id"])
+ assert wav_path.is_file()
+ assert wav_path.stat().st_size > 44
+
+
+def test_fifteen_listen_windows_write_fifteen_wavs(tmp_path, monkeypatch):
+ from pathlib import Path
+ from core.app_runtime.runtime import save_shadowing_listen_audio
+
+ conn = _shadowing_conn(tmp_path, monkeypatch)
+ conn.shadowing_clips = CLIPS[:1] * 15
+ saved = []
+ for i in range(15):
+ conn.shadowing_assess_index = i
+ conn.shadowing_index = i
+ conn.shadowing_audio_saved_index = -1
+ conn.shadowing_listen_pcm = [b"\x00\x00" * (80 + i)]
+ save_shadowing_listen_audio(conn, i)
+ turn = conn.task_session.find_turn(i)
+ assert turn.get("audio_id")
+ saved.append(turn["audio_id"])
+ assert (tmp_path / "data" / "recordings" / Path(turn["audio_id"])).is_file()
+ assert len(saved) == 15
+ assert len(set(saved)) == 15
+
+
+def test_shadowing_save_does_not_merge_previous_asr_audio(tmp_path, monkeypatch):
+ from pathlib import Path
+ import wave
+ from core.app_runtime.runtime import save_shadowing_listen_audio
+
+ conn = _shadowing_conn(tmp_path, monkeypatch)
+ leftover = b"\x11\x00" * 8000
+ current = b"\x22\x00" * 160
+ conn.asr_audio = [leftover]
+ conn.shadowing_listen_pcm = [current]
+ conn.shadowing_assess_index = 0
+ conn.shadowing_audio_saved_index = -1
+ save_shadowing_listen_audio(conn, 0)
+ turn = conn.task_session.find_turn(0)
+ wav_path = tmp_path / "data" / "recordings" / Path(turn["audio_id"])
+ with wave.open(str(wav_path), "rb") as handle:
+ pcm = handle.readframes(handle.getnframes())
+ assert leftover not in pcm
+ assert current in pcm
+ assert len(pcm) == len(current)
+
+
+def test_one_step_writes_only_one_wav(tmp_path, monkeypatch):
+ from core.app_runtime.runtime import save_shadowing_listen_audio
+
+ conn = _shadowing_conn(tmp_path, monkeypatch)
+ conn.shadowing_listen_pcm = [b"\x00\x00" * 80]
+ save_shadowing_listen_audio(conn, 0)
+ first = conn.task_session.find_turn(0)["audio_id"]
+ conn.shadowing_listen_pcm = [b"\x00\x00" * 800]
+ conn.asr_audio = [b"\x00\x00" * 8000]
+ save_shadowing_listen_audio(conn, 0)
+ assert conn.task_session.find_turn(0)["audio_id"] == first
+ recordings = list((tmp_path / "data" / "recordings").rglob("*.wav"))
+ assert len(recordings) == 1
+
+
+def test_listen_stop_drains_queued_pcm_instead_of_silent_placeholder(tmp_path, monkeypatch):
+ import queue as queue_mod
+ import wave
+ from pathlib import Path
+ from core.app_runtime.runtime import close_shadowing_listen_window
+
+ conn = _shadowing_conn(tmp_path, monkeypatch)
+ conn.shadowing_awaiting_user = True
+ conn.shadowing_index = 0
+ conn.shadowing_listen_pcm = []
+ conn.asr_audio_queue = queue_mod.Queue()
+ real = b"\x22\x00" * 1600
+ conn.asr_audio_queue.put(real)
+ conn.asr_audio_queue.put(b"\x33\x00" * 1600)
+ asyncio.run(close_shadowing_listen_window(conn))
+ turn = conn.task_session.find_turn(0)
+ wav_path = tmp_path / "data" / "recordings" / Path(turn["audio_id"])
+ with wave.open(str(wav_path), "rb") as handle:
+ pcm = handle.readframes(handle.getnframes())
+ assert real in pcm
+ assert len(pcm) == 1600 * 2 * 2
+ assert conn.asr_audio_queue.empty()
+
+
+def test_listen_stop_advances_index_for_next_wav(tmp_path, monkeypatch):
+ from pathlib import Path
+ from core.app_runtime.runtime import close_shadowing_listen_window, save_shadowing_listen_audio
+
+ conn = _shadowing_conn(tmp_path, monkeypatch)
+ conn.shadowing_clips = CLIPS[:1] * 15
+ conn.shadowing_awaiting_user = True
+ conn.shadowing_index = 0
+ conn.shadowing_listen_pcm = [b"\x00\x00" * 80]
+ asyncio.run(close_shadowing_listen_window(conn))
+ assert conn.shadowing_index == 1
+ assert conn.shadowing_awaiting_user is False
+ first = conn.task_session.find_turn(0)["audio_id"]
+ conn.shadowing_listen_pcm = [b"\x11\x00" * 80]
+ conn.shadowing_audio_saved_index = -1
+ save_shadowing_listen_audio(conn, 1)
+ second = conn.task_session.find_turn(1)["audio_id"]
+ assert first != second
+ recordings = list((tmp_path / "data" / "recordings").rglob("*.wav"))
+ assert len(recordings) == 2
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/tests/test_audio_archive.py b/xiaozhi-esp32-server/main/xiaozhi-server/tests/test_audio_archive.py
new file mode 100644
index 0000000..0ed7d39
--- /dev/null
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/tests/test_audio_archive.py
@@ -0,0 +1,59 @@
+"""Local audio archive path: device / scene / date."""
+
+from pathlib import Path
+from types import SimpleNamespace
+
+from core.audio_archive import archive_pcm_snapshot, archive_wav
+from core.app_runtime.assessment import pcm_snapshot_to_wav, save_turn_wav
+
+
+def test_archive_path_uses_device_mode_and_date(tmp_path, monkeypatch):
+ monkeypatch.setattr("core.audio_archive.get_project_dir", lambda: str(tmp_path))
+ conn = SimpleNamespace(
+ device_id="AA:BB:CC:DD:EE:FF",
+ interaction_mode="SHADOWING",
+ config={"audio_archive": {"enabled": True, "dir": "data/recordings"}},
+ )
+ wav = pcm_snapshot_to_wav([b"\x00\x00" * 80])
+ relative = archive_wav(conn, wav, kind="asr", extra="sess_0")
+ assert relative is not None
+ assert relative.startswith("AA-BB-CC-DD-EE-FF/shadowing/")
+ assert "_asr_" in relative
+ assert (tmp_path / "data" / "recordings" / Path(relative)).is_file()
+
+
+def test_archive_disabled(tmp_path, monkeypatch):
+ monkeypatch.setattr("core.audio_archive.get_project_dir", lambda: str(tmp_path))
+ conn = SimpleNamespace(
+ device_id="dev1",
+ interaction_mode="CHAT",
+ config={"audio_archive": {"enabled": False}},
+ )
+ wav = pcm_snapshot_to_wav([b"\x00\x00" * 80])
+ assert archive_wav(conn, wav) is None
+
+
+def test_save_turn_wav_uses_archive_layout(tmp_path, monkeypatch):
+ monkeypatch.setattr("core.audio_archive.get_project_dir", lambda: str(tmp_path))
+ conn = SimpleNamespace(
+ device_id="mac1",
+ interaction_mode="SHADOWING",
+ config={"audio_archive": {"enabled": True, "dir": "data/recordings"}},
+ )
+ wav = pcm_snapshot_to_wav([b"\x00\x00" * 160])
+ audio_id = save_turn_wav("sess", 3, wav, conn=conn)
+ assert audio_id is not None
+ assert "mac1/shadowing/" in audio_id
+ assert audio_id.endswith(".wav")
+
+
+def test_chat_pcm_archive_uses_chat_dir(tmp_path, monkeypatch):
+ monkeypatch.setattr("core.audio_archive.get_project_dir", lambda: str(tmp_path))
+ conn = SimpleNamespace(
+ device_id="dev2",
+ interaction_mode="CHAT",
+ config={"audio_archive": {"enabled": True, "dir": "data/recordings"}},
+ )
+ relative = archive_pcm_snapshot(conn, [b"\x00\x00" * 80])
+ assert relative is not None
+ assert relative.startswith("dev2/chat/")
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/tests/test_playback.py b/xiaozhi-esp32-server/main/xiaozhi-server/tests/test_playback.py
new file mode 100644
index 0000000..527fc0f
--- /dev/null
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/tests/test_playback.py
@@ -0,0 +1,384 @@
+"""Playback speed clamp and repeater speed gradients."""
+
+from core.app_runtime.playback import (
+ PLAY_SPEED_DEFAULT,
+ clamp_play_speed,
+ daily_base_speed,
+ lesson_day_index,
+ mark_lesson_played_today,
+ parse_playlist,
+ speeds_for_repeats,
+)
+from core.app_runtime.runtime import (
+ _apply_playback_speed,
+ _begin_repeater_playlist,
+ _begin_shadowing_playlist,
+ _continue_repeater_if_needed,
+)
+from types import SimpleNamespace
+import asyncio
+
+
+def test_clamp_play_speed_range():
+ assert clamp_play_speed(None) == PLAY_SPEED_DEFAULT
+ assert clamp_play_speed("nope") == PLAY_SPEED_DEFAULT
+ assert clamp_play_speed(0.3) == 0.5
+ assert clamp_play_speed(3.0) == 2.0
+ assert clamp_play_speed(0.8) == 0.8
+ assert clamp_play_speed(1.05) == 1.05
+ assert clamp_play_speed(1.0) == 1.0
+
+
+def test_daily_speeds_use_practice_day():
+ task = {"playback": {"daily_speeds": [0.8, 0.9, 1.0, 1.05, 1.1, 1.15, 1.2]}}
+ assert daily_base_speed(task, 0) == 0.8
+ assert daily_base_speed(task, 3) == 1.05
+ assert daily_base_speed(task, 99) == 1.2
+
+
+def test_repeat_speeds_five_passes():
+ task = {
+ "playback": {
+ "daily_speeds": [0.8, 0.9, 1.0],
+ "repeat_speeds": [0.8, 0.9, 1.0, 1.1, 1.2],
+ }
+ }
+ assert speeds_for_repeats(task, 1, day_index=0) == [0.8]
+ assert speeds_for_repeats(task, 5, day_index=0) == [0.8, 0.9, 1.0, 1.1, 1.2]
+
+
+def test_repeat_one_does_not_use_full_curve():
+ task = {"playback": {"speed": 1.0, "repeat_speeds": [0.8, 0.9, 1.0, 1.1, 1.2]}}
+ assert speeds_for_repeats(task, 1) == [1.0]
+
+
+def test_lesson_day_index_counts_distinct_days(tmp_path, monkeypatch):
+ monkeypatch.setattr("core.app_runtime.playback.get_project_dir", lambda: str(tmp_path))
+ assert lesson_day_index("mac1", "english_learning", "repeater", "unit01.mp3") == 0
+ mark_lesson_played_today("mac1", "english_learning", "repeater", "unit01.mp3")
+ assert lesson_day_index("mac1", "english_learning", "repeater", "unit01.mp3") == 1
+ mark_lesson_played_today("mac1", "english_learning", "repeater", "unit01.mp3")
+ assert lesson_day_index("mac1", "english_learning", "repeater", "unit01.mp3") == 1
+
+
+def test_apply_playback_speed_queues_gradient():
+ task = {"playback": {"repeat_speeds": [0.8, 0.9, 1.0, 1.1, 1.2]}}
+ conn = SimpleNamespace(
+ device_id="mac",
+ active_app_id="english_learning",
+ current_task_id="repeater",
+ repeater_speed_queue=[],
+ repeater_play_name="",
+ repeater_power_off_after=False,
+ )
+ play = _apply_playback_speed(
+ conn, task, {"name": "/english_learning/repeater/lessons/unit01/full.mp3", "repeat": 5}
+ )
+ assert play["speed"] == 0.8
+ assert play["repeat"] == 1
+ assert play["power_off_after"] is False
+ assert conn.repeater_speed_queue == [0.9, 1.0, 1.1, 1.2]
+
+
+def test_explicit_speed_skips_gradient():
+ task = {"playback": {"repeat_speeds": [0.8, 0.9, 1.0, 1.1, 1.2]}}
+ conn = SimpleNamespace(
+ device_id="mac",
+ active_app_id="english_learning",
+ current_task_id="repeater",
+ repeater_speed_queue=[],
+ repeater_play_name="",
+ repeater_power_off_after=False,
+ )
+ play = _apply_playback_speed(conn, task, {"name": "a.mp3", "repeat": 5, "speed": 1.05})
+ assert play["speed"] == 1.05
+ assert play["repeat"] == 5
+ assert conn.repeater_speed_queue == []
+
+
+def test_continue_repeater_plays_next_speed(monkeypatch):
+ played = []
+
+ async def fake_play(conn, params):
+ played.append(params["speed"])
+ return None
+
+ monkeypatch.setattr("core.app_runtime.runtime._execute_named_tool_play", fake_play)
+ conn = SimpleNamespace(
+ repeater_speed_queue=[0.9, 1.0],
+ repeater_play_name="a.mp3",
+ repeater_power_off_after=False,
+ interaction_mode="PLAY_AUDIO",
+ )
+ assert asyncio.run(_continue_repeater_if_needed(conn)) is True
+ assert played == [0.9]
+ assert conn.repeater_speed_queue == [1.0]
+ assert asyncio.run(_continue_repeater_if_needed(conn)) is True
+ assert played == [0.9, 1.0]
+ assert conn.repeater_speed_queue == []
+ assert asyncio.run(_continue_repeater_if_needed(conn)) is False
+
+
+def test_parse_playlist_from_items_and_urls():
+ items = parse_playlist(
+ {
+ "items": [
+ {
+ "url": "http://cdn.example/static/english_learning/repeater/lessons/a.mp3",
+ "repeat": 3,
+ },
+ {
+ "path": "/english_learning/repeater/lessons/b.mp3",
+ "count": 2,
+ "speed": 0.9,
+ },
+ ]
+ }
+ )
+ assert items[0]["name"] == "/english_learning/repeater/lessons/a.mp3"
+ assert items[0]["repeat"] == 3
+ assert items[1]["name"] == "/english_learning/repeater/lessons/b.mp3"
+ assert items[1]["repeat"] == 2
+ assert items[1]["speed"] == 0.9
+
+
+def test_begin_repeater_queues_remaining_items():
+ conn = SimpleNamespace(
+ device_id="mac",
+ active_app_id="english_learning",
+ current_task_id="repeater",
+ repeater_speed_queue=[],
+ repeater_play_name="",
+ repeater_power_off_after=False,
+ repeater_playlist=[],
+ )
+ task = {"playback": {"speed": 1.0}}
+ first = _begin_repeater_playlist(
+ conn,
+ task,
+ {
+ "items": [
+ {"name": "/english_learning/repeater/lessons/a.mp3", "repeat": 2, "speed": 1.0},
+ {"name": "/english_learning/repeater/lessons/b.mp3", "repeat": 1, "speed": 0.9},
+ ]
+ },
+ )
+ assert first["name"].endswith("/a.mp3")
+ assert first["repeat"] == 2
+ assert first.get("finish_task") is False
+ assert len(first["items"]) == 2
+ assert first["items"][1]["name"].endswith("/b.mp3")
+ assert first["items"][1]["repeat"] == 1
+ assert conn.repeater_playlist == []
+
+
+def test_continue_repeater_plays_next_playlist_file(monkeypatch):
+ played = []
+
+ async def fake_play(conn, params):
+ played.append((params.get("name"), params.get("repeat"), params.get("speed"), params.get("finish_task")))
+ return None
+
+ monkeypatch.setattr("core.app_runtime.runtime._execute_named_tool_play", fake_play)
+ conn = SimpleNamespace(
+ device_id="mac",
+ active_app_id="english_learning",
+ current_task_id="repeater",
+ repeater_speed_queue=[],
+ repeater_play_name="/english_learning/repeater/lessons/a.mp3",
+ repeater_power_off_after=False,
+ repeater_playlist=[{"name": "/english_learning/repeater/lessons/b.mp3", "repeat": 2, "speed": 0.9}],
+ interaction_mode="PLAY_AUDIO",
+ )
+ assert asyncio.run(_continue_repeater_if_needed(conn)) is True
+ assert played == [("/english_learning/repeater/lessons/b.mp3", 2, 0.9, False)]
+ assert conn.repeater_playlist == []
+ assert asyncio.run(_continue_repeater_if_needed(conn)) is False
+
+
+def test_begin_repeater_uses_temp_unit01_playlist():
+ from core.app_runtime.playback import TEMP_REPEATER_PLAYLIST
+
+ conn = SimpleNamespace(
+ device_id="mac",
+ active_app_id="english_learning",
+ current_task_id="repeater",
+ repeater_speed_queue=[],
+ repeater_play_name="",
+ repeater_power_off_after=False,
+ repeater_playlist=[],
+ )
+ first = _begin_repeater_playlist(conn, {"playback": {"speed": 1.0}}, {})
+ assert first["name"] == TEMP_REPEATER_PLAYLIST[0]["name"]
+ assert first["repeat"] == 10
+ assert first.get("finish_task") is False
+ assert len(first["items"]) == 2
+ assert first["items"][0]["repeat"] == 10
+ assert first["items"][1]["name"] == TEMP_REPEATER_PLAYLIST[1]["name"]
+ assert first["items"][1]["repeat"] == 5
+ assert conn.repeater_playlist == []
+
+
+def test_begin_shadowing_playlist_one_shot():
+ from core.app_runtime.playback import TEMP_SHADOWING_PLAYLIST
+
+ conn = SimpleNamespace(
+ device_id="mac",
+ active_app_id="english_learning",
+ current_task_id="shadowing",
+ config={},
+ shadowing_clips=[],
+ shadowing_index=0,
+ shadowing_prompt="",
+ shadowing_awaiting_user=True,
+ )
+ play = _begin_shadowing_playlist(conn, {})
+ assert play["name"] == TEMP_SHADOWING_PLAYLIST[0]["name"]
+ assert play["repeat"] == 5
+ assert play.get("finish_task") is False
+ assert play.get("listen_ratio") == 1.2
+ assert len(play["items"]) == 2
+ assert play["items"][0]["repeat"] == 5
+ assert play["items"][1]["name"] == TEMP_SHADOWING_PLAYLIST[1]["name"]
+ assert play["items"][1]["repeat"] == 10
+ assert conn.shadowing_clips.count(TEMP_SHADOWING_PLAYLIST[0]["name"]) == 5
+ assert conn.shadowing_clips.count(TEMP_SHADOWING_PLAYLIST[1]["name"]) == 10
+ assert len(conn.shadowing_clips) == 15
+ assert conn.shadowing_awaiting_user is False
+
+
+def test_expand_shadowing_playlist_one_clip_per_listen_window():
+ from core.app_runtime.playback import (
+ TEMP_SHADOWING_PLAYLIST,
+ default_shadowing_playlist,
+ expand_playlist_plays,
+ )
+
+ clips = expand_playlist_plays(default_shadowing_playlist())
+ assert clips.count(TEMP_SHADOWING_PLAYLIST[0]["name"]) == 5
+ assert clips.count(TEMP_SHADOWING_PLAYLIST[1]["name"]) == 10
+ assert len(clips) == 15
+
+
+def test_audio_completion_timeout_scales_with_queue():
+ from core.utils.audioRateController import audio_completion_timeout_sec
+
+ assert audio_completion_timeout_sec(0) == 8.0
+ assert audio_completion_timeout_sec(82) == 8.0 + 82 * 0.06
+ assert audio_completion_timeout_sec(5000) == 180.0
+
+
+def test_execute_named_tool_play_sends_active_task(monkeypatch):
+ from core.app_runtime.runtime import _execute_named_tool_play
+
+ captured = {}
+
+ async def fake_tool(conn, tool_name, arguments):
+ captured["tool"] = tool_name
+ captured["arguments"] = arguments
+ return None
+
+ monkeypatch.setattr("core.app_runtime.runtime._execute_named_tool", fake_tool)
+ conn = SimpleNamespace(
+ active_app_id="english_learning",
+ current_task_id="repeater",
+ )
+ asyncio.run(
+ _execute_named_tool_play(
+ conn,
+ {
+ "name": "/english_learning/shadowing/lessons/unit01/02.mp3",
+ "repeat": 5,
+ "speed": 1.0,
+ "finish_task": True,
+ "items": [
+ {
+ "name": "/english_learning/shadowing/lessons/unit01/01.mp3",
+ "repeat": 10,
+ "speed": 1.0,
+ },
+ {
+ "name": "/english_learning/shadowing/lessons/unit01/02.mp3",
+ "repeat": 5,
+ "speed": 1.0,
+ },
+ ],
+ },
+ )
+ )
+ assert captured["arguments"]["task_id"] == "repeater"
+ assert captured["arguments"]["app_id"] == "english_learning"
+ assert captured["arguments"]["finish_task"] is False
+ assert captured["arguments"]["force"] is True
+ assert "/01.mp3" in captured["arguments"]["items"]
+ assert "/02.mp3" in captured["arguments"]["items"]
+
+
+def test_execute_named_tool_play_retries_duplicate_ignored(monkeypatch):
+ from core.app_runtime.runtime import _execute_named_tool_play
+ from plugins_func.register import Action, ActionResponse
+
+ calls = []
+
+ async def fake_tool(conn, tool_name, arguments):
+ calls.append(dict(arguments))
+ if len(calls) == 1:
+ return ActionResponse(
+ action=Action.RESPONSE,
+ response='{"accepted":true,"duplicate_ignored":true}',
+ )
+ return ActionResponse(
+ action=Action.RESPONSE,
+ response='{"accepted":true,"repeat":1,"power_off_after":false}',
+ )
+
+ monkeypatch.setattr("core.app_runtime.runtime._execute_named_tool", fake_tool)
+ conn = SimpleNamespace(active_app_id="english_learning", current_task_id="shadowing")
+ notice = asyncio.run(
+ _execute_named_tool_play(
+ conn,
+ {
+ "name": "/english_learning/shadowing/lessons/unit01/01.mp3",
+ "repeat": 1,
+ "speed": 1.0,
+ "finish_task": False,
+ },
+ )
+ )
+ assert notice is None
+ assert len(calls) == 2
+ assert calls[0]["task_id"] == "shadowing"
+ assert calls[0]["finish_task"] is False
+ assert calls[1]["speed"] == 1.02
+ assert calls[1]["force"] is True
+
+
+def test_play_shadowing_clip_keeps_task_open(monkeypatch):
+ from core.app_runtime.runtime import _play_shadowing_clip
+
+ captured = {}
+
+ async def fake_play(conn, params):
+ captured["params"] = params
+ return None
+
+ monkeypatch.setattr("core.app_runtime.runtime._execute_named_tool_play", fake_play)
+ conn = SimpleNamespace(
+ interaction_mode="SHADOWING",
+ shadowing_clip_gen=0,
+ active_app_id="english_learning",
+ current_task_id="shadowing",
+ config={},
+ )
+ ok = asyncio.run(_play_shadowing_clip(conn, "/english_learning/shadowing/lessons/unit01/01.mp3"))
+ assert ok is True
+ assert captured["params"]["finish_task"] is False
+ assert captured["params"]["repeat"] == 1
+
+
+def test_lesson_end_speech_includes_chat_greeting():
+ from core.app_runtime.runtime import _lesson_end_speech
+
+ conn = SimpleNamespace(config={})
+ assert _lesson_end_speech(conn, "澶嶈缁撴潫") == "澶嶈缁撴潫銆傛垜浠潵鑱婂ぉ鍚с��"
+ assert _lesson_end_speech(conn, "璺熻缁撴潫") == "璺熻缁撴潫銆傛垜浠潵鑱婂ぉ鍚с��"
--
Gitblit v1.9.1