From 659c470eba542383987aad227b7f918100f131fa Mon Sep 17 00:00:00 2001
From: wangzhibo <wangzhibo@shsening.com>
Date: 星期五, 11 九月 2026 15:09:05 +0800
Subject: [PATCH] 小智终端 5种 场景模式 OK
---
xiaozhi-esp32-server/main/xiaozhi-server/core/connection.py | 90 ++++++++++++++++++++++++++++++++++++++++-----
1 files changed, 80 insertions(+), 10 deletions(-)
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/core/connection.py b/xiaozhi-esp32-server/main/xiaozhi-server/core/connection.py
index 6e5f417..69a53f3 100644
--- a/xiaozhi-esp32-server/main/xiaozhi-server/core/connection.py
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/core/connection.py
@@ -45,6 +45,7 @@
from core.utils.voiceprint_provider import VoiceprintProvider
from core.utils.util import get_system_error_response
from core.utils import textUtils
+from core.websocket_handshake import is_client_hello, is_priority_listen_detect
TAG = __name__
@@ -122,6 +123,7 @@
self.loop = None # 鍦� handle_connection 涓幏鍙栬繍琛屼腑鐨勪簨浠跺惊鐜�
self.stop_event = threading.Event()
self.executor = ThreadPoolExecutor(max_workers=5)
+ self._tts_channel_lock = threading.Lock()
# 娣诲姞涓婃姤绾跨▼姹�
self.report_queue = queue.Queue()
@@ -209,6 +211,18 @@
self.tool_allowlist = []
self.original_system_prompt = None
self.awaiting_voice_app_choice = False
+ self.shadowing_index = 0
+ self.shadowing_prompt = ""
+ self.shadowing_clips = []
+ self.shadowing_awaiting_user = False
+ self.shadowing_clip_gen = 0
+ self.shadowing_timeout_task = None
+ self.shadowing_assess_index = 0
+ self.last_asr_language = None
+ self.task_session = None
+ self.last_asr_audio_snapshot = None
+ self.shadowing_listen_pcm = []
+ self.shadowing_audio_saved_index = -1
async def handle_connection(self, ws: websockets.ServerConnection):
try:
@@ -232,6 +246,8 @@
# 璁よ瘉閫氳繃,缁х画澶勭悊
self.websocket = ws
+ if self.server is not None:
+ await self.server.register_device_handler(self.device_id, self)
# 妫�鏌ユ槸鍚︽潵鑷狹QTT杩炴帴
request_path = ws.request.path
@@ -249,7 +265,7 @@
# 鍚姩AEC缂撳瓨娓呯悊浠诲姟
self._aec_cache_cleanup_task = asyncio.create_task(self._check_aec_cache_expiry())
- self.welcome_msg = self.config["xiaozhi"]
+ self.welcome_msg = copy.deepcopy(self.config.get("xiaozhi") or {})
self.welcome_msg["session_id"] = self.session_id
# 浠庨厤缃腑璇诲彇閲囨牱鐜�
@@ -354,6 +370,12 @@
async def _route_message(self, message):
"""娑堟伅璺敱"""
+ # hello 蹇呴』绔嬪埢搴旂瓟銆傜粦瀹氱姸鎬佽繕鍦ㄦ媺閰嶇疆鏃跺鏋滀涪鎺� hello锛�
+ # 璁惧浼氬崱鍦� handshake/hello 绛夊緟锛岃〃鐜颁负杩炰笉涓婅闊虫湇鍔°��
+ if is_client_hello(message) or is_priority_listen_detect(message):
+ await handleTextMessage(self, message)
+ return
+
# 妫�鏌ユ槸鍚﹀凡缁忚幏鍙栧埌鐪熷疄鐨勭粦瀹氱姸鎬�
if not self.bind_completed_event.is_set():
# 杩樻病鏈夎幏鍙栧埌鐪熷疄鐘舵�侊紝绛夊緟鐩村埌鑾峰彇鍒扮湡瀹炵姸鎬佹垨瓒呮椂
@@ -610,14 +632,45 @@
)
)
- def _initialize_components(self):
- try:
+ def _tts_config_ready(self) -> bool:
+ selected = (self.config.get("selected_module") or {}).get("TTS")
+ providers = self.config.get("TTS") or {}
+ return bool(selected) and selected in providers
+
+ async def _open_tts_channels(self):
+ """Open TTS worker threads. Must run on the connection event loop after TTS config exists."""
+ if self.tts is None and not self._tts_config_ready():
+ self.logger.bind(tag=TAG).info("TTS閰嶇疆灏氭湭浠庢櫤鎺у彴鍔犺浇锛屽欢鍚庢墦寮�閫氶亾")
+ return
+ with self._tts_channel_lock:
if self.tts is None:
self.tts = self._initialize_tts()
- # 鎵撳紑璇煶鍚堟垚閫氶亾
- asyncio.run_coroutine_threadsafe(
- self.tts.open_audio_channels(self), self.loop
- )
+ tts = self.tts
+ if tts is None:
+ self.logger.bind(tag=TAG).warning("TTS瀹炰緥涓嶅彲鐢紝鏃犳硶鎵撳紑閫氶亾")
+ return
+ await tts.open_audio_channels(self)
+ play = getattr(tts, "audio_play_priority_thread", None)
+ text_th = getattr(tts, "tts_priority_thread", None)
+ if play and play.is_alive() and text_th and text_th.is_alive():
+ self.logger.bind(tag=TAG).info("TTS閫氶亾宸叉墦寮�")
+ else:
+ self.logger.bind(tag=TAG).error("TTS閫氶亾鎵撳紑鍚庡伐浣滅嚎绋嬫湭杩愯")
+
+ def _ensure_tts_channels(self):
+ """Start TTS worker threads once; used after private-config init."""
+ try:
+ if self.loop is None:
+ return
+ fut = asyncio.run_coroutine_threadsafe(self._open_tts_channels(), self.loop)
+ fut.result(timeout=15)
+ except Exception as e:
+ self.logger.bind(tag=TAG).error(f"鎵撳紑TTS閫氶亾澶辫触: {e}")
+
+ def _initialize_components(self):
+ try:
+ if self.tts is not None:
+ self._ensure_tts_channels()
if self.need_bind:
self.bind_completed_event.set()
return
@@ -755,8 +808,11 @@
def _initialize_tts(self):
"""鍒濆鍖朤TS"""
tts = None
- if not self.need_bind:
+ if not self.need_bind and self._tts_config_ready():
tts = initialize_tts(self.config)
+ elif not self._tts_config_ready():
+ self.logger.bind(tag=TAG).info("璺宠繃TTS鍒濆鍖栵細閰嶇疆涓病鏈夊彲鐢ㄧ殑TTS妯″潡")
+ return None
if tts is None:
tts = DefaultTTS(self.config, delete_audio_file=True)
@@ -954,6 +1010,11 @@
modules = {}
if modules.get("tts", None) is not None:
self.tts = modules["tts"]
+ if self.tts is not None:
+ try:
+ await self._open_tts_channels()
+ except Exception as e:
+ self.logger.bind(tag=TAG).error(f"鎵撳紑TTS閫氶亾澶辫触: {e}")
if modules.get("vad", None) is not None:
self.vad = modules["vad"]
if modules.get("asr", None) is not None:
@@ -1534,11 +1595,11 @@
self.logger.bind(tag=TAG).info("鑱婂ぉ璁板綍涓婃姤绾跨▼宸查��鍑�")
- def _process_report(self, type, text, audio_data, report_time):
+ def _process_report(self, type, text, audio_data, report_time, task_instance_id=None, step_index=None):
"""澶勭悊涓婃姤浠诲姟"""
try:
# 鎵ц寮傛涓婃姤锛堝湪浜嬩欢寰幆涓繍琛岋級
- asyncio.run(report(self, type, text, audio_data, report_time))
+ asyncio.run(report(self, type, text, audio_data, report_time, task_instance_id, step_index))
except Exception as e:
self.logger.bind(tag=TAG).error(f"涓婃姤澶勭悊寮傚父: {e}")
finally:
@@ -1552,6 +1613,15 @@
async def close(self, ws=None):
"""璧勬簮娓呯悊鏂规硶"""
try:
+ if self.server is not None:
+ await self.server.unregister_device_handler(self.device_id, self)
+ try:
+ from core.app_runtime.session import end_session
+
+ end_session(self, "disconnected")
+ except Exception as session_error:
+ self.logger.bind(tag=TAG).warning(f"鏂嚎 flush TaskSession 澶辫触: {session_error}")
+
# 娓呯悊 VAD 杩炴帴璧勬簮
if (
hasattr(self, "vad")
--
Gitblit v1.9.1