From 659c470eba542383987aad227b7f918100f131fa Mon Sep 17 00:00:00 2001
From: wangzhibo <wangzhibo@shsening.com>
Date: 星期五, 11 九月 2026 15:09:05 +0800
Subject: [PATCH] 小智终端 5种 场景模式 OK
---
xiaozhi-esp32-server/main/xiaozhi-server/core/handle/receiveAudioHandle.py | 84 +++++++++++++++++++++++++++++++----------
1 files changed, 63 insertions(+), 21 deletions(-)
diff --git a/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/receiveAudioHandle.py b/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/receiveAudioHandle.py
index 2985600..e26fea1 100644
--- a/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/receiveAudioHandle.py
+++ b/xiaozhi-esp32-server/main/xiaozhi-server/core/handle/receiveAudioHandle.py
@@ -17,16 +17,21 @@
async def handleAudioMessage(conn: "ConnectionHandler", pcm_frame):
# 褰撳墠鐗囨鏄惁鏈変汉璇磋瘽
have_voice = conn.vad.is_vad(conn, pcm_frame)
- # 濡傛灉璁惧鍒氬垰琚敜閱掞紝鐭殏蹇界暐VAD妫�娴�
- if hasattr(conn, "just_woken_up") and conn.just_woken_up:
+ mode = getattr(conn, "interaction_mode", "CHAT")
+ # 璺熻鍚獥蹇呴』鏀堕綈 PCM銆俲ust_woken_up 鐨� 2s 涓㈠抚鍙敤浜庡璇濓紝閬垮厤璺熻鍓嶅嚑鍙ヨ惤鎴愮┖ wav銆�
+ if mode != "SHADOWING" and hasattr(conn, "just_woken_up") and conn.just_woken_up:
have_voice = False
# 璁剧疆涓�涓煭鏆傚欢杩熷悗鎭㈠VAD妫�娴�
if not hasattr(conn, "vad_resume_task") or conn.vad_resume_task.done():
conn.vad_resume_task = asyncio.create_task(resume_vad_detection(conn))
return
- # 鏈嶅姟绔疉EC鍔熻兘闇�瑕佸疄鏃惰Е鍙戞墦鏂�
+ # 鏈嶅姟绔疉EC锛氫粎瀵硅瘽绫绘ā寮忓厑璁告墦鏂�傚璇�/璺熻/缈昏瘧鏃跺枃鍙洖澹颁笉鑳藉綋鎴愭彃璇濄��
if conn.client_aec and have_voice:
- if conn.client_is_speaking and conn.client_listen_mode != "manual":
+ if (
+ conn.client_is_speaking
+ and conn.client_listen_mode != "manual"
+ and mode in ("CHAT", "BILINGUAL")
+ ):
await handleAbortMessage(conn)
# 璁惧闀挎椂闂寸┖闂叉娴嬶紝鐢ㄤ簬say goodbye
await no_voice_close_connect(conn, have_voice)
@@ -44,6 +49,7 @@
# 妫�鏌ヨ緭鍏ユ槸鍚︽槸JSON鏍煎紡锛堝寘鍚璇濅汉淇℃伅锛�
speaker_name = None
actual_text = text
+ conn.last_asr_language = getattr(conn, "last_asr_language", None)
try:
# FunASR 绛変細鎶婄粨鏋滃寘鎴� {"content","language","emotion"}锛涘０绾瑰満鏅繕鏈� speaker
@@ -52,6 +58,8 @@
if isinstance(data, dict) and "content" in data:
speaker_name = data.get("speaker")
actual_content = data.get("content") or ""
+ if data.get("language"):
+ conn.last_asr_language = data.get("language")
if speaker_name:
conn.logger.bind(tag=TAG).info(f"瑙ｆ瀽鍒拌璇濅汉淇℃伅: {speaker_name}")
# 浠呭湪璇ヨ璇濅汉棣栨鍑虹幇鏃朵繚鐣� {"speaker":...} JSON锛岃妯″瀷鑷劧绉板懠涓�娆★紱
@@ -86,15 +94,28 @@
return
from core.app_runtime.runtime import (
- is_exit_translation,
+ detect_user_language,
+ handle_shadowing_turn,
+ is_exit_to_chat,
match_voice_switch,
run_translation,
start_app_task,
- VOICE_MENU,
+ wrap_bilingual_user_text,
+ voice_menu_text,
)
from core.handle.intentHandler import speak_txt
mode = getattr(conn, "interaction_mode", "CHAT")
+
+ # 鑿滃崟 TTS 閲岀殑銆岃窡璇�/澶嶈鏈�/缈昏瘧銆嶄笉鑳藉綋鎴愬垏 App锛涢��鍑鸿亰澶╁彛浠や粛瑕佺敓鏁�
+ if conn.client_is_speaking and getattr(conn, "awaiting_voice_app_choice", False):
+ mapped = match_voice_switch(actual_text)
+ if mapped and mapped[0] == "chat":
+ conn.awaiting_voice_app_choice = False
+ await start_app_task(conn, "chat", "conversation", {}, source="voice")
+ return
+ conn.logger.bind(tag=TAG).info("璇煶鑿滃崟鎾姤涓紝蹇界暐鍒� App")
+ return
if getattr(conn, "awaiting_voice_app_choice", False):
mapped = match_voice_switch(actual_text)
@@ -105,39 +126,50 @@
return
if mapped and mapped[0] == "__menu__":
conn.awaiting_voice_app_choice = True
- speak_txt(conn, VOICE_MENU)
+ speak_txt(conn, voice_menu_text())
return
- if mapped and mapped[0] != "__menu__":
- params = {}
- await start_app_task(conn, mapped[0], mapped[1], params, source="voice")
+ if mapped and mapped[0] == "chat":
+ await start_app_task(conn, "chat", "conversation", {}, source="voice")
return
- speak_txt(conn, VOICE_MENU)
- conn.awaiting_voice_app_choice = True
+ if mapped:
+ await start_app_task(conn, mapped[0], mapped[1], {}, source="voice")
+ return
+ conn.logger.bind(tag=TAG).info("璇煶鑿滃崟鏈尮閰嶏紝淇濇寔 chat")
return
mapped = match_voice_switch(actual_text)
if mapped and mapped[0] == "__menu__":
conn.awaiting_voice_app_choice = True
- speak_txt(conn, VOICE_MENU)
+ speak_txt(conn, voice_menu_text())
return
if mapped and mapped[0] == "__reserved__":
speak_txt(conn, "杩欎釜杩樻病寮�鏀撅紝鍏堢户缁綋鍓嶅唴瀹瑰惂銆�")
return
- if mapped and mapped[0] in ("english_translation", "english_learning", "chat"):
- if mapped[0] == "english_learning" and mapped[1] == "repeater":
- speak_txt(conn, "澶嶈闇�瑕佹寚瀹� SD 鍗′笂鐨勬枃浠跺悕锛屾垨鐢ㄥ皬绋嬪簭寮�濮嬨��")
- return
+ if conn.client_is_speaking and mapped and mapped[0] not in ("chat",):
+ conn.logger.bind(tag=TAG).info(f"TTS 鎾斁涓拷鐣ュ垏 App: {mapped}")
+ return
+ if mapped and mapped[0] in (
+ "english_translation",
+ "english_learning",
+ "english_conversation",
+ "chat",
+ ):
await start_app_task(conn, mapped[0], mapped[1], {}, source="voice")
return
+ if mode != "CHAT" and is_exit_to_chat(actual_text):
+ await start_app_task(conn, "chat", "conversation", {}, source="voice")
+ return
+
if mode == "PLAY_AUDIO":
- conn.logger.bind(tag=TAG).info("PLAY_AUDIO 妯″紡蹇界暐涓婅瀵硅瘽")
+ conn.logger.bind(tag=TAG).info("澶嶈鏈烘ā寮忓拷鐣ヤ笂琛屽璇�")
+ return
+
+ if mode == "SHADOWING":
+ await handle_shadowing_turn(conn, actual_text)
return
if mode == "TRANSLATION":
- if is_exit_translation(actual_text):
- await start_app_task(conn, "chat", "conversation", {}, source="voice")
- return
await send_stt_message(conn, actual_text)
conn.client_abort = False
translated = await run_translation(conn, actual_text)
@@ -149,6 +181,16 @@
speak_txt(conn, translated)
return
+ if mode == "BILINGUAL":
+ await send_stt_message(conn, actual_text)
+ conn.client_abort = False
+ user_lang = detect_user_language(
+ actual_text, getattr(conn, "last_asr_language", None)
+ )
+ wrapped = wrap_bilingual_user_text(actual_text, user_lang)
+ conn.executor.submit(conn.chat, wrapped)
+ return
+
# manual 妯″紡涓嬩笉鎵撴柇姝ｅ湪鎾斁鐨勫唴瀹�
if conn.client_is_speaking and conn.client_listen_mode != "manual":
await handleAbortMessage(conn)
--
Gitblit v1.9.1