import time
|
import json
|
import asyncio
|
from typing import TYPE_CHECKING
|
|
if TYPE_CHECKING:
|
from core.connection import ConnectionHandler
|
from core.utils.util import audio_to_data
|
from core.handle.abortHandle import handleAbortMessage
|
from core.handle.intentHandler import handle_user_intent
|
from core.utils.output_counter import check_device_output_limit
|
from core.handle.sendAudioHandle import send_stt_message, SentenceType
|
|
TAG = __name__
|
|
|
async def handleAudioMessage(conn: "ConnectionHandler", pcm_frame):
|
# 当前片段是否有人说话
|
have_voice = conn.vad.is_vad(conn, pcm_frame)
|
mode = getattr(conn, "interaction_mode", "CHAT")
|
# 跟读听窗必须收齐 PCM。just_woken_up 的 2s 丢帧只用于对话，避免跟读前几句落成空 wav。
|
if mode != "SHADOWING" and hasattr(conn, "just_woken_up") and conn.just_woken_up:
|
have_voice = False
|
# 设置一个短暂延迟后恢复VAD检测
|
if not hasattr(conn, "vad_resume_task") or conn.vad_resume_task.done():
|
conn.vad_resume_task = asyncio.create_task(resume_vad_detection(conn))
|
return
|
# 服务端AEC：仅对话类模式允许打断。复读/跟读/翻译时喇叭回声不能当成插话。
|
if conn.client_aec and have_voice:
|
if (
|
conn.client_is_speaking
|
and conn.client_listen_mode != "manual"
|
and mode in ("CHAT", "BILINGUAL")
|
):
|
await handleAbortMessage(conn)
|
# 设备长时间空闲检测，用于say goodbye
|
await no_voice_close_connect(conn, have_voice)
|
# 接收音频
|
await conn.asr.receive_audio(conn, pcm_frame, have_voice)
|
|
|
async def resume_vad_detection(conn: "ConnectionHandler"):
|
# 等待2秒后恢复VAD检测
|
await asyncio.sleep(2)
|
conn.just_woken_up = False
|
|
|
async def startToChat(conn: "ConnectionHandler", text):
|
# 检查输入是否是JSON格式（包含说话人信息）
|
speaker_name = None
|
actual_text = text
|
conn.last_asr_language = getattr(conn, "last_asr_language", None)
|
|
try:
|
# FunASR 等会把结果包成 {"content","language","emotion"}；声纹场景还有 speaker
|
if text.strip().startswith("{") and text.strip().endswith("}"):
|
data = json.loads(text)
|
if isinstance(data, dict) and "content" in data:
|
speaker_name = data.get("speaker")
|
actual_content = data.get("content") or ""
|
if data.get("language"):
|
conn.last_asr_language = data.get("language")
|
if speaker_name:
|
conn.logger.bind(tag=TAG).info(f"解析到说话人信息: {speaker_name}")
|
# 仅在该说话人首次出现时保留 {"speaker":...} JSON，让模型自然称呼一次；
|
# 后续轮降为纯文本，避免每轮重复出现名字诱导模型反复称呼
|
if speaker_name not in conn.introduced_speakers:
|
conn.introduced_speakers.add(speaker_name)
|
actual_text = text
|
else:
|
actual_text = actual_content
|
elif actual_content:
|
actual_text = actual_content
|
except (json.JSONDecodeError, KeyError, TypeError):
|
# 如果解析失败，继续使用原始文本
|
pass
|
|
# 保存说话人信息到连接对象
|
if speaker_name:
|
conn.current_speaker = speaker_name
|
else:
|
conn.current_speaker = None
|
|
if conn.need_bind:
|
await check_bind_device(conn)
|
return
|
|
# 如果当日的输出字数大于限定的字数
|
if conn.max_output_size > 0:
|
if check_device_output_limit(
|
conn.headers.get("device-id"), conn.max_output_size
|
):
|
await max_out_size(conn)
|
return
|
|
from core.app_runtime.runtime import (
|
detect_user_language,
|
handle_shadowing_turn,
|
is_exit_to_chat,
|
match_voice_switch,
|
run_translation,
|
start_app_task,
|
wrap_bilingual_user_text,
|
voice_menu_text,
|
)
|
from core.handle.intentHandler import speak_txt
|
|
mode = getattr(conn, "interaction_mode", "CHAT")
|
|
# 菜单 TTS 里的「跟读/复读机/翻译」不能当成切 App；退出聊天口令仍要生效
|
if conn.client_is_speaking and getattr(conn, "awaiting_voice_app_choice", False):
|
mapped = match_voice_switch(actual_text)
|
if mapped and mapped[0] == "chat":
|
conn.awaiting_voice_app_choice = False
|
await start_app_task(conn, "chat", "conversation", {}, source="voice")
|
return
|
conn.logger.bind(tag=TAG).info("语音菜单播报中，忽略切 App")
|
return
|
|
if getattr(conn, "awaiting_voice_app_choice", False):
|
mapped = match_voice_switch(actual_text)
|
conn.awaiting_voice_app_choice = False
|
if mapped and mapped[0] == "__reserved__":
|
speak_txt(conn, "这个游戏还没开放，我们先聊天吧。")
|
await start_app_task(conn, "chat", "conversation", {}, source="voice")
|
return
|
if mapped and mapped[0] == "__menu__":
|
conn.awaiting_voice_app_choice = True
|
speak_txt(conn, voice_menu_text())
|
return
|
if mapped and mapped[0] == "chat":
|
await start_app_task(conn, "chat", "conversation", {}, source="voice")
|
return
|
if mapped:
|
await start_app_task(conn, mapped[0], mapped[1], {}, source="voice")
|
return
|
conn.logger.bind(tag=TAG).info("语音菜单未匹配，保持 chat")
|
return
|
|
mapped = match_voice_switch(actual_text)
|
if mapped and mapped[0] == "__menu__":
|
conn.awaiting_voice_app_choice = True
|
speak_txt(conn, voice_menu_text())
|
return
|
if mapped and mapped[0] == "__reserved__":
|
speak_txt(conn, "这个还没开放，先继续当前内容吧。")
|
return
|
if conn.client_is_speaking and mapped and mapped[0] not in ("chat",):
|
conn.logger.bind(tag=TAG).info(f"TTS 播放中忽略切 App: {mapped}")
|
return
|
if mapped and mapped[0] in (
|
"english_translation",
|
"english_learning",
|
"english_conversation",
|
"chat",
|
):
|
await start_app_task(conn, mapped[0], mapped[1], {}, source="voice")
|
return
|
|
if mode != "CHAT" and is_exit_to_chat(actual_text):
|
await start_app_task(conn, "chat", "conversation", {}, source="voice")
|
return
|
|
if mode == "PLAY_AUDIO":
|
conn.logger.bind(tag=TAG).info("复读机模式忽略上行对话")
|
return
|
|
if mode == "SHADOWING":
|
await handle_shadowing_turn(conn, actual_text)
|
return
|
|
if mode == "TRANSLATION":
|
await send_stt_message(conn, actual_text)
|
conn.client_abort = False
|
translated = await run_translation(conn, actual_text)
|
if not translated:
|
translated = "没有听清要翻译的内容，请再说一次。"
|
if not conn.sentence_id:
|
import uuid as _uuid
|
conn.sentence_id = _uuid.uuid4().hex
|
speak_txt(conn, translated)
|
return
|
|
if mode == "BILINGUAL":
|
await send_stt_message(conn, actual_text)
|
conn.client_abort = False
|
user_lang = detect_user_language(
|
actual_text, getattr(conn, "last_asr_language", None)
|
)
|
wrapped = wrap_bilingual_user_text(actual_text, user_lang)
|
conn.executor.submit(conn.chat, wrapped)
|
return
|
|
# manual 模式下不打断正在播放的内容
|
if conn.client_is_speaking and conn.client_listen_mode != "manual":
|
await handleAbortMessage(conn)
|
|
# 首先进行意图分析，使用实际文本内容
|
intent_handled = await handle_user_intent(conn, actual_text)
|
|
if intent_handled:
|
# 如果意图已被处理，不再进行聊天
|
return
|
|
# 意图未被处理，继续常规聊天流程，使用实际文本内容
|
await send_stt_message(conn, actual_text)
|
|
# 准备开始新会话
|
conn.client_abort = False
|
|
conn.executor.submit(conn.chat, actual_text)
|
|
|
async def no_voice_close_connect(conn: "ConnectionHandler", have_voice):
|
if have_voice:
|
conn.last_activity_time = time.time() * 1000
|
return
|
# 只有在已经初始化过时间戳的情况下才进行超时检查
|
if conn.last_activity_time > 0.0:
|
no_voice_time = time.time() * 1000 - conn.last_activity_time
|
close_connection_no_voice_time = int(
|
conn.config.get("close_connection_no_voice_time", 120)
|
)
|
if (
|
not conn.close_after_chat
|
and no_voice_time > 1000 * close_connection_no_voice_time
|
):
|
conn.close_after_chat = True
|
conn.client_abort = False
|
end_prompt = conn.config.get("end_prompt", {})
|
if end_prompt and end_prompt.get("enable", True) is False:
|
conn.logger.bind(tag=TAG).info("结束对话，无需发送结束提示语")
|
await conn.close()
|
return
|
prompt = end_prompt.get("prompt")
|
if not prompt:
|
prompt = "请你以```时间过得真快```未来头，用富有感情、依依不舍的话来结束这场对话吧。！"
|
await startToChat(conn, prompt)
|
|
|
async def max_out_size(conn: "ConnectionHandler"):
|
# 播放超出最大输出字数的提示
|
conn.client_abort = False
|
text = "不好意思，我现在有点事情要忙，明天这个时候我们再聊，约好了哦！明天不见不散，拜拜！"
|
await send_stt_message(conn, text)
|
file_path = "config/assets/max_output_size.wav"
|
opus_packets = await audio_to_data(file_path)
|
conn.tts.tts_audio_queue.put((SentenceType.LAST, opus_packets, text))
|
conn.close_after_chat = True
|
|
|
async def check_bind_device(conn: "ConnectionHandler"):
|
if conn.bind_code:
|
# 确保bind_code是6位数字
|
if len(conn.bind_code) != 6:
|
conn.logger.bind(tag=TAG).error(f"无效的绑定码格式: {conn.bind_code}")
|
text = "绑定码格式错误，请检查配置。"
|
await send_stt_message(conn, text)
|
return
|
|
text = f"请登录控制面板，输入{conn.bind_code}，绑定设备。"
|
await send_stt_message(conn, text)
|
|
# 播放提示音
|
music_path = "config/assets/bind_code.wav"
|
opus_packets = await audio_to_data(music_path)
|
conn.tts.tts_audio_queue.put((SentenceType.FIRST, opus_packets, text))
|
|
# 逐个播放数字
|
for i in range(6): # 确保只播放6位数字
|
try:
|
digit = conn.bind_code[i]
|
num_path = f"config/assets/bind_code/{digit}.wav"
|
num_packets = await audio_to_data(num_path)
|
conn.tts.tts_audio_queue.put((SentenceType.MIDDLE, num_packets, None))
|
except Exception as e:
|
conn.logger.bind(tag=TAG).error(f"播放数字音频失败: {e}")
|
continue
|
conn.tts.tts_audio_queue.put((SentenceType.LAST, [], None))
|
else:
|
# 播放未绑定提示
|
conn.client_abort = False
|
text = f"没有找到该设备的版本信息，请正确配置 OTA地址，然后重新编译固件。"
|
await send_stt_message(conn, text)
|
music_path = "config/assets/bind_not_found.wav"
|
opus_packets = await audio_to_data(music_path)
|
conn.tts.tts_audio_queue.put((SentenceType.LAST, opus_packets, text))
|
