import time import json import asyncio from typing import TYPE_CHECKING if TYPE_CHECKING: from core.connection import ConnectionHandler from core.utils.util import audio_to_data from core.handle.abortHandle import handleAbortMessage from core.handle.intentHandler import handle_user_intent from core.utils.output_counter import check_device_output_limit from core.handle.sendAudioHandle import send_stt_message, SentenceType TAG = __name__ async def handleAudioMessage(conn: "ConnectionHandler", pcm_frame): # 当前片段是否有人说话 have_voice = conn.vad.is_vad(conn, pcm_frame) mode = getattr(conn, "interaction_mode", "CHAT") # 跟读听窗必须收齐 PCM。just_woken_up 的 2s 丢帧只用于对话,避免跟读前几句落成空 wav。 if mode != "SHADOWING" and hasattr(conn, "just_woken_up") and conn.just_woken_up: have_voice = False # 设置一个短暂延迟后恢复VAD检测 if not hasattr(conn, "vad_resume_task") or conn.vad_resume_task.done(): conn.vad_resume_task = asyncio.create_task(resume_vad_detection(conn)) return # 服务端AEC:仅对话类模式允许打断。泛听/跟读/翻译时喇叭回声不能当成插话。 if conn.client_aec and have_voice: if ( conn.client_is_speaking and conn.client_listen_mode != "manual" and mode in ("CHAT", "BILINGUAL") ): await handleAbortMessage(conn) # 设备长时间空闲检测,用于say goodbye await no_voice_close_connect(conn, have_voice) # 接收音频 await conn.asr.receive_audio(conn, pcm_frame, have_voice) async def resume_vad_detection(conn: "ConnectionHandler"): # 等待2秒后恢复VAD检测 await asyncio.sleep(2) conn.just_woken_up = False async def startToChat(conn: "ConnectionHandler", text): # 检查输入是否是JSON格式(包含说话人信息) speaker_name = None actual_text = text conn.last_asr_language = getattr(conn, "last_asr_language", None) try: # FunASR 等会把结果包成 {"content","language","emotion"};声纹场景还有 speaker if text.strip().startswith("{") and text.strip().endswith("}"): data = json.loads(text) if isinstance(data, dict) and "content" in data: speaker_name = data.get("speaker") actual_content = data.get("content") or "" if data.get("language"): conn.last_asr_language = data.get("language") if speaker_name: conn.logger.bind(tag=TAG).info(f"解析到说话人信息: {speaker_name}") # 仅在该说话人首次出现时保留 {"speaker":...} JSON,让模型自然称呼一次; # 后续轮降为纯文本,避免每轮重复出现名字诱导模型反复称呼 if speaker_name not in conn.introduced_speakers: conn.introduced_speakers.add(speaker_name) actual_text = text else: actual_text = actual_content elif actual_content: actual_text = actual_content except (json.JSONDecodeError, KeyError, TypeError): # 如果解析失败,继续使用原始文本 pass # 保存说话人信息到连接对象 if speaker_name: conn.current_speaker = speaker_name else: conn.current_speaker = None if conn.need_bind: await check_bind_device(conn) return # 如果当日的输出字数大于限定的字数 if conn.max_output_size > 0: if check_device_output_limit( conn.headers.get("device-id"), conn.max_output_size ): await max_out_size(conn) return from core.app_runtime.runtime import ( detect_user_language, handle_shadowing_turn, is_exit_to_chat, match_voice_switch, run_translation, start_app_task, wrap_bilingual_user_text, voice_menu_text, ) from core.handle.intentHandler import speak_txt mode = getattr(conn, "interaction_mode", "CHAT") # 菜单 TTS 里的「跟读/泛听/翻译」不能当成切 App;退出聊天口令仍要生效 if conn.client_is_speaking and getattr(conn, "awaiting_voice_app_choice", False): mapped = match_voice_switch(actual_text) if mapped and mapped[0] == "chat": conn.awaiting_voice_app_choice = False await start_app_task(conn, "chat", "conversation", {}, source="voice") return conn.logger.bind(tag=TAG).info("语音菜单播报中,忽略切 App") return if getattr(conn, "awaiting_voice_app_choice", False): mapped = match_voice_switch(actual_text) conn.awaiting_voice_app_choice = False if mapped and mapped[0] == "__reserved__": speak_txt(conn, "这个游戏还没开放,我们先聊天吧。") await start_app_task(conn, "chat", "conversation", {}, source="voice") return if mapped and mapped[0] == "__menu__": conn.awaiting_voice_app_choice = True speak_txt(conn, voice_menu_text()) return if mapped and mapped[0] == "chat": await start_app_task(conn, "chat", "conversation", {}, source="voice") return if mapped: await start_app_task(conn, mapped[0], mapped[1], {}, source="voice") return conn.logger.bind(tag=TAG).info("语音菜单未匹配,保持 chat") return mapped = match_voice_switch(actual_text) if mapped and mapped[0] == "__menu__": conn.awaiting_voice_app_choice = True speak_txt(conn, voice_menu_text()) return if mapped and mapped[0] == "__reserved__": speak_txt(conn, "这个还没开放,先继续当前内容吧。") return if conn.client_is_speaking and mapped and mapped[0] not in ("chat",): conn.logger.bind(tag=TAG).info(f"TTS 播放中忽略切 App: {mapped}") return if mapped and mapped[0] in ( "english_translation", "english_learning", "english_conversation", "chat", ): await start_app_task(conn, mapped[0], mapped[1], {}, source="voice") return if mode != "CHAT" and is_exit_to_chat(actual_text): await start_app_task(conn, "chat", "conversation", {}, source="voice") return if mode == "PLAY_AUDIO": conn.logger.bind(tag=TAG).info("泛听模式忽略上行对话") return if mode == "SHADOWING": await handle_shadowing_turn(conn, actual_text) return if mode == "TRANSLATION": await send_stt_message(conn, actual_text) conn.client_abort = False translated = await run_translation(conn, actual_text) if not translated: translated = "没有听清要翻译的内容,请再说一次。" if not conn.sentence_id: import uuid as _uuid conn.sentence_id = _uuid.uuid4().hex speak_txt(conn, translated) return if mode == "BILINGUAL": await send_stt_message(conn, actual_text) conn.client_abort = False user_lang = detect_user_language( actual_text, getattr(conn, "last_asr_language", None) ) wrapped = wrap_bilingual_user_text(actual_text, user_lang) conn.executor.submit(conn.chat, wrapped) return # manual 模式下不打断正在播放的内容 if conn.client_is_speaking and conn.client_listen_mode != "manual": await handleAbortMessage(conn) # 首先进行意图分析,使用实际文本内容 intent_handled = await handle_user_intent(conn, actual_text) if intent_handled: # 如果意图已被处理,不再进行聊天 return # 意图未被处理,继续常规聊天流程,使用实际文本内容 await send_stt_message(conn, actual_text) # 准备开始新会话 conn.client_abort = False conn.executor.submit(conn.chat, actual_text) async def no_voice_close_connect(conn: "ConnectionHandler", have_voice): if have_voice: conn.last_activity_time = time.time() * 1000 return # 只有在已经初始化过时间戳的情况下才进行超时检查 if conn.last_activity_time > 0.0: no_voice_time = time.time() * 1000 - conn.last_activity_time close_connection_no_voice_time = int( conn.config.get("close_connection_no_voice_time", 120) ) if ( not conn.close_after_chat and no_voice_time > 1000 * close_connection_no_voice_time ): conn.close_after_chat = True conn.client_abort = False end_prompt = conn.config.get("end_prompt", {}) if end_prompt and end_prompt.get("enable", True) is False: conn.logger.bind(tag=TAG).info("结束对话,无需发送结束提示语") await conn.close() return prompt = end_prompt.get("prompt") if not prompt: prompt = "请你以```时间过得真快```未来头,用富有感情、依依不舍的话来结束这场对话吧。!" await startToChat(conn, prompt) async def max_out_size(conn: "ConnectionHandler"): # 播放超出最大输出字数的提示 conn.client_abort = False text = "不好意思,我现在有点事情要忙,明天这个时候我们再聊,约好了哦!明天不见不散,拜拜!" await send_stt_message(conn, text) file_path = "config/assets/max_output_size.wav" opus_packets = await audio_to_data(file_path) conn.tts.tts_audio_queue.put((SentenceType.LAST, opus_packets, text)) conn.close_after_chat = True async def check_bind_device(conn: "ConnectionHandler"): conn.last_bind_prompt_time = time.time() if getattr(conn, "tts", None) is None: conn.logger.bind(tag=TAG).error("TTS未就绪,无法播放绑定码") return if conn.bind_code: # 确保bind_code是6位数字 if len(conn.bind_code) != 6: conn.logger.bind(tag=TAG).error(f"无效的绑定码格式: {conn.bind_code}") text = "绑定码格式错误,请检查配置。" await send_stt_message(conn, text) return text = f"请登录控制面板,输入{conn.bind_code},绑定设备。" await send_stt_message(conn, text) # 播放提示音 music_path = "config/assets/bind_code.wav" opus_packets = await audio_to_data(music_path) conn.tts.tts_audio_queue.put((SentenceType.FIRST, opus_packets, text)) # 逐个播放数字 for i in range(6): # 确保只播放6位数字 try: digit = conn.bind_code[i] num_path = f"config/assets/bind_code/{digit}.wav" num_packets = await audio_to_data(num_path) conn.tts.tts_audio_queue.put((SentenceType.MIDDLE, num_packets, None)) except Exception as e: conn.logger.bind(tag=TAG).error(f"播放数字音频失败: {e}") continue conn.tts.tts_audio_queue.put((SentenceType.LAST, [], None)) else: # 播放未绑定提示 conn.client_abort = False text = f"没有找到该设备的版本信息,请正确配置 OTA地址,然后重新编译固件。" await send_stt_message(conn, text) music_path = "config/assets/bind_not_found.wav" opus_packets = await audio_to_data(music_path) conn.tts.tts_audio_queue.put((SentenceType.LAST, opus_packets, text))