From 659c470eba542383987aad227b7f918100f131fa Mon Sep 17 00:00:00 2001
From: wangzhibo <wangzhibo@shsening.com>
Date: 星期五, 11 九月 2026 15:09:05 +0800
Subject: [PATCH] 小智终端 5种 场景模式 OK
---
xiaozhi-esp32-2.2.6/main/application.cc | 609 ++++++++++++++++++++++++++++++++++++++++++------------
1 files changed, 468 insertions(+), 141 deletions(-)
diff --git a/xiaozhi-esp32-2.2.6/main/application.cc b/xiaozhi-esp32-2.2.6/main/application.cc
index 643d8db..f4cb306 100644
--- a/xiaozhi-esp32-2.2.6/main/application.cc
+++ b/xiaozhi-esp32-2.2.6/main/application.cc
@@ -6,6 +6,7 @@
#include "mqtt_protocol.h"
#include "websocket_protocol.h"
#include "boards/sening-xiaozhi-s3/mqtt.h"
+#include "boards/sening-xiaozhi-s3/app_task.h"
#include "assets/lang_config.h"
#include "mcp_server.h"
#include "assets.h"
@@ -97,7 +98,7 @@
.arg = this,
.dispatch_method = ESP_TIMER_TASK,
.name = "clock_timer",
- .skip_unhandled_events = true
+ .skip_unhandled_events = false
};
esp_timer_create(&clock_timer_args, &clock_timer_handle_);
}
@@ -143,9 +144,14 @@
}
};
callbacks.on_offline_command_detected = [this](const std::string& command) {
- if (!power_off_standby_active_.load()) {
- Board_Handle_Offline_Command(command.c_str());
+ if (power_off_standby_active_.load()) {
+ return;
}
+ // AFE audio_detection only has a 4KB stack. Starting repeater (NVS, cJSON,
+ // SD list, MP3, JPG) on that task overflows and reboots.
+ Schedule([command]() {
+ Board_Handle_Offline_Command(command.c_str());
+ });
};
callbacks.on_vad_change = [this](bool speaking) {
xEventGroupSetBits(event_group_, MAIN_EVENT_VAD_CHANGE);
@@ -314,13 +320,66 @@
for (auto& task : tasks) {
task();
}
+ if (GetDeviceState() == kDeviceStateSpeaking) {
+ MaybeFinishTtsAfterPlayback();
+ }
}
if (bits & MAIN_EVENT_CLOCK_TICK) {
clock_ticks_++;
auto display = Board::GetInstance().GetDisplay();
display->UpdateStatusBar();
-
+ auto state = GetDeviceState();
+ if (state == kDeviceStateConnecting || state == kDeviceStateSpeaking ||
+ (state == kDeviceStateIdle && suppress_wake_until_listen_)) {
+ audio_service_.KeepCodecReadyForPlayback();
+ }
+ MaybeFinishTtsAfterPlayback();
+ if (suppress_wake_until_listen_ && state == kDeviceStateIdle) {
+ if (protocol_ && protocol_->IsAudioChannelConnected()) {
+ boot_wake_restore_ticks_ = 0;
+ if (greeting_listen_wait_ticks_ > 0) {
+ greeting_listen_wait_ticks_--;
+ if (greeting_listen_wait_ticks_ == 0) {
+ ESP_LOGW(TAG, "greeting TTS missing, start listening");
+ SetListeningMode(GetDefaultListeningMode());
+ }
+ }
+ } else if (++boot_wake_restore_ticks_ >= 20) {
+ ESP_LOGW(TAG, "boot websocket not opened, restore wake word");
+ suppress_wake_until_listen_ = false;
+ boot_wake_restore_ticks_ = 0;
+ audio_service_.EnableWakeWordDetection(true);
+ }
+ } else {
+ boot_wake_restore_ticks_ = 0;
+ }
+ if (state == kDeviceStateSpeaking && tts_audio_expected_ && !tts_audio_received_ &&
+ speaking_started_us_ > 0) {
+ const int64_t wait_ms = (esp_timer_get_time() - speaking_started_us_) / 1000;
+ if (wait_ms >= 1000 && (wait_ms / 1000) <= 8) {
+ ESP_LOGW(TAG, "TTS audio still missing after %lldms", (long long)wait_ms);
+ }
+ }
+ if (protocol_ && IsSdCardMp3Playing() && protocol_->IsAudioChannelConnected()) {
+ protocol_->MarkIncomingActivity();
+ }
+ if (!pending_detect_text_.empty() &&
+ GetDeviceState() == kDeviceStateIdle &&
+ protocol_ && !protocol_->IsAudioChannelConnected()) {
+ if (pending_detect_retry_ticks_ > 0) {
+ pending_detect_retry_ticks_--;
+ } else if (pending_detect_retries_left_ > 0) {
+ pending_detect_retries_left_--;
+ pending_detect_retry_ticks_ = 25;
+ ESP_LOGW(TAG, "retry listen detect after handshake fail: %s left=%d",
+ pending_detect_text_.c_str(), pending_detect_retries_left_);
+ SendListenDetect(pending_detect_text_, true);
+ } else {
+ pending_detect_text_.clear();
+ }
+ }
+
// Print debug info every 10 seconds
if (clock_ticks_ % 10 == 0) {
SystemInfo::PrintHeapStats();
@@ -376,6 +435,10 @@
ESP_LOGI(TAG, "Activation done");
SystemInfo::PrintHeapStats();
+ // 寮�鏈洪┈涓婅杩炶闊� WebSocket銆傚厛涓嶈鎷夎捣 AFE锛氬唴閮� SRAM 琚� WakeNet 鍗犳弧鏃�
+ // 绗竴娆℃彙鎵嬩細 10 绉掕秴鏃讹紝閲嶈瘯铏借兘杩炰笂锛屾杩庤瘝 opus 涔熷鏄撲涪銆�
+ suppress_wake_until_listen_ = true;
+ boot_wake_restore_ticks_ = 0;
SetDeviceState(kDeviceStateIdle);
has_server_time_ = ota_->HasServerTime();
@@ -388,7 +451,9 @@
// Release OTA object after activation is complete
ota_.reset();
auto& board = Board::GetInstance();
- board.SetPowerSaveLevel(PowerSaveLevel::LOW_POWER);
+ // 寮�鏈洪┈涓婂氨瑕佽繛璇煶 WebSocket 鎾杩庤瘝銆傝繖閲屽垏 LOW_POWER 浼氳闅忓悗鐨勬彙鎵�
+ // 鍗″湪 handshake timeout锛屾杩庤瘝鏍规湰鍙戜笉鍑烘潵銆備繚鎸侀珮鎬ц兘鐩村埌閫氶亾鎵撳紑銆�
+ board.SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE);
Schedule([this]() {
// Play the success sound to indicate the device is ready
@@ -582,6 +647,10 @@
protocol_->OnNetworkError([this](const std::string& message) {
last_error_message_ = message;
+ if (network_error_suppressed_) {
+ ESP_LOGW(TAG, "network error suppressed during retry: %s", message.c_str());
+ return;
+ }
xEventGroupSetBits(event_group_, MAIN_EVENT_ERROR);
});
@@ -589,14 +658,25 @@
auto shared_packet = std::shared_ptr<AudioStreamPacket>(packet.release());
Schedule([this, shared_packet]() {
auto state = GetDeviceState();
- if (state != kDeviceStateSpeaking && !tts_audio_expected_) {
+ // speak_txt 涓嶅彂鐙珛 tts start锛岄鍙� opus 鍙兘鏃╀簬 sentence_start銆�
+ // 寰呭懡/鑱嗗惉鏃朵笉瑕佷涪鍖咃紝浜ょ粰 ScheduleTtsAudio 绛夊埌 speaking 灏辩华鍐嶆挱銆�
+ if (state != kDeviceStateSpeaking && !tts_audio_expected_ &&
+ state != kDeviceStateListening && state != kDeviceStateIdle) {
return;
+ }
+ if (state == kDeviceStateListening || state == kDeviceStateIdle) {
+ tts_audio_expected_ = true;
}
tts_response_started_ = true;
+ ESP_LOGI(TAG, "TTS audio packet bytes=%u expected=%d ready=%d state=%d",
+ (unsigned)shared_packet->payload.size(),
+ tts_audio_expected_ ? 1 : 0,
+ tts_speaking_state_ready_ ? 1 : 0,
+ static_cast<int>(state));
ScheduleTtsAudio(shared_packet);
- if (state == kDeviceStateListening) {
+ if (state == kDeviceStateListening || state == kDeviceStateIdle) {
SetDeviceState(kDeviceStateSpeaking);
}
});
@@ -611,10 +691,25 @@
});
protocol_->OnAudioChannelClosed([this, &board]() {
- board.SetPowerSaveLevel(PowerSaveLevel::LOW_POWER);
+ if (opening_audio_channel_) {
+ ESP_LOGI(TAG, "Skip channel-closed teardown while opening/reusing websocket");
+ return;
+ }
+ // 璺熻/澶嶈杩樺湪鎾椂涓嶈鍒� LOW_POWER锛氬悗闈㈣ˉ鍙� playlist_completed 浼� handshake timeout銆�
+ if ((Board_IsShadowingTask() || Board_IsRepeaterTask()) && IsSdCardMp3Playing()) {
+ board.SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE);
+ } else {
+ board.SetPowerSaveLevel(PowerSaveLevel::LOW_POWER);
+ }
Schedule([this]() {
auto display = Board::GetInstance().GetDisplay();
opening_audio_channel_ = false;
+ if ((Board_IsShadowingTask() || Board_IsRepeaterTask()) && IsSdCardMp3Playing()) {
+ if (GetDeviceState() == kDeviceStateListening) {
+ SetDeviceState(kDeviceStateIdle);
+ }
+ return;
+ }
tts_audio_expected_ = false;
tts_response_started_ = false;
tts_audio_received_ = false;
@@ -656,14 +751,24 @@
tts_audio_expected_ = true;
tts_response_started_ = false;
tts_audio_received_ = false;
+ tts_stop_pending_ = false;
+ tts_stop_waiting_ = false;
+ tts_playback_empty_ticks_ = 0;
+ tts_stop_wait_ticks_ = 0;
tts_activity_generation_++;
+ audio_service_.SetAudioPowerInhibit(true);
+ audio_service_.KeepCodecReadyForPlayback();
+ auto state = GetDeviceState();
+ if (state == kDeviceStateListening || state == kDeviceStateIdle) {
+ SetDeviceState(kDeviceStateSpeaking);
+ }
+ ESP_LOGI(TAG, "TTS start output=%d",
+ Board::GetInstance().GetAudioCodec()->output_enabled() ? 1 : 0);
});
} else if (strcmp(state->valuestring, "stop") == 0) {
Schedule([this]() {
- if (!tts_response_started_ && !standby_requested_ && !power_off_requested_) {
- tts_stop_pending_ = true;
- return;
- }
+ ESP_LOGI(TAG, "TTS stop started=%d received=%d",
+ tts_response_started_, tts_audio_received_);
tts_stop_pending_ = true;
FinishTtsResponseAfterPlayback();
});
@@ -683,14 +788,17 @@
tts_audio_expected_ = true;
tts_response_started_ = true;
tts_activity_generation_++;
+ tts_playback_empty_ticks_ = 0;
+ audio_service_.SetAudioPowerInhibit(true);
+ audio_service_.KeepCodecReadyForPlayback();
if (IsSdCardMp3Playing() && !IsSdCardMp3Paused()) {
PauseSdCardMp3();
resume_sdcard_mp3_after_tts_ = true;
}
- if (GetDeviceState() == kDeviceStateListening) {
+ auto state = GetDeviceState();
+ if (state == kDeviceStateListening || state == kDeviceStateIdle) {
SetDeviceState(kDeviceStateSpeaking);
}
- ScheduleTtsIdleFallback();
display->SetChatMessage("assistant", message.c_str());
});
}
@@ -844,18 +952,58 @@
xEventGroupSetBits(event_group_, MAIN_EVENT_STOP_LISTENING);
}
+bool Application::SendListenDetectNow(const std::string& text) {
+ if (!protocol_ || text.empty()) {
+ return false;
+ }
+ ESP_LOGI(TAG, "Send listen detect now: %s connected=%d",
+ text.c_str(), protocol_->IsAudioChannelConnected() ? 1 : 0);
+ if (!protocol_->IsAudioChannelConnected()) {
+ return false;
+ }
+ Board::GetInstance().SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE);
+ protocol_->MarkIncomingActivity();
+ return protocol_->SendWakeWordDetected(text);
+}
+
void Application::SendListenDetect(const std::string& text, bool open_if_idle) {
Schedule([this, text, open_if_idle]() {
if (!protocol_ || text.empty()) {
return;
}
- if (protocol_->IsAudioChannelOpened()) {
- protocol_->SendWakeWordDetected(text);
+ ESP_LOGI(TAG, "Send listen detect: %s connected=%d",
+ text.c_str(), protocol_->IsAudioChannelConnected() ? 1 : 0);
+ if (protocol_->IsAudioChannelConnected()) {
+ Board::GetInstance().SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE);
+ protocol_->MarkIncomingActivity();
+ if (protocol_->SendWakeWordDetected(text)) {
+ return;
+ }
+ ESP_LOGW(TAG, "detect send failed, reopen audio channel");
+ }
+ if (!open_if_idle) {
return;
}
- if (open_if_idle) {
- WakeWordInvoke(text);
+ if (!Board::GetInstance().IsNetworkReady()) {
+ ESP_LOGW(TAG, "skip listen detect, network not ready: %s", text.c_str());
+ return;
}
+ auto state = GetDeviceState();
+ // 寮�鏈�/TTS/閰嶇綉鏃朵笉瑕� WakeWordInvoke锛氶偅浼氳繘 listening銆佸紑 AFE銆丷esetDecoder锛�
+ // 娆㈣繋璇嶈娓呮帀锛孎EED 鐜紦鍐蹭篃浼氱珛鍒绘拺婊°��
+ if (state == kDeviceStateSpeaking || state == kDeviceStateWifiConfiguring ||
+ state == kDeviceStateAudioTesting || state == kDeviceStateConnecting ||
+ state == kDeviceStateStarting || state == kDeviceStateActivating) {
+ ESP_LOGI(TAG, "defer listen detect until idle: state=%d", static_cast<int>(state));
+ return;
+ }
+ if (state != kDeviceStateIdle) {
+ return;
+ }
+ SetDeviceState(kDeviceStateConnecting);
+ Schedule([this, text]() {
+ ContinueOpenAudioChannelForDetect(text);
+ });
});
}
@@ -913,26 +1061,103 @@
if (GetDeviceState() != kDeviceStateConnecting) {
return;
}
- if (opening_audio_channel_) {
+ if (!OpenAudioChannelWithRetry(2)) {
+ SetDeviceState(kDeviceStateIdle);
return;
}
+ SetListeningMode(mode);
+}
+
+bool Application::OpenAudioChannelWithRetry(int attempts) {
+ if (!protocol_) {
+ return false;
+ }
+ if (protocol_->IsAudioChannelConnected()) {
+ protocol_->MarkIncomingActivity();
+ return true;
+ }
+ if (opening_audio_channel_) {
+ return false;
+ }
+ if (attempts < 1) {
+ attempts = 1;
+ }
+
opening_audio_channel_ = true;
- if (!protocol_->IsAudioChannelOpened()) {
- if (!Board::GetInstance().EnsureNetworkReadyForWake(30000)) {
- opening_audio_channel_ = false;
- SetDeviceState(kDeviceStateIdle);
- return;
+ // 鎻℃墜鏈熼棿鍋� AFE锛岄伩鍏嶅拰 Wi-Fi 鎶㈠唴閮� SRAM/CPU銆備笉瑕佸彟寤轰换鍔★細
+ // AFE 璧锋潵鍚庡唴閮ㄥ爢鍙墿鍑� KB锛寈TaskCreate(8KB) 浼氱洿鎺ュけ璐ャ��
+ audio_service_.EnableVoiceProcessing(false);
+ audio_service_.EnableWakeWordDetection(false);
+ Board::GetInstance().SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE);
+ audio_service_.SetAudioPowerInhibit(true);
+
+ bool ok = false;
+ for (int i = 0; i < attempts; ++i) {
+ if (protocol_->IsAudioChannelConnected()) {
+ ok = true;
+ break;
}
- if (!protocol_->OpenAudioChannel()) {
- opening_audio_channel_ = false;
- SetDeviceState(kDeviceStateIdle);
- return;
+ if (!Board::GetInstance().EnsureNetworkReadyForWake(30000)) {
+ ESP_LOGW(TAG, "OpenAudioChannel aborted, network not ready");
+ break;
+ }
+ Board::GetInstance().SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE);
+ const bool last = (i == attempts - 1);
+ network_error_suppressed_ = !last;
+ ok = protocol_->OpenAudioChannel();
+ network_error_suppressed_ = false;
+ if (ok) {
+ break;
+ }
+ ESP_LOGW(TAG, "OpenAudioChannel failed (%d/%d)", i + 1, attempts);
+ protocol_->CloseAudioChannel(false);
+ if (!last) {
+ ESP_LOGI(TAG, "Retry websocket handshake in 500ms");
+ vTaskDelay(pdMS_TO_TICKS(500));
+ audio_service_.EnableWakeWordDetection(false);
+ Board::GetInstance().SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE);
+ audio_service_.KeepCodecReadyForPlayback();
}
}
- opening_audio_channel_ = false;
- SetListeningMode(mode);
+ opening_audio_channel_ = false;
+ audio_service_.SetAudioPowerInhibit(false);
+ return ok;
+}
+
+void Application::ContinueOpenAudioChannelForDetect(const std::string& detect_text) {
+ if (power_off_standby_active_.load()) {
+ return;
+ }
+ if (GetDeviceState() != kDeviceStateConnecting) {
+ return;
+ }
+ listening_mode_ = GetDefaultListeningMode();
+ suppress_wake_until_listen_ = true;
+ if (!OpenAudioChannelWithRetry(2)) {
+ pending_detect_text_ = detect_text;
+ if (pending_detect_retries_left_ <= 0) {
+ pending_detect_retries_left_ = 3;
+ }
+ pending_detect_retry_ticks_ = 3;
+ suppress_wake_until_listen_ = false;
+ audio_service_.EnableWakeWordDetection(true);
+ SetDeviceState(kDeviceStateIdle);
+ ESP_LOGW(TAG, "boot websocket failed, will retry detect in 3s");
+ return;
+ }
+
+ pending_detect_text_.clear();
+ pending_detect_retries_left_ = 0;
+ pending_detect_retry_ticks_ = 0;
+ greeting_listen_wait_ticks_ = 8;
+ if (!detect_text.empty()) {
+ protocol_->MarkIncomingActivity();
+ protocol_->SendWakeWordDetected(detect_text);
+ }
+ // 淇濇寔寰呭懡涓斾笉寮� AFE锛氭杩庤瘝浠� idle 杩� speaking銆傝瀹屽啀 listening 寮�楹︺��
+ SetDeviceState(kDeviceStateIdle);
}
void Application::PauseSdCardMp3ForVoice() {
@@ -946,8 +1171,87 @@
if (resume_sdcard_mp3_after_voice_ && !sdcard_mp3_stop_requested_) {
resume_sdcard_mp3_after_voice_ = false;
ResumeSdCardMp3();
+ ApplyMp3CommandOnlyMic();
} else {
resume_sdcard_mp3_after_voice_ = false;
+ }
+}
+
+void Application::ApplyMp3CommandOnlyMic() {
+ if (GetDeviceState() == kDeviceStateWifiConfiguring ||
+ GetDeviceState() == kDeviceStateSpeaking) {
+ return;
+ }
+ if (!IsSdCardMp3Playing() || IsSdCardMp3Paused()) {
+ return;
+ }
+ // 闊抽鍦ㄦ挱锛氫笉涓婅褰曢煶/闂茶亰锛屽彧鐣欑绾垮彛浠わ紙閫�鍑哄璇�/璺熻銆佹垜浠亰澶╁惂锛夈��
+ audio_service_.EnableVoiceProcessing(false);
+ audio_service_.EnableWakeWordDetection(audio_service_.IsAfeWakeWord());
+ if (GetDeviceState() != kDeviceStateListening) {
+ return;
+ }
+ if (protocol_ && protocol_->IsAudioChannelOpened()) {
+ protocol_->SendStopListening();
+ }
+ // 鎾鏂囨椂鍒囧緟鍛斤細鍏虫帀 AFE/涓婅锛屽彧鐣欑绾垮彛浠ゃ�傝窡璇诲彞闂寸敱鎾畬鍚庡啀 ResumeListening銆�
+ Schedule([this]() {
+ if (GetDeviceState() == kDeviceStateListening &&
+ IsSdCardMp3Playing() && !IsSdCardMp3Paused()) {
+ SetDeviceState(kDeviceStateIdle);
+ }
+ });
+}
+
+void Application::ResumeListeningAfterMp3Clip() {
+ if (power_off_standby_active_.load()) {
+ return;
+ }
+ if (IsSdCardMp3Playing() && !IsSdCardMp3Paused() && !Board_IsShadowingTask()) {
+ return;
+ }
+ auto state = GetDeviceState();
+ // 寮�鏈烘杩庤瘝 / 杩涘満 TTS 鏈熼棿绂佹寮�楹︼細EnableVoiceProcessing 浼� ResetDecoder 骞跺杺 AFE銆�
+ if (state == kDeviceStateSpeaking || state == kDeviceStateWifiConfiguring ||
+ state == kDeviceStateAudioTesting || state == kDeviceStateConnecting ||
+ state == kDeviceStateStarting || state == kDeviceStateActivating) {
+ return;
+ }
+ if (!protocol_) {
+ return;
+ }
+ ESP_LOGI(TAG, "Resume listening after MP3 clip");
+ if (state == kDeviceStateListening) {
+ protocol_->SendStartListening(listening_mode_);
+ audio_service_.EnableVoiceProcessing(true);
+#ifdef CONFIG_WAKE_WORD_DETECTION_IN_LISTENING
+ audio_service_.EnableWakeWordDetection(audio_service_.IsAfeWakeWord());
+#else
+ audio_service_.EnableWakeWordDetection(false);
+#endif
+ return;
+ }
+ if (state == kDeviceStateIdle && !protocol_->IsAudioChannelConnected()) {
+ return;
+ }
+ skip_playback_wait_on_listen_ = true;
+ SetListeningMode(GetDefaultListeningMode());
+}
+
+void Application::CloseShadowingListenWindow()
+{
+ audio_service_.EnableVoiceProcessing(false);
+ audio_service_.EnableWakeWordDetection(audio_service_.IsAfeWakeWord());
+ if (protocol_ && protocol_->IsAudioChannelOpened()) {
+ while (auto packet = audio_service_.PopPacketFromSendQueue()) {
+ if (!protocol_->SendAudio(std::move(packet))) {
+ break;
+ }
+ }
+ protocol_->SendStopListening();
+ }
+ if (GetDeviceState() == kDeviceStateListening) {
+ SetDeviceState(kDeviceStateIdle);
}
}
@@ -1074,37 +1378,23 @@
return;
}
- opening_audio_channel_ = true;
- if (!Board::GetInstance().EnsureNetworkReadyForWake(30000)) {
- opening_audio_channel_ = false;
- SetDeviceState(kDeviceStateIdle);
+ if (!OpenAudioChannelWithRetry(2)) {
+ ResumeSdCardMp3AfterVoice();
audio_service_.EnableWakeWordDetection(true);
+ SetDeviceState(kDeviceStateIdle);
return;
}
- if (!protocol_->IsAudioChannelOpened()) {
- if (!protocol_->OpenAudioChannel()) {
- opening_audio_channel_ = false;
- ResumeSdCardMp3AfterVoice();
- audio_service_.EnableWakeWordDetection(true);
- return;
- }
- }
- opening_audio_channel_ = false;
ESP_LOGI(TAG, "Wake word detected: %s", wake_word.c_str());
#if CONFIG_SEND_WAKE_WORD_DATA
- // Encode and send the wake word data to the server
while (auto packet = audio_service_.PopWakeWordPacket()) {
if (!protocol_->SendAudio(std::move(packet))) {
break;
}
}
- // Set the chat state to wake word detected
protocol_->SendWakeWordDetected(wake_word);
SetListeningMode(GetDefaultListeningMode());
#else
- // Set flag to play popup sound after state changes to listening
- // (PlaySound here would be cleared by ResetDecoder in EnableVoiceProcessing)
play_popup_on_listening_ = true;
SetListeningMode(GetDefaultListeningMode());
#endif
@@ -1132,20 +1422,35 @@
case kDeviceStateUnknown:
case kDeviceStateIdle:
opening_audio_channel_ = false;
- tts_audio_expected_ = false;
- tts_response_started_ = false;
- tts_audio_received_ = false;
- tts_stop_pending_ = false;
- tts_stop_waiting_ = false;
- tts_idle_waiting_ = false;
- tts_speaking_state_ready_ = false;
- tts_activity_generation_++;
+ if (!suppress_wake_until_listen_) {
+ tts_audio_expected_ = false;
+ tts_response_started_ = false;
+ tts_audio_received_ = false;
+ tts_stop_pending_ = false;
+ tts_stop_waiting_ = false;
+ tts_idle_waiting_ = false;
+ tts_speaking_state_ready_ = false;
+ tts_activity_generation_++;
+ }
ResumeSdCardMp3AfterVoice();
display->SetStatus(Lang::Strings::STANDBY);
display->ClearChatMessages(); // Clear messages first
display->SetEmotion("neutral"); // Then set emotion (wechat mode checks child count)
audio_service_.EnableVoiceProcessing(false);
- audio_service_.EnableWakeWordDetection(true);
+ if (suppress_wake_until_listen_) {
+ audio_service_.EnableWakeWordDetection(false);
+ audio_service_.SetAudioPowerInhibit(true);
+ audio_service_.KeepCodecReadyForPlayback();
+ } else if (Board_IsRepeaterTask()) {
+ audio_service_.EnableVoiceProcessing(false);
+ audio_service_.EnableWakeWordDetection(audio_service_.IsAfeWakeWord());
+ audio_service_.SetAudioPowerInhibit(false);
+ } else if (IsSdCardMp3Playing() && !IsSdCardMp3Paused()) {
+ ApplyMp3CommandOnlyMic();
+ } else {
+ audio_service_.EnableWakeWordDetection(true);
+ audio_service_.SetAudioPowerInhibit(false);
+ }
break;
case kDeviceStateConnecting:
display->SetStatus(Lang::Strings::CONNECTING);
@@ -1157,28 +1462,25 @@
display->SetStatus(Lang::Strings::LISTENING);
display->SetEmotion("neutral");
- // Make sure the audio processor is running
- if (play_popup_on_listening_ || !audio_service_.IsAudioProcessorRunning()) {
- // For auto mode, wait for playback queue to be empty before enabling voice processing
- // This prevents audio truncation when STOP arrives late due to network jitter
- if (listening_mode_ == kListeningModeAutoStop) {
- audio_service_.WaitForPlaybackQueueEmpty();
+ if (IsSdCardMp3Playing() && !IsSdCardMp3Paused() && !Board_IsShadowingTask()) {
+ ApplyMp3CommandOnlyMic();
+ } else {
+ if (play_popup_on_listening_ || !audio_service_.IsAudioProcessorRunning()) {
+ if (listening_mode_ == kListeningModeAutoStop && !skip_playback_wait_on_listen_) {
+ audio_service_.WaitForPlaybackQueueEmpty();
+ }
+ protocol_->SendStartListening(listening_mode_);
}
-
- // Send the start listening command
- protocol_->SendStartListening(listening_mode_);
audio_service_.EnableVoiceProcessing(true);
- }
-
#ifdef CONFIG_WAKE_WORD_DETECTION_IN_LISTENING
- // Enable wake word detection in listening mode (configured via Kconfig)
- audio_service_.EnableWakeWordDetection(audio_service_.IsAfeWakeWord());
+ audio_service_.EnableWakeWordDetection(audio_service_.IsAfeWakeWord());
#else
- // Disable wake word detection in listening mode
- audio_service_.EnableWakeWordDetection(false);
+ audio_service_.EnableWakeWordDetection(false);
#endif
-
- // Play popup sound after ResetDecoder (in EnableVoiceProcessing) has been called
+ }
+ audio_service_.SetAudioPowerInhibit(false);
+ suppress_wake_until_listen_ = false;
+ skip_playback_wait_on_listen_ = false;
if (play_popup_on_listening_) {
play_popup_on_listening_ = false;
audio_service_.PlaySound(Lang::Sounds::OGG_POPUP);
@@ -1186,19 +1488,35 @@
break;
case kDeviceStateSpeaking:
tts_speaking_state_ready_ = false;
+ tts_playback_empty_ticks_ = 0;
+ tts_stop_wait_ticks_ = 0;
+ greeting_listen_wait_ticks_ = 0;
+ speaking_started_us_ = esp_timer_get_time();
display->SetStatus(Lang::Strings::SPEAKING);
if (listening_mode_ != kListeningModeRealtime) {
audio_service_.EnableVoiceProcessing(false);
- // Only AFE wake word can be detected in speaking mode
- audio_service_.EnableWakeWordDetection(audio_service_.IsAfeWakeWord());
+ // TTS 鍥炲０浼氭妸鑿滃崟閲岀殑銆岃窡璇�/鎴戜滑鑱婂ぉ鍚с�嶅惉鎴愬彛浠わ紝鑰屼笖 AFE 浼氭尋鐖嗗唴閮� SRAM銆�
+ audio_service_.EnableWakeWordDetection(false);
}
- audio_service_.ResetDecoder();
+ // 娆㈣繋璇嶇殑 opus 鍙兘宸茬粡鍦ㄥ緟鍛芥椂鍏ラ槦銆傝繖閲� ResetDecoder 浼氭竻鎺夊垰鍒扮殑 TTS锛�
+ // 鍚捣鏉ュ儚娌℃挱锛岄殢鍚庡張绛変笉鍒� tts stop锛屼竴鐩村仠鍦ㄨ璇濅腑銆�
+ if (!tts_audio_expected_ && !tts_audio_received_ && audio_service_.IsPlaybackQueueEmpty()) {
+ audio_service_.ResetDecoder();
+ }
+ audio_service_.SetAudioPowerInhibit(true);
+ audio_service_.KeepCodecReadyForPlayback();
+ ESP_LOGI(TAG, "Speaking: codec ready output=%d",
+ Board::GetInstance().GetAudioCodec()->output_enabled() ? 1 : 0);
tts_speaking_state_ready_ = true;
break;
case kDeviceStateWifiConfiguring:
audio_service_.EnableVoiceProcessing(false);
- audio_service_.EnableWakeWordDetection(true);
+ // MultiNet/AFE wake word consumes internal SRAM. The captive portal HTML
+ // and /scan both fail when free SRAM drops to hundreds of bytes
+ // (httpd send EAGAIN, blank page, empty AP list). Keep it off until
+ // WiFi is configured; Idle re-enables detection afterwards.
+ audio_service_.EnableWakeWordDetection(false);
break;
default:
// Do nothing
@@ -1459,19 +1777,31 @@
tts_stop_waiting_ = false;
tts_idle_waiting_ = false;
tts_speaking_state_ready_ = false;
- if (GetDeviceState() == kDeviceStateSpeaking) {
- if (listening_mode_ == kListeningModeManualStop) {
- SetDeviceState(kDeviceStateIdle);
- } else {
- SetDeviceState(kDeviceStateListening);
- }
- }
+ tts_playback_empty_ticks_ = 0;
+ tts_stop_wait_ticks_ = 0;
+ speaking_started_us_ = 0;
+ audio_service_.SetAudioPowerInhibit(false);
+ suppress_wake_until_listen_ = false;
+ pending_detect_text_.clear();
+ pending_detect_retries_left_ = 0;
ResumeSdCardMp3AfterVoice();
if (resume_sdcard_mp3_after_tts_ && !sdcard_mp3_stop_requested_) {
resume_sdcard_mp3_after_tts_ = false;
ResumeSdCardMp3();
} else {
resume_sdcard_mp3_after_tts_ = false;
+ }
+ if (GetDeviceState() == kDeviceStateSpeaking) {
+ if (listening_mode_ == kListeningModeManualStop ||
+ (IsSdCardMp3Playing() && !IsSdCardMp3Paused())) {
+ SetDeviceState(kDeviceStateIdle);
+ } else {
+ listening_mode_ = GetDefaultListeningMode();
+ SetListeningMode(listening_mode_);
+ }
+ }
+ if (IsSdCardMp3Playing() && !IsSdCardMp3Paused()) {
+ ApplyMp3CommandOnlyMic();
}
if (power_off_requested_) {
power_off_requested_ = false;
@@ -1486,18 +1816,22 @@
void Application::ScheduleTtsAudio(std::shared_ptr<AudioStreamPacket> packet) {
Schedule([this, packet]() {
- if (GetDeviceState() != kDeviceStateSpeaking || !tts_audio_expected_) {
+ auto state = GetDeviceState();
+ if (state != kDeviceStateSpeaking && state != kDeviceStateIdle &&
+ state != kDeviceStateListening) {
+ ESP_LOGW(TAG, "TTS audio ignored state=%d expected=%d",
+ static_cast<int>(state), tts_audio_expected_ ? 1 : 0);
return;
}
- if (!tts_speaking_state_ready_) {
- ScheduleTtsAudio(packet);
- return;
- }
+ tts_audio_expected_ = true;
tts_audio_received_ = true;
tts_activity_generation_++;
+ tts_playback_empty_ticks_ = 0;
+ audio_service_.KeepCodecReadyForPlayback();
auto audio_packet = std::make_unique<AudioStreamPacket>(std::move(*packet));
- audio_service_.PushPacketToDecodeQueue(std::move(audio_packet));
- ScheduleTtsIdleFallback();
+ if (!audio_service_.PushPacketToDecodeQueue(std::move(audio_packet), true)) {
+ ESP_LOGW(TAG, "TTS audio dropped, decode queue full");
+ }
if (tts_stop_pending_) {
FinishTtsResponseAfterPlayback();
}
@@ -1505,61 +1839,54 @@
}
void Application::FinishTtsResponseAfterPlayback() {
- if (tts_stop_waiting_) {
- return;
- }
- if (tts_audio_expected_ && !tts_audio_received_) {
- ScheduleTtsIdleFallback();
- return;
- }
+ tts_stop_pending_ = true;
tts_stop_waiting_ = true;
- xTaskCreate([](void* arg) {
- auto* app = static_cast<Application*>(arg);
- uint32_t generation = app->tts_activity_generation_;
- app->audio_service_.WaitForPlaybackQueueEmpty();
- vTaskDelay(pdMS_TO_TICKS(800));
- app->Schedule([app, generation]() {
- if (app->tts_activity_generation_ == generation) {
- app->FinishTtsResponse();
- } else {
- app->tts_stop_waiting_ = false;
- if (app->GetDeviceState() == kDeviceStateSpeaking) {
- if (app->tts_stop_pending_) {
- app->FinishTtsResponseAfterPlayback();
- } else {
- app->ScheduleTtsIdleFallback();
- }
- }
- }
- });
- vTaskDelete(NULL);
- }, "tts_finish", 4096, this, 2, NULL);
+ tts_stop_wait_ticks_ = 0;
}
-void Application::ScheduleTtsIdleFallback() {
- if (tts_idle_waiting_ || tts_stop_waiting_) {
+void Application::MaybeFinishTtsAfterPlayback() {
+ if (GetDeviceState() != kDeviceStateSpeaking) {
+ tts_playback_empty_ticks_ = 0;
+ tts_stop_wait_ticks_ = 0;
return;
}
- tts_idle_waiting_ = true;
- xTaskCreate([](void* arg) {
- auto* app = static_cast<Application*>(arg);
- uint32_t generation = app->tts_activity_generation_;
- vTaskDelay(pdMS_TO_TICKS(1500));
- app->Schedule([app, generation]() {
- app->tts_idle_waiting_ = false;
- if (app->tts_activity_generation_ != generation || app->GetDeviceState() != kDeviceStateSpeaking) {
- return;
- }
- if (app->tts_audio_received_) {
- app->FinishTtsResponseAfterPlayback();
- return;
- }
- if (app->tts_stop_pending_ || app->tts_audio_expected_) {
- app->FinishTtsResponse();
- }
- });
- vTaskDelete(NULL);
- }, "tts_idle", 4096, this, 2, NULL);
+
+ const bool playback_empty = audio_service_.IsPlaybackQueueEmpty();
+ if (playback_empty) {
+ tts_playback_empty_ticks_++;
+ } else {
+ tts_playback_empty_ticks_ = 0;
+ }
+
+ const int64_t elapsed_ms = speaking_started_us_ > 0
+ ? (esp_timer_get_time() - speaking_started_us_) / 1000
+ : 0;
+
+ if (tts_stop_waiting_ || tts_stop_pending_) {
+ tts_stop_wait_ticks_++;
+ const int empty_needed = tts_audio_received_ ? 1 : 2;
+ if (playback_empty && tts_playback_empty_ticks_ >= empty_needed) {
+ ESP_LOGI(TAG, "TTS done, return to listen empty=%d wait=%d received=%d",
+ tts_playback_empty_ticks_, tts_stop_wait_ticks_, tts_audio_received_);
+ FinishTtsResponse();
+ } else if (!playback_empty && tts_stop_wait_ticks_ >= 30) {
+ ESP_LOGW(TAG, "TTS playback stuck after stop, return to listen");
+ FinishTtsResponse();
+ }
+ return;
+ }
+
+ // 娆㈣繋璇�/闀垮洖绛旓細stop 娌″埌鏃讹紝鍙湪銆屼竴鐩存病鏀跺埌 opus銆嶆垨銆屾挱瀹屽悗闀挎椂闂寸┖闃熷垪銆嶆墠鍥炲惉銆�
+ // 涓嶈鐢ㄨ璇濆閽� 15s/闃熷垪绌� 2s锛氫腑鑻卞璇濋暱鍙ヤ細鍦ㄥ彞闂寸┖闅欐垨 15 绉掑琚帎鏂��
+ if (!tts_audio_received_ && elapsed_ms >= 5000) {
+ ESP_LOGW(TAG, "TTS stop missing, return to listen elapsed=%lldms received=0 empty=%d",
+ (long long)elapsed_ms, tts_playback_empty_ticks_);
+ FinishTtsResponse();
+ } else if (playback_empty && tts_audio_received_ && tts_playback_empty_ticks_ >= 8) {
+ ESP_LOGW(TAG, "TTS stop missing after idle playback elapsed=%lldms empty=%d",
+ (long long)elapsed_ms, tts_playback_empty_ticks_);
+ FinishTtsResponse();
+ }
}
bool Application::CanEnterSleepMode() {
--
Gitblit v1.9.1