#include "application.h" #include "board.h" #include "display.h" #include "system_info.h" #include "audio_codec.h" #include "mqtt_protocol.h" #include "websocket_protocol.h" #include "boards/sening-xiaozhi-s3/mqtt.h" #include "boards/sening-xiaozhi-s3/app_task.h" #include "assets/lang_config.h" #include "mcp_server.h" #include "assets.h" #include "settings.h" #include "boards/sening-xiaozhi-s3/system.h" #include #include #include #include #include #include #include #include #define TAG "Application" extern "C" void SDCard_Pause_Play(void) __attribute__((weak)); extern "C" void SDCard_Resume_Play(void) __attribute__((weak)); extern "C" bool SDCard_Is_Playing(void) __attribute__((weak)); extern "C" bool SDCard_Is_Paused(void) __attribute__((weak)); static bool IsSdCardMp3Playing() { return SDCard_Is_Playing && SDCard_Is_Playing(); } static bool IsSdCardMp3Paused() { return SDCard_Is_Paused && SDCard_Is_Paused(); } static bool ShouldHoldIdleForLessonWork() { return Board_IsLessonSyncing() || Board_IsRepeaterTask() || Board_IsShadowingTask() || Board_IsWaitingStartPhrase(); } static bool IsLessonDetectText(const std::string& text) { return text.rfind("[lesson_sync]", 0) == 0 || text.rfind("[app_task]english_learning|", 0) == 0; } static void PauseSdCardMp3() { if (SDCard_Pause_Play) { SDCard_Pause_Play(); } } static void ResumeSdCardMp3() { if (SDCard_Resume_Play) { SDCard_Resume_Play(); } } static bool IsInternalToolSpeech(const char* text) { if (!text) { return false; } while (*text == ' ' || *text == '\t') { ++text; } if (strncmp(text, "% self.", 7) == 0 || strncmp(text, "%self.", 6) == 0) { return true; } return (strstr(text, "\"arguments\"") && strstr(text, "\"trigger_time\"")) || strstr(text, "\"label\""); } static bool IsStandbyRequestText(const char* text) { if (!text) { return false; } return strstr(text, "待命") || strstr(text, "进入待机") || strstr(text, "进入待机状态") || strstr(text, "进入休眠") || strstr(text, "休眠状态") || strstr(text, "停止监听") || strstr(text, "别听了") || strstr(text, "不要听了"); } Application::Application() { event_group_ = xEventGroupCreate(); #if CONFIG_USE_DEVICE_AEC && CONFIG_USE_SERVER_AEC #error "CONFIG_USE_DEVICE_AEC and CONFIG_USE_SERVER_AEC cannot be enabled at the same time" #elif CONFIG_USE_DEVICE_AEC aec_mode_ = kAecOnDeviceSide; #elif CONFIG_USE_SERVER_AEC aec_mode_ = kAecOnServerSide; #else aec_mode_ = kAecOff; #endif esp_timer_create_args_t clock_timer_args = { .callback = [](void* arg) { Application* app = (Application*)arg; xEventGroupSetBits(app->event_group_, MAIN_EVENT_CLOCK_TICK); }, .arg = this, .dispatch_method = ESP_TIMER_TASK, .name = "clock_timer", .skip_unhandled_events = false }; esp_timer_create(&clock_timer_args, &clock_timer_handle_); } Application::~Application() { if (clock_timer_handle_ != nullptr) { esp_timer_stop(clock_timer_handle_); esp_timer_delete(clock_timer_handle_); } vEventGroupDelete(event_group_); } bool Application::SetDeviceState(DeviceState state) { return state_machine_.TransitionTo(state); } void Application::Initialize() { auto& board = Board::GetInstance(); ota_ = std::make_unique(); ota_->MarkCurrentVersionValid(); SetDeviceState(kDeviceStateStarting); // Setup the display auto display = board.GetDisplay(); display->SetupUI(); ESP_LOGI(TAG, "Application firmware version: %s", APP_FIRMWARE_VERSION); // Print board name/version info display->SetChatMessage("system", SystemInfo::GetUserAgent().c_str()); // Setup the audio service auto codec = board.GetAudioCodec(); audio_service_.Initialize(codec); audio_service_.Start(); AudioServiceCallbacks callbacks; callbacks.on_send_queue_available = [this]() { xEventGroupSetBits(event_group_, MAIN_EVENT_SEND_AUDIO); }; callbacks.on_wake_word_detected = [this](const std::string& wake_word) { if (!power_off_standby_active_.load()) { xEventGroupSetBits(event_group_, MAIN_EVENT_WAKE_WORD_DETECTED); } }; callbacks.on_offline_command_detected = [this](const std::string& command) { if (power_off_standby_active_.load()) { return; } // AFE audio_detection only has a 4KB stack. Starting repeater (NVS, cJSON, // SD list, MP3, JPG) on that task overflows and reboots. Schedule([command]() { Board_Handle_Offline_Command(command.c_str()); }); }; callbacks.on_vad_change = [this](bool speaking) { xEventGroupSetBits(event_group_, MAIN_EVENT_VAD_CHANGE); }; audio_service_.SetCallbacks(callbacks); // Add state change listeners state_machine_.AddStateChangeListener([this](DeviceState old_state, DeviceState new_state) { xEventGroupSetBits(event_group_, MAIN_EVENT_STATE_CHANGED); }); // Start the clock timer to update the status bar esp_timer_start_periodic(clock_timer_handle_, 1000000); // Add MCP common tools (only once during initialization) auto& mcp_server = McpServer::GetInstance(); mcp_server.AddCommonTools(); mcp_server.AddUserOnlyTools(); // Set network event callback for UI updates and network state handling board.SetNetworkEventCallback([this](NetworkEvent event, const std::string& data) { auto display = Board::GetInstance().GetDisplay(); switch (event) { case NetworkEvent::Scanning: display->ShowNotification(Lang::Strings::SCANNING_WIFI, 30000); xEventGroupSetBits(event_group_, MAIN_EVENT_NETWORK_DISCONNECTED); break; case NetworkEvent::Connecting: { if (data.empty()) { // Cellular network - registering without carrier info yet display->SetStatus(Lang::Strings::REGISTERING_NETWORK); } else { // WiFi or cellular with carrier info std::string msg = Lang::Strings::CONNECT_TO; msg += data; msg += "..."; display->ShowNotification(msg.c_str(), 30000); } break; } case NetworkEvent::Connected: { std::string msg = Lang::Strings::CONNECTED_TO; msg += data; display->ShowNotification(msg.c_str(), 30000); xEventGroupSetBits(event_group_, MAIN_EVENT_NETWORK_CONNECTED); break; } case NetworkEvent::Disconnected: xEventGroupSetBits(event_group_, MAIN_EVENT_NETWORK_DISCONNECTED); break; case NetworkEvent::ConnectionFailed: break; case NetworkEvent::WifiConfigModeEnter: // WiFi config mode enter is handled by WifiBoard internally break; case NetworkEvent::WifiConfigModeExit: // WiFi config mode exit is handled by WifiBoard internally break; // Cellular modem specific events case NetworkEvent::ModemDetecting: display->SetStatus(Lang::Strings::DETECTING_MODULE); break; case NetworkEvent::ModemErrorNoSim: Alert(Lang::Strings::ERROR, Lang::Strings::PIN_ERROR, "triangle_exclamation", Lang::Sounds::OGG_ERR_PIN); break; case NetworkEvent::ModemErrorRegDenied: Alert(Lang::Strings::ERROR, Lang::Strings::REG_ERROR, "triangle_exclamation", Lang::Sounds::OGG_ERR_REG); break; case NetworkEvent::ModemErrorInitFailed: Alert(Lang::Strings::ERROR, Lang::Strings::MODEM_INIT_ERROR, "triangle_exclamation", Lang::Sounds::OGG_EXCLAMATION); break; case NetworkEvent::ModemErrorTimeout: display->SetStatus(Lang::Strings::REGISTERING_NETWORK); break; } }); LoadLocalAssetsForWakeWord(); // Start network asynchronously board.StartNetwork(); // Update the status bar immediately to show the network state display->UpdateStatusBar(true); } void Application::Run() { main_loop_task_ = xTaskGetCurrentTaskHandle(); // Set the priority of the main task to 10 vTaskPrioritySet(nullptr, 10); const EventBits_t ALL_EVENTS = MAIN_EVENT_SCHEDULE | MAIN_EVENT_SEND_AUDIO | MAIN_EVENT_WAKE_WORD_DETECTED | MAIN_EVENT_VAD_CHANGE | MAIN_EVENT_CLOCK_TICK | MAIN_EVENT_ERROR | MAIN_EVENT_NETWORK_CONNECTED | MAIN_EVENT_NETWORK_DISCONNECTED | MAIN_EVENT_TOGGLE_CHAT | MAIN_EVENT_START_LISTENING | MAIN_EVENT_STOP_LISTENING | MAIN_EVENT_ACTIVATION_DONE | MAIN_EVENT_STATE_CHANGED; while (true) { auto bits = xEventGroupWaitBits(event_group_, ALL_EVENTS, pdTRUE, pdFALSE, portMAX_DELAY); if (bits & MAIN_EVENT_ERROR) { SetDeviceState(kDeviceStateIdle); Alert(Lang::Strings::ERROR, last_error_message_.c_str(), "circle_xmark", Lang::Sounds::OGG_EXCLAMATION); } if (bits & MAIN_EVENT_NETWORK_CONNECTED) { HandleNetworkConnectedEvent(); } if (bits & MAIN_EVENT_NETWORK_DISCONNECTED) { HandleNetworkDisconnectedEvent(); } if (bits & MAIN_EVENT_ACTIVATION_DONE) { HandleActivationDoneEvent(); } if (bits & MAIN_EVENT_STATE_CHANGED) { HandleStateChangedEvent(); } if (bits & MAIN_EVENT_TOGGLE_CHAT) { HandleToggleChatEvent(); } if (bits & MAIN_EVENT_START_LISTENING) { HandleStartListeningEvent(); } if (bits & MAIN_EVENT_STOP_LISTENING) { HandleStopListeningEvent(); } if (bits & MAIN_EVENT_SEND_AUDIO) { while (auto packet = audio_service_.PopPacketFromSendQueue()) { if (protocol_ && !protocol_->SendAudio(std::move(packet))) { break; } } } if (bits & MAIN_EVENT_WAKE_WORD_DETECTED) { HandleWakeWordDetectedEvent(); } if (bits & MAIN_EVENT_VAD_CHANGE) { if (GetDeviceState() == kDeviceStateListening) { auto led = Board::GetInstance().GetLed(); led->OnStateChanged(); } } if (bits & MAIN_EVENT_SCHEDULE) { std::unique_lock lock(mutex_); auto tasks = std::move(main_tasks_); lock.unlock(); for (auto& task : tasks) { task(); } if (GetDeviceState() == kDeviceStateSpeaking) { MaybeFinishTtsAfterPlayback(); } } if (bits & MAIN_EVENT_CLOCK_TICK) { clock_ticks_++; auto display = Board::GetInstance().GetDisplay(); display->UpdateStatusBar(); auto state = GetDeviceState(); if (state == kDeviceStateConnecting || state == kDeviceStateSpeaking || state == kDeviceStateActivating || (state == kDeviceStateIdle && suppress_wake_until_listen_)) { audio_service_.KeepCodecReadyForPlayback(); } MaybeFinishTtsAfterPlayback(); if (suppress_wake_until_listen_ && state == kDeviceStateIdle) { if (protocol_ && protocol_->IsAudioChannelConnected()) { boot_wake_restore_ticks_ = 0; if (greeting_listen_wait_ticks_ > 0) { greeting_listen_wait_ticks_--; if (greeting_listen_wait_ticks_ == 0) { if (ShouldHoldIdleForLessonWork()) { ESP_LOGI(TAG, "skip greeting listen while lesson task"); } else { ESP_LOGW(TAG, "greeting TTS missing, start listening"); SetListeningMode(GetDefaultListeningMode()); } } } } else if (Board_IsLessonSyncing()) { boot_wake_restore_ticks_ = 0; } else if (++boot_wake_restore_ticks_ >= 20) { ESP_LOGW(TAG, "boot websocket not opened, restore wake word"); suppress_wake_until_listen_ = false; boot_wake_restore_ticks_ = 0; audio_service_.EnableWakeWordDetection(true); } } else { boot_wake_restore_ticks_ = 0; } if (state == kDeviceStateSpeaking && tts_audio_expected_ && !tts_audio_received_ && speaking_started_us_ > 0) { const int64_t wait_ms = (esp_timer_get_time() - speaking_started_us_) / 1000; if (wait_ms >= 1000 && (wait_ms / 1000) <= 8) { ESP_LOGW(TAG, "TTS audio still missing after %dms", (int)wait_ms); } } if (protocol_ && IsSdCardMp3Playing() && protocol_->IsAudioChannelConnected()) { protocol_->MarkIncomingActivity(); } if (!pending_detect_text_.empty() && GetDeviceState() == kDeviceStateIdle && protocol_ && !protocol_->IsAudioChannelConnected()) { if (pending_detect_retry_ticks_ > 0) { pending_detect_retry_ticks_--; } else if (pending_detect_retries_left_ > 0) { pending_detect_retries_left_--; pending_detect_retry_ticks_ = 25; ESP_LOGW(TAG, "retry listen detect after handshake fail: %s left=%d", pending_detect_text_.c_str(), pending_detect_retries_left_); SendListenDetect(pending_detect_text_, true); } else { pending_detect_text_.clear(); } } if (hold_idle_for_prompt_ && !pending_detect_text_.empty() && GetDeviceState() == kDeviceStateIdle && protocol_ && protocol_->IsAudioChannelConnected()) { if (pending_detect_retry_ticks_ > 0) { pending_detect_retry_ticks_--; } else if (pending_detect_retries_left_ > 0) { pending_detect_retries_left_--; pending_detect_retry_ticks_ = 2; ESP_LOGW(TAG, "retry app_task detect: %s left=%d", pending_detect_text_.c_str(), pending_detect_retries_left_); SendListenDetectNow(pending_detect_text_); } else if (IsLessonDetectText(pending_detect_text_) || ShouldHoldIdleForLessonWork()) { ESP_LOGW(TAG, "lesson detect unanswered, keep idle: %s", pending_detect_text_.c_str()); pending_detect_text_.clear(); pending_detect_retries_left_ = 0; pending_detect_retry_ticks_ = 0; } else { ESP_LOGW(TAG, "app_task detect unanswered, start listening"); hold_idle_for_prompt_ = false; suppress_wake_until_listen_ = false; pending_detect_text_.clear(); SetListeningMode(GetDefaultListeningMode()); } } Board_PollPendingLessonPlayback(); // Print debug info every 10 seconds if (clock_ticks_ % 10 == 0) { SystemInfo::PrintHeapStats(); } } } } void Application::HandleNetworkConnectedEvent() { ESP_LOGI(TAG, "Network connected"); auto state = GetDeviceState(); if (state == kDeviceStateStarting || state == kDeviceStateWifiConfiguring) { // Network is ready, start activation SetDeviceState(kDeviceStateActivating); if (activation_task_handle_ != nullptr) { ESP_LOGW(TAG, "Activation task already running"); return; } xTaskCreate([](void* arg) { Application* app = static_cast(arg); app->ActivationTask(); app->activation_task_handle_ = nullptr; vTaskDelete(NULL); }, "activation", 4096 * 2, this, 2, &activation_task_handle_); } // Update the status bar immediately to show the network state auto display = Board::GetInstance().GetDisplay(); display->UpdateStatusBar(true); } void Application::HandleNetworkDisconnectedEvent() { auto& board = Board::GetInstance(); if (board.IsNetworkReady()) { return; } // Close current conversation when network disconnected auto state = GetDeviceState(); if (state == kDeviceStateConnecting || state == kDeviceStateListening || state == kDeviceStateSpeaking) { ESP_LOGI(TAG, "Closing audio channel due to network disconnection"); protocol_->CloseAudioChannel(); } // Update the status bar immediately to show the network state auto display = board.GetDisplay(); display->UpdateStatusBar(true); } void Application::HandleActivationDoneEvent() { ESP_LOGI(TAG, "Activation done"); SystemInfo::PrintHeapStats(); // 开机马上要连语音 WebSocket。先不要拉起 AFE:内部 SRAM 被 WakeNet 占满时 // 第一次握手会 10 秒超时,重试虽能连上,欢迎词 opus 也容易丢。 suppress_wake_until_listen_ = true; boot_wake_restore_ticks_ = 0; SetDeviceState(kDeviceStateIdle); has_server_time_ = ota_->HasServerTime(); auto display = Board::GetInstance().GetDisplay(); std::string message = std::string(Lang::Strings::VERSION) + ota_->GetCurrentVersion(); display->ShowNotification(message.c_str()); display->SetChatMessage("system", ""); // Release OTA object after activation is complete ota_.reset(); auto& board = Board::GetInstance(); // 开机马上就要连语音 WebSocket 播欢迎词。这里切 LOW_POWER 会让随后的握手 // 卡在 handshake timeout,欢迎词根本发不出来。保持高性能直到通道打开。 board.SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE); Schedule([this]() { // Play the success sound to indicate the device is ready audio_service_.PlaySound(Lang::Sounds::OGG_SUCCESS); Board::GetInstance().OnDeviceReady(); }); } void Application::LoadLocalAssetsForWakeWord() { auto& assets = Assets::GetInstance(); if (!assets.partition_valid() || local_assets_applied_) { return; } local_assets_applied_ = assets.Apply(false); } void Application::ActivationTask() { if (!ota_) { ota_ = std::make_unique(); } // Check for new assets version CheckAssetsVersion(); // Check for new firmware version CheckNewVersion(); // Initialize the protocol InitializeProtocol(); // Signal completion to main loop xEventGroupSetBits(event_group_, MAIN_EVENT_ACTIVATION_DONE); } void Application::CheckAssetsVersion() { // Only allow CheckAssetsVersion to be called once if (assets_version_checked_) { return; } assets_version_checked_ = true; auto& board = Board::GetInstance(); auto display = board.GetDisplay(); auto& assets = Assets::GetInstance(); if (!assets.partition_valid()) { ESP_LOGW(TAG, "Assets partition is disabled for board %s", BOARD_NAME); return; } Settings settings("assets", true); // Check if there is a new assets need to be downloaded std::string download_url = settings.GetString("download_url"); if (!download_url.empty()) { settings.EraseKey("download_url"); char message[256]; snprintf(message, sizeof(message), Lang::Strings::FOUND_NEW_ASSETS, download_url.c_str()); Alert(Lang::Strings::LOADING_ASSETS, message, "cloud_arrow_down", Lang::Sounds::OGG_UPGRADE); // Wait for the audio service to be idle for 3 seconds vTaskDelay(pdMS_TO_TICKS(3000)); SetDeviceState(kDeviceStateUpgrading); board.SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE); display->SetChatMessage("system", Lang::Strings::PLEASE_WAIT); bool success = assets.Download(download_url, [this, display](int progress, size_t speed) -> void { char buffer[32]; snprintf(buffer, sizeof(buffer), "%d%% %uKB/s", progress, static_cast(speed / 1024)); Schedule([display, message = std::string(buffer)]() { display->SetChatMessage("system", message.c_str()); }); }); board.SetPowerSaveLevel(PowerSaveLevel::LOW_POWER); vTaskDelay(pdMS_TO_TICKS(1000)); if (!success) { Alert(Lang::Strings::ERROR, Lang::Strings::DOWNLOAD_ASSETS_FAILED, "circle_xmark", Lang::Sounds::OGG_EXCLAMATION); vTaskDelay(pdMS_TO_TICKS(2000)); SetDeviceState(kDeviceStateActivating); return; } local_assets_applied_ = false; } // Apply assets if (!local_assets_applied_) { local_assets_applied_ = assets.Apply(); } display->SetChatMessage("system", ""); display->SetEmotion("microchip_ai"); } void Application::CheckNewVersion() { const int MAX_RETRY = 10; int retry_count = 0; int retry_delay = 10; // Initial retry delay in seconds auto& board = Board::GetInstance(); while (true) { auto display = board.GetDisplay(); display->SetStatus(Lang::Strings::CHECKING_NEW_VERSION); esp_err_t err = ota_->CheckVersion(); if (err != ESP_OK) { retry_count++; if (retry_count >= MAX_RETRY) { ESP_LOGE(TAG, "Too many retries, exit version check"); return; } char error_message[128]; snprintf(error_message, sizeof(error_message), "code=%d, url=%s", err, ota_->GetCheckVersionUrl().c_str()); char buffer[256]; snprintf(buffer, sizeof(buffer), Lang::Strings::CHECK_NEW_VERSION_FAILED, retry_delay, error_message); Alert(Lang::Strings::ERROR, buffer, "cloud_slash", Lang::Sounds::OGG_EXCLAMATION); ESP_LOGW(TAG, "Check new version failed, retry in %d seconds (%d/%d)", retry_delay, retry_count, MAX_RETRY); for (int i = 0; i < retry_delay; i++) { vTaskDelay(pdMS_TO_TICKS(1000)); if (GetDeviceState() == kDeviceStateIdle) { break; } } retry_delay *= 2; // Double the retry delay continue; } retry_count = 0; retry_delay = 10; // Reset retry delay if (ota_->HasNewVersion()) { if (UpgradeFirmware(ota_->GetFirmwareUrl(), ota_->GetFirmwareVersion())) { return; // This line will never be reached after reboot } // If upgrade failed, continue to normal operation } // No new version, mark the current version as valid ota_->MarkCurrentVersionValid(); if (!ota_->HasActivationCode() && !ota_->HasActivationChallenge()) { // Exit the loop if done checking new version break; } display->SetStatus(Lang::Strings::ACTIVATION); // Activation code is shown to the user and waiting for the user to input if (ota_->HasActivationCode()) { ShowActivationCode(ota_->GetActivationCode(), ota_->GetActivationMessage()); } // This will block the loop until the activation is done or timeout for (int i = 0; i < 10; ++i) { ESP_LOGI(TAG, "Activating... %d/%d", i + 1, 10); esp_err_t err = ota_->Activate(); if (err == ESP_OK) { break; } else if (err == ESP_ERR_TIMEOUT) { vTaskDelay(pdMS_TO_TICKS(3000)); } else { vTaskDelay(pdMS_TO_TICKS(10000)); } if (GetDeviceState() == kDeviceStateIdle) { break; } // 激活请求失败时再播一遍 6 位码,避免只在开机瞬间播一次被听漏。 if (ota_->HasActivationCode() && i < 9) { ShowActivationCode(ota_->GetActivationCode(), ota_->GetActivationMessage()); } } } } void Application::InitializeProtocol() { auto& board = Board::GetInstance(); auto display = board.GetDisplay(); auto codec = board.GetAudioCodec(); display->SetStatus(Lang::Strings::LOADING_PROTOCOL); if (ota_->HasMqttConfig()) { protocol_ = std::make_unique(); } else if (ota_->HasWebsocketConfig()) { protocol_ = std::make_unique(); } else { ESP_LOGW(TAG, "No protocol specified in the OTA config, using MQTT"); protocol_ = std::make_unique(); } protocol_->OnConnected([this]() { DismissAlert(); }); protocol_->OnNetworkError([this](const std::string& message) { last_error_message_ = message; if (network_error_suppressed_) { ESP_LOGW(TAG, "network error suppressed during retry: %s", message.c_str()); return; } xEventGroupSetBits(event_group_, MAIN_EVENT_ERROR); }); protocol_->OnIncomingAudio([this](std::unique_ptr packet) { auto shared_packet = std::shared_ptr(packet.release()); Schedule([this, shared_packet]() { auto state = GetDeviceState(); // speak_txt 不发独立 tts start,首句 opus 可能早于 sentence_start。 // 待命/聆听时不要丢包,交给 ScheduleTtsAudio 等到 speaking 就绪再播。 if (state != kDeviceStateSpeaking && !tts_audio_expected_ && state != kDeviceStateListening && state != kDeviceStateIdle) { return; } if (state == kDeviceStateListening || state == kDeviceStateIdle) { tts_audio_expected_ = true; if (!ShouldHoldIdleForLessonWork()) { hold_idle_for_prompt_ = false; } } tts_response_started_ = true; ESP_LOGD(TAG, "TTS audio packet bytes=%u expected=%d ready=%d state=%d", (unsigned)shared_packet->payload.size(), tts_audio_expected_ ? 1 : 0, tts_speaking_state_ready_ ? 1 : 0, static_cast(state)); ScheduleTtsAudio(shared_packet); if (state == kDeviceStateListening || state == kDeviceStateIdle) { SetDeviceState(kDeviceStateSpeaking); } }); }); protocol_->OnAudioChannelOpened([this, codec, &board]() { board.SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE); if (protocol_->server_sample_rate() != codec->output_sample_rate()) { ESP_LOGW(TAG, "Server sample rate %d does not match device output sample rate %d, resampling may cause distortion", protocol_->server_sample_rate(), codec->output_sample_rate()); } }); protocol_->OnAudioChannelClosed([this, &board]() { if (opening_audio_channel_) { ESP_LOGI(TAG, "Skip channel-closed teardown while opening/reusing websocket"); return; } // 通道关掉后会立刻切 idle、加载唤醒词。这时再进 MAX_MODEM, // MQTT 重连任务和 phy_track_pll 会把内部 SRAM 打穿,ESP_ERROR_CHECK 直接重启。 // 保持 PERFORMANCE,等 PowerSaveTimer 超时后再真正断网待命。 board.SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE); Schedule([this]() { auto display = Board::GetInstance().GetDisplay(); opening_audio_channel_ = false; if ((Board_IsShadowingTask() || Board_IsRepeaterTask()) && IsSdCardMp3Playing()) { if (GetDeviceState() == kDeviceStateListening) { SetDeviceState(kDeviceStateIdle); } return; } tts_audio_expected_ = false; tts_response_started_ = false; tts_audio_received_ = false; tts_stop_pending_ = false; tts_stop_waiting_ = false; tts_idle_waiting_ = false; tts_session_end_ = false; tts_speaking_state_ready_ = false; tts_activity_generation_++; audio_service_.ResetDecoder(); display->SetChatMessage("system", ""); SetDeviceState(kDeviceStateIdle); if (power_off_requested_) { power_off_requested_ = false; resume_sdcard_mp3_after_tts_ = false; resume_sdcard_mp3_after_voice_ = false; Board_Enter_Power_Off(); return; } if (standby_requested_) { standby_requested_ = false; } if (resume_sdcard_mp3_after_tts_ && !sdcard_mp3_stop_requested_) { resume_sdcard_mp3_after_tts_ = false; ResumeSdCardMp3(); } else { resume_sdcard_mp3_after_tts_ = false; } }); }); protocol_->OnIncomingJson([this, display](const cJSON* root) { // Parse JSON data auto type = cJSON_GetObjectItem(root, "type"); if (strcmp(type->valuestring, "tts") == 0) { auto state = cJSON_GetObjectItem(root, "state"); if (strcmp(state->valuestring, "start") == 0) { Schedule([this]() { aborted_ = false; if (!ShouldHoldIdleForLessonWork()) { hold_idle_for_prompt_ = false; } pending_detect_text_.clear(); pending_detect_retries_left_ = 0; pending_detect_retry_ticks_ = 0; tts_audio_expected_ = true; tts_response_started_ = false; tts_audio_received_ = false; tts_stop_pending_ = false; tts_stop_waiting_ = false; tts_session_end_ = false; tts_playback_empty_ticks_ = 0; tts_stop_wait_ticks_ = 0; tts_activity_generation_++; audio_service_.SetAudioPowerInhibit(true); audio_service_.KeepCodecReadyForPlayback(); auto state = GetDeviceState(); if (state == kDeviceStateListening || state == kDeviceStateIdle) { SetDeviceState(kDeviceStateSpeaking); } ESP_LOGI(TAG, "TTS start output=%d", Board::GetInstance().GetAudioCodec()->output_enabled() ? 1 : 0); }); } else if (strcmp(state->valuestring, "stop") == 0) { auto session_end_item = cJSON_GetObjectItem(root, "session_end"); const bool session_end = cJSON_IsTrue(session_end_item) || (cJSON_IsNumber(session_end_item) && session_end_item->valuedouble != 0); Schedule([this, session_end]() { ESP_LOGI(TAG, "TTS stop started=%d received=%d session_end=%d", tts_response_started_, tts_audio_received_, session_end ? 1 : 0); if (session_end) { tts_session_end_ = true; } tts_stop_pending_ = true; FinishTtsResponseAfterPlayback(); }); } else if (strcmp(state->valuestring, "sentence_start") == 0) { auto text = cJSON_GetObjectItem(root, "text"); if (cJSON_IsString(text)) { ESP_LOGI(TAG, "<< %s", text->valuestring); if (IsInternalToolSpeech(text->valuestring)) { ESP_LOGW(TAG, "Ignore internal tool speech: %s", text->valuestring); Schedule([this]() { tts_audio_expected_ = false; tts_response_started_ = false; }); return; } Schedule([this, display, message = std::string(text->valuestring)]() { if (!ShouldHoldIdleForLessonWork()) { hold_idle_for_prompt_ = false; } pending_detect_text_.clear(); pending_detect_retries_left_ = 0; pending_detect_retry_ticks_ = 0; tts_audio_expected_ = true; tts_response_started_ = true; tts_activity_generation_++; tts_playback_empty_ticks_ = 0; audio_service_.SetAudioPowerInhibit(true); audio_service_.KeepCodecReadyForPlayback(); if (Board_OnStartPhraseSentence(message.c_str())) { resume_sdcard_mp3_after_tts_ = false; resume_sdcard_mp3_after_voice_ = false; } else if (IsSdCardMp3Playing() && !IsSdCardMp3Paused()) { PauseSdCardMp3(); resume_sdcard_mp3_after_tts_ = true; } auto state = GetDeviceState(); if (state == kDeviceStateListening || state == kDeviceStateIdle) { SetDeviceState(kDeviceStateSpeaking); } display->SetChatMessage("assistant", message.c_str()); }); } } } else if (strcmp(type->valuestring, "stt") == 0) { auto text = cJSON_GetObjectItem(root, "text"); if (cJSON_IsString(text)) { if (IsSdCardMp3Playing() && !IsSdCardMp3Paused()) { PauseSdCardMp3(); resume_sdcard_mp3_after_tts_ = true; } ESP_LOGI(TAG, ">> %s", text->valuestring); BoardMqtt_PublishSttEvent(text->valuestring); if (IsStandbyRequestText(text->valuestring)) { EnterStandby(); } Schedule([display, message = std::string(text->valuestring)]() { display->SetChatMessage("user", message.c_str()); }); } } else if (strcmp(type->valuestring, "llm") == 0) { auto emotion = cJSON_GetObjectItem(root, "emotion"); if (cJSON_IsString(emotion)) { Schedule([display, emotion_str = std::string(emotion->valuestring)]() { display->SetEmotion(emotion_str.c_str()); }); } } else if (strcmp(type->valuestring, "mcp") == 0) { auto payload = cJSON_GetObjectItem(root, "payload"); if (cJSON_IsObject(payload)) { auto method = cJSON_GetObjectItem(payload, "method"); auto params = cJSON_GetObjectItem(payload, "params"); auto name = cJSON_IsObject(params) ? cJSON_GetObjectItem(params, "name") : nullptr; if (cJSON_IsString(method) && strcmp(method->valuestring, "tools/call") == 0 && cJSON_IsString(name)) { if (strcmp(name->valuestring, "self.mp3.stop") == 0) { sdcard_mp3_stop_requested_ = true; BoardMqtt_StopPlaylistPlayback(); resume_sdcard_mp3_after_tts_ = false; resume_sdcard_mp3_after_voice_ = false; } else if (strcmp(name->valuestring, "self.mp3.play") == 0) { sdcard_mp3_stop_requested_ = false; if (GetDeviceState() == kDeviceStateSpeaking) { PauseSdCardMp3(); resume_sdcard_mp3_after_tts_ = true; } } } McpServer::GetInstance().ParseMessage(payload); } } else if (strcmp(type->valuestring, "system") == 0) { auto command = cJSON_GetObjectItem(root, "command"); if (cJSON_IsString(command)) { ESP_LOGI(TAG, "System command: %s", command->valuestring); if (strcmp(command->valuestring, "reboot") == 0) { // Do a reboot if user requests a OTA update Schedule([this]() { Reboot(); }); } else { ESP_LOGW(TAG, "Unknown system command: %s", command->valuestring); } } } else if (strcmp(type->valuestring, "alert") == 0) { auto status = cJSON_GetObjectItem(root, "status"); auto message = cJSON_GetObjectItem(root, "message"); auto emotion = cJSON_GetObjectItem(root, "emotion"); if (cJSON_IsString(status) && cJSON_IsString(message) && cJSON_IsString(emotion)) { Alert(status->valuestring, message->valuestring, emotion->valuestring, Lang::Sounds::OGG_VIBRATION); } else { ESP_LOGW(TAG, "Alert command requires status, message and emotion"); } #if CONFIG_RECEIVE_CUSTOM_MESSAGE } else if (strcmp(type->valuestring, "custom") == 0) { auto payload = cJSON_GetObjectItem(root, "payload"); ESP_LOGI(TAG, "Received custom message: %s", cJSON_PrintUnformatted(root)); if (cJSON_IsObject(payload)) { Schedule([this, display, payload_str = std::string(cJSON_PrintUnformatted(payload))]() { display->SetChatMessage("system", payload_str.c_str()); }); } else { ESP_LOGW(TAG, "Invalid custom message format: missing payload"); } #endif } else { ESP_LOGW(TAG, "Unknown message type: %s", type->valuestring); } }); protocol_->Start(); } static void WaitUntilActivationPlaybackIdle(AudioService& audio) { audio.KeepCodecReadyForPlayback(); for (int i = 0; i < 150; ++i) { if (audio.IsPlaybackQueueEmpty()) { vTaskDelay(pdMS_TO_TICKS(150)); return; } audio.KeepCodecReadyForPlayback(); vTaskDelay(pdMS_TO_TICKS(100)); } } void Application::ShowActivationCode(const std::string& code, const std::string& message) { struct digit_sound { char digit; const std::string_view& sound; }; static const std::array digit_sounds{{ digit_sound{'0', Lang::Sounds::OGG_0}, digit_sound{'1', Lang::Sounds::OGG_1}, digit_sound{'2', Lang::Sounds::OGG_2}, digit_sound{'3', Lang::Sounds::OGG_3}, digit_sound{'4', Lang::Sounds::OGG_4}, digit_sound{'5', Lang::Sounds::OGG_5}, digit_sound{'6', Lang::Sounds::OGG_6}, digit_sound{'7', Lang::Sounds::OGG_7}, digit_sound{'8', Lang::Sounds::OGG_8}, digit_sound{'9', Lang::Sounds::OGG_9} }}; ESP_LOGI(TAG, "Speaking activation code: %s", code.c_str()); audio_service_.KeepCodecReadyForPlayback(); // 功放刚打开时前几百毫秒常是静音,先稳住再播,避免只听到提示音听不到数字。 vTaskDelay(pdMS_TO_TICKS(400)); // This sentence uses 9KB of SRAM, so we need to wait for it to finish Alert(Lang::Strings::ACTIVATION, message.c_str(), "link", Lang::Sounds::OGG_ACTIVATION); WaitUntilActivationPlaybackIdle(audio_service_); for (const auto& digit : code) { auto it = std::find_if(digit_sounds.begin(), digit_sounds.end(), [digit](const digit_sound& ds) { return ds.digit == digit; }); if (it != digit_sounds.end()) { ESP_LOGI(TAG, "Playing activation digit: %c", digit); audio_service_.KeepCodecReadyForPlayback(); audio_service_.PlaySound(it->sound); WaitUntilActivationPlaybackIdle(audio_service_); vTaskDelay(pdMS_TO_TICKS(250)); } } } void Application::Alert(const char* status, const char* message, const char* emotion, const std::string_view& sound) { ESP_LOGW(TAG, "Alert [%s] %s: %s", emotion, status, message); auto display = Board::GetInstance().GetDisplay(); display->SetStatus(status); display->SetEmotion(emotion); display->SetChatMessage("system", message); if (!sound.empty()) { audio_service_.PlaySound(sound); } } void Application::DismissAlert() { if (GetDeviceState() == kDeviceStateIdle) { auto display = Board::GetInstance().GetDisplay(); display->SetStatus(Lang::Strings::STANDBY); display->SetEmotion("neutral"); display->SetChatMessage("system", ""); } } void Application::ToggleChatState() { xEventGroupSetBits(event_group_, MAIN_EVENT_TOGGLE_CHAT); } void Application::StartListening() { xEventGroupSetBits(event_group_, MAIN_EVENT_START_LISTENING); } void Application::StopListening() { xEventGroupSetBits(event_group_, MAIN_EVENT_STOP_LISTENING); } bool Application::SendListenDetectNow(const std::string& text) { if (!protocol_ || text.empty()) { return false; } ESP_LOGI(TAG, "Send listen detect now: %s connected=%d", text.c_str(), protocol_->IsAudioChannelConnected() ? 1 : 0); if (!protocol_->IsAudioChannelConnected()) { return false; } Board::GetInstance().SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE); protocol_->MarkIncomingActivity(); return protocol_->SendWakeWordDetected(text); } void Application::HoldIdleForServerPrompt() { hold_idle_for_prompt_ = true; suppress_wake_until_listen_ = true; auto state = GetDeviceState(); if (state == kDeviceStateListening) { SetDeviceState(kDeviceStateIdle); } } void Application::ReleaseHoldIdle() { hold_idle_for_prompt_ = false; suppress_wake_until_listen_ = false; boot_wake_restore_ticks_ = 0; } void Application::StartListeningAfterPrompt() { SetListeningMode(GetDefaultListeningMode()); } void Application::ArmPendingDetectRetry(const std::string& text, int retries) { if (text.empty() || retries <= 0) { return; } pending_detect_text_ = text; pending_detect_retries_left_ = retries; pending_detect_retry_ticks_ = 2; } bool Application::IsAudioChannelConnected() const { return protocol_ && protocol_->IsAudioChannelConnected(); } void Application::AnnounceReturnedToChat() { pending_detect_text_.clear(); pending_detect_retries_left_ = 0; pending_detect_retry_ticks_ = 0; suppress_wake_until_listen_ = false; audio_service_.KeepCodecReadyForPlayback(); auto display = Board::GetInstance().GetDisplay(); display->SetStatus("对话模式"); display->SetEmotion("happy"); display->SetChatMessage("assistant", "我们来聊天吧。"); audio_service_.PlaySound(Lang::Sounds::OGG_SUCCESS); ESP_LOGI(TAG, "announced local return to chat"); } void Application::ReconnectAfterLesson(const std::string& detect) { // 课文结束 WS 已断:先释放 AFE,再握手补发 skill_event。 // Server 会播「泛听/跟读结束。我们来聊天吧。」说完再 listening。 pending_detect_text_.clear(); pending_detect_retries_left_ = 0; pending_detect_retry_ticks_ = 0; suppress_wake_until_listen_ = true; audio_service_.SetAudioPowerInhibit(false); audio_service_.KeepCodecReadyForPlayback(); audio_service_.EnableVoiceProcessing(false); audio_service_.EnableWakeWordDetection(false); audio_service_.ReleaseAfeRuntime(); auto state = GetDeviceState(); if (state == kDeviceStateListening || state == kDeviceStateSpeaking) { SetDeviceState(kDeviceStateIdle); } if (detect.empty() || !protocol_) { ESP_LOGW(TAG, "lesson reconnect skipped, stay local chat"); AnnounceReturnedToChat(); audio_service_.EnableWakeWordDetection(true); return; } // 泛听时长会把 WiFi 待机关掉(跟读有听窗所以不会)。不要用 IsNetworkReady() // 直接放弃:后面 OpenAudioChannel 会 EnsureNetworkReadyForWake,连上再播结束语。 ESP_LOGI(TAG, "lesson reconnect for %s", detect.c_str()); Board::GetInstance().SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE); SetDeviceState(kDeviceStateConnecting); Schedule([this, detect]() { ContinueOpenAudioChannelForDetect(detect); }); } void Application::SendListenDetect(const std::string& text, bool open_if_idle) { Schedule([this, text, open_if_idle]() { if (!protocol_ || text.empty()) { return; } ESP_LOGI(TAG, "Send listen detect: %s connected=%d", text.c_str(), protocol_->IsAudioChannelConnected() ? 1 : 0); if (protocol_->IsAudioChannelConnected()) { Board::GetInstance().SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE); protocol_->MarkIncomingActivity(); if (protocol_->SendWakeWordDetected(text)) { Board_NoteLessonStartDetectSent(); return; } ESP_LOGW(TAG, "detect send failed, reopen audio channel"); } if (!open_if_idle) { return; } if (!Board::GetInstance().IsNetworkReady()) { ESP_LOGW(TAG, "skip listen detect, network not ready: %s", text.c_str()); return; } auto state = GetDeviceState(); // 开机/TTS/配网时不要 WakeWordInvoke:那会进 listening、开 AFE、ResetDecoder, // 欢迎词被清掉,FEED 环缓冲也会立刻撑满。 if (state == kDeviceStateSpeaking || state == kDeviceStateWifiConfiguring || state == kDeviceStateAudioTesting || state == kDeviceStateConnecting || state == kDeviceStateStarting || state == kDeviceStateActivating) { ESP_LOGI(TAG, "defer listen detect until idle: state=%d", static_cast(state)); return; } if (state != kDeviceStateIdle) { return; } SetDeviceState(kDeviceStateConnecting); Schedule([this, text]() { ContinueOpenAudioChannelForDetect(text); }); }); } void Application::HandleToggleChatEvent() { if (power_off_standby_active_.load()) { ESP_LOGI(TAG, "Ignore toggle chat while power-off standby is active"); return; } auto state = GetDeviceState(); if (state == kDeviceStateActivating) { SetDeviceState(kDeviceStateIdle); return; } else if (state == kDeviceStateWifiConfiguring) { audio_service_.EnableAudioTesting(true); SetDeviceState(kDeviceStateAudioTesting); return; } else if (state == kDeviceStateAudioTesting) { audio_service_.EnableAudioTesting(false); SetDeviceState(kDeviceStateWifiConfiguring); return; } if (!protocol_) { ESP_LOGE(TAG, "Protocol not initialized"); return; } if (state == kDeviceStateIdle) { ListeningMode mode = GetDefaultListeningMode(); if (!protocol_->IsAudioChannelOpened()) { SetDeviceState(kDeviceStateConnecting); // Schedule to let the state change be processed first (UI update) Schedule([this, mode]() { ContinueOpenAudioChannel(mode); }); return; } SetListeningMode(mode); } else if (state == kDeviceStateSpeaking) { AbortSpeaking(kAbortReasonNone); } else if (state == kDeviceStateListening) { protocol_->CloseAudioChannel(); } } void Application::ContinueOpenAudioChannel(ListeningMode mode) { if (power_off_standby_active_.load()) { ESP_LOGI(TAG, "Ignore open audio channel while power-off standby is active"); return; } // Check state again in case it was changed during scheduling if (GetDeviceState() != kDeviceStateConnecting) { return; } if (!OpenAudioChannelWithRetry(2)) { SetDeviceState(kDeviceStateIdle); return; } SetListeningMode(mode); } bool Application::OpenAudioChannelWithRetry(int attempts) { if (!protocol_) { return false; } if (protocol_->IsAudioChannelConnected()) { protocol_->MarkIncomingActivity(); return true; } if (opening_audio_channel_) { return false; } if (attempts < 1) { attempts = 1; } opening_audio_channel_ = true; // 握手期间停 AFE,避免和 Wi-Fi 抢内部 SRAM/CPU。不要另建任务: // AFE 起来后内部堆只剩几 KB,xTaskCreate(8KB) 会直接失败。 audio_service_.EnableVoiceProcessing(false); audio_service_.EnableWakeWordDetection(false); const size_t dma_before = heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL | MALLOC_CAP_DMA); ESP_LOGI(TAG, "OpenAudioChannel DMA largest=%u free=%u", (unsigned)dma_before, (unsigned)heap_caps_get_free_size(MALLOC_CAP_INTERNAL)); if (dma_before < 8192) { ESP_LOGW(TAG, "low DMA, release AFE before websocket"); audio_service_.ClearWakeWordBuffers(); audio_service_.ReleaseAfeRuntime(); vTaskDelay(pdMS_TO_TICKS(50)); ESP_LOGI(TAG, "OpenAudioChannel DMA after AFE release=%u", (unsigned)heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL | MALLOC_CAP_DMA)); } Board::GetInstance().SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE); audio_service_.SetAudioPowerInhibit(true); bool ok = false; for (int i = 0; i < attempts; ++i) { if (protocol_->IsAudioChannelConnected()) { ok = true; break; } if (!Board::GetInstance().EnsureNetworkReadyForWake(30000)) { ESP_LOGW(TAG, "OpenAudioChannel aborted, network not ready"); break; } Board::GetInstance().SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE); const bool last = (i == attempts - 1); network_error_suppressed_ = !last; ok = protocol_->OpenAudioChannel(); network_error_suppressed_ = false; if (ok) { break; } ESP_LOGW(TAG, "OpenAudioChannel failed (%d/%d)", i + 1, attempts); protocol_->CloseAudioChannel(false); if (!last) { ESP_LOGI(TAG, "Retry websocket handshake in 500ms"); vTaskDelay(pdMS_TO_TICKS(500)); audio_service_.EnableWakeWordDetection(false); Board::GetInstance().SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE); audio_service_.KeepCodecReadyForPlayback(); } } opening_audio_channel_ = false; audio_service_.SetAudioPowerInhibit(false); return ok; } void Application::ContinueOpenAudioChannelForDetect(const std::string& detect_text) { if (power_off_standby_active_.load()) { return; } if (GetDeviceState() != kDeviceStateConnecting) { return; } listening_mode_ = GetDefaultListeningMode(); suppress_wake_until_listen_ = true; if (!OpenAudioChannelWithRetry(2)) { // 课文结束后通道已死时不要反复 handshake:会卡在「连接中」并关掉唤醒词。 const bool skip_retry = detect_text.rfind("[skill_event]", 0) == 0; if (skip_retry) { pending_detect_text_.clear(); pending_detect_retries_left_ = 0; pending_detect_retry_ticks_ = 0; } else { pending_detect_text_ = detect_text; if (pending_detect_retries_left_ <= 0) { pending_detect_retries_left_ = 3; } pending_detect_retry_ticks_ = 3; } suppress_wake_until_listen_ = false; SetDeviceState(kDeviceStateIdle); if (skip_retry) { ESP_LOGW(TAG, "websocket failed after lesson, keep idle chat without retry"); AnnounceReturnedToChat(); } else { ESP_LOGW(TAG, "boot websocket failed, will retry detect in 3s"); } if (IsLessonDetectText(detect_text) || ShouldHoldIdleForLessonWork()) { suppress_wake_until_listen_ = true; audio_service_.EnableWakeWordDetection(false); } else { audio_service_.EnableWakeWordDetection(true); } return; } pending_detect_text_.clear(); pending_detect_retries_left_ = 0; pending_detect_retry_ticks_ = 0; greeting_listen_wait_ticks_ = (IsLessonDetectText(detect_text) || ShouldHoldIdleForLessonWork()) ? 0 : 8; if (!detect_text.empty()) { protocol_->MarkIncomingActivity(); if (protocol_->SendWakeWordDetected(detect_text)) { Board_NoteLessonStartDetectSent(); } } // 保持待命且不开 AFE:欢迎词从 idle 进 speaking。说完再 listening 开麦。 SetDeviceState(kDeviceStateIdle); } void Application::PauseSdCardMp3ForVoice() { if (IsSdCardMp3Playing() && !IsSdCardMp3Paused()) { PauseSdCardMp3(); resume_sdcard_mp3_after_voice_ = true; } } void Application::ResumeSdCardMp3AfterVoice() { if (resume_sdcard_mp3_after_voice_ && !sdcard_mp3_stop_requested_) { resume_sdcard_mp3_after_voice_ = false; ResumeSdCardMp3(); ApplyMp3CommandOnlyMic(); } else { resume_sdcard_mp3_after_voice_ = false; } } void Application::ApplyMp3CommandOnlyMic() { if (GetDeviceState() == kDeviceStateWifiConfiguring || GetDeviceState() == kDeviceStateSpeaking) { return; } if (!IsSdCardMp3Playing() || IsSdCardMp3Paused()) { return; } // 音频在播:不上行录音/闲聊,只留离线口令(退出泛听/跟读、我们聊天吧)。 // 课文还在读 SD 时禁止拉起 AFE:会把 SDMMC DMA 需要的连续内部 SRAM 吃光,fopen abort。 audio_service_.EnableVoiceProcessing(false); const size_t largest_internal = heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL); if (!SDCard_Mp3OfflineWakeAllowed() || largest_internal < 32768) { audio_service_.EnableWakeWordDetection(false); if (!SDCard_Mp3OfflineWakeAllowed()) { ESP_LOGD(TAG, "offline wake deferred until MP3 cached"); } else { ESP_LOGW(TAG, "skip offline wake, internal largest=%u", (unsigned)largest_internal); } } else { audio_service_.EnableWakeWordDetection(audio_service_.IsAfeWakeWord()); } if (GetDeviceState() != kDeviceStateListening) { return; } if (protocol_ && protocol_->IsAudioChannelOpened()) { protocol_->SendStopListening(); } // 播课文时切待命:关掉 AFE/上行,只留离线口令。跟读句间由播完后再 ResumeListening。 Schedule([this]() { if (GetDeviceState() == kDeviceStateListening && IsSdCardMp3Playing() && !IsSdCardMp3Paused()) { SetDeviceState(kDeviceStateIdle); } }); } void Application::ResumeListeningAfterMp3Clip() { if (power_off_standby_active_.load()) { return; } if (IsSdCardMp3Playing() && !IsSdCardMp3Paused() && !Board_IsShadowingTask()) { return; } auto state = GetDeviceState(); // 开机欢迎词 / 进场 TTS 期间禁止开麦:EnableVoiceProcessing 会 ResetDecoder 并喂 AFE。 if (state == kDeviceStateSpeaking || state == kDeviceStateWifiConfiguring || state == kDeviceStateAudioTesting || state == kDeviceStateConnecting || state == kDeviceStateStarting || state == kDeviceStateActivating) { return; } if (!protocol_) { return; } ESP_LOGI(TAG, "Resume listening after MP3 clip"); if (state == kDeviceStateListening) { protocol_->SendStartListening(listening_mode_); audio_service_.EnableVoiceProcessing(true); #ifdef CONFIG_WAKE_WORD_DETECTION_IN_LISTENING audio_service_.EnableWakeWordDetection(audio_service_.IsAfeWakeWord()); #else audio_service_.EnableWakeWordDetection(false); #endif return; } if (state == kDeviceStateIdle && !protocol_->IsAudioChannelConnected()) { return; } skip_playback_wait_on_listen_ = true; SetListeningMode(GetDefaultListeningMode()); } void Application::CloseShadowingListenWindow() { audio_service_.EnableVoiceProcessing(false); // 下一句还要 fopen/解码,「跟读结束」TTS 还要 mbedtls AES。这里不要立刻拉起 // AFE 唤醒词,否则内部 SRAM 只剩 1KB 连续块,SD DMA / SSL 都会失败。 audio_service_.EnableWakeWordDetection(false); if (protocol_ && protocol_->IsAudioChannelOpened()) { while (auto packet = audio_service_.PopPacketFromSendQueue()) { if (!protocol_->SendAudio(std::move(packet))) { break; } } protocol_->SendStopListening(); } if (GetDeviceState() == kDeviceStateListening) { SetDeviceState(kDeviceStateIdle); } } void Application::HandleStartListeningEvent() { if (power_off_standby_active_.load()) { ESP_LOGI(TAG, "Ignore start listening while power-off standby is active"); return; } auto state = GetDeviceState(); if (state == kDeviceStateActivating) { SetDeviceState(kDeviceStateIdle); return; } else if (state == kDeviceStateWifiConfiguring) { audio_service_.EnableAudioTesting(true); SetDeviceState(kDeviceStateAudioTesting); return; } if (!protocol_) { ESP_LOGE(TAG, "Protocol not initialized"); return; } if (state == kDeviceStateIdle) { if (!protocol_->IsAudioChannelOpened()) { PauseSdCardMp3ForVoice(); SetDeviceState(kDeviceStateConnecting); // Schedule to let the state change be processed first (UI update) Schedule([this]() { ContinueOpenAudioChannel(kListeningModeManualStop); }); return; } PauseSdCardMp3ForVoice(); SetListeningMode(kListeningModeManualStop); } else if (state == kDeviceStateSpeaking) { AbortSpeaking(kAbortReasonNone); SetListeningMode(kListeningModeManualStop); } } void Application::HandleStopListeningEvent() { auto state = GetDeviceState(); if (state == kDeviceStateAudioTesting) { audio_service_.EnableAudioTesting(false); SetDeviceState(kDeviceStateWifiConfiguring); return; } else if (state == kDeviceStateListening) { if (protocol_) { protocol_->SendStopListening(); } SetDeviceState(kDeviceStateIdle); } } void Application::HandleWakeWordDetectedEvent() { if (power_off_standby_active_.load()) { ESP_LOGI(TAG, "Ignore wake word while power-off standby is active"); return; } if (!protocol_) { return; } auto state = GetDeviceState(); auto wake_word = audio_service_.GetLastWakeWord(); ESP_LOGI(TAG, "Wake word detected: %s (state: %d)", wake_word.c_str(), (int)state); if (state == kDeviceStateIdle) { PauseSdCardMp3ForVoice(); size_t dma_largest = heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL | MALLOC_CAP_DMA); // 再见后唤醒词 AFE 还占着内部堆,DMA 往往不够编码 opus。 // 旧逻辑会跳过编码却继续阻塞等 opus,主循环卡死,15s 后 codec 掉电,像软关机。 if (dma_largest < 12288) { ESP_LOGW(TAG, "low DMA before wake handshake, release AFE largest=%u", (unsigned)dma_largest); audio_service_.ReleaseAfeRuntime(); dma_largest = heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL | MALLOC_CAP_DMA); } if (dma_largest >= 8192) { audio_service_.EncodeWakeWord(); } else { ESP_LOGW(TAG, "skip wake opus encode, DMA largest=%u", (unsigned)dma_largest); audio_service_.AbortWakeWordEncode(); } auto wake_word = audio_service_.GetLastWakeWord(); if (!protocol_->IsAudioChannelOpened()) { SetDeviceState(kDeviceStateConnecting); // Schedule to let the state change be processed first (UI update), // then continue with OpenAudioChannel which may block for ~1 second Schedule([this, wake_word]() { ContinueWakeWordInvoke(wake_word); }); return; } Schedule([this, wake_word]() { ContinueWakeWordInvoke(wake_word); }); } else if (state == kDeviceStateSpeaking || state == kDeviceStateListening) { AbortSpeaking(kAbortReasonWakeWordDetected); // Clear send queue to avoid sending residues to server while (audio_service_.PopPacketFromSendQueue()); if (state == kDeviceStateListening) { protocol_->SendStartListening(GetDefaultListeningMode()); audio_service_.ResetDecoder(); audio_service_.PlaySound(Lang::Sounds::OGG_POPUP); // Re-enable wake word detection as it was stopped by the detection itself audio_service_.EnableWakeWordDetection(true); } else { // Play popup sound and start listening again play_popup_on_listening_ = true; SetListeningMode(GetDefaultListeningMode()); } } else if (state == kDeviceStateActivating) { // Restart the activation check if the wake word is detected during activation SetDeviceState(kDeviceStateIdle); } } void Application::ContinueWakeWordInvoke(const std::string& wake_word) { if (power_off_standby_active_.load()) { ESP_LOGI(TAG, "Ignore wake word invoke while power-off standby is active"); return; } // Check state again in case it was changed during scheduling auto state = GetDeviceState(); if (state != kDeviceStateConnecting && state != kDeviceStateIdle) { return; } if (opening_audio_channel_) { return; } if (!OpenAudioChannelWithRetry(2)) { ResumeSdCardMp3AfterVoice(); audio_service_.EnableWakeWordDetection(true); SetDeviceState(kDeviceStateIdle); return; } ESP_LOGI(TAG, "Wake word detected: %s", wake_word.c_str()); #if CONFIG_SEND_WAKE_WORD_DATA while (auto packet = audio_service_.PopWakeWordPacket()) { if (!protocol_->SendAudio(std::move(packet))) { break; } } protocol_->SendWakeWordDetected(wake_word); SetListeningMode(GetDefaultListeningMode()); #else play_popup_on_listening_ = true; SetListeningMode(GetDefaultListeningMode()); #endif } void Application::HandleStateChangedEvent() { DeviceState new_state = state_machine_.GetState(); clock_ticks_ = 0; auto& board = Board::GetInstance(); auto display = board.GetDisplay(); auto led = board.GetLed(); led->OnStateChanged(); if (power_off_standby_active_.load()) { opening_audio_channel_ = false; play_popup_on_listening_ = false; audio_service_.EnableVoiceProcessing(false); audio_service_.EnableWakeWordDetection(false); audio_service_.ResetDecoder(); return; } switch (new_state) { case kDeviceStateUnknown: case kDeviceStateIdle: opening_audio_channel_ = false; if (!suppress_wake_until_listen_) { tts_audio_expected_ = false; tts_response_started_ = false; tts_audio_received_ = false; tts_stop_pending_ = false; tts_stop_waiting_ = false; tts_idle_waiting_ = false; tts_session_end_ = false; tts_speaking_state_ready_ = false; tts_activity_generation_++; } ResumeSdCardMp3AfterVoice(); display->SetStatus(Lang::Strings::STANDBY); display->ClearChatMessages(); // Clear messages first display->SetEmotion("neutral"); // Then set emotion (wechat mode checks child count) audio_service_.EnableVoiceProcessing(false); // 再见断线会 speaking→listening→idle:processor AFE 没释放就把唤醒词 AFE 拉起来, // audio_detection 建不起来,feed 刷屏直到内部 SRAM 打穿。 if (!suppress_wake_until_listen_) { audio_service_.ReleaseVoiceProcessorRuntime(); } if (IsSdCardMp3Playing() && !IsSdCardMp3Paused()) { ApplyMp3CommandOnlyMic(); } else if (suppress_wake_until_listen_ || Board_IsLessonSyncing() || Board_IsRepeaterTask() || Board_IsShadowingTask()) { audio_service_.EnableWakeWordDetection(false); audio_service_.SetAudioPowerInhibit(true); audio_service_.KeepCodecReadyForPlayback(); } else { audio_service_.EnableWakeWordDetection(true); audio_service_.SetAudioPowerInhibit(false); } break; case kDeviceStateConnecting: display->SetStatus(Lang::Strings::CONNECTING); display->SetEmotion("neutral"); display->SetChatMessage("system", ""); break; case kDeviceStateListening: tts_speaking_state_ready_ = false; display->SetStatus(Lang::Strings::LISTENING); display->SetEmotion("neutral"); if (IsSdCardMp3Playing() && !IsSdCardMp3Paused() && !Board_IsShadowingTask()) { ApplyMp3CommandOnlyMic(); } else { if (play_popup_on_listening_ || !audio_service_.IsAudioProcessorRunning()) { if (listening_mode_ == kListeningModeAutoStop && !skip_playback_wait_on_listen_) { audio_service_.WaitForPlaybackQueueEmpty(); } protocol_->SendStartListening(listening_mode_); } audio_service_.EnableVoiceProcessing(true); #ifdef CONFIG_WAKE_WORD_DETECTION_IN_LISTENING audio_service_.EnableWakeWordDetection(audio_service_.IsAfeWakeWord()); #else audio_service_.EnableWakeWordDetection(false); #endif } audio_service_.SetAudioPowerInhibit(false); suppress_wake_until_listen_ = false; skip_playback_wait_on_listen_ = false; if (play_popup_on_listening_) { play_popup_on_listening_ = false; audio_service_.PlaySound(Lang::Sounds::OGG_POPUP); } break; case kDeviceStateSpeaking: tts_speaking_state_ready_ = false; tts_playback_empty_ticks_ = 0; tts_stop_wait_ticks_ = 0; greeting_listen_wait_ticks_ = 0; speaking_started_us_ = esp_timer_get_time(); display->SetStatus(Lang::Strings::SPEAKING); if (listening_mode_ != kListeningModeRealtime) { audio_service_.EnableVoiceProcessing(false); // TTS 回声会把菜单里的「跟读/我们聊天吧」听成口令,而且 AFE 会挤爆内部 SRAM。 audio_service_.EnableWakeWordDetection(false); } // 欢迎词的 opus 可能已经在待命时入队。这里 ResetDecoder 会清掉刚到的 TTS, // 听起来像没播,随后又等不到 tts stop,一直停在说话中。 if (!tts_audio_expected_ && !tts_audio_received_ && audio_service_.IsPlaybackQueueEmpty()) { audio_service_.ResetDecoder(); } audio_service_.SetAudioPowerInhibit(true); audio_service_.KeepCodecReadyForPlayback(); ESP_LOGI(TAG, "Speaking: codec ready output=%d", Board::GetInstance().GetAudioCodec()->output_enabled() ? 1 : 0); tts_speaking_state_ready_ = true; break; case kDeviceStateWifiConfiguring: audio_service_.EnableVoiceProcessing(false); // MultiNet/AFE wake word consumes internal SRAM. The captive portal HTML // and /scan both fail when free SRAM drops to hundreds of bytes // (httpd send EAGAIN, blank page, empty AP list). Keep it off until // WiFi is configured; Idle re-enables detection afterwards. audio_service_.EnableWakeWordDetection(false); break; default: // Do nothing break; } } bool Application::IsOnMainLoop() const { return main_loop_task_ != nullptr && xTaskGetCurrentTaskHandle() == main_loop_task_; } void Application::Schedule(std::function&& callback) { { std::lock_guard lock(mutex_); main_tasks_.push_back(std::move(callback)); } xEventGroupSetBits(event_group_, MAIN_EVENT_SCHEDULE); } void Application::AbortSpeaking(AbortReason reason) { ESP_LOGI(TAG, "Abort speaking"); aborted_ = true; if (protocol_) { protocol_->SendAbortSpeaking(reason); } } void Application::SetListeningMode(ListeningMode mode) { listening_mode_ = mode; SetDeviceState(kDeviceStateListening); } ListeningMode Application::GetDefaultListeningMode() const { return aec_mode_ == kAecOff ? kListeningModeAutoStop : kListeningModeRealtime; } void Application::Reboot() { ESP_LOGI(TAG, "Rebooting..."); // Disconnect the audio channel if (protocol_ && protocol_->IsAudioChannelOpened()) { protocol_->CloseAudioChannel(); } protocol_.reset(); audio_service_.Stop(); vTaskDelay(pdMS_TO_TICKS(1000)); esp_restart(); } bool Application::UpgradeFirmware(const std::string& url, const std::string& version, std::function progress_callback, std::function complete_callback) { auto& board = Board::GetInstance(); auto display = board.GetDisplay(); std::string upgrade_url = url; std::string version_info = version.empty() ? "(Manual upgrade)" : version; // Close audio channel if it's open if (protocol_ && protocol_->IsAudioChannelOpened()) { ESP_LOGI(TAG, "Closing audio channel before firmware upgrade"); protocol_->CloseAudioChannel(); } ESP_LOGI(TAG, "Starting firmware upgrade from URL: %s", upgrade_url.c_str()); Alert(Lang::Strings::OTA_UPGRADE, Lang::Strings::UPGRADING, "download", Lang::Sounds::OGG_UPGRADE); vTaskDelay(pdMS_TO_TICKS(3000)); AcquireSleepInhibit(); SetDeviceState(kDeviceStateUpgrading); std::string message = std::string(Lang::Strings::NEW_VERSION) + version_info; display->SetChatMessage("system", message.c_str()); board.SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE); audio_service_.Stop(); vTaskDelay(pdMS_TO_TICKS(1000)); bool upgrade_success = Ota::Upgrade(upgrade_url, [this, display, progress_callback](int progress, size_t speed) { char buffer[32]; snprintf(buffer, sizeof(buffer), "%d%% %uKB/s", progress, static_cast(speed / 1024)); Schedule([display, message = std::string(buffer)]() { display->SetChatMessage("system", message.c_str()); }); if (progress_callback) { progress_callback(progress, speed); } }); if (!upgrade_success) { // Upgrade failed, restart audio service and continue running ESP_LOGE(TAG, "Firmware upgrade failed, restarting audio service and continuing operation..."); audio_service_.Start(); // Restart audio service board.SetPowerSaveLevel(PowerSaveLevel::LOW_POWER); // Restore power save level ReleaseSleepInhibit(); if (complete_callback) { complete_callback(false); } Alert(Lang::Strings::ERROR, Lang::Strings::UPGRADE_FAILED, "circle_xmark", Lang::Sounds::OGG_EXCLAMATION); vTaskDelay(pdMS_TO_TICKS(3000)); return false; } else { // Upgrade success, reboot immediately ESP_LOGI(TAG, "Firmware upgrade successful, rebooting..."); display->SetChatMessage("system", "Upgrade successful, rebooting..."); vTaskDelay(pdMS_TO_TICKS(1000)); // Brief pause to show message ReleaseSleepInhibit(); if (complete_callback) { complete_callback(true); } Reboot(); return true; } } void Application::WakeWordInvoke(const std::string& wake_word) { if (power_off_standby_active_.load()) { ESP_LOGI(TAG, "Ignore wake word invoke while power-off standby is active"); return; } if (!protocol_) { return; } auto state = GetDeviceState(); if (state == kDeviceStateIdle) { audio_service_.EncodeWakeWord(); if (!protocol_->IsAudioChannelOpened()) { SetDeviceState(kDeviceStateConnecting); // Schedule to let the state change be processed first (UI update) Schedule([this, wake_word]() { ContinueWakeWordInvoke(wake_word); }); return; } Schedule([this, wake_word]() { ContinueWakeWordInvoke(wake_word); }); } else if (state == kDeviceStateSpeaking) { Schedule([this]() { AbortSpeaking(kAbortReasonNone); }); } else if (state == kDeviceStateListening) { Schedule([this]() { if (protocol_) { protocol_->CloseAudioChannel(); } }); } } void Application::EnterStandby() { Schedule([this]() { standby_requested_ = true; auto state = GetDeviceState(); if (state == kDeviceStateIdle) { standby_requested_ = false; } else if (state == kDeviceStateSpeaking && tts_stop_pending_) { FinishTtsResponseAfterPlayback(); } }); } void Application::RequestPowerOffStandby() { Schedule([this]() { power_off_requested_ = true; auto state = GetDeviceState(); if (state == kDeviceStateIdle) { power_off_requested_ = false; Board_Enter_Power_Off(); } else if (state == kDeviceStateSpeaking && tts_stop_pending_) { FinishTtsResponseAfterPlayback(); } }); } void Application::EnterPowerOffStandby() { power_off_standby_active_.store(true); Schedule([this]() { ESP_LOGI(TAG, "Entering power-off standby"); standby_requested_ = false; power_off_requested_ = false; opening_audio_channel_ = false; play_popup_on_listening_ = false; resume_sdcard_mp3_after_tts_ = false; resume_sdcard_mp3_after_voice_ = false; sdcard_mp3_stop_requested_ = false; tts_audio_expected_ = false; tts_response_started_ = false; tts_audio_received_ = false; tts_stop_pending_ = false; tts_stop_waiting_ = false; tts_idle_waiting_ = false; tts_session_end_ = false; tts_speaking_state_ready_ = false; tts_activity_generation_++; if (GetDeviceState() == kDeviceStateSpeaking) { AbortSpeaking(kAbortReasonNone); } if (protocol_ && protocol_->IsAudioChannelOpened()) { protocol_->CloseAudioChannel(); } audio_service_.EnableVoiceProcessing(false); audio_service_.EnableWakeWordDetection(false); audio_service_.ResetDecoder(); while (audio_service_.PopPacketFromSendQueue()) { } BoardMqtt_StopPlaylistPlayback(); auto state = GetDeviceState(); if (state == kDeviceStateConnecting || state == kDeviceStateListening || state == kDeviceStateSpeaking) { SetDeviceState(kDeviceStateIdle); } }); } void Application::RestoreFromPowerOffStandby(bool network_ready) { Schedule([this, network_ready]() { ESP_LOGI(TAG, "Restoring from power-off standby, network_ready=%d", network_ready); power_off_standby_active_.store(false); standby_requested_ = false; power_off_requested_ = false; opening_audio_channel_ = false; play_popup_on_listening_ = false; tts_activity_generation_++; audio_service_.EnableVoiceProcessing(false); audio_service_.ResetDecoder(); if (GetDeviceState() != kDeviceStateIdle) { SetDeviceState(kDeviceStateIdle); } audio_service_.EnableWakeWordDetection(true); if (!Board::GetInstance().OnRestoredFromPowerOff(network_ready)) { BoardMqtt_ResumePlaylistIfUnfinished(); } }); } void Application::AcquireSleepInhibit() { std::lock_guard lock(mutex_); sleep_inhibit_count_++; } void Application::ReleaseSleepInhibit() { std::lock_guard lock(mutex_); if (sleep_inhibit_count_ > 0) { sleep_inhibit_count_--; } } void Application::FinishTtsResponse() { const bool end_session = tts_session_end_; const bool hold_idle = hold_idle_for_prompt_; tts_audio_expected_ = false; tts_response_started_ = false; tts_audio_received_ = false; tts_stop_pending_ = false; tts_stop_waiting_ = false; tts_idle_waiting_ = false; tts_session_end_ = false; tts_speaking_state_ready_ = false; tts_playback_empty_ticks_ = 0; tts_stop_wait_ticks_ = 0; speaking_started_us_ = 0; audio_service_.SetAudioPowerInhibit(false); if (hold_idle || ShouldHoldIdleForLessonWork()) { suppress_wake_until_listen_ = true; } else { suppress_wake_until_listen_ = false; pending_detect_text_.clear(); pending_detect_retries_left_ = 0; pending_detect_retry_ticks_ = 0; } ResumeSdCardMp3AfterVoice(); if (resume_sdcard_mp3_after_tts_ && !sdcard_mp3_stop_requested_) { resume_sdcard_mp3_after_tts_ = false; ResumeSdCardMp3(); } else { resume_sdcard_mp3_after_tts_ = false; } if (GetDeviceState() == kDeviceStateSpeaking) { if (end_session || hold_idle || ShouldHoldIdleForLessonWork() || listening_mode_ == kListeningModeManualStop || (IsSdCardMp3Playing() && !IsSdCardMp3Paused())) { ESP_LOGI(TAG, "TTS done, return to idle session_end=%d hold=%d lesson=%d", end_session ? 1 : 0, hold_idle ? 1 : 0, ShouldHoldIdleForLessonWork() ? 1 : 0); SetDeviceState(kDeviceStateIdle); } else { listening_mode_ = GetDefaultListeningMode(); SetListeningMode(listening_mode_); } } if (IsSdCardMp3Playing() && !IsSdCardMp3Paused()) { ApplyMp3CommandOnlyMic(); } Board_FlushPendingLessonPlayback(); if (power_off_requested_) { power_off_requested_ = false; Board_Enter_Power_Off(); return; } if (standby_requested_) { standby_requested_ = false; SetDeviceState(kDeviceStateIdle); } } void Application::ScheduleTtsAudio(std::shared_ptr packet) { Schedule([this, packet]() { auto state = GetDeviceState(); if (state != kDeviceStateSpeaking && state != kDeviceStateIdle && state != kDeviceStateListening) { ESP_LOGW(TAG, "TTS audio ignored state=%d expected=%d", static_cast(state), tts_audio_expected_ ? 1 : 0); return; } tts_audio_expected_ = true; tts_audio_received_ = true; tts_activity_generation_++; tts_playback_empty_ticks_ = 0; audio_service_.KeepCodecReadyForPlayback(); auto audio_packet = std::make_unique(std::move(*packet)); if (!audio_service_.PushPacketToDecodeQueue(std::move(audio_packet), true)) { ESP_LOGW(TAG, "TTS audio dropped, decode queue full"); } if (tts_stop_pending_) { FinishTtsResponseAfterPlayback(); } }); } void Application::FinishTtsResponseAfterPlayback() { tts_stop_pending_ = true; tts_stop_waiting_ = true; tts_stop_wait_ticks_ = 0; } void Application::MaybeFinishTtsAfterPlayback() { if (GetDeviceState() != kDeviceStateSpeaking) { tts_playback_empty_ticks_ = 0; tts_stop_wait_ticks_ = 0; return; } const bool playback_empty = audio_service_.IsPlaybackQueueEmpty(); if (playback_empty) { tts_playback_empty_ticks_++; } else { tts_playback_empty_ticks_ = 0; } const int64_t elapsed_ms = speaking_started_us_ > 0 ? (esp_timer_get_time() - speaking_started_us_) / 1000 : 0; if (tts_stop_waiting_ || tts_stop_pending_) { tts_stop_wait_ticks_++; const int empty_needed = tts_audio_received_ ? 1 : 2; if (playback_empty && tts_playback_empty_ticks_ >= empty_needed) { ESP_LOGI(TAG, "TTS done empty=%d wait=%d received=%d session_end=%d", tts_playback_empty_ticks_, tts_stop_wait_ticks_, tts_audio_received_, tts_session_end_ ? 1 : 0); FinishTtsResponse(); } else if (!playback_empty && tts_stop_wait_ticks_ >= 30) { ESP_LOGW(TAG, "TTS playback stuck after stop, return to listen"); FinishTtsResponse(); } return; } // 欢迎词/长回答:stop 没到时,只在「一直没收到 opus」或「播完后长时间空队列」才回听。 // 不要用说话墙钟 15s/队列空 2s:中英对话长句会在句间空隙或 15 秒处被掐断。 if (!tts_audio_received_ && elapsed_ms >= 10000) { ESP_LOGW(TAG, "TTS stop missing, return to listen elapsed=%dms received=0 empty=%d", (int)elapsed_ms, tts_playback_empty_ticks_); FinishTtsResponse(); } else if (playback_empty && tts_audio_received_ && tts_playback_empty_ticks_ >= 8) { ESP_LOGW(TAG, "TTS stop missing after idle playback elapsed=%dms empty=%d", (int)elapsed_ms, tts_playback_empty_ticks_); FinishTtsResponse(); } } bool Application::CanEnterSleepMode() { { std::lock_guard lock(mutex_); if (sleep_inhibit_count_ > 0) { return false; } } if (GetDeviceState() != kDeviceStateIdle) { return false; } // 泛听是 idle 播 SD MP3,AudioService 队列是空的,会被当成可休眠。 // 跟读每句有听窗所以不会进 WiFi 待机;泛听会把网关掉,结束语就发不出去。 if (IsSdCardMp3Playing() || Board_IsRepeaterTask() || Board_IsShadowingTask()) { return false; } if (!audio_service_.IsIdle()) { return false; } // Now it is safe to enter sleep mode return true; } void Application::PrepareProtocolForNetworkStandby() { if (protocol_) { protocol_->PrepareForNetworkStandby(); } } void Application::SendMcpMessage(const std::string& payload) { // Always schedule to run in main task for thread safety Schedule([this, payload = std::move(payload)]() { if (protocol_ && !power_off_standby_active_.load()) { protocol_->SendMcpMessage(payload); } }); } void Application::SetAecMode(AecMode mode) { aec_mode_ = mode; Schedule([this]() { auto& board = Board::GetInstance(); auto display = board.GetDisplay(); switch (aec_mode_) { case kAecOff: audio_service_.EnableDeviceAec(false); display->ShowNotification(Lang::Strings::RTC_MODE_OFF); break; case kAecOnServerSide: audio_service_.EnableDeviceAec(false); display->ShowNotification(Lang::Strings::RTC_MODE_ON); break; case kAecOnDeviceSide: audio_service_.EnableDeviceAec(true); display->ShowNotification(Lang::Strings::RTC_MODE_ON); break; } // If the AEC mode is changed, close the audio channel if (protocol_ && protocol_->IsAudioChannelOpened()) { protocol_->CloseAudioChannel(); } }); } void Application::PlaySound(const std::string_view& sound) { audio_service_.PlaySound(sound); } void Application::ResetProtocol() { Schedule([this]() { // Close audio channel if opened if (protocol_ && protocol_->IsAudioChannelOpened()) { protocol_->CloseAudioChannel(); } // Reset protocol protocol_.reset(); }); }