#include "application.h"
|
#include "board.h"
|
#include "display.h"
|
#include "system_info.h"
|
#include "audio_codec.h"
|
#include "mqtt_protocol.h"
|
#include "websocket_protocol.h"
|
#include "boards/sening-xiaozhi-s3/mqtt.h"
|
#include "boards/sening-xiaozhi-s3/app_task.h"
|
#include "assets/lang_config.h"
|
#include "mcp_server.h"
|
#include "assets.h"
|
#include "settings.h"
|
#include "boards/sening-xiaozhi-s3/system.h"
|
|
#include <cstring>
|
#include <memory>
|
#include <esp_log.h>
|
#include <esp_heap_caps.h>
|
#include <cJSON.h>
|
#include <driver/gpio.h>
|
#include <arpa/inet.h>
|
#include <font_awesome.h>
|
|
#define TAG "Application"
|
|
extern "C" void SDCard_Pause_Play(void) __attribute__((weak));
|
extern "C" void SDCard_Resume_Play(void) __attribute__((weak));
|
extern "C" bool SDCard_Is_Playing(void) __attribute__((weak));
|
extern "C" bool SDCard_Is_Paused(void) __attribute__((weak));
|
|
static bool IsSdCardMp3Playing() {
|
return SDCard_Is_Playing && SDCard_Is_Playing();
|
}
|
|
static bool IsSdCardMp3Paused() {
|
return SDCard_Is_Paused && SDCard_Is_Paused();
|
}
|
|
static bool ShouldHoldIdleForLessonWork() {
|
return Board_IsLessonSyncing() || Board_IsRepeaterTask() || Board_IsShadowingTask() ||
|
Board_IsWaitingStartPhrase();
|
}
|
|
static bool IsLessonDetectText(const std::string& text) {
|
return text.rfind("[lesson_sync]", 0) == 0 ||
|
text.rfind("[app_task]english_learning|", 0) == 0;
|
}
|
|
static void PauseSdCardMp3() {
|
if (SDCard_Pause_Play) {
|
SDCard_Pause_Play();
|
}
|
}
|
|
static void ResumeSdCardMp3() {
|
if (SDCard_Resume_Play) {
|
SDCard_Resume_Play();
|
}
|
}
|
|
static bool IsInternalToolSpeech(const char* text) {
|
if (!text) {
|
return false;
|
}
|
while (*text == ' ' || *text == '\t') {
|
++text;
|
}
|
if (strncmp(text, "% self.", 7) == 0 || strncmp(text, "%self.", 6) == 0) {
|
return true;
|
}
|
return (strstr(text, "\"arguments\"") && strstr(text, "\"trigger_time\"")) || strstr(text, "\"label\"");
|
}
|
|
static bool IsStandbyRequestText(const char* text) {
|
if (!text) {
|
return false;
|
}
|
return strstr(text, "待命") || strstr(text, "进入待机") || strstr(text, "进入待机状态") ||
|
strstr(text, "进入休眠") || strstr(text, "休眠状态") || strstr(text, "停止监听") ||
|
strstr(text, "别听了") || strstr(text, "不要听了");
|
}
|
|
Application::Application() {
|
event_group_ = xEventGroupCreate();
|
|
#if CONFIG_USE_DEVICE_AEC && CONFIG_USE_SERVER_AEC
|
#error "CONFIG_USE_DEVICE_AEC and CONFIG_USE_SERVER_AEC cannot be enabled at the same time"
|
#elif CONFIG_USE_DEVICE_AEC
|
aec_mode_ = kAecOnDeviceSide;
|
#elif CONFIG_USE_SERVER_AEC
|
aec_mode_ = kAecOnServerSide;
|
#else
|
aec_mode_ = kAecOff;
|
#endif
|
|
esp_timer_create_args_t clock_timer_args = {
|
.callback = [](void* arg) {
|
Application* app = (Application*)arg;
|
xEventGroupSetBits(app->event_group_, MAIN_EVENT_CLOCK_TICK);
|
},
|
.arg = this,
|
.dispatch_method = ESP_TIMER_TASK,
|
.name = "clock_timer",
|
.skip_unhandled_events = false
|
};
|
esp_timer_create(&clock_timer_args, &clock_timer_handle_);
|
}
|
|
Application::~Application() {
|
if (clock_timer_handle_ != nullptr) {
|
esp_timer_stop(clock_timer_handle_);
|
esp_timer_delete(clock_timer_handle_);
|
}
|
vEventGroupDelete(event_group_);
|
}
|
|
bool Application::SetDeviceState(DeviceState state) {
|
return state_machine_.TransitionTo(state);
|
}
|
|
void Application::Initialize() {
|
auto& board = Board::GetInstance();
|
ota_ = std::make_unique<Ota>();
|
ota_->MarkCurrentVersionValid();
|
|
SetDeviceState(kDeviceStateStarting);
|
|
// Setup the display
|
auto display = board.GetDisplay();
|
display->SetupUI();
|
ESP_LOGI(TAG, "Application firmware version: %s", APP_FIRMWARE_VERSION);
|
// Print board name/version info
|
display->SetChatMessage("system", SystemInfo::GetUserAgent().c_str());
|
|
// Setup the audio service
|
auto codec = board.GetAudioCodec();
|
audio_service_.Initialize(codec);
|
audio_service_.Start();
|
|
AudioServiceCallbacks callbacks;
|
callbacks.on_send_queue_available = [this]() {
|
xEventGroupSetBits(event_group_, MAIN_EVENT_SEND_AUDIO);
|
};
|
callbacks.on_wake_word_detected = [this](const std::string& wake_word) {
|
if (!power_off_standby_active_.load()) {
|
xEventGroupSetBits(event_group_, MAIN_EVENT_WAKE_WORD_DETECTED);
|
}
|
};
|
callbacks.on_offline_command_detected = [this](const std::string& command) {
|
if (power_off_standby_active_.load()) {
|
return;
|
}
|
// AFE audio_detection only has a 4KB stack. Starting repeater (NVS, cJSON,
|
// SD list, MP3, JPG) on that task overflows and reboots.
|
Schedule([command]() {
|
Board_Handle_Offline_Command(command.c_str());
|
});
|
};
|
callbacks.on_vad_change = [this](bool speaking) {
|
xEventGroupSetBits(event_group_, MAIN_EVENT_VAD_CHANGE);
|
};
|
audio_service_.SetCallbacks(callbacks);
|
|
// Add state change listeners
|
state_machine_.AddStateChangeListener([this](DeviceState old_state, DeviceState new_state) {
|
xEventGroupSetBits(event_group_, MAIN_EVENT_STATE_CHANGED);
|
});
|
|
// Start the clock timer to update the status bar
|
esp_timer_start_periodic(clock_timer_handle_, 1000000);
|
|
// Add MCP common tools (only once during initialization)
|
auto& mcp_server = McpServer::GetInstance();
|
mcp_server.AddCommonTools();
|
mcp_server.AddUserOnlyTools();
|
|
// Set network event callback for UI updates and network state handling
|
board.SetNetworkEventCallback([this](NetworkEvent event, const std::string& data) {
|
auto display = Board::GetInstance().GetDisplay();
|
|
switch (event) {
|
case NetworkEvent::Scanning:
|
display->ShowNotification(Lang::Strings::SCANNING_WIFI, 30000);
|
xEventGroupSetBits(event_group_, MAIN_EVENT_NETWORK_DISCONNECTED);
|
break;
|
case NetworkEvent::Connecting: {
|
if (data.empty()) {
|
// Cellular network - registering without carrier info yet
|
display->SetStatus(Lang::Strings::REGISTERING_NETWORK);
|
} else {
|
// WiFi or cellular with carrier info
|
std::string msg = Lang::Strings::CONNECT_TO;
|
msg += data;
|
msg += "...";
|
display->ShowNotification(msg.c_str(), 30000);
|
}
|
break;
|
}
|
case NetworkEvent::Connected: {
|
std::string msg = Lang::Strings::CONNECTED_TO;
|
msg += data;
|
display->ShowNotification(msg.c_str(), 30000);
|
xEventGroupSetBits(event_group_, MAIN_EVENT_NETWORK_CONNECTED);
|
break;
|
}
|
case NetworkEvent::Disconnected:
|
xEventGroupSetBits(event_group_, MAIN_EVENT_NETWORK_DISCONNECTED);
|
break;
|
case NetworkEvent::ConnectionFailed:
|
break;
|
case NetworkEvent::WifiConfigModeEnter:
|
// WiFi config mode enter is handled by WifiBoard internally
|
break;
|
case NetworkEvent::WifiConfigModeExit:
|
// WiFi config mode exit is handled by WifiBoard internally
|
break;
|
// Cellular modem specific events
|
case NetworkEvent::ModemDetecting:
|
display->SetStatus(Lang::Strings::DETECTING_MODULE);
|
break;
|
case NetworkEvent::ModemErrorNoSim:
|
Alert(Lang::Strings::ERROR, Lang::Strings::PIN_ERROR, "triangle_exclamation", Lang::Sounds::OGG_ERR_PIN);
|
break;
|
case NetworkEvent::ModemErrorRegDenied:
|
Alert(Lang::Strings::ERROR, Lang::Strings::REG_ERROR, "triangle_exclamation", Lang::Sounds::OGG_ERR_REG);
|
break;
|
case NetworkEvent::ModemErrorInitFailed:
|
Alert(Lang::Strings::ERROR, Lang::Strings::MODEM_INIT_ERROR, "triangle_exclamation", Lang::Sounds::OGG_EXCLAMATION);
|
break;
|
case NetworkEvent::ModemErrorTimeout:
|
display->SetStatus(Lang::Strings::REGISTERING_NETWORK);
|
break;
|
}
|
});
|
|
LoadLocalAssetsForWakeWord();
|
|
// Start network asynchronously
|
board.StartNetwork();
|
|
// Update the status bar immediately to show the network state
|
display->UpdateStatusBar(true);
|
}
|
|
void Application::Run() {
|
main_loop_task_ = xTaskGetCurrentTaskHandle();
|
// Set the priority of the main task to 10
|
vTaskPrioritySet(nullptr, 10);
|
|
const EventBits_t ALL_EVENTS =
|
MAIN_EVENT_SCHEDULE |
|
MAIN_EVENT_SEND_AUDIO |
|
MAIN_EVENT_WAKE_WORD_DETECTED |
|
MAIN_EVENT_VAD_CHANGE |
|
MAIN_EVENT_CLOCK_TICK |
|
MAIN_EVENT_ERROR |
|
MAIN_EVENT_NETWORK_CONNECTED |
|
MAIN_EVENT_NETWORK_DISCONNECTED |
|
MAIN_EVENT_TOGGLE_CHAT |
|
MAIN_EVENT_START_LISTENING |
|
MAIN_EVENT_STOP_LISTENING |
|
MAIN_EVENT_ACTIVATION_DONE |
|
MAIN_EVENT_STATE_CHANGED;
|
|
while (true) {
|
auto bits = xEventGroupWaitBits(event_group_, ALL_EVENTS, pdTRUE, pdFALSE, portMAX_DELAY);
|
|
if (bits & MAIN_EVENT_ERROR) {
|
SetDeviceState(kDeviceStateIdle);
|
Alert(Lang::Strings::ERROR, last_error_message_.c_str(), "circle_xmark", Lang::Sounds::OGG_EXCLAMATION);
|
}
|
|
if (bits & MAIN_EVENT_NETWORK_CONNECTED) {
|
HandleNetworkConnectedEvent();
|
}
|
|
if (bits & MAIN_EVENT_NETWORK_DISCONNECTED) {
|
HandleNetworkDisconnectedEvent();
|
}
|
|
if (bits & MAIN_EVENT_ACTIVATION_DONE) {
|
HandleActivationDoneEvent();
|
}
|
|
if (bits & MAIN_EVENT_STATE_CHANGED) {
|
HandleStateChangedEvent();
|
}
|
|
if (bits & MAIN_EVENT_TOGGLE_CHAT) {
|
HandleToggleChatEvent();
|
}
|
|
if (bits & MAIN_EVENT_START_LISTENING) {
|
HandleStartListeningEvent();
|
}
|
|
if (bits & MAIN_EVENT_STOP_LISTENING) {
|
HandleStopListeningEvent();
|
}
|
|
if (bits & MAIN_EVENT_SEND_AUDIO) {
|
while (auto packet = audio_service_.PopPacketFromSendQueue()) {
|
if (protocol_ && !protocol_->SendAudio(std::move(packet))) {
|
break;
|
}
|
}
|
}
|
|
if (bits & MAIN_EVENT_WAKE_WORD_DETECTED) {
|
HandleWakeWordDetectedEvent();
|
}
|
|
if (bits & MAIN_EVENT_VAD_CHANGE) {
|
if (GetDeviceState() == kDeviceStateListening) {
|
auto led = Board::GetInstance().GetLed();
|
led->OnStateChanged();
|
}
|
}
|
|
if (bits & MAIN_EVENT_SCHEDULE) {
|
std::unique_lock<std::mutex> lock(mutex_);
|
auto tasks = std::move(main_tasks_);
|
lock.unlock();
|
for (auto& task : tasks) {
|
task();
|
}
|
if (GetDeviceState() == kDeviceStateSpeaking) {
|
MaybeFinishTtsAfterPlayback();
|
}
|
}
|
|
if (bits & MAIN_EVENT_CLOCK_TICK) {
|
clock_ticks_++;
|
auto display = Board::GetInstance().GetDisplay();
|
display->UpdateStatusBar();
|
auto state = GetDeviceState();
|
if (state == kDeviceStateConnecting || state == kDeviceStateSpeaking ||
|
state == kDeviceStateActivating ||
|
(state == kDeviceStateIdle && suppress_wake_until_listen_)) {
|
audio_service_.KeepCodecReadyForPlayback();
|
}
|
MaybeFinishTtsAfterPlayback();
|
if (suppress_wake_until_listen_ && state == kDeviceStateIdle) {
|
if (protocol_ && protocol_->IsAudioChannelConnected()) {
|
boot_wake_restore_ticks_ = 0;
|
if (greeting_listen_wait_ticks_ > 0) {
|
greeting_listen_wait_ticks_--;
|
if (greeting_listen_wait_ticks_ == 0) {
|
if (ShouldHoldIdleForLessonWork()) {
|
ESP_LOGI(TAG, "skip greeting listen while lesson task");
|
} else {
|
ESP_LOGW(TAG, "greeting TTS missing, start listening");
|
SetListeningMode(GetDefaultListeningMode());
|
}
|
}
|
}
|
} else if (Board_IsLessonSyncing()) {
|
boot_wake_restore_ticks_ = 0;
|
} else if (++boot_wake_restore_ticks_ >= 20) {
|
ESP_LOGW(TAG, "boot websocket not opened, restore wake word");
|
suppress_wake_until_listen_ = false;
|
boot_wake_restore_ticks_ = 0;
|
audio_service_.EnableWakeWordDetection(true);
|
}
|
} else {
|
boot_wake_restore_ticks_ = 0;
|
}
|
if (state == kDeviceStateSpeaking && tts_audio_expected_ && !tts_audio_received_ &&
|
speaking_started_us_ > 0) {
|
const int64_t wait_ms = (esp_timer_get_time() - speaking_started_us_) / 1000;
|
if (wait_ms >= 1000 && (wait_ms / 1000) <= 8) {
|
ESP_LOGW(TAG, "TTS audio still missing after %dms", (int)wait_ms);
|
}
|
}
|
if (protocol_ && IsSdCardMp3Playing() && protocol_->IsAudioChannelConnected()) {
|
protocol_->MarkIncomingActivity();
|
}
|
if (!pending_detect_text_.empty() &&
|
GetDeviceState() == kDeviceStateIdle &&
|
protocol_ && !protocol_->IsAudioChannelConnected()) {
|
if (pending_detect_retry_ticks_ > 0) {
|
pending_detect_retry_ticks_--;
|
} else if (pending_detect_retries_left_ > 0) {
|
pending_detect_retries_left_--;
|
pending_detect_retry_ticks_ = 25;
|
ESP_LOGW(TAG, "retry listen detect after handshake fail: %s left=%d",
|
pending_detect_text_.c_str(), pending_detect_retries_left_);
|
SendListenDetect(pending_detect_text_, true);
|
} else {
|
pending_detect_text_.clear();
|
}
|
}
|
if (hold_idle_for_prompt_ && !pending_detect_text_.empty() &&
|
GetDeviceState() == kDeviceStateIdle &&
|
protocol_ && protocol_->IsAudioChannelConnected()) {
|
if (pending_detect_retry_ticks_ > 0) {
|
pending_detect_retry_ticks_--;
|
} else if (pending_detect_retries_left_ > 0) {
|
pending_detect_retries_left_--;
|
pending_detect_retry_ticks_ = 2;
|
ESP_LOGW(TAG, "retry app_task detect: %s left=%d",
|
pending_detect_text_.c_str(), pending_detect_retries_left_);
|
SendListenDetectNow(pending_detect_text_);
|
} else if (IsLessonDetectText(pending_detect_text_) || ShouldHoldIdleForLessonWork()) {
|
ESP_LOGW(TAG, "lesson detect unanswered, keep idle: %s",
|
pending_detect_text_.c_str());
|
pending_detect_text_.clear();
|
pending_detect_retries_left_ = 0;
|
pending_detect_retry_ticks_ = 0;
|
} else {
|
ESP_LOGW(TAG, "app_task detect unanswered, start listening");
|
hold_idle_for_prompt_ = false;
|
suppress_wake_until_listen_ = false;
|
pending_detect_text_.clear();
|
SetListeningMode(GetDefaultListeningMode());
|
}
|
}
|
|
Board_PollPendingLessonPlayback();
|
// Print debug info every 10 seconds
|
if (clock_ticks_ % 10 == 0) {
|
SystemInfo::PrintHeapStats();
|
}
|
}
|
}
|
}
|
|
void Application::HandleNetworkConnectedEvent() {
|
ESP_LOGI(TAG, "Network connected");
|
auto state = GetDeviceState();
|
|
if (state == kDeviceStateStarting || state == kDeviceStateWifiConfiguring) {
|
// Network is ready, start activation
|
SetDeviceState(kDeviceStateActivating);
|
if (activation_task_handle_ != nullptr) {
|
ESP_LOGW(TAG, "Activation task already running");
|
return;
|
}
|
|
xTaskCreate([](void* arg) {
|
Application* app = static_cast<Application*>(arg);
|
app->ActivationTask();
|
app->activation_task_handle_ = nullptr;
|
vTaskDelete(NULL);
|
}, "activation", 4096 * 2, this, 2, &activation_task_handle_);
|
}
|
|
// Update the status bar immediately to show the network state
|
auto display = Board::GetInstance().GetDisplay();
|
display->UpdateStatusBar(true);
|
}
|
|
void Application::HandleNetworkDisconnectedEvent() {
|
auto& board = Board::GetInstance();
|
if (board.IsNetworkReady()) {
|
return;
|
}
|
|
// Close current conversation when network disconnected
|
auto state = GetDeviceState();
|
if (state == kDeviceStateConnecting || state == kDeviceStateListening || state == kDeviceStateSpeaking) {
|
ESP_LOGI(TAG, "Closing audio channel due to network disconnection");
|
protocol_->CloseAudioChannel();
|
}
|
|
// Update the status bar immediately to show the network state
|
auto display = board.GetDisplay();
|
display->UpdateStatusBar(true);
|
}
|
|
void Application::HandleActivationDoneEvent() {
|
ESP_LOGI(TAG, "Activation done");
|
|
SystemInfo::PrintHeapStats();
|
// 开机马上要连语音 WebSocket。先不要拉起 AFE：内部 SRAM 被 WakeNet 占满时
|
// 第一次握手会 10 秒超时，重试虽能连上，欢迎词 opus 也容易丢。
|
suppress_wake_until_listen_ = true;
|
boot_wake_restore_ticks_ = 0;
|
SetDeviceState(kDeviceStateIdle);
|
|
has_server_time_ = ota_->HasServerTime();
|
|
auto display = Board::GetInstance().GetDisplay();
|
std::string message = std::string(Lang::Strings::VERSION) + ota_->GetCurrentVersion();
|
display->ShowNotification(message.c_str());
|
display->SetChatMessage("system", "");
|
|
// Release OTA object after activation is complete
|
ota_.reset();
|
auto& board = Board::GetInstance();
|
// 开机马上就要连语音 WebSocket 播欢迎词。这里切 LOW_POWER 会让随后的握手
|
// 卡在 handshake timeout，欢迎词根本发不出来。保持高性能直到通道打开。
|
board.SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE);
|
|
Schedule([this]() {
|
// Play the success sound to indicate the device is ready
|
audio_service_.PlaySound(Lang::Sounds::OGG_SUCCESS);
|
Board::GetInstance().OnDeviceReady();
|
});
|
}
|
|
void Application::LoadLocalAssetsForWakeWord() {
|
auto& assets = Assets::GetInstance();
|
if (!assets.partition_valid() || local_assets_applied_) {
|
return;
|
}
|
|
local_assets_applied_ = assets.Apply(false);
|
}
|
|
void Application::ActivationTask() {
|
if (!ota_) {
|
ota_ = std::make_unique<Ota>();
|
}
|
|
// Check for new assets version
|
CheckAssetsVersion();
|
|
// Check for new firmware version
|
CheckNewVersion();
|
|
// Initialize the protocol
|
InitializeProtocol();
|
|
// Signal completion to main loop
|
xEventGroupSetBits(event_group_, MAIN_EVENT_ACTIVATION_DONE);
|
}
|
|
void Application::CheckAssetsVersion() {
|
// Only allow CheckAssetsVersion to be called once
|
if (assets_version_checked_) {
|
return;
|
}
|
assets_version_checked_ = true;
|
|
auto& board = Board::GetInstance();
|
auto display = board.GetDisplay();
|
auto& assets = Assets::GetInstance();
|
|
if (!assets.partition_valid()) {
|
ESP_LOGW(TAG, "Assets partition is disabled for board %s", BOARD_NAME);
|
return;
|
}
|
|
Settings settings("assets", true);
|
// Check if there is a new assets need to be downloaded
|
std::string download_url = settings.GetString("download_url");
|
|
if (!download_url.empty()) {
|
settings.EraseKey("download_url");
|
|
char message[256];
|
snprintf(message, sizeof(message), Lang::Strings::FOUND_NEW_ASSETS, download_url.c_str());
|
Alert(Lang::Strings::LOADING_ASSETS, message, "cloud_arrow_down", Lang::Sounds::OGG_UPGRADE);
|
|
// Wait for the audio service to be idle for 3 seconds
|
vTaskDelay(pdMS_TO_TICKS(3000));
|
SetDeviceState(kDeviceStateUpgrading);
|
board.SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE);
|
display->SetChatMessage("system", Lang::Strings::PLEASE_WAIT);
|
|
bool success = assets.Download(download_url, [this, display](int progress, size_t speed) -> void {
|
char buffer[32];
|
snprintf(buffer, sizeof(buffer), "%d%% %uKB/s", progress, static_cast<unsigned>(speed / 1024));
|
Schedule([display, message = std::string(buffer)]() {
|
display->SetChatMessage("system", message.c_str());
|
});
|
});
|
|
board.SetPowerSaveLevel(PowerSaveLevel::LOW_POWER);
|
vTaskDelay(pdMS_TO_TICKS(1000));
|
|
if (!success) {
|
Alert(Lang::Strings::ERROR, Lang::Strings::DOWNLOAD_ASSETS_FAILED, "circle_xmark", Lang::Sounds::OGG_EXCLAMATION);
|
vTaskDelay(pdMS_TO_TICKS(2000));
|
SetDeviceState(kDeviceStateActivating);
|
return;
|
}
|
local_assets_applied_ = false;
|
}
|
|
// Apply assets
|
if (!local_assets_applied_) {
|
local_assets_applied_ = assets.Apply();
|
}
|
display->SetChatMessage("system", "");
|
display->SetEmotion("microchip_ai");
|
}
|
|
void Application::CheckNewVersion() {
|
const int MAX_RETRY = 10;
|
int retry_count = 0;
|
int retry_delay = 10; // Initial retry delay in seconds
|
|
auto& board = Board::GetInstance();
|
while (true) {
|
auto display = board.GetDisplay();
|
display->SetStatus(Lang::Strings::CHECKING_NEW_VERSION);
|
|
esp_err_t err = ota_->CheckVersion();
|
if (err != ESP_OK) {
|
retry_count++;
|
if (retry_count >= MAX_RETRY) {
|
ESP_LOGE(TAG, "Too many retries, exit version check");
|
return;
|
}
|
|
char error_message[128];
|
snprintf(error_message, sizeof(error_message), "code=%d, url=%s", err, ota_->GetCheckVersionUrl().c_str());
|
char buffer[256];
|
snprintf(buffer, sizeof(buffer), Lang::Strings::CHECK_NEW_VERSION_FAILED, retry_delay, error_message);
|
Alert(Lang::Strings::ERROR, buffer, "cloud_slash", Lang::Sounds::OGG_EXCLAMATION);
|
|
ESP_LOGW(TAG, "Check new version failed, retry in %d seconds (%d/%d)", retry_delay, retry_count, MAX_RETRY);
|
for (int i = 0; i < retry_delay; i++) {
|
vTaskDelay(pdMS_TO_TICKS(1000));
|
if (GetDeviceState() == kDeviceStateIdle) {
|
break;
|
}
|
}
|
retry_delay *= 2; // Double the retry delay
|
continue;
|
}
|
retry_count = 0;
|
retry_delay = 10; // Reset retry delay
|
|
if (ota_->HasNewVersion()) {
|
if (UpgradeFirmware(ota_->GetFirmwareUrl(), ota_->GetFirmwareVersion())) {
|
return; // This line will never be reached after reboot
|
}
|
// If upgrade failed, continue to normal operation
|
}
|
|
// No new version, mark the current version as valid
|
ota_->MarkCurrentVersionValid();
|
if (!ota_->HasActivationCode() && !ota_->HasActivationChallenge()) {
|
// Exit the loop if done checking new version
|
break;
|
}
|
|
display->SetStatus(Lang::Strings::ACTIVATION);
|
// Activation code is shown to the user and waiting for the user to input
|
if (ota_->HasActivationCode()) {
|
ShowActivationCode(ota_->GetActivationCode(), ota_->GetActivationMessage());
|
}
|
|
// This will block the loop until the activation is done or timeout
|
for (int i = 0; i < 10; ++i) {
|
ESP_LOGI(TAG, "Activating... %d/%d", i + 1, 10);
|
esp_err_t err = ota_->Activate();
|
if (err == ESP_OK) {
|
break;
|
} else if (err == ESP_ERR_TIMEOUT) {
|
vTaskDelay(pdMS_TO_TICKS(3000));
|
} else {
|
vTaskDelay(pdMS_TO_TICKS(10000));
|
}
|
if (GetDeviceState() == kDeviceStateIdle) {
|
break;
|
}
|
// 激活请求失败时再播一遍 6 位码，避免只在开机瞬间播一次被听漏。
|
if (ota_->HasActivationCode() && i < 9) {
|
ShowActivationCode(ota_->GetActivationCode(), ota_->GetActivationMessage());
|
}
|
}
|
}
|
}
|
|
void Application::InitializeProtocol() {
|
auto& board = Board::GetInstance();
|
auto display = board.GetDisplay();
|
auto codec = board.GetAudioCodec();
|
|
display->SetStatus(Lang::Strings::LOADING_PROTOCOL);
|
|
if (ota_->HasMqttConfig()) {
|
protocol_ = std::make_unique<MqttProtocol>();
|
} else if (ota_->HasWebsocketConfig()) {
|
protocol_ = std::make_unique<WebsocketProtocol>();
|
} else {
|
ESP_LOGW(TAG, "No protocol specified in the OTA config, using MQTT");
|
protocol_ = std::make_unique<MqttProtocol>();
|
}
|
|
protocol_->OnConnected([this]() {
|
DismissAlert();
|
});
|
|
protocol_->OnNetworkError([this](const std::string& message) {
|
last_error_message_ = message;
|
if (network_error_suppressed_) {
|
ESP_LOGW(TAG, "network error suppressed during retry: %s", message.c_str());
|
return;
|
}
|
xEventGroupSetBits(event_group_, MAIN_EVENT_ERROR);
|
});
|
|
protocol_->OnIncomingAudio([this](std::unique_ptr<AudioStreamPacket> packet) {
|
auto shared_packet = std::shared_ptr<AudioStreamPacket>(packet.release());
|
Schedule([this, shared_packet]() {
|
auto state = GetDeviceState();
|
// speak_txt 不发独立 tts start，首句 opus 可能早于 sentence_start。
|
// 待命/聆听时不要丢包，交给 ScheduleTtsAudio 等到 speaking 就绪再播。
|
if (state != kDeviceStateSpeaking && !tts_audio_expected_ &&
|
state != kDeviceStateListening && state != kDeviceStateIdle) {
|
return;
|
}
|
if (state == kDeviceStateListening || state == kDeviceStateIdle) {
|
tts_audio_expected_ = true;
|
if (!ShouldHoldIdleForLessonWork()) {
|
hold_idle_for_prompt_ = false;
|
}
|
}
|
|
tts_response_started_ = true;
|
ESP_LOGD(TAG, "TTS audio packet bytes=%u expected=%d ready=%d state=%d",
|
(unsigned)shared_packet->payload.size(),
|
tts_audio_expected_ ? 1 : 0,
|
tts_speaking_state_ready_ ? 1 : 0,
|
static_cast<int>(state));
|
|
ScheduleTtsAudio(shared_packet);
|
if (state == kDeviceStateListening || state == kDeviceStateIdle) {
|
SetDeviceState(kDeviceStateSpeaking);
|
}
|
});
|
});
|
|
protocol_->OnAudioChannelOpened([this, codec, &board]() {
|
board.SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE);
|
if (protocol_->server_sample_rate() != codec->output_sample_rate()) {
|
ESP_LOGW(TAG, "Server sample rate %d does not match device output sample rate %d, resampling may cause distortion",
|
protocol_->server_sample_rate(), codec->output_sample_rate());
|
}
|
});
|
|
protocol_->OnAudioChannelClosed([this, &board]() {
|
if (opening_audio_channel_) {
|
ESP_LOGI(TAG, "Skip channel-closed teardown while opening/reusing websocket");
|
return;
|
}
|
// 通道关掉后会立刻切 idle、加载唤醒词。这时再进 MAX_MODEM，
|
// MQTT 重连任务和 phy_track_pll 会把内部 SRAM 打穿，ESP_ERROR_CHECK 直接重启。
|
// 保持 PERFORMANCE，等 PowerSaveTimer 超时后再真正断网待命。
|
board.SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE);
|
Schedule([this]() {
|
auto display = Board::GetInstance().GetDisplay();
|
opening_audio_channel_ = false;
|
if ((Board_IsShadowingTask() || Board_IsRepeaterTask()) && IsSdCardMp3Playing()) {
|
if (GetDeviceState() == kDeviceStateListening) {
|
SetDeviceState(kDeviceStateIdle);
|
}
|
return;
|
}
|
tts_audio_expected_ = false;
|
tts_response_started_ = false;
|
tts_audio_received_ = false;
|
tts_stop_pending_ = false;
|
tts_stop_waiting_ = false;
|
tts_idle_waiting_ = false;
|
tts_session_end_ = false;
|
tts_speaking_state_ready_ = false;
|
tts_activity_generation_++;
|
audio_service_.ResetDecoder();
|
display->SetChatMessage("system", "");
|
SetDeviceState(kDeviceStateIdle);
|
if (power_off_requested_) {
|
power_off_requested_ = false;
|
resume_sdcard_mp3_after_tts_ = false;
|
resume_sdcard_mp3_after_voice_ = false;
|
Board_Enter_Power_Off();
|
return;
|
}
|
if (standby_requested_) {
|
standby_requested_ = false;
|
}
|
if (resume_sdcard_mp3_after_tts_ && !sdcard_mp3_stop_requested_) {
|
resume_sdcard_mp3_after_tts_ = false;
|
ResumeSdCardMp3();
|
} else {
|
resume_sdcard_mp3_after_tts_ = false;
|
}
|
});
|
});
|
|
protocol_->OnIncomingJson([this, display](const cJSON* root) {
|
// Parse JSON data
|
auto type = cJSON_GetObjectItem(root, "type");
|
if (strcmp(type->valuestring, "tts") == 0) {
|
auto state = cJSON_GetObjectItem(root, "state");
|
if (strcmp(state->valuestring, "start") == 0) {
|
Schedule([this]() {
|
aborted_ = false;
|
if (!ShouldHoldIdleForLessonWork()) {
|
hold_idle_for_prompt_ = false;
|
}
|
pending_detect_text_.clear();
|
pending_detect_retries_left_ = 0;
|
pending_detect_retry_ticks_ = 0;
|
tts_audio_expected_ = true;
|
tts_response_started_ = false;
|
tts_audio_received_ = false;
|
tts_stop_pending_ = false;
|
tts_stop_waiting_ = false;
|
tts_session_end_ = false;
|
tts_playback_empty_ticks_ = 0;
|
tts_stop_wait_ticks_ = 0;
|
tts_activity_generation_++;
|
audio_service_.SetAudioPowerInhibit(true);
|
audio_service_.KeepCodecReadyForPlayback();
|
auto state = GetDeviceState();
|
if (state == kDeviceStateListening || state == kDeviceStateIdle) {
|
SetDeviceState(kDeviceStateSpeaking);
|
}
|
ESP_LOGI(TAG, "TTS start output=%d",
|
Board::GetInstance().GetAudioCodec()->output_enabled() ? 1 : 0);
|
});
|
} else if (strcmp(state->valuestring, "stop") == 0) {
|
auto session_end_item = cJSON_GetObjectItem(root, "session_end");
|
const bool session_end = cJSON_IsTrue(session_end_item) ||
|
(cJSON_IsNumber(session_end_item) && session_end_item->valuedouble != 0);
|
Schedule([this, session_end]() {
|
ESP_LOGI(TAG, "TTS stop started=%d received=%d session_end=%d",
|
tts_response_started_, tts_audio_received_, session_end ? 1 : 0);
|
if (session_end) {
|
tts_session_end_ = true;
|
}
|
tts_stop_pending_ = true;
|
FinishTtsResponseAfterPlayback();
|
});
|
} else if (strcmp(state->valuestring, "sentence_start") == 0) {
|
auto text = cJSON_GetObjectItem(root, "text");
|
if (cJSON_IsString(text)) {
|
ESP_LOGI(TAG, "<< %s", text->valuestring);
|
if (IsInternalToolSpeech(text->valuestring)) {
|
ESP_LOGW(TAG, "Ignore internal tool speech: %s", text->valuestring);
|
Schedule([this]() {
|
tts_audio_expected_ = false;
|
tts_response_started_ = false;
|
});
|
return;
|
}
|
Schedule([this, display, message = std::string(text->valuestring)]() {
|
if (!ShouldHoldIdleForLessonWork()) {
|
hold_idle_for_prompt_ = false;
|
}
|
pending_detect_text_.clear();
|
pending_detect_retries_left_ = 0;
|
pending_detect_retry_ticks_ = 0;
|
tts_audio_expected_ = true;
|
tts_response_started_ = true;
|
tts_activity_generation_++;
|
tts_playback_empty_ticks_ = 0;
|
audio_service_.SetAudioPowerInhibit(true);
|
audio_service_.KeepCodecReadyForPlayback();
|
if (Board_OnStartPhraseSentence(message.c_str())) {
|
resume_sdcard_mp3_after_tts_ = false;
|
resume_sdcard_mp3_after_voice_ = false;
|
} else if (IsSdCardMp3Playing() && !IsSdCardMp3Paused()) {
|
PauseSdCardMp3();
|
resume_sdcard_mp3_after_tts_ = true;
|
}
|
auto state = GetDeviceState();
|
if (state == kDeviceStateListening || state == kDeviceStateIdle) {
|
SetDeviceState(kDeviceStateSpeaking);
|
}
|
display->SetChatMessage("assistant", message.c_str());
|
});
|
}
|
}
|
} else if (strcmp(type->valuestring, "stt") == 0) {
|
auto text = cJSON_GetObjectItem(root, "text");
|
if (cJSON_IsString(text)) {
|
if (IsSdCardMp3Playing() && !IsSdCardMp3Paused()) {
|
PauseSdCardMp3();
|
resume_sdcard_mp3_after_tts_ = true;
|
}
|
ESP_LOGI(TAG, ">> %s", text->valuestring);
|
BoardMqtt_PublishSttEvent(text->valuestring);
|
if (IsStandbyRequestText(text->valuestring)) {
|
EnterStandby();
|
}
|
Schedule([display, message = std::string(text->valuestring)]() {
|
display->SetChatMessage("user", message.c_str());
|
});
|
}
|
} else if (strcmp(type->valuestring, "llm") == 0) {
|
auto emotion = cJSON_GetObjectItem(root, "emotion");
|
if (cJSON_IsString(emotion)) {
|
Schedule([display, emotion_str = std::string(emotion->valuestring)]() {
|
display->SetEmotion(emotion_str.c_str());
|
});
|
}
|
} else if (strcmp(type->valuestring, "mcp") == 0) {
|
auto payload = cJSON_GetObjectItem(root, "payload");
|
if (cJSON_IsObject(payload)) {
|
auto method = cJSON_GetObjectItem(payload, "method");
|
auto params = cJSON_GetObjectItem(payload, "params");
|
auto name = cJSON_IsObject(params) ? cJSON_GetObjectItem(params, "name") : nullptr;
|
if (cJSON_IsString(method) && strcmp(method->valuestring, "tools/call") == 0 && cJSON_IsString(name)) {
|
if (strcmp(name->valuestring, "self.mp3.stop") == 0) {
|
sdcard_mp3_stop_requested_ = true;
|
BoardMqtt_StopPlaylistPlayback();
|
resume_sdcard_mp3_after_tts_ = false;
|
resume_sdcard_mp3_after_voice_ = false;
|
} else if (strcmp(name->valuestring, "self.mp3.play") == 0) {
|
sdcard_mp3_stop_requested_ = false;
|
if (GetDeviceState() == kDeviceStateSpeaking) {
|
PauseSdCardMp3();
|
resume_sdcard_mp3_after_tts_ = true;
|
}
|
}
|
}
|
McpServer::GetInstance().ParseMessage(payload);
|
}
|
} else if (strcmp(type->valuestring, "system") == 0) {
|
auto command = cJSON_GetObjectItem(root, "command");
|
if (cJSON_IsString(command)) {
|
ESP_LOGI(TAG, "System command: %s", command->valuestring);
|
if (strcmp(command->valuestring, "reboot") == 0) {
|
// Do a reboot if user requests a OTA update
|
Schedule([this]() {
|
Reboot();
|
});
|
} else {
|
ESP_LOGW(TAG, "Unknown system command: %s", command->valuestring);
|
}
|
}
|
} else if (strcmp(type->valuestring, "alert") == 0) {
|
auto status = cJSON_GetObjectItem(root, "status");
|
auto message = cJSON_GetObjectItem(root, "message");
|
auto emotion = cJSON_GetObjectItem(root, "emotion");
|
if (cJSON_IsString(status) && cJSON_IsString(message) && cJSON_IsString(emotion)) {
|
Alert(status->valuestring, message->valuestring, emotion->valuestring, Lang::Sounds::OGG_VIBRATION);
|
} else {
|
ESP_LOGW(TAG, "Alert command requires status, message and emotion");
|
}
|
#if CONFIG_RECEIVE_CUSTOM_MESSAGE
|
} else if (strcmp(type->valuestring, "custom") == 0) {
|
auto payload = cJSON_GetObjectItem(root, "payload");
|
ESP_LOGI(TAG, "Received custom message: %s", cJSON_PrintUnformatted(root));
|
if (cJSON_IsObject(payload)) {
|
Schedule([this, display, payload_str = std::string(cJSON_PrintUnformatted(payload))]() {
|
display->SetChatMessage("system", payload_str.c_str());
|
});
|
} else {
|
ESP_LOGW(TAG, "Invalid custom message format: missing payload");
|
}
|
#endif
|
} else {
|
ESP_LOGW(TAG, "Unknown message type: %s", type->valuestring);
|
}
|
});
|
|
protocol_->Start();
|
}
|
|
static void WaitUntilActivationPlaybackIdle(AudioService& audio) {
|
audio.KeepCodecReadyForPlayback();
|
for (int i = 0; i < 150; ++i) {
|
if (audio.IsPlaybackQueueEmpty()) {
|
vTaskDelay(pdMS_TO_TICKS(150));
|
return;
|
}
|
audio.KeepCodecReadyForPlayback();
|
vTaskDelay(pdMS_TO_TICKS(100));
|
}
|
}
|
|
void Application::ShowActivationCode(const std::string& code, const std::string& message) {
|
struct digit_sound {
|
char digit;
|
const std::string_view& sound;
|
};
|
static const std::array<digit_sound, 10> digit_sounds{{
|
digit_sound{'0', Lang::Sounds::OGG_0},
|
digit_sound{'1', Lang::Sounds::OGG_1},
|
digit_sound{'2', Lang::Sounds::OGG_2},
|
digit_sound{'3', Lang::Sounds::OGG_3},
|
digit_sound{'4', Lang::Sounds::OGG_4},
|
digit_sound{'5', Lang::Sounds::OGG_5},
|
digit_sound{'6', Lang::Sounds::OGG_6},
|
digit_sound{'7', Lang::Sounds::OGG_7},
|
digit_sound{'8', Lang::Sounds::OGG_8},
|
digit_sound{'9', Lang::Sounds::OGG_9}
|
}};
|
|
ESP_LOGI(TAG, "Speaking activation code: %s", code.c_str());
|
audio_service_.KeepCodecReadyForPlayback();
|
// 功放刚打开时前几百毫秒常是静音，先稳住再播，避免只听到提示音听不到数字。
|
vTaskDelay(pdMS_TO_TICKS(400));
|
|
// This sentence uses 9KB of SRAM, so we need to wait for it to finish
|
Alert(Lang::Strings::ACTIVATION, message.c_str(), "link", Lang::Sounds::OGG_ACTIVATION);
|
WaitUntilActivationPlaybackIdle(audio_service_);
|
|
for (const auto& digit : code) {
|
auto it = std::find_if(digit_sounds.begin(), digit_sounds.end(),
|
[digit](const digit_sound& ds) { return ds.digit == digit; });
|
if (it != digit_sounds.end()) {
|
ESP_LOGI(TAG, "Playing activation digit: %c", digit);
|
audio_service_.KeepCodecReadyForPlayback();
|
audio_service_.PlaySound(it->sound);
|
WaitUntilActivationPlaybackIdle(audio_service_);
|
vTaskDelay(pdMS_TO_TICKS(250));
|
}
|
}
|
}
|
|
void Application::Alert(const char* status, const char* message, const char* emotion, const std::string_view& sound) {
|
ESP_LOGW(TAG, "Alert [%s] %s: %s", emotion, status, message);
|
auto display = Board::GetInstance().GetDisplay();
|
display->SetStatus(status);
|
display->SetEmotion(emotion);
|
display->SetChatMessage("system", message);
|
if (!sound.empty()) {
|
audio_service_.PlaySound(sound);
|
}
|
}
|
|
void Application::DismissAlert() {
|
if (GetDeviceState() == kDeviceStateIdle) {
|
auto display = Board::GetInstance().GetDisplay();
|
display->SetStatus(Lang::Strings::STANDBY);
|
display->SetEmotion("neutral");
|
display->SetChatMessage("system", "");
|
}
|
}
|
|
void Application::ToggleChatState() {
|
xEventGroupSetBits(event_group_, MAIN_EVENT_TOGGLE_CHAT);
|
}
|
|
void Application::StartListening() {
|
xEventGroupSetBits(event_group_, MAIN_EVENT_START_LISTENING);
|
}
|
|
void Application::StopListening() {
|
xEventGroupSetBits(event_group_, MAIN_EVENT_STOP_LISTENING);
|
}
|
|
bool Application::SendListenDetectNow(const std::string& text) {
|
if (!protocol_ || text.empty()) {
|
return false;
|
}
|
ESP_LOGI(TAG, "Send listen detect now: %s connected=%d",
|
text.c_str(), protocol_->IsAudioChannelConnected() ? 1 : 0);
|
if (!protocol_->IsAudioChannelConnected()) {
|
return false;
|
}
|
Board::GetInstance().SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE);
|
protocol_->MarkIncomingActivity();
|
return protocol_->SendWakeWordDetected(text);
|
}
|
|
void Application::HoldIdleForServerPrompt() {
|
hold_idle_for_prompt_ = true;
|
suppress_wake_until_listen_ = true;
|
auto state = GetDeviceState();
|
if (state == kDeviceStateListening) {
|
SetDeviceState(kDeviceStateIdle);
|
}
|
}
|
|
void Application::ReleaseHoldIdle() {
|
hold_idle_for_prompt_ = false;
|
suppress_wake_until_listen_ = false;
|
boot_wake_restore_ticks_ = 0;
|
}
|
|
void Application::StartListeningAfterPrompt() {
|
SetListeningMode(GetDefaultListeningMode());
|
}
|
|
void Application::ArmPendingDetectRetry(const std::string& text, int retries) {
|
if (text.empty() || retries <= 0) {
|
return;
|
}
|
pending_detect_text_ = text;
|
pending_detect_retries_left_ = retries;
|
pending_detect_retry_ticks_ = 2;
|
}
|
|
bool Application::IsAudioChannelConnected() const {
|
return protocol_ && protocol_->IsAudioChannelConnected();
|
}
|
|
void Application::AnnounceReturnedToChat() {
|
pending_detect_text_.clear();
|
pending_detect_retries_left_ = 0;
|
pending_detect_retry_ticks_ = 0;
|
suppress_wake_until_listen_ = false;
|
audio_service_.KeepCodecReadyForPlayback();
|
auto display = Board::GetInstance().GetDisplay();
|
display->SetStatus("对话模式");
|
display->SetEmotion("happy");
|
display->SetChatMessage("assistant", "我们来聊天吧。");
|
audio_service_.PlaySound(Lang::Sounds::OGG_SUCCESS);
|
ESP_LOGI(TAG, "announced local return to chat");
|
}
|
|
void Application::ReconnectAfterLesson(const std::string& detect) {
|
// 课文结束 WS 已断：先释放 AFE，再握手补发 skill_event。
|
// Server 会播「泛听/跟读结束。我们来聊天吧。」说完再 listening。
|
pending_detect_text_.clear();
|
pending_detect_retries_left_ = 0;
|
pending_detect_retry_ticks_ = 0;
|
suppress_wake_until_listen_ = true;
|
audio_service_.SetAudioPowerInhibit(false);
|
audio_service_.KeepCodecReadyForPlayback();
|
audio_service_.EnableVoiceProcessing(false);
|
audio_service_.EnableWakeWordDetection(false);
|
audio_service_.ReleaseAfeRuntime();
|
|
auto state = GetDeviceState();
|
if (state == kDeviceStateListening || state == kDeviceStateSpeaking) {
|
SetDeviceState(kDeviceStateIdle);
|
}
|
|
if (detect.empty() || !protocol_) {
|
ESP_LOGW(TAG, "lesson reconnect skipped, stay local chat");
|
AnnounceReturnedToChat();
|
audio_service_.EnableWakeWordDetection(true);
|
return;
|
}
|
|
// 泛听时长会把 WiFi 待机关掉（跟读有听窗所以不会）。不要用 IsNetworkReady()
|
// 直接放弃：后面 OpenAudioChannel 会 EnsureNetworkReadyForWake，连上再播结束语。
|
ESP_LOGI(TAG, "lesson reconnect for %s", detect.c_str());
|
Board::GetInstance().SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE);
|
SetDeviceState(kDeviceStateConnecting);
|
Schedule([this, detect]() {
|
ContinueOpenAudioChannelForDetect(detect);
|
});
|
}
|
|
void Application::SendListenDetect(const std::string& text, bool open_if_idle) {
|
Schedule([this, text, open_if_idle]() {
|
if (!protocol_ || text.empty()) {
|
return;
|
}
|
ESP_LOGI(TAG, "Send listen detect: %s connected=%d",
|
text.c_str(), protocol_->IsAudioChannelConnected() ? 1 : 0);
|
if (protocol_->IsAudioChannelConnected()) {
|
Board::GetInstance().SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE);
|
protocol_->MarkIncomingActivity();
|
if (protocol_->SendWakeWordDetected(text)) {
|
Board_NoteLessonStartDetectSent();
|
return;
|
}
|
ESP_LOGW(TAG, "detect send failed, reopen audio channel");
|
}
|
if (!open_if_idle) {
|
return;
|
}
|
if (!Board::GetInstance().IsNetworkReady()) {
|
ESP_LOGW(TAG, "skip listen detect, network not ready: %s", text.c_str());
|
return;
|
}
|
auto state = GetDeviceState();
|
// 开机/TTS/配网时不要 WakeWordInvoke：那会进 listening、开 AFE、ResetDecoder，
|
// 欢迎词被清掉，FEED 环缓冲也会立刻撑满。
|
if (state == kDeviceStateSpeaking || state == kDeviceStateWifiConfiguring ||
|
state == kDeviceStateAudioTesting || state == kDeviceStateConnecting ||
|
state == kDeviceStateStarting || state == kDeviceStateActivating) {
|
ESP_LOGI(TAG, "defer listen detect until idle: state=%d", static_cast<int>(state));
|
return;
|
}
|
if (state != kDeviceStateIdle) {
|
return;
|
}
|
SetDeviceState(kDeviceStateConnecting);
|
Schedule([this, text]() {
|
ContinueOpenAudioChannelForDetect(text);
|
});
|
});
|
}
|
|
void Application::HandleToggleChatEvent() {
|
if (power_off_standby_active_.load()) {
|
ESP_LOGI(TAG, "Ignore toggle chat while power-off standby is active");
|
return;
|
}
|
|
auto state = GetDeviceState();
|
|
if (state == kDeviceStateActivating) {
|
SetDeviceState(kDeviceStateIdle);
|
return;
|
} else if (state == kDeviceStateWifiConfiguring) {
|
audio_service_.EnableAudioTesting(true);
|
SetDeviceState(kDeviceStateAudioTesting);
|
return;
|
} else if (state == kDeviceStateAudioTesting) {
|
audio_service_.EnableAudioTesting(false);
|
SetDeviceState(kDeviceStateWifiConfiguring);
|
return;
|
}
|
|
if (!protocol_) {
|
ESP_LOGE(TAG, "Protocol not initialized");
|
return;
|
}
|
|
if (state == kDeviceStateIdle) {
|
ListeningMode mode = GetDefaultListeningMode();
|
if (!protocol_->IsAudioChannelOpened()) {
|
SetDeviceState(kDeviceStateConnecting);
|
// Schedule to let the state change be processed first (UI update)
|
Schedule([this, mode]() {
|
ContinueOpenAudioChannel(mode);
|
});
|
return;
|
}
|
SetListeningMode(mode);
|
} else if (state == kDeviceStateSpeaking) {
|
AbortSpeaking(kAbortReasonNone);
|
} else if (state == kDeviceStateListening) {
|
protocol_->CloseAudioChannel();
|
}
|
}
|
|
void Application::ContinueOpenAudioChannel(ListeningMode mode) {
|
if (power_off_standby_active_.load()) {
|
ESP_LOGI(TAG, "Ignore open audio channel while power-off standby is active");
|
return;
|
}
|
|
// Check state again in case it was changed during scheduling
|
if (GetDeviceState() != kDeviceStateConnecting) {
|
return;
|
}
|
if (!OpenAudioChannelWithRetry(2)) {
|
SetDeviceState(kDeviceStateIdle);
|
return;
|
}
|
|
SetListeningMode(mode);
|
}
|
|
bool Application::OpenAudioChannelWithRetry(int attempts) {
|
if (!protocol_) {
|
return false;
|
}
|
if (protocol_->IsAudioChannelConnected()) {
|
protocol_->MarkIncomingActivity();
|
return true;
|
}
|
if (opening_audio_channel_) {
|
return false;
|
}
|
if (attempts < 1) {
|
attempts = 1;
|
}
|
|
opening_audio_channel_ = true;
|
// 握手期间停 AFE，避免和 Wi-Fi 抢内部 SRAM/CPU。不要另建任务：
|
// AFE 起来后内部堆只剩几 KB，xTaskCreate(8KB) 会直接失败。
|
audio_service_.EnableVoiceProcessing(false);
|
audio_service_.EnableWakeWordDetection(false);
|
const size_t dma_before = heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL | MALLOC_CAP_DMA);
|
ESP_LOGI(TAG, "OpenAudioChannel DMA largest=%u free=%u",
|
(unsigned)dma_before,
|
(unsigned)heap_caps_get_free_size(MALLOC_CAP_INTERNAL));
|
if (dma_before < 8192) {
|
ESP_LOGW(TAG, "low DMA, release AFE before websocket");
|
audio_service_.ClearWakeWordBuffers();
|
audio_service_.ReleaseAfeRuntime();
|
vTaskDelay(pdMS_TO_TICKS(50));
|
ESP_LOGI(TAG, "OpenAudioChannel DMA after AFE release=%u",
|
(unsigned)heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL | MALLOC_CAP_DMA));
|
}
|
Board::GetInstance().SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE);
|
audio_service_.SetAudioPowerInhibit(true);
|
|
bool ok = false;
|
for (int i = 0; i < attempts; ++i) {
|
if (protocol_->IsAudioChannelConnected()) {
|
ok = true;
|
break;
|
}
|
if (!Board::GetInstance().EnsureNetworkReadyForWake(30000)) {
|
ESP_LOGW(TAG, "OpenAudioChannel aborted, network not ready");
|
break;
|
}
|
Board::GetInstance().SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE);
|
const bool last = (i == attempts - 1);
|
network_error_suppressed_ = !last;
|
ok = protocol_->OpenAudioChannel();
|
network_error_suppressed_ = false;
|
if (ok) {
|
break;
|
}
|
ESP_LOGW(TAG, "OpenAudioChannel failed (%d/%d)", i + 1, attempts);
|
protocol_->CloseAudioChannel(false);
|
if (!last) {
|
ESP_LOGI(TAG, "Retry websocket handshake in 500ms");
|
vTaskDelay(pdMS_TO_TICKS(500));
|
audio_service_.EnableWakeWordDetection(false);
|
Board::GetInstance().SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE);
|
audio_service_.KeepCodecReadyForPlayback();
|
}
|
}
|
|
opening_audio_channel_ = false;
|
audio_service_.SetAudioPowerInhibit(false);
|
return ok;
|
}
|
|
void Application::ContinueOpenAudioChannelForDetect(const std::string& detect_text) {
|
if (power_off_standby_active_.load()) {
|
return;
|
}
|
if (GetDeviceState() != kDeviceStateConnecting) {
|
return;
|
}
|
listening_mode_ = GetDefaultListeningMode();
|
suppress_wake_until_listen_ = true;
|
if (!OpenAudioChannelWithRetry(2)) {
|
// 课文结束后通道已死时不要反复 handshake：会卡在「连接中」并关掉唤醒词。
|
const bool skip_retry = detect_text.rfind("[skill_event]", 0) == 0;
|
if (skip_retry) {
|
pending_detect_text_.clear();
|
pending_detect_retries_left_ = 0;
|
pending_detect_retry_ticks_ = 0;
|
} else {
|
pending_detect_text_ = detect_text;
|
if (pending_detect_retries_left_ <= 0) {
|
pending_detect_retries_left_ = 3;
|
}
|
pending_detect_retry_ticks_ = 3;
|
}
|
suppress_wake_until_listen_ = false;
|
SetDeviceState(kDeviceStateIdle);
|
if (skip_retry) {
|
ESP_LOGW(TAG, "websocket failed after lesson, keep idle chat without retry");
|
AnnounceReturnedToChat();
|
} else {
|
ESP_LOGW(TAG, "boot websocket failed, will retry detect in 3s");
|
}
|
if (IsLessonDetectText(detect_text) || ShouldHoldIdleForLessonWork()) {
|
suppress_wake_until_listen_ = true;
|
audio_service_.EnableWakeWordDetection(false);
|
} else {
|
audio_service_.EnableWakeWordDetection(true);
|
}
|
return;
|
}
|
|
pending_detect_text_.clear();
|
pending_detect_retries_left_ = 0;
|
pending_detect_retry_ticks_ = 0;
|
greeting_listen_wait_ticks_ = (IsLessonDetectText(detect_text) || ShouldHoldIdleForLessonWork()) ? 0 : 8;
|
if (!detect_text.empty()) {
|
protocol_->MarkIncomingActivity();
|
if (protocol_->SendWakeWordDetected(detect_text)) {
|
Board_NoteLessonStartDetectSent();
|
}
|
}
|
// 保持待命且不开 AFE：欢迎词从 idle 进 speaking。说完再 listening 开麦。
|
SetDeviceState(kDeviceStateIdle);
|
}
|
|
void Application::PauseSdCardMp3ForVoice() {
|
if (IsSdCardMp3Playing() && !IsSdCardMp3Paused()) {
|
PauseSdCardMp3();
|
resume_sdcard_mp3_after_voice_ = true;
|
}
|
}
|
|
void Application::ResumeSdCardMp3AfterVoice() {
|
if (resume_sdcard_mp3_after_voice_ && !sdcard_mp3_stop_requested_) {
|
resume_sdcard_mp3_after_voice_ = false;
|
ResumeSdCardMp3();
|
ApplyMp3CommandOnlyMic();
|
} else {
|
resume_sdcard_mp3_after_voice_ = false;
|
}
|
}
|
|
void Application::ApplyMp3CommandOnlyMic() {
|
if (GetDeviceState() == kDeviceStateWifiConfiguring ||
|
GetDeviceState() == kDeviceStateSpeaking) {
|
return;
|
}
|
if (!IsSdCardMp3Playing() || IsSdCardMp3Paused()) {
|
return;
|
}
|
// 音频在播：不上行录音/闲聊，只留离线口令（退出泛听/跟读、我们聊天吧）。
|
// 课文还在读 SD 时禁止拉起 AFE：会把 SDMMC DMA 需要的连续内部 SRAM 吃光，fopen abort。
|
audio_service_.EnableVoiceProcessing(false);
|
const size_t largest_internal = heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL);
|
if (!SDCard_Mp3OfflineWakeAllowed() || largest_internal < 32768) {
|
audio_service_.EnableWakeWordDetection(false);
|
if (!SDCard_Mp3OfflineWakeAllowed()) {
|
ESP_LOGD(TAG, "offline wake deferred until MP3 cached");
|
} else {
|
ESP_LOGW(TAG, "skip offline wake, internal largest=%u", (unsigned)largest_internal);
|
}
|
} else {
|
audio_service_.EnableWakeWordDetection(audio_service_.IsAfeWakeWord());
|
}
|
if (GetDeviceState() != kDeviceStateListening) {
|
return;
|
}
|
if (protocol_ && protocol_->IsAudioChannelOpened()) {
|
protocol_->SendStopListening();
|
}
|
// 播课文时切待命：关掉 AFE/上行，只留离线口令。跟读句间由播完后再 ResumeListening。
|
Schedule([this]() {
|
if (GetDeviceState() == kDeviceStateListening &&
|
IsSdCardMp3Playing() && !IsSdCardMp3Paused()) {
|
SetDeviceState(kDeviceStateIdle);
|
}
|
});
|
}
|
|
void Application::ResumeListeningAfterMp3Clip() {
|
if (power_off_standby_active_.load()) {
|
return;
|
}
|
if (IsSdCardMp3Playing() && !IsSdCardMp3Paused() && !Board_IsShadowingTask()) {
|
return;
|
}
|
auto state = GetDeviceState();
|
// 开机欢迎词 / 进场 TTS 期间禁止开麦：EnableVoiceProcessing 会 ResetDecoder 并喂 AFE。
|
if (state == kDeviceStateSpeaking || state == kDeviceStateWifiConfiguring ||
|
state == kDeviceStateAudioTesting || state == kDeviceStateConnecting ||
|
state == kDeviceStateStarting || state == kDeviceStateActivating) {
|
return;
|
}
|
if (!protocol_) {
|
return;
|
}
|
ESP_LOGI(TAG, "Resume listening after MP3 clip");
|
if (state == kDeviceStateListening) {
|
protocol_->SendStartListening(listening_mode_);
|
audio_service_.EnableVoiceProcessing(true);
|
#ifdef CONFIG_WAKE_WORD_DETECTION_IN_LISTENING
|
audio_service_.EnableWakeWordDetection(audio_service_.IsAfeWakeWord());
|
#else
|
audio_service_.EnableWakeWordDetection(false);
|
#endif
|
return;
|
}
|
if (state == kDeviceStateIdle && !protocol_->IsAudioChannelConnected()) {
|
return;
|
}
|
skip_playback_wait_on_listen_ = true;
|
SetListeningMode(GetDefaultListeningMode());
|
}
|
|
void Application::CloseShadowingListenWindow()
|
{
|
audio_service_.EnableVoiceProcessing(false);
|
// 下一句还要 fopen/解码，「跟读结束」TTS 还要 mbedtls AES。这里不要立刻拉起
|
// AFE 唤醒词，否则内部 SRAM 只剩 1KB 连续块，SD DMA / SSL 都会失败。
|
audio_service_.EnableWakeWordDetection(false);
|
if (protocol_ && protocol_->IsAudioChannelOpened()) {
|
while (auto packet = audio_service_.PopPacketFromSendQueue()) {
|
if (!protocol_->SendAudio(std::move(packet))) {
|
break;
|
}
|
}
|
protocol_->SendStopListening();
|
}
|
if (GetDeviceState() == kDeviceStateListening) {
|
SetDeviceState(kDeviceStateIdle);
|
}
|
}
|
|
void Application::HandleStartListeningEvent() {
|
if (power_off_standby_active_.load()) {
|
ESP_LOGI(TAG, "Ignore start listening while power-off standby is active");
|
return;
|
}
|
|
auto state = GetDeviceState();
|
|
if (state == kDeviceStateActivating) {
|
SetDeviceState(kDeviceStateIdle);
|
return;
|
} else if (state == kDeviceStateWifiConfiguring) {
|
audio_service_.EnableAudioTesting(true);
|
SetDeviceState(kDeviceStateAudioTesting);
|
return;
|
}
|
|
if (!protocol_) {
|
ESP_LOGE(TAG, "Protocol not initialized");
|
return;
|
}
|
|
if (state == kDeviceStateIdle) {
|
if (!protocol_->IsAudioChannelOpened()) {
|
PauseSdCardMp3ForVoice();
|
SetDeviceState(kDeviceStateConnecting);
|
// Schedule to let the state change be processed first (UI update)
|
Schedule([this]() {
|
ContinueOpenAudioChannel(kListeningModeManualStop);
|
});
|
return;
|
}
|
PauseSdCardMp3ForVoice();
|
SetListeningMode(kListeningModeManualStop);
|
} else if (state == kDeviceStateSpeaking) {
|
AbortSpeaking(kAbortReasonNone);
|
SetListeningMode(kListeningModeManualStop);
|
}
|
}
|
|
void Application::HandleStopListeningEvent() {
|
auto state = GetDeviceState();
|
|
if (state == kDeviceStateAudioTesting) {
|
audio_service_.EnableAudioTesting(false);
|
SetDeviceState(kDeviceStateWifiConfiguring);
|
return;
|
} else if (state == kDeviceStateListening) {
|
if (protocol_) {
|
protocol_->SendStopListening();
|
}
|
SetDeviceState(kDeviceStateIdle);
|
}
|
}
|
|
void Application::HandleWakeWordDetectedEvent() {
|
if (power_off_standby_active_.load()) {
|
ESP_LOGI(TAG, "Ignore wake word while power-off standby is active");
|
return;
|
}
|
|
if (!protocol_) {
|
return;
|
}
|
|
auto state = GetDeviceState();
|
auto wake_word = audio_service_.GetLastWakeWord();
|
ESP_LOGI(TAG, "Wake word detected: %s (state: %d)", wake_word.c_str(), (int)state);
|
|
if (state == kDeviceStateIdle) {
|
PauseSdCardMp3ForVoice();
|
size_t dma_largest = heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL | MALLOC_CAP_DMA);
|
// 再见后唤醒词 AFE 还占着内部堆，DMA 往往不够编码 opus。
|
// 旧逻辑会跳过编码却继续阻塞等 opus，主循环卡死，15s 后 codec 掉电，像软关机。
|
if (dma_largest < 12288) {
|
ESP_LOGW(TAG, "low DMA before wake handshake, release AFE largest=%u",
|
(unsigned)dma_largest);
|
audio_service_.ReleaseAfeRuntime();
|
dma_largest = heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL | MALLOC_CAP_DMA);
|
}
|
if (dma_largest >= 8192) {
|
audio_service_.EncodeWakeWord();
|
} else {
|
ESP_LOGW(TAG, "skip wake opus encode, DMA largest=%u", (unsigned)dma_largest);
|
audio_service_.AbortWakeWordEncode();
|
}
|
auto wake_word = audio_service_.GetLastWakeWord();
|
|
if (!protocol_->IsAudioChannelOpened()) {
|
SetDeviceState(kDeviceStateConnecting);
|
// Schedule to let the state change be processed first (UI update),
|
// then continue with OpenAudioChannel which may block for ~1 second
|
Schedule([this, wake_word]() {
|
ContinueWakeWordInvoke(wake_word);
|
});
|
return;
|
}
|
Schedule([this, wake_word]() {
|
ContinueWakeWordInvoke(wake_word);
|
});
|
} else if (state == kDeviceStateSpeaking || state == kDeviceStateListening) {
|
AbortSpeaking(kAbortReasonWakeWordDetected);
|
// Clear send queue to avoid sending residues to server
|
while (audio_service_.PopPacketFromSendQueue());
|
|
if (state == kDeviceStateListening) {
|
protocol_->SendStartListening(GetDefaultListeningMode());
|
audio_service_.ResetDecoder();
|
audio_service_.PlaySound(Lang::Sounds::OGG_POPUP);
|
// Re-enable wake word detection as it was stopped by the detection itself
|
audio_service_.EnableWakeWordDetection(true);
|
} else {
|
// Play popup sound and start listening again
|
play_popup_on_listening_ = true;
|
SetListeningMode(GetDefaultListeningMode());
|
}
|
} else if (state == kDeviceStateActivating) {
|
// Restart the activation check if the wake word is detected during activation
|
SetDeviceState(kDeviceStateIdle);
|
}
|
}
|
|
void Application::ContinueWakeWordInvoke(const std::string& wake_word) {
|
if (power_off_standby_active_.load()) {
|
ESP_LOGI(TAG, "Ignore wake word invoke while power-off standby is active");
|
return;
|
}
|
|
// Check state again in case it was changed during scheduling
|
auto state = GetDeviceState();
|
if (state != kDeviceStateConnecting && state != kDeviceStateIdle) {
|
return;
|
}
|
if (opening_audio_channel_) {
|
return;
|
}
|
|
if (!OpenAudioChannelWithRetry(2)) {
|
ResumeSdCardMp3AfterVoice();
|
audio_service_.EnableWakeWordDetection(true);
|
SetDeviceState(kDeviceStateIdle);
|
return;
|
}
|
|
ESP_LOGI(TAG, "Wake word detected: %s", wake_word.c_str());
|
#if CONFIG_SEND_WAKE_WORD_DATA
|
while (auto packet = audio_service_.PopWakeWordPacket()) {
|
if (!protocol_->SendAudio(std::move(packet))) {
|
break;
|
}
|
}
|
protocol_->SendWakeWordDetected(wake_word);
|
SetListeningMode(GetDefaultListeningMode());
|
#else
|
play_popup_on_listening_ = true;
|
SetListeningMode(GetDefaultListeningMode());
|
#endif
|
}
|
|
void Application::HandleStateChangedEvent() {
|
DeviceState new_state = state_machine_.GetState();
|
clock_ticks_ = 0;
|
|
auto& board = Board::GetInstance();
|
auto display = board.GetDisplay();
|
auto led = board.GetLed();
|
led->OnStateChanged();
|
|
if (power_off_standby_active_.load()) {
|
opening_audio_channel_ = false;
|
play_popup_on_listening_ = false;
|
audio_service_.EnableVoiceProcessing(false);
|
audio_service_.EnableWakeWordDetection(false);
|
audio_service_.ResetDecoder();
|
return;
|
}
|
|
switch (new_state) {
|
case kDeviceStateUnknown:
|
case kDeviceStateIdle:
|
opening_audio_channel_ = false;
|
if (!suppress_wake_until_listen_) {
|
tts_audio_expected_ = false;
|
tts_response_started_ = false;
|
tts_audio_received_ = false;
|
tts_stop_pending_ = false;
|
tts_stop_waiting_ = false;
|
tts_idle_waiting_ = false;
|
tts_session_end_ = false;
|
tts_speaking_state_ready_ = false;
|
tts_activity_generation_++;
|
}
|
ResumeSdCardMp3AfterVoice();
|
display->SetStatus(Lang::Strings::STANDBY);
|
display->ClearChatMessages(); // Clear messages first
|
display->SetEmotion("neutral"); // Then set emotion (wechat mode checks child count)
|
audio_service_.EnableVoiceProcessing(false);
|
// 再见断线会 speaking→listening→idle：processor AFE 没释放就把唤醒词 AFE 拉起来，
|
// audio_detection 建不起来，feed 刷屏直到内部 SRAM 打穿。
|
if (!suppress_wake_until_listen_) {
|
audio_service_.ReleaseVoiceProcessorRuntime();
|
}
|
if (IsSdCardMp3Playing() && !IsSdCardMp3Paused()) {
|
ApplyMp3CommandOnlyMic();
|
} else if (suppress_wake_until_listen_ || Board_IsLessonSyncing() ||
|
Board_IsRepeaterTask() || Board_IsShadowingTask()) {
|
audio_service_.EnableWakeWordDetection(false);
|
audio_service_.SetAudioPowerInhibit(true);
|
audio_service_.KeepCodecReadyForPlayback();
|
} else {
|
audio_service_.EnableWakeWordDetection(true);
|
audio_service_.SetAudioPowerInhibit(false);
|
}
|
break;
|
case kDeviceStateConnecting:
|
display->SetStatus(Lang::Strings::CONNECTING);
|
display->SetEmotion("neutral");
|
display->SetChatMessage("system", "");
|
break;
|
case kDeviceStateListening:
|
tts_speaking_state_ready_ = false;
|
display->SetStatus(Lang::Strings::LISTENING);
|
display->SetEmotion("neutral");
|
|
if (IsSdCardMp3Playing() && !IsSdCardMp3Paused() && !Board_IsShadowingTask()) {
|
ApplyMp3CommandOnlyMic();
|
} else {
|
if (play_popup_on_listening_ || !audio_service_.IsAudioProcessorRunning()) {
|
if (listening_mode_ == kListeningModeAutoStop && !skip_playback_wait_on_listen_) {
|
audio_service_.WaitForPlaybackQueueEmpty();
|
}
|
protocol_->SendStartListening(listening_mode_);
|
}
|
audio_service_.EnableVoiceProcessing(true);
|
#ifdef CONFIG_WAKE_WORD_DETECTION_IN_LISTENING
|
audio_service_.EnableWakeWordDetection(audio_service_.IsAfeWakeWord());
|
#else
|
audio_service_.EnableWakeWordDetection(false);
|
#endif
|
}
|
audio_service_.SetAudioPowerInhibit(false);
|
suppress_wake_until_listen_ = false;
|
skip_playback_wait_on_listen_ = false;
|
if (play_popup_on_listening_) {
|
play_popup_on_listening_ = false;
|
audio_service_.PlaySound(Lang::Sounds::OGG_POPUP);
|
}
|
break;
|
case kDeviceStateSpeaking:
|
tts_speaking_state_ready_ = false;
|
tts_playback_empty_ticks_ = 0;
|
tts_stop_wait_ticks_ = 0;
|
greeting_listen_wait_ticks_ = 0;
|
speaking_started_us_ = esp_timer_get_time();
|
display->SetStatus(Lang::Strings::SPEAKING);
|
|
if (listening_mode_ != kListeningModeRealtime) {
|
audio_service_.EnableVoiceProcessing(false);
|
// TTS 回声会把菜单里的「跟读/我们聊天吧」听成口令，而且 AFE 会挤爆内部 SRAM。
|
audio_service_.EnableWakeWordDetection(false);
|
}
|
// 欢迎词的 opus 可能已经在待命时入队。这里 ResetDecoder 会清掉刚到的 TTS，
|
// 听起来像没播，随后又等不到 tts stop，一直停在说话中。
|
if (!tts_audio_expected_ && !tts_audio_received_ && audio_service_.IsPlaybackQueueEmpty()) {
|
audio_service_.ResetDecoder();
|
}
|
audio_service_.SetAudioPowerInhibit(true);
|
audio_service_.KeepCodecReadyForPlayback();
|
ESP_LOGI(TAG, "Speaking: codec ready output=%d",
|
Board::GetInstance().GetAudioCodec()->output_enabled() ? 1 : 0);
|
tts_speaking_state_ready_ = true;
|
break;
|
case kDeviceStateWifiConfiguring:
|
audio_service_.EnableVoiceProcessing(false);
|
// MultiNet/AFE wake word consumes internal SRAM. The captive portal HTML
|
// and /scan both fail when free SRAM drops to hundreds of bytes
|
// (httpd send EAGAIN, blank page, empty AP list). Keep it off until
|
// WiFi is configured; Idle re-enables detection afterwards.
|
audio_service_.EnableWakeWordDetection(false);
|
break;
|
default:
|
// Do nothing
|
break;
|
}
|
}
|
|
bool Application::IsOnMainLoop() const {
|
return main_loop_task_ != nullptr && xTaskGetCurrentTaskHandle() == main_loop_task_;
|
}
|
|
void Application::Schedule(std::function<void()>&& callback) {
|
{
|
std::lock_guard<std::mutex> lock(mutex_);
|
main_tasks_.push_back(std::move(callback));
|
}
|
xEventGroupSetBits(event_group_, MAIN_EVENT_SCHEDULE);
|
}
|
|
void Application::AbortSpeaking(AbortReason reason) {
|
ESP_LOGI(TAG, "Abort speaking");
|
aborted_ = true;
|
if (protocol_) {
|
protocol_->SendAbortSpeaking(reason);
|
}
|
}
|
|
void Application::SetListeningMode(ListeningMode mode) {
|
listening_mode_ = mode;
|
SetDeviceState(kDeviceStateListening);
|
}
|
|
ListeningMode Application::GetDefaultListeningMode() const {
|
return aec_mode_ == kAecOff ? kListeningModeAutoStop : kListeningModeRealtime;
|
}
|
|
void Application::Reboot() {
|
ESP_LOGI(TAG, "Rebooting...");
|
// Disconnect the audio channel
|
if (protocol_ && protocol_->IsAudioChannelOpened()) {
|
protocol_->CloseAudioChannel();
|
}
|
protocol_.reset();
|
audio_service_.Stop();
|
|
vTaskDelay(pdMS_TO_TICKS(1000));
|
esp_restart();
|
}
|
|
bool Application::UpgradeFirmware(const std::string& url, const std::string& version,
|
std::function<void(int progress, size_t speed)> progress_callback,
|
std::function<void(bool success)> complete_callback) {
|
auto& board = Board::GetInstance();
|
auto display = board.GetDisplay();
|
|
std::string upgrade_url = url;
|
std::string version_info = version.empty() ? "(Manual upgrade)" : version;
|
|
// Close audio channel if it's open
|
if (protocol_ && protocol_->IsAudioChannelOpened()) {
|
ESP_LOGI(TAG, "Closing audio channel before firmware upgrade");
|
protocol_->CloseAudioChannel();
|
}
|
ESP_LOGI(TAG, "Starting firmware upgrade from URL: %s", upgrade_url.c_str());
|
|
Alert(Lang::Strings::OTA_UPGRADE, Lang::Strings::UPGRADING, "download", Lang::Sounds::OGG_UPGRADE);
|
vTaskDelay(pdMS_TO_TICKS(3000));
|
|
AcquireSleepInhibit();
|
SetDeviceState(kDeviceStateUpgrading);
|
|
std::string message = std::string(Lang::Strings::NEW_VERSION) + version_info;
|
display->SetChatMessage("system", message.c_str());
|
|
board.SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE);
|
audio_service_.Stop();
|
vTaskDelay(pdMS_TO_TICKS(1000));
|
|
bool upgrade_success = Ota::Upgrade(upgrade_url, [this, display, progress_callback](int progress, size_t speed) {
|
char buffer[32];
|
snprintf(buffer, sizeof(buffer), "%d%% %uKB/s", progress, static_cast<unsigned>(speed / 1024));
|
Schedule([display, message = std::string(buffer)]() {
|
display->SetChatMessage("system", message.c_str());
|
});
|
if (progress_callback) {
|
progress_callback(progress, speed);
|
}
|
});
|
|
if (!upgrade_success) {
|
// Upgrade failed, restart audio service and continue running
|
ESP_LOGE(TAG, "Firmware upgrade failed, restarting audio service and continuing operation...");
|
audio_service_.Start(); // Restart audio service
|
board.SetPowerSaveLevel(PowerSaveLevel::LOW_POWER); // Restore power save level
|
ReleaseSleepInhibit();
|
if (complete_callback) {
|
complete_callback(false);
|
}
|
Alert(Lang::Strings::ERROR, Lang::Strings::UPGRADE_FAILED, "circle_xmark", Lang::Sounds::OGG_EXCLAMATION);
|
vTaskDelay(pdMS_TO_TICKS(3000));
|
return false;
|
} else {
|
// Upgrade success, reboot immediately
|
ESP_LOGI(TAG, "Firmware upgrade successful, rebooting...");
|
display->SetChatMessage("system", "Upgrade successful, rebooting...");
|
vTaskDelay(pdMS_TO_TICKS(1000)); // Brief pause to show message
|
ReleaseSleepInhibit();
|
if (complete_callback) {
|
complete_callback(true);
|
}
|
Reboot();
|
return true;
|
}
|
}
|
|
void Application::WakeWordInvoke(const std::string& wake_word) {
|
if (power_off_standby_active_.load()) {
|
ESP_LOGI(TAG, "Ignore wake word invoke while power-off standby is active");
|
return;
|
}
|
|
if (!protocol_) {
|
return;
|
}
|
|
auto state = GetDeviceState();
|
|
if (state == kDeviceStateIdle) {
|
audio_service_.EncodeWakeWord();
|
|
if (!protocol_->IsAudioChannelOpened()) {
|
SetDeviceState(kDeviceStateConnecting);
|
// Schedule to let the state change be processed first (UI update)
|
Schedule([this, wake_word]() {
|
ContinueWakeWordInvoke(wake_word);
|
});
|
return;
|
}
|
Schedule([this, wake_word]() {
|
ContinueWakeWordInvoke(wake_word);
|
});
|
} else if (state == kDeviceStateSpeaking) {
|
Schedule([this]() {
|
AbortSpeaking(kAbortReasonNone);
|
});
|
} else if (state == kDeviceStateListening) {
|
Schedule([this]() {
|
if (protocol_) {
|
protocol_->CloseAudioChannel();
|
}
|
});
|
}
|
}
|
|
void Application::EnterStandby() {
|
Schedule([this]() {
|
standby_requested_ = true;
|
auto state = GetDeviceState();
|
if (state == kDeviceStateIdle) {
|
standby_requested_ = false;
|
} else if (state == kDeviceStateSpeaking && tts_stop_pending_) {
|
FinishTtsResponseAfterPlayback();
|
}
|
});
|
}
|
|
void Application::RequestPowerOffStandby() {
|
Schedule([this]() {
|
power_off_requested_ = true;
|
auto state = GetDeviceState();
|
if (state == kDeviceStateIdle) {
|
power_off_requested_ = false;
|
Board_Enter_Power_Off();
|
} else if (state == kDeviceStateSpeaking && tts_stop_pending_) {
|
FinishTtsResponseAfterPlayback();
|
}
|
});
|
}
|
|
void Application::EnterPowerOffStandby() {
|
power_off_standby_active_.store(true);
|
Schedule([this]() {
|
ESP_LOGI(TAG, "Entering power-off standby");
|
|
standby_requested_ = false;
|
power_off_requested_ = false;
|
opening_audio_channel_ = false;
|
play_popup_on_listening_ = false;
|
resume_sdcard_mp3_after_tts_ = false;
|
resume_sdcard_mp3_after_voice_ = false;
|
sdcard_mp3_stop_requested_ = false;
|
tts_audio_expected_ = false;
|
tts_response_started_ = false;
|
tts_audio_received_ = false;
|
tts_stop_pending_ = false;
|
tts_stop_waiting_ = false;
|
tts_idle_waiting_ = false;
|
tts_session_end_ = false;
|
tts_speaking_state_ready_ = false;
|
tts_activity_generation_++;
|
|
if (GetDeviceState() == kDeviceStateSpeaking) {
|
AbortSpeaking(kAbortReasonNone);
|
}
|
if (protocol_ && protocol_->IsAudioChannelOpened()) {
|
protocol_->CloseAudioChannel();
|
}
|
|
audio_service_.EnableVoiceProcessing(false);
|
audio_service_.EnableWakeWordDetection(false);
|
audio_service_.ResetDecoder();
|
while (audio_service_.PopPacketFromSendQueue()) {
|
}
|
BoardMqtt_StopPlaylistPlayback();
|
|
auto state = GetDeviceState();
|
if (state == kDeviceStateConnecting || state == kDeviceStateListening || state == kDeviceStateSpeaking) {
|
SetDeviceState(kDeviceStateIdle);
|
}
|
});
|
}
|
|
void Application::RestoreFromPowerOffStandby(bool network_ready) {
|
Schedule([this, network_ready]() {
|
ESP_LOGI(TAG, "Restoring from power-off standby, network_ready=%d", network_ready);
|
|
power_off_standby_active_.store(false);
|
standby_requested_ = false;
|
power_off_requested_ = false;
|
opening_audio_channel_ = false;
|
play_popup_on_listening_ = false;
|
tts_activity_generation_++;
|
|
audio_service_.EnableVoiceProcessing(false);
|
audio_service_.ResetDecoder();
|
if (GetDeviceState() != kDeviceStateIdle) {
|
SetDeviceState(kDeviceStateIdle);
|
}
|
audio_service_.EnableWakeWordDetection(true);
|
if (!Board::GetInstance().OnRestoredFromPowerOff(network_ready)) {
|
BoardMqtt_ResumePlaylistIfUnfinished();
|
}
|
});
|
}
|
|
void Application::AcquireSleepInhibit() {
|
std::lock_guard<std::mutex> lock(mutex_);
|
sleep_inhibit_count_++;
|
}
|
|
void Application::ReleaseSleepInhibit() {
|
std::lock_guard<std::mutex> lock(mutex_);
|
if (sleep_inhibit_count_ > 0) {
|
sleep_inhibit_count_--;
|
}
|
}
|
|
void Application::FinishTtsResponse() {
|
const bool end_session = tts_session_end_;
|
const bool hold_idle = hold_idle_for_prompt_;
|
tts_audio_expected_ = false;
|
tts_response_started_ = false;
|
tts_audio_received_ = false;
|
tts_stop_pending_ = false;
|
tts_stop_waiting_ = false;
|
tts_idle_waiting_ = false;
|
tts_session_end_ = false;
|
tts_speaking_state_ready_ = false;
|
tts_playback_empty_ticks_ = 0;
|
tts_stop_wait_ticks_ = 0;
|
speaking_started_us_ = 0;
|
audio_service_.SetAudioPowerInhibit(false);
|
if (hold_idle || ShouldHoldIdleForLessonWork()) {
|
suppress_wake_until_listen_ = true;
|
} else {
|
suppress_wake_until_listen_ = false;
|
pending_detect_text_.clear();
|
pending_detect_retries_left_ = 0;
|
pending_detect_retry_ticks_ = 0;
|
}
|
ResumeSdCardMp3AfterVoice();
|
if (resume_sdcard_mp3_after_tts_ && !sdcard_mp3_stop_requested_) {
|
resume_sdcard_mp3_after_tts_ = false;
|
ResumeSdCardMp3();
|
} else {
|
resume_sdcard_mp3_after_tts_ = false;
|
}
|
if (GetDeviceState() == kDeviceStateSpeaking) {
|
if (end_session || hold_idle || ShouldHoldIdleForLessonWork() ||
|
listening_mode_ == kListeningModeManualStop ||
|
(IsSdCardMp3Playing() && !IsSdCardMp3Paused())) {
|
ESP_LOGI(TAG, "TTS done, return to idle session_end=%d hold=%d lesson=%d",
|
end_session ? 1 : 0, hold_idle ? 1 : 0,
|
ShouldHoldIdleForLessonWork() ? 1 : 0);
|
SetDeviceState(kDeviceStateIdle);
|
} else {
|
listening_mode_ = GetDefaultListeningMode();
|
SetListeningMode(listening_mode_);
|
}
|
}
|
if (IsSdCardMp3Playing() && !IsSdCardMp3Paused()) {
|
ApplyMp3CommandOnlyMic();
|
}
|
Board_FlushPendingLessonPlayback();
|
if (power_off_requested_) {
|
power_off_requested_ = false;
|
Board_Enter_Power_Off();
|
return;
|
}
|
if (standby_requested_) {
|
standby_requested_ = false;
|
SetDeviceState(kDeviceStateIdle);
|
}
|
}
|
|
void Application::ScheduleTtsAudio(std::shared_ptr<AudioStreamPacket> packet) {
|
Schedule([this, packet]() {
|
auto state = GetDeviceState();
|
if (state != kDeviceStateSpeaking && state != kDeviceStateIdle &&
|
state != kDeviceStateListening) {
|
ESP_LOGW(TAG, "TTS audio ignored state=%d expected=%d",
|
static_cast<int>(state), tts_audio_expected_ ? 1 : 0);
|
return;
|
}
|
tts_audio_expected_ = true;
|
tts_audio_received_ = true;
|
tts_activity_generation_++;
|
tts_playback_empty_ticks_ = 0;
|
audio_service_.KeepCodecReadyForPlayback();
|
auto audio_packet = std::make_unique<AudioStreamPacket>(std::move(*packet));
|
if (!audio_service_.PushPacketToDecodeQueue(std::move(audio_packet), true)) {
|
ESP_LOGW(TAG, "TTS audio dropped, decode queue full");
|
}
|
if (tts_stop_pending_) {
|
FinishTtsResponseAfterPlayback();
|
}
|
});
|
}
|
|
void Application::FinishTtsResponseAfterPlayback() {
|
tts_stop_pending_ = true;
|
tts_stop_waiting_ = true;
|
tts_stop_wait_ticks_ = 0;
|
}
|
|
void Application::MaybeFinishTtsAfterPlayback() {
|
if (GetDeviceState() != kDeviceStateSpeaking) {
|
tts_playback_empty_ticks_ = 0;
|
tts_stop_wait_ticks_ = 0;
|
return;
|
}
|
|
const bool playback_empty = audio_service_.IsPlaybackQueueEmpty();
|
if (playback_empty) {
|
tts_playback_empty_ticks_++;
|
} else {
|
tts_playback_empty_ticks_ = 0;
|
}
|
|
const int64_t elapsed_ms = speaking_started_us_ > 0
|
? (esp_timer_get_time() - speaking_started_us_) / 1000
|
: 0;
|
|
if (tts_stop_waiting_ || tts_stop_pending_) {
|
tts_stop_wait_ticks_++;
|
const int empty_needed = tts_audio_received_ ? 1 : 2;
|
if (playback_empty && tts_playback_empty_ticks_ >= empty_needed) {
|
ESP_LOGI(TAG, "TTS done empty=%d wait=%d received=%d session_end=%d",
|
tts_playback_empty_ticks_, tts_stop_wait_ticks_, tts_audio_received_,
|
tts_session_end_ ? 1 : 0);
|
FinishTtsResponse();
|
} else if (!playback_empty && tts_stop_wait_ticks_ >= 30) {
|
ESP_LOGW(TAG, "TTS playback stuck after stop, return to listen");
|
FinishTtsResponse();
|
}
|
return;
|
}
|
|
// 欢迎词/长回答：stop 没到时，只在「一直没收到 opus」或「播完后长时间空队列」才回听。
|
// 不要用说话墙钟 15s/队列空 2s：中英对话长句会在句间空隙或 15 秒处被掐断。
|
if (!tts_audio_received_ && elapsed_ms >= 10000) {
|
ESP_LOGW(TAG, "TTS stop missing, return to listen elapsed=%dms received=0 empty=%d",
|
(int)elapsed_ms, tts_playback_empty_ticks_);
|
FinishTtsResponse();
|
} else if (playback_empty && tts_audio_received_ && tts_playback_empty_ticks_ >= 8) {
|
ESP_LOGW(TAG, "TTS stop missing after idle playback elapsed=%dms empty=%d",
|
(int)elapsed_ms, tts_playback_empty_ticks_);
|
FinishTtsResponse();
|
}
|
}
|
|
bool Application::CanEnterSleepMode() {
|
{
|
std::lock_guard<std::mutex> lock(mutex_);
|
if (sleep_inhibit_count_ > 0) {
|
return false;
|
}
|
}
|
|
if (GetDeviceState() != kDeviceStateIdle) {
|
return false;
|
}
|
|
// 泛听是 idle 播 SD MP3，AudioService 队列是空的，会被当成可休眠。
|
// 跟读每句有听窗所以不会进 WiFi 待机；泛听会把网关掉，结束语就发不出去。
|
if (IsSdCardMp3Playing() || Board_IsRepeaterTask() || Board_IsShadowingTask()) {
|
return false;
|
}
|
|
if (!audio_service_.IsIdle()) {
|
return false;
|
}
|
|
// Now it is safe to enter sleep mode
|
return true;
|
}
|
|
void Application::PrepareProtocolForNetworkStandby() {
|
if (protocol_) {
|
protocol_->PrepareForNetworkStandby();
|
}
|
}
|
|
void Application::SendMcpMessage(const std::string& payload) {
|
// Always schedule to run in main task for thread safety
|
Schedule([this, payload = std::move(payload)]() {
|
if (protocol_ && !power_off_standby_active_.load()) {
|
protocol_->SendMcpMessage(payload);
|
}
|
});
|
}
|
|
void Application::SetAecMode(AecMode mode) {
|
aec_mode_ = mode;
|
Schedule([this]() {
|
auto& board = Board::GetInstance();
|
auto display = board.GetDisplay();
|
switch (aec_mode_) {
|
case kAecOff:
|
audio_service_.EnableDeviceAec(false);
|
display->ShowNotification(Lang::Strings::RTC_MODE_OFF);
|
break;
|
case kAecOnServerSide:
|
audio_service_.EnableDeviceAec(false);
|
display->ShowNotification(Lang::Strings::RTC_MODE_ON);
|
break;
|
case kAecOnDeviceSide:
|
audio_service_.EnableDeviceAec(true);
|
display->ShowNotification(Lang::Strings::RTC_MODE_ON);
|
break;
|
}
|
|
// If the AEC mode is changed, close the audio channel
|
if (protocol_ && protocol_->IsAudioChannelOpened()) {
|
protocol_->CloseAudioChannel();
|
}
|
});
|
}
|
|
void Application::PlaySound(const std::string_view& sound) {
|
audio_service_.PlaySound(sound);
|
}
|
|
void Application::ResetProtocol() {
|
Schedule([this]() {
|
// Close audio channel if opened
|
if (protocol_ && protocol_->IsAudioChannelOpened()) {
|
protocol_->CloseAudioChannel();
|
}
|
// Reset protocol
|
protocol_.reset();
|
});
|
}
|
