#include "afe_wake_word.h" #include "audio_service.h" #include "boards/sening-xiaozhi-s3/system.h" #include #include #include #include #include #include #define DETECTION_RUNNING_EVENT 1 #define TAG "AfeWakeWord" AfeWakeWord::AfeWakeWord() : afe_data_(nullptr), wake_word_pcm_(), wake_word_opus_() { event_group_ = xEventGroupCreate(); } AfeWakeWord::~AfeWakeWord() { if (afe_data_ != nullptr) { afe_iface_->destroy(afe_data_); } if (multinet_data_ != nullptr && multinet_ != nullptr) { multinet_->destroy(multinet_data_); } if (wake_word_encode_task_stack_ != nullptr) { heap_caps_free(wake_word_encode_task_stack_); } if (wake_word_encode_task_buffer_ != nullptr) { heap_caps_free(wake_word_encode_task_buffer_); } if (models_ != nullptr) { esp_srmodel_deinit(models_); } vEventGroupDelete(event_group_); } bool AfeWakeWord::Initialize(AudioCodec* codec, srmodel_list_t* models_list) { codec_ = codec; if (models_list == nullptr) { models_ = esp_srmodel_init("model"); } else { models_ = models_list; } if (models_ == nullptr || models_->num == -1) { ESP_LOGE(TAG, "Failed to initialize wakenet model"); return false; } for (int i = 0; i < models_->num; i++) { ESP_LOGI(TAG, "Model %d: %s", i, models_->model_name[i]); if (strstr(models_->model_name[i], ESP_WN_PREFIX) != NULL) { wakenet_model_ = models_->model_name[i]; auto words = esp_srmodel_get_wake_words(models_, wakenet_model_); // split by ";" to get all wake words std::stringstream ss(words); std::string word; while (std::getline(ss, word, ';')) { wake_words_.push_back(word); } } } multinet_model_ = esp_srmodel_filter(models_, ESP_MN_PREFIX, "cn"); if (multinet_model_ == nullptr) { multinet_model_ = esp_srmodel_filter(models_, ESP_MN_PREFIX, NULL); } if (!EnsureDetectionTask()) { ESP_LOGE(TAG, "Failed to start audio detection task"); return false; } if (multinet_model_ != nullptr) { multinet_ = esp_mn_handle_from_name(multinet_model_); multinet_data_ = multinet_->create(multinet_model_, 3000); multinet_->set_det_threshold(multinet_data_, 0.2f); esp_mn_commands_clear(); esp_mn_commands_add(1, OFFLINE_COMMAND_WORD_PINYIN); esp_mn_commands_add(2, OFFLINE_REPEATER_WORD_PINYIN); esp_mn_commands_add(3, OFFLINE_START_REPEATER_WORD_PINYIN); esp_mn_commands_add(4, OFFLINE_EXIT_REPEATER_WORD_PINYIN); esp_mn_commands_add(5, OFFLINE_COMMAND_WORD_ALT_PINYIN); esp_mn_commands_add(6, OFFLINE_CHAT_WORD_PINYIN); esp_mn_commands_add(7, OFFLINE_EXIT_SHADOWING_WORD_PINYIN); esp_mn_commands_add(8, OFFLINE_CHAT_WORD_FULL_PINYIN); esp_mn_commands_update(); multinet_->print_active_speech_commands(multinet_data_); } else { ESP_LOGW(TAG, "MultiNet model not found, offline command word disabled"); } if (!CreateAfe()) { ESP_LOGE(TAG, "Failed to create AFE runtime"); return false; } return true; } bool AfeWakeWord::EnsureDetectionTask() { if (detection_task_started_ && detection_task_handle_ != nullptr) { return true; } const size_t stack_size = 8192; if (detection_task_stack_ == nullptr) { detection_task_stack_ = (StackType_t*)heap_caps_malloc(stack_size, MALLOC_CAP_SPIRAM); if (detection_task_stack_ == nullptr) { ESP_LOGE(TAG, "Failed to alloc audio detection stack"); return false; } } if (detection_task_buffer_ == nullptr) { detection_task_buffer_ = (StaticTask_t*)heap_caps_malloc(sizeof(StaticTask_t), MALLOC_CAP_INTERNAL); if (detection_task_buffer_ == nullptr) { ESP_LOGE(TAG, "Failed to alloc audio detection TCB"); return false; } } detection_task_handle_ = xTaskCreateStatic([](void* arg) { auto this_ = (AfeWakeWord*)arg; this_->AudioDetectionTask(); vTaskDelete(NULL); }, "audio_detection", stack_size, this, 3, detection_task_stack_, detection_task_buffer_); if (detection_task_handle_ == nullptr) { ESP_LOGE(TAG, "Failed to create audio detection task"); return false; } detection_task_started_ = true; return true; } bool AfeWakeWord::CreateAfe() { if (afe_data_ != nullptr) { return true; } if (codec_ == nullptr) { return false; } int ref_num = codec_->input_reference() ? 1 : 0; std::string input_format; for (int i = 0; i < codec_->input_channels() - ref_num; i++) { input_format.push_back('M'); } for (int i = 0; i < ref_num; i++) { input_format.push_back('R'); } afe_config_t* afe_config = afe_config_init(input_format.c_str(), models_, AFE_TYPE_SR, AFE_MODE_HIGH_PERF); afe_config->aec_init = codec_->input_reference(); afe_config->aec_mode = AEC_MODE_SR_HIGH_PERF; afe_config->afe_perferred_core = 1; afe_config->afe_perferred_priority = 1; afe_config->memory_alloc_mode = AFE_MEMORY_ALLOC_MORE_PSRAM; afe_iface_ = esp_afe_handle_from_config(afe_config); afe_data_ = afe_iface_->create_from_config(afe_config); if (afe_data_ == nullptr) { ESP_LOGE(TAG, "create_from_config failed"); return false; } ESP_LOGI(TAG, "AFE runtime created"); return true; } void AfeWakeWord::ReleaseRuntime() { Stop(); if (afe_data_ == nullptr) { return; } // fetch 用短超时,停检测后再销毁,避免和 AudioDetectionTask 抢 afe_data_ vTaskDelay(pdMS_TO_TICKS(120)); { std::lock_guard lock(input_buffer_mutex_); if (afe_data_ != nullptr && afe_iface_ != nullptr) { afe_iface_->destroy(afe_data_); afe_data_ = nullptr; } input_buffer_.clear(); multinet_input_buffer_.clear(); } ClearWakeBuffers(); ESP_LOGI(TAG, "AFE runtime released"); } bool AfeWakeWord::EnsureRuntime() { return CreateAfe(); } void AfeWakeWord::ClearWakeBuffers() { std::lock_guard lock(wake_word_mutex_); wake_word_pcm_.clear(); wake_word_opus_.clear(); } void AfeWakeWord::AbortWakeWordEncode() { std::lock_guard lock(wake_word_mutex_); wake_word_pcm_.clear(); wake_word_opus_.clear(); wake_word_opus_.emplace_back(); wake_word_cv_.notify_all(); } void AfeWakeWord::OnWakeWordDetected(std::function callback) { wake_word_detected_callback_ = callback; } void AfeWakeWord::OnOfflineCommandDetected(std::function callback) { offline_command_detected_callback_ = callback; } void AfeWakeWord::Start() { xEventGroupSetBits(event_group_, DETECTION_RUNNING_EVENT); } void AfeWakeWord::Stop() { xEventGroupClearBits(event_group_, DETECTION_RUNNING_EVENT); std::lock_guard lock(input_buffer_mutex_); if (afe_data_ != nullptr) { afe_iface_->reset_buffer(afe_data_); } if (multinet_data_ != nullptr) { multinet_->clean(multinet_data_); } input_buffer_.clear(); multinet_input_buffer_.clear(); } void AfeWakeWord::Feed(const std::vector& data) { if (afe_data_ == nullptr || detection_task_handle_ == nullptr) { return; } std::lock_guard lock(input_buffer_mutex_); // Check running state inside lock to avoid TOCTOU race with Stop() if (!(xEventGroupGetBits(event_group_) & DETECTION_RUNNING_EVENT)) { return; } input_buffer_.insert(input_buffer_.end(), data.begin(), data.end()); size_t chunk_size = afe_iface_->get_feed_chunksize(afe_data_) * codec_->input_channels(); while (input_buffer_.size() >= chunk_size) { afe_iface_->feed(afe_data_, input_buffer_.data()); input_buffer_.erase(input_buffer_.begin(), input_buffer_.begin() + chunk_size); } } size_t AfeWakeWord::GetFeedSize() { if (afe_data_ == nullptr) { return 0; } return afe_iface_->get_feed_chunksize(afe_data_); } void AfeWakeWord::AudioDetectionTask() { ESP_LOGI(TAG, "Audio detection task started"); while (true) { xEventGroupWaitBits(event_group_, DETECTION_RUNNING_EVENT, pdFALSE, pdTRUE, portMAX_DELAY); esp_afe_sr_data_t* afe_data = nullptr; const esp_afe_sr_iface_t* afe_iface = nullptr; { std::lock_guard lock(input_buffer_mutex_); afe_data = afe_data_; afe_iface = afe_iface_; } if (afe_data == nullptr || afe_iface == nullptr) { vTaskDelay(pdMS_TO_TICKS(20)); continue; } auto res = afe_iface->fetch_with_delay(afe_data, pdMS_TO_TICKS(80)); if (!(xEventGroupGetBits(event_group_) & DETECTION_RUNNING_EVENT)) { continue; } if (res == nullptr || res->ret_value == ESP_FAIL) { continue; } // Store the wake word data for voice recognition, like who is speaking StoreWakeWordData(res->data, res->data_size / sizeof(int16_t)); if (multinet_data_ != nullptr) { multinet_input_buffer_.insert(multinet_input_buffer_.end(), res->data, res->data + res->data_size / sizeof(int16_t)); int chunksize = multinet_->get_samp_chunksize(multinet_data_); while (multinet_input_buffer_.size() >= chunksize) { esp_mn_state_t mn_state = multinet_->detect(multinet_data_, multinet_input_buffer_.data()); if (mn_state == ESP_MN_STATE_DETECTED) { esp_mn_results_t *mn_result = multinet_->get_results(multinet_data_); for (int i = 0; i < mn_result->num; i++) { int command_id = mn_result->command_id[i]; const char *command_text = nullptr; if (command_id == 1 || command_id == 5) { command_text = OFFLINE_COMMAND_WORD; } else if (command_id == 2) { command_text = OFFLINE_REPEATER_WORD; } else if (command_id == 3) { command_text = OFFLINE_START_REPEATER_WORD; } else if (command_id == 4) { command_text = OFFLINE_EXIT_REPEATER_WORD; } else if (command_id == 6) { command_text = OFFLINE_CHAT_WORD; } else if (command_id == 7) { command_text = OFFLINE_EXIT_SHADOWING_WORD; } else if (command_id == 8) { command_text = OFFLINE_CHAT_WORD_FULL; } ESP_LOGI(TAG, "Offline command detected: command_id=%d, string=%s, prob=%f", command_id, mn_result->string, mn_result->prob[i]); if (command_text && offline_command_detected_callback_) { offline_command_detected_callback_(command_text); } } multinet_->clean(multinet_data_); multinet_input_buffer_.clear(); break; } else if (mn_state == ESP_MN_STATE_TIMEOUT) { multinet_->clean(multinet_data_); multinet_input_buffer_.clear(); break; } multinet_input_buffer_.erase(multinet_input_buffer_.begin(), multinet_input_buffer_.begin() + chunksize); } } if (res->wakeup_state == WAKENET_DETECTED) { Stop(); last_detected_wake_word_ = wake_words_[res->wakenet_model_index - 1]; if (wake_word_detected_callback_) { wake_word_detected_callback_(last_detected_wake_word_); } } } } void AfeWakeWord::StoreWakeWordData(const int16_t* data, size_t samples) { // 512 sample * 2B = 1KB,默认会进内部 SRAM,播完本地 MP3 后 DMA 最大块会被切成 ~1KB。 std::vector chunk; chunk.reserve(2048); chunk.assign(data, data + samples); wake_word_pcm_.push_back(std::move(chunk)); // keep about 2 seconds of data, detect duration is 30ms (sample_rate == 16000, chunksize == 512) while (wake_word_pcm_.size() > 2000 / 30) { wake_word_pcm_.pop_front(); } } void AfeWakeWord::EncodeWakeWordData() { const size_t stack_size = 4096 * 6; wake_word_opus_.clear(); if (wake_word_encode_task_stack_ == nullptr) { wake_word_encode_task_stack_ = (StackType_t*)heap_caps_malloc(stack_size, MALLOC_CAP_SPIRAM); assert(wake_word_encode_task_stack_ != nullptr); } if (wake_word_encode_task_buffer_ == nullptr) { wake_word_encode_task_buffer_ = (StaticTask_t*)heap_caps_malloc(sizeof(StaticTask_t), MALLOC_CAP_INTERNAL); assert(wake_word_encode_task_buffer_ != nullptr); } wake_word_encode_task_ = xTaskCreateStatic([](void* arg) { auto this_ = (AfeWakeWord*)arg; { auto start_time = esp_timer_get_time(); // Create encoder esp_opus_enc_config_t opus_enc_cfg = AS_OPUS_ENC_CONFIG(); void* encoder_handle = nullptr; auto ret = esp_opus_enc_open(&opus_enc_cfg, sizeof(esp_opus_enc_config_t), &encoder_handle); if (encoder_handle == nullptr) { ESP_LOGE(TAG, "Failed to create audio encoder, error code: %d", ret); std::lock_guard lock(this_->wake_word_mutex_); this_->wake_word_opus_.push_back(std::vector()); this_->wake_word_cv_.notify_all(); return; } // Get frame size int frame_size = 0; int outbuf_size = 0; esp_opus_enc_get_frame_size(encoder_handle, &frame_size, &outbuf_size); frame_size = frame_size / sizeof(int16_t); // Encode all PCM data int packets = 0; std::vector in_buffer; esp_audio_enc_in_frame_t in = {}; esp_audio_enc_out_frame_t out = {}; for (auto& pcm: this_->wake_word_pcm_) { if (in_buffer.empty()) { in_buffer = std::move(pcm); } else { in_buffer.reserve(in_buffer.size() + pcm.size()); in_buffer.insert(in_buffer.end(), pcm.begin(), pcm.end()); } while (in_buffer.size() >= frame_size) { std::vector opus_buf(outbuf_size); in.buffer = (uint8_t *)(in_buffer.data()); in.len = (uint32_t)(frame_size * sizeof(int16_t)); out.buffer = opus_buf.data(); out.len = outbuf_size; out.encoded_bytes = 0; ret = esp_opus_enc_process(encoder_handle, &in, &out); if (ret == ESP_AUDIO_ERR_OK) { std::lock_guard lock(this_->wake_word_mutex_); this_->wake_word_opus_.emplace_back(opus_buf.data(), opus_buf.data() + out.encoded_bytes); this_->wake_word_cv_.notify_all(); packets++; } else { ESP_LOGE(TAG, "Failed to encode audio, error code: %d", ret); } in_buffer.erase(in_buffer.begin(), in_buffer.begin() + frame_size); } } this_->wake_word_pcm_.clear(); // Close encoder esp_opus_enc_close(encoder_handle); auto end_time = esp_timer_get_time(); ESP_LOGI(TAG, "Encode wake word opus %d packets in %ld ms", packets, (long)((end_time - start_time) / 1000)); std::lock_guard lock(this_->wake_word_mutex_); this_->wake_word_opus_.push_back(std::vector()); this_->wake_word_cv_.notify_all(); } vTaskDelete(NULL); }, "encode_wake_word", stack_size, this, 2, wake_word_encode_task_stack_, wake_word_encode_task_buffer_); if (wake_word_encode_task_ == nullptr) { ESP_LOGE(TAG, "Failed to create encode_wake_word task"); std::lock_guard lock(wake_word_mutex_); wake_word_opus_.push_back(std::vector()); wake_word_cv_.notify_all(); } } bool AfeWakeWord::GetWakeWordOpus(std::vector& opus) { std::unique_lock lock(wake_word_mutex_); if (!wake_word_cv_.wait_for(lock, std::chrono::milliseconds(2000), [this]() { return !wake_word_opus_.empty(); })) { ESP_LOGW(TAG, "wake opus wait timeout"); return false; } opus.swap(wake_word_opus_.front()); wake_word_opus_.pop_front(); return !opus.empty(); }