From 220e4fab27b7f419b3e89e2f2d28ae789aac09d7 Mon Sep 17 00:00:00 2001
From: wangzhibo <wangzhibo@shsening.com>
Date: 星期四, 08 十月 2026 16:29:55 +0800
Subject: [PATCH] 5种场景模式家长端ok
---
xiaozhi-esp32-2.2.6/main/audio/wake_words/afe_wake_word.cc | 157 +++++++++++++++++++++++++++++++++++++++++++++------
1 files changed, 137 insertions(+), 20 deletions(-)
diff --git a/xiaozhi-esp32-2.2.6/main/audio/wake_words/afe_wake_word.cc b/xiaozhi-esp32-2.2.6/main/audio/wake_words/afe_wake_word.cc
index 3436fac..0cebc1b 100644
--- a/xiaozhi-esp32-2.2.6/main/audio/wake_words/afe_wake_word.cc
+++ b/xiaozhi-esp32-2.2.6/main/audio/wake_words/afe_wake_word.cc
@@ -1,9 +1,11 @@
#include "afe_wake_word.h"
#include "audio_service.h"
#include "boards/sening-xiaozhi-s3/system.h"
+#include <esp_heap_caps.h>
#include <esp_log.h>
#include <esp_mn_models.h>
#include <esp_mn_speech_commands.h>
+#include <chrono>
#include <sstream>
#define DETECTION_RUNNING_EVENT 1
@@ -43,7 +45,6 @@
bool AfeWakeWord::Initialize(AudioCodec* codec, srmodel_list_t* models_list) {
codec_ = codec;
- int ref_num = codec_->input_reference() ? 1 : 0;
if (models_list == nullptr) {
models_ = esp_srmodel_init("model");
@@ -73,6 +74,11 @@
if (multinet_model_ == nullptr) {
multinet_model_ = esp_srmodel_filter(models_, ESP_MN_PREFIX, NULL);
}
+ if (!EnsureDetectionTask()) {
+ ESP_LOGE(TAG, "Failed to start audio detection task");
+ return false;
+ }
+
if (multinet_model_ != nullptr) {
multinet_ = esp_mn_handle_from_name(multinet_model_);
multinet_data_ = multinet_->create(multinet_model_, 3000);
@@ -92,6 +98,57 @@
ESP_LOGW(TAG, "MultiNet model not found, offline command word disabled");
}
+ if (!CreateAfe()) {
+ ESP_LOGE(TAG, "Failed to create AFE runtime");
+ return false;
+ }
+
+ return true;
+}
+
+bool AfeWakeWord::EnsureDetectionTask() {
+ if (detection_task_started_ && detection_task_handle_ != nullptr) {
+ return true;
+ }
+
+ const size_t stack_size = 8192;
+ if (detection_task_stack_ == nullptr) {
+ detection_task_stack_ = (StackType_t*)heap_caps_malloc(stack_size, MALLOC_CAP_SPIRAM);
+ if (detection_task_stack_ == nullptr) {
+ ESP_LOGE(TAG, "Failed to alloc audio detection stack");
+ return false;
+ }
+ }
+ if (detection_task_buffer_ == nullptr) {
+ detection_task_buffer_ = (StaticTask_t*)heap_caps_malloc(sizeof(StaticTask_t), MALLOC_CAP_INTERNAL);
+ if (detection_task_buffer_ == nullptr) {
+ ESP_LOGE(TAG, "Failed to alloc audio detection TCB");
+ return false;
+ }
+ }
+
+ detection_task_handle_ = xTaskCreateStatic([](void* arg) {
+ auto this_ = (AfeWakeWord*)arg;
+ this_->AudioDetectionTask();
+ vTaskDelete(NULL);
+ }, "audio_detection", stack_size, this, 3, detection_task_stack_, detection_task_buffer_);
+ if (detection_task_handle_ == nullptr) {
+ ESP_LOGE(TAG, "Failed to create audio detection task");
+ return false;
+ }
+ detection_task_started_ = true;
+ return true;
+}
+
+bool AfeWakeWord::CreateAfe() {
+ if (afe_data_ != nullptr) {
+ return true;
+ }
+ if (codec_ == nullptr) {
+ return false;
+ }
+
+ int ref_num = codec_->input_reference() ? 1 : 0;
std::string input_format;
for (int i = 0; i < codec_->input_channels() - ref_num; i++) {
input_format.push_back('M');
@@ -105,17 +162,53 @@
afe_config->afe_perferred_core = 1;
afe_config->afe_perferred_priority = 1;
afe_config->memory_alloc_mode = AFE_MEMORY_ALLOC_MORE_PSRAM;
-
+
afe_iface_ = esp_afe_handle_from_config(afe_config);
afe_data_ = afe_iface_->create_from_config(afe_config);
-
- xTaskCreate([](void* arg) {
- auto this_ = (AfeWakeWord*)arg;
- this_->AudioDetectionTask();
- vTaskDelete(NULL);
- }, "audio_detection", 8192, this, 3, nullptr);
-
+ if (afe_data_ == nullptr) {
+ ESP_LOGE(TAG, "create_from_config failed");
+ return false;
+ }
+ ESP_LOGI(TAG, "AFE runtime created");
return true;
+}
+
+void AfeWakeWord::ReleaseRuntime() {
+ Stop();
+ if (afe_data_ == nullptr) {
+ return;
+ }
+ // fetch 鐢ㄧ煭瓒呮椂锛屽仠妫�娴嬪悗鍐嶉攢姣侊紝閬垮厤鍜� AudioDetectionTask 鎶� afe_data_
+ vTaskDelay(pdMS_TO_TICKS(120));
+ {
+ std::lock_guard<std::mutex> lock(input_buffer_mutex_);
+ if (afe_data_ != nullptr && afe_iface_ != nullptr) {
+ afe_iface_->destroy(afe_data_);
+ afe_data_ = nullptr;
+ }
+ input_buffer_.clear();
+ multinet_input_buffer_.clear();
+ }
+ ClearWakeBuffers();
+ ESP_LOGI(TAG, "AFE runtime released");
+}
+
+bool AfeWakeWord::EnsureRuntime() {
+ return CreateAfe();
+}
+
+void AfeWakeWord::ClearWakeBuffers() {
+ std::lock_guard<std::mutex> lock(wake_word_mutex_);
+ wake_word_pcm_.clear();
+ wake_word_opus_.clear();
+}
+
+void AfeWakeWord::AbortWakeWordEncode() {
+ std::lock_guard<std::mutex> lock(wake_word_mutex_);
+ wake_word_pcm_.clear();
+ wake_word_opus_.clear();
+ wake_word_opus_.emplace_back();
+ wake_word_cv_.notify_all();
}
void AfeWakeWord::OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback) {
@@ -145,7 +238,7 @@
}
void AfeWakeWord::Feed(const std::vector<int16_t>& data) {
- if (afe_data_ == nullptr) {
+ if (afe_data_ == nullptr || detection_task_handle_ == nullptr) {
return;
}
@@ -170,17 +263,29 @@
}
void AfeWakeWord::AudioDetectionTask() {
- auto fetch_size = afe_iface_->get_fetch_chunksize(afe_data_);
- auto feed_size = afe_iface_->get_feed_chunksize(afe_data_);
- ESP_LOGI(TAG, "Audio detection task started, feed size: %d fetch size: %d",
- feed_size, fetch_size);
+ ESP_LOGI(TAG, "Audio detection task started");
while (true) {
xEventGroupWaitBits(event_group_, DETECTION_RUNNING_EVENT, pdFALSE, pdTRUE, portMAX_DELAY);
- auto res = afe_iface_->fetch_with_delay(afe_data_, portMAX_DELAY);
+ esp_afe_sr_data_t* afe_data = nullptr;
+ const esp_afe_sr_iface_t* afe_iface = nullptr;
+ {
+ std::lock_guard<std::mutex> lock(input_buffer_mutex_);
+ afe_data = afe_data_;
+ afe_iface = afe_iface_;
+ }
+ if (afe_data == nullptr || afe_iface == nullptr) {
+ vTaskDelay(pdMS_TO_TICKS(20));
+ continue;
+ }
+
+ auto res = afe_iface->fetch_with_delay(afe_data, pdMS_TO_TICKS(80));
+ if (!(xEventGroupGetBits(event_group_) & DETECTION_RUNNING_EVENT)) {
+ continue;
+ }
if (res == nullptr || res->ret_value == ESP_FAIL) {
- continue;;
+ continue;
}
// Store the wake word data for voice recognition, like who is speaking
@@ -241,8 +346,11 @@
}
void AfeWakeWord::StoreWakeWordData(const int16_t* data, size_t samples) {
- // store audio data to wake_word_pcm_
- wake_word_pcm_.emplace_back(std::vector<int16_t>(data, data + samples));
+ // 512 sample * 2B = 1KB锛岄粯璁や細杩涘唴閮� SRAM锛屾挱瀹屾湰鍦� MP3 鍚� DMA 鏈�澶у潡浼氳鍒囨垚 ~1KB銆�
+ std::vector<int16_t> chunk;
+ chunk.reserve(2048);
+ chunk.assign(data, data + samples);
+ wake_word_pcm_.push_back(std::move(chunk));
// keep about 2 seconds of data, detect duration is 30ms (sample_rate == 16000, chunksize == 512)
while (wake_word_pcm_.size() > 2000 / 30) {
wake_word_pcm_.pop_front();
@@ -330,13 +438,22 @@
}
vTaskDelete(NULL);
}, "encode_wake_word", stack_size, this, 2, wake_word_encode_task_stack_, wake_word_encode_task_buffer_);
+ if (wake_word_encode_task_ == nullptr) {
+ ESP_LOGE(TAG, "Failed to create encode_wake_word task");
+ std::lock_guard<std::mutex> lock(wake_word_mutex_);
+ wake_word_opus_.push_back(std::vector<uint8_t>());
+ wake_word_cv_.notify_all();
+ }
}
bool AfeWakeWord::GetWakeWordOpus(std::vector<uint8_t>& opus) {
std::unique_lock<std::mutex> lock(wake_word_mutex_);
- wake_word_cv_.wait(lock, [this]() {
+ if (!wake_word_cv_.wait_for(lock, std::chrono::milliseconds(2000), [this]() {
return !wake_word_opus_.empty();
- });
+ })) {
+ ESP_LOGW(TAG, "wake opus wait timeout");
+ return false;
+ }
opus.swap(wake_word_opus_.front());
wake_word_opus_.pop_front();
return !opus.empty();
--
Gitblit v1.9.1