#include "afe_wake_word.h"
|
#include "audio_service.h"
|
#include "boards/sening-xiaozhi-s3/system.h"
|
#include <esp_heap_caps.h>
|
#include <esp_log.h>
|
#include <esp_mn_models.h>
|
#include <esp_mn_speech_commands.h>
|
#include <chrono>
|
#include <sstream>
|
|
#define DETECTION_RUNNING_EVENT 1
|
|
#define TAG "AfeWakeWord"
|
|
AfeWakeWord::AfeWakeWord()
|
: afe_data_(nullptr),
|
wake_word_pcm_(),
|
wake_word_opus_() {
|
|
event_group_ = xEventGroupCreate();
|
}
|
|
AfeWakeWord::~AfeWakeWord() {
|
if (afe_data_ != nullptr) {
|
afe_iface_->destroy(afe_data_);
|
}
|
if (multinet_data_ != nullptr && multinet_ != nullptr) {
|
multinet_->destroy(multinet_data_);
|
}
|
|
if (wake_word_encode_task_stack_ != nullptr) {
|
heap_caps_free(wake_word_encode_task_stack_);
|
}
|
|
if (wake_word_encode_task_buffer_ != nullptr) {
|
heap_caps_free(wake_word_encode_task_buffer_);
|
}
|
|
if (models_ != nullptr) {
|
esp_srmodel_deinit(models_);
|
}
|
|
vEventGroupDelete(event_group_);
|
}
|
|
bool AfeWakeWord::Initialize(AudioCodec* codec, srmodel_list_t* models_list) {
|
codec_ = codec;
|
|
if (models_list == nullptr) {
|
models_ = esp_srmodel_init("model");
|
} else {
|
models_ = models_list;
|
}
|
|
if (models_ == nullptr || models_->num == -1) {
|
ESP_LOGE(TAG, "Failed to initialize wakenet model");
|
return false;
|
}
|
for (int i = 0; i < models_->num; i++) {
|
ESP_LOGI(TAG, "Model %d: %s", i, models_->model_name[i]);
|
if (strstr(models_->model_name[i], ESP_WN_PREFIX) != NULL) {
|
wakenet_model_ = models_->model_name[i];
|
auto words = esp_srmodel_get_wake_words(models_, wakenet_model_);
|
// split by ";" to get all wake words
|
std::stringstream ss(words);
|
std::string word;
|
while (std::getline(ss, word, ';')) {
|
wake_words_.push_back(word);
|
}
|
}
|
}
|
|
multinet_model_ = esp_srmodel_filter(models_, ESP_MN_PREFIX, "cn");
|
if (multinet_model_ == nullptr) {
|
multinet_model_ = esp_srmodel_filter(models_, ESP_MN_PREFIX, NULL);
|
}
|
if (!EnsureDetectionTask()) {
|
ESP_LOGE(TAG, "Failed to start audio detection task");
|
return false;
|
}
|
|
if (multinet_model_ != nullptr) {
|
multinet_ = esp_mn_handle_from_name(multinet_model_);
|
multinet_data_ = multinet_->create(multinet_model_, 3000);
|
multinet_->set_det_threshold(multinet_data_, 0.2f);
|
esp_mn_commands_clear();
|
esp_mn_commands_add(1, OFFLINE_COMMAND_WORD_PINYIN);
|
esp_mn_commands_add(2, OFFLINE_REPEATER_WORD_PINYIN);
|
esp_mn_commands_add(3, OFFLINE_START_REPEATER_WORD_PINYIN);
|
esp_mn_commands_add(4, OFFLINE_EXIT_REPEATER_WORD_PINYIN);
|
esp_mn_commands_add(5, OFFLINE_COMMAND_WORD_ALT_PINYIN);
|
esp_mn_commands_add(6, OFFLINE_CHAT_WORD_PINYIN);
|
esp_mn_commands_add(7, OFFLINE_EXIT_SHADOWING_WORD_PINYIN);
|
esp_mn_commands_add(8, OFFLINE_CHAT_WORD_FULL_PINYIN);
|
esp_mn_commands_update();
|
multinet_->print_active_speech_commands(multinet_data_);
|
} else {
|
ESP_LOGW(TAG, "MultiNet model not found, offline command word disabled");
|
}
|
|
if (!CreateAfe()) {
|
ESP_LOGE(TAG, "Failed to create AFE runtime");
|
return false;
|
}
|
|
return true;
|
}
|
|
bool AfeWakeWord::EnsureDetectionTask() {
|
if (detection_task_started_ && detection_task_handle_ != nullptr) {
|
return true;
|
}
|
|
const size_t stack_size = 8192;
|
if (detection_task_stack_ == nullptr) {
|
detection_task_stack_ = (StackType_t*)heap_caps_malloc(stack_size, MALLOC_CAP_SPIRAM);
|
if (detection_task_stack_ == nullptr) {
|
ESP_LOGE(TAG, "Failed to alloc audio detection stack");
|
return false;
|
}
|
}
|
if (detection_task_buffer_ == nullptr) {
|
detection_task_buffer_ = (StaticTask_t*)heap_caps_malloc(sizeof(StaticTask_t), MALLOC_CAP_INTERNAL);
|
if (detection_task_buffer_ == nullptr) {
|
ESP_LOGE(TAG, "Failed to alloc audio detection TCB");
|
return false;
|
}
|
}
|
|
detection_task_handle_ = xTaskCreateStatic([](void* arg) {
|
auto this_ = (AfeWakeWord*)arg;
|
this_->AudioDetectionTask();
|
vTaskDelete(NULL);
|
}, "audio_detection", stack_size, this, 3, detection_task_stack_, detection_task_buffer_);
|
if (detection_task_handle_ == nullptr) {
|
ESP_LOGE(TAG, "Failed to create audio detection task");
|
return false;
|
}
|
detection_task_started_ = true;
|
return true;
|
}
|
|
bool AfeWakeWord::CreateAfe() {
|
if (afe_data_ != nullptr) {
|
return true;
|
}
|
if (codec_ == nullptr) {
|
return false;
|
}
|
|
int ref_num = codec_->input_reference() ? 1 : 0;
|
std::string input_format;
|
for (int i = 0; i < codec_->input_channels() - ref_num; i++) {
|
input_format.push_back('M');
|
}
|
for (int i = 0; i < ref_num; i++) {
|
input_format.push_back('R');
|
}
|
afe_config_t* afe_config = afe_config_init(input_format.c_str(), models_, AFE_TYPE_SR, AFE_MODE_HIGH_PERF);
|
afe_config->aec_init = codec_->input_reference();
|
afe_config->aec_mode = AEC_MODE_SR_HIGH_PERF;
|
afe_config->afe_perferred_core = 1;
|
afe_config->afe_perferred_priority = 1;
|
afe_config->memory_alloc_mode = AFE_MEMORY_ALLOC_MORE_PSRAM;
|
|
afe_iface_ = esp_afe_handle_from_config(afe_config);
|
afe_data_ = afe_iface_->create_from_config(afe_config);
|
if (afe_data_ == nullptr) {
|
ESP_LOGE(TAG, "create_from_config failed");
|
return false;
|
}
|
ESP_LOGI(TAG, "AFE runtime created");
|
return true;
|
}
|
|
void AfeWakeWord::ReleaseRuntime() {
|
Stop();
|
if (afe_data_ == nullptr) {
|
return;
|
}
|
// fetch 用短超时，停检测后再销毁，避免和 AudioDetectionTask 抢 afe_data_
|
vTaskDelay(pdMS_TO_TICKS(120));
|
{
|
std::lock_guard<std::mutex> lock(input_buffer_mutex_);
|
if (afe_data_ != nullptr && afe_iface_ != nullptr) {
|
afe_iface_->destroy(afe_data_);
|
afe_data_ = nullptr;
|
}
|
input_buffer_.clear();
|
multinet_input_buffer_.clear();
|
}
|
ClearWakeBuffers();
|
ESP_LOGI(TAG, "AFE runtime released");
|
}
|
|
bool AfeWakeWord::EnsureRuntime() {
|
return CreateAfe();
|
}
|
|
void AfeWakeWord::ClearWakeBuffers() {
|
std::lock_guard<std::mutex> lock(wake_word_mutex_);
|
wake_word_pcm_.clear();
|
wake_word_opus_.clear();
|
}
|
|
void AfeWakeWord::AbortWakeWordEncode() {
|
std::lock_guard<std::mutex> lock(wake_word_mutex_);
|
wake_word_pcm_.clear();
|
wake_word_opus_.clear();
|
wake_word_opus_.emplace_back();
|
wake_word_cv_.notify_all();
|
}
|
|
void AfeWakeWord::OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback) {
|
wake_word_detected_callback_ = callback;
|
}
|
|
void AfeWakeWord::OnOfflineCommandDetected(std::function<void(const std::string& command)> callback) {
|
offline_command_detected_callback_ = callback;
|
}
|
|
void AfeWakeWord::Start() {
|
xEventGroupSetBits(event_group_, DETECTION_RUNNING_EVENT);
|
}
|
|
void AfeWakeWord::Stop() {
|
xEventGroupClearBits(event_group_, DETECTION_RUNNING_EVENT);
|
|
std::lock_guard<std::mutex> lock(input_buffer_mutex_);
|
if (afe_data_ != nullptr) {
|
afe_iface_->reset_buffer(afe_data_);
|
}
|
if (multinet_data_ != nullptr) {
|
multinet_->clean(multinet_data_);
|
}
|
input_buffer_.clear();
|
multinet_input_buffer_.clear();
|
}
|
|
void AfeWakeWord::Feed(const std::vector<int16_t>& data) {
|
if (afe_data_ == nullptr || detection_task_handle_ == nullptr) {
|
return;
|
}
|
|
std::lock_guard<std::mutex> lock(input_buffer_mutex_);
|
// Check running state inside lock to avoid TOCTOU race with Stop()
|
if (!(xEventGroupGetBits(event_group_) & DETECTION_RUNNING_EVENT)) {
|
return;
|
}
|
input_buffer_.insert(input_buffer_.end(), data.begin(), data.end());
|
size_t chunk_size = afe_iface_->get_feed_chunksize(afe_data_) * codec_->input_channels();
|
while (input_buffer_.size() >= chunk_size) {
|
afe_iface_->feed(afe_data_, input_buffer_.data());
|
input_buffer_.erase(input_buffer_.begin(), input_buffer_.begin() + chunk_size);
|
}
|
}
|
|
size_t AfeWakeWord::GetFeedSize() {
|
if (afe_data_ == nullptr) {
|
return 0;
|
}
|
return afe_iface_->get_feed_chunksize(afe_data_);
|
}
|
|
void AfeWakeWord::AudioDetectionTask() {
|
ESP_LOGI(TAG, "Audio detection task started");
|
|
while (true) {
|
xEventGroupWaitBits(event_group_, DETECTION_RUNNING_EVENT, pdFALSE, pdTRUE, portMAX_DELAY);
|
|
esp_afe_sr_data_t* afe_data = nullptr;
|
const esp_afe_sr_iface_t* afe_iface = nullptr;
|
{
|
std::lock_guard<std::mutex> lock(input_buffer_mutex_);
|
afe_data = afe_data_;
|
afe_iface = afe_iface_;
|
}
|
if (afe_data == nullptr || afe_iface == nullptr) {
|
vTaskDelay(pdMS_TO_TICKS(20));
|
continue;
|
}
|
|
auto res = afe_iface->fetch_with_delay(afe_data, pdMS_TO_TICKS(80));
|
if (!(xEventGroupGetBits(event_group_) & DETECTION_RUNNING_EVENT)) {
|
continue;
|
}
|
if (res == nullptr || res->ret_value == ESP_FAIL) {
|
continue;
|
}
|
|
// Store the wake word data for voice recognition, like who is speaking
|
StoreWakeWordData(res->data, res->data_size / sizeof(int16_t));
|
|
if (multinet_data_ != nullptr) {
|
multinet_input_buffer_.insert(multinet_input_buffer_.end(), res->data, res->data + res->data_size / sizeof(int16_t));
|
int chunksize = multinet_->get_samp_chunksize(multinet_data_);
|
while (multinet_input_buffer_.size() >= chunksize) {
|
esp_mn_state_t mn_state = multinet_->detect(multinet_data_, multinet_input_buffer_.data());
|
if (mn_state == ESP_MN_STATE_DETECTED) {
|
esp_mn_results_t *mn_result = multinet_->get_results(multinet_data_);
|
for (int i = 0; i < mn_result->num; i++) {
|
int command_id = mn_result->command_id[i];
|
const char *command_text = nullptr;
|
if (command_id == 1 || command_id == 5) {
|
command_text = OFFLINE_COMMAND_WORD;
|
} else if (command_id == 2) {
|
command_text = OFFLINE_REPEATER_WORD;
|
} else if (command_id == 3) {
|
command_text = OFFLINE_START_REPEATER_WORD;
|
} else if (command_id == 4) {
|
command_text = OFFLINE_EXIT_REPEATER_WORD;
|
} else if (command_id == 6) {
|
command_text = OFFLINE_CHAT_WORD;
|
} else if (command_id == 7) {
|
command_text = OFFLINE_EXIT_SHADOWING_WORD;
|
} else if (command_id == 8) {
|
command_text = OFFLINE_CHAT_WORD_FULL;
|
}
|
ESP_LOGI(TAG, "Offline command detected: command_id=%d, string=%s, prob=%f",
|
command_id, mn_result->string, mn_result->prob[i]);
|
if (command_text && offline_command_detected_callback_) {
|
offline_command_detected_callback_(command_text);
|
}
|
}
|
multinet_->clean(multinet_data_);
|
multinet_input_buffer_.clear();
|
break;
|
} else if (mn_state == ESP_MN_STATE_TIMEOUT) {
|
multinet_->clean(multinet_data_);
|
multinet_input_buffer_.clear();
|
break;
|
}
|
multinet_input_buffer_.erase(multinet_input_buffer_.begin(), multinet_input_buffer_.begin() + chunksize);
|
}
|
}
|
|
if (res->wakeup_state == WAKENET_DETECTED) {
|
Stop();
|
last_detected_wake_word_ = wake_words_[res->wakenet_model_index - 1];
|
|
if (wake_word_detected_callback_) {
|
wake_word_detected_callback_(last_detected_wake_word_);
|
}
|
}
|
}
|
}
|
|
void AfeWakeWord::StoreWakeWordData(const int16_t* data, size_t samples) {
|
// 512 sample * 2B = 1KB，默认会进内部 SRAM，播完本地 MP3 后 DMA 最大块会被切成 ~1KB。
|
std::vector<int16_t> chunk;
|
chunk.reserve(2048);
|
chunk.assign(data, data + samples);
|
wake_word_pcm_.push_back(std::move(chunk));
|
// keep about 2 seconds of data, detect duration is 30ms (sample_rate == 16000, chunksize == 512)
|
while (wake_word_pcm_.size() > 2000 / 30) {
|
wake_word_pcm_.pop_front();
|
}
|
}
|
|
void AfeWakeWord::EncodeWakeWordData() {
|
const size_t stack_size = 4096 * 6;
|
wake_word_opus_.clear();
|
if (wake_word_encode_task_stack_ == nullptr) {
|
wake_word_encode_task_stack_ = (StackType_t*)heap_caps_malloc(stack_size, MALLOC_CAP_SPIRAM);
|
assert(wake_word_encode_task_stack_ != nullptr);
|
}
|
if (wake_word_encode_task_buffer_ == nullptr) {
|
wake_word_encode_task_buffer_ = (StaticTask_t*)heap_caps_malloc(sizeof(StaticTask_t), MALLOC_CAP_INTERNAL);
|
assert(wake_word_encode_task_buffer_ != nullptr);
|
}
|
|
wake_word_encode_task_ = xTaskCreateStatic([](void* arg) {
|
auto this_ = (AfeWakeWord*)arg;
|
{
|
auto start_time = esp_timer_get_time();
|
// Create encoder
|
esp_opus_enc_config_t opus_enc_cfg = AS_OPUS_ENC_CONFIG();
|
void* encoder_handle = nullptr;
|
auto ret = esp_opus_enc_open(&opus_enc_cfg, sizeof(esp_opus_enc_config_t), &encoder_handle);
|
if (encoder_handle == nullptr) {
|
ESP_LOGE(TAG, "Failed to create audio encoder, error code: %d", ret);
|
std::lock_guard<std::mutex> lock(this_->wake_word_mutex_);
|
this_->wake_word_opus_.push_back(std::vector<uint8_t>());
|
this_->wake_word_cv_.notify_all();
|
return;
|
}
|
|
// Get frame size
|
int frame_size = 0;
|
int outbuf_size = 0;
|
esp_opus_enc_get_frame_size(encoder_handle, &frame_size, &outbuf_size);
|
frame_size = frame_size / sizeof(int16_t);
|
|
// Encode all PCM data
|
int packets = 0;
|
std::vector<int16_t> in_buffer;
|
esp_audio_enc_in_frame_t in = {};
|
esp_audio_enc_out_frame_t out = {};
|
|
for (auto& pcm: this_->wake_word_pcm_) {
|
if (in_buffer.empty()) {
|
in_buffer = std::move(pcm);
|
} else {
|
in_buffer.reserve(in_buffer.size() + pcm.size());
|
in_buffer.insert(in_buffer.end(), pcm.begin(), pcm.end());
|
}
|
|
while (in_buffer.size() >= frame_size) {
|
std::vector<uint8_t> opus_buf(outbuf_size);
|
in.buffer = (uint8_t *)(in_buffer.data());
|
in.len = (uint32_t)(frame_size * sizeof(int16_t));
|
out.buffer = opus_buf.data();
|
out.len = outbuf_size;
|
out.encoded_bytes = 0;
|
|
ret = esp_opus_enc_process(encoder_handle, &in, &out);
|
if (ret == ESP_AUDIO_ERR_OK) {
|
std::lock_guard<std::mutex> lock(this_->wake_word_mutex_);
|
this_->wake_word_opus_.emplace_back(opus_buf.data(), opus_buf.data() + out.encoded_bytes);
|
this_->wake_word_cv_.notify_all();
|
packets++;
|
} else {
|
ESP_LOGE(TAG, "Failed to encode audio, error code: %d", ret);
|
}
|
|
in_buffer.erase(in_buffer.begin(), in_buffer.begin() + frame_size);
|
}
|
}
|
this_->wake_word_pcm_.clear();
|
// Close encoder
|
esp_opus_enc_close(encoder_handle);
|
auto end_time = esp_timer_get_time();
|
ESP_LOGI(TAG, "Encode wake word opus %d packets in %ld ms", packets, (long)((end_time - start_time) / 1000));
|
|
std::lock_guard<std::mutex> lock(this_->wake_word_mutex_);
|
this_->wake_word_opus_.push_back(std::vector<uint8_t>());
|
this_->wake_word_cv_.notify_all();
|
}
|
vTaskDelete(NULL);
|
}, "encode_wake_word", stack_size, this, 2, wake_word_encode_task_stack_, wake_word_encode_task_buffer_);
|
if (wake_word_encode_task_ == nullptr) {
|
ESP_LOGE(TAG, "Failed to create encode_wake_word task");
|
std::lock_guard<std::mutex> lock(wake_word_mutex_);
|
wake_word_opus_.push_back(std::vector<uint8_t>());
|
wake_word_cv_.notify_all();
|
}
|
}
|
|
bool AfeWakeWord::GetWakeWordOpus(std::vector<uint8_t>& opus) {
|
std::unique_lock<std::mutex> lock(wake_word_mutex_);
|
if (!wake_word_cv_.wait_for(lock, std::chrono::milliseconds(2000), [this]() {
|
return !wake_word_opus_.empty();
|
})) {
|
ESP_LOGW(TAG, "wake opus wait timeout");
|
return false;
|
}
|
opus.swap(wake_word_opus_.front());
|
wake_word_opus_.pop_front();
|
return !opus.empty();
|
}
|
