#include "afe_audio_processor.h"
|
#include <esp_log.h>
|
|
#define PROCESSOR_RUNNING 0x01
|
|
#define TAG "AfeAudioProcessor"
|
|
AfeAudioProcessor::AfeAudioProcessor()
|
: afe_data_(nullptr) {
|
event_group_ = xEventGroupCreate();
|
}
|
|
void AfeAudioProcessor::Initialize(AudioCodec* codec, int frame_duration_ms, srmodel_list_t* models_list) {
|
codec_ = codec;
|
models_list_ = models_list;
|
frame_duration_ms_ = frame_duration_ms;
|
frame_samples_ = frame_duration_ms * 16000 / 1000;
|
|
// Pre-allocate output buffer capacity
|
output_buffer_.reserve(frame_samples_);
|
|
if (!CreateAfe()) {
|
ESP_LOGE(TAG, "Failed to create AFE runtime");
|
return;
|
}
|
|
if (!processor_task_started_) {
|
xTaskCreate([](void* arg) {
|
auto this_ = (AfeAudioProcessor*)arg;
|
this_->AudioProcessorTask();
|
vTaskDelete(NULL);
|
}, "audio_communication", 4096, this, 3, NULL);
|
processor_task_started_ = true;
|
}
|
}
|
|
bool AfeAudioProcessor::CreateAfe() {
|
if (afe_data_ != nullptr) {
|
return true;
|
}
|
if (codec_ == nullptr) {
|
return false;
|
}
|
|
int ref_num = codec_->input_reference() ? 1 : 0;
|
std::string input_format;
|
for (int i = 0; i < codec_->input_channels() - ref_num; i++) {
|
input_format.push_back('M');
|
}
|
for (int i = 0; i < ref_num; i++) {
|
input_format.push_back('R');
|
}
|
|
srmodel_list_t *models;
|
if (models_list_ == nullptr) {
|
models = esp_srmodel_init("model");
|
} else {
|
models = models_list_;
|
}
|
|
char* ns_model_name = esp_srmodel_filter(models, ESP_NSNET_PREFIX, NULL);
|
char* vad_model_name = esp_srmodel_filter(models, ESP_VADN_PREFIX, NULL);
|
|
afe_config_t* afe_config = afe_config_init(input_format.c_str(), NULL, AFE_TYPE_VC, AFE_MODE_HIGH_PERF);
|
afe_config->aec_mode = AEC_MODE_VOIP_HIGH_PERF;
|
afe_config->vad_mode = VAD_MODE_0;
|
afe_config->vad_min_noise_ms = 100;
|
if (vad_model_name != nullptr) {
|
afe_config->vad_model_name = vad_model_name;
|
}
|
|
if (ns_model_name != nullptr) {
|
afe_config->ns_init = true;
|
afe_config->ns_model_name = ns_model_name;
|
afe_config->afe_ns_mode = AFE_NS_MODE_NET;
|
} else {
|
afe_config->ns_init = false;
|
}
|
|
afe_config->agc_init = false;
|
afe_config->memory_alloc_mode = AFE_MEMORY_ALLOC_MORE_PSRAM;
|
|
#ifdef CONFIG_USE_DEVICE_AEC
|
afe_config->aec_init = true;
|
afe_config->vad_init = false;
|
#else
|
afe_config->aec_init = false;
|
afe_config->vad_init = true;
|
#endif
|
|
afe_iface_ = esp_afe_handle_from_config(afe_config);
|
afe_data_ = afe_iface_->create_from_config(afe_config);
|
if (afe_data_ == nullptr) {
|
ESP_LOGE(TAG, "create_from_config failed");
|
return false;
|
}
|
ESP_LOGI(TAG, "AFE runtime created");
|
return true;
|
}
|
|
void AfeAudioProcessor::ReleaseRuntime() {
|
Stop();
|
if (afe_data_ == nullptr) {
|
return;
|
}
|
vTaskDelay(pdMS_TO_TICKS(120));
|
std::lock_guard<std::mutex> lock(input_buffer_mutex_);
|
if (afe_data_ != nullptr && afe_iface_ != nullptr) {
|
afe_iface_->destroy(afe_data_);
|
afe_data_ = nullptr;
|
}
|
input_buffer_.clear();
|
ESP_LOGI(TAG, "AFE runtime released");
|
}
|
|
bool AfeAudioProcessor::EnsureRuntime() {
|
return CreateAfe();
|
}
|
|
AfeAudioProcessor::~AfeAudioProcessor() {
|
if (afe_data_ != nullptr) {
|
afe_iface_->destroy(afe_data_);
|
}
|
vEventGroupDelete(event_group_);
|
}
|
|
size_t AfeAudioProcessor::GetFeedSize() {
|
if (afe_data_ == nullptr) {
|
return 0;
|
}
|
return afe_iface_->get_feed_chunksize(afe_data_);
|
}
|
|
void AfeAudioProcessor::Feed(std::vector<int16_t>&& data) {
|
if (afe_data_ == nullptr) {
|
return;
|
}
|
|
std::lock_guard<std::mutex> lock(input_buffer_mutex_);
|
// Check running state inside lock to avoid TOCTOU race with Stop()
|
if (!IsRunning()) {
|
return;
|
}
|
input_buffer_.insert(input_buffer_.end(), data.begin(), data.end());
|
size_t chunk_size = afe_iface_->get_feed_chunksize(afe_data_) * codec_->input_channels();
|
while (input_buffer_.size() >= chunk_size) {
|
afe_iface_->feed(afe_data_, input_buffer_.data());
|
input_buffer_.erase(input_buffer_.begin(), input_buffer_.begin() + chunk_size);
|
}
|
}
|
|
void AfeAudioProcessor::Start() {
|
xEventGroupSetBits(event_group_, PROCESSOR_RUNNING);
|
}
|
|
void AfeAudioProcessor::Stop() {
|
xEventGroupClearBits(event_group_, PROCESSOR_RUNNING);
|
|
std::lock_guard<std::mutex> lock(input_buffer_mutex_);
|
if (afe_data_ != nullptr) {
|
afe_iface_->reset_buffer(afe_data_);
|
}
|
input_buffer_.clear();
|
}
|
|
bool AfeAudioProcessor::IsRunning() {
|
return xEventGroupGetBits(event_group_) & PROCESSOR_RUNNING;
|
}
|
|
void AfeAudioProcessor::OnOutput(std::function<void(std::vector<int16_t>&& data)> callback) {
|
output_callback_ = callback;
|
}
|
|
void AfeAudioProcessor::OnVadStateChange(std::function<void(bool speaking)> callback) {
|
vad_state_change_callback_ = callback;
|
}
|
|
void AfeAudioProcessor::AudioProcessorTask() {
|
ESP_LOGI(TAG, "Audio communication task started");
|
|
while (true) {
|
xEventGroupWaitBits(event_group_, PROCESSOR_RUNNING, pdFALSE, pdTRUE, portMAX_DELAY);
|
|
esp_afe_sr_data_t* afe_data = nullptr;
|
const esp_afe_sr_iface_t* afe_iface = nullptr;
|
{
|
std::lock_guard<std::mutex> lock(input_buffer_mutex_);
|
afe_data = afe_data_;
|
afe_iface = afe_iface_;
|
}
|
if (afe_data == nullptr || afe_iface == nullptr) {
|
vTaskDelay(pdMS_TO_TICKS(20));
|
continue;
|
}
|
|
auto res = afe_iface->fetch_with_delay(afe_data, pdMS_TO_TICKS(80));
|
if ((xEventGroupGetBits(event_group_) & PROCESSOR_RUNNING) == 0) {
|
continue;
|
}
|
if (res == nullptr || res->ret_value == ESP_FAIL) {
|
if (res != nullptr) {
|
ESP_LOGD(TAG, "Error code: %d", res->ret_value);
|
}
|
continue;
|
}
|
|
// VAD state change
|
if (vad_state_change_callback_) {
|
if (res->vad_state == VAD_SPEECH && !is_speaking_) {
|
is_speaking_ = true;
|
vad_state_change_callback_(true);
|
} else if (res->vad_state == VAD_SILENCE && is_speaking_) {
|
is_speaking_ = false;
|
vad_state_change_callback_(false);
|
}
|
}
|
|
if (output_callback_) {
|
size_t samples = res->data_size / sizeof(int16_t);
|
|
// Add data to buffer
|
output_buffer_.insert(output_buffer_.end(), res->data, res->data + samples);
|
|
// Output complete frames when buffer has enough data
|
while (output_buffer_.size() >= frame_samples_) {
|
if (output_buffer_.size() == frame_samples_) {
|
// If buffer size equals frame size, move the entire buffer
|
output_callback_(std::move(output_buffer_));
|
output_buffer_.clear();
|
output_buffer_.reserve(frame_samples_);
|
} else {
|
// If buffer size exceeds frame size, copy one frame and remove it
|
output_callback_(std::vector<int16_t>(output_buffer_.begin(), output_buffer_.begin() + frame_samples_));
|
output_buffer_.erase(output_buffer_.begin(), output_buffer_.begin() + frame_samples_);
|
}
|
}
|
}
|
}
|
}
|
|
void AfeAudioProcessor::EnableDeviceAec(bool enable) {
|
if (afe_data_ == nullptr || afe_iface_ == nullptr) {
|
return;
|
}
|
if (enable) {
|
#if CONFIG_USE_DEVICE_AEC
|
afe_iface_->disable_vad(afe_data_);
|
afe_iface_->enable_aec(afe_data_);
|
#else
|
ESP_LOGE(TAG, "Device AEC is not supported");
|
#endif
|
} else {
|
afe_iface_->disable_aec(afe_data_);
|
afe_iface_->enable_vad(afe_data_);
|
}
|
}
|
