robot/xiaozhi-esp32
1
0
Fork 0
mirror of https://github.com/78/xiaozhi-esp32.git synced 2025-05-17 07:08:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

fit the memory usage for esp32c3

Browse Source
This commit is contained in:
Terrence
2024-11-29 11:06:05 +08:00
parent ff28586c35
commit 436ff2b906
35 changed files with 754 additions and 360 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
CMakeLists.txt
View File

@ -4,7 +4,7 @@
# CMakeLists in this exact order for cmake to work correctly
cmake_minimum_required(VERSION 3.16)
set(PROJECT_VER "0.9.2")
set(PROJECT_VER "0.9.3")
include($ENV{IDF_PATH}/tools/cmake/project.cmake)
project(xiaozhi)

10
main/CMakeLists.txt
View File

@ -1,6 +1,7 @@
set(SOURCES "audio_codecs/audio_codec.cc"
"audio_codecs/no_audio_codec.cc"
"audio_codecs/box_audio_codec.cc"
"audio_codecs/es8311_audio_codec.cc"
"display/display.cc"
"display/no_display.cc"
"display/st7789_display.cc"
@ -12,10 +13,11 @@ set(SOURCES "audio_codecs/audio_codec.cc"
"application.cc"
"ota.cc"
"settings.cc"
"background_task.cc"
"main.cc"
)
set(INCLUDE_DIRS "." "display" "audio_codecs" "protocols")
set(INCLUDE_DIRS "." "display" "audio_codecs" "protocols" "audio_processing")
# 字体
file(GLOB FONT_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/fonts/*.c)
@ -38,14 +40,16 @@ elseif(CONFIG_BOARD_TYPE_KEVIN_BOX_1)
set(BOARD_TYPE "kevin-box-1")
elseif(CONFIG_BOARD_TYPE_KEVIN_BOX_2)
set(BOARD_TYPE "kevin-box-2")
elseif(CONFIG_BOARD_TYPE_KEVIN_C3)
set(BOARD_TYPE "kevin-c3")
elseif(CONFIG_BOARD_TYPE_LICHUANG_DEV)
set(BOARD_TYPE "lichuang-dev")
endif()
file(GLOB BOARD_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/boards/${BOARD_TYPE}/*.cc)
list(APPEND SOURCES ${BOARD_SOURCES})
if(CONFIG_USE_AFE_SR)
list(APPEND SOURCES "audio_processor.cc" "wake_word_detect.cc")
if(CONFIG_IDF_TARGET_ESP32S3)
list(APPEND SOURCES "audio_processing/audio_processor.cc" "audio_processing/wake_word_detect.cc")
endif()
idf_component_register(SRCS ${SOURCES}

2
main/Kconfig.projbuild
View File

@ -46,6 +46,8 @@ choice BOARD_TYPE
bool "Kevin Box 1"
config BOARD_TYPE_KEVIN_BOX_2
bool "Kevin Box 2"
config BOARD_TYPE_KEVIN_C3
bool "Kevin C3"
config BOARD_TYPE_LICHUANG_DEV
bool "立创开发板"
endchoice

317
main/application.cc
View File

@ -22,7 +22,7 @@ extern const char p3_err_wificonfig_start[] asm("_binary_err_wificonfig_p3_start
extern const char p3_err_wificonfig_end[] asm("_binary_err_wificonfig_p3_end");
Application::Application() {
Application::Application() : background_task_(4096 * 8) {
event_group_ = xEventGroupCreate();
ota_.SetCheckVersionUrl(CONFIG_OTA_VERSION_URL);
@ -36,9 +36,6 @@ Application::~Application() {
if (opus_decoder_ != nullptr) {
opus_decoder_destroy(opus_decoder_);
}
if (audio_encode_task_stack_ != nullptr) {
heap_caps_free(audio_encode_task_stack_);
}
vEventGroupDelete(event_group_);
}
@ -116,7 +113,6 @@ void Application::PlayLocalFile(const char* data, size_t size) {
std::lock_guard<std::mutex> lock(mutex_);
audio_decode_queue_.emplace_back(std::move(opus));
}
cv_.notify_all();
}
void Application::ToggleChatState() {
@ -157,6 +153,8 @@ void Application::StartListening() {
} else if (chat_state_ == kChatStateSpeaking) {
AbortSpeaking(kAbortReasonNone);
protocol_->SendStartListening(kListeningModeManualStop);
// FIXME: Wait for the speaker to empty the buffer
vTaskDelay(pdMS_TO_TICKS(120));
SetChatState(kChatStateListening);
}
});
@ -164,8 +162,10 @@ void Application::StartListening() {
void Application::StopListening() {
Schedule([this]() {
protocol_->SendStopListening();
SetChatState(kChatStateIdle);
if (chat_state_ == kChatStateListening) {
protocol_->SendStopListening();
SetChatState(kChatStateIdle);
}
});
}
@ -184,71 +184,32 @@ void Application::Start() {
auto codec = board.GetAudioCodec();
opus_decode_sample_rate_ = codec->output_sample_rate();
opus_decoder_ = opus_decoder_create(opus_decode_sample_rate_, 1, NULL);
opus_encoder_.Configure(16000, 1);
opus_encoder_.Configure(16000, 1, OPUS_FRAME_DURATION_MS);
if (codec->input_sample_rate() != 16000) {
input_resampler_.Configure(codec->input_sample_rate(), 16000);
reference_resampler_.Configure(codec->input_sample_rate(), 16000);
}
codec->OnInputData([this, codec](std::vector<int16_t>&& data) {
if (codec->input_sample_rate() != 16000) {
if (codec->input_channels() == 2) {
auto mic_channel = std::vector<int16_t>(data.size() / 2);
auto reference_channel = std::vector<int16_t>(data.size() / 2);
for (size_t i = 0, j = 0; i < mic_channel.size(); ++i, j += 2) {
mic_channel[i] = data[j];
reference_channel[i] = data[j + 1];
}
auto resampled_mic = std::vector<int16_t>(input_resampler_.GetOutputSamples(mic_channel.size()));
auto resampled_reference = std::vector<int16_t>(reference_resampler_.GetOutputSamples(reference_channel.size()));
input_resampler_.Process(mic_channel.data(), mic_channel.size(), resampled_mic.data());
reference_resampler_.Process(reference_channel.data(), reference_channel.size(), resampled_reference.data());
data.resize(resampled_mic.size() + resampled_reference.size());
for (size_t i = 0, j = 0; i < resampled_mic.size(); ++i, j += 2) {
data[j] = resampled_mic[i];
data[j + 1] = resampled_reference[i];
}
} else {
auto resampled = std::vector<int16_t>(input_resampler_.GetOutputSamples(data.size()));
input_resampler_.Process(data.data(), data.size(), resampled.data());
data = std::move(resampled);
}
}
#ifdef CONFIG_USE_AFE_SR
if (audio_processor_.IsRunning()) {
audio_processor_.Input(data);
}
if (wake_word_detect_.IsDetectionRunning()) {
wake_word_detect_.Feed(data);
}
#else
Schedule([this, data = std::move(data)]() {
if (chat_state_ == kChatStateListening) {
std::lock_guard<std::mutex> lock(mutex_);
audio_encode_queue_.emplace_back(std::move(data));
cv_.notify_all();
}
});
#endif
codec->OnInputReady([this, codec]() {
BaseType_t higher_priority_task_woken = pdFALSE;
xEventGroupSetBitsFromISR(event_group_, AUDIO_INPUT_READY_EVENT, &higher_priority_task_woken);
return higher_priority_task_woken == pdTRUE;
});
codec->OnOutputReady([this]() {
BaseType_t higher_priority_task_woken = pdFALSE;
xEventGroupSetBitsFromISR(event_group_, AUDIO_OUTPUT_READY_EVENT, &higher_priority_task_woken);
return higher_priority_task_woken == pdTRUE;
});
const size_t opus_stack_size = 4096 * 8; // OPUS encoder / decoder use a lot of stack memory
audio_encode_task_stack_ = (StackType_t*)heap_caps_malloc(opus_stack_size, MALLOC_CAP_SPIRAM);
audio_encode_task_ = xTaskCreateStatic([](void* arg) {
Application* app = (Application*)arg;
app->AudioEncodeTask();
vTaskDelete(NULL);
}, "opus_encode", opus_stack_size, this, 1, audio_encode_task_stack_, &audio_encode_task_buffer_);
codec->Start();
/* Wait for the network to be ready */
board.StartNetwork();
/* Start the main loop */
xTaskCreate([](void* arg) {
Application* app = (Application*)arg;
app->MainLoop();
vTaskDelete(NULL);
}, "main_loop", 4096 * 2, this, 1, nullptr);
}, "main_loop", 4096 * 2, this, 2, nullptr);
/* Wait for the network to be ready */
board.StartNetwork();
// Check for new firmware version or get the MQTT broker address
xTaskCreate([](void* arg) {
@ -257,12 +218,16 @@ void Application::Start() {
vTaskDelete(NULL);
}, "check_new_version", 4096 * 2, this, 1, nullptr);
#ifdef CONFIG_USE_AFE_SR
#if CONFIG_IDF_TARGET_ESP32S3
audio_processor_.Initialize(codec->input_channels(), codec->input_reference());
audio_processor_.OnOutput([this](std::vector<int16_t>&& data) {
std::lock_guard<std::mutex> lock(mutex_);
audio_encode_queue_.emplace_back(std::move(data));
cv_.notify_all();
background_task_.Schedule([this, data = std::move(data)]() {
opus_encoder_.Encode(data, [this](const uint8_t* opus, size_t opus_size) {
Schedule([this, opus = std::string(reinterpret_cast<const char*>(opus), opus_size)]() {
protocol_->SendAudio(opus);
});
});
});
});
wake_word_detect_.Initialize(codec->input_channels(), codec->input_reference());
@ -326,8 +291,9 @@ void Application::Start() {
});
protocol_->OnIncomingAudio([this](const std::string& data) {
std::lock_guard<std::mutex> lock(mutex_);
audio_decode_queue_.emplace_back(std::move(data));
cv_.notify_all();
if (chat_state_ == kChatStateSpeaking) {
audio_decode_queue_.emplace_back(std::move(data));
}
});
protocol_->OnAudioChannelOpened([this, codec, &board]() {
if (protocol_->server_sample_rate() != codec->output_sample_rate()) {
@ -350,17 +316,15 @@ void Application::Start() {
auto state = cJSON_GetObjectItem(root, "state");
if (strcmp(state->valuestring, "start") == 0) {
Schedule([this]() {
aborted_ = false;
if (chat_state_ == kChatStateIdle || chat_state_ == kChatStateListening) {
skip_to_end_ = false;
opus_decoder_ctl(opus_decoder_, OPUS_RESET_STATE);
SetChatState(kChatStateSpeaking);
}
});
} else if (strcmp(state->valuestring, "stop") == 0) {
Schedule([this]() {
auto codec = Board::GetInstance().GetAudioCodec();
codec->WaitForOutputDone();
if (chat_state_ == kChatStateSpeaking) {
background_task_.WaitForCompletion();
if (keep_listening_) {
protocol_->SendStartListening(kListeningModeAutoStop);
SetChatState(kChatStateListening);
@ -399,9 +363,10 @@ void Application::Start() {
}
void Application::Schedule(std::function<void()> callback) {
std::lock_guard<std::mutex> lock(mutex_);
mutex_.lock();
main_tasks_.push_back(callback);
cv_.notify_all();
mutex_.unlock();
xEventGroupSetBits(event_group_, SCHEDULE_EVENT);
}
// The Main Loop controls the chat state and websocket connection
@ -409,24 +374,140 @@ void Application::Schedule(std::function<void()> callback) {
// they should use Schedule to call this function
void Application::MainLoop() {
while (true) {
std::unique_lock<std::mutex> lock(mutex_);
cv_.wait(lock, [this]() {
return !main_tasks_.empty();
});
auto task = std::move(main_tasks_.front());
main_tasks_.pop_front();
lock.unlock();
task();
auto bits = xEventGroupWaitBits(event_group_,
SCHEDULE_EVENT | AUDIO_INPUT_READY_EVENT | AUDIO_OUTPUT_READY_EVENT,
pdTRUE, pdFALSE, portMAX_DELAY);
if (bits & AUDIO_INPUT_READY_EVENT) {
InputAudio();
}
if (bits & AUDIO_OUTPUT_READY_EVENT) {
OutputAudio();
}
if (bits & SCHEDULE_EVENT) {
mutex_.lock();
std::list<std::function<void()>> tasks = std::move(main_tasks_);
mutex_.unlock();
for (auto& task : tasks) {
task();
}
}
}
}
void Application::ResetDecoder() {
std::lock_guard<std::mutex> lock(mutex_);
opus_decoder_ctl(opus_decoder_, OPUS_RESET_STATE);
audio_decode_queue_.clear();
last_output_time_ = std::chrono::steady_clock::now();
Board::GetInstance().GetAudioCodec()->EnableOutput(true);
}
void Application::OutputAudio() {
auto now = std::chrono::steady_clock::now();
auto codec = Board::GetInstance().GetAudioCodec();
const int max_silence_seconds = 10;
std::unique_lock<std::mutex> lock(mutex_);
if (audio_decode_queue_.empty()) {
// Disable the output if there is no audio data for a long time
auto duration = std::chrono::duration_cast<std::chrono::seconds>(now - last_output_time_).count();
if (duration > max_silence_seconds) {
codec->EnableOutput(false);
}
return;
}
if (chat_state_ == kChatStateListening) {
audio_decode_queue_.clear();
return;
}
last_output_time_ = now;
auto opus = std::move(audio_decode_queue_.front());
audio_decode_queue_.pop_front();
lock.unlock();
background_task_.Schedule([this, codec, opus = std::move(opus)]() {
if (aborted_) {
return;
}
int frame_size = opus_decode_sample_rate_ * OPUS_FRAME_DURATION_MS / 1000;
std::vector<int16_t> pcm(frame_size);
int ret = opus_decode(opus_decoder_, (const unsigned char*)opus.data(), opus.size(), pcm.data(), frame_size, 0);
if (ret < 0) {
ESP_LOGE(TAG, "Failed to decode audio, error code: %d", ret);
return;
}
// Resample if the sample rate is different
if (opus_decode_sample_rate_ != codec->output_sample_rate()) {
int target_size = output_resampler_.GetOutputSamples(frame_size);
std::vector<int16_t> resampled(target_size);
output_resampler_.Process(pcm.data(), frame_size, resampled.data());
pcm = std::move(resampled);
}
codec->OutputData(pcm);
});
}
void Application::InputAudio() {
auto codec = Board::GetInstance().GetAudioCodec();
std::vector<int16_t> data;
if (!codec->InputData(data)) {
return;
}
if (codec->input_sample_rate() != 16000) {
if (codec->input_channels() == 2) {
auto mic_channel = std::vector<int16_t>(data.size() / 2);
auto reference_channel = std::vector<int16_t>(data.size() / 2);
for (size_t i = 0, j = 0; i < mic_channel.size(); ++i, j += 2) {
mic_channel[i] = data[j];
reference_channel[i] = data[j + 1];
}
auto resampled_mic = std::vector<int16_t>(input_resampler_.GetOutputSamples(mic_channel.size()));
auto resampled_reference = std::vector<int16_t>(reference_resampler_.GetOutputSamples(reference_channel.size()));
input_resampler_.Process(mic_channel.data(), mic_channel.size(), resampled_mic.data());
reference_resampler_.Process(reference_channel.data(), reference_channel.size(), resampled_reference.data());
data.resize(resampled_mic.size() + resampled_reference.size());
for (size_t i = 0, j = 0; i < resampled_mic.size(); ++i, j += 2) {
data[j] = resampled_mic[i];
data[j + 1] = resampled_reference[i];
}
} else {
auto resampled = std::vector<int16_t>(input_resampler_.GetOutputSamples(data.size()));
input_resampler_.Process(data.data(), data.size(), resampled.data());
data = std::move(resampled);
}
}
#if CONFIG_IDF_TARGET_ESP32S3
if (audio_processor_.IsRunning()) {
audio_processor_.Input(data);
}
if (wake_word_detect_.IsDetectionRunning()) {
wake_word_detect_.Feed(data);
}
#else
if (chat_state_ == kChatStateListening) {
background_task_.Schedule([this, data = std::move(data)]() {
opus_encoder_.Encode(data, [this](const uint8_t* opus, size_t opus_size) {
Schedule([this, opus = std::string(reinterpret_cast<const char*>(opus), opus_size)]() {
protocol_->SendAudio(opus);
});
});
});
}
#endif
}
void Application::AbortSpeaking(AbortReason reason) {
ESP_LOGI(TAG, "Abort speaking");
aborted_ = true;
protocol_->SendAbortSpeaking(reason);
skip_to_end_ = true;
auto codec = Board::GetInstance().GetAudioCodec();
codec->ClearOutputQueue();
}
void Application::SetChatState(ChatState state) {
@ -444,6 +525,11 @@ void Application::SetChatState(ChatState state) {
return;
}
chat_state_ = state;
ESP_LOGI(TAG, "STATE: %s", state_str[chat_state_]);
// The state is changed, wait for all background tasks to finish
background_task_.WaitForCompletion();
auto display = Board::GetInstance().GetDisplay();
auto builtin_led = Board::GetInstance().GetBuiltinLed();
switch (state) {
@ -452,7 +538,7 @@ void Application::SetChatState(ChatState state) {
builtin_led->TurnOff();
display->SetStatus("待命");
display->SetEmotion("neutral");
#ifdef CONFIG_USE_AFE_SR
#ifdef CONFIG_IDF_TARGET_ESP32S3
audio_processor_.Stop();
#endif
break;
@ -466,8 +552,9 @@ void Application::SetChatState(ChatState state) {
builtin_led->TurnOn();
display->SetStatus("聆听中...");
display->SetEmotion("neutral");
ResetDecoder();
opus_encoder_.ResetState();
#ifdef CONFIG_USE_AFE_SR
#if CONFIG_IDF_TARGET_ESP32S3
audio_processor_.Start();
#endif
break;
@ -475,7 +562,8 @@ void Application::SetChatState(ChatState state) {
builtin_led->SetGreen();
builtin_led->TurnOn();
display->SetStatus("说话中...");
#ifdef CONFIG_USE_AFE_SR
ResetDecoder();
#if CONFIG_IDF_TARGET_ESP32S3
audio_processor_.Stop();
#endif
break;
@ -487,59 +575,6 @@ void Application::SetChatState(ChatState state) {
ESP_LOGE(TAG, "Invalid chat state: %d", chat_state_);
return;
}
chat_state_ = state;
ESP_LOGI(TAG, "STATE: %s", state_str[chat_state_]);
}
void Application::AudioEncodeTask() {
ESP_LOGI(TAG, "Audio encode task started");
auto codec = Board::GetInstance().GetAudioCodec();
while (true) {
std::unique_lock<std::mutex> lock(mutex_);
cv_.wait(lock, [this]() {
return !audio_encode_queue_.empty() || !audio_decode_queue_.empty();
});
if (!audio_encode_queue_.empty()) {
auto pcm = std::move(audio_encode_queue_.front());
audio_encode_queue_.pop_front();
lock.unlock();
opus_encoder_.Encode(pcm, [this](const uint8_t* opus, size_t opus_size) {
Schedule([this, data = std::string(reinterpret_cast<const char*>(opus), opus_size)]() {
protocol_->SendAudio(data);
});
});
} else if (!audio_decode_queue_.empty()) {
auto opus = std::move(audio_decode_queue_.front());
audio_decode_queue_.pop_front();
lock.unlock();
if (skip_to_end_) {
continue;
}
int frame_size = opus_decode_sample_rate_ * OPUS_FRAME_DURATION_MS / 1000;
std::vector<int16_t> pcm(frame_size);
int ret = opus_decode(opus_decoder_, (const unsigned char*)opus.data(), opus.size(), pcm.data(), frame_size, 0);
if (ret < 0) {
ESP_LOGE(TAG, "Failed to decode audio, error code: %d", ret);
continue;
}
// Resample if the sample rate is different
if (opus_decode_sample_rate_ != codec->output_sample_rate()) {
int target_size = output_resampler_.GetOutputSamples(frame_size);
std::vector<int16_t> resampled(target_size);
output_resampler_.Process(pcm.data(), frame_size, resampled.data());
pcm = std::move(resampled);
}
codec->OutputData(pcm);
}
}
}
void Application::SetDecodeSampleRate(int sample_rate) {

21
main/application.h
View File

@ -16,12 +16,16 @@
#include "display.h"
#include "board.h"
#include "ota.h"
#include "background_task.h"
#ifdef CONFIG_USE_AFE_SR
#if CONFIG_IDF_TARGET_ESP32S3
#include "wake_word_detect.h"
#include "audio_processor.h"
#endif
#define SCHEDULE_EVENT (1 << 0)
#define AUDIO_INPUT_READY_EVENT (1 << 1)
#define AUDIO_OUTPUT_READY_EVENT (1 << 2)
enum ChatState {
kChatStateUnknown,
@ -58,25 +62,22 @@ private:
Application();
~Application();
#ifdef CONFIG_USE_AFE_SR
#if CONFIG_IDF_TARGET_ESP32S3
WakeWordDetect wake_word_detect_;
AudioProcessor audio_processor_;
#endif
Ota ota_;
std::mutex mutex_;
std::condition_variable_any cv_;
std::list<std::function<void()>> main_tasks_;
Protocol* protocol_ = nullptr;
EventGroupHandle_t event_group_;
volatile ChatState chat_state_ = kChatStateUnknown;
bool keep_listening_ = false;
bool skip_to_end_ = false;
bool aborted_ = false;
// Audio encode / decode
TaskHandle_t audio_encode_task_ = nullptr;
StaticTask_t audio_encode_task_buffer_;
StackType_t* audio_encode_task_stack_ = nullptr;
std::list<std::vector<int16_t>> audio_encode_queue_;
BackgroundTask background_task_;
std::chrono::steady_clock::time_point last_output_time_;
std::list<std::string> audio_decode_queue_;
OpusEncoder opus_encoder_;
@ -88,10 +89,12 @@ private:
OpusResampler output_resampler_;
void MainLoop();
void InputAudio();
void OutputAudio();
void ResetDecoder();
void SetDecodeSampleRate(int sample_rate);
void CheckNewVersion();
void AudioEncodeTask();
void PlayLocalFile(const char* data, size_t size);
};

127
main/audio_codecs/audio_codec.cc
View File

@ -9,34 +9,48 @@
#define TAG "AudioCodec"
AudioCodec::AudioCodec() {
audio_event_group_ = xEventGroupCreate();
}
AudioCodec::~AudioCodec() {
if (audio_input_task_ != nullptr) {
vTaskDelete(audio_input_task_);
}
if (audio_output_task_ != nullptr) {
vTaskDelete(audio_output_task_);
}
if (audio_event_group_ != nullptr) {
vEventGroupDelete(audio_event_group_);
}
}
void AudioCodec::OnInputData(std::function<void(std::vector<int16_t>&& data)> callback) {
on_input_data_ = callback;
void AudioCodec::OnInputReady(std::function<bool()> callback) {
on_input_ready_ = callback;
}
void AudioCodec::OnOutputReady(std::function<bool()> callback) {
on_output_ready_ = callback;
}
void AudioCodec::OutputData(std::vector<int16_t>& data) {
std::lock_guard<std::mutex> lock(audio_output_queue_mutex_);
audio_output_queue_.emplace_back(std::move(data));
audio_output_queue_cv_.notify_one();
Write(data.data(), data.size());
}
bool AudioCodec::InputData(std::vector<int16_t>& data) {
int duration = 30;
int input_frame_size = input_sample_rate_ / 1000 * duration * input_channels_;
data.resize(input_frame_size);
int samples = Read(data.data(), data.size());
if (samples > 0) {
return true;
}
return false;
}
IRAM_ATTR bool AudioCodec::on_sent(i2s_chan_handle_t handle, i2s_event_data_t *event, void *user_ctx) {
auto audio_codec = (AudioCodec*)user_ctx;
xEventGroupSetBits(audio_codec->audio_event_group_, AUDIO_EVENT_OUTPUT_DONE);
if (audio_codec->output_enabled_ && audio_codec->on_output_ready_) {
return audio_codec->on_output_ready_();
}
return false;
}
IRAM_ATTR bool AudioCodec::on_recv(i2s_chan_handle_t handle, i2s_event_data_t *event, void *user_ctx) {
auto audio_codec = (AudioCodec*)user_ctx;
if (audio_codec->input_enabled_ && audio_codec->on_input_ready_) {
return audio_codec->on_input_ready_();
}
return false;
}
@ -44,85 +58,20 @@ void AudioCodec::Start() {
Settings settings("audio", false);
output_volume_ = settings.GetInt("output_volume", output_volume_);
// 注册音频输出回调
i2s_event_callbacks_t callbacks = {};
callbacks.on_sent = on_sent;
i2s_channel_register_event_callback(tx_handle_, &callbacks, this);
// 注册音频数据回调
i2s_event_callbacks_t rx_callbacks = {};
rx_callbacks.on_recv = on_recv;
i2s_channel_register_event_callback(rx_handle_, &rx_callbacks, this);
i2s_event_callbacks_t tx_callbacks = {};
tx_callbacks.on_sent = on_sent;
i2s_channel_register_event_callback(tx_handle_, &tx_callbacks, this);
ESP_ERROR_CHECK(i2s_channel_enable(tx_handle_));
ESP_ERROR_CHECK(i2s_channel_enable(rx_handle_));
EnableInput(true);
EnableOutput(true);
// 创建音频输入任务
if (audio_input_task_ == nullptr) {
xTaskCreate([](void* arg) {
auto audio_device = (AudioCodec*)arg;
audio_device->InputTask();
}, "audio_input", 4096 * 2, this, 3, &audio_input_task_);
}
// 创建音频输出任务
if (audio_output_task_ == nullptr) {
xTaskCreate([](void* arg) {
auto audio_device = (AudioCodec*)arg;
audio_device->OutputTask();
}, "audio_output", 4096 * 2, this, 3, &audio_output_task_);
}
}
void AudioCodec::InputTask() {
int duration = 30;
int input_frame_size = input_sample_rate_ / 1000 * duration * input_channels_;
while (true) {
std::vector<int16_t> input_data(input_frame_size);
int samples = Read(input_data.data(), input_data.size());
if (samples > 0) {
if (on_input_data_) {
on_input_data_(std::move(input_data));
}
}
}
}
void AudioCodec::OutputTask() {
while (true) {
std::unique_lock<std::mutex> lock(audio_output_queue_mutex_);
if (!audio_output_queue_cv_.wait_for(lock, std::chrono::seconds(30), [this]() {
return !audio_output_queue_.empty();
})) {
// If timeout, disable output
EnableOutput(false);
continue;
}
auto data = std::move(audio_output_queue_.front());
audio_output_queue_.pop_front();
lock.unlock();
if (!output_enabled_) {
EnableOutput(true);
}
xEventGroupClearBits(audio_event_group_, AUDIO_EVENT_OUTPUT_DONE);
Write(data.data(), data.size());
audio_output_queue_cv_.notify_all();
}
}
void AudioCodec::WaitForOutputDone() {
// Wait for the output queue to be empty and the output is done
std::unique_lock<std::mutex> lock(audio_output_queue_mutex_);
audio_output_queue_cv_.wait(lock, [this]() {
return audio_output_queue_.empty();
});
lock.unlock();
xEventGroupWaitBits(audio_event_group_, AUDIO_EVENT_OUTPUT_DONE, pdFALSE, pdFALSE, portMAX_DELAY);
}
void AudioCodec::ClearOutputQueue() {
std::lock_guard<std::mutex> lock(audio_output_queue_mutex_);
audio_output_queue_.clear();
}
void AudioCodec::SetOutputVolume(int volume) {

26
main/audio_codecs/audio_codec.h
View File

@ -2,21 +2,15 @@
#define _AUDIO_CODEC_H
#include <freertos/FreeRTOS.h>
#include <freertos/task.h>
#include <freertos/event_groups.h>
#include <driver/i2s_std.h>
#include <vector>
#include <string>
#include <functional>
#include <list>
#include <mutex>
#include <condition_variable>
#include "board.h"
#define AUDIO_EVENT_OUTPUT_DONE (1 << 0)
class AudioCodec {
public:
AudioCodec();
@ -27,10 +21,10 @@ public:
virtual void EnableOutput(bool enable);
void Start();
void OnInputData(std::function<void(std::vector<int16_t>&& data)> callback);
void OutputData(std::vector<int16_t>& data);
void WaitForOutputDone();
void ClearOutputQueue();
bool InputData(std::vector<int16_t>& data);
void OnOutputReady(std::function<bool()> callback);
void OnInputReady(std::function<bool()> callback);
inline bool duplex() const { return duplex_; }
inline bool input_reference() const { return input_reference_; }
@ -41,18 +35,12 @@ public:
inline int output_volume() const { return output_volume_; }
private:
TaskHandle_t audio_input_task_ = nullptr;
TaskHandle_t audio_output_task_ = nullptr;
std::function<void(std::vector<int16_t>&& data)> on_input_data_;
std::list<std::vector<int16_t>> audio_output_queue_;
std::mutex audio_output_queue_mutex_;
std::condition_variable audio_output_queue_cv_;
EventGroupHandle_t audio_event_group_ = nullptr;
std::function<bool()> on_input_ready_;
std::function<bool()> on_output_ready_;
IRAM_ATTR static bool on_recv(i2s_chan_handle_t handle, i2s_event_data_t *event, void *user_ctx);
IRAM_ATTR static bool on_sent(i2s_chan_handle_t handle, i2s_event_data_t *event, void *user_ctx);
void InputTask();
void OutputTask();
protected:
i2s_chan_handle_t tx_handle_ = nullptr;
i2s_chan_handle_t rx_handle_ = nullptr;

6
main/audio_codecs/box_audio_codec.cc
View File

@ -28,7 +28,7 @@ BoxAudioCodec::BoxAudioCodec(void* i2c_master_handle, int input_sample_rate, int
// Output
audio_codec_i2c_cfg_t i2c_cfg = {
.port = I2C_NUM_1,
.port = (i2c_port_t)1,
.addr = es8311_addr,
.bus_handle = i2c_master_handle,
};
@ -96,8 +96,8 @@ void BoxAudioCodec::CreateDuplexChannels(gpio_num_t mclk, gpio_num_t bclk, gpio_
i2s_chan_config_t chan_cfg = {
.id = I2S_NUM_0,
.role = I2S_ROLE_MASTER,
.dma_desc_num = 6,
.dma_frame_num = 240,
.dma_desc_num = 2,
.dma_frame_num = 240 * 3,
.auto_clear_after_cb = true,
.auto_clear_before_cb = false,
.intr_priority = 0,

186
main/audio_codecs/es8311_audio_codec.cc Normal file
View File

@ -0,0 +1,186 @@
#include "es8311_audio_codec.h"
#include <esp_log.h>
static const char TAG[] = "Es8311AudioCodec";
Es8311AudioCodec::Es8311AudioCodec(void* i2c_master_handle, i2c_port_t i2c_port, int input_sample_rate, int output_sample_rate,
gpio_num_t mclk, gpio_num_t bclk, gpio_num_t ws, gpio_num_t dout, gpio_num_t din,
gpio_num_t pa_pin, uint8_t es8311_addr) {
duplex_ = true; // 是否双工
input_reference_ = false; // 是否使用参考输入，实现回声消除
input_channels_ = 1; // 输入通道数
input_sample_rate_ = input_sample_rate;
output_sample_rate_ = output_sample_rate;
CreateDuplexChannels(mclk, bclk, ws, dout, din);
// Do initialize of related interface: data_if, ctrl_if and gpio_if
audio_codec_i2s_cfg_t i2s_cfg = {
.port = I2S_NUM_0,
.rx_handle = rx_handle_,
.tx_handle = tx_handle_,
};
data_if_ = audio_codec_new_i2s_data(&i2s_cfg);
assert(data_if_ != NULL);
// Output
audio_codec_i2c_cfg_t i2c_cfg = {
.port = i2c_port,
.addr = es8311_addr,
.bus_handle = i2c_master_handle,
};
ctrl_if_ = audio_codec_new_i2c_ctrl(&i2c_cfg);
assert(ctrl_if_ != NULL);
gpio_if_ = audio_codec_new_gpio();
assert(gpio_if_ != NULL);
es8311_codec_cfg_t es8311_cfg = {};
es8311_cfg.ctrl_if = ctrl_if_;
es8311_cfg.gpio_if = gpio_if_;
es8311_cfg.codec_mode = ESP_CODEC_DEV_WORK_MODE_BOTH;
es8311_cfg.pa_pin = pa_pin;
es8311_cfg.use_mclk = true;
es8311_cfg.hw_gain.pa_voltage = 5.0;
es8311_cfg.hw_gain.codec_dac_voltage = 3.3;
codec_if_ = es8311_codec_new(&es8311_cfg);
assert(codec_if_ != NULL);
esp_codec_dev_cfg_t dev_cfg = {
.dev_type = ESP_CODEC_DEV_TYPE_OUT,
.codec_if = codec_if_,
.data_if = data_if_,
};
output_dev_ = esp_codec_dev_new(&dev_cfg);
assert(output_dev_ != NULL);
dev_cfg.dev_type = ESP_CODEC_DEV_TYPE_IN;
input_dev_ = esp_codec_dev_new(&dev_cfg);
assert(input_dev_ != NULL);
ESP_LOGI(TAG, "Es8311AudioCodec initialized");
}
Es8311AudioCodec::~Es8311AudioCodec() {
ESP_ERROR_CHECK(esp_codec_dev_close(output_dev_));
esp_codec_dev_delete(output_dev_);
ESP_ERROR_CHECK(esp_codec_dev_close(input_dev_));
esp_codec_dev_delete(input_dev_);
audio_codec_delete_codec_if(codec_if_);
audio_codec_delete_ctrl_if(ctrl_if_);
audio_codec_delete_gpio_if(gpio_if_);
audio_codec_delete_data_if(data_if_);
}
void Es8311AudioCodec::CreateDuplexChannels(gpio_num_t mclk, gpio_num_t bclk, gpio_num_t ws, gpio_num_t dout, gpio_num_t din) {
assert(input_sample_rate_ == output_sample_rate_);
i2s_chan_config_t chan_cfg = {
.id = I2S_NUM_0,
.role = I2S_ROLE_MASTER,
.dma_desc_num = 2,
.dma_frame_num = 240 * 3,
.auto_clear_after_cb = true,
.auto_clear_before_cb = false,
.intr_priority = 0,
};
ESP_ERROR_CHECK(i2s_new_channel(&chan_cfg, &tx_handle_, &rx_handle_));
i2s_std_config_t std_cfg = {
.clk_cfg = {
.sample_rate_hz = (uint32_t)output_sample_rate_,
.clk_src = I2S_CLK_SRC_DEFAULT,
.ext_clk_freq_hz = 0,
.mclk_multiple = I2S_MCLK_MULTIPLE_256
},
.slot_cfg = {
.data_bit_width = I2S_DATA_BIT_WIDTH_16BIT,
.slot_bit_width = I2S_SLOT_BIT_WIDTH_AUTO,
.slot_mode = I2S_SLOT_MODE_STEREO,
.slot_mask = I2S_STD_SLOT_BOTH,
.ws_width = I2S_DATA_BIT_WIDTH_16BIT,
.ws_pol = false,
.bit_shift = true,
.left_align = true,
.big_endian = false,
.bit_order_lsb = false
},
.gpio_cfg = {
.mclk = mclk,
.bclk = bclk,
.ws = ws,
.dout = dout,
.din = din,
.invert_flags = {
.mclk_inv = false,
.bclk_inv = false,
.ws_inv = false
}
}
};
ESP_ERROR_CHECK(i2s_channel_init_std_mode(tx_handle_, &std_cfg));
ESP_ERROR_CHECK(i2s_channel_init_std_mode(rx_handle_, &std_cfg));
ESP_LOGI(TAG, "Duplex channels created");
}
void Es8311AudioCodec::SetOutputVolume(int volume) {
ESP_ERROR_CHECK(esp_codec_dev_set_out_vol(output_dev_, volume));
AudioCodec::SetOutputVolume(volume);
}
void Es8311AudioCodec::EnableInput(bool enable) {
if (enable == input_enabled_) {
return;
}
if (enable) {
esp_codec_dev_sample_info_t fs = {
.bits_per_sample = 16,
.channel = 1,
.channel_mask = 0,
.sample_rate = (uint32_t)output_sample_rate_,
.mclk_multiple = 0,
};
ESP_ERROR_CHECK(esp_codec_dev_open(input_dev_, &fs));
ESP_ERROR_CHECK(esp_codec_dev_set_in_gain(input_dev_, 40.0));
} else {
ESP_ERROR_CHECK(esp_codec_dev_close(input_dev_));
}
AudioCodec::EnableInput(enable);
}
void Es8311AudioCodec::EnableOutput(bool enable) {
if (enable == output_enabled_) {
return;
}
if (enable) {
// Play 16bit 1 channel
esp_codec_dev_sample_info_t fs = {
.bits_per_sample = 16,
.channel = 1,
.channel_mask = 0,
.sample_rate = (uint32_t)output_sample_rate_,
.mclk_multiple = 0,
};
ESP_ERROR_CHECK(esp_codec_dev_open(output_dev_, &fs));
ESP_ERROR_CHECK(esp_codec_dev_set_out_vol(output_dev_, output_volume_));
} else {
ESP_ERROR_CHECK(esp_codec_dev_close(output_dev_));
}
AudioCodec::EnableOutput(enable);
}
int Es8311AudioCodec::Read(int16_t* dest, int samples) {
if (input_enabled_) {
ESP_ERROR_CHECK_WITHOUT_ABORT(esp_codec_dev_read(input_dev_, (void*)dest, samples * sizeof(int16_t)));
}
return samples;
}
int Es8311AudioCodec::Write(const int16_t* data, int samples) {
if (output_enabled_) {
ESP_ERROR_CHECK_WITHOUT_ABORT(esp_codec_dev_write(output_dev_, (void*)data, samples * sizeof(int16_t)));
}
return samples;
}

36
main/audio_codecs/es8311_audio_codec.h Normal file
View File

@ -0,0 +1,36 @@
#ifndef _ES8311_AUDIO_CODEC_H
#define _ES8311_AUDIO_CODEC_H
#include "audio_codec.h"
#include <driver/i2c.h>
#include <esp_codec_dev.h>
#include <esp_codec_dev_defaults.h>
class Es8311AudioCodec : public AudioCodec {
private:
const audio_codec_data_if_t* data_if_ = nullptr;
const audio_codec_ctrl_if_t* ctrl_if_ = nullptr;
const audio_codec_if_t* codec_if_ = nullptr;
const audio_codec_gpio_if_t* gpio_if_ = nullptr;
esp_codec_dev_handle_t output_dev_ = nullptr;
esp_codec_dev_handle_t input_dev_ = nullptr;
void CreateDuplexChannels(gpio_num_t mclk, gpio_num_t bclk, gpio_num_t ws, gpio_num_t dout, gpio_num_t din);
virtual int Read(int16_t* dest, int samples) override;
virtual int Write(const int16_t* data, int samples) override;
public:
Es8311AudioCodec(void* i2c_master_handle, i2c_port_t i2c_port, int input_sample_rate, int output_sample_rate,
gpio_num_t mclk, gpio_num_t bclk, gpio_num_t ws, gpio_num_t dout, gpio_num_t din,
gpio_num_t pa_pin, uint8_t es8311_addr);
virtual ~Es8311AudioCodec();
virtual void SetOutputVolume(int volume) override;
virtual void EnableInput(bool enable) override;
virtual void EnableOutput(bool enable) override;
};
#endif // _ES8311_AUDIO_CODEC_H

8
main/audio_codecs/no_audio_codec.cc
View File

@ -23,8 +23,8 @@ NoAudioCodec::NoAudioCodec(int input_sample_rate, int output_sample_rate, gpio_n
i2s_chan_config_t chan_cfg = {
.id = I2S_NUM_0,
.role = I2S_ROLE_MASTER,
.dma_desc_num = 6,
.dma_frame_num = 240,
.dma_desc_num = 2,
.dma_frame_num = 240 * 3,
.auto_clear_after_cb = false,
.auto_clear_before_cb = false,
.intr_priority = 0,
@ -75,7 +75,7 @@ NoAudioCodec::NoAudioCodec(int input_sample_rate, int output_sample_rate, gpio_n
// Create a new channel for speaker
i2s_chan_config_t chan_cfg = {
.id = I2S_NUM_0,
.id = (i2s_port_t)0,
.role = I2S_ROLE_MASTER,
.dma_desc_num = 6,
.dma_frame_num = 240,
@ -120,7 +120,7 @@ NoAudioCodec::NoAudioCodec(int input_sample_rate, int output_sample_rate, gpio_n
ESP_ERROR_CHECK(i2s_channel_init_std_mode(tx_handle_, &std_cfg));
// Create a new channel for MIC
chan_cfg.id = I2S_NUM_1;
chan_cfg.id = (i2s_port_t)1;
ESP_ERROR_CHECK(i2s_new_channel(&chan_cfg, nullptr, &rx_handle_));
std_cfg.clk_cfg.sample_rate_hz = (uint32_t)input_sample_rate_;
std_cfg.gpio_cfg.bclk = mic_sck;

0
main/audio_processor.cc → main/audio_processing/audio_processor.cc
View File

0
main/audio_processor.h → main/audio_processing/audio_processor.h
View File

0
main/wake_word_detect.cc → main/audio_processing/wake_word_detect.cc
View File

0
main/wake_word_detect.h → main/audio_processing/wake_word_detect.h
View File

63
main/background_task.cc Normal file
View File

@ -0,0 +1,63 @@
#include "background_task.h"
#include <esp_log.h>
#define TAG "BackgroundTask"
BackgroundTask::BackgroundTask(uint32_t stack_size) {
#if CONFIG_IDF_TARGET_ESP32S3
task_stack_ = (StackType_t*)heap_caps_malloc(stack_size, MALLOC_CAP_SPIRAM);
background_task_handle_ = xTaskCreateStatic([](void* arg) {
BackgroundTask* task = (BackgroundTask*)arg;
task->BackgroundTaskLoop();
}, "background_task", stack_size, this, 1, task_stack_, &task_buffer_);
#else
xTaskCreate([](void* arg) {
BackgroundTask* task = (BackgroundTask*)arg;
task->BackgroundTaskLoop();
}, "background_task", stack_size, this, 1, &background_task_handle_);
#endif
}
BackgroundTask::~BackgroundTask() {
if (background_task_handle_ != nullptr) {
vTaskDelete(background_task_handle_);
}
}
void BackgroundTask::Schedule(std::function<void()> callback) {
std::lock_guard<std::mutex> lock(mutex_);
if (active_tasks_ >= 30) {
ESP_LOGW(TAG, "active_tasks_ == %u", active_tasks_.load());
}
active_tasks_++;
auto wrapped_callback = [this, callback]() {
callback();
active_tasks_--;
condition_variable_.notify_all();
};
main_tasks_.push_back(wrapped_callback);
condition_variable_.notify_all();
}
void BackgroundTask::WaitForCompletion() {
std::unique_lock<std::mutex> lock(mutex_);
condition_variable_.wait(lock, [this]() {
return main_tasks_.empty() && active_tasks_ == 0;
});
}
void BackgroundTask::BackgroundTaskLoop() {
ESP_LOGI(TAG, "background_task started");
while (true) {
std::unique_lock<std::mutex> lock(mutex_);
condition_variable_.wait(lock, [this]() { return !main_tasks_.empty(); });
std::list<std::function<void()>> tasks = std::move(main_tasks_);
lock.unlock();
for (auto& task : tasks) {
task();
}
}
}

33
main/background_task.h Normal file
View File

@ -0,0 +1,33 @@
#ifndef BACKGROUND_TASK_H
#define BACKGROUND_TASK_H
#include <freertos/FreeRTOS.h>
#include <freertos/task.h>
#include <mutex>
#include <list>
#include <condition_variable>
#include <atomic>
class BackgroundTask {
public:
BackgroundTask(uint32_t stack_size = 4096 * 2);
~BackgroundTask();
void Schedule(std::function<void()> callback);
void WaitForCompletion();
private:
std::mutex mutex_;
std::list<std::function<void()>> main_tasks_;
std::condition_variable condition_variable_;
TaskHandle_t background_task_handle_ = nullptr;
std::atomic<size_t> active_tasks_{0};
TaskHandle_t task_ = nullptr;
StaticTask_t task_buffer_;
StackType_t* task_stack_ = nullptr;
void BackgroundTaskLoop();
};
#endif

2
main/boards/bread-compact-ml307/compact_ml307_board.cc
View File

@ -22,7 +22,7 @@ private:
void InitializeDisplayI2c() {
i2c_master_bus_config_t bus_config = {
.i2c_port = I2C_NUM_0,
.i2c_port = (i2c_port_t)0,
.sda_io_num = DISPLAY_SDA_PIN,
.scl_io_num = DISPLAY_SCL_PIN,
.clk_source = I2C_CLK_SRC_DEFAULT,

2
main/boards/bread-compact-wifi/compact_wifi_board.cc
View File

@ -23,7 +23,7 @@ private:
void InitializeDisplayI2c() {
i2c_master_bus_config_t bus_config = {
.i2c_port = I2C_NUM_0,
.i2c_port = (i2c_port_t)0,
.sda_io_num = DISPLAY_SDA_PIN,
.scl_io_num = DISPLAY_SCL_PIN,
.clk_source = I2C_CLK_SRC_DEFAULT,

9
main/boards/common/board.cc
View File

@ -1,5 +1,6 @@
#include "board.h"
#include "system_info.h"
#include "display/no_display.h"
#include <esp_log.h>
#include <esp_ota_ops.h>
@ -11,6 +12,12 @@ bool Board::GetBatteryLevel(int &level, bool& charging) {
return false;
}
Display* Board::GetDisplay() {
static NoDisplay display;
return &display;
}
std::string Board::GetJson() {
/*
{
@ -101,4 +108,4 @@ std::string Board::GetJson() {
// Close the JSON object
json += "}";
return json;
}
}

2
main/boards/common/board.h
View File

@ -35,7 +35,7 @@ public:
virtual ~Board() = default;
virtual Led* GetBuiltinLed() = 0;
virtual AudioCodec* GetAudioCodec() = 0;
virtual Display* GetDisplay() = 0;
virtual Display* GetDisplay();
virtual Http* CreateHttp() = 0;
virtual WebSocket* CreateWebSocket() = 0;
virtual Mqtt* CreateMqtt() = 0;

90
main/boards/common/led.cc
View File

@ -7,10 +7,6 @@
#define TAG "Led"
Led::Led(gpio_num_t gpio) {
mutex_ = xSemaphoreCreateMutex();
blink_event_group_ = xEventGroupCreate();
xEventGroupSetBits(blink_event_group_, BLINK_TASK_STOPPED_BIT);
if (gpio == GPIO_NUM_NC) {
ESP_LOGI(TAG, "Builtin LED not connected");
return;
@ -29,19 +25,25 @@ Led::Led(gpio_num_t gpio) {
led_strip_clear(led_strip_);
SetGrey();
esp_timer_create_args_t blink_timer_args = {
.callback = [](void *arg) {
auto led = static_cast<Led*>(arg);
led->OnBlinkTimer();
},
.arg = this,
.dispatch_method = ESP_TIMER_TASK,
.name = "Blink Timer",
.skip_unhandled_events = false,
};
ESP_ERROR_CHECK(esp_timer_create(&blink_timer_args, &blink_timer_));
}
Led::~Led() {
StopBlinkInternal();
esp_timer_stop(blink_timer_);
if (led_strip_ != nullptr) {
led_strip_del(led_strip_);
}
if (mutex_ != nullptr) {
vSemaphoreDelete(mutex_);
}
if (blink_event_group_ != nullptr) {
vEventGroupDelete(blink_event_group_);
}
}
void Led::SetColor(uint8_t r, uint8_t g, uint8_t b) {
@ -54,21 +56,21 @@ void Led::TurnOn() {
if (led_strip_ == nullptr) {
return;
}
StopBlinkInternal();
xSemaphoreTake(mutex_, portMAX_DELAY);
std::lock_guard<std::mutex> lock(mutex_);
esp_timer_stop(blink_timer_);
led_strip_set_pixel(led_strip_, 0, r_, g_, b_);
led_strip_refresh(led_strip_);
xSemaphoreGive(mutex_);
}
void Led::TurnOff() {
if (led_strip_ == nullptr) {
return;
}
StopBlinkInternal();
xSemaphoreTake(mutex_, portMAX_DELAY);
std::lock_guard<std::mutex> lock(mutex_);
esp_timer_stop(blink_timer_);
led_strip_clear(led_strip_);
xSemaphoreGive(mutex_);
}
void Led::BlinkOnce() {
@ -87,45 +89,27 @@ void Led::StartBlinkTask(int times, int interval_ms) {
if (led_strip_ == nullptr) {
return;
}
StopBlinkInternal();
xSemaphoreTake(mutex_, portMAX_DELAY);
std::lock_guard<std::mutex> lock(mutex_);
esp_timer_stop(blink_timer_);
blink_times_ = times;
led_strip_clear(led_strip_);
blink_counter_ = times * 2;
blink_interval_ms_ = interval_ms;
should_blink_ = true;
esp_timer_start_periodic(blink_timer_, interval_ms * 1000);
}
xEventGroupClearBits(blink_event_group_, BLINK_TASK_STOPPED_BIT);
xEventGroupSetBits(blink_event_group_, BLINK_TASK_RUNNING_BIT);
void Led::OnBlinkTimer() {
std::lock_guard<std::mutex> lock(mutex_);
blink_counter_--;
if (blink_counter_ & 1) {
led_strip_set_pixel(led_strip_, 0, r_, g_, b_);
led_strip_refresh(led_strip_);
} else {
led_strip_clear(led_strip_);
xTaskCreate([](void* obj) {
auto this_ = static_cast<Led*>(obj);
int count = 0;
while (this_->should_blink_ && (this_->blink_times_ == BLINK_INFINITE || count < this_->blink_times_)) {
xSemaphoreTake(this_->mutex_, portMAX_DELAY);
led_strip_set_pixel(this_->led_strip_, 0, this_->r_, this_->g_, this_->b_);
led_strip_refresh(this_->led_strip_);
xSemaphoreGive(this_->mutex_);
vTaskDelay(this_->blink_interval_ms_ / portTICK_PERIOD_MS);
if (!this_->should_blink_) break;
xSemaphoreTake(this_->mutex_, portMAX_DELAY);
led_strip_clear(this_->led_strip_);
xSemaphoreGive(this_->mutex_);
vTaskDelay(this_->blink_interval_ms_ / portTICK_PERIOD_MS);
if (this_->blink_times_ != BLINK_INFINITE) count++;
if (blink_counter_ == 0) {
esp_timer_stop(blink_timer_);
}
this_->blink_task_ = nullptr;
xEventGroupClearBits(this_->blink_event_group_, BLINK_TASK_RUNNING_BIT);
xEventGroupSetBits(this_->blink_event_group_, BLINK_TASK_STOPPED_BIT);
vTaskDelete(NULL);
}, "blink", 2048, this, tskIDLE_PRIORITY, &blink_task_);
xSemaphoreGive(mutex_);
}
void Led::StopBlinkInternal() {
should_blink_ = false;
xEventGroupWaitBits(blink_event_group_, BLINK_TASK_STOPPED_BIT, pdFALSE, pdTRUE, portMAX_DELAY);
}
}

14
main/boards/common/led.h
View File

@ -2,10 +2,9 @@
#define _LED_H_
#include <led_strip.h>
#include <freertos/semphr.h>
#include <freertos/task.h>
#include <freertos/event_groups.h>
#include <esp_timer.h>
#include <atomic>
#include <mutex>
#define BLINK_INFINITE -1
#define BLINK_TASK_STOPPED_BIT BIT0
@ -33,17 +32,16 @@ public:
void SetBlue(uint8_t brightness = DEFAULT_BRIGHTNESS) { SetColor(0, 0, brightness); }
private:
SemaphoreHandle_t mutex_;
EventGroupHandle_t blink_event_group_;
std::mutex mutex_;
TaskHandle_t blink_task_ = nullptr;
led_strip_handle_t led_strip_ = nullptr;
uint8_t r_ = 0, g_ = 0, b_ = 0;
int blink_times_ = 0;
int blink_counter_ = 0;
int blink_interval_ms_ = 0;
std::atomic<bool> should_blink_{false};
esp_timer_handle_t blink_timer_ = nullptr;
void StartBlinkTask(int times, int interval_ms);
void StopBlinkInternal();
void OnBlinkTimer();
};
#endif // _LED_H_

5
main/boards/common/wifi_board.cc
View File

@ -62,7 +62,10 @@ void WifiBoard::StartNetwork() {
// Wait forever until reset after configuration
while (true) {
vTaskDelay(pdMS_TO_TICKS(1000));
int free_sram = heap_caps_get_free_size(MALLOC_CAP_INTERNAL);
int min_free_sram = heap_caps_get_minimum_free_size(MALLOC_CAP_INTERNAL);
ESP_LOGI(TAG, "Free internal: %u minimal internal: %u", free_sram, min_free_sram);
vTaskDelay(pdMS_TO_TICKS(10000));
}
}
}

2
main/boards/esp-box-3/esp_box3_board.cc
View File

@ -19,7 +19,7 @@ private:
void InitializeI2c() {
// Initialize I2C peripheral
i2c_master_bus_config_t i2c_bus_cfg = {
.i2c_port = I2C_NUM_1,
.i2c_port = (i2c_port_t)1,
.sda_io_num = AUDIO_CODEC_I2C_SDA_PIN,
.scl_io_num = AUDIO_CODEC_I2C_SCL_PIN,
.clk_source = I2C_CLK_SRC_DEFAULT,

4
main/boards/kevin-box-1/kevin_box_board.cc
View File

@ -48,7 +48,7 @@ private:
void InitializeDisplayI2c() {
i2c_master_bus_config_t bus_config = {
.i2c_port = I2C_NUM_0,
.i2c_port = (i2c_port_t)0,
.sda_io_num = DISPLAY_SDA_PIN,
.scl_io_num = DISPLAY_SCL_PIN,
.clk_source = I2C_CLK_SRC_DEFAULT,
@ -65,7 +65,7 @@ private:
void InitializeCodecI2c() {
// Initialize I2C peripheral
i2c_master_bus_config_t i2c_bus_cfg = {
.i2c_port = I2C_NUM_1,
.i2c_port = (i2c_port_t)1,
.sda_io_num = AUDIO_CODEC_I2C_SDA_PIN,
.scl_io_num = AUDIO_CODEC_I2C_SCL_PIN,
.clk_source = I2C_CLK_SRC_DEFAULT,

8
main/boards/kevin-box-2/kevin_box_board.cc
View File

@ -86,7 +86,7 @@ private:
void InitializeDisplayI2c() {
i2c_master_bus_config_t bus_config = {
.i2c_port = I2C_NUM_0,
.i2c_port = (i2c_port_t)0,
.sda_io_num = DISPLAY_SDA_PIN,
.scl_io_num = DISPLAY_SCL_PIN,
.clk_source = I2C_CLK_SRC_DEFAULT,
@ -103,7 +103,7 @@ private:
void InitializeCodecI2c() {
// Initialize I2C peripheral
i2c_master_bus_config_t i2c_bus_cfg = {
.i2c_port = I2C_NUM_1,
.i2c_port = (i2c_port_t)1,
.sda_io_num = AUDIO_CODEC_I2C_SDA_PIN,
.scl_io_num = AUDIO_CODEC_I2C_SCL_PIN,
.clk_source = I2C_CLK_SRC_DEFAULT,
@ -118,10 +118,6 @@ private:
}
void InitializeButtons() {
// 测试按住说话
// boot_button_.OnClick([this]() {
// Application::GetInstance().ToggleChatState();
// });
boot_button_.OnPressDown([this]() {
Application::GetInstance().StartListening();
});

24
main/boards/kevin-c3/config.h Normal file
View File

@ -0,0 +1,24 @@
#ifndef _BOARD_CONFIG_H_
#define _BOARD_CONFIG_H_
#include <driver/gpio.h>
#define AUDIO_INPUT_SAMPLE_RATE 24000
#define AUDIO_OUTPUT_SAMPLE_RATE 24000
#define AUDIO_I2S_GPIO_MCLK GPIO_NUM_10
#define AUDIO_I2S_GPIO_WS GPIO_NUM_12
#define AUDIO_I2S_GPIO_BCLK GPIO_NUM_8
#define AUDIO_I2S_GPIO_DIN GPIO_NUM_7
#define AUDIO_I2S_GPIO_DOUT GPIO_NUM_11
#define AUDIO_CODEC_PA_PIN GPIO_NUM_13
#define AUDIO_CODEC_I2C_SDA_PIN GPIO_NUM_0
#define AUDIO_CODEC_I2C_SCL_PIN GPIO_NUM_1
#define AUDIO_CODEC_ES8311_ADDR ES8311_CODEC_DEFAULT_ADDR
#define BUILTIN_LED_GPIO GPIO_NUM_2
#define BOOT_BUTTON_GPIO GPIO_NUM_9
#endif // _BOARD_CONFIG_H_

78
main/boards/kevin-c3/kevin_box_board.cc Normal file
View File

@ -0,0 +1,78 @@
#include "wifi_board.h"
#include "audio_codecs/es8311_audio_codec.h"
#include "application.h"
#include "button.h"
#include "led.h"
#include "config.h"
#include <wifi_station.h>
#include <esp_log.h>
#include <driver/i2c_master.h>
#define TAG "KevinBoxBoard"
class KevinBoxBoard : public WifiBoard {
private:
i2c_master_bus_handle_t codec_i2c_bus_;
Button boot_button_;
void InitializeCodecI2c() {
// Initialize I2C peripheral
i2c_master_bus_config_t i2c_bus_cfg = {
.i2c_port = I2C_NUM_0,
.sda_io_num = AUDIO_CODEC_I2C_SDA_PIN,
.scl_io_num = AUDIO_CODEC_I2C_SCL_PIN,
.clk_source = I2C_CLK_SRC_DEFAULT,
.glitch_ignore_cnt = 7,
.intr_priority = 0,
.trans_queue_depth = 0,
.flags = {
.enable_internal_pullup = 1,
},
};
ESP_ERROR_CHECK(i2c_new_master_bus(&i2c_bus_cfg, &codec_i2c_bus_));
}
void InitializeButtons() {
boot_button_.OnClick([this]() {
auto& app = Application::GetInstance();
if (app.GetChatState() == kChatStateUnknown && !WifiStation::GetInstance().IsConnected()) {
ResetWifiConfiguration();
}
});
boot_button_.OnPressDown([this]() {
Application::GetInstance().StartListening();
});
boot_button_.OnPressUp([this]() {
Application::GetInstance().StopListening();
});
}
public:
KevinBoxBoard() :
boot_button_(BOOT_BUTTON_GPIO) {
}
virtual void Initialize() override {
ESP_LOGI(TAG, "Initializing KevinBoxBoard");
InitializeCodecI2c();
InitializeButtons();
WifiBoard::Initialize();
}
virtual Led* GetBuiltinLed() override {
static Led led(BUILTIN_LED_GPIO);
return &led;
}
virtual AudioCodec* GetAudioCodec() override {
static Es8311AudioCodec audio_codec(codec_i2c_bus_, I2C_NUM_0, AUDIO_INPUT_SAMPLE_RATE, AUDIO_OUTPUT_SAMPLE_RATE,
AUDIO_I2S_GPIO_MCLK, AUDIO_I2S_GPIO_BCLK, AUDIO_I2S_GPIO_WS, AUDIO_I2S_GPIO_DOUT, AUDIO_I2S_GPIO_DIN,
AUDIO_CODEC_PA_PIN, AUDIO_CODEC_ES8311_ADDR);
return &audio_codec;
}
};
DECLARE_BOARD(KevinBoxBoard);

9
main/boards/lichuang-dev/lichuang_dev_board.cc
View File

@ -42,7 +42,7 @@ private:
void InitializeI2c() {
// Initialize I2C peripheral
i2c_master_bus_config_t i2c_bus_cfg = {
.i2c_port = I2C_NUM_1,
.i2c_port = (i2c_port_t)1,
.sda_io_num = AUDIO_CODEC_I2C_SDA_PIN,
.scl_io_num = AUDIO_CODEC_I2C_SCL_PIN,
.clk_source = I2C_CLK_SRC_DEFAULT,
@ -76,7 +76,12 @@ private:
if (app.GetChatState() == kChatStateUnknown && !WifiStation::GetInstance().IsConnected()) {
ResetWifiConfiguration();
}
app.ToggleChatState();
});
boot_button_.OnPressDown([this]() {
Application::GetInstance().StartListening();
});
boot_button_.OnPressUp([this]() {
Application::GetInstance().StopListening();
});
}

2
main/idf_component.yml
View File

@ -2,7 +2,7 @@
dependencies:
78/esp-wifi-connect: "~1.4.1"
78/esp-opus-encoder: "~1.1.0"
78/esp-ml307: "~1.6.3"
78/esp-ml307: "~1.7.0"
espressif/led_strip: "^2.4.1"
espressif/esp_codec_dev: "^1.3.1"
espressif/esp-sr: "^1.9.0"

10
main/ota.cc
View File

@ -50,11 +50,11 @@ bool Ota::CheckVersion() {
}
http->SetHeader("Content-Type", "application/json");
if (post_data_.length() > 0) {
http->SetContent(post_data_);
http->Open("POST", check_version_url_);
} else {
http->Open("GET", check_version_url_);
std::string method = post_data_.length() > 0 ? "POST" : "GET";
if (!http->Open(method, check_version_url_, post_data_)) {
ESP_LOGE(TAG, "Failed to open HTTP connection");
delete http;
return false;
}
auto response = http->GetBody();

8
sdkconfig.defaults
View File

@ -10,13 +10,13 @@ CONFIG_PARTITION_TABLE_CUSTOM=y
CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv"
CONFIG_PARTITION_TABLE_OFFSET=0x8000
CONFIG_USE_WAKENET=y
CONFIG_SR_WN_WN9_NIHAOXIAOZHI_TTS=y
CONFIG_USE_MULTINET=n
ESP_TASK_WDT_TIMEOUT_S=10
CONFIG_FREERTOS_GENERATE_RUN_TIME_STATS=y
CONFIG_FREERTOS_USE_STATS_FORMATTING_FUNCTIONS=y
CONFIG_LV_COLOR_16_SWAP=y
CONFIG_LV_MEM_CUSTOM=y
CONFIG_MBEDTLS_DYNAMIC_BUFFER=y
CONFIG_ESP_WIFI_IRAM_OPT=n
CONFIG_ESP_WIFI_RX_IRAM_OPT=n

4
sdkconfig.defaults.esp32c3
View File

@ -1,6 +1,2 @@
CONFIG_ESPTOOLPY_FLASHSIZE_16MB=y
CONFIG_PARTITION_TABLE_CUSTOM=y
CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions_4M.csv"
CONFIG_PARTITION_TABLE_OFFSET=0x8000

4
sdkconfig.defaults.esp32s3
View File

@ -16,3 +16,7 @@ CONFIG_MBEDTLS_EXTERNAL_MEM_ALLOC=y
CONFIG_ESP32S3_INSTRUCTION_CACHE_32KB=y
CONFIG_ESP32S3_DATA_CACHE_64KB=y
CONFIG_ESP32S3_DATA_CACHE_LINE_64B=y
CONFIG_USE_WAKENET=y
CONFIG_SR_WN_WN9_NIHAOXIAOZHI_TTS=y
CONFIG_USE_MULTINET=n