#include "sdkconfig.h"
|
|
#include <esp_heap_caps.h>
|
#include <cstdio>
|
#include <cstring>
|
#include <esp_log.h>
|
#include <img_converters.h>
|
|
#include "esp32_camera.h"
|
#include "board.h"
|
#include "display.h"
|
#include "lvgl_display.h"
|
#include "mcp_server.h"
|
#include "system_info.h"
|
#include "jpg/image_to_jpeg.h"
|
#include "esp_timer.h"
|
|
#define TAG "Esp32Camera"
|
|
Esp32Camera::Esp32Camera(const camera_config_t &config) {
|
esp_err_t err = esp_camera_init(&config);
|
if (err != ESP_OK) {
|
ESP_LOGE(TAG, "esp_camera_init failed with error 0x%x", err);
|
return;
|
}
|
|
sensor_t *s = esp_camera_sensor_get();
|
if (s) {
|
if (s->id.PID == GC0308_PID) {
|
s->set_hmirror(s, 0); // Control camera mirror: 1 for mirror, 0 for normal
|
}
|
ESP_LOGI(TAG, "Camera initialized: format=%d", config.pixel_format);
|
}
|
|
streaming_on_ = true;
|
}
|
|
Esp32Camera::~Esp32Camera() {
|
if (streaming_on_) {
|
if (current_fb_) {
|
esp_camera_fb_return(current_fb_);
|
current_fb_ = nullptr;
|
}
|
if (encode_buf_) {
|
heap_caps_free(encode_buf_);
|
encode_buf_ = nullptr;
|
encode_buf_size_ = 0;
|
}
|
esp_camera_deinit();
|
streaming_on_ = false;
|
}
|
}
|
|
void Esp32Camera::SetExplainUrl(const std::string &url, const std::string &token) {
|
explain_url_ = url;
|
explain_token_ = token;
|
}
|
|
bool Esp32Camera::Capture() {
|
if (encoder_thread_.joinable()) {
|
encoder_thread_.join();
|
}
|
|
if (!streaming_on_) {
|
return false;
|
}
|
|
// Get the latest frame, discard old frames for real-time performance
|
for (int i = 0; i < 2; i++) {
|
if (current_fb_) {
|
esp_camera_fb_return(current_fb_);
|
}
|
current_fb_ = esp_camera_fb_get();
|
if (!current_fb_) {
|
ESP_LOGE(TAG, "Camera capture failed");
|
return false;
|
}
|
}
|
|
// Prepare encode buffer for RGB565 format (with optional byte swapping)
|
if (current_fb_->format == PIXFORMAT_RGB565) {
|
size_t pixel_count = current_fb_->width * current_fb_->height;
|
size_t data_size = pixel_count * 2;
|
|
// Allocate or reallocate encode buffer if needed
|
if (encode_buf_size_ < data_size) {
|
if (encode_buf_) {
|
heap_caps_free(encode_buf_);
|
}
|
encode_buf_ = (uint8_t *)heap_caps_malloc(data_size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
|
if (encode_buf_ == nullptr) {
|
ESP_LOGE(TAG, "Failed to allocate memory for encode buffer");
|
encode_buf_size_ = 0;
|
return false;
|
}
|
encode_buf_size_ = data_size;
|
}
|
|
// Copy data to encode buffer with optional byte swapping
|
uint16_t *src = (uint16_t *)current_fb_->buf;
|
uint16_t *dst = (uint16_t *)encode_buf_;
|
if (swap_bytes_enabled_) {
|
for (size_t i = 0; i < pixel_count; i++) {
|
dst[i] = __builtin_bswap16(src[i]);
|
}
|
} else {
|
memcpy(encode_buf_, current_fb_->buf, data_size);
|
}
|
|
// Allocate separate buffer for preview display
|
uint8_t *preview_data = (uint8_t *)heap_caps_malloc(data_size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
|
if (preview_data != nullptr) {
|
memcpy(preview_data, encode_buf_, data_size);
|
auto display = dynamic_cast<LvglDisplay *>(Board::GetInstance().GetDisplay());
|
if (display != nullptr) {
|
display->SetPreviewImage(std::make_unique<LvglAllocatedImage>(preview_data, data_size, current_fb_->width, current_fb_->height, current_fb_->width * 2, LV_COLOR_FORMAT_RGB565));
|
} else {
|
heap_caps_free(preview_data);
|
}
|
}
|
} else if (current_fb_->format == PIXFORMAT_JPEG) {
|
// JPEG format preview usually requires decoding, skip preview display for now, just log
|
ESP_LOGW(TAG, "JPEG capture success, len=%zu, but not supported for preview", current_fb_->len);
|
}
|
|
ESP_LOGI(TAG, "Captured frame: %dx%d, len=%zu, format=%d",
|
current_fb_->width, current_fb_->height, current_fb_->len, current_fb_->format);
|
|
return true;
|
}
|
|
bool Esp32Camera::SetHMirror(bool enabled) {
|
sensor_t *s = esp_camera_sensor_get();
|
if (!s) {
|
return false;
|
}
|
s->set_hmirror(s, enabled ? 1 : 0);
|
return true;
|
}
|
|
bool Esp32Camera::SetVFlip(bool enabled) {
|
sensor_t *s = esp_camera_sensor_get();
|
if (!s) {
|
return false;
|
}
|
s->set_vflip(s, enabled ? 1 : 0);
|
return true;
|
}
|
|
bool Esp32Camera::SetSwapBytes(bool enabled) {
|
swap_bytes_enabled_ = enabled;
|
return true;
|
}
|
|
std::string Esp32Camera::Explain(const std::string &question) {
|
if (explain_url_.empty()) {
|
throw std::runtime_error("Image explain URL or token is not set");
|
}
|
|
if (current_fb_ == nullptr) {
|
throw std::runtime_error("No camera frame captured");
|
}
|
|
// Create local JPEG queue
|
QueueHandle_t jpeg_queue = xQueueCreate(40, sizeof(JpegChunk));
|
if (jpeg_queue == nullptr) {
|
ESP_LOGE(TAG, "Failed to create JPEG queue");
|
throw std::runtime_error("Failed to create JPEG queue");
|
}
|
|
// Start encoding thread
|
encoder_thread_ = std::thread([this, jpeg_queue]() {
|
int64_t start_time = esp_timer_get_time();
|
uint16_t w = current_fb_->width;
|
uint16_t h = current_fb_->height;
|
v4l2_pix_fmt_t enc_fmt;
|
switch (current_fb_->format) {
|
case PIXFORMAT_RGB565:
|
enc_fmt = V4L2_PIX_FMT_RGB565;
|
break;
|
case PIXFORMAT_YUV422:
|
enc_fmt = V4L2_PIX_FMT_YUYV; // YUV422 is actually YUYV format
|
break;
|
case PIXFORMAT_YUV420:
|
enc_fmt = V4L2_PIX_FMT_YUV420;
|
break;
|
case PIXFORMAT_GRAYSCALE:
|
enc_fmt = V4L2_PIX_FMT_GREY;
|
break;
|
case PIXFORMAT_JPEG:
|
enc_fmt = V4L2_PIX_FMT_JPEG;
|
break;
|
case PIXFORMAT_RGB888:
|
enc_fmt = V4L2_PIX_FMT_RGB24;
|
break;
|
default:
|
ESP_LOGE(TAG, "Unsupported pixel format: %d", current_fb_->format);
|
return;
|
}
|
|
// Use encode buffer for RGB565, otherwise use original frame buffer
|
uint8_t *jpeg_src_buf = current_fb_->buf;
|
size_t jpeg_src_len = current_fb_->len;
|
if (current_fb_->format == PIXFORMAT_RGB565 && encode_buf_ != nullptr) {
|
jpeg_src_buf = encode_buf_;
|
jpeg_src_len = encode_buf_size_;
|
}
|
|
bool ok = image_to_jpeg_cb(jpeg_src_buf, jpeg_src_len, w, h, enc_fmt, 80,
|
[](void* arg, size_t index, const void* data, size_t len) -> size_t {
|
auto jpeg_queue = static_cast<QueueHandle_t>(arg);
|
JpegChunk chunk = {.data = nullptr, .len = len};
|
if (index == 0 && data != nullptr && len > 0) {
|
chunk.data = (uint8_t*)heap_caps_aligned_alloc(16, len, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
|
if (chunk.data == nullptr) {
|
ESP_LOGE(TAG, "Failed to allocate %zu bytes for JPEG chunk", len);
|
chunk.len = 0;
|
} else {
|
memcpy(chunk.data, data, len);
|
}
|
} else {
|
chunk.len = 0; // Sentinel or error
|
}
|
xQueueSend(jpeg_queue, &chunk, portMAX_DELAY);
|
return len;
|
}, jpeg_queue);
|
|
if (!ok) {
|
JpegChunk chunk = {.data = nullptr, .len = 0};
|
xQueueSend(jpeg_queue, &chunk, portMAX_DELAY);
|
}
|
int64_t end_time = esp_timer_get_time();
|
ESP_LOGI(TAG, "JPEG encoding time: %ld ms", int((end_time - start_time) / 1000));
|
});
|
|
auto network = Board::GetInstance().GetNetwork();
|
auto http = network->CreateHttp(3);
|
std::string boundary = "----ESP32_CAMERA_BOUNDARY";
|
|
http->SetHeader("Device-Id", SystemInfo::GetMacAddress().c_str());
|
http->SetHeader("Client-Id", Board::GetInstance().GetUuid().c_str());
|
if (!explain_token_.empty()) {
|
http->SetHeader("Authorization", "Bearer " + explain_token_);
|
}
|
http->SetHeader("Content-Type", "multipart/form-data; boundary=" + boundary);
|
http->SetHeader("Transfer-Encoding", "chunked");
|
if (!http->Open("POST", explain_url_)) {
|
ESP_LOGE(TAG, "Failed to connect to explain URL");
|
encoder_thread_.join();
|
JpegChunk chunk;
|
while (xQueueReceive(jpeg_queue, &chunk, portMAX_DELAY) == pdPASS) {
|
if (chunk.data != nullptr) {
|
heap_caps_free(chunk.data);
|
} else {
|
break;
|
}
|
}
|
vQueueDelete(jpeg_queue);
|
throw std::runtime_error("Failed to connect to explain URL");
|
}
|
|
{
|
std::string question_field;
|
question_field += "--" + boundary + "\r\n";
|
question_field += "Content-Disposition: form-data; name=\"question\"\r\n";
|
question_field += "\r\n";
|
question_field += question + "\r\n";
|
http->Write(question_field.c_str(), question_field.size());
|
}
|
{
|
std::string file_header;
|
file_header += "--" + boundary + "\r\n";
|
file_header += "Content-Disposition: form-data; name=\"file\"; filename=\"camera.jpg\"\r\n";
|
file_header += "Content-Type: image/jpeg\r\n";
|
file_header += "\r\n";
|
http->Write(file_header.c_str(), file_header.size());
|
}
|
|
size_t total_sent = 0;
|
bool saw_terminator = false;
|
while (true) {
|
JpegChunk chunk;
|
if (xQueueReceive(jpeg_queue, &chunk, portMAX_DELAY) != pdPASS) {
|
ESP_LOGE(TAG, "Failed to receive JPEG chunk");
|
break;
|
}
|
if (chunk.data == nullptr) {
|
saw_terminator = true;
|
break;
|
}
|
http->Write((const char *)chunk.data, chunk.len);
|
total_sent += chunk.len;
|
heap_caps_free(chunk.data);
|
}
|
encoder_thread_.join();
|
vQueueDelete(jpeg_queue);
|
|
if (!saw_terminator || total_sent == 0) {
|
ESP_LOGE(TAG, "JPEG encoder failed or produced empty output");
|
throw std::runtime_error("Failed to encode image to JPEG");
|
}
|
|
{
|
std::string multipart_footer;
|
multipart_footer += "\r\n--" + boundary + "--\r\n";
|
http->Write(multipart_footer.c_str(), multipart_footer.size());
|
}
|
http->Write("", 0);
|
|
if (http->GetStatusCode() != 200) {
|
ESP_LOGE(TAG, "Failed to upload photo, status code: %d", http->GetStatusCode());
|
throw std::runtime_error("Failed to upload photo");
|
}
|
|
std::string result = http->ReadAll();
|
http->Close();
|
|
size_t remain_stack_size = uxTaskGetStackHighWaterMark(nullptr);
|
ESP_LOGI(TAG, "Explain image size=%dx%d, compressed size=%d, remain stack size=%d, question=%s\n%s",
|
current_fb_->width, current_fb_->height, (int)total_sent, (int)remain_stack_size, question.c_str(), result.c_str());
|
return result;
|
}
|
