diff --git a/host/class/uac/usb_host_uac/CHANGELOG.md b/host/class/uac/usb_host_uac/CHANGELOG.md index aa15a8bc..2bba3f63 100644 --- a/host/class/uac/usb_host_uac/CHANGELOG.md +++ b/host/class/uac/usb_host_uac/CHANGELOG.md @@ -2,7 +2,13 @@ All notable changes to this component will be documented in this file. -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [1.3.3] - 2025-11-4 + +### Changed + +- Fixed a playback stuttering issue when bInterval was not equal to 1 in the full-speed device audio endpoint descriptor, and forced bInterval = 1 to make it compatible with these non-standard devices. ## [1.3.2] - 2025-10-21 diff --git a/host/class/uac/usb_host_uac/examples/audio_player/CMakeLists.txt b/host/class/uac/usb_host_uac/examples/audio_player/CMakeLists.txt new file mode 100644 index 00000000..80038fb7 --- /dev/null +++ b/host/class/uac/usb_host_uac/examples/audio_player/CMakeLists.txt @@ -0,0 +1,6 @@ +# The following lines of boilerplate have to be in your project's CMakeLists +# in this exact order for cmake to work correctly +cmake_minimum_required(VERSION 3.5) + +include($ENV{IDF_PATH}/tools/cmake/project.cmake) +project(audio_player) diff --git a/host/class/uac/usb_host_uac/examples/audio_player/README.md b/host/class/uac/usb_host_uac/examples/audio_player/README.md new file mode 100644 index 00000000..f5aa7ba4 --- /dev/null +++ b/host/class/uac/usb_host_uac/examples/audio_player/README.md @@ -0,0 +1,12 @@ +| Supported Targets | ESP32-S2 | ESP32-S3 | ESP32-P4 | +| ----------------- | -------- | -------- | -------- | + +# UVC driver example: Video stream + +## Selecting the USB Component + +To manually select which USB Component shall be used to build this example, please refer to the following documentation page: [Manual USB component selection](../../../../../../docs/host/usb_host_lib/usb_component_manual_selection.md). + +## Enable MIC playback + +To enable MIC playback in menuconfig `(Top) → Example USB Audio Player → Playback audio from microphone` diff --git a/host/class/uac/usb_host_uac/examples/audio_player/main/CMakeLists.txt b/host/class/uac/usb_host_uac/examples/audio_player/main/CMakeLists.txt new file mode 100644 index 00000000..804c5dd7 --- /dev/null +++ b/host/class/uac/usb_host_uac/examples/audio_player/main/CMakeLists.txt @@ -0,0 +1,3 @@ +idf_component_register(SRC_DIRS . + INCLUDE_DIRS "." + EMBED_FILES ../spiffs/output_8k.wav ../spiffs/output_16k.wav) diff --git a/host/class/uac/usb_host_uac/examples/audio_player/main/Kconfig.projbuild b/host/class/uac/usb_host_uac/examples/audio_player/main/Kconfig.projbuild new file mode 100644 index 00000000..266c6a4a --- /dev/null +++ b/host/class/uac/usb_host_uac/examples/audio_player/main/Kconfig.projbuild @@ -0,0 +1,8 @@ +menu "Example USB Audio Player" + config EXAMPLE_MIC_PLAYBACK + bool "Playback audio from microphone" + default n + help + Enabled this to playback audio from microphone + +endmenu diff --git a/host/class/uac/usb_host_uac/examples/audio_player/main/idf_component.yml b/host/class/uac/usb_host_uac/examples/audio_player/main/idf_component.yml new file mode 100644 index 00000000..e3f7d99a --- /dev/null +++ b/host/class/uac/usb_host_uac/examples/audio_player/main/idf_component.yml @@ -0,0 +1,10 @@ +dependencies: + idf: ">=5.0" # The version of the ESP-IDF that esp-audio-player requires + usb_host_uac: + override_path: ../../../../usb_host_uac/ + espressif/usb: + version: "*" + override_path: "../../../../../../usb" + rules: # Both if clauses must be fulfilled to override the component + - if: "$ENV_VAR_USB_COMP_MANAGED == yes" # Environmental variable to select between managed (esp-usb) and native (esp-idf) USB Component + - if: "idf_version >=5.4" # Use managed component only for 5.4 and above diff --git a/host/class/uac/usb_host_uac/examples/audio_player/main/main.c b/host/class/uac/usb_host_uac/examples/audio_player/main/main.c new file mode 100644 index 00000000..32c96a06 --- /dev/null +++ b/host/class/uac/usb_host_uac/examples/audio_player/main/main.c @@ -0,0 +1,461 @@ +/* + * SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include "freertos/FreeRTOS.h" +#include "freertos/event_groups.h" +#include "freertos/task.h" +#include "freertos/queue.h" +#include "esp_err.h" +#include "esp_log.h" +#include "usb/usb_host.h" +#include "usb/uac_host.h" +#include +#include + +static const char *TAG = "usb_audio_player"; + +#define USB_HOST_TASK_PRIORITY 5 +#define UAC_TASK_PRIORITY 5 +#define USER_TASK_PRIORITY 2 + +static QueueHandle_t s_event_queue = NULL; +static uac_host_device_handle_t s_spk_dev_handle = NULL; +static uac_host_device_handle_t s_mic_dev_handle = NULL; +static void uac_device_callback(uac_host_device_handle_t uac_device_handle, const uac_host_device_event_t event, void *arg); + +// WAV playback resources +static TaskHandle_t s_play_task_handle = NULL; +extern const uint8_t wav_8kfile_start[] asm("_binary_output_8k_wav_start"); +extern const uint8_t wav_8kfile_end[] asm("_binary_output_8k_wav_end"); +extern const uint8_t wav_16kfile_start[] asm("_binary_output_16k_wav_start"); +extern const uint8_t wav_16kfile_end[] asm("_binary_output_16k_wav_end"); + +// MIC recording resources +static uint8_t *s_mic_record_buf = NULL; +static size_t s_mic_record_buf_size = 0; // total capacity +static size_t s_mic_record_wr = 0; // ring write index +static bool is_playing_back = false; +static bool s_mic_recording = false; // recording state + + +static bool parse_wav_header(const uint8_t *buf, size_t len, uint32_t *sample_rate, + uint8_t *num_channels, uint8_t *bits_per_sample, + const uint8_t **data_ptr, size_t *data_size) +{ + if (len < 44) { + return false; + } + if (memcmp(buf + 0, "RIFF", 4) != 0 || memcmp(buf + 8, "WAVE", 4) != 0) { + return false; + } + const uint8_t *p = buf + 12; + const uint8_t *end = buf + len; + uint16_t audio_format = 0; + bool have_fmt = false; + bool have_data = false; + while (p + 8 <= end) { + const uint8_t *chunk_id_ptr = p; p += 4; + uint32_t chunk_size = *(const uint32_t *)p; p += 4; + if (p + chunk_size > end) { + return false; + } + if (memcmp(chunk_id_ptr, "fmt ", 4) == 0) { + if (chunk_size < 16) { + return false; + } + audio_format = *(const uint16_t *)(p + 0); + *num_channels = *(const uint16_t *)(p + 2); + *sample_rate = *(const uint32_t *)(p + 4); + *bits_per_sample = *(const uint16_t *)(p + 14); + have_fmt = true; + } else if (memcmp(chunk_id_ptr, "data", 4) == 0) { + *data_ptr = p; + *data_size = chunk_size; + have_data = true; + } + p += chunk_size + (chunk_size & 1); // chunks are word aligned + if (have_fmt && have_data) { + break; + } + } + if (!have_fmt || !have_data) { + return false; + } + if (audio_format != 1) { // PCM only + return false; + } + return true; +} + +static esp_err_t get_wav_data(int req_sample_freq, const uint8_t **pcm_ptr, size_t *pcm_size, uint32_t *rate, uint8_t *ch, uint8_t *bits) +{ + const uint8_t *s_wav_data = NULL; + const uint8_t *s_wav_data_end = NULL; + if (req_sample_freq == 8000) { + s_wav_data = wav_8kfile_start; + s_wav_data_end = wav_8kfile_end; + } else if (req_sample_freq == 16000) { + s_wav_data = wav_16kfile_start; + s_wav_data_end = wav_16kfile_end; + } else { + ESP_LOGE(TAG, "Unsupported sample rate: %u", req_sample_freq); + return ESP_FAIL; + } + const uint8_t *wav_start = s_wav_data; + const size_t wav_size = (size_t)(s_wav_data_end - s_wav_data); + if (!parse_wav_header(wav_start, wav_size, rate, ch, bits, pcm_ptr, pcm_size)) { + ESP_LOGE(TAG, "Failed to parse embedded WAV file"); + return ESP_FAIL; + } + return ESP_OK; +} + +typedef struct { + const uint8_t *pcm_ptr; + size_t pcm_size; + bool is_loop; + void (*complete_cb)(void); +} player_config_t; + + +static void pcm_play_task(void *arg) +{ + + player_config_t *config = (player_config_t *)arg; + size_t byte_offset = 0; + const size_t chunk_bytes = 2048; + ESP_LOGI(TAG, "PCM play task started"); + while (true) { + if (s_spk_dev_handle == NULL) { + break; + } + if (config->pcm_ptr == NULL || config->pcm_size == 0) { + vTaskDelay(10); + continue; + } + size_t remaining = config->pcm_size - byte_offset; + size_t bytes_to_write = remaining > chunk_bytes ? chunk_bytes : remaining; + const uint8_t *buf = config->pcm_ptr + byte_offset; + ESP_LOGI(TAG, "Writing %d bytes", bytes_to_write); + esp_err_t ret = uac_host_device_write(s_spk_dev_handle, (void *)buf, bytes_to_write, 1000); + if (ret != ESP_OK) { + ESP_LOGE(TAG, "uac_host_device_write failed: %s", esp_err_to_name(ret)); + break; + } + byte_offset += bytes_to_write; + if (byte_offset >= config->pcm_size) { + if (!config->is_loop) { + ESP_LOGI(TAG, "PCM playback completed"); + if (config->complete_cb) { + config->complete_cb(); + } + break; + } + byte_offset = 0; // loop + } + } + s_play_task_handle = NULL; + is_playing_back = false; + vTaskDelete(NULL); +} + +static void start_pcm_playback(player_config_t *config) +{ + static player_config_t s_player_config; + s_player_config = *config; + if (s_play_task_handle == NULL) { + BaseType_t ret = xTaskCreatePinnedToCore(pcm_play_task, "pcm_play", 4096, (void *)&s_player_config, USER_TASK_PRIORITY, &s_play_task_handle, 0); + if (ret != pdTRUE) { + ESP_LOGE(TAG, "Failed to create PCM play task"); + } + } +} + +/** + * @brief event group + * + * APP_EVENT - General control event + * UAC_DRIVER_EVENT - UAC Host Driver event, such as device connection + * UAC_DEVICE_EVENT - UAC Host Device event, such as rx/tx completion, device disconnection + */ +typedef enum { + APP_EVENT = 0, + UAC_DRIVER_EVENT, + UAC_DEVICE_EVENT, +} event_group_t; + +/** + * @brief event queue + * + * This event is used for delivering the UAC Host event from callback to the uac_lib_task + */ +typedef struct { + event_group_t event_group; + union { + struct { + uint8_t addr; + uint8_t iface_num; + uac_host_driver_event_t event; + void *arg; + } driver_evt; + struct { + uac_host_device_handle_t handle; + uac_host_driver_event_t event; + void *arg; + } device_evt; + }; +} s_event_queue_t; + +// removed audio_player dependent code + +static void uac_device_callback(uac_host_device_handle_t uac_device_handle, const uac_host_device_event_t event, void *arg) +{ + if (event == UAC_HOST_DRIVER_EVENT_DISCONNECTED) { + if (uac_device_handle == s_spk_dev_handle) { + s_spk_dev_handle = NULL; + } + if (uac_device_handle == s_mic_dev_handle) { + s_mic_dev_handle = NULL; + if (s_mic_record_buf) { + free(s_mic_record_buf); + s_mic_record_buf = NULL; + } + s_mic_record_buf_size = 0; + s_mic_record_wr = 0; + } + ESP_LOGI(TAG, "UAC Device disconnected"); + ESP_ERROR_CHECK(uac_host_device_close(uac_device_handle)); + return; + } + // Send uac device event to the event queue + s_event_queue_t evt_queue = { + .event_group = UAC_DEVICE_EVENT, + .device_evt.handle = uac_device_handle, + .device_evt.event = event, + .device_evt.arg = arg + }; + // should not block here + xQueueSend(s_event_queue, &evt_queue, 0); +} + +static void uac_host_lib_callback(uint8_t addr, uint8_t iface_num, const uac_host_driver_event_t event, void *arg) +{ + // Send uac driver event to the event queue + s_event_queue_t evt_queue = { + .event_group = UAC_DRIVER_EVENT, + .driver_evt.addr = addr, + .driver_evt.iface_num = iface_num, + .driver_evt.event = event, + .driver_evt.arg = arg + }; + xQueueSend(s_event_queue, &evt_queue, 0); +} + +static void mic_palyback_done_cb(void) +{ + uac_host_device_resume(s_mic_dev_handle); + s_mic_recording = true; + s_mic_record_wr = 0; +} + +/** + * @brief Start USB Host install and handle common USB host library events while app pin not low + * + * @param[in] arg Not used + */ +static void usb_lib_task(void *arg) +{ + const usb_host_config_t host_config = { + .skip_phy_setup = false, + .intr_flags = ESP_INTR_FLAG_LEVEL1, + }; + + ESP_ERROR_CHECK(usb_host_install(&host_config)); + ESP_LOGI(TAG, "USB Host installed"); + xTaskNotifyGive(arg); + + while (true) { + uint32_t event_flags; + usb_host_lib_handle_events(portMAX_DELAY, &event_flags); + // In this example, there is only one client registered + // So, once we deregister the client, this call must succeed with ESP_OK + if (event_flags & USB_HOST_LIB_EVENT_FLAGS_NO_CLIENTS) { + ESP_ERROR_CHECK(usb_host_device_free_all()); + break; + } + } + + ESP_LOGI(TAG, "USB Host shutdown"); + // Clean up USB Host + vTaskDelay(10); // Short delay to allow clients clean-up + ESP_ERROR_CHECK(usb_host_uninstall()); + vTaskDelete(NULL); +} + +static void uac_lib_task(void *arg) +{ + ulTaskNotifyTake(pdTRUE, portMAX_DELAY); + uac_host_driver_config_t uac_config = { + .create_background_task = true, + .task_priority = UAC_TASK_PRIORITY, + .stack_size = 4096, + .core_id = 0, + .callback = uac_host_lib_callback, + .callback_arg = NULL + }; + + ESP_ERROR_CHECK(uac_host_install(&uac_config)); + ESP_LOGI(TAG, "UAC Class Driver installed"); + s_event_queue_t evt_queue = {0}; + while (1) { + if (xQueueReceive(s_event_queue, &evt_queue, portMAX_DELAY)) { + if (UAC_DRIVER_EVENT == evt_queue.event_group) { + uac_host_driver_event_t event = evt_queue.driver_evt.event; + uint8_t addr = evt_queue.driver_evt.addr; + uint8_t iface_num = evt_queue.driver_evt.iface_num; + switch (event) { + case UAC_HOST_DRIVER_EVENT_TX_CONNECTED: { + uac_host_dev_info_t dev_info; + uac_host_device_handle_t uac_device_handle = NULL; + const uac_host_device_config_t dev_config = { + .addr = addr, + .iface_num = iface_num, + .buffer_size = 16000, + .buffer_threshold = 4000, + .callback = uac_device_callback, + .callback_arg = NULL, + }; + ESP_ERROR_CHECK(uac_host_device_open(&dev_config, &uac_device_handle)); + ESP_ERROR_CHECK(uac_host_get_device_info(uac_device_handle, &dev_info)); + ESP_LOGI(TAG, "UAC Device connected: SPK"); + uac_host_printf_device_param(uac_device_handle); + uac_host_dev_alt_param_t iface_alt_params; + uac_host_get_device_alt_param(uac_device_handle, 1, &iface_alt_params); + // Parse embedded WAV data + player_config_t player_config = {0}; + uac_host_stream_config_t stm_config = {0}; + get_wav_data(iface_alt_params.sample_freq[0], &player_config.pcm_ptr, &player_config.pcm_size, + &stm_config.sample_freq, &stm_config.channels, &stm_config.bit_resolution); + + ESP_LOGI(TAG, "Start UAC speaker with %"PRIu32" Hz, %u bits, %s (PCM bytes=%u)", + stm_config.sample_freq, stm_config.bit_resolution, stm_config.channels == 1 ? "Mono" : "Stereo", player_config.pcm_size); + ESP_ERROR_CHECK(uac_host_device_start(uac_device_handle, &stm_config)); + s_spk_dev_handle = uac_device_handle; + uac_host_device_set_volume(uac_device_handle, 50); // set volume + uac_host_device_set_mute(uac_device_handle, false); // set mute off +#ifndef CONFIG_EXAMPLE_MIC_PLAYBACK + player_config.is_loop = true; + start_pcm_playback(&player_config); // start playback in loop +#endif + break; + } + case UAC_HOST_DRIVER_EVENT_RX_CONNECTED: { + ESP_LOGI(TAG, "UAC Device connected: MIC"); + // Open MIC device and start recording into a ring buffer +#ifdef CONFIG_EXAMPLE_MIC_PLAYBACK + uac_host_device_handle_t uac_device_handle = NULL; + const uint32_t rx_buffer_size = 19200; // internal driver buffer + const uac_host_device_config_t dev_config = { + .addr = addr, + .iface_num = iface_num, + .buffer_size = rx_buffer_size, + .callback = uac_device_callback, + .callback_arg = NULL, + }; + ESP_ERROR_CHECK(uac_host_device_open(&dev_config, &uac_device_handle)); + uac_host_dev_alt_param_t mic_alt_params; + ESP_ERROR_CHECK(uac_host_get_device_alt_param(uac_device_handle, 1, &mic_alt_params)); + const uac_host_stream_config_t mic_stream_config = { + .channels = mic_alt_params.channels, + .bit_resolution = mic_alt_params.bit_resolution, + .sample_freq = mic_alt_params.sample_freq[0], + }; + ESP_LOGI(TAG, "Start UAC microphone with %"PRIu32" Hz, %u bits, channels=%u", + mic_stream_config.sample_freq, + mic_stream_config.bit_resolution, + mic_stream_config.channels); + ESP_ERROR_CHECK(uac_host_device_start(uac_device_handle, &mic_stream_config)); + s_mic_dev_handle = uac_device_handle; + if (s_mic_record_buf == NULL) { + // Allocate a linear buffer to store ~5 seconds of PCM; round down to threshold multiple + uint32_t bytes_per_sec = mic_stream_config.sample_freq * mic_stream_config.channels * (mic_stream_config.bit_resolution / 8); + s_mic_record_buf_size = bytes_per_sec * 5; + s_mic_record_buf = (uint8_t *)calloc(1, s_mic_record_buf_size); + if (s_mic_record_buf == NULL) { + ESP_LOGE(TAG, "Failed to allocate MIC record buffer (%u bytes)", (unsigned)s_mic_record_buf_size); + } + } + // Initialize recording state + s_mic_record_wr = 0; + s_mic_recording = true; +#endif + break; + } + default: + break; + } + } else if (UAC_DEVICE_EVENT == evt_queue.event_group) { + uac_host_device_event_t event = evt_queue.device_evt.event; + switch (event) { + case UAC_HOST_DRIVER_EVENT_DISCONNECTED: + ESP_LOGI(TAG, "UAC Device disconnected"); + if (s_play_task_handle) { + // let the task exit by noticing NULL handle + s_spk_dev_handle = NULL; + } + break; + case UAC_HOST_DEVICE_EVENT_RX_DONE: + if (s_mic_dev_handle && s_mic_record_buf && s_mic_recording) { + uint32_t rx_size = 0; + esp_err_t ret = ESP_OK; + uac_host_device_read(s_mic_dev_handle, s_mic_record_buf + s_mic_record_wr, s_mic_record_buf_size - s_mic_record_wr, &rx_size, 0); + s_mic_record_wr += rx_size; + if (s_mic_record_wr >= s_mic_record_buf_size) { + s_mic_recording = false; + // Stop/suspend microphone streaming before playback + uac_host_device_suspend(s_mic_dev_handle); + // Prepare playback of the recorded PCM + player_config_t player_config = { + .pcm_ptr = s_mic_record_buf, + .pcm_size = s_mic_record_buf_size, + .complete_cb = mic_palyback_done_cb, + }; + start_pcm_playback(&player_config); + } + } + break; + case UAC_HOST_DEVICE_EVENT_TX_DONE: + break; + case UAC_HOST_DEVICE_EVENT_TRANSFER_ERROR: + break; + default: + break; + } + } else if (APP_EVENT == evt_queue.event_group) { + break; + } + } + } + + ESP_LOGI(TAG, "UAC Driver uninstall"); + ESP_ERROR_CHECK(uac_host_uninstall()); +} + +void app_main(void) +{ + s_event_queue = xQueueCreate(10, sizeof(s_event_queue_t)); + assert(s_event_queue != NULL); + + static TaskHandle_t uac_task_handle = NULL; + BaseType_t ret = xTaskCreatePinnedToCore(uac_lib_task, "uac_events", 4096, NULL, + USER_TASK_PRIORITY, &uac_task_handle, 0); + assert(ret == pdTRUE); + ret = xTaskCreatePinnedToCore(usb_lib_task, "usb_events", 4096, (void *)uac_task_handle, + USB_HOST_TASK_PRIORITY, NULL, 0); + assert(ret == pdTRUE); + +} diff --git a/host/class/uac/usb_host_uac/examples/audio_player/partitions.csv b/host/class/uac/usb_host_uac/examples/audio_player/partitions.csv new file mode 100644 index 00000000..5b6a6734 --- /dev/null +++ b/host/class/uac/usb_host_uac/examples/audio_player/partitions.csv @@ -0,0 +1,5 @@ +# Note: if you have increased the bootloader size, make sure to update the offsets to avoid overlap +# Name, Type, SubType, Offset, Size, Flags +nvs, data, nvs, 0x9000, 24k, +phy_init, data, phy, 0xf000, 4k, +factory, app, factory, , 2M, diff --git a/host/class/uac/usb_host_uac/examples/audio_player/sdkconfig.defaults b/host/class/uac/usb_host_uac/examples/audio_player/sdkconfig.defaults new file mode 100644 index 00000000..3a6eb136 --- /dev/null +++ b/host/class/uac/usb_host_uac/examples/audio_player/sdkconfig.defaults @@ -0,0 +1,8 @@ +# This file was generated using idf.py save-defconfig. It can be edited manually. +# Espressif IoT Development Framework (ESP-IDF) Project Minimal Configuration +# +CONFIG_ESPTOOLPY_FLASHMODE_QIO=y +CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_FREERTOS_HZ=1000 +CONFIG_USB_HOST_CONTROL_TRANSFER_MAX_SIZE=2048 diff --git a/host/class/uac/usb_host_uac/examples/audio_player/spiffs/new_epic.mp3 b/host/class/uac/usb_host_uac/examples/audio_player/spiffs/new_epic.mp3 new file mode 100644 index 00000000..b5be07f3 Binary files /dev/null and b/host/class/uac/usb_host_uac/examples/audio_player/spiffs/new_epic.mp3 differ diff --git a/host/class/uac/usb_host_uac/examples/audio_player/spiffs/output_16k.wav b/host/class/uac/usb_host_uac/examples/audio_player/spiffs/output_16k.wav new file mode 100644 index 00000000..c34f799e Binary files /dev/null and b/host/class/uac/usb_host_uac/examples/audio_player/spiffs/output_16k.wav differ diff --git a/host/class/uac/usb_host_uac/examples/audio_player/spiffs/output_8k.wav b/host/class/uac/usb_host_uac/examples/audio_player/spiffs/output_8k.wav new file mode 100644 index 00000000..994a7d47 Binary files /dev/null and b/host/class/uac/usb_host_uac/examples/audio_player/spiffs/output_8k.wav differ diff --git a/host/class/uac/usb_host_uac/idf_component.yml b/host/class/uac/usb_host_uac/idf_component.yml index b30bc118..31ef36ee 100644 --- a/host/class/uac/usb_host_uac/idf_component.yml +++ b/host/class/uac/usb_host_uac/idf_component.yml @@ -1,5 +1,5 @@ ## IDF Component Manager Manifest File -version: "1.3.2" +version: "1.3.3" description: USB Host UAC driver url: https://github.com/espressif/esp-usb/tree/master/host/class/uac/usb_host_uac dependencies: diff --git a/host/class/uac/usb_host_uac/uac_host.c b/host/class/uac/usb_host_uac/uac_host.c index a5736128..219cbba2 100644 --- a/host/class/uac/usb_host_uac/uac_host.c +++ b/host/class/uac/usb_host_uac/uac_host.c @@ -16,6 +16,7 @@ #include "sdkconfig.h" #include "esp_log.h" #include "esp_check.h" +#include "esp_memory_utils.h" #include "freertos/FreeRTOS.h" #include "freertos/task.h" #include "freertos/semphr.h" @@ -739,6 +740,11 @@ static esp_err_t uac_host_interface_add(uac_device_t *uac_device, uint8_t iface_ usb_host_get_active_config_descriptor(uac_device->dev_hdl, &config_desc); UAC_GOTO_ON_FALSE(config_desc, ESP_ERR_INVALID_STATE, "No active configuration descriptor"); + // Query device speed once to decide whether to force bInterval for Full-Speed + usb_device_info_t dev_info_local; + ESP_RETURN_ON_ERROR(usb_host_device_info(uac_device->dev_hdl, &dev_info_local), TAG, "Failed to get device info"); + ESP_RETURN_ON_FALSE(dev_info_local.speed != USB_SPEED_LOW, ESP_ERR_NOT_SUPPORTED, TAG, "Low-Speed device not supported"); + const bool is_full_speed = (dev_info_local.speed == USB_SPEED_FULL); const size_t total_length = config_desc->wTotalLength; int iface_alt_offset = 0; int iface_alt_idx = 0; @@ -803,6 +809,21 @@ static esp_err_t uac_host_interface_add(uac_device_t *uac_device, uint8_t iface_ iface_alt->ep_addr = ep_desc->bEndpointAddress; iface_alt->ep_mps = ep_desc->wMaxPacketSize; iface_alt->ep_attr = ep_desc->bmAttributes; + if (is_full_speed && ep_desc->bInterval != 1) { + // Full-Speed isochronous endpoints must have bInterval = 1 + uint8_t *_bInterval = (uint8_t *) & (ep_desc->bInterval); + // Check if we can modify the bInterval value + if (esp_ptr_in_dram(_bInterval) || esp_ptr_in_diram_dram(_bInterval) +#if CONFIG_SPIRAM + || esp_ptr_in_psram(_bInterval) +#endif + ) { + ESP_LOGW(TAG, "UAC Full-Speed device, Endpoint %d, bInterval %d, set to 1", USB_EP_DESC_GET_EP_NUM(ep_desc), ep_desc->bInterval); + *_bInterval = 1; + } else { + ESP_LOGW(TAG, "UAC Full-Speed device, Endpoint %d, bInterval %d, can't set to 1", USB_EP_DESC_GET_EP_NUM(ep_desc), ep_desc->bInterval); + } + } iface_alt->interval = ep_desc->bInterval; uac_iface->dev_info.type = (ep_desc->bEndpointAddress & UAC_EP_DIR_IN) ? UAC_STREAM_RX : UAC_STREAM_TX; uac_ac_feature_unit_desc_t *feature_unit_desc = _uac_host_device_find_feature_unit((uint8_t *)uac_device->cs_ac_desc, @@ -823,7 +844,7 @@ static esp_err_t uac_host_interface_add(uac_device_t *uac_device, uint8_t iface_ ESP_LOGD(TAG, "UAC %s Feature Unit ID %d, Volume Ch Map %02X, Mute Ch Map %02X", uac_iface->dev_info.type == UAC_STREAM_RX ? "RX" : "TX", feature_unit_desc->bUnitID, iface_alt->vol_ch_map, iface_alt->mute_ch_map); } - ESP_LOGD(TAG, "UAC Endpoint 0x%02X, Max Packet Size %d, Attributes 0x%02X, Interval %d", ep_desc->bEndpointAddress, ep_desc->wMaxPacketSize, ep_desc->bmAttributes, ep_desc->bInterval); + ESP_LOGD(TAG, "UAC Endpoint 0x%02X, Max Packet Size %d, Attributes 0x%02X, Interval %d", USB_EP_DESC_GET_EP_NUM(ep_desc), ep_desc->wMaxPacketSize, ep_desc->bmAttributes, ep_desc->bInterval); break; } case UAC_CS_ENDPOINT: { @@ -1305,9 +1326,9 @@ static esp_err_t uac_host_interface_suspend(uac_iface_t *iface) memcpy(&uac_request, &usb_request, sizeof(usb_setup_packet_t)); esp_err_t ret = uac_cs_request_set(iface->parent, &uac_request); if (ret != ESP_OK) { - ESP_LOGW(TAG, "Set Interface %d-%d Failed", iface->dev_info.iface_num, 0); + ESP_LOGW(TAG, "Suspend Interface %d-%d Failed", iface->dev_info.iface_num, 0); } else { - ESP_LOGI(TAG, "Set Interface %d-%d", iface->dev_info.iface_num, 0); + ESP_LOGI(TAG, "Suspend Interface %d-%d", iface->dev_info.iface_num, 0); } uint8_t ep_addr = iface->iface_alt[iface->cur_alt].ep_addr; @@ -1353,16 +1374,16 @@ static esp_err_t uac_host_interface_resume(uac_iface_t *iface) uac_cs_request_t uac_request = {0}; memcpy(&uac_request, &usb_request, sizeof(usb_setup_packet_t)); UAC_RETURN_ON_ERROR(uac_cs_request_set(iface->parent, &uac_request), "Unable to set Interface alternate"); - ESP_LOGI(TAG, "Set Interface %d-%d", iface->dev_info.iface_num, iface->cur_alt + 1); + ESP_LOGI(TAG, "Resume Interface %d-%d", iface->dev_info.iface_num, iface->cur_alt + 1); // Set endpoint frequency control if (iface->iface_alt[iface->cur_alt].freq_ctrl_supported) { - ESP_LOGI(TAG, "Set EP %02X frequency %"PRIu32, iface->iface_alt[iface->cur_alt].ep_addr, iface->iface_alt[iface->cur_alt].cur_sampling_freq); + ESP_LOGI(TAG, "Set EP %d frequency %"PRIu32, iface->iface_alt[iface->cur_alt].ep_addr & USB_B_ENDPOINT_ADDRESS_EP_NUM_MASK, iface->iface_alt[iface->cur_alt].cur_sampling_freq); UAC_RETURN_ON_ERROR(uac_cs_request_set_ep_frequency(iface, iface->iface_alt[iface->cur_alt].ep_addr, iface->iface_alt[iface->cur_alt].cur_sampling_freq), "Unable to set endpoint frequency"); } // for RX, we just submit all the transfers if (iface->dev_info.type == UAC_STREAM_RX) { - assert(iface->iface_alt[iface->cur_alt].ep_addr & 0x80); + assert(iface->iface_alt[iface->cur_alt].ep_addr & USB_B_ENDPOINT_ADDRESS_EP_DIR_MASK); for (int i = 0; i < iface->xfer_num; i++) { assert(iface->free_xfer_list[i]); iface->free_xfer_list[i]->device_handle = iface->parent->dev_hdl; @@ -1381,7 +1402,7 @@ static esp_err_t uac_host_interface_resume(uac_iface_t *iface) UAC_RETURN_ON_ERROR(usb_host_transfer_submit(iface->xfer_list[i]), "Unable to submit RX transfer"); } } else if (iface->dev_info.type == UAC_STREAM_TX) { - assert(!(iface->iface_alt[iface->cur_alt].ep_addr & 0x80)); + assert(!(iface->iface_alt[iface->cur_alt].ep_addr & USB_B_ENDPOINT_ADDRESS_EP_DIR_MASK)); // for TX, we submit the first transfer with data 0 to make the speaker quiet for (int i = 0; i < iface->xfer_num; i++) { assert(iface->free_xfer_list[i]); diff --git a/host/class/uvc/usb_host_uvc/CHANGELOG.md b/host/class/uvc/usb_host_uvc/CHANGELOG.md index b05a0a91..cfc8ce1f 100644 --- a/host/class/uvc/usb_host_uvc/CHANGELOG.md +++ b/host/class/uvc/usb_host_uvc/CHANGELOG.md @@ -1,3 +1,9 @@ +## 2.3.2 + +- Added SOI check to prevent output of corrupted MJPEG frames +- Moved `usb_types_uvc.h` to `private_include` directory +- Fixed bulk transfer EOF handling to improve robustness + ## 2.3.1 - Added support for ESP32-H4 diff --git a/host/class/uvc/usb_host_uvc/examples/basic_uvc_stream/main/basic_uvc_stream.c b/host/class/uvc/usb_host_uvc/examples/basic_uvc_stream/main/basic_uvc_stream.c index 4b2456d6..0f79a19e 100644 --- a/host/class/uvc/usb_host_uvc/examples/basic_uvc_stream/main/basic_uvc_stream.c +++ b/host/class/uvc/usb_host_uvc/examples/basic_uvc_stream/main/basic_uvc_stream.c @@ -36,10 +36,20 @@ #define EXAMPLE_NUMBER_OF_STREAMS (1) // This example shows how to control multiple UVC streams. Set this to 1 if you camera offers only 1 stream #define EXAMPLE_USE_SDCARD (0) // SD card on P4 evaluation board will be initialized +#define UVC_DESC_DWFRAMEINTERVAL_TO_FPS(dwFrameInterval) (((dwFrameInterval) != 0) ? 10000000 / ((float)(dwFrameInterval)) : 0) + static const char *TAG = "UVC example"; static QueueHandle_t rx_frames_queue[EXAMPLE_NUMBER_OF_STREAMS]; static bool dev_connected = false; +static const char *FORMAT_STR[] = { + "FORMAT_UNDEFINED", + "FORMAT_MJPEG", + "FORMAT_YUY2", + "FORMAT_H264", + "FORMAT_H265", +}; + bool frame_callback(const uvc_host_frame_t *frame, void *user_ctx) { assert(frame); @@ -279,6 +289,27 @@ void app_init_sdcard(void) } #endif // EXAMPLE_USE_SDCARD +static void uvc_event_cb(const uvc_host_driver_event_data_t *event, void *user_ctx) +{ + switch (event->type) { + case UVC_HOST_DRIVER_EVENT_DEVICE_CONNECTED: { + ESP_LOGI(TAG, "Device connected, addr: %d", event->device_connected.dev_addr); + + size_t list_size = event->device_connected.frame_info_num; + uvc_host_frame_info_t *frame_info = (uvc_host_frame_info_t *)calloc(list_size, sizeof(uvc_host_frame_info_t)); + assert(frame_info); + uvc_host_get_frame_list(event->device_connected.dev_addr, event->device_connected.uvc_stream_index, (uvc_host_frame_info_t (*)[])frame_info, &list_size); + for (int i = 0; i < list_size; i++) { + ESP_LOGI(TAG, "Camera format: %s %d*%d@%.1ffps", FORMAT_STR[frame_info[i].format], frame_info[i].h_res, frame_info[i].v_res, UVC_DESC_DWFRAMEINTERVAL_TO_FPS(frame_info[i].default_interval)); + } + free(frame_info); + break; + } + default: + break; + } +} + /** * @brief Main application */ @@ -311,6 +342,7 @@ void app_main(void) .driver_task_priority = EXAMPLE_USB_HOST_PRIORITY + 1, .xCoreID = tskNO_AFFINITY, .create_background_task = true, + .event_cb = uvc_event_cb, }; ESP_ERROR_CHECK(uvc_host_install(&uvc_driver_config)); @@ -321,5 +353,5 @@ void app_main(void) vTaskDelay(pdMS_TO_TICKS(1000)); task_created = xTaskCreatePinnedToCore(frame_handling_task, "h265_handling", 4096, (void *)&stream_h265_config, EXAMPLE_USB_HOST_PRIORITY - 3, NULL, tskNO_AFFINITY); assert(task_created == pdTRUE); -#endif // EXAMPLE_USE_SDCARD +#endif // EXAMPLE_NUMBER_OF_STREAMS > 1 } diff --git a/host/class/uvc/usb_host_uvc/examples/basic_uvc_stream/sdkconfig.defaults b/host/class/uvc/usb_host_uvc/examples/basic_uvc_stream/sdkconfig.defaults index 85f51400..6332811c 100644 --- a/host/class/uvc/usb_host_uvc/examples/basic_uvc_stream/sdkconfig.defaults +++ b/host/class/uvc/usb_host_uvc/examples/basic_uvc_stream/sdkconfig.defaults @@ -1,12 +1,9 @@ # This file was generated using idf.py save-defconfig. It can be edited manually. # Espressif IoT Development Framework (ESP-IDF) 5.4.0 Project Minimal Configuration # -CONFIG_IDF_TARGET="esp32p4" -CONFIG_ESP32P4_REV_MIN_0=y -CONFIG_RTC_CLK_SRC_EXT_CRYS=y -CONFIG_RTC_CLK_CAL_CYCLES=1024 -CONFIG_SPIRAM=y -CONFIG_SPIRAM_SPEED_200M=y +CONFIG_FREERTOS_HZ=1000 +CONFIG_COMPILER_OPTIMIZATION_PERF=y + CONFIG_LOG_COLORS=y CONFIG_USB_HOST_CONTROL_TRANSFER_MAX_SIZE=4096 CONFIG_USB_HOST_HW_BUFFER_BIAS_IN=y diff --git a/host/class/uvc/usb_host_uvc/examples/basic_uvc_stream/sdkconfig.defaults.esp32p4 b/host/class/uvc/usb_host_uvc/examples/basic_uvc_stream/sdkconfig.defaults.esp32p4 new file mode 100644 index 00000000..7d9fa4c9 --- /dev/null +++ b/host/class/uvc/usb_host_uvc/examples/basic_uvc_stream/sdkconfig.defaults.esp32p4 @@ -0,0 +1,6 @@ + +CONFIG_ESP32P4_REV_MIN_0=y +CONFIG_RTC_CLK_SRC_EXT_CRYS=y +CONFIG_RTC_CLK_CAL_CYCLES=1024 +CONFIG_SPIRAM=y +CONFIG_SPIRAM_SPEED_200M=y diff --git a/host/class/uvc/usb_host_uvc/examples/basic_uvc_stream/sdkconfig.defaults.esp32s2 b/host/class/uvc/usb_host_uvc/examples/basic_uvc_stream/sdkconfig.defaults.esp32s2 new file mode 100644 index 00000000..6b43e22a --- /dev/null +++ b/host/class/uvc/usb_host_uvc/examples/basic_uvc_stream/sdkconfig.defaults.esp32s2 @@ -0,0 +1,6 @@ + +CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y +CONFIG_SPIRAM=y +CONFIG_SPIRAM_SPEED_80M=y +CONFIG_SPIRAM_FETCH_INSTRUCTIONS=y +CONFIG_SPIRAM_RODATA=y diff --git a/host/class/uvc/usb_host_uvc/examples/basic_uvc_stream/sdkconfig.defaults.esp32s3 b/host/class/uvc/usb_host_uvc/examples/basic_uvc_stream/sdkconfig.defaults.esp32s3 new file mode 100644 index 00000000..1ca2a2b1 --- /dev/null +++ b/host/class/uvc/usb_host_uvc/examples/basic_uvc_stream/sdkconfig.defaults.esp32s3 @@ -0,0 +1,7 @@ + +CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y +CONFIG_SPIRAM=y +CONFIG_SPIRAM_MODE_OCT=y +CONFIG_SPIRAM_SPEED_80M=y +CONFIG_SPIRAM_FETCH_INSTRUCTIONS=y +CONFIG_SPIRAM_RODATA=y diff --git a/host/class/uvc/usb_host_uvc/host_test/main/streaming/test_streaming_helpers.hpp b/host/class/uvc/usb_host_uvc/host_test/main/streaming/test_streaming_helpers.hpp index cb113136..8aeb4755 100644 --- a/host/class/uvc/usb_host_uvc/host_test/main/streaming/test_streaming_helpers.hpp +++ b/host/class/uvc/usb_host_uvc/host_test/main/streaming/test_streaming_helpers.hpp @@ -12,7 +12,7 @@ #include #include "usb/usb_types_stack.h" -#include "usb/usb_types_uvc.h" +#include "usb_types_uvc.h" extern "C" { #include "Mockusb_host.h" diff --git a/host/class/uvc/usb_host_uvc/idf_component.yml b/host/class/uvc/usb_host_uvc/idf_component.yml index 216ac52f..e8b0fb3b 100644 --- a/host/class/uvc/usb_host_uvc/idf_component.yml +++ b/host/class/uvc/usb_host_uvc/idf_component.yml @@ -1,5 +1,5 @@ ## IDF Component Manager Manifest File -version: "2.3.1" +version: "2.3.2" description: USB Host UVC driver url: https://github.com/espressif/esp-usb/tree/master/host/class/uvc/usb_host_uvc dependencies: diff --git a/host/class/uvc/usb_host_uvc/include/esp_private/uvc_control.h b/host/class/uvc/usb_host_uvc/include/esp_private/uvc_control.h index bfd5e1e5..ffb9955b 100644 --- a/host/class/uvc/usb_host_uvc/include/esp_private/uvc_control.h +++ b/host/class/uvc/usb_host_uvc/include/esp_private/uvc_control.h @@ -7,6 +7,7 @@ #pragma once #include "usb/uvc_host.h" +#include "usb_types_uvc.h" #ifdef __cplusplus extern "C" { diff --git a/host/class/uvc/usb_host_uvc/include/usb/uvc_host.h b/host/class/uvc/usb_host_uvc/include/usb/uvc_host.h index 4b443cd6..b9347859 100644 --- a/host/class/uvc/usb_host_uvc/include/usb/uvc_host.h +++ b/host/class/uvc/usb_host_uvc/include/usb/uvc_host.h @@ -8,7 +8,6 @@ #include #include "usb/usb_host.h" -#include "usb/usb_types_uvc.h" #include "esp_err.h" // Use this macros for opening a UVC stream with any VID or PID diff --git a/host/class/uvc/usb_host_uvc/include/usb/usb_types_uvc.h b/host/class/uvc/usb_host_uvc/private_include/usb_types_uvc.h similarity index 100% rename from host/class/uvc/usb_host_uvc/include/usb/usb_types_uvc.h rename to host/class/uvc/usb_host_uvc/private_include/usb_types_uvc.h diff --git a/host/class/uvc/usb_host_uvc/private_include/uvc_descriptors_priv.h b/host/class/uvc/usb_host_uvc/private_include/uvc_descriptors_priv.h index 07fe1702..e9b55fee 100644 --- a/host/class/uvc/usb_host_uvc/private_include/uvc_descriptors_priv.h +++ b/host/class/uvc/usb_host_uvc/private_include/uvc_descriptors_priv.h @@ -10,7 +10,7 @@ // So we include only files with USB specification definitions // This interface is also used in host_tests #include "usb/usb_types_ch9.h" -#include "usb/usb_types_uvc.h" +#include "usb_types_uvc.h" #define UVC_DESC_FPS_TO_DWFRAMEINTERVAL(fps) (((fps) != 0) ? 10000000.0f / (fps) : 0) #define UVC_DESC_DWFRAMEINTERVAL_TO_FPS(dwFrameInterval) (((dwFrameInterval) != 0) ? 10000000.0f / ((float)(dwFrameInterval)) : 0) diff --git a/host/class/uvc/usb_host_uvc/private_include/uvc_frame_priv.h b/host/class/uvc/usb_host_uvc/private_include/uvc_frame_priv.h index a8075989..02ea6db9 100644 --- a/host/class/uvc/usb_host_uvc/private_include/uvc_frame_priv.h +++ b/host/class/uvc/usb_host_uvc/private_include/uvc_frame_priv.h @@ -89,6 +89,17 @@ static inline void uvc_frame_reset(uvc_host_frame_t *frame) */ void uvc_frame_format_update(uvc_stream_t *uvc_stream, const uvc_host_stream_format_t *vs_format); +/** + * @brief Check if UVC payload header is valid + * + * @param[in] hdr Pointer to UVC payload header + * @param[in] packet_len Length of packet in bytes + * @return + * - true: Header is valid + * - false: Header is invalid + */ +bool uvc_frame_payload_header_validate(const uvc_payload_header_t *hdr, size_t packet_len); + #ifdef __cplusplus } #endif diff --git a/host/class/uvc/usb_host_uvc/private_include/uvc_types_priv.h b/host/class/uvc/usb_host_uvc/private_include/uvc_types_priv.h index 3581fee3..acf01815 100644 --- a/host/class/uvc/usb_host_uvc/private_include/uvc_types_priv.h +++ b/host/class/uvc/usb_host_uvc/private_include/uvc_types_priv.h @@ -12,6 +12,7 @@ #include "usb/usb_host.h" #include "usb/uvc_host.h" +#include "usb_types_uvc.h" #include "freertos/FreeRTOS.h" #include "freertos/queue.h" diff --git a/host/class/uvc/usb_host_uvc/uvc_bulk.c b/host/class/uvc/usb_host_uvc/uvc_bulk.c index 83e75a3b..639a62fd 100644 --- a/host/class/uvc/usb_host_uvc/uvc_bulk.c +++ b/host/class/uvc/usb_host_uvc/uvc_bulk.c @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD + * SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD * * SPDX-License-Identifier: Apache-2.0 */ @@ -81,49 +81,36 @@ void bulk_transfer_callback(usb_transfer_t *transfer) // Consequently, it becomes impossible to distinguish between the last data packet and the EoF header. // To address this, the hack discards all frames whose last data packet has the MPS size. switch (uvc_stream->single_thread.next_bulk_packet) { - case UVC_STREAM_BULK_PACKET_EOF: { - const uvc_payload_header_t *payload_header = (const uvc_payload_header_t *)payload; - uvc_stream->single_thread.next_bulk_packet = UVC_STREAM_BULK_PACKET_SOF; - - if (payload_header->bmHeaderInfo.end_of_frame) { - assert(payload_header->bmHeaderInfo.frame_id == uvc_stream->single_thread.current_frame_id); - if (payload_header->bmHeaderInfo.error) { - uvc_stream->single_thread.skip_current_frame = true; - } - - // Get the current frame being processed and clear it from the stream, - // so no more data is written to this frame after the end of frame - UVC_ENTER_CRITICAL(); // Enter critical section to safely check and modify the stream state. - uvc_host_frame_t *this_frame = uvc_stream->dynamic.current_frame; - uvc_stream->dynamic.current_frame = NULL; - - // Determine if we should invoke the frame callback: - // Only invoke the callback if streaming is active, a frame callback exists, - // and we have a valid frame to pass to the user. - const bool invoke_fb_callback = (uvc_stream->dynamic.streaming && uvc_stream->constant.frame_cb && this_frame && !uvc_stream->single_thread.skip_current_frame); - UVC_EXIT_CRITICAL(); - - bool return_frame = true; // Default to returning the frame in case streaming has been stopped - if (invoke_fb_callback) { - // Call the user's frame callback. If the callback returns false, - // we do not return the frame to the empty queue (i.e., the user wants to keep it for processing) - return_frame = uvc_stream->constant.frame_cb(this_frame, uvc_stream->constant.cb_arg); - } - if (return_frame) { - // If the user has processed the frame (or the stream is stopped), return it to the empty frame queue - uvc_host_frame_return(uvc_stream, this_frame); - } - break; - } - - __attribute__((fallthrough)); // Fall through! This is not EoF but SoF! - } case UVC_STREAM_BULK_PACKET_SOF: { const uvc_payload_header_t *payload_header = (const uvc_payload_header_t *)payload; // We detected start of new frame. Update Frame ID and start fetching this frame uvc_stream->single_thread.current_frame_id = payload_header->bmHeaderInfo.frame_id; uvc_stream->single_thread.skip_current_frame = payload_header->bmHeaderInfo.error; // Check for error flag + // Validate header before accessing it + if (!uvc_frame_payload_header_validate(payload_header, transfer->actual_num_bytes)) { + ESP_LOGD(TAG, "invalid UVC payload header, %02x, %02x, len:%d", payload[0], payload[1], transfer->actual_num_bytes); + uvc_stream->single_thread.skip_current_frame = true; + goto skip_sof; + } + payload_data += payload_header->bHeaderLength; // Pointer arithmetic! + payload_data_len -= payload_header->bHeaderLength; + + // Drop frame if device signals error in header + if (payload_header->bmHeaderInfo.error) { + ESP_LOGW(TAG, "frame error flag set"); + uvc_stream->single_thread.skip_current_frame = true; + goto skip_sof; + } + + // Check mjpeg frame start + if (uvc_stream->dynamic.vs_format.format == UVC_VS_FORMAT_MJPEG && + payload_data[0] != 0xff && payload_data[1] != 0xd8) { + // We received frame with invalid frame, skip this frame + uvc_stream->single_thread.skip_current_frame = true; + ESP_LOGW(TAG, "invalid MJPEG SOI"); + goto skip_sof; + } // Get free frame buffer for this new frame UVC_ENTER_CRITICAL(); @@ -149,17 +136,11 @@ void bulk_transfer_callback(usb_transfer_t *transfer) uvc_frame_reset(uvc_stream->dynamic.current_frame); UVC_EXIT_CRITICAL(); } - - payload_data += payload_header->bHeaderLength; // Pointer arithmetic! - payload_data_len -= payload_header->bHeaderLength; +skip_sof: uvc_stream->single_thread.next_bulk_packet = UVC_STREAM_BULK_PACKET_DATA; __attribute__((fallthrough)); // Fall through! There can be data after SoF! } case UVC_STREAM_BULK_PACKET_DATA: { - // We got short packet in data section, next packet is EoF - if (transfer->data_buffer_size > transfer->actual_num_bytes) { - uvc_stream->single_thread.next_bulk_packet = UVC_STREAM_BULK_PACKET_EOF; - } // Add received data to frame buffer if (!uvc_stream->single_thread.skip_current_frame) { uvc_host_frame_t *current_frame = UVC_ATOMIC_LOAD(uvc_stream->dynamic.current_frame); @@ -178,7 +159,40 @@ void bulk_transfer_callback(usb_transfer_t *transfer) } } } + // We got short packet in data section, this packet is EoF + if (transfer->data_buffer_size > transfer->actual_num_bytes) { + uvc_stream->single_thread.next_bulk_packet = UVC_STREAM_BULK_PACKET_EOF; + } else { + break; //only break if we have got full packet + } + } + case UVC_STREAM_BULK_PACKET_EOF: { + uvc_stream->single_thread.next_bulk_packet = UVC_STREAM_BULK_PACKET_SOF; + + // Get the current frame being processed and clear it from the stream, + // so no more data is written to this frame after the end of frame + UVC_ENTER_CRITICAL(); // Enter critical section to safely check and modify the stream state. + uvc_host_frame_t *this_frame = uvc_stream->dynamic.current_frame; + uvc_stream->dynamic.current_frame = NULL; + + // Determine if we should invoke the frame callback: + // Only invoke the callback if streaming is active, a frame callback exists, + // and we have a valid frame to pass to the user. + const bool invoke_fb_callback = (uvc_stream->dynamic.streaming && uvc_stream->constant.frame_cb && this_frame && !uvc_stream->single_thread.skip_current_frame); + UVC_EXIT_CRITICAL(); + + bool return_frame = true; // Default to returning the frame in case streaming has been stopped + if (invoke_fb_callback) { + // Call the user's frame callback. If the callback returns false, + // we do not return the frame to the empty queue (i.e., the user wants to keep it for processing) + return_frame = uvc_stream->constant.frame_cb(this_frame, uvc_stream->constant.cb_arg); + } + if (return_frame) { + // If the user has processed the frame (or the stream is stopped), return it to the empty frame queue + uvc_host_frame_return(uvc_stream, this_frame); + } break; + } default: abort(); } diff --git a/host/class/uvc/usb_host_uvc/uvc_control.c b/host/class/uvc/usb_host_uvc/uvc_control.c index 93063638..f31f9e9a 100644 --- a/host/class/uvc/usb_host_uvc/uvc_control.c +++ b/host/class/uvc/usb_host_uvc/uvc_control.c @@ -12,7 +12,7 @@ #include "uvc_control.h" #include "usb/usb_types_ch9.h" -#include "usb/usb_types_uvc.h" +#include "usb_types_uvc.h" #include "uvc_types_priv.h" #include "uvc_descriptors_priv.h" #include "uvc_check_priv.h" diff --git a/host/class/uvc/usb_host_uvc/uvc_frame.c b/host/class/uvc/usb_host_uvc/uvc_frame.c index a03f2786..1f96d64c 100644 --- a/host/class/uvc/usb_host_uvc/uvc_frame.c +++ b/host/class/uvc/usb_host_uvc/uvc_frame.c @@ -133,3 +133,43 @@ void uvc_frame_format_update(uvc_stream_t *uvc_stream, const uvc_host_stream_for uvc_frame_format_update(uvc_stream, vs_format); uvc_host_frame_return(uvc_stream, this_frame); } + +bool uvc_frame_payload_header_validate(const uvc_payload_header_t *hdr, size_t packet_len) +{ + // Need at least base header struct (bHeaderLength + bmHeaderInfo) + if (packet_len < sizeof(uvc_payload_header_t)) { + return false; + } + + const uint8_t header_len = hdr->bHeaderLength; + + // Basic structural checks + if (header_len < sizeof(uvc_payload_header_t)) { + return false; + } + + if (packet_len <= header_len) { + return false; + } + + // UVC spec: End of Header (EOH) bit must be set + if (!hdr->bmHeaderInfo.end_of_header) { + return false; + } + + // Optional fields minimal size checks (per UVC spec): + // - presentation_time -> +4 bytes + // - source_clock_reference -> +6 bytes + size_t min_len = sizeof(uvc_payload_header_t); + if (hdr->bmHeaderInfo.presentation_time) { + min_len += 4; + } + if (hdr->bmHeaderInfo.source_clock_reference) { + min_len += 6; + } + if (header_len < min_len) { + return false; + } + + return true; +} diff --git a/host/class/uvc/usb_host_uvc/uvc_isoc.c b/host/class/uvc/usb_host_uvc/uvc_isoc.c index 3618925c..f0dd2d3c 100644 --- a/host/class/uvc/usb_host_uvc/uvc_isoc.c +++ b/host/class/uvc/usb_host_uvc/uvc_isoc.c @@ -80,19 +80,38 @@ void isoc_transfer_callback(usb_transfer_t *transfer) assert(false); } - // Check for Zero Length Packet - if (isoc_desc->actual_num_bytes == 0) { + // Check for start of new frame + const uvc_payload_header_t *payload_header = (const uvc_payload_header_t *)payload; + if (!uvc_frame_payload_header_validate(payload_header, isoc_desc->actual_num_bytes)) { + ESP_LOGD(TAG, "invalid UVC payload header, %02x, %02x, len:%d", payload[0], payload[1], transfer->actual_num_bytes); + uvc_stream->single_thread.skip_current_frame = true; goto next_isoc_packet; } - // Check for start of new frame - const uvc_payload_header_t *payload_header = (const uvc_payload_header_t *)payload; + // Derive payload data pointer/length once and reuse below + const uint8_t *payload_data = payload + payload_header->bHeaderLength; + const size_t payload_data_len = isoc_desc->actual_num_bytes - payload_header->bHeaderLength; + + // Check for error flag + if (payload_header->bmHeaderInfo.error) { + ESP_LOGW(TAG, "frame error"); + uvc_stream->single_thread.skip_current_frame = true; + } + const bool start_of_frame = (uvc_stream->single_thread.current_frame_id != payload_header->bmHeaderInfo.frame_id); if (start_of_frame) { // We detected start of new frame. Update Frame ID and start fetching this frame uvc_stream->single_thread.current_frame_id = payload_header->bmHeaderInfo.frame_id; uvc_stream->single_thread.skip_current_frame = payload_header->bmHeaderInfo.error; + // Check mjpeg frame start + if (uvc_stream->dynamic.vs_format.format == UVC_VS_FORMAT_MJPEG && + payload_data[0] != 0xff && payload_data[1] != 0xd8) { + // We received frame with invalid frame, skip this frame + uvc_stream->single_thread.skip_current_frame = true; + ESP_LOGW(TAG, "invalid MJPEG SOI"); + } + // Get free frame buffer for this new frame UVC_ENTER_CRITICAL(); const bool need_new_frame = (uvc_stream->dynamic.streaming && !uvc_stream->dynamic.current_frame); @@ -115,20 +134,15 @@ void isoc_transfer_callback(usb_transfer_t *transfer) } } else { // We received SoF but current_frame is not NULL: We missed EoF - reset the frame buffer + ESP_EARLY_LOGW(TAG, "missed EoF"); + uvc_stream->single_thread.skip_current_frame = true; uvc_frame_reset(uvc_stream->dynamic.current_frame); UVC_EXIT_CRITICAL(); } } - // Check for error flag - if (payload_header->bmHeaderInfo.error) { - uvc_stream->single_thread.skip_current_frame = true; - } - // Add received data to frame buffer if (!uvc_stream->single_thread.skip_current_frame) { - const uint8_t *payload_data = payload + payload_header->bHeaderLength; - const size_t payload_data_len = isoc_desc->actual_num_bytes - payload_header->bHeaderLength; uvc_host_frame_t *current_frame = UVC_ATOMIC_LOAD(uvc_stream->dynamic.current_frame); esp_err_t ret = uvc_frame_add_data(current_frame, payload_data, payload_data_len); @@ -157,6 +171,11 @@ void isoc_transfer_callback(usb_transfer_t *transfer) uvc_host_frame_t *this_frame = uvc_stream->dynamic.current_frame; uvc_stream->dynamic.current_frame = NULL; // Stop writing more data to this frame + if (this_frame && this_frame->data_len <= 2) { + // We received too small frame, skip this frame + uvc_stream->single_thread.skip_current_frame = true; + } + // Determine if we should invoke the frame callback: // Only invoke the callback if streaming is active, a frame callback exists, // and we have a valid frame to pass to the user.