diff --git a/.github/workflows/build_examples.yml b/.github/workflows/build_examples.yml new file mode 100644 index 0000000..864a39b --- /dev/null +++ b/.github/workflows/build_examples.yml @@ -0,0 +1,28 @@ +name: Build Examples +on: + workflow_call: {} + workflow_dispatch: {} +jobs: + build-examples: + name: Build Examples + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # Add additional targets later + device: [esp32s3] + steps: + - name: Checkout + uses: actions/checkout@v4 + with: { submodules: recursive } + - name: Build Examples + uses: espressif/esp-idf-ci-action@v1 + with: + esp_idf_version: v5.4 + command: | + export WIFI_SSID="---" + export WIFI_PASSWORD="---" + export LK_SERVER_URL="---" + export LK_TOKEN="---" + pip install idf-build-apps + idf-build-apps build -p ./examples --recursive --target ${{ matrix.device }} \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..d865d82 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,16 @@ +name: CI +on: + schedule: + - cron: 0 0 * * 1 + workflow_dispatch: + pull_request: + types: [opened, reopened, synchronize] + push: + branches: [main] +concurrency: + group: "ci" + cancel-in-progress: true +jobs: + build-examples: + name: Build Examples + uses: ./.github/workflows/build_examples.yml \ No newline at end of file diff --git a/components/livekit/core/engine.c b/components/livekit/core/engine.c index 0e60410..278db75 100644 --- a/components/livekit/core/engine.c +++ b/components/livekit/core/engine.c @@ -1,5 +1,4 @@ #include "esp_log.h" -#include "webrtc_utils_time.h" #include "media_lib_os.h" #include "esp_codec_dev.h" @@ -39,21 +38,6 @@ typedef struct { esp_peer_audio_stream_info_t sub_audio_info; } engine_t; -/// @brief Performs one-time system initialization. -static void sys_init(void) -{ - static bool is_initialized = false; - if (is_initialized) { - ESP_LOGI(TAG, "System already initialized"); - return; - } - is_initialized = webrtc_utils_time_sync_init() == ESP_OK; - if (!is_initialized) { - ESP_LOGE(TAG, "System initialization failed"); - return; - } -} - static esp_capture_codec_type_t capture_audio_codec_type(esp_peer_audio_codec_t peer_codec) { switch (peer_codec) { @@ -153,7 +137,7 @@ static engine_err_t media_stream_begin(engine_t *eng) } media_lib_thread_handle_t handle = NULL; eng->is_media_streaming = true; - if (media_lib_thread_create_from_scheduler(&handle, "lk_stream", media_stream_task, eng) != ESP_OK) { + if (media_lib_thread_create_from_scheduler(&handle, STREAM_THREAD_NAME, media_stream_task, eng) != ESP_OK) { ESP_LOGE(TAG, "Failed to create media stream thread"); eng->is_media_streaming = false; return ENGINE_ERR_MEDIA; @@ -267,6 +251,7 @@ static void free_ice_servers(engine_t *eng) eng->ice_server_count = 0; } +__attribute__((unused)) static engine_err_t set_ice_servers(engine_t* eng, livekit_pb_ice_server_t *servers, int count) { if (eng == NULL || servers == NULL || count <= 0) { @@ -327,7 +312,6 @@ static void on_peer_pub_state_changed(peer_state_t state, void *ctx) static void on_peer_sub_state_changed(peer_state_t state, void *ctx) { - engine_t *eng = (engine_t *)ctx; if (state == PEER_STATE_CONNECTED) { // TODO: Subscribe } @@ -347,7 +331,6 @@ static void on_peer_sub_answer(const char *sdp, void *ctx) static void on_peer_ice_candidate(const char *candidate, void *ctx) { - engine_t *eng = (engine_t *)ctx; ESP_LOGI(TAG, "Peer generated ice candidate: %s", candidate); } @@ -389,21 +372,18 @@ static void on_peer_sub_audio_frame(esp_peer_audio_frame_t* frame, void *ctx) static void on_sig_connect(void *ctx) { - engine_t *eng = (engine_t *)ctx; ESP_LOGI(TAG, "Signaling connected"); // TODO: Implement } static void on_sig_disconnect(void *ctx) { - engine_t *eng = (engine_t *)ctx; ESP_LOGI(TAG, "Signaling disconnected"); // TODO: Implement } static void on_sig_error(void *ctx) { - engine_t *eng = (engine_t *)ctx; ESP_LOGI(TAG, "Signaling error"); // TODO: Implement } @@ -582,8 +562,6 @@ engine_err_t engine_connect(engine_handle_t handle, const char* server_url, cons } engine_t *eng = (engine_t *)handle; - sys_init(); - if (signal_connect(eng->sig, server_url, token) != SIGNAL_ERR_NONE) { ESP_LOGE(TAG, "Failed to connect signaling client"); return ENGINE_ERR_SIGNALING; diff --git a/components/livekit/core/engine.h b/components/livekit/core/engine.h index c00d1ba..c93d918 100644 --- a/components/livekit/core/engine.h +++ b/components/livekit/core/engine.h @@ -9,6 +9,8 @@ #include "common.h" #include "protocol.h" +#define STREAM_THREAD_NAME "lk_stream" + #ifdef __cplusplus extern "C" { #endif diff --git a/components/livekit/core/livekit.c b/components/livekit/core/livekit.c index b75082a..3fe29ad 100644 --- a/components/livekit/core/livekit.c +++ b/components/livekit/core/livekit.c @@ -3,7 +3,7 @@ #include "esp_peer.h" #include "engine.h" #include "rpc_manager.h" - +#include "system.h" #include "livekit.h" static const char *TAG = "livekit"; @@ -141,6 +141,10 @@ livekit_err_t livekit_room_create(livekit_room_handle_t *handle, const livekit_r if (handle == NULL || options == NULL) { return LIVEKIT_ERR_INVALID_ARG; } + if (!system_is_media_lib_setup()) { + ESP_LOGE(TAG, "Must perform system initialization before creating a room"); + return LIVEKIT_ERR_SYSTEM_INIT; + } // Validate options if (options->publish.kind != LIVEKIT_MEDIA_TYPE_NONE && @@ -308,4 +312,12 @@ livekit_err_t livekit_room_rpc_unregister(livekit_room_handle_t handle, const ch return LIVEKIT_ERR_INVALID_STATE; } return LIVEKIT_ERR_NONE; +} + +livekit_err_t livekit_system_init(void) +{ + if (!system_setup_media_lib()) { + return LIVEKIT_ERR_SYSTEM_INIT; + } + return LIVEKIT_ERR_NONE; } \ No newline at end of file diff --git a/components/livekit/core/peer.c b/components/livekit/core/peer.c index 490886e..e093d19 100644 --- a/components/livekit/core/peer.c +++ b/components/livekit/core/peer.c @@ -13,6 +13,9 @@ static const char *SUB_TAG = "livekit_peer.sub"; static const char *PUB_TAG = "livekit_peer.pub"; #define TAG(peer) (peer->options.target == LIVEKIT_PB_SIGNAL_TARGET_SUBSCRIBER ? SUB_TAG : PUB_TAG) +#define SUB_THREAD_NAME (PEER_THREAD_NAME_PREFIX "sub") +#define PUB_THREAD_NAME (PEER_THREAD_NAME_PREFIX "pub") + #define RELIABLE_CHANNEL_LABEL "_reliable" #define LOSSY_CHANNEL_LABEL "_lossy" #define STREAM_ID_INVALID 0xFFFF @@ -332,9 +335,9 @@ peer_err_t peer_connect(peer_handle_t handle) peer->running = true; media_lib_thread_handle_t thread; const char* thread_name = peer->options.target == LIVEKIT_PB_SIGNAL_TARGET_SUBSCRIBER ? - "lk_sub_task" : "lk_pub_task"; + SUB_THREAD_NAME : PUB_THREAD_NAME; if (media_lib_thread_create_from_scheduler(&thread, thread_name, peer_task, peer) != ESP_PEER_ERR_NONE) { - ESP_LOGE(TAG(peer), "Failed to create task"); + ESP_LOGE(TAG(peer), "Failed to create thread"); return PEER_ERR_RTC; } diff --git a/components/livekit/core/peer.h b/components/livekit/core/peer.h index 51e3101..dcb598d 100644 --- a/components/livekit/core/peer.h +++ b/components/livekit/core/peer.h @@ -5,6 +5,8 @@ #include "engine.h" #include "protocol.h" +#define PEER_THREAD_NAME_PREFIX "lk_peer_" + #ifdef __cplusplus extern "C" { #endif diff --git a/components/livekit/core/protocol.c b/components/livekit/core/protocol.c deleted file mode 100644 index 9f47896..0000000 --- a/components/livekit/core/protocol.c +++ /dev/null @@ -1,31 +0,0 @@ -#include "protocol.h" - -const char* livekit_protocol_sig_res_name(pb_size_t which_message) -{ - switch (which_message) { - case LIVEKIT_PB_SIGNAL_RESPONSE_JOIN_TAG: return "Join"; - case LIVEKIT_PB_SIGNAL_RESPONSE_ANSWER_TAG: return "Answer"; - case LIVEKIT_PB_SIGNAL_RESPONSE_OFFER_TAG: return "Offer"; - case LIVEKIT_PB_SIGNAL_RESPONSE_TRICKLE_TAG: return "Trickle"; - case LIVEKIT_PB_SIGNAL_RESPONSE_UPDATE_TAG: return "Update"; - case LIVEKIT_PB_SIGNAL_RESPONSE_TRACK_PUBLISHED_TAG: return "TrackPublished"; - case LIVEKIT_PB_SIGNAL_RESPONSE_LEAVE_TAG: return "Leave"; - case LIVEKIT_PB_SIGNAL_RESPONSE_MUTE_TAG: return "Mute"; - case LIVEKIT_PB_SIGNAL_RESPONSE_SPEAKERS_CHANGED_TAG: return "SpeakersChanged"; - case LIVEKIT_PB_SIGNAL_RESPONSE_ROOM_UPDATE_TAG: return "RoomUpdate"; - case LIVEKIT_PB_SIGNAL_RESPONSE_CONNECTION_QUALITY_TAG: return "ConnectionQuality"; - case LIVEKIT_PB_SIGNAL_RESPONSE_STREAM_STATE_UPDATE_TAG: return "StreamStateUpdate"; - case LIVEKIT_PB_SIGNAL_RESPONSE_SUBSCRIBED_QUALITY_UPDATE_TAG: return "SubscribedQualityUpdate"; - case LIVEKIT_PB_SIGNAL_RESPONSE_SUBSCRIPTION_PERMISSION_UPDATE_TAG: return "SubscriptionPermissionUpdate"; - case LIVEKIT_PB_SIGNAL_RESPONSE_REFRESH_TOKEN_TAG: return "RefreshToken"; - case LIVEKIT_PB_SIGNAL_RESPONSE_TRACK_UNPUBLISHED_TAG: return "TrackUnpublished"; - case LIVEKIT_PB_SIGNAL_RESPONSE_PONG_TAG: return "Pong"; - case LIVEKIT_PB_SIGNAL_RESPONSE_RECONNECT_TAG: return "Reconnect"; - case LIVEKIT_PB_SIGNAL_RESPONSE_PONG_RESP_TAG: return "PongResp"; - case LIVEKIT_PB_SIGNAL_RESPONSE_SUBSCRIPTION_RESPONSE_TAG: return "SubscriptionResponse"; - case LIVEKIT_PB_SIGNAL_RESPONSE_REQUEST_RESPONSE_TAG: return "RequestResponse"; - case LIVEKIT_PB_SIGNAL_RESPONSE_TRACK_SUBSCRIBED_TAG: return "TrackSubscribed"; - case LIVEKIT_PB_SIGNAL_RESPONSE_ROOM_MOVED_TAG: return "RoomMoved"; - default: return "Unknown"; - } -} \ No newline at end of file diff --git a/components/livekit/core/protocol.h b/components/livekit/core/protocol.h index 67aef15..7ef6933 100644 --- a/components/livekit/core/protocol.h +++ b/components/livekit/core/protocol.h @@ -1,21 +1,10 @@ #pragma once -#include -#include +#include "pb_encode.h" +#include "pb_decode.h" #include "livekit_rtc.pb.h" #include "livekit_models.pb.h" #include "livekit_metrics.pb.h" -#include "timestamp.pb.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/// @brief Gets the name of the signaling response type. -const char* livekit_protocol_sig_res_name(pb_size_t which_message); - -#ifdef __cplusplus -} -#endif \ No newline at end of file +#include "timestamp.pb.h" \ No newline at end of file diff --git a/components/livekit/core/signaling.c b/components/livekit/core/signaling.c index 11fcf36..9c0f5e9 100644 --- a/components/livekit/core/signaling.c +++ b/components/livekit/core/signaling.c @@ -171,9 +171,7 @@ static void on_data(signal_t *sg, const char *data, size_t len) return; } - ESP_LOGI(TAG, "Decoded res: type=%s(%d)", - livekit_protocol_sig_res_name(res.which_message), - res.which_message); + ESP_LOGI(TAG, "Decoded res: type=%d", res.which_message); handle_res(sg, &res); } diff --git a/components/livekit/core/system.c b/components/livekit/core/system.c new file mode 100644 index 0000000..6d6982a --- /dev/null +++ b/components/livekit/core/system.c @@ -0,0 +1,96 @@ +#include +#include "esp_log.h" +#include "webrtc_utils_time.h" +#include "media_lib_os.h" +#include "media_lib_adapter.h" +#include "system.h" +#include "peer.h" +#include "engine.h" + +#define VIDEO_ENCODE_THREAD_NAME "venc" +#define AUDIO_ENCODE_THREAD_NAME "aenc" +#define AUDIO_DECODE_THREAD_NAME "Adec" +#define AEC_SRC_READ_THREAD_NAME "SrcRead" +#define AEC_BUFFER_IN_THREAD_NAME "buffer_in" + +static const char *TAG = "livekit_system"; +static bool is_media_lib_setup = false; + +static void thread_scheduler(const char *thread_name, media_lib_thread_cfg_t *thread_cfg) +{ + ESP_LOGI(TAG, "Scheduling thread '%s'", thread_name); + + // LiveKit threads + if (strncmp(thread_name, PEER_THREAD_NAME_PREFIX, strlen(PEER_THREAD_NAME_PREFIX)) == 0) { + thread_cfg->stack_size = 25 * 1024; + thread_cfg->priority = 18; + thread_cfg->core_id = 1; + return; + } + if (strcmp(thread_name, STREAM_THREAD_NAME) == 0) { + thread_cfg->stack_size = 4 * 1024; + thread_cfg->priority = 15; + thread_cfg->core_id = 1; + return; + } + + // Media lib threads + if (strcmp(thread_name, AUDIO_DECODE_THREAD_NAME) == 0) { + thread_cfg->stack_size = 40 * 1024; + thread_cfg->priority = 10; + thread_cfg->core_id = 1; + return; + } + if (strcmp(thread_name, AUDIO_ENCODE_THREAD_NAME) == 0) { + // Required for Opus + thread_cfg->stack_size = 40 * 1024; + thread_cfg->priority = 10; + return; + } + if (strcmp(thread_name, AEC_SRC_READ_THREAD_NAME) == 0) { + thread_cfg->stack_size = 40 * 1024; + thread_cfg->priority = 16; + thread_cfg->core_id = 0; + return; + } + if (strcmp(thread_name, AEC_BUFFER_IN_THREAD_NAME) == 0) { + thread_cfg->stack_size = 6 * 1024; + thread_cfg->priority = 10; + thread_cfg->core_id = 0; + return; + } + if (strcmp(thread_name, VIDEO_ENCODE_THREAD_NAME) == 0) { +#if CONFIG_IDF_TARGET_ESP32S3 + thread_cfg->stack_size = 20 * 1024; +#endif + thread_cfg->priority = 10; + return; + } +} + +bool system_setup_media_lib(void) +{ + esp_err_t ret = media_lib_add_default_adapter(); + if (ret != ESP_OK) { + ESP_LOGE(TAG, "Failed to setup media lib"); + return false; + } + media_lib_thread_set_schedule_cb(thread_scheduler); + is_media_lib_setup = true; + return true; +} + +bool system_is_media_lib_setup(void) +{ + return is_media_lib_setup; +} + +bool system_sync_time(void) +{ + esp_err_t ret = webrtc_utils_time_sync_init(); + if (ret != ESP_OK) { + ESP_LOGE(TAG, "Failed to sync time"); + return false; + } + return true; +} \ No newline at end of file diff --git a/components/livekit/core/system.h b/components/livekit/core/system.h new file mode 100644 index 0000000..a8bd7c0 --- /dev/null +++ b/components/livekit/core/system.h @@ -0,0 +1,15 @@ + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +bool system_setup_media_lib(void); +bool system_is_media_lib_setup(void); + +bool system_sync_time(void); + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/components/livekit/include/livekit.h b/components/livekit/include/livekit.h index ccbfe08..d91007e 100644 --- a/components/livekit/include/livekit.h +++ b/components/livekit/include/livekit.h @@ -1,8 +1,10 @@ #pragma once +#include "media_lib_os.h" #include "esp_capture.h" #include "av_render.h" + #include "livekit_rpc.h" #ifdef __cplusplus @@ -17,6 +19,7 @@ typedef enum { LIVEKIT_ERR_ENGINE = -3, ///< Engine LIVEKIT_ERR_OTHER = -4, ///< Other error LIVEKIT_ERR_INVALID_STATE = -5, ///< Invalid state + LIVEKIT_ERR_SYSTEM_INIT = -6 ///< System not initialized } livekit_err_t; /// Video codec to use within a room. @@ -128,6 +131,21 @@ typedef struct { void* ctx; } livekit_room_options_t; +/// @defgroup System System initialization +/// Perform required one-time system initialization. +/// @{ + +/// Performs one-time system initialization. +/// +/// @return @ref LIVEKIT_ERR_NONE if successful, otherwise LIVEKIT_ERR_SYSTEM_INIT. +/// +/// Invoke this function early in the application's main function before +/// creating a room. Internally, this will setup the media library's thread scheduler. +/// +livekit_err_t livekit_system_init(void); + +/// @} + /// @defgroup Lifecycle /// Create and destroy room objects. /// @{ diff --git a/examples/voice_agent/main/main.c b/examples/voice_agent/main/main.c index b0c145c..3c11a6a 100644 --- a/examples/voice_agent/main/main.c +++ b/examples/voice_agent/main/main.c @@ -81,52 +81,6 @@ static int init_console() return 0; } -static void thread_scheduler(const char *thread_name, media_lib_thread_cfg_t *thread_cfg) -{ - // TODO: Handle internally - if (strcmp(thread_name, "lk_pub_task") == 0 || - strcmp(thread_name, "lk_sub_task") == 0) { - thread_cfg->stack_size = 25 * 1024; - thread_cfg->priority = 18; - thread_cfg->core_id = 1; - } - if (strcmp(thread_name, "lk_stream") == 0) { - thread_cfg->stack_size = 4 * 1024; - thread_cfg->priority = 15; - thread_cfg->core_id = 1; - } - if (strcmp(thread_name, "start") == 0) { - thread_cfg->stack_size = 6 * 1024; - } - if (strcmp(thread_name, "Adec") == 0) { - thread_cfg->stack_size = 40 * 1024; - thread_cfg->priority = 10; - thread_cfg->core_id = 1; - } - if (strcmp(thread_name, "venc") == 0) { -#if CONFIG_IDF_TARGET_ESP32S3 - thread_cfg->stack_size = 20 * 1024; -#endif - thread_cfg->priority = 10; - } - - // Required for Opus - if (strcmp(thread_name, "aenc") == 0) { - thread_cfg->stack_size = 40 * 1024; - thread_cfg->priority = 10; - } - if (strcmp(thread_name, "SrcRead") == 0) { - thread_cfg->stack_size = 40 * 1024; - thread_cfg->priority = 16; - thread_cfg->core_id = 0; - } - if (strcmp(thread_name, "buffer_in") == 0) { - thread_cfg->stack_size = 6 * 1024; - thread_cfg->priority = 10; - thread_cfg->core_id = 0; - } -} - static int network_event_handler(bool connected) { // Auto-join when network is connected @@ -141,8 +95,7 @@ static int network_event_handler(bool connected) void app_main(void) { esp_log_level_set("*", ESP_LOG_INFO); - media_lib_add_default_adapter(); - media_lib_thread_set_schedule_cb(thread_scheduler); + livekit_system_init(); board_init(); media_setup_init(); init_console(); diff --git a/examples/voice_agent/sdkconfig.defaults b/examples/voice_agent/sdkconfig.defaults index c3bd54a..ace1b34 100644 --- a/examples/voice_agent/sdkconfig.defaults +++ b/examples/voice_agent/sdkconfig.defaults @@ -32,4 +32,7 @@ CONFIG_SPIRAM_MALLOC_ALWAYSINTERNAL=256 CONFIG_SPIRAM_TRY_ALLOCATE_WIFI_LWIP=y CONFIG_MBEDTLS_EXTERNAL_MEM_ALLOC=y -CONFIG_IDF_TARGET_ESP32P4=1 \ No newline at end of file +CONFIG_IDF_TARGET_ESP32P4=1 + +# Board support package +CONFIG_BSP_I2C_NUM=0 \ No newline at end of file