OpenAI Realtime Voice API support plus gpt-4o and gpt-4o-mini

OpenAIAPI.uplugin

@@ -1,16 +1,16 @@
 {
 	"FileVersion": 3,
 	"Version": 1,
-	"VersionName": "1.0",
+	"VersionName": "1.1",
 	"FriendlyName": "OpenAIAPI",
 	"Description": "Provides easy access to OpenAI's API",
 	"Category": "Other",
-	"CreatedBy": "Kellan Mythen",
+	"CreatedBy": "Kellan Mythen, Robert Bjarnason (Citizens Foundation)",
 	"CreatedByURL": "",
 	"DocsURL": "https://github.com/KellanM/OpenAI-Api-Unreal/",
 	"MarketplaceURL": "",
 	"SupportURL": "",
-	"EngineVersion": "5.3",
+	"EngineVersion": "5.4",
 	"CanContainContent": true,
 	"Installed": true,
 	"Modules": [
@@ -21,8 +21,19 @@
 			"WhitelistPlatforms": [
 				"Win64",
 				"Mac",
+				"Linux",
 				"Android"
 			]
 		}
+	],
+	"Plugins": [
+		{
+			"Name": "AudioCapture",
+			"Enabled": true
+		},
+		{
+			"Name": "Synthesis",
+			"Enabled": true
+		}
 	]
 }

README.md

@@ -1,5 +1,5 @@
 # OpenAI API Plugin
-The **OpenAIApi Plugin** gives you access to the **OpenAI API** in **Unreal Engine**. It is compatible with 4.26, 4.27, 5.0, 5.1, 5.2, and 5.3
+The **OpenAIApi Plugin** gives you access to the **OpenAI API** in **Unreal Engine**. It is compatible with at least  5.3 and 5.4
 
 This is a community Plugin. No affiliation with OpenAI
 
@@ -72,9 +72,15 @@ This example shows you how to generate a 1024x1024 image using DALL·E 2 in blue
 This example shows you how to record and transcribe speech using Whisper v2-Large in blueprints.
 
 ![](https://i.imgur.com/ameqz1L.png)
+
+## Blueprint for Realtime Voice API using the Runtime Audio Importer plugin
+![image](https://github.com/user-attachments/assets/18564ac6-d333-4566-ae83-1671c1cc2c92)
+
 ## References
 - [OpenAI's API Engine Documentation](https://beta.openai.com/docs/engines)
 - [OpenAI's API Reference](https://beta.openai.com/docs/api-reference/completions)
 
 ## Supported Platforms
-Windows, Mac, Android
+Windows, Mac*, Android*
+
+  *Realtime Voice API mic audio capture has only been tested on Windows

Source/OpenAIAPI/OpenAIAPI.Build.cs

@@ -7,45 +7,55 @@ public class OpenAIAPI : ModuleRules
 	public OpenAIAPI(ReadOnlyTargetRules Target) : base(Target)
 	{
 		PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs;
-		
-		
+
+
 		PublicIncludePaths.AddRange(
 			new string[] {
 				// ... add public include paths required here ...
 			}
 			);
-				
-		
+
+
 		PrivateIncludePaths.AddRange(
 			new string[] {
 				// ... add other private include paths required here ...
 			}
 			);
-			
-		
+
+
 		PublicDependencyModuleNames.AddRange(
 			new string[]
 			{
 				"Core",
+				 "WebSockets",
+				 "AudioCapture",
+				 "AudioCaptureCore",
+				 "SignalProcessing",
+				 	"AudioMixer",
 				// ... add other public dependencies that you statically link with here ...
 			}
 			);
-			
-		
+
+
 		PrivateDependencyModuleNames.AddRange(
 			new string[]
 			{
 				"CoreUObject",
 				"Engine",
+			  "Synthesis",
+				"AudioMixer",
 				"Slate",
+				"InputCore",
+				"AudioCapture",
+				"AudioCaptureCore",
 				"SlateCore",
 				"Json",
-				"Http"
-				// ... add private dependencies that you statically link with here ...	
+				"HTTP"
+				// ... add private dependencies that you statically link with here ...
 			}
 			);
-		
-		
+
+
 		DynamicallyLoadedModuleNames.AddRange(
 			new string[]
 			{

Source/OpenAIAPI/Private/OpenAIAudioCapture.cpp

@@ -0,0 +1,151 @@
+#include "OpenAIAudioCapture.h"
+#include "Kismet/GameplayStatics.h"
+
+UOpenAIAudioCapture::UOpenAIAudioCapture()
+{
+    PrimaryComponentTick.bCanEverTick = false;
+    AudioCapture = nullptr;
+    bIsCapturing = false;
+    bAutoActivate = true;
+    UE_LOG(LogTemp, Log, TEXT("UOpenAIAudioCapture constructor called"));
+}
+
+
+void UOpenAIAudioCapture::Activate(bool bReset)
+{
+    UE_LOG(LogTemp, Log, TEXT("UOpenAIAudioCapture Activate called"));
+    Super::Activate(bReset);
+
+    // Move initialization logic from BeginPlay to here
+    if (!AudioCapture)
+    {
+        AudioCapture = NewObject<UAudioCapture>(this);
+        if (AudioCapture)
+        {
+            AudioCapture->AddGeneratorDelegate([this](const float* InAudio, int32 NumSamples) {
+                this->OnAudioGenerate(InAudio, NumSamples);
+            });
+
+            AudioCapture->OpenDefaultAudioStream();
+            StartCapturing();
+            UE_LOG(LogTemp, Log, TEXT("-------------------> AudioCapture started from Activate on GameThread"));
+        }
+        else
+        {
+            UE_LOG(LogTemp, Error, TEXT("Failed to create AudioCapture object"));
+        }
+    } else {
+        UE_LOG(LogTemp, Log, TEXT("AudioCapture is already created"));
+    }
+}
+
+void UOpenAIAudioCapture::StartCapturing()
+{
+    if (AudioCapture && !bIsCapturing)
+    {
+        if (AudioCapture) {
+            UE_LOG(LogTemp, Log, TEXT("AudioCapture is valid"));
+            AudioCapture->StartCapturingAudio();
+            bIsCapturing = true;
+            UE_LOG(LogTemp, Log, TEXT("Audio capture started successfully"));
+        } else {
+            UE_LOG(LogTemp, Log, TEXT("AudioCapture is null"));
+        }
+    } else {
+        UE_LOG(LogTemp, Log, TEXT("AudioCapture is null or already capturing"));
+    }
+}
+
+void UOpenAIAudioCapture::StopCapturing()
+{
+    UE_LOG(LogTemp, Log, TEXT("Attempting to stop audio capture"));
+    if (AudioCapture)
+    {
+        UE_LOG(LogTemp, Log, TEXT("AudioCapture is valid"));
+        AudioCapture->StopCapturingAudio();
+         bIsCapturing = false;
+         ProcessAndBroadcastBuffer(); // Broadcast any remaining data
+
+        UE_LOG(LogTemp, Log, TEXT("Audio capture stopped successfully"));
+    }
+    else
+    {
+        UE_LOG(LogTemp, Warning, TEXT("AudioCapture is null"));
+    }
+}
+
+void UOpenAIAudioCapture::DestroyAudioCapture() {
+    UE_LOG(LogTemp, Log, TEXT("DestroyAudioCapture called"));
+    if (AudioCapture)
+    {
+        AudioCapture->ConditionalBeginDestroy();
+        AudioCapture = nullptr;
+        UE_LOG(LogTemp, Log, TEXT("AudioCapture destroyed"));
+    } else {
+        UE_LOG(LogTemp, Log, TEXT("AudioCapture is null"));
+    }
+}
+
+void UOpenAIAudioCapture::OnAudioGenerate(const float* InAudio, int32 NumSamples)
+{
+    if (InAudio) {
+        if (bIsCapturing)
+        {
+            const int32 DownsampleFactor = 2; // Factor by which to downsample
+            const int32 InterpolatedSamples = NumSamples / DownsampleFactor;
+
+            for (int32 i = 0; i < InterpolatedSamples - 1; i++)
+            {
+               // Indices for original samples
+                int32 index1 = i * DownsampleFactor;
+                int32 index2 = index1 + DownsampleFactor;
+
+                // Boundary check to prevent accessing out-of-range samples
+                if (index2 >= NumSamples)
+                {
+                    // Handle the last sample if NumSamples is not perfectly divisible
+                    index2 = NumSamples - 1;
+                }
+
+                // Original sample values
+                float sample1 = InAudio[index1];
+                float sample2 = InAudio[index2];
+
+                // Calculate the interpolated sample using linear interpolation
+                float interpolatedSample = sample1 + ((sample2 - sample1) * 0.5f);
+
+                // Add the interpolated sample to the AudioBuffer
+                AudioBuffer.Add(interpolatedSample);
+            }
+            double CurrentTime = FPlatformTime::Seconds();
+            if (CurrentTime - LastBroadcastTime >= MaxBufferTime || AudioBuffer.Num() >= MaxBufferSize)
+            {
+                ProcessAndBroadcastBuffer();
+                LastBroadcastTime = CurrentTime;
+            }
+
+        } else {
+            UE_LOG(LogTemp, Log, TEXT("bIsCapturing is false"));
+        }
+    } else {
+        UE_LOG(LogTemp, Log, TEXT("InAudio is null"));
+    }
+}
+
+void UOpenAIAudioCapture::ProcessAndBroadcastBuffer()
+{
+    TArray<float> BufferCopy;
+    {
+        if (AudioBuffer.Num() > 0)
+        {
+            BufferCopy = AudioBuffer;
+            AudioBuffer.Empty();
+        }
+    }
+
+    if (BufferCopy.Num() > 0)
+    {
+        // Broadcast the captured audio data
+        OnAudioBufferCaptured.Broadcast(BufferCopy);
+    }
+}

Source/OpenAIAPI/Private/OpenAICallChat.cpp

@@ -30,14 +30,14 @@ void UOpenAICallChat::Activate()
 		_apiKey = UOpenAIUtils::GetEnvironmentVariable(TEXT("OPENAI_API_KEY"));
 	else
 		_apiKey = UOpenAIUtils::getApiKey();
-	
+
 	// checking parameters are valid
 	if (_apiKey.IsEmpty())
 	{
 		Finished.Broadcast({}, TEXT("Api key is not set"), false);
 	}	else
 	{
-	
+
 		auto HttpRequest = FHttpModule::Get().CreateRequest();
 
 		FString apiMethod;
@@ -49,16 +49,22 @@ void UOpenAICallChat::Activate()
 		case EOAChatEngineType::GPT_4:
 			apiMethod = "gpt-4";
 			break;
+		case EOAChatEngineType::GPT_4o:
+			apiMethod = "gpt-4o";
+			break;
+		case EOAChatEngineType::GPT_4o_mini:
+			apiMethod = "gpt-4o-mini";
+			break;
 		case EOAChatEngineType::GPT_4_32k:
 			apiMethod = "gpt-4-32k";
 			break;
 			case EOAChatEngineType::GPT_4_TURBO:
 				apiMethod = "gpt-4-0125-preview";
 			break;
 		}
-		
+
 		//TODO: add aditional params to match the ones listed in the curl response in: https://platform.openai.com/docs/api-reference/making-requests
-	
+
 		// convert parameters to strings
 		FString tempHeader = "Bearer ";
 		tempHeader += _apiKey;
@@ -74,7 +80,7 @@ void UOpenAICallChat::Activate()
 		_payloadObject->SetStringField(TEXT("model"), apiMethod);
 		_payloadObject->SetNumberField(TEXT("max_tokens"), chatSettings.maxTokens);
 
-		
+
 		// convert role enum to model string
 		if (!(chatSettings.messages.Num() == 0))
 		{