System audio

HaishinKit.xcodeproj/project.pbxproj

@@ -286,6 +286,11 @@
 		2EC97B7227880FF400D8BE32 /* OnTapGestureView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2EC97B6E27880FF400D8BE32 /* OnTapGestureView.swift */; };
 		2EC97B7327880FF400D8BE32 /* Views.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2EC97B6F27880FF400D8BE32 /* Views.swift */; };
 		2EC97B7427880FF400D8BE32 /* MTHKSwiftUiView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2EC97B7027880FF400D8BE32 /* MTHKSwiftUiView.swift */; };
+		6C0F263E2AFBCD2F00D86136 /* SystemAudio.h in Headers */ = {isa = PBXBuildFile; fileRef = 6C0F263A2AFBCD2F00D86136 /* SystemAudio.h */; settings = {ATTRIBUTES = (Public, ); }; };
+		6C0F263F2AFBCD2F00D86136 /* SystemAudio.mm in Sources */ = {isa = PBXBuildFile; fileRef = 6C0F263B2AFBCD2F00D86136 /* SystemAudio.mm */; };
+		6C0F26402AFBCD2F00D86136 /* SystemAudioMixer.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 6C0F263C2AFBCD2F00D86136 /* SystemAudioMixer.cpp */; };
+		6C0F26412AFBCD2F00D86136 /* SystemAudioMixer.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 6C0F263D2AFBCD2F00D86136 /* SystemAudioMixer.hpp */; };
+		6C0F26442AFBE6FB00D86136 /* SystemAudioUtilities.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 6C0F26432AFBE6FB00D86136 /* SystemAudioUtilities.hpp */; };
 		BC0BF4F22985FA9000D72CB4 /* HaishinKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 2945CBBD1B4BE66000104112 /* HaishinKit.framework */; };
 		BC0BF4F529866FDE00D72CB4 /* IOMixerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC0BF4F429866FDE00D72CB4 /* IOMixerTests.swift */; };
 		BC0BF4F72986CE8700D72CB4 /* VideoCodecTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC0BF4F62986CE8700D72CB4 /* VideoCodecTests.swift */; };
@@ -732,6 +737,11 @@
 		2EC97B6E27880FF400D8BE32 /* OnTapGestureView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OnTapGestureView.swift; sourceTree = "<group>"; };
 		2EC97B6F27880FF400D8BE32 /* Views.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Views.swift; sourceTree = "<group>"; };
 		2EC97B7027880FF400D8BE32 /* MTHKSwiftUiView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MTHKSwiftUiView.swift; sourceTree = "<group>"; };
+		6C0F263A2AFBCD2F00D86136 /* SystemAudio.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SystemAudio.h; sourceTree = "<group>"; };
+		6C0F263B2AFBCD2F00D86136 /* SystemAudio.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = SystemAudio.mm; sourceTree = "<group>"; };
+		6C0F263C2AFBCD2F00D86136 /* SystemAudioMixer.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = SystemAudioMixer.cpp; sourceTree = "<group>"; };
+		6C0F263D2AFBCD2F00D86136 /* SystemAudioMixer.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = SystemAudioMixer.hpp; sourceTree = "<group>"; };
+		6C0F26432AFBE6FB00D86136 /* SystemAudioUtilities.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = SystemAudioUtilities.hpp; sourceTree = "<group>"; };
 		BC0BF4F429866FDE00D72CB4 /* IOMixerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOMixerTests.swift; sourceTree = "<group>"; };
 		BC0BF4F62986CE8700D72CB4 /* VideoCodecTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VideoCodecTests.swift; sourceTree = "<group>"; };
 		BC0D236C26331BAB001DDA0C /* DataBuffer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = DataBuffer.swift; sourceTree = "<group>"; };
@@ -1058,6 +1068,7 @@
 				297C16881CC5382600117ADF /* Net */,
 				29C0E0591C2EB00A009DD8E8 /* RTMP */,
 				290907CE1C3961BC00F2E80C /* Util */,
+				6C0F26392AFBCCCB00D86136 /* SystemAudio */,
 			);
 			path = Sources;
 			sourceTree = "<group>";
@@ -1350,6 +1361,18 @@
 			path = View;
 			sourceTree = "<group>";
 		};
+		6C0F26392AFBCCCB00D86136 /* SystemAudio */ = {
+			isa = PBXGroup;
+			children = (
+				6C0F26432AFBE6FB00D86136 /* SystemAudioUtilities.hpp */,
+				6C0F263A2AFBCD2F00D86136 /* SystemAudio.h */,
+				6C0F263B2AFBCD2F00D86136 /* SystemAudio.mm */,
+				6C0F263C2AFBCD2F00D86136 /* SystemAudioMixer.cpp */,
+				6C0F263D2AFBCD2F00D86136 /* SystemAudioMixer.hpp */,
+			);
+			path = SystemAudio;
+			sourceTree = "<group>";
+		};
 		BC0BF4F329866FB700D72CB4 /* Media */ = {
 			isa = PBXGroup;
 			children = (
@@ -1435,6 +1458,9 @@
 			isa = PBXHeadersBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
+				6C0F263E2AFBCD2F00D86136 /* SystemAudio.h in Headers */,
+				6C0F26442AFBE6FB00D86136 /* SystemAudioUtilities.hpp in Headers */,
+				6C0F26412AFBCD2F00D86136 /* SystemAudioMixer.hpp in Headers */,
 				296543621D62FE8100734698 /* HaishinKit.h in Headers */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -2152,11 +2178,13 @@
 				29B877151CD70D5A00FC07DA /* RTMPMuxer.swift in Sources */,
 				29EA87E31E79A1E90043A5F8 /* CMVideoFormatDescription+Extension.swift in Sources */,
 				29B877171CD70D5A00FC07DA /* RTMPSharedObject.swift in Sources */,
+				6C0F26402AFBCD2F00D86136 /* SystemAudioMixer.cpp in Sources */,
 				29B877181CD70D5A00FC07DA /* RTMPSocket.swift in Sources */,
 				29EA87DD1E79A0460043A5F8 /* Data+Extension.swift in Sources */,
 				BCB976E026107B5600C9A649 /* TSField.swift in Sources */,
 				29B877191CD70D5A00FC07DA /* RTMPStream.swift in Sources */,
 				29B8771B1CD70D5A00FC07DA /* ByteArray.swift in Sources */,
+				6C0F263F2AFBCD2F00D86136 /* SystemAudio.mm in Sources */,
 				BCCBCE9829A90D880095B51C /* AVCNALUnit.swift in Sources */,
 				295891231EEB8EC500CE51E1 /* FLVAVCPacketType.swift in Sources */,
 				29EA87DA1E79A00E0043A5F8 /* ExpressibleByIntegerLiteral+Extension.swift in Sources */,

Sources/SystemAudio/SystemAudio.h

@@ -0,0 +1,24 @@
+//
+//  SystemAudio.h
+//  CloudApp
+//
+//  Created by Gordon Childs on 10/29/2023.
+//
+
+#ifndef SystemAudio_h
+#define SystemAudio_h
+
+#import <CoreMedia/CoreMedia.h>   // CMSampleBufferRef
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+    CMSampleBufferRef SystemAudioProcessSampleBuffer(CMSampleBufferRef sampleBuffer);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
+#endif /* SystemAudio_h */

Sources/SystemAudio/SystemAudio.mm

@@ -0,0 +1,225 @@
+//
+//  SystemAudio.mm
+//  CloudApp
+//
+//  Created by Gordon Childs on 10/29/2023.
+//
+
+#import "SystemAudio.h"
+#include "SystemAudioMixer.hpp"
+#include <memory> // std::unique_ptr
+#include <os/log.h>
+#include <string.h> // memset, memcmp
+
+#include "SystemAudioUtilities.hpp"
+
+class SystemAudio
+{
+  public:
+    SystemAudio();
+
+    CMSampleBufferRef processSampleBuffer(CMSampleBufferRef sampleBuffer);
+
+  private:
+    AudioStreamBasicDescription sampleBufferASBD_;
+    std::unique_ptr<SystemAudioMixer> mixer_;
+
+    auto_CMFormatDescription outputSampleBufferFormatDesc_;
+
+    unique_AudioBufferList_ptr inputABL_;
+    UInt32 inputABLSize_;
+
+    unique_AudioBufferList_ptr outputABL_;
+    UInt32 outputABLBufferBytesPerSample_;
+    UInt32 outputABLTotalBufferBytesPerSample_;
+
+    bool createSystemAudioMixer(const AudioStreamBasicDescription &newInputASBD);
+};
+
+SystemAudio::SystemAudio()
+{
+    memset(&sampleBufferASBD_, 0, sizeof(sampleBufferASBD_));
+}
+
+CMSampleBufferRef SystemAudio::processSampleBuffer(CMSampleBufferRef sampleBuffer)
+{
+    CMFormatDescriptionRef format = CMSampleBufferGetFormatDescription(sampleBuffer);
+    if (!format)
+        return nullptr;
+
+    const AudioStreamBasicDescription *asbd = CMAudioFormatDescriptionGetStreamBasicDescription(format);
+    if (!asbd)
+        return nullptr;
+
+    if (memcmp(asbd, &sampleBufferASBD_, sizeof(sampleBufferASBD_)) != 0)
+    {
+        os_log(OS_LOG_DEFAULT, "sample format changed");
+        if (!createSystemAudioMixer(*asbd))
+            return nullptr;
+
+        sampleBufferASBD_ = *asbd;
+    }
+
+    OSStatus status;
+
+    auto_CMBlockBuffer autoInputBlockBuffer;
+    {
+        CMBlockBufferRef blockBuffer;
+        status = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(
+            sampleBuffer, nullptr, inputABL_.get(), inputABLSize_, kCFAllocatorDefault, kCFAllocatorDefault,
+            0 /* flags */, &blockBuffer);
+        if (noErr != status)
+            return nullptr;
+
+        autoInputBlockBuffer.reset(blockBuffer);
+    }
+
+    CMSampleTimingInfo timingInfo;
+    status = CMSampleBufferGetSampleTimingInfo(sampleBuffer, 0, &timingInfo);
+    if (noErr != status)
+        return nullptr;
+
+    CMItemCount numSamples = CMSampleBufferGetNumSamples(sampleBuffer);
+
+    const size_t kOutputBlockSize = numSamples * outputABLTotalBufferBytesPerSample_;
+
+    auto outputBuffer = std::unique_ptr<uint8_t[]>(new uint8_t[kOutputBlockSize]);
+
+    // setup pointers and sizes in output ABL
+    uint8_t *dst = outputBuffer.get();
+    const UInt32 kPerChannelByteSize = outputABLBufferBytesPerSample_ * static_cast<UInt32>(numSamples);
+
+    for (UInt32 i = 0; i < outputABL_->mNumberBuffers; i++)
+    {
+        AudioBuffer &ab = outputABL_->mBuffers[i];
+        ab.mData = dst;
+        ab.mDataByteSize = kPerChannelByteSize;
+        dst += kPerChannelByteSize;
+    }
+
+    // mix down input and store it in output buffer
+    status = mixer_->mix(static_cast<UInt32>(numSamples), inputABL_.get(), outputABL_.get());
+    if (noErr != status)
+        return nullptr;
+
+    // create a CMSampleBufferRef from the mixed down 2 channel sample data
+    auto_CMBlockBuffer outputBlockBuffer;
+    {
+        CMBlockBufferRef blockBuffer;
+        status = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault, nullptr, kOutputBlockSize, kCFAllocatorDefault,
+                                                    nullptr, 0, kOutputBlockSize, kCMBlockBufferAssureMemoryNowFlag,
+                                                    &blockBuffer);
+        if (noErr != status)
+            return nullptr;
+
+        outputBlockBuffer.reset(blockBuffer);
+    }
+
+    status = CMBlockBufferReplaceDataBytes(outputBuffer.get(), outputBlockBuffer.get(), 0, kOutputBlockSize);
+    if (noErr != status)
+        return nullptr;
+
+    auto_CMSampleBuffer outputSampleBuffer;
+    {
+        CMSampleBufferRef sampleBuffer;
+        status = CMSampleBufferCreate(kCFAllocatorDefault, outputBlockBuffer.get(), true, nullptr, nullptr,
+                                      outputSampleBufferFormatDesc_.get(), numSamples, 1, &timingInfo, 0, nullptr,
+                                      &sampleBuffer);
+        if (noErr != status)
+            return nullptr;
+
+        outputSampleBuffer.reset(sampleBuffer);
+    }
+
+    return outputSampleBuffer.release();
+}
+
+bool SystemAudio::createSystemAudioMixer(const AudioStreamBasicDescription &inputASBD)
+{
+    OSStatus status;
+
+    if (mixer_)
+    {
+        status = mixer_->uninitialize();
+        if (noErr != status)
+            os_log(OS_LOG_DEFAULT, "failed to uninit mixer: %x", status);
+
+        memset(&sampleBufferASBD_, 0, sizeof(sampleBufferASBD_));
+    }
+
+    outputSampleBufferFormatDesc_.reset(nullptr);
+
+    const bool isInterleaved = !(inputASBD.mFormatFlags & kAudioFormatFlagIsNonInterleaved);
+
+    // stereo output
+    const UInt32 kOutputNumChannels = 2;
+    AudioStreamBasicDescription outputASBD = inputASBD;
+    outputASBD.mChannelsPerFrame = kOutputNumChannels;
+
+    // let input and output interleaved-ness match, but if it is interleaved,
+    // fix up the packet and frame sizes. mostly non-interleaved however.
+    if (isInterleaved)
+    {
+        outputASBD.mBytesPerPacket = kOutputNumChannels * outputASBD.mBitsPerChannel / 8;
+        outputASBD.mBytesPerFrame = kOutputNumChannels * outputASBD.mBitsPerChannel / 8;
+    }
+
+    {
+        CMFormatDescriptionRef formatDesc;
+
+        status = CMAudioFormatDescriptionCreate(kCFAllocatorDefault, &outputASBD, 0, nullptr, 0, nullptr, nullptr,
+                                                &formatDesc);
+        if (noErr != status)
+            return false;
+
+        outputSampleBufferFormatDesc_.reset(formatDesc);
+    }
+
+    std::unique_ptr<SystemAudioMixer> newMixer(new SystemAudioMixer);
+
+    status = newMixer->initialize(inputASBD, outputASBD);
+    if (noErr != status)
+    {
+        os_log(OS_LOG_DEFAULT, "failed to initialize mixer: %x", status);
+        return false;
+    }
+
+    mixer_ = std::move(newMixer);
+
+    {
+        const UInt32 numAudioBuffers = isInterleaved ? 1 : inputASBD.mChannelsPerFrame;
+
+        UInt32 ablSize = sizeof(AudioBufferList) + (numAudioBuffers - 1) * sizeof(AudioBuffer);
+        inputABL_.reset(reinterpret_cast<AudioBufferList *>(malloc(ablSize)));
+        inputABLSize_ = ablSize;
+
+        inputABL_->mNumberBuffers = numAudioBuffers;
+    }
+    {
+        const UInt32 numAudioBuffers = isInterleaved ? 1 : outputASBD.mChannelsPerFrame;
+
+        UInt32 ablSize = sizeof(AudioBufferList) + (numAudioBuffers - 1) * sizeof(AudioBuffer);
+        outputABL_.reset(reinterpret_cast<AudioBufferList *>(malloc(ablSize)));
+
+        // init non-pointer and channel size fields of output ABL
+        outputABL_->mNumberBuffers = numAudioBuffers;
+
+        UInt32 outputChannelsPerBuffer = outputASBD.mChannelsPerFrame / numAudioBuffers;
+
+        for (UInt32 i = 0; i < numAudioBuffers; i++)
+        {
+            outputABL_->mBuffers[i].mNumberChannels = outputChannelsPerBuffer;
+        }
+
+        outputABLBufferBytesPerSample_ = outputASBD.mBytesPerFrame;
+        outputABLTotalBufferBytesPerSample_ = numAudioBuffers * outputASBD.mBytesPerFrame;
+    }
+    return true;
+}
+
+static SystemAudio gSystemAudioBufferProcesser;
+
+CMSampleBufferRef SystemAudioProcessSampleBuffer(CMSampleBufferRef sampleBuffer)
+{
+    return gSystemAudioBufferProcesser.processSampleBuffer(sampleBuffer);
+}