Skip to content

Commit 917defa

Browse files
authored
feat: preconnect audio buffer (#256)
* feat: preconnect audio buffer * chore: lint * fix: 0.80 react-native compatibility * chore: update livekit-client peer dependency to 2.15.4 * chore: cleanup export
1 parent 1b7124f commit 917defa

File tree

16 files changed

+390
-37
lines changed

16 files changed

+390
-37
lines changed

android/src/main/java/com/livekit/reactnative/LivekitReactNativeModule.kt

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,19 @@ import com.facebook.react.bridge.ReactApplicationContext
88
import com.facebook.react.bridge.ReactContextBaseJavaModule
99
import com.facebook.react.bridge.ReactMethod
1010
import com.facebook.react.bridge.ReadableMap
11+
import com.facebook.react.modules.core.DeviceEventManagerModule
1112
import com.livekit.reactnative.audio.AudioDeviceKind
1213
import com.livekit.reactnative.audio.AudioManagerUtils
1314
import com.livekit.reactnative.audio.AudioSwitchManager
15+
import com.livekit.reactnative.audio.events.Events
1416
import com.livekit.reactnative.audio.processing.AudioSinkManager
17+
import com.livekit.reactnative.audio.processing.AudioSinkProcessor
1518
import com.livekit.reactnative.audio.processing.MultibandVolumeProcessor
1619
import com.livekit.reactnative.audio.processing.VolumeProcessor
1720
import com.oney.WebRTCModule.WebRTCModuleOptions
1821
import org.webrtc.audio.WebRtcAudioTrackHelper
22+
import java.lang.Thread.sleep
23+
import kotlin.concurrent.thread
1924
import kotlin.time.Duration.Companion.milliseconds
2025

2126
// NOTE: As of 0.80 react-native new architecture requires all
@@ -134,6 +139,23 @@ class LivekitReactNativeModule(reactContext: ReactApplicationContext) : ReactCon
134139
promise.resolve(null)
135140
}
136141

142+
@ReactMethod(isBlockingSynchronousMethod = true)
143+
fun createAudioSinkListener(pcId: Int, trackId: String): String {
144+
val processor = AudioSinkProcessor(reactApplicationContext)
145+
val reactTag = audioSinkManager.registerSink(processor)
146+
audioSinkManager.attachSinkToTrack(processor, pcId, trackId)
147+
processor.reactTag = reactTag
148+
149+
return reactTag
150+
}
151+
152+
@ReactMethod(isBlockingSynchronousMethod = true)
153+
fun deleteAudioSinkListener(reactTag: String, pcId: Int, trackId: String): Boolean {
154+
audioSinkManager.detachSinkFromTrack(reactTag, pcId, trackId)
155+
audioSinkManager.unregisterSink(reactTag)
156+
return true
157+
}
158+
137159
@ReactMethod(isBlockingSynchronousMethod = true)
138160
fun createVolumeProcessor(pcId: Int, trackId: String): String {
139161
val processor = VolumeProcessor(reactApplicationContext)

android/src/main/java/com/livekit/reactnative/audio/events/Events.kt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,5 @@ package com.livekit.reactnative.audio.events
33
enum class Events {
44
LK_VOLUME_PROCESSED,
55
LK_MULTIBAND_PROCESSED,
6+
LK_AUDIO_DATA,
67
}
Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
package com.livekit.reactnative.audio.processing
2+
3+
import com.facebook.react.bridge.Arguments
4+
import com.facebook.react.bridge.ReactContext
5+
import com.facebook.react.modules.core.DeviceEventManagerModule
6+
import com.livekit.reactnative.audio.events.Events
7+
import org.webrtc.AudioTrackSink
8+
import java.nio.ByteBuffer
9+
import java.util.Arrays
10+
import kotlin.io.encoding.Base64
11+
import kotlin.io.encoding.ExperimentalEncodingApi
12+
13+
class AudioSinkProcessor(private val reactContext: ReactContext) : BaseAudioSinkProcessor() {
14+
var reactTag: String? = null
15+
16+
@OptIn(ExperimentalEncodingApi::class)
17+
override fun onAudioData(byteArray: ByteArray) {
18+
val reactTag = this.reactTag ?: return
19+
20+
val encodedString = Base64.encode(byteArray)
21+
val event = Arguments.createMap().apply {
22+
putString("data", encodedString)
23+
putString("id", reactTag)
24+
}
25+
reactContext.getJSModule(DeviceEventManagerModule.RCTDeviceEventEmitter::class.java)
26+
.emit(Events.LK_AUDIO_DATA.name, event)
27+
}
28+
}
29+
30+
abstract class BaseAudioSinkProcessor : AudioTrackSink {
31+
abstract fun onAudioData(byteArray: ByteArray)
32+
33+
override fun onData(
34+
audioData: ByteBuffer,
35+
bitsPerSample: Int,
36+
sampleRate: Int,
37+
numberOfChannels: Int,
38+
numberOfFrames: Int,
39+
absoluteCaptureTimestampMs: Long
40+
) {
41+
val byteArray: ByteArray
42+
43+
if (audioData.hasArray()) {
44+
val audioArray = audioData.array()
45+
byteArray = Arrays.copyOfRange(audioArray, audioData.arrayOffset(), audioArray.size)
46+
} else {
47+
audioData.mark()
48+
audioData.position(0)
49+
50+
byteArray = ByteArray(audioData.remaining())
51+
audioData.get(byteArray)
52+
audioData.reset()
53+
}
54+
55+
onAudioData(byteArray)
56+
}
57+
}

ci/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
"dependencies": {
1414
"@livekit/react-native": "*",
1515
"@livekit/react-native-webrtc": "^125.0.12",
16-
"livekit-client": "^2.9.8",
16+
"livekit-client": "^2.15.4",
1717
"react": "18.2.0",
1818
"react-native": "0.74.2"
1919
},

ci/yarn.lock

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2165,12 +2165,12 @@ __metadata:
21652165
languageName: node
21662166
linkType: hard
21672167

2168-
"@livekit/protocol@npm:1.34.0":
2169-
version: 1.34.0
2170-
resolution: "@livekit/protocol@npm:1.34.0"
2168+
"@livekit/protocol@npm:1.39.3":
2169+
version: 1.39.3
2170+
resolution: "@livekit/protocol@npm:1.39.3"
21712171
dependencies:
21722172
"@bufbuild/protobuf": ^1.10.0
2173-
checksum: e08d0ceb20e5434b352e0c42f89d13f29edbf853a968924aea9792660d79edc13a535288d05a47da97843e1e3a66917adb79ee51b8652442d42123e5f75f649c
2173+
checksum: 72aa3b9e82203ccec039047518d841d65b10dcc11ab9d661f269eab8a8672aed7b2d072f4f09cc6fb94a2579542b0a9216d7f66476d34cd1675c13e1ebcc119f
21742174
languageName: node
21752175
linkType: hard
21762176

@@ -3883,7 +3883,7 @@ __metadata:
38833883
eslint: ^8.19.0
38843884
install-local: ^3.0.1
38853885
jest: ^29.6.3
3886-
livekit-client: ^2.9.8
3886+
livekit-client: ^2.15.4
38873887
prettier: 2.8.8
38883888
react: 18.2.0
38893889
react-native: 0.74.2
@@ -6956,20 +6956,22 @@ __metadata:
69566956
languageName: node
69576957
linkType: hard
69586958

6959-
"livekit-client@npm:^2.9.8":
6960-
version: 2.9.8
6961-
resolution: "livekit-client@npm:2.9.8"
6959+
"livekit-client@npm:^2.15.4":
6960+
version: 2.15.4
6961+
resolution: "livekit-client@npm:2.15.4"
69626962
dependencies:
69636963
"@livekit/mutex": 1.1.1
6964-
"@livekit/protocol": 1.34.0
6964+
"@livekit/protocol": 1.39.3
69656965
events: ^3.3.0
69666966
loglevel: ^1.9.2
69676967
sdp-transform: ^2.15.0
69686968
ts-debounce: ^4.0.0
69696969
tslib: 2.8.1
69706970
typed-emitter: ^2.1.0
69716971
webrtc-adapter: ^9.0.1
6972-
checksum: cc01e43fbf5fa298c6696a3a084852d04cd818c554e9b3b9469ad22e85b8d29a975d96072d8432aff79651337f2950881c266117d0490c8ab94112dfda789ca4
6972+
peerDependencies:
6973+
"@types/dom-mediacapture-record": ^1
6974+
checksum: 7ae2e2f5326c8f7925b359277fd12ddeefb0bc5069b0563a9e184f2b860351bf91733c1fa5fd7b65467b643776368058d485af888e2771fd68c61c409b950e05
69736975
languageName: node
69746976
linkType: hard
69756977

example/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
"@react-navigation/native": "^6.0.8",
1616
"@react-navigation/native-stack": "^6.5.0",
1717
"@supersami/rn-foreground-service": "^2.2.5",
18-
"livekit-client": "^2.9.8",
18+
"livekit-client": "^2.15.4",
1919
"react": "18.2.0",
2020
"react-native": "0.74.2",
2121
"react-native-callkeep": "^4.3.14",

example/yarn.lock

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2163,12 +2163,12 @@ __metadata:
21632163
languageName: node
21642164
linkType: hard
21652165

2166-
"@livekit/protocol@npm:1.39.1":
2167-
version: 1.39.1
2168-
resolution: "@livekit/protocol@npm:1.39.1"
2166+
"@livekit/protocol@npm:1.39.3":
2167+
version: 1.39.3
2168+
resolution: "@livekit/protocol@npm:1.39.3"
21692169
dependencies:
21702170
"@bufbuild/protobuf": ^1.10.0
2171-
checksum: 2ca25994ac28c05c618272029f4af82197897545096402548b5ae75b2bd97de993d345b0cca902d2b0fb4f0110602c667844a7a540824d6a12b726f6f495b988
2171+
checksum: 72aa3b9e82203ccec039047518d841d65b10dcc11ab9d661f269eab8a8672aed7b2d072f4f09cc6fb94a2579542b0a9216d7f66476d34cd1675c13e1ebcc119f
21722172
languageName: node
21732173
linkType: hard
21742174

@@ -6102,12 +6102,12 @@ __metadata:
61026102
languageName: node
61036103
linkType: hard
61046104

6105-
"livekit-client@npm:^2.9.8":
6106-
version: 2.13.5
6107-
resolution: "livekit-client@npm:2.13.5"
6105+
"livekit-client@npm:^2.15.4":
6106+
version: 2.15.4
6107+
resolution: "livekit-client@npm:2.15.4"
61086108
dependencies:
61096109
"@livekit/mutex": 1.1.1
6110-
"@livekit/protocol": 1.39.1
6110+
"@livekit/protocol": 1.39.3
61116111
events: ^3.3.0
61126112
loglevel: ^1.9.2
61136113
sdp-transform: ^2.15.0
@@ -6117,7 +6117,7 @@ __metadata:
61176117
webrtc-adapter: ^9.0.1
61186118
peerDependencies:
61196119
"@types/dom-mediacapture-record": ^1
6120-
checksum: aaff2e0729a7dd51b8641153043ab77825614c12af80662c4f1b98b713af7b7d2565210eed1a56c9e068c45afe4b03bb60a49de02117769868b516a9d1a3a1de
6120+
checksum: 7ae2e2f5326c8f7925b359277fd12ddeefb0bc5069b0563a9e184f2b860351bf91733c1fa5fd7b65467b643776368058d485af888e2771fd68c61c409b950e05
61216121
languageName: node
61226122
linkType: hard
61236123

@@ -6140,7 +6140,7 @@ __metadata:
61406140
"@types/fastestsmallesttextencoderdecoder": ^1.0.0
61416141
"@types/react": ^18.2.6
61426142
babel-plugin-module-resolver: ^4.1.0
6143-
livekit-client: ^2.9.8
6143+
livekit-client: ^2.15.4
61446144
patch-package: ^8.0.0
61456145
react: 18.2.0
61466146
react-native: 0.74.2

ios/LiveKitReactNativeModule.swift

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import React
66
struct LKEvents {
77
static let kEventVolumeProcessed = "LK_VOLUME_PROCESSED";
88
static let kEventMultibandProcessed = "LK_MULTIBAND_PROCESSED";
9+
static let kEventAudioData = "LK_AUDIO_DATA";
910
}
1011

1112
@objc(LivekitReactNativeModule)
@@ -178,6 +179,24 @@ public class LivekitReactNativeModule: RCTEventEmitter {
178179
session.unlockForConfiguration()
179180
}
180181

182+
@objc(createAudioSinkListener:trackId:)
183+
public func createAudioSinkListener(_ pcId: NSNumber, trackId: String) -> String {
184+
let renderer = AudioSinkRenderer(eventEmitter: self)
185+
let reactTag = self.audioRendererManager.registerRenderer(renderer)
186+
renderer.reactTag = reactTag
187+
self.audioRendererManager.attach(renderer: renderer, pcId: pcId, trackId: trackId)
188+
189+
return reactTag
190+
}
191+
192+
@objc(deleteAudioSinkListener:pcId:trackId:)
193+
public func deleteAudioSinkListener(_ reactTag: String, pcId: NSNumber, trackId: String) -> Any? {
194+
self.audioRendererManager.detach(rendererByTag: reactTag, pcId: pcId, trackId: trackId)
195+
self.audioRendererManager.unregisterRenderer(forReactTag: reactTag)
196+
197+
return nil
198+
}
199+
181200
@objc(createVolumeProcessor:trackId:)
182201
public func createVolumeProcessor(_ pcId: NSNumber, trackId: String) -> String {
183202
let renderer = VolumeAudioRenderer(intervalMs: 40.0, eventEmitter: self)
@@ -195,7 +214,7 @@ public class LivekitReactNativeModule: RCTEventEmitter {
195214

196215
return nil
197216
}
198-
217+
199218
@objc(createMultibandVolumeProcessor:pcId:trackId:)
200219
public func createMultibandVolumeProcessor(_ options: NSDictionary, pcId: NSNumber, trackId: String) -> String {
201220
let bands = (options["bands"] as? NSNumber)?.intValue ?? 5
@@ -237,6 +256,7 @@ public class LivekitReactNativeModule: RCTEventEmitter {
237256
return [
238257
LKEvents.kEventVolumeProcessed,
239258
LKEvents.kEventMultibandProcessed,
259+
LKEvents.kEventAudioData,
240260
]
241261
}
242262
}

ios/LivekitReactNativeModule.m

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,12 @@ @interface RCT_EXTERN_MODULE(LivekitReactNativeModule, RCTEventEmitter)
2121
/// Configure audio config for WebRTC
2222
RCT_EXTERN_METHOD(setAppleAudioConfiguration:(NSDictionary *) configuration)
2323

24+
RCT_EXTERN__BLOCKING_SYNCHRONOUS_METHOD(createAudioSinkListener:(nonnull NSNumber *)pcId
25+
trackId:(nonnull NSString *)trackId)
26+
27+
RCT_EXTERN__BLOCKING_SYNCHRONOUS_METHOD(deleteAudioSinkListener:(nonnull NSString *)reactTag
28+
pcId:(nonnull NSNumber *)pcId
29+
trackId:(nonnull NSString *)trackId)
2430

2531
RCT_EXTERN__BLOCKING_SYNCHRONOUS_METHOD(createVolumeProcessor:(nonnull NSNumber *)pcId
2632
trackId:(nonnull NSString *)trackId)

ios/audio/AudioSinkRenderer.swift

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
import livekit_react_native_webrtc
2+
import React
3+
4+
@objc
5+
public class AudioSinkRenderer: BaseAudioSinkRenderer {
6+
private let eventEmitter: RCTEventEmitter
7+
8+
@objc
9+
public var reactTag: String? = nil
10+
11+
@objc
12+
public init(eventEmitter: RCTEventEmitter) {
13+
self.eventEmitter = eventEmitter
14+
super.init()
15+
}
16+
17+
override public func onData(_ pcmBuffer: AVAudioPCMBuffer) {
18+
guard pcmBuffer.format.commonFormat == .pcmFormatInt16,
19+
let channelData = pcmBuffer.int16ChannelData else {
20+
return
21+
}
22+
let channelCount = Int(pcmBuffer.format.channelCount)
23+
let channels = UnsafeBufferPointer(start: channelData, count: channelCount)
24+
let length = Int(pcmBuffer.frameCapacity * pcmBuffer.format.streamDescription.pointee.mBytesPerFrame)
25+
let data = NSData(bytes: channels[0], length: length)
26+
let base64 = data.base64EncodedString()
27+
NSLog("AUDIO DATA!!!!")
28+
NSLog("\(data.length)")
29+
NSLog(base64)
30+
NSLog("\(base64.count)")
31+
NSLog("\(length)")
32+
eventEmitter.sendEvent(withName: LKEvents.kEventAudioData, body: [
33+
"data": base64,
34+
"id": reactTag
35+
])
36+
}
37+
}
38+
39+
public class BaseAudioSinkRenderer: NSObject, RTCAudioRenderer {
40+
41+
public override init() {
42+
super.init()
43+
}
44+
45+
public func render(pcmBuffer: AVAudioPCMBuffer) {
46+
onData(pcmBuffer)
47+
}
48+
49+
public func onData(_ pcmBuffer: AVAudioPCMBuffer) {
50+
}
51+
}

0 commit comments

Comments
 (0)