Files
buzz/mobile/ios/Runner/HuddleAudioEngine.swift
klopez4212 8c0f42e187 Add mobile Huddles voice MVP (#6056)
## Summary
- add foreground mobile Huddles on Android and iOS with native Opus
capture/playback, mute, speaker routing, participants, lifecycle, and
minimized drawer UI
- keep mobile Huddle cards and roster state live, including ended rooms,
relay-resolved profiles, and agents
- broadcast desktop agent TTS through the existing Huddle audio protocol

## Scope
Foreground human-to-human voice MVP only. Agent setup/transcripts,
background calling, recording, and advanced device controls remain out
of scope.

## Validation
- `just mobile-check`
- `just mobile-test` — 1,500 passed
- `just desktop-check` and `just desktop-test` — 4,957 passed
- desktop typecheck, strict Clippy, and Tauri tests — 2,445 passed, 15
ignored
- mobile worktree identity contract checks
- physical Pixel/iPhone behavior reviewed during development

---------

Signed-off-by: kenny lopez <klopez4212@gmail.com>
Signed-off-by: Kenny Lopez <klopez4212@gmail.com>
Signed-off-by: Princess Donut <b238ea756dee4d98afa5883fc7f1de61eeabe65bf700e3a5a5a80db5e42e2c2b@buzz.block.builderlab.xyz>
Signed-off-by: Tom Brow <tomb@block.xyz>
Co-authored-by: Carl <3c4caeafb646d23867f1c4832e68211d77e2561946171625f75c3ce1a3f2670f@buzz.block.builderlab.xyz>
Co-authored-by: Princess Donut <b238ea756dee4d98afa5883fc7f1de61eeabe65bf700e3a5a5a80db5e42e2c2b@buzz.block.builderlab.xyz>
Co-authored-by: leader <71e9f2c44a6932b6772caaaccda1911d010463c3e2c6c40410b8329956046801@buzz.block.builderlab.xyz>
Co-authored-by: Tom Brow <tomb@block.xyz>
Co-authored-by: Mongo <9cfd347903944d5b85aa6c93d2ab67381b978a92a31914bca69998968752a1d7@buzz.block.builderlab.xyz>
2026-08-22 05:28:16 -07:00

1014 lines
29 KiB
Swift

import AVFoundation
import Foundation
struct HuddleLocalOpusPacket {
let sequence: Int
let timestamp48k: Int
let levelDbov: Int
let flags: Int
let opus: Data
}
struct HuddleRemoteOpusPacket {
let peerIndex: Int
let sequence: Int
let timestamp48k: Int64
let levelDbov: Int
let opus: Data
}
struct HuddlePacketJitterQueue {
private(set) var packets: [HuddleRemoteOpusPacket] = []
private var lastDrainedSequence: Int?
private var started = false
let capacity: Int
let startPackets: Int
init(capacity: Int, startPackets: Int) {
self.capacity = capacity
self.startPackets = startPackets
}
mutating func enqueue(_ packet: HuddleRemoteOpusPacket) {
if let last = lastDrainedSequence, !sequenceAfter(packet.sequence, last) {
return
}
guard !packets.contains(where: { $0.sequence == packet.sequence }) else {
return
}
let index = packets.firstIndex {
sequenceBefore(packet.sequence, $0.sequence)
} ?? packets.endIndex
packets.insert(packet, at: index)
if packets.count > capacity { packets.removeFirst() }
if packets.count >= startPackets { started = true }
}
mutating func drainOne() -> HuddleRemoteOpusPacket? {
guard started, !packets.isEmpty else { return nil }
let packet = packets.removeFirst()
lastDrainedSequence = packet.sequence
return packet
}
private func sequenceAfter(_ sequence: Int, _ reference: Int) -> Bool {
let distance = (sequence - reference) & 0xffff
return distance > 0 && distance <= 0x7fff
}
private func sequenceBefore(_ left: Int, _ right: Int) -> Bool {
sequenceAfter(right, left)
}
}
struct HuddleCaptureDiagnostics {
let frameCount: Int64
let rmsDbovHistogram: [String: Int64]
let peakDbovHistogram: [String: Int64]
let maxPeakDbov: Int
let deviceLabel: String?
let voiceProcessingEnabled: Bool
}
enum HuddleNativeMediaError: LocalizedError {
case invalidState(String)
case unsupported(String)
case conversion(String)
case malformedPacket(String)
var errorDescription: String? {
switch self {
case .invalidState(let message),
.unsupported(let message),
.conversion(let message),
.malformedPacket(let message):
message
}
}
}
enum HuddleAudioFormats {
static let sampleRate = 48_000.0
static let channels: AVAudioChannelCount = 1
static let frameSamples = 960
static let maximumOpusPacketBytes = 4_088
static func makePCM() throws -> AVAudioFormat {
guard
let format = AVAudioFormat(
commonFormat: .pcmFormatFloat32,
sampleRate: sampleRate,
channels: channels,
interleaved: false
)
else {
throw HuddleNativeMediaError.unsupported(
"iOS does not expose 48 kHz mono PCM audio."
)
}
return format
}
static func makeOpus() throws -> AVAudioFormat {
var description = AudioStreamBasicDescription(
mSampleRate: sampleRate,
mFormatID: kAudioFormatOpus,
mFormatFlags: 0,
mBytesPerPacket: 0,
mFramesPerPacket: UInt32(frameSamples),
mBytesPerFrame: 0,
mChannelsPerFrame: channels,
mBitsPerChannel: 0,
mReserved: 0
)
guard let format = AVAudioFormat(streamDescription: &description) else {
throw HuddleNativeMediaError.unsupported(
"iOS does not expose the native Opus audio format."
)
}
return format
}
}
final class HuddleOpusEncoder {
private let pcmFormat: AVAudioFormat
private let opusFormat: AVAudioFormat
private let converter: AVAudioConverter
private let maximumPacketSize: Int
init() throws {
let pcmFormat = try HuddleAudioFormats.makePCM()
let opusFormat = try HuddleAudioFormats.makeOpus()
guard let converter = AVAudioConverter(from: pcmFormat, to: opusFormat) else {
throw HuddleNativeMediaError.unsupported(
"This iOS device does not expose an Opus encoder."
)
}
converter.bitRate = 32_000
converter.primeMethod = .none
self.pcmFormat = pcmFormat
self.opusFormat = opusFormat
self.converter = converter
maximumPacketSize = max(1_275, converter.maximumOutputPacketSize)
}
func encode(_ samples: [Float]) throws -> Data {
guard samples.count == HuddleAudioFormats.frameSamples else {
throw HuddleNativeMediaError.conversion(
"Opus input must contain exactly 960 samples."
)
}
guard
let pcm = AVAudioPCMBuffer(
pcmFormat: pcmFormat,
frameCapacity: AVAudioFrameCount(HuddleAudioFormats.frameSamples)
),
let channel = pcm.floatChannelData?.pointee
else {
throw HuddleNativeMediaError.conversion(
"Unable to allocate the iOS Opus input buffer."
)
}
pcm.frameLength = AVAudioFrameCount(HuddleAudioFormats.frameSamples)
for index in samples.indices {
channel[index] = samples[index]
}
let compressed = AVAudioCompressedBuffer(
format: opusFormat,
packetCapacity: 1,
maximumPacketSize: maximumPacketSize
)
var suppliedInput = false
var conversionError: NSError?
let status = converter.convert(
to: compressed,
error: &conversionError
) { _, inputStatus in
guard !suppliedInput else {
inputStatus.pointee = .noDataNow
return nil
}
suppliedInput = true
inputStatus.pointee = .haveData
return pcm
}
if status == .error || conversionError != nil {
throw HuddleNativeMediaError.conversion(
conversionError?.localizedDescription ?? "iOS Opus encoding failed."
)
}
let byteCount = Int(compressed.byteLength)
guard
compressed.packetCount == 1,
byteCount > 0,
byteCount <= HuddleAudioFormats.maximumOpusPacketBytes
else {
throw HuddleNativeMediaError.conversion(
"iOS produced an invalid Opus packet."
)
}
return Data(bytes: compressed.data, count: byteCount)
}
}
final class HuddleOpusDecoder {
private let pcmFormat: AVAudioFormat
private let opusFormat: AVAudioFormat
private let converter: AVAudioConverter
init() throws {
let pcmFormat = try HuddleAudioFormats.makePCM()
let opusFormat = try HuddleAudioFormats.makeOpus()
guard let converter = AVAudioConverter(from: opusFormat, to: pcmFormat) else {
throw HuddleNativeMediaError.unsupported(
"This iOS device does not expose an Opus decoder."
)
}
converter.primeMethod = .none
self.pcmFormat = pcmFormat
self.opusFormat = opusFormat
self.converter = converter
}
func decode(_ packet: Data) throws -> AVAudioPCMBuffer {
guard
!packet.isEmpty,
packet.count <= HuddleAudioFormats.maximumOpusPacketBytes
else {
throw HuddleNativeMediaError.malformedPacket(
"Remote Huddle Opus packet has an invalid length."
)
}
let compressed = AVAudioCompressedBuffer(
format: opusFormat,
packetCapacity: 1,
maximumPacketSize: HuddleAudioFormats.maximumOpusPacketBytes
)
packet.withUnsafeBytes { bytes in
guard let baseAddress = bytes.baseAddress else { return }
compressed.data.copyMemory(from: baseAddress, byteCount: packet.count)
}
compressed.byteLength = UInt32(packet.count)
compressed.packetCount = 1
if let descriptions = compressed.packetDescriptions {
descriptions.pointee = AudioStreamPacketDescription(
mStartOffset: 0,
mVariableFramesInPacket: UInt32(HuddleAudioFormats.frameSamples),
mDataByteSize: UInt32(packet.count)
)
}
guard
let pcm = AVAudioPCMBuffer(
pcmFormat: pcmFormat,
frameCapacity: AVAudioFrameCount(HuddleAudioFormats.frameSamples)
)
else {
throw HuddleNativeMediaError.conversion(
"Unable to allocate the iOS Opus output buffer."
)
}
var suppliedInput = false
var conversionError: NSError?
let status = converter.convert(
to: pcm,
error: &conversionError
) { _, inputStatus in
guard !suppliedInput else {
inputStatus.pointee = .noDataNow
return nil
}
suppliedInput = true
inputStatus.pointee = .haveData
return compressed
}
if status == .error || conversionError != nil || pcm.frameLength == 0 {
throw HuddleNativeMediaError.conversion(
conversionError?.localizedDescription ?? "iOS Opus decoding failed."
)
}
return pcm
}
}
enum HuddleAudioLevels {
static func rmsDbov(_ samples: [Float]) -> Int {
guard !samples.isEmpty else { return -127 }
let squareSum = samples.reduce(0.0) { partial, sample in
partial + Double(sample * sample)
}
let meanSquare = squareSum / Double(samples.count)
guard meanSquare > 0 else { return -127 }
return min(0, max(-127, Int((10 * log10(meanSquare)).rounded())))
}
static func peakDbov(_ samples: [Float]) -> Int {
let peak = samples.reduce(Float.zero) { max($0, abs($1)) }
guard peak > 0 else { return -127 }
return min(0, max(-127, Int((20 * log10(Double(peak))).rounded())))
}
}
struct HuddleTalkerSelection: Equatable {
let accepted: Bool
let evictedPeerIndex: Int?
func allocateIfAccepted<T>(_ allocate: () throws -> T) rethrows -> T? {
guard accepted else { return nil }
return try allocate()
}
}
/// Fixed-capacity active-talker selector. The UI roster is independent; native
/// decoder/player resources exist only for peers that actually send packets.
struct HuddleActiveTalkerSelector {
struct Activity {
let peerIndex: Int
let lastPacketOrdinal: UInt64
let lastPacketAt: TimeInterval
let levelDbov: Int
}
let capacity: Int
let now: () -> TimeInterval
let inactivityTimeout: TimeInterval
private(set) var active: [Int: Activity] = [:]
private var ordinal: UInt64 = 0
init(
capacity: Int,
now: @escaping () -> TimeInterval = { ProcessInfo.processInfo.systemUptime },
inactivityTimeout: TimeInterval = 1
) {
self.capacity = capacity
self.now = now
self.inactivityTimeout = inactivityTimeout
}
mutating func activate(peerIndex: Int, levelDbov: Int) -> HuddleTalkerSelection {
ordinal &+= 1
let packetAt = now()
if active[peerIndex] != nil {
active[peerIndex] = Activity(
peerIndex: peerIndex,
lastPacketOrdinal: ordinal,
lastPacketAt: packetAt,
levelDbov: levelDbov
)
return HuddleTalkerSelection(accepted: true, evictedPeerIndex: nil)
}
let weakest = active.count >= capacity
? active.values.min { left, right in
if left.levelDbov != right.levelDbov {
return left.levelDbov < right.levelDbov
}
if left.lastPacketOrdinal != right.lastPacketOrdinal {
return left.lastPacketOrdinal < right.lastPacketOrdinal
}
return left.peerIndex < right.peerIndex
}
: nil
let expired = active.values
.filter { packetAt - $0.lastPacketAt >= inactivityTimeout }
.min { left, right in
if left.lastPacketAt != right.lastPacketAt {
return left.lastPacketAt < right.lastPacketAt
}
return left.peerIndex < right.peerIndex
}
// Equal-level packets (especially continuous -127 dBov silence) must not
// churn decoder/jitter state. Admit a new peer when any selected slot is
// inactive or the new packet is louder than the quietest selected peer.
if let weakest, expired == nil, levelDbov <= weakest.levelDbov {
return HuddleTalkerSelection(accepted: false, evictedPeerIndex: nil)
}
let evicted = expired?.peerIndex ?? weakest?.peerIndex
if let evicted { active.removeValue(forKey: evicted) }
active[peerIndex] = Activity(
peerIndex: peerIndex,
lastPacketOrdinal: ordinal,
lastPacketAt: packetAt,
levelDbov: levelDbov
)
return HuddleTalkerSelection(accepted: true, evictedPeerIndex: evicted)
}
mutating func remove(_ peerIndex: Int) {
active.removeValue(forKey: peerIndex)
}
mutating func removeAll() {
active.removeAll()
ordinal = 0
}
}
/// Foreground-only iOS realtime media engine for the fixed Huddle Opus v2 path.
///
/// AVAudioEngine owns voice-processed capture and per-peer mixed playout.
/// AVAudioConverter keeps PCM native and moves only Opus packets across Flutter.
final class HuddleAudioEngine {
private let onLocalPacket: (HuddleLocalOpusPacket) -> Void
private let onFailure: (String, String) -> Void
private let onDiagnostics: (HuddleCaptureDiagnostics) -> Void
private let diagnosticsEnabled: Bool
private let audioEngine = AVAudioEngine()
private let processingQueue = DispatchQueue(
label: "xyz.block.buzz.huddle.ios-media",
qos: .userInteractive
)
private let stateLock = NSLock()
private let pcmFormat: AVAudioFormat
private let encoder: HuddleOpusEncoder
private var running = false
private var muted = false
private var interrupted = false
private var failureReported = false
private var pendingCaptureBuffers = 0
private var pendingRemotePackets = 0
private var captureConverter: AVAudioConverter?
private var captureSourceFormat: AVAudioFormat?
private var captureTapInstalled = false
private var captureSamples: [Float] = []
private var captureSampleOffset = 0
private var sequence = 0
private var frameIndex: Int64 = 0
private var diagnosticFrameCount: Int64 = 0
private var diagnosticMaxPeakDbov = -127
private var rmsDbovHistogram: [Int: Int64] = [:]
private var peakDbovHistogram: [Int: Int64] = [:]
private var peerPlaybacks: [Int: HuddlePeerPlayback] = [:]
private var activeTalkers = HuddleActiveTalkerSelector(capacity: 15)
private var playbackTimer: DispatchSourceTimer?
private var configurationObserver: NSObjectProtocol?
init(
onLocalPacket: @escaping (HuddleLocalOpusPacket) -> Void,
onFailure: @escaping (String, String) -> Void,
onDiagnostics: @escaping (HuddleCaptureDiagnostics) -> Void,
diagnosticsEnabled: Bool
) throws {
pcmFormat = try HuddleAudioFormats.makePCM()
encoder = try HuddleOpusEncoder()
self.onLocalPacket = onLocalPacket
self.onFailure = onFailure
self.onDiagnostics = onDiagnostics
self.diagnosticsEnabled = diagnosticsEnabled
configurationObserver = NotificationCenter.default.addObserver(
forName: .AVAudioEngineConfigurationChange,
object: audioEngine,
queue: nil
) { [weak self] _ in
self?.processingQueue.async {
self?.restartAfterConfigurationChange()
}
}
}
deinit {
stop()
if let configurationObserver {
NotificationCenter.default.removeObserver(configurationObserver)
}
}
static func isSupported() -> Bool {
do {
_ = try HuddleOpusEncoder()
_ = try HuddleOpusDecoder()
return true
} catch {
return false
}
}
func start() throws {
var startError: Error?
processingQueue.sync {
do {
try startOnQueue()
} catch {
startError = error
}
}
if let startError { throw startError }
}
func setMuted(_ value: Bool) throws {
stateLock.lock()
defer { stateLock.unlock() }
guard running else {
throw HuddleNativeMediaError.invalidState(
"Huddle audio is not running."
)
}
muted = value
}
func setInterrupted(_ value: Bool) {
stateLock.lock()
let shouldUpdate = running
interrupted = value
stateLock.unlock()
guard shouldUpdate else { return }
processingQueue.async { [weak self] in
guard let self else { return }
if value {
for playback in peerPlaybacks.values {
playback.pause()
}
return
}
do {
if !audioEngine.isRunning {
try audioEngine.start()
}
for playback in peerPlaybacks.values {
playback.resume()
}
} catch {
reportFailure(code: "audio_resume_failed", error: error)
}
}
}
func enqueueRemote(_ packet: HuddleRemoteOpusPacket) throws {
stateLock.lock()
guard running else {
stateLock.unlock()
throw HuddleNativeMediaError.invalidState(
"Huddle audio is not running."
)
}
guard pendingRemotePackets < 50 else {
stateLock.unlock()
return
}
pendingRemotePackets += 1
stateLock.unlock()
processingQueue.async { [weak self] in
guard let self else { return }
defer { completeRemotePacket() }
do {
let selection = activeTalkers.activate(
peerIndex: packet.peerIndex,
levelDbov: packet.levelDbov
)
if let evictedIndex = selection.evictedPeerIndex,
let evicted = peerPlaybacks.removeValue(forKey: evictedIndex)
{
evicted.release(from: audioEngine)
}
let playback: HuddlePeerPlayback
if let existing = peerPlaybacks[packet.peerIndex] {
playback = existing
} else if let allocated = try selection.allocateIfAccepted({
try HuddlePeerPlayback(
engine: self.audioEngine,
pcmFormat: self.pcmFormat
)
}) {
playback = allocated
peerPlaybacks[packet.peerIndex] = allocated
} else {
return
}
playback.enqueue(packet)
} catch {
reportFailure(code: "playback_failed", error: error)
}
}
}
private func completeRemotePacket() {
stateLock.lock()
pendingRemotePackets = max(0, pendingRemotePackets - 1)
stateLock.unlock()
}
func removeRemotePeer(_ peerIndex: Int) {
processingQueue.async { [weak self] in
guard let self else { return }
activeTalkers.remove(peerIndex)
guard let playback = peerPlaybacks.removeValue(forKey: peerIndex)
else { return }
playback.release(from: audioEngine)
}
}
func stop() {
processingQueue.sync {
stopOnQueue()
}
}
private func startOnQueue() throws {
stateLock.lock()
let wasRunning = running
if !wasRunning {
running = true
muted = false
interrupted = false
failureReported = false
}
stateLock.unlock()
guard !wasRunning else {
throw HuddleNativeMediaError.invalidState(
"Huddle audio is already running."
)
}
do {
let input = audioEngine.inputNode
try input.setVoiceProcessingEnabled(true)
let inputFormat = input.outputFormat(forBus: 0)
guard inputFormat.sampleRate > 0, inputFormat.channelCount > 0 else {
throw HuddleNativeMediaError.unsupported(
"The iOS microphone does not expose an input format."
)
}
input.installTap(
onBus: 0,
bufferSize: AVAudioFrameCount(HuddleAudioFormats.frameSamples),
format: inputFormat
) { [weak self] buffer, _ in
self?.acceptCapturedBuffer(buffer)
}
captureTapInstalled = true
audioEngine.prepare()
try audioEngine.start()
startPlaybackTimer()
if diagnosticsEnabled {
onDiagnostics(makeDiagnostics())
}
} catch {
stopOnQueue()
throw error
}
}
private func stopOnQueue() {
stateLock.lock()
running = false
muted = false
interrupted = false
pendingCaptureBuffers = 0
pendingRemotePackets = 0
stateLock.unlock()
playbackTimer?.cancel()
playbackTimer = nil
if captureTapInstalled {
audioEngine.inputNode.removeTap(onBus: 0)
captureTapInstalled = false
}
for playback in peerPlaybacks.values {
playback.release(from: audioEngine)
}
peerPlaybacks.removeAll()
activeTalkers.removeAll()
audioEngine.stop()
audioEngine.reset()
try? audioEngine.inputNode.setVoiceProcessingEnabled(false)
captureConverter = nil
captureSourceFormat = nil
captureSamples.removeAll(keepingCapacity: false)
captureSampleOffset = 0
sequence = 0
frameIndex = 0
diagnosticFrameCount = 0
diagnosticMaxPeakDbov = -127
rmsDbovHistogram.removeAll()
peakDbovHistogram.removeAll()
}
private func acceptCapturedBuffer(_ buffer: AVAudioPCMBuffer) {
stateLock.lock()
guard running, pendingCaptureBuffers < 8 else {
stateLock.unlock()
return
}
pendingCaptureBuffers += 1
stateLock.unlock()
guard let copied = copyPCMBuffer(buffer) else {
completeCaptureBuffer()
return
}
processingQueue.async { [weak self] in
guard let self else { return }
defer { completeCaptureBuffer() }
do {
try processCapturedBuffer(copied)
} catch {
reportFailure(code: "capture_failed", error: error)
}
}
}
private func completeCaptureBuffer() {
stateLock.lock()
pendingCaptureBuffers = max(0, pendingCaptureBuffers - 1)
stateLock.unlock()
}
private func processCapturedBuffer(_ input: AVAudioPCMBuffer) throws {
stateLock.lock()
let isRunning = running
stateLock.unlock()
guard isRunning else { return }
let normalized = try normalizeCapturedBuffer(input)
guard let channel = normalized.floatChannelData?.pointee else {
throw HuddleNativeMediaError.conversion(
"iOS microphone PCM data is unavailable."
)
}
captureSamples.append(
contentsOf: UnsafeBufferPointer(
start: channel,
count: Int(normalized.frameLength)
)
)
while captureSamples.count - captureSampleOffset >= HuddleAudioFormats.frameSamples {
let end = captureSampleOffset + HuddleAudioFormats.frameSamples
let frame = Array(captureSamples[captureSampleOffset..<end])
captureSampleOffset = end
try processCaptureFrame(frame)
}
if captureSampleOffset >= HuddleAudioFormats.frameSamples * 4 {
captureSamples.removeFirst(captureSampleOffset)
captureSampleOffset = 0
}
}
private func normalizeCapturedBuffer(
_ input: AVAudioPCMBuffer
) throws -> AVAudioPCMBuffer {
if matchesTargetPCM(input.format) {
return input
}
if captureSourceFormat?.isEqual(input.format) != true {
guard let converter = AVAudioConverter(from: input.format, to: pcmFormat) else {
throw HuddleNativeMediaError.unsupported(
"iOS cannot convert the active microphone route to Huddle audio."
)
}
converter.primeMethod = .none
captureConverter = converter
captureSourceFormat = input.format
}
guard let converter = captureConverter else {
throw HuddleNativeMediaError.conversion(
"The iOS microphone converter is unavailable."
)
}
let ratio = HuddleAudioFormats.sampleRate / input.format.sampleRate
let capacity = max(
HuddleAudioFormats.frameSamples,
Int(ceil(Double(input.frameLength) * ratio)) + 64
)
guard
let output = AVAudioPCMBuffer(
pcmFormat: pcmFormat,
frameCapacity: AVAudioFrameCount(capacity)
)
else {
throw HuddleNativeMediaError.conversion(
"Unable to allocate converted iOS microphone audio."
)
}
var suppliedInput = false
var conversionError: NSError?
let status = converter.convert(
to: output,
error: &conversionError
) { _, inputStatus in
guard !suppliedInput else {
inputStatus.pointee = .noDataNow
return nil
}
suppliedInput = true
inputStatus.pointee = .haveData
return input
}
if status == .error || conversionError != nil {
throw HuddleNativeMediaError.conversion(
conversionError?.localizedDescription
?? "iOS microphone conversion failed."
)
}
return output
}
private func processCaptureFrame(_ frame: [Float]) throws {
stateLock.lock()
let shouldSend = running && !muted && !interrupted
stateLock.unlock()
guard shouldSend else { return }
let level = HuddleAudioLevels.rmsDbov(frame)
if diagnosticsEnabled {
let peak = HuddleAudioLevels.peakDbov(frame)
diagnosticFrameCount += 1
diagnosticMaxPeakDbov = max(diagnosticMaxPeakDbov, peak)
rmsDbovHistogram[level, default: 0] += 1
peakDbovHistogram[peak, default: 0] += 1
if diagnosticFrameCount % 50 == 0 {
onDiagnostics(makeDiagnostics())
}
}
let opus = try encoder.encode(frame)
let packet = HuddleLocalOpusPacket(
sequence: sequence,
timestamp48k: Int(
UInt32(truncatingIfNeeded: frameIndex * Int64(HuddleAudioFormats.frameSamples))
),
levelDbov: level,
flags: 0,
opus: opus
)
sequence = (sequence + 1) & 0xffff
frameIndex += 1
onLocalPacket(packet)
}
private func startPlaybackTimer() {
let timer = DispatchSource.makeTimerSource(queue: processingQueue)
timer.schedule(
deadline: .now() + .milliseconds(20),
repeating: .milliseconds(20),
leeway: .milliseconds(2)
)
timer.setEventHandler { [weak self] in
self?.drainPlaybackTick()
}
playbackTimer = timer
timer.resume()
}
private func drainPlaybackTick() {
stateLock.lock()
let shouldPlay = running && !interrupted
stateLock.unlock()
guard shouldPlay else { return }
do {
for playback in peerPlaybacks.values {
try playback.drainOne()
}
} catch {
reportFailure(code: "playback_failed", error: error)
}
}
private func restartAfterConfigurationChange() {
stateLock.lock()
let shouldRestart = running && !interrupted
stateLock.unlock()
guard shouldRestart else { return }
do {
if !audioEngine.isRunning {
try audioEngine.start()
}
for playback in peerPlaybacks.values {
playback.resume()
}
} catch {
reportFailure(code: "audio_route_failed", error: error)
}
}
private func makeDiagnostics() -> HuddleCaptureDiagnostics {
let route = AVAudioSession.sharedInstance().currentRoute
return HuddleCaptureDiagnostics(
frameCount: diagnosticFrameCount,
rmsDbovHistogram: numericHistogram(rmsDbovHistogram),
peakDbovHistogram: numericHistogram(peakDbovHistogram),
maxPeakDbov: diagnosticMaxPeakDbov,
deviceLabel: route.inputs.first?.portName,
voiceProcessingEnabled: audioEngine.inputNode.isVoiceProcessingEnabled
)
}
private func reportFailure(code: String, error: Error) {
stateLock.lock()
guard running, !failureReported else {
stateLock.unlock()
return
}
let message =
(error as? LocalizedError)?.errorDescription
?? error.localizedDescription
failureReported = true
stateLock.unlock()
// Fail closed on the native queue. Flutter will perform the durable session
// teardown too, but microphone and playout lifetime must not depend on a
// platform-channel callback making a successful round trip.
stopOnQueue()
onFailure(code, message)
}
private func matchesTargetPCM(_ format: AVAudioFormat) -> Bool {
format.commonFormat == .pcmFormatFloat32
&& format.sampleRate == HuddleAudioFormats.sampleRate
&& format.channelCount == HuddleAudioFormats.channels
&& !format.isInterleaved
}
private func numericHistogram(_ values: [Int: Int64]) -> [String: Int64] {
Dictionary(uniqueKeysWithValues: values.map { (String($0.key), $0.value) })
}
private func copyPCMBuffer(_ source: AVAudioPCMBuffer) -> AVAudioPCMBuffer? {
guard
let copy = AVAudioPCMBuffer(
pcmFormat: source.format,
frameCapacity: source.frameLength
)
else { return nil }
copy.frameLength = source.frameLength
let sourceBuffers = UnsafeMutableAudioBufferListPointer(
source.mutableAudioBufferList
)
let destinationBuffers = UnsafeMutableAudioBufferListPointer(
copy.mutableAudioBufferList
)
guard sourceBuffers.count == destinationBuffers.count else { return nil }
for index in sourceBuffers.indices {
let sourceBuffer = sourceBuffers[index]
let destinationBuffer = destinationBuffers[index]
guard
let sourceData = sourceBuffer.mData,
let destinationData = destinationBuffer.mData
else { return nil }
let byteCount = Int(sourceBuffer.mDataByteSize)
memcpy(destinationData, sourceData, byteCount)
}
return copy
}
}
private final class HuddlePeerPlayback {
private let player: AVAudioPlayerNode
private let decoder: HuddleOpusDecoder
private var jitterQueue = HuddlePacketJitterQueue(
capacity: 10,
startPackets: 3
)
init(
engine: AVAudioEngine,
pcmFormat: AVAudioFormat
) throws {
decoder = try HuddleOpusDecoder()
let player = AVAudioPlayerNode()
self.player = player
engine.attach(player)
do {
engine.connect(player, to: engine.mainMixerNode, format: pcmFormat)
player.play()
} catch {
player.stop()
engine.disconnectNodeOutput(player)
engine.detach(player)
throw error
}
}
func enqueue(_ packet: HuddleRemoteOpusPacket) {
jitterQueue.enqueue(packet)
}
func drainOne() throws {
guard let packet = jitterQueue.drainOne() else { return }
let decoded = try decoder.decode(packet.opus)
player.scheduleBuffer(decoded)
if !player.isPlaying {
player.play()
}
}
func pause() {
if player.isPlaying {
player.pause()
}
}
func resume() {
if !player.isPlaying {
player.play()
}
}
func release(from engine: AVAudioEngine) {
player.stop()
engine.disconnectNodeOutput(player)
engine.detach(player)
jitterQueue = HuddlePacketJitterQueue(capacity: 10, startPackets: 3)
}
}