feat(huddle): fix concurrent-speaker mixing + jitter buffer + protocol v2 (up to 10 peers) (#609)

Signed-off-by: tlongwell-block <109685178+tlongwell-block@users.noreply.github.com>
This commit is contained in:
tlongwell-block
2026-05-18 16:34:52 -04:00
committed by GitHub
parent c0eb5affcf
commit 6e1418e571
17 changed files with 1966 additions and 236 deletions

View File

@@ -65,12 +65,27 @@ pub async fn ws_audio_handler(
// ── Auth message shape ────────────────────────────────────────────────────────
/// Highest huddle audio protocol version this relay understands. Clients are
/// allowed to negotiate any version in `1..=CURRENT_PROTOCOL_VERSION`; older
/// versions stay supported indefinitely for staged rollouts.
const CURRENT_PROTOCOL_VERSION: u8 = 2;
#[derive(Deserialize)]
struct AuthMsg {
#[serde(rename = "type")]
msg_type: String,
event: nostr::Event,
parent_channel_id: Option<Uuid>,
/// Huddle audio protocol version requested by the client. Defaults to 1
/// when missing so existing clients keep working without recompile. A
/// room is pinned to whichever version its first peer requested; later
/// peers must match or get `upgrade_required`.
#[serde(default = "default_protocol_version")]
protocol_version: u8,
}
fn default_protocol_version() -> u8 {
1
}
// ── Core connection lifecycle ─────────────────────────────────────────────────
@@ -208,16 +223,81 @@ async fn handle_audio_connection(socket: WebSocket, state: Arc<AppState>, channe
Ok(_) => {} // Channel exists and is not archived — proceed.
}
let (peer_id, peer_index, audio_rx, peer_ctrl_rx) = match room.add_peer(pubkey_hex.clone()) {
Some(v) => v,
None => {
// Reject unsupported future versions up-front so we don't accidentally
// pin a room to a version we can't speak. Versions 1..=CURRENT are OK.
let requested_version = auth_msg.protocol_version;
if requested_version == 0 || requested_version > CURRENT_PROTOCOL_VERSION {
warn!(
channel_id = %channel_id,
pubkey = %pubkey_hex,
requested_version,
current = CURRENT_PROTOCOL_VERSION,
"audio: client requested unsupported protocol version"
);
let _ = ws_send
.send(WsMessage::Text(
serde_json::json!({
"type": "error",
"code": "unsupported_version",
"message": format!(
"huddle audio protocol v{requested_version} not supported; relay max is v{CURRENT_PROTOCOL_VERSION}"
),
"current_version": CURRENT_PROTOCOL_VERSION,
})
.to_string()
.into(),
))
.await;
return;
}
let (peer_id, peer_index, audio_rx, peer_ctrl_rx) = match room
.add_peer(pubkey_hex.clone(), requested_version)
{
Ok(v) => v,
Err(crate::audio::room::AdmissionError::Full) => {
warn!(channel_id = %channel_id, "audio room full (255 peers exhausted)");
let _ = ws_send
.send(WsMessage::Text(
serde_json::json!({"type":"error","code":"room_full","message":"peer index space exhausted"})
.to_string().into(),
))
.await;
.send(WsMessage::Text(
serde_json::json!({"type":"error","code":"room_full","message":"peer index space exhausted"})
.to_string().into(),
))
.await;
return;
}
Err(crate::audio::room::AdmissionError::Ended) => {
debug!(channel_id = %channel_id, "room ended before admission");
let _ = ws_send
.send(WsMessage::Text(
serde_json::json!({"type":"error","code":"room_ended","message":"huddle has ended"})
.to_string().into(),
))
.await;
return;
}
Err(crate::audio::room::AdmissionError::VersionMismatch { pinned, requested }) => {
info!(
channel_id = %channel_id,
pubkey = %pubkey_hex,
pinned,
requested,
"audio: protocol version mismatch — upgrade required"
);
let _ = ws_send
.send(WsMessage::Text(
serde_json::json!({
"type": "error",
"code": "upgrade_required",
"message": format!(
"this huddle is using audio protocol v{pinned}; your client requested v{requested}"
),
"pinned_version": pinned,
"requested_version": requested,
})
.to_string()
.into(),
))
.await;
return;
}
};
@@ -288,6 +368,7 @@ async fn handle_audio_connection(socket: WebSocket, state: Arc<AppState>, channe
ws_recv,
Arc::clone(&room),
peer_id,
requested_version,
ctrl_tx,
Arc::clone(&missed_pongs),
cancel.clone(),
@@ -364,10 +445,13 @@ async fn recv_loop(
mut ws_recv: futures_util::stream::SplitStream<WebSocket>,
room: Arc<crate::audio::room::Room>,
peer_id: Uuid,
protocol_version: u8,
ctrl_tx: mpsc::Sender<WsMessage>,
missed_pongs: Arc<AtomicU8>,
cancel: CancellationToken,
) {
use crate::audio::wire::{FrameHeader, V2_HEADER_LEN};
loop {
tokio::select! {
biased;
@@ -379,6 +463,52 @@ async fn recv_loop(
warn!(peer_id = %peer_id, bytes = data.len(), "audio frame too large — dropping");
continue;
}
// Protocol v2 sanity-parse: validate the header is
// present and well-shaped, then forward opaquely.
// We never strip, rewrite, or re-encode bytes — the
// header is sender-authored telemetry only — but we
// do refuse to broadcast frames that are clearly
// malformed for the room's pinned protocol so we
// don't help v2 peers feed garbage to other v2 peers.
if protocol_version >= 2 {
// Frame must carry at least the 8-byte header
// plus a non-empty Opus payload.
if data.len() <= V2_HEADER_LEN {
warn!(
peer_id = %peer_id,
bytes = data.len(),
"v2 frame missing header or payload — dropping"
);
continue;
}
match FrameHeader::parse(&data) {
Some((header, payload)) if !payload.is_empty() => {
// Header is well-formed. `level_dbov` is
// already clamped by `parse` — bad values
// do not drop the frame, they just lose
// the metric (which the relay does not
// trust for anything anyway).
tracing::trace!(
peer_id = %peer_id,
seq = header.seq,
ts_48k = header.ts_48k,
level_dbov = header.level_dbov,
is_dtx = header.is_dtx(),
"v2 audio frame"
);
}
_ => {
warn!(
peer_id = %peer_id,
bytes = data.len(),
"v2 frame failed header parse — dropping"
);
continue;
}
}
}
room.broadcast_frame(peer_id, data);
}
Some(Ok(WsMessage::Text(text))) => {

View File

@@ -11,6 +11,7 @@
pub mod handler;
pub mod room;
pub mod wire;
pub use handler::ws_audio_handler;
pub use room::AudioRoomManager;

View File

@@ -47,6 +47,24 @@ const CTRL_CHANNEL_CAPACITY: usize = 32;
/// is reasonable. The 255 index space is the hard limit; this is the soft one.
const MAX_PEERS_PER_ROOM: usize = 25;
/// Reason a peer was refused entry to a room.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AdmissionError {
/// The room has been ended (or is shutting down) and no longer admits peers.
Ended,
/// The room has hit the soft peer cap or exhausted the 255-index space.
Full,
/// The room is pinned to a different protocol version than the requested one.
/// The caller should reply to the WS client with an `upgrade_required` error
/// and the room's actual `pinned` version, then close the socket.
VersionMismatch {
/// Version the room is currently pinned to.
pinned: u8,
/// Version the joining client requested.
requested: u8,
},
}
/// Peer index allocator + room lifecycle gate.
///
/// The `ended` flag and peer admission are synchronized under the same mutex.
@@ -58,6 +76,25 @@ struct AdmissionGuard {
next_fresh: u8,
free: Vec<u8>,
ended: bool,
/// Pinned huddle audio protocol version for this room.
///
/// `None` until the first peer admits. The first admission pins the room
/// to that version; subsequent peers MUST present the same version or
/// they're rejected with `AdmissionError::VersionMismatch`. The relay
/// forwards binary frames opaquely either way, so allowing a v1 client
/// into a v2 room would silently corrupt v2 peers' decode (they'd see
/// no header where one is expected, and vice versa).
///
/// Pin is per-`Room`-instance and clears when the manager evicts the
/// Room via [`AudioRoomManager::cleanup_if_empty`] — the next
/// `get_or_create` for the same channel id then constructs a fresh
/// `Room` with a fresh `AdmissionGuard` (and therefore `None` pin),
/// so a new generation of joiners can negotiate a new version.
/// A momentarily-empty-but-not-yet-cleaned-up Room keeps its pin so
/// reconnecting peers don't accidentally renegotiate mid-call. See
/// `version_pin_persists_across_peer_churn` for the test that pins
/// this behavior.
pinned_version: Option<u8>,
}
impl AdmissionGuard {
@@ -66,6 +103,7 @@ impl AdmissionGuard {
next_fresh: 0,
free: Vec::new(),
ended: false,
pinned_version: None,
}
}
@@ -126,26 +164,53 @@ impl Room {
}
}
/// Add a peer. Returns `(peer_id, peer_index, audio_rx, ctrl_rx)`, or
/// `None` if the room has ended, hit the peer cap, or exhausted the index space.
/// Add a peer. Returns `(peer_id, peer_index, audio_rx, ctrl_rx)` on
/// success, or an [`AdmissionError`] explaining why the peer was rejected.
///
/// The ended check, index allocation, and peer insert all happen under
/// the admission guard lock — mutually exclusive with `mark_ended`.
/// `requested_version` is the huddle audio protocol version the peer
/// negotiated in its WS auth message. The first successful admission
/// pins the room to that version; later admits must match the pin or
/// they receive [`AdmissionError::VersionMismatch`].
///
/// The cap check, ended check, version pin, index allocation, and peer
/// insert all happen under the admission guard lock — mutually exclusive
/// with `mark_ended` and with any concurrent `add_peer` that might race
/// the version pin.
///
/// Error precedence is deliberate: `Ended` > `Full` > `VersionMismatch`.
/// A "no seat available" error wins over version mismatch because a
/// client that couldn't join either way shouldn't learn the room's
/// pinned protocol version — that's a (mild) information leak. The cap
/// check lives inside the lock so two concurrent joiners can't both
/// pass it; the per-room index space (255) plus the soft cap
/// (`MAX_PEERS_PER_ROOM`) is then a single, race-free invariant.
pub fn add_peer(
&self,
pubkey: String,
) -> Option<(Uuid, u8, mpsc::Receiver<Bytes>, mpsc::Receiver<PeerCtrl>)> {
if self.peers.len() >= MAX_PEERS_PER_ROOM {
return None;
}
// Hold the guard across ended check + index alloc + peer insert.
// This makes add_peer mutually exclusive with mark_ended — the lock
// is the single synchronization point that closes the race.
let mut g = self.guard.lock().ok()?;
requested_version: u8,
) -> Result<(Uuid, u8, mpsc::Receiver<Bytes>, mpsc::Receiver<PeerCtrl>), AdmissionError> {
let mut g = self.guard.lock().map_err(
|_| AdmissionError::Ended, /* poisoned ≈ shutting down */
)?;
if g.ended {
return None;
return Err(AdmissionError::Ended);
}
let peer_index = g.alloc()?;
if self.peers.len() >= MAX_PEERS_PER_ROOM {
return Err(AdmissionError::Full);
}
if let Some(pinned) = g.pinned_version {
if pinned != requested_version {
return Err(AdmissionError::VersionMismatch {
pinned,
requested: requested_version,
});
}
}
let peer_index = g.alloc().ok_or(AdmissionError::Full)?;
// Pin the room version on the first successful index allocation. We
// pin *after* alloc so a Full error doesn't accidentally set the
// version for a peer that didn't actually join.
g.pinned_version.get_or_insert(requested_version);
let peer_id = Uuid::new_v4();
let (audio_tx, audio_rx) = mpsc::channel(AUDIO_CHANNEL_CAPACITY);
let (ctrl_tx, ctrl_rx) = mpsc::channel(CTRL_CHANNEL_CAPACITY);
@@ -159,7 +224,7 @@ impl Room {
},
);
drop(g); // Release lock after insert.
Some((peer_id, peer_index, audio_rx, ctrl_rx))
Ok((peer_id, peer_index, audio_rx, ctrl_rx))
}
/// Remove a peer and recycle its index.
@@ -288,3 +353,154 @@ impl Default for AudioRoomManager {
Self::new()
}
}
#[cfg(test)]
mod tests {
use super::*;
fn fresh_room() -> Room {
Room::new(Uuid::new_v4())
}
/// First peer's `requested_version` becomes the room's pin; later peers
/// requesting the same version are admitted normally.
#[test]
fn first_admit_pins_version_and_matching_admits_succeed() {
let room = fresh_room();
let first = room
.add_peer("alice".to_string(), 2)
.expect("first peer admits");
let second = room
.add_peer("bob".to_string(), 2)
.expect("matching version admits");
assert_eq!(room.peers.len(), 2);
// peer_index allocation is monotonic from 0 inside a fresh room.
assert_eq!(first.1, 0);
assert_eq!(second.1, 1);
}
/// A peer requesting a different protocol version than the pinned one
/// is refused with `VersionMismatch` and never appears in the peer map.
#[test]
fn admit_rejects_mismatched_version() {
let room = fresh_room();
let _ = room
.add_peer("alice".to_string(), 2)
.expect("first peer admits");
let err = room
.add_peer("bob".to_string(), 1)
.expect_err("mismatched version must be rejected");
match err {
AdmissionError::VersionMismatch { pinned, requested } => {
assert_eq!(pinned, 2);
assert_eq!(requested, 1);
}
other => panic!("expected VersionMismatch, got {other:?}"),
}
// The rejected peer must not appear in the peer map, and the index
// space must not have been consumed by the failed admit.
assert_eq!(room.peers.len(), 1);
}
/// `add_peer` rejects requests after the room is marked ended even if the
/// version matches.
#[test]
fn admit_after_mark_ended_returns_ended() {
let room = fresh_room();
assert!(room.mark_ended());
let err = room
.add_peer("alice".to_string(), 1)
.expect_err("ended room must refuse");
assert!(matches!(err, AdmissionError::Ended));
}
/// Per Max's review checklist: an empty-and-cleaned-up room (via the
/// manager's `cleanup_if_empty`) becomes a fresh room on the next
/// `get_or_create`, with no pin carried over.
#[test]
fn manager_cleanup_resets_version_pin() {
let manager = AudioRoomManager::new();
let channel_id = Uuid::new_v4();
let room1 = manager.get_or_create(channel_id);
let (peer_id, _, _, _) = room1
.add_peer("alice".to_string(), 2)
.expect("first peer admits");
// Last peer leaves and ends the room atomically.
let (_, ended) = room1
.remove_peer_and_check_ended(peer_id)
.expect("peer existed");
assert!(ended, "single-peer room should end on its last departure");
assert!(manager.cleanup_if_empty(channel_id));
// Next joiner with a different version on the same channel id gets a
// brand-new room (no v=2 pin carried over from the prior generation).
let room2 = manager.get_or_create(channel_id);
let _ = room2
.add_peer("bob".to_string(), 1)
.expect("fresh room must accept any version");
}
/// Peer-index reuse: after a peer leaves, their index is released; a new
/// peer joining the same (still-pinned) room reuses the freed index.
/// Version pin must persist across this reuse — the room generation
/// hasn't ended.
#[test]
fn version_pin_persists_across_peer_churn() {
let room = fresh_room();
let (alice_id, alice_idx, _, _) =
room.add_peer("alice".to_string(), 2).expect("alice admits");
room.remove_peer(alice_id);
// Room is non-empty thanks to nothing yet — wait, alice left and
// nobody else is here. Add bob with the same version: should work.
// Then add carol with a different version: should fail with the
// *original* pin, even though alice already left.
let (_, bob_idx, _, _) = room
.add_peer("bob".to_string(), 2)
.expect("bob admits at v=2");
assert_eq!(
bob_idx, alice_idx,
"freed peer index should be recycled by the next admit",
);
let err = room
.add_peer("carol".to_string(), 1)
.expect_err("v=1 must still be refused — room is pinned v=2");
assert!(matches!(
err,
AdmissionError::VersionMismatch {
pinned: 2,
requested: 1
}
));
}
/// Per Sami/Perci's review: when a room is both at-capacity AND the
/// joiner's protocol version doesn't match the pin, the error must be
/// `Full` — not `VersionMismatch`. A client that couldn't get a seat
/// either way shouldn't learn the room's pinned protocol version.
/// This also pins the in-lock cap check (the old code's outside-lock
/// cap check meant the version error could win in a race).
#[test]
fn admit_full_wins_over_version_mismatch() {
let room = fresh_room();
// Fill the room with v=2 peers right up to the soft cap.
for i in 0..MAX_PEERS_PER_ROOM {
room.add_peer(format!("peer-{i}"), 2)
.expect("seed admit must succeed");
}
// Next joiner is BOTH over the cap AND requests the wrong version.
let err = room
.add_peer("over-cap-and-wrong-version".to_string(), 1)
.expect_err("over-cap + wrong-version joiner must be rejected");
assert!(
matches!(err, AdmissionError::Full),
"expected Full to win over VersionMismatch, got {err:?}",
);
// And the room state must be unchanged.
assert_eq!(room.peers.len(), MAX_PEERS_PER_ROOM);
}
}

View File

@@ -0,0 +1,168 @@
//! Huddle audio wire protocol — relay-side parse helpers.
//!
//! Mirrors the desktop client's `huddle::wire` for the half of the protocol
//! the relay cares about: validating that v2 frames carry an 8-byte header,
//! parsing it into structured metrics, and clamping
//! caller-authored telemetry to safe ranges. The relay never *generates*
//! v2 frames (clients author them) and never *re-encodes* them (broadcast
//! is opaque byte forwarding) — so only the parse half of the protocol
//! lives here, not the encode half.
//!
//! # Threat-model invariant
//!
//! `level_dbov` is client-authored telemetry. Anything we surface from it
//! (logs, active-speaker UI hints, dominant-talker decisions) must treat
//! it as untrusted. This module's [`FrameHeader::parse`] clamps out-of-range
//! values into the canonical `-127..=0` range but never drops the audio
//! frame on bad VU data — bad metadata must not cause audible loss.
//! Trust decisions (admission, moderation, kicks) MUST NOT consume
//! `level_dbov`.
/// Length of the v2 per-frame header in bytes. The wire layout is, in
/// network byte order:
///
/// ```text
/// byte 0..=1 : seq u16
/// byte 2..=5 : ts_48k u32
/// byte 6 : level_dbov i8 range [-127, 0]
/// byte 7 : flags u8 bit 0 = DTX; other bits reserved
/// ```
pub const V2_HEADER_LEN: usize = 8;
/// `flags & FLAG_DTX` indicates a DTX/comfort-noise frame.
pub const FLAG_DTX: u8 = 0x01;
/// Parsed v2 header view. Cheap (Copy).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct FrameHeader {
/// Sender-authored sequence number; wraps every 2^16 frames.
pub seq: u16,
/// Sender-authored 48 kHz RTP-style media timestamp.
pub ts_48k: u32,
/// Audio level in dBov. Always parsed into the canonical `-127..=0`
/// range — out-of-range inputs are clamped to `-127` (silence floor).
/// Untrusted; for diagnostics only.
pub level_dbov: i8,
/// Raw flags byte. Use bit masks; reserved bits MAY be set.
pub flags: u8,
}
impl FrameHeader {
/// True if `FLAG_DTX` is set.
pub fn is_dtx(&self) -> bool {
(self.flags & FLAG_DTX) != 0
}
/// Parse a v2 header from the leading 8 bytes of `bytes`. Returns the
/// header and the remaining payload slice (the opaque Opus body the
/// relay forwards). On `None`, the caller should treat the frame as
/// malformed and drop it.
///
/// `level_dbov` is clamped into `-127..=0`; out-of-range inputs become
/// `-127`. The audio frame is never rejected for bad telemetry — only
/// the metric is suppressed.
pub fn parse(bytes: &[u8]) -> Option<(Self, &[u8])> {
if bytes.len() < V2_HEADER_LEN {
return None;
}
let seq = u16::from_be_bytes([bytes[0], bytes[1]]);
let ts_48k = u32::from_be_bytes([bytes[2], bytes[3], bytes[4], bytes[5]]);
let raw_level = bytes[6] as i8;
let level_dbov = if (-127..=0).contains(&raw_level) {
raw_level
} else {
-127
};
let flags = bytes[7];
Some((
Self {
seq,
ts_48k,
level_dbov,
flags,
},
&bytes[V2_HEADER_LEN..],
))
}
}
#[cfg(test)]
mod tests {
use super::*;
/// Canonical layout: BE u16 seq, BE u32 ts_48k, i8 level, u8 flags.
/// This test pins the byte order so an accidental endianness flip on
/// either side of the protocol is caught immediately.
#[test]
fn parse_reads_network_byte_order() {
let bytes = [
0x01, 0x02, // seq = 0x0102
0x03, 0x04, 0x05, 0x06, // ts_48k = 0x0304_0506
0xFF, // level_dbov = -1
0x01, // flags = FLAG_DTX
0xAA, 0xBB, 0xCC, // opaque payload — what the relay forwards
];
let (h, payload) = FrameHeader::parse(&bytes).expect("parse");
assert_eq!(h.seq, 0x0102);
assert_eq!(h.ts_48k, 0x0304_0506);
assert_eq!(h.level_dbov, -1);
assert!(h.is_dtx());
assert_eq!(payload, &[0xAA, 0xBB, 0xCC]);
}
/// Anything shorter than the fixed 8-byte header is malformed.
#[test]
fn parse_rejects_short_input() {
for len in 0..V2_HEADER_LEN {
let buf = vec![0u8; len];
assert!(
FrameHeader::parse(&buf).is_none(),
"{len}-byte input must fail"
);
}
}
/// Out-of-range `level_dbov` must not drop the frame. The metric is
/// suppressed (clamped to -127, the silence floor) but the audio
/// payload is preserved for forwarding. This pins the "bad VU
/// metadata is not audible loss" invariant relay-side as well as
/// desktop-side.
#[test]
fn parse_clamps_out_of_range_level_keeps_frame() {
// i8 = +127 → outside [-127, 0]
let bytes = [
0x00, 0x07, // seq = 7
0x00, 0x00, 0x03, 0xC0, // ts_48k = 960
0x7F, // level_dbov raw = +127 (invalid)
0x00, // flags
b'o', b'p', b'u', b's',
];
let (h, payload) = FrameHeader::parse(&bytes).expect("parse must succeed");
assert_eq!(h.level_dbov, -127, "invalid level clamps to silence floor");
assert_eq!(h.seq, 7, "valid fields preserved alongside clamp");
assert_eq!(
payload, b"opus",
"audio payload still available for forwarding"
);
}
/// Reserved flag bits are passed through untouched. Receivers (the
/// other half of the protocol) are responsible for ignoring them;
/// the relay's only job is to forward the bytes faithfully.
#[test]
fn parse_preserves_reserved_flag_bits() {
let bytes = [
0x00,
0x00,
0x00,
0x00,
0x00,
0x00,
0x00,
0b1010_1010, // FLAG_DTX clear, reserved bits set
];
let (h, _) = FrameHeader::parse(&bytes).expect("parse");
assert_eq!(h.flags, 0b1010_1010);
assert!(!h.is_dtx());
}
}

View File

@@ -1312,7 +1312,7 @@ dependencies = [
"libc",
"option-ext",
"redox_users",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -1525,7 +1525,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -2992,7 +2992,7 @@ dependencies = [
"png 0.18.1",
"serde",
"thiserror 2.0.18",
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@@ -3037,6 +3037,19 @@ version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43a88da9dd148bbcdce323dd6ac47d369b4769d4a3b78c6c52389b9269f77932"
[[package]]
name = "neteq"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57fbe9601485d95750187c92344fa056e75055027b00e375c53d9f4b8b883112"
dependencies = [
"log",
"ringbuf",
"serde",
"thiserror 1.0.69",
"web-time",
]
[[package]]
name = "new_debug_unreachable"
version = "1.0.6"
@@ -4473,6 +4486,15 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "ringbuf"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "79abed428d1fd2a128201cec72c5f6938e2da607c6f3745f769fabea399d950a"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "rodio"
version = "0.22.2"
@@ -4560,7 +4582,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -4619,7 +4641,7 @@ dependencies = [
"security-framework",
"security-framework-sys",
"webpki-root-certs",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -5151,7 +5173,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
dependencies = [
"libc",
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@@ -5218,6 +5240,7 @@ dependencies = [
"hex",
"infer",
"libc",
"neteq",
"nostr 0.36.0",
"nostr 0.37.0",
"opus",
@@ -6139,7 +6162,7 @@ dependencies = [
"getrandom 0.4.2",
"once_cell",
"rustix",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -6562,7 +6585,7 @@ dependencies = [
"png 0.18.1",
"serde",
"thiserror 2.0.18",
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@@ -6628,7 +6651,7 @@ checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e"
dependencies = [
"memoffset",
"tempfile",
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@@ -7141,7 +7164,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]

View File

@@ -42,6 +42,7 @@ tokio-tungstenite = { version = "0.29", features = ["rustls-tls-webpki-roots"] }
tokio-util = { version = "0.7", features = ["rt"] }
futures-util = "0.3"
opus = "0.3"
neteq = { version = "0.8", default-features = false }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
nostr = { version = "0.37", features = ["nip44"] }

View File

@@ -0,0 +1,267 @@
//! Per-peer NetEQ-style jitter buffer for huddle remote audio.
//!
//! Wraps the [`neteq`] crate's `NetEq` state machine and registers a thin
//! `AudioDecoder` impl over our existing `opus = "0.3"` dependency. Default
//! features on `neteq` are disabled so we don't pull in axum/cpal/clap; this
//! module owns the Opus side of the trait instead.
//!
//! ## Lifecycle
//!
//! One [`PeerJitterBuffer`] per active remote peer. It owns:
//! * a `NetEq` configured for 48 kHz mono with conservative jitter bounds,
//! * a stable synthetic RTP SSRC (the relay-assigned `peer_index`), and
//! * a 16-bit sequence number that wraps with the wire protocol's `seq`.
//!
//! On `joined` / first packet, construct one. On `left` / rejoin, drop and
//! recreate so NetEq's internal delay state doesn't carry over.
//!
//! ## Decode path
//!
//! Each inbound Opus frame becomes a `neteq::AudioPacket` with:
//! * sequence number from the protocol v2 header,
//! * 48 kHz RTP-style media timestamp (frame_index * 960 for 20 ms frames),
//! * a synthetic SSRC == peer_index,
//! * payload type [`OPUS_PAYLOAD_TYPE`] (any value, only NetEq uses it to
//! dispatch to its registered decoder),
//! * the raw Opus bytes as `payload`,
//! * `duration_ms = 20` (our encoder frame size).
//!
//! The playout side calls [`PeerJitterBuffer::get_audio`] on a 10 ms tick;
//! NetEq returns 10 ms of decoded PCM with adaptive-delay / expand / accelerate
//! decisions already applied. For 20 ms input frames at 48 kHz mono, that's
//! 480 f32 samples per call.
use neteq::{
codec::AudioDecoder, neteq::SpeechType, AudioPacket, NetEq, NetEqConfig, NetEqError, RtpHeader,
};
/// RTP payload type used internally. NetEq only needs it to look up the decoder
/// registered via `register_decoder`; the relay forwards Opus payloads opaquely
/// so the wire never carries a payload-type byte.
pub const OPUS_PAYLOAD_TYPE: u8 = 111;
/// Audio sample rate for huddle (Opus VoIP, mono).
pub const SAMPLE_RATE_HZ: u32 = 48_000;
/// One channel (mono).
pub const CHANNELS: u8 = 1;
/// Opus encoder frame size in milliseconds. The wire format is one Opus packet
/// per 20 ms frame; the receiver still drains NetEq at 10 ms granularity.
pub const FRAME_DURATION_MS: u32 = 20;
/// 48 kHz media-time increment per encoder frame.
pub const FRAME_TIMESTAMP_DELTA: u32 = SAMPLE_RATE_HZ / 1000 * FRAME_DURATION_MS; // 960
/// NetEq returns 10 ms frames; this is the sample count per `get_audio` call.
/// Kept as a documented constant for the protocol-v2 follow-up and consumers
/// that want to size their own buffers around the playout contract.
#[allow(dead_code)]
pub const PLAYOUT_SAMPLES: usize = (SAMPLE_RATE_HZ as usize / 1000) * 10; // 480
/// Minimum jitter delay NetEq is allowed to converge to (ms).
const MIN_DELAY_MS: u32 = 40;
/// Maximum jitter delay NetEq is allowed to converge to (ms).
const MAX_DELAY_MS: u32 = 200;
/// Buffer cap in packets — generous, but bounded so a runaway sender can't
/// drive memory growth.
const MAX_PACKETS_IN_BUFFER: usize = 50;
/// Thin `AudioDecoder` over `opus::Decoder` so `neteq` can drive Opus payloads.
///
/// NetEq's `decode` trait is sample-based, not FEC-aware (no `fec` bool). The
/// stock `neteq` `NativeOpusDecoder` (gated behind the `native` feature, which
/// pulls in cpal/axum) calls `decode_float(..., false)` unconditionally; this
/// matches that behavior so we don't need the feature.
///
/// Receive-side FEC ("decode this frame's redundant copy on a known-lost prior
/// frame") would require either a trait change upstream or NetEq calling decode
/// twice with different `fec` values — out of scope for the initial 10-person
/// fix. Tracked as a follow-up alongside encoder-side `set_inband_fec`.
struct OpusFrameDecoder {
inner: opus::Decoder,
sample_rate: u32,
channels: u8,
/// Scratch buffer reused across calls to avoid per-frame allocation.
scratch: Vec<f32>,
}
impl OpusFrameDecoder {
fn new(sample_rate: u32, channels: u8) -> Result<Self, String> {
let opus_channels = match channels {
1 => opus::Channels::Mono,
2 => opus::Channels::Stereo,
n => return Err(format!("unsupported channel count: {n}")),
};
let inner = opus::Decoder::new(sample_rate, opus_channels)
.map_err(|e| format!("opus decoder init: {e}"))?;
// Largest plausible frame: 60 ms at 48 kHz stereo. We always use
// 20 ms mono in practice; this is just a one-shot allocation cap.
let cap = (sample_rate as usize / 1000) * 60 * channels as usize;
Ok(Self {
inner,
sample_rate,
channels,
scratch: vec![0.0; cap],
})
}
}
impl AudioDecoder for OpusFrameDecoder {
fn sample_rate(&self) -> u32 {
self.sample_rate
}
fn channels(&self) -> u8 {
self.channels
}
fn decode(&mut self, encoded: &[u8]) -> Result<Vec<f32>, NetEqError> {
match self.inner.decode_float(encoded, &mut self.scratch, false) {
Ok(samples_per_channel) => {
let total = samples_per_channel * self.channels as usize;
Ok(self.scratch[..total].to_vec())
}
Err(e) => Err(NetEqError::DecoderError(format!("opus decode: {e}"))),
}
}
}
/// One peer's jitter buffer + decoder pair.
pub struct PeerJitterBuffer {
neteq: NetEq,
ssrc: u32,
}
impl PeerJitterBuffer {
/// Construct a new jitter buffer for the peer identified by `peer_index`.
///
/// `peer_index` is the relay-assigned 0..=254 stable index for the peer's
/// lifetime in the room; we promote it to a 32-bit synthetic SSRC so NetEq
/// has a stable stream identity for its delay-manager state.
pub fn new(peer_index: u8) -> Result<Self, NetEqError> {
let config = NetEqConfig {
sample_rate: SAMPLE_RATE_HZ,
channels: CHANNELS,
max_packets_in_buffer: MAX_PACKETS_IN_BUFFER,
min_delay_ms: MIN_DELAY_MS,
max_delay_ms: MAX_DELAY_MS,
..Default::default()
};
let mut neteq = NetEq::new(config)?;
let decoder =
OpusFrameDecoder::new(SAMPLE_RATE_HZ, CHANNELS).map_err(NetEqError::DecoderError)?;
neteq.register_decoder(OPUS_PAYLOAD_TYPE, Box::new(decoder));
Ok(Self {
neteq,
ssrc: peer_index as u32,
})
}
/// Insert a received Opus packet.
///
/// `seq` and `ts_48k` come from the protocol v2 header (sender-authored,
/// 16-bit wrapping sequence + 48 kHz RTP-style media time).
pub fn insert_packet(
&mut self,
seq: u16,
ts_48k: u32,
opus_payload: &[u8],
) -> Result<(), NetEqError> {
let header = RtpHeader::new(seq, ts_48k, self.ssrc, OPUS_PAYLOAD_TYPE, false);
let packet = AudioPacket::new(
header,
opus_payload.to_vec(),
SAMPLE_RATE_HZ,
CHANNELS,
FRAME_DURATION_MS,
);
self.neteq.insert_packet(packet)
}
/// Drain one 10 ms playout frame. Always succeeds: NetEq emits PLC /
/// comfort-noise / silence rather than an error when there's nothing to
/// play.
///
/// Returns `(samples, vad_active)`. `samples.len() == PLAYOUT_SAMPLES`.
pub fn get_audio(&mut self) -> Result<(Vec<f32>, bool), NetEqError> {
let frame = self.neteq.get_audio()?;
let vad = frame.vad_activity && !matches!(frame.speech_type, SpeechType::Expand);
Ok((frame.samples, vad))
}
/// True when NetEq's buffer has no packets queued and the next
/// `get_audio` call would emit expand/silence. Used by
/// `huddle::playout` as part of the idle-peer gate — if a peer has
/// gone quiet (`last_packet_at` is stale) *and* there's nothing
/// buffered to drain, we stop appending frames to that peer's rodio
/// Player. The check is conservative: a peer who just sent a burst
/// then disconnected may have buffered audio that should still play
/// out, so this method drives that decision.
pub fn is_empty(&self) -> bool {
self.neteq.is_empty()
}
}
#[cfg(test)]
mod tests {
use super::*;
fn opus_silence_frame() -> Vec<u8> {
// Encode 20 ms of silence at 48 kHz mono so the test doesn't rely on
// synthetic Opus byte sequences.
let mut encoder = opus::Encoder::new(
SAMPLE_RATE_HZ,
opus::Channels::Mono,
opus::Application::Voip,
)
.expect("opus encoder");
let pcm = vec![0.0_f32; FRAME_TIMESTAMP_DELTA as usize];
let mut out = vec![0u8; 4000];
let n = encoder
.encode_float(&pcm, &mut out)
.expect("opus encode silence");
out.truncate(n);
out
}
#[test]
fn jitter_buffer_construction_succeeds_for_each_peer_index() {
for idx in [0u8, 1, 7, 42, 254] {
let jb = PeerJitterBuffer::new(idx).expect("construct jitter buffer");
assert!(jb.is_empty(), "peer {idx} should start empty");
// SSRC is the peer_index as u32 — verifies the stable-identity contract.
assert_eq!(jb.ssrc, idx as u32);
}
}
#[test]
fn insert_packet_then_get_audio_returns_playout_frame() {
let mut jb = PeerJitterBuffer::new(3).expect("jitter buffer");
let payload = opus_silence_frame();
// Insert a few sequential packets so NetEq has enough buffer to start
// emitting Normal-decoded frames instead of pure Expand/silence.
for i in 0..6u16 {
let ts = (i as u32) * FRAME_TIMESTAMP_DELTA;
jb.insert_packet(i, ts, &payload).expect("insert");
}
// Pull a playout frame — should be 10 ms = 480 mono samples.
let (samples, _vad) = jb.get_audio().expect("get_audio");
assert_eq!(
samples.len(),
PLAYOUT_SAMPLES,
"NetEq must emit exactly one 10 ms frame per get_audio call",
);
}
#[test]
fn empty_buffer_get_audio_still_produces_a_frame() {
// Even with no packets inserted, NetEq's contract is to always emit
// 10 ms — as Expand/silence — so the playout clock keeps ticking.
let mut jb = PeerJitterBuffer::new(0).expect("jitter buffer");
let (samples, vad) = jb.get_audio().expect("get_audio");
assert_eq!(samples.len(), PLAYOUT_SAMPLES);
assert!(
!vad,
"empty buffer should not report voice activity (Expand frame)",
);
}
}

View File

@@ -25,14 +25,17 @@
pub mod agents;
pub mod audio_output;
pub mod jitter;
pub mod models;
pub mod pipeline;
pub mod playout;
pub mod pocket;
pub mod preprocessing;
pub mod relay_api;
pub mod state;
pub mod stt;
pub mod tts;
pub mod wire;
// ── Shared utilities ──────────────────────────────────────────────────────────

View File

@@ -0,0 +1,362 @@
//! Receive-side playout loop for the huddle audio relay.
//!
//! Owns the per-peer state map (one `NetEq` + one `rodio::Player` per remote
//! peer), the 10 ms playout clock, and the 500 ms active-speaker tick. Sibling
//! to [`relay_api`](super::relay_api), which keeps the encode/send half.
//!
//! ## Architecture
//!
//! ```text
//! WS binary frame ──► insert_packet ──► NetEq jitter buffer
//! │
//! playout_tick (10 ms) ──┘──► get_audio ─► per-peer
//! rodio::Player
//! │
//! ▼
//! device mixer (sums
//! concurrent peers)
//! ```
//!
//! The pre-fix shape used a single `rodio::Player` shared across every peer.
//! `Player` is a FIFO queue, so 3+ simultaneous speakers serialized into one
//! voice flipping speakers every 20 ms with unbounded queue growth. See
//! `desktop/src-tauri/tests/rodio_mixer_diagnostic.rs` for the deterministic
//! repro that pins this diagnosis in CI.
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use futures_util::{SinkExt, StreamExt};
use tokio_tungstenite::tungstenite::Message as WsMsg;
use tokio_util::sync::CancellationToken;
use super::jitter::{PeerJitterBuffer, SAMPLE_RATE_HZ};
use super::relay_api::{WsStream, REMOTE_SPEECH_THRESHOLD};
use super::wire::{FrameHeader, FLAG_DTX, V2_HEADER_LEN};
/// Speaker-tick window for emitting `huddle-active-speakers`. Active set is
/// cleared each tick — peers that didn't send a frame in the last window are
/// considered silent.
const SPEAKER_TICK_MS: u64 = 500;
/// Per-peer arrival window for the TTS interrupt frame counter.
const FRAME_WINDOW: std::time::Duration = std::time::Duration::from_millis(500);
/// Playout clock: NetEq emits 10 ms frames, so we tick at 10 ms.
const PLAYOUT_TICK_MS: u64 = 10;
/// How long after the last received packet we keep pulling frames out of a
/// peer's NetEq into its rodio Player. NetEq always emits a frame on every
/// `get_audio` call — silent/Expand when there are no packets — so without
/// this bound an idle peer (one that disconnected without sending `left`)
/// would have its Player queue 100 silence buffers/sec forever. We pull for
/// a short grace window past the last packet so brief DTX gaps still feed
/// NetEq's PLC/expand path normally.
const IDLE_PEER_GRACE: std::time::Duration = std::time::Duration::from_millis(500);
/// Drift bound on per-peer rodio `Player` queue depth.
///
/// The playout pipeline has two clocks: the producer is a `tokio` 10 ms
/// interval (this loop) that pulls from NetEq and appends to each peer's
/// `Player`; the consumer is the `cpal` audio callback that pulls samples
/// from the device `Mixer` at hardware sample rate. NetEq does rate-adapt
/// (accelerate / expand) but only to its own input pattern — it cannot
/// see the actual device-side consumption rate.
///
/// In steady state producer ≈ consumer, but scheduler jitter or small
/// clock skew can leave the producer slightly ahead, and rodio's
/// `Player` queue is an unbounded MPSC under the hood. Over a long call
/// that drift would accumulate as monotonic added latency (and eventually
/// memory).
///
/// We bound it explicitly: before each append, if the queue is already
/// at or above this threshold, drop the oldest queued frame with
/// `Player::skip_one()` so the new frame replaces it. 4 frames × 10 ms
/// = 40 ms, far below NetEq's `max_delay_ms = 200 ms`, so the audible
/// effect is negligible while the worst-case latency stays bounded.
const PLAYOUT_QUEUE_HIGH_WATER: usize = 4;
/// One remote peer's slot: jitter buffer + dedicated rodio Player.
///
/// Per-frame seq/timestamp come from the v2 wire header (sender-authored).
/// The relay forwards `peer_index | header | opus_bytes` opaquely; we parse
/// the header here and pass the sender's own monotonic seq + 48 kHz media
/// timestamp into NetEq.
struct PeerSlot {
jitter: PeerJitterBuffer,
player: rodio::Player,
/// Wall-clock time of the most recent inbound packet for this peer. Read
/// by the playout tick to decide whether to keep draining NetEq into the
/// Player. Updated on every successful `insert_packet`.
last_packet_at: tokio::time::Instant,
}
impl PeerSlot {
fn new(peer_idx: u8, sink_mixer: &rodio::mixer::Mixer) -> Option<Self> {
match PeerJitterBuffer::new(peer_idx) {
Ok(jitter) => Some(Self {
jitter,
player: rodio::Player::connect_new(sink_mixer),
last_packet_at: tokio::time::Instant::now(),
}),
Err(e) => {
eprintln!("sprout-desktop: jitter buffer init peer {peer_idx}: {e}");
None
}
}
}
/// Whether this peer is still actively sending — used by the playout tick
/// to gate the rodio append so disconnected peers don't pump silence
/// indefinitely.
///
/// The gate is `recent packet OR jitter buffer not empty`. The recent-
/// packet half covers the common case: brief speech gaps and DTX cadence
/// (≤400 ms) stay inside the [`IDLE_PEER_GRACE`] window so PLC/expand
/// frames keep flowing. The buffer-not-empty half is a safety net for
/// the edge case Mari called out: a peer who sends a burst then
/// disconnects has real audio queued in NetEq that should still play
/// out, even if `last_packet_at` ages past the grace before the buffer
/// finishes draining. The grace alone is enough today because NetEq's
/// `max_delay_ms` (200 ms) is well inside the grace (500 ms), but the
/// OR keeps the invariant robust against future config tuning.
fn is_active(&self) -> bool {
self.last_packet_at.elapsed() < IDLE_PEER_GRACE || !self.jitter.is_empty()
}
}
/// Drive the receive loop until cancelled or the WS closes.
///
/// `ws_tx_for_pongs` is shared with the encode-side task and only used here to
/// reply to Pings; it is locked briefly per Ping and never held across the
/// audio fast path.
#[allow(clippy::too_many_arguments)]
pub(crate) async fn run_playout_recv_loop(
mut ws_rx: futures_util::stream::SplitStream<WsStream>,
ws_tx_for_pongs: Arc<tokio::sync::Mutex<futures_util::stream::SplitSink<WsStream, WsMsg>>>,
sink_handle: rodio::MixerDeviceSink,
cancel: CancellationToken,
app_handle: Option<tauri::AppHandle>,
initial_peers: Vec<(u8, String)>,
tts_active: Arc<AtomicBool>,
tts_cancel: Arc<AtomicBool>,
) {
use rodio::buffer::SamplesBuffer;
use std::num::NonZero;
let mut peers: std::collections::HashMap<u8, PeerSlot> = std::collections::HashMap::new();
let channels = NonZero::new(1u16).expect("1 is non-zero");
let rate = NonZero::new(SAMPLE_RATE_HZ).expect("48k is non-zero");
let mut index_to_pubkey: std::collections::HashMap<u8, String> =
initial_peers.into_iter().collect();
let mut active_indices: std::collections::HashSet<u8> = std::collections::HashSet::new();
let mut frame_counts: std::collections::HashMap<u8, u16> = std::collections::HashMap::new();
let mut last_frame_reset = tokio::time::Instant::now();
let mut tts_was_active = false;
let mut speaker_tick = tokio::time::interval(std::time::Duration::from_millis(SPEAKER_TICK_MS));
speaker_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
let mut playout_tick = tokio::time::interval(std::time::Duration::from_millis(PLAYOUT_TICK_MS));
// `Delay` (not `Skip`) so a brief stall in another select arm — e.g. the
// ws_tx_for_pongs mutex contending with the encode-side task on a Ping —
// doesn't drop a playout tick outright. Dropped ticks would leave the
// per-peer Player queues empty for 10 ms and the device mixer would
// produce audible silence. `Delay` catches up immediately when the loop
// returns to the select.
playout_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
loop {
tokio::select! {
biased;
_ = cancel.cancelled() => break,
_ = playout_tick.tick() => {
// Drain one 10 ms frame from each *active* peer's NetEq into
// its Player. NetEq always emits a frame (Expand/silence when
// empty), so for peers that recently sent we keep the device
// mixer fed without starving; for peers that have stopped
// sending — disconnected without a `left`, or simply quiet —
// we skip the append so we don't pump 100 silence buffers/sec
// per idle peer into rodio forever. `is_active` is a 500 ms
// grace past the last received packet, far longer than typical
// DTX comfort-noise cadence.
for (peer_idx, slot) in peers.iter_mut() {
if !slot.is_active() {
// Still drain the frame to keep NetEq's internal clock
// advancing; just don't enqueue it for playback.
let _ = slot.jitter.get_audio();
continue;
}
match slot.jitter.get_audio() {
Ok((samples, _vad)) => {
// Bound producer-vs-device-clock drift. If our
// tokio tick has gotten ahead of the audio
// callback's actual consumption rate, drop the
// oldest queued frame rather than letting the
// queue grow without bound.
if slot.player.len() >= PLAYOUT_QUEUE_HIGH_WATER {
eprintln!(
"sprout-desktop: playout queue high-water for peer {peer_idx} \
(depth={}) — dropping oldest frame",
slot.player.len(),
);
slot.player.skip_one();
}
slot.player.append(SamplesBuffer::new(channels, rate, samples));
}
Err(e) => {
eprintln!(
"sprout-desktop: jitter get_audio peer {peer_idx}: {e}"
);
}
}
}
}
_ = speaker_tick.tick() => {
if let Some(ref app) = app_handle {
use tauri::Emitter;
let pubkeys: Vec<String> = active_indices
.iter()
.filter_map(|idx| index_to_pubkey.get(idx).cloned())
.collect();
let _ = app.emit("huddle-active-speakers", &pubkeys);
}
active_indices.clear();
}
msg = ws_rx.next() => {
match msg {
Some(Ok(WsMsg::Binary(data))) => {
// Wire shape (v2): [peer_index: u8][header: 8 bytes][opus payload...]
// The minimum size is 1 (peer_index) + 8 (header) + ≥1 Opus byte.
if data.len() <= 1 + V2_HEADER_LEN {
continue;
}
let peer_idx = data[0];
let after_idx = &data[1..];
let Some((header, opus_bytes)) = FrameHeader::parse(after_idx)
else {
// Malformed v2 frame: header parse only fails when
// the slice is too short, which `if data.len() <= ...`
// already guards. Defensive log + drop.
eprintln!(
"sprout-desktop: dropping malformed audio frame from peer {peer_idx} ({} bytes)",
data.len(),
);
continue;
};
if opus_bytes.is_empty() {
continue;
}
let is_dtx = (header.flags & FLAG_DTX) != 0;
// Only count non-DTX arrivals toward the UI's
// active-speaker set. DTX/comfort packets are emitted
// by an idle peer to keep the codec alive — they
// don't mean the peer is speaking, and shouldn't
// make their tile flash for the 500 ms speaker tick.
if !is_dtx {
active_indices.insert(peer_idx);
}
// TTS interrupt frame counter — reset on TTS rising edge.
let tts_now = tts_active.load(Ordering::Acquire);
if tts_now && !tts_was_active {
frame_counts.clear();
last_frame_reset = tokio::time::Instant::now();
}
tts_was_active = tts_now;
let slot = match peers.entry(peer_idx) {
std::collections::hash_map::Entry::Occupied(e) => e.into_mut(),
std::collections::hash_map::Entry::Vacant(e) => {
let Some(slot) = PeerSlot::new(peer_idx, sink_handle.mixer())
else {
continue;
};
e.insert(slot)
}
};
// Sender-authored seq/ts: NetEq can detect real
// packet reordering & loss, not just arrival jitter.
if let Err(err) =
slot.jitter
.insert_packet(header.seq, header.ts_48k, opus_bytes)
{
eprintln!(
"sprout-desktop: jitter insert peer {peer_idx}: {err}"
);
} else {
// Heartbeat for the playout tick's idle-peer
// guard — only on successful insert so a stream
// of bad packets can't keep a dead peer "active".
slot.last_packet_at = tokio::time::Instant::now();
}
// Count remote-speech frame arrivals for the TTS
// interrupt. DTX/comfort frames don't count — they
// mean the peer is silent, just keeping the codec
// state alive.
if tts_now && !is_dtx {
if last_frame_reset.elapsed() >= FRAME_WINDOW {
frame_counts.clear();
last_frame_reset = tokio::time::Instant::now();
}
let count = frame_counts.entry(peer_idx).or_insert(0);
*count = count.saturating_add(1);
if *count >= REMOTE_SPEECH_THRESHOLD {
tts_cancel.store(true, Ordering::Release);
}
}
}
Some(Ok(WsMsg::Text(text))) => {
if let Ok(v) = serde_json::from_str::<serde_json::Value>(&text) {
match v["type"].as_str() {
Some("joined") => {
if let Some(peer_list) = v["peers"].as_array() {
for p in peer_list {
if let (Some(pk), Some(idx)) = (
p["pubkey"].as_str(),
p["peer_index"].as_u64(),
) {
let key = idx as u8;
// peer_index reuse with a new pubkey:
// flush the old peer's NetEq + Player so
// the next frame starts clean.
if index_to_pubkey
.get(&key)
.map(|s| s.as_str())
!= Some(pk)
{
peers.remove(&key);
frame_counts.remove(&key);
active_indices.remove(&key);
}
index_to_pubkey.insert(key, pk.to_string());
}
}
}
}
Some("left") => {
if let Some(idx) = v["peer_index"].as_u64() {
let key = idx as u8;
index_to_pubkey.remove(&key);
frame_counts.remove(&key);
// Dropping Player detaches its queue from the
// device mixer, freeing the per-peer slot.
peers.remove(&key);
}
}
_ => {}
}
}
}
Some(Ok(WsMsg::Ping(data))) => {
let mut tx = ws_tx_for_pongs.lock().await;
let _ = tx.send(WsMsg::Pong(data)).await;
}
Some(Ok(WsMsg::Close(_))) | None => break,
Some(Ok(_)) => {} // non-binary/text frame
Some(Err(_)) => break,
}
}
}
}
}

View File

@@ -11,10 +11,7 @@
//! ```
use futures_util::{SinkExt, StreamExt};
use std::sync::{
atomic::{AtomicBool, Ordering},
Arc,
};
use std::sync::{atomic::AtomicBool, Arc};
use tokio_tungstenite::{connect_async, tungstenite::Message as WsMsg};
use tokio_util::sync::CancellationToken;
use uuid::Uuid;
@@ -112,6 +109,12 @@ pub(crate) async fn connect_audio_relay(
"type": "auth",
"event": event_json,
"parent_channel_id": parent_channel_id,
// Negotiate huddle audio protocol v2 (8-byte sender-authored header
// per Opus frame: seq | ts_48k | level_dbov | flags). See
// huddle::wire for the layout. The relay pins the first joiner's
// version per-room and rejects mismatched joiners with
// `upgrade_required`.
"protocol_version": super::wire::PROTOCOL_VERSION,
});
ws_tx
.send(WsMsg::Text(auth_msg.to_string().into()))
@@ -198,12 +201,12 @@ pub(crate) async fn connect_audio_relay(
}
/// Background Opus encode/decode pipeline spawned by `connect_audio_relay`.
type WsStream =
pub(crate) type WsStream =
tokio_tungstenite::WebSocketStream<tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>>;
async fn audio_relay_pipeline(
ws_tx: futures_util::stream::SplitSink<WsStream, WsMsg>,
mut ws_rx: futures_util::stream::SplitStream<WsStream>,
ws_rx: futures_util::stream::SplitStream<WsStream>,
mut pcm_rx: tokio::sync::mpsc::Receiver<Vec<u8>>,
cancel: CancellationToken,
app_handle: Option<tauri::AppHandle>,
@@ -222,11 +225,6 @@ async fn audio_relay_pipeline(
.map_err(|e| format!("opus dtx: {e}"))?;
let sink_handle = super::audio_output::open_output_sink_by_name(output_device_name.as_deref())?;
let player = rodio::Player::connect_new(&sink_handle.mixer());
let decoders: std::collections::HashMap<u8, opus::Decoder> = std::collections::HashMap::new();
const FRAME_SAMPLES: usize = 960; // 20 ms at 48 kHz
let decode_buf = vec![0f32; FRAME_SAMPLES];
use std::sync::Arc as StdArc;
let ws_tx = StdArc::new(tokio::sync::Mutex::new(ws_tx));
@@ -234,9 +232,15 @@ async fn audio_relay_pipeline(
let cancel_send = cancel.clone();
let send_task = tokio::spawn(async move {
use super::wire::{audio_level_dbov, FrameHeader, V2_HEADER_LEN};
let mut encoder = encoder; // Move encoder into task.
const FRAME_SAMPLES: usize = 960;
let mut out_buf = vec![0u8; 4000];
// Per-frame wire-protocol state. We send v2 frames now: each Opus
// payload is preceded by an 8-byte header carrying our own seq +
// 48 kHz timestamp + audio level + flags.
let mut seq: u16 = 0;
let mut ts_48k: u32 = 0;
loop {
let pcm_bytes = {
@@ -260,6 +264,10 @@ async fn audio_relay_pipeline(
let mut tx = ws_tx_send.lock().await;
for chunk in samples.chunks(FRAME_SAMPLES) {
// dBov is computed from the pre-encode PCM. Opus DTX may
// produce a 1-2 byte comfort packet; computing level from
// the encoded payload would be meaningless.
let level = audio_level_dbov(chunk);
let encode_result = if chunk.len() == FRAME_SAMPLES {
encoder.encode_float(chunk, &mut out_buf)
} else {
@@ -275,14 +283,28 @@ async fn audio_relay_pipeline(
}
};
if n > 0 {
// Send raw Opus bytes — the relay prepends the peer_index.
if tx
.send(WsMsg::Binary(out_buf[..n].to_vec().into()))
.await
.is_err()
{
// Opus DTX packets are very small (≤2 bytes). Flag them
// explicitly so the receiver can elide DTX from speaker
// detection without re-parsing the Opus payload.
let flags = if n <= 2 { super::wire::FLAG_DTX } else { 0 };
let header = FrameHeader {
seq,
ts_48k,
level_dbov: level,
flags,
}
.encode();
// Build the v2 wire frame: 8-byte header + Opus payload.
let mut frame = Vec::with_capacity(V2_HEADER_LEN + n);
frame.extend_from_slice(&header);
frame.extend_from_slice(&out_buf[..n]);
if tx.send(WsMsg::Binary(frame.into())).await.is_err() {
return; // WS closed.
}
seq = seq.wrapping_add(1);
ts_48k = ts_48k.wrapping_add(super::jitter::FRAME_TIMESTAMP_DELTA);
}
}
}
@@ -290,154 +312,16 @@ async fn audio_relay_pipeline(
let _ = tx.send(WsMsg::Close(None)).await;
});
let recv_task = tokio::spawn(async move {
let mut decoders = decoders;
let mut decode_buf = decode_buf;
let cancel_recv = cancel;
let ws_tx_recv = ws_tx;
// Active speaker tracking: peer_index → pubkey, emit every 500ms.
let mut index_to_pubkey: std::collections::HashMap<u8, String> =
initial_peers.into_iter().collect();
let mut active_indices: std::collections::HashSet<u8> = std::collections::HashSet::new();
// Per-peer frame counter with Instant-based window (starvation-proof).
let mut frame_counts: std::collections::HashMap<u8, u16> = std::collections::HashMap::new();
let mut last_frame_reset = tokio::time::Instant::now();
const FRAME_WINDOW: std::time::Duration = std::time::Duration::from_millis(500);
let mut tts_was_active = false;
let mut speaker_tick = tokio::time::interval(std::time::Duration::from_millis(500));
speaker_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
loop {
use futures_util::future::Either;
let cancelled = std::pin::pin!(cancel_recv.cancelled());
let next = std::pin::pin!(ws_rx.next());
let tick = std::pin::pin!(speaker_tick.tick());
// Three-way select: cancel, WS message, or speaker tick.
let ws_or_tick = std::pin::pin!(futures_util::future::select(next, tick));
match futures_util::future::select(cancelled, ws_or_tick).await {
Either::Left(_) => break, // Cancelled.
Either::Right((Either::Right((_, _)), _)) => {
// Speaker tick: emit active speakers and reset.
if let Some(ref app) = app_handle {
use tauri::Emitter;
let pubkeys: Vec<String> = active_indices
.iter()
.filter_map(|idx| index_to_pubkey.get(idx).cloned())
.collect();
let _ = app.emit("huddle-active-speakers", &pubkeys);
}
active_indices.clear();
continue;
}
Either::Right((Either::Left((Some(Ok(msg)), _)), _)) => {
// WS message — process below.
match msg {
WsMsg::Binary(data) => {
if data.len() < 2 {
continue;
}
let peer_idx = data[0];
let opus_bytes = &data[1..];
active_indices.insert(peer_idx);
// Track TTS transitions at frame level (not decode level).
let tts_now = tts_active.load(Ordering::Acquire);
if tts_now && !tts_was_active {
frame_counts.clear();
last_frame_reset = tokio::time::Instant::now();
}
tts_was_active = tts_now;
let decoder = match decoders.entry(peer_idx) {
std::collections::hash_map::Entry::Occupied(e) => e.into_mut(),
std::collections::hash_map::Entry::Vacant(e) => {
match opus::Decoder::new(48000, opus::Channels::Mono) {
Ok(d) => e.insert(d),
Err(err) => {
eprintln!("sprout-desktop: opus decoder init peer {peer_idx}: {err}");
continue;
}
}
}
};
match decoder.decode_float(opus_bytes, &mut decode_buf, false) {
Ok(n) if n > 0 => {
if tts_now {
if last_frame_reset.elapsed() >= FRAME_WINDOW {
frame_counts.clear();
last_frame_reset = tokio::time::Instant::now();
}
let count = frame_counts.entry(peer_idx).or_insert(0);
*count = count.saturating_add(1);
if *count >= REMOTE_SPEECH_THRESHOLD {
tts_cancel.store(true, Ordering::Release);
}
}
use rodio::buffer::SamplesBuffer;
use std::num::NonZero;
let channels = NonZero::new(1u16).unwrap();
let rate = NonZero::new(48000u32).unwrap();
player.append(SamplesBuffer::new(
channels,
rate,
decode_buf[..n].to_vec(),
));
}
Ok(_) => {} // DTX silence — not counted.
Err(e) => {
eprintln!("sprout-desktop: opus decode peer {peer_idx}: {e}");
}
}
}
WsMsg::Text(text) => {
if let Ok(v) = serde_json::from_str::<serde_json::Value>(&text) {
match v["type"].as_str() {
Some("joined") => {
if let Some(peers) = v["peers"].as_array() {
for p in peers {
if let (Some(pk), Some(idx)) =
(p["pubkey"].as_str(), p["peer_index"].as_u64())
{
let key = idx as u8;
if index_to_pubkey.get(&key).map(|s| s.as_str())
!= Some(pk)
{
decoders.remove(&key);
frame_counts.remove(&key);
active_indices.remove(&key);
}
index_to_pubkey.insert(key, pk.to_string());
}
}
}
}
Some("left") => {
if let Some(idx) = v["peer_index"].as_u64() {
let key = idx as u8;
index_to_pubkey.remove(&key);
frame_counts.remove(&key);
decoders.remove(&key);
}
}
_ => {}
}
}
}
WsMsg::Ping(data) => {
let mut tx = ws_tx_recv.lock().await;
let _ = tx.send(WsMsg::Pong(data)).await;
}
WsMsg::Close(_) => break,
_ => {}
}
}
// WS error or closed.
Either::Right((Either::Left(_), _)) => break,
}
}
});
let recv_task = tokio::spawn(super::playout::run_playout_recv_loop(
ws_rx,
ws_tx,
sink_handle,
cancel,
app_handle,
initial_peers,
tts_active,
tts_cancel,
));
// Wait for either task to finish, then abort the survivor.
use futures_util::future::Either;

View File

@@ -0,0 +1,292 @@
//! Huddle audio wire protocol — versioning + frame header.
//!
//! ## v1 (legacy)
//!
//! Client → relay: `<opus_bytes>`
//! Relay → client: `<peer_index: u8><opus_bytes>`
//!
//! No per-frame metadata; receiver synthesizes sequence/timestamp on arrival.
//! Kept for backward compatibility — relay still admits v1 clients into
//! v1-pinned rooms — but new clients always speak v2.
//!
//! ## v2 (this commit)
//!
//! Client → relay: `<header: [u8; 8]><opus_bytes>`
//! Relay → client: `<peer_index: u8><header: [u8; 8]><opus_bytes>`
//!
//! Header layout (8 bytes, network byte order, big-endian):
//!
//! ```text
//! byte 0..=1 : seq u16 wrapping sequence number, +1 per packet
//! byte 2..=5 : ts_48k u32 sender RTP-style 48 kHz media time,
//! +960 per 20 ms encoded frame
//! byte 6 : level_dbov i8 audio level in dBov, range [-127, 0]
//! byte 7 : flags u8 bit 0 = DTX/comfort frame; other bits
//! MUST be ignored on decode
//! ```
//!
//! Notes for reviewers (Max's spec):
//! * Header is fixed-size. No extension mechanism in v2 — future fields go
//! into a v3 with a new pinned version.
//! * `level_dbov` is client-authored telemetry. The relay parses it for
//! logging/active-speaker hints, clamps invalid values into range, and
//! **never** uses it for trust decisions (admission, moderation, etc.).
//! * Negotiation lives in the WS auth message (`protocol_version: 2`), not
//! in any bit of `flags`. Mixed-version rooms are rejected at the relay
//! with `upgrade_required`.
/// Wire protocol version this client speaks. Bumped only when the frame
/// layout itself changes; the relay tracks pinned per-room.
pub const PROTOCOL_VERSION: u8 = 2;
/// Length of the v2 per-frame header in bytes.
pub const V2_HEADER_LEN: usize = 8;
/// Flag bit indicating a DTX / comfort-noise frame. Receivers MAY skip
/// arrival accounting for these; encoders MUST set it for any Opus packet
/// they tag as DTX. Unset on normal speech frames.
pub const FLAG_DTX: u8 = 0x01;
/// Reserved bit-mask. Any bits outside this set MUST be ignored by the
/// receiver — they're available for the next minor protocol bump within v2.
#[allow(dead_code)] // Public constant for diagnostics / future flag additions.
pub const RESERVED_FLAG_MASK: u8 = !FLAG_DTX;
/// Parsed view of one v2 header. Cheap (Copy).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct FrameHeader {
/// Sender-authored sequence number; wraps every 2^16 frames (~21.8 min
/// at 50 Hz).
pub seq: u16,
/// Sender-authored 48 kHz RTP-style media timestamp.
pub ts_48k: u32,
/// Audio level in dBov. Always parsed into the canonical `-127..=0`
/// range — out-of-range inputs are clamped to `-127` (silence).
pub level_dbov: i8,
/// Raw flags byte. Use [`Self::is_dtx`] / explicit masks; bits outside
/// `FLAG_DTX | reserved` are not inspected.
pub flags: u8,
}
impl FrameHeader {
/// Encode `self` into 8 bytes, network byte order. Suitable for direct
/// concatenation onto an Opus payload.
pub fn encode(self) -> [u8; V2_HEADER_LEN] {
let mut out = [0u8; V2_HEADER_LEN];
out[0..2].copy_from_slice(&self.seq.to_be_bytes());
out[2..6].copy_from_slice(&self.ts_48k.to_be_bytes());
out[6] = self.level_dbov as u8;
out[7] = self.flags;
out
}
/// Parse a v2 header from the leading 8 bytes of `bytes`. Returns
/// `(header, remainder)` so callers can pass the remainder straight to
/// the Opus decoder.
///
/// `level_dbov` is clamped into `-127..=0`; values outside that range
/// are coerced to `-127`. Per Max's spec, malformed VU metadata must
/// *not* cause the audio frame to be dropped — only the metric is
/// suppressed.
pub fn parse(bytes: &[u8]) -> Option<(Self, &[u8])> {
if bytes.len() < V2_HEADER_LEN {
return None;
}
let seq = u16::from_be_bytes([bytes[0], bytes[1]]);
let ts_48k = u32::from_be_bytes([bytes[2], bytes[3], bytes[4], bytes[5]]);
let raw_level = bytes[6] as i8;
let level_dbov = if (-127..=0).contains(&raw_level) {
raw_level
} else {
// Out-of-range telemetry is coerced to "silent floor"; the
// audio packet is still delivered. This matches Max's "don't
// drop audio on bad VU" guidance.
-127
};
let flags = bytes[7];
Some((
Self {
seq,
ts_48k,
level_dbov,
flags,
},
&bytes[V2_HEADER_LEN..],
))
}
/// True if [`FLAG_DTX`] is set.
#[allow(dead_code)]
pub fn is_dtx(&self) -> bool {
(self.flags & FLAG_DTX) != 0
}
}
/// Compute a dBov audio level for a normalized f32 PCM frame.
///
/// "dBov" is RMS expressed in dB relative to full scale (where full scale =
/// peak amplitude 1.0), clamped into the v2 header's canonical `-127..=0`
/// range. A silent frame returns `-127`; full-scale ±1.0 returns `0`.
///
/// This is cheap (one pass over the samples, one log) and runs once per
/// 20 ms frame, so it's nowhere near the audio fast-path budget.
pub fn audio_level_dbov(samples: &[f32]) -> i8 {
if samples.is_empty() {
return -127;
}
let mean_square: f64 = samples
.iter()
.map(|&s| {
let v = s as f64;
v * v
})
.sum::<f64>()
/ samples.len() as f64;
if mean_square <= 0.0 {
return -127;
}
let rms = mean_square.sqrt();
let db = 20.0 * rms.log10();
// Clamp into the canonical range. -127 = silence floor, 0 = full scale.
if !db.is_finite() || db <= -127.0 {
-127
} else if db >= 0.0 {
0
} else {
db.round() as i8
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn round_trip_preserves_fields() {
let h = FrameHeader {
seq: 0xABCD,
ts_48k: 0x12_34_56_78,
level_dbov: -40,
flags: FLAG_DTX,
};
let bytes = h.encode();
let (parsed, tail) = FrameHeader::parse(&bytes).expect("parse");
assert_eq!(parsed, h);
assert!(tail.is_empty());
assert!(parsed.is_dtx());
}
/// Header is fixed 8 bytes; anything shorter is rejected.
#[test]
fn parse_rejects_short_input() {
for len in 0..V2_HEADER_LEN {
let buf = vec![0u8; len];
assert!(
FrameHeader::parse(&buf).is_none(),
"{len} bytes must be rejected",
);
}
}
/// Trailing bytes pass through as the remainder, which is what the
/// receive path hands to the Opus decoder.
#[test]
fn parse_returns_payload_remainder() {
let mut buf = FrameHeader {
seq: 1,
ts_48k: 960,
level_dbov: -20,
flags: 0,
}
.encode()
.to_vec();
buf.extend_from_slice(b"opus-bytes");
let (_h, tail) = FrameHeader::parse(&buf).expect("parse");
assert_eq!(tail, b"opus-bytes");
}
/// Bytes in big-endian network order, matching Max's spec. This pins
/// the byte layout against accidental endianness changes.
#[test]
fn encoded_byte_order_is_network_byte_order() {
let h = FrameHeader {
seq: 0x0102,
ts_48k: 0x0304_0506,
level_dbov: -1,
flags: 0,
};
let bytes = h.encode();
assert_eq!(bytes[0..2], [0x01, 0x02]);
assert_eq!(bytes[2..6], [0x03, 0x04, 0x05, 0x06]);
assert_eq!(bytes[6], 0xFF); // -1 as i8 -> 0xFF
assert_eq!(bytes[7], 0x00);
}
/// Per Max: malformed `level_dbov` must not drop the frame; only the
/// metric is suppressed. We coerce to -127 (silence floor) and keep
/// the payload available.
#[test]
fn out_of_range_level_dbov_is_clamped_not_rejected() {
// i8(127) is +127 which is outside [-127, 0].
let mut bytes = FrameHeader {
seq: 7,
ts_48k: 7 * 960,
level_dbov: 0,
flags: 0,
}
.encode();
bytes[6] = 0x7F; // raw +127 — out of range
let (parsed, _) = FrameHeader::parse(&bytes).expect("parse must succeed");
assert_eq!(parsed.level_dbov, -127, "clamped to silence floor");
assert_eq!(parsed.seq, 7, "other fields preserved");
}
#[test]
fn audio_level_dbov_clamps_silence_to_minus_127() {
assert_eq!(audio_level_dbov(&[]), -127);
assert_eq!(audio_level_dbov(&[0.0_f32; 960]), -127);
}
#[test]
fn audio_level_dbov_full_scale_returns_zero() {
let frame: Vec<f32> = (0..960).map(|_| 1.0_f32).collect();
assert_eq!(audio_level_dbov(&frame), 0);
}
#[test]
fn audio_level_dbov_is_in_canonical_range_for_realistic_speech() {
// A 1 kHz sine at amplitude 0.3 — typical normalized speech peak.
// RMS ≈ 0.3 / sqrt(2) ≈ 0.212 → ≈ -13 dBov.
let frame: Vec<f32> = (0..960)
.map(|i| {
let t = i as f32 / 48_000.0;
0.3 * (2.0 * std::f32::consts::PI * 1_000.0 * t).sin()
})
.collect();
let db = audio_level_dbov(&frame);
assert!(
(-20..=-8).contains(&db),
"expected -20..=-8 dBov for 0.3-amp sine, got {db}",
);
}
/// Reserved flag bits MUST be ignored on parse (no error, just preserved
/// for forward-compat). The contract is that *receivers* don't inspect
/// them; the parse path keeps them so introspection still works.
#[test]
fn reserved_flag_bits_are_preserved_for_inspection() {
let mut bytes = FrameHeader {
seq: 0,
ts_48k: 0,
level_dbov: 0,
flags: 0,
}
.encode();
bytes[7] = 0b1010_1010; // FLAG_DTX bit clear, reserved bits set
let (parsed, _) = FrameHeader::parse(&bytes).expect("parse");
assert_eq!(parsed.flags, 0b1010_1010);
assert!(!parsed.is_dtx());
// Caller can still check which reserved bits were set if needed.
assert_eq!(parsed.flags & RESERVED_FLAG_MASK, 0b1010_1010);
}
}

View File

@@ -0,0 +1,142 @@
//! Diagnostic regression test: rodio's `Player` is a single-queue FIFO, not a summing mixer.
//!
//! This pins the failure mode we hit at 3+ simultaneous speakers in `huddle`:
//! `Player::connect_new(&device_mixer)` calls `device_mixer.add(queue_source)` exactly once,
//! and every `player.append(SamplesBuffer)` enqueues onto that single queue. Multiple peers'
//! 20 ms frames therefore serialize — peer B plays after peer A finishes, etc. — instead of
//! mixing.
//!
//! The fix is per-peer `Player` (each peer gets its own queue added to the device mixer), so
//! the device mixer actually sums across queues. This test asserts both halves of the
//! diagnosis with a deterministic, device-free harness:
//!
//! * A single queue receiving two sources → samples arrive **serially**.
//! * A summing mixer fed by two sources → samples arrive **concurrently and summed**.
//!
//! If rodio ever changes either invariant under us we want CI to scream, not the next
//! 3-person huddle.
use rodio::buffer::SamplesBuffer;
use rodio::mixer;
use rodio::queue;
use rodio::source::Source;
use std::num::NonZero;
const SR: u32 = 48_000;
const CH: u16 = 1;
/// 20 ms at 48 kHz mono — one Opus frame's worth of decoded samples.
const FRAME_SAMPLES: usize = 960;
fn channels() -> NonZero<u16> {
NonZero::new(CH).unwrap()
}
fn sample_rate() -> NonZero<u32> {
NonZero::new(SR).unwrap()
}
/// Build a `SamplesBuffer` of length `n` samples filled with `value`.
fn buf(value: f32, n: usize) -> SamplesBuffer {
SamplesBuffer::new(channels(), sample_rate(), vec![value; n])
}
/// Drain at most `limit` samples from any `Source` and return them.
fn drain<S: Source<Item = f32>>(src: S, limit: usize) -> Vec<f32> {
src.take(limit).collect()
}
/// A single `queue` plays sources **one after the other** (FIFO).
///
/// This is the shape `rodio::Player::connect_new` produces: one queue is added to the
/// device mixer at start, and every `Player::append` enqueues onto that one queue.
/// In the huddle path, every peer's decoded frame was appended here — serializing them.
#[test]
fn single_queue_serializes_sources() {
let (input, output) = queue::queue(false /* don't keep alive when empty */);
// Two "peers": A produces all 1.0s, B produces all -1.0s. If they were summed, the
// overlap would be ~0.0; if they're serialized we see a clean 1.0 → -1.0 transition.
input.append(buf(1.0, FRAME_SAMPLES));
input.append(buf(-1.0, FRAME_SAMPLES));
let samples = drain(output, FRAME_SAMPLES * 4);
// Total samples produced = sum of inputs (serial). If it were summed we'd only get
// FRAME_SAMPLES of output (and at amplitude 0.0).
assert_eq!(
samples.len(),
FRAME_SAMPLES * 2,
"single queue should drain both sources back-to-back (serial), not mix them",
);
// The first source's samples come first, then the second's. No interleaving, no summing.
assert!(
samples[..FRAME_SAMPLES].iter().all(|&s| s == 1.0),
"first half of queue output should be source A's samples (1.0), not a mix",
);
assert!(
samples[FRAME_SAMPLES..].iter().all(|&s| s == -1.0),
"second half of queue output should be source B's samples (-1.0)",
);
}
/// A `mixer` plays sources **concurrently** and **sums** overlapping samples.
///
/// This is the shape we want for huddle playout: each peer gets its own source added to
/// the device mixer, so simultaneous speakers actually mix instead of serializing.
#[test]
fn mixer_sums_overlapping_sources() {
let (controller, mixer_source) = mixer::mixer(channels(), sample_rate());
// Same two "peers" — 1.0 and -1.0. If the mixer sums correctly, overlap == 0.0.
controller.add(buf(1.0, FRAME_SAMPLES));
controller.add(buf(-1.0, FRAME_SAMPLES));
// Take exactly one frame's worth — both sources should be active across that window.
let samples = drain(mixer_source, FRAME_SAMPLES);
assert_eq!(
samples.len(),
FRAME_SAMPLES,
"mixer should produce one frame's worth of mixed output, not two frames serialized",
);
// Both sources are active over this window. Their sum is 0.0 at every sample.
// Allow for f32 rounding (rodio's UniformSourceIterator + sum can introduce a few ULPs).
let max_abs = samples.iter().map(|s| s.abs()).fold(0.0_f32, f32::max);
assert!(
max_abs < 1e-5,
"mixer should sum 1.0 + -1.0 ≈ 0.0 per sample; saw max |sample| = {max_abs} \
which means the sources weren't actually mixed concurrently",
);
}
/// One mixer fed by **two distinct queues** (= two distinct `Player`s on one device sink)
/// behaves the same way: the queues drain concurrently, the mixer sums them.
///
/// This is the exact shape the huddle fix moves to: `HashMap<peer_index, Player>`, each
/// added to `MixerDeviceSink::mixer()`. The test pins that this composes correctly.
#[test]
fn mixer_of_queues_mixes_per_peer_streams() {
let (mixer_in, mixer_source) = mixer::mixer(channels(), sample_rate());
// Two queues — one per "peer".
let (peer_a_in, peer_a_out) =
queue::queue(true /* keep alive — players outlive frames */);
let (peer_b_in, peer_b_out) = queue::queue(true);
mixer_in.add(peer_a_out);
mixer_in.add(peer_b_out);
// Each peer pushes one 20 ms frame at the same wall-clock moment.
peer_a_in.append(buf(1.0, FRAME_SAMPLES));
peer_b_in.append(buf(-1.0, FRAME_SAMPLES));
let samples = drain(mixer_source, FRAME_SAMPLES);
assert_eq!(samples.len(), FRAME_SAMPLES);
let max_abs = samples.iter().map(|s| s.abs()).fold(0.0_f32, f32::max);
assert!(
max_abs < 1e-5,
"per-peer queues into one mixer should sum concurrently; saw max |sample| = {max_abs}",
);
}

View File

@@ -0,0 +1,72 @@
import { Headphones } from "lucide-react";
import {
AlertDialog,
AlertDialogAction,
AlertDialogCancel,
AlertDialogContent,
AlertDialogDescription,
AlertDialogFooter,
AlertDialogHeader,
AlertDialogTitle,
} from "@/shared/ui/alert-dialog";
/**
* Pre-join "use headphones" confirmation dialog.
*
* Shown once per browser-tab session when the user clicks the start/join
* huddle button while echo cancellation is missing. The mic isn't opened
* and `start_huddle` / `join_huddle` is not called until the user clicks
* "Continue". After a confirmed continue (or an explicit cancel) the gate
* remembers its decision in `sessionStorage` so the same user isn't
* nagged on every channel hop.
*
* Pair with [`useHeadphonesGate`]. The hook owns the storage + decision
* state; this component is presentational.
*
* Removable in one diff alongside `HeadphonesNotice` when the WebAudio
* AEC follow-up flips `aecMissing` to false in `HuddleBar`.
*/
export function HeadphonesGate({
open,
onContinue,
onCancel,
}: {
open: boolean;
onContinue: () => void;
onCancel: () => void;
}) {
return (
<AlertDialog
open={open}
onOpenChange={(next) => {
// Radix calls onOpenChange(false) when the user clicks outside or
// presses Escape. Treat that as cancel — never as a silent join.
if (!next) onCancel();
}}
>
<AlertDialogContent>
<AlertDialogHeader>
<AlertDialogTitle className="flex items-center gap-2">
<Headphones className="h-4 w-4" />
Headphones recommended
</AlertDialogTitle>
<AlertDialogDescription>
Echo cancellation lands in the next release. Until then, two or more
people on speakers in the same room will hear themselves echo. Use
headphones for the best experience.
</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
<AlertDialogCancel onClick={onCancel}>Cancel</AlertDialogCancel>
<AlertDialogAction
data-testid="huddle-headphones-gate-continue"
onClick={onContinue}
>
Continue
</AlertDialogAction>
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialog>
);
}

View File

@@ -0,0 +1,41 @@
import { Headphones } from "lucide-react";
/**
* "Headphones recommended — echo cancellation lands in the next release."
*
* Session-dismissable banner shown on the active HuddleBar while the desktop
* client lacks an echo-cancellation reference. The current play path is
* native rodio (outside the WebView render graph), so the browser's WebRTC
* AEC cannot suppress local-speaker → mic feedback for users on speakers.
*
* The follow-up PR moves remote-peer playout into WebAudio inside the same
* `AudioContext` as `getUserMedia({ echoCancellation: true })`. When that
* lands, the parent component flips `aecMissing` to false and this banner
* (and this file) become removable in a single diff.
*
* State (`dismissed`, `aecMissing`) lives in the parent so the parent can
* decide whether to render at all. Keep this component dumb so the
* deletion later is mechanical.
*/
export function HeadphonesNotice({ onDismiss }: { onDismiss: () => void }) {
return (
<div
role="status"
data-testid="huddle-headphones-notice"
className="flex items-center gap-1.5 rounded bg-amber-500/10 px-2 py-1 text-xs text-amber-700 dark:text-amber-300"
>
<Headphones className="h-3 w-3" />
<span className="max-w-[260px] truncate">
Headphones recommended — echo cancellation lands in the next release.
</span>
<button
aria-label="Dismiss headphones notice for this session"
className="ml-1 opacity-60 hover:opacity-100"
onClick={onDismiss}
type="button"
>
✕
</button>
</div>
);
}

View File

@@ -8,6 +8,7 @@ import { Button } from "@/shared/ui/button";
import { Tooltip, TooltipContent, TooltipTrigger } from "@/shared/ui/tooltip";
import { useHuddle } from "../HuddleContext";
import { AddAgentDialog, type AgentAddResult } from "./AddAgentDialog";
import { HeadphonesNotice } from "./HeadphonesNotice";
import { MicControls, SpeakerControls } from "./MicControls";
import { ParticipantList } from "./ParticipantList";
@@ -60,6 +61,16 @@ export function HuddleBar({ className }: HuddleBarProps) {
const isPttMode = voiceInputMode === "push_to_talk";
const [state, setState] = React.useState<HuddleState | null>(null);
const [isMuted, setIsMuted] = React.useState(false);
// Session-dismissable "use headphones" notice. The desktop client plays
// remote peer audio via native rodio, outside the WebView render graph,
// so the browser's echo canceller has no far-end reference to cancel
// against. Until the WebAudio AEC follow-up PR lands, two people on
// speakers in the same physical room will hear themselves echo. The
// banner stays visible across huddle start/stop within a single page
// load; it auto-removes when the AEC plumbing lands and the detection
// below flips to false.
const [headphonesNoticeDismissed, setHeadphonesNoticeDismissed] =
React.useState(false);
const ttsEnabled = state?.tts_enabled ?? true;
const [isLeaving, setIsLeaving] = React.useState(false);
const [showAddAgent, setShowAddAgent] = React.useState(false);
@@ -165,6 +176,12 @@ export function HuddleBar({ className }: HuddleBarProps) {
if (!state || (state.phase !== "active" && state.phase !== "connected"))
return null;
// Self-removing detection: remote-peer audio plays through native rodio
// today (outside the WebView render graph), so the browser's AEC has no
// far-end reference. The AEC follow-up PR flips this constant in the
// same diff that moves playout into WebAudio.
const aecMissing = true;
async function handleLeave() {
if (isLeaving) return;
setIsLeaving(true);
@@ -226,6 +243,13 @@ export function HuddleBar({ className }: HuddleBarProps) {
</div>
)}
{/* Echo-cancellation pre-PR notice. Removed when AEC plumbing lands. */}
{aecMissing && !headphonesNoticeDismissed && (
<HeadphonesNotice
onDismiss={() => setHeadphonesNoticeDismissed(true)}
/>
)}
{/* Model download progress */}
{modelStatus &&
(modelStatus.stt !== "ready" || modelStatus.tts !== "ready") && (

View File

@@ -9,6 +9,8 @@ import { cn } from "@/shared/lib/cn";
import { Button } from "@/shared/ui/button";
import { Tooltip, TooltipContent, TooltipTrigger } from "@/shared/ui/tooltip";
import { useHuddle } from "../HuddleContext";
import { useHeadphonesGate } from "../lib/useHeadphonesGate";
import { HeadphonesGate } from "./HeadphonesGate";
/** Huddle lifecycle event kinds */
const KIND_HUDDLE_STARTED = 48100;
@@ -48,6 +50,13 @@ export function HuddleIndicator({
);
const [isJoining, setIsJoining] = React.useState(false);
// Pre-join "use headphones" confirmation while echo cancellation is
// missing. Mirrors the `aecMissing` constant in HuddleBar — both flip
// to false in the AEC follow-up PR, after which this hook is a no-op
// and the gate UI/components are mechanically deletable.
const aecMissing = true;
const headphonesGate = useHeadphonesGate(aecMissing);
React.useEffect(() => {
if (!channelId) return;
@@ -203,22 +212,36 @@ export function HuddleIndicator({
};
}, []);
// Pre-join headphones confirmation. The same dialog instance serves
// both the start and the join paths — `gate` decides which deferred
// action to fire on Continue.
const gateDialog = (
<HeadphonesGate
open={headphonesGate.dialogOpen}
onContinue={headphonesGate.onContinue}
onCancel={headphonesGate.onCancel}
/>
);
// No active huddle — render the start button (if onStart provided).
if (!activeHuddle) {
if (!onStart) return null;
return (
<Button
aria-label="Start huddle"
className={cn("h-7 w-7 rounded-full", className)}
data-testid="channel-start-huddle-trigger"
disabled={startDisabled || isStarting}
onClick={() => onStart()}
size="icon"
type="button"
variant="outline"
>
<Headphones className="h-3 w-3" />
</Button>
<>
<Button
aria-label="Start huddle"
className={cn("h-7 w-7 rounded-full", className)}
data-testid="channel-start-huddle-trigger"
disabled={startDisabled || isStarting}
onClick={() => headphonesGate.gate(() => onStart())}
size="icon"
type="button"
variant="outline"
>
<Headphones className="h-3 w-3" />
</Button>
{gateDialog}
</>
);
}
@@ -227,7 +250,7 @@ export function HuddleIndicator({
// reconstructed set — floor at 1 to avoid showing "0 participants".
const participantCount = Math.max(1, activeHuddle.participants.size);
async function handleJoin() {
async function doJoin() {
if (!activeHuddle || isJoining) return;
setIsJoining(true);
try {
@@ -242,30 +265,33 @@ export function HuddleIndicator({
}
return (
<Tooltip>
<TooltipTrigger asChild>
<Button
aria-label={`Join active huddle (${participantCount} participant${participantCount !== 1 ? "s" : ""})`}
className={cn("relative h-7 w-7 rounded-full", className)}
disabled={isJoining || isStarting}
onClick={() => void handleJoin()}
size="icon"
type="button"
variant="outline"
>
<Headphones className="h-3 w-3 text-muted-foreground" />
<span className="absolute inset-0 animate-pulse rounded-full ring-2 ring-border/70" />
{/* Participant count badge */}
{participantCount > 0 && (
<span className="absolute -right-1 -top-1 flex h-3.5 min-w-3.5 items-center justify-center rounded-full border border-border bg-background px-0.5 text-[9px] font-bold text-muted-foreground">
{participantCount}
</span>
)}
</Button>
</TooltipTrigger>
<TooltipContent>
{`Huddle active — ${participantCount} participant${participantCount !== 1 ? "s" : ""}`}
</TooltipContent>
</Tooltip>
<>
<Tooltip>
<TooltipTrigger asChild>
<Button
aria-label={`Join active huddle (${participantCount} participant${participantCount !== 1 ? "s" : ""})`}
className={cn("relative h-7 w-7 rounded-full", className)}
disabled={isJoining || isStarting}
onClick={() => headphonesGate.gate(() => void doJoin())}
size="icon"
type="button"
variant="outline"
>
<Headphones className="h-3 w-3 text-muted-foreground" />
<span className="absolute inset-0 animate-pulse rounded-full ring-2 ring-border/70" />
{/* Participant count badge */}
{participantCount > 0 && (
<span className="absolute -right-1 -top-1 flex h-3.5 min-w-3.5 items-center justify-center rounded-full border border-border bg-background px-0.5 text-[9px] font-bold text-muted-foreground">
{participantCount}
</span>
)}
</Button>
</TooltipTrigger>
<TooltipContent>
{`Huddle active — ${participantCount} participant${participantCount !== 1 ? "s" : ""}`}
</TooltipContent>
</Tooltip>
{gateDialog}
</>
);
}

View File

@@ -0,0 +1,78 @@
import * as React from "react";
/**
* Owns the "should we show the pre-join headphones warning?" decision.
*
* Wraps an arbitrary click handler — `gate(action)` returns a function that
* either runs `action` directly (already confirmed this session) or shows
* the confirmation dialog and runs `action` only after the user clicks
* Continue. Cancel/Escape/click-outside all decline silently.
*
* The "confirmed this session" bit is stored in `sessionStorage`, so it
* resets when the desktop window is closed but persists across channel
* navigation in the same window. That matches Quinn's recipe in the
* design thread: "session-dismissable, self-removing once AEC lands."
*
* `aecMissing` is a prop, not a feature-detect inside the hook, because
* the AEC follow-up PR flips it from a parent constant; centralizing the
* predicate makes the future deletion mechanical.
*
* Returns an object with `dialogOpen`, `gate(action)`, `onContinue`, and
* `onCancel`. Render the `HeadphonesGate` component with the dialog props.
*/
const STORAGE_KEY = "huddle.headphones-gate-confirmed";
function readConfirmed(): boolean {
try {
return sessionStorage.getItem(STORAGE_KEY) === "1";
} catch {
// sessionStorage can throw in some embedded WebView contexts; treat
// failure as "not confirmed" so the gate still warns conservatively.
return false;
}
}
function writeConfirmed() {
try {
sessionStorage.setItem(STORAGE_KEY, "1");
} catch {
/* best-effort; if storage is denied the user just sees the gate next time */
}
}
export function useHeadphonesGate(aecMissing: boolean) {
const [dialogOpen, setDialogOpen] = React.useState(false);
const pendingActionRef = React.useRef<(() => void) | null>(null);
const gate = React.useCallback(
(action: () => void) => {
// Once AEC works (predicate flips to false), this hook is a no-op:
// run the action immediately. The component using this hook can stay
// in place; deleting the hook + gate + notice happens in one diff
// when the follow-up PR makes aecMissing always false.
if (!aecMissing || readConfirmed()) {
action();
return;
}
pendingActionRef.current = action;
setDialogOpen(true);
},
[aecMissing],
);
const onContinue = React.useCallback(() => {
writeConfirmed();
const action = pendingActionRef.current;
pendingActionRef.current = null;
setDialogOpen(false);
action?.();
}, []);
const onCancel = React.useCallback(() => {
pendingActionRef.current = null;
setDialogOpen(false);
}, []);
return { dialogOpen, gate, onContinue, onCancel };
}