From d0cc02ed3a506602782c0887e7cb0f75772178dc Mon Sep 17 00:00:00 2001 From: Meli <5aaa86bce934fc3445fc254aab560a40923f10252f92107e665073dede0e04d3@buzz.block.builderlab.xyz> Date: Thu, 10 Sep 2026 20:46:14 -0400 Subject: [PATCH] feat(agent): stream Realtime voice through ACP with a portable Frankie demo Signed-off-by: Meli <5aaa86bce934fc3445fc254aab560a40923f10252f92107e665073dede0e04d3@buzz.block.builderlab.xyz> --- .github/workflows/ci.yml | 2 + Justfile | 6 +- crates/buzz-agent/README.md | 44 +- crates/buzz-agent/src/lib.rs | 41 +- crates/buzz-agent/src/realtime.rs | 37 +- crates/buzz-agent/src/realtime_media.rs | 199 ++++ crates/buzz-agent/src/realtime_session.rs | 101 +- .../buzz-agent/src/realtime_session/media.rs | 683 ++++++++++++ crates/buzz-agent/src/wire.rs | 4 +- crates/buzz-agent/tests/fake_llm.rs | 18 +- crates/buzz-agent/tests/realtime_acp.rs | 721 +++++++++++++ desktop/src-tauri/Cargo.lock | 2 + examples/realtime-audio/README.md | 170 +++ examples/realtime-audio/audio-worklet.mjs | 185 ++++ .../realtime-audio/audio-worklet.test.mjs | 141 +++ examples/realtime-audio/capture-queue.mjs | 31 + .../realtime-audio/capture-queue.test.mjs | 25 + examples/realtime-audio/client.mjs | 162 +++ examples/realtime-audio/index.html | 22 + examples/realtime-audio/launch.mjs | 272 +++++ examples/realtime-audio/mcp-shell.mjs | 141 +++ examples/realtime-audio/server.mjs | 250 +++++ examples/realtime-audio/server.test.mjs | 206 ++++ examples/realtime-audio/style.css | 988 ++++++++++++++++++ examples/realtime-audio/ui.mjs | 240 +++++ examples/realtime-audio/voice.mjs | 528 ++++++++++ 26 files changed, 5177 insertions(+), 42 deletions(-) create mode 100644 crates/buzz-agent/src/realtime_media.rs create mode 100644 crates/buzz-agent/src/realtime_session/media.rs create mode 100644 examples/realtime-audio/README.md create mode 100644 examples/realtime-audio/audio-worklet.mjs create mode 100644 examples/realtime-audio/audio-worklet.test.mjs create mode 100644 examples/realtime-audio/capture-queue.mjs create mode 100644 examples/realtime-audio/capture-queue.test.mjs create mode 100644 examples/realtime-audio/client.mjs create mode 100644 examples/realtime-audio/index.html create mode 100644 examples/realtime-audio/launch.mjs create mode 100755 examples/realtime-audio/mcp-shell.mjs create mode 100644 examples/realtime-audio/server.mjs create mode 100644 examples/realtime-audio/server.test.mjs create mode 100644 examples/realtime-audio/style.css create mode 100644 examples/realtime-audio/ui.mjs create mode 100644 examples/realtime-audio/voice.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a663ef957c..7f538fafe5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,6 +88,8 @@ jobs: run: | scripts/test-postgres-test-discovery.sh scripts/test-postgres-test-wrapper.sh + - name: Realtime audio example + run: just realtime-audio-test - name: Release workflow source contract run: scripts/test-release-ref-contract.sh - name: Relay image eligibility contract diff --git a/Justfile b/Justfile index 00c4fecc2f..f44ca69511 100644 --- a/Justfile +++ b/Justfile @@ -350,7 +350,7 @@ desktop-e2e-pre-push: _ensure-migrations cd {{desktop_dir}} && pnpm build:e2e && pnpm exec playwright test --only-changed=origin/main # Run all checks suitable for CI / pre-push (no infra needed) -ci: check test-unit desktop-test desktop-build desktop-tauri-check desktop-tauri-test web-build mobile-test +ci: realtime-audio-test check test-unit desktop-test desktop-build desktop-tauri-check desktop-tauri-test web-build mobile-test # ─── Test ───────────────────────────────────────────────────────────────────── @@ -1167,3 +1167,7 @@ benchmark-check: # Stop the benchmark Docker stack (state and channels are kept) benchmark-down: docker compose --project-name buzz-benchmark down + +# Exercise the dependency-free local Realtime demo. +realtime-audio-test: + node --test examples/realtime-audio/*.test.mjs diff --git a/crates/buzz-agent/README.md b/crates/buzz-agent/README.md index fdd927a04d..30782cdbad 100644 --- a/crates/buzz-agent/README.md +++ b/crates/buzz-agent/README.md @@ -384,7 +384,7 @@ Test strategy is **real subprocess, no mocks**: - **Fake MCP server** — `tests/bin/fake_mcp.rs` is a separate binary controlled by env vars: `FAKE_MCP_HANG_INIT`, `FAKE_MCP_TOOL_DELAY`, `FAKE_MCP_SPAWN_GRANDCHILD`, etc. Each fault path is a real process being abused. - **Regression tests are the changelog.** Each `#[test]` in `regressions.rs` is named for the bug it locks down: `assistant_text_preserved_across_prompts`, `cancel_leaves_history_valid_for_next_prompt`, `mcp_init_timeout_kills_child`, `oversize_line_kills_connection`. Read them in order to learn the protocol's failure modes. -### Manual Realtime sessions +### Realtime sessions `OPENAI_COMPAT_API=realtime` selects the GA OpenAI Realtime WebSocket protocol. The same configured model, API key and base URL apply. HTTP API roots receive @@ -401,7 +401,45 @@ include a transcript and a complete WAV content block, not incremental playback. The server owns transient conversation state. Buzz retains ACP, MCP execution and permissions. Rejected mutations, cancellation, transport loss or expiry end the ACP Realtime session; create a new session rather than replaying possible -tool effects. Manual turns only: no VAD/barge-in/truncation, steer, model changes, -hooks or require-reply policies in this mode. Hosted OpenAI interoperability is +tool effects. Steer, model changes, hooks and require-reply policies are not +supported in this mode. Hosted OpenAI interoperability is not yet live-verified; local protocol fixtures and native inference are separate gates. + + +#### Opt-in live audio + +The [local voice demo](../../examples/realtime-audio/README.md) uses a versioned, +unstable ACP extension for continuous microphone capture and incremental playback. +Existing clients keep manual turns unless they opt in: + +1. Send `clientCapabilities._meta.buzz.realtimeAudio: 1` in `initialize` and + require `agentCapabilities._meta.buzz.realtimeAudio: 1` in its result. +2. Create a fresh session and send `session/prompt` with + `_meta.buzz.realtimeAudio: 1`. Its prompt may contain initial text context. + This request stays pending until the live conversation ends. +3. Wait for `_buzz/unstable/realtime/update` with `update.type: "ready"`. + Its parameters include `sessionId`, `streamId`, and the negotiated PCM limits. +4. Send requests to `_buzz/unstable/realtime/append` with both IDs, an increasing + `sequence` starting at zero, and base64 `data` containing mono PCM16 LE at + 24 kHz (at most 4,800 bytes / 100 ms). Optional `playbackData` contains the + aligned, actually rendered PCM for providers that advertise duplex input. +5. Play `audio` updates in order. Report actual device-clock positions through + `.../playback` with `playback: {responseId, itemId, contentIndex: 0, playedSamples}`. + A `clear` update requires stopping queued output and reporting the final + position with `stopped: true`. `.../interrupt` accepts the same final position + for client-initiated interruption. Positions cannot exceed emitted samples. +6. Send `.../close` to end the live prompt, or use ordinary `session/cancel`. + +Provider VAD commits user turns. Buzz coordinates response creation, cancellation, +playback truncation acknowledgment, and the existing tool executor and permission +broker. Capture and control queues are separate and bounded; stale stream IDs, +invalid positions, unsupported provider configuration, and stalled clients fail +visibly. Output token/audio exhaustion ends that response while capture remains +live. A fresh connection never replays tools from a previous one. + +`BUZZ_AGENT_THINKING_EFFORT=none|minimal|low|medium|high|xhigh|max` is forwarded in +`session.update.reasoning.effort` and must be acknowledged by the provider. +Frankie's optional duplex and backchannel extensions are used only when the +provider advertises them. The browser renders audio at the native 24 kHz rate and +has bounded buffering for capture stalls and playback. diff --git a/crates/buzz-agent/src/lib.rs b/crates/buzz-agent/src/lib.rs index 7f964a8c9c..ad070fefed 100644 --- a/crates/buzz-agent/src/lib.rs +++ b/crates/buzz-agent/src/lib.rs @@ -12,6 +12,7 @@ pub mod model_capabilities; mod permission; pub mod realtime; mod realtime_audio; +mod realtime_media; mod realtime_session; pub mod types; mod wire; @@ -38,7 +39,7 @@ pub const WINDOWS_SHELL_RESOLUTION_ENV: &[&str] = &[ use std::collections::HashMap; use std::path::Path; -use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use std::sync::Arc; use serde_json::{json, Value}; @@ -68,6 +69,7 @@ struct App { /// [`PROTOCOL_VERSION`] before `initialize`; no prompt (and thus no /// permission ask) can run before then. negotiated_version: AtomicU32, + realtime_audio: AtomicBool, /// Owns the entire `session/request_permission` correlation lifecycle: /// process-wide admission, id allocation, response delivery, and abort-safe /// cleanup. See [`permission::PermissionBroker`]. @@ -82,6 +84,7 @@ struct App { struct Session { realtime: realtime_session::RealtimeSession, + media: Option, id: String, mcp: Arc, /// Skills discovered at session creation; used by the built-in `load_skill` tool. @@ -221,6 +224,7 @@ async fn async_main() { llm, sessions: Mutex::new(HashMap::new()), negotiated_version: AtomicU32::new(PROTOCOL_VERSION), + realtime_audio: AtomicBool::new(false), permissions, models_cache: tokio::sync::OnceCell::new(), }); @@ -308,6 +312,10 @@ async fn handle_request( params: Value, wire_tx: &WireSender, ) { + if let Some(operation) = method.strip_prefix(realtime_media::PREFIX) { + realtime_media::dispatch(app, operation, id, params, wire_tx).await; + return; + } match method.as_str() { "initialize" => initialize(app, id, params, wire_tx).await, "session/new" => { @@ -362,6 +370,9 @@ async fn initialize(app: &Arc, id: Value, params: Value, wire_tx: &WireSend // RFD. Revisit when that RFD merges; otherwise a genuine upstream-v2 agent // would silently lose `[Base]`. let negotiated_version = p.protocol_version.min(PROTOCOL_VERSION); + let live_audio = app.cfg.openai_api == config::OpenAiApi::Realtime + && realtime_media::enabled(&p.client_capabilities["_meta"]); + app.realtime_audio.store(live_audio, Ordering::Relaxed); // Store the negotiated version for the connection lifetime: the // `session/request_permission` wire shape derives from this value, never // from a later mutable session field, so a strict client always receives @@ -376,6 +387,7 @@ async fn initialize(app: &Arc, id: Value, params: Value, wire_tx: &WireSend "protocolVersion": negotiated_version, "agentCapabilities": { "loadSession": false, + "_meta": {"buzz": {"realtimeAudio": if live_audio { 1 } else { 0 }}}, "promptCapabilities": { "image": app.cfg.openai_api == config::OpenAiApi::Realtime, "audio": app.cfg.openai_api == config::OpenAiApi::Realtime, "embeddedContext": false }, "mcpCapabilities": { "http": false, "sse": false }, }, @@ -561,6 +573,7 @@ async fn session_new(app: &Arc, id: Value, params: Value, wire_tx: &WireSen sessions.insert( session_id.clone(), Session { + media: None, id: session_id.clone(), mcp, skills, @@ -772,6 +785,16 @@ async fn run_prompt(app: Arc, id: Value, params: Value, wire_tx: WireSender Ok(p) => p, Err(m) => return reject(&wire_tx, id, INVALID_PARAMS, &m).await, }; + let live_audio = realtime_media::enabled(&p.meta); + if live_audio && !app.realtime_audio.load(Ordering::Relaxed) { + return reject( + &wire_tx, + id, + INVALID_PARAMS, + "realtime audio extension not negotiated", + ) + .await; + } let ( sid, mcp, @@ -799,6 +822,15 @@ async fn run_prompt(app: Arc, id: Value, params: Value, wire_tx: WireSender .await } }; + let media = if live_audio { + let (ingress, receiver) = realtime_media::channel(run_id.clone()); + if let Some(session) = app.sessions.lock().await.get_mut(&sid) { + session.media = Some(ingress); + } + Some(receiver) + } else { + None + }; // Advertise the active run id so steer-capable clients can target this turn // via `expectedRunId`. Mirrors goose's `send_active_run_update`. wire::send( @@ -864,7 +896,11 @@ async fn run_prompt(app: Arc, id: Value, params: Value, wire_tx: WireSender }; match realtime.as_mut() { Some(state) => { - let result = state.run(&mut ctx, p.prompt).await; + let result = if let Some(media) = media { + state.run_media(&mut ctx, p.prompt, media).await + } else { + state.run(&mut ctx, p.prompt).await + }; if let Some(s) = app.sessions.lock().await.get_mut(&sid) { if let Some(state) = realtime.take() { s.realtime = state; @@ -879,6 +915,7 @@ async fn run_prompt(app: Arc, id: Value, params: Value, wire_tx: WireSender }; if let Some(s) = app.sessions.lock().await.get_mut(&sid) { s.busy = false; + s.media = None; // Clear run state so a late steer can't queue into a finished turn. s.active_run_id = None; s.steer_tx = None; diff --git a/crates/buzz-agent/src/realtime.rs b/crates/buzz-agent/src/realtime.rs index a78bdd009c..bf44910bea 100644 --- a/crates/buzz-agent/src/realtime.rs +++ b/crates/buzz-agent/src/realtime.rs @@ -137,11 +137,27 @@ impl RealtimeSender { /// Append at most one second of 24 kHz mono signed little-endian PCM16. /// The caller must configure the matching audio/pcm session format first. pub async fn append_audio(&mut self, pcm: &[u8]) -> Result<(), AgentError> { + self.append_duplex_audio(pcm, None).await + } + + /// Optional system PCM must come from the same capture/render clock. + /// Callers may send it only after the provider advertises frankie.duplex_audio. + pub(crate) async fn append_duplex_audio( + &mut self, + pcm: &[u8], + playback: Option<&[u8]>, + ) -> Result<(), AgentError> { if pcm.is_empty() || pcm.len() > MAX_AUDIO_BYTES || !pcm.len().is_multiple_of(2) { return Err(error("invalid PCM16 chunk length")); } - self.send(json!({"type":"input_audio_buffer.append", "audio":STANDARD.encode(pcm)})) - .await + let mut event = json!({"type":"input_audio_buffer.append", "audio":STANDARD.encode(pcm)}); + if let Some(played) = playback { + if played.len() != pcm.len() { + return Err(error("duplex PCM lengths differ")); + } + event["playback_audio"] = Value::String(STANDARD.encode(played)); + } + self.send(event).await } /// Commit buffered input without implicitly requesting a response. @@ -230,7 +246,22 @@ impl RealtimeReceiver { .next() .await .ok_or_else(|| error("websocket closed"))? - .map_err(|_| error("websocket read failed"))?; + .map_err(|cause| { + // Classify transport failures without logging frame contents or credentials. + let detail = match cause { + tokio_tungstenite::tungstenite::Error::Protocol(problem) => { + problem.to_string() + } + tokio_tungstenite::tungstenite::Error::Io(problem) => { + format!("I/O {:?}", problem.kind()) + } + tokio_tungstenite::tungstenite::Error::Capacity(_) => { + "frame capacity exceeded".into() + } + _ => "transport error".into(), + }; + error(&format!("websocket read failed: {detail}")) + })?; match message { Message::Text(text) => { let event: Value = diff --git a/crates/buzz-agent/src/realtime_media.rs b/crates/buzz-agent/src/realtime_media.rs new file mode 100644 index 0000000000..50a21a6ff2 --- /dev/null +++ b/crates/buzz-agent/src/realtime_media.rs @@ -0,0 +1,199 @@ +//! Opt-in ACP media boundary. Capture and control use separate bounded queues. + +use base64::{engine::general_purpose::STANDARD, Engine}; +use serde::Deserialize; +use serde_json::{json, Value}; +use tokio::sync::mpsc; + +use crate::{wire, App}; + +pub(crate) const VERSION: u64 = 1; +pub(crate) const FRAME_BYTES: usize = 4800; +pub(crate) const QUEUE_FRAMES: usize = 8; +pub(crate) const PREFIX: &str = "_buzz/unstable/realtime/"; + +pub(crate) struct Command { + pub id: Value, + pub action: Action, +} + +pub(crate) enum Action { + Append { + sequence: u64, + pcm: Vec, + playback_pcm: Option>, + }, + Playback(Playback), + Interrupt(Option), + Close, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct Playback { + pub response_id: String, + pub item_id: String, + pub content_index: u32, + pub played_samples: u64, + #[serde(default)] + pub stopped: bool, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Params { + session_id: String, + stream_id: String, + #[serde(default)] + sequence: Option, + #[serde(default)] + data: Option, + #[serde(default)] + playback_data: Option, + #[serde(default)] + playback: Option, +} + +pub(crate) struct Ingress { + pub stream_id: String, + audio: mpsc::Sender, + control: mpsc::Sender, +} + +pub(crate) struct Receiver { + pub audio: mpsc::Receiver, + pub control: mpsc::Receiver, +} + +pub(crate) fn channel(stream_id: String) -> (Ingress, Receiver) { + let (audio, audio_rx) = mpsc::channel(QUEUE_FRAMES); + let (control, control_rx) = mpsc::channel(QUEUE_FRAMES); + ( + Ingress { + stream_id, + audio, + control, + }, + Receiver { + audio: audio_rx, + control: control_rx, + }, + ) +} + +pub(crate) fn enabled(meta: &Value) -> bool { + meta.get("buzz") + .and_then(|v| v.get("realtimeAudio")) + .and_then(Value::as_u64) + == Some(VERSION) +} + +fn action(operation: &str, params: &mut Params) -> Result { + if operation != "append" && params.playback_data.is_some() { + return Err("playback audio is only valid with capture"); + } + match operation { + "append" if params.playback.is_none() => { + let sequence = params.sequence.ok_or("missing sequence")?; + let data = params.data.take().ok_or("missing PCM data")?; + if data.len() > FRAME_BYTES.div_ceil(3) * 4 { + return Err("PCM frame exceeds 100 ms"); + } + let pcm = STANDARD.decode(data).map_err(|_| "invalid PCM base64")?; + if pcm.is_empty() || pcm.len() > FRAME_BYTES || !pcm.len().is_multiple_of(2) { + return Err("invalid PCM16 frame length"); + } + let playback_pcm = params + .playback_data + .take() + .map(|data| { + if data.len() > FRAME_BYTES.div_ceil(3) * 4 { + return Err("playback frame exceeds 100 ms"); + } + let samples = STANDARD + .decode(data) + .map_err(|_| "invalid playback PCM base64")?; + if samples.len() != pcm.len() { + return Err("duplex capture lengths differ"); + } + Ok(samples) + }) + .transpose()?; + Ok(Action::Append { + sequence, + pcm, + playback_pcm, + }) + } + "playback" if params.sequence.is_none() && params.data.is_none() => Ok(Action::Playback( + params.playback.take().ok_or("missing playback position")?, + )), + "interrupt" if params.sequence.is_none() && params.data.is_none() => { + Ok(Action::Interrupt(params.playback.take())) + } + "close" + if params.sequence.is_none() && params.data.is_none() && params.playback.is_none() => + { + Ok(Action::Close) + } + _ => Err("unsupported media operation or fields"), + } +} + +pub(crate) async fn dispatch( + app: &App, + operation: &str, + id: Value, + value: Value, + output: &wire::WireSender, +) { + let result = async { + let mut params: Params = + serde_json::from_value(value).map_err(|_| "invalid media params")?; + let action = action(operation, &mut params)?; + let sessions = app.sessions.lock().await; + let ingress = sessions + .get(¶ms.session_id) + .and_then(|s| s.media.as_ref()) + .ok_or("no active media session")?; + if ingress.stream_id != params.stream_id { + return Err("stale media stream"); + } + let queue = if matches!(action, Action::Append { .. }) { + &ingress.audio + } else { + &ingress.control + }; + queue + .try_send(Command { + id: id.clone(), + action, + }) + .map_err(|_| "media queue full or closed") + } + .await; + if let Err(message) = result { + wire::send(output, wire::err(id, wire::INVALID_PARAMS, message)).await; + } +} + +pub(crate) async fn notify( + output: &wire::WireSender, + session_id: &str, + stream_id: &str, + update: Value, +) -> Result<(), crate::AgentError> { + tokio::time::timeout( + std::time::Duration::from_secs(2), + wire::send_checked( + output, + json!({ + "jsonrpc":"2.0", "method":"_buzz/unstable/realtime/update", + "params":{"sessionId":session_id,"streamId":stream_id,"update":update} + }), + ), + ) + .await + .map_err(|_| crate::AgentError::Llm("realtime: media consumer stalled".into()))? + .map_err(|_| crate::AgentError::Cancelled) +} diff --git a/crates/buzz-agent/src/realtime_session.rs b/crates/buzz-agent/src/realtime_session.rs index 943fd6b5f6..5ceca6ffc6 100644 --- a/crates/buzz-agent/src/realtime_session.rs +++ b/crates/buzz-agent/src/realtime_session.rs @@ -1,6 +1,8 @@ //! Manual-turn ACP driver for a persistent, server-owned Realtime conversation. //! Transport loss is terminal: never replay possibly committed tool effects. +mod media; + use std::{collections::HashSet, time::Duration}; use crate::realtime_audio::{decode_wav, encode_wav, MAX_PCM}; @@ -47,6 +49,9 @@ struct Live { expires: Instant, ids: HashSet, active_response: Option, + duplex_audio: bool, + backchannels: bool, + output_audio_limit: u64, } impl Drop for Live { @@ -155,15 +160,12 @@ impl RealtimeSession { } } - async fn run_inner( + async fn connect( &mut self, ctx: &mut RunCtx<'_>, - inputs: Vec, audio_output: bool, - ) -> Result { - if *ctx.cancel.borrow() { - return Err(AgentError::Cancelled); - } + media: bool, + ) -> Result<(), AgentError> { if self.live.is_none() { let endpoint = endpoint(&ctx.cfg.base_url, ctx.effective_model)?; let connection = tokio::select! { @@ -200,6 +202,9 @@ impl RealtimeSession { expires, ids: HashSet::new(), active_response: None, + duplex_audio: false, + backchannels: false, + output_audio_limit: MAX_PCM as u64 / 2, }); let live = self.live.as_mut().ok_or_else(|| error("missing session"))?; let mut tools = ctx.mcp.tools(); @@ -215,29 +220,67 @@ impl RealtimeSession { }) }) .collect(); - live.sender - .update_session(json!({ - "type":"realtime", "model":ctx.effective_model, - "instructions":ctx.system_prompt, "tools":tools, - "output_modalities":[if audio_output { "audio" } else { "text" }], - "audio":{ - "input":{"turn_detection":null,"format":{"type":"audio/pcm","rate":24000}}, - "output":{"format":{"type":"audio/pcm","rate":24000}}, - }, - })) - .await?; + let mut session = json!({ + "type":"realtime", "model":ctx.effective_model, + "instructions":ctx.system_prompt, "tools":tools, + "output_modalities":[if audio_output { "audio" } else { "text" }], + "audio":{ + "input":{"turn_detection":if media { json!({"type":"server_vad","create_response":false,"interrupt_response":false}) } else { Value::Null },"format":{"type":"audio/pcm","rate":24000}}, + "output":{"format":{"type":"audio/pcm","rate":24000}}, + }, + }); + if let Some(effort) = ctx.cfg.thinking_effort { + session["reasoning"] = json!({"effort": effort.openai_effort_str()}); + } + live.sender.update_session(session).await?; let deadline = Instant::now() + ctx.cfg.llm_timeout; let updated = live.next(ctx, deadline).await?; + live.duplex_audio = updated["session"]["frankie"]["duplex_audio"] == true; + live.backchannels = updated["session"]["frankie"]["backchannels"] == true; + if let Some(limit) = updated["session"]["frankie"].get("max_output_audio_samples") { + let limit = limit + .as_u64() + .filter(|n| (1..=24000 * 3600).contains(n)) + .ok_or_else(|| error("invalid provider live audio budget"))?; + live.output_audio_limit = limit; + } if updated["type"] != "session.updated" + || ctx.cfg.thinking_effort.is_some_and(|effort| { + updated["session"]["reasoning"]["effort"] != effort.openai_effort_str() + }) || updated["session"]["output_modalities"] != json!([if audio_output { "audio" } else { "text" }]) - || updated["session"]["audio"]["input"] - .get("turn_detection") - .is_some_and(|value| !value.is_null()) + || if media { + let detection = &updated["session"]["audio"]["input"]["turn_detection"]; + detection["type"] != "server_vad" + || detection["create_response"] != false + || detection["interrupt_response"] != false + } else { + updated["session"]["audio"]["input"] + .get("turn_detection") + .is_some_and(|value| !value.is_null()) + } { - return Err(error("provider did not acknowledge manual session")); + return Err(error(if media { + "provider did not acknowledge live media session" + } else { + "provider did not acknowledge manual session" + })); } } + Ok(()) + } + + async fn run_inner( + &mut self, + ctx: &mut RunCtx<'_>, + inputs: Vec, + audio_output: bool, + ) -> Result { + if *ctx.cancel.borrow() { + return Err(AgentError::Cancelled); + } + self.connect(ctx, audio_output, false).await?; let live = self.live.as_mut().ok_or_else(|| error("missing session"))?; if Instant::now() >= live.expires || live.reader.is_finished() { return Err(error("session disconnected or expired")); @@ -498,9 +541,7 @@ impl Live { if event["type"] == "error" { // This manual driver cannot reconcile rejected mutations yet. // Keep that policy here, not in the reusable transport. - let message = event["error"]["message"].as_str().unwrap_or("unspecified provider error"); - let bounded: String = message.chars().filter(|c| !c.is_control()).take(512).collect(); - return Err(error(&format!("provider rejected request: {bounded}"))); + return Err(provider_error("provider rejected request", &event["error"])); } Ok(event) } @@ -508,6 +549,18 @@ impl Live { } } +fn provider_error(context: &str, detail: &Value) -> AgentError { + let message = detail["message"] + .as_str() + .unwrap_or("unspecified provider error"); + let bounded: String = message + .chars() + .filter(|c| !c.is_control()) + .take(512) + .collect(); + error(&format!("{context}: {bounded}")) +} + fn field<'a>(value: &'a Value, key: &str) -> Result<&'a str, AgentError> { value[key] .as_str() diff --git a/crates/buzz-agent/src/realtime_session/media.rs b/crates/buzz-agent/src/realtime_session/media.rs new file mode 100644 index 0000000000..22c02b82ea --- /dev/null +++ b/crates/buzz-agent/src/realtime_session/media.rs @@ -0,0 +1,683 @@ +//! Persistent full-duplex interaction. Provider VAD commits input; this client +//! schedules responses only after playback/history and tool fences settle. +use super::*; +use crate::realtime_media::{self as media, Action, Command, Playback}; + +struct Output { + response: String, + item: String, + emitted: u64, + played: u64, + sequence: u64, +} + +struct State { + output: Option, + aside: Option, + completed_playback: Option, + input_sequence: u64, + interrupted: bool, + awaiting_playback: bool, + awaiting_truncate: bool, + pending_input: bool, + response_requested: bool, + fence_deadline: Option, + response_deadline: Option, +} + +impl State { + fn new() -> Self { + Self { + output: None, + aside: None, + completed_playback: None, + input_sequence: 0, + interrupted: false, + awaiting_playback: false, + awaiting_truncate: false, + pending_input: false, + response_requested: false, + fence_deadline: None, + response_deadline: None, + } + } + + fn playback(&mut self, p: &Playback) -> Result { + // A device clock can repeat the final position after scheduling retired its item. + if self.completed_playback.as_ref().is_some_and(|o| { + o.response == p.response_id + && o.item == p.item_id + && p.content_index == 0 + && p.played_samples == o.emitted + && o.played == o.emitted + }) { + return Ok(false); + } + let output = self + .output + .as_mut() + .ok_or_else(|| error("no playback item"))?; + if output.response != p.response_id + || output.item != p.item_id + || p.content_index != 0 + || p.played_samples < output.played + || p.played_samples > output.emitted + { + return Err(error("stale or invalid playback position")); + } + if self.interrupted && !self.awaiting_playback { + return Err(error("playback already stopped")); + } + output.played = p.played_samples; + Ok(true) + } + + async fn interrupt( + &mut self, + live: &mut Live, + output: &wire::WireSender, + sid: &str, + stream: &str, + ) -> Result<(), AgentError> { + if self.interrupted { + return Ok(()); + } + self.interrupted = true; + self.fence_deadline = Some(Instant::now() + Duration::from_secs(5)); + if let Some(id) = live.active_response.as_ref() { + live.sender.cancel_response(id).await?; + } + self.awaiting_playback = self.output.is_some(); + media::notify( + output, + sid, + stream, + json!({"type":"clear", "responseId":live.active_response, + "itemId":self.output.as_ref().map(|o| &o.item)}), + ) + .await + } + + async fn truncate(&mut self, live: &mut Live) -> Result<(), AgentError> { + if let Some(output) = self.output.as_ref() { + live.sender + .truncate_audio(&output.item, 0, (output.played / 24) as u32) + .await?; + self.awaiting_truncate = true; + self.fence_deadline = Some(Instant::now() + Duration::from_secs(5)); + } + self.awaiting_playback = false; + Ok(()) + } + + fn acknowledge_truncate(&mut self, event: &Value) -> Result<(), AgentError> { + if !self.awaiting_truncate + || self.output.as_ref().map(|o| o.item.as_str()) != event["item_id"].as_str() + || event["content_index"] != 0 + || self.output.as_ref().map(|o| o.played / 24) != event["audio_end_ms"].as_u64() + { + return Err(error("uncorrelated truncation")); + } + self.awaiting_truncate = false; + Ok(()) + } + + async fn schedule(&mut self, live: &mut Live, ctx: &RunCtx<'_>) -> Result<(), AgentError> { + if self.awaiting_playback + || self.awaiting_truncate + || live.active_response.is_some() + || self.response_requested + { + return Ok(()); + } + if self.interrupted { + self.output = None; + self.interrupted = false; + self.fence_deadline = None; + } + if self.pending_input { + // Never overwrite accounting for an item that is still playing. + if self.output.as_ref().is_some_and(|o| o.played < o.emitted) { + return Ok(()); + } + if let Some(output) = self.output.take() { + self.completed_playback = Some(output); + } + self.pending_input = false; + self.response_requested = true; + self.response_deadline = Some(Instant::now() + ctx.cfg.llm_timeout); + live.sender.create_response().await?; + } + Ok(()) + } +} + +impl RealtimeSession { + pub(crate) async fn run_media( + &mut self, + ctx: &mut RunCtx<'_>, + prompt: Vec, + mut ingress: media::Receiver, + ) -> Result { + // Mode changes cannot silently discard an existing provider conversation. + if self.terminal || self.live.is_some() { + return Err(error("live media requires a fresh ACP session")); + } + if prompt.len() > 16 { + return Err(error("initial media context limit")); + } + let mut parts = Vec::new(); + let mut bytes = 0; + for block in prompt { + match block { + ContentBlock::Text { text } => { + bytes += text.len(); + parts.push(json!({"type":"input_text","text":text})); + } + _ => return Err(error("live media initial context supports text only")), + } + } + if bytes > 16384 { + return Err(error("initial media context limit")); + } + let result = async { + self.connect(ctx, true, true).await?; + let live = self.live.as_mut().ok_or_else(|| error("missing session"))?; + if !parts.is_empty() { + live.sender.create_item(json!({"type":"message","role":"user","content":parts})).await?; + } + media::notify(ctx.wire, ctx.session_id, &ctx.run_id, json!({"type":"ready","format":"s16le","rate":24000,"channels":1, + "maxFrameBytes":media::FRAME_BYTES,"queueFrames":media::QUEUE_FRAMES})).await?; + let mut state = State::new(); + let mut heartbeat = tokio::time::interval(Duration::from_secs(15)); + heartbeat.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + loop { + let deadline = state.fence_deadline.into_iter().chain(state.response_deadline).fold(live.expires, Instant::min); + let command = tokio::select! { + biased; + _ = ctx.cancel.changed() => return Err(AgentError::Cancelled), + _ = tokio::time::sleep_until(deadline) => return Err(error("media session or response deadline exceeded")), + command = ingress.control.recv() => Some(command.ok_or_else(|| error("media controls closed"))?), + _ = heartbeat.tick() => { + media::notify(ctx.wire, ctx.session_id, &ctx.run_id, json!({"type":"heartbeat"})).await?; + None + }, + event = live.events.recv() => { + let event = event.ok_or_else(|| error("session disconnected"))??; + if let Some(calls) = state.event(live, ctx, event).await? { + state.tools(live, ctx, &mut ingress, &calls).await?; + } + None + }, + command = ingress.audio.recv() => Some(command.ok_or_else(|| error("media capture closed"))?), + }; + if let Some(command) = command { + if state.command(live, ctx.wire, ctx.session_id, &ctx.run_id, command).await? { return Ok(StopReason::EndTurn); } + } + state.schedule(live, ctx).await?; + } + }.await; + self.terminal = true; + if let Some(live) = self.live.as_mut() { + if let Some(id) = live.active_response.take() { + let _ = live.sender.cancel_response(&id).await; + } + } + self.live = None; + let _ = media::notify( + ctx.wire, + ctx.session_id, + &ctx.run_id, + json!({"type":"closed","failed":result.is_err()}), + ) + .await; + match result { + Err(AgentError::Cancelled) => Ok(StopReason::Cancelled), + other => other, + } + } +} + +impl State { + async fn command( + &mut self, + live: &mut Live, + output: &wire::WireSender, + sid: &str, + stream: &str, + command: Command, + ) -> Result { + let close = matches!(command.action, Action::Close); + let result: Result<(), AgentError> = async { + match command.action { + Action::Append { + sequence, + pcm, + playback_pcm, + } => { + if sequence != self.input_sequence { + Err(error("out-of-order capture frame")) + } else { + let playback = if live.duplex_audio { + playback_pcm.as_deref() + } else { + None + }; + live.sender.append_duplex_audio(&pcm, playback).await?; + self.input_sequence = self + .input_sequence + .checked_add(1) + .ok_or_else(|| error("input sequence exhausted"))?; + Ok(()) + } + } + Action::Playback(position) => { + let current = self.playback(&position)?; + if current && position.stopped && self.awaiting_playback { + self.truncate(live).await?; + } + Ok(()) + } + Action::Interrupt(position) => { + let current = if let Some(position) = position.as_ref() { + self.playback(position)? + } else { + false + }; + self.interrupt(live, output, sid, stream).await?; + if current { + self.truncate(live).await?; + } + Ok(()) + } + Action::Close => Ok(()), + } + } + .await; + let reply = match result { + Ok(()) => wire::ok(command.id, Value::Null), + Err(e) => wire::err(command.id, wire::INVALID_PARAMS, &e.to_string()), + }; + tokio::time::timeout(Duration::from_secs(2), wire::send_checked(output, reply)) + .await + .map_err(|_| error("media consumer stalled"))? + .map_err(|_| AgentError::Cancelled)?; + Ok(close) + } + + async fn event( + &mut self, + live: &mut Live, + ctx: &RunCtx<'_>, + event: Value, + ) -> Result>, AgentError> { + match event["type"].as_str() { + Some("error") => { + return Err(provider_error( + "provider rejected media operation", + &event["error"], + )) + } + Some("input_audio_buffer.speech_started") => { + // Also interrupt completed generation whose audio has not finished playing. + if live.active_response.is_some() + || self.response_requested + || self.output.as_ref().is_some_and(|o| o.played < o.emitted) + { + self.interrupt(live, ctx.wire, ctx.session_id, &ctx.run_id) + .await?; + } + media::notify( + ctx.wire, + ctx.session_id, + &ctx.run_id, + json!({"type":"speech_started"}), + ) + .await?; + } + Some("conversation.item.deleted") => { + media::notify( + ctx.wire, + ctx.session_id, + &ctx.run_id, + json!({"type":"input_removed", "itemId":field(&event,"item_id")?}), + ) + .await?; + } + Some("conversation.item.input_audio_transcription.completed") => { + let text = field(&event, "transcript")?; + if text.len() > 16384 { + return Err(error("input transcript limit")); + } + media::notify( + ctx.wire, + ctx.session_id, + &ctx.run_id, + json!({ + "type":"input_transcript", "itemId":field(&event,"item_id")?, "text":text + }), + ) + .await?; + } + Some("input_audio_buffer.speech_stopped") => { + media::notify( + ctx.wire, + ctx.session_id, + &ctx.run_id, + json!({"type":"speech_stopped", "lastSpeechMs":event["frankie_last_speech_ms"]}), + ) + .await?; + } + Some("input_audio_buffer.cleared") => { + media::notify( + ctx.wire, + ctx.session_id, + &ctx.run_id, + json!({"type":"input_cleared"}), + ) + .await?; + } + Some("input_audio_buffer.committed") => { + self.pending_input = true; + } + Some("conversation.item.truncated") => { + self.acknowledge_truncate(&event)?; + } + Some("frankie.backchannel.delta") => { + if !live.backchannels { + return Err(error("unnegotiated backchannel audio")); + } + let id = field(&event, "id")?; + if self.aside.is_none() { + live.remember(id)?; + self.aside = Some(Output { + response: id.into(), + item: id.into(), + emitted: 0, + played: 0, + sequence: 0, + }); + } + let aside = self + .aside + .as_mut() + .ok_or_else(|| error("missing backchannel"))?; + if aside.item != id || event["sequence"].as_u64() != Some(aside.sequence) { + return Err(error("uncorrelated backchannel audio")); + } + let delta = field(&event, "delta")?; + if delta.len() > 12000 { + return Err(error("backchannel frame too large")); + } + let pcm = STANDARD + .decode(delta) + .map_err(|_| error("invalid backchannel PCM"))?; + if pcm.is_empty() + || !pcm.len().is_multiple_of(2) + || aside.emitted + pcm.len() as u64 / 2 > 30720 + { + return Err(error("backchannel audio limit")); + } + media::notify(ctx.wire, ctx.session_id, &ctx.run_id, + json!({"type":"backchannel_audio","id":id,"startSample":aside.emitted,"data":delta})).await?; + aside.emitted += pcm.len() as u64 / 2; + aside.sequence += 1; + } + Some("frankie.backchannel.done") => { + if !live.backchannels { + return Err(error("unnegotiated backchannel completion")); + } + let id = field(&event, "id")?; + if let Some(aside) = self.aside.take() { + if aside.item != id || event["samples"].as_u64() != Some(aside.emitted) { + return Err(error("backchannel completion mismatch")); + } + } else if event["samples"].as_u64() == Some(0) { + live.remember(id)?; + } else { + return Err(error("missing backchannel audio")); + } + media::notify( + ctx.wire, + ctx.session_id, + &ctx.run_id, + json!({"type":"backchannel_done","id":id}), + ) + .await?; + } + Some("response.created") => { + if live.active_response.is_some() || !self.response_requested { + return Err(error("unsolicited media response")); + } + let id = field(&event["response"], "id")?; + live.remember(id)?; + live.active_response = Some(id.to_owned()); + self.response_requested = false; + if self.interrupted { + live.sender.cancel_response(id).await?; + } + } + Some("response.output_audio.delta") => { + let response = field(&event, "response_id")?; + if live.active_response.as_deref() != Some(response) || event["content_index"] != 0 + { + return Err(error("uncorrelated media audio")); + } + if self.interrupted { + return Ok(None); + } + let item = field(&event, "item_id")?; + let output = self.output.get_or_insert_with(|| Output { + response: response.to_owned(), + item: item.to_owned(), + emitted: 0, + played: 0, + sequence: 0, + }); + if output.item != item || output.response != response { + return Err(error("multiple media audio items")); + } + let delta = field(&event, "delta")?; + let pcm = STANDARD + .decode(delta) + .map_err(|_| error("invalid audio base64"))?; + if pcm.is_empty() + || !pcm.len().is_multiple_of(2) + || output.emitted + pcm.len() as u64 / 2 > live.output_audio_limit + { + return Err(error("media output limit")); + } + media::notify( + ctx.wire, + ctx.session_id, + &ctx.run_id, + json!({"type":"audio","responseId":response,"itemId":item,"contentIndex":0, + "sequence":output.sequence,"startSample":output.emitted,"data":delta}), + ) + .await?; + output.emitted += pcm.len() as u64 / 2; + output.sequence += 1; + } + Some("response.done") => { + let response = &event["response"]; + if live.active_response.as_deref() != Some(field(response, "id")?) { + return Err(error("uncorrelated media response.done")); + } + live.active_response = None; + self.response_deadline = None; + // Output budgets end a response, not the live microphone session. + // Never execute even apparently complete calls from a truncated response. + let limited = response["status"] == "incomplete" + && matches!( + response["status_details"]["reason"].as_str(), + Some("max_output_tokens" | "max_output_audio") + ); + if response["status"] != "completed" + && !limited + && !(self.interrupted && response["status"] == "cancelled") + { + return Err(provider_error( + "media response did not complete", + &response["status_details"]["error"], + )); + } + let mut calls = Vec::new(); + let items = response["output"] + .as_array() + .ok_or_else(|| error("missing response output"))?; + if items.len() > MAX_CALLS { + return Err(error("too many output items")); + } + if limited { + media::notify(ctx.wire, ctx.session_id, &ctx.run_id, json!({"type":"response_limited","responseId":response["id"],"reason":response["status_details"]["reason"]})).await?; + } + for item in items.iter().filter(|_| !limited) { + live.remember(field(item, "id")?)?; + if item["type"] == "function_call" { + if item["status"] != "completed" { + return Err(error("incomplete tool call")); + } + let id = field(item, "call_id")?; + live.remember(id)?; + calls.push(ToolCall { + provider_id: id.to_owned(), + name: field(item, "name")?.to_owned(), + arguments: serde_json::from_str(field(item, "arguments")?) + .map_err(|_| error("invalid function arguments"))?, + provider_extra: Default::default(), + }); + } else if item["type"] != "message" || item["role"] != "assistant" { + return Err(error("unsupported media output")); + } else if response["status"] == "completed" { + // Completed provider text is display-only; playback positions remain + // sample-clock reports and never derive from this transcript. + if let Some(content) = item["content"].as_array() { + for part in content { + let text = match part["type"].as_str() { + Some("output_audio") => part["transcript"].as_str(), + Some("output_text") => part["text"].as_str(), + _ => None, + }; + if let Some(text) = text.filter(|text| !text.is_empty()) { + wire::send( + ctx.wire, + wire::session_update( + ctx.session_id, + json!({ + "sessionUpdate":"agent_message_chunk", + "content":{"type":"text","text":text} + }), + ), + ) + .await; + } + } + } + } + } + media::notify(ctx.wire, ctx.session_id, &ctx.run_id, json!({"type":"response_done","responseId":response["id"],"status":response["status"]})).await?; + if !calls.is_empty() { + return Ok(Some(calls)); + } + } + _ => {} + } + Ok(None) + } + + async fn tools( + &mut self, + live: &mut Live, + ctx: &mut RunCtx<'_>, + ingress: &mut media::Receiver, + calls: &[ToolCall], + ) -> Result<(), AgentError> { + // The existing executor must finish its cancellation cleanup before this returns. + let mut original = ctx.cancel.clone(); + let (cancel, local) = watch::channel(self.interrupted); + let saved = std::mem::replace(ctx.cancel, local); + let start = ctx.history.len(); + let result = { + let output = ctx.wire.clone(); + let sid = ctx.session_id.to_owned(); + let stream = ctx.run_id.clone(); + let execute = ctx.execute_calls(calls); + tokio::pin!(execute); + let mut heartbeat = tokio::time::interval(Duration::from_secs(15)); + heartbeat.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + loop { + tokio::select! { + biased; + _ = original.changed() => { let _ = cancel.send(true); execute.await; break Err(AgentError::Cancelled); } + _ = tokio::time::sleep_until(self.fence_deadline.map_or(live.expires, |d| d.min(live.expires))) => { let _ = cancel.send(true); execute.await; break Err(error("session or playback fence expired during tools")); } + _ = heartbeat.tick() => { + if let Err(e) = media::notify(&output, &sid, &stream, json!({"type":"heartbeat"})).await { + let _ = cancel.send(true); execute.await; break Err(e); + } + } + command = ingress.control.recv() => { + let Some(command) = command else { let _ = cancel.send(true); execute.await; break Err(AgentError::Cancelled); }; + if matches!(command.action, Action::Close | Action::Interrupt(_)) { + let _ = cancel.send(true); + } + match self.command(live, &output, &sid, &stream, command).await { + Ok(true) => { execute.await; break Err(AgentError::Cancelled); } + Ok(false) => {}, + Err(e) => { let _ = cancel.send(true); execute.await; break Err(e); } + } + } + command = ingress.audio.recv() => { + let Some(command) = command else { let _ = cancel.send(true); execute.await; break Err(AgentError::Cancelled); }; + if let Err(e) = self.command(live, &output, &sid, &stream, command).await { + let _ = cancel.send(true); execute.await; break Err(e); + } + } + event = live.events.recv() => { + match event { + Some(Ok(event)) => match event["type"].as_str() { + Some("input_audio_buffer.speech_started") => { + let _ = cancel.send(true); + if let Err(e) = self.interrupt(live, &output, &sid, &stream).await { + execute.await; break Err(e); + } + if let Err(e) = media::notify(&output, &sid, &stream, json!({"type":"speech_started"})).await { + execute.await; break Err(e); + } + } + Some("conversation.item.truncated") => { + if let Err(e) = self.acknowledge_truncate(&event) { + let _ = cancel.send(true); execute.await; break Err(e); + } + } + Some("input_audio_buffer.speech_stopped") => { + if let Err(e) = media::notify(&output, &sid, &stream, json!({"type":"speech_stopped", "lastSpeechMs":event["frankie_last_speech_ms"]})).await { + let _ = cancel.send(true); execute.await; break Err(e); + } + } + Some("input_audio_buffer.committed") => self.pending_input = true, + Some("error") => { let _ = cancel.send(true); execute.await; break Err(provider_error("provider rejected media operation", &event["error"])); } + Some("rate_limits.updated" | "conversation.item.created" | "conversation.item.added" | "conversation.item.done") => {}, + _ => { let _ = cancel.send(true); execute.await; break Err(error("unexpected provider event during tools")); } + }, + _ => { let _ = cancel.send(true); execute.await; break Err(error("provider disconnected during tools")); } + } + } + _ = &mut execute => break Ok(()), + } + } + }; + *ctx.cancel = saved; + result?; + let results: Vec<_> = ctx.history.drain(start..).collect(); + for result in results { + let HistoryItem::ToolResult(result) = result else { + return Err(error("invalid executor result")); + }; + live.sender.create_item(json!({"type":"function_call_output","call_id":result.provider_id,"output":result.text()})).await?; + } + if !self.interrupted { + self.pending_input = true; + } + Ok(()) + } +} diff --git a/crates/buzz-agent/src/wire.rs b/crates/buzz-agent/src/wire.rs index e6fe89e07c..d994097842 100644 --- a/crates/buzz-agent/src/wire.rs +++ b/crates/buzz-agent/src/wire.rs @@ -52,7 +52,7 @@ pub struct InitializeParams { #[serde(rename = "protocolVersion")] pub protocol_version: u32, #[serde(default, rename = "clientCapabilities")] - pub _client_capabilities: Value, + pub client_capabilities: Value, } #[derive(Debug, Deserialize)] @@ -70,6 +70,8 @@ pub struct SessionNewParams { pub struct SessionPromptParams { pub session_id: String, pub prompt: Vec, + #[serde(default, rename = "_meta")] + pub meta: Value, } #[derive(Debug, Deserialize)] diff --git a/crates/buzz-agent/tests/fake_llm.rs b/crates/buzz-agent/tests/fake_llm.rs index 9822243d5f..cd7faf759a 100644 --- a/crates/buzz-agent/tests/fake_llm.rs +++ b/crates/buzz-agent/tests/fake_llm.rs @@ -1320,7 +1320,7 @@ async fn mid_turn_usage_includes_earlier_turns() { /// Setup: round 1 is a tool call WITH usage (tokens are captured). After the /// tool_call_update notification (proving round 1 is fully processed), we gate /// the round-2 LLM response behind a `oneshot` barrier that only releases after -/// cancel is sent. This guarantees the turn exits with `stopReason: "cancelled"` +/// cancel is acknowledged. This guarantees the turn exits with `stopReason: "cancelled"` /// deterministically, even on a slow CI worker. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn cancelled_turn_with_usage_emits_notification_before_response() { @@ -1334,8 +1334,8 @@ async fn cancelled_turn_with_usage_emits_notification_before_response() { // Round 2: gated — blocked until cancel fires, then released so the // in-flight TCP request can resolve. The queue is empty for round 2, so the // agent receives the fallback "no canned response" body which it treats as - // an LLM error; the cancel check at the round boundary fires first because - // the gate is only released after cancel is enqueued. + // an LLM error. Release only after the agent acknowledges cancellation: + // writing cancel to stdin does not prove the child has processed it. let responses = vec![openai_tool_call_with_usage( "call_cancel_test", "fake__noop", @@ -1371,8 +1371,7 @@ async fn cancelled_turn_with_usage_emits_notification_before_response() { } } // For request 2+ (round 2), wait for the gate to open before - // responding. This ensures cancel is sent before round 2 resolves, - // making stopReason: cancelled deterministic. + // responding. The client opens the gate only after cancel is acknowledged. if req_num >= 2 { let rx = gate.lock().await.take(); if let Some(rx) = rx { @@ -1415,10 +1414,9 @@ async fn cancelled_turn_with_usage_emits_notification_before_response() { }) .await; - // Now send cancel and release the round-2 gate. Cancel is enqueued before - // round 2 can respond, so the turn exits with stopReason: cancelled. + // Keep reading usage and prompt events while awaiting cancellation acknowledgement. let c_id = h.send("session/cancel", json!({"sessionId": sid})).await; - let _ = gate_tx.send(()); // unblock round 2 + let mut gate_tx = Some(gate_tx); let mut saw_usage_before_prompt_response = false; let mut saw_usage = false; @@ -1427,6 +1425,10 @@ async fn cancelled_turn_with_usage_emits_notification_before_response() { for _ in 0..40 { let v = h.recv().await; if v["id"] == json!(c_id) { + assert!(v.get("error").is_none(), "cancel failed: {v}"); + if let Some(gate) = gate_tx.take() { + let _ = gate.send(()); + } saw_cancel_ok = true; } else if is_usage_update(&v) { saw_usage = true; diff --git a/crates/buzz-agent/tests/realtime_acp.rs b/crates/buzz-agent/tests/realtime_acp.rs index c26366a4a5..3e3a440bba 100644 --- a/crates/buzz-agent/tests/realtime_acp.rs +++ b/crates/buzz-agent/tests/realtime_acp.rs @@ -113,6 +113,57 @@ fn options() -> Vec<(&'static str, &'static str)> { ] } +#[tokio::test] +async fn realtime_thinking_effort_is_sent_and_must_be_acknowledged() { + for acknowledged in [true, false] { + let dir = tempfile::tempdir().unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("ws://{}/v1/realtime", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + let (socket, _) = listener.accept().await.unwrap(); + let mut ws = accept_async(socket).await.unwrap(); + send( + &mut ws, + json!({"type":"session.created","session":{"id":"s1"}}), + ) + .await; + let update = recv(&mut ws).await; + assert_eq!(update["session"]["reasoning"]["effort"], "low"); + let mut session = update["session"].clone(); + if !acknowledged { + session.as_object_mut().unwrap().remove("reasoning"); + } + send(&mut ws, json!({"type":"session.updated","session":session})).await; + if acknowledged { + modality_ack(&mut ws).await; + assert_eq!(recv(&mut ws).await["type"], "conversation.item.create"); + assert_eq!(recv(&mut ws).await["type"], "response.create"); + text(&mut ws, "r1").await; + } + tokio::time::sleep(Duration::from_millis(200)).await; + }); + let mut env = options(); + env.push(("BUZZ_AGENT_THINKING_EFFORT", "low")); + let mut harness = Harness::spawn_with_env(&url, &env).await; + let sid = init(&mut harness, dir.path()).await; + let prompt_id = prompt(&mut harness, &sid).await; + let result = harness.recv_until(|v| v["id"] == prompt_id).await; + if acknowledged { + assert_eq!(result["result"]["stopReason"], "end_turn", "{result}"); + } else { + assert!( + result["error"]["message"] + .as_str() + .unwrap() + .contains("acknowledge"), + "{result}" + ); + } + assert!(!dir.path().join("calls.log").exists()); + server.await.unwrap(); + } +} + #[tokio::test] async fn realtime_acp_permission_and_persistent_conversation() { for allowed in [true, false] { @@ -541,3 +592,673 @@ async fn realtime_invalid_image_fails_before_connect_and_keeps_session_usable() provider.await.unwrap(); h.shutdown().await; } + +#[tokio::test] +async fn realtime_acp_live_duplex_and_playback_fence() { + let dir = tempfile::tempdir().unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("ws://{}/v1/realtime", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + let (socket, _) = listener.accept().await.unwrap(); + let mut ws = accept_async(socket).await.unwrap(); + send( + &mut ws, + json!({"type":"session.created","session":{"id":"persistent"}}), + ) + .await; + let update = recv(&mut ws).await; + assert_eq!( + update["session"]["audio"]["input"]["turn_detection"], + json!({"type":"server_vad","create_response":false,"interrupt_response":false}) + ); + send( + &mut ws, + json!({"type":"session.updated","session":update["session"]}), + ) + .await; + assert_eq!(recv(&mut ws).await["type"], "input_audio_buffer.append"); + send( + &mut ws, + json!({"type":"input_audio_buffer.committed","item_id":"user1"}), + ) + .await; + assert_eq!(recv(&mut ws).await["type"], "response.create"); + created(&mut ws, "live1").await; + send(&mut ws,json!({"type":"response.output_audio.delta","response_id":"live1","item_id":"audio1","content_index":0,"delta":STANDARD.encode(vec![0u8;4800])})).await; + // The client can only send this after receiving PCM. The response is + // deliberately still in flight: buffering until done deadlocks the test. + assert_eq!(recv(&mut ws).await["type"], "input_audio_buffer.append"); + send(&mut ws, json!({"type":"input_audio_buffer.speech_started"})).await; + let cancel = recv(&mut ws).await; + assert_eq!(cancel["type"], "response.cancel"); + assert_eq!(cancel["response_id"], "live1"); + send(&mut ws,json!({"type":"response.output_audio.delta","response_id":"live1","item_id":"audio1","content_index":0,"delta":STANDARD.encode(vec![1u8;4800])})).await; + let truncate = recv(&mut ws).await; + assert_eq!(truncate["type"], "conversation.item.truncate"); + assert_eq!(truncate["item_id"], "audio1"); + assert_eq!(truncate["audio_end_ms"], 25); + send( + &mut ws, + json!({"type":"input_audio_buffer.committed","item_id":"user2"}), + ) + .await; + send(&mut ws,json!({"type":"response.done","response":{"id":"live1","status":"cancelled","output":[]}})).await; + // No response may be created until the provider acknowledges truncation. + assert!(tokio::time::timeout(Duration::from_millis(100), ws.next()) + .await + .is_err()); + send(&mut ws,json!({"type":"conversation.item.truncated","item_id":"audio1","content_index":0,"audio_end_ms":25})).await; + assert_eq!(recv(&mut ws).await["type"], "response.create"); + created(&mut ws, "live2").await; + send(&mut ws,json!({"type":"response.output_audio.delta","response_id":"live2","item_id":"audio2","content_index":0,"delta":STANDARD.encode(vec![2u8;4800])})).await; + done(&mut ws,"live2",vec![json!({"id":"audio2","type":"message","role":"assistant","status":"completed","content":[{"type":"output_audio","transcript":"second"}]})]).await; + assert_eq!(recv(&mut ws).await["type"], "input_audio_buffer.append"); + send( + &mut ws, + json!({"type":"input_audio_buffer.committed","item_id":"user3"}), + ) + .await; + assert_eq!(recv(&mut ws).await["type"], "response.create"); + created(&mut ws, "live3").await; + // Ordering marker: the previous output has been retired, no new PCM yet. + send(&mut ws, json!({"type":"input_audio_buffer.speech_stopped"})).await; + assert_eq!(recv(&mut ws).await["type"], "input_audio_buffer.append"); + send(&mut ws,json!({"type":"response.output_audio.delta","response_id":"live3","item_id":"audio3","content_index":0,"delta":STANDARD.encode(vec![2u8;4800])})).await; + done(&mut ws,"live3",vec![json!({"id":"audio3","type":"message","role":"assistant","status":"completed","content":[{"type":"output_audio","transcript":"third"}]})]).await; + // Keep the connection open until the ACP client explicitly closes it. + let _ = tokio::time::timeout(Duration::from_secs(10), ws.next()).await; + }); + let mut h = Harness::spawn_with_env(&url, &options()).await; + let i = h.send("initialize",json!({"protocolVersion":1,"clientCapabilities":{"_meta":{"buzz":{"realtimeAudio":1}}}})).await; + let initialized = h.recv_until(|v| v["id"] == i).await; + assert_eq!( + initialized["result"]["agentCapabilities"]["_meta"]["buzz"]["realtimeAudio"], + 1 + ); + let i = h + .send("session/new", json!({"cwd":dir.path(),"mcpServers":[]})) + .await; + let sid = h.recv_until(|v| v["id"] == i).await["result"]["sessionId"] + .as_str() + .unwrap() + .to_owned(); + let prompt = h + .send( + "session/prompt", + json!({"sessionId":sid,"prompt":[],"_meta":{"buzz":{"realtimeAudio":1}}}), + ) + .await; + let ready = h + .recv_until(|v| v["params"]["update"]["type"] == "ready" || v["id"] == prompt) + .await; + assert_eq!(ready["params"]["update"]["type"], "ready", "{ready}"); + let stream = ready["params"]["streamId"].as_str().unwrap(); + let append = json!({"sessionId":sid,"streamId":stream,"sequence":0,"data":STANDARD.encode(vec![0u8;960])}); + h.send("_buzz/unstable/realtime/append", append).await; + let audio = h + .recv_until(|v| v["params"]["update"]["type"] == "audio" || v["id"] == prompt) + .await; + assert_eq!(audio["params"]["update"]["responseId"], "live1"); + h.send("_buzz/unstable/realtime/append",json!({"sessionId":sid,"streamId":stream,"sequence":1,"data":STANDARD.encode(vec![1u8;960])})).await; + h.recv_until(|v| v["params"]["update"]["type"] == "clear") + .await; + // A position already in transit when clear arrives is not the stopped clock. + // It must not close the truncation fence before the explicit worklet stop. + let tick = h.send("_buzz/unstable/realtime/playback",json!({"sessionId":sid,"streamId":stream,"playback":{"responseId":"live1","itemId":"audio1","contentIndex":0,"playedSamples":400}})).await; + assert!(h + .recv_until(|v| v["id"] == tick) + .await + .get("error") + .is_none()); + h.send("_buzz/unstable/realtime/playback",json!({"sessionId":sid,"streamId":stream,"playback":{"responseId":"live1","itemId":"audio1","contentIndex":0,"playedSamples":600,"stopped":true}})).await; + let mut transcript = String::new(); + loop { + let event = h.recv().await; + if event["params"]["update"]["sessionUpdate"] == "agent_message_chunk" { + transcript.push_str( + event["params"]["update"]["content"]["text"] + .as_str() + .unwrap(), + ); + } + assert_ne!( + event["id"], prompt, + "long-lived prompt ended before close: {event}" + ); + if event["params"]["update"]["type"] == "audio" { + assert_eq!( + event["params"]["update"]["responseId"], "live2", + "late interrupted PCM leaked" + ); + } + if event["params"]["update"]["type"] == "response_done" + && event["params"]["update"]["responseId"] == "live2" + { + break; + } + } + assert_eq!( + transcript, "second", + "completed live transcript was not forwarded" + ); + let stale = h.send("_buzz/unstable/realtime/playback",json!({"sessionId":sid,"streamId":stream,"playback":{"responseId":"live1","itemId":"audio1","contentIndex":0,"playedSamples":800}})).await; + let rejected = h.recv_until(|v| v["id"] == stale).await; + assert!( + rejected.get("error").is_some(), + "stale stopped playback was accepted: {rejected}" + ); + let final_position = json!({"sessionId":sid,"streamId":stream,"playback":{"responseId":"live2","itemId":"audio2","contentIndex":0,"playedSamples":2400}}); + let id = h + .send("_buzz/unstable/realtime/playback", final_position.clone()) + .await; + assert!(h.recv_until(|v| v["id"] == id).await.get("error").is_none()); + h.send("_buzz/unstable/realtime/append",json!({"sessionId":sid,"streamId":stream,"sequence":2,"data":STANDARD.encode(vec![0u8;960])})).await; + h.recv_until(|v| v["params"]["update"]["type"] == "speech_stopped") + .await; + let repeat = h + .send("_buzz/unstable/realtime/playback", final_position.clone()) + .await; + let ack = h.recv_until(|v| v["id"] == repeat).await; + assert!( + ack.get("error").is_none(), + "final playback repeat rejected after retirement: {ack}" + ); + let mut invalid = final_position.clone(); + invalid["playback"]["playedSamples"] = json!(2399); + let id = h.send("_buzz/unstable/realtime/playback", invalid).await; + assert!(h.recv_until(|v| v["id"] == id).await.get("error").is_some()); + h.send("_buzz/unstable/realtime/append",json!({"sessionId":sid,"streamId":stream,"sequence":3,"data":STANDARD.encode(vec![0u8;960])})).await; + h.recv_until(|v| { + v["params"]["update"]["type"] == "response_done" + && v["params"]["update"]["responseId"] == "live3" + }) + .await; + let repeat = h + .send("_buzz/unstable/realtime/playback", final_position) + .await; + let ack = h.recv_until(|v| v["id"] == repeat).await; + assert!( + ack.get("error").is_none(), + "final playback repeat rejected with newer PCM: {ack}" + ); + let close = h + .send( + "_buzz/unstable/realtime/close", + json!({"sessionId":sid,"streamId":stream}), + ) + .await; + h.recv_until(|v| v["id"] == close).await; + let ended = h.recv_until(|v| v["id"] == prompt).await; + assert_eq!(ended["result"]["stopReason"], "end_turn", "{ended}"); + server.await.unwrap(); +} + +#[tokio::test] +async fn realtime_live_tools_keep_capture_open_and_revoke_approval_on_speech() { + for interrupt in [false, true] { + let dir = tempfile::tempdir().unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("ws://{}/v1/realtime", listener.local_addr().unwrap()); + let (speech_tx, speech_rx) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + let (socket, _) = listener.accept().await.unwrap(); + let mut ws = accept_async(socket).await.unwrap(); + send( + &mut ws, + json!({"type":"session.created","session":{"id":"s"}}), + ) + .await; + let update = recv(&mut ws).await; + send( + &mut ws, + json!({"type":"session.updated","session":update["session"]}), + ) + .await; + assert_eq!(recv(&mut ws).await["type"], "input_audio_buffer.append"); + send( + &mut ws, + json!({"type":"input_audio_buffer.committed","item_id":"user1"}), + ) + .await; + assert_eq!(recv(&mut ws).await["type"], "response.create"); + created(&mut ws, "toolresponse").await; + send( + &mut ws, + json!({"type":"response.output_audio.delta","response_id":"toolresponse", + "item_id":"spoken","content_index":0,"delta":STANDARD.encode(vec![0;2400])}), + ) + .await; + done(&mut ws, "toolresponse", vec![ + json!({"id":"spoken","type":"message","role":"assistant","status":"completed","content":[]}), + call("livecall")]).await; + // Capture must reach the provider while approval is still pending. + assert_eq!(recv(&mut ws).await["type"], "input_audio_buffer.append"); + if interrupt { + send(&mut ws, json!({"type":"input_audio_buffer.speech_started"})).await; + } + speech_tx.send(()).unwrap(); + let mut result = recv(&mut ws).await; + if interrupt { + let other = recv(&mut ws).await; + let truncate = if result["type"] == "conversation.item.truncate" { + std::mem::replace(&mut result, other) + } else { + other + }; + assert_eq!(truncate["type"], "conversation.item.truncate"); + assert_eq!(truncate["item_id"], "spoken"); + assert_eq!(truncate["audio_end_ms"], 25); + send(&mut ws, json!({"type":"conversation.item.truncated","item_id":"spoken","content_index":0,"audio_end_ms":25})).await; + } + assert_eq!(result["type"], "conversation.item.create"); + assert_eq!(result["item"]["call_id"], "livecall"); + if interrupt { + assert!(result["item"]["output"] + .as_str() + .unwrap() + .contains("cancel")); + } else { + assert_eq!(recv(&mut ws).await["type"], "response.create"); + created(&mut ws, "aftertool").await; + done(&mut ws, "aftertool", vec![call("silentcall")]).await; + let result = recv(&mut ws).await; + assert_eq!(result["type"], "conversation.item.create"); + assert_eq!(result["item"]["call_id"], "silentcall"); + assert_eq!(recv(&mut ws).await["type"], "response.create"); + created(&mut ws, "aftersilenttool").await; + done(&mut ws, "aftersilenttool", vec![]).await; + } + let _ = tokio::time::timeout(Duration::from_secs(10), ws.next()).await; + }); + let mut h = Harness::spawn_with_env(&url, &options()).await; + let sid = init(&mut h, dir.path()).await; + let i = h.send("initialize",json!({"protocolVersion":1,"clientCapabilities":{"_meta":{"buzz":{"realtimeAudio":1}}}})).await; + h.recv_until(|v| v["id"] == i).await; + let prompt = h + .send( + "session/prompt", + json!({"sessionId":sid,"prompt":[],"_meta":{"buzz":{"realtimeAudio":1}}}), + ) + .await; + let ready = h + .recv_until(|v| v["params"]["update"]["type"] == "ready" || v["id"] == prompt) + .await; + let stream = ready["params"]["streamId"].as_str().unwrap().to_owned(); + h.send("_buzz/unstable/realtime/append",json!({"sessionId":sid,"streamId":stream,"sequence":0,"data":STANDARD.encode(vec![0;960])})).await; + let ask = h + .recv_until(|v| v["method"] == "session/request_permission" || v["id"] == prompt) + .await; + assert_eq!(ask["method"], "session/request_permission"); + assert!(!dir.path().join("calls.log").exists()); + let position = json!({"responseId":"toolresponse","itemId":"spoken","contentIndex":0, + "playedSamples":if interrupt {240} else {1200}}); + let playback = h + .send( + "_buzz/unstable/realtime/playback", + json!({"sessionId":sid,"streamId":stream,"playback":position}), + ) + .await; + let ack = h.recv_until(|v| v["id"] == playback).await; + assert!( + ack.get("error").is_none(), + "playback during approval failed: {ack}" + ); + h.send("_buzz/unstable/realtime/append",json!({"sessionId":sid,"streamId":stream,"sequence":1,"data":STANDARD.encode(vec![0;960])})).await; + speech_rx.await.unwrap(); + if interrupt { + h.recv_until(|v| v["params"]["update"]["type"] == "speech_started") + .await; + h.send("_buzz/unstable/realtime/playback",json!({"sessionId":sid,"streamId":stream, + "playback":{"responseId":"toolresponse","itemId":"spoken","contentIndex":0,"playedSamples":600,"stopped":true}})).await; + h.recv_until(|v| { + v["params"]["update"]["sessionUpdate"] == "tool_call_update" + && v["params"]["update"]["status"] == "failed" + }) + .await; + } + h.write(approve_permission(&ask)).await; + if !interrupt { + let ask = h + .recv_until(|v| v["method"] == "session/request_permission" || v["id"] == prompt) + .await; + assert_eq!(ask["method"], "session/request_permission"); + h.write(approve_permission(&ask)).await; + h.recv_until(|v| { + v["params"]["update"]["type"] == "response_done" + && v["params"]["update"]["responseId"] == "aftersilenttool" + }) + .await; + // An audio-less tool continuation must not erase the last speaker position. + for (samples, valid) in [(1200, true), (1199, false), (1201, false)] { + let id = h.send("_buzz/unstable/realtime/playback",json!({"sessionId":sid,"streamId":stream, + "playback":{"responseId":"toolresponse","itemId":"spoken","contentIndex":0,"playedSamples":samples}})).await; + let ack = h.recv_until(|v| v["id"] == id).await; + assert_eq!( + ack.get("error").is_none(), + valid, + "playback after silent tool continuation: {ack}" + ); + } + } + let close = h + .send( + "_buzz/unstable/realtime/close", + json!({"sessionId":sid,"streamId":stream}), + ) + .await; + h.recv_until(|v| v["id"] == close).await; + h.recv_until(|v| v["id"] == prompt).await; + let calls = std::fs::read_to_string(dir.path().join("calls.log")).unwrap_or_default(); + assert_eq!(calls.lines().count(), if interrupt { 0 } else { 2 }); + server.await.unwrap(); + } +} + +#[tokio::test] +async fn realtime_live_failure_preserves_bounded_provider_reason() { + for failed_response in [false, true] { + let dir = tempfile::tempdir().unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("ws://{}/v1/realtime", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + let (socket, _) = listener.accept().await.unwrap(); + let mut ws = accept_async(socket).await.unwrap(); + send( + &mut ws, + json!({"type":"session.created","session":{"id":"failure"}}), + ) + .await; + let update = recv(&mut ws).await; + send( + &mut ws, + json!({"type":"session.updated","session":update["session"]}), + ) + .await; + assert_eq!(recv(&mut ws).await["type"], "input_audio_buffer.append"); + let detail = json!({"code":"inference_error","message":format!("mouth\n priming failed {}TAIL", "x".repeat(600))}); + if failed_response { + send( + &mut ws, + json!({"type":"input_audio_buffer.committed","item_id":"user"}), + ) + .await; + assert_eq!(recv(&mut ws).await["type"], "response.create"); + created(&mut ws, "failed").await; + send(&mut ws, json!({"type":"response.done","response":{"id":"failed","status":"failed","output":[],"status_details":{"type":"failed","error":detail}}})).await; + } else { + send(&mut ws, json!({"type":"error","error":detail})).await; + } + let _ = tokio::time::timeout(Duration::from_secs(5), ws.next()).await; + }); + let mut h = Harness::spawn_with_env(&url, &options()).await; + let i = h.send("initialize", json!({"protocolVersion":1,"clientCapabilities":{"_meta":{"buzz":{"realtimeAudio":1}}}})).await; + h.recv_until(|v| v["id"] == i).await; + let i = h + .send("session/new", json!({"cwd":dir.path(),"mcpServers":[]})) + .await; + let sid = h.recv_until(|v| v["id"] == i).await["result"]["sessionId"] + .as_str() + .unwrap() + .to_owned(); + let prompt = h + .send( + "session/prompt", + json!({"sessionId":sid,"prompt":[],"_meta":{"buzz":{"realtimeAudio":1}}}), + ) + .await; + let ready = h + .recv_until(|v| v["params"]["update"]["type"] == "ready" || v["id"] == prompt) + .await; + assert_eq!(ready["params"]["update"]["type"], "ready", "{ready}"); + h.send("_buzz/unstable/realtime/append", json!({"sessionId":sid,"streamId":ready["params"]["streamId"],"sequence":0,"data":STANDARD.encode(vec![0u8;960])})).await; + let result = h.recv_until(|v| v["id"] == prompt).await; + let message = result["error"]["message"].as_str().unwrap(); + assert!(message.contains("mouth priming failed"), "{result}"); + assert!( + !message.contains("TAIL") && !message.contains('\n') && message.len() < 650, + "{result}" + ); + server.await.unwrap(); + h.shutdown().await; + } +} + +#[tokio::test] +async fn realtime_acp_live_output_limits_preserve_session_without_tools() { + for reason in ["max_output_tokens", "max_output_audio", "unknown_limit"] { + let dir = tempfile::tempdir().unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("ws://{}/v1/realtime", listener.local_addr().unwrap()); + let known = reason != "unknown_limit"; + let server = tokio::spawn(async move { + let (socket, _) = listener.accept().await.unwrap(); + let mut ws = accept_async(socket).await.unwrap(); + send( + &mut ws, + json!({"type":"session.created","session":{"id":"limits"}}), + ) + .await; + modality_ack(&mut ws).await; + assert_eq!(recv(&mut ws).await["type"], "input_audio_buffer.append"); + send( + &mut ws, + json!({"type":"input_audio_buffer.committed","item_id":"input1"}), + ) + .await; + assert_eq!(recv(&mut ws).await["type"], "response.create"); + created(&mut ws, "limited").await; + send(&mut ws,json!({"type":"response.output_audio.delta","response_id":"limited","item_id":"audio_limit","content_index":0,"delta":STANDARD.encode(vec![0u8;4800])})).await; + // Even a syntactically complete call in an incomplete response must not run. + send(&mut ws,json!({"type":"response.done","response":{"id":"limited","status":"incomplete","status_details":{"type":"incomplete","reason":reason},"output":[call("discarded")]}})).await; + if known { + assert_eq!(recv(&mut ws).await["type"], "input_audio_buffer.append"); + send( + &mut ws, + json!({"type":"input_audio_buffer.committed","item_id":"input2"}), + ) + .await; + assert_eq!(recv(&mut ws).await["type"], "response.create"); + text(&mut ws, "recovered").await; + } + let _ = tokio::time::timeout(Duration::from_secs(10), ws.next()).await; + }); + let mut h = Harness::spawn_with_env(&url, &options()).await; + let i=h.send("initialize",json!({"protocolVersion":1,"clientCapabilities":{"_meta":{"buzz":{"realtimeAudio":1}}}})).await; + h.recv_until(|v| v["id"] == i).await; + let i = h + .send("session/new", json!({"cwd":dir.path(),"mcpServers":[]})) + .await; + let sid = h.recv_until(|v| v["id"] == i).await["result"]["sessionId"] + .as_str() + .unwrap() + .to_owned(); + let prompt = h + .send( + "session/prompt", + json!({"sessionId":sid,"prompt":[],"_meta":{"buzz":{"realtimeAudio":1}}}), + ) + .await; + let ready = h + .recv_until(|v| v["params"]["update"]["type"] == "ready" || v["id"] == prompt) + .await; + let stream = ready["params"]["streamId"].as_str().unwrap(); + h.send("_buzz/unstable/realtime/append",json!({"sessionId":sid,"streamId":stream,"sequence":0,"data":STANDARD.encode(vec![0u8;960])})).await; + let mut notice = false; + loop { + let event = h.recv().await; + assert_ne!( + event["method"], "session/request_permission", + "truncated tool reached permissions" + ); + assert_ne!(event["params"]["update"]["sessionUpdate"], "tool_call"); + if event["params"]["update"]["type"] == "response_limited" { + notice = true; + assert_eq!(event["params"]["update"]["reason"], reason); + } + if event["id"] == prompt { + assert!(!known, "bounded response killed live session: {event}"); + assert!(event.get("error").is_some()); + break; + } + if event["params"]["update"]["type"] == "response_done" { + assert!(known && notice); + break; + } + } + if known { + let id=h.send("_buzz/unstable/realtime/playback",json!({"sessionId":sid,"streamId":stream,"playback":{"responseId":"limited","itemId":"audio_limit","contentIndex":0,"playedSamples":2400}})).await; + assert!(h.recv_until(|v| v["id"] == id).await.get("error").is_none()); + h.send("_buzz/unstable/realtime/append",json!({"sessionId":sid,"streamId":stream,"sequence":1,"data":STANDARD.encode(vec![0u8;960])})).await; + let recovered = h + .recv_until(|v| { + v["params"]["update"]["type"] == "response_done" || v["id"] == prompt + }) + .await; + assert_eq!(recovered["params"]["update"]["responseId"], "recovered"); + let close = h + .send( + "_buzz/unstable/realtime/close", + json!({"sessionId":sid,"streamId":stream}), + ) + .await; + h.recv_until(|v| v["id"] == close).await; + assert_eq!( + h.recv_until(|v| v["id"] == prompt).await["result"]["stopReason"], + "end_turn" + ); + } + server.await.unwrap(); + h.shutdown().await; + } +} + +#[tokio::test] +async fn realtime_duplex_capture_and_backchannels_preserve_turn_ownership() { + for (supported, bad_sequence) in [(true, false), (false, false), (true, true)] { + let dir = tempfile::tempdir().unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("ws://{}/v1/realtime", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + let (socket, _) = listener.accept().await.unwrap(); + let mut ws = accept_async(socket).await.unwrap(); + send( + &mut ws, + json!({"type":"session.created","session":{"id":"duplex"}}), + ) + .await; + let update = recv(&mut ws).await; + let mut session = update["session"].clone(); + if supported { + session["frankie"] = json!({"duplex_audio":true,"backchannels":true}); + } + send(&mut ws, json!({"type":"session.updated","session":session})).await; + let input = recv(&mut ws).await; + assert_eq!(input["type"], "input_audio_buffer.append"); + assert_eq!( + STANDARD.decode(input["audio"].as_str().unwrap()).unwrap(), + vec![1u8; 960] + ); + send(&mut ws, json!({"type":"input_audio_buffer.cleared"})).await; + if supported { + assert_eq!( + STANDARD + .decode(input["playback_audio"].as_str().unwrap()) + .unwrap(), + vec![2u8; 960] + ); + send(&mut ws,json!({"type":"frankie.backchannel.delta","id":"aside","sequence":if bad_sequence {1} else {0},"delta":STANDARD.encode(vec![3u8;3840])})).await; + if !bad_sequence { + send( + &mut ws, + json!({"type":"frankie.backchannel.done","id":"aside","samples":1920}), + ) + .await; + } + } + if !supported { + assert!(input.get("playback_audio").is_none()); + } + if !bad_sequence { + // Listener audio must never cause response.create or a conversation item. + assert_eq!(recv(&mut ws).await["type"], "input_audio_buffer.append"); + send( + &mut ws, + json!({"type":"input_audio_buffer.committed","item_id":"user"}), + ) + .await; + assert_eq!(recv(&mut ws).await["type"], "response.create"); + text(&mut ws, "reply").await; + } + let _ = tokio::time::timeout(Duration::from_secs(10), ws.next()).await; + }); + let mut h = Harness::spawn_with_env(&url, &options()).await; + let id = h.send("initialize",json!({"protocolVersion":1,"clientCapabilities":{"_meta":{"buzz":{"realtimeAudio":1}}}})).await; + h.recv_until(|v| v["id"] == id).await; + let id = h + .send("session/new", json!({"cwd":dir.path(),"mcpServers":[]})) + .await; + let sid = h.recv_until(|v| v["id"] == id).await["result"]["sessionId"] + .as_str() + .unwrap() + .to_owned(); + let prompt = h + .send( + "session/prompt", + json!({"sessionId":sid,"prompt":[],"_meta":{"buzz":{"realtimeAudio":1}}}), + ) + .await; + let ready = h + .recv_until(|v| v["params"]["update"]["type"] == "ready" || v["id"] == prompt) + .await; + let stream = ready["params"]["streamId"].as_str().unwrap(); + let id=h.send("_buzz/unstable/realtime/append",json!({"sessionId":sid,"streamId":stream,"sequence":0,"data":STANDARD.encode(vec![1u8;960]),"playbackData":STANDARD.encode(vec![2u8;958])})).await; + assert!(h.recv_until(|v| v["id"] == id).await.get("error").is_some()); + h.send("_buzz/unstable/realtime/append",json!({"sessionId":sid,"streamId":stream,"sequence":0,"data":STANDARD.encode(vec![1u8;960]),"playbackData":STANDARD.encode(vec![2u8;960])})).await; + h.recv_until(|v| v["params"]["update"]["type"] == "input_cleared") + .await; + if bad_sequence { + assert!(h + .recv_until(|v| v["id"] == prompt) + .await + .get("error") + .is_some()); + } else { + if supported { + let audio = h + .recv_until(|v| { + v["params"]["update"]["type"] == "backchannel_audio" || v["id"] == prompt + }) + .await; + assert_eq!(audio["params"]["update"]["id"], "aside"); + assert_eq!( + STANDARD + .decode(audio["params"]["update"]["data"].as_str().unwrap()) + .unwrap(), + vec![3u8; 3840] + ); + h.recv_until(|v| v["params"]["update"]["type"] == "backchannel_done") + .await; + } + h.send("_buzz/unstable/realtime/append",json!({"sessionId":sid,"streamId":stream,"sequence":1,"data":STANDARD.encode(vec![0u8;960])})).await; + let reply = h + .recv_until(|v| { + v["params"]["update"]["type"] == "response_done" || v["id"] == prompt + }) + .await; + assert_eq!(reply["params"]["update"]["responseId"], "reply"); + let close = h + .send( + "_buzz/unstable/realtime/close", + json!({"sessionId":sid,"streamId":stream}), + ) + .await; + h.recv_until(|v| v["id"] == close).await; + assert_eq!( + h.recv_until(|v| v["id"] == prompt).await["result"]["stopReason"], + "end_turn" + ); + } + server.await.unwrap(); + h.shutdown().await; + } +} diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index 6af27ee243..888fcd5b3b 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -1042,6 +1042,7 @@ dependencies = [ "base64 0.22.1", "dirs", "fs2", + "futures-util", "getrandom 0.4.3", "hex", "nix 0.31.3", @@ -1052,6 +1053,7 @@ dependencies = [ "serde_yaml", "sha2 0.11.0", "tokio", + "tokio-tungstenite 0.29.0", "tracing", "tracing-subscriber", "url", diff --git a/examples/realtime-audio/README.md b/examples/realtime-audio/README.md new file mode 100644 index 0000000000..1978300ffa --- /dev/null +++ b/examples/realtime-audio/README.md @@ -0,0 +1,170 @@ +# Frankie through Buzz + +Run a local voice conversation through the native llama.cpp Frankie server, Buzz +agent, and this webpage. The browser captures and plays mono PCM16 at 24 kHz. +Buzz owns MCP tools and their permission decisions; llama.cpp loads the single +GGUF and runs the brain, Parakeet, Qwen3-TTS, vision, and turn models through ggml. +No Python inference service or hosted model API is needed. + +The page includes live transcripts, interruption, thinking controls, tool +approval, and latency measured from the end of speech to actual playback. + +## Get the model + +Download the [shared Frankie GGUF](https://drive.google.com/file/d/1oMUgJteaoBvgQicYvz6pWRpcSxun4KA-/view?usp=sharing) +and save it as `models/frankie-studio-c10-q8-audio.gguf` outside the source repositories. +Use Drive's download button; its preview is not the model file. Access to this +externally hosted download is managed separately from this repository. + +The tested September 10, 2026 package has: + +- Size: **20,326,799,488 bytes** (20.33 GB / 18.93 GiB). +- SHA-256: `1822228c2caa051fb4fa8535c9a3105c2bb67440968dc5a9f6ee44ffbc439c50`. +- Custom Q4_1 brain; supported Parakeet and Qwen3-TTS matrices in Q8_0. + Convolutions and sensitive tensors retain their original types; vision is F16. +- The packaged c10 reference voice. A custom reference can replace it at launch. + +Verify the download with `shasum -a 256 models/frankie-studio-c10-q8-audio.gguf` +(macOS) or `sha256sum` (Linux). A different hash identifies a different package; +the compatibility results here apply to the hash above. Model and reference-voice +licensing are separate from the code license; consult the package's source and +voice metadata before redistribution. + +## Build llama.cpp + +Prerequisites: Git, CMake, a C/C++ toolchain, ICU development headers/libraries, +and OpenSSL development headers/libraries. For CUDA, install the CUDA toolkit; +for Metal, use macOS with Xcode command-line tools. See the fork's +[Frankie build guide](https://github.com/tlongwell-block/llama.cpp/blob/d0aa7352535cbdc2bddfd5512b5f95d731761f72/tools/frankie/README.md) +for native dependencies and tuning. + +Use the Frankie fork, including [PR #1](https://github.com/tlongwell-block/llama.cpp/pull/1). +The pinned revision below is the one used to validate this GGUF; stock llama.cpp +cannot run this package. + +```sh +git clone https://github.com/tlongwell-block/llama.cpp.git llama-frankie +cd llama-frankie +git checkout d0aa7352535cbdc2bddfd5512b5f95d731761f72 + +# Metal (macOS) +cmake -S . -B build-frankie -DCMAKE_BUILD_TYPE=Release \ + -DLLAMA_BUILD_FRANKIE=ON -DGGML_METAL=ON +cmake --build build-frankie --target llama-frankie-realtime -j4 +cd .. +``` + +For CPU, replace `-DGGML_METAL=ON` with +`-DGGML_METAL=OFF -DGGML_CUDA=OFF`. For CUDA, use +`-DGGML_METAL=OFF -DGGML_CUDA=ON`. If CMake cannot find ICU, add +`-DICU_ROOT=/path/to/icu/prefix` using the prefix of your installation. + +## Build Buzz and launch + +The example uses Node 24 and the repository's Rust toolchain through Hermit. +Follow the root [contributing guide](../../CONTRIBUTING.md) for any platform +build prerequisites. The page has no npm runtime dependencies and needs no +frontend build. + +```sh +git clone https://github.com/block/buzz.git buzz-frankie +cd buzz-frankie +git fetch origin pull/7519/head:frankie-demo +git checkout frankie-demo +. ./bin/activate-hermit +cargo build --release -p buzz-agent -p buzz-dev-mcp + +node examples/realtime-audio/launch.mjs \ + --native ../llama-frankie/build-frankie/bin/llama-frankie-realtime \ + --model ../models/frankie-studio-c10-q8-audio.gguf \ + --device gpu +``` + +Use `--device cpu` with a CPU build. Open the complete `http://127.0.0.1:18796/#…` +URL printed by the launcher in Chrome, click **Start talking**, and allow the +microphone. Model loading and warm-up happen before this URL appears. Stop with +Ctrl-C. The launcher supports macOS and Linux. + +Try “What is two plus two?” and then “Run uptime.” A tool request displays its +arguments: choose **Allow once** or **Deny**. The demo advertises one short shell +tool through a thin adapter to the existing `buzz-dev-mcp`; approval and execution +stay in Buzz. Nothing is executed by recognizing speech alone. + +By default tools run in an empty temporary directory. Add `--cwd /path/to/project` +to choose their working directory. The launcher uses neutral instructions and +disables local hints/skills; it does not load a personal persona. Each connection +gets a new agent session. A disconnect does not replay previous tool actions. + +## Voice, thinking, and memory + +Add a non-silent WAV up to 20 seconds to replace the bundled voice: + +```sh +node examples/realtime-audio/launch.mjs \ + --native ../llama-frankie/build-frankie/bin/llama-frankie-realtime \ + --model ../models/frankie-studio-c10-q8-audio.gguf \ + --voice /path/to/my-voice.wav +``` + +WAV-only conditioning uses the native speaker encoder; rebuilding the GGUF is +unnecessary. Optional `--voice-text-file` and `--voice-codes` must be supplied +together with the matching WAV. Codec IDs must already exist; the native runtime +does not generate these optional ICL codes from arbitrary WAVs. + +Choose **Off**, **Minimal**, **Low**, **Medium**, **High**, **Very high**, or +**Maximum** thinking before connecting. Off is the default. Buzz forwards the +setting and requires the provider to acknowledge it. Reasoning is not spoken; +higher levels can substantially delay the first answer. End and reconnect to +change the level. + +The launcher defaults to `--context 131072 --cache-type q4_0 --threads 4`. +`--context`, `--cache-type q4_0|q8_0|f16`, and `--threads` are configurable. File +size is not peak GPU memory: KV/recurrent state and compute buffers also count. +CPU and Metal have native integration coverage. CUDA and a complete 128K workload +on a 24 GiB consumer GPU still require hardware validation; the default context +allocation does not establish that memory or throughput target. + +## Operation and troubleshooting + +- Only one browser connection can use this demo at a time. Close an existing + conversation before opening another. The page requires a fresh connection + after transport errors; tool effects are never automatically replayed. +- Use `--port` and `--native-port` if 18796 or 18793 is already occupied. +- The browser adapter and native server bind to loopback. Keep the printed URL + private: its random fragment authorizes that local demo. Provider credentials + stay server-side. There are no external page assets or browser storage writes. +- Temporary logs are printed at startup and retained for troubleshooting. + The adapter does not record conversations or audio to disk. Normal tool commands + may read or write files when approved. Do not add runtime logs or models to Git. +- With Bluetooth headphones, selecting their microphone can switch the device + into lower-quality call audio. Try speakers or a separate microphone if playback + sounds uniformly muffled. Check Chrome's microphone selection. +- Brief capture stalls are buffered for up to five seconds. Longer stalls close + the session visibly instead of silently dropping accepted samples. Slow inference + can still underrun playback; reduce thinking or use a faster backend. +- The launcher expires after 24 hours (`--hours`, up to 168). Each conversation + also has a one-hour limit. Use `--help` for binary and port overrides. + +For a separately managed native server, `server.mjs` can run directly. Configure +`BUZZ_AGENT_PROVIDER=openai`, `OPENAI_COMPAT_API=realtime`, +`OPENAI_COMPAT_MODEL=frankie`, `OPENAI_COMPAT_BASE_URL=ws://127.0.0.1:18793/v1/realtime`, +and `OPENAI_COMPAT_API_KEY` matching the native `FRANKIE_REALTIME_TOKEN`. Set +`BUZZ_AGENT_BIN`, `BUZZ_MCP_BIN` to this directory's executable `mcp-shell.mjs`, +`FRANKIE_DEV_MCP_BIN` to `buzz-dev-mcp`, and `BUZZ_AGENT_NO_HINTS=1`. Run +`node /path/to/server.mjs` from the desired tool working directory. The launcher +is the recommended path because it supplies neutral configuration and owns cleanup. + +## Tests + +```sh +node --test examples/realtime-audio/*.test.mjs +cargo test -p buzz-agent +``` + +The Node tests exercise the production capture queue, AudioWorklet, loopback +authentication, permission forwarding, disconnect cleanup, and launcher rejection +paths. Rust tests exercise the actual ACP subprocess and controlled WebSocket/MCP +peers, including stale playback, cancellation, thinking acknowledgment, output +limits, and reconnect isolation. Native model inference is a separate end-to-end +check requiring the GGUF and the fork binary; unit fixtures are not inference +benchmarks. The example tests also run in `just ci` and GitHub CI. diff --git a/examples/realtime-audio/audio-worklet.mjs b/examples/realtime-audio/audio-worklet.mjs new file mode 100644 index 0000000000..30b5cd4d51 --- /dev/null +++ b/examples/realtime-audio/audio-worklet.mjs @@ -0,0 +1,185 @@ +// Both directions share the device sample clock, not message-arrival timestamps. +class DuplexAudio extends AudioWorkletProcessor { + constructor() { + super(); + this.input = new Int16Array(480); + this.systemInput = new Int16Array(480); + this.inputUsed = 0; + this.capture = false; + this.queue = []; + this.queued = 0; + this.played = 0; + this.item = null; + this.blocked = new Set(); + this.aside = []; + this.asideQueued = 0; + this.asideId = null; + this.captured = 0; + this.rendered = 0; + this.startedAt = null; + this.gapFrame = null; + this.tick = 0; + this.reportedItem = null; + this.reportedSamples = 0; + this.port.onmessage = ({ data }) => { + if (data.type === "capture") this.capture = data.enabled; + if (data.type === "backchannel") { + if (this.queued || this.blocked.has(data.id)) return; + if (this.asideId !== data.id) { + if (this.asideQueued) { + this.fail("overlapping backchannels"); + return; + } + this.asideId = data.id; + } + if (this.asideQueued + data.pcm.length > 30720) { + this.fail("backchannel buffer exceeded 1.28 seconds"); + return; + } + this.aside.push({ pcm: data.pcm, offset: 0 }); + this.asideQueued += data.pcm.length; + } + if (data.type === "audio") { + if (this.asideId) this.blocked.add(this.asideId); + this.aside = []; + this.asideQueued = 0; + if (this.blocked.has(data.itemId)) return; + if (this.item && this.item.itemId !== data.itemId && this.queued) { + this.fail("overlapping playback items"); + return; + } + if (!this.item || this.item.itemId !== data.itemId) { + this.item = data; + this.played = 0; + this.startedAt = null; + this.gapFrame = null; + } + // Bound queued playback independently of response length; native speech is paced. + if (this.queued + data.pcm.length > 24000 * 30) { + this.fail("playback buffer exceeded thirty seconds"); + return; + } + this.queue.push({ pcm: data.pcm, offset: 0 }); + this.queued += data.pcm.length; + } + if (data.type === "clear") { + if (this.item) this.blocked.add(this.item.itemId); + if (this.blocked.size > 4096) { + this.fail("playback item limit"); + return; + } + this.queue = []; + this.queued = 0; + if (this.asideId) this.blocked.add(this.asideId); + this.aside = []; + this.asideQueued = 0; + this.position("stopped", data.requestId); + this.item = null; + this.gapFrame = null; + } + }; + } + fail(message) { + this.queue = []; + this.queued = 0; + this.aside = []; + this.asideQueued = 0; + this.capture = false; + this.port.postMessage({ type: "error", message }); + } + position(type, requestId) { + if ( + type === "position" && + (!this.item || + (this.reportedItem === this.item.itemId && + this.reportedSamples === this.played)) + ) + return; + this.reportedItem = this.item?.itemId; + this.reportedSamples = this.played; + this.port.postMessage({ + type, + requestId, + startedAt: this.startedAt, + playback: this.item + ? { + responseId: this.item.responseId, + itemId: this.item.itemId, + contentIndex: 0, + playedSamples: this.played, + } + : null, + }); + } + process(inputs, outputs) { + if (sampleRate !== 24000) { + this.fail("device context must run at 24 kHz"); + return false; + } + const input = inputs[0]?.[0], + output = outputs[0][0]; + const frame = + typeof currentFrame === "number" ? currentFrame : this.rendered; + for (let i = 0; i < output.length; i++) { + const head = this.queue[0]; + if (head) { + if (this.gapFrame !== null) { + this.port.postMessage({ + type: "playback_gap", + itemId: this.item.itemId, + playedSamples: this.played, + missingSamples: frame + i - this.gapFrame, + }); + this.gapFrame = null; + } + if (this.startedAt === null) this.startedAt = (frame + i) / sampleRate; + output[i] = head.pcm[head.offset++] / 32768; + this.played++; + this.queued--; + if (head.offset === head.pcm.length) this.queue.shift(); + } else { + // Count a gap only if more audio resumes in this same item; idle time is not an underrun. + if (this.item && this.played && this.gapFrame === null) + this.gapFrame = frame + i; + const aside = this.aside[0]; + if (aside) { + output[i] = aside.pcm[aside.offset++] / 32768; + this.asideQueued--; + if (aside.offset === aside.pcm.length) this.aside.shift(); + } else output[i] = 0; + } + if (this.capture) { + this.captured++; + this.systemInput[this.inputUsed] = Math.round(output[i] * 32768); + this.input[this.inputUsed++] = Math.round( + Math.max(-1, Math.min(1, input?.[i] || 0)) * 32767, + ); + if (this.inputUsed === 480) { + const pcm = this.input, + playback = this.systemInput; + this.port.postMessage( + { + type: "capture", + pcm, + playback, + captureSamples: this.captured, + captureTime: (frame + i + 1) / sampleRate, + }, + [pcm.buffer, playback.buffer], + ); + this.input = new Int16Array(480); + this.systemInput = new Int16Array(480); + this.inputUsed = 0; + } + } + } + this.rendered += output.length; + this.tick += output.length; + if (this.tick >= 2400) { + this.tick = 0; + this.position("position"); + } + return true; + } +} +registerProcessor("duplex-audio", DuplexAudio); diff --git a/examples/realtime-audio/audio-worklet.test.mjs b/examples/realtime-audio/audio-worklet.test.mjs new file mode 100644 index 0000000000..8a6e9aa28f --- /dev/null +++ b/examples/realtime-audio/audio-worklet.test.mjs @@ -0,0 +1,141 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { test } from "node:test"; +import vm from "node:vm"; + +function processor() { + let Audio; + const events = []; + const scope = { + sampleRate: 24000, + AudioWorkletProcessor: class { + constructor() { + this.port = { postMessage: (e) => events.push(e) }; + } + }, + registerProcessor: (_, type) => { + Audio = type; + }, + }; + vm.runInNewContext( + readFileSync(new URL("./audio-worklet.mjs", import.meta.url), "utf8"), + scope, + ); + const instance = new Audio(); + return { + instance, + events, + send: (data) => instance.port.onmessage({ data }), + }; +} +const audio = (seconds, itemId = "one") => ({ + type: "audio", + responseId: itemId, + itemId, + pcm: new Int16Array(24000 * seconds).fill(1000), +}); + +test("a valid fast response can queue beyond three seconds and remain interruptible", () => { + const { instance, events, send } = processor(); + send(audio(10)); + assert.equal(events.length, 0); + assert.equal(instance.queued, 240000); + send({ type: "capture", enabled: true }); + const output = new Float32Array(2400); + instance.process([[new Float32Array(2400)]], [[output]]); + assert.ok(output[0] > 0); + assert.ok(events.some((e) => e.type === "capture")); + send({ type: "clear", requestId: 7 }); + assert.equal(instance.queued, 0); + const stopped = events.find((e) => e.type === "stopped"); + assert.equal(stopped.playback.playedSamples, 2400); + send(audio(1)); + assert.equal(instance.queued, 0); + send(audio(1, "two")); + assert.equal(instance.queued, 24000); +}); + +test("playback retains a hard thirty-second cap", () => { + const { instance, events, send } = processor(); + send(audio(30)); + assert.equal(events.length, 0); + send(audio(1)); + assert.ok(events.some((e) => e.type === "error")); + assert.equal(instance.queued, 0); +}); + +test("idle playback does not report the previous response as a new first sound", () => { + const { instance, events, send } = processor(); + const output = new Float32Array(2400); + send(audio(0.1)); + instance.process([], [[output]]); + for (let i = 0; i < 10; i++) instance.process([], [[output]]); + assert.equal(events.filter((e) => e.type === "position").length, 1); + send(audio(0.1, "two")); + instance.process([], [[output]]); + const positions = events.filter((e) => e.type === "position"); + assert.deepEqual( + positions.map((e) => e.playback.itemId), + ["one", "two"], + ); + send({ type: "clear", requestId: 9 }); + assert.equal(events.find((e) => e.type === "stopped").playback.itemId, "two"); +}); + +test("duplex capture reports exactly rendered audio, including underruns and backchannels", () => { + const { instance, events, send } = processor(); + send({ type: "capture", enabled: true }); + send({ + type: "backchannel", + id: "aside", + pcm: new Int16Array(240).fill(3276), + }); + const out = new Float32Array(480); + instance.process([[new Float32Array(480).fill(0.2)]], [[out]]); + const capture = events.find((e) => e.type === "capture"); + assert.equal(capture.pcm.length, 480); + assert.equal(capture.playback.length, 480); + assert.deepEqual( + Array.from(capture.playback.slice(0, 240)), + Array(240).fill(3276), + ); + assert.deepEqual(Array.from(capture.playback.slice(240)), Array(240).fill(0)); + assert.ok(capture.pcm.every((x) => x === 6553)); + // A normal reply supersedes an aside and late aside frames stay suppressed. + send({ + type: "backchannel", + id: "aside", + pcm: new Int16Array(480).fill(3276), + }); + send(audio(1)); + send({ + type: "backchannel", + id: "aside", + pcm: new Int16Array(480).fill(3276), + }); + instance.process([[new Float32Array(480)]], [[out]]); + assert.ok(out.every((x) => x === 1000 / 32768)); +}); + +test("playback gap diagnostics measure inserted silence without counting idle time between replies", () => { + const { instance, events, send } = processor(); + const output = new Float32Array(480); + send(audio(0.01)); + instance.process([], [[output]]); + assert.equal(events.filter((e) => e.type === "playback_gap").length, 0); + send(audio(0.02)); + instance.process([], [[output]]); + const gaps = events.filter((e) => e.type === "playback_gap"); + assert.equal(gaps.length, 1); + assert.equal(gaps[0].missingSamples, 240); + assert.equal(gaps[0].playedSamples, 240); + assert.ok(output.every((v) => v === 1000 / 32768)); + for (let i = 0; i < 10; i++) instance.process([], [[output]]); + send(audio(0.02, "two")); + instance.process([], [[output]]); + send({ type: "clear", requestId: 3 }); + instance.process([], [[output]]); + send(audio(0.02, "three")); + instance.process([], [[output]]); + assert.equal(events.filter((e) => e.type === "playback_gap").length, 1); +}); diff --git a/examples/realtime-audio/capture-queue.mjs b/examples/realtime-audio/capture-queue.mjs new file mode 100644 index 0000000000..5c00ec95fa --- /dev/null +++ b/examples/realtime-audio/capture-queue.mjs @@ -0,0 +1,31 @@ +// Bound latency/memory, not scheduler jitter. Never drop or reorder microphone samples. +export class CaptureQueue { + constructor() { + this.frames = []; + this.samples = 0; + } + push(pcm) { + if (this.samples + pcm.length > 120000) + throw Error("microphone transport stalled for over five seconds"); + this.frames.push(pcm); + this.samples += pcm.length; + } + shift() { + const count = Math.min(this.samples, 2400), + out = new Int16Array(count); + let used = 0; + while (used < count) { + const head = this.frames[0], + n = Math.min(head.length, count - used); + out.set(head.subarray(0, n), used); + used += n; + if (n === head.length) this.frames.shift(); + else this.frames[0] = head.subarray(n); + } + this.samples -= count; + return out; + } + get length() { + return this.samples; + } +} diff --git a/examples/realtime-audio/capture-queue.test.mjs b/examples/realtime-audio/capture-queue.test.mjs new file mode 100644 index 0000000000..a10fef9294 --- /dev/null +++ b/examples/realtime-audio/capture-queue.test.mjs @@ -0,0 +1,25 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { CaptureQueue } from "./capture-queue.mjs"; +test("1500ms scheduler stall retains ordered samples and drains in bounded ACP frames", () => { + const q = new CaptureQueue(); + for (let i = 0; i < 75; i++) q.push(new Int16Array(480).fill(i)); + const values = []; + while (q.length) { + const pcm = q.shift(); + assert.ok(pcm.length <= 2400); + values.push(...pcm); + } + assert.equal(values.length, 75 * 480); + for (let i = 0; i < values.length; i++) + assert.equal(values[i], Math.floor(i / 480)); +}); +test("five seconds is a hard bound, overflow is visible and no accepted audio is lost", () => { + const q = new CaptureQueue(); + for (let i = 0; i < 250; i++) q.push(new Int16Array(480)); + assert.throws(() => q.push(new Int16Array(480)), /stalled/); + assert.equal(q.length, 120000); + let n = 0; + while (q.length) n += q.shift().length; + assert.equal(n, 120000); +}); diff --git a/examples/realtime-audio/client.mjs b/examples/realtime-audio/client.mjs new file mode 100644 index 0000000000..ee940944b2 --- /dev/null +++ b/examples/realtime-audio/client.mjs @@ -0,0 +1,162 @@ +import * as ui from "./ui.mjs"; +import { openVoice } from "./voice.mjs"; +const $ = (id) => document.getElementById(id); +// The launch fragment stays in this tab for refresh; it is never sent in HTTP requests. +const token = location.hash.slice(1); +let voice, + connecting, + busy = false, + savedFocus, + received = 0, + finished = false, + measured = false, + outputItem; +window.realtimeEvidence = []; +function permission(tool, decide) { + $("permission").hidden = !tool; + if (!tool) { + savedFocus?.focus(); + return; + } + savedFocus = document.activeElement; + $("tool").textContent = JSON.stringify(tool, null, 2); + $("allow").onclick = () => decide("allow_once").catch(error); + $("deny").onclick = () => decide("reject_once").catch(error); + $("deny").focus(); +} +$("permission").onkeydown = (e) => { + if (e.key === "Escape") { + $("deny").click(); + e.preventDefault(); + } + if (e.key === "Tab") { + e.preventDefault(); + (document.activeElement === $("deny") ? $("allow") : $("deny")).focus(); + } +}; +function error(e) { + ui.note(e.message); + $("help").textContent = e.message; +} +$("go").onclick = async () => { + if (busy) { + if (connecting) { + connecting.abort(); + $("go").disabled = true; + } + return; + } + busy = true; + $("go").disabled = true; + $("thinking").disabled = true; + try { + if (voice) { + const old = voice; + voice = null; + await old.stop(); + return; + } + connecting = new AbortController(); + $("go-label").textContent = "Cancel connection"; + $("go").disabled = false; + const opened = await openVoice( + token, + { + evidence: (e) => { + window.realtimeEvidence = e; + }, + status: (t) => { + $("help").textContent = t; + }, + analyzers: ui.analyzers, + transcript: ui.transcript, + userTranscript: ui.userTranscript, + removeInput: ui.removeInput, + permission, + ended: () => { + voice = null; + $("thinking").disabled = false; + permission(null); + ui.analyzers(null, null); + ui.connection(false, $("help").textContent); + }, + event: (type, d) => { + if (type === "ready") { + ui.connection( + true, + "Speak naturally. You can interrupt. Headphones recommended.", + ); + ui.state("LISTEN"); + } + if (type === "speech_started") { + ui.state("LISTEN"); + received = 0; + finished = false; + measured = false; + outputItem = null; + } + if (type === "speech_stopped") { + ui.state("SPEC"); + } + if (type === "input_cleared") { + ui.state("LISTEN"); + } + if (type === "audio_received") { + if (outputItem !== d.itemId) { + outputItem = d.itemId; + received = 0; + finished = false; + } + received = Math.max(received, d.startSample + d.samples); + ui.state("REPLY"); + } + if (type === "response_done") { + finished = true; + } + if ( + type === "played" && + d.itemId === outputItem && + !measured && + Number.isFinite(d.firstSoundLatencyMs) && + d.firstSoundLatencyMs >= 0 + ) { + $("latency").textContent = + `FIRST SOUND ${Math.round(d.firstSoundLatencyMs)} ms`; + measured = true; + } + if ( + type === "played" && + d.itemId === outputItem && + finished && + d.playedSamples >= received + ) + ui.state("LISTEN"); + if (type === "playback_stopped") { + ui.state("LISTEN"); + ui.note("↳ Playback stopped."); + } + if (type === "error") ui.note(d.message); + if (type === "response_limited") + ui.note( + "Reply stopped at its output limit. Keep talking; the session is still live.", + ); + }, + }, + { thinking: $("thinking").value, signal: connecting.signal }, + ); + voice = opened; + } catch (e) { + if (connecting?.signal.aborted) { + ui.connection(false, "Connection cancelled."); + } else error(e); + } finally { + connecting = null; + busy = false; + $("go").disabled = false; + $("thinking").disabled = !!voice; + } +}; +window.addEventListener("pagehide", () => { + connecting?.abort(); + voice?.stop(false); +}); diff --git a/examples/realtime-audio/index.html b/examples/realtime-audio/index.html new file mode 100644 index 0000000000..f9b912ebff --- /dev/null +++ b/examples/realtime-audio/index.html @@ -0,0 +1,22 @@ + + + + +Frankie — voice interface + + + +
+
Frankie
VOICE LINK / 01OFFLINE
+
An open channel. A little personality.

Frankie.

Less interface. More conversation.

+
+
NEURAL AUDIO / LIVEAWAITING CONNECTION
FRANKIE IS
Ready.
YOUR VOICE STARTS HERE
YOU / MICSTANDBY
FRANKIE / VOICESTANDBY
+

TRANSCRIPT

00 TURNS

Channel is quiet.

Open the mic.
Frankie will meet you here.

FIRST AUDIO —LIVE TRANSCRIPT
+
+

Make yourself heard.

Start the connection and allow your microphone. Headphones recommended.

+
BUILT TO LISTEN. NOT JUST TO ANSWER.FRANKIE / EXPERIMENTAL VOICE INTERFACE
+ +
+
+ + diff --git a/examples/realtime-audio/launch.mjs b/examples/realtime-audio/launch.mjs new file mode 100644 index 0000000000..a77da4ac4a --- /dev/null +++ b/examples/realtime-audio/launch.mjs @@ -0,0 +1,272 @@ +// Start an isolated local demo using the existing native and Buzz binaries. +import { parseArgs } from "node:util"; +import { access, mkdtemp, open, stat } from "node:fs/promises"; +import { constants } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { spawn } from "node:child_process"; +import { randomBytes } from "node:crypto"; +import net from "node:net"; +import { setTimeout as delay } from "node:timers/promises"; + +const directory = path.dirname(fileURLToPath(import.meta.url)); +const { values } = parseArgs({ + options: { + help: { type: "boolean" }, + native: { type: "string" }, + model: { type: "string" }, + agent: { + type: "string", + default: path.resolve(directory, "../../target/release/buzz-agent"), + }, + mcp: { + type: "string", + default: path.resolve(directory, "../../target/release/buzz-dev-mcp"), + }, + cwd: { type: "string" }, + voice: { type: "string" }, + "voice-text-file": { type: "string" }, + "voice-codes": { type: "string" }, + device: { type: "string", default: "gpu" }, + context: { type: "string", default: "131072" }, + "cache-type": { type: "string", default: "q4_0" }, + threads: { type: "string", default: "4" }, + port: { type: "string", default: "18796" }, + "native-port": { type: "string", default: "18793" }, + hours: { type: "string", default: "24" }, + }, +}); +if (values.help) { + console.log(`Usage: node examples/realtime-audio/launch.mjs --native PATH --model GGUF + --device cpu|gpu CPU, or the GPU backend compiled into llama.cpp + --context 131072 --cache-type q4_0|q8_0|f16 --threads 4 + --voice WAV Override the bundled voice with your reference + --voice-text-file TXT --voice-codes I32 Optional matching ICL conditioning + --cwd DIR Directory tools operate in (default: empty temporary directory) + --agent PATH --mcp PATH Override target/release Buzz binaries + --port 18796 --native-port 18793 --hours 24 +Open the printed URL in Chrome. Ctrl-C stops the demo. macOS and Linux only.`); +} else { + await main().catch((error) => { + console.error(error.message); + process.exitCode = 1; + }); +} + +async function available(port) { + const server = net.createServer(); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(port, "127.0.0.1", resolve); + }); + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); +} + +async function main() { + if (process.platform === "win32") + throw Error("This launcher supports macOS and Linux."); + if (!values.native || !values.model) + throw Error("--native and --model are required; see --help."); + if (!["cpu", "gpu"].includes(values.device)) + throw Error("--device must be cpu or gpu"); + if (!["q4_0", "q8_0", "f16"].includes(values["cache-type"])) + throw Error("invalid --cache-type"); + for (const [key, min, max] of [ + ["port", 1, 65535], + ["native-port", 1, 65535], + ["context", 4096, 262144], + ["threads", 1, 1024], + ]) { + if ( + !Number.isInteger(Number(values[key])) || + +values[key] < min || + +values[key] > max + ) + throw Error(`invalid --${key}`); + } + if (values.port === values["native-port"]) + throw Error("The two ports must differ."); + if (!(+values.hours > 0 && +values.hours <= 168)) + throw Error("--hours must be greater than 0 and at most 168"); + if ( + Boolean(values["voice-text-file"]) !== Boolean(values["voice-codes"]) || + (values["voice-codes"] && !values.voice) + ) { + throw Error( + "ICL conditioning requires --voice, --voice-text-file and --voice-codes together", + ); + } + for (const key of [ + "native", + "agent", + "mcp", + "model", + "voice", + "voice-text-file", + "voice-codes", + ]) { + if (!values[key]) continue; + values[key] = path.resolve(values[key]); + await access( + values[key], + ["native", "agent", "mcp"].includes(key) + ? constants.X_OK + : constants.R_OK, + ); + if (!(await stat(values[key])).isFile()) + throw Error(`--${key} must name a file`); + } + if (values.cwd && !(await stat(values.cwd)).isDirectory()) + throw Error("--cwd must be a directory"); + await available(+values.port); + await available(+values["native-port"]); + + const runtime = await mkdtemp(path.join(tmpdir(), "frankie-demo-")); + const cwd = values.cwd + ? path.resolve(values.cwd) + : await mkdtemp(path.join(runtime, "workspace-")); + const env = Object.fromEntries( + Object.entries(process.env).filter( + ([key]) => !/^(BUZZ_|OPENAI_|ANTHROPIC_|MCP_|FRANKIE_)/.test(key), + ), + ); + // The MCP adapter's shebang uses the same Node runtime as this launcher. + env.PATH = `${path.dirname(process.execPath)}${path.delimiter}${env.PATH || "/usr/bin:/bin"}`; + const credential = randomBytes(32).toString("hex"); + const children = [], + logs = []; + let stopping = false; + const stop = () => { + stopping = true; + }; + process.on("SIGINT", stop); + process.on("SIGTERM", stop); + const lifetime = setTimeout(stop, +values.hours * 3600000); + async function start( + command, + args, + environment, + logName, + stdout = "inherit", + ) { + const log = await open(path.join(runtime, logName), "wx", 0o600); + logs.push(log); + const child = spawn(command, args, { + cwd, + env: environment, + detached: true, + stdio: ["ignore", stdout, log.fd], + }); + children.push(child); + child.on("error", (error) => { + console.error(error.message); + process.exitCode = 1; + stop(); + }); + return child; + } + const ended = (child) => child.exitCode !== null || child.signalCode !== null; + function signal(child, name) { + if (!child.pid) return; + try { + process.kill(-child.pid, name); + } catch (error) { + if (error.code !== "ESRCH") console.error(error.message); + } + } + try { + console.log( + `Loading Frankie. Runtime logs: ${runtime}\nTool working directory: ${cwd}`, + ); + const args = [ + values.model, + values["native-port"], + "--device", + values.device, + "--ctx-size", + values.context, + "--cache-type", + values["cache-type"], + "--threads", + values.threads, + "--thinking", + "none", + ]; + for (const key of ["voice", "voice-text-file", "voice-codes"]) + if (values[key]) args.push(`--${key}`, values[key]); + const native = await start( + values.native, + args, + { ...env, FRANKIE_REALTIME_TOKEN: credential }, + "native.log", + "ignore", + ); + const deadline = Date.now() + 300000; + while (!stopping) { + if (ended(native)) + throw Error(`Native startup failed; inspect ${runtime}/native.log`); + try { + const response = await fetch( + `http://127.0.0.1:${values["native-port"]}/health`, + { signal: AbortSignal.timeout(1000) }, + ); + await response.body?.cancel(); + if (response.ok) break; + } catch { + /* Loading the GGUF precedes binding the health endpoint. */ + } + if (Date.now() > deadline) + throw Error(`Native readiness deadline; inspect ${runtime}/native.log`); + await delay(200); + } + if (stopping) return; + const web = await start( + process.execPath, + [path.join(directory, "server.mjs")], + { + ...env, + PORT: values.port, + FRANKIE_LIFETIME_HOURS: values.hours, + BUZZ_AGENT_BIN: values.agent, + BUZZ_MCP_BIN: path.join(directory, "mcp-shell.mjs"), + FRANKIE_DEV_MCP_BIN: values.mcp, + BUZZ_AGENT_PROVIDER: "openai", + OPENAI_COMPAT_API: "realtime", + OPENAI_COMPAT_MODEL: "frankie", + OPENAI_COMPAT_BASE_URL: `ws://127.0.0.1:${values["native-port"]}/v1/realtime`, + OPENAI_COMPAT_API_KEY: credential, + BUZZ_AGENT_NO_HINTS: "1", + BUZZ_AGENT_LLM_TIMEOUT_SECS: "240", + BUZZ_AGENT_REALTIME_OUTPUT: "audio", + BUZZ_AGENT_SYSTEM_PROMPT: + "You are Frankie, a helpful voice assistant. Be concise and conversational. " + + "For simple questions, answer directly. When asked to run a command, use the shell tool and wait for its result. " + + "Never claim a tool ran without its result. If permission is denied, acknowledge it and do not retry that action or an equivalent command unless the user asks again. Do not read tool syntax aloud. Speak plain text without Markdown.", + }, + "web.log", + ); + console.log( + "Open the URL below. Choose a thinking level, then connect. Tool calls ask for approval. Ctrl-C stops the demo.", + ); + while (!stopping) { + if (ended(native) || ended(web)) + throw Error(`A demo process exited; inspect ${runtime}`); + await delay(250); + } + } finally { + clearTimeout(lifetime); + // Let the adapter close ACP stdin first so Buzz can cancel tools and reap MCP groups. + for (const child of children.toReversed()) + if (!ended(child)) child.kill("SIGTERM"); + const deadline = Date.now() + 8000; + while (children.some((child) => !ended(child)) && Date.now() < deadline) + await delay(100); + for (const child of children) if (!ended(child)) signal(child, "SIGKILL"); + for (const log of logs) await log.close(); + process.off("SIGINT", stop); + process.off("SIGTERM", stop); + } +} diff --git a/examples/realtime-audio/mcp-shell.mjs b/examples/realtime-audio/mcp-shell.mjs new file mode 100755 index 0000000000..92dfb69e96 --- /dev/null +++ b/examples/realtime-audio/mcp-shell.mjs @@ -0,0 +1,141 @@ +#!/usr/bin/env node +// Isolated demo adapter: one advertised shell, executed by the existing dev MCP. +import { spawn } from "node:child_process"; +const binary = process.env.FRANKIE_DEV_MCP_BIN; +if (!binary) { + console.error("FRANKIE_DEV_MCP_BIN required"); + process.exit(1); +} +const child = spawn(binary, [], { stdio: ["pipe", "pipe", "inherit"] }); +const listing = new Set(), + calls = new Set(); +let ended = false; +function stop(code) { + if (ended) return; + ended = true; + for (const id of calls) + child.stdin.write( + `${JSON.stringify({ + jsonrpc: "2.0", + method: "notifications/cancelled", + params: { requestId: id, reason: "Voice demo closed" }, + })}\n`, + ); + child.stdin.end(); + setTimeout(() => { + child.kill("SIGKILL"); + process.exit(code); + }, 1500).unref(); +} +async function write(out, event) { + if (!out.write(`${JSON.stringify(event)}\n`)) + await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + cleanup(); + reject(Error("MCP consumer stalled")); + }, 2000); + const cleanup = () => { + clearTimeout(timer); + out.off("drain", drain); + out.off("error", error); + }; + const drain = () => { + cleanup(); + resolve(); + }; + const error = (e) => { + cleanup(); + reject(e); + }; + out.once("drain", drain); + out.once("error", error); + }); +} +async function relay(input, output, transform) { + let buffer = ""; + input.setEncoding("utf8"); + for await (const chunk of input) { + buffer += chunk; + if (buffer.length > 2 * 1024 * 1024) throw Error("MCP frame limit"); + for (;;) { + const i = buffer.indexOf("\n"); + if (i < 0) break; + const line = buffer.slice(0, i); + buffer = buffer.slice(i + 1); + if (line.trim()) { + const event = await transform(JSON.parse(line)); + if (event) await write(output, event); + } + } + } + if (buffer.trim()) throw Error("truncated MCP frame"); +} +relay(process.stdin, child.stdin, async (e) => { + if (e.method === "tools/list") { + if (listing.size >= 32) throw Error("MCP pending list limit"); + listing.add(e.id); + } + if (e.method === "tools/call" && e.params?.name !== "shell") { + await write(process.stdout, { + jsonrpc: "2.0", + id: e.id, + error: { code: -32602, message: "This voice demo exposes only shell" }, + }); + return null; + } + if (e.method === "tools/call") { + if (calls.size >= 32) throw Error("MCP pending call limit"); + calls.add(e.id); + } + return e; +}) + .then(() => stop(0)) + .catch((e) => { + console.error(e.message); + stop(1); + }); +relay(child.stdout, process.stdout, async (e) => { + calls.delete(e.id); + if (listing.delete(e.id) && e.result) { + const tool = e.result.tools?.find((t) => t.name === "shell"); + if (!tool) throw Error("dev MCP omitted shell"); + e.result = { + tools: [ + { + name: tool.name, + inputSchema: { + type: "object", + properties: { + command: { + type: "string", + description: "The bash command to execute.", + }, + }, + required: ["command"], + additionalProperties: false, + }, + description: + "Run a short shell command in the selected working directory and return stdout and stderr. Examples: uptime (system uptime), df -h / (disk usage), ls (list files). Use the exact command requested; keep commands read-only unless the user requests a change.", + }, + ], + }; + } + return e; +}).catch((e) => { + console.error(e.message); + stop(1); +}); +child.stdin.on("error", (e) => { + console.error(e.message); + stop(1); +}); +child.on("error", (e) => { + console.error(e.message); + stop(1); +}); +child.on("exit", (code) => { + ended = true; + process.exitCode = code ?? 1; + process.stdin.destroy(); +}); +for (const signal of ["SIGTERM", "SIGINT"]) process.on(signal, () => stop(0)); diff --git a/examples/realtime-audio/server.mjs b/examples/realtime-audio/server.mjs new file mode 100644 index 0000000000..9095fb7ded --- /dev/null +++ b/examples/realtime-audio/server.mjs @@ -0,0 +1,250 @@ +// Loopback ACP presentation adapter; provider credentials never reach the browser. +import http from "node:http"; +import { spawn } from "node:child_process"; +import { randomBytes, timingSafeEqual } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import path from "node:path"; + +const directory = path.dirname(fileURLToPath(import.meta.url)); +const port = Number(process.env.PORT || 18796); +const lifetimeHours = Number(process.env.FRANKIE_LIFETIME_HOURS || 24); +if (!Number.isInteger(port) || port < 1 || port > 65535) + throw Error("invalid port"); +if ( + !Number.isFinite(lifetimeHours) || + lifetimeHours <= 0 || + lifetimeHours > 168 +) + throw Error("lifetime must be 0..168 hours"); +const origin = `http://127.0.0.1:${port}`; +const token = randomBytes(24).toString("hex"); +const agentPath = + process.env.BUZZ_AGENT_BIN || + path.resolve(directory, "../../target/debug/buzz-agent"); +const mcpPath = process.env.BUZZ_MCP_BIN; +let agent, + consumer, + stopping = false; +const methods = new Set([ + "initialize", + "session/new", + "session/prompt", + "session/cancel", + ...["append", "playback", "interrupt", "close"].map( + (x) => `_buzz/unstable/realtime/${x}`, + ), +]); + +const closing = new WeakSet(); +function stopAgent(child) { + if ( + closing.has(child) || + child.exitCode !== null || + child.signalCode !== null + ) + return; + closing.add(child); + child.stdin.end(); + const timer = setTimeout(() => child.kill("SIGKILL"), 5000); + timer.unref(); + child.once("exit", () => clearTimeout(timer)); +} +function stop() { + if (stopping) return; + stopping = true; + if (agent) stopAgent(agent); + consumer?.end(); + server.close(); + setTimeout(() => { + agent?.kill("SIGKILL"); + process.exit(process.exitCode || 0); + }, 6000).unref(); +} +async function write(stream, bytes) { + if (stream.destroyed) throw Error("stream closed"); + if (stream.write(bytes)) return; + await new Promise((resolve, reject) => { + const cleanup = () => { + clearTimeout(timer); + stream.off("drain", drained); + stream.off("close", closed); + stream.off("error", closed); + }; + const drained = () => { + cleanup(); + resolve(); + }; + const closed = () => { + cleanup(); + reject(Error("stream closed")); + }; + const timer = setTimeout(() => { + cleanup(); + reject(Error("consumer stalled")); + }, 2000); + stream.once("drain", drained); + stream.once("close", closed); + stream.once("error", closed); + }); +} +const server = http.createServer(async (req, res) => { + res.setHeader("Cache-Control", "no-store"); + res.setHeader("X-Content-Type-Options", "nosniff"); + res.setHeader( + "Content-Security-Policy", + "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; connect-src 'self'; object-src 'none'; frame-ancestors 'none'", + ); + if ( + req.headers.host !== `127.0.0.1:${port}` || + (req.headers.origin && req.headers.origin !== origin) + ) { + res.writeHead(403).end(); + return; + } + const pathname = new URL(req.url, origin).pathname; + const staticFiles = { + "/": ["index.html", "text/html"], + "/capture-queue.mjs": ["capture-queue.mjs", "text/javascript"], + "/voice.mjs": ["voice.mjs", "text/javascript"], + "/ui.mjs": ["ui.mjs", "text/javascript"], + "/client.mjs": ["client.mjs", "text/javascript"], + "/audio-worklet.mjs": ["audio-worklet.mjs", "text/javascript"], + "/style.css": ["style.css", "text/css"], + }; + if (req.method === "GET" && staticFiles[pathname]) { + const [name, type] = staticFiles[pathname]; + res.setHeader("Content-Type", type); + res.end(readFileSync(path.join(directory, name))); + return; + } + const supplied = Buffer.from( + (req.headers.authorization || "").replace(/^Bearer /, ""), + ); + if ( + supplied.length !== token.length || + !timingSafeEqual(supplied, Buffer.from(token)) + ) { + res.writeHead(403).end(); + return; + } + if (pathname === "/events" && req.method === "GET") { + if (agent || stopping) { + res.writeHead(409).end(); + return; + } + const thinking = + new URL(req.url, origin).searchParams.get("thinking") || "none"; + if ( + !["none", "minimal", "low", "medium", "high", "xhigh", "max"].includes( + thinking, + ) + ) { + res.writeHead(400).end("invalid thinking level"); + return; + } + consumer = res; + res.setHeader("Content-Type", "application/x-ndjson"); + res.flushHeaders(); + agent = spawn(agentPath, [], { + env: { ...process.env, BUZZ_AGENT_THINKING_EFFORT: thinking }, + stdio: ["pipe", "pipe", "inherit"], + }); + const child = agent; + child.stdin.on("error", () => { + stopAgent(child); + res.end(); + }); + const finish = () => { + if (agent === child) { + agent = null; + consumer = null; + } + res.end(); + }; + child.on("error", (error) => { + console.error(error); + finish(); + }); + child.on("exit", finish); + res.on("close", () => stopAgent(child)); + (async () => { + let buffer = ""; + child.stdout.setEncoding("utf8"); + for await (const chunk of child.stdout) { + buffer += chunk; + if (buffer.length > 2 * 1024 * 1024) throw Error("ACP frame limit"); + for (;;) { + const index = buffer.indexOf("\n"); + if (index < 0) break; + const line = buffer.slice(0, index); + buffer = buffer.slice(index + 1); + JSON.parse(line); + await write(res, `${line}\n`); + } + } + })().catch((error) => { + console.error(error); + stopAgent(child); + res.end(); + }); + return; + } + if (pathname === "/rpc" && req.method === "POST" && agent) { + try { + let size = 0; + const chunks = []; + for await (const chunk of req) { + size += chunk.length; + if (size > 256 * 1024) throw Error("request limit"); + chunks.push(chunk); + } + const event = JSON.parse(Buffer.concat(chunks)); + if ( + event.jsonrpc !== "2.0" || + (event.method && !methods.has(event.method)) + ) + throw Error("method"); + if (event.method === "session/new") { + event.params = { + cwd: process.cwd(), + mcpServers: mcpPath + ? [ + { + name: "dev", + command: mcpPath, + args: [], + env: process.env.FRANKIE_DEV_MCP_BIN + ? [ + { + name: "FRANKIE_DEV_MCP_BIN", + value: process.env.FRANKIE_DEV_MCP_BIN, + }, + ] + : [], + }, + ] + : [], + }; + } + await write(agent.stdin, `${JSON.stringify(event)}\n`); + res.writeHead(204).end(); + } catch { + res.writeHead(400).end("invalid ACP request"); + } + return; + } + res.writeHead(404).end(); +}); +server.requestTimeout = 10000; +server.headersTimeout = 10000; +server.maxConnections = 8; +process.on("SIGTERM", stop); +process.on("SIGINT", stop); +setTimeout(stop, lifetimeHours * 60 * 60 * 1000).unref(); +server.on("error", (error) => { + console.error(error.message); + process.exitCode = 1; + stop(); +}); +server.listen(port, "127.0.0.1", () => console.log(`${origin}/#${token}`)); diff --git a/examples/realtime-audio/server.test.mjs b/examples/realtime-audio/server.test.mjs new file mode 100644 index 0000000000..0dda18dfa0 --- /dev/null +++ b/examples/realtime-audio/server.test.mjs @@ -0,0 +1,206 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtemp, writeFile, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { spawn } from "node:child_process"; +import { once } from "node:events"; +import net from "node:net"; +import { setTimeout as delay } from "node:timers/promises"; + +async function freePort() { + const server = net.createServer(); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const port = server.address().port; + await new Promise((resolve) => server.close(resolve)); + return port; +} +async function fixture(t, body) { + const dir = await mkdtemp(path.join(tmpdir(), "realtime-adapter-test-")); + t.after(() => rm(dir, { recursive: true, force: true })); + const binary = path.join(dir, "fixture.mjs"); + await writeFile(binary, `#!${process.execPath}\n${body}`, { mode: 0o700 }); + return { dir, binary }; +} +function run(t, file, args = [], env = {}) { + const child = spawn( + process.execPath, + [new URL(file, import.meta.url).pathname, ...args], + { + env: { ...process.env, ...env }, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + let stdout = "", + stderr = ""; + child.stdout.on("data", (bytes) => { + stdout += bytes; + }); + child.stderr.on("data", (bytes) => { + stderr += bytes; + }); + t.after(async () => { + if (child.exitCode === null && child.signalCode === null) { + child.kill("SIGTERM"); + const timer = setTimeout(() => child.kill("SIGKILL"), 9000); + await once(child, "exit"); + clearTimeout(timer); + } + }); + return { child, output: () => stdout, errors: () => stderr }; +} +async function until(check) { + const deadline = Date.now() + 10000; + while (!(await check())) { + if (Date.now() > deadline) throw Error("test deadline"); + await delay(20); + } +} + +test("adapter fences origin/auth, preserves approval, constrains tools, and reaps on disconnect", { + timeout: 20000, +}, async (t) => { + const { dir, binary } = await fixture( + t, + ` +import {createInterface} from 'node:readline'; import {writeFileSync} from 'node:fs'; +const send = value => console.log(JSON.stringify(value)); +const lines = createInterface({input:process.stdin}); +lines.on('line', line => { + const event = JSON.parse(line); + if (event.method === 'session/new') send({jsonrpc:'2.0',id:event.id,result:event.params}); + else if (event.method === 'session/prompt') send({jsonrpc:'2.0',id:'approval',method:'session/request_permission',params:{options:[{kind:'allow_once',optionId:'allow'}]}}); + else send({jsonrpc:'2.0',id:'observed',result:event}); +}); +lines.on('close',()=>writeFileSync(new URL('eof',import.meta.url),'closed')); +`, + ); + const port = await freePort(); + const app = run(t, "server.mjs", [], { + PORT: String(port), + BUZZ_AGENT_BIN: binary, + BUZZ_MCP_BIN: binary, + FRANKIE_TOOLS_PREAUTHORIZED: "1", + }); + await until(() => app.output().includes("/#")); + const url = new URL(app.output().trim()), + origin = url.origin; + const headers = { + Authorization: `Bearer ${url.hash.slice(1)}`, + "Content-Type": "application/json", + }; + assert.equal((await fetch(origin)).status, 200); + assert.equal((await fetch(`${origin}/events`)).status, 403); + assert.equal( + ( + await fetch(`${origin}/events`, { + headers: { ...headers, Origin: "https://example.com" }, + }) + ).status, + 403, + ); + assert.equal( + (await fetch(`${origin}/events?thinking=invalid`, { headers })).status, + 400, + ); + const abort = new AbortController(); + t.after(() => abort.abort()); + const events = await fetch(`${origin}/events?thinking=high`, { + headers, + signal: abort.signal, + }); + assert.equal(events.status, 200); + const reader = events.body.getReader(), + decoder = new TextDecoder(); + let buffer = ""; + async function next() { + while (!buffer.includes("\n")) { + const { value, done } = await reader.read(); + assert.equal(done, false); + buffer += decoder.decode(value, { stream: true }); + } + const index = buffer.indexOf("\n"), + line = buffer.slice(0, index); + buffer = buffer.slice(index + 1); + return JSON.parse(line); + } + const rpc = (event) => + fetch(`${origin}/rpc`, { + method: "POST", + headers, + body: JSON.stringify(event), + }); + assert.equal((await fetch(`${origin}/events`, { headers })).status, 409); + assert.equal( + (await rpc({ jsonrpc: "2.0", id: 1, method: "untrusted" })).status, + 400, + ); + assert.equal( + ( + await rpc({ + jsonrpc: "2.0", + id: 2, + method: "session/new", + params: { cwd: "/untrusted", mcpServers: [{ command: "/untrusted" }] }, + }) + ).status, + 204, + ); + const session = (await next()).result; + assert.equal(session.cwd, process.cwd()); + assert.equal(session.mcpServers[0].command, binary); + assert.equal( + (await rpc({ jsonrpc: "2.0", id: 3, method: "session/prompt" })).status, + 204, + ); + assert.equal((await next()).method, "session/request_permission"); + const denial = { + jsonrpc: "2.0", + id: "approval", + result: { outcome: { outcome: "cancelled" } }, + }; + await rpc(denial); + assert.deepEqual((await next()).result, denial); + abort.abort(); + await until(async () => { + try { + return (await readFile(path.join(dir, "eof"), "utf8")) === "closed"; + } catch { + return false; + } + }); +}); + +test("launcher rejects invalid conditioning and occupied ports before model startup", { + timeout: 10000, +}, async (t) => { + const { binary, dir } = await fixture(t, "throw Error('must not start');"); + const model = path.join(dir, "model.gguf"); + await writeFile(model, "fixture"); + const common = [ + "--native", + binary, + "--agent", + binary, + "--mcp", + binary, + "--model", + model, + ]; + const invalid = run(t, "launch.mjs", [...common, "--voice-codes", model]); + assert.equal((await once(invalid.child, "exit"))[0], 1); + assert.match(invalid.errors(), /ICL conditioning requires/); + const server = net.createServer(); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + t.after(() => new Promise((resolve) => server.close(resolve))); + const occupied = run(t, "launch.mjs", [ + ...common, + "--port", + String(server.address().port), + ]); + assert.equal((await once(occupied.child, "exit"))[0], 1); + assert.match(occupied.errors(), /EADDRINUSE/); + assert.doesNotMatch(occupied.errors(), /must not start/); +}); diff --git a/examples/realtime-audio/style.css b/examples/realtime-audio/style.css new file mode 100644 index 0000000000..200f3fa6df --- /dev/null +++ b/examples/realtime-audio/style.css @@ -0,0 +1,988 @@ +:root { + --paper: #eeece4; + --ink: #242822; + --muted: #666b60; + --line: #c9cbbf; + --accent: #d13d21; + --dark: #212821; + --mono: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; +} +* { + box-sizing: border-box; +} +body { + margin: 0; + background: var(--paper); + color: var(--ink); + font-family: -apple-system, BlinkMacSystemFont, "Helvetica Neue", sans-serif; + -webkit-font-smoothing: antialiased; +} +button { + font: inherit; + cursor: pointer; +} +button:disabled { + cursor: wait; +} +button:focus-visible, +summary:focus-visible { + outline: 3px solid var(--accent); + outline-offset: 5px; +} +::selection { + background: var(--accent); + color: white; +} +.shell { + max-width: 1800px; + margin: auto; + padding: 28px 44px 20px; +} +.mono { + font: 11px var(--mono); + letter-spacing: 0.12em; + text-transform: uppercase; +} +.top { + display: flex; + align-items: center; + justify-content: space-between; + border-bottom: 1px solid var(--ink); + padding-bottom: 20px; +} +.brand { + display: flex; + align-items: center; + gap: 14px; + font-size: 13px; + font-weight: 650; + letter-spacing: 0.06em; +} +.insignia { + width: 27px; + height: 27px; + background: var(--accent); + clip-path: polygon( + 0 0, + 100% 0, + 100% 27%, + 29% 27%, + 29% 44%, + 83% 44%, + 83% 70%, + 29% 70%, + 29% 100%, + 0 100% + ); +} +.top-right { + display: flex; + gap: 35px; + align-items: center; +} +.status-dot { + display: inline-block; + width: 6px; + height: 6px; + border-radius: 50%; + background: var(--muted); + margin-right: 8px; +} +body[data-connected="true"] .status-dot { + background: var(--accent); +} +.hero { + display: grid; + grid-template-columns: 1fr auto; + align-items: end; + padding: 35px 0 27px; +} +.eyebrow { + color: var(--accent); + margin: 0 0 9px; +} +.hero h1 { + font-size: clamp(64px, 8vw, 120px); + font-weight: 450; + letter-spacing: -0.075em; + line-height: 0.92; + margin: 0 0 17px; +} +.hero p { + font-size: 16px; + color: var(--muted); + margin: 0; +} +.hero-side { + text-align: right; + padding-bottom: 4px; +} +.serial { + font: 54px var(--mono); + letter-spacing: -0.1em; + font-weight: 200; +} +.hero-side p { + font: 10px var(--mono); + letter-spacing: 0.16em; + margin-top: 10px; +} +.deck { + display: grid; + grid-template-columns: minmax(0, 1.32fr) minmax(340px, 1fr); + gap: 22px; +} +.instrument { + background: var(--dark); + color: #edeee2; + border-radius: 3px; + position: relative; + overflow: hidden; + min-height: 440px; + display: flex; + flex-direction: column; +} +.instrument:before { + content: ""; + position: absolute; + inset: 0; + background-image: + linear-gradient(#a0b4910b 1px, transparent 1px), + linear-gradient(90deg, #a0b4910b 1px, transparent 1px); + background-size: 32px 32px; + pointer-events: none; +} +.instrument-head { + padding: 22px 24px; + display: flex; + justify-content: space-between; + z-index: 1; + color: #a8b19d; +} +.instrument-head b { + color: #e8eddf; + font-weight: 400; +} +.scope { + position: relative; + flex: 1; + min-height: 260px; + display: grid; + place-items: center; +} +.scope canvas { + position: absolute; + inset: 0; + width: 100%; + height: 100%; +} +.core-label { + position: relative; + text-align: center; + pointer-events: none; +} +.core-label .mono { + color: #afb9a6; + font-size: 10px; +} +.core-label strong { + display: block; + font-size: 36px; + font-weight: 400; + letter-spacing: -0.055em; + margin: 7px 0; +} +.core-label .sub { + font: 10px var(--mono); + color: #aeb7a7; + letter-spacing: 0.08em; +} +.axis { + position: absolute; + left: 24px; + bottom: 20px; + color: #9aa58f; + font: 10px var(--mono); +} +.axis.right { + left: auto; + right: 24px; +} +.phases { + display: grid; + grid-template-columns: repeat(3, 1fr); + padding: 0 24px 22px; + gap: 10px; +} +.phase { + border-top: 2px solid #52604d; + padding-top: 12px; + color: #9da993; + font: 10px var(--mono); + letter-spacing: 0.08em; +} +.phase span { + margin-right: 8px; + opacity: 0.65; +} +.phase.active { + border-color: #ef785c; + color: #ffab94; +} +.telemetry { + display: grid; + grid-template-columns: 1fr 1fr; + border-top: 1px solid #4a5544; + position: relative; +} +.meter { + padding: 18px 24px; +} +.meter + .meter { + border-left: 1px solid #4a5544; +} +.meter-top { + display: flex; + justify-content: space-between; + color: #b6c0aa; + font: 10px var(--mono); + letter-spacing: 0.08em; +} +.meter canvas { + display: block; + width: 100%; + height: 24px; + margin-top: 8px; +} +.conversation { + border: 1px solid var(--line); + border-radius: 3px; + display: flex; + flex-direction: column; + min-width: 0; + height: 440px; +} +.conversation-head { + display: flex; + justify-content: space-between; + align-items: center; + padding: 21px 23px; + border-bottom: 1px solid var(--line); +} +.conversation-head h2 { + font: 11px var(--mono); + letter-spacing: 0.12em; + margin: 0; +} +.conversation-head span { + font: 10px var(--mono); + color: var(--muted); +} +#log { + overflow: auto; + padding: 22px 24px; + flex: 1; + scrollbar-width: thin; +} +.empty { + height: 100%; + display: flex; + flex-direction: column; + justify-content: center; + align-items: flex-start; +} +.empty-mark { + font-size: 49px; + font-weight: 200; + color: var(--accent); + margin-bottom: 20px; + line-height: 1; +} +.empty h3 { + font-size: 30px; + font-weight: 400; + letter-spacing: -0.045em; + margin: 0 0 12px; +} +.empty p { + font-size: 14px; + line-height: 1.7; + color: var(--muted); + max-width: 280px; + margin: 0; +} +.entry { + margin-bottom: 24px; + animation: arrive 0.25s ease-out; +} +.entry .label { + font: 10px var(--mono); + letter-spacing: 0.12em; + color: var(--muted); + margin-bottom: 8px; +} +.entry .label.frankie { + color: var(--accent); +} +.entry p { + font-size: 18px; + letter-spacing: -0.02em; + line-height: 1.5; + margin: 0; + white-space: pre-wrap; + overflow-wrap: anywhere; +} +.entry.user p { + font-size: 14px; + color: var(--muted); +} +.turn { + border-bottom: 1px solid var(--line); + padding-bottom: 5px; + margin-bottom: 22px; +} +.note { + font: 11px / 1.6 var(--mono); + color: var(--accent); + border-left: 2px solid var(--accent); + padding: 8px 12px; + margin: 12px 0; + white-space: pre-wrap; + overflow-wrap: anywhere; +} +details { + font: 10px / 1.7 var(--mono); + color: var(--muted); + margin-bottom: 15px; + overflow-wrap: anywhere; +} +summary { + cursor: pointer; +} +.conversation-foot { + border-top: 1px solid var(--line); + padding: 12px 23px; + display: flex; + justify-content: space-between; + font: 10px var(--mono); + color: var(--muted); +} +.controlbar { + display: flex; + justify-content: space-between; + align-items: center; + padding: 25px 0; + border-bottom: 1px solid var(--ink); + gap: 24px; +} +.control-copy { + display: flex; + align-items: center; + gap: 17px; +} +.mic-icon { + width: 40px; + height: 40px; + border: 1px solid var(--line); + border-radius: 50%; + display: grid; + place-items: center; +} +.control-copy h2 { + font-size: 16px; + font-weight: 500; + margin: 0 0 5px; + letter-spacing: -0.02em; +} +.control-copy p { + font-size: 12px; + color: var(--muted); + margin: 0; + max-width: 550px; +} +.actions { + display: flex; + align-items: center; + gap: 15px; +} +#go { + background: var(--accent); + color: white; + border: 0; + border-radius: 3px; + padding: 17px 25px; + min-width: 190px; + display: flex; + justify-content: space-between; + gap: 24px; + font-size: 13px; +} +#go:hover { + background: #b63219; +} +#go:disabled { + opacity: 0.6; +} +.footer { + display: flex; + justify-content: space-between; + padding: 17px 0 0; + color: var(--muted); + font: 10px var(--mono); + letter-spacing: 0.09em; +} +.footer strong { + font-weight: 400; + color: var(--ink); +} +.sr-only { + position: absolute; + width: 1px; + height: 1px; + padding: 0; + overflow: hidden; + clip: rect(0, 0, 0, 0); + white-space: nowrap; + border: 0; +} +@keyframes arrive { + from { + opacity: 0; + transform: translateY(5px); + } + to { + opacity: 1; + transform: none; + } +} +@media (min-height: 900px) { + .instrument, + .conversation { + min-height: 510px; + height: 510px; + } + .hero { + padding-top: 45px; + padding-bottom: 35px; + } +} +@media (max-width: 900px) { + .shell { + padding: 22px; + } + .deck { + grid-template-columns: 1fr 1fr; + gap: 14px; + } + .top-right .unit { + display: none; + } + .hero h1 { + font-size: 84px; + } + .serial { + font-size: 42px; + } + .instrument, + .conversation { + min-height: 410px; + height: 410px; + } + .instrument-head { + padding: 20px 16px; + } + .meter { + padding: 15px; + } + .control-copy p { + max-width: 320px; + } +} +@media (max-width: 650px) { + .shell { + padding: 18px; + } + .top-right { + font-size: 9px; + } + .hero { + padding: 28px 0 24px; + } + .hero h1 { + font-size: 76px; + } + .hero p { + font-size: 13px; + } + .hero-side { + display: none; + } + .deck { + grid-template-columns: 1fr; + } + .instrument { + min-height: 400px; + } + .conversation { + height: 360px; + min-height: 360px; + } + .controlbar { + align-items: stretch; + flex-direction: column; + } + .actions, + #go { + width: 100%; + } + .footer { + font-size: 8px; + gap: 15px; + } + .core-label strong { + font-size: 32px; + } +} +@media (prefers-reduced-motion: reduce) { + .entry { + animation: none; + } +} + +/* Full-viewport flight surface. No landing-page furniture. */ +:root { + --paper: #080b0f; + --ink: #dce8ed; + --muted: #84929c; + --line: #263139; + --accent: #72ebdf; + --dark: #080b0f; +} +html, +body { + height: 100%; + overflow: hidden; +} +.shell { + height: 100dvh; + max-width: none; + padding: 0; + position: relative; + background: + radial-gradient(ellipse at 37% 49%, #142328 0, transparent 49%), #080b0f; +} +.top { + height: 76px; + padding: 0 32px; + border-bottom: 1px solid #253137; + position: absolute; + inset: 0 0 auto; + z-index: 3; +} +.brand { + font-size: 18px; + letter-spacing: -0.035em; + font-weight: 450; + gap: 12px; +} +.insignia { + height: 18px; + width: 18px; +} +.top-right { + font-size: 10px; + gap: 28px; +} +.top-right .unit { + color: #6c828c; +} +.hero, +.footer { + display: none; +} +.deck { + display: block; + position: absolute; + inset: 76px 0 90px; +} +.instrument { + position: absolute; + inset: 0 350px 0 0; + height: auto; + min-height: 0; + background: transparent; + border-radius: 0; + overflow: hidden; +} +.instrument:before { + background-size: 64px 64px; + opacity: 0.25; + mask-image: radial-gradient(ellipse, black, transparent 70%); +} +.instrument-head { + padding: 24px 30px; + font-size: 9px; + color: #748a94; + position: absolute; + inset: 0 0 auto; + z-index: 1; +} +.instrument-head b { + color: #8faab4; +} +.scope { + position: absolute; + inset: 0; + min-height: 0; +} +.scope .axis { + bottom: 104px; + color: #6a828a; + font-size: 9px; +} +.scope .axis.right { + bottom: 104px; +} +.core-label strong { + font-size: clamp(38px, 4.2vw, 66px); + letter-spacing: -0.055em; + font-weight: 250; + margin: 10px 0; + color: #e7f9f8; +} +.core-label .mono { + color: #7b969e; + letter-spacing: 0.24em; + font-size: 9px; +} +.core-label .sub { + color: #6b9eaa; + font-size: 9px; + letter-spacing: 0.15em; +} +.phases { + position: absolute; + left: 50%; + bottom: 138px; + transform: translateX(-50%); + padding: 0; + width: 300px; + gap: 22px; +} +.phase { + font-size: 9px; + color: #66808a; + border-top: 1px solid #30434b; + padding-top: 9px; +} +.phase.active { + color: #9ef7ec; + border-color: #72ebdf; +} +.telemetry { + position: absolute; + inset: auto 30px 25px; + border: 0; + gap: 35px; +} +.meter { + padding: 0; +} +.meter + .meter { + border: 0; +} +.meter-top { + font-size: 9px; + color: #67828c; +} +.meter canvas { + height: 19px; + margin-top: 10px; +} +.conversation { + position: absolute; + inset: 0 0 0 auto; + width: 350px; + height: auto; + min-height: 0; + border: 0; + border-left: 1px solid #253137; + background: #080c1080; + border-radius: 0; +} +.conversation-head { + height: 63px; + padding: 0 23px; + border-bottom: 0; +} +.conversation-head h2 { + font-size: 9px; + color: #8298a2; + letter-spacing: 0.17em; +} +.conversation-head span { + font-size: 9px; + color: #78949f; +} +#log { + padding: 24px 23px; +} +.empty { + justify-content: flex-end; + padding-bottom: 34px; +} +.empty-mark { + font-size: 25px; + opacity: 0.65; + margin-bottom: 18px; +} +.empty h3 { + font-size: 20px; + letter-spacing: -0.035em; + color: #a8c1ca; +} +.empty p { + font-size: 12px; + line-height: 1.8; + color: #657f8a; +} +.entry p { + font-size: 17px; + line-height: 1.55; + color: #d2e2e7; +} +.entry.user p { + font-size: 13px; + color: #879da7; +} +.entry .label { + color: #6a838e; + font-size: 9px; +} +.entry .label.frankie { + color: #72ebdf; +} +.turn { + border-color: #203039; +} +.note { + color: #87c5c0; + font-size: 10px; + border-color: #518c87; + background: #16282655; +} +.conversation-foot { + padding: 18px 23px; + border-color: #253137; + font-size: 9px; + color: #77929d; +} +.controlbar { + position: absolute; + inset: auto 0 0; + height: 90px; + padding: 0 32px; + border-top: 1px solid #253137; + border-bottom: 0; + background: #080b0fd9; + z-index: 4; +} +.mic-icon { + width: 34px; + height: 34px; + border-color: #334751; + color: #72ebdf; +} +.control-copy { + gap: 13px; +} +.control-copy h2 { + font-size: 13px; + color: #b2c7ce; +} +.control-copy p { + font-size: 11px; + color: #738b95; +} +.actions { + gap: 12px; +} +#go { + background: #72ebdf; + color: #071310; + padding: 14px 20px; + font-size: 12px; + border-radius: 2px; + min-width: 175px; +} +#go:hover { + background: #abfff5; +} +#fullscreen { + border: 1px solid #334751; + color: #b1c7ce; + background: transparent; + width: 42px; + height: 42px; + border-radius: 2px; + font-size: 20px; +} +#fullscreen:hover { + border-color: #72ebdf; +} +@media (min-height: 900px) { + .scope .axis { + bottom: 108px; + } + .phases { + bottom: 148px; + } +} +@media (max-width: 1000px) { + .instrument { + right: 290px; + } + .conversation { + width: 290px; + } + .top { + padding: 0 24px; + } + .phases { + width: 240px; + } + .scope .axis { + display: none; + } + .telemetry { + left: 24px; + right: 24px; + gap: 20px; + } + .controlbar { + padding: 0 24px; + } + .control-copy p { + max-width: 420px; + } +} +@media (max-width: 650px) { + .top { + height: 62px; + padding: 0 20px; + } + .brand { + font-size: 18px; + } + .top-right { + font-size: 9px; + } + .deck { + inset: 62px 0 112px; + } + .instrument { + inset: 0 0 36%; + } + .conversation { + inset: 64% 0 0; + width: auto; + border-left: 0; + border-top: 1px solid #253137; + background: #080c10; + } + .instrument-head { + padding: 17px 20px; + } + .scope { + inset: 0; + } + .core-label strong { + font-size: 34px; + } + .core-label .sub { + font-size: 8px; + } + .phases, + .telemetry, + .empty, + .conversation-foot { + display: none; + } + .conversation-head { + height: 35px; + padding: 0 20px; + } + #log { + padding: 6px 20px; + } + .entry { + margin-bottom: 14px; + } + .entry p { + font-size: 15px; + } + .controlbar { + height: 112px; + padding: 12px 20px; + gap: 8px; + flex-direction: column; + align-items: stretch; + } + .control-copy { + display: none; + } + .actions { + height: 100%; + gap: 12px; + } + #go { + flex: 1; + width: auto; + } + #fullscreen { + flex-shrink: 0; + } + .top-right .unit { + display: none; + } +} + +#permission { + position: fixed; + z-index: 10; + inset: 15% 10% auto; + padding: 24px; + background: #142328; + border: 1px solid #72ebdf; + max-height: 70vh; + overflow: auto; +} +#permission pre { + white-space: pre-wrap; + overflow-wrap: anywhere; +} +#permission button { + padding: 12px; +} +[hidden] { + /* biome-ignore lint/complexity/noImportantStyles: hidden must override the dialog layout. */ + display: none !important; +} + +.thinking-control { + display: flex; + align-items: center; + gap: 0.5rem; + font-size: 0.75rem; +} +.thinking-control select { + color: inherit; + background: transparent; + border: 1px solid currentColor; + border-radius: 6px; + padding: 0.45rem; + max-width: 8rem; +} +.thinking-control select:disabled { + opacity: 0.5; +} diff --git a/examples/realtime-audio/ui.mjs b/examples/realtime-audio/ui.mjs new file mode 100644 index 0000000000..13f20ef669 --- /dev/null +++ b/examples/realtime-audio/ui.mjs @@ -0,0 +1,240 @@ +const $ = (id) => document.getElementById(id), + st = $("st"), + log = $("log"); +let turns = 0, + micAnalyser, + voiceAnalyser, + connected = false; +const phases = { + LISTEN: ["Listening.", "TAKE YOUR TIME", "listen"], + SPEC: ["Thinking.", "FOLLOWING YOUR THOUGHT", "think"], + REPLY: ["Speaking.", "A THOUGHT TAKES VOICE", "speak"], + PLAY: ["Speaking.", "FINISHING THE THOUGHT", "speak"], + OFF: ["Ready.", "YOUR VOICE STARTS HERE", ""], +}; +function setState(state) { + const v = phases[state] || phases.OFF; + document.body.dataset.state = state; + st.textContent = v[0]; + $("state-sub").textContent = v[1]; + for (const p of ["listen", "think", "speak"]) + $(`phase-${p}`).classList.toggle("active", p === v[2]); + $("announcement").textContent = `Frankie: ${v[0]}`; + $("voice-label").textContent = v[2] === "speak" ? "OUTPUT" : "STANDBY"; +} +function clearEmpty() { + $("empty")?.remove(); +} +function follow() { + log.scrollTop = log.scrollHeight; +} +function entry(who, text) { + const d = document.createElement("div"); + d.className = `entry ${who === "You" ? "user" : ""}`; + const l = document.createElement("div"); + l.className = `label ${who === "Frankie" ? "frankie" : ""}`; + l.textContent = who; + const p = document.createElement("p"); + p.textContent = text; + d.append(l, p); + return d; +} +function newTurn() { + clearEmpty(); + const d = document.createElement("article"); + d.className = "turn"; + log.append(d); + while (log.children.length > 60) log.firstElementChild.remove(); + return d; +} +function note(text) { + clearEmpty(); + const d = document.createElement("div"); + d.className = "note"; + d.textContent = text; + log.append(d); + while (log.children.length > 60) log.firstElementChild.remove(); + follow(); +} + +$("fullscreen").onclick = async () => { + try { + if (document.fullscreenElement) await document.exitFullscreen(); + else if (document.documentElement.requestFullscreen) + await document.documentElement.requestFullscreen(); + else + $("help").textContent = + "Use your browser’s fullscreen command for browser fullscreen."; + } catch { + $("help").textContent = + "Use your browser’s fullscreen command for browser fullscreen."; + } +}; + +const reduced = matchMedia("(prefers-reduced-motion: reduce)"), + canvases = ["scope", "mic-meter", "voice-meter"].map((id) => $(id)), + pens = canvases.map((c) => c.getContext("2d")), + micData = new Uint8Array(256).fill(128), + voiceData = new Uint8Array(256).fill(128); +let micLevel = 0, + voiceLevel = 0; +function resize() { + const d = Math.min(devicePixelRatio || 1, 2); + canvases.forEach((c, i) => { + const b = c.getBoundingClientRect(); + c.width = Math.round(b.width * d); + c.height = Math.round(b.height * d); + pens[i].setTransform(d, 0, 0, d, 0, 0); + }); +} +new ResizeObserver(resize).observe($("scope").parentElement); +window.addEventListener("resize", resize); +resize(); +function level(analyser, data) { + if (!analyser) return 0; + analyser.getByteTimeDomainData(data); + let v = 0; + for (const x of data) v += ((x - 128) / 128) ** 2; + return Math.min(1, Math.sqrt(v / data.length) * 5); +} +function draw(now) { + if (!document.hidden) { + micLevel = 0.7 * micLevel + 0.3 * level(micAnalyser, micData); + voiceLevel = 0.7 * voiceLevel + 0.3 * level(voiceAnalyser, voiceData); + const c = canvases[0], + p = pens[0], + w = c.clientWidth, + h = c.clientHeight, + x = w / 2, + y = h / 2, + r = Math.min(h * 0.3, w * 0.31), + t = reduced.matches ? 0 : now * 0.0002; + p.clearRect(0, 0, w, h); + p.strokeStyle = "#34505c"; + p.lineWidth = 0.6; + p.beginPath(); + p.moveTo(20, y); + p.lineTo(x - r - 18, y); + p.moveTo(x + r + 18, y); + p.lineTo(w - 20, y); + p.moveTo(x, 10); + p.lineTo(x, y - r - 12); + p.moveTo(x, y + r + 12); + p.lineTo(x, h - 10); + p.stroke(); + for (let k = 0; k < 120; k++) { + const a = (k * Math.PI) / 60 - Math.PI / 2, + major = k % 10 === 0; + const amp = voiceLevel || micLevel; + const len = major + ? 11 + : 4 + amp * 12 * (0.5 + 0.5 * Math.sin(k * 0.8 + t * 20)); + p.strokeStyle = k < Math.round(amp * 120) ? "#72ebdf" : "#34505c"; + p.lineWidth = major ? 1.5 : 1; + p.beginPath(); + p.moveTo(x + Math.cos(a) * r, y + Math.sin(a) * r); + p.lineTo(x + Math.cos(a) * (r + len), y + Math.sin(a) * (r + len)); + p.stroke(); + } + p.strokeStyle = "#65cfc8"; + p.lineWidth = 1; + p.beginPath(); + for (let k = 0; k <= 180; k++) { + const a = (k * Math.PI) / 90; + const data = voiceLevel > 0.02 ? voiceData : micData; + const v = (data[Math.floor((k * 255) / 180)] - 128) / 128; + const rr = r - 28 + (connected ? v * 35 : 0); + const xx = x + Math.cos(a) * rr, + yy = y + Math.sin(a) * rr; + if (k === 0) p.moveTo(xx, yy); + else p.lineTo(xx, yy); + } + p.stroke(); + p.strokeStyle = "#315762"; + p.lineWidth = 0.5; + p.beginPath(); + p.arc(x, y, r - 13, 0, Math.PI * 2); + p.stroke(); + p.strokeStyle = "#72ebdf"; + p.lineWidth = 2; + p.beginPath(); + p.arc(x, y, r + 20, t, t + 0.6 + voiceLevel * 2); + p.stroke(); + p.strokeStyle = "#a2e8eb"; + p.lineWidth = 1; + p.beginPath(); + p.arc(x, y, r - 19, -t + Math.PI, -t + Math.PI + 0.35); + p.stroke(); + [micLevel, voiceLevel].forEach((v, j) => { + const cc = canvases[j + 1], + q = pens[j + 1], + ww = cc.clientWidth, + hh = cc.clientHeight, + n = 42; + q.clearRect(0, 0, ww, hh); + for (let i = 0; i < n; i++) { + q.fillStyle = i < Math.round(v * n) ? "#72ebdf" : "#233943"; + q.fillRect( + (i * ww) / n, + hh - 6 - (i / n) * 14, + Math.max(1, ww / n - 3), + 6 + (i / n) * 14, + ); + } + }); + } + requestAnimationFrame(draw); +} +requestAnimationFrame(draw); + +export function state(s) { + setState(s); +} +export function analyzers(mic, voice) { + micAnalyser = mic; + voiceAnalyser = voice; +} +export function connection(value, message) { + if (value && !connected) { + // Provider item IDs restart with each fresh agent session. + log.replaceChildren(); + turns = 0; + $("turn-count").textContent = "00 TURNS"; + $("latency").textContent = "FIRST SOUND —"; + } + connected = value; + document.body.dataset.connected = String(value); + $("connection").textContent = value ? "CONNECTED" : "OFFLINE"; + $("signal").textContent = value + ? "BUZZ AGENT → LLAMA.CPP" + : "AWAITING CONNECTION"; + $("mic-label").textContent = value ? "LIVE · 24 kHz" : "STANDBY"; + $("go-label").textContent = value ? "End conversation" : "Start talking"; + $("go").disabled = false; + $("help").textContent = message; + $("instruction").textContent = value + ? "Ready to listen." + : "Make yourself heard."; + if (!value) setState("OFF"); +} +export function transcript(text) { + const row = newTurn(); + row.append(entry("Frankie", text)); + turns++; + $("turn-count").textContent = `${String(turns).padStart(2, "0")} TURNS`; + follow(); +} +export { note }; + +export function userTranscript(text, itemId) { + let row = [...log.children].find((r) => r.dataset.inputId === itemId); + if (!row) { + row = newTurn(); + row.dataset.inputId = itemId; + } + row.replaceChildren(entry("You", text || "[No speech recognized]")); + follow(); +} +export function removeInput(itemId) { + [...log.children].find((r) => r.dataset.inputId === itemId)?.remove(); +} diff --git a/examples/realtime-audio/voice.mjs b/examples/realtime-audio/voice.mjs new file mode 100644 index 0000000000..8e14e753c6 --- /dev/null +++ b/examples/realtime-audio/voice.mjs @@ -0,0 +1,528 @@ +import { CaptureQueue } from "./capture-queue.mjs"; +export async function openVoice(token, ui, options = {}) { + const pending = new Map(), + stops = new Map(); + const eventsAbort = new AbortController(); + let serial = 0, + sid, + stream, + context, + node, + tracks, + permission, + closed = false; + let sequence = 0, + captureQueue = new CaptureQueue(), + systemQueue = new CaptureQueue(), + sending = false, + playbackPending = false, + latestPlayback; + let captureOrigin, lastSpeechMs, microphone; + let capturedSamples = 0, + sentSamples = 0, + captureEnergy = 0, + capturePeak = 0, + lastCaptureAt = 0, + lastDiagnostic = 0, + diagnosticSamples = 0; + let playbackGaps = 0, + playbackGapSamples = 0, + maxPlaybackGapSamples = 0; + let lastEvent = performance.now(), + active = false, + suppressPlayback = false; + const blocked = new Set(); + const evidence = []; + // Test instrumentation observes the actual media client; it does not replace its queues or clock. + ui.evidence(evidence); + function status(text) { + ui.status(text); + } + function record(type, detail = {}) { + evidence.push({ time: performance.now(), type, ...detail }); + if (evidence.length > 20000) evidence.shift(); + ui.event(type, detail); + } + async function rpc(event) { + const response = await fetch("/rpc", { + method: "POST", + headers: { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(event), + signal: AbortSignal.timeout(10000), + }); + if (!response.ok) throw Error(`ACP request rejected (${response.status})`); + } + function request(method, params, lifetime = 10000) { + const id = ++serial; + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + pending.delete(id); + reject(Error(`${method} deadline`)); + }, lifetime); + pending.set(id, { resolve, reject, timer }); + rpc({ jsonrpc: "2.0", id, method, params }).catch((error) => { + clearTimeout(timer); + pending.delete(id); + reject(error); + }); + }); + } + function media(operation, params = {}) { + return request(`_buzz/unstable/realtime/${operation}`, { + sessionId: sid, + streamId: stream, + ...params, + }); + } + function base64(pcm) { + const bytes = new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength); + let text = ""; + for (const byte of bytes) text += String.fromCharCode(byte); + return btoa(text); + } + function pcm16(data) { + const bytes = Uint8Array.from(atob(data), (c) => c.charCodeAt(0)); + if (!bytes.length || bytes.length % 2) throw Error("invalid PCM"); + const view = new DataView(bytes.buffer); + return Int16Array.from({ length: bytes.length / 2 }, (_, i) => + view.getInt16(i * 2, true), + ); + } + async function drainCapture() { + if (sending) return; + sending = true; + try { + while (captureQueue.length && !closed) { + const pcm = captureQueue.shift(), + playback = systemQueue.shift(); + if (pcm.length !== playback.length) + throw Error("duplex capture alignment"); + await media("append", { + sequence: sequence++, + data: base64(pcm), + playbackData: base64(playback), + }); + sentSamples += pcm.length; + record("capture_sent", { samples: pcm.length }); + } + } catch (error) { + fail(error); + } finally { + sending = false; + } + } + async function reportPlayback() { + if (playbackPending || suppressPlayback || !latestPlayback || closed) + return; + playbackPending = true; + const playback = latestPlayback; + latestPlayback = null; + try { + await media("playback", { playback }); + } catch (error) { + if (!suppressPlayback) fail(error); + } finally { + playbackPending = false; + } + } + function clearPlayback() { + suppressPlayback = true; + latestPlayback = null; + const requestId = ++serial; + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + stops.delete(requestId); + reject(Error("playback stop deadline")); + }, 2000); + stops.set(requestId, { resolve, timer }); + node.port.postMessage({ type: "clear", requestId }); + }); + } + async function interrupt(serverInitiated) { + const playback = await clearPlayback(); + if (playback) blocked.add(playback.itemId); + if (serverInitiated) { + if (playback) + await media("playback", { playback: { ...playback, stopped: true } }); + } else await media("interrupt", playback ? { playback } : {}); + record("playback_stopped", { playback }); + suppressPlayback = false; + } + function fail(error) { + if (closed) return; + status(error.message); + record("error", { message: error.message }); + stop(false).catch(() => {}); + } + async function stop(graceful = true) { + if (closed) return; + closed = true; + active = false; + captureQueue = new CaptureQueue(); + systemQueue = new CaptureQueue(); + tracks?.getTracks().forEach((track) => { + track.stop(); + }); + node?.port.postMessage({ type: "capture", enabled: false }); + if (node) await clearPlayback().catch(() => {}); + try { + if (stream) { + if (graceful) { + await media("close"); + const deadline = performance.now() + 5000; + while (!evidence.some((e) => e.type === "prompt_done")) { + if (performance.now() > deadline) throw Error("ACP close deadline"); + await new Promise((r) => setTimeout(r, 10)); + } + } else + await rpc({ + jsonrpc: "2.0", + method: "session/cancel", + params: { sessionId: sid }, + }); + } + } catch (error) { + status(error.message); + record("error", { message: error.message }); + throw error; + } finally { + await context?.close(); + eventsAbort.abort(); + for (const p of pending.values()) { + clearTimeout(p.timer); + p.reject(Error("Session closed")); + } + pending.clear(); + ui.ended(); + } + if (graceful) { + status("Connection ended. Start again whenever you like."); + record("client_closed"); + } + } + async function onEvent(event) { + lastEvent = performance.now(); + if (event.method === "session/request_permission") { + permission = event; + ui.permission( + event.params.subject?.toolCall || event.params.toolCall, + decide, + ); + record("permission"); + return; + } + if (event.id !== undefined && pending.has(event.id)) { + const item = pending.get(event.id); + pending.delete(event.id); + clearTimeout(item.timer); + event.error + ? item.reject(Error(event.error.message)) + : item.resolve(event.result); + return; + } + if (event.method === "session/update") { + const u = event.params.update; + if ( + u.sessionUpdate === "agent_message_chunk" && + u.content?.type === "text" + ) + ui.transcript(u.content.text); + if ( + u.sessionUpdate === "tool_call" || + u.sessionUpdate === "tool_call_update" + ) + record(u.sessionUpdate, u); + } + if (event.method !== "_buzz/unstable/realtime/update") return; + const u = event.params.update; + if (u.type === "ready") { + stream = event.params.streamId; + active = true; + record("ready"); + status( + `Listening through ${microphone?.label || "your microphone"}. You can interrupt Frankie.`, + ); + + return; + } + if (u.type === "backchannel_audio") { + if (closed) return; + const pcm = pcm16(u.data); + record("backchannel_audio", { + id: u.id, + startSample: u.startSample, + samples: pcm.length, + }); + node.port.postMessage({ type: "backchannel", id: u.id, pcm }, [ + pcm.buffer, + ]); + } else if (u.type === "audio") { + if (blocked.has(u.itemId) || suppressPlayback || closed) return; + const pcm = pcm16(u.data); + record("audio_received", { + responseId: u.responseId, + itemId: u.itemId, + samples: pcm.length, + startSample: u.startSample, + }); + node.port.postMessage( + { type: "audio", responseId: u.responseId, itemId: u.itemId, pcm }, + [pcm.buffer], + ); + } else if (u.type === "clear") { + if (u.itemId) blocked.add(u.itemId); + // Do not await an ACP reply inside its event reader. + interrupt(true).catch(fail); + } else if (u.type === "closed") { + record("provider_closed", { failed: u.failed }); + if (!closed) { + if (u.failed) + setTimeout( + () => fail(Error("Provider session failed without an ACP error")), + 500, + ); + else fail(Error("Provider session closed")); + } + } else if (u.type === "response_limited") { + status("Frankie reached the reply limit. You can keep talking."); + record(u.type, { responseId: u.responseId, reason: u.reason }); + } else if (u.type === "input_removed") { + ui.removeInput(u.itemId); + record(u.type, { itemId: u.itemId }); + } else if (u.type === "input_transcript") { + ui.userTranscript(u.text, u.itemId); + record(u.type, { itemId: u.itemId, text: u.text }); + } else { + if (u.type === "speech_started") lastSpeechMs = undefined; + if (u.type === "speech_stopped" && Number.isFinite(u.lastSpeechMs)) + lastSpeechMs = u.lastSpeechMs; + record(u.type, { + lastSpeechMs: u.lastSpeechMs, + responseId: u.responseId, + status: u.status, + }); + } + } + async function connect() { + const response = await fetch( + `/events?thinking=${encodeURIComponent(options.thinking || "none")}`, + { + signal: eventsAbort.signal, + headers: { Authorization: `Bearer ${token}` }, + }, + ); + if (!response.ok) throw Error("Use the complete local launch URL"); + const reader = response.body.getReader(), + decoder = new TextDecoder(); + (async () => { + let buffer = ""; + for (;;) { + const { value, done } = await reader.read(); + if (done) throw Error("ACP disconnected"); + buffer += decoder.decode(value, { stream: true }); + if (buffer.length > 2 * 1024 * 1024) throw Error("ACP output limit"); + for (;;) { + const index = buffer.indexOf("\n"); + if (index < 0) break; + const line = buffer.slice(0, index); + buffer = buffer.slice(index + 1); + await onEvent(JSON.parse(line)); + } + } + })().catch(fail); + const init = await request("initialize", { + protocolVersion: 1, + clientCapabilities: { _meta: { buzz: { realtimeAudio: 1 } } }, + }); + if (init.agentCapabilities?._meta?.buzz?.realtimeAudio !== 1) + throw Error("Agent does not support live PCM"); + sid = (await request("session/new", {})).sessionId; + } + async function start() { + context = new AudioContext({ sampleRate: 24000 }); + await context.resume(); + await context.audioWorklet.addModule("/audio-worklet.mjs"); + node = new AudioWorkletNode(context, "duplex-audio", { + numberOfInputs: 1, + numberOfOutputs: 1, + outputChannelCount: [1], + }); + node.connect(context.destination); + node.port.onmessage = ({ data }) => { + if (data.type === "capture" && !closed) { + lastCaptureAt = performance.now(); + capturedSamples += data.pcm.length; + for (const sample of data.pcm) { + const value = sample / 32768; + captureEnergy += value * value; + capturePeak = Math.max(capturePeak, Math.abs(value)); + } + if (Number.isFinite(data.captureTime)) + captureOrigin = data.captureTime - data.captureSamples / 24000; + try { + captureQueue.push(data.pcm); + systemQueue.push(data.playback || new Int16Array(data.pcm.length)); + drainCapture(); + } catch (error) { + fail(error); + } + } else if ( + data.type === "position" && + data.playback && + !suppressPlayback + ) { + latestPlayback = data.playback; + if (data.playback.playedSamples > 0) { + const latency = + Number.isFinite(captureOrigin) && + Number.isFinite(lastSpeechMs) && + Number.isFinite(data.startedAt) + ? (data.startedAt - captureOrigin) * 1000 - lastSpeechMs + : undefined; + record("played", { ...data.playback, firstSoundLatencyMs: latency }); + } + reportPlayback(); + } else if (data.type === "playback_gap") { + playbackGaps++; + playbackGapSamples += data.missingSamples; + maxPlaybackGapSamples = Math.max( + maxPlaybackGapSamples, + data.missingSamples, + ); + record("playback_gap", { + itemId: data.itemId, + playedSamples: data.playedSamples, + gapMs: data.missingSamples / 24, + }); + } else if (data.type === "stopped" && stops.has(data.requestId)) { + const item = stops.get(data.requestId); + stops.delete(data.requestId); + clearTimeout(item.timer); + item.resolve(data.playback); + } else if (data.type === "error") fail(Error(data.message)); + }; + { + tracks = await navigator.mediaDevices.getUserMedia({ + audio: { + channelCount: 1, + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + }); + if (closed) { + tracks.getTracks().forEach((t) => { + t.stop(); + }); + throw Error("Session closed during microphone setup"); + } + microphone = tracks.getAudioTracks()[0]; + record("microphone", { sampleRate: microphone.getSettings().sampleRate }); + microphone.addEventListener("ended", () => { + if (!closed) + fail( + Error("The microphone disconnected. Start again to reconnect it."), + ); + }); + microphone.addEventListener("mute", () => record("microphone_muted")); + microphone.addEventListener("unmute", () => record("microphone_unmuted")); + const source = context.createMediaStreamSource(tracks); + source.connect(node); + const mic = context.createAnalyser(), + voice = context.createAnalyser(); + mic.fftSize = voice.fftSize = 256; + source.connect(mic); + node.connect(voice); + ui.analyzers(mic, voice); + } + const text = ui.context || ""; + request( + "session/prompt", + { + sessionId: sid, + prompt: text ? [{ type: "text", text }] : [], + _meta: { buzz: { realtimeAudio: 1 } }, + }, + 60 * 60 * 1000, + ) + .then((result) => record("prompt_done", result)) + .catch(fail); + status("Preparing Frankie's conversation…"); + const deadline = performance.now() + 120000; + while (!active) { + if (closed || performance.now() > deadline) + throw Error("media readiness deadline"); + await new Promise((resolve) => setTimeout(resolve, 10)); + } + node.port.postMessage({ type: "capture", enabled: true }); + } + async function decide(kind) { + if (!permission) return; + const event = permission; + permission = null; + ui.permission(null); + const option = event.params.options.find((o) => o.kind === kind); + if (!option) throw Error("missing permission option"); + await rpc({ + jsonrpc: "2.0", + id: event.id, + result: { outcome: { outcome: "selected", optionId: option.optionId } }, + }); + } + + const idle = setInterval(() => { + if (!active) return; + const now = performance.now(); + if (now - lastDiagnostic >= 5000) { + const detail = { + capturedSamples, + sentSamples, + queuedSamples: captureQueue.length, + captureAgeMs: lastCaptureAt ? now - lastCaptureAt : null, + rms: Math.sqrt( + captureEnergy / Math.max(1, capturedSamples - diagnosticSamples), + ), + peak: capturePeak, + contextState: context?.state, + muted: microphone?.muted, + trackState: microphone?.readyState, + sampleRate: context?.sampleRate, + outputLatency: context?.outputLatency, + playbackGaps, + playbackGapMs: playbackGapSamples / 24, + maxPlaybackGapMs: maxPlaybackGapSamples / 24, + }; + record("capture_health", detail); + lastDiagnostic = now; + diagnosticSamples = capturedSamples; + captureEnergy = 0; + capturePeak = 0; + } + if (now - lastEvent > 45000) fail(Error("media idle deadline")); + }, 1000); + const cancel = () => { + stop(false).catch(() => {}); + }; + options.signal?.addEventListener("abort", cancel, { once: true }); + const end = ui.ended; + ui.ended = () => { + clearInterval(idle); + options.signal?.removeEventListener("abort", cancel); + end(); + }; + if (options.signal?.aborted) { + await stop(false); + throw Error("Connection cancelled"); + } + try { + await connect(); + await start(); + } catch (error) { + fail(error); + throw error; + } + return { stop, interrupt: () => interrupt(false), decide }; +}