mirror of
https://github.com/block/buzz.git
synced 2026-10-30 22:11:36 +00:00
refactor(voice): extract reusable Pocket primitives + Pocket voice settings (relands #2467 + #3208) (#3910)
Relands **#2467** (extract `buzz-voice` crate) and **#3208** (Pocket
voice settings) onto main, after #3266 and #3180 merged.
## Why a fresh PR
The repo is squash-only with delete-branch-on-merge. Squashing #3266
deleted `jtennant/pocket-tts-2026-04`, which was #2467's base — GitHub
auto-closed #2467 and it cannot be reopened. Squash merges also sever
ancestry, so GitHub's natural merge-base reports phantom conflicts for
the whole remaining stack.
## Content provenance
- Byte-identical to the blessed `jt/buzz-voice-refactor` branch
(`93029c577`, tree `6729e0eff` — reviewed by Dawn (#2467) and Max
(#3208) at exact heads) **except** the three files where #3180 and #3208
genuinely interact.
- Three-file resolution (union of both sides):
- `huddle/mod.rs` — #3180's pipeline re-exports + #3208's
`agent_tts_routing` imports.
- `huddle/state.rs` — `reset_preserving_generation` preserves both
`huddle_generation` (#3180) and `tts_enabled` (#3208); test sets merged
into one `tests` module.
- `desktop/src/testing/e2eBridge.ts` — both switch arms kept; no
duplicate case labels.
## Verification at cf32dacd7
- `cargo test` (desktop/src-tauri, pinned 1.95.0): **2047 + 3 pass / 0
fail** (14 ignored: 8 keychain, 4 real_relay, 2 flag-gated)
- `cargo clippy --all-targets -- -D warnings`: clean; `cargo fmt
--check`: clean
- `cargo check --workspace` (root, includes new `buzz-voice` member):
clean; `cargo test -p buzz-voice`: 5/0
- `pnpm test`: **3885 / 0**; `tsc --noEmit`: clean; lint: clean
The 3180×3208 interaction resolution is getting an independent team
re-review before merge.
Buzz channel: buzz-desktop-voice `fd5fb402-b651-4238-89b1-bb3e2fa4dc96`,
thread `b4798ecc`.
Signed-off-by: npub1qyvc0c5kl4gqv2fd97fsk46tu378sqgy35vc83rvgfwne90sel7s0ed67d <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz>
Co-authored-by: npub1qyvc0c5kl4gqv2fd97fsk46tu378sqgy35vc83rvgfwne90sel7s0ed67d <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz>
This commit is contained in:
Tyler
@@ -53,15 +53,13 @@ pub struct AppState {
|
||||
pub channel_templates_store_lock: Mutex<()>,
|
||||
pub managed_agent_processes: Mutex<HashMap<ManagedAgentRuntimeKey, ManagedAgentPairRuntime>>,
|
||||
pub huddle_state: Mutex<HuddleState>,
|
||||
pub huddle_audio: crate::huddle::tts_settings::HuddleAudioSettingsState,
|
||||
/// Tauri app handle — stored after setup so huddle commands can emit
|
||||
/// `huddle-state-changed` events without needing the handle threaded
|
||||
/// through every call site.
|
||||
///
|
||||
/// Set once during `setup()` in `lib.rs`; never cleared.
|
||||
pub app_handle: Mutex<Option<AppHandle>>,
|
||||
/// Selected audio output device name. `None` = system default.
|
||||
/// Used by `connect_audio_relay` and TTS pipeline when opening sinks.
|
||||
pub audio_output_device: Mutex<Option<String>>,
|
||||
/// Port of the localhost media streaming proxy (set during setup).
|
||||
pub media_proxy_port: AtomicU16,
|
||||
/// Set when identity resolution detected a "keyring-locked" state: the
|
||||
@@ -219,8 +217,8 @@ pub fn build_app_state() -> AppState {
|
||||
managed_agent_processes: Mutex::new(HashMap::new()),
|
||||
session_config_cache: Mutex::new(HashMap::new()),
|
||||
huddle_state: Mutex::new(HuddleState::default()),
|
||||
huddle_audio: Default::default(),
|
||||
app_handle: Mutex::new(None),
|
||||
audio_output_device: Mutex::new(None),
|
||||
media_proxy_port: AtomicU16::new(0),
|
||||
prevent_sleep: Arc::new(Mutex::new(
|
||||
crate::prevent_sleep::PreventSleepState::default(),
|
||||
|
||||
56
desktop/src-tauri/src/huddle/agent_tts_routing.rs
Normal file
56
desktop/src-tauri/src/huddle/agent_tts_routing.rs
Normal file
@@ -0,0 +1,56 @@
|
||||
use super::HuddlePhase;
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub(super) enum AgentTtsRuntimeGate {
|
||||
Disabled,
|
||||
Inactive,
|
||||
NeedsPipeline,
|
||||
Ready,
|
||||
}
|
||||
|
||||
pub(super) fn classify_agent_tts_runtime(
|
||||
enabled: bool,
|
||||
phase: &HuddlePhase,
|
||||
has_pipeline: bool,
|
||||
) -> AgentTtsRuntimeGate {
|
||||
if !enabled {
|
||||
AgentTtsRuntimeGate::Disabled
|
||||
} else if !matches!(phase, HuddlePhase::Connected | HuddlePhase::Active) {
|
||||
AgentTtsRuntimeGate::Inactive
|
||||
} else if has_pipeline {
|
||||
AgentTtsRuntimeGate::Ready
|
||||
} else {
|
||||
AgentTtsRuntimeGate::NeedsPipeline
|
||||
}
|
||||
}
|
||||
|
||||
/// Maximum text length accepted for TTS synthesis.
|
||||
/// ~2000 chars is 1–2 minutes of speech. Longer messages are truncated.
|
||||
pub(super) const MAX_TTS_TEXT_LEN: usize = 2000;
|
||||
|
||||
pub(super) fn normalize_agent_tts_text(text: String) -> String {
|
||||
if text.chars().count() > MAX_TTS_TEXT_LEN {
|
||||
let mut truncated: String = text.chars().take(MAX_TTS_TEXT_LEN).collect();
|
||||
truncated.push_str("... message truncated.");
|
||||
truncated
|
||||
} else {
|
||||
text
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn enqueue_agent_tts_text<F>(
|
||||
route_id: u64,
|
||||
text: String,
|
||||
enqueue: F,
|
||||
) -> Result<(), String>
|
||||
where
|
||||
F: FnOnce(u64, String) -> Result<(), String> + Send + 'static,
|
||||
{
|
||||
tokio::task::spawn_blocking(move || enqueue(route_id, text))
|
||||
.await
|
||||
.map_err(|error| format!("TTS enqueue task failed: {error}"))?
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "agent_tts_routing_tests.rs"]
|
||||
mod tests;
|
||||
57
desktop/src-tauri/src/huddle/agent_tts_routing_tests.rs
Normal file
57
desktop/src-tauri/src/huddle/agent_tts_routing_tests.rs
Normal file
@@ -0,0 +1,57 @@
|
||||
use super::{
|
||||
classify_agent_tts_runtime, enqueue_agent_tts_text, normalize_agent_tts_text,
|
||||
AgentTtsRuntimeGate, MAX_TTS_TEXT_LEN,
|
||||
};
|
||||
use crate::huddle::HuddlePhase;
|
||||
|
||||
#[tokio::test]
|
||||
async fn assistant_plain_text_routes_unchanged_into_voice_pipeline_boundary() {
|
||||
let (sender, receiver) = std::sync::mpsc::channel();
|
||||
let text = "A newly submitted assistant reply.".to_string();
|
||||
let route_id = 42;
|
||||
|
||||
enqueue_agent_tts_text(route_id, text.clone(), move |route_id, queued| {
|
||||
sender
|
||||
.send((route_id, queued))
|
||||
.map_err(|error| error.to_string())
|
||||
})
|
||||
.await
|
||||
.expect("route assistant text");
|
||||
|
||||
assert_eq!(
|
||||
receiver.recv().expect("queued text"),
|
||||
(route_id, text),
|
||||
"route correlation must survive the native queue boundary"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disabled_is_the_only_intentional_runtime_no_op() {
|
||||
assert_eq!(
|
||||
classify_agent_tts_runtime(false, &HuddlePhase::Connected, false),
|
||||
AgentTtsRuntimeGate::Disabled
|
||||
);
|
||||
assert_eq!(
|
||||
classify_agent_tts_runtime(true, &HuddlePhase::Idle, false),
|
||||
AgentTtsRuntimeGate::Inactive
|
||||
);
|
||||
assert_eq!(
|
||||
classify_agent_tts_runtime(true, &HuddlePhase::Connected, false),
|
||||
AgentTtsRuntimeGate::NeedsPipeline
|
||||
);
|
||||
assert_eq!(
|
||||
classify_agent_tts_runtime(true, &HuddlePhase::Connected, true),
|
||||
AgentTtsRuntimeGate::Ready
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assistant_text_truncation_is_unicode_safe_before_voice_routing() {
|
||||
let input = "🦀".repeat(MAX_TTS_TEXT_LEN + 1);
|
||||
let output = normalize_agent_tts_text(input);
|
||||
assert_eq!(
|
||||
output.chars().count(),
|
||||
MAX_TTS_TEXT_LEN + "... message truncated.".chars().count()
|
||||
);
|
||||
assert!(output.ends_with("... message truncated."));
|
||||
}
|
||||
@@ -39,7 +39,8 @@ fn list_audio_output_devices_blocking() -> Result<Vec<AudioOutputDevice>, String
|
||||
#[tauri::command]
|
||||
pub fn set_audio_output_device(name: String, state: State<'_, AppState>) -> Result<(), String> {
|
||||
let mut guard = state
|
||||
.audio_output_device
|
||||
.huddle_audio
|
||||
.output_device
|
||||
.lock()
|
||||
.map_err(|e| e.to_string())?;
|
||||
*guard = if name.is_empty() { None } else { Some(name) };
|
||||
@@ -50,7 +51,8 @@ pub fn set_audio_output_device(name: String, state: State<'_, AppState>) -> Resu
|
||||
#[tauri::command]
|
||||
pub fn get_audio_output_device(state: State<'_, AppState>) -> Result<String, String> {
|
||||
let guard = state
|
||||
.audio_output_device
|
||||
.huddle_audio
|
||||
.output_device
|
||||
.lock()
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(guard.clone().unwrap_or_default())
|
||||
|
||||
@@ -23,6 +23,7 @@
|
||||
//! takes `stt_pipeline`/`tts_pipeline` out of the lock, then calls `shutdown()`
|
||||
//! and drops them outside the lock (thread joins can block ~200ms).
|
||||
|
||||
mod agent_tts_routing;
|
||||
pub mod agents;
|
||||
pub mod audio_output;
|
||||
pub mod jitter;
|
||||
@@ -37,6 +38,8 @@ pub mod state;
|
||||
pub mod stt;
|
||||
pub mod transcription;
|
||||
pub mod tts;
|
||||
pub mod tts_settings;
|
||||
mod tts_voice_registry;
|
||||
pub mod wire;
|
||||
|
||||
// ── Shared utilities ──────────────────────────────────────────────────────────
|
||||
@@ -63,6 +66,7 @@ pub(super) fn drain_until_shutdown<T>(
|
||||
|
||||
pub use state::{HuddleJoinInfo, HuddlePhase, HuddleState, VoiceInputMode};
|
||||
pub use transcription::{set_huddle_transcription_enabled, start_stt_pipeline};
|
||||
pub use tts_settings::set_tts_enabled;
|
||||
|
||||
// ── Imports ───────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -71,10 +75,15 @@ use tauri::State;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::{app_state::AppState, events, relay::submit_event};
|
||||
|
||||
use agent_tts_routing::{
|
||||
classify_agent_tts_runtime, enqueue_agent_tts_text, normalize_agent_tts_text,
|
||||
AgentTtsRuntimeGate,
|
||||
};
|
||||
pub use pipeline::check_pipeline_hotstart;
|
||||
use pipeline::{
|
||||
maybe_start_stt_pipeline, maybe_start_tts_pipeline, post_connect_setup,
|
||||
start_auto_enabled_transcription, PostConnectOutcome,
|
||||
await_inflight_tts_start, maybe_start_stt_pipeline, maybe_start_tts_pipeline,
|
||||
post_connect_setup, start_auto_enabled_transcription, PostConnectOutcome,
|
||||
};
|
||||
use relay_api::{
|
||||
count_human_members, fetch_channel_members, parse_channel_uuid, validate_pubkey_hex,
|
||||
@@ -745,91 +754,90 @@ pub fn get_model_status(_state: State<'_, AppState>) -> Result<models::VoiceMode
|
||||
})
|
||||
}
|
||||
|
||||
/// Enable or disable TTS output.
|
||||
///
|
||||
/// When disabled, the TTS pipeline is shut down and audio output stops.
|
||||
/// When re-enabled, the pipeline is restarted if TTS models are available.
|
||||
///
|
||||
/// Takes the pipeline handle out of the lock before calling shutdown() — the
|
||||
/// thread join in Drop can block for ~200 ms (ONNX inference) and we don't
|
||||
/// want to hold the HuddleState mutex during that time.
|
||||
#[tauri::command]
|
||||
pub async fn set_tts_enabled(enabled: bool, state: State<'_, AppState>) -> Result<(), String> {
|
||||
let old_pipeline = {
|
||||
let mut hs = state.huddle()?;
|
||||
hs.tts_enabled = enabled;
|
||||
if !enabled {
|
||||
hs.tts_pipeline.take() // Take out of lock.
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
// Shut down outside the lock — thread join happens here.
|
||||
if let Some(ref pipeline) = old_pipeline {
|
||||
pipeline.shutdown();
|
||||
}
|
||||
drop(old_pipeline);
|
||||
|
||||
if enabled {
|
||||
// Re-start TTS pipeline if models are available and huddle is active.
|
||||
let phase = {
|
||||
let hs = state.huddle()?;
|
||||
hs.phase.clone()
|
||||
};
|
||||
if matches!(phase, HuddlePhase::Connected | HuddlePhase::Active) {
|
||||
if let Err(e) = maybe_start_tts_pipeline(&state).await {
|
||||
eprintln!("buzz-desktop: TTS pipeline restart failed: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Speak an agent message via TTS.
|
||||
///
|
||||
/// Maximum text length accepted for TTS synthesis.
|
||||
/// ~2000 chars ≈ 1–2 minutes of speech. Longer messages are truncated.
|
||||
const MAX_TTS_TEXT_LEN: usize = 2000;
|
||||
|
||||
/// Called by the WebView when it receives an incoming agent kind:9 message.
|
||||
/// Called by the WebView when it receives an eligible live agent message.
|
||||
/// Lazily starts the TTS pipeline if models are ready but the pipeline hasn't
|
||||
/// been created yet (e.g. models finished downloading after huddle started).
|
||||
///
|
||||
/// No-op if TTS is disabled or models aren't ready.
|
||||
/// Disabled is the only intentional no-op. Enabled-but-unavailable speech
|
||||
/// returns an error so the caller cannot mistake a dropped message for success.
|
||||
#[tauri::command]
|
||||
pub async fn speak_agent_message(text: String, state: State<'_, AppState>) -> Result<(), String> {
|
||||
pub async fn speak_agent_message(
|
||||
text: String,
|
||||
route_id: u64,
|
||||
state: State<'_, AppState>,
|
||||
) -> Result<(), String> {
|
||||
eprintln!("buzz-desktop: tts stage=invoke status=started route_id={route_id}");
|
||||
// Truncate oversized messages — agents shouldn't monologue in a voice huddle.
|
||||
// Use char count (not byte length) to avoid panicking on multi-byte UTF-8.
|
||||
let text = if text.chars().count() > MAX_TTS_TEXT_LEN {
|
||||
let mut truncated: String = text.chars().take(MAX_TTS_TEXT_LEN).collect();
|
||||
truncated.push_str("... message truncated.");
|
||||
truncated
|
||||
} else {
|
||||
text
|
||||
};
|
||||
let text = normalize_agent_tts_text(text);
|
||||
|
||||
let needs_pipeline = {
|
||||
let hs = state.huddle()?;
|
||||
hs.tts_enabled
|
||||
&& hs.tts_pipeline.is_none()
|
||||
&& matches!(hs.phase, HuddlePhase::Connected | HuddlePhase::Active)
|
||||
let mut hs = state.huddle()?;
|
||||
if hs
|
||||
.tts_pipeline
|
||||
.as_ref()
|
||||
.is_some_and(|pipeline| pipeline.is_finished())
|
||||
{
|
||||
hs.tts_pipeline = None;
|
||||
}
|
||||
match classify_agent_tts_runtime(hs.tts_enabled, &hs.phase, hs.tts_pipeline.is_some()) {
|
||||
AgentTtsRuntimeGate::Disabled => {
|
||||
eprintln!(
|
||||
"buzz-desktop: tts stage=invoke status=no_op reason=disabled route_id={route_id}"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
AgentTtsRuntimeGate::Inactive => {
|
||||
eprintln!(
|
||||
"buzz-desktop: tts stage=invoke status=failed reason=inactive_huddle route_id={route_id}"
|
||||
);
|
||||
return Err(
|
||||
"Agent text to speech is unavailable outside an active huddle".to_string(),
|
||||
);
|
||||
}
|
||||
AgentTtsRuntimeGate::NeedsPipeline => true,
|
||||
AgentTtsRuntimeGate::Ready => false,
|
||||
}
|
||||
};
|
||||
|
||||
// Lazy-start: models may have finished downloading after the huddle began.
|
||||
if needs_pipeline {
|
||||
if let Err(e) = maybe_start_tts_pipeline(&state).await {
|
||||
eprintln!("buzz-desktop: TTS lazy-start failed: {e}");
|
||||
}
|
||||
maybe_start_tts_pipeline(&state).await.inspect_err(|_| {
|
||||
eprintln!(
|
||||
"buzz-desktop: tts stage=invoke status=failed reason=startup_failed route_id={route_id}"
|
||||
);
|
||||
})?;
|
||||
await_inflight_tts_start(&state).await.inspect_err(|_| {
|
||||
eprintln!(
|
||||
"buzz-desktop: tts stage=invoke status=failed reason=startup_timeout route_id={route_id}"
|
||||
);
|
||||
})?;
|
||||
}
|
||||
|
||||
let hs = state.huddle()?;
|
||||
if hs.tts_enabled {
|
||||
if let Some(ref pipeline) = hs.tts_pipeline {
|
||||
pipeline.speak(text)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
let sender = {
|
||||
let hs = state.huddle()?;
|
||||
hs.tts_pipeline
|
||||
.as_ref()
|
||||
.map(|pipeline| pipeline.text_sender())
|
||||
};
|
||||
let Some(sender) = sender else {
|
||||
eprintln!(
|
||||
"buzz-desktop: tts stage=invoke status=failed reason=unavailable route_id={route_id}"
|
||||
);
|
||||
return Err("Agent text to speech is enabled but its audio pipeline is unavailable".into());
|
||||
};
|
||||
enqueue_agent_tts_text(route_id, text, move |route_id, text| {
|
||||
sender
|
||||
.send(route_id, text)
|
||||
.map_err(|error| format!("TTS queue closed while waiting to enqueue: {error}"))
|
||||
})
|
||||
.await
|
||||
.inspect(|_| eprintln!("buzz-desktop: tts stage=queue status=accepted route_id={route_id}"))
|
||||
.inspect_err(|_| {
|
||||
eprintln!("buzz-desktop: tts stage=queue status=failed reason=closed route_id={route_id}")
|
||||
})
|
||||
}
|
||||
|
||||
/// Add an agent to the active huddle.
|
||||
|
||||
@@ -27,6 +27,10 @@ use sha2::{Digest, Sha256};
|
||||
use super::pocket::{
|
||||
april_model_info, PocketModelArtifact, APRIL_BUNDLE_ID, APRIL_MODEL_ID, APRIL_MODEL_REVISION,
|
||||
};
|
||||
use super::tts_voice_registry::POCKET_VOICES;
|
||||
|
||||
#[path = "models_voice_upgrade.rs"]
|
||||
mod voice_upgrade;
|
||||
|
||||
// ── Integrity verification ────────────────────────────────────────────────────
|
||||
//
|
||||
@@ -91,8 +95,8 @@ const TTS_REFERENCE_ARTIFACT: PocketModelArtifact = PocketModelArtifact {
|
||||
/// honest (each version tag identifies one specific set of model bytes).
|
||||
const STT_MODEL_VERSION: &str = "2";
|
||||
|
||||
/// Identifies the exact April INT8 asset set expected by readiness checks.
|
||||
const TTS_MODEL_VERSION: &str = "4";
|
||||
/// Identifies the April INT8 asset set plus the official VCTK presets.
|
||||
const TTS_MODEL_VERSION: &str = "5";
|
||||
|
||||
/// Filename for the version manifest written alongside model files.
|
||||
const MANIFEST_FILENAME: &str = ".buzz-model-manifest";
|
||||
@@ -160,39 +164,6 @@ const TTS_MODEL_DIR_NAME: &str = "pocket-tts";
|
||||
/// Attribution sidecar written next to the Pocket TTS model files.
|
||||
const TTS_LICENSE_FILE_NAME: &str = "MODEL_LICENSE.txt";
|
||||
|
||||
/// CC-BY-4.0 §3(a)(1) attribution block for Pocket TTS, its ONNX packaging,
|
||||
/// and the bundled reference voice WAV.
|
||||
const TTS_LICENSE_TEXT: &str = "\
|
||||
Pocket TTS
|
||||
© Kyutai.
|
||||
|
||||
Licensed under the Creative Commons Attribution 4.0 International License
|
||||
(CC-BY-4.0). License text: https://creativecommons.org/licenses/by/4.0/
|
||||
|
||||
Original model by Kyutai: https://huggingface.co/kyutai/pocket-tts
|
||||
Paper: Charles, Roebel, et al., Pocket TTS (arXiv:2509.06926).
|
||||
Mimi neural codec by Kyutai is bundled as part of the model.
|
||||
|
||||
April 2026 ONNX export by KevinAHM:
|
||||
https://huggingface.co/KevinAHM/pocket-tts-onnx
|
||||
Pinned revision: 58a6d00cf13d239b6748cb0769f35c580a8f606c
|
||||
|
||||
Bundled reference voice (reference_sample.wav):
|
||||
\"Mary (f, conversation)\" preset from the Kyutai TTS demo voice catalogue
|
||||
(https://kyutai.org/tts), distributed via
|
||||
https://huggingface.co/kyutai/tts-voices as `vctk/p333_023_enhanced.wav`.
|
||||
Original recording from the Voice Cloning Toolkit (VCTK) corpus, speaker p333:
|
||||
https://datashare.ed.ac.uk/handle/10283/3443 (CC-BY-4.0).
|
||||
Recording enhancement (denoise/dereverb) by ai-coustics:
|
||||
https://ai-coustics.com/
|
||||
|
||||
Buzz ships all ONNX/model artifacts and the reference voice WAV unmodified,
|
||||
renamed only by placement in the local model directory.
|
||||
|
||||
Provided \"AS IS\", without warranty of any kind, express or implied. See the
|
||||
license text for full warranty disclaimer.
|
||||
";
|
||||
|
||||
/// All files that must be present for Pocket TTS to be considered ready.
|
||||
const TTS_EXPECTED_FILES: &[&str] = &[
|
||||
"bundle.json",
|
||||
@@ -205,6 +176,17 @@ const TTS_EXPECTED_FILES: &[&str] = &[
|
||||
"tokenizer.model",
|
||||
"LICENSE",
|
||||
"reference_sample.wav",
|
||||
"anna.wav",
|
||||
"vera.wav",
|
||||
"fantine.wav",
|
||||
"charles.wav",
|
||||
"paul.wav",
|
||||
"eponine.wav",
|
||||
"azelma.wav",
|
||||
"george.wav",
|
||||
"jane.wav",
|
||||
"michael.wav",
|
||||
"eve.wav",
|
||||
TTS_LICENSE_FILE_NAME,
|
||||
];
|
||||
|
||||
@@ -705,6 +687,9 @@ impl ModelManager {
|
||||
|
||||
/// Start a background Pocket TTS download. No-op if already ready or downloading.
|
||||
pub fn start_tts_download(&self, http_client: reqwest::Client) {
|
||||
if let Err(error) = voice_upgrade::install_vctk_presets_into_v4_model(&self.models_dir) {
|
||||
eprintln!("buzz-desktop: could not upgrade existing Pocket voices in place: {error}");
|
||||
}
|
||||
let manager = self.clone();
|
||||
self.tts.start_download(
|
||||
&self.models_dir,
|
||||
@@ -822,7 +807,7 @@ impl ModelManager {
|
||||
/// - five ONNX sessions selected by the April INT8 bundle
|
||||
/// - bundle metadata, SentencePiece tokenizer, and learned voice BOS
|
||||
/// - upstream `LICENSE` plus Buzz's `MODEL_LICENSE.txt` attribution sidecar
|
||||
/// - `reference_sample.wav` as the bundled default voice
|
||||
/// - `reference_sample.wav` plus the embedded official VCTK presets
|
||||
///
|
||||
/// Files are written to a temp directory first, then moved atomically.
|
||||
async fn download_tts_model(&self, http_client: reqwest::Client) -> Result<(), String> {
|
||||
@@ -904,9 +889,20 @@ impl ModelManager {
|
||||
});
|
||||
}
|
||||
|
||||
tokio::fs::write(temp_dir.join(TTS_LICENSE_FILE_NAME), TTS_LICENSE_TEXT)
|
||||
.await
|
||||
.map_err(|e| format!("write TTS model license sidecar: {e}"))?;
|
||||
tokio::fs::write(
|
||||
temp_dir.join(TTS_LICENSE_FILE_NAME),
|
||||
voice_upgrade::TTS_LICENSE_TEXT,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| format!("write TTS model license sidecar: {e}"))?;
|
||||
for voice in POCKET_VOICES {
|
||||
let Some(bytes) = voice.bytes else {
|
||||
continue;
|
||||
};
|
||||
tokio::fs::write(temp_dir.join(voice.reference_file), bytes)
|
||||
.await
|
||||
.map_err(|e| format!("install bundled {} voice: {e}", voice.display_name))?;
|
||||
}
|
||||
|
||||
self.tts.set_status(ModelStatus::Downloading {
|
||||
progress_percent: 90,
|
||||
|
||||
@@ -22,11 +22,8 @@ fn expected_files_match_april_int8_metadata() {
|
||||
.artifacts
|
||||
.iter()
|
||||
.map(|artifact| artifact.filename)
|
||||
.chain([
|
||||
TTS_LICENSE_ARTIFACT.filename,
|
||||
TTS_REFERENCE_ARTIFACT.filename,
|
||||
TTS_LICENSE_FILE_NAME,
|
||||
])
|
||||
.chain([TTS_LICENSE_ARTIFACT.filename, TTS_LICENSE_FILE_NAME])
|
||||
.chain(POCKET_VOICES.iter().map(|voice| voice.reference_file))
|
||||
.collect::<Vec<_>>();
|
||||
expected.sort_unstable();
|
||||
let mut actual = TTS_EXPECTED_FILES.to_vec();
|
||||
@@ -36,6 +33,7 @@ fn expected_files_match_april_int8_metadata() {
|
||||
assert!(!actual.contains(&"flow_lm_main.onnx"));
|
||||
assert!(!actual.contains(&"flow_lm_flow.onnx"));
|
||||
assert!(!actual.contains(&"mimi_decoder.onnx"));
|
||||
assert!(!actual.contains(&"marius.wav"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
128
desktop/src-tauri/src/huddle/models_voice_upgrade.rs
Normal file
128
desktop/src-tauri/src/huddle/models_voice_upgrade.rs
Normal file
@@ -0,0 +1,128 @@
|
||||
use super::*;
|
||||
use crate::huddle::tts_voice_registry::POCKET_VOICES;
|
||||
|
||||
const PRESET_VOICE_TTS_MODEL_VERSION: &str = "4";
|
||||
|
||||
/// Attribution written beside every installed Pocket model and voice asset.
|
||||
pub(super) const TTS_LICENSE_TEXT: &str = "\
|
||||
Pocket TTS
|
||||
© Kyutai.
|
||||
|
||||
Licensed under the Creative Commons Attribution 4.0 International License
|
||||
(CC-BY-4.0). License text: https://creativecommons.org/licenses/by/4.0/
|
||||
|
||||
Original model by Kyutai: https://huggingface.co/kyutai/pocket-tts
|
||||
Paper: Charles, Roebel, et al., Pocket TTS (arXiv:2509.06926).
|
||||
Mimi neural codec by Kyutai is bundled as part of the model.
|
||||
|
||||
April 2026 ONNX export by KevinAHM:
|
||||
https://huggingface.co/KevinAHM/pocket-tts-onnx
|
||||
Pinned revision: 58a6d00cf13d239b6748cb0769f35c580a8f606c
|
||||
|
||||
Bundled English VCTK presets: Anna (p228), Vera (p229), Fantine (p244),
|
||||
Charles (p254), Paul (p259), Eponine (p262), Azelma (p303), George (p315),
|
||||
Mary (p333), Jane (p339), Michael (p360), and Eve (p361). These exact,
|
||||
ai-coustics-enhanced WAVs come from Kyutai's tts-voices repository at revision
|
||||
323332d33f997de8394f24a193e1a76df720e01a.
|
||||
Source: https://huggingface.co/kyutai/tts-voices/tree/323332d33f997de8394f24a193e1a76df720e01a/vctk
|
||||
Original recordings: Voice Cloning Toolkit (VCTK) corpus,
|
||||
https://datashare.ed.ac.uk/handle/10283/3443 (CC-BY-4.0).
|
||||
Enhancement (denoise/dereverb): ai-coustics, https://ai-coustics.com/
|
||||
|
||||
Buzz ships the ONNX/model artifacts and voice WAVs unmodified, renamed only
|
||||
by placement in the local model directory.
|
||||
|
||||
Provided \"AS IS\", without warranty of any kind, express or implied. See the
|
||||
license text for full warranty disclaimer.
|
||||
";
|
||||
|
||||
fn is_embedded_voice_file(filename: &str) -> bool {
|
||||
POCKET_VOICES
|
||||
.iter()
|
||||
.any(|voice| voice.bytes.is_some() && voice.reference_file == filename)
|
||||
}
|
||||
|
||||
/// Add the official VCTK presets to an otherwise-ready v4 install.
|
||||
///
|
||||
/// Model artifacts and Mary already exist in v4. The manifest is written last,
|
||||
/// so interruption leaves v4 intact and the next launch retries.
|
||||
pub(super) fn install_vctk_presets_into_v4_model(models_dir: &Path) -> Result<(), String> {
|
||||
let model_dir = models_dir.join(TTS_MODEL_DIR_NAME);
|
||||
let manifest_path = model_dir.join(MANIFEST_FILENAME);
|
||||
let version = match std::fs::read_to_string(&manifest_path) {
|
||||
Ok(version) => version,
|
||||
Err(_) => return Ok(()),
|
||||
};
|
||||
if version.trim() != PRESET_VOICE_TTS_MODEL_VERSION {
|
||||
return Ok(());
|
||||
}
|
||||
if !TTS_EXPECTED_FILES
|
||||
.iter()
|
||||
.filter(|filename| !is_embedded_voice_file(filename))
|
||||
.all(|filename| model_dir.join(filename).is_file())
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
for voice in POCKET_VOICES {
|
||||
let Some(bytes) = voice.bytes else {
|
||||
continue;
|
||||
};
|
||||
std::fs::write(model_dir.join(voice.reference_file), bytes)
|
||||
.map_err(|error| format!("write bundled {} voice: {error}", voice.display_name))?;
|
||||
}
|
||||
let retired_marius = model_dir.join("marius.wav");
|
||||
if retired_marius.is_file() {
|
||||
std::fs::remove_file(retired_marius)
|
||||
.map_err(|error| format!("remove retired Marius voice: {error}"))?;
|
||||
}
|
||||
std::fs::write(model_dir.join(TTS_LICENSE_FILE_NAME), TTS_LICENSE_TEXT)
|
||||
.map_err(|error| format!("update Pocket voice notice: {error}"))?;
|
||||
std::fs::write(manifest_path, TTS_MODEL_VERSION)
|
||||
.map_err(|error| format!("update Pocket model manifest: {error}"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn v4_install_adds_presets_without_redownloading_models() {
|
||||
let temp = tempfile::tempdir().expect("tempdir");
|
||||
let model_dir = temp.path().join(TTS_MODEL_DIR_NAME);
|
||||
std::fs::create_dir_all(&model_dir).expect("create model dir");
|
||||
for file in TTS_EXPECTED_FILES
|
||||
.iter()
|
||||
.filter(|filename| !is_embedded_voice_file(filename))
|
||||
{
|
||||
std::fs::write(model_dir.join(file), b"existing").expect("write prior file");
|
||||
}
|
||||
std::fs::write(
|
||||
model_dir.join(MANIFEST_FILENAME),
|
||||
PRESET_VOICE_TTS_MODEL_VERSION,
|
||||
)
|
||||
.expect("write prior manifest");
|
||||
std::fs::write(model_dir.join("marius.wav"), b"retired").expect("write retired voice");
|
||||
|
||||
install_vctk_presets_into_v4_model(temp.path()).expect("in-place upgrade");
|
||||
|
||||
for voice in POCKET_VOICES {
|
||||
if let Some(bytes) = voice.bytes {
|
||||
assert_eq!(
|
||||
std::fs::read(model_dir.join(voice.reference_file))
|
||||
.expect("bundled voice installed"),
|
||||
bytes
|
||||
);
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(model_dir.join(MANIFEST_FILENAME)).expect("updated manifest"),
|
||||
TTS_MODEL_VERSION
|
||||
);
|
||||
assert!(!model_dir.join("marius.wav").exists());
|
||||
assert!(
|
||||
ModelSlot::new(TTS_MODEL_DIR_NAME, TTS_EXPECTED_FILES, TTS_MODEL_VERSION)
|
||||
.is_ready(temp.path())
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -3,9 +3,12 @@
|
||||
//! Handles starting, hot-starting, and spawning transcription tasks for
|
||||
//! the voice pipelines. Extracted from mod.rs to keep the command layer thin.
|
||||
|
||||
use std::sync::{
|
||||
atomic::{AtomicU64, Ordering},
|
||||
Arc, Mutex,
|
||||
use std::{
|
||||
sync::{
|
||||
atomic::{AtomicU64, Ordering},
|
||||
Arc, Mutex,
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use nostr::JsonUtil;
|
||||
@@ -17,7 +20,7 @@ use crate::events;
|
||||
|
||||
use super::models;
|
||||
use super::relay_api::{self, fetch_channel_members, parse_channel_uuid};
|
||||
use super::state::{HuddlePhase, VoiceInputMode};
|
||||
use super::state::{HuddlePhase, HuddleState, VoiceInputMode};
|
||||
use super::stt;
|
||||
use super::tts;
|
||||
|
||||
@@ -392,7 +395,7 @@ pub(crate) async fn maybe_start_tts_pipeline(state: &AppState) -> Result<bool, S
|
||||
// Atomically check preconditions and claim the construction slot.
|
||||
// The sentinel prevents a second caller from starting construction
|
||||
// while we're building outside the lock.
|
||||
let (tts_active, tts_cancel) = {
|
||||
let (tts_active, tts_cancel, tts_starting) = {
|
||||
let hs = state.huddle()?;
|
||||
if hs.tts_pipeline.is_some() {
|
||||
return Ok(false);
|
||||
@@ -403,18 +406,39 @@ pub(crate) async fn maybe_start_tts_pipeline(state: &AppState) -> Result<bool, S
|
||||
if hs.tts_starting.swap(true, Ordering::AcqRel) {
|
||||
return Ok(false); // Another caller is already constructing.
|
||||
}
|
||||
(Arc::clone(&hs.tts_active), Arc::clone(&hs.tts_cancel))
|
||||
(
|
||||
Arc::clone(&hs.tts_active),
|
||||
Arc::clone(&hs.tts_cancel),
|
||||
Arc::clone(&hs.tts_starting),
|
||||
)
|
||||
};
|
||||
let _starting_guard = TtsStartingGuard(tts_starting);
|
||||
|
||||
// Construct outside the lock — this spawns the TTS worker thread and
|
||||
// loads ONNX sessions (~200ms). If this fails, clear the sentinel.
|
||||
let output_device = state
|
||||
.audio_output_device
|
||||
.huddle_audio
|
||||
.output_device
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.clone();
|
||||
let initial_voice = state
|
||||
.huddle_audio
|
||||
.tts
|
||||
.lock()
|
||||
.map_err(|error| format!("text-to-speech settings lock poisoned: {error}"))
|
||||
.map(|settings| {
|
||||
super::tts_settings::pocket_voice_name(&settings.voice_preferences).to_string()
|
||||
})?;
|
||||
let constructed_voice = initial_voice.clone();
|
||||
let constructed = tokio::task::spawn_blocking(move || {
|
||||
tts::TtsPipeline::new(model_dir, tts_active, tts_cancel, output_device)
|
||||
tts::TtsPipeline::new_with_voice(
|
||||
model_dir,
|
||||
tts_active,
|
||||
tts_cancel,
|
||||
&initial_voice,
|
||||
output_device,
|
||||
)
|
||||
})
|
||||
.await;
|
||||
let pipeline = match constructed {
|
||||
@@ -431,23 +455,80 @@ pub(crate) async fn maybe_start_tts_pipeline(state: &AppState) -> Result<bool, S
|
||||
}
|
||||
};
|
||||
|
||||
{
|
||||
let mut hs = state.huddle()?;
|
||||
hs.tts_starting.store(false, Ordering::Release);
|
||||
// Phase check: huddle may have been torn down during construction.
|
||||
if !matches!(hs.phase, HuddlePhase::Connected | HuddlePhase::Active) {
|
||||
return Ok(false);
|
||||
finalize_tts_pipeline_start(state, move |voice, huddle| {
|
||||
if should_reselect_constructed_voice(&constructed_voice, voice) {
|
||||
pipeline.select_voice_before_publish(voice);
|
||||
}
|
||||
// Final check: another path may have created a pipeline while we were constructing.
|
||||
if hs.tts_pipeline.is_some() {
|
||||
return Ok(false);
|
||||
}
|
||||
hs.tts_pipeline = Some(pipeline);
|
||||
}
|
||||
huddle.tts_pipeline = Some(pipeline);
|
||||
})
|
||||
}
|
||||
|
||||
/// Wait for a concurrent TTS constructor to publish or fail.
|
||||
///
|
||||
/// `maybe_start_tts_pipeline` deliberately lets only one caller construct the
|
||||
/// engine. A live message that loses that race must wait for the owner instead
|
||||
/// of observing the temporary empty slot and being dropped.
|
||||
pub(crate) async fn await_inflight_tts_start(state: &AppState) -> Result<(), String> {
|
||||
let starting = {
|
||||
let huddle = state.huddle()?;
|
||||
Arc::clone(&huddle.tts_starting)
|
||||
};
|
||||
tokio::time::timeout(Duration::from_secs(15), async {
|
||||
while starting.load(Ordering::Acquire) {
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| "TTS pipeline startup did not finish before timeout".to_string())?;
|
||||
// The owner clears the sentinel while holding the huddle lock, before it
|
||||
// publishes. Reacquiring that lock ensures publication is visible before
|
||||
// the losing caller looks up the sender.
|
||||
drop(state.huddle()?);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct TtsStartingGuard(Arc<std::sync::atomic::AtomicBool>);
|
||||
|
||||
impl Drop for TtsStartingGuard {
|
||||
fn drop(&mut self) {
|
||||
self.0.store(false, Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
/// Publish a constructed TTS pipeline against the latest settings.
|
||||
///
|
||||
/// Construction happens outside locks and can overlap a voice change or OFF
|
||||
/// transition. Holding the huddle lock while re-reading settings gives either
|
||||
/// transition a safe ordering: it updates the installed pipeline afterward,
|
||||
/// or this finalizer observes the new setting before publishing.
|
||||
fn finalize_tts_pipeline_start(
|
||||
state: &AppState,
|
||||
publish: impl FnOnce(&str, &mut HuddleState),
|
||||
) -> Result<bool, String> {
|
||||
let mut huddle = state.huddle()?;
|
||||
huddle.tts_starting.store(false, Ordering::Release);
|
||||
if !huddle.tts_enabled
|
||||
|| !matches!(huddle.phase, HuddlePhase::Connected | HuddlePhase::Active)
|
||||
|| huddle.tts_pipeline.is_some()
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
let voice = state
|
||||
.huddle_audio
|
||||
.tts
|
||||
.lock()
|
||||
.map_err(|error| format!("text-to-speech settings lock poisoned: {error}"))
|
||||
.map(|settings| {
|
||||
super::tts_settings::pocket_voice_name(&settings.voice_preferences).to_string()
|
||||
})?;
|
||||
publish(&voice, &mut huddle);
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
fn should_reselect_constructed_voice(constructed_voice: &str, latest_voice: &str) -> bool {
|
||||
constructed_voice != latest_voice
|
||||
}
|
||||
|
||||
/// Sign an STT transcript event and produce the guarded POST body.
|
||||
///
|
||||
/// Factored out of the transcription loop so egress boundary 5 (huddle STT)
|
||||
@@ -570,3 +651,148 @@ pub(crate) fn spawn_transcription_task(
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tts_start_race_tests {
|
||||
use std::sync::{
|
||||
atomic::{AtomicBool, Ordering},
|
||||
Arc, Barrier, Mutex,
|
||||
};
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::app_state::build_app_state;
|
||||
|
||||
use super::{
|
||||
await_inflight_tts_start, finalize_tts_pipeline_start, should_reselect_constructed_voice,
|
||||
HuddlePhase,
|
||||
};
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_losing_starter_observes_publication_before_resuming() {
|
||||
let state = Arc::new(build_app_state());
|
||||
{
|
||||
let mut huddle = state.huddle().expect("huddle state");
|
||||
huddle.phase = HuddlePhase::Active;
|
||||
huddle.tts_enabled = true;
|
||||
huddle.tts_starting.store(true, Ordering::Release);
|
||||
}
|
||||
let published = Arc::new(AtomicBool::new(false));
|
||||
let owner_state = Arc::clone(&state);
|
||||
let owner_published = Arc::clone(&published);
|
||||
let owner = std::thread::spawn(move || {
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
finalize_tts_pipeline_start(&owner_state, |_, _| {
|
||||
owner_published.store(true, Ordering::Release);
|
||||
})
|
||||
});
|
||||
|
||||
await_inflight_tts_start(&state)
|
||||
.await
|
||||
.expect("wait for pipeline owner");
|
||||
assert!(published.load(Ordering::Acquire));
|
||||
assert!(owner.join().expect("pipeline owner").expect("finalize"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn constructor_fallback_survives_unchanged_preference_at_publication() {
|
||||
let selected_voice = Mutex::new(super::super::pocket::DEFAULT_VOICE.to_string());
|
||||
let constructed_voice = "eve";
|
||||
let latest_voice = "eve";
|
||||
|
||||
if should_reselect_constructed_voice(constructed_voice, latest_voice) {
|
||||
*selected_voice.lock().expect("selected voice") = latest_voice.to_string();
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
selected_voice.lock().expect("selected voice").as_str(),
|
||||
super::super::pocket::DEFAULT_VOICE
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn construction_reconciles_a_voice_selected_while_starting() {
|
||||
let state = Arc::new(build_app_state());
|
||||
{
|
||||
let mut huddle = state.huddle().expect("huddle state");
|
||||
huddle.phase = HuddlePhase::Active;
|
||||
huddle.tts_enabled = true;
|
||||
huddle.tts_starting.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
let constructed = Arc::new(Barrier::new(2));
|
||||
let publish = Arc::new(Barrier::new(2));
|
||||
let selected_voice = Arc::new(Mutex::new(None));
|
||||
let worker_state = Arc::clone(&state);
|
||||
let worker_constructed = Arc::clone(&constructed);
|
||||
let worker_publish = Arc::clone(&publish);
|
||||
let worker_voice = Arc::clone(&selected_voice);
|
||||
let worker = std::thread::spawn(move || {
|
||||
worker_constructed.wait();
|
||||
worker_publish.wait();
|
||||
finalize_tts_pipeline_start(&worker_state, |voice, _| {
|
||||
*worker_voice.lock().expect("selected voice") = Some(voice.to_string());
|
||||
})
|
||||
});
|
||||
|
||||
constructed.wait();
|
||||
assert!(state
|
||||
.huddle()
|
||||
.expect("huddle state")
|
||||
.tts_starting
|
||||
.load(Ordering::Acquire));
|
||||
state
|
||||
.huddle_audio
|
||||
.tts
|
||||
.lock()
|
||||
.expect("text-to-speech settings")
|
||||
.voice_preferences = vec!["pocket:eve".to_string()];
|
||||
publish.wait();
|
||||
|
||||
assert!(worker.join().expect("starter thread").expect("finalize"));
|
||||
assert_eq!(
|
||||
*selected_voice.lock().expect("selected voice"),
|
||||
Some("eve".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn construction_is_discarded_when_disabled_while_starting() {
|
||||
let state = Arc::new(build_app_state());
|
||||
{
|
||||
let mut huddle = state.huddle().expect("huddle state");
|
||||
huddle.phase = HuddlePhase::Active;
|
||||
huddle.tts_enabled = true;
|
||||
huddle.tts_starting.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
let constructed = Arc::new(Barrier::new(2));
|
||||
let publish = Arc::new(Barrier::new(2));
|
||||
let did_publish = Arc::new(Mutex::new(false));
|
||||
let worker_state = Arc::clone(&state);
|
||||
let worker_constructed = Arc::clone(&constructed);
|
||||
let worker_publish = Arc::clone(&publish);
|
||||
let worker_did_publish = Arc::clone(&did_publish);
|
||||
let worker = std::thread::spawn(move || {
|
||||
worker_constructed.wait();
|
||||
worker_publish.wait();
|
||||
finalize_tts_pipeline_start(&worker_state, |_, _| {
|
||||
*worker_did_publish.lock().expect("publish flag") = true;
|
||||
})
|
||||
});
|
||||
|
||||
constructed.wait();
|
||||
{
|
||||
let mut huddle = state.huddle().expect("huddle state");
|
||||
huddle.tts_enabled = false;
|
||||
}
|
||||
publish.wait();
|
||||
|
||||
assert!(!worker.join().expect("starter thread").expect("finalize"));
|
||||
assert!(!*did_publish.lock().expect("publish flag"));
|
||||
assert!(!state
|
||||
.huddle()
|
||||
.expect("huddle state")
|
||||
.tts_starting
|
||||
.load(Ordering::Acquire));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,166 +1,4 @@
|
||||
//! April 2026 Pocket TTS engine for Buzz Desktop.
|
||||
//!
|
||||
//! The `english_2026-04` bundle uses SentencePiece tokenization, a learned
|
||||
//! voice BOS embedding, recurrent FlowLM state, and stateful Mimi decoding.
|
||||
//! Buzz selects the upstream three-graph INT8 variant while retaining the
|
||||
//! full-precision Mimi encoder and text conditioner specified by that variant.
|
||||
//!
|
||||
//! ## Attribution
|
||||
//!
|
||||
//! - Pocket TTS and Mimi: Kyutai, CC-BY-4.0.
|
||||
//! - ONNX export: KevinAHM/pocket-tts-onnx, CC-BY-4.0.
|
||||
//! - Reference voice: Kyutai's Mary preset (VCTK p333), CC-BY-4.0.
|
||||
//!
|
||||
//! `huddle::models` writes the complete attribution beside the cached bytes.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Mutex;
|
||||
|
||||
use sherpa_onnx::Wave;
|
||||
|
||||
#[path = "pocket_april.rs"]
|
||||
mod pocket_april;
|
||||
#[path = "pocket_models.rs"]
|
||||
mod pocket_models;
|
||||
|
||||
use pocket_april::{prepare_april_prompt, AprilPocketTts};
|
||||
pub(crate) use pocket_models::{
|
||||
pub use buzz_voice_pkg::pocket::*;
|
||||
pub(crate) use buzz_voice_pkg::{
|
||||
april_model_info, PocketModelArtifact, APRIL_BUNDLE_ID, APRIL_MODEL_ID, APRIL_MODEL_REVISION,
|
||||
};
|
||||
|
||||
/// Pocket TTS emits 24 kHz mono PCM.
|
||||
pub const SAMPLE_RATE: u32 = 24_000;
|
||||
|
||||
/// Bundled reference voice name without its extension.
|
||||
pub const DEFAULT_VOICE: &str = "reference_sample";
|
||||
|
||||
/// Pocket voice files are reference WAVs.
|
||||
pub const VOICE_FILE_EXT: &str = "wav";
|
||||
|
||||
const TTS_NUM_THREADS: usize = 1;
|
||||
|
||||
/// Loaded reference voice samples and their original sample rate.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct VoiceStyle {
|
||||
samples: Vec<f32>,
|
||||
sample_rate: i32,
|
||||
}
|
||||
|
||||
/// Load a Pocket reference voice WAV from disk.
|
||||
pub fn load_voice_style(path: &Path) -> Result<VoiceStyle, String> {
|
||||
let path_str = path
|
||||
.to_str()
|
||||
.ok_or_else(|| format!("voice path is not valid UTF-8: {}", path.display()))?;
|
||||
let wave = Wave::read(path_str)
|
||||
.ok_or_else(|| format!("could not read voice WAV at {}", path.display()))?;
|
||||
let samples = wave.samples().to_vec();
|
||||
if samples.is_empty() {
|
||||
return Err(format!("voice WAV is empty: {}", path.display()));
|
||||
}
|
||||
Ok(VoiceStyle {
|
||||
samples,
|
||||
sample_rate: wave.sample_rate(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Resident April INT8 Pocket TTS engine.
|
||||
pub struct PocketTts {
|
||||
inner: Mutex<AprilPocketTts>,
|
||||
}
|
||||
|
||||
/// Load Buzz Desktop's pinned April INT8 model.
|
||||
pub fn load_text_to_speech(model_dir: &str) -> Result<PocketTts, String> {
|
||||
let dir = PathBuf::from(model_dir);
|
||||
for artifact in april_model_info().artifacts {
|
||||
let path = dir.join(artifact.filename);
|
||||
if !path.is_file() {
|
||||
return Err(format!(
|
||||
"incomplete Pocket TTS {} INT8 bundle: missing {}",
|
||||
APRIL_BUNDLE_ID,
|
||||
path.display()
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(PocketTts {
|
||||
inner: Mutex::new(AprilPocketTts::load(&dir, TTS_NUM_THREADS)?),
|
||||
})
|
||||
}
|
||||
|
||||
impl PocketTts {
|
||||
/// Split text into synthesis units that satisfy the bundle's exact
|
||||
/// 50-token input limit.
|
||||
pub fn split_text_into_chunks(&self, text: &str) -> Result<Vec<String>, String> {
|
||||
let Some(prepared) = prepare_april_prompt(text) else {
|
||||
return Ok(Vec::new());
|
||||
};
|
||||
self.inner
|
||||
.lock()
|
||||
.map_err(|_| "Pocket TTS engine lock poisoned".to_string())?
|
||||
.split_prompt(&prepared)
|
||||
}
|
||||
|
||||
/// Synthesize text with the supplied reference voice.
|
||||
///
|
||||
/// Pocket detects language from text and this model uses one synthesis
|
||||
/// step, so `_lang` and `_steps` intentionally do not affect output.
|
||||
pub fn synth_chunk(
|
||||
&self,
|
||||
text: &str,
|
||||
_lang: &str,
|
||||
style: &VoiceStyle,
|
||||
_steps: usize,
|
||||
) -> Result<Vec<f32>, String> {
|
||||
let Some(prepared) = prepare_april_prompt(text) else {
|
||||
return Ok(Vec::new());
|
||||
};
|
||||
let mut engine = self
|
||||
.inner
|
||||
.lock()
|
||||
.map_err(|_| "Pocket TTS engine lock poisoned".to_string())?;
|
||||
let chunks = engine.split_prompt(&prepared)?;
|
||||
let mut samples = Vec::new();
|
||||
for chunk in chunks {
|
||||
let prepared = prepare_april_prompt(&chunk)
|
||||
.ok_or_else(|| "Pocket TTS prompt chunk became empty".to_string())?;
|
||||
samples.extend(engine.synth_chunk(&prepared, style)?);
|
||||
}
|
||||
Ok(samples)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn desktop_model_is_april_int8_only() {
|
||||
let info = april_model_info();
|
||||
assert_eq!(info.max_token_per_chunk, 50);
|
||||
assert_eq!(info.sample_rate, SAMPLE_RATE);
|
||||
assert!(info
|
||||
.artifacts
|
||||
.iter()
|
||||
.any(|artifact| artifact.filename == "flow_lm_main_int8.onnx"));
|
||||
assert!(!info
|
||||
.artifacts
|
||||
.iter()
|
||||
.any(|artifact| artifact.filename == "flow_lm_main.onnx"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires BUZZ_POCKET_TEST_MODEL_DIR"]
|
||||
fn production_api_emits_non_silent_april_int8_pcm() {
|
||||
let dir = std::env::var("BUZZ_POCKET_TEST_MODEL_DIR")
|
||||
.expect("set BUZZ_POCKET_TEST_MODEL_DIR to an April INT8 model directory");
|
||||
let engine = load_text_to_speech(&dir).expect("load April INT8 engine");
|
||||
let style = load_voice_style(&Path::new(&dir).join("reference_sample.wav"))
|
||||
.expect("load reference voice");
|
||||
let samples = engine
|
||||
.synth_chunk("Bright birds begin beside the bay.", "en", &style, 1)
|
||||
.expect("synthesize through the production API");
|
||||
|
||||
assert!(!samples.is_empty());
|
||||
assert!(samples.iter().all(|sample| sample.is_finite()));
|
||||
assert!(samples.iter().any(|sample| sample.abs() > 1.0e-6));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,940 +0,0 @@
|
||||
//! Native ONNX loader for Pocket TTS `english_2026-04`.
|
||||
//!
|
||||
//! The bundle uses SentencePiece, prepends a learned BOS voice embedding, and
|
||||
//! describes recurrent state tensors in `bundle.json`. This module supplies
|
||||
//! that frontend and state loop while reusing the ONNX Runtime linked by the
|
||||
//! Desktop speech stack.
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::f32::consts::TAU;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use ort::session::{Session, SessionInputValue};
|
||||
use ort::value::{DynValue, Tensor};
|
||||
use rand::{Rng, RngExt};
|
||||
use sentencepiece_model::SentencePieceModel;
|
||||
use serde::Deserialize;
|
||||
use sherpa_onnx::LinearResampler;
|
||||
use tokenizers::models::unigram::Unigram;
|
||||
use tokenizers::pre_tokenizers::metaspace::{Metaspace, PrependScheme};
|
||||
use tokenizers::Tokenizer;
|
||||
|
||||
use super::VoiceStyle;
|
||||
|
||||
const FILE_BUNDLE: &str = "bundle.json";
|
||||
const FILE_MIMI_ENCODER: &str = "mimi_encoder.onnx";
|
||||
const FILE_TEXT_CONDITIONER: &str = "text_conditioner.onnx";
|
||||
const FILE_FLOW_MAIN_INT8: &str = "flow_lm_main_int8.onnx";
|
||||
const FILE_FLOW_INT8: &str = "flow_lm_flow_int8.onnx";
|
||||
const FILE_MIMI_DECODER_INT8: &str = "mimi_decoder_int8.onnx";
|
||||
|
||||
const MODEL_LANGUAGE: &str = "english_2026-04";
|
||||
const DEFAULT_TEMPERATURE: f32 = 0.7;
|
||||
const EOS_LOGIT_THRESHOLD: f32 = -4.0;
|
||||
const DECODER_CHUNK_FRAMES: usize = 12;
|
||||
const TOKENS_PER_SECOND_ESTIMATE: f32 = 3.0;
|
||||
const GENERATION_SECONDS_PADDING: f32 = 2.0;
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct Bundle {
|
||||
schema_version: u32,
|
||||
language: String,
|
||||
sample_rate: usize,
|
||||
frame_rate: f32,
|
||||
samples_per_frame: usize,
|
||||
latent_dim: usize,
|
||||
conditioning_dim: usize,
|
||||
insert_bos_before_voice: bool,
|
||||
pad_with_spaces_for_short_inputs: bool,
|
||||
remove_semicolons: bool,
|
||||
model_recommended_frames_after_eos: Option<usize>,
|
||||
max_token_per_chunk: usize,
|
||||
tokenizer_file: String,
|
||||
bos_before_voice_file: String,
|
||||
flow_lm_state_manifest: Vec<StateSpec>,
|
||||
mimi_state_manifest: Vec<StateSpec>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
struct StateSpec {
|
||||
input_name: String,
|
||||
output_name: String,
|
||||
dtype: StateDtype,
|
||||
shape: Vec<i64>,
|
||||
fill: StateFill,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
enum StateDtype {
|
||||
#[serde(rename = "float32")]
|
||||
Float32,
|
||||
#[serde(rename = "int64")]
|
||||
Int64,
|
||||
Bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
enum StateFill {
|
||||
Empty,
|
||||
Nan,
|
||||
Ones,
|
||||
Zeros,
|
||||
}
|
||||
|
||||
struct StateValue {
|
||||
spec: StateSpec,
|
||||
value: DynValue,
|
||||
}
|
||||
|
||||
struct CachedVoice {
|
||||
samples_ptr: usize,
|
||||
samples_len: usize,
|
||||
sample_rate: i32,
|
||||
embeddings: Vec<f32>,
|
||||
}
|
||||
|
||||
pub(crate) struct AprilPocketTts {
|
||||
bundle: Bundle,
|
||||
tokenizer: Tokenizer,
|
||||
bos_embedding: Vec<f32>,
|
||||
mimi_encoder: Session,
|
||||
text_conditioner: Session,
|
||||
flow_main: Session,
|
||||
flow: Session,
|
||||
mimi_decoder: Session,
|
||||
cached_voice: Option<CachedVoice>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub(crate) struct AprilPreparedPrompt {
|
||||
pub(crate) text: String,
|
||||
pub(crate) frames_after_eos: usize,
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_april_prompt(input: &str) -> Option<AprilPreparedPrompt> {
|
||||
let trimmed = input.trim();
|
||||
if trimmed.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut cleaned = String::with_capacity(trimmed.len());
|
||||
let mut last_was_space = false;
|
||||
for ch in trimmed.chars() {
|
||||
if ch.is_whitespace() {
|
||||
if !last_was_space {
|
||||
cleaned.push(' ');
|
||||
}
|
||||
last_was_space = true;
|
||||
} else {
|
||||
cleaned.push(ch);
|
||||
last_was_space = false;
|
||||
}
|
||||
}
|
||||
|
||||
let first = cleaned.chars().next().expect("cleaned non-empty above");
|
||||
if first.is_lowercase() {
|
||||
let upper: String = first.to_uppercase().collect();
|
||||
let mut iter = cleaned.chars();
|
||||
iter.next();
|
||||
cleaned = upper + iter.as_str();
|
||||
}
|
||||
|
||||
let last = cleaned
|
||||
.chars()
|
||||
.next_back()
|
||||
.expect("cleaned non-empty above");
|
||||
if last.is_alphanumeric() {
|
||||
cleaned.push('.');
|
||||
}
|
||||
|
||||
let word_count = cleaned.split_whitespace().count();
|
||||
Some(AprilPreparedPrompt {
|
||||
text: cleaned,
|
||||
// Mirror the bundle's upstream heuristic: three generated frames plus
|
||||
// two trailing frames for short prompts, one plus two otherwise.
|
||||
frames_after_eos: if word_count <= 4 { 5 } else { 3 },
|
||||
})
|
||||
}
|
||||
|
||||
impl AprilPocketTts {
|
||||
pub(crate) fn load(dir: &Path, num_threads: usize) -> Result<Self, String> {
|
||||
if num_threads == 0 {
|
||||
return Err("Pocket TTS num_threads must be at least 1".to_string());
|
||||
}
|
||||
let bundle_path = dir.join(FILE_BUNDLE);
|
||||
let bundle: Bundle = serde_json::from_slice(
|
||||
&fs::read(&bundle_path)
|
||||
.map_err(|err| format!("read {}: {err}", bundle_path.display()))?,
|
||||
)
|
||||
.map_err(|err| format!("parse {}: {err}", bundle_path.display()))?;
|
||||
|
||||
if bundle.schema_version != 2 {
|
||||
return Err(format!(
|
||||
"unsupported Pocket TTS bundle schema {} in {}",
|
||||
bundle.schema_version,
|
||||
bundle_path.display()
|
||||
));
|
||||
}
|
||||
if bundle.language != MODEL_LANGUAGE {
|
||||
return Err(format!(
|
||||
"expected Pocket TTS language {MODEL_LANGUAGE}, got {}",
|
||||
bundle.language
|
||||
));
|
||||
}
|
||||
if bundle.sample_rate != 24_000
|
||||
|| bundle.frame_rate != 12.5
|
||||
|| bundle.samples_per_frame != 1_920
|
||||
|| bundle.latent_dim != 32
|
||||
|| bundle.conditioning_dim != 1024
|
||||
{
|
||||
return Err(format!(
|
||||
"unexpected Pocket TTS dimensions: sample_rate={}, frame_rate={}, samples_per_frame={}, latent_dim={}, conditioning_dim={}",
|
||||
bundle.sample_rate,
|
||||
bundle.frame_rate,
|
||||
bundle.samples_per_frame,
|
||||
bundle.latent_dim,
|
||||
bundle.conditioning_dim
|
||||
));
|
||||
}
|
||||
if !bundle.insert_bos_before_voice {
|
||||
return Err("April Pocket TTS bundle must insert BOS before voice".to_string());
|
||||
}
|
||||
if bundle.pad_with_spaces_for_short_inputs
|
||||
|| bundle.remove_semicolons
|
||||
|| bundle.model_recommended_frames_after_eos.is_some()
|
||||
|| bundle.max_token_per_chunk != 50
|
||||
{
|
||||
return Err("unsupported April Pocket TTS prompt-policy metadata".to_string());
|
||||
}
|
||||
|
||||
let tokenizer_path = dir.join(&bundle.tokenizer_file);
|
||||
let tokenizer = load_tokenizer(&tokenizer_path)?;
|
||||
let bos_path = dir.join(&bundle.bos_before_voice_file);
|
||||
let bos_embedding = read_npy_f32(&bos_path)?;
|
||||
if bos_embedding.len() != bundle.conditioning_dim {
|
||||
return Err(format!(
|
||||
"{} has {} values; expected {}",
|
||||
bos_path.display(),
|
||||
bos_embedding.len(),
|
||||
bundle.conditioning_dim
|
||||
));
|
||||
}
|
||||
|
||||
let flow_main = FILE_FLOW_MAIN_INT8;
|
||||
let flow = FILE_FLOW_INT8;
|
||||
let mimi_decoder = FILE_MIMI_DECODER_INT8;
|
||||
|
||||
Ok(Self {
|
||||
// The INT8 layout quantizes only the three generation graphs;
|
||||
// voice encoding and text conditioning remain full precision.
|
||||
mimi_encoder: load_session(dir.join(FILE_MIMI_ENCODER), num_threads)?,
|
||||
text_conditioner: load_session(dir.join(FILE_TEXT_CONDITIONER), num_threads)?,
|
||||
flow_main: load_session(dir.join(flow_main), num_threads)?,
|
||||
flow: load_session(dir.join(flow), num_threads)?,
|
||||
mimi_decoder: load_session(dir.join(mimi_decoder), num_threads)?,
|
||||
bundle,
|
||||
tokenizer,
|
||||
bos_embedding,
|
||||
cached_voice: None,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn split_prompt(
|
||||
&self,
|
||||
prepared: &AprilPreparedPrompt,
|
||||
) -> Result<Vec<String>, String> {
|
||||
if self.token_count(&prepared.text)? <= self.bundle.max_token_per_chunk {
|
||||
return Ok(vec![prepared.text.clone()]);
|
||||
}
|
||||
|
||||
let mut chunks = Vec::new();
|
||||
let mut current = String::new();
|
||||
for word in prepared.text.split_whitespace() {
|
||||
let candidate = if current.is_empty() {
|
||||
word.to_string()
|
||||
} else {
|
||||
format!("{current} {word}")
|
||||
};
|
||||
if self.prepared_token_count(&candidate)? <= self.bundle.max_token_per_chunk {
|
||||
current = candidate;
|
||||
continue;
|
||||
}
|
||||
if !current.is_empty() {
|
||||
chunks.push(std::mem::take(&mut current));
|
||||
}
|
||||
|
||||
if self.prepared_token_count(word)? <= self.bundle.max_token_per_chunk {
|
||||
current = word.to_string();
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut fragment = String::new();
|
||||
for ch in word.chars() {
|
||||
let candidate = format!("{fragment}{ch}");
|
||||
if !fragment.is_empty()
|
||||
&& self.prepared_token_count(&candidate)? > self.bundle.max_token_per_chunk
|
||||
{
|
||||
chunks.push(std::mem::take(&mut fragment));
|
||||
}
|
||||
fragment.push(ch);
|
||||
}
|
||||
current = fragment;
|
||||
}
|
||||
if !current.is_empty() {
|
||||
chunks.push(current);
|
||||
}
|
||||
|
||||
chunks
|
||||
.into_iter()
|
||||
.map(|text| {
|
||||
let chunk = prepare_april_prompt(&text)
|
||||
.ok_or_else(|| "Pocket TTS prompt chunk became empty".to_string())?;
|
||||
let token_count = self.token_count(&chunk.text)?;
|
||||
if token_count > self.bundle.max_token_per_chunk {
|
||||
return Err(format!(
|
||||
"Pocket TTS prompt chunk has {token_count} tokens; maximum is {}",
|
||||
self.bundle.max_token_per_chunk
|
||||
));
|
||||
}
|
||||
Ok(chunk.text)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) fn synth_chunk(
|
||||
&mut self,
|
||||
prepared: &AprilPreparedPrompt,
|
||||
style: &VoiceStyle,
|
||||
) -> Result<Vec<f32>, String> {
|
||||
let voice_embeddings = self.voice_embeddings(style)?;
|
||||
let mut flow_state = self.condition_voice(&voice_embeddings)?;
|
||||
let token_ids = self
|
||||
.tokenizer
|
||||
.encode(prepared.text.as_str(), false)
|
||||
.map_err(|err| format!("tokenize Pocket TTS prompt: {err}"))?
|
||||
.get_ids()
|
||||
.iter()
|
||||
.copied()
|
||||
.map(i64::from)
|
||||
.collect::<Vec<_>>();
|
||||
if token_ids.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
if token_ids.len() > self.bundle.max_token_per_chunk {
|
||||
return Err(format!(
|
||||
"Pocket TTS prompt has {} tokens; split_text_into_chunks maximum is {}",
|
||||
token_ids.len(),
|
||||
self.bundle.max_token_per_chunk
|
||||
));
|
||||
}
|
||||
|
||||
let token_count = token_ids.len();
|
||||
let text_embeddings = self.text_embeddings(token_ids)?;
|
||||
self.run_flow_main_prefix(&text_embeddings, &mut flow_state)?;
|
||||
let max_frames = estimate_max_frames(token_count, self.bundle.frame_rate);
|
||||
let latents =
|
||||
self.generate_latents(max_frames, prepared.frames_after_eos, &mut flow_state)?;
|
||||
self.decode_latents(&latents)
|
||||
}
|
||||
|
||||
fn prepared_token_count(&self, text: &str) -> Result<usize, String> {
|
||||
let prepared = prepare_april_prompt(text)
|
||||
.ok_or_else(|| "Pocket TTS prompt chunk became empty".to_string())?;
|
||||
self.token_count(&prepared.text)
|
||||
}
|
||||
|
||||
fn token_count(&self, text: &str) -> Result<usize, String> {
|
||||
Ok(self
|
||||
.tokenizer
|
||||
.encode(text, false)
|
||||
.map_err(|err| format!("tokenize Pocket TTS prompt: {err}"))?
|
||||
.get_ids()
|
||||
.len())
|
||||
}
|
||||
|
||||
fn voice_embeddings(&mut self, style: &VoiceStyle) -> Result<Vec<f32>, String> {
|
||||
let key = (
|
||||
style.samples.as_ptr() as usize,
|
||||
style.samples.len(),
|
||||
style.sample_rate,
|
||||
);
|
||||
if let Some(cached) = &self.cached_voice {
|
||||
if (cached.samples_ptr, cached.samples_len, cached.sample_rate) == key {
|
||||
return Ok(cached.embeddings.clone());
|
||||
}
|
||||
}
|
||||
|
||||
let samples = if style.sample_rate == self.bundle.sample_rate as i32 {
|
||||
style.samples.clone()
|
||||
} else {
|
||||
LinearResampler::create(style.sample_rate, self.bundle.sample_rate as i32)
|
||||
.ok_or_else(|| {
|
||||
format!(
|
||||
"create Pocket TTS resampler {}Hz -> {}Hz",
|
||||
style.sample_rate, self.bundle.sample_rate
|
||||
)
|
||||
})?
|
||||
.resample(&style.samples, true)
|
||||
};
|
||||
let audio = Tensor::from_array((
|
||||
vec![1_i64, 1, samples.len() as i64],
|
||||
samples.into_boxed_slice(),
|
||||
))
|
||||
.map_err(ort_error("create voice audio tensor"))?;
|
||||
let outputs = self
|
||||
.mimi_encoder
|
||||
.run(ort::inputs!["audio" => audio])
|
||||
.map_err(ort_error("run Mimi encoder"))?;
|
||||
let (_, encoded) = outputs[0]
|
||||
.try_extract_tensor::<f32>()
|
||||
.map_err(ort_error("extract Mimi encoder output"))?;
|
||||
if !encoded.len().is_multiple_of(self.bundle.conditioning_dim) {
|
||||
return Err(format!(
|
||||
"Mimi encoder returned {} values, not divisible by {}",
|
||||
encoded.len(),
|
||||
self.bundle.conditioning_dim
|
||||
));
|
||||
}
|
||||
let mut embeddings =
|
||||
Vec::with_capacity(self.bos_embedding.len().saturating_add(encoded.len()));
|
||||
embeddings.extend_from_slice(&self.bos_embedding);
|
||||
embeddings.extend_from_slice(encoded);
|
||||
self.cached_voice = Some(CachedVoice {
|
||||
samples_ptr: key.0,
|
||||
samples_len: key.1,
|
||||
sample_rate: key.2,
|
||||
embeddings: embeddings.clone(),
|
||||
});
|
||||
Ok(embeddings)
|
||||
}
|
||||
|
||||
fn condition_voice(&mut self, embeddings: &[f32]) -> Result<Vec<StateValue>, String> {
|
||||
let frames = embeddings.len() / self.bundle.conditioning_dim;
|
||||
let sequence = Tensor::<f32>::new(
|
||||
&ort::memory::Allocator::default(),
|
||||
[1_i64, 0, self.bundle.latent_dim as i64],
|
||||
)
|
||||
.map_err(ort_error("create empty voice sequence"))?;
|
||||
let text_embeddings = Tensor::from_array((
|
||||
vec![1_i64, frames as i64, self.bundle.conditioning_dim as i64],
|
||||
embeddings.to_vec().into_boxed_slice(),
|
||||
))
|
||||
.map_err(ort_error("create voice embedding tensor"))?;
|
||||
let mut state = initialize_state(&self.bundle.flow_lm_state_manifest)?;
|
||||
let mut inputs = vec![
|
||||
(Cow::Borrowed("sequence"), SessionInputValue::from(sequence)),
|
||||
(
|
||||
Cow::Borrowed("text_embeddings"),
|
||||
SessionInputValue::from(text_embeddings),
|
||||
),
|
||||
];
|
||||
append_state_inputs(&mut inputs, &state);
|
||||
let mut outputs = self
|
||||
.flow_main
|
||||
.run(inputs)
|
||||
.map_err(ort_error("condition Pocket TTS voice"))?;
|
||||
replace_state_from_outputs(&mut state, &mut outputs)?;
|
||||
Ok(state)
|
||||
}
|
||||
|
||||
fn text_embeddings(&mut self, token_ids: Vec<i64>) -> Result<Vec<f32>, String> {
|
||||
let tokens = Tensor::from_array((
|
||||
vec![1_i64, token_ids.len() as i64],
|
||||
token_ids.into_boxed_slice(),
|
||||
))
|
||||
.map_err(ort_error("create token tensor"))?;
|
||||
let outputs = self
|
||||
.text_conditioner
|
||||
.run(ort::inputs!["token_ids" => tokens])
|
||||
.map_err(ort_error("run text conditioner"))?;
|
||||
let (_, embeddings) = outputs[0]
|
||||
.try_extract_tensor::<f32>()
|
||||
.map_err(ort_error("extract text embeddings"))?;
|
||||
Ok(embeddings.to_vec())
|
||||
}
|
||||
|
||||
fn run_flow_main_prefix(
|
||||
&mut self,
|
||||
text_embeddings: &[f32],
|
||||
state: &mut [StateValue],
|
||||
) -> Result<(), String> {
|
||||
if !text_embeddings
|
||||
.len()
|
||||
.is_multiple_of(self.bundle.conditioning_dim)
|
||||
{
|
||||
return Err(format!(
|
||||
"text conditioner returned {} values, not divisible by {}",
|
||||
text_embeddings.len(),
|
||||
self.bundle.conditioning_dim
|
||||
));
|
||||
}
|
||||
let frames = text_embeddings.len() / self.bundle.conditioning_dim;
|
||||
let sequence = Tensor::<f32>::new(
|
||||
&ort::memory::Allocator::default(),
|
||||
[1_i64, 0, self.bundle.latent_dim as i64],
|
||||
)
|
||||
.map_err(ort_error("create empty text sequence"))?;
|
||||
let text_embeddings = Tensor::from_array((
|
||||
vec![1_i64, frames as i64, self.bundle.conditioning_dim as i64],
|
||||
text_embeddings.to_vec().into_boxed_slice(),
|
||||
))
|
||||
.map_err(ort_error("create text embedding tensor"))?;
|
||||
let mut inputs = vec![
|
||||
(Cow::Borrowed("sequence"), SessionInputValue::from(sequence)),
|
||||
(
|
||||
Cow::Borrowed("text_embeddings"),
|
||||
SessionInputValue::from(text_embeddings),
|
||||
),
|
||||
];
|
||||
append_state_inputs(&mut inputs, state);
|
||||
let mut outputs = self
|
||||
.flow_main
|
||||
.run(inputs)
|
||||
.map_err(ort_error("prime Pocket TTS text state"))?;
|
||||
replace_state_from_outputs(state, &mut outputs)
|
||||
}
|
||||
|
||||
fn generate_latents(
|
||||
&mut self,
|
||||
max_frames: usize,
|
||||
frames_after_eos: usize,
|
||||
state: &mut [StateValue],
|
||||
) -> Result<Vec<f32>, String> {
|
||||
let mut current = vec![f32::NAN; self.bundle.latent_dim];
|
||||
let mut latents = Vec::with_capacity(max_frames * self.bundle.latent_dim);
|
||||
let mut eos_step = None;
|
||||
let mut rng = rand::rng();
|
||||
|
||||
for step in 0..max_frames {
|
||||
let sequence = Tensor::from_array((
|
||||
vec![1_i64, 1, self.bundle.latent_dim as i64],
|
||||
current.clone().into_boxed_slice(),
|
||||
))
|
||||
.map_err(ort_error("create latent input"))?;
|
||||
let text_embeddings = Tensor::<f32>::new(
|
||||
&ort::memory::Allocator::default(),
|
||||
[1_i64, 0, self.bundle.conditioning_dim as i64],
|
||||
)
|
||||
.map_err(ort_error("create empty text input"))?;
|
||||
let mut inputs = vec![
|
||||
(Cow::Borrowed("sequence"), SessionInputValue::from(sequence)),
|
||||
(
|
||||
Cow::Borrowed("text_embeddings"),
|
||||
SessionInputValue::from(text_embeddings),
|
||||
),
|
||||
];
|
||||
append_state_inputs(&mut inputs, state);
|
||||
let mut outputs = self
|
||||
.flow_main
|
||||
.run(inputs)
|
||||
.map_err(ort_error("run Pocket TTS Flow LM"))?;
|
||||
let conditioning = outputs[0]
|
||||
.try_extract_tensor::<f32>()
|
||||
.map_err(ort_error("extract Flow LM conditioning"))?
|
||||
.1
|
||||
.to_vec();
|
||||
let eos_logit = outputs[1]
|
||||
.try_extract_tensor::<f32>()
|
||||
.map_err(ort_error("extract Flow LM EOS logit"))?
|
||||
.1
|
||||
.first()
|
||||
.copied()
|
||||
.ok_or_else(|| "Flow LM returned empty EOS logit".to_string())?;
|
||||
replace_state_from_outputs(state, &mut outputs)?;
|
||||
|
||||
if eos_logit > EOS_LOGIT_THRESHOLD && eos_step.is_none() {
|
||||
eos_step = Some(step);
|
||||
}
|
||||
if eos_step.is_some_and(|eos| step >= eos + frames_after_eos) {
|
||||
break;
|
||||
}
|
||||
|
||||
let mut noise =
|
||||
normal_noise(&mut rng, self.bundle.latent_dim, DEFAULT_TEMPERATURE.sqrt());
|
||||
let conditioning = Tensor::from_array((
|
||||
vec![1_i64, self.bundle.conditioning_dim as i64],
|
||||
conditioning.into_boxed_slice(),
|
||||
))
|
||||
.map_err(ort_error("create flow conditioning"))?;
|
||||
let s = Tensor::from_array((vec![1_i64, 1], vec![0.0_f32].into_boxed_slice()))
|
||||
.map_err(ort_error("create flow start tensor"))?;
|
||||
let t = Tensor::from_array((vec![1_i64, 1], vec![1.0_f32].into_boxed_slice()))
|
||||
.map_err(ort_error("create flow end tensor"))?;
|
||||
let x = Tensor::from_array((
|
||||
vec![1_i64, self.bundle.latent_dim as i64],
|
||||
noise.clone().into_boxed_slice(),
|
||||
))
|
||||
.map_err(ort_error("create flow noise tensor"))?;
|
||||
let outputs = self
|
||||
.flow
|
||||
.run(ort::inputs![
|
||||
"c" => conditioning,
|
||||
"s" => s,
|
||||
"t" => t,
|
||||
"x" => x,
|
||||
])
|
||||
.map_err(ort_error("run Pocket TTS flow"))?;
|
||||
let flow = outputs[0]
|
||||
.try_extract_tensor::<f32>()
|
||||
.map_err(ort_error("extract Pocket TTS flow"))?
|
||||
.1;
|
||||
if flow.len() != noise.len() {
|
||||
return Err(format!(
|
||||
"flow returned {} values; expected {}",
|
||||
flow.len(),
|
||||
noise.len()
|
||||
));
|
||||
}
|
||||
for (sample, delta) in noise.iter_mut().zip(flow) {
|
||||
*sample += *delta;
|
||||
}
|
||||
current.clone_from(&noise);
|
||||
latents.extend_from_slice(&noise);
|
||||
}
|
||||
Ok(latents)
|
||||
}
|
||||
|
||||
fn decode_latents(&mut self, latents: &[f32]) -> Result<Vec<f32>, String> {
|
||||
if latents.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
if !latents.len().is_multiple_of(self.bundle.latent_dim) {
|
||||
return Err(format!(
|
||||
"latent buffer has {} values, not divisible by {}",
|
||||
latents.len(),
|
||||
self.bundle.latent_dim
|
||||
));
|
||||
}
|
||||
let frame_count = latents.len() / self.bundle.latent_dim;
|
||||
let mut state = initialize_state(&self.bundle.mimi_state_manifest)?;
|
||||
let mut audio = Vec::new();
|
||||
|
||||
for start in (0..frame_count).step_by(DECODER_CHUNK_FRAMES) {
|
||||
let end = (start + DECODER_CHUNK_FRAMES).min(frame_count);
|
||||
let values =
|
||||
latents[start * self.bundle.latent_dim..end * self.bundle.latent_dim].to_vec();
|
||||
let latent = Tensor::from_array((
|
||||
vec![1_i64, (end - start) as i64, self.bundle.latent_dim as i64],
|
||||
values.into_boxed_slice(),
|
||||
))
|
||||
.map_err(ort_error("create Mimi latent tensor"))?;
|
||||
let mut inputs = vec![(Cow::Borrowed("latent"), SessionInputValue::from(latent))];
|
||||
append_state_inputs(&mut inputs, &state);
|
||||
let mut outputs = self
|
||||
.mimi_decoder
|
||||
.run(inputs)
|
||||
.map_err(ort_error("run Mimi decoder"))?;
|
||||
let samples = outputs[0]
|
||||
.try_extract_tensor::<f32>()
|
||||
.map_err(ort_error("extract Mimi audio"))?
|
||||
.1;
|
||||
audio.extend_from_slice(samples);
|
||||
replace_state_from_outputs(&mut state, &mut outputs)?;
|
||||
}
|
||||
Ok(audio)
|
||||
}
|
||||
}
|
||||
|
||||
fn load_session(path: PathBuf, num_threads: usize) -> Result<Session, String> {
|
||||
if !path.is_file() {
|
||||
return Err(format!("missing Pocket TTS file: {}", path.display()));
|
||||
}
|
||||
Session::builder()
|
||||
.map_err(ort_error("create ONNX session builder"))?
|
||||
.with_intra_threads(num_threads)
|
||||
.map_err(|err| format!("configure ONNX intra-op threads: {err}"))?
|
||||
.with_inter_threads(1)
|
||||
.map_err(|err| format!("configure ONNX inter-op threads: {err}"))?
|
||||
.commit_from_file(&path)
|
||||
.map_err(|err| format!("load {}: {err}", path.display()))
|
||||
}
|
||||
|
||||
fn load_tokenizer(path: &Path) -> Result<Tokenizer, String> {
|
||||
let sentencepiece = SentencePieceModel::from_file(path)
|
||||
.map_err(|err| format!("load {}: {err}", path.display()))?;
|
||||
let trainer = sentencepiece
|
||||
.trainer()
|
||||
.ok_or_else(|| format!("{} has no SentencePiece trainer metadata", path.display()))?;
|
||||
let normalizer = sentencepiece.normalizer().ok_or_else(|| {
|
||||
format!(
|
||||
"{} has no SentencePiece normalizer metadata",
|
||||
path.display()
|
||||
)
|
||||
})?;
|
||||
if normalizer.name() != "identity" {
|
||||
return Err(format!(
|
||||
"{} uses unsupported SentencePiece normalizer {:?}",
|
||||
path.display(),
|
||||
normalizer.name()
|
||||
));
|
||||
}
|
||||
|
||||
let vocab = sentencepiece
|
||||
.pieces()
|
||||
.iter()
|
||||
.map(|piece| (piece.piece().to_owned(), f64::from(piece.score())))
|
||||
.collect();
|
||||
let mut tokenizer = Tokenizer::new(
|
||||
Unigram::from(
|
||||
vocab,
|
||||
Some(trainer.unk_id() as usize),
|
||||
trainer.byte_fallback(),
|
||||
)
|
||||
.map_err(|err| format!("construct tokenizer from {}: {err}", path.display()))?,
|
||||
);
|
||||
// SentencePiece's identity normalizer still escapes spaces as U+2581 and
|
||||
// prepends one marker to the input before unigram segmentation.
|
||||
tokenizer.with_pre_tokenizer(Some(Metaspace::new('▁', PrependScheme::Always, false)));
|
||||
Ok(tokenizer)
|
||||
}
|
||||
|
||||
fn initialize_state(specs: &[StateSpec]) -> Result<Vec<StateValue>, String> {
|
||||
specs
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|spec| {
|
||||
let len = shape_len(&spec.shape)?;
|
||||
let value = match spec.dtype {
|
||||
StateDtype::Float32 => {
|
||||
let fill = match spec.fill {
|
||||
StateFill::Nan => f32::NAN,
|
||||
StateFill::Empty | StateFill::Zeros => 0.0,
|
||||
StateFill::Ones => 1.0,
|
||||
};
|
||||
if len == 0 {
|
||||
Tensor::<f32>::new(&ort::memory::Allocator::default(), spec.shape.clone())
|
||||
.map_err(ort_error("create empty float state tensor"))?
|
||||
.into_dyn()
|
||||
} else {
|
||||
Tensor::from_array((spec.shape.clone(), vec![fill; len].into_boxed_slice()))
|
||||
.map_err(ort_error("create float state tensor"))?
|
||||
.into_dyn()
|
||||
}
|
||||
}
|
||||
StateDtype::Int64 => {
|
||||
let fill = i64::from(matches!(spec.fill, StateFill::Ones));
|
||||
if len == 0 {
|
||||
Tensor::<i64>::new(&ort::memory::Allocator::default(), spec.shape.clone())
|
||||
.map_err(ort_error("create empty integer state tensor"))?
|
||||
.into_dyn()
|
||||
} else {
|
||||
Tensor::from_array((spec.shape.clone(), vec![fill; len].into_boxed_slice()))
|
||||
.map_err(ort_error("create integer state tensor"))?
|
||||
.into_dyn()
|
||||
}
|
||||
}
|
||||
StateDtype::Bool => {
|
||||
let fill = matches!(spec.fill, StateFill::Ones);
|
||||
if len == 0 {
|
||||
Tensor::<bool>::new(&ort::memory::Allocator::default(), spec.shape.clone())
|
||||
.map_err(ort_error("create empty bool state tensor"))?
|
||||
.into_dyn()
|
||||
} else {
|
||||
Tensor::from_array((spec.shape.clone(), vec![fill; len].into_boxed_slice()))
|
||||
.map_err(ort_error("create bool state tensor"))?
|
||||
.into_dyn()
|
||||
}
|
||||
}
|
||||
};
|
||||
Ok(StateValue { spec, value })
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn append_state_inputs<'a>(
|
||||
inputs: &mut Vec<(Cow<'a, str>, SessionInputValue<'a>)>,
|
||||
state: &'a [StateValue],
|
||||
) {
|
||||
for value in state {
|
||||
inputs.push((
|
||||
Cow::Borrowed(value.spec.input_name.as_str()),
|
||||
SessionInputValue::from(&value.value),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
fn replace_state_from_outputs(
|
||||
state: &mut [StateValue],
|
||||
outputs: &mut ort::session::SessionOutputs<'_>,
|
||||
) -> Result<(), String> {
|
||||
for value in state {
|
||||
value.value = outputs
|
||||
.remove(&value.spec.output_name)
|
||||
.ok_or_else(|| format!("missing state output {}", value.spec.output_name))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn shape_len(shape: &[i64]) -> Result<usize, String> {
|
||||
shape.iter().try_fold(1_usize, |len, &dim| {
|
||||
let dim = usize::try_from(dim).map_err(|_| format!("negative state dimension {dim}"))?;
|
||||
len.checked_mul(dim)
|
||||
.ok_or_else(|| format!("state shape overflows usize: {shape:?}"))
|
||||
})
|
||||
}
|
||||
|
||||
fn estimate_max_frames(token_count: usize, frame_rate: f32) -> usize {
|
||||
((token_count as f32 / TOKENS_PER_SECOND_ESTIMATE + GENERATION_SECONDS_PADDING) * frame_rate)
|
||||
.ceil() as usize
|
||||
}
|
||||
|
||||
fn normal_noise(rng: &mut impl Rng, len: usize, std_dev: f32) -> Vec<f32> {
|
||||
let mut out = Vec::with_capacity(len);
|
||||
while out.len() < len {
|
||||
let u1 = rng.random::<f32>().max(f32::MIN_POSITIVE);
|
||||
let u2 = rng.random::<f32>();
|
||||
let radius = (-2.0_f32 * u1.ln()).sqrt() * std_dev;
|
||||
out.push(radius * (TAU * u2).cos());
|
||||
if out.len() < len {
|
||||
out.push(radius * (TAU * u2).sin());
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn read_npy_f32(path: &Path) -> Result<Vec<f32>, String> {
|
||||
let bytes = fs::read(path).map_err(|err| format!("read {}: {err}", path.display()))?;
|
||||
if bytes.len() < 10 || &bytes[..6] != b"\x93NUMPY" {
|
||||
return Err(format!("{} is not a NumPy array", path.display()));
|
||||
}
|
||||
let major = bytes[6];
|
||||
let header_len_bytes = match major {
|
||||
1 => 2,
|
||||
2 | 3 => 4,
|
||||
_ => {
|
||||
return Err(format!(
|
||||
"unsupported NumPy version {major} in {}",
|
||||
path.display()
|
||||
))
|
||||
}
|
||||
};
|
||||
let header_start = 8 + header_len_bytes;
|
||||
if bytes.len() < header_start {
|
||||
return Err(format!("truncated NumPy header in {}", path.display()));
|
||||
}
|
||||
let header_len = if header_len_bytes == 2 {
|
||||
u16::from_le_bytes([bytes[8], bytes[9]]) as usize
|
||||
} else {
|
||||
u32::from_le_bytes([bytes[8], bytes[9], bytes[10], bytes[11]]) as usize
|
||||
};
|
||||
let data_start = header_start
|
||||
.checked_add(header_len)
|
||||
.ok_or_else(|| format!("NumPy header overflow in {}", path.display()))?;
|
||||
if data_start > bytes.len() {
|
||||
return Err(format!("truncated NumPy data in {}", path.display()));
|
||||
}
|
||||
let header = std::str::from_utf8(&bytes[header_start..data_start])
|
||||
.map_err(|err| format!("invalid NumPy header in {}: {err}", path.display()))?;
|
||||
if !(header.contains("'descr': '<f4'") || header.contains("\"descr\": \"<f4\""))
|
||||
|| header.contains("fortran_order': True")
|
||||
|| header.contains("fortran_order\": true")
|
||||
{
|
||||
return Err(format!(
|
||||
"{} must be a little-endian, C-order float32 NumPy array",
|
||||
path.display()
|
||||
));
|
||||
}
|
||||
let data = &bytes[data_start..];
|
||||
if data.len() % 4 != 0 {
|
||||
return Err(format!("misaligned float32 data in {}", path.display()));
|
||||
}
|
||||
Ok(data
|
||||
.chunks_exact(4)
|
||||
.map(|chunk| f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]))
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn ort_error(context: &'static str) -> impl FnOnce(ort::Error) -> String {
|
||||
move |err| format!("{context}: {err}")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn shape_len_supports_empty_state_dimensions() {
|
||||
assert_eq!(shape_len(&[1, 128, 0]).expect("shape"), 0);
|
||||
assert_eq!(shape_len(&[2, 1, 8, 1000, 64]).expect("shape"), 1_024_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normal_noise_has_requested_length() {
|
||||
let mut rng = rand::rng();
|
||||
assert_eq!(normal_noise(&mut rng, 1, 1.0).len(), 1);
|
||||
assert_eq!(normal_noise(&mut rng, 32, 1.0).len(), 32);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generation_frame_estimate_scales_with_token_count() {
|
||||
assert_eq!(estimate_max_frames(3, 12.5), 38);
|
||||
assert_eq!(estimate_max_frames(300, 12.5), 1_275);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires BUZZ_POCKET_TEST_MODEL_DIR"]
|
||||
fn tokenizer_matches_sentencepiece_reference_including_unknown_words() {
|
||||
let dir = std::env::var("BUZZ_POCKET_TEST_MODEL_DIR")
|
||||
.expect("set BUZZ_POCKET_TEST_MODEL_DIR to the verified April bundle");
|
||||
let tokenizer =
|
||||
load_tokenizer(&Path::new(&dir).join("tokenizer.model")).expect("load April tokenizer");
|
||||
let cases: &[(&str, &[u32])] = &[
|
||||
("Yep.", &[2462, 263]),
|
||||
("Hello there.", &[2994, 310, 263]),
|
||||
(
|
||||
"quizzaciously xyzzy.",
|
||||
&[
|
||||
260, 1157, 1818, 362, 1814, 323, 260, 568, 327, 1818, 327, 263,
|
||||
],
|
||||
),
|
||||
("I'm listening.", &[268, 264, 283, 260, 604, 273, 263]),
|
||||
];
|
||||
for (text, expected) in cases {
|
||||
let encoding = tokenizer.encode(*text, false).expect("tokenize");
|
||||
assert_eq!(encoding.get_ids(), *expected, "{text}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires BUZZ_POCKET_TEST_MODEL_DIR"]
|
||||
fn loader_splits_oversized_prompts_at_bundle_token_limit() {
|
||||
let dir = std::env::var("BUZZ_POCKET_TEST_MODEL_DIR")
|
||||
.expect("set BUZZ_POCKET_TEST_MODEL_DIR to the verified April bundle");
|
||||
let engine = AprilPocketTts::load(Path::new(&dir), 1).expect("load April bundle");
|
||||
let text = "This deliberately long sentence repeats ordinary English words so the exact SentencePiece token limit is exercised without relying on punctuation, and it keeps adding more material until the prompt must be divided into multiple independently safe generation chunks before the recurrent state cache can be exhausted.";
|
||||
let prepared = prepare_april_prompt(text).expect("prepare prompt");
|
||||
let chunks = engine.split_prompt(&prepared).expect("split prompt");
|
||||
|
||||
assert!(chunks.len() > 1);
|
||||
assert!(chunks.iter().all(|chunk| {
|
||||
engine.token_count(chunk).expect("tokenize chunk") <= engine.bundle.max_token_per_chunk
|
||||
}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires BUZZ_POCKET_TEST_MODEL_DIR"]
|
||||
fn gary_provost_long_sentence_respects_bundle_token_limit() {
|
||||
let dir = std::env::var("BUZZ_POCKET_TEST_MODEL_DIR")
|
||||
.expect("set BUZZ_POCKET_TEST_MODEL_DIR to the verified April bundle");
|
||||
let engine = AprilPocketTts::load(Path::new(&dir), 1).expect("load April bundle");
|
||||
let text = "And sometimes, when I am certain the reader is rested, I will engage him with a sentence of considerable length, a sentence that burns with energy and builds with all the impetus of a crescendo, the roll of the drums, the crash of the cymbals–sounds that say listen to this, it is important.";
|
||||
let prepared = prepare_april_prompt(text).expect("prepare prompt");
|
||||
let chunks = engine.split_prompt(&prepared).expect("split long sentence");
|
||||
let token_counts: Vec<_> = chunks
|
||||
.iter()
|
||||
.map(|chunk| engine.token_count(chunk).expect("count tokens"))
|
||||
.collect();
|
||||
|
||||
assert_eq!(
|
||||
chunks,
|
||||
[
|
||||
"And sometimes, when I am certain the reader is rested, I will engage him with a sentence of considerable length, a sentence that burns with energy and builds with all the.",
|
||||
"Impetus of a crescendo, the roll of the drums, the crash of the cymbals–sounds that say listen to this, it is important.",
|
||||
]
|
||||
);
|
||||
assert_eq!(token_counts, [48, 44]);
|
||||
}
|
||||
}
|
||||
@@ -1,130 +0,0 @@
|
||||
//! Immutable capabilities for Buzz Desktop's April Pocket TTS bundle.
|
||||
|
||||
/// Pinned upstream export repository.
|
||||
pub const APRIL_MODEL_ID: &str = "KevinAHM/pocket-tts-onnx";
|
||||
|
||||
/// Pinned revision containing the `english_2026-04` bundle.
|
||||
pub const APRIL_MODEL_REVISION: &str = "58a6d00cf13d239b6748cb0769f35c580a8f606c";
|
||||
|
||||
/// Language bundle selected from the pinned export.
|
||||
pub const APRIL_BUNDLE_ID: &str = "english_2026-04";
|
||||
|
||||
/// Maximum input size declared by the April bundle.
|
||||
pub const APRIL_MAX_TOKEN_PER_CHUNK: usize = 50;
|
||||
|
||||
/// One immutable artifact required by the April INT8 runtime.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct PocketModelArtifact {
|
||||
pub filename: &'static str,
|
||||
pub sha256: &'static str,
|
||||
pub size_bytes: u64,
|
||||
pub quantized: bool,
|
||||
}
|
||||
|
||||
/// Capabilities of Buzz Desktop's sole Pocket model.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct PocketModelInfo {
|
||||
pub bundle_id: &'static str,
|
||||
pub source_model_id: &'static str,
|
||||
pub revision: &'static str,
|
||||
pub sample_rate: u32,
|
||||
pub max_token_per_chunk: usize,
|
||||
pub artifacts: &'static [PocketModelArtifact],
|
||||
pub quantized_components: &'static [&'static str],
|
||||
}
|
||||
|
||||
const INT8_ARTIFACTS: [PocketModelArtifact; 8] = [
|
||||
PocketModelArtifact {
|
||||
filename: "bundle.json",
|
||||
sha256: "bab643150f437f37df080a710520ff39ed9ebd9a339f8ebdc739f7eddfc28b3f",
|
||||
size_bytes: 24_381,
|
||||
quantized: false,
|
||||
},
|
||||
PocketModelArtifact {
|
||||
filename: "bos_before_voice.npy",
|
||||
sha256: "f46edf4f7007b7ba4ea58831f49d003e59e167b4641c44bb3addfe9231a780b1",
|
||||
size_bytes: 4_224,
|
||||
quantized: false,
|
||||
},
|
||||
PocketModelArtifact {
|
||||
filename: "tokenizer.model",
|
||||
sha256: "d461765ae179566678c93091c5fa6f2984c31bbe990bf1aa62d92c64d91bc3f6",
|
||||
size_bytes: 59_339,
|
||||
quantized: false,
|
||||
},
|
||||
PocketModelArtifact {
|
||||
filename: "flow_lm_main_int8.onnx",
|
||||
sha256: "f9bd8106b79a0192c1c43399ab938fb24900a95c1c599870d75a884e99000116",
|
||||
size_bytes: 76_341_079,
|
||||
quantized: true,
|
||||
},
|
||||
PocketModelArtifact {
|
||||
filename: "flow_lm_flow_int8.onnx",
|
||||
sha256: "3dd781ee5abee9e195320bf0106bebd6372a852b3b36352524ee78b40554635d",
|
||||
size_bytes: 9_962_530,
|
||||
quantized: true,
|
||||
},
|
||||
PocketModelArtifact {
|
||||
filename: "mimi_decoder_int8.onnx",
|
||||
sha256: "3630450a3297a101792a6ac66619ebc70ab916b265e6220c2afaef8b1673f925",
|
||||
size_bytes: 22_684_077,
|
||||
quantized: true,
|
||||
},
|
||||
PocketModelArtifact {
|
||||
filename: "mimi_encoder.onnx",
|
||||
sha256: "853e2ca623b8782d94c3745ec6133bfdff7ce33d9b11128bd29ea03f28d76e3d",
|
||||
size_bytes: 39_768_446,
|
||||
quantized: false,
|
||||
},
|
||||
PocketModelArtifact {
|
||||
filename: "text_conditioner.onnx",
|
||||
sha256: "4ecee995fb69f85c7a7493d11f7b5ee15d9950facc7ab3f5c9c49ef1e03847bb",
|
||||
size_bytes: 16_388_344,
|
||||
quantized: false,
|
||||
},
|
||||
];
|
||||
|
||||
const INT8_COMPONENTS: [&str; 3] = ["flow_lm_main", "flow_lm_flow", "mimi_decoder"];
|
||||
|
||||
/// Return immutable metadata for Buzz Desktop's April INT8 model.
|
||||
pub const fn april_model_info() -> PocketModelInfo {
|
||||
PocketModelInfo {
|
||||
bundle_id: APRIL_BUNDLE_ID,
|
||||
source_model_id: APRIL_MODEL_ID,
|
||||
revision: APRIL_MODEL_REVISION,
|
||||
sample_rate: 24_000,
|
||||
max_token_per_chunk: APRIL_MAX_TOKEN_PER_CHUNK,
|
||||
artifacts: &INT8_ARTIFACTS,
|
||||
quantized_components: &INT8_COMPONENTS,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn metadata_matches_pinned_int8_layout() {
|
||||
let info = april_model_info();
|
||||
assert_eq!(info.artifacts.len(), 8);
|
||||
assert_eq!(
|
||||
info.quantized_components,
|
||||
["flow_lm_main", "flow_lm_flow", "mimi_decoder"]
|
||||
);
|
||||
assert_eq!(
|
||||
info.artifacts
|
||||
.iter()
|
||||
.map(|artifact| artifact.size_bytes)
|
||||
.sum::<u64>(),
|
||||
165_232_420
|
||||
);
|
||||
assert!(info
|
||||
.artifacts
|
||||
.iter()
|
||||
.any(|artifact| { artifact.filename == "mimi_encoder.onnx" && !artifact.quantized }));
|
||||
assert!(!info
|
||||
.artifacts
|
||||
.iter()
|
||||
.any(|artifact| artifact.filename == "mimi_encoder_int8.onnx"));
|
||||
}
|
||||
}
|
||||
@@ -164,7 +164,8 @@ pub(crate) async fn connect_audio_relay(
|
||||
let cancel_clone = cancel.clone();
|
||||
let (pcm_tx, pcm_rx) = tokio::sync::mpsc::channel::<Vec<u8>>(50);
|
||||
let output_device_name = state
|
||||
.audio_output_device
|
||||
.huddle_audio
|
||||
.output_device
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.clone();
|
||||
|
||||
@@ -288,9 +288,11 @@ impl HuddleState {
|
||||
pub(crate) fn reset_preserving_generation(&mut self) {
|
||||
let gen = Arc::clone(&self.session_generation);
|
||||
let huddle_generation = self.huddle_generation;
|
||||
let tts_enabled = self.tts_enabled;
|
||||
*self = Self::default();
|
||||
self.session_generation = gen;
|
||||
self.huddle_generation = huddle_generation;
|
||||
self.tts_enabled = tts_enabled;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -430,6 +432,18 @@ mod tests {
|
||||
assert!(state.owns_huddle_lifetime(replacement_generation, super::HuddlePhase::Connecting));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn teardown_preserves_installation_global_tts_preference() {
|
||||
let mut state = HuddleState {
|
||||
tts_enabled: false,
|
||||
phase: super::HuddlePhase::Active,
|
||||
..HuddleState::default()
|
||||
};
|
||||
state.reset_preserving_generation();
|
||||
assert!(!state.tts_enabled);
|
||||
assert_eq!(state.phase, super::HuddlePhase::Idle);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_constructor_cannot_clear_replacement_sentinel() {
|
||||
let mut state = HuddleState::default();
|
||||
|
||||
@@ -35,10 +35,11 @@
|
||||
//! can gate microphone input while the agent is speaking.
|
||||
|
||||
use std::{
|
||||
collections::VecDeque,
|
||||
num::NonZero,
|
||||
path::PathBuf,
|
||||
sync::{
|
||||
atomic::{AtomicBool, Ordering},
|
||||
atomic::{AtomicBool, AtomicU64, Ordering},
|
||||
mpsc::{self, SyncSender},
|
||||
Arc, Mutex, MutexGuard, PoisonError,
|
||||
},
|
||||
@@ -46,9 +47,21 @@ use std::{
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use super::pocket::{load_text_to_speech, load_voice_style, SAMPLE_RATE, VOICE_FILE_EXT};
|
||||
use super::pocket::{
|
||||
load_text_to_speech, load_voice_style, DEFAULT_VOICE, SAMPLE_RATE, VOICE_FILE_EXT,
|
||||
};
|
||||
use super::preprocessing::{preprocess_for_tts, split_sentences};
|
||||
|
||||
#[path = "tts_voice_transition.rs"]
|
||||
mod voice_transition;
|
||||
use voice_transition::*;
|
||||
#[path = "tts_startup.rs"]
|
||||
mod startup;
|
||||
use startup::await_worker_startup;
|
||||
#[path = "tts_audio.rs"]
|
||||
mod audio;
|
||||
use audio::*;
|
||||
|
||||
// ── Constants ─────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Maximum number of queued text items.
|
||||
@@ -56,15 +69,15 @@ use super::preprocessing::{preprocess_for_tts, split_sentences};
|
||||
/// TTS can play it. Excess items are dropped with a warning.
|
||||
const TEXT_QUEUE_DEPTH: usize = 8;
|
||||
|
||||
/// How long the worker waits on the text channel before checking the shutdown flag.
|
||||
/// How long the worker waits before checking the shutdown flag.
|
||||
const RECV_TIMEOUT: Duration = Duration::from_millis(100);
|
||||
|
||||
/// Poll interval of the barge-in monitor thread. Bounds flag-to-silence
|
||||
/// latency: a cancel is noticed within one tick, and rodio's internal
|
||||
/// `periodic_access` wrapper stops the in-flight source within a further
|
||||
/// ~5 ms — so playing audio dies ~15 ms after the flag is set, even while
|
||||
/// the worker is blocked inside `synth_chunk`.
|
||||
const MONITOR_TICK: Duration = Duration::from_millis(10);
|
||||
const AUDIO_PRIME_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
|
||||
/// Pocket TTS is a one-step consistency model, not diffusion. Kept for API compat.
|
||||
const SYNTH_STEPS: usize = 1;
|
||||
@@ -109,6 +122,8 @@ const MAX_CHUNK_CHARS: usize = 200;
|
||||
/// Injected as a silent buffer between each synthesized sentence chunk.
|
||||
const INTER_SENTENCE_SILENCE: f32 = 0.1;
|
||||
|
||||
type WorkerControlState = (Arc<AtomicBool>, Arc<AtomicBool>, WorkerCancelSignals);
|
||||
|
||||
// ── Public pipeline handle ────────────────────────────────────────────────────
|
||||
|
||||
/// Handle to the running TTS pipeline.
|
||||
@@ -117,7 +132,7 @@ const INTER_SENTENCE_SILENCE: f32 = 0.1;
|
||||
#[derive(Debug)]
|
||||
pub struct TtsPipeline {
|
||||
/// Send preprocessed text into the pipeline.
|
||||
text_tx: SyncSender<String>,
|
||||
text_tx: SyncSender<QueuedText>,
|
||||
/// `true` while the agent is speaking. Shared with the STT pipeline for gating.
|
||||
#[allow(dead_code)]
|
||||
pub tts_active: Arc<AtomicBool>,
|
||||
@@ -127,38 +142,25 @@ pub struct TtsPipeline {
|
||||
/// Kept alive here so the Arc isn't dropped — the worker holds a clone.
|
||||
#[allow(dead_code)]
|
||||
cancel: Arc<AtomicBool>,
|
||||
/// Voice name (e.g. "reference_sample"). Stored for future voice-switching support.
|
||||
#[allow(dead_code)]
|
||||
voice: String,
|
||||
/// Internal cancellation used only for voice changes. Kept separate so a
|
||||
/// concurrent human barge-in always clears every queued message.
|
||||
voice_cancel: Arc<AtomicBool>,
|
||||
/// Selected manifest voice. The worker reloads only the lightweight style
|
||||
/// when this changes; the warmed Pocket engine and audio player stay alive.
|
||||
voice: Arc<Mutex<String>>,
|
||||
/// Tags messages so a voice change drops only pre-change queue entries.
|
||||
voice_generation: Arc<AtomicU64>,
|
||||
/// Completed after the worker drains pre-change text and installs the new style.
|
||||
voice_change_ack: VoiceChangeAck,
|
||||
/// Worker thread handle — taken on drop to join cleanly.
|
||||
thread: Option<thread::JoinHandle<()>>,
|
||||
}
|
||||
|
||||
impl TtsPipeline {
|
||||
/// Spawn the TTS pipeline thread using the default voice.
|
||||
/// Spawn the TTS pipeline thread with a manifest-backed voice name.
|
||||
///
|
||||
/// `model_dir` must contain the Pocket TTS files declared by `huddle::models`
|
||||
/// (the five ONNX sessions, the two JSON tables, and `<voice>.wav`).
|
||||
///
|
||||
/// `tts_active` is set to `true` while audio is playing and `false` when idle.
|
||||
/// Pass the same `Arc` to the STT pipeline to gate microphone input.
|
||||
///
|
||||
/// `cancel` is the shared barge-in flag from `HuddleState.tts_cancel`. Pass the
|
||||
/// same `Arc` to the STT pipeline so both sides reference the same flag for the
|
||||
/// entire huddle session — no stale references after pipeline restarts.
|
||||
pub fn new(
|
||||
model_dir: PathBuf,
|
||||
tts_active: Arc<AtomicBool>,
|
||||
cancel: Arc<AtomicBool>,
|
||||
output_device: Option<String>,
|
||||
) -> Result<Self, String> {
|
||||
use super::pocket::DEFAULT_VOICE;
|
||||
Self::new_with_voice(model_dir, tts_active, cancel, DEFAULT_VOICE, output_device)
|
||||
}
|
||||
|
||||
/// Spawn the TTS pipeline thread with a specific voice name. Today only the
|
||||
/// bundled default voice (see `pocket::DEFAULT_VOICE`) is shipped; other
|
||||
/// names will surface a clear error from `load_voice_style`.
|
||||
/// `cancel` is shared with STT for barge-in. The same handle survives voice
|
||||
/// changes so the warmed Pocket engine is retained.
|
||||
pub fn new_with_voice(
|
||||
model_dir: PathBuf,
|
||||
tts_active: Arc<AtomicBool>,
|
||||
@@ -166,37 +168,56 @@ impl TtsPipeline {
|
||||
voice: &str,
|
||||
output_device: Option<String>,
|
||||
) -> Result<Self, String> {
|
||||
let (text_tx, text_rx) = mpsc::sync_channel::<String>(TEXT_QUEUE_DEPTH);
|
||||
let (text_tx, text_rx) = mpsc::sync_channel::<QueuedText>(TEXT_QUEUE_DEPTH);
|
||||
let shutdown = Arc::new(AtomicBool::new(false));
|
||||
// cancel is passed in from HuddleState.tts_cancel — shared with STT for barge-in.
|
||||
|
||||
let shutdown_worker = Arc::clone(&shutdown);
|
||||
let cancel_worker = Arc::clone(&cancel);
|
||||
let voice_cancel = Arc::new(AtomicBool::new(false));
|
||||
let worker_voice_cancel = Arc::clone(&voice_cancel);
|
||||
let tts_active_worker = Arc::clone(&tts_active);
|
||||
let voice_name = voice.to_string();
|
||||
let voice = Arc::new(Mutex::new(voice.to_string()));
|
||||
let voice_worker = Arc::clone(&voice);
|
||||
let voice_generation = Arc::new(AtomicU64::new(1));
|
||||
let worker_voice_generation = Arc::clone(&voice_generation);
|
||||
let voice_change_ack = Arc::new(Mutex::new(None));
|
||||
let worker_voice_change_ack = Arc::clone(&voice_change_ack);
|
||||
let model_dir_worker = model_dir.clone();
|
||||
let (startup_tx, startup_rx) = mpsc::sync_channel(1);
|
||||
|
||||
let handle = thread::Builder::new()
|
||||
.name("tts-worker".into())
|
||||
.spawn(move || {
|
||||
tts_worker(
|
||||
model_dir_worker,
|
||||
voice_name,
|
||||
(
|
||||
voice_worker,
|
||||
worker_voice_generation,
|
||||
worker_voice_change_ack,
|
||||
),
|
||||
text_rx,
|
||||
tts_active_worker,
|
||||
shutdown_worker,
|
||||
cancel_worker,
|
||||
(
|
||||
tts_active_worker,
|
||||
shutdown_worker,
|
||||
(cancel_worker, worker_voice_cancel),
|
||||
),
|
||||
output_device,
|
||||
startup_tx,
|
||||
)
|
||||
})
|
||||
.map_err(|e| format!("failed to spawn tts-worker thread: {e}"))?;
|
||||
let handle = await_worker_startup(handle, startup_rx)?;
|
||||
|
||||
Ok(Self {
|
||||
text_tx,
|
||||
tts_active,
|
||||
shutdown,
|
||||
cancel,
|
||||
voice: voice.to_string(),
|
||||
voice_cancel,
|
||||
voice,
|
||||
voice_generation,
|
||||
voice_change_ack,
|
||||
thread: Some(handle),
|
||||
})
|
||||
}
|
||||
@@ -206,14 +227,59 @@ impl TtsPipeline {
|
||||
/// Non-blocking. Returns `Err` if the queue is full (bounded at
|
||||
/// `TEXT_QUEUE_DEPTH`) — caller may log and discard.
|
||||
pub fn speak(&self, text: String) -> Result<(), String> {
|
||||
self.text_tx.try_send(text).map_err(|e| {
|
||||
eprintln!("buzz-desktop: TTS queue saturated, dropping message: {e}");
|
||||
format!("TTS queue full, dropping: {e}")
|
||||
})
|
||||
self.text_tx
|
||||
.try_send(QueuedText {
|
||||
generation: self.voice_generation.load(Ordering::Acquire),
|
||||
route_id: 0,
|
||||
text,
|
||||
})
|
||||
.map_err(|e| {
|
||||
eprintln!("buzz-desktop: TTS queue saturated, dropping message: {e}");
|
||||
format!("TTS queue full, dropping: {e}")
|
||||
})
|
||||
}
|
||||
|
||||
/// Clone the bounded queue sender so callers can apply backpressure without
|
||||
/// holding the huddle mutex. Disabling TTS drops the receiver and unblocks
|
||||
/// any waiting sender while the shared cancellation flag stops playback.
|
||||
pub(crate) fn text_sender(&self) -> TtsTextSender {
|
||||
TtsTextSender {
|
||||
text_tx: self.text_tx.clone(),
|
||||
generation: self.voice_generation.load(Ordering::Acquire),
|
||||
}
|
||||
}
|
||||
|
||||
/// Select a bundled Pocket voice for subsequent speech.
|
||||
///
|
||||
/// Current playback and queued text are cancelled immediately so content
|
||||
/// cannot continue in the old voice. The worker keeps its warmed inference
|
||||
/// engine and reloads only the reference style before the next utterance.
|
||||
pub fn select_voice(&self, voice: &str) -> Option<tokio::sync::oneshot::Receiver<()>> {
|
||||
let acknowledged = begin_voice_change(
|
||||
&self.voice,
|
||||
&self.voice_generation,
|
||||
&self.voice_cancel,
|
||||
&self.voice_change_ack,
|
||||
voice,
|
||||
);
|
||||
if acknowledged.is_some() {
|
||||
eprintln!("buzz-desktop: tts stage=cancellation reason=voice_switch route_id=0");
|
||||
}
|
||||
acknowledged
|
||||
}
|
||||
|
||||
/// Reconcile the voice of a pipeline that has not been published yet.
|
||||
///
|
||||
/// No caller can enqueue text before publication, so raising the shared
|
||||
/// cancellation flag here would create a race that could discard the first
|
||||
/// message queued immediately after installation.
|
||||
pub(crate) fn select_voice_before_publish(&self, voice: &str) {
|
||||
*self.voice.lock().unwrap_or_else(|error| error.into_inner()) = voice.to_string();
|
||||
}
|
||||
|
||||
/// Signal the worker thread to stop.
|
||||
pub fn shutdown(&self) {
|
||||
eprintln!("buzz-desktop: tts stage=cancellation reason=shutdown route_id=0");
|
||||
self.shutdown.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
@@ -239,40 +305,52 @@ impl Drop for TtsPipeline {
|
||||
|
||||
fn tts_worker(
|
||||
model_dir: PathBuf,
|
||||
voice_name: String,
|
||||
text_rx: mpsc::Receiver<String>,
|
||||
tts_active: Arc<AtomicBool>,
|
||||
shutdown: Arc<AtomicBool>,
|
||||
cancel: Arc<AtomicBool>,
|
||||
voice_state: WorkerVoiceState,
|
||||
text_rx: mpsc::Receiver<QueuedText>,
|
||||
control_state: WorkerControlState,
|
||||
output_device: Option<String>,
|
||||
startup_tx: mpsc::SyncSender<Result<(), String>>,
|
||||
) {
|
||||
let (selected_voice, voice_generation, voice_change_ack) = voice_state;
|
||||
let (tts_active, shutdown, cancel_signals) = control_state;
|
||||
let (cancel, voice_cancel) = cancel_signals;
|
||||
// ── 1. Initialise TTS engine ──────────────────────────────────────────────
|
||||
let model_dir_str = model_dir.to_string_lossy().to_string();
|
||||
|
||||
let engine = match load_text_to_speech(&model_dir_str) {
|
||||
Ok(e) => e,
|
||||
Err(e) => {
|
||||
eprintln!(
|
||||
"buzz-desktop: TTS engine init failed (model_dir={}): {e}. TTS disabled.",
|
||||
model_dir.display()
|
||||
);
|
||||
drain_until_shutdown(text_rx, &shutdown);
|
||||
let error = format!("TTS engine initialization failed: {e}");
|
||||
eprintln!("buzz-desktop: tts stage=startup status=failed reason=engine_load");
|
||||
let _ = startup_tx.send(Err(error));
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// ── 2. Load voice style ───────────────────────────────────────────────────
|
||||
let voice_path = model_dir.join(format!("{voice_name}.{VOICE_FILE_EXT}"));
|
||||
let style = match load_voice_style(&voice_path) {
|
||||
let requested_voice = selected_voice
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner())
|
||||
.clone();
|
||||
let mut voice_name = DEFAULT_VOICE.to_string();
|
||||
let fallback_path = model_dir.join(format!("{DEFAULT_VOICE}.{VOICE_FILE_EXT}"));
|
||||
let mut style = match load_voice_style(&fallback_path) {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
eprintln!(
|
||||
"buzz-desktop: TTS voice style load failed ({voice_name}): {e}. TTS disabled."
|
||||
);
|
||||
drain_until_shutdown(text_rx, &shutdown);
|
||||
let error = format!("TTS voice style initialization failed: {e}");
|
||||
eprintln!("buzz-desktop: tts stage=startup status=failed reason=fallback_voice_style");
|
||||
let _ = startup_tx.send(Err(error));
|
||||
return;
|
||||
}
|
||||
};
|
||||
if requested_voice != DEFAULT_VOICE
|
||||
&& !reconcile_selected_voice(&model_dir, &selected_voice, &mut voice_name, &mut style)
|
||||
{
|
||||
let _ = startup_tx.send(Err(
|
||||
"TTS selected voice and Mary fallback are unavailable".to_string()
|
||||
));
|
||||
return;
|
||||
}
|
||||
|
||||
// ── 2b. Warmup inference ─────────────────────────────────────────────────
|
||||
// The first ONNX inference on any session is significantly slower than
|
||||
@@ -280,15 +358,10 @@ fn tts_worker(
|
||||
// pool allocation, and graph-specific caches. Run a short dummy synthesis
|
||||
// and discard the output so the first real utterance runs at warm-session speed.
|
||||
{
|
||||
let t = std::time::Instant::now();
|
||||
match engine.synth_chunk("warmup", "en", &style, SYNTH_STEPS) {
|
||||
Ok(_) => eprintln!(
|
||||
"buzz-desktop: TTS warmup completed in {:.0}ms",
|
||||
t.elapsed().as_millis()
|
||||
),
|
||||
Err(e) => eprintln!(
|
||||
"buzz-desktop: TTS warmup failed after {:.0}ms: {e} — first utterance may be slow",
|
||||
t.elapsed().as_millis()
|
||||
Ok(_) => eprintln!("buzz-desktop: tts stage=warmup status=ready"),
|
||||
Err(_) => eprintln!(
|
||||
"buzz-desktop: tts stage=warmup status=failed reason=inference first_utterance_may_be_slow=true"
|
||||
),
|
||||
}
|
||||
}
|
||||
@@ -301,8 +374,9 @@ fn tts_worker(
|
||||
{
|
||||
Ok(h) => h,
|
||||
Err(e) => {
|
||||
eprintln!("buzz-desktop: TTS audio output failed: {e}. TTS disabled.");
|
||||
drain_until_shutdown(text_rx, &shutdown);
|
||||
let error = format!("TTS audio output initialization failed: {e}");
|
||||
eprintln!("buzz-desktop: tts stage=startup status=failed reason=output_open");
|
||||
let _ = startup_tx.send(Err(error));
|
||||
return;
|
||||
}
|
||||
};
|
||||
@@ -310,14 +384,14 @@ fn tts_worker(
|
||||
let channels = match NonZero::new(1u16) {
|
||||
Some(c) => c,
|
||||
None => {
|
||||
eprintln!("buzz-desktop: TTS channel count invariant violated");
|
||||
let _ = startup_tx.send(Err("TTS channel count invariant violated".to_string()));
|
||||
return;
|
||||
}
|
||||
};
|
||||
let rate = match NonZero::new(SAMPLE_RATE) {
|
||||
Some(r) => r,
|
||||
None => {
|
||||
eprintln!("buzz-desktop: TTS sample rate invariant violated");
|
||||
let _ = startup_tx.send(Err("TTS sample rate invariant violated".to_string()));
|
||||
return;
|
||||
}
|
||||
};
|
||||
@@ -341,10 +415,22 @@ fn tts_worker(
|
||||
player.append(SamplesBuffer::new(channels, rate, silence));
|
||||
// Wait for the silent buffer to drain — this ensures the output stream
|
||||
// is fully initialized before the first real utterance.
|
||||
let deadline = std::time::Instant::now() + AUDIO_PRIME_TIMEOUT;
|
||||
while !player.empty() {
|
||||
if std::time::Instant::now() >= deadline {
|
||||
eprintln!("buzz-desktop: tts stage=startup status=failed reason=output_prime");
|
||||
let _ = startup_tx.send(Err(
|
||||
"TTS audio output did not become ready before timeout".to_string(),
|
||||
));
|
||||
return;
|
||||
}
|
||||
thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
}
|
||||
if startup_tx.send(Ok(())).is_err() {
|
||||
return;
|
||||
}
|
||||
eprintln!("buzz-desktop: tts stage=startup status=ready");
|
||||
|
||||
// ── 3b. Barge-in monitor thread ───────────────────────────────────────────
|
||||
//
|
||||
@@ -372,6 +458,7 @@ fn tts_worker(
|
||||
let monitor = {
|
||||
let player = Arc::clone(&player);
|
||||
let cancel = Arc::clone(&cancel);
|
||||
let voice_cancel = Arc::clone(&voice_cancel);
|
||||
let tts_active = Arc::clone(&tts_active);
|
||||
let stop = Arc::clone(&monitor_stop);
|
||||
let player_ops = Arc::clone(&player_ops);
|
||||
@@ -379,12 +466,12 @@ fn tts_worker(
|
||||
.name("tts-barge-in-monitor".into())
|
||||
.spawn(move || {
|
||||
while !stop.load(Ordering::Acquire) {
|
||||
if cancel.load(Ordering::Acquire) {
|
||||
if cancel.load(Ordering::Acquire) || voice_cancel.load(Ordering::Acquire) {
|
||||
let _ops = lock_player_ops(&player_ops);
|
||||
// Re-check under the lock: the worker may have
|
||||
// consumed this cancel (and appended fresh audio)
|
||||
// between the load above and the lock acquisition.
|
||||
if cancel.load(Ordering::Acquire) {
|
||||
if cancel.load(Ordering::Acquire) || voice_cancel.load(Ordering::Acquire) {
|
||||
// clear() pauses the persistent player; play()
|
||||
// un-pauses (see handle_cancel_or_shutdown).
|
||||
// Idempotent — safe to repeat every tick until
|
||||
@@ -418,13 +505,45 @@ fn tts_worker(
|
||||
// idle branch below uses it to decide when to drop `tts_active` and to
|
||||
// arm a fresh lead-in cushion for the next utterance.
|
||||
let mut first_append = true;
|
||||
let mut last_route_id = 0;
|
||||
let mut deferred_text = VecDeque::new();
|
||||
let append_audio = |prepared: PreparedModelAudio, route_id: u64| {
|
||||
let _ops = lock_player_ops(&player_ops);
|
||||
if cancel.load(Ordering::Acquire)
|
||||
|| voice_cancel.load(Ordering::Acquire)
|
||||
|| shutdown.load(Ordering::Acquire)
|
||||
{
|
||||
let reason = if shutdown.load(Ordering::Acquire) {
|
||||
"shutdown"
|
||||
} else if cancel.load(Ordering::Acquire) {
|
||||
"barge_in"
|
||||
} else {
|
||||
"voice_switch"
|
||||
};
|
||||
eprintln!(
|
||||
"buzz-desktop: tts stage=synthesis status=cancelled reason={reason} route_id={route_id}"
|
||||
);
|
||||
return false;
|
||||
}
|
||||
player.append(SamplesBuffer::new(channels, rate, prepared.buffer));
|
||||
eprintln!(
|
||||
"buzz-desktop: tts stage=player status=append_accepted route_id={route_id} chunk_index={} sample_count={}",
|
||||
prepared.chunk_index, prepared.sample_count
|
||||
);
|
||||
// Set this only after append so STT remains open during synthesis.
|
||||
tts_active.store(true, Ordering::Release);
|
||||
true
|
||||
};
|
||||
|
||||
loop {
|
||||
let mut no_current_text = None;
|
||||
if handle_cancel_or_shutdown(
|
||||
&cancel,
|
||||
(&cancel, &voice_cancel),
|
||||
&shutdown,
|
||||
&tts_active,
|
||||
&text_rx,
|
||||
(&text_rx, &mut deferred_text, &mut no_current_text),
|
||||
&voice_change_ack,
|
||||
None,
|
||||
Some((&player, &player_ops)),
|
||||
) {
|
||||
if shutdown.load(Ordering::Acquire) {
|
||||
@@ -436,28 +555,47 @@ fn tts_worker(
|
||||
continue;
|
||||
}
|
||||
|
||||
let raw_text = match text_rx.recv_timeout(RECV_TIMEOUT) {
|
||||
Ok(t) => t,
|
||||
Err(mpsc::RecvTimeoutError::Timeout) => {
|
||||
// Nothing queued. If playback has also finished, the agent
|
||||
// has gone quiet — release the mic gate and reset the
|
||||
// lead-in so the next utterance gets a fresh cushion.
|
||||
if player.empty() && !first_append {
|
||||
tts_active.store(false, Ordering::Release);
|
||||
first_append = true;
|
||||
// Voice changes cancel the old utterance/queue and are observed here,
|
||||
// before receiving subsequent text. A bad bundled asset falls back to
|
||||
// Mary without discarding the already-warmed Pocket engine.
|
||||
let voice_ready =
|
||||
reconcile_selected_voice(&model_dir, &selected_voice, &mut voice_name, &mut style);
|
||||
acknowledge_voice_change(&voice_change_ack, &voice_cancel);
|
||||
if !voice_ready {
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut queued_text = Some(match deferred_text.pop_front() {
|
||||
Some(text) => text,
|
||||
None => match text_rx.recv_timeout(RECV_TIMEOUT) {
|
||||
Ok(text) => text,
|
||||
Err(mpsc::RecvTimeoutError::Timeout) => {
|
||||
// Nothing queued. If playback has also finished, the agent
|
||||
// has gone quiet — release the mic gate and reset the
|
||||
// lead-in so the next utterance gets a fresh cushion.
|
||||
if player.empty() && !first_append {
|
||||
tts_active.store(false, Ordering::Release);
|
||||
eprintln!(
|
||||
"buzz-desktop: tts stage=player status=drained route_id={last_route_id}"
|
||||
);
|
||||
first_append = true;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
Err(mpsc::RecvTimeoutError::Disconnected) => break,
|
||||
};
|
||||
Err(mpsc::RecvTimeoutError::Disconnected) => break,
|
||||
},
|
||||
});
|
||||
|
||||
// Check cancel again after unblocking — a cancel may have arrived
|
||||
// while we were waiting.
|
||||
let pending_route_id = queued_text.as_ref().map(|queued| queued.route_id);
|
||||
if handle_cancel_or_shutdown(
|
||||
&cancel,
|
||||
(&cancel, &voice_cancel),
|
||||
&shutdown,
|
||||
&tts_active,
|
||||
&text_rx,
|
||||
(&text_rx, &mut deferred_text, &mut queued_text),
|
||||
&voice_change_ack,
|
||||
pending_route_id,
|
||||
Some((&player, &player_ops)),
|
||||
) {
|
||||
if shutdown.load(Ordering::Acquire) {
|
||||
@@ -466,6 +604,30 @@ fn tts_worker(
|
||||
first_append = true;
|
||||
continue;
|
||||
}
|
||||
let Some(queued_text) = queued_text else {
|
||||
continue;
|
||||
};
|
||||
if queued_text.generation < voice_generation.load(Ordering::Acquire) {
|
||||
eprintln!(
|
||||
"buzz-desktop: tts stage=queue status=dropped reason=voice_switch route_id={}",
|
||||
queued_text.route_id
|
||||
);
|
||||
continue;
|
||||
}
|
||||
let raw_text = queued_text.text;
|
||||
let route_id = queued_text.route_id;
|
||||
eprintln!("buzz-desktop: tts stage=synthesis status=started route_id={route_id}");
|
||||
|
||||
// The selected voice can change while this worker is blocked in
|
||||
// recv_timeout. Reconcile again after receipt so the first message
|
||||
// queued after an unpublished pipeline is installed cannot use the
|
||||
// voice captured when construction began.
|
||||
if !reconcile_selected_voice(&model_dir, &selected_voice, &mut voice_name, &mut style) {
|
||||
eprintln!(
|
||||
"buzz-desktop: tts stage=synthesis status=failed reason=voice_unavailable route_id={route_id}"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
// If playback already drained while we were waiting for this item,
|
||||
// the agent is silent — release the mic gate BEFORE preprocessing/
|
||||
@@ -477,12 +639,16 @@ fn tts_worker(
|
||||
// stays set across items.)
|
||||
if player.empty() && !first_append {
|
||||
tts_active.store(false, Ordering::Release);
|
||||
eprintln!("buzz-desktop: tts stage=player status=drained route_id={last_route_id}");
|
||||
first_append = true;
|
||||
}
|
||||
|
||||
// Preprocess text.
|
||||
let text = preprocess_for_tts(&raw_text);
|
||||
if text.is_empty() {
|
||||
eprintln!(
|
||||
"buzz-desktop: tts stage=synthesis status=empty reason=preprocess route_id={route_id}"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -497,16 +663,29 @@ fn tts_worker(
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.collect();
|
||||
let chunks = group_sentences_into_chunks(&sentences, MAX_CHUNK_CHARS);
|
||||
if chunks.is_empty() {
|
||||
eprintln!(
|
||||
"buzz-desktop: tts stage=synthesis status=empty reason=no_chunks route_id={route_id}"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut synthesis_outcome = "completed";
|
||||
let mut appended_audio = false;
|
||||
let mut model_unit_index = 0_usize;
|
||||
'playback_chunks: for chunk in &chunks {
|
||||
let mut no_current_text = None;
|
||||
if handle_cancel_or_shutdown(
|
||||
&cancel,
|
||||
(&cancel, &voice_cancel),
|
||||
&shutdown,
|
||||
&tts_active,
|
||||
&text_rx,
|
||||
(&text_rx, &mut deferred_text, &mut no_current_text),
|
||||
&voice_change_ack,
|
||||
Some(route_id),
|
||||
Some((&player, &player_ops)),
|
||||
) {
|
||||
first_append = true;
|
||||
synthesis_outcome = "cancelled";
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -517,76 +696,111 @@ fn tts_worker(
|
||||
|
||||
let model_chunks = match engine.split_text_into_chunks(text) {
|
||||
Ok(model_chunks) => model_chunks,
|
||||
Err(error) => {
|
||||
eprintln!("buzz-desktop: TTS chunking failed: {error}");
|
||||
break;
|
||||
Err(_) => {
|
||||
eprintln!(
|
||||
"buzz-desktop: tts stage=synthesis status=failed reason=chunking route_id={route_id}"
|
||||
);
|
||||
synthesis_outcome = "failed";
|
||||
break 'playback_chunks;
|
||||
}
|
||||
};
|
||||
let model_chunk_count = model_chunks.len();
|
||||
for (model_chunk_index, model_chunk) in model_chunks.iter().enumerate() {
|
||||
if model_chunks.is_empty() {
|
||||
eprintln!(
|
||||
"buzz-desktop: tts stage=synthesis status=empty reason=no_chunks route_id={route_id}"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
let mut playback_audio = PlaybackChunkAudio::new();
|
||||
for model_chunk in &model_chunks {
|
||||
let chunk_index = model_unit_index;
|
||||
model_unit_index += 1;
|
||||
let mut no_current_text = None;
|
||||
if handle_cancel_or_shutdown(
|
||||
&cancel,
|
||||
(&cancel, &voice_cancel),
|
||||
&shutdown,
|
||||
&tts_active,
|
||||
&text_rx,
|
||||
(&text_rx, &mut deferred_text, &mut no_current_text),
|
||||
&voice_change_ack,
|
||||
Some(route_id),
|
||||
Some((&player, &player_ops)),
|
||||
) {
|
||||
first_append = true;
|
||||
synthesis_outcome = "cancelled";
|
||||
break 'playback_chunks;
|
||||
}
|
||||
|
||||
let ends_playback_chunk = model_chunk_index + 1 == model_chunk_count;
|
||||
match engine.synth_chunk(model_chunk, "en", &style, SYNTH_STEPS) {
|
||||
let synthesis = engine.synth_chunk(model_chunk, "en", &style, SYNTH_STEPS);
|
||||
if cancel.load(Ordering::Acquire)
|
||||
|| voice_cancel.load(Ordering::Acquire)
|
||||
|| shutdown.load(Ordering::Acquire)
|
||||
{
|
||||
let reason = if shutdown.load(Ordering::Acquire) {
|
||||
"shutdown"
|
||||
} else if cancel.load(Ordering::Acquire) {
|
||||
"barge_in"
|
||||
} else {
|
||||
"voice_switch"
|
||||
};
|
||||
eprintln!(
|
||||
"buzz-desktop: tts stage=synthesis status=cancelled reason={reason} route_id={route_id}"
|
||||
);
|
||||
// The monitor already stopped any queued playback. Discard
|
||||
// synthesis that completed after cancellation so stale audio
|
||||
// never reaches the player, while keeping buzz-voice's
|
||||
// extracted April engine API unchanged.
|
||||
first_append = true;
|
||||
synthesis_outcome = "cancelled";
|
||||
break 'playback_chunks;
|
||||
}
|
||||
match synthesis {
|
||||
Ok(samples) if !samples.is_empty() => {
|
||||
let mut audio = clamp_to_full_scale(samples);
|
||||
if ends_playback_chunk {
|
||||
// Fade only at the playback-chunk boundary. Applying
|
||||
// it at the model's internal token boundary would
|
||||
// create an audible dip between contiguous units.
|
||||
apply_fade_out(&mut audio);
|
||||
}
|
||||
|
||||
let buf = build_sentence_append_buffer(
|
||||
if let Some(prepared) = playback_audio.push(
|
||||
samples,
|
||||
chunk_index,
|
||||
&mut first_append,
|
||||
audio,
|
||||
silence_buf_len,
|
||||
model_chunk_index == 0 || player.empty(),
|
||||
ends_playback_chunk,
|
||||
);
|
||||
|
||||
// Check-and-append under `player_ops`, serialized with
|
||||
// the monitor: a barge-in may have arrived during
|
||||
// synthesis (the blocking window the monitor thread
|
||||
// exists for). Don't append the now-stale sentence — the
|
||||
// human interrupted; speaking it anyway would talk over
|
||||
// them. Holding the lock for the check + append means the
|
||||
// monitor can never clear between our check passing and
|
||||
// the buffer landing. The flag is deliberately NOT
|
||||
// consumed here: the loop-top handle_cancel_or_shutdown
|
||||
// does the full consume (drain queue, reset lead-in) on
|
||||
// the next iteration.
|
||||
let _ops = lock_player_ops(&player_ops);
|
||||
if cancel.load(Ordering::Acquire) {
|
||||
// Nothing appended; the loop-top consume re-arms
|
||||
// `first_append` (the flag is still set — the worker
|
||||
// is its only consumer).
|
||||
break;
|
||||
player.empty(),
|
||||
) {
|
||||
if !append_audio(prepared, route_id) {
|
||||
first_append = true;
|
||||
synthesis_outcome = "cancelled";
|
||||
break 'playback_chunks;
|
||||
}
|
||||
appended_audio = true;
|
||||
last_route_id = route_id;
|
||||
}
|
||||
player.append(SamplesBuffer::new(channels, rate, buf));
|
||||
// NOTE: tts_active is set AFTER player.append(), not
|
||||
// before. Setting it before synthesis would cause STT to
|
||||
// discard user speech during the synthesis window as
|
||||
// "echo" even though no audio is actually playing yet.
|
||||
// See crossfire review C3.
|
||||
tts_active.store(true, Ordering::Release);
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
eprintln!("buzz-desktop: TTS synth failed: {e}");
|
||||
Ok(_) => {
|
||||
eprintln!(
|
||||
"buzz-desktop: tts stage=synthesis status=empty route_id={route_id} chunk_index={chunk_index}"
|
||||
);
|
||||
}
|
||||
Err(_) => {
|
||||
eprintln!(
|
||||
"buzz-desktop: tts stage=synthesis status=failed reason=inference route_id={route_id} chunk_index={chunk_index}"
|
||||
);
|
||||
synthesis_outcome = "failed";
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(prepared) =
|
||||
playback_audio.finish(&mut first_append, silence_buf_len, player.empty())
|
||||
{
|
||||
if !append_audio(prepared, route_id) {
|
||||
first_append = true;
|
||||
synthesis_outcome = "cancelled";
|
||||
break 'playback_chunks;
|
||||
}
|
||||
appended_audio = true;
|
||||
last_route_id = route_id;
|
||||
}
|
||||
if synthesis_outcome == "failed" {
|
||||
break 'playback_chunks;
|
||||
}
|
||||
}
|
||||
if synthesis_outcome == "completed" && appended_audio {
|
||||
eprintln!("buzz-desktop: tts stage=synthesis status=completed route_id={route_id}");
|
||||
}
|
||||
|
||||
if shutdown.load(Ordering::Acquire) {
|
||||
@@ -601,6 +815,7 @@ fn tts_worker(
|
||||
let _ = handle.join();
|
||||
}
|
||||
|
||||
finish_voice_change_ack(&voice_change_ack);
|
||||
tts_active.store(false, Ordering::Release);
|
||||
}
|
||||
|
||||
@@ -614,13 +829,21 @@ fn tts_worker(
|
||||
/// it is serialized with the monitor's stale-branch re-check (see the monitor
|
||||
/// block in `tts_worker`).
|
||||
fn handle_cancel_or_shutdown(
|
||||
cancel: &AtomicBool,
|
||||
cancel_signals: CancelSignals<'_>,
|
||||
shutdown: &AtomicBool,
|
||||
tts_active: &AtomicBool,
|
||||
text_rx: &mpsc::Receiver<String>,
|
||||
text_state: CancelTextState<'_>,
|
||||
voice_change_ack: &VoiceChangeAck,
|
||||
active_route_id: Option<u64>,
|
||||
player: Option<(&rodio::Player, &Mutex<()>)>,
|
||||
) -> bool {
|
||||
let (cancel, voice_cancel) = cancel_signals;
|
||||
let (text_rx, deferred_text, current_text) = text_state;
|
||||
if shutdown.load(Ordering::Acquire) {
|
||||
eprintln!(
|
||||
"buzz-desktop: tts stage=cancellation reason=shutdown route_id={}",
|
||||
active_route_id.unwrap_or(0)
|
||||
);
|
||||
if let Some((p, ops)) = player {
|
||||
let _ops = lock_player_ops(ops);
|
||||
p.clear();
|
||||
@@ -628,7 +851,29 @@ fn handle_cancel_or_shutdown(
|
||||
tts_active.store(false, Ordering::Release);
|
||||
return true;
|
||||
}
|
||||
if cancel.load(Ordering::Acquire) {
|
||||
if cancel.load(Ordering::Acquire) || voice_cancel.load(Ordering::Acquire) {
|
||||
// Serialize with begin_voice_change so the generation boundary and
|
||||
// cancel consumption are observed as one transition.
|
||||
let pending_voice_change = voice_change_ack
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner());
|
||||
// Consume at the serialization point. A later barge-in remains true
|
||||
// for the next pass instead of being overwritten after queue cleanup.
|
||||
let barge_in = cancel.swap(false, Ordering::AcqRel);
|
||||
voice_cancel.store(false, Ordering::Release);
|
||||
eprintln!(
|
||||
"buzz-desktop: tts stage=cancellation reason={} route_id={}",
|
||||
if barge_in { "barge_in" } else { "voice_switch" },
|
||||
active_route_id.unwrap_or(0)
|
||||
);
|
||||
let preserve_generation = (!barge_in)
|
||||
.then(|| {
|
||||
pending_voice_change
|
||||
.as_ref()
|
||||
.map(|pending| pending.generation)
|
||||
})
|
||||
.flatten();
|
||||
retain_cancelled_text(deferred_text, current_text, text_rx, preserve_generation);
|
||||
if let Some((p, ops)) = player {
|
||||
let _ops = lock_player_ops(ops);
|
||||
// `Player::clear()` removes queued sources AND pauses the player
|
||||
@@ -641,11 +886,6 @@ fn handle_cancel_or_shutdown(
|
||||
// Consume the flag under the lock: once released with
|
||||
// `cancel == false`, the monitor's stale branch no-ops instead
|
||||
// of clearing the fresh post-cancel utterance.
|
||||
while text_rx.try_recv().is_ok() {}
|
||||
cancel.store(false, Ordering::Release);
|
||||
} else {
|
||||
while text_rx.try_recv().is_ok() {}
|
||||
cancel.store(false, Ordering::Release);
|
||||
}
|
||||
tts_active.store(false, Ordering::Release);
|
||||
return true;
|
||||
@@ -663,140 +903,11 @@ fn lock_player_ops(ops: &Mutex<()>) -> MutexGuard<'_, ()> {
|
||||
ops.lock().unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
|
||||
/// Hard-clamp samples to ±1.0 full scale.
|
||||
///
|
||||
/// No gain is applied because Pocket TTS already emits speech-level audio and
|
||||
/// the reference pipeline applies no output scaling. Normalizing each sentence
|
||||
/// would cause level pumping between chunks. The clamp remains only as a safety
|
||||
/// net against outlier transients.
|
||||
fn clamp_to_full_scale(samples: Vec<f32>) -> Vec<f32> {
|
||||
samples.into_iter().map(|s| s.clamp(-1.0, 1.0)).collect()
|
||||
}
|
||||
|
||||
/// Apply a short linear fade-out at the *end* of `samples`.
|
||||
///
|
||||
/// Uses `FADE_OUT_SAMPLES` (8 ms) or half the buffer length, whichever is
|
||||
/// smaller. Eliminates the click that occurs when a non-zero waveform
|
||||
/// terminates abruptly at a sentence boundary.
|
||||
///
|
||||
/// # Why no fade-in
|
||||
///
|
||||
/// A symmetric fade-in would attenuate the leading consonant attack because
|
||||
/// Pocket TTS produces real audio energy inside the first millisecond. A
|
||||
/// linear 0→1 ramp over 192 samples scales those onset samples by ≤50% for the
|
||||
/// first ~4 ms, which can make the first phoneme sound clipped.
|
||||
///
|
||||
/// The first sample of Pocket output measures ≈ 0.0018 (≈ −54 dBFS) — well
|
||||
/// below the threshold at which a DC-jump would be audible as a click — so
|
||||
/// no fade-in is needed. The OS audio device gets its quiet ramp-up window
|
||||
/// from `SENTENCE_LEAD_IN_SAMPLES` instead, inserted as pure silence before
|
||||
/// each sentence buffer.
|
||||
fn apply_fade_out(samples: &mut [f32]) {
|
||||
let len = samples.len();
|
||||
let fade = FADE_OUT_SAMPLES.min(len / 2);
|
||||
for i in 0..fade {
|
||||
samples[len - 1 - i] *= i as f32 / fade as f32;
|
||||
}
|
||||
}
|
||||
|
||||
/// Build one buffer appended to the rodio `Player` for a synthesis unit.
|
||||
///
|
||||
/// Every playback boundary gets a short lead-in pad immediately before its
|
||||
/// audio. This matters for chunks that start with soft first phonemes (`I'm`,
|
||||
/// `I've`): the synthesized buffer can begin with speech within the first
|
||||
/// millisecond, so the playback layer must provide the device/mixer cushion.
|
||||
/// To keep the audible gap unchanged, the trailing silence after this chunk is
|
||||
/// shortened by the same amount (`silence_buf_len - SENTENCE_LEAD_IN_SAMPLES`):
|
||||
/// sentence N contributes 80 ms of post-speech silence and sentence N+1
|
||||
/// contributes the remaining 20 ms of pre-speech cushion.
|
||||
///
|
||||
/// The lead-in, audio, and trailing silence are concatenated into one
|
||||
/// `SamplesBuffer` before appending. This keeps rodio's queue shape at one
|
||||
/// tracked source per synthesized sentence, avoiding source-boundary/drain
|
||||
/// regressions from enqueueing the lead-in, audio, and tail as separate sounds.
|
||||
///
|
||||
/// A playback chunk may contain several model-sized synthesis units. Only the
|
||||
/// first unit receives the onset cushion and only the last receives the
|
||||
/// remaining gap. If playback underruns while the next unit is synthesized,
|
||||
/// that unit becomes a new playback boundary and receives a fresh cushion.
|
||||
///
|
||||
/// `first_append` is flipped on the first call after the player goes idle.
|
||||
/// The worker uses it in the idle branch of the main loop to distinguish
|
||||
/// "never queued anything since last drain" from "drained after speaking",
|
||||
/// which controls when `tts_active` is released and the lead-in re-armed.
|
||||
fn build_sentence_append_buffer(
|
||||
first_append: &mut bool,
|
||||
audio: Vec<f32>,
|
||||
silence_buf_len: usize,
|
||||
starts_playback_chunk: bool,
|
||||
ends_playback_chunk: bool,
|
||||
) -> Vec<f32> {
|
||||
if *first_append {
|
||||
*first_append = false;
|
||||
}
|
||||
|
||||
let lead_in_len = if starts_playback_chunk {
|
||||
SENTENCE_LEAD_IN_SAMPLES
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let trailing_silence_len = if ends_playback_chunk {
|
||||
silence_buf_len.saturating_sub(SENTENCE_LEAD_IN_SAMPLES)
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let mut buf = Vec::with_capacity(lead_in_len + audio.len() + trailing_silence_len);
|
||||
buf.extend(std::iter::repeat_n(0.0_f32, lead_in_len));
|
||||
buf.extend(audio);
|
||||
buf.extend(std::iter::repeat_n(0.0_f32, trailing_silence_len));
|
||||
buf
|
||||
}
|
||||
|
||||
/// Group sentences into synthesis chunks.
|
||||
///
|
||||
/// The first sentence always stands alone — it is what the listener hears
|
||||
/// first, and synthesizing it by itself keeps time-to-first-audio at the
|
||||
/// single-sentence cost. Subsequent sentences pack greedily: a sentence
|
||||
/// joins the current chunk while the combined length stays within
|
||||
/// `max_chars`; otherwise it starts a new chunk. A single sentence longer
|
||||
/// than `max_chars` becomes its own chunk here, then the Pocket engine splits
|
||||
/// it at the April bundle's exact token limit before synthesis.
|
||||
///
|
||||
/// Sentences within a chunk are joined with a single space; sentence-ending
|
||||
/// punctuation is preserved by `split_sentences`, so the model sees natural
|
||||
/// multi-sentence prose — the same shape upstream's ~50-token chunker feeds it.
|
||||
fn group_sentences_into_chunks(sentences: &[String], max_chars: usize) -> Vec<String> {
|
||||
let mut chunks: Vec<String> = Vec::new();
|
||||
for (i, sentence) in sentences.iter().enumerate() {
|
||||
let sentence = sentence.trim();
|
||||
if sentence.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if i == 0 || chunks.is_empty() {
|
||||
chunks.push(sentence.to_string());
|
||||
continue;
|
||||
}
|
||||
// Never merge into the first chunk — it's the latency-critical one.
|
||||
let can_merge = chunks.len() > 1
|
||||
&& chunks
|
||||
.last()
|
||||
.is_some_and(|c| c.len() + 1 + sentence.len() <= max_chars);
|
||||
if can_merge {
|
||||
let last = chunks.last_mut().expect("non-empty checked above");
|
||||
last.push(' ');
|
||||
last.push_str(sentence);
|
||||
} else {
|
||||
chunks.push(sentence.to_string());
|
||||
}
|
||||
}
|
||||
chunks
|
||||
}
|
||||
|
||||
// drain_until_shutdown lives in super (huddle/mod.rs) — shared with stt.rs.
|
||||
use super::drain_until_shutdown;
|
||||
|
||||
// ── Tests ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "tts_tests.rs"]
|
||||
mod tests;
|
||||
#[cfg(test)]
|
||||
#[path = "tts_voice_selection_tests.rs"]
|
||||
mod voice_selection_tests;
|
||||
|
||||
235
desktop/src-tauri/src/huddle/tts_audio.rs
Normal file
235
desktop/src-tauri/src/huddle/tts_audio.rs
Normal file
@@ -0,0 +1,235 @@
|
||||
use super::{FADE_OUT_SAMPLES, SENTENCE_LEAD_IN_SAMPLES};
|
||||
|
||||
pub(super) struct PreparedModelAudio {
|
||||
pub(super) buffer: Vec<f32>,
|
||||
pub(super) sample_count: usize,
|
||||
pub(super) chunk_index: usize,
|
||||
}
|
||||
|
||||
/// Holds one synthesized model unit so playback-boundary decoration is based
|
||||
/// on the first and last unit that actually produced audio.
|
||||
pub(super) struct PlaybackChunkAudio {
|
||||
pending: Option<(Vec<f32>, usize)>,
|
||||
appended: bool,
|
||||
}
|
||||
|
||||
impl PlaybackChunkAudio {
|
||||
pub(super) fn new() -> Self {
|
||||
Self {
|
||||
pending: None,
|
||||
appended: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn push(
|
||||
&mut self,
|
||||
samples: Vec<f32>,
|
||||
chunk_index: usize,
|
||||
first_append: &mut bool,
|
||||
silence_buf_len: usize,
|
||||
playback_idle: bool,
|
||||
) -> Option<PreparedModelAudio> {
|
||||
if samples.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let previous = self.pending.replace((samples, chunk_index))?;
|
||||
let prepared = prepare_model_audio(
|
||||
previous,
|
||||
first_append,
|
||||
silence_buf_len,
|
||||
!self.appended || playback_idle,
|
||||
false,
|
||||
);
|
||||
self.appended = true;
|
||||
Some(prepared)
|
||||
}
|
||||
|
||||
pub(super) fn finish(
|
||||
&mut self,
|
||||
first_append: &mut bool,
|
||||
silence_buf_len: usize,
|
||||
playback_idle: bool,
|
||||
) -> Option<PreparedModelAudio> {
|
||||
let pending = self.pending.take()?;
|
||||
Some(prepare_model_audio(
|
||||
pending,
|
||||
first_append,
|
||||
silence_buf_len,
|
||||
!self.appended || playback_idle,
|
||||
true,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
fn prepare_model_audio(
|
||||
(samples, chunk_index): (Vec<f32>, usize),
|
||||
first_append: &mut bool,
|
||||
silence_buf_len: usize,
|
||||
starts_playback_chunk: bool,
|
||||
ends_playback_chunk: bool,
|
||||
) -> PreparedModelAudio {
|
||||
let sample_count = samples.len();
|
||||
let mut audio = clamp_to_full_scale(samples);
|
||||
if ends_playback_chunk {
|
||||
apply_fade_out(&mut audio);
|
||||
}
|
||||
PreparedModelAudio {
|
||||
buffer: build_sentence_append_buffer(
|
||||
first_append,
|
||||
audio,
|
||||
silence_buf_len,
|
||||
starts_playback_chunk,
|
||||
ends_playback_chunk,
|
||||
),
|
||||
sample_count,
|
||||
chunk_index,
|
||||
}
|
||||
}
|
||||
|
||||
/// Hard-clamp samples to ±1.0 full scale.
|
||||
pub(super) fn clamp_to_full_scale(samples: Vec<f32>) -> Vec<f32> {
|
||||
samples.into_iter().map(|s| s.clamp(-1.0, 1.0)).collect()
|
||||
}
|
||||
|
||||
/// Apply a short linear fade-out to avoid a discontinuity at playback boundaries.
|
||||
pub(super) fn apply_fade_out(samples: &mut [f32]) {
|
||||
let len = samples.len();
|
||||
let fade = FADE_OUT_SAMPLES.min(len / 2);
|
||||
for i in 0..fade {
|
||||
samples[len - 1 - i] *= i as f32 / fade as f32;
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn build_sentence_append_buffer(
|
||||
first_append: &mut bool,
|
||||
audio: Vec<f32>,
|
||||
silence_buf_len: usize,
|
||||
starts_playback_chunk: bool,
|
||||
ends_playback_chunk: bool,
|
||||
) -> Vec<f32> {
|
||||
if *first_append {
|
||||
*first_append = false;
|
||||
}
|
||||
|
||||
let lead_in_len = if starts_playback_chunk {
|
||||
SENTENCE_LEAD_IN_SAMPLES
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let trailing_silence_len = if ends_playback_chunk {
|
||||
silence_buf_len.saturating_sub(SENTENCE_LEAD_IN_SAMPLES)
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let mut buffer = Vec::with_capacity(lead_in_len + audio.len() + trailing_silence_len);
|
||||
buffer.extend(std::iter::repeat_n(0.0_f32, lead_in_len));
|
||||
buffer.extend(audio);
|
||||
buffer.extend(std::iter::repeat_n(0.0_f32, trailing_silence_len));
|
||||
buffer
|
||||
}
|
||||
|
||||
pub(super) fn group_sentences_into_chunks(sentences: &[String], max_chars: usize) -> Vec<String> {
|
||||
let mut chunks: Vec<String> = Vec::new();
|
||||
for (index, sentence) in sentences.iter().enumerate() {
|
||||
let sentence = sentence.trim();
|
||||
if sentence.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if index == 0 || chunks.is_empty() {
|
||||
chunks.push(sentence.to_string());
|
||||
continue;
|
||||
}
|
||||
let can_merge = chunks.len() > 1
|
||||
&& chunks
|
||||
.last()
|
||||
.is_some_and(|chunk| chunk.len() + 1 + sentence.len() <= max_chars);
|
||||
if can_merge {
|
||||
if let Some(last) = chunks.last_mut() {
|
||||
last.push(' ');
|
||||
last.push_str(sentence);
|
||||
}
|
||||
} else {
|
||||
chunks.push(sentence.to_string());
|
||||
}
|
||||
}
|
||||
chunks
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn multi_unit_audio_decorates_only_outer_playback_boundaries() {
|
||||
let mut chunk = PlaybackChunkAudio::new();
|
||||
let mut first_append = true;
|
||||
let silence = SENTENCE_LEAD_IN_SAMPLES + 100;
|
||||
|
||||
assert!(chunk
|
||||
.push(vec![0.4; 16], 0, &mut first_append, silence, false)
|
||||
.is_none());
|
||||
let first = chunk
|
||||
.push(vec![0.5; 16], 1, &mut first_append, silence, false)
|
||||
.expect("first ready model unit");
|
||||
assert_eq!(first.buffer.len(), SENTENCE_LEAD_IN_SAMPLES + 16);
|
||||
assert!(first.buffer[..SENTENCE_LEAD_IN_SAMPLES]
|
||||
.iter()
|
||||
.all(|sample| *sample == 0.0));
|
||||
assert_eq!(first.buffer[SENTENCE_LEAD_IN_SAMPLES], 0.4);
|
||||
|
||||
let last = chunk
|
||||
.finish(&mut first_append, silence, false)
|
||||
.expect("last ready model unit");
|
||||
assert_eq!(last.buffer.len(), 16 + 100);
|
||||
assert_eq!(last.buffer.last(), Some(&0.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_edge_units_do_not_steal_lead_in_or_trailing_boundary() {
|
||||
let mut chunk = PlaybackChunkAudio::new();
|
||||
let mut first_append = true;
|
||||
let silence = SENTENCE_LEAD_IN_SAMPLES + 100;
|
||||
|
||||
assert!(chunk
|
||||
.push(Vec::new(), 0, &mut first_append, silence, false)
|
||||
.is_none());
|
||||
assert!(chunk
|
||||
.push(vec![0.5; 16], 1, &mut first_append, silence, false)
|
||||
.is_none());
|
||||
assert!(chunk
|
||||
.push(Vec::new(), 2, &mut first_append, silence, false)
|
||||
.is_none());
|
||||
|
||||
let only = chunk
|
||||
.finish(&mut first_append, silence, false)
|
||||
.expect("only audible model unit");
|
||||
assert_eq!(only.buffer.len(), SENTENCE_LEAD_IN_SAMPLES + 16 + 100);
|
||||
assert!(only.buffer[..SENTENCE_LEAD_IN_SAMPLES]
|
||||
.iter()
|
||||
.all(|sample| *sample == 0.0));
|
||||
assert_eq!(only.buffer.last(), Some(&0.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn playback_underrun_rearms_the_onset_cushion() {
|
||||
let mut chunk = PlaybackChunkAudio::new();
|
||||
let mut first_append = true;
|
||||
let silence = SENTENCE_LEAD_IN_SAMPLES + 100;
|
||||
|
||||
assert!(chunk
|
||||
.push(vec![0.4; 16], 0, &mut first_append, silence, false)
|
||||
.is_none());
|
||||
let first = chunk
|
||||
.push(vec![0.5; 16], 1, &mut first_append, silence, false)
|
||||
.expect("first model unit");
|
||||
assert_eq!(first.buffer.len(), SENTENCE_LEAD_IN_SAMPLES + 16);
|
||||
|
||||
let after_underrun = chunk
|
||||
.push(vec![0.6; 16], 2, &mut first_append, silence, true)
|
||||
.expect("model unit after underrun");
|
||||
assert_eq!(after_underrun.buffer.len(), SENTENCE_LEAD_IN_SAMPLES + 16);
|
||||
assert!(after_underrun.buffer[..SENTENCE_LEAD_IN_SAMPLES]
|
||||
.iter()
|
||||
.all(|sample| *sample == 0.0));
|
||||
}
|
||||
}
|
||||
831
desktop/src-tauri/src/huddle/tts_settings.rs
Normal file
831
desktop/src-tauri/src/huddle/tts_settings.rs
Normal file
@@ -0,0 +1,831 @@
|
||||
//! Installation-global text-to-speech preferences and the local voice registry.
|
||||
//!
|
||||
//! Voice keys are backend-qualified (`pocket:mary`, `siri:aaron`) and
|
||||
//! preferences are ordered. A client resolves the first compatible entry for
|
||||
//! its one active playback backend. The same [`VoicePreferences`] value can be
|
||||
//! embedded in installation-global settings or future agent identity without a
|
||||
//! schema change. Availability is intentionally client-local.
|
||||
|
||||
use std::{
|
||||
path::{Path, PathBuf},
|
||||
sync::{Arc, Mutex},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tauri::{AppHandle, Manager, State};
|
||||
|
||||
use crate::{app_state::AppState, managed_agents::storage::atomic_write_json_restricted};
|
||||
|
||||
use super::{
|
||||
models,
|
||||
pocket::DEFAULT_VOICE,
|
||||
tts_voice_registry::{source_url, MARY_VOICE_KEY, POCKET_VOICES},
|
||||
HuddlePhase, HuddleState,
|
||||
};
|
||||
|
||||
const SETTINGS_FILE: &str = "tts-settings.json";
|
||||
const CURRENT_VERSION: u32 = 1;
|
||||
const VOICE_CHANGE_ACK_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
pub const POCKET_BACKEND_ID: &str = "pocket";
|
||||
|
||||
type VoiceChangeWait = (
|
||||
Arc<super::tts::TtsPipeline>,
|
||||
tokio::sync::oneshot::Receiver<()>,
|
||||
);
|
||||
|
||||
const VOICE_AVAILABILITY_BUNDLED: &str = "bundled";
|
||||
const VOICE_AVAILABILITY_INSTALLED: &str = "installed";
|
||||
|
||||
/// Installation-global huddle audio and speech preferences.
|
||||
#[derive(Default)]
|
||||
pub struct HuddleAudioSettingsState {
|
||||
pub tts: Mutex<TtsSettings>,
|
||||
pub tts_load_error: Mutex<Option<String>>,
|
||||
pub tts_transition: tokio::sync::Mutex<()>,
|
||||
/// Selected huddle output device. `None` uses the system default.
|
||||
pub output_device: Mutex<Option<String>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct VoiceRegistryEntry {
|
||||
/// Stable identity, never derived from or merged by the display name.
|
||||
///
|
||||
/// Built-ins use `backend:slug`. Future imports use
|
||||
/// `pocket:imported:<audio-content-sha256>` so two clips with the same
|
||||
/// editable label remain distinct.
|
||||
pub key: String,
|
||||
pub display_name: String,
|
||||
pub backend: String,
|
||||
pub backend_name: String,
|
||||
/// Client-local state: bundled, installed, downloadable, or unavailable.
|
||||
pub availability: String,
|
||||
pub fallback_key: Option<String>,
|
||||
pub reference_file: Option<String>,
|
||||
pub provenance: VoiceProvenance,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct VoiceProvenance {
|
||||
pub source: String,
|
||||
pub content_hash: Option<String>,
|
||||
pub license: Option<String>,
|
||||
pub source_url: Option<String>,
|
||||
}
|
||||
|
||||
/// Ordered, backend-qualified preferences shared by global and agent settings.
|
||||
///
|
||||
/// Unknown but well-formed keys remain persisted because a different client
|
||||
/// may have that backend installed. Resolution is always local.
|
||||
pub type VoicePreferences = Vec<String>;
|
||||
|
||||
/// Cross-backend registry for voices known to this client.
|
||||
///
|
||||
/// V1 contains Pocket entries only. Siri, Kokoro, imported voices, and
|
||||
/// per-agent assignment can add entries or reuse the preference type without
|
||||
/// changing the registry/settings boundary.
|
||||
pub fn voice_registry() -> Vec<VoiceRegistryEntry> {
|
||||
POCKET_VOICES
|
||||
.iter()
|
||||
.map(|voice| VoiceRegistryEntry {
|
||||
key: voice.key.to_string(),
|
||||
display_name: voice.display_name.to_string(),
|
||||
backend: POCKET_BACKEND_ID.to_string(),
|
||||
backend_name: "Pocket TTS".to_string(),
|
||||
availability: VOICE_AVAILABILITY_BUNDLED.to_string(),
|
||||
fallback_key: (voice.key != MARY_VOICE_KEY).then(|| MARY_VOICE_KEY.to_string()),
|
||||
reference_file: Some(voice.reference_file.to_string()),
|
||||
provenance: VoiceProvenance {
|
||||
source: "bundled".to_string(),
|
||||
content_hash: Some(voice.sha256.to_string()),
|
||||
license: Some("CC-BY-4.0".to_string()),
|
||||
source_url: Some(source_url(voice)),
|
||||
},
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct TtsSettings {
|
||||
pub version: u32,
|
||||
pub agent_text_to_speech: bool,
|
||||
pub voice_preferences: VoicePreferences,
|
||||
}
|
||||
|
||||
impl Default for TtsSettings {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
version: CURRENT_VERSION,
|
||||
agent_text_to_speech: true,
|
||||
voice_preferences: vec![MARY_VOICE_KEY.to_string()],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn voice_by_key(key: &str) -> Option<VoiceRegistryEntry> {
|
||||
voice_registry().into_iter().find(|voice| voice.key == key)
|
||||
}
|
||||
|
||||
fn is_qualified_voice_key(key: &str) -> bool {
|
||||
key.split_once(':')
|
||||
.is_some_and(|(backend, voice)| !backend.is_empty() && !voice.is_empty())
|
||||
}
|
||||
|
||||
fn is_locally_available(availability: &str) -> bool {
|
||||
matches!(
|
||||
availability,
|
||||
VOICE_AVAILABILITY_BUNDLED | VOICE_AVAILABILITY_INSTALLED
|
||||
)
|
||||
}
|
||||
|
||||
pub fn resolve_voice_for_backend(
|
||||
preferences: &[String],
|
||||
backend: &str,
|
||||
) -> Result<VoiceRegistryEntry, String> {
|
||||
let registry = voice_registry();
|
||||
preferences
|
||||
.iter()
|
||||
.filter_map(|key| registry.iter().find(|voice| voice.key == *key))
|
||||
.find(|voice| voice.backend == backend && is_locally_available(voice.availability.as_str()))
|
||||
.or_else(|| {
|
||||
registry.iter().find(|voice| {
|
||||
voice.backend == backend
|
||||
&& voice.fallback_key.is_none()
|
||||
&& is_locally_available(voice.availability.as_str())
|
||||
})
|
||||
})
|
||||
.cloned()
|
||||
.ok_or_else(|| format!("No locally available fallback voice for backend {backend}"))
|
||||
}
|
||||
|
||||
pub fn pocket_voice_name(preferences: &[String]) -> String {
|
||||
resolve_voice_for_backend(preferences, POCKET_BACKEND_ID)
|
||||
.ok()
|
||||
.and_then(|voice| voice.reference_file)
|
||||
.and_then(|file| file.strip_suffix(".wav").map(str::to_string))
|
||||
.unwrap_or_else(|| DEFAULT_VOICE.to_string())
|
||||
}
|
||||
|
||||
pub(crate) fn settings_path(app: &AppHandle) -> Result<PathBuf, String> {
|
||||
app.path()
|
||||
.app_data_dir()
|
||||
.map(|dir| dir.join(SETTINGS_FILE))
|
||||
.map_err(|error| format!("could not locate Buzz settings storage: {error}"))
|
||||
}
|
||||
|
||||
pub(crate) fn load_from_path(path: &Path) -> Result<TtsSettings, String> {
|
||||
if !path.exists() {
|
||||
return Ok(TtsSettings::default());
|
||||
}
|
||||
let bytes = std::fs::read(path)
|
||||
.map_err(|error| format!("could not read text-to-speech settings: {error}"))?;
|
||||
let value: serde_json::Value = serde_json::from_slice(&bytes)
|
||||
.map_err(|error| format!("text-to-speech settings are not valid JSON: {error}"))?;
|
||||
|
||||
// Unversioned settings are incompatible with the V1 schema. Use
|
||||
// deterministic V1 defaults rather than interpreting ambiguous fields.
|
||||
if value.get("version").is_none() {
|
||||
return Ok(TtsSettings::default());
|
||||
}
|
||||
|
||||
let version = value
|
||||
.get("version")
|
||||
.and_then(serde_json::Value::as_u64)
|
||||
.ok_or("text-to-speech settings version is invalid")?;
|
||||
if version > u64::from(CURRENT_VERSION) {
|
||||
return Err(format!(
|
||||
"text-to-speech settings version {version} is newer than this Buzz build supports"
|
||||
));
|
||||
}
|
||||
|
||||
// Legacy V1 settings may contain one bare Pocket `voiceId`. Preserve the
|
||||
// toggle and qualify it into the ordered cross-backend preference schema.
|
||||
if value.get("voicePreferences").is_none() {
|
||||
let legacy_voice = value
|
||||
.get("voiceId")
|
||||
.or_else(|| value.get("voice_id"))
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.unwrap_or("mary");
|
||||
let voice_key = if is_qualified_voice_key(legacy_voice) {
|
||||
legacy_voice.to_string()
|
||||
} else {
|
||||
format!("{POCKET_BACKEND_ID}:{legacy_voice}")
|
||||
};
|
||||
return Ok(TtsSettings {
|
||||
version: CURRENT_VERSION,
|
||||
agent_text_to_speech: value
|
||||
.get("agentTextToSpeech")
|
||||
.and_then(serde_json::Value::as_bool)
|
||||
.unwrap_or(true),
|
||||
voice_preferences: vec![voice_key],
|
||||
});
|
||||
}
|
||||
|
||||
let mut settings: TtsSettings = serde_json::from_value(value)
|
||||
.map_err(|error| format!("text-to-speech settings are invalid: {error}"))?;
|
||||
settings.version = CURRENT_VERSION;
|
||||
if settings.voice_preferences.is_empty()
|
||||
|| settings
|
||||
.voice_preferences
|
||||
.iter()
|
||||
.any(|key| !is_qualified_voice_key(key))
|
||||
{
|
||||
settings.voice_preferences = TtsSettings::default().voice_preferences;
|
||||
}
|
||||
Ok(settings)
|
||||
}
|
||||
|
||||
pub(crate) fn save_to_path(path: &Path, settings: &TtsSettings) -> Result<(), String> {
|
||||
if settings.voice_preferences.is_empty() {
|
||||
return Err("At least one voice preference is required".to_string());
|
||||
}
|
||||
if let Some(key) = settings
|
||||
.voice_preferences
|
||||
.iter()
|
||||
.find(|key| !is_qualified_voice_key(key))
|
||||
{
|
||||
return Err(format!(
|
||||
"Voice preference keys must be backend-qualified: {key}"
|
||||
));
|
||||
}
|
||||
let payload = serde_json::to_vec_pretty(settings)
|
||||
.map_err(|error| format!("could not encode text-to-speech settings: {error}"))?;
|
||||
atomic_write_json_restricted(path, &payload)
|
||||
.map_err(|error| format!("could not save text-to-speech settings: {error}"))
|
||||
}
|
||||
|
||||
pub fn load_for_app(app: &AppHandle) -> (TtsSettings, Option<String>) {
|
||||
let result = settings_path(app).and_then(|path| load_from_path(&path));
|
||||
match result {
|
||||
Ok(settings) => (settings, None),
|
||||
Err(error) => {
|
||||
eprintln!("buzz-desktop: {error}; preserving the file and using Mary for this session");
|
||||
(TtsSettings::default(), Some(error))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
pub fn get_tts_settings(state: State<'_, AppState>) -> Result<TtsSettings, String> {
|
||||
if let Some(error) = state
|
||||
.huddle_audio
|
||||
.tts_load_error
|
||||
.lock()
|
||||
.map_err(|lock_error| format!("text-to-speech settings lock poisoned: {lock_error}"))?
|
||||
.clone()
|
||||
{
|
||||
return Err(format!(
|
||||
"Voice settings could not be loaded and were left unchanged: {error}"
|
||||
));
|
||||
}
|
||||
state
|
||||
.huddle_audio
|
||||
.tts
|
||||
.lock()
|
||||
.map(|settings| settings.clone())
|
||||
.map_err(|error| format!("text-to-speech settings lock poisoned: {error}"))
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
pub fn list_voice_registry() -> Vec<VoiceRegistryEntry> {
|
||||
voice_registry()
|
||||
}
|
||||
|
||||
fn ensure_settings_writable(state: &AppState) -> Result<(), String> {
|
||||
if let Some(error) = state
|
||||
.huddle_audio
|
||||
.tts_load_error
|
||||
.lock()
|
||||
.map_err(|lock_error| format!("text-to-speech settings lock poisoned: {lock_error}"))?
|
||||
.as_ref()
|
||||
{
|
||||
return Err(format!(
|
||||
"Voice settings were not saved because the existing file could not be loaded: {error}"
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cancel_huddle_speech(
|
||||
huddle: &mut super::HuddleState,
|
||||
) -> Option<std::sync::Arc<super::tts::TtsPipeline>> {
|
||||
huddle.tts_enabled = false;
|
||||
huddle
|
||||
.tts_cancel
|
||||
.store(true, std::sync::atomic::Ordering::Release);
|
||||
huddle.tts_pipeline.take()
|
||||
}
|
||||
|
||||
fn disable_tts_runtime(state: &AppState) -> Result<(), String> {
|
||||
let old_pipeline = {
|
||||
let mut huddle = state.huddle()?;
|
||||
cancel_huddle_speech(&mut huddle)
|
||||
};
|
||||
if let Some(ref pipeline) = old_pipeline {
|
||||
pipeline.shutdown();
|
||||
}
|
||||
drop(old_pipeline);
|
||||
state.emit_huddle_state_changed();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn commit_effective_off(state: &AppState) -> Result<(), String> {
|
||||
state
|
||||
.huddle_audio
|
||||
.tts
|
||||
.lock()
|
||||
.map_err(|error| format!("text-to-speech settings lock poisoned: {error}"))?
|
||||
.agent_text_to_speech = false;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn enable_tts_runtime(huddle: &mut HuddleState, voice: &str) -> Option<VoiceChangeWait> {
|
||||
huddle.tts_enabled = true;
|
||||
// OFF removes the pipeline. Clear a prior cancellation only when enabling
|
||||
// a fresh pipeline; an idempotent ON write must not erase a voice
|
||||
// transition that the existing worker still needs to drain.
|
||||
prepare_enable_cancel(&huddle.tts_cancel, huddle.tts_pipeline.is_some());
|
||||
huddle.tts_pipeline.as_ref().and_then(|pipeline| {
|
||||
pipeline
|
||||
.select_voice(voice)
|
||||
.map(|acknowledged| (Arc::clone(pipeline), acknowledged))
|
||||
})
|
||||
}
|
||||
|
||||
fn prepare_enable_cancel(cancel: &std::sync::atomic::AtomicBool, has_pipeline: bool) {
|
||||
if !has_pipeline {
|
||||
cancel.store(false, std::sync::atomic::Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
async fn apply_tts_settings(
|
||||
settings: TtsSettings,
|
||||
app: &AppHandle,
|
||||
state: &AppState,
|
||||
) -> Result<Option<VoiceChangeWait>, String> {
|
||||
if settings.version != CURRENT_VERSION {
|
||||
return Err(format!(
|
||||
"Unsupported text-to-speech settings version: {}",
|
||||
settings.version
|
||||
));
|
||||
}
|
||||
|
||||
// OFF is safety-sensitive: stop current and queued speech before any disk
|
||||
// I/O, and never resume it merely because persistence fails.
|
||||
if !settings.agent_text_to_speech {
|
||||
disable_tts_runtime(state)?;
|
||||
commit_effective_off(state)?;
|
||||
}
|
||||
|
||||
ensure_settings_writable(state)?;
|
||||
save_to_path(&settings_path(app)?, &settings)?;
|
||||
|
||||
*state
|
||||
.huddle_audio
|
||||
.tts
|
||||
.lock()
|
||||
.map_err(|error| format!("text-to-speech settings lock poisoned: {error}"))? =
|
||||
settings.clone();
|
||||
|
||||
let mut voice_change_wait = None;
|
||||
if settings.agent_text_to_speech {
|
||||
let (active, voice_change_ack) = {
|
||||
let mut huddle = state.huddle()?;
|
||||
let voice_change_ack =
|
||||
enable_tts_runtime(&mut huddle, &pocket_voice_name(&settings.voice_preferences));
|
||||
(
|
||||
matches!(huddle.phase, HuddlePhase::Connected | HuddlePhase::Active),
|
||||
voice_change_ack,
|
||||
)
|
||||
};
|
||||
voice_change_wait = voice_change_ack;
|
||||
if active {
|
||||
if let Err(error) = super::pipeline::maybe_start_tts_pipeline(state).await {
|
||||
eprintln!("buzz-desktop: could not hot-start text to speech: {error}");
|
||||
}
|
||||
}
|
||||
state.emit_huddle_state_changed();
|
||||
}
|
||||
Ok(voice_change_wait)
|
||||
}
|
||||
|
||||
fn current_settings(state: &AppState) -> Result<TtsSettings, String> {
|
||||
state
|
||||
.huddle_audio
|
||||
.tts
|
||||
.lock()
|
||||
.map_err(|error| format!("text-to-speech settings lock poisoned: {error}"))
|
||||
.map(|settings| settings.clone())
|
||||
}
|
||||
|
||||
async fn finish_voice_change(voice_change: Option<VoiceChangeWait>) -> Result<(), String> {
|
||||
let Some((pipeline, acknowledged)) = voice_change else {
|
||||
return Ok(());
|
||||
};
|
||||
wait_for_voice_change_ack(acknowledged, VOICE_CHANGE_ACK_TIMEOUT, || {
|
||||
pipeline.is_finished()
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn wait_for_voice_change_ack(
|
||||
mut acknowledged: tokio::sync::oneshot::Receiver<()>,
|
||||
timeout: Duration,
|
||||
mut worker_is_finished: impl FnMut() -> bool,
|
||||
) -> Result<(), String> {
|
||||
let deadline = tokio::time::sleep(timeout);
|
||||
tokio::pin!(deadline);
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = &mut acknowledged => return Ok(()),
|
||||
_ = &mut deadline => {
|
||||
return Err(
|
||||
"Pocket TTS is still finishing the previous voice. Turn Agent text to speech off and try again."
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
_ = tokio::time::sleep(Duration::from_millis(25)) => {
|
||||
if worker_is_finished() {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Compatibility command for the huddle speaker button. It updates the same
|
||||
/// installation-global preference as Settings; there is no per-huddle override.
|
||||
#[tauri::command]
|
||||
pub async fn set_tts_enabled(
|
||||
enabled: bool,
|
||||
app: AppHandle,
|
||||
state: State<'_, AppState>,
|
||||
) -> Result<TtsSettings, String> {
|
||||
let transition = state.huddle_audio.tts_transition.lock().await;
|
||||
let mut settings = state
|
||||
.huddle_audio
|
||||
.tts
|
||||
.lock()
|
||||
.map_err(|error| format!("text-to-speech settings lock poisoned: {error}"))?
|
||||
.clone();
|
||||
settings.agent_text_to_speech = enabled;
|
||||
let voice_change = apply_tts_settings(settings, &app, &state).await?;
|
||||
drop(transition);
|
||||
finish_voice_change(voice_change).await?;
|
||||
current_settings(&state)
|
||||
}
|
||||
|
||||
fn settings_with_pocket_voice(
|
||||
mut settings: TtsSettings,
|
||||
voice_key: &str,
|
||||
) -> Result<TtsSettings, String> {
|
||||
let voice = voice_by_key(voice_key).ok_or_else(|| format!("Unknown voice: {voice_key}"))?;
|
||||
if voice.backend != POCKET_BACKEND_ID || !is_locally_available(&voice.availability) {
|
||||
return Err("The selected Pocket voice is not available on this device".to_string());
|
||||
}
|
||||
let first_pocket_index = settings
|
||||
.voice_preferences
|
||||
.iter()
|
||||
.position(|key| key.starts_with("pocket:"));
|
||||
settings
|
||||
.voice_preferences
|
||||
.retain(|key| !key.starts_with("pocket:"));
|
||||
let insert_at = first_pocket_index
|
||||
.unwrap_or(settings.voice_preferences.len())
|
||||
.min(settings.voice_preferences.len());
|
||||
settings
|
||||
.voice_preferences
|
||||
.insert(insert_at, voice_key.to_string());
|
||||
Ok(settings)
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
pub async fn set_pocket_voice(
|
||||
voice_key: String,
|
||||
app: AppHandle,
|
||||
state: State<'_, AppState>,
|
||||
) -> Result<TtsSettings, String> {
|
||||
let transition = state.huddle_audio.tts_transition.lock().await;
|
||||
let settings = state
|
||||
.huddle_audio
|
||||
.tts
|
||||
.lock()
|
||||
.map_err(|error| format!("text-to-speech settings lock poisoned: {error}"))?
|
||||
.clone();
|
||||
let settings = settings_with_pocket_voice(settings, &voice_key)?;
|
||||
let voice_change = apply_tts_settings(settings, &app, &state).await?;
|
||||
drop(transition);
|
||||
if let Err(error) = finish_voice_change(voice_change).await {
|
||||
// The preference is already durable. Report the delayed live
|
||||
// transition diagnostically without telling the UI that saving failed;
|
||||
// the next pipeline start resolves the persisted voice normally.
|
||||
eprintln!(
|
||||
"buzz-desktop: tts stage=voice_switch status=delayed reason=ack_timeout error={error}"
|
||||
);
|
||||
}
|
||||
current_settings(&state)
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
pub async fn preview_pocket_voice(
|
||||
voice_key: String,
|
||||
state: State<'_, AppState>,
|
||||
) -> Result<(), String> {
|
||||
let voice = voice_by_key(&voice_key).ok_or_else(|| format!("Unknown voice: {voice_key}"))?;
|
||||
if voice.backend != POCKET_BACKEND_ID {
|
||||
return Err("Only Pocket voices can be previewed in this build".to_string());
|
||||
}
|
||||
if !models::is_tts_ready() {
|
||||
return Err("Voice files are still downloading. Try preview again shortly.".to_string());
|
||||
}
|
||||
let model_dir = models::tts_model_dir().ok_or("Pocket voice files are unavailable")?;
|
||||
let output_device = state
|
||||
.huddle_audio
|
||||
.output_device
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner())
|
||||
.clone();
|
||||
let voice_name = voice
|
||||
.reference_file
|
||||
.and_then(|file| file.strip_suffix(".wav").map(str::to_string))
|
||||
.ok_or_else(|| format!("Voice {voice_key} has no local Pocket reference file"))?;
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let active = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let cancel = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let pipeline = super::tts::TtsPipeline::new_with_voice(
|
||||
model_dir,
|
||||
active.clone(),
|
||||
cancel,
|
||||
&voice_name,
|
||||
output_device,
|
||||
)?;
|
||||
pipeline.speak("Hello! This is how I’ll read agent responses.".to_string())?;
|
||||
let started = std::time::Instant::now();
|
||||
let mut heard_audio = false;
|
||||
while started.elapsed() < std::time::Duration::from_secs(30) {
|
||||
let is_active = active.load(std::sync::atomic::Ordering::Acquire);
|
||||
heard_audio |= is_active;
|
||||
if heard_audio && !is_active {
|
||||
return Ok(());
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(25));
|
||||
}
|
||||
Err("Voice preview timed out. Check your audio output and try again.".to_string())
|
||||
})
|
||||
.await
|
||||
.map_err(|error| format!("Voice preview task failed: {error}"))?
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
const EVE_VOICE_KEY: &str = "pocket:eve";
|
||||
|
||||
#[tokio::test]
|
||||
async fn stalled_voice_change_returns_an_actionable_error() {
|
||||
let (_keep_pending, acknowledged) = tokio::sync::oneshot::channel();
|
||||
|
||||
let error = wait_for_voice_change_ack(acknowledged, Duration::from_millis(1), || false)
|
||||
.await
|
||||
.expect_err("stalled worker should time out");
|
||||
|
||||
assert!(error.contains("Turn Agent text to speech off"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn idempotent_enable_preserves_an_existing_pipeline_cancel() {
|
||||
let cancel = std::sync::atomic::AtomicBool::new(true);
|
||||
prepare_enable_cancel(&cancel, true);
|
||||
assert!(cancel.load(std::sync::atomic::Ordering::Acquire));
|
||||
prepare_enable_cancel(&cancel, false);
|
||||
assert!(!cancel.load(std::sync::atomic::Ordering::Acquire));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn defaults_are_backwards_compatible_and_use_mary() {
|
||||
assert_eq!(
|
||||
TtsSettings::default(),
|
||||
TtsSettings {
|
||||
version: 1,
|
||||
agent_text_to_speech: true,
|
||||
voice_preferences: vec!["pocket:mary".to_string()],
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_has_all_official_english_vctk_presets() {
|
||||
assert_eq!(
|
||||
voice_registry()
|
||||
.iter()
|
||||
.map(|voice| {
|
||||
(
|
||||
voice.key.as_str(),
|
||||
voice.display_name.as_str(),
|
||||
voice.reference_file.as_deref(),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
vec![
|
||||
("pocket:anna", "Anna", Some("anna.wav")),
|
||||
("pocket:vera", "Vera", Some("vera.wav")),
|
||||
("pocket:fantine", "Fantine", Some("fantine.wav")),
|
||||
("pocket:charles", "Charles", Some("charles.wav")),
|
||||
("pocket:paul", "Paul", Some("paul.wav")),
|
||||
("pocket:eponine", "Eponine", Some("eponine.wav")),
|
||||
("pocket:azelma", "Azelma", Some("azelma.wav")),
|
||||
("pocket:george", "George", Some("george.wav")),
|
||||
("pocket:mary", "Mary", Some("reference_sample.wav")),
|
||||
("pocket:jane", "Jane", Some("jane.wav")),
|
||||
("pocket:michael", "Michael", Some("michael.wav")),
|
||||
("pocket:eve", "Eve", Some("eve.wav")),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_backend_resolution_uses_first_compatible_preference() {
|
||||
let preferences = vec![
|
||||
"siri:aaron".to_string(),
|
||||
EVE_VOICE_KEY.to_string(),
|
||||
MARY_VOICE_KEY.to_string(),
|
||||
"kokoro:af_heart".to_string(),
|
||||
];
|
||||
assert_eq!(
|
||||
resolve_voice_for_backend(&preferences, POCKET_BACKEND_ID)
|
||||
.expect("Pocket fallback")
|
||||
.key,
|
||||
EVE_VOICE_KEY
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsupported_or_missing_preferences_fall_back_to_backend_default() {
|
||||
let preferences = vec![
|
||||
"siri:aaron".to_string(),
|
||||
"pocket:imported:deadbeef".to_string(),
|
||||
];
|
||||
assert_eq!(
|
||||
resolve_voice_for_backend(&preferences, POCKET_BACKEND_ID)
|
||||
.expect("Pocket fallback")
|
||||
.key,
|
||||
MARY_VOICE_KEY
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identity_is_qualified_key_not_display_label() {
|
||||
assert!(is_qualified_voice_key("pocket:imported:audio-content-hash"));
|
||||
assert_ne!(MARY_VOICE_KEY, EVE_VOICE_KEY);
|
||||
let mut registry = voice_registry();
|
||||
registry[0].display_name = "Jim".to_string();
|
||||
registry[1].display_name = "Jim".to_string();
|
||||
assert_eq!(registry[0].display_name, registry[1].display_name);
|
||||
assert_ne!(registry[0].key, registry[1].key);
|
||||
assert_eq!(
|
||||
registry
|
||||
.iter()
|
||||
.map(|voice| voice.key.as_str())
|
||||
.collect::<std::collections::HashSet<_>>()
|
||||
.len(),
|
||||
registry.len()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bundled_vctk_assets_match_the_registry_manifest() {
|
||||
for voice in POCKET_VOICES {
|
||||
let Some(bytes) = voice.bytes else {
|
||||
continue;
|
||||
};
|
||||
assert_eq!(&bytes[0..4], b"RIFF", "{}", voice.display_name);
|
||||
assert_eq!(&bytes[8..12], b"WAVE", "{}", voice.display_name);
|
||||
assert_eq!(
|
||||
hex::encode(<sha2::Sha256 as sha2::Digest>::digest(bytes)),
|
||||
voice.sha256,
|
||||
"{}",
|
||||
voice.display_name
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn migrates_unversioned_experiment_settings_to_v1_defaults() {
|
||||
let dir = tempfile::tempdir().expect("temp dir");
|
||||
let path = dir.path().join(SETTINGS_FILE);
|
||||
std::fs::write(&path, r#"{"voice":"legacy-experiment"}"#).expect("fixture write");
|
||||
assert_eq!(
|
||||
load_from_path(&path).expect("migration"),
|
||||
TtsSettings::default()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn migrates_bare_pocket_voice_id_to_qualified_preferences() {
|
||||
let dir = tempfile::tempdir().expect("temp dir");
|
||||
let path = dir.path().join(SETTINGS_FILE);
|
||||
std::fs::write(
|
||||
&path,
|
||||
r#"{"version":1,"agentTextToSpeech":false,"voiceId":"eve"}"#,
|
||||
)
|
||||
.expect("fixture write");
|
||||
assert_eq!(
|
||||
load_from_path(&path).expect("migration"),
|
||||
TtsSettings {
|
||||
version: 1,
|
||||
agent_text_to_speech: false,
|
||||
voice_preferences: vec![EVE_VOICE_KEY.to_string()],
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_qualified_preferences_are_preserved_for_other_clients() {
|
||||
let dir = tempfile::tempdir().expect("temp dir");
|
||||
let path = dir.path().join(SETTINGS_FILE);
|
||||
std::fs::write(
|
||||
&path,
|
||||
r#"{"version":1,"agentTextToSpeech":false,"voicePreferences":["siri:aaron","pocket:imported:abc123"]}"#,
|
||||
)
|
||||
.expect("fixture write");
|
||||
let settings = load_from_path(&path).expect("load");
|
||||
assert!(!settings.agent_text_to_speech);
|
||||
assert_eq!(
|
||||
settings.voice_preferences,
|
||||
vec!["siri:aaron", "pocket:imported:abc123"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_future_schema_versions_clearly() {
|
||||
let dir = tempfile::tempdir().expect("temp dir");
|
||||
let path = dir.path().join(SETTINGS_FILE);
|
||||
std::fs::write(
|
||||
&path,
|
||||
r#"{"version":99,"agentTextToSpeech":true,"voicePreferences":["pocket:mary"]}"#,
|
||||
)
|
||||
.expect("fixture write");
|
||||
assert!(load_from_path(&path)
|
||||
.expect_err("future version should fail")
|
||||
.contains("newer than this Buzz build supports"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disabling_cancels_runtime_before_persistence_can_fail() {
|
||||
let mut huddle = super::super::HuddleState {
|
||||
tts_enabled: true,
|
||||
..super::super::HuddleState::default()
|
||||
};
|
||||
assert!(!huddle.tts_cancel.load(std::sync::atomic::Ordering::Acquire));
|
||||
assert!(cancel_huddle_speech(&mut huddle).is_none());
|
||||
assert!(!huddle.tts_enabled);
|
||||
assert!(huddle.tts_cancel.load(std::sync::atomic::Ordering::Acquire));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pocket_voice_update_preserves_the_latest_toggle_and_other_backends() {
|
||||
let current = TtsSettings {
|
||||
agent_text_to_speech: false,
|
||||
voice_preferences: vec!["siri:aaron".to_string(), MARY_VOICE_KEY.to_string()],
|
||||
..TtsSettings::default()
|
||||
};
|
||||
let updated = settings_with_pocket_voice(current, EVE_VOICE_KEY).expect("available voice");
|
||||
assert!(!updated.agent_text_to_speech);
|
||||
assert_eq!(updated.voice_preferences, vec!["siri:aaron", EVE_VOICE_KEY]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failed_off_persistence_cannot_be_undone_by_a_later_voice_update() {
|
||||
let state = crate::app_state::build_app_state();
|
||||
commit_effective_off(&state).expect("commit effective OFF state");
|
||||
|
||||
// This models the next command after the OFF save fails: it must merge
|
||||
// from effective memory state, not the stale last-persisted ON value.
|
||||
let current = state.huddle_audio.tts.lock().expect("settings").clone();
|
||||
let voice_update =
|
||||
settings_with_pocket_voice(current, EVE_VOICE_KEY).expect("available voice");
|
||||
assert!(!voice_update.agent_text_to_speech);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failed_disabled_voice_save_does_not_change_the_remembered_voice() {
|
||||
let state = crate::app_state::build_app_state();
|
||||
state
|
||||
.huddle_audio
|
||||
.tts
|
||||
.lock()
|
||||
.expect("settings")
|
||||
.agent_text_to_speech = false;
|
||||
let current = state.huddle_audio.tts.lock().expect("settings").clone();
|
||||
let unsaved = settings_with_pocket_voice(current, EVE_VOICE_KEY).expect("available voice");
|
||||
|
||||
// This is the only pre-persistence mutation for an OFF candidate.
|
||||
commit_effective_off(&state).expect("commit effective OFF state");
|
||||
let remembered = state.huddle_audio.tts.lock().expect("settings").clone();
|
||||
assert_eq!(remembered.voice_preferences, vec![MARY_VOICE_KEY]);
|
||||
assert_eq!(unsaved.voice_preferences, vec![EVE_VOICE_KEY]);
|
||||
}
|
||||
}
|
||||
24
desktop/src-tauri/src/huddle/tts_startup.rs
Normal file
24
desktop/src-tauri/src/huddle/tts_startup.rs
Normal file
@@ -0,0 +1,24 @@
|
||||
use std::{sync::mpsc, thread};
|
||||
|
||||
pub(super) fn await_worker_startup(
|
||||
handle: thread::JoinHandle<()>,
|
||||
startup_rx: mpsc::Receiver<Result<(), String>>,
|
||||
) -> Result<thread::JoinHandle<()>, String> {
|
||||
match startup_rx.recv() {
|
||||
Ok(Ok(())) => Ok(handle),
|
||||
Ok(Err(error)) => {
|
||||
let _ = handle.join();
|
||||
Err(error)
|
||||
}
|
||||
Err(error) => {
|
||||
let _ = handle.join();
|
||||
Err(format!(
|
||||
"TTS worker exited before reporting readiness: {error}"
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "tts_startup_tests.rs"]
|
||||
mod tests;
|
||||
44
desktop/src-tauri/src/huddle/tts_startup_tests.rs
Normal file
44
desktop/src-tauri/src/huddle/tts_startup_tests.rs
Normal file
@@ -0,0 +1,44 @@
|
||||
use super::*;
|
||||
use std::sync::{
|
||||
atomic::{AtomicBool, Ordering},
|
||||
Arc,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn startup_failure_is_returned_after_worker_exit() {
|
||||
let (tx, rx) = mpsc::sync_channel(1);
|
||||
let exited = Arc::new(AtomicBool::new(false));
|
||||
let exited_worker = Arc::clone(&exited);
|
||||
let handle = std::thread::spawn(move || {
|
||||
tx.send(Err("output unavailable".to_string()))
|
||||
.expect("startup receiver");
|
||||
exited_worker.store(true, Ordering::Release);
|
||||
});
|
||||
|
||||
assert_eq!(
|
||||
await_worker_startup(handle, rx).expect_err("startup must fail"),
|
||||
"output unavailable"
|
||||
);
|
||||
assert!(exited.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worker_exit_before_readiness_is_a_startup_error() {
|
||||
let (tx, rx) = mpsc::sync_channel::<Result<(), String>>(1);
|
||||
let handle = std::thread::spawn(move || drop(tx));
|
||||
|
||||
assert!(await_worker_startup(handle, rx)
|
||||
.expect_err("closed startup channel must fail")
|
||||
.contains("before reporting readiness"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ready_ack_precedes_pipeline_publication_boundary() {
|
||||
let (tx, rx) = mpsc::sync_channel(1);
|
||||
let handle = std::thread::spawn(move || {
|
||||
tx.send(Ok(())).expect("startup receiver");
|
||||
});
|
||||
|
||||
let handle = await_worker_startup(handle, rx).expect("ready worker");
|
||||
handle.join().expect("worker exits");
|
||||
}
|
||||
129
desktop/src-tauri/src/huddle/tts_voice_registry.rs
Normal file
129
desktop/src-tauri/src/huddle/tts_voice_registry.rs
Normal file
@@ -0,0 +1,129 @@
|
||||
//! Built-in Pocket voice identities and immutable asset metadata.
|
||||
//!
|
||||
//! Stable keys identify audio, not display labels. Future imported voices use
|
||||
//! `pocket:imported:<audio-content-sha256>` and may share editable labels.
|
||||
|
||||
pub(super) const MARY_VOICE_KEY: &str = "pocket:mary";
|
||||
pub(super) const VCTK_REVISION: &str = "323332d33f997de8394f24a193e1a76df720e01a";
|
||||
|
||||
pub(super) struct PocketVoiceSpec {
|
||||
pub key: &'static str,
|
||||
pub display_name: &'static str,
|
||||
pub reference_file: &'static str,
|
||||
pub upstream_file: &'static str,
|
||||
pub sha256: &'static str,
|
||||
pub bytes: Option<&'static [u8]>,
|
||||
}
|
||||
|
||||
macro_rules! bundled_voice {
|
||||
($key:literal, $name:literal, $file:literal, $upstream:literal, $hash:literal) => {
|
||||
PocketVoiceSpec {
|
||||
key: $key,
|
||||
display_name: $name,
|
||||
reference_file: concat!($file, ".wav"),
|
||||
upstream_file: concat!("vctk/", $upstream),
|
||||
sha256: $hash,
|
||||
bytes: Some(include_bytes!(concat!(
|
||||
"../../resources/pocket-voices/",
|
||||
$file,
|
||||
".wav"
|
||||
))),
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/// Official English Pocket presets, in the order published by Kyutai.
|
||||
pub(super) static POCKET_VOICES: &[PocketVoiceSpec] = &[
|
||||
bundled_voice!(
|
||||
"pocket:anna",
|
||||
"Anna",
|
||||
"anna",
|
||||
"p228_023_enhanced.wav",
|
||||
"0a6de25cf12bf1540beb85979f306a92be81fecc051c547c5395e7e5237a3856"
|
||||
),
|
||||
bundled_voice!(
|
||||
"pocket:vera",
|
||||
"Vera",
|
||||
"vera",
|
||||
"p229_023_enhanced.wav",
|
||||
"309cf91a895830f15842b398f69a4962cb1f7e0bfab10e25dd27838e826c204b"
|
||||
),
|
||||
bundled_voice!(
|
||||
"pocket:fantine",
|
||||
"Fantine",
|
||||
"fantine",
|
||||
"p244_023_enhanced.wav",
|
||||
"5f07d4e2a3f20a15572aae885156b43ef3fc12ef3812996fd135680d9956448b"
|
||||
),
|
||||
bundled_voice!(
|
||||
"pocket:charles",
|
||||
"Charles",
|
||||
"charles",
|
||||
"p254_023_enhanced.wav",
|
||||
"6b681a429198f16e378d53bccb08d06939da7b00144a7696111d4f8f76be7756"
|
||||
),
|
||||
bundled_voice!(
|
||||
"pocket:paul",
|
||||
"Paul",
|
||||
"paul",
|
||||
"p259_023_enhanced.wav",
|
||||
"7aba504fe0b3b16478b69eb27ce6007e3cb42b0c1915b5f1c6a6024ae37d679b"
|
||||
),
|
||||
bundled_voice!(
|
||||
"pocket:eponine",
|
||||
"Eponine",
|
||||
"eponine",
|
||||
"p262_023_enhanced.wav",
|
||||
"a13c27fb47627b05223691a0ef2974358a18c886e6c2f9d2762ff1d02c20926b"
|
||||
),
|
||||
bundled_voice!(
|
||||
"pocket:azelma",
|
||||
"Azelma",
|
||||
"azelma",
|
||||
"p303_023_enhanced.wav",
|
||||
"60e3d26cdf2efdec5df712152c839928f4d5522821e6554ae11fd96c57ab1026"
|
||||
),
|
||||
bundled_voice!(
|
||||
"pocket:george",
|
||||
"George",
|
||||
"george",
|
||||
"p315_023_enhanced.wav",
|
||||
"29a41f93bf5236e5b21501091d7774c255d5f3d4e62fa4f9fdf0a92a793c84ae"
|
||||
),
|
||||
PocketVoiceSpec {
|
||||
key: MARY_VOICE_KEY,
|
||||
display_name: "Mary",
|
||||
reference_file: "reference_sample.wav",
|
||||
upstream_file: "vctk/p333_023_enhanced.wav",
|
||||
sha256: "a35b0468382218e9f37a9a7494d1e4b74deaf18d7ced22265b4e325bb55c183f",
|
||||
bytes: None,
|
||||
},
|
||||
bundled_voice!(
|
||||
"pocket:jane",
|
||||
"Jane",
|
||||
"jane",
|
||||
"p339_023_enhanced.wav",
|
||||
"2f12e7f155eb3118f55425394f1b049e5b1b67bdc9b3932c8ba4521420aeb84a"
|
||||
),
|
||||
bundled_voice!(
|
||||
"pocket:michael",
|
||||
"Michael",
|
||||
"michael",
|
||||
"p360_023_enhanced.wav",
|
||||
"b6743e9195e5e3fd34fe9d1633ae93f7ffab787b249e45f6467d7d6f7a6ee6ad"
|
||||
),
|
||||
bundled_voice!(
|
||||
"pocket:eve",
|
||||
"Eve",
|
||||
"eve",
|
||||
"p361_023_enhanced.wav",
|
||||
"396e7cbd066b0f3fb6d67fa26e7904076958239d736d4390f15b5fe88feb14cd"
|
||||
),
|
||||
];
|
||||
|
||||
pub(super) fn source_url(voice: &PocketVoiceSpec) -> String {
|
||||
format!(
|
||||
"https://huggingface.co/kyutai/tts-voices/blob/{VCTK_REVISION}/{}",
|
||||
voice.upstream_file
|
||||
)
|
||||
}
|
||||
385
desktop/src-tauri/src/huddle/tts_voice_selection_tests.rs
Normal file
385
desktop/src-tauri/src/huddle/tts_voice_selection_tests.rs
Normal file
@@ -0,0 +1,385 @@
|
||||
use super::*;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
|
||||
fn inert_pipeline(cancel: Arc<AtomicBool>) -> TtsPipeline {
|
||||
let (text_tx, text_rx) = std::sync::mpsc::sync_channel(TEXT_QUEUE_DEPTH);
|
||||
let shutdown = Arc::new(AtomicBool::new(false));
|
||||
let worker_shutdown = Arc::clone(&shutdown);
|
||||
let thread = std::thread::spawn(move || {
|
||||
while !worker_shutdown.load(Ordering::Acquire) {
|
||||
let _ = text_rx.recv_timeout(RECV_TIMEOUT);
|
||||
}
|
||||
});
|
||||
TtsPipeline {
|
||||
text_tx,
|
||||
tts_active: Arc::new(AtomicBool::new(false)),
|
||||
shutdown,
|
||||
cancel,
|
||||
voice_cancel: Arc::new(AtomicBool::new(false)),
|
||||
voice: Arc::new(std::sync::Mutex::new("reference_sample".to_string())),
|
||||
voice_generation: Arc::new(AtomicU64::new(1)),
|
||||
voice_change_ack: Arc::new(std::sync::Mutex::new(None)),
|
||||
thread: Some(thread),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selecting_a_voice_raises_only_the_internal_cancel_and_retains_the_engine_handle() {
|
||||
let cancel = Arc::new(AtomicBool::new(false));
|
||||
let pipeline = inert_pipeline(Arc::clone(&cancel));
|
||||
|
||||
let _acknowledged = pipeline.select_voice("eve");
|
||||
|
||||
assert!(!cancel.load(Ordering::Acquire));
|
||||
assert!(pipeline.voice_cancel.load(Ordering::Acquire));
|
||||
assert_eq!(
|
||||
pipeline
|
||||
.voice
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner())
|
||||
.as_str(),
|
||||
"eve"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reconciling_an_unpublished_pipeline_does_not_cancel_its_first_message() {
|
||||
let cancel = Arc::new(AtomicBool::new(false));
|
||||
let pipeline = inert_pipeline(Arc::clone(&cancel));
|
||||
|
||||
pipeline.select_voice_before_publish("eve");
|
||||
|
||||
assert!(!cancel.load(Ordering::Acquire));
|
||||
assert_eq!(
|
||||
pipeline
|
||||
.voice
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner())
|
||||
.as_str(),
|
||||
"eve"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn received_text_reconciles_a_voice_changed_while_the_worker_was_waiting() {
|
||||
let model_dir = tempfile::tempdir().expect("temp model dir");
|
||||
let bundled_voice =
|
||||
std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("resources/pocket-voices/eve.wav");
|
||||
std::fs::copy(
|
||||
&bundled_voice,
|
||||
model_dir.path().join("reference_sample.wav"),
|
||||
)
|
||||
.expect("Mary test voice");
|
||||
std::fs::copy(&bundled_voice, model_dir.path().join("eve.wav")).expect("Eve test voice");
|
||||
|
||||
let selected_voice = Arc::new(std::sync::Mutex::new("reference_sample".to_string()));
|
||||
let mut style =
|
||||
load_voice_style(&model_dir.path().join("reference_sample.wav")).expect("initial style");
|
||||
let waiting = Arc::new(std::sync::Barrier::new(2));
|
||||
let (text_tx, text_rx) = std::sync::mpsc::channel();
|
||||
let worker_voice = Arc::clone(&selected_voice);
|
||||
let worker_waiting = Arc::clone(&waiting);
|
||||
let worker_model_dir = model_dir.path().to_path_buf();
|
||||
let worker = std::thread::spawn(move || {
|
||||
let mut voice_name = "reference_sample".to_string();
|
||||
worker_waiting.wait();
|
||||
let text = text_rx.recv().expect("first queued text");
|
||||
assert!(reconcile_selected_voice(
|
||||
&worker_model_dir,
|
||||
&worker_voice,
|
||||
&mut voice_name,
|
||||
&mut style,
|
||||
));
|
||||
(text, voice_name)
|
||||
});
|
||||
|
||||
waiting.wait();
|
||||
*selected_voice.lock().expect("selected voice") = "eve".to_string();
|
||||
text_tx
|
||||
.send("first message".to_string())
|
||||
.expect("queue first message");
|
||||
|
||||
assert_eq!(
|
||||
worker.join().expect("worker"),
|
||||
("first message".to_string(), "eve".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corrupt_selected_voice_falls_back_to_mary() {
|
||||
let model_dir = tempfile::tempdir().expect("temp model dir");
|
||||
let bundled_voice =
|
||||
std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("resources/pocket-voices/eve.wav");
|
||||
std::fs::copy(bundled_voice, model_dir.path().join("reference_sample.wav"))
|
||||
.expect("Mary test voice");
|
||||
std::fs::write(model_dir.path().join("eve.wav"), b"not a wave")
|
||||
.expect("corrupt selected voice");
|
||||
|
||||
let selected_voice = std::sync::Mutex::new("eve".to_string());
|
||||
let mut voice_name = "reference_sample".to_string();
|
||||
let mut style =
|
||||
load_voice_style(&model_dir.path().join("reference_sample.wav")).expect("Mary style");
|
||||
|
||||
assert!(reconcile_selected_voice(
|
||||
model_dir.path(),
|
||||
&selected_voice,
|
||||
&mut voice_name,
|
||||
&mut style,
|
||||
));
|
||||
assert_eq!(voice_name, DEFAULT_VOICE);
|
||||
assert_eq!(
|
||||
selected_voice
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner())
|
||||
.as_str(),
|
||||
DEFAULT_VOICE
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_in_hand_post_change_message_survives_cancellation() {
|
||||
let selected_voice = Arc::new(std::sync::Mutex::new("reference_sample".to_string()));
|
||||
let voice_generation = AtomicU64::new(1);
|
||||
let barge_in = AtomicBool::new(false);
|
||||
let voice_cancel = Arc::new(AtomicBool::new(false));
|
||||
let voice_change_ack = Arc::new(std::sync::Mutex::new(None));
|
||||
let (text_tx, text_rx) = std::sync::mpsc::sync_channel(1);
|
||||
let mut acknowledged = begin_voice_change(
|
||||
&selected_voice,
|
||||
&voice_generation,
|
||||
&voice_cancel,
|
||||
&voice_change_ack,
|
||||
"eve",
|
||||
)
|
||||
.expect("voice changed");
|
||||
assert!(voice_cancel.load(Ordering::Acquire));
|
||||
assert!(matches!(
|
||||
acknowledged.try_recv(),
|
||||
Err(tokio::sync::oneshot::error::TryRecvError::Empty)
|
||||
));
|
||||
acknowledge_voice_change(&voice_change_ack, &voice_cancel);
|
||||
assert!(matches!(
|
||||
acknowledged.try_recv(),
|
||||
Err(tokio::sync::oneshot::error::TryRecvError::Empty)
|
||||
));
|
||||
text_tx
|
||||
.send(QueuedText {
|
||||
generation: voice_generation.load(Ordering::Acquire),
|
||||
route_id: 1,
|
||||
text: "new message".to_string(),
|
||||
})
|
||||
.expect("new message");
|
||||
let mut current_text = Some(text_rx.recv().expect("in-hand new message"));
|
||||
|
||||
let shutdown = AtomicBool::new(false);
|
||||
let active = AtomicBool::new(true);
|
||||
let mut deferred_text = VecDeque::from([
|
||||
QueuedText {
|
||||
generation: 1,
|
||||
route_id: 2,
|
||||
text: "old message".to_string(),
|
||||
},
|
||||
QueuedText {
|
||||
generation: voice_generation.load(Ordering::Acquire),
|
||||
route_id: 3,
|
||||
text: "later new message".to_string(),
|
||||
},
|
||||
]);
|
||||
assert!(handle_cancel_or_shutdown(
|
||||
(&barge_in, &voice_cancel),
|
||||
&shutdown,
|
||||
&active,
|
||||
(&text_rx, &mut deferred_text, &mut current_text),
|
||||
&voice_change_ack,
|
||||
None,
|
||||
None,
|
||||
));
|
||||
acknowledge_voice_change(&voice_change_ack, &voice_cancel);
|
||||
acknowledged.blocking_recv().expect("voice change ack");
|
||||
|
||||
assert_eq!(
|
||||
deferred_text
|
||||
.pop_front()
|
||||
.expect("preserved post-change message")
|
||||
.text,
|
||||
"new message"
|
||||
);
|
||||
assert_eq!(
|
||||
deferred_text
|
||||
.pop_front()
|
||||
.expect("later post-change message")
|
||||
.text,
|
||||
"later new message"
|
||||
);
|
||||
assert!(text_rx.try_recv().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn superseding_voice_change_removes_earlier_deferred_messages() {
|
||||
let selected_voice = std::sync::Mutex::new("reference_sample".to_string());
|
||||
let voice_generation = AtomicU64::new(1);
|
||||
let barge_in = AtomicBool::new(false);
|
||||
let voice_cancel = AtomicBool::new(false);
|
||||
let voice_change_ack = Arc::new(std::sync::Mutex::new(None));
|
||||
let (_text_tx, text_rx) = std::sync::mpsc::channel();
|
||||
let shutdown = AtomicBool::new(false);
|
||||
let active = AtomicBool::new(true);
|
||||
let mut deferred_text = VecDeque::new();
|
||||
let mut current_text = None;
|
||||
|
||||
let first = begin_voice_change(
|
||||
&selected_voice,
|
||||
&voice_generation,
|
||||
&voice_cancel,
|
||||
&voice_change_ack,
|
||||
"eve",
|
||||
)
|
||||
.expect("first voice change");
|
||||
deferred_text.push_back(QueuedText {
|
||||
generation: voice_generation.load(Ordering::Acquire),
|
||||
route_id: 4,
|
||||
text: "message for Eve".to_string(),
|
||||
});
|
||||
assert!(handle_cancel_or_shutdown(
|
||||
(&barge_in, &voice_cancel),
|
||||
&shutdown,
|
||||
&active,
|
||||
(&text_rx, &mut deferred_text, &mut current_text),
|
||||
&voice_change_ack,
|
||||
None,
|
||||
None,
|
||||
));
|
||||
acknowledge_voice_change(&voice_change_ack, &voice_cancel);
|
||||
first.blocking_recv().expect("first acknowledgement");
|
||||
|
||||
let _second = begin_voice_change(
|
||||
&selected_voice,
|
||||
&voice_generation,
|
||||
&voice_cancel,
|
||||
&voice_change_ack,
|
||||
"reference_sample",
|
||||
)
|
||||
.expect("second voice change");
|
||||
assert!(handle_cancel_or_shutdown(
|
||||
(&barge_in, &voice_cancel),
|
||||
&shutdown,
|
||||
&active,
|
||||
(&text_rx, &mut deferred_text, &mut current_text),
|
||||
&voice_change_ack,
|
||||
None,
|
||||
None,
|
||||
));
|
||||
|
||||
assert!(deferred_text.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn barge_in_clears_deferred_voice_change_messages() {
|
||||
let barge_in = AtomicBool::new(true);
|
||||
let voice_cancel = AtomicBool::new(false);
|
||||
let shutdown = AtomicBool::new(false);
|
||||
let active = AtomicBool::new(true);
|
||||
let voice_change_ack = Arc::new(std::sync::Mutex::new(None));
|
||||
let (_text_tx, text_rx) = std::sync::mpsc::channel();
|
||||
let mut deferred_text = VecDeque::from([QueuedText {
|
||||
generation: 2,
|
||||
route_id: 5,
|
||||
text: "deferred message".to_string(),
|
||||
}]);
|
||||
let mut current_text = None;
|
||||
|
||||
assert!(handle_cancel_or_shutdown(
|
||||
(&barge_in, &voice_cancel),
|
||||
&shutdown,
|
||||
&active,
|
||||
(&text_rx, &mut deferred_text, &mut current_text),
|
||||
&voice_change_ack,
|
||||
None,
|
||||
None,
|
||||
));
|
||||
|
||||
assert!(deferred_text.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn barge_in_during_a_voice_change_clears_post_change_messages() {
|
||||
let selected_voice = std::sync::Mutex::new("reference_sample".to_string());
|
||||
let voice_generation = AtomicU64::new(1);
|
||||
let barge_in = AtomicBool::new(false);
|
||||
let voice_cancel = AtomicBool::new(false);
|
||||
let voice_change_ack = Arc::new(std::sync::Mutex::new(None));
|
||||
let (_text_tx, text_rx) = std::sync::mpsc::channel();
|
||||
let shutdown = AtomicBool::new(false);
|
||||
let active = AtomicBool::new(true);
|
||||
let mut deferred_text = VecDeque::new();
|
||||
let mut current_text = None;
|
||||
|
||||
let _acknowledged = begin_voice_change(
|
||||
&selected_voice,
|
||||
&voice_generation,
|
||||
&voice_cancel,
|
||||
&voice_change_ack,
|
||||
"eve",
|
||||
)
|
||||
.expect("voice change");
|
||||
deferred_text.push_back(QueuedText {
|
||||
generation: voice_generation.load(Ordering::Acquire),
|
||||
route_id: 6,
|
||||
text: "post-change message".to_string(),
|
||||
});
|
||||
barge_in.store(true, Ordering::Release);
|
||||
|
||||
assert!(handle_cancel_or_shutdown(
|
||||
(&barge_in, &voice_cancel),
|
||||
&shutdown,
|
||||
&active,
|
||||
(&text_rx, &mut deferred_text, &mut current_text),
|
||||
&voice_change_ack,
|
||||
None,
|
||||
None,
|
||||
));
|
||||
assert!(deferred_text.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_sender_captured_before_voice_change_is_stale_even_if_it_sends_after_drain() {
|
||||
let selected_voice = std::sync::Mutex::new("reference_sample".to_string());
|
||||
let voice_generation = Arc::new(AtomicU64::new(1));
|
||||
let barge_in = AtomicBool::new(false);
|
||||
let voice_cancel = AtomicBool::new(false);
|
||||
let voice_change_ack = Arc::new(std::sync::Mutex::new(None));
|
||||
let (text_tx, text_rx) = std::sync::mpsc::sync_channel(1);
|
||||
let old_sender = TtsTextSender {
|
||||
text_tx,
|
||||
generation: voice_generation.load(Ordering::Acquire),
|
||||
};
|
||||
let shutdown = AtomicBool::new(false);
|
||||
let active = AtomicBool::new(true);
|
||||
let mut deferred_text = VecDeque::new();
|
||||
let mut current_text = None;
|
||||
|
||||
let _acknowledged = begin_voice_change(
|
||||
&selected_voice,
|
||||
&voice_generation,
|
||||
&voice_cancel,
|
||||
&voice_change_ack,
|
||||
"eve",
|
||||
)
|
||||
.expect("voice change");
|
||||
assert!(handle_cancel_or_shutdown(
|
||||
(&barge_in, &voice_cancel),
|
||||
&shutdown,
|
||||
&active,
|
||||
(&text_rx, &mut deferred_text, &mut current_text),
|
||||
&voice_change_ack,
|
||||
None,
|
||||
None,
|
||||
));
|
||||
old_sender
|
||||
.send(7, "late old message".to_string())
|
||||
.expect("late send");
|
||||
let late = text_rx.recv().expect("late queued text");
|
||||
|
||||
assert!(late.generation < voice_generation.load(Ordering::Acquire));
|
||||
}
|
||||
197
desktop/src-tauri/src/huddle/tts_voice_transition.rs
Normal file
197
desktop/src-tauri/src/huddle/tts_voice_transition.rs
Normal file
@@ -0,0 +1,197 @@
|
||||
use std::{
|
||||
collections::VecDeque,
|
||||
path::Path,
|
||||
sync::{
|
||||
atomic::{AtomicBool, AtomicU64, Ordering},
|
||||
mpsc::{self, SyncSender},
|
||||
Arc, Mutex,
|
||||
},
|
||||
};
|
||||
|
||||
use crate::huddle::pocket::{load_voice_style, VoiceStyle, DEFAULT_VOICE, VOICE_FILE_EXT};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(super) struct PendingVoiceChange {
|
||||
pub(super) generation: u64,
|
||||
acknowledged: tokio::sync::oneshot::Sender<()>,
|
||||
}
|
||||
|
||||
pub(super) type VoiceChangeAck = Arc<Mutex<Option<PendingVoiceChange>>>;
|
||||
pub(super) type WorkerVoiceState = (Arc<Mutex<String>>, Arc<AtomicU64>, VoiceChangeAck);
|
||||
pub(super) type WorkerCancelSignals = (Arc<AtomicBool>, Arc<AtomicBool>);
|
||||
pub(super) type CancelTextState<'a> = (
|
||||
&'a mpsc::Receiver<QueuedText>,
|
||||
&'a mut VecDeque<QueuedText>,
|
||||
&'a mut Option<QueuedText>,
|
||||
);
|
||||
pub(super) type CancelSignals<'a> = (&'a AtomicBool, &'a AtomicBool);
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(super) struct QueuedText {
|
||||
pub(super) generation: u64,
|
||||
pub(super) route_id: u64,
|
||||
pub(super) text: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct TtsTextSender {
|
||||
pub(super) text_tx: SyncSender<QueuedText>,
|
||||
pub(super) generation: u64,
|
||||
}
|
||||
|
||||
impl TtsTextSender {
|
||||
pub(crate) fn send(&self, route_id: u64, text: String) -> Result<(), String> {
|
||||
self.text_tx
|
||||
.send(QueuedText {
|
||||
generation: self.generation,
|
||||
route_id,
|
||||
text,
|
||||
})
|
||||
.map_err(|error| error.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn begin_voice_change(
|
||||
selected_voice: &Mutex<String>,
|
||||
voice_generation: &AtomicU64,
|
||||
voice_cancel: &AtomicBool,
|
||||
voice_change_ack: &VoiceChangeAck,
|
||||
voice: &str,
|
||||
) -> Option<tokio::sync::oneshot::Receiver<()>> {
|
||||
let mut pending_ack = voice_change_ack
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner());
|
||||
let mut selected = selected_voice
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner());
|
||||
if selected.as_str() == voice {
|
||||
return None;
|
||||
}
|
||||
|
||||
let (sender, receiver) = tokio::sync::oneshot::channel();
|
||||
voice_cancel.store(true, Ordering::Release);
|
||||
let generation = voice_generation.fetch_add(1, Ordering::AcqRel) + 1;
|
||||
if let Some(superseded) = pending_ack.replace(PendingVoiceChange {
|
||||
generation,
|
||||
acknowledged: sender,
|
||||
}) {
|
||||
let _ = superseded.acknowledged.send(());
|
||||
}
|
||||
*selected = voice.to_string();
|
||||
Some(receiver)
|
||||
}
|
||||
|
||||
pub(super) fn acknowledge_voice_change(
|
||||
voice_change_ack: &VoiceChangeAck,
|
||||
voice_cancel: &AtomicBool,
|
||||
) {
|
||||
let mut pending_ack = voice_change_ack
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner());
|
||||
if voice_cancel.load(Ordering::Acquire) {
|
||||
return;
|
||||
}
|
||||
if let Some(pending) = pending_ack.take() {
|
||||
let _ = pending.acknowledged.send(());
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn finish_voice_change_ack(voice_change_ack: &VoiceChangeAck) {
|
||||
if let Some(pending) = voice_change_ack
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner())
|
||||
.take()
|
||||
{
|
||||
let _ = pending.acknowledged.send(());
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn reconcile_selected_voice(
|
||||
model_dir: &Path,
|
||||
selected_voice: &Mutex<String>,
|
||||
voice_name: &mut String,
|
||||
style: &mut VoiceStyle,
|
||||
) -> bool {
|
||||
let requested_voice = selected_voice
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner())
|
||||
.clone();
|
||||
if requested_voice == *voice_name {
|
||||
return true;
|
||||
}
|
||||
|
||||
let requested_path = model_dir.join(format!("{requested_voice}.{VOICE_FILE_EXT}"));
|
||||
match load_voice_style(&requested_path) {
|
||||
Ok(requested_style) => {
|
||||
*style = requested_style;
|
||||
*voice_name = requested_voice;
|
||||
true
|
||||
}
|
||||
Err(_) => {
|
||||
eprintln!("buzz-desktop: tts stage=voice_switch status=fallback reason=voice_style");
|
||||
let fallback_path = model_dir.join(format!("{DEFAULT_VOICE}.{VOICE_FILE_EXT}"));
|
||||
match load_voice_style(&fallback_path) {
|
||||
Ok(fallback_style) => {
|
||||
*style = fallback_style;
|
||||
*voice_name = DEFAULT_VOICE.to_string();
|
||||
*selected_voice
|
||||
.lock()
|
||||
.unwrap_or_else(|lock_error| lock_error.into_inner()) =
|
||||
DEFAULT_VOICE.to_string();
|
||||
true
|
||||
}
|
||||
Err(_) => {
|
||||
eprintln!(
|
||||
"buzz-desktop: tts stage=voice_switch status=failed reason=fallback_voice_style"
|
||||
);
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn retain_cancelled_text(
|
||||
deferred_text: &mut VecDeque<QueuedText>,
|
||||
current_text: &mut Option<QueuedText>,
|
||||
text_rx: &mpsc::Receiver<QueuedText>,
|
||||
preserve_generation: Option<u64>,
|
||||
) {
|
||||
if let Some(generation) = preserve_generation {
|
||||
deferred_text.retain(|text| {
|
||||
let preserve = text.generation >= generation;
|
||||
if !preserve {
|
||||
log_cancelled_route(text.route_id, "voice_switch");
|
||||
}
|
||||
preserve
|
||||
});
|
||||
if let Some(text) = current_text.take() {
|
||||
if text.generation >= generation {
|
||||
deferred_text.push_front(text);
|
||||
} else {
|
||||
log_cancelled_route(text.route_id, "voice_switch");
|
||||
}
|
||||
}
|
||||
while let Ok(text) = text_rx.try_recv() {
|
||||
if text.generation >= generation {
|
||||
deferred_text.push_back(text);
|
||||
} else {
|
||||
log_cancelled_route(text.route_id, "voice_switch");
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for text in deferred_text.drain(..) {
|
||||
log_cancelled_route(text.route_id, "barge_in");
|
||||
}
|
||||
if let Some(text) = current_text.take() {
|
||||
log_cancelled_route(text.route_id, "barge_in");
|
||||
}
|
||||
while let Ok(text) = text_rx.try_recv() {
|
||||
log_cancelled_route(text.route_id, "barge_in");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn log_cancelled_route(route_id: u64, reason: &str) {
|
||||
eprintln!("buzz-desktop: tts stage=queue status=dropped reason={reason} route_id={route_id}");
|
||||
}
|
||||
@@ -467,6 +467,18 @@ pub fn run() {
|
||||
*guard = Some(app_handle.clone());
|
||||
}
|
||||
|
||||
let (tts_settings, tts_settings_load_error) =
|
||||
huddle::tts_settings::load_for_app(&app_handle);
|
||||
if let Ok(mut guard) = state.huddle_audio.tts.lock() {
|
||||
*guard = tts_settings.clone();
|
||||
}
|
||||
if let Ok(mut guard) = state.huddle_audio.tts_load_error.lock() {
|
||||
*guard = tts_settings_load_error;
|
||||
}
|
||||
if let Ok(mut huddle) = state.huddle_state.lock() {
|
||||
huddle.tts_enabled = tts_settings.agent_text_to_speech;
|
||||
}
|
||||
|
||||
// Bring up the runtime-owned shared-compute coordinator before
|
||||
// saved agents are restored. Its lifetime is tied to the app, not
|
||||
// a UI mount; it publishes discovery and reconciles membership for
|
||||
@@ -877,6 +889,10 @@ pub fn run() {
|
||||
download_voice_models,
|
||||
get_model_status,
|
||||
set_tts_enabled,
|
||||
huddle::tts_settings::get_tts_settings,
|
||||
huddle::tts_settings::list_voice_registry,
|
||||
huddle::tts_settings::set_pocket_voice,
|
||||
huddle::tts_settings::preview_pocket_voice,
|
||||
speak_agent_message,
|
||||
add_agent_to_huddle,
|
||||
check_pipeline_hotstart,
|
||||
|
||||
Reference in New Issue
Block a user