From 0367f751dd00e404f3831293ed798411656ad2b0 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Thu, 30 Jul 2026 11:23:23 -0400 Subject: [PATCH] feat(desktop): add voice dictation to composer Co-authored-by: Kenny Lopez Signed-off-by: John Tennant --- desktop/src/app/AppShell.tsx | 2 + .../dictation/hooks/useComposerDictation.ts | 191 ++++++ .../features/dictation/hooks/useDictation.ts | 90 +++ .../dictation/hooks/useLocalDictation.ts | 619 ++++++++++++++++++ .../hooks/useMessageComposerDictation.tsx | 49 ++ .../hooks/useVoiceDictationShortcut.ts | 53 ++ desktop/src/features/dictation/index.ts | 8 + .../features/dictation/lib/activeComposer.ts | 23 + .../dictation/lib/voiceInput.test.mjs | 150 +++++ .../src/features/dictation/lib/voiceInput.ts | 114 ++++ .../features/dictation/ui/DictationButton.tsx | 73 +++ .../features/messages/ui/MessageComposer.tsx | 100 +-- .../messages/ui/MessageComposerOverlays.tsx | 72 ++ .../messages/ui/useComposerScrollToBottom.ts | 13 + .../settings/ui/KeyboardShortcutsCard.tsx | 39 +- desktop/src/shared/lib/keyboard-shortcuts.ts | 8 + desktop/tests/e2e/messaging.spec.ts | 67 +- preview-features.json | 6 + 18 files changed, 1610 insertions(+), 67 deletions(-) create mode 100644 desktop/src/features/dictation/hooks/useComposerDictation.ts create mode 100644 desktop/src/features/dictation/hooks/useDictation.ts create mode 100644 desktop/src/features/dictation/hooks/useLocalDictation.ts create mode 100644 desktop/src/features/dictation/hooks/useMessageComposerDictation.tsx create mode 100644 desktop/src/features/dictation/hooks/useVoiceDictationShortcut.ts create mode 100644 desktop/src/features/dictation/index.ts create mode 100644 desktop/src/features/dictation/lib/activeComposer.ts create mode 100644 desktop/src/features/dictation/lib/voiceInput.test.mjs create mode 100644 desktop/src/features/dictation/lib/voiceInput.ts create mode 100644 desktop/src/features/dictation/ui/DictationButton.tsx create mode 100644 desktop/src/features/messages/ui/MessageComposerOverlays.tsx create mode 100644 desktop/src/features/messages/ui/useComposerScrollToBottom.ts diff --git a/desktop/src/app/AppShell.tsx b/desktop/src/app/AppShell.tsx index 4eb0a42bbe..fac09191bd 100644 --- a/desktop/src/app/AppShell.tsx +++ b/desktop/src/app/AppShell.tsx @@ -86,6 +86,7 @@ import { useIdentityQuery } from "@/shared/api/hooks"; import { useRelayAutoHeal } from "@/shared/api/useRelayAutoHeal"; import { useDeferredStartup } from "@/shared/hooks/useDeferredStartup"; import { useWebviewScrollBoundaryLock } from "@/shared/hooks/useWebviewScrollBoundaryLock"; +import { useVoiceDictationShortcut } from "@/features/dictation"; import { joinChannel } from "@/shared/api/tauri"; import type { ChannelVisibility, SearchHit } from "@/shared/api/types"; import { ChannelNavigationProvider } from "@/shared/context/ChannelNavigationContext"; @@ -153,6 +154,7 @@ export function AppShell() { }); // Settings lives in history so back returns to the previous app entry. const settingsOpen = location.pathname === "/settings"; + useVoiceDictationShortcut(settingsOpen); const locationSearchSection = (location.search as { section?: unknown }) .section; const settingsSection: SettingsSection = isSettingsSection( diff --git a/desktop/src/features/dictation/hooks/useComposerDictation.ts b/desktop/src/features/dictation/hooks/useComposerDictation.ts new file mode 100644 index 0000000000..d11b2b3042 --- /dev/null +++ b/desktop/src/features/dictation/hooks/useComposerDictation.ts @@ -0,0 +1,191 @@ +import type * as React from "react"; +import { useEffect, useId, useRef } from "react"; +import { + clearActiveDictationComposer, + isActiveDictationComposer, + setActiveDictationComposer, +} from "../lib/activeComposer"; +import { useDictation } from "./useDictation"; + +interface UseComposerDictationOptions { + /** Whether the voice-dictation preview feature is enabled. */ + enabled?: boolean; + /** Ref to a function that syncs contentRef from the Tiptap editor and returns it. */ + syncContentRef: React.MutableRefObject<() => string>; + /** Whether the composer is currently disabled (read-only, etc.). */ + disabled?: boolean; + disabledRef: React.MutableRefObject; + isSendingRef: React.MutableRefObject; + isUploadingRef: React.MutableRefObject; + /** Updates contentRef + isContentEmpty state. */ + setComposerContent: (text: string) => void; + /** Ref to a function that updates the Tiptap editor document. */ + setEditorContentRef: React.MutableRefObject<(text: string) => void>; + submitMessageRef: React.MutableRefObject<() => void>; + /** When this key changes (channel/thread switch), active dictation is stopped. */ + draftKey?: string | null; + /** Ref to the composer's container element for focus tracking. */ + composerRef?: React.RefObject; +} + +/** + * Thin wrapper around `useDictation` pre-wired for the MessageComposer's + * state management (syncContentRef, setComposerContent, editor, submitMessageRef). + * + * Uses the local Parakeet STT engine — fully offline, no relay or API key needed. + */ +export function useComposerDictation({ + enabled = true, + syncContentRef, + disabled = false, + disabledRef, + isSendingRef, + isUploadingRef, + setComposerContent, + setEditorContentRef, + submitMessageRef, + draftKey, + composerRef, +}: UseComposerDictationOptions) { + const instanceId = useId(); + const isSendBlockedRef = useRef(false); + isSendBlockedRef.current = + !enabled || + disabledRef.current || + isSendingRef.current || + isUploadingRef.current; + + const dictation = useDictation({ + disabled: !enabled, + getText: () => syncContentRef.current(), + setText: (text) => { + setComposerContent(text); + setEditorContentRef.current(text); + }, + onSend: (text) => { + setComposerContent(text); + setEditorContentRef.current(text); + // Submit synchronously — the content ref is already set above, so + // syncComposerContentFromEditor() will serialize the editor which now + // holds the dictated text. + submitMessageRef.current(); + }, + isSendBlockedRef, + }); + + // Track which composer is active (most recently focused) so that the global + // ⌘D shortcut only dispatches to one instance when multiple are mounted. + useEffect(() => { + const el = composerRef?.current; + if (!el) { + // If no ref provided, this composer is always considered active (single-composer case). + setActiveDictationComposer(instanceId); + return () => clearActiveDictationComposer(instanceId); + } + + function handleFocusIn() { + setActiveDictationComposer(instanceId); + } + + el.addEventListener("focusin", handleFocusIn); + return () => { + el.removeEventListener("focusin", handleFocusIn); + clearActiveDictationComposer(instanceId); + }; + }, [instanceId, composerRef]); + + // Cancel dictation when the channel/thread changes so that transcript events + // from a stale local STT session don't leak into the wrong draft. + // Only cancel if this instance owns a live/pending session — avoids killing + // another composer's session since the native engine is a singleton. + // + // Includes `isTranscribing`, not just `isRecording`/`isStarting`: + // `stopRecording()` clears `isRecording` immediately but deliberately keeps + // the transcript listener alive (isTranscribing=true) until the native + // `stopped` event, so the final local STT flush can still be appended. If the + // draftKey changes during that grace window, we must still cancel — otherwise + // the late transcript is appended through this same composer instance into the + // newly restored draft. `cancelRecording()` unlistens the transcript handler + // and performs a session-scoped native stop, so this is safe. + const isOwningSessionRef = useRef(false); + isOwningSessionRef.current = + dictation.isRecording || dictation.isStarting || dictation.isTranscribing; + // biome-ignore lint/correctness/useExhaustiveDependencies: draftKey is the sole trigger + useEffect(() => { + if (isOwningSessionRef.current) { + dictation.cancelRecording(); + } + }, [draftKey]); + + // Auto-cancel dictation when the composer becomes disabled mid-session + // (e.g. channel becomes read-only, parent send state disables thread composer). + // Without this, the STT session keeps running with no way to stop it. + // + // Covers the full ownership window — `isStarting` and `isTranscribing`, not + // just `isRecording`. If the composer is disabled while `startRecording()` is + // still resolving (mic permission / AudioWorklet setup), the pending start + // would otherwise finish and begin microphone capture in a disabled composer. + // `cancelRecording()` aborts the in-flight start as well as a live recording, + // and re-running when any of these flags flip catches a start that completes + // after the disable. + useEffect(() => { + const owningSession = + dictation.isRecording || dictation.isStarting || dictation.isTranscribing; + if ((!enabled || disabled) && owningSession) { + dictation.cancelRecording(); + } + }, [ + enabled, + disabled, + dictation.isRecording, + dictation.isStarting, + dictation.isTranscribing, + dictation.cancelRecording, + ]); + + // ⌘D push-to-talk — hold to record, release to stop. + // Dispatched from AppShell's keydown/keyup handlers. + // Only the active (most recently focused) composer responds, and only while + // focus remains inside it and it is not disabled/send-blocked. + // biome-ignore lint/correctness/useExhaustiveDependencies: disabledRef/isSendBlockedRef are stable refs read at call time + useEffect(() => { + function handleKeyDown() { + if (!enabled) return; + // Only respond if this is the active composer instance. + if (!isActiveDictationComposer(instanceId)) return; + // Only respond if focus is still inside this composer. The active-composer + // registration persists after focusout (until unmount), so without this + // check, focusing a composer and then moving to another mounted input or + // dialog (quick search, create-channel, etc.) would still let ⌘D start + // microphone capture and append transcript into the background draft. + const el = composerRef?.current; + if (el && !el.contains(document.activeElement)) return; + // Don't start dictation in disabled/blocked composers. + if (disabledRef.current || isSendBlockedRef.current) return; + if (!dictation.isRecording && !dictation.isStarting) { + dictation.startRecording(); + } + } + function handleKeyUp() { + if (dictation.isRecording || dictation.isStarting) { + dictation.stopRecording(); + } + } + window.addEventListener("buzz:dictation-key-down", handleKeyDown); + window.addEventListener("buzz:dictation-key-up", handleKeyUp); + return () => { + window.removeEventListener("buzz:dictation-key-down", handleKeyDown); + window.removeEventListener("buzz:dictation-key-up", handleKeyUp); + }; + }, [ + instanceId, + composerRef, + enabled, + dictation.isRecording, + dictation.isStarting, + dictation.startRecording, + dictation.stopRecording, + ]); + + return dictation; +} diff --git a/desktop/src/features/dictation/hooks/useDictation.ts b/desktop/src/features/dictation/hooks/useDictation.ts new file mode 100644 index 0000000000..c11fc4b3be --- /dev/null +++ b/desktop/src/features/dictation/hooks/useDictation.ts @@ -0,0 +1,90 @@ +import type * as React from "react"; +import { useCallback, useMemo, useRef } from "react"; +import { + DEFAULT_AUTO_SUBMIT_PHRASE, + getAutoSubmitMatch, + parseAutoSubmitPhrases, + replaceTrailingTranscribedText, +} from "../lib/voiceInput"; +import { useLocalDictation } from "./useLocalDictation"; + +interface UseDictationOptions { + /** Disable native availability checks and recording entry points. */ + disabled?: boolean; + /** Returns the current composer text (must be fresh — synced from editor). */ + getText: () => string; + /** Set composer text */ + setText: (value: string) => void; + /** Send the message */ + onSend: (text: string) => void; + /** Ref that is `true` when sending is blocked (uploading, preparing mention, etc.) */ + isSendBlockedRef?: React.MutableRefObject; +} + +export function useDictation({ + disabled = false, + getText, + setText, + onSend, + isSendBlockedRef, +}: UseDictationOptions) { + const autoSubmitPhrases = useMemo( + () => parseAutoSubmitPhrases(DEFAULT_AUTO_SUBMIT_PHRASE), + [], + ); + const stopRecordingRef = useRef<() => void>(() => {}); + const lastTranscriptRef = useRef(""); + + const handleTranscript = useCallback( + (transcript: string) => { + const previous = lastTranscriptRef.current; + const latest = getText(); + const merged = replaceTrailingTranscribedText( + latest, + previous, + transcript, + ); + const match = getAutoSubmitMatch(transcript, autoSubmitPhrases); + + if (!match) { + setText(merged); + // Reset to empty — each streaming partial is an independent segment + // (the native engine flushes and clears its buffer). The next transcript + // should be appended, not replace this one. + lastTranscriptRef.current = ""; + return; + } + + const textWithoutPhrase = replaceTrailingTranscribedText( + latest, + previous, + match.textWithoutPhrase, + ); + if (!textWithoutPhrase.trim()) return; + + stopRecordingRef.current(); + + if (isSendBlockedRef?.current) { + setText(textWithoutPhrase); + return; + } + + setText(textWithoutPhrase.trim()); + onSend(textWithoutPhrase.trim()); + lastTranscriptRef.current = ""; + }, + [autoSubmitPhrases, getText, onSend, isSendBlockedRef, setText], + ); + + const dictation = useLocalDictation({ + disabled, + onRecordingStart: () => { + lastTranscriptRef.current = ""; + }, + onTranscriptText: handleTranscript, + }); + + stopRecordingRef.current = dictation.stopRecording; + + return dictation; +} diff --git a/desktop/src/features/dictation/hooks/useLocalDictation.ts b/desktop/src/features/dictation/hooks/useLocalDictation.ts new file mode 100644 index 0000000000..2a4e3ba441 --- /dev/null +++ b/desktop/src/features/dictation/hooks/useLocalDictation.ts @@ -0,0 +1,619 @@ +import { invoke } from "@tauri-apps/api/core"; +import { listen, type UnlistenFn } from "@tauri-apps/api/event"; +import { useCallback, useEffect, useRef, useState } from "react"; +import { toast } from "sonner"; + +/** + * Raw binary invoke — uses Tauri's internal IPC for zero-copy ArrayBuffer transfer. + * Same pattern as huddle's audioWorklet.ts. + */ +function invokeRawBinary(cmd: string, payload: Uint8Array): Promise { + // biome-ignore lint/suspicious/noExplicitAny: Tauri internals have no public type definition + const internals = (window as any).__TAURI_INTERNALS__; + if (!internals?.invoke) { + return Promise.reject(new Error("Tauri internals not available")); + } + return internals.invoke(cmd, payload); +} + +interface UseLocalDictationOptions { + disabled?: boolean; + onRecordingStart?: () => void; + onTranscriptText: (text: string) => void; +} + +interface DictationStatus { + available: boolean; + active: boolean; +} + +const DICTATION_TRANSCRIPT_EVENT = "dictation-transcript"; +const DICTATION_STATE_EVENT = "dictation-state"; + +/** Interval (ms) to poll model availability after initial unavailability. */ +const MODEL_POLL_INTERVAL_MS = 5_000; + +/** + * Batching interval for audio IPC (ms). The worklet posts every render quantum + * (~2.67ms at 48kHz/128 samples). Sending each one individually overloads IPC. + * We accumulate ~100ms of audio before sending to reduce IPC overhead from + * ~375 calls/s to ~10 calls/s. + */ +const AUDIO_BATCH_MS = 100; + +/** + * Max samples per `push_dictation_audio` IPC call. The native command rejects + * any raw audio payload over 100 KB (`MAX_AUDIO_BATCH_BYTES` in `dictation.rs`); + * at 48 kHz f32 mono that is 25,600 samples (~0.53s). We chunk under that cap + * (24,000 samples / 96 KB, leaving headroom) so a stalled main thread that + * lets the batch grow past ~0.5s can't produce a single oversized buffer that + * native rejects and we silently drop. Chunks are sent in order. + */ +const MAX_IPC_SAMPLES = 24_000; + +/** + * Size (bytes) of the little-endian `u64` session header prepended to each + * `push_dictation_audio` payload. Native reads this header and only feeds audio + * whose session matches the currently-active one, so late chunks from a + * just-stopped session can't leak into a newer session's transcript. + */ +const SESSION_HEADER_BYTES = 8; + +/** + * Local STT dictation hook using the Parakeet model via Tauri native commands. + * + * Works fully offline — no relay or OpenAI API key needed. Uses the same + * sherpa-onnx Parakeet TDT-CTC 110M model as huddle transcription. + * + * Audio capture uses the Web Audio API (AudioWorklet) on the frontend side, + * then sends batched raw PCM bytes to the native STT engine via + * `push_dictation_audio`. + */ +export function useLocalDictation({ + disabled = false, + onRecordingStart, + onTranscriptText, +}: UseLocalDictationOptions) { + const [isRecording, setIsRecording] = useState(false); + const [isStarting, setIsStarting] = useState(false); + const [isTranscribing, setIsTranscribing] = useState(false); + const [isAvailable, setIsAvailable] = useState(false); + + const streamRef = useRef(null); + const audioContextRef = useRef(null); + const workletRef = useRef(null); + const outputGainRef = useRef(null); + const batchTimerRef = useRef | null>(null); + const audioBatchRef = useRef([]); + // Tail of the serialized flush chain. Each flush appends its IPC work to + // this promise so flushes run strictly one-after-another; awaiting the tail + // guarantees every already-started flush's chunks are enqueued native-side. + const flushChainRef = useRef>(Promise.resolve()); + const unlistenTranscriptRef = useRef(null); + const unlistenStateRef = useRef(null); + const onRecordingStartRef = useRef(onRecordingStart); + const onTranscriptTextRef = useRef(onTranscriptText); + // Native session ID — set after `start_dictation` returns. Transcript and + // state events include this ID so we can definitively ignore stale events + // from a previous session's forwarder. + const nativeSessionRef = useRef(0); + // Abort flag — set when stop/cancel is called while startRecording is still + // awaiting async setup. The start resumes and bails before activating. + const startAbortedRef = useRef(false); + + onRecordingStartRef.current = onRecordingStart; + onTranscriptTextRef.current = onTranscriptText; + + const isEnabled = !disabled && isAvailable; + + // Check availability on mount and poll until available (model may be downloading). + useEffect(() => { + if (disabled) { + setIsAvailable(false); + return; + } + + let cancelled = false; + let pollTimer: ReturnType | null = null; + + function checkAvailability() { + invoke("get_dictation_status") + .then((status) => { + if (cancelled) return; + setIsAvailable(status.available); + // Stop polling once the model is ready. + if (status.available && pollTimer) { + clearInterval(pollTimer); + pollTimer = null; + } + }) + .catch(() => { + if (!cancelled) setIsAvailable(false); + }); + } + + checkAvailability(); + + // Poll periodically until available (handles background model download). + pollTimer = setInterval(() => { + if (cancelled) return; + checkAvailability(); + }, MODEL_POLL_INTERVAL_MS); + + return () => { + cancelled = true; + if (pollTimer) clearInterval(pollTimer); + }; + }, [disabled]); + + /** Flush accumulated audio batch to the native STT engine. Returns a promise + * that resolves once the IPC call completes (or immediately if nothing to flush). + * + * Flushes are serialized through `flushChainRef`: the batch is drained + * synchronously here (so each flush owns a distinct set of samples in FIFO + * order), but the actual IPC send is chained after any prior in-flight + * flush. This keeps chunks strictly ordered and lets `stopRecording`/ + * `cleanup` await the chain tail so an earlier timer-tick flush can't still + * be enqueuing chunks when `stop_dictation` fires. */ + const flushAudioBatch = useCallback((): Promise => { + const batch = audioBatchRef.current; + if (batch.length === 0) return flushChainRef.current; + + // Tag this flush with the session that owns the buffered audio. Captured + // once here so every chunk of this batch carries the same session; native + // drops chunks whose session no longer matches the active one, so a late + // flush from a just-stopped session can't leak into a newer session. + const session = nativeSessionRef.current; + + // Calculate total byte length and merge into a single buffer. Drain the + // batch synchronously so a concurrent timer tick can't grab the same + // samples — the merged buffer captured here is this flush's exclusive + // payload. + let totalSamples = 0; + for (const chunk of batch) { + totalSamples += chunk.length; + } + const merged = new Float32Array(totalSamples); + let offset = 0; + for (const chunk of batch) { + merged.set(chunk, offset); + offset += chunk.length; + } + audioBatchRef.current = []; + + // Split into chunks under the native IPC cap and send them in order. + // A single unbounded buffer can exceed the 100 KB native limit if the + // batch timer was delayed (e.g. main-thread stall), in which case native + // rejects it and the whole chunk is silently lost. + const sendChunks = async (): Promise => { + for (let start = 0; start < merged.length; start += MAX_IPC_SAMPLES) { + const slice = merged.subarray( + start, + Math.min(start + MAX_IPC_SAMPLES, merged.length), + ); + // Build the IPC payload: an 8-byte LE u64 session header followed by + // the audio bytes. Copy the slice into the header-prefixed buffer so + // the raw payload is exactly this chunk (a subarray view shares the + // parent's ArrayBuffer). + const payload = new Uint8Array(SESSION_HEADER_BYTES + slice.byteLength); + new DataView(payload.buffer).setBigUint64( + 0, + BigInt(session), + true, // little-endian + ); + payload.set( + new Uint8Array( + slice.buffer.slice( + slice.byteOffset, + slice.byteOffset + slice.byteLength, + ), + ), + SESSION_HEADER_BYTES, + ); + await invokeRawBinary("push_dictation_audio", payload).catch(() => {}); + } + }; + + // Chain this flush's IPC work after any prior in-flight flush so chunks + // stay strictly ordered and awaiting the tail drains everything. + const chained = flushChainRef.current.then(sendChunks); + flushChainRef.current = chained; + return chained; + }, []); + + const cleanup = useCallback(() => { + // Abort any in-flight startRecording so it bails after its next await + // instead of resuming and opening the mic/worklet or leaving the native + // session/listeners running after teardown. `cleanup` is the unmount + // handler (and the catch-path teardown); without this, unmounting while + // startRecording awaits `start_dictation`/`listen`/`getUserMedia`/ + // `addModule` would let the async start finish against a torn-down + // instance. A fresh startRecording clears this flag before its first + // await, so it never wrongly aborts a subsequent start. + startAbortedRef.current = true; + // Flush any remaining audio before teardown, then stop the native engine. + // Scope the stop to THIS hook instance's session. `cleanup` runs as every + // instance's unmount handler, so an unscoped stop here would let a + // non-recording composer (e.g. a thread reply composer closing) tear down + // the singleton engine owned by another composer that is actively + // recording. `nativeSessionRef.current` is 0 for an instance that never + // started a session (native IDs start at 1), so its stop can never match + // the live session and correctly no-ops. + const stoppingSession = nativeSessionRef.current; + void flushAudioBatch().then(() => { + invoke("stop_dictation", { session: stoppingSession }).catch(() => {}); + }); + // Stop batch timer. + if (batchTimerRef.current) { + clearInterval(batchTimerRef.current); + batchTimerRef.current = null; + } + // Stop mic. + if (streamRef.current) { + for (const track of streamRef.current.getTracks()) { + track.stop(); + } + streamRef.current = null; + } + // Disconnect audio worklet. Clear the port handler first so any PCM + // messages still queued on the main thread are dropped instead of + // appended to the (reused) audio batch. + if (workletRef.current) { + workletRef.current.port.onmessage = null; + workletRef.current.disconnect(); + workletRef.current = null; + } + if (outputGainRef.current) { + outputGainRef.current.disconnect(); + outputGainRef.current = null; + } + // Drop any audio still buffered so it can't leak into the next session. + audioBatchRef.current = []; + // Close audio context. + if (audioContextRef.current) { + void audioContextRef.current.close(); + audioContextRef.current = null; + } + // Unlisten events. + if (unlistenTranscriptRef.current) { + unlistenTranscriptRef.current(); + unlistenTranscriptRef.current = null; + } + if (unlistenStateRef.current) { + unlistenStateRef.current(); + unlistenStateRef.current = null; + } + }, [flushAudioBatch]); + + // Cleanup on unmount. + useEffect(() => cleanup, [cleanup]); + + const startRecording = useCallback(async () => { + // Also guard on `isTranscribing`: after `stopRecording()` the previous + // session has cleared `isRecording` but is still awaiting its native + // `stopped` event, which delivers the final transcript before its + // listeners are unregistered. Starting a new session in that window would + // unlisten the old session's handlers (below) before its last words + // arrived, dropping them. Wait for the prior session to fully stop. + if (!isEnabled || isStarting || isRecording || isTranscribing) return; + + // Clear abort flag for this new start attempt. + startAbortedRef.current = false; + // Reset any leftover audio buffer so a stale batch from a prior session + // can't be flushed into this new session/draft. + audioBatchRef.current = []; + // Reset the flush chain so this session's flushes don't chain behind a + // prior session's (already-settled) tail. + flushChainRef.current = Promise.resolve(); + + setIsStarting(true); + onRecordingStartRef.current?.(); + + try { + // 1. Start the native STT engine — returns the session ID used to tag events. + const sessionId = await invoke("start_dictation"); + nativeSessionRef.current = sessionId; + + // Bail if aborted during engine start. + if (startAbortedRef.current) { + invoke("stop_dictation", { session: sessionId }).catch(() => {}); + return; + } + + // Unregister any lingering listeners from a previous session so they + // can't match the new session ID via the shared ref. + if (unlistenTranscriptRef.current) { + unlistenTranscriptRef.current(); + unlistenTranscriptRef.current = null; + } + if (unlistenStateRef.current) { + unlistenStateRef.current(); + unlistenStateRef.current = null; + } + + // 2. Listen for transcript events from the native layer. + // Each listener captures `sessionId` by value (closure) so it only + // matches events from this specific session — immune to ref mutation. + const unlistenTranscript = await listen<{ + text: string; + session: number; + }>(DICTATION_TRANSCRIPT_EVENT, (event) => { + const { text, session } = event.payload; + if (session !== sessionId) return; + if (text) { + onTranscriptTextRef.current(text); + } + }); + // Bail if stop/cancel was called while we were awaiting. + if (startAbortedRef.current) { + unlistenTranscript(); + invoke("stop_dictation", { session: sessionId }).catch(() => {}); + return; + } + unlistenTranscriptRef.current = unlistenTranscript; + + const unlistenState = await listen<{ + state: string; + session: number; + }>(DICTATION_STATE_EVENT, (event) => { + const { state, session } = event.payload; + if (session !== sessionId) return; + if (state === "stopped") { + setIsRecording(false); + setIsTranscribing(false); + // Tear down this instance's local capture pipeline. The native + // engine is a singleton: when another mounted composer calls + // `start_dictation`, it stops this session's engine, so we can + // receive `stopped` without our own `stopRecording()`/`cleanup()` + // having run. Without tearing down here, `streamRef`/`workletRef`/ + // `batchTimerRef` stay alive and this composer keeps the mic open, + // pushing stale audio, until it unmounts. Don't re-invoke + // `stop_dictation` — the native side already stopped (that's why + // this event fired). + if (batchTimerRef.current) { + clearInterval(batchTimerRef.current); + batchTimerRef.current = null; + } + if (streamRef.current) { + for (const track of streamRef.current.getTracks()) { + track.stop(); + } + streamRef.current = null; + } + if (workletRef.current) { + workletRef.current.port.onmessage = null; + workletRef.current.disconnect(); + workletRef.current = null; + } + if (outputGainRef.current) { + outputGainRef.current.disconnect(); + outputGainRef.current = null; + } + audioBatchRef.current = []; + if (audioContextRef.current) { + void audioContextRef.current.close(); + audioContextRef.current = null; + } + // Clean up event listeners now that the session is fully done. + if (unlistenTranscriptRef.current) { + unlistenTranscriptRef.current(); + unlistenTranscriptRef.current = null; + } + if (unlistenStateRef.current) { + unlistenStateRef.current(); + unlistenStateRef.current = null; + } + } + }); + // Bail if stop/cancel was called while we were awaiting. + if (startAbortedRef.current) { + unlistenTranscript(); + unlistenState(); + invoke("stop_dictation", { session: sessionId }).catch(() => {}); + return; + } + unlistenStateRef.current = unlistenState; + + // 3. Capture mic audio. + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { + autoGainControl: true, + echoCancellation: true, + noiseSuppression: true, + }, + }); + streamRef.current = stream; + + // Bail if aborted during mic permission prompt. + if (startAbortedRef.current) { + for (const track of stream.getTracks()) track.stop(); + streamRef.current = null; + invoke("stop_dictation", { session: sessionId }).catch(() => {}); + return; + } + + // 4. Set up AudioWorklet to send PCM to native layer. + const audioContext = new AudioContext({ sampleRate: 48000 }); + audioContextRef.current = audioContext; + + // Resume if the WebView created the context suspended (autoplay policy). + // A suspended context never pulls the worklet's `process()`, so no PCM + // would reach `push_dictation_audio` while the UI shows an active + // recording. Mirrors the huddle capture path (`huddle/lib/audioWorklet.ts`). + if (audioContext.state === "suspended") { + await audioContext.resume(); + } + + // Create a processor that accumulates audio frames and posts them + // to the main thread. Batching happens on the main thread side via + // a timer to reduce IPC overhead. + const processorCode = ` + class DictationProcessor extends AudioWorkletProcessor { + process(inputs) { + const input = inputs[0]; + if (input && input[0] && input[0].length > 0) { + this.port.postMessage(input[0].buffer); + } + return true; + } + } + registerProcessor('dictation-processor', DictationProcessor); + `; + const blob = new Blob([processorCode], { + type: "application/javascript", + }); + const blobUrl = URL.createObjectURL(blob); + try { + await audioContext.audioWorklet.addModule(blobUrl); + } finally { + URL.revokeObjectURL(blobUrl); + } + + // Bail if stop/cancel was called while the worklet module was loading. + // Without this the worklet/flush timer would start and `isRecording` + // would be set true, leaving dictation running after it was stopped. + if (startAbortedRef.current) { + for (const track of stream.getTracks()) track.stop(); + streamRef.current = null; + void audioContext.close(); + audioContextRef.current = null; + invoke("stop_dictation", { session: sessionId }).catch(() => {}); + return; + } + + const source = audioContext.createMediaStreamSource(stream); + const worklet = new AudioWorkletNode(audioContext, "dictation-processor"); + workletRef.current = worklet; + + // Accumulate audio frames in a batch array; a timer flushes to native. + worklet.port.onmessage = (event: MessageEvent) => { + audioBatchRef.current.push(new Float32Array(event.data)); + }; + + // Start the batch flush timer (~10 IPC calls/s instead of ~375). + batchTimerRef.current = setInterval(flushAudioBatch, AUDIO_BATCH_MS); + + const outputGain = audioContext.createGain(); + outputGain.gain.value = 0; + outputGainRef.current = outputGain; + + source.connect(worklet); + worklet.connect(outputGain); + outputGain.connect(audioContext.destination); + + setIsRecording(true); + setIsTranscribing(true); + } catch (error) { + // Tear down and stop the native engine if it was started but a later + // step failed (e.g. mic permission denied, AudioWorklet setup error). + // `cleanup()` performs the session-scoped stop, so it only tears down + // this instance's own session — never another composer's. + cleanup(); + setIsRecording(false); + setIsTranscribing(false); + + const message = + error instanceof Error ? error.message : "Local dictation failed"; + if (/not allowed|denied|permission/i.test(message)) { + toast.error("Microphone access denied", { + description: + "Allow microphone access in System Settings to use dictation.", + }); + } else if (/not found|no audio/i.test(message)) { + toast.error("No microphone found", { + description: "Connect a microphone and try again.", + }); + } else if (/model not ready/i.test(message)) { + toast.error("Voice model downloading", { + description: + "The speech model is still downloading. Try again shortly.", + }); + } else { + toast.error("Dictation failed", { description: message }); + } + } finally { + setIsStarting(false); + } + }, [ + cleanup, + flushAudioBatch, + isEnabled, + isRecording, + isStarting, + isTranscribing, + ]); + + const stopRecording = useCallback(() => { + // Signal any in-flight startRecording to bail after its next await. + startAbortedRef.current = true; + // Stop batch timer immediately. + if (batchTimerRef.current) { + clearInterval(batchTimerRef.current); + batchTimerRef.current = null; + } + // Stop mic and audio pipeline immediately so the user gets visual feedback. + if (streamRef.current) { + for (const track of streamRef.current.getTracks()) { + track.stop(); + } + streamRef.current = null; + } + if (workletRef.current) { + // Clear the port handler so PCM messages still queued on the main + // thread are dropped rather than appended to the batch after the + // final flush below (and leaking into the next session). + workletRef.current.port.onmessage = null; + workletRef.current.disconnect(); + workletRef.current = null; + } + if (outputGainRef.current) { + outputGainRef.current.disconnect(); + outputGainRef.current = null; + } + if (audioContextRef.current) { + void audioContextRef.current.close(); + audioContextRef.current = null; + } + setIsRecording(false); + // Flush remaining audio and THEN stop the native engine, ensuring the + // final batch arrives before the engine shuts down and flushes its buffer. + // isTranscribing stays true — cleared when `dictation-state: stopped` arrives. + // Scope the stop to THIS session: if the user restarts during the flush + // window, a new session may already be stored by the time this resolves — + // passing the session keeps this stop from tearing down the new engine. + const stoppingSession = nativeSessionRef.current; + void flushAudioBatch().then(() => { + invoke("stop_dictation", { session: stoppingSession }).catch(() => {}); + }); + }, [flushAudioBatch]); + + const cancelRecording = useCallback(() => { + // Signal any in-flight startRecording to bail after its next await. + startAbortedRef.current = true; + // `cleanup()` performs the session-scoped native stop, so cancelling a + // composer that isn't the active recorder can't tear down another + // composer's session. + cleanup(); + setIsRecording(false); + setIsTranscribing(false); + }, [cleanup]); + + const toggleRecording = useCallback(() => { + if (isRecording || isStarting) { + stopRecording(); + return; + } + void startRecording(); + }, [isRecording, isStarting, startRecording, stopRecording]); + + return { + isEnabled, + isRecording, + isStarting, + isTranscribing, + startRecording, + stopRecording, + cancelRecording, + toggleRecording, + }; +} diff --git a/desktop/src/features/dictation/hooks/useMessageComposerDictation.tsx b/desktop/src/features/dictation/hooks/useMessageComposerDictation.tsx new file mode 100644 index 0000000000..4f81656351 --- /dev/null +++ b/desktop/src/features/dictation/hooks/useMessageComposerDictation.tsx @@ -0,0 +1,49 @@ +import type * as React from "react"; +import { useRef } from "react"; +import { useFeatureEnabled } from "@/shared/features"; +import { DictationButton } from "../ui/DictationButton"; +import { useComposerDictation } from "./useComposerDictation"; + +interface UseMessageComposerDictationOptions { + syncContentRef: React.MutableRefObject<() => string>; + disabled: boolean; + disabledRef: React.MutableRefObject; + isSendingRef: React.MutableRefObject; + isUploadingRef: React.MutableRefObject; + setComposerContent: (text: string) => void; + setEditorContent: (text: string) => void; + submitMessageRef: React.MutableRefObject<() => void>; + draftKey: string | null; + composerRef: React.RefObject; +} + +export function useMessageComposerDictation({ + setEditorContent, + ...options +}: UseMessageComposerDictationOptions) { + const enabled = useFeatureEnabled("voiceDictation"); + const setEditorContentRef = useRef(setEditorContent); + setEditorContentRef.current = setEditorContent; + return useComposerDictation({ + ...options, + enabled, + setEditorContentRef, + }); +} + +export function MessageComposerDictationAction({ + children, + dictation, + disabled, +}: { + children?: React.ReactNode; + dictation: ReturnType; + disabled: boolean; +}) { + return ( + <> + + {children} + + ); +} diff --git a/desktop/src/features/dictation/hooks/useVoiceDictationShortcut.ts b/desktop/src/features/dictation/hooks/useVoiceDictationShortcut.ts new file mode 100644 index 0000000000..31597ed7ae --- /dev/null +++ b/desktop/src/features/dictation/hooks/useVoiceDictationShortcut.ts @@ -0,0 +1,53 @@ +import * as React from "react"; +import { useFeatureEnabled } from "@/shared/features"; +import { hasPrimaryShortcutModifier } from "@/shared/lib/platform"; + +export function useVoiceDictationShortcut(disabled: boolean) { + const enabled = useFeatureEnabled("voiceDictation"); + + React.useLayoutEffect(() => { + if (!enabled || disabled) { + return; + } + + let keyHeld = false; + const release = () => { + if (!keyHeld) return; + keyHeld = false; + window.dispatchEvent(new CustomEvent("buzz:dictation-key-up")); + }; + const handleKeyDown = (event: KeyboardEvent) => { + if ( + event.defaultPrevented || + event.repeat || + event.altKey || + event.shiftKey || + event.key.toLowerCase() !== "d" || + !hasPrimaryShortcutModifier(event) + ) { + return; + } + event.preventDefault(); + keyHeld = true; + window.dispatchEvent(new CustomEvent("buzz:dictation-key-down")); + }; + const handleKeyUp = (event: KeyboardEvent) => { + if (event.key.toLowerCase() === "d") release(); + }; + const handleVisibilityChange = () => { + if (document.hidden) release(); + }; + + window.addEventListener("keydown", handleKeyDown); + window.addEventListener("keyup", handleKeyUp); + window.addEventListener("blur", release); + document.addEventListener("visibilitychange", handleVisibilityChange); + return () => { + release(); + window.removeEventListener("keydown", handleKeyDown); + window.removeEventListener("keyup", handleKeyUp); + window.removeEventListener("blur", release); + document.removeEventListener("visibilitychange", handleVisibilityChange); + }; + }, [disabled, enabled]); +} diff --git a/desktop/src/features/dictation/index.ts b/desktop/src/features/dictation/index.ts new file mode 100644 index 0000000000..12964f17af --- /dev/null +++ b/desktop/src/features/dictation/index.ts @@ -0,0 +1,8 @@ +export { useComposerDictation } from "./hooks/useComposerDictation"; +export { useDictation } from "./hooks/useDictation"; +export { + MessageComposerDictationAction, + useMessageComposerDictation, +} from "./hooks/useMessageComposerDictation"; +export { useVoiceDictationShortcut } from "./hooks/useVoiceDictationShortcut"; +export { DictationButton } from "./ui/DictationButton"; diff --git a/desktop/src/features/dictation/lib/activeComposer.ts b/desktop/src/features/dictation/lib/activeComposer.ts new file mode 100644 index 0000000000..1fb1b3f104 --- /dev/null +++ b/desktop/src/features/dictation/lib/activeComposer.ts @@ -0,0 +1,23 @@ +/** + * Tracks which composer instance should handle global dictation shortcuts. + * + * When multiple composers are mounted (e.g. channel + thread), only the + * most recently focused one should respond to ⌘D. Each composer registers + * on focus and the global shortcut only dispatches to the active instance. + */ + +let activeInstanceId: string | null = null; + +export function setActiveDictationComposer(id: string): void { + activeInstanceId = id; +} + +export function clearActiveDictationComposer(id: string): void { + if (activeInstanceId === id) { + activeInstanceId = null; + } +} + +export function isActiveDictationComposer(id: string): boolean { + return activeInstanceId === id; +} diff --git a/desktop/src/features/dictation/lib/voiceInput.test.mjs b/desktop/src/features/dictation/lib/voiceInput.test.mjs new file mode 100644 index 0000000000..151104cbd8 --- /dev/null +++ b/desktop/src/features/dictation/lib/voiceInput.test.mjs @@ -0,0 +1,150 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + DEFAULT_AUTO_SUBMIT_PHRASE, + getAutoSubmitMatch, + parseAutoSubmitPhrases, + replaceTrailingTranscribedText, +} from "./voiceInput.ts"; + +// ── parseAutoSubmitPhrases ────────────────────────────────────────────────── + +test("parseAutoSubmitPhrases_returnsEmptyForNullish", () => { + assert.deepEqual(parseAutoSubmitPhrases(null), []); + assert.deepEqual(parseAutoSubmitPhrases(undefined), []); + assert.deepEqual(parseAutoSubmitPhrases(""), []); +}); + +test("parseAutoSubmitPhrases_splitsNormalizesAndDedupes", () => { + assert.deepEqual(parseAutoSubmitPhrases("Submit, send it, submit, "), [ + "submit", + "send it", + ]); +}); + +test("parseAutoSubmitPhrases_stripsTrailingPunctuation", () => { + assert.deepEqual(parseAutoSubmitPhrases("submit!"), ["submit"]); +}); + +// ── replaceTrailingTranscribedText ────────────────────────────────────────── + +test("replaceTrailingTranscribedText_appendsWhenNoPrevious", () => { + assert.equal( + replaceTrailingTranscribedText("Hello", "", "world"), + "Hello world", + ); +}); + +test("replaceTrailingTranscribedText_appendsToEmptyBase", () => { + assert.equal(replaceTrailingTranscribedText("", "", "hello"), "hello"); +}); + +test("replaceTrailingTranscribedText_replacesTrailingInterim", () => { + // Interim "hello wor" is refined to "hello world". + assert.equal( + replaceTrailingTranscribedText("hello wor", "hello wor", "hello world"), + "hello world", + ); +}); + +test("replaceTrailingTranscribedText_preservesTextTypedBeforeDictation", () => { + // User typed "Note: " then dictated; the manual prefix must survive. + assert.equal( + replaceTrailingTranscribedText("Note: hi", "hi", "hi there"), + "Note: hi there", + ); +}); + +test("replaceTrailingTranscribedText_appendsWhenPreviousNoLongerMatches", () => { + // If the previous transcript isn't the trailing text anymore, append. + assert.equal( + replaceTrailingTranscribedText("edited text", "old", "new"), + "edited text new", + ); +}); + +test("replaceTrailingTranscribedText_noDoubleSpaceBeforePunctuation", () => { + assert.equal( + replaceTrailingTranscribedText("Hello", "", ", world"), + "Hello, world", + ); +}); + +// ── getAutoSubmitMatch ────────────────────────────────────────────────────── + +test("getAutoSubmitMatch_returnsNullWhenPhraseAbsent", () => { + assert.equal( + getAutoSubmitMatch("hello there", parseAutoSubmitPhrases("submit")), + null, + ); +}); + +test("getAutoSubmitMatch_returnsNullWhenPhrasesEmpty", () => { + // DEFAULT_AUTO_SUBMIT_PHRASE is empty (auto-submit disabled by default). + assert.equal( + getAutoSubmitMatch( + "send this message submit", + parseAutoSubmitPhrases(DEFAULT_AUTO_SUBMIT_PHRASE), + ), + null, + ); +}); + +test("getAutoSubmitMatch_matchesTrailingPhraseAndStripsIt", () => { + const match = getAutoSubmitMatch( + "send this message submit", + parseAutoSubmitPhrases("submit"), + ); + assert.ok(match); + assert.equal(match.matchedPhrase, "submit"); + assert.equal(match.textWithoutPhrase, "send this message"); +}); + +test("getAutoSubmitMatch_ignoresPhraseMidSentence", () => { + // "submit" is not at the end, so it must not auto-send. + assert.equal( + getAutoSubmitMatch( + "submit the form later", + parseAutoSubmitPhrases("submit"), + ), + null, + ); +}); + +test("getAutoSubmitMatch_requiresWordBoundaryBeforePhrase", () => { + // "resubmit" ends with "submit" but is not a standalone word → no match. + assert.equal( + getAutoSubmitMatch("resubmit", parseAutoSubmitPhrases("submit")), + null, + ); +}); + +test("getAutoSubmitMatch_toleratesTrailingPunctuation", () => { + const match = getAutoSubmitMatch( + "ship it submit.", + parseAutoSubmitPhrases("submit"), + ); + assert.ok(match); + assert.equal(match.textWithoutPhrase, "ship it"); +}); + +test("getAutoSubmitMatch_matchesMultiWordPhrase", () => { + const match = getAutoSubmitMatch( + "please do this send it", + parseAutoSubmitPhrases("send it"), + ); + assert.ok(match); + assert.equal(match.matchedPhrase, "send it"); + assert.equal(match.textWithoutPhrase, "please do this"); +}); + +test("getAutoSubmitMatch_prefersLongestPhrase", () => { + const match = getAutoSubmitMatch( + "text please submit now", + parseAutoSubmitPhrases("submit now, now"), + ); + assert.ok(match); + assert.equal(match.matchedPhrase, "submit now"); + assert.equal(match.textWithoutPhrase, "text please"); +}); diff --git a/desktop/src/features/dictation/lib/voiceInput.ts b/desktop/src/features/dictation/lib/voiceInput.ts new file mode 100644 index 0000000000..1c287cef72 --- /dev/null +++ b/desktop/src/features/dictation/lib/voiceInput.ts @@ -0,0 +1,114 @@ +/** + * Default auto-submit phrase. Empty string disables auto-submit — the user + * must manually press Enter/Send after dictation. The infrastructure for + * configurable phrases is in place (parseAutoSubmitPhrases, getAutoSubmitMatch) + * and can be wired to a user setting when we're ready to ship auto-submit. + */ +export const DEFAULT_AUTO_SUBMIT_PHRASE = ""; + +const TRAILING_PUNCTUATION_REGEX = /[\s"'`.,!?;:)\]}]+$/u; + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +function normalizePhrase(value: string): string { + return value + .toLowerCase() + .replace(/\s+/g, " ") + .trim() + .replace(TRAILING_PUNCTUATION_REGEX, "") + .trim(); +} + +export function parseAutoSubmitPhrases( + rawValue: string | null | undefined, +): string[] { + if (!rawValue) return []; + return Array.from( + new Set( + rawValue + .split(",") + .map((value) => normalizePhrase(value)) + .filter(Boolean), + ), + ); +} + +function appendTranscribedText(baseText: string, fragment: string): string { + const normalizedFragment = fragment.replace(/\s+/g, " ").trim(); + if (!normalizedFragment) return baseText; + if (!baseText.trim()) return normalizedFragment; + if (/[\s([{/-]$/.test(baseText) || /^[,.;!?)]/.test(normalizedFragment)) { + return `${baseText}${normalizedFragment}`; + } + return `${baseText} ${normalizedFragment}`; +} + +export function replaceTrailingTranscribedText( + fullText: string, + previousTranscribedText: string, + nextTranscribedText: string, +): string { + if (!previousTranscribedText) { + return appendTranscribedText(fullText, nextTranscribedText); + } + + if (fullText.endsWith(previousTranscribedText)) { + return appendTranscribedText( + fullText.slice(0, -previousTranscribedText.length), + nextTranscribedText, + ); + } + + const trimmedPreviousText = previousTranscribedText.trim(); + if (trimmedPreviousText && fullText.endsWith(trimmedPreviousText)) { + return appendTranscribedText( + fullText.slice(0, -trimmedPreviousText.length), + nextTranscribedText, + ); + } + + return appendTranscribedText(fullText, nextTranscribedText); +} + +export function getAutoSubmitMatch( + transcribedText: string, + autoSubmitPhrases: string[], +): { matchedPhrase: string; textWithoutPhrase: string } | null { + const normalizedTranscribedText = normalizePhrase(transcribedText); + if (!normalizedTranscribedText) return null; + + const sortedPhrases = [...autoSubmitPhrases].sort( + (left, right) => right.length - left.length, + ); + + for (const phrase of sortedPhrases) { + if (!normalizedTranscribedText.endsWith(phrase)) continue; + + const phraseStartIndex = normalizedTranscribedText.length - phrase.length; + if ( + phraseStartIndex > 0 && + normalizedTranscribedText[phraseStartIndex - 1] !== " " + ) { + continue; + } + + const trimmedText = transcribedText.replace(TRAILING_PUNCTUATION_REGEX, ""); + const phraseWords = phrase.split(" ").filter(Boolean).map(escapeRegExp); + const phrasePattern = new RegExp( + `(^|\\s)(${phraseWords.join("\\s+")})\\s*$`, + "iu", + ); + const rawMatch = trimmedText.match(phrasePattern); + const phraseStartOffset = + rawMatch && rawMatch.index !== undefined + ? rawMatch.index + (rawMatch[1]?.length ?? 0) + : trimmedText.length - phrase.length; + const textWithoutPhrase = trimmedText.slice(0, phraseStartOffset).trimEnd(); + + return { matchedPhrase: phrase, textWithoutPhrase }; + } + + return null; +} diff --git a/desktop/src/features/dictation/ui/DictationButton.tsx b/desktop/src/features/dictation/ui/DictationButton.tsx new file mode 100644 index 0000000000..f284957e17 --- /dev/null +++ b/desktop/src/features/dictation/ui/DictationButton.tsx @@ -0,0 +1,73 @@ +import { Mic, Square } from "lucide-react"; +import { Button } from "@/shared/ui/button"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/shared/ui/tooltip"; +import { cn } from "@/shared/lib/cn"; +import { isMacPlatform } from "@/shared/lib/platform"; + +interface DictationState { + isEnabled: boolean; + isRecording: boolean; + isStarting: boolean; + isTranscribing: boolean; + toggleRecording: () => void; +} + +interface DictationButtonProps { + dictation: DictationState; + disabled?: boolean; +} + +export function DictationButton({ + dictation, + disabled = false, +}: DictationButtonProps) { + if (!dictation.isEnabled) return null; + + const shortcutHint = isMacPlatform() ? "⌘D" : "Ctrl+D"; + + const tooltipText = dictation.isRecording + ? "Stop recording" + : dictation.isTranscribing + ? "Transcribing…" + : null; + + return ( + + + + + + {tooltipText ?? ( + + Voice Dictation + + {shortcutHint} + + + )} + + + ); +} diff --git a/desktop/src/features/messages/ui/MessageComposer.tsx b/desktop/src/features/messages/ui/MessageComposer.tsx index 69e4ec67b5..ef040c68dd 100644 --- a/desktop/src/features/messages/ui/MessageComposer.tsx +++ b/desktop/src/features/messages/ui/MessageComposer.tsx @@ -2,9 +2,9 @@ import * as React from "react"; import { EditorContent } from "@tiptap/react"; import { useChannelLinks } from "@/features/messages/lib/useChannelLinks"; +import type { ChannelSuggestion } from "@/features/messages/lib/useChannelLinks"; import { handleAgentSnapshotPaste } from "@/features/messages/lib/agentSnapshotClipboard"; import { useComposerAutofocus } from "@/features/messages/lib/useComposerAutofocus"; -import type { ChannelSuggestion } from "@/features/messages/lib/useChannelLinks"; import { useDrafts } from "@/features/messages/lib/useDrafts"; import { resolveSentDraftKey } from "@/features/messages/ui/draftSubmitKey"; import { useEmojiAutocomplete } from "@/features/messages/lib/useEmojiAutocomplete"; @@ -39,22 +39,26 @@ import { import { useLinkEditor } from "@/features/messages/lib/useLinkEditor"; import { useComposerSpoilerParticles } from "@/features/messages/lib/useComposerSpoilerParticles"; import { useTypingBroadcast } from "@/features/messages/useTypingBroadcast"; +import { + MessageComposerDictationAction, + useMessageComposerDictation, +} from "@/features/dictation"; import { getBuzzCodeBlockClipboardText } from "@/shared/lib/codeBlockClipboard"; import { cn } from "@/shared/lib/cn"; -import { ChannelAutocomplete } from "./ChannelAutocomplete"; import { ComposerReplyEditBanner } from "./ComposerReplyEditBanner"; import { ComposerAttachments, DropZoneOverlay } from "./ComposerAttachments"; -import { EmojiAutocomplete } from "./EmojiAutocomplete"; -import { - MentionAutocomplete, - type MentionSuggestion, -} from "./MentionAutocomplete"; +import type { MentionSuggestion } from "./MentionAutocomplete"; import { ComposerDockToolbar } from "./ComposerDockToolbar"; import { NonMemberMentionDialog } from "./NonMemberMentionDialog"; import { useMentionSendFlow } from "./useMentionSendFlow"; import { usePersistentAgentMentionHydration } from "./usePersistentAgentMentionHydration"; import { useComposerContentState } from "./useComposerContentState"; import { useDraftPersistLifecycle } from "./useDraftPersistSnapshot"; +import { + MessageComposerAutocompletes, + MessageComposerUploadError, +} from "./MessageComposerOverlays"; +import { useComposerScrollToBottom } from "./useComposerScrollToBottom"; import type { MessageComposerProps } from "./MessageComposer.types"; @@ -206,6 +210,7 @@ function MessageComposerImpl({ emojiAutocomplete.isEmojiAutocompleteOpen; const submitMessageRef = React.useRef<() => void>(() => {}); + const stopDictationRef = React.useRef<() => void>(() => {}); const composerScrollRef = React.useRef(null); // Set after `useLinkEditor` exists below; the editor's link-click handler @@ -219,13 +224,7 @@ function MessageComposerImpl({ >(null); const onLinkShortcutRef = React.useRef<(() => boolean) | null>(null); - const scrollComposerToBottom = React.useCallback(() => { - window.requestAnimationFrame(() => { - const scrollElement = composerScrollRef.current; - if (!scrollElement) return; - scrollElement.scrollTop = scrollElement.scrollHeight; - }); - }, []); + const scrollComposerToBottom = useComposerScrollToBottom(composerScrollRef); const computedPlaceholder = editTarget ? "Edit your message" @@ -268,6 +267,19 @@ function MessageComposerImpl({ }, }); + const dictation = useMessageComposerDictation({ + syncContentRef: syncContentRefFromEditorRef, + disabled, + disabledRef, + isSendingRef, + isUploadingRef, + setComposerContent, + setEditorContent: richText.setContent, + submitMessageRef, + draftKey: effectiveDraftKey, + composerRef: composerScrollRef, + }); + stopDictationRef.current = dictation.cancelRecording; const linkEditor = useLinkEditor(richText); syncContentRefFromEditorRef.current = () => { const markdown = richText.getMarkdown(); @@ -278,7 +290,6 @@ function MessageComposerImpl({ onLinkSelectionChangeRef.current = linkEditor.showFromCursor; onLinkShortcutRef.current = linkEditor.openFromShortcut; useComposerSpoilerParticles(richText.editor, composerScrollRef); - const persistentMentionHydration = usePersistentAgentMentionHydration({ audienceScope, hydrationKey: effectiveDraftKey, @@ -512,6 +523,7 @@ function MessageComposerImpl({ // Edit mode if (editTargetRef.current && onEditSaveRef.current) { if (isSendingRef.current || isUploadingRef.current) return; + stopDictationRef.current(); const currentPendingImeta = media.pendingImetaRef.current; // No empty-edit guard here: clearing an edit to empty (no text, no // attachments) flows through to onEditSave as empty content, which @@ -587,6 +599,7 @@ function MessageComposerImpl({ ) { return; } + stopDictationRef.current(); const capturedThreadContext = onCaptureSendContext?.() ?? null; if ( @@ -919,42 +932,22 @@ function MessageComposerImpl({ }} > {ownsDropZone && media.isDragOver && } - + media.setUploadState({ status: "idle" })} /> - - - {media.uploadState.status === "error" ? ( -
- Upload failed: {media.uploadState.message} - -
- ) : null} {(media.pendingImeta.length > 0 || media.isUploading) && (
@@ -988,7 +981,14 @@ function MessageComposerImpl({ layoutMode={layoutMode} composerDisabled={disabled} editor={richText.editor} - extraActions={toolbarExtraActions} + extraActions={ + + {toolbarExtraActions} + + } formattingDisabled={disabled} isEmojiPickerOpen={isEmojiPickerOpen} isFormattingOpen={isFormattingOpen} diff --git a/desktop/src/features/messages/ui/MessageComposerOverlays.tsx b/desktop/src/features/messages/ui/MessageComposerOverlays.tsx new file mode 100644 index 0000000000..67b4f030b3 --- /dev/null +++ b/desktop/src/features/messages/ui/MessageComposerOverlays.tsx @@ -0,0 +1,72 @@ +import type { ChannelSuggestion } from "@/features/messages/lib/useChannelLinks"; +import type { useChannelLinks } from "@/features/messages/lib/useChannelLinks"; +import type { EmojiSuggestion } from "@/features/messages/lib/useEmojiAutocomplete"; +import type { useEmojiAutocomplete } from "@/features/messages/lib/useEmojiAutocomplete"; +import type { useMentions } from "@/features/messages/lib/useMentions"; +import { ChannelAutocomplete } from "./ChannelAutocomplete"; +import { EmojiAutocomplete } from "./EmojiAutocomplete"; +import { + MentionAutocomplete, + type MentionSuggestion, +} from "./MentionAutocomplete"; + +export function MessageComposerAutocompletes({ + applyChannelInsert, + applyEmojiInsert, + applyMentionInsert, + channelLinks, + emojiAutocomplete, + mentions, +}: { + applyChannelInsert: (suggestion: ChannelSuggestion) => void; + applyEmojiInsert: (suggestion: EmojiSuggestion) => void; + applyMentionInsert: (suggestion: MentionSuggestion) => void; + channelLinks: ReturnType; + emojiAutocomplete: ReturnType; + mentions: ReturnType; +}) { + return ( + <> + + + + + ); +} + +export function MessageComposerUploadError({ + message, + onDismiss, +}: { + message: string | null | undefined; + onDismiss: () => void; +}) { + if (message == null) return null; + return ( +
+ Upload failed: {message} + +
+ ); +} diff --git a/desktop/src/features/messages/ui/useComposerScrollToBottom.ts b/desktop/src/features/messages/ui/useComposerScrollToBottom.ts new file mode 100644 index 0000000000..9d07efb3a5 --- /dev/null +++ b/desktop/src/features/messages/ui/useComposerScrollToBottom.ts @@ -0,0 +1,13 @@ +import * as React from "react"; + +export function useComposerScrollToBottom( + composerScrollRef: React.RefObject, +) { + return React.useCallback(() => { + window.requestAnimationFrame(() => { + const scrollElement = composerScrollRef.current; + if (!scrollElement) return; + scrollElement.scrollTop = scrollElement.scrollHeight; + }); + }, [composerScrollRef]); +} diff --git a/desktop/src/features/settings/ui/KeyboardShortcutsCard.tsx b/desktop/src/features/settings/ui/KeyboardShortcutsCard.tsx index c71e62d5d9..af7bfafbb3 100644 --- a/desktop/src/features/settings/ui/KeyboardShortcutsCard.tsx +++ b/desktop/src/features/settings/ui/KeyboardShortcutsCard.tsx @@ -3,6 +3,7 @@ import { getPlatformKeys, type KeyboardShortcut, } from "@/shared/lib/keyboard-shortcuts"; +import { useFeatureEnabled } from "@/shared/features"; import { SettingsOptionGroup, SettingsOptionRow } from "./SettingsOptionGroup"; import { SettingsSectionHeader } from "./SettingsSectionHeader"; @@ -29,6 +30,7 @@ function KeyCombo({ shortcut }: { shortcut: KeyboardShortcut }) { } export function KeyboardShortcutsCard() { + const voiceDictationEnabled = useFeatureEnabled("voiceDictation"); const categories = getShortcutsByCategory(); return ( @@ -45,22 +47,27 @@ export function KeyboardShortcutsCard() { {category} - {shortcuts.map((shortcut) => ( - -
- - {shortcut.label} - - - {shortcut.description} - -
- - - ))} + {shortcuts + .filter( + (shortcut) => + voiceDictationEnabled || shortcut.id !== "voice-dictation", + ) + .map((shortcut) => ( + +
+ + {shortcut.label} + + + {shortcut.description} + +
+ + + ))}
))} diff --git a/desktop/src/shared/lib/keyboard-shortcuts.ts b/desktop/src/shared/lib/keyboard-shortcuts.ts index 422c45c5f3..377e4ae616 100644 --- a/desktop/src/shared/lib/keyboard-shortcuts.ts +++ b/desktop/src/shared/lib/keyboard-shortcuts.ts @@ -181,6 +181,14 @@ export const KEYBOARD_SHORTCUTS: KeyboardShortcut[] = [ keysWindows: "Ctrl+Space", category: "Messages", }, + { + id: "voice-dictation", + label: "Voice Dictation", + description: "Hold to dictate, release to stop (push-to-talk)", + keys: "⌘D", + keysWindows: "Ctrl+D", + category: "Messages", + }, // Formatting { diff --git a/desktop/tests/e2e/messaging.spec.ts b/desktop/tests/e2e/messaging.spec.ts index c6f5aefb9b..3e6a0ae40c 100644 --- a/desktop/tests/e2e/messaging.spec.ts +++ b/desktop/tests/e2e/messaging.spec.ts @@ -109,7 +109,72 @@ test.beforeEach(async ({ page }, testInfo) => { ], } : undefined; - await installMockBridge(page, mock); + await installMockBridge(page, mock, { + seedPreviewFeatures: !testInfo.title.includes("dictation experiment"), + }); +}); + +test("voice dictation stays behind the dictation experiment", async ({ + page, +}) => { + await page.goto("/"); + await page.getByTestId("channel-general").click(); + + await page.evaluate(() => { + let dictationStarts = 0; + window.addEventListener("buzz:dictation-key-down", () => { + dictationStarts += 1; + }); + ( + window as typeof window & { __BUZZ_DICTATION_STARTS__?: () => number } + ).__BUZZ_DICTATION_STARTS__ = () => dictationStarts; + }); + + const dispatchDictationShortcut = () => + page.evaluate(() => { + const isMac = /mac|iphone|ipad|ipod/i.test(navigator.platform); + window.dispatchEvent( + new KeyboardEvent("keydown", { + bubbles: true, + ctrlKey: !isMac, + key: "d", + metaKey: isMac, + }), + ); + window.dispatchEvent( + new KeyboardEvent("keyup", { + bubbles: true, + ctrlKey: !isMac, + key: "d", + metaKey: isMac, + }), + ); + }); + const getDictationStarts = () => + page.evaluate( + () => + ( + window as typeof window & { + __BUZZ_DICTATION_STARTS__?: () => number; + } + ).__BUZZ_DICTATION_STARTS__?.() ?? 0, + ); + + await page.getByTestId("message-input").click(); + await dispatchDictationShortcut(); + await expect.poll(getDictationStarts).toBe(0); + + await openSettings(page, "experimental"); + const dictationToggle = page.getByTestId("feature-toggle-voiceDictation"); + await expect(dictationToggle).not.toBeChecked(); + await dictationToggle.click(); + await expect(dictationToggle).toBeChecked(); + + await page.getByTestId("settings-back-to-app").click(); + await expect(page.getByTestId("message-input")).toBeVisible(); + await page.getByTestId("message-input").click(); + await dispatchDictationShortcut(); + await expect.poll(getDictationStarts).toBe(1); }); test("agent owner label identifies the agent and owner", async ({ page }) => { diff --git a/preview-features.json b/preview-features.json index 388f1c39b0..cfd37ee873 100644 --- a/preview-features.json +++ b/preview-features.json @@ -30,6 +30,12 @@ "name": "Agent-managed profiles", "description": "Let agents manage their own relay name and avatar instead of restoring the desktop copy", "platforms": ["desktop"] + }, + { + "id": "voiceDictation", + "name": "Voice dictation", + "description": "Compose messages with local, real-time speech recognition", + "platforms": ["desktop"] } ] }