diff --git a/apps/server/src/speech/SpeechService.test.ts b/apps/server/src/speech/SpeechService.test.ts index 3bec6b27a5148..2a4ba9752a6f4 100644 --- a/apps/server/src/speech/SpeechService.test.ts +++ b/apps/server/src/speech/SpeechService.test.ts @@ -106,6 +106,7 @@ const pcm = () => new Uint8Array(new Float32Array([0.25]).buffer); beforeEach(() => { vi.clearAllMocks(); + native.finish.mockReset(); native.backend = "Vulkan0"; native.supportsInitialPrompt = false; readyModels.clear(); @@ -215,6 +216,58 @@ it.effect("unloads after the chosen idle period and reloads on the next recordin }).pipe(Effect.provide(layer)), ); +it.effect.each([ + { cue: "um", remove: true, text: "Make it red, um, blue.", expected: "Make it red, um, blue." }, + { + cue: "uh no", + remove: true, + text: "Make it red, uh no, blue.", + expected: "Make it red, uh no, blue.", + }, + { + cue: "uh no", + remove: true, + text: "Uh, Make it red, UH NO, blue. Uh nope.", + expected: "Make it red, UH NO, blue. nope.", + }, + { + cue: " UM ", + remove: true, + text: "Uh, Make it red, um, blue.", + expected: "Make it red, um, blue.", + }, + { + cue: undefined, + remove: true, + text: "Uh, Make it red, um, blue.", + expected: "Make it red, blue.", + }, + { + cue: "um", + remove: false, + text: "Uh, Make it red, um, blue.", + expected: "Uh, Make it red, um, blue.", + }, +])( + "preserves only an explicit correction cue during batch and stream filler removal: $cue / $remove", + ({ cue, remove, text, expected }) => + Effect.gen(function* () { + const speech = yield* SpeechService.SpeechService; + const options = { + ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, + speechLanguage: "en", + speechRemoveFillerWords: remove, + speechCustomWords: [{ term: "um", aliases: [] }], + ...(cue === undefined ? {} : { speechCorrectionWord: cue }), + }; + native.transcribe.mockResolvedValueOnce({ text }); + native.finish.mockResolvedValueOnce(text); + expect(yield* speech.transcribe(pcm(), options)).toBe(expected); + const stream = yield* speech.startStream(options); + expect(yield* stream.finish).toBe(expected); + }).pipe(Effect.scoped, Effect.provide(layer)), +); + it.effect("holds model ownership until the stream scope closes and preserves silence", () => Effect.gen(function* () { const speech = yield* SpeechService.SpeechService; diff --git a/apps/server/src/speech/SpeechService.ts b/apps/server/src/speech/SpeechService.ts index 8b91677131948..6390e44df06a3 100644 --- a/apps/server/src/speech/SpeechService.ts +++ b/apps/server/src/speech/SpeechService.ts @@ -494,7 +494,9 @@ const make = Effect.gen(function* () { language: string | undefined, ) => { return ( - options.speechRemoveFillerWords ? removeSpeechFillerWords(text, language) : text + options.speechRemoveFillerWords + ? removeSpeechFillerWords(text, language, options.speechCorrectionWord) + : text ).trim(); }; diff --git a/apps/server/src/speech/fillerWords.ts b/apps/server/src/speech/fillerWords.ts index 0181db1bf916c..5061341833c06 100644 --- a/apps/server/src/speech/fillerWords.ts +++ b/apps/server/src/speech/fillerWords.ts @@ -22,19 +22,21 @@ const LANGUAGE_FILLER_WORDS: Readonly> = { const escapeRegex = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); -export function removeSpeechFillerWords(text: string, language?: string): string { +export function removeSpeechFillerWords( + text: string, + language?: string, + correctionCue?: string, +): string { const baseLanguage = language?.split(/[-_]/, 1)[0]?.toLowerCase(); const words = [ ...UNIVERSAL_FILLER_WORDS, ...(baseLanguage ? (LANGUAGE_FILLER_WORDS[baseLanguage] ?? []) : []), ]; - let filtered = text; - for (const word of words) { - const pattern = new RegExp( - `(? cue ?? ""); return filtered.replace(/\s{2,}/g, " ").trim(); } diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx index c47b0259046e8..398229f784907 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx +++ b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx @@ -402,19 +402,29 @@ it("cleans the transcript only after cleanup is separately enabled", async () => }); // The settings model test must show what the model heard, not what a provider rewrote. -it("keeps a cleanup-free recording off the provider even when cleanup is on", async () => { - setup({ destination: "env-a" }); - chooseTranscriptCleanup(true); - mocks.settings.speechCorrectionWord = "sorry"; - mocks.projectWords = { "proj-1": [{ term: "Acme", aliases: ["ack me"] }] }; - await using _screen = await render(); - await act(() => mocks.voice.start()); - await act(() => mocks.voice.stop()); - expect(mocks.cleanupCalls).toEqual([]); - expect(mocks.recordingOptions.at(-1)?.speechCustomWords).toEqual(mocks.projectWords["proj-1"]); - expect(mocks.transcribeCalls).toEqual(["env-a"]); - expect(mocks.committed).toHaveBeenCalledExactlyOnceWith("dictated", expect.anything()); -}); +it.each([ + { cleanup: false, authorized: true }, + { cleanup: undefined, authorized: false }, +])( + "keeps a recording without authorized cleanup off the provider: $cleanup / $authorized", + async ({ cleanup, authorized }) => { + setup({ destination: "env-a" }); + chooseTranscriptCleanup(true); + mocks.settings.speechCorrectionWord = "sorry"; + if (!authorized) mocks.canDictate.delete(THREAD_ENV); + mocks.projectWords = { "proj-1": [{ term: "Acme", aliases: ["ack me"] }] }; + await using _screen = await render( + , + ); + await act(() => mocks.voice.start()); + await act(() => mocks.voice.stop()); + expect(mocks.cleanupCalls).toEqual([]); + expect(mocks.recordingOptions.at(-1)?.speechCustomWords).toEqual(mocks.projectWords["proj-1"]); + expect(mocks.recordingOptions.at(-1)?.speechCorrectionWord).toBeUndefined(); + expect(mocks.transcribeCalls).toEqual(["env-a"]); + expect(mocks.committed).toHaveBeenCalledExactlyOnceWith("dictated", expect.anything()); + }, +); it("sends the project's vocabulary and name as hints, and the cue only with cleanup on", async () => { setup({ destination: "env-a" }); @@ -441,6 +451,14 @@ it("sends the project's vocabulary and name as hints, and the cue only with clea "effect", ]); expect(mocks.cleanupOptions.at(-1)).toMatchObject({ speechCorrectionWord: "scratch that" }); + expect(mocks.recordingOptions.at(-1)?.speechCorrectionWord).toBe("scratch that"); + chooseTranscriptCleanup(false); + await act(() => mocks.voice.start()); + await act(() => mocks.voice.stop()); + expect(mocks.recordingOptions.at(-1)?.speechCorrectionWord).toBeUndefined(); + expect(mocks.recordingOptions.at(-1)?.speechCustomWords.map(({ term }) => term)).not.toContain( + "scratch that", + ); }); it("shares one status probe per destination across every mounted editor", async () => { diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.ts b/apps/web/src/speech/useEnvironmentSpeechInput.ts index 1265577a7e0af..c14d2b176bdd4 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.ts +++ b/apps/web/src/speech/useEnvironmentSpeechInput.ts @@ -240,7 +240,8 @@ export function useEnvironmentSpeechInput(input: HookInput) { } = latestRef.current; const cleanupGrant = current.cleanup === false ? null : readCleanupGrant(); const cleanupEnabled = cleanupGrant !== null; - const cue = cleanupEnabled ? settings.speechCorrectionWord : ""; + const cue = + cleanupEnabled && latestRef.current.cleanupPrepared ? settings.speechCorrectionWord : ""; const speechCustomWords = resolveSpeechVocabulary({ personal: settings.speechCustomWords, project, @@ -262,6 +263,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { speechTranslateToEnglish: settings.speechTranslateToEnglish, speechCustomWords, speechRemoveFillerWords: settings.speechRemoveFillerWords, + ...(cue.trim() ? { speechCorrectionWord: cue } : {}), }; }, getMicrophoneId: () => latestRef.current.clientSettings.voiceMicrophone, diff --git a/packages/contracts/src/speech.ts b/packages/contracts/src/speech.ts index a78f131019c74..1fc1ecaa4ceef 100644 --- a/packages/contracts/src/speech.ts +++ b/packages/contracts/src/speech.ts @@ -117,12 +117,17 @@ export const SpeechPostProcessingPromptSettings = Schema.Struct({ }); export type SpeechPostProcessingPromptSettings = typeof SpeechPostProcessingPromptSettings.Type; +/** A word the person says to flag a spoken correction, such as "sorry" or "err". */ +export const SpeechCorrectionCue = Schema.String.check(Schema.isMaxLength(50)); + /** Recognition preferences that travel with every recording. */ export const SpeechTranscriptionOptions = Schema.Struct({ speechLanguage: SpeechLanguage, speechTranslateToEnglish: Schema.Boolean, speechCustomWords: SpeechCustomWords, speechRemoveFillerWords: Schema.Boolean, + /** Preserve this cue for separately consented transcript cleanup. */ + speechCorrectionWord: Schema.optionalKey(SpeechCorrectionCue), }); export type SpeechTranscriptionOptions = typeof SpeechTranscriptionOptions.Type; export const DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS: SpeechTranscriptionOptions = { @@ -133,9 +138,6 @@ export const DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS: SpeechTranscriptionOptions = }; export const SPEECH_MAX_OPTIONS_BYTES = 128_000; -/** A word the person says to flag a spoken correction, such as "sorry" or "err". */ -export const SpeechCorrectionCue = Schema.String.check(Schema.isMaxLength(50)); - export const SpeechPostProcessingOptions = Schema.Struct({ speechCustomWords: SpeechCustomWords, speechPostProcessingPrompt: SpeechPostProcessingPromptSettings,