feat: add voice dictation using OpenAI Whisper & ElevenLabs (#3079)

Co-authored-by: jack <>
This commit is contained in:
jack
2025-06-27 07:36:35 +01:00
committed by GitHub
parent 0962041696
commit 6ad95fe0d0
16 changed files with 1457 additions and 42 deletions
+150 -37
View File
@@ -2,11 +2,14 @@ import React, { useRef, useState, useEffect, useMemo } from 'react';
import { Button } from './ui/button';
import type { View } from '../App';
import Stop from './ui/Stop';
import { Attach, Send, Close } from './icons';
import { Attach, Send, Close, Microphone } from './icons';
import { debounce } from 'lodash';
import BottomMenu from './bottom_menu/BottomMenu';
import { LocalMessageStorage } from '../utils/localMessageStorage';
import { Message } from '../types/message';
import { useWhisper } from '../hooks/useWhisper';
import { WaveformVisualizer } from './WaveformVisualizer';
import { toastError } from '../toasts';
interface PastedImage {
id: string;
@@ -63,6 +66,39 @@ export default function ChatInput({
const [isFocused, setIsFocused] = useState(false);
const [pastedImages, setPastedImages] = useState<PastedImage[]>([]);
// Whisper hook for voice dictation
const {
isRecording,
isTranscribing,
canUseDictation,
audioContext,
analyser,
startRecording,
stopRecording,
recordingDuration,
estimatedSize,
} = useWhisper({
onTranscription: (text) => {
// Append transcribed text to the current input
const newValue = displayValue.trim() ? `${displayValue.trim()} ${text}` : text;
setDisplayValue(newValue);
setValue(newValue);
textAreaRef.current?.focus();
},
onError: (error) => {
toastError({
title: 'Dictation Error',
msg: error.message,
});
},
onSizeWarning: (sizeMB) => {
toastError({
title: 'Recording Size Warning',
msg: `Recording is ${sizeMB.toFixed(1)}MB. Maximum size is 25MB.`,
});
},
});
// Update internal value when initialValue changes
useEffect(() => {
setValue(initialValue);
@@ -451,28 +487,40 @@ export default function ChatInput({
} bg-bgApp z-10`}
>
<form onSubmit={onFormSubmit}>
<textarea
data-testid="chat-input"
autoFocus
id="dynamic-textarea"
placeholder="What can goose help with? ⌘↑/⌘↓"
value={displayValue}
onChange={handleChange}
onCompositionStart={handleCompositionStart}
onCompositionEnd={handleCompositionEnd}
onKeyDown={handleKeyDown}
onPaste={handlePaste}
onFocus={() => setIsFocused(true)}
onBlur={() => setIsFocused(false)}
ref={textAreaRef}
rows={1}
style={{
minHeight: `${minHeight}px`,
maxHeight: `${maxHeight}px`,
overflowY: 'auto',
}}
className="w-full pl-4 pr-[68px] outline-none border-none focus:ring-0 bg-transparent pt-3 pb-1.5 text-sm resize-none text-textStandard placeholder:text-textPlaceholder"
/>
<div className="relative">
<textarea
data-testid="chat-input"
autoFocus
id="dynamic-textarea"
placeholder={isRecording ? '' : 'What can goose help with? ⌘↑/⌘↓'}
value={displayValue}
onChange={handleChange}
onCompositionStart={handleCompositionStart}
onCompositionEnd={handleCompositionEnd}
onKeyDown={handleKeyDown}
onPaste={handlePaste}
onFocus={() => setIsFocused(true)}
onBlur={() => setIsFocused(false)}
ref={textAreaRef}
rows={1}
style={{
minHeight: `${minHeight}px`,
maxHeight: `${maxHeight}px`,
overflowY: 'auto',
opacity: isRecording ? 0 : 1,
}}
className="w-full pl-4 pr-[108px] outline-none border-none focus:ring-0 bg-transparent pt-3 pb-1.5 text-sm resize-none text-textStandard placeholder:text-textPlaceholder"
/>
{isRecording && (
<div className="absolute inset-0 flex items-center pl-4 pr-[108px] pt-3 pb-1.5">
<WaveformVisualizer
audioContext={audioContext}
analyser={analyser}
isRecording={isRecording}
/>
</div>
)}
</div>
{pastedImages.length > 0 && (
<div className="flex flex-wrap gap-2 p-2 border-t border-borderSubtle">
@@ -537,20 +585,85 @@ export default function ChatInput({
<Stop size={24} />
</Button>
) : (
<Button
type="submit"
size="icon"
variant="ghost"
disabled={!hasSubmittableContent || isAnyImageLoading} // Disable if no content or if images are still loading/saving
className={`absolute right-3 top-2 transition-colors rounded-full w-7 h-7 [&_svg]:size-4 ${
!hasSubmittableContent || isAnyImageLoading
? 'text-textSubtle cursor-not-allowed'
: 'bg-bgAppInverse text-textProminentInverse hover:cursor-pointer'
}`}
title={isAnyImageLoading ? 'Waiting for images to save...' : 'Send'}
>
<Send />
</Button>
<>
{/* Microphone button - only show if dictation is enabled and configured */}
{canUseDictation && (
<>
<Button
type="button"
size="icon"
variant="ghost"
onClick={() => {
if (isRecording) {
stopRecording();
} else {
startRecording();
}
}}
disabled={isTranscribing}
className={`absolute right-12 top-2 transition-colors rounded-full w-7 h-7 [&_svg]:size-4 ${
isRecording
? 'bg-red-500 text-white hover:bg-red-600'
: isTranscribing
? 'text-textSubtle cursor-not-allowed animate-pulse'
: 'text-textSubtle hover:text-textStandard'
}`}
title={
isRecording
? `Stop recording (${Math.floor(recordingDuration)}s, ~${estimatedSize.toFixed(1)}MB)`
: isTranscribing
? 'Transcribing...'
: 'Start dictation'
}
>
<Microphone />
</Button>
{/* Recording/transcribing status indicator - positioned above the input */}
{(isRecording || isTranscribing) && (
<div className="absolute right-0 -top-8 bg-bgApp px-2 py-1 rounded text-xs whitespace-nowrap shadow-md border border-borderSubtle">
{isTranscribing ? (
<span className="text-blue-500 flex items-center gap-1">
<span className="inline-block w-2 h-2 bg-blue-500 rounded-full animate-pulse" />
Transcribing...
</span>
) : (
<span
className={`flex items-center gap-2 ${estimatedSize > 20 ? 'text-orange-500' : 'text-textSubtle'}`}
>
<span className="inline-block w-2 h-2 bg-red-500 rounded-full animate-pulse" />
{Math.floor(recordingDuration)}s ~{estimatedSize.toFixed(1)}MB
{estimatedSize > 20 && <span className="text-xs">(near 25MB limit)</span>}
</span>
)}
</div>
)}
</>
)}
<Button
type="submit"
size="icon"
variant="ghost"
disabled={
!hasSubmittableContent || isAnyImageLoading || isRecording || isTranscribing
}
className={`absolute right-3 top-2 transition-colors rounded-full w-7 h-7 [&_svg]:size-4 ${
!hasSubmittableContent || isAnyImageLoading || isRecording || isTranscribing
? 'text-textSubtle cursor-not-allowed'
: 'bg-bgAppInverse text-textProminentInverse hover:cursor-pointer'
}`}
title={
isAnyImageLoading
? 'Waiting for images to save...'
: isRecording
? 'Recording...'
: isTranscribing
? 'Transcribing...'
: 'Send'
}
>
<Send />
</Button>
</>
)}
</form>
@@ -1,4 +1,4 @@
/* global Blob, ClipboardItem */
/* global ClipboardItem */
import React, { useState } from 'react';
import { Copy } from './icons';
@@ -0,0 +1,113 @@
import React, { useEffect, useRef } from 'react';
interface WaveformVisualizerProps {
audioContext: AudioContext | null;
analyser: AnalyserNode | null;
isRecording: boolean;
}
export const WaveformVisualizer: React.FC<WaveformVisualizerProps> = ({
analyser,
isRecording,
}) => {
const canvasRef = useRef<HTMLCanvasElement>(null);
const animationRef = useRef<number>();
useEffect(() => {
if (!canvasRef.current || !analyser || !isRecording) return;
const canvas = canvasRef.current;
const ctx = canvas.getContext('2d');
if (!ctx) return;
// Set canvas size
const dpr = window.devicePixelRatio || 1;
const rect = canvas.getBoundingClientRect();
canvas.width = rect.width * dpr;
canvas.height = rect.height * dpr;
ctx.scale(dpr, dpr);
// Configure analyser
analyser.fftSize = 256;
const bufferLength = analyser.frequencyBinCount;
const dataArray = new Uint8Array(bufferLength);
// Visual settings
const barWidth = 3;
const barSpacing = 2;
const barCount = Math.floor(rect.width / (barWidth + barSpacing));
const barMaxHeight = rect.height * 0.8;
const barMinHeight = 2;
// Smoothing for bars
const smoothedHeights = new Array(barCount).fill(0);
const targetHeights = new Array(barCount).fill(0);
const draw = () => {
if (!isRecording) return;
animationRef.current = requestAnimationFrame(draw);
// Get frequency data
analyser.getByteFrequencyData(dataArray);
// Clear canvas
ctx.clearRect(0, 0, rect.width, rect.height);
// Calculate target heights based on frequency data
for (let i = 0; i < barCount; i++) {
const dataIndex = Math.floor((i / barCount) * bufferLength * 0.5); // Use lower frequencies
const value = dataArray[dataIndex] / 255;
// Apply some randomness and minimum height for visual interest
const randomFactor = 0.85 + Math.random() * 0.3;
targetHeights[i] = Math.max(barMinHeight, value * barMaxHeight * randomFactor);
}
// Smooth the bar heights
for (let i = 0; i < barCount; i++) {
const diff = targetHeights[i] - smoothedHeights[i];
smoothedHeights[i] += diff * 0.3; // Smoothing factor
}
// Draw bars
for (let i = 0; i < barCount; i++) {
const x = i * (barWidth + barSpacing) + barSpacing;
const barHeight = smoothedHeights[i];
const y = (rect.height - barHeight) / 2;
// Create gradient for each bar
const gradient = ctx.createLinearGradient(0, y, 0, y + barHeight);
// Dynamic color based on height
const intensity = barHeight / barMaxHeight;
const hue = 200 + intensity * 20; // Blue to cyan
const saturation = 50 + intensity * 50;
const lightness = 50 + intensity * 20;
gradient.addColorStop(0, `hsla(${hue}, ${saturation}%, ${lightness}%, 0.3)`);
gradient.addColorStop(0.5, `hsla(${hue}, ${saturation}%, ${lightness}%, 0.8)`);
gradient.addColorStop(1, `hsla(${hue}, ${saturation}%, ${lightness}%, 0.3)`);
ctx.fillStyle = gradient;
ctx.fillRect(x, y, barWidth, barHeight);
}
};
draw();
return () => {
if (animationRef.current) {
cancelAnimationFrame(animationRef.current);
}
};
}, [analyser, isRecording]);
return (
<canvas
ref={canvasRef}
className="absolute inset-0 w-full h-full pointer-events-none"
style={{ opacity: 0.9 }}
/>
);
};
@@ -0,0 +1,48 @@
import React from 'react';
interface MicrophoneProps {
className?: string;
size?: number;
}
export const Microphone: React.FC<MicrophoneProps> = ({ className = '', size = 24 }) => {
return (
<svg
width={size}
height={size}
viewBox="0 0 24 24"
fill="none"
xmlns="http://www.w3.org/2000/svg"
className={className}
>
<path
d="M12 14.5C13.66 14.5 15 13.16 15 11.5V5.5C15 3.84 13.66 2.5 12 2.5C10.34 2.5 9 3.84 9 5.5V11.5C9 13.16 10.34 14.5 12 14.5Z"
stroke="currentColor"
strokeWidth="2"
strokeLinecap="round"
strokeLinejoin="round"
/>
<path
d="M19 11.5C19 15.09 16.09 18 12.5 18C8.91 18 6 15.09 6 11.5"
stroke="currentColor"
strokeWidth="2"
strokeLinecap="round"
strokeLinejoin="round"
/>
<path
d="M12 18V21.5"
stroke="currentColor"
strokeWidth="2"
strokeLinecap="round"
strokeLinejoin="round"
/>
<path
d="M8 21.5H16"
stroke="currentColor"
strokeWidth="2"
strokeLinecap="round"
strokeLinejoin="round"
/>
</svg>
);
};
@@ -20,6 +20,7 @@ import Send from './Send';
import Settings from './Settings';
import Time from './Time';
import { Gear } from './Gear';
import { Microphone } from './Microphone';
export {
ArrowDown,
@@ -37,6 +38,7 @@ export {
Edit,
Idea,
Gear,
Microphone,
More,
Refresh,
SensitiveHidden,
@@ -9,6 +9,7 @@ import SessionSharingSection from './sessions/SessionSharingSection';
import { ResponseStylesSection } from './response_styles/ResponseStylesSection';
import AppSettingsSection from './app/AppSettingsSection';
import SchedulerSection from './scheduler/SchedulerSection';
import DictationSection from './dictation/DictationSection';
import { ExtensionConfig } from '../../api';
import MoreMenuLayout from '../more_menu/MoreMenuLayout';
@@ -56,6 +57,8 @@ export default function SettingsView({
<SessionSharingSection />
{/* Response Styles */}
<ResponseStylesSection />
{/* Voice Dictation */}
<DictationSection />
{/* Tool Selection Strategy */}
<ToolSelectionStrategySection setView={setView} />
{/* App Settings */}
@@ -0,0 +1,276 @@
import { useState, useEffect, useRef } from 'react';
import { Switch } from '../../ui/switch';
import { ChevronDown } from 'lucide-react';
import { Input } from '../../ui/input';
import { useConfig } from '../../ConfigContext';
type DictationProvider = 'openai' | 'elevenlabs';
interface DictationSettings {
enabled: boolean;
provider: DictationProvider;
}
const DICTATION_SETTINGS_KEY = 'dictation_settings';
const ELEVENLABS_API_KEY = 'ELEVENLABS_API_KEY';
export default function DictationSection() {
const [settings, setSettings] = useState<DictationSettings>({
enabled: true,
provider: 'openai',
});
const [hasOpenAIKey, setHasOpenAIKey] = useState(false);
const [showProviderDropdown, setShowProviderDropdown] = useState(false);
const [showElevenLabsKey, setShowElevenLabsKey] = useState(false);
const [elevenLabsApiKey, setElevenLabsApiKey] = useState('');
const [isLoadingKey, setIsLoadingKey] = useState(false);
const [hasElevenLabsKey, setHasElevenLabsKey] = useState(false);
const elevenLabsApiKeyRef = useRef('');
const { getProviders, upsert, read } = useConfig();
// Load settings from localStorage and ElevenLabs API key from secure storage
useEffect(() => {
const loadSettings = async () => {
const savedSettings = localStorage.getItem(DICTATION_SETTINGS_KEY);
if (savedSettings) {
const parsed = JSON.parse(savedSettings);
setSettings(parsed);
setShowElevenLabsKey(parsed.provider === 'elevenlabs');
} else {
// Default settings
const defaultSettings: DictationSettings = {
enabled: true,
provider: 'openai',
};
setSettings(defaultSettings);
localStorage.setItem(DICTATION_SETTINGS_KEY, JSON.stringify(defaultSettings));
}
// Load ElevenLabs API key from storage
setIsLoadingKey(true);
try {
// Try reading as secret - will return true if exists
const keyExists = await read(ELEVENLABS_API_KEY, true);
if (keyExists === true) {
setHasElevenLabsKey(true);
// Don't set the actual key since we can't read secrets
}
} catch (error) {
console.error('Error checking ElevenLabs API key:', error);
} finally {
setIsLoadingKey(false);
}
};
loadSettings();
}, [read]);
// Save ElevenLabs key on unmount if it has changed
useEffect(() => {
return () => {
if (showElevenLabsKey && elevenLabsApiKeyRef.current) {
// We can't use async in cleanup, so we'll use the promise directly
const keyToSave = elevenLabsApiKeyRef.current;
if (keyToSave.trim()) {
upsert(ELEVENLABS_API_KEY, keyToSave, true).catch((error) => {
console.error('Error saving ElevenLabs API key on unmount:', error);
});
}
}
};
}, [showElevenLabsKey, upsert]);
// Check if OpenAI is configured
useEffect(() => {
const checkOpenAIKey = async () => {
try {
const providers = await getProviders(false);
const openAIProvider = providers.find((p) => p.name === 'openai');
setHasOpenAIKey(openAIProvider?.is_configured || false);
} catch (error) {
console.error('Error checking OpenAI configuration:', error);
setHasOpenAIKey(false);
}
};
checkOpenAIKey();
}, [getProviders]);
const saveSettings = (newSettings: DictationSettings) => {
setSettings(newSettings);
localStorage.setItem(DICTATION_SETTINGS_KEY, JSON.stringify(newSettings));
};
const handleToggle = (enabled: boolean) => {
saveSettings({ ...settings, enabled });
};
const handleProviderChange = (provider: DictationProvider) => {
saveSettings({ ...settings, provider });
setShowProviderDropdown(false);
setShowElevenLabsKey(provider === 'elevenlabs');
};
const handleElevenLabsKeyChange = (key: string) => {
setElevenLabsApiKey(key);
elevenLabsApiKeyRef.current = key;
};
const saveElevenLabsKey = async () => {
// Save to secure storage
try {
if (elevenLabsApiKey.trim()) {
await upsert(ELEVENLABS_API_KEY, elevenLabsApiKey, true);
setHasElevenLabsKey(true);
} else {
// If key is empty, remove it from storage
await upsert(ELEVENLABS_API_KEY, null, true);
setHasElevenLabsKey(false);
}
} catch (error) {
console.error('Error saving ElevenLabs API key:', error);
}
};
const getProviderLabel = (provider: DictationProvider): string => {
switch (provider) {
case 'openai':
return 'OpenAI Whisper';
case 'elevenlabs':
return 'ElevenLabs';
default:
return provider;
}
};
return (
<section id="dictation" className="px-8">
<div className="flex justify-between items-center mb-2">
<h2 className="text-xl font-medium text-textStandard">Voice Dictation</h2>
</div>
<div className="border-b border-borderSubtle pb-8">
<p className="text-sm text-textStandard mb-6">Configure voice input for messages</p>
{/* Enable/Disable Toggle */}
<div className="flex items-center justify-between mb-4">
<div>
<h3 className="text-textStandard">Enable Voice Dictation</h3>
<p className="text-xs text-textSubtle max-w-md mt-[2px]">
Show microphone button for voice input
</p>
</div>
<div className="flex items-center">
<Switch checked={settings.enabled} onCheckedChange={handleToggle} variant="mono" />
</div>
</div>
{/* Provider Selection */}
{settings.enabled && (
<>
<div className="flex items-center justify-between mb-4">
<div>
<h3 className="text-textStandard">Dictation Provider</h3>
<p className="text-xs text-textSubtle max-w-md mt-[2px]">
Choose how voice is converted to text
</p>
</div>
<div className="relative">
<button
onClick={() => setShowProviderDropdown(!showProviderDropdown)}
className="flex items-center gap-2 px-3 py-1.5 text-sm border border-borderSubtle rounded-md hover:border-borderStandard transition-colors text-textStandard bg-bgApp"
>
{getProviderLabel(settings.provider)}
<ChevronDown className="w-4 h-4" />
</button>
{showProviderDropdown && (
<div className="absolute right-0 mt-1 w-48 bg-bgApp border border-borderStandard rounded-md shadow-lg z-10">
<button
onClick={() => handleProviderChange('openai')}
disabled={!hasOpenAIKey}
className={`w-full px-3 py-2 text-left text-sm transition-colors first:rounded-t-md ${
hasOpenAIKey
? 'hover:bg-bgSubtle text-textStandard'
: 'text-textSubtle cursor-not-allowed'
}`}
>
OpenAI Whisper
{!hasOpenAIKey && <span className="text-xs ml-1">(not configured)</span>}
{settings.provider === 'openai' && <span className="float-right"></span>}
</button>
{/* ElevenLabs option */}
<button
onClick={() => handleProviderChange('elevenlabs')}
className="w-full px-3 py-2 text-left text-sm hover:bg-bgSubtle transition-colors text-textStandard last:rounded-b-md"
>
ElevenLabs
{settings.provider === 'elevenlabs' && <span className="float-right"></span>}
</button>
</div>
)}
</div>
</div>
{/* ElevenLabs API Key */}
{showElevenLabsKey && (
<div className="mb-4">
<div className="mb-2">
<h3 className="text-textStandard">ElevenLabs API Key</h3>
<p className="text-xs text-textSubtle max-w-md mt-[2px]">
Required for ElevenLabs voice recognition
{hasElevenLabsKey && <span className="text-green-600 ml-2">(Configured)</span>}
</p>
</div>
<Input
type="password"
value={elevenLabsApiKey}
onChange={(e) => handleElevenLabsKeyChange(e.target.value)}
onBlur={saveElevenLabsKey}
placeholder={
hasElevenLabsKey
? 'Enter new API key to update'
: 'Enter your ElevenLabs API key'
}
className="max-w-md"
disabled={isLoadingKey}
/>
</div>
)}
{/* Provider-specific information */}
<div className="mt-4 p-3 bg-bgSubtle rounded-md">
{settings.provider === 'openai' && (
<p className="text-xs text-textSubtle">
Uses OpenAI's Whisper API for high-quality transcription. Requires an OpenAI API
key configured in the Models section.
</p>
)}
{settings.provider === 'elevenlabs' && (
<div>
<p className="text-xs text-textSubtle">
Uses ElevenLabs speech-to-text API for high-quality transcription.
</p>
<p className="text-xs text-textSubtle mt-2">
<strong>Features:</strong>
</p>
<ul className="text-xs text-textSubtle ml-4 mt-1 list-disc">
<li>Advanced voice processing</li>
<li>High accuracy transcription</li>
<li>Multiple language support</li>
<li>Fast processing</li>
</ul>
<p className="text-xs text-textSubtle mt-2">
<strong>Note:</strong> Requires an ElevenLabs API key with speech-to-text
access.
</p>
</div>
)}
</div>
</>
)}
</div>
</section>
);
}