Careless whisper (#6877)
Co-authored-by: Douwe Osinga <douwe@squareup.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
Douwe Osinga
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -190,7 +190,7 @@ export type DetectProviderResponse = {
|
||||
provider_name: string;
|
||||
};
|
||||
|
||||
export type DictationProvider = 'openai' | 'elevenlabs';
|
||||
export type DictationProvider = 'openai' | 'elevenlabs' | 'groq' | 'local';
|
||||
|
||||
export type DictationProviderStatus = {
|
||||
/**
|
||||
@@ -219,6 +219,40 @@ export type DictationProviderStatus = {
|
||||
uses_provider_config: boolean;
|
||||
};
|
||||
|
||||
export type DownloadProgress = {
|
||||
/**
|
||||
* Bytes downloaded so far
|
||||
*/
|
||||
bytes_downloaded: number;
|
||||
/**
|
||||
* Error message if failed
|
||||
*/
|
||||
error?: string | null;
|
||||
/**
|
||||
* Estimated time remaining in seconds
|
||||
*/
|
||||
eta_seconds?: number | null;
|
||||
/**
|
||||
* Model ID being downloaded
|
||||
*/
|
||||
model_id: string;
|
||||
/**
|
||||
* Download progress percentage (0-100)
|
||||
*/
|
||||
progress_percent: number;
|
||||
/**
|
||||
* Download speed in bytes per second
|
||||
*/
|
||||
speed_bps?: number | null;
|
||||
status: DownloadStatus;
|
||||
/**
|
||||
* Total bytes to download
|
||||
*/
|
||||
total_bytes: number;
|
||||
};
|
||||
|
||||
export type DownloadStatus = 'downloading' | 'completed' | 'failed' | 'cancelled';
|
||||
|
||||
export type EmbeddedResource = {
|
||||
_meta?: {
|
||||
[key: string]: unknown;
|
||||
@@ -1311,6 +1345,28 @@ export type UpsertPermissionsQuery = {
|
||||
tool_permissions: Array<ToolPermission>;
|
||||
};
|
||||
|
||||
export type WhisperModelResponse = {
|
||||
/**
|
||||
* Description
|
||||
*/
|
||||
description: string;
|
||||
/**
|
||||
* Model identifier (e.g., "tiny", "base", "small")
|
||||
*/
|
||||
id: string;
|
||||
/**
|
||||
* Model file size in MB
|
||||
*/
|
||||
size_mb: number;
|
||||
/**
|
||||
* Download URL from HuggingFace
|
||||
*/
|
||||
url: string;
|
||||
} & {
|
||||
downloaded: boolean;
|
||||
recommended: boolean;
|
||||
};
|
||||
|
||||
export type WindowProps = {
|
||||
height: number;
|
||||
resizable: boolean;
|
||||
@@ -2522,6 +2578,124 @@ export type GetDictationConfigResponses = {
|
||||
|
||||
export type GetDictationConfigResponse = GetDictationConfigResponses[keyof GetDictationConfigResponses];
|
||||
|
||||
export type ListModelsData = {
|
||||
body?: never;
|
||||
path?: never;
|
||||
query?: never;
|
||||
url: '/dictation/models';
|
||||
};
|
||||
|
||||
export type ListModelsResponses = {
|
||||
/**
|
||||
* List of available Whisper models
|
||||
*/
|
||||
200: Array<WhisperModelResponse>;
|
||||
};
|
||||
|
||||
export type ListModelsResponse = ListModelsResponses[keyof ListModelsResponses];
|
||||
|
||||
export type DeleteModelData = {
|
||||
body?: never;
|
||||
path: {
|
||||
model_id: string;
|
||||
};
|
||||
query?: never;
|
||||
url: '/dictation/models/{model_id}';
|
||||
};
|
||||
|
||||
export type DeleteModelErrors = {
|
||||
/**
|
||||
* Model not found or not downloaded
|
||||
*/
|
||||
404: unknown;
|
||||
/**
|
||||
* Failed to delete model
|
||||
*/
|
||||
500: unknown;
|
||||
};
|
||||
|
||||
export type DeleteModelResponses = {
|
||||
/**
|
||||
* Model deleted
|
||||
*/
|
||||
200: unknown;
|
||||
};
|
||||
|
||||
export type CancelDownloadData = {
|
||||
body?: never;
|
||||
path: {
|
||||
model_id: string;
|
||||
};
|
||||
query?: never;
|
||||
url: '/dictation/models/{model_id}/download';
|
||||
};
|
||||
|
||||
export type CancelDownloadErrors = {
|
||||
/**
|
||||
* Download not found
|
||||
*/
|
||||
404: unknown;
|
||||
};
|
||||
|
||||
export type CancelDownloadResponses = {
|
||||
/**
|
||||
* Download cancelled
|
||||
*/
|
||||
200: unknown;
|
||||
};
|
||||
|
||||
export type GetDownloadProgressData = {
|
||||
body?: never;
|
||||
path: {
|
||||
model_id: string;
|
||||
};
|
||||
query?: never;
|
||||
url: '/dictation/models/{model_id}/download';
|
||||
};
|
||||
|
||||
export type GetDownloadProgressErrors = {
|
||||
/**
|
||||
* Download not found
|
||||
*/
|
||||
404: unknown;
|
||||
};
|
||||
|
||||
export type GetDownloadProgressResponses = {
|
||||
/**
|
||||
* Download progress
|
||||
*/
|
||||
200: DownloadProgress;
|
||||
};
|
||||
|
||||
export type GetDownloadProgressResponse = GetDownloadProgressResponses[keyof GetDownloadProgressResponses];
|
||||
|
||||
export type DownloadModelData = {
|
||||
body?: never;
|
||||
path: {
|
||||
model_id: string;
|
||||
};
|
||||
query?: never;
|
||||
url: '/dictation/models/{model_id}/download';
|
||||
};
|
||||
|
||||
export type DownloadModelErrors = {
|
||||
/**
|
||||
* Download already in progress
|
||||
*/
|
||||
400: unknown;
|
||||
/**
|
||||
* Internal server error
|
||||
*/
|
||||
500: unknown;
|
||||
};
|
||||
|
||||
export type DownloadModelResponses = {
|
||||
/**
|
||||
* Download started
|
||||
*/
|
||||
202: unknown;
|
||||
};
|
||||
|
||||
export type TranscribeDictationData = {
|
||||
body: TranscribeRequest;
|
||||
path?: never;
|
||||
@@ -2539,7 +2713,7 @@ export type TranscribeDictationErrors = {
|
||||
*/
|
||||
401: unknown;
|
||||
/**
|
||||
* DictationProvider not configured
|
||||
* Provider not configured
|
||||
*/
|
||||
412: unknown;
|
||||
/**
|
||||
@@ -2555,7 +2729,7 @@ export type TranscribeDictationErrors = {
|
||||
*/
|
||||
500: unknown;
|
||||
/**
|
||||
* DictationProvider API error
|
||||
* Provider API error
|
||||
*/
|
||||
502: unknown;
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
// AudioWorklet processor for capturing audio samples
|
||||
class AudioCaptureProcessor extends AudioWorkletProcessor {
|
||||
process(inputs) {
|
||||
const ch = inputs[0]?.[0];
|
||||
if (ch?.length > 0) {
|
||||
this.port.postMessage(new Float32Array(ch));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
registerProcessor('audio-capture', AudioCaptureProcessor);
|
||||
@@ -260,16 +260,37 @@ export default function ChatInput({
|
||||
isTranscribing,
|
||||
startRecording,
|
||||
stopRecording,
|
||||
recordingDuration,
|
||||
estimatedSize,
|
||||
} = useAudioRecorder({
|
||||
onTranscription: (text) => {
|
||||
trackVoiceDictation('transcribed');
|
||||
// Append transcribed text to the current input
|
||||
const newValue = displayValue.trim() ? `${displayValue.trim()} ${text}` : text;
|
||||
|
||||
let filteredText = text.replace(/\([^)]*\)/g, '').trim();
|
||||
|
||||
if (!filteredText) {
|
||||
return;
|
||||
}
|
||||
|
||||
const shouldAutoSubmit = /\bsubmit[.,!?;'"\s]*$/i.test(filteredText);
|
||||
|
||||
const cleanedText = shouldAutoSubmit
|
||||
? filteredText.replace(/\bsubmit[.,!?;'"\s]*$/i, '').trim()
|
||||
: filteredText;
|
||||
|
||||
const newValue = displayValue.trim() && cleanedText
|
||||
? `${displayValue.trim()} ${cleanedText}`
|
||||
: displayValue.trim() || cleanedText;
|
||||
|
||||
setDisplayValue(newValue);
|
||||
setValue(newValue);
|
||||
textAreaRef.current?.focus();
|
||||
|
||||
if (shouldAutoSubmit && newValue.trim()) {
|
||||
trackVoiceDictation('auto_submit');
|
||||
setTimeout(() => {
|
||||
performSubmit(newValue);
|
||||
}, 100);
|
||||
} else {
|
||||
textAreaRef.current?.focus();
|
||||
}
|
||||
},
|
||||
onError: (message) => {
|
||||
const errorType = 'DictationError';
|
||||
@@ -907,8 +928,8 @@ export default function ChatInput({
|
||||
]
|
||||
);
|
||||
|
||||
|
||||
const handleKeyDown = (evt: React.KeyboardEvent<HTMLTextAreaElement>) => {
|
||||
// If mention popover is open, handle arrow keys and enter
|
||||
if (mentionPopover.isOpen && mentionPopoverRef.current) {
|
||||
if (evt.key === 'ArrowDown') {
|
||||
evt.preventDefault();
|
||||
@@ -940,7 +961,6 @@ export default function ChatInput({
|
||||
}
|
||||
}
|
||||
|
||||
// Handle history navigation first
|
||||
handleHistoryNavigation(evt);
|
||||
|
||||
if (evt.key === 'Enter') {
|
||||
@@ -1221,23 +1241,15 @@ export default function ChatInput({
|
||||
onBlur={() => setIsFocused(false)}
|
||||
ref={textAreaRef}
|
||||
rows={1}
|
||||
readOnly={isRecording}
|
||||
style={{
|
||||
minHeight: `${minTextareaHeight}px`,
|
||||
maxHeight: `${maxHeight}px`,
|
||||
overflowY: 'auto',
|
||||
opacity: isRecording ? 0 : 1,
|
||||
paddingRight: dictationProvider ? '180px' : '120px',
|
||||
}}
|
||||
className="w-full outline-none border-none focus:ring-0 bg-transparent px-3 pt-3 pb-1.5 text-sm resize-none text-textStandard placeholder:text-textPlaceholder"
|
||||
/>
|
||||
{isRecording && (
|
||||
<div className="absolute inset-0 flex items-center pl-4 pr-32 pt-3 pb-1.5">
|
||||
<div className="flex items-center gap-2 text-textSubtle">
|
||||
<span className="inline-block w-2 h-2 bg-red-500 rounded-full animate-pulse" />
|
||||
<span>Recording...</span>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Inline action buttons - absolutely positioned on the right */}
|
||||
<div className="absolute right-2 top-1/2 -translate-y-1/2 flex items-center gap-1">
|
||||
@@ -1272,37 +1284,52 @@ export default function ChatInput({
|
||||
ElevenLabs API key is not configured. Set it up in <b>Settings</b> {'>'}{' '}
|
||||
<b>Chat</b> {'>'} <b>Voice Dictation.</b>
|
||||
</p>
|
||||
) : dictationProvider === 'local' ? (
|
||||
<p>
|
||||
Local Whisper model not found. Download a model in{' '}
|
||||
<b>Settings > Dictation > Local (Offline)</b>
|
||||
</p>
|
||||
) : (
|
||||
<p>Dictation provider is not properly configured.</p>
|
||||
)}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
) : (
|
||||
<Button
|
||||
type="button"
|
||||
size="sm"
|
||||
shape="round"
|
||||
variant="outline"
|
||||
onClick={() => {
|
||||
if (isRecording) {
|
||||
trackVoiceDictation('stop', Math.floor(recordingDuration));
|
||||
stopRecording();
|
||||
} else {
|
||||
trackVoiceDictation('start');
|
||||
startRecording();
|
||||
}
|
||||
}}
|
||||
disabled={isTranscribing}
|
||||
className={`rounded-full px-6 py-2 ${
|
||||
isRecording
|
||||
? 'bg-red-500 text-white hover:bg-red-600 border-red-500'
|
||||
: isTranscribing
|
||||
? 'bg-slate-600 text-white cursor-not-allowed animate-pulse border-slate-600'
|
||||
: 'bg-slate-600 text-white hover:bg-slate-700 border-slate-600'
|
||||
}`}
|
||||
>
|
||||
<Microphone />
|
||||
</Button>
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<Button
|
||||
type="button"
|
||||
size="sm"
|
||||
shape="round"
|
||||
variant="outline"
|
||||
onClick={() => {
|
||||
if (isRecording) {
|
||||
trackVoiceDictation('stop');
|
||||
stopRecording();
|
||||
} else {
|
||||
trackVoiceDictation('start');
|
||||
startRecording();
|
||||
}
|
||||
}}
|
||||
disabled={isTranscribing}
|
||||
className={`rounded-full px-6 py-2 ${
|
||||
isRecording
|
||||
? 'bg-red-500 text-white hover:bg-red-600 border-red-500'
|
||||
: isTranscribing
|
||||
? 'bg-slate-600 text-white cursor-not-allowed animate-pulse border-slate-600'
|
||||
: 'bg-slate-600 text-white hover:bg-slate-700 border-slate-600'
|
||||
}`}
|
||||
>
|
||||
<Microphone />
|
||||
</Button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>
|
||||
<p>
|
||||
Voice dictation
|
||||
{isRecording ? '' : ' • Say "submit" to send'}
|
||||
</p>
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
@@ -1349,20 +1376,23 @@ export default function ChatInput({
|
||||
{/* Recording/transcribing status indicator - positioned above the button row */}
|
||||
{(isRecording || isTranscribing) && (
|
||||
<div className="absolute right-0 -top-8 bg-background-default px-2 py-1 rounded text-xs whitespace-nowrap shadow-md border border-borderSubtle">
|
||||
{isTranscribing ? (
|
||||
<span className="text-blue-500 flex items-center gap-1">
|
||||
<span className="inline-block w-2 h-2 bg-blue-500 rounded-full animate-pulse" />
|
||||
Transcribing...
|
||||
</span>
|
||||
) : (
|
||||
<span
|
||||
className={`flex items-center gap-2 ${estimatedSize > 20 ? 'text-orange-500' : 'text-textSubtle'}`}
|
||||
>
|
||||
<span className="inline-block w-2 h-2 bg-red-500 rounded-full animate-pulse" />
|
||||
{Math.floor(recordingDuration)}s • ~{estimatedSize.toFixed(1)}MB
|
||||
{estimatedSize > 20 && <span className="text-xs">(near 25MB limit)</span>}
|
||||
</span>
|
||||
)}
|
||||
<span className="flex items-center gap-2">
|
||||
{isRecording && (
|
||||
<span className="flex items-center gap-1 text-textSubtle">
|
||||
<span className="inline-block w-2 h-2 bg-red-500 rounded-full animate-pulse" />
|
||||
Listening
|
||||
</span>
|
||||
)}
|
||||
{isRecording && isTranscribing && (
|
||||
<span className="text-textSubtle">•</span>
|
||||
)}
|
||||
{isTranscribing && (
|
||||
<span className="flex items-center gap-1 text-blue-500">
|
||||
<span className="inline-block w-2 h-2 bg-blue-500 rounded-full animate-pulse" />
|
||||
Transcribing
|
||||
</span>
|
||||
)}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
@@ -5,6 +5,7 @@ import { useConfig } from '../../ConfigContext';
|
||||
import { Input } from '../../ui/input';
|
||||
import { Button } from '../../ui/button';
|
||||
import { trackSettingToggled } from '../../../utils/analytics';
|
||||
import { LocalModelManager } from './LocalModelManager';
|
||||
|
||||
export const DictationSettings = () => {
|
||||
const [provider, setProvider] = useState<DictationProvider | null>(null);
|
||||
@@ -163,11 +164,11 @@ export const DictationSettings = () => {
|
||||
|
||||
{provider && providerStatuses[provider] && (
|
||||
<>
|
||||
<div className="py-2 px-2">
|
||||
<p className="text-xs text-text-muted">{providerStatuses[provider].description}</p>
|
||||
</div>
|
||||
|
||||
{providerStatuses[provider].uses_provider_config ? (
|
||||
{provider === 'local' ? (
|
||||
<div className="py-2 px-2">
|
||||
<LocalModelManager />
|
||||
</div>
|
||||
) : providerStatuses[provider].uses_provider_config ? (
|
||||
<div className="py-2 px-2 bg-background-subtle rounded-lg">
|
||||
{!providerStatuses[provider].configured ? (
|
||||
<p className="text-xs text-text-muted">
|
||||
|
||||
@@ -0,0 +1,305 @@
|
||||
import { useState, useEffect } from 'react';
|
||||
import { Download, Trash2, X, Check, ChevronDown, ChevronUp } from 'lucide-react';
|
||||
import { Button } from '../../ui/button';
|
||||
import { useConfig } from '../../ConfigContext';
|
||||
import {
|
||||
listModels,
|
||||
downloadModel,
|
||||
getDownloadProgress,
|
||||
cancelDownload as cancelDownloadApi,
|
||||
deleteModel as deleteModelApi,
|
||||
type WhisperModelResponse,
|
||||
type DownloadProgress,
|
||||
} from '../../../api';
|
||||
|
||||
const LOCAL_WHISPER_MODEL_CONFIG_KEY = 'LOCAL_WHISPER_MODEL';
|
||||
|
||||
const formatBytes = (bytes: number): string => {
|
||||
if (bytes < 1024) return `${bytes}B`;
|
||||
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(0)}KB`;
|
||||
if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(0)}MB`;
|
||||
return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)}GB`;
|
||||
};
|
||||
|
||||
const capitalize = (str: string): string => {
|
||||
return str.charAt(0).toUpperCase() + str.slice(1);
|
||||
};
|
||||
|
||||
export const LocalModelManager = () => {
|
||||
const [models, setModels] = useState<WhisperModelResponse[]>([]);
|
||||
const [downloads, setDownloads] = useState<Map<string, DownloadProgress>>(new Map());
|
||||
const [selectedModelId, setSelectedModelId] = useState<string | null>(null);
|
||||
const [showAllModels, setShowAllModels] = useState(false);
|
||||
const { read, upsert } = useConfig();
|
||||
|
||||
useEffect(() => {
|
||||
loadModels();
|
||||
loadSelectedModel();
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, []);
|
||||
|
||||
// Determine if we should show all models by default (if non-recommended models are downloaded)
|
||||
useEffect(() => {
|
||||
if (models.length === 0) return;
|
||||
|
||||
const hasDownloadedNonRecommended = models.some(
|
||||
(model) => model.downloaded && !model.recommended
|
||||
);
|
||||
|
||||
if (hasDownloadedNonRecommended && !showAllModels) {
|
||||
setShowAllModels(true);
|
||||
}
|
||||
}, [models, showAllModels]);
|
||||
|
||||
const loadSelectedModel = async () => {
|
||||
try {
|
||||
const value = await read(LOCAL_WHISPER_MODEL_CONFIG_KEY, false);
|
||||
if (value && typeof value === 'string') {
|
||||
setSelectedModelId(value);
|
||||
} else {
|
||||
setSelectedModelId(null);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to load selected model:', error);
|
||||
setSelectedModelId(null);
|
||||
}
|
||||
};
|
||||
|
||||
const selectModel = async (modelId: string) => {
|
||||
await upsert(LOCAL_WHISPER_MODEL_CONFIG_KEY, modelId, false);
|
||||
setSelectedModelId(modelId);
|
||||
};
|
||||
|
||||
const loadModels = async () => {
|
||||
try {
|
||||
const response = await listModels();
|
||||
if (response.data) {
|
||||
setModels(response.data);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to load models:', error);
|
||||
}
|
||||
};
|
||||
|
||||
const startDownload = async (modelId: string) => {
|
||||
try {
|
||||
await downloadModel({ path: { model_id: modelId } });
|
||||
pollDownloadProgress(modelId);
|
||||
} catch (error) {
|
||||
console.error('Failed to start download:', error);
|
||||
}
|
||||
};
|
||||
|
||||
const pollDownloadProgress = (modelId: string) => {
|
||||
const interval = setInterval(async () => {
|
||||
try {
|
||||
const response = await getDownloadProgress({ path: { model_id: modelId } });
|
||||
if (response.data) {
|
||||
const progress = response.data;
|
||||
setDownloads((prev) => new Map(prev).set(modelId, progress));
|
||||
|
||||
if (progress.status === 'completed') {
|
||||
clearInterval(interval);
|
||||
await loadModels(); // Refresh model list
|
||||
// Backend auto-selects, but also update frontend state
|
||||
await loadSelectedModel();
|
||||
} else if (progress.status === 'failed') {
|
||||
clearInterval(interval);
|
||||
await loadModels();
|
||||
}
|
||||
} else {
|
||||
clearInterval(interval);
|
||||
}
|
||||
} catch {
|
||||
clearInterval(interval);
|
||||
}
|
||||
}, 500);
|
||||
};
|
||||
|
||||
const cancelDownload = async (modelId: string) => {
|
||||
try {
|
||||
await cancelDownloadApi({ path: { model_id: modelId } });
|
||||
setDownloads((prev) => {
|
||||
const next = new Map(prev);
|
||||
next.delete(modelId);
|
||||
return next;
|
||||
});
|
||||
loadModels();
|
||||
} catch (error) {
|
||||
console.error('Failed to cancel download:', error);
|
||||
}
|
||||
};
|
||||
|
||||
const deleteModel = async (modelId: string) => {
|
||||
if (!window.confirm('Delete this model? You can re-download it later.')) return;
|
||||
|
||||
try {
|
||||
await deleteModelApi({ path: { model_id: modelId } });
|
||||
if (selectedModelId === modelId) {
|
||||
await upsert(LOCAL_WHISPER_MODEL_CONFIG_KEY, '', false);
|
||||
setSelectedModelId(null);
|
||||
}
|
||||
loadModels();
|
||||
} catch (error) {
|
||||
console.error('Failed to delete model:', error);
|
||||
}
|
||||
};
|
||||
|
||||
const displayedModels = showAllModels ? models : models.filter((m) => m.recommended);
|
||||
const hasNonRecommendedModels = models.some((m) => !m.recommended);
|
||||
|
||||
return (
|
||||
<div className="space-y-3">
|
||||
<div className="text-xs text-text-muted mb-2">
|
||||
<p>Supports GPU acceleration (CUDA for NVIDIA, Metal for Apple Silicon). GPU features must be enabled at build time for hardware acceleration.</p>
|
||||
</div>
|
||||
|
||||
<div className="space-y-2">
|
||||
{displayedModels.map((model) => {
|
||||
const progress = downloads.get(model.id);
|
||||
const isDownloading = progress?.status === 'downloading';
|
||||
const isSelected = selectedModelId === model.id;
|
||||
const canSelect = model.downloaded && !isDownloading;
|
||||
|
||||
return (
|
||||
<div
|
||||
key={model.id}
|
||||
className={`border rounded-lg p-3 transition-colors ${
|
||||
isSelected
|
||||
? 'border-accent-primary bg-accent-primary/5'
|
||||
: 'border-border-subtle bg-background-default hover:border-border-default'
|
||||
}`}
|
||||
>
|
||||
<div className="flex items-start justify-between gap-3">
|
||||
<div className="flex-1 min-w-0">
|
||||
<div className="flex items-center gap-2">
|
||||
{canSelect && (
|
||||
<input
|
||||
type="radio"
|
||||
checked={isSelected}
|
||||
onChange={() => selectModel(model.id)}
|
||||
className="cursor-pointer"
|
||||
/>
|
||||
)}
|
||||
<h4 className="text-sm font-medium text-text-default">
|
||||
{capitalize(model.id)}
|
||||
</h4>
|
||||
<span className="text-xs text-text-muted">
|
||||
{model.size_mb}MB
|
||||
</span>
|
||||
{model.recommended && (
|
||||
<span className="text-xs bg-blue-500 text-white px-2 py-0.5 rounded">
|
||||
Recommended
|
||||
</span>
|
||||
)}
|
||||
{isSelected && (
|
||||
<span className="text-xs bg-accent-primary text-white px-2 py-0.5 rounded">
|
||||
Active
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<p className="text-xs text-text-muted mt-1">
|
||||
{model.description}
|
||||
</p>
|
||||
{model.recommended && (
|
||||
<p className="text-xs text-blue-600 mt-1 font-medium">
|
||||
Recommended for your hardware
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="flex items-center gap-2">
|
||||
{model.downloaded ? (
|
||||
<>
|
||||
<div className="flex items-center gap-1 text-xs text-green-600">
|
||||
<Check className="w-4 h-4" />
|
||||
<span>Downloaded</span>
|
||||
</div>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={() => deleteModel(model.id)}
|
||||
className="text-destructive hover:text-destructive"
|
||||
>
|
||||
<Trash2 className="w-4 h-4" />
|
||||
</Button>
|
||||
</>
|
||||
) : isDownloading ? (
|
||||
<>
|
||||
<div className="text-xs text-text-muted min-w-[60px]">
|
||||
{progress.progress_percent.toFixed(0)}%
|
||||
</div>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={() => cancelDownload(model.id)}
|
||||
>
|
||||
<X className="w-4 h-4" />
|
||||
</Button>
|
||||
</>
|
||||
) : (
|
||||
<Button variant="outline" size="sm" onClick={() => startDownload(model.id)}>
|
||||
<Download className="w-4 h-4 mr-1" />
|
||||
Download
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{isDownloading && progress && (
|
||||
<div className="mt-2 space-y-1">
|
||||
<div className="w-full bg-background-subtle rounded-full h-1.5">
|
||||
<div
|
||||
className="bg-accent-primary h-1.5 rounded-full transition-all"
|
||||
style={{ width: `${progress.progress_percent}%` }}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex justify-between text-xs text-text-muted">
|
||||
<span>
|
||||
{formatBytes(progress.bytes_downloaded)} / {formatBytes(progress.total_bytes)}
|
||||
</span>
|
||||
{progress.speed_bps && (
|
||||
<span>{formatBytes(progress.speed_bps)}/s</span>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{progress?.status === 'failed' && progress.error && (
|
||||
<div className="mt-2 text-xs text-destructive">{progress.error}</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
|
||||
{hasNonRecommendedModels && (
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={() => setShowAllModels(!showAllModels)}
|
||||
className="w-full text-text-muted hover:text-text-default"
|
||||
>
|
||||
{showAllModels ? (
|
||||
<>
|
||||
<ChevronUp className="w-4 h-4 mr-1" />
|
||||
Show recommended only
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<ChevronDown className="w-4 h-4 mr-1" />
|
||||
Show all models
|
||||
</>
|
||||
)}
|
||||
</Button>
|
||||
)}
|
||||
|
||||
{models.length === 0 && (
|
||||
<div className="text-center py-6 text-text-muted text-sm">
|
||||
No models available
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -8,136 +8,192 @@ interface UseAudioRecorderOptions {
|
||||
onError: (message: string) => void;
|
||||
}
|
||||
|
||||
const MAX_AUDIO_SIZE_MB = 25;
|
||||
const MAX_RECORDING_DURATION_SECONDS = 10 * 60;
|
||||
const SAMPLE_RATE = 16000;
|
||||
const SILENCE_MS = 800;
|
||||
const MIN_SPEECH_MS = 200;
|
||||
// RMS threshold for speech detection. Audio samples are Float32 in [-1, 1] range.
|
||||
// 0.015 (~1.5% of full-scale) distinguishes normal speech from background noise
|
||||
// without clipping early speech onsets. Determined empirically for 16kHz mono input.
|
||||
const RMS_THRESHOLD = 0.015;
|
||||
|
||||
// Import the worklet module - Vite will handle this correctly
|
||||
const WORKLET_URL = new URL('../audio-capture-worklet.js', import.meta.url).href;
|
||||
|
||||
function encodeWav(samples: Float32Array, sampleRate: number): ArrayBuffer {
|
||||
const buf = new ArrayBuffer(44 + samples.length * 2);
|
||||
const v = new DataView(buf);
|
||||
const w = (o: number, s: string) => {
|
||||
for (let i = 0; i < s.length; i++) v.setUint8(o + i, s.charCodeAt(i));
|
||||
};
|
||||
w(0, 'RIFF');
|
||||
v.setUint32(4, 36 + samples.length * 2, true);
|
||||
w(8, 'WAVE');
|
||||
w(12, 'fmt ');
|
||||
v.setUint32(16, 16, true);
|
||||
v.setUint16(20, 1, true);
|
||||
v.setUint16(22, 1, true);
|
||||
v.setUint32(24, sampleRate, true);
|
||||
v.setUint32(28, sampleRate * 2, true);
|
||||
v.setUint16(32, 2, true);
|
||||
v.setUint16(34, 16, true);
|
||||
w(36, 'data');
|
||||
v.setUint32(40, samples.length * 2, true);
|
||||
let o = 44;
|
||||
for (let i = 0; i < samples.length; i++) {
|
||||
const s = Math.max(-1, Math.min(1, samples[i]));
|
||||
v.setInt16(o, s < 0 ? s * 0x8000 : s * 0x7fff, true);
|
||||
o += 2;
|
||||
}
|
||||
return buf;
|
||||
}
|
||||
|
||||
function rms(samples: Float32Array): number {
|
||||
let sum = 0;
|
||||
for (let i = 0; i < samples.length; i++) sum += samples[i] * samples[i];
|
||||
return Math.sqrt(sum / samples.length);
|
||||
}
|
||||
|
||||
function blobToBase64(blob: Blob): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const r = new FileReader();
|
||||
r.onloadend = () => resolve((r.result as string).split(',')[1]);
|
||||
r.onerror = reject;
|
||||
r.readAsDataURL(blob);
|
||||
});
|
||||
}
|
||||
|
||||
export const useAudioRecorder = ({ onTranscription, onError }: UseAudioRecorderOptions) => {
|
||||
const [isRecording, setIsRecording] = useState(false);
|
||||
const [isTranscribing, setIsTranscribing] = useState(false);
|
||||
const [recordingDuration, setRecordingDuration] = useState(0);
|
||||
const [estimatedSize, setEstimatedSize] = useState(0);
|
||||
const [isEnabled, setIsEnabled] = useState(false);
|
||||
const [provider, setProvider] = useState<DictationProvider | null>(null);
|
||||
|
||||
const { read } = useConfig();
|
||||
const { read, config } = useConfig();
|
||||
|
||||
const mediaRecorderRef = useRef<MediaRecorder | null>(null);
|
||||
const audioChunksRef = useRef<Blob[]>([]);
|
||||
const audioContextRef = useRef<AudioContext | null>(null);
|
||||
const streamRef = useRef<MediaStream | null>(null);
|
||||
const durationIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||
|
||||
// VAD state (all refs to avoid re-render/stale closure issues)
|
||||
const samplesRef = useRef<Float32Array[]>([]);
|
||||
const isSpeakingRef = useRef(false);
|
||||
const silenceStartRef = useRef(0);
|
||||
const speechStartRef = useRef(0);
|
||||
const pendingTranscriptions = useRef(0);
|
||||
const providerRef = useRef(provider);
|
||||
providerRef.current = provider;
|
||||
|
||||
// Keep callback refs fresh
|
||||
const onTranscriptionRef = useRef(onTranscription);
|
||||
onTranscriptionRef.current = onTranscription;
|
||||
const onErrorRef = useRef(onError);
|
||||
onErrorRef.current = onError;
|
||||
|
||||
useEffect(() => {
|
||||
const checkProviderConfig = async () => {
|
||||
const check = async () => {
|
||||
try {
|
||||
const providerValue = await read('voice_dictation_provider', false);
|
||||
const preferredProvider = (providerValue as DictationProvider) || null;
|
||||
|
||||
if (!preferredProvider) {
|
||||
const val = await read('voice_dictation_provider', false);
|
||||
const pref = (val as DictationProvider) || null;
|
||||
if (!pref) {
|
||||
setIsEnabled(false);
|
||||
setProvider(null);
|
||||
return;
|
||||
}
|
||||
|
||||
const audioConfigResponse = await getDictationConfig();
|
||||
const providerStatus = audioConfigResponse.data?.[preferredProvider];
|
||||
|
||||
setIsEnabled(!!providerStatus?.configured);
|
||||
setProvider(preferredProvider);
|
||||
const resp = await getDictationConfig();
|
||||
setIsEnabled(!!resp.data?.[pref]?.configured);
|
||||
setProvider(pref);
|
||||
} catch (error) {
|
||||
console.error('Error checking audio config:', error);
|
||||
console.error('Failed to check dictation config:', error);
|
||||
setIsEnabled(false);
|
||||
setProvider(null);
|
||||
}
|
||||
};
|
||||
check();
|
||||
}, [read, config]);
|
||||
|
||||
checkProviderConfig();
|
||||
}, [read]);
|
||||
const transcribeChunk = useCallback(async (samples: Float32Array) => {
|
||||
const prov = providerRef.current;
|
||||
if (!prov) return;
|
||||
|
||||
pendingTranscriptions.current++;
|
||||
setIsTranscribing(true);
|
||||
|
||||
try {
|
||||
const wav = new Blob([encodeWav(samples, SAMPLE_RATE)], { type: 'audio/wav' });
|
||||
const base64 = await blobToBase64(wav);
|
||||
const result = await transcribeDictation({
|
||||
body: { audio: base64, mime_type: 'audio/wav', provider: prov },
|
||||
throwOnError: true,
|
||||
});
|
||||
if (result.data?.text) {
|
||||
onTranscriptionRef.current(result.data.text);
|
||||
}
|
||||
} catch (error) {
|
||||
onErrorRef.current(errorMessage(error));
|
||||
} finally {
|
||||
pendingTranscriptions.current--;
|
||||
if (pendingTranscriptions.current === 0) setIsTranscribing(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const flush = useCallback(() => {
|
||||
const chunks = samplesRef.current;
|
||||
if (chunks.length === 0) return;
|
||||
|
||||
const total = chunks.reduce((n, c) => n + c.length, 0);
|
||||
const merged = new Float32Array(total);
|
||||
let off = 0;
|
||||
for (const c of chunks) {
|
||||
merged.set(c, off);
|
||||
off += c.length;
|
||||
}
|
||||
samplesRef.current = [];
|
||||
transcribeChunk(merged);
|
||||
}, [transcribeChunk]);
|
||||
|
||||
const flushRef = useRef(flush);
|
||||
flushRef.current = flush;
|
||||
|
||||
const handleSamples = useCallback((samples: Float32Array) => {
|
||||
const now = Date.now();
|
||||
|
||||
if (rms(samples) > RMS_THRESHOLD) {
|
||||
if (!isSpeakingRef.current) {
|
||||
isSpeakingRef.current = true;
|
||||
speechStartRef.current = now;
|
||||
}
|
||||
silenceStartRef.current = 0;
|
||||
samplesRef.current.push(new Float32Array(samples));
|
||||
} else if (isSpeakingRef.current) {
|
||||
samplesRef.current.push(new Float32Array(samples));
|
||||
|
||||
if (silenceStartRef.current === 0) {
|
||||
silenceStartRef.current = now;
|
||||
} else if (now - silenceStartRef.current > SILENCE_MS) {
|
||||
if (now - speechStartRef.current > MIN_SPEECH_MS) {
|
||||
flushRef.current();
|
||||
} else {
|
||||
samplesRef.current = [];
|
||||
}
|
||||
isSpeakingRef.current = false;
|
||||
silenceStartRef.current = 0;
|
||||
}
|
||||
}
|
||||
}, []);
|
||||
|
||||
const stopRecording = useCallback(() => {
|
||||
if (isSpeakingRef.current && samplesRef.current.length > 0) {
|
||||
flushRef.current();
|
||||
}
|
||||
isSpeakingRef.current = false;
|
||||
silenceStartRef.current = 0;
|
||||
|
||||
audioContextRef.current?.close();
|
||||
audioContextRef.current = null;
|
||||
streamRef.current?.getTracks().forEach((t) => t.stop());
|
||||
streamRef.current = null;
|
||||
setIsRecording(false);
|
||||
|
||||
if (mediaRecorderRef.current && mediaRecorderRef.current.state !== 'inactive') {
|
||||
mediaRecorderRef.current.stop();
|
||||
}
|
||||
|
||||
if (durationIntervalRef.current) {
|
||||
clearInterval(durationIntervalRef.current);
|
||||
durationIntervalRef.current = null;
|
||||
}
|
||||
|
||||
if (streamRef.current) {
|
||||
streamRef.current.getTracks().forEach((track) => track.stop());
|
||||
streamRef.current = null;
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (durationIntervalRef.current) {
|
||||
clearInterval(durationIntervalRef.current);
|
||||
}
|
||||
if (streamRef.current) {
|
||||
streamRef.current.getTracks().forEach((track) => track.stop());
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
const transcribeAudio = useCallback(
|
||||
async (audioBlob: Blob) => {
|
||||
if (!provider) {
|
||||
onError('No transcription provider configured');
|
||||
return;
|
||||
}
|
||||
|
||||
setIsTranscribing(true);
|
||||
|
||||
try {
|
||||
const sizeMB = audioBlob.size / (1024 * 1024);
|
||||
if (sizeMB > MAX_AUDIO_SIZE_MB) {
|
||||
onError(
|
||||
`Audio file too large (${sizeMB.toFixed(1)}MB). Maximum size is ${MAX_AUDIO_SIZE_MB}MB.`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const reader = new FileReader();
|
||||
const base64Audio = await new Promise<string>((resolve, reject) => {
|
||||
reader.onloadend = () => {
|
||||
const base64 = reader.result as string;
|
||||
resolve(base64.split(',')[1]);
|
||||
};
|
||||
reader.onerror = reject;
|
||||
reader.readAsDataURL(audioBlob);
|
||||
});
|
||||
|
||||
const mimeType = audioBlob.type;
|
||||
if (!mimeType) {
|
||||
throw new Error('Unable to determine audio format');
|
||||
}
|
||||
|
||||
const result = await transcribeDictation({
|
||||
body: {
|
||||
audio: base64Audio,
|
||||
mime_type: mimeType,
|
||||
provider: provider,
|
||||
},
|
||||
throwOnError: true,
|
||||
});
|
||||
|
||||
if (result.data?.text) {
|
||||
onTranscription(result.data.text);
|
||||
}
|
||||
} catch (error) {
|
||||
onError(errorMessage(error));
|
||||
} finally {
|
||||
setIsTranscribing(false);
|
||||
setRecordingDuration(0);
|
||||
setEstimatedSize(0);
|
||||
}
|
||||
},
|
||||
[provider, onTranscription, onError]
|
||||
);
|
||||
|
||||
const startRecording = useCallback(async () => {
|
||||
if (!isEnabled) {
|
||||
onError('Voice dictation is not enabled');
|
||||
@@ -146,73 +202,47 @@ export const useAudioRecorder = ({ onTranscription, onError }: UseAudioRecorderO
|
||||
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
autoGainControl: true,
|
||||
},
|
||||
audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true },
|
||||
});
|
||||
streamRef.current = stream;
|
||||
|
||||
const supportedTypes = ['audio/webm;codecs=opus', 'audio/webm', 'audio/mp4', 'audio/wav'];
|
||||
const mimeType = supportedTypes.find((type) => MediaRecorder.isTypeSupported(type)) || '';
|
||||
const ctx = new AudioContext({ sampleRate: SAMPLE_RATE });
|
||||
audioContextRef.current = ctx;
|
||||
|
||||
const mediaRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : {});
|
||||
mediaRecorderRef.current = mediaRecorder;
|
||||
audioChunksRef.current = [];
|
||||
await ctx.audioWorklet.addModule(WORKLET_URL);
|
||||
|
||||
const startTime = Date.now();
|
||||
durationIntervalRef.current = setInterval(() => {
|
||||
const elapsed = (Date.now() - startTime) / 1000;
|
||||
setRecordingDuration(elapsed);
|
||||
const source = ctx.createMediaStreamSource(stream);
|
||||
// eslint-disable-next-line no-undef
|
||||
const worklet = new AudioWorkletNode(ctx, 'audio-capture');
|
||||
|
||||
const estimatedSizeMB = (elapsed * 128 * 1024) / (8 * 1024 * 1024);
|
||||
setEstimatedSize(estimatedSizeMB);
|
||||
worklet.port.onmessage = (e: MessageEvent<Float32Array>) => handleSamples(e.data);
|
||||
|
||||
if (elapsed >= MAX_RECORDING_DURATION_SECONDS) {
|
||||
stopRecording();
|
||||
onError(
|
||||
`Maximum recording duration (${MAX_RECORDING_DURATION_SECONDS / 60} minutes) reached`
|
||||
);
|
||||
}
|
||||
}, 100);
|
||||
// Connect through silent gain to keep worklet processing alive
|
||||
const silence = ctx.createGain();
|
||||
silence.gain.value = 0;
|
||||
source.connect(worklet);
|
||||
worklet.connect(silence);
|
||||
silence.connect(ctx.destination);
|
||||
|
||||
mediaRecorder.ondataavailable = (event) => {
|
||||
if (event.data.size > 0) {
|
||||
audioChunksRef.current.push(event.data);
|
||||
}
|
||||
};
|
||||
|
||||
mediaRecorder.onstop = async () => {
|
||||
const audioBlob = new Blob(audioChunksRef.current, { type: mimeType || 'audio/webm' });
|
||||
|
||||
if (audioBlob.size === 0) {
|
||||
onError('No audio data was recorded. Please check your microphone.');
|
||||
return;
|
||||
}
|
||||
|
||||
await transcribeAudio(audioBlob);
|
||||
};
|
||||
|
||||
mediaRecorder.onerror = (_event) => {
|
||||
onError('Recording failed');
|
||||
};
|
||||
|
||||
mediaRecorder.start(100);
|
||||
setIsRecording(true);
|
||||
} catch (error) {
|
||||
stopRecording();
|
||||
onError(errorMessage(error));
|
||||
}
|
||||
}, [isEnabled, onError, transcribeAudio, stopRecording]);
|
||||
}, [isEnabled, onError, handleSamples, stopRecording]);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
audioContextRef.current?.close();
|
||||
streamRef.current?.getTracks().forEach((t) => t.stop());
|
||||
};
|
||||
}, []);
|
||||
|
||||
return {
|
||||
isEnabled,
|
||||
dictationProvider: provider,
|
||||
isRecording,
|
||||
isTranscribing,
|
||||
recordingDuration,
|
||||
estimatedSize,
|
||||
startRecording,
|
||||
stopRecording,
|
||||
};
|
||||
|
||||
@@ -168,7 +168,7 @@ export type AnalyticsEvent =
|
||||
| {
|
||||
name: 'input_voice_dictation';
|
||||
properties: {
|
||||
action: 'start' | 'stop' | 'transcribed' | 'error';
|
||||
action: 'start' | 'stop' | 'transcribed' | 'error' | 'auto_submit';
|
||||
duration_seconds?: number;
|
||||
error_type?: string;
|
||||
};
|
||||
@@ -593,7 +593,7 @@ export function trackFileAttached(fileType: 'file' | 'directory'): void {
|
||||
}
|
||||
|
||||
export function trackVoiceDictation(
|
||||
action: 'start' | 'stop' | 'transcribed' | 'error',
|
||||
action: 'start' | 'stop' | 'transcribed' | 'error' | 'auto_submit',
|
||||
durationSeconds?: number,
|
||||
errorType?: string
|
||||
): void {
|
||||
|
||||
Reference in New Issue
Block a user