tkmind/tkmind_go
1
0
Fork 0
Code Issues Pull Requests Actions 8 Packages Projects Releases Wiki Activity

Fix eleven labs audio transcription and added more logging (#4358)

Browse Source
This commit is contained in:
Zane
2025-08-27 12:53:05 -07:00
committed by GitHub
parent 862dd97ca4
commit 1014d9e519
7 changed files with 316 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ui/desktop/src/components/ChatInput.tsx
+2 -2
View File
@@ -1259,8 +1259,8 @@ export default function ChatInput({
{/* Inline action buttons on the right */}
<div className="flex items-center gap-1 px-2 relative">
{/* Microphone button - show if dictation is enabled, disable if not configured */}
{(dictationSettings?.enabled || dictationSettings?.provider === null) && (
{/* Microphone button - show only if dictation is enabled */}
{dictationSettings?.enabled && (
<>
{!canUseDictation ? (
<Tooltip>
ui/desktop/src/components/settings/dictation/DictationSection.tsx
+23 -13
View File
@@ -4,9 +4,11 @@ import { ChevronDown } from 'lucide-react';
import { Input } from '../../ui/input';
import { useConfig } from '../../ConfigContext';
import { DictationProvider, DictationSettings } from '../../../hooks/useDictationSettings';
const DICTATION_SETTINGS_KEY = 'dictation_settings';
const ELEVENLABS_API_KEY = 'ELEVENLABS_API_KEY';
import {
DICTATION_SETTINGS_KEY,
ELEVENLABS_API_KEY,
getDefaultDictationSettings,
} from '../../../hooks/dictationConstants';
export default function DictationSection() {
const [settings, setSettings] = useState<DictationSettings>({
@@ -27,20 +29,19 @@ export default function DictationSection() {
useEffect(() => {
const loadSettings = async () => {
const savedSettings = localStorage.getItem(DICTATION_SETTINGS_KEY);
let loadedSettings: DictationSettings;
if (savedSettings) {
const parsed = JSON.parse(savedSettings);
setSettings(parsed);
setShowElevenLabsKey(parsed.provider === 'elevenlabs');
loadedSettings = parsed;
} else {
// Default settings
const defaultSettings: DictationSettings = {
enabled: true,
provider: 'openai',
};
setSettings(defaultSettings);
localStorage.setItem(DICTATION_SETTINGS_KEY, JSON.stringify(defaultSettings));
loadedSettings = await getDefaultDictationSettings(getProviders);
}
setSettings(loadedSettings);
setShowElevenLabsKey(loadedSettings.provider === 'elevenlabs');
// Load ElevenLabs API key from storage
setIsLoadingKey(true);
try {
@@ -58,7 +59,7 @@ export default function DictationSection() {
};
loadSettings();
}, [read]);
}, [read, getProviders]);
// Save ElevenLabs key on unmount if it has changed
useEffect(() => {
@@ -109,6 +110,7 @@ export default function DictationSection() {
};
const saveSettings = (newSettings: DictationSettings) => {
console.log('Saving dictation settings to localStorage:', newSettings);
setSettings(newSettings);
localStorage.setItem(DICTATION_SETTINGS_KEY, JSON.stringify(newSettings));
};
@@ -130,18 +132,26 @@ export default function DictationSection() {
const handleElevenLabsKeyChange = (key: string) => {
setElevenLabsApiKey(key);
elevenLabsApiKeyRef.current = key;
// If user starts typing, they're updating the key
if (key.length > 0) {
setHasElevenLabsKey(false); // Hide "configured" while typing
}
};
const saveElevenLabsKey = async () => {
// Save to secure storage
try {
if (elevenLabsApiKey.trim()) {
console.log('Saving ElevenLabs API key to secure storage...');
await upsert(ELEVENLABS_API_KEY, elevenLabsApiKey, true);
setHasElevenLabsKey(true);
console.log('ElevenLabs API key saved successfully');
} else {
// If key is empty, remove it from storage
console.log('Removing ElevenLabs API key from secure storage...');
await upsert(ELEVENLABS_API_KEY, null, true);
setHasElevenLabsKey(false);
console.log('ElevenLabs API key removed successfully');
}
} catch (error) {
console.error('Error saving ElevenLabs API key:', error);
ui/desktop/src/hooks/dictationConstants.ts
+25
View File
@@ -0,0 +1,25 @@
import { DictationSettings, DictationProvider } from './useDictationSettings';
export const DICTATION_SETTINGS_KEY = 'dictation_settings';
export const ELEVENLABS_API_KEY = 'ELEVENLABS_API_KEY';
export const getDefaultDictationSettings = async (
getProviders: (refresh: boolean) => Promise<Array<{ name: string; is_configured: boolean }>>
): Promise<DictationSettings> => {
const providers = await getProviders(false);
// Check if we have an OpenAI API key as primary default
const openAIProvider = providers.find((p) => p.name === 'openai');
if (openAIProvider && openAIProvider.is_configured) {
return {
enabled: true,
provider: 'openai' as DictationProvider,
};
} else {
return {
enabled: false,
provider: null as DictationProvider,
};
}
};
ui/desktop/src/hooks/useDictationSettings.ts
+10 -18
View File
@@ -1,5 +1,10 @@
import { useState, useEffect } from 'react';
import { useConfig } from '../components/ConfigContext';
import {
DICTATION_SETTINGS_KEY,
ELEVENLABS_API_KEY,
getDefaultDictationSettings,
} from './dictationConstants';
export type DictationProvider = 'openai' | 'elevenlabs' | null;
@@ -8,9 +13,6 @@ export interface DictationSettings {
provider: DictationProvider;
}
const DICTATION_SETTINGS_KEY = 'dictation_settings';
const ELEVENLABS_API_KEY = 'ELEVENLABS_API_KEY';
export const useDictationSettings = () => {
const [settings, setSettings] = useState<DictationSettings | null>(null);
const [hasElevenLabsKey, setHasElevenLabsKey] = useState<boolean>(false);
@@ -20,23 +22,13 @@ export const useDictationSettings = () => {
const loadSettings = async () => {
// Load settings from localStorage
const saved = localStorage.getItem(DICTATION_SETTINGS_KEY);
if (saved) {
setSettings(JSON.parse(saved));
const parsedSettings = JSON.parse(saved);
setSettings(parsedSettings);
} else {
const providers = await getProviders(false);
// Check if we have an OpenAI API key as primary default
const openAIProvider = providers.find((p) => p.name === 'openai');
if (openAIProvider && openAIProvider.is_configured) {
setSettings({
enabled: true,
provider: 'openai',
});
} else {
setSettings({
enabled: false,
provider: null,
});
}
const defaultSettings = await getDefaultDictationSettings(getProviders);
setSettings(defaultSettings);
}
// Load ElevenLabs API key from storage (non-secret for frontend access)
ui/desktop/src/hooks/useWhisper.ts
+70 -18
View File
@@ -87,7 +87,7 @@ export const useWhisper = ({ onTranscription, onError, onSizeWarning }: UseWhisp
// Define stopRecording before startRecording to avoid circular dependency
const stopRecording = useCallback(() => {
setIsRecording(false); // Always update the visual state
setIsRecording(false);
if (mediaRecorderRef.current && mediaRecorderRef.current.state !== 'inactive') {
mediaRecorderRef.current.stop();
@@ -159,14 +159,20 @@ export const useWhisper = ({ onTranscription, onError, onSizeWarning }: UseWhisp
reader.readAsDataURL(audioBlob);
});
const mimeType = audioBlob.type;
if (!mimeType) {
throw new Error('Unable to determine audio format. Please try again.');
}
let endpoint = '';
let headers: Record<string, string> = {
'Content-Type': 'application/json',
'X-Secret-Key': await window.electron.getSecretKey(),
};
let body: Record<string, string> = {
audio: base64Audio,
mime_type: 'audio/webm',
mime_type: mimeType,
};
// Choose endpoint based on provider
@@ -234,23 +240,32 @@ export const useWhisper = ({ onTranscription, onError, onSizeWarning }: UseWhisp
try {
// Request microphone permission
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
const stream = await navigator.mediaDevices.getUserMedia({
audio: {
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true,
sampleRate: 44100,
},
});
streamRef.current = stream;
// Create audio context and analyser for visualization
const context = new AudioContext();
const source = context.createMediaStreamSource(stream);
const analyserNode = context.createAnalyser();
analyserNode.fftSize = 2048;
source.connect(analyserNode);
// Verify we have valid audio tracks
const audioTracks = stream.getAudioTracks();
if (audioTracks.length === 0) {
throw new Error('No audio tracks available in the microphone stream');
}
setAudioContext(context);
setAnalyser(analyserNode);
// AudioContext creation is disabled to prevent MediaRecorder conflicts
setAudioContext(null);
setAnalyser(null);
// Create MediaRecorder
const mediaRecorder = new MediaRecorder(stream, {
mimeType: 'audio/webm',
});
// Determine best supported MIME type
const supportedTypes = ['audio/webm;codecs=opus', 'audio/webm', 'audio/mp4', 'audio/wav'];
const mimeType = supportedTypes.find((type) => MediaRecorder.isTypeSupported(type)) || '';
const mediaRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : {});
mediaRecorderRef.current = mediaRecorder;
audioChunksRef.current = [];
@@ -297,12 +312,49 @@ export const useWhisper = ({ onTranscription, onError, onSizeWarning }: UseWhisp
};
mediaRecorder.onstop = async () => {
const audioBlob = new Blob(audioChunksRef.current, { type: 'audio/webm' });
const audioBlob = new Blob(audioChunksRef.current, { type: mimeType || 'audio/webm' });
// Check if the blob is empty
if (audioBlob.size === 0) {
onError?.(
new Error(
'No audio data was recorded. Please check your microphone permissions and try again.'
)
);
return;
}
await transcribeAudio(audioBlob);
};
mediaRecorder.start(1000); // Collect data every second for size monitoring
setIsRecording(true);
// Add error handler for MediaRecorder
mediaRecorder.onerror = (event) => {
console.error('MediaRecorder error:', event);
onError?.(new Error('Recording failed: Unknown error'));
};
if (!stream.active) {
throw new Error('Audio stream became inactive before recording could start');
}
// Check audio tracks again before starting recording
if (audioTracks.length === 0) {
throw new Error('No audio tracks available in the stream');
}
const activeAudioTracks = audioTracks.filter((track) => track.readyState === 'live');
if (activeAudioTracks.length === 0) {
throw new Error('No live audio tracks available');
}
try {
mediaRecorder.start(100);
setIsRecording(true);
} catch (startError) {
console.error('Error calling mediaRecorder.start():', startError);
const errorMessage = startError instanceof Error ? startError.message : String(startError);
throw new Error(`Failed to start recording: ${errorMessage}`);
}
} catch (error) {
console.error('Error starting recording:', error);
stopRecording();