Fix eleven labs audio transcription and added more logging (#4358)

ui/desktop/src/hooks/dictationConstants.ts

@@ -0,0 +1,25 @@
+import { DictationSettings, DictationProvider } from './useDictationSettings';
+
+export const DICTATION_SETTINGS_KEY = 'dictation_settings';
+export const ELEVENLABS_API_KEY = 'ELEVENLABS_API_KEY';
+
+export const getDefaultDictationSettings = async (
+  getProviders: (refresh: boolean) => Promise<Array<{ name: string; is_configured: boolean }>>
+): Promise<DictationSettings> => {
+  const providers = await getProviders(false);
+
+  // Check if we have an OpenAI API key as primary default
+  const openAIProvider = providers.find((p) => p.name === 'openai');
+
+  if (openAIProvider && openAIProvider.is_configured) {
+    return {
+      enabled: true,
+      provider: 'openai' as DictationProvider,
+    };
+  } else {
+    return {
+      enabled: false,
+      provider: null as DictationProvider,
+    };
+  }
+};

ui/desktop/src/hooks/useDictationSettings.ts

@@ -1,5 +1,10 @@
 import { useState, useEffect } from 'react';
 import { useConfig } from '../components/ConfigContext';
+import {
+  DICTATION_SETTINGS_KEY,
+  ELEVENLABS_API_KEY,
+  getDefaultDictationSettings,
+} from './dictationConstants';
 
 export type DictationProvider = 'openai' | 'elevenlabs' | null;
 
@@ -8,9 +13,6 @@ export interface DictationSettings {
   provider: DictationProvider;
 }
 
-const DICTATION_SETTINGS_KEY = 'dictation_settings';
-const ELEVENLABS_API_KEY = 'ELEVENLABS_API_KEY';
-
 export const useDictationSettings = () => {
   const [settings, setSettings] = useState<DictationSettings | null>(null);
   const [hasElevenLabsKey, setHasElevenLabsKey] = useState<boolean>(false);
@@ -20,23 +22,13 @@ export const useDictationSettings = () => {
     const loadSettings = async () => {
       // Load settings from localStorage
       const saved = localStorage.getItem(DICTATION_SETTINGS_KEY);
+
       if (saved) {
-        setSettings(JSON.parse(saved));
+        const parsedSettings = JSON.parse(saved);
+        setSettings(parsedSettings);
       } else {
-        const providers = await getProviders(false);
-        // Check if we have an OpenAI API key as primary default
-        const openAIProvider = providers.find((p) => p.name === 'openai');
-        if (openAIProvider && openAIProvider.is_configured) {
-          setSettings({
-            enabled: true,
-            provider: 'openai',
-          });
-        } else {
-          setSettings({
-            enabled: false,
-            provider: null,
-          });
-        }
+        const defaultSettings = await getDefaultDictationSettings(getProviders);
+        setSettings(defaultSettings);
       }
 
       // Load ElevenLabs API key from storage (non-secret for frontend access)

ui/desktop/src/hooks/useWhisper.ts

@@ -87,7 +87,7 @@ export const useWhisper = ({ onTranscription, onError, onSizeWarning }: UseWhisp
 
   // Define stopRecording before startRecording to avoid circular dependency
   const stopRecording = useCallback(() => {
-    setIsRecording(false); // Always update the visual state
+    setIsRecording(false);
 
     if (mediaRecorderRef.current && mediaRecorderRef.current.state !== 'inactive') {
       mediaRecorderRef.current.stop();
@@ -159,14 +159,20 @@ export const useWhisper = ({ onTranscription, onError, onSizeWarning }: UseWhisp
           reader.readAsDataURL(audioBlob);
         });
 
+        const mimeType = audioBlob.type;
+        if (!mimeType) {
+          throw new Error('Unable to determine audio format. Please try again.');
+        }
+
         let endpoint = '';
         let headers: Record<string, string> = {
           'Content-Type': 'application/json',
           'X-Secret-Key': await window.electron.getSecretKey(),
         };
+
         let body: Record<string, string> = {
           audio: base64Audio,
-          mime_type: 'audio/webm',
+          mime_type: mimeType,
         };
 
         // Choose endpoint based on provider
@@ -234,23 +240,32 @@ export const useWhisper = ({ onTranscription, onError, onSizeWarning }: UseWhisp
 
     try {
       // Request microphone permission
-      const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+      const stream = await navigator.mediaDevices.getUserMedia({
+        audio: {
+          echoCancellation: true,
+          noiseSuppression: true,
+          autoGainControl: true,
+          sampleRate: 44100,
+        },
+      });
       streamRef.current = stream;
 
-      // Create audio context and analyser for visualization
-      const context = new AudioContext();
-      const source = context.createMediaStreamSource(stream);
-      const analyserNode = context.createAnalyser();
-      analyserNode.fftSize = 2048;
-      source.connect(analyserNode);
+      // Verify we have valid audio tracks
+      const audioTracks = stream.getAudioTracks();
+      if (audioTracks.length === 0) {
+        throw new Error('No audio tracks available in the microphone stream');
+      }
 
-      setAudioContext(context);
-      setAnalyser(analyserNode);
+      // AudioContext creation is disabled to prevent MediaRecorder conflicts
+      setAudioContext(null);
+      setAnalyser(null);
 
-      // Create MediaRecorder
-      const mediaRecorder = new MediaRecorder(stream, {
-        mimeType: 'audio/webm',
-      });
+      // Determine best supported MIME type
+      const supportedTypes = ['audio/webm;codecs=opus', 'audio/webm', 'audio/mp4', 'audio/wav'];
+
+      const mimeType = supportedTypes.find((type) => MediaRecorder.isTypeSupported(type)) || '';
+
+      const mediaRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : {});
 
       mediaRecorderRef.current = mediaRecorder;
       audioChunksRef.current = [];
@@ -297,12 +312,49 @@ export const useWhisper = ({ onTranscription, onError, onSizeWarning }: UseWhisp
       };
 
       mediaRecorder.onstop = async () => {
-        const audioBlob = new Blob(audioChunksRef.current, { type: 'audio/webm' });
+        const audioBlob = new Blob(audioChunksRef.current, { type: mimeType || 'audio/webm' });
+
+        // Check if the blob is empty
+        if (audioBlob.size === 0) {
+          onError?.(
+            new Error(
+              'No audio data was recorded. Please check your microphone permissions and try again.'
+            )
+          );
+          return;
+        }
+
         await transcribeAudio(audioBlob);
       };
 
-      mediaRecorder.start(1000); // Collect data every second for size monitoring
-      setIsRecording(true);
+      // Add error handler for MediaRecorder
+      mediaRecorder.onerror = (event) => {
+        console.error('MediaRecorder error:', event);
+        onError?.(new Error('Recording failed: Unknown error'));
+      };
+
+      if (!stream.active) {
+        throw new Error('Audio stream became inactive before recording could start');
+      }
+
+      // Check audio tracks again before starting recording
+      if (audioTracks.length === 0) {
+        throw new Error('No audio tracks available in the stream');
+      }
+
+      const activeAudioTracks = audioTracks.filter((track) => track.readyState === 'live');
+      if (activeAudioTracks.length === 0) {
+        throw new Error('No live audio tracks available');
+      }
+
+      try {
+        mediaRecorder.start(100);
+        setIsRecording(true);
+      } catch (startError) {
+        console.error('Error calling mediaRecorder.start():', startError);
+        const errorMessage = startError instanceof Error ? startError.message : String(startError);
+        throw new Error(`Failed to start recording: ${errorMessage}`);
+      }
     } catch (error) {
       console.error('Error starting recording:', error);
       stopRecording();