feat: add voice dictation using OpenAI Whisper & ElevenLabs (#3079)
Co-authored-by: jack <>
This commit is contained in:
jack
Generated
+1
@@ -7021,6 +7021,7 @@ dependencies = [
|
||||
"js-sys",
|
||||
"log",
|
||||
"mime",
|
||||
"mime_guess",
|
||||
"once_cell",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
|
||||
@@ -40,7 +40,7 @@ serde_yaml = "0.9.34"
|
||||
axum-extra = "0.10.0"
|
||||
utoipa = { version = "4.1", features = ["axum_extras", "chrono"] }
|
||||
dirs = "6.0.0"
|
||||
reqwest = { version = "0.12.9", features = ["json", "rustls-tls", "blocking"], default-features = false }
|
||||
reqwest = { version = "0.12.9", features = ["json", "rustls-tls", "blocking", "multipart"], default-features = false }
|
||||
|
||||
[[bin]]
|
||||
name = "goosed"
|
||||
|
||||
@@ -0,0 +1,474 @@
|
||||
/// Audio transcription route handler
|
||||
///
|
||||
/// This module provides endpoints for audio transcription using OpenAI's Whisper API.
|
||||
/// The OpenAI API key must be configured in the backend for this to work.
|
||||
use super::utils::verify_secret_key;
|
||||
use crate::state::AppState;
|
||||
use axum::{
|
||||
extract::State,
|
||||
http::{HeaderMap, StatusCode},
|
||||
routing::{get, post},
|
||||
Json, Router,
|
||||
};
|
||||
use base64::{engine::general_purpose::STANDARD as BASE64, Engine};
|
||||
use reqwest::Client;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
// Constants
|
||||
const MAX_AUDIO_SIZE_BYTES: usize = 25 * 1024 * 1024; // 25MB
|
||||
const OPENAI_TIMEOUT_SECONDS: u64 = 30;
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct TranscribeRequest {
|
||||
audio: String, // Base64 encoded audio data
|
||||
mime_type: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct TranscribeElevenLabsRequest {
|
||||
audio: String, // Base64 encoded audio data
|
||||
mime_type: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct TranscribeResponse {
|
||||
text: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct WhisperResponse {
|
||||
text: String,
|
||||
}
|
||||
|
||||
/// Transcribe audio using OpenAI's Whisper API
|
||||
///
|
||||
/// # Request
|
||||
/// - `audio`: Base64 encoded audio data
|
||||
/// - `mime_type`: MIME type of the audio (e.g., "audio/webm", "audio/wav")
|
||||
///
|
||||
/// # Response
|
||||
/// - `text`: Transcribed text from the audio
|
||||
///
|
||||
/// # Errors
|
||||
/// - 401: Unauthorized (missing or invalid X-Secret-Key header)
|
||||
/// - 412: Precondition Failed (OpenAI API key not configured)
|
||||
/// - 400: Bad Request (invalid base64 audio data)
|
||||
/// - 413: Payload Too Large (audio file exceeds 25MB limit)
|
||||
/// - 415: Unsupported Media Type (unsupported audio format)
|
||||
/// - 502: Bad Gateway (OpenAI API error)
|
||||
/// - 503: Service Unavailable (network error)
|
||||
async fn transcribe_handler(
|
||||
State(state): State<Arc<AppState>>,
|
||||
headers: HeaderMap,
|
||||
Json(request): Json<TranscribeRequest>,
|
||||
) -> Result<Json<TranscribeResponse>, StatusCode> {
|
||||
verify_secret_key(&headers, &state)?;
|
||||
|
||||
// Get the OpenAI API key from config
|
||||
let config = goose::config::Config::global();
|
||||
let api_key: String = config
|
||||
.get_secret("OPENAI_API_KEY")
|
||||
.map_err(|_| StatusCode::PRECONDITION_FAILED)?;
|
||||
|
||||
// Get the OpenAI host from config (with default)
|
||||
let openai_host = match config.get("OPENAI_HOST", false) {
|
||||
Ok(value) => value
|
||||
.as_str()
|
||||
.map(|s| s.to_string())
|
||||
.unwrap_or_else(|| "https://api.openai.com".to_string()),
|
||||
Err(_) => "https://api.openai.com".to_string(),
|
||||
};
|
||||
|
||||
tracing::debug!("Using OpenAI host: {}", openai_host);
|
||||
|
||||
// Decode the base64 audio data
|
||||
let audio_bytes = BASE64
|
||||
.decode(&request.audio)
|
||||
.map_err(|_| StatusCode::BAD_REQUEST)?;
|
||||
|
||||
// Check file size
|
||||
if audio_bytes.len() > MAX_AUDIO_SIZE_BYTES {
|
||||
tracing::warn!(
|
||||
"Audio file too large: {} bytes (max: {} bytes)",
|
||||
audio_bytes.len(),
|
||||
MAX_AUDIO_SIZE_BYTES
|
||||
);
|
||||
return Err(StatusCode::PAYLOAD_TOO_LARGE);
|
||||
}
|
||||
|
||||
// Determine file extension based on MIME type
|
||||
let file_extension = match request.mime_type.as_str() {
|
||||
"audio/webm" => "webm",
|
||||
"audio/mp4" => "mp4",
|
||||
"audio/mpeg" => "mp3",
|
||||
"audio/mpga" => "mpga",
|
||||
"audio/m4a" => "m4a",
|
||||
"audio/wav" => "wav",
|
||||
"audio/x-wav" => "wav",
|
||||
_ => return Err(StatusCode::UNSUPPORTED_MEDIA_TYPE),
|
||||
};
|
||||
|
||||
// Create a multipart form with the audio file
|
||||
let part = reqwest::multipart::Part::bytes(audio_bytes)
|
||||
.file_name(format!("audio.{}", file_extension))
|
||||
.mime_str(&request.mime_type)
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
|
||||
let form = reqwest::multipart::Form::new()
|
||||
.part("file", part)
|
||||
.text("model", "whisper-1")
|
||||
.text("response_format", "json");
|
||||
|
||||
// Make request to OpenAI Whisper API
|
||||
let client = Client::builder()
|
||||
.timeout(Duration::from_secs(OPENAI_TIMEOUT_SECONDS))
|
||||
.build()
|
||||
.map_err(|e| {
|
||||
tracing::error!("Failed to create HTTP client: {}", e);
|
||||
StatusCode::INTERNAL_SERVER_ERROR
|
||||
})?;
|
||||
|
||||
let response = client
|
||||
.post(format!("{}/v1/audio/transcriptions", openai_host))
|
||||
.header("Authorization", format!("Bearer {}", api_key))
|
||||
.multipart(form)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| {
|
||||
if e.is_timeout() {
|
||||
tracing::error!(
|
||||
"OpenAI API request timed out after {}s",
|
||||
OPENAI_TIMEOUT_SECONDS
|
||||
);
|
||||
StatusCode::GATEWAY_TIMEOUT
|
||||
} else {
|
||||
tracing::error!("Failed to send request to OpenAI: {}", e);
|
||||
StatusCode::SERVICE_UNAVAILABLE
|
||||
}
|
||||
})?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
let error_text = response.text().await.unwrap_or_default();
|
||||
tracing::error!("OpenAI API error: {}", error_text);
|
||||
return Err(StatusCode::BAD_GATEWAY);
|
||||
}
|
||||
|
||||
let whisper_response: WhisperResponse = response.json().await.map_err(|e| {
|
||||
tracing::error!("Failed to parse OpenAI response: {}", e);
|
||||
StatusCode::INTERNAL_SERVER_ERROR
|
||||
})?;
|
||||
|
||||
Ok(Json(TranscribeResponse {
|
||||
text: whisper_response.text,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Transcribe audio using ElevenLabs Speech-to-Text API
|
||||
///
|
||||
/// Uses ElevenLabs' speech-to-text endpoint for transcription.
|
||||
/// Requires an ElevenLabs API key with speech-to-text access.
|
||||
async fn transcribe_elevenlabs_handler(
|
||||
State(state): State<Arc<AppState>>,
|
||||
headers: HeaderMap,
|
||||
Json(request): Json<TranscribeElevenLabsRequest>,
|
||||
) -> Result<Json<TranscribeResponse>, StatusCode> {
|
||||
verify_secret_key(&headers, &state)?;
|
||||
|
||||
// Get the ElevenLabs API key from config
|
||||
let config = goose::config::Config::global();
|
||||
|
||||
// First try to get it as a secret
|
||||
let api_key: String = match config.get_secret("ELEVENLABS_API_KEY") {
|
||||
Ok(key) => key,
|
||||
Err(_) => {
|
||||
// Try to get it as non-secret (for backward compatibility)
|
||||
match config.get("ELEVENLABS_API_KEY", false) {
|
||||
Ok(value) => {
|
||||
match value.as_str() {
|
||||
Some(key_str) => {
|
||||
tracing::info!("Migrating ElevenLabs API key to secret storage");
|
||||
let key = key_str.to_string();
|
||||
// Migrate to secret storage
|
||||
if let Err(e) = config.set(
|
||||
"ELEVENLABS_API_KEY",
|
||||
serde_json::Value::String(key.clone()),
|
||||
true,
|
||||
) {
|
||||
tracing::error!("Failed to migrate ElevenLabs API key: {:?}", e);
|
||||
}
|
||||
// Delete the non-secret version
|
||||
let _ = config.delete("ELEVENLABS_API_KEY");
|
||||
key
|
||||
}
|
||||
None => {
|
||||
tracing::error!("ElevenLabs API key is not a string");
|
||||
return Err(StatusCode::PRECONDITION_FAILED);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to get ElevenLabs API key from config: {:?}", e);
|
||||
return Err(StatusCode::PRECONDITION_FAILED);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Decode the base64 audio data
|
||||
let audio_bytes = BASE64
|
||||
.decode(&request.audio)
|
||||
.map_err(|_| StatusCode::BAD_REQUEST)?;
|
||||
|
||||
// Check file size
|
||||
if audio_bytes.len() > MAX_AUDIO_SIZE_BYTES {
|
||||
tracing::warn!(
|
||||
"Audio file too large: {} bytes (max: {} bytes)",
|
||||
audio_bytes.len(),
|
||||
MAX_AUDIO_SIZE_BYTES
|
||||
);
|
||||
return Err(StatusCode::PAYLOAD_TOO_LARGE);
|
||||
}
|
||||
|
||||
// Determine file extension and content type based on MIME type
|
||||
let (file_extension, content_type) = match request.mime_type.as_str() {
|
||||
"audio/webm" => ("webm", "audio/webm"),
|
||||
"audio/mp4" => ("mp4", "audio/mp4"),
|
||||
"audio/mpeg" => ("mp3", "audio/mpeg"),
|
||||
"audio/mpga" => ("mp3", "audio/mpeg"),
|
||||
"audio/m4a" => ("m4a", "audio/m4a"),
|
||||
"audio/wav" => ("wav", "audio/wav"),
|
||||
"audio/x-wav" => ("wav", "audio/wav"),
|
||||
_ => return Err(StatusCode::UNSUPPORTED_MEDIA_TYPE),
|
||||
};
|
||||
|
||||
// Create multipart form for ElevenLabs API
|
||||
let part = reqwest::multipart::Part::bytes(audio_bytes)
|
||||
.file_name(format!("audio.{}", file_extension))
|
||||
.mime_str(content_type)
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
|
||||
let form = reqwest::multipart::Form::new()
|
||||
.part("file", part) // Changed from "audio" to "file"
|
||||
.text("model_id", "scribe_v1") // Use the correct model_id for speech-to-text
|
||||
.text("tag_audio_events", "false")
|
||||
.text("diarize", "false");
|
||||
|
||||
// Make request to ElevenLabs Speech-to-Text API
|
||||
let client = Client::builder()
|
||||
.timeout(Duration::from_secs(OPENAI_TIMEOUT_SECONDS))
|
||||
.build()
|
||||
.map_err(|e| {
|
||||
tracing::error!("Failed to create HTTP client: {}", e);
|
||||
StatusCode::INTERNAL_SERVER_ERROR
|
||||
})?;
|
||||
|
||||
let response = client
|
||||
.post("https://api.elevenlabs.io/v1/speech-to-text")
|
||||
.header("xi-api-key", &api_key)
|
||||
.multipart(form)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| {
|
||||
if e.is_timeout() {
|
||||
tracing::error!(
|
||||
"ElevenLabs API request timed out after {}s",
|
||||
OPENAI_TIMEOUT_SECONDS
|
||||
);
|
||||
StatusCode::GATEWAY_TIMEOUT
|
||||
} else {
|
||||
tracing::error!("Failed to send request to ElevenLabs: {}", e);
|
||||
StatusCode::SERVICE_UNAVAILABLE
|
||||
}
|
||||
})?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
let error_text = response.text().await.unwrap_or_default();
|
||||
tracing::error!("ElevenLabs API error: {}", error_text);
|
||||
|
||||
// Check for specific error codes
|
||||
if error_text.contains("Unauthorized") || error_text.contains("Invalid API key") {
|
||||
return Err(StatusCode::UNAUTHORIZED);
|
||||
} else if error_text.contains("quota") || error_text.contains("limit") {
|
||||
return Err(StatusCode::PAYMENT_REQUIRED);
|
||||
}
|
||||
|
||||
return Err(StatusCode::BAD_GATEWAY);
|
||||
}
|
||||
|
||||
// Parse ElevenLabs response
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct ElevenLabsResponse {
|
||||
text: String,
|
||||
#[serde(rename = "chunks")]
|
||||
#[allow(dead_code)]
|
||||
_chunks: Option<Vec<serde_json::Value>>,
|
||||
}
|
||||
|
||||
let elevenlabs_response: ElevenLabsResponse = response.json().await.map_err(|e| {
|
||||
tracing::error!("Failed to parse ElevenLabs response: {}", e);
|
||||
StatusCode::INTERNAL_SERVER_ERROR
|
||||
})?;
|
||||
|
||||
Ok(Json(TranscribeResponse {
|
||||
text: elevenlabs_response.text,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Check if dictation providers are configured
|
||||
///
|
||||
/// Returns configuration status for dictation providers
|
||||
async fn check_dictation_config(
|
||||
State(state): State<Arc<AppState>>,
|
||||
headers: HeaderMap,
|
||||
) -> Result<Json<serde_json::Value>, StatusCode> {
|
||||
verify_secret_key(&headers, &state)?;
|
||||
|
||||
let config = goose::config::Config::global();
|
||||
|
||||
// Check if ElevenLabs API key is configured
|
||||
let has_elevenlabs = config
|
||||
.get_secret::<String>("ELEVENLABS_API_KEY")
|
||||
.map(|_| true)
|
||||
.unwrap_or_else(|_| {
|
||||
// Check non-secret for backward compatibility
|
||||
config
|
||||
.get("ELEVENLABS_API_KEY", false)
|
||||
.map(|_| true)
|
||||
.unwrap_or(false)
|
||||
});
|
||||
|
||||
Ok(Json(serde_json::json!({
|
||||
"elevenlabs": has_elevenlabs
|
||||
})))
|
||||
}
|
||||
|
||||
pub fn routes(state: Arc<AppState>) -> Router {
|
||||
Router::new()
|
||||
.route("/audio/transcribe", post(transcribe_handler))
|
||||
.route(
|
||||
"/audio/transcribe/elevenlabs",
|
||||
post(transcribe_elevenlabs_handler),
|
||||
)
|
||||
.route("/audio/config", get(check_dictation_config))
|
||||
.with_state(state)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use axum::{body::Body, http::Request};
|
||||
use tower::ServiceExt;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_transcribe_endpoint_requires_auth() {
|
||||
let state = AppState::new(
|
||||
Arc::new(goose::agents::Agent::new()),
|
||||
"test-secret".to_string(),
|
||||
)
|
||||
.await;
|
||||
let app = routes(state);
|
||||
|
||||
// Test without auth header
|
||||
let request = Request::builder()
|
||||
.uri("/audio/transcribe")
|
||||
.method("POST")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
serde_json::to_string(&serde_json::json!({
|
||||
"audio": "dGVzdA==",
|
||||
"mime_type": "audio/webm"
|
||||
}))
|
||||
.unwrap(),
|
||||
))
|
||||
.unwrap();
|
||||
|
||||
let response = app.oneshot(request).await.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_transcribe_endpoint_validates_size() {
|
||||
let state = AppState::new(
|
||||
Arc::new(goose::agents::Agent::new()),
|
||||
"test-secret".to_string(),
|
||||
)
|
||||
.await;
|
||||
let app = routes(state);
|
||||
|
||||
// Create a large base64 string (simulating > 25MB audio)
|
||||
let large_audio = BASE64.encode(vec![0u8; MAX_AUDIO_SIZE_BYTES + 1]);
|
||||
|
||||
let request = Request::builder()
|
||||
.uri("/audio/transcribe")
|
||||
.method("POST")
|
||||
.header("content-type", "application/json")
|
||||
.header("x-secret-key", "test-secret")
|
||||
.body(Body::from(
|
||||
serde_json::to_string(&serde_json::json!({
|
||||
"audio": large_audio,
|
||||
"mime_type": "audio/webm"
|
||||
}))
|
||||
.unwrap(),
|
||||
))
|
||||
.unwrap();
|
||||
|
||||
let response = app.oneshot(request).await.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_transcribe_endpoint_validates_mime_type() {
|
||||
let state = AppState::new(
|
||||
Arc::new(goose::agents::Agent::new()),
|
||||
"test-secret".to_string(),
|
||||
)
|
||||
.await;
|
||||
let app = routes(state);
|
||||
|
||||
let request = Request::builder()
|
||||
.uri("/audio/transcribe")
|
||||
.method("POST")
|
||||
.header("content-type", "application/json")
|
||||
.header("x-secret-key", "test-secret")
|
||||
.body(Body::from(
|
||||
serde_json::to_string(&serde_json::json!({
|
||||
"audio": "dGVzdA==",
|
||||
"mime_type": "application/pdf" // Invalid MIME type
|
||||
}))
|
||||
.unwrap(),
|
||||
))
|
||||
.unwrap();
|
||||
|
||||
let response = app.oneshot(request).await.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::UNSUPPORTED_MEDIA_TYPE);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_transcribe_endpoint_handles_invalid_base64() {
|
||||
let state = AppState::new(
|
||||
Arc::new(goose::agents::Agent::new()),
|
||||
"test-secret".to_string(),
|
||||
)
|
||||
.await;
|
||||
let app = routes(state);
|
||||
|
||||
let request = Request::builder()
|
||||
.uri("/audio/transcribe")
|
||||
.method("POST")
|
||||
.header("content-type", "application/json")
|
||||
.header("x-secret-key", "test-secret")
|
||||
.body(Body::from(
|
||||
serde_json::to_string(&serde_json::json!({
|
||||
"audio": "invalid-base64-!@#$%",
|
||||
"mime_type": "audio/webm"
|
||||
}))
|
||||
.unwrap(),
|
||||
))
|
||||
.unwrap();
|
||||
|
||||
let response = app.oneshot(request).await.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
// Export route modules
|
||||
pub mod agent;
|
||||
pub mod audio;
|
||||
pub mod config_management;
|
||||
pub mod context;
|
||||
pub mod extension;
|
||||
@@ -19,6 +20,7 @@ pub fn configure(state: Arc<crate::state::AppState>) -> Router {
|
||||
.merge(health::routes())
|
||||
.merge(reply::routes(state.clone()))
|
||||
.merge(agent::routes(state.clone()))
|
||||
.merge(audio::routes(state.clone()))
|
||||
.merge(context::routes(state.clone()))
|
||||
.merge(extension::routes(state.clone()))
|
||||
.merge(config_management::routes(state.clone()))
|
||||
|
||||
@@ -2,7 +2,8 @@
|
||||
"root": true,
|
||||
"env": {
|
||||
"browser": true,
|
||||
"es2020": true
|
||||
"es2020": true,
|
||||
"node": true
|
||||
},
|
||||
"extends": [
|
||||
"eslint:recommended",
|
||||
|
||||
@@ -70,6 +70,7 @@ module.exports = [
|
||||
HTMLTextAreaElement: 'readonly',
|
||||
HTMLButtonElement: 'readonly',
|
||||
HTMLDivElement: 'readonly',
|
||||
HTMLCanvasElement: 'readonly',
|
||||
File: 'readonly',
|
||||
FileList: 'readonly',
|
||||
FileReader: 'readonly',
|
||||
@@ -87,10 +88,17 @@ module.exports = [
|
||||
React: 'readonly',
|
||||
handleAction: 'readonly',
|
||||
requestAnimationFrame: 'readonly',
|
||||
cancelAnimationFrame: 'readonly',
|
||||
ResizeObserver: 'readonly',
|
||||
MutationObserver: 'readonly',
|
||||
NodeFilter: 'readonly',
|
||||
Text: 'readonly',
|
||||
AudioContext: 'readonly',
|
||||
AnalyserNode: 'readonly',
|
||||
MediaRecorder: 'readonly',
|
||||
MediaStream: 'readonly',
|
||||
Blob: 'readonly',
|
||||
FormData: 'readonly',
|
||||
},
|
||||
},
|
||||
plugins: {
|
||||
|
||||
@@ -2,11 +2,14 @@ import React, { useRef, useState, useEffect, useMemo } from 'react';
|
||||
import { Button } from './ui/button';
|
||||
import type { View } from '../App';
|
||||
import Stop from './ui/Stop';
|
||||
import { Attach, Send, Close } from './icons';
|
||||
import { Attach, Send, Close, Microphone } from './icons';
|
||||
import { debounce } from 'lodash';
|
||||
import BottomMenu from './bottom_menu/BottomMenu';
|
||||
import { LocalMessageStorage } from '../utils/localMessageStorage';
|
||||
import { Message } from '../types/message';
|
||||
import { useWhisper } from '../hooks/useWhisper';
|
||||
import { WaveformVisualizer } from './WaveformVisualizer';
|
||||
import { toastError } from '../toasts';
|
||||
|
||||
interface PastedImage {
|
||||
id: string;
|
||||
@@ -63,6 +66,39 @@ export default function ChatInput({
|
||||
const [isFocused, setIsFocused] = useState(false);
|
||||
const [pastedImages, setPastedImages] = useState<PastedImage[]>([]);
|
||||
|
||||
// Whisper hook for voice dictation
|
||||
const {
|
||||
isRecording,
|
||||
isTranscribing,
|
||||
canUseDictation,
|
||||
audioContext,
|
||||
analyser,
|
||||
startRecording,
|
||||
stopRecording,
|
||||
recordingDuration,
|
||||
estimatedSize,
|
||||
} = useWhisper({
|
||||
onTranscription: (text) => {
|
||||
// Append transcribed text to the current input
|
||||
const newValue = displayValue.trim() ? `${displayValue.trim()} ${text}` : text;
|
||||
setDisplayValue(newValue);
|
||||
setValue(newValue);
|
||||
textAreaRef.current?.focus();
|
||||
},
|
||||
onError: (error) => {
|
||||
toastError({
|
||||
title: 'Dictation Error',
|
||||
msg: error.message,
|
||||
});
|
||||
},
|
||||
onSizeWarning: (sizeMB) => {
|
||||
toastError({
|
||||
title: 'Recording Size Warning',
|
||||
msg: `Recording is ${sizeMB.toFixed(1)}MB. Maximum size is 25MB.`,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
// Update internal value when initialValue changes
|
||||
useEffect(() => {
|
||||
setValue(initialValue);
|
||||
@@ -451,28 +487,40 @@ export default function ChatInput({
|
||||
} bg-bgApp z-10`}
|
||||
>
|
||||
<form onSubmit={onFormSubmit}>
|
||||
<textarea
|
||||
data-testid="chat-input"
|
||||
autoFocus
|
||||
id="dynamic-textarea"
|
||||
placeholder="What can goose help with? ⌘↑/⌘↓"
|
||||
value={displayValue}
|
||||
onChange={handleChange}
|
||||
onCompositionStart={handleCompositionStart}
|
||||
onCompositionEnd={handleCompositionEnd}
|
||||
onKeyDown={handleKeyDown}
|
||||
onPaste={handlePaste}
|
||||
onFocus={() => setIsFocused(true)}
|
||||
onBlur={() => setIsFocused(false)}
|
||||
ref={textAreaRef}
|
||||
rows={1}
|
||||
style={{
|
||||
minHeight: `${minHeight}px`,
|
||||
maxHeight: `${maxHeight}px`,
|
||||
overflowY: 'auto',
|
||||
}}
|
||||
className="w-full pl-4 pr-[68px] outline-none border-none focus:ring-0 bg-transparent pt-3 pb-1.5 text-sm resize-none text-textStandard placeholder:text-textPlaceholder"
|
||||
/>
|
||||
<div className="relative">
|
||||
<textarea
|
||||
data-testid="chat-input"
|
||||
autoFocus
|
||||
id="dynamic-textarea"
|
||||
placeholder={isRecording ? '' : 'What can goose help with? ⌘↑/⌘↓'}
|
||||
value={displayValue}
|
||||
onChange={handleChange}
|
||||
onCompositionStart={handleCompositionStart}
|
||||
onCompositionEnd={handleCompositionEnd}
|
||||
onKeyDown={handleKeyDown}
|
||||
onPaste={handlePaste}
|
||||
onFocus={() => setIsFocused(true)}
|
||||
onBlur={() => setIsFocused(false)}
|
||||
ref={textAreaRef}
|
||||
rows={1}
|
||||
style={{
|
||||
minHeight: `${minHeight}px`,
|
||||
maxHeight: `${maxHeight}px`,
|
||||
overflowY: 'auto',
|
||||
opacity: isRecording ? 0 : 1,
|
||||
}}
|
||||
className="w-full pl-4 pr-[108px] outline-none border-none focus:ring-0 bg-transparent pt-3 pb-1.5 text-sm resize-none text-textStandard placeholder:text-textPlaceholder"
|
||||
/>
|
||||
{isRecording && (
|
||||
<div className="absolute inset-0 flex items-center pl-4 pr-[108px] pt-3 pb-1.5">
|
||||
<WaveformVisualizer
|
||||
audioContext={audioContext}
|
||||
analyser={analyser}
|
||||
isRecording={isRecording}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{pastedImages.length > 0 && (
|
||||
<div className="flex flex-wrap gap-2 p-2 border-t border-borderSubtle">
|
||||
@@ -537,20 +585,85 @@ export default function ChatInput({
|
||||
<Stop size={24} />
|
||||
</Button>
|
||||
) : (
|
||||
<Button
|
||||
type="submit"
|
||||
size="icon"
|
||||
variant="ghost"
|
||||
disabled={!hasSubmittableContent || isAnyImageLoading} // Disable if no content or if images are still loading/saving
|
||||
className={`absolute right-3 top-2 transition-colors rounded-full w-7 h-7 [&_svg]:size-4 ${
|
||||
!hasSubmittableContent || isAnyImageLoading
|
||||
? 'text-textSubtle cursor-not-allowed'
|
||||
: 'bg-bgAppInverse text-textProminentInverse hover:cursor-pointer'
|
||||
}`}
|
||||
title={isAnyImageLoading ? 'Waiting for images to save...' : 'Send'}
|
||||
>
|
||||
<Send />
|
||||
</Button>
|
||||
<>
|
||||
{/* Microphone button - only show if dictation is enabled and configured */}
|
||||
{canUseDictation && (
|
||||
<>
|
||||
<Button
|
||||
type="button"
|
||||
size="icon"
|
||||
variant="ghost"
|
||||
onClick={() => {
|
||||
if (isRecording) {
|
||||
stopRecording();
|
||||
} else {
|
||||
startRecording();
|
||||
}
|
||||
}}
|
||||
disabled={isTranscribing}
|
||||
className={`absolute right-12 top-2 transition-colors rounded-full w-7 h-7 [&_svg]:size-4 ${
|
||||
isRecording
|
||||
? 'bg-red-500 text-white hover:bg-red-600'
|
||||
: isTranscribing
|
||||
? 'text-textSubtle cursor-not-allowed animate-pulse'
|
||||
: 'text-textSubtle hover:text-textStandard'
|
||||
}`}
|
||||
title={
|
||||
isRecording
|
||||
? `Stop recording (${Math.floor(recordingDuration)}s, ~${estimatedSize.toFixed(1)}MB)`
|
||||
: isTranscribing
|
||||
? 'Transcribing...'
|
||||
: 'Start dictation'
|
||||
}
|
||||
>
|
||||
<Microphone />
|
||||
</Button>
|
||||
{/* Recording/transcribing status indicator - positioned above the input */}
|
||||
{(isRecording || isTranscribing) && (
|
||||
<div className="absolute right-0 -top-8 bg-bgApp px-2 py-1 rounded text-xs whitespace-nowrap shadow-md border border-borderSubtle">
|
||||
{isTranscribing ? (
|
||||
<span className="text-blue-500 flex items-center gap-1">
|
||||
<span className="inline-block w-2 h-2 bg-blue-500 rounded-full animate-pulse" />
|
||||
Transcribing...
|
||||
</span>
|
||||
) : (
|
||||
<span
|
||||
className={`flex items-center gap-2 ${estimatedSize > 20 ? 'text-orange-500' : 'text-textSubtle'}`}
|
||||
>
|
||||
<span className="inline-block w-2 h-2 bg-red-500 rounded-full animate-pulse" />
|
||||
{Math.floor(recordingDuration)}s • ~{estimatedSize.toFixed(1)}MB
|
||||
{estimatedSize > 20 && <span className="text-xs">(near 25MB limit)</span>}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
<Button
|
||||
type="submit"
|
||||
size="icon"
|
||||
variant="ghost"
|
||||
disabled={
|
||||
!hasSubmittableContent || isAnyImageLoading || isRecording || isTranscribing
|
||||
}
|
||||
className={`absolute right-3 top-2 transition-colors rounded-full w-7 h-7 [&_svg]:size-4 ${
|
||||
!hasSubmittableContent || isAnyImageLoading || isRecording || isTranscribing
|
||||
? 'text-textSubtle cursor-not-allowed'
|
||||
: 'bg-bgAppInverse text-textProminentInverse hover:cursor-pointer'
|
||||
}`}
|
||||
title={
|
||||
isAnyImageLoading
|
||||
? 'Waiting for images to save...'
|
||||
: isRecording
|
||||
? 'Recording...'
|
||||
: isTranscribing
|
||||
? 'Transcribing...'
|
||||
: 'Send'
|
||||
}
|
||||
>
|
||||
<Send />
|
||||
</Button>
|
||||
</>
|
||||
)}
|
||||
</form>
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
/* global Blob, ClipboardItem */
|
||||
/* global ClipboardItem */
|
||||
|
||||
import React, { useState } from 'react';
|
||||
import { Copy } from './icons';
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
import React, { useEffect, useRef } from 'react';
|
||||
|
||||
interface WaveformVisualizerProps {
|
||||
audioContext: AudioContext | null;
|
||||
analyser: AnalyserNode | null;
|
||||
isRecording: boolean;
|
||||
}
|
||||
|
||||
export const WaveformVisualizer: React.FC<WaveformVisualizerProps> = ({
|
||||
analyser,
|
||||
isRecording,
|
||||
}) => {
|
||||
const canvasRef = useRef<HTMLCanvasElement>(null);
|
||||
const animationRef = useRef<number>();
|
||||
|
||||
useEffect(() => {
|
||||
if (!canvasRef.current || !analyser || !isRecording) return;
|
||||
|
||||
const canvas = canvasRef.current;
|
||||
const ctx = canvas.getContext('2d');
|
||||
if (!ctx) return;
|
||||
|
||||
// Set canvas size
|
||||
const dpr = window.devicePixelRatio || 1;
|
||||
const rect = canvas.getBoundingClientRect();
|
||||
canvas.width = rect.width * dpr;
|
||||
canvas.height = rect.height * dpr;
|
||||
ctx.scale(dpr, dpr);
|
||||
|
||||
// Configure analyser
|
||||
analyser.fftSize = 256;
|
||||
const bufferLength = analyser.frequencyBinCount;
|
||||
const dataArray = new Uint8Array(bufferLength);
|
||||
|
||||
// Visual settings
|
||||
const barWidth = 3;
|
||||
const barSpacing = 2;
|
||||
const barCount = Math.floor(rect.width / (barWidth + barSpacing));
|
||||
const barMaxHeight = rect.height * 0.8;
|
||||
const barMinHeight = 2;
|
||||
|
||||
// Smoothing for bars
|
||||
const smoothedHeights = new Array(barCount).fill(0);
|
||||
const targetHeights = new Array(barCount).fill(0);
|
||||
|
||||
const draw = () => {
|
||||
if (!isRecording) return;
|
||||
|
||||
animationRef.current = requestAnimationFrame(draw);
|
||||
|
||||
// Get frequency data
|
||||
analyser.getByteFrequencyData(dataArray);
|
||||
|
||||
// Clear canvas
|
||||
ctx.clearRect(0, 0, rect.width, rect.height);
|
||||
|
||||
// Calculate target heights based on frequency data
|
||||
for (let i = 0; i < barCount; i++) {
|
||||
const dataIndex = Math.floor((i / barCount) * bufferLength * 0.5); // Use lower frequencies
|
||||
const value = dataArray[dataIndex] / 255;
|
||||
|
||||
// Apply some randomness and minimum height for visual interest
|
||||
const randomFactor = 0.85 + Math.random() * 0.3;
|
||||
targetHeights[i] = Math.max(barMinHeight, value * barMaxHeight * randomFactor);
|
||||
}
|
||||
|
||||
// Smooth the bar heights
|
||||
for (let i = 0; i < barCount; i++) {
|
||||
const diff = targetHeights[i] - smoothedHeights[i];
|
||||
smoothedHeights[i] += diff * 0.3; // Smoothing factor
|
||||
}
|
||||
|
||||
// Draw bars
|
||||
for (let i = 0; i < barCount; i++) {
|
||||
const x = i * (barWidth + barSpacing) + barSpacing;
|
||||
const barHeight = smoothedHeights[i];
|
||||
const y = (rect.height - barHeight) / 2;
|
||||
|
||||
// Create gradient for each bar
|
||||
const gradient = ctx.createLinearGradient(0, y, 0, y + barHeight);
|
||||
|
||||
// Dynamic color based on height
|
||||
const intensity = barHeight / barMaxHeight;
|
||||
const hue = 200 + intensity * 20; // Blue to cyan
|
||||
const saturation = 50 + intensity * 50;
|
||||
const lightness = 50 + intensity * 20;
|
||||
|
||||
gradient.addColorStop(0, `hsla(${hue}, ${saturation}%, ${lightness}%, 0.3)`);
|
||||
gradient.addColorStop(0.5, `hsla(${hue}, ${saturation}%, ${lightness}%, 0.8)`);
|
||||
gradient.addColorStop(1, `hsla(${hue}, ${saturation}%, ${lightness}%, 0.3)`);
|
||||
|
||||
ctx.fillStyle = gradient;
|
||||
ctx.fillRect(x, y, barWidth, barHeight);
|
||||
}
|
||||
};
|
||||
|
||||
draw();
|
||||
|
||||
return () => {
|
||||
if (animationRef.current) {
|
||||
cancelAnimationFrame(animationRef.current);
|
||||
}
|
||||
};
|
||||
}, [analyser, isRecording]);
|
||||
|
||||
return (
|
||||
<canvas
|
||||
ref={canvasRef}
|
||||
className="absolute inset-0 w-full h-full pointer-events-none"
|
||||
style={{ opacity: 0.9 }}
|
||||
/>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,48 @@
|
||||
import React from 'react';
|
||||
|
||||
interface MicrophoneProps {
|
||||
className?: string;
|
||||
size?: number;
|
||||
}
|
||||
|
||||
export const Microphone: React.FC<MicrophoneProps> = ({ className = '', size = 24 }) => {
|
||||
return (
|
||||
<svg
|
||||
width={size}
|
||||
height={size}
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
className={className}
|
||||
>
|
||||
<path
|
||||
d="M12 14.5C13.66 14.5 15 13.16 15 11.5V5.5C15 3.84 13.66 2.5 12 2.5C10.34 2.5 9 3.84 9 5.5V11.5C9 13.16 10.34 14.5 12 14.5Z"
|
||||
stroke="currentColor"
|
||||
strokeWidth="2"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
/>
|
||||
<path
|
||||
d="M19 11.5C19 15.09 16.09 18 12.5 18C8.91 18 6 15.09 6 11.5"
|
||||
stroke="currentColor"
|
||||
strokeWidth="2"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
/>
|
||||
<path
|
||||
d="M12 18V21.5"
|
||||
stroke="currentColor"
|
||||
strokeWidth="2"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
/>
|
||||
<path
|
||||
d="M8 21.5H16"
|
||||
stroke="currentColor"
|
||||
strokeWidth="2"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
/>
|
||||
</svg>
|
||||
);
|
||||
};
|
||||
@@ -20,6 +20,7 @@ import Send from './Send';
|
||||
import Settings from './Settings';
|
||||
import Time from './Time';
|
||||
import { Gear } from './Gear';
|
||||
import { Microphone } from './Microphone';
|
||||
|
||||
export {
|
||||
ArrowDown,
|
||||
@@ -37,6 +38,7 @@ export {
|
||||
Edit,
|
||||
Idea,
|
||||
Gear,
|
||||
Microphone,
|
||||
More,
|
||||
Refresh,
|
||||
SensitiveHidden,
|
||||
|
||||
@@ -9,6 +9,7 @@ import SessionSharingSection from './sessions/SessionSharingSection';
|
||||
import { ResponseStylesSection } from './response_styles/ResponseStylesSection';
|
||||
import AppSettingsSection from './app/AppSettingsSection';
|
||||
import SchedulerSection from './scheduler/SchedulerSection';
|
||||
import DictationSection from './dictation/DictationSection';
|
||||
import { ExtensionConfig } from '../../api';
|
||||
import MoreMenuLayout from '../more_menu/MoreMenuLayout';
|
||||
|
||||
@@ -56,6 +57,8 @@ export default function SettingsView({
|
||||
<SessionSharingSection />
|
||||
{/* Response Styles */}
|
||||
<ResponseStylesSection />
|
||||
{/* Voice Dictation */}
|
||||
<DictationSection />
|
||||
{/* Tool Selection Strategy */}
|
||||
<ToolSelectionStrategySection setView={setView} />
|
||||
{/* App Settings */}
|
||||
|
||||
@@ -0,0 +1,276 @@
|
||||
import { useState, useEffect, useRef } from 'react';
|
||||
import { Switch } from '../../ui/switch';
|
||||
import { ChevronDown } from 'lucide-react';
|
||||
import { Input } from '../../ui/input';
|
||||
import { useConfig } from '../../ConfigContext';
|
||||
|
||||
type DictationProvider = 'openai' | 'elevenlabs';
|
||||
|
||||
interface DictationSettings {
|
||||
enabled: boolean;
|
||||
provider: DictationProvider;
|
||||
}
|
||||
|
||||
const DICTATION_SETTINGS_KEY = 'dictation_settings';
|
||||
const ELEVENLABS_API_KEY = 'ELEVENLABS_API_KEY';
|
||||
|
||||
export default function DictationSection() {
|
||||
const [settings, setSettings] = useState<DictationSettings>({
|
||||
enabled: true,
|
||||
provider: 'openai',
|
||||
});
|
||||
const [hasOpenAIKey, setHasOpenAIKey] = useState(false);
|
||||
const [showProviderDropdown, setShowProviderDropdown] = useState(false);
|
||||
const [showElevenLabsKey, setShowElevenLabsKey] = useState(false);
|
||||
const [elevenLabsApiKey, setElevenLabsApiKey] = useState('');
|
||||
const [isLoadingKey, setIsLoadingKey] = useState(false);
|
||||
const [hasElevenLabsKey, setHasElevenLabsKey] = useState(false);
|
||||
const elevenLabsApiKeyRef = useRef('');
|
||||
|
||||
const { getProviders, upsert, read } = useConfig();
|
||||
|
||||
// Load settings from localStorage and ElevenLabs API key from secure storage
|
||||
useEffect(() => {
|
||||
const loadSettings = async () => {
|
||||
const savedSettings = localStorage.getItem(DICTATION_SETTINGS_KEY);
|
||||
if (savedSettings) {
|
||||
const parsed = JSON.parse(savedSettings);
|
||||
setSettings(parsed);
|
||||
setShowElevenLabsKey(parsed.provider === 'elevenlabs');
|
||||
} else {
|
||||
// Default settings
|
||||
const defaultSettings: DictationSettings = {
|
||||
enabled: true,
|
||||
provider: 'openai',
|
||||
};
|
||||
setSettings(defaultSettings);
|
||||
localStorage.setItem(DICTATION_SETTINGS_KEY, JSON.stringify(defaultSettings));
|
||||
}
|
||||
|
||||
// Load ElevenLabs API key from storage
|
||||
setIsLoadingKey(true);
|
||||
try {
|
||||
// Try reading as secret - will return true if exists
|
||||
const keyExists = await read(ELEVENLABS_API_KEY, true);
|
||||
if (keyExists === true) {
|
||||
setHasElevenLabsKey(true);
|
||||
// Don't set the actual key since we can't read secrets
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error checking ElevenLabs API key:', error);
|
||||
} finally {
|
||||
setIsLoadingKey(false);
|
||||
}
|
||||
};
|
||||
|
||||
loadSettings();
|
||||
}, [read]);
|
||||
|
||||
// Save ElevenLabs key on unmount if it has changed
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (showElevenLabsKey && elevenLabsApiKeyRef.current) {
|
||||
// We can't use async in cleanup, so we'll use the promise directly
|
||||
const keyToSave = elevenLabsApiKeyRef.current;
|
||||
if (keyToSave.trim()) {
|
||||
upsert(ELEVENLABS_API_KEY, keyToSave, true).catch((error) => {
|
||||
console.error('Error saving ElevenLabs API key on unmount:', error);
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
}, [showElevenLabsKey, upsert]);
|
||||
|
||||
// Check if OpenAI is configured
|
||||
useEffect(() => {
|
||||
const checkOpenAIKey = async () => {
|
||||
try {
|
||||
const providers = await getProviders(false);
|
||||
const openAIProvider = providers.find((p) => p.name === 'openai');
|
||||
setHasOpenAIKey(openAIProvider?.is_configured || false);
|
||||
} catch (error) {
|
||||
console.error('Error checking OpenAI configuration:', error);
|
||||
setHasOpenAIKey(false);
|
||||
}
|
||||
};
|
||||
|
||||
checkOpenAIKey();
|
||||
}, [getProviders]);
|
||||
|
||||
const saveSettings = (newSettings: DictationSettings) => {
|
||||
setSettings(newSettings);
|
||||
localStorage.setItem(DICTATION_SETTINGS_KEY, JSON.stringify(newSettings));
|
||||
};
|
||||
|
||||
const handleToggle = (enabled: boolean) => {
|
||||
saveSettings({ ...settings, enabled });
|
||||
};
|
||||
|
||||
const handleProviderChange = (provider: DictationProvider) => {
|
||||
saveSettings({ ...settings, provider });
|
||||
setShowProviderDropdown(false);
|
||||
setShowElevenLabsKey(provider === 'elevenlabs');
|
||||
};
|
||||
|
||||
const handleElevenLabsKeyChange = (key: string) => {
|
||||
setElevenLabsApiKey(key);
|
||||
elevenLabsApiKeyRef.current = key;
|
||||
};
|
||||
|
||||
const saveElevenLabsKey = async () => {
|
||||
// Save to secure storage
|
||||
try {
|
||||
if (elevenLabsApiKey.trim()) {
|
||||
await upsert(ELEVENLABS_API_KEY, elevenLabsApiKey, true);
|
||||
setHasElevenLabsKey(true);
|
||||
} else {
|
||||
// If key is empty, remove it from storage
|
||||
await upsert(ELEVENLABS_API_KEY, null, true);
|
||||
setHasElevenLabsKey(false);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error saving ElevenLabs API key:', error);
|
||||
}
|
||||
};
|
||||
|
||||
const getProviderLabel = (provider: DictationProvider): string => {
|
||||
switch (provider) {
|
||||
case 'openai':
|
||||
return 'OpenAI Whisper';
|
||||
case 'elevenlabs':
|
||||
return 'ElevenLabs';
|
||||
default:
|
||||
return provider;
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<section id="dictation" className="px-8">
|
||||
<div className="flex justify-between items-center mb-2">
|
||||
<h2 className="text-xl font-medium text-textStandard">Voice Dictation</h2>
|
||||
</div>
|
||||
<div className="border-b border-borderSubtle pb-8">
|
||||
<p className="text-sm text-textStandard mb-6">Configure voice input for messages</p>
|
||||
|
||||
{/* Enable/Disable Toggle */}
|
||||
<div className="flex items-center justify-between mb-4">
|
||||
<div>
|
||||
<h3 className="text-textStandard">Enable Voice Dictation</h3>
|
||||
<p className="text-xs text-textSubtle max-w-md mt-[2px]">
|
||||
Show microphone button for voice input
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex items-center">
|
||||
<Switch checked={settings.enabled} onCheckedChange={handleToggle} variant="mono" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Provider Selection */}
|
||||
{settings.enabled && (
|
||||
<>
|
||||
<div className="flex items-center justify-between mb-4">
|
||||
<div>
|
||||
<h3 className="text-textStandard">Dictation Provider</h3>
|
||||
<p className="text-xs text-textSubtle max-w-md mt-[2px]">
|
||||
Choose how voice is converted to text
|
||||
</p>
|
||||
</div>
|
||||
<div className="relative">
|
||||
<button
|
||||
onClick={() => setShowProviderDropdown(!showProviderDropdown)}
|
||||
className="flex items-center gap-2 px-3 py-1.5 text-sm border border-borderSubtle rounded-md hover:border-borderStandard transition-colors text-textStandard bg-bgApp"
|
||||
>
|
||||
{getProviderLabel(settings.provider)}
|
||||
<ChevronDown className="w-4 h-4" />
|
||||
</button>
|
||||
|
||||
{showProviderDropdown && (
|
||||
<div className="absolute right-0 mt-1 w-48 bg-bgApp border border-borderStandard rounded-md shadow-lg z-10">
|
||||
<button
|
||||
onClick={() => handleProviderChange('openai')}
|
||||
disabled={!hasOpenAIKey}
|
||||
className={`w-full px-3 py-2 text-left text-sm transition-colors first:rounded-t-md ${
|
||||
hasOpenAIKey
|
||||
? 'hover:bg-bgSubtle text-textStandard'
|
||||
: 'text-textSubtle cursor-not-allowed'
|
||||
}`}
|
||||
>
|
||||
OpenAI Whisper
|
||||
{!hasOpenAIKey && <span className="text-xs ml-1">(not configured)</span>}
|
||||
{settings.provider === 'openai' && <span className="float-right">✓</span>}
|
||||
</button>
|
||||
|
||||
{/* ElevenLabs option */}
|
||||
<button
|
||||
onClick={() => handleProviderChange('elevenlabs')}
|
||||
className="w-full px-3 py-2 text-left text-sm hover:bg-bgSubtle transition-colors text-textStandard last:rounded-b-md"
|
||||
>
|
||||
ElevenLabs
|
||||
{settings.provider === 'elevenlabs' && <span className="float-right">✓</span>}
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* ElevenLabs API Key */}
|
||||
{showElevenLabsKey && (
|
||||
<div className="mb-4">
|
||||
<div className="mb-2">
|
||||
<h3 className="text-textStandard">ElevenLabs API Key</h3>
|
||||
<p className="text-xs text-textSubtle max-w-md mt-[2px]">
|
||||
Required for ElevenLabs voice recognition
|
||||
{hasElevenLabsKey && <span className="text-green-600 ml-2">(Configured)</span>}
|
||||
</p>
|
||||
</div>
|
||||
<Input
|
||||
type="password"
|
||||
value={elevenLabsApiKey}
|
||||
onChange={(e) => handleElevenLabsKeyChange(e.target.value)}
|
||||
onBlur={saveElevenLabsKey}
|
||||
placeholder={
|
||||
hasElevenLabsKey
|
||||
? 'Enter new API key to update'
|
||||
: 'Enter your ElevenLabs API key'
|
||||
}
|
||||
className="max-w-md"
|
||||
disabled={isLoadingKey}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Provider-specific information */}
|
||||
<div className="mt-4 p-3 bg-bgSubtle rounded-md">
|
||||
{settings.provider === 'openai' && (
|
||||
<p className="text-xs text-textSubtle">
|
||||
Uses OpenAI's Whisper API for high-quality transcription. Requires an OpenAI API
|
||||
key configured in the Models section.
|
||||
</p>
|
||||
)}
|
||||
{settings.provider === 'elevenlabs' && (
|
||||
<div>
|
||||
<p className="text-xs text-textSubtle">
|
||||
Uses ElevenLabs speech-to-text API for high-quality transcription.
|
||||
</p>
|
||||
<p className="text-xs text-textSubtle mt-2">
|
||||
<strong>Features:</strong>
|
||||
</p>
|
||||
<ul className="text-xs text-textSubtle ml-4 mt-1 list-disc">
|
||||
<li>Advanced voice processing</li>
|
||||
<li>High accuracy transcription</li>
|
||||
<li>Multiple language support</li>
|
||||
<li>Fast processing</li>
|
||||
</ul>
|
||||
<p className="text-xs text-textSubtle mt-2">
|
||||
<strong>Note:</strong> Requires an ElevenLabs API key with speech-to-text
|
||||
access.
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
import { useState, useEffect } from 'react';
|
||||
import { useConfig } from '../components/ConfigContext';
|
||||
|
||||
export type DictationProvider = 'openai' | 'elevenlabs';
|
||||
|
||||
export interface DictationSettings {
|
||||
enabled: boolean;
|
||||
provider: DictationProvider;
|
||||
}
|
||||
|
||||
const DICTATION_SETTINGS_KEY = 'dictation_settings';
|
||||
const ELEVENLABS_API_KEY = 'ELEVENLABS_API_KEY';
|
||||
|
||||
export const useDictationSettings = () => {
|
||||
const [settings, setSettings] = useState<DictationSettings | null>(null);
|
||||
const [hasElevenLabsKey, setHasElevenLabsKey] = useState<boolean>(false);
|
||||
const { read } = useConfig();
|
||||
|
||||
useEffect(() => {
|
||||
const loadSettings = async () => {
|
||||
// Load settings from localStorage
|
||||
const saved = localStorage.getItem(DICTATION_SETTINGS_KEY);
|
||||
if (saved) {
|
||||
setSettings(JSON.parse(saved));
|
||||
} else {
|
||||
// Default settings
|
||||
const defaultSettings: DictationSettings = {
|
||||
enabled: true,
|
||||
provider: 'openai',
|
||||
};
|
||||
setSettings(defaultSettings);
|
||||
}
|
||||
|
||||
// Load ElevenLabs API key from storage (non-secret for frontend access)
|
||||
try {
|
||||
const keyExists = await read(ELEVENLABS_API_KEY, true);
|
||||
if (keyExists === true) {
|
||||
setHasElevenLabsKey(true);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[useDictationSettings] Error loading ElevenLabs API key:', error);
|
||||
}
|
||||
};
|
||||
|
||||
loadSettings();
|
||||
|
||||
// Listen for storage changes from other tabs/windows
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const handleStorageChange = (e: any) => {
|
||||
if (e.key === DICTATION_SETTINGS_KEY && e.newValue) {
|
||||
setSettings(JSON.parse(e.newValue));
|
||||
}
|
||||
};
|
||||
|
||||
window.addEventListener('storage', handleStorageChange);
|
||||
return () => window.removeEventListener('storage', handleStorageChange);
|
||||
}, [read]);
|
||||
|
||||
return { settings, hasElevenLabsKey };
|
||||
};
|
||||
@@ -0,0 +1,298 @@
|
||||
import { useState, useRef, useCallback, useEffect } from 'react';
|
||||
import { useConfig } from '../components/ConfigContext';
|
||||
import { getApiUrl, getSecretKey } from '../config';
|
||||
import { useDictationSettings } from './useDictationSettings';
|
||||
|
||||
interface UseWhisperOptions {
|
||||
onTranscription?: (text: string) => void;
|
||||
onError?: (error: Error) => void;
|
||||
onSizeWarning?: (sizeInMB: number) => void;
|
||||
}
|
||||
|
||||
// Constants
|
||||
const MAX_AUDIO_SIZE_MB = 25;
|
||||
const MAX_RECORDING_DURATION_SECONDS = 600; // 10 minutes
|
||||
const WARNING_SIZE_MB = 20; // Warn at 20MB
|
||||
|
||||
export const useWhisper = ({ onTranscription, onError, onSizeWarning }: UseWhisperOptions = {}) => {
|
||||
const [isRecording, setIsRecording] = useState(false);
|
||||
const [isTranscribing, setIsTranscribing] = useState(false);
|
||||
const [hasOpenAIKey, setHasOpenAIKey] = useState(false);
|
||||
const [canUseDictation, setCanUseDictation] = useState(false);
|
||||
const [audioContext, setAudioContext] = useState<AudioContext | null>(null);
|
||||
const [analyser, setAnalyser] = useState<AnalyserNode | null>(null);
|
||||
const [recordingDuration, setRecordingDuration] = useState(0);
|
||||
const [estimatedSize, setEstimatedSize] = useState(0);
|
||||
|
||||
const mediaRecorderRef = useRef<MediaRecorder | null>(null);
|
||||
const audioChunksRef = useRef<Blob[]>([]);
|
||||
const streamRef = useRef<MediaStream | null>(null);
|
||||
const recordingStartTimeRef = useRef<number | null>(null);
|
||||
const durationIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||
const currentSizeRef = useRef<number>(0);
|
||||
|
||||
const { getProviders } = useConfig();
|
||||
const { settings: dictationSettings, hasElevenLabsKey } = useDictationSettings();
|
||||
|
||||
// Check if OpenAI API key is configured (regardless of current provider)
|
||||
useEffect(() => {
|
||||
const checkOpenAIKey = async () => {
|
||||
try {
|
||||
// Get all configured providers
|
||||
const providers = await getProviders(false);
|
||||
|
||||
// Find OpenAI provider
|
||||
const openAIProvider = providers.find((p) => p.name === 'openai');
|
||||
|
||||
// Check if OpenAI is configured
|
||||
if (openAIProvider && openAIProvider.is_configured) {
|
||||
setHasOpenAIKey(true);
|
||||
} else {
|
||||
setHasOpenAIKey(false);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error checking OpenAI configuration:', error);
|
||||
setHasOpenAIKey(false);
|
||||
}
|
||||
};
|
||||
|
||||
checkOpenAIKey();
|
||||
}, [getProviders]); // Re-check when providers change
|
||||
|
||||
// Check if dictation can be used based on settings
|
||||
useEffect(() => {
|
||||
if (!dictationSettings) {
|
||||
setCanUseDictation(false);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!dictationSettings.enabled) {
|
||||
setCanUseDictation(false);
|
||||
return;
|
||||
}
|
||||
|
||||
// Check provider availability
|
||||
switch (dictationSettings.provider) {
|
||||
case 'openai':
|
||||
setCanUseDictation(hasOpenAIKey);
|
||||
break;
|
||||
case 'elevenlabs':
|
||||
setCanUseDictation(hasElevenLabsKey);
|
||||
break;
|
||||
default:
|
||||
setCanUseDictation(false);
|
||||
}
|
||||
}, [dictationSettings, hasOpenAIKey, hasElevenLabsKey]);
|
||||
|
||||
const transcribeAudio = useCallback(
|
||||
async (audioBlob: Blob) => {
|
||||
if (!dictationSettings) {
|
||||
onError?.(new Error('Dictation settings not loaded'));
|
||||
return;
|
||||
}
|
||||
|
||||
setIsTranscribing(true);
|
||||
|
||||
try {
|
||||
// Check final size
|
||||
const sizeMB = audioBlob.size / (1024 * 1024);
|
||||
if (sizeMB > MAX_AUDIO_SIZE_MB) {
|
||||
throw new Error(
|
||||
`Audio file too large (${sizeMB.toFixed(1)}MB). Maximum size is ${MAX_AUDIO_SIZE_MB}MB.`
|
||||
);
|
||||
}
|
||||
|
||||
// Convert blob to base64 for easier transport
|
||||
const reader = new FileReader();
|
||||
const base64Audio = await new Promise<string>((resolve, reject) => {
|
||||
reader.onloadend = () => {
|
||||
const base64 = reader.result as string;
|
||||
resolve(base64.split(',')[1]); // Remove data:audio/webm;base64, prefix
|
||||
};
|
||||
reader.onerror = reject;
|
||||
reader.readAsDataURL(audioBlob);
|
||||
});
|
||||
|
||||
let endpoint = '';
|
||||
let headers: Record<string, string> = {
|
||||
'Content-Type': 'application/json',
|
||||
'X-Secret-Key': getSecretKey(),
|
||||
};
|
||||
let body: Record<string, string> = {
|
||||
audio: base64Audio,
|
||||
mime_type: 'audio/webm',
|
||||
};
|
||||
|
||||
// Choose endpoint based on provider
|
||||
switch (dictationSettings.provider) {
|
||||
case 'openai':
|
||||
endpoint = '/audio/transcribe';
|
||||
break;
|
||||
case 'elevenlabs':
|
||||
endpoint = '/audio/transcribe/elevenlabs';
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Unsupported provider: ${dictationSettings.provider}`);
|
||||
}
|
||||
|
||||
const response = await fetch(getApiUrl(endpoint), {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) {
|
||||
throw new Error(
|
||||
`Audio transcription endpoint not found. Please implement ${endpoint} endpoint in the Goose backend.`
|
||||
);
|
||||
} else if (response.status === 401) {
|
||||
throw new Error('Invalid API key. Please check your API key is correct.');
|
||||
} else if (response.status === 402) {
|
||||
throw new Error('API quota exceeded. Please check your account limits.');
|
||||
}
|
||||
const errorData = await response
|
||||
.json()
|
||||
.catch(() => ({ error: { message: 'Transcription failed' } }));
|
||||
throw new Error(errorData.error?.message || 'Transcription failed');
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
if (data.text) {
|
||||
onTranscription?.(data.text);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error transcribing audio:', error);
|
||||
onError?.(error as Error);
|
||||
} finally {
|
||||
setIsTranscribing(false);
|
||||
setRecordingDuration(0);
|
||||
setEstimatedSize(0);
|
||||
}
|
||||
},
|
||||
[onTranscription, onError, dictationSettings]
|
||||
);
|
||||
|
||||
// Define stopRecording before startRecording to avoid circular dependency
|
||||
const stopRecording = useCallback(() => {
|
||||
if (mediaRecorderRef.current && mediaRecorderRef.current.state !== 'inactive') {
|
||||
mediaRecorderRef.current.stop();
|
||||
setIsRecording(false);
|
||||
}
|
||||
|
||||
// Clear interval
|
||||
if (durationIntervalRef.current) {
|
||||
clearInterval(durationIntervalRef.current);
|
||||
durationIntervalRef.current = null;
|
||||
}
|
||||
|
||||
// Stop all tracks in the stream
|
||||
if (streamRef.current) {
|
||||
streamRef.current.getTracks().forEach((track) => track.stop());
|
||||
streamRef.current = null;
|
||||
}
|
||||
|
||||
// Close audio context
|
||||
if (audioContext) {
|
||||
audioContext.close();
|
||||
setAudioContext(null);
|
||||
setAnalyser(null);
|
||||
}
|
||||
}, [audioContext]);
|
||||
|
||||
const startRecording = useCallback(async () => {
|
||||
if (!dictationSettings) {
|
||||
onError?.(new Error('Dictation settings not loaded'));
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// Request microphone permission
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
streamRef.current = stream;
|
||||
|
||||
// Create audio context and analyser for visualization
|
||||
const context = new AudioContext();
|
||||
const source = context.createMediaStreamSource(stream);
|
||||
const analyserNode = context.createAnalyser();
|
||||
analyserNode.fftSize = 2048;
|
||||
source.connect(analyserNode);
|
||||
|
||||
setAudioContext(context);
|
||||
setAnalyser(analyserNode);
|
||||
|
||||
// Create MediaRecorder
|
||||
const mediaRecorder = new MediaRecorder(stream, {
|
||||
mimeType: 'audio/webm',
|
||||
});
|
||||
|
||||
mediaRecorderRef.current = mediaRecorder;
|
||||
audioChunksRef.current = [];
|
||||
currentSizeRef.current = 0;
|
||||
recordingStartTimeRef.current = Date.now();
|
||||
|
||||
// Start duration and size tracking
|
||||
durationIntervalRef.current = setInterval(() => {
|
||||
const elapsed = (Date.now() - (recordingStartTimeRef.current || 0)) / 1000;
|
||||
setRecordingDuration(elapsed);
|
||||
|
||||
// Estimate size based on typical webm bitrate (~128kbps)
|
||||
const estimatedSizeMB = (elapsed * 128 * 1024) / (8 * 1024 * 1024);
|
||||
setEstimatedSize(estimatedSizeMB);
|
||||
|
||||
// Check if we're approaching the limit
|
||||
if (estimatedSizeMB > WARNING_SIZE_MB) {
|
||||
onSizeWarning?.(estimatedSizeMB);
|
||||
}
|
||||
|
||||
// Auto-stop if we hit the duration limit
|
||||
if (elapsed >= MAX_RECORDING_DURATION_SECONDS) {
|
||||
stopRecording();
|
||||
onError?.(
|
||||
new Error(
|
||||
`Maximum recording duration (${MAX_RECORDING_DURATION_SECONDS / 60} minutes) reached.`
|
||||
)
|
||||
);
|
||||
}
|
||||
}, 100);
|
||||
|
||||
mediaRecorder.ondataavailable = (event) => {
|
||||
if (event.data.size > 0) {
|
||||
audioChunksRef.current.push(event.data);
|
||||
currentSizeRef.current += event.data.size;
|
||||
|
||||
// Check actual size
|
||||
const actualSizeMB = currentSizeRef.current / (1024 * 1024);
|
||||
if (actualSizeMB > MAX_AUDIO_SIZE_MB) {
|
||||
stopRecording();
|
||||
onError?.(new Error(`Maximum file size (${MAX_AUDIO_SIZE_MB}MB) reached.`));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
mediaRecorder.onstop = async () => {
|
||||
const audioBlob = new Blob(audioChunksRef.current, { type: 'audio/webm' });
|
||||
await transcribeAudio(audioBlob);
|
||||
};
|
||||
|
||||
mediaRecorder.start(1000); // Collect data every second for size monitoring
|
||||
setIsRecording(true);
|
||||
} catch (error) {
|
||||
console.error('Error starting recording:', error);
|
||||
onError?.(error as Error);
|
||||
}
|
||||
}, [onError, onSizeWarning, transcribeAudio, stopRecording, dictationSettings]);
|
||||
|
||||
return {
|
||||
isRecording,
|
||||
isTranscribing,
|
||||
hasOpenAIKey,
|
||||
canUseDictation,
|
||||
audioContext,
|
||||
analyser,
|
||||
startRecording,
|
||||
stopRecording,
|
||||
recordingDuration,
|
||||
estimatedSize,
|
||||
};
|
||||
};
|
||||
+18
-2
@@ -549,6 +549,10 @@ const createChat = async (
|
||||
webPreferences: {
|
||||
spellcheck: true,
|
||||
preload: path.join(__dirname, 'preload.js'),
|
||||
// Enable features needed for Web Speech API
|
||||
webSecurity: true,
|
||||
nodeIntegration: false,
|
||||
contextIsolation: true,
|
||||
additionalArguments: [
|
||||
JSON.stringify({
|
||||
...appConfig, // Use the potentially updated appConfig
|
||||
@@ -1444,6 +1448,18 @@ app.whenReady().then(async () => {
|
||||
// Register update IPC handlers once (but don't setup auto-updater yet)
|
||||
registerUpdateIpcHandlers();
|
||||
|
||||
// Handle microphone permission requests
|
||||
session.defaultSession.setPermissionRequestHandler((_webContents, permission, callback) => {
|
||||
console.log('Permission requested:', permission);
|
||||
// Allow microphone and media access
|
||||
if (permission === 'media') {
|
||||
callback(true);
|
||||
} else {
|
||||
// Default behavior for other permissions
|
||||
callback(true);
|
||||
}
|
||||
});
|
||||
|
||||
// Add CSP headers to all sessions
|
||||
session.defaultSession.webRequest.onHeadersReceived((details, callback) => {
|
||||
callback({
|
||||
@@ -1465,8 +1481,8 @@ app.whenReady().then(async () => {
|
||||
"frame-src 'none';" +
|
||||
// Font sources
|
||||
"font-src 'self';" +
|
||||
// Media sources
|
||||
"media-src 'none';" +
|
||||
// Media sources - allow microphone
|
||||
"media-src 'self' mediastream:;" +
|
||||
// Form actions
|
||||
"form-action 'none';" +
|
||||
// Base URI restriction
|
||||
|
||||
Reference in New Issue
Block a user