tkmind/tkmind_go
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

Rejig dictation (#6844)

Browse Source 
Co-authored-by: Douwe Osinga <douwe@squareup.com>
This commit is contained in:
Douwe Osinga
2026-02-02 14:25:23 +01:00
committed by GitHub
parent dd15fd0dd8
commit 2661454426
22 changed files with 1177 additions and 1448 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/goose-server/src/openapi.rs
+6
View File
@@ -412,6 +412,8 @@ derive_utoipa!(Icon as IconSchema);
super::routes::tunnel::stop_tunnel,
super::routes::tunnel::get_tunnel_status,
super::routes::telemetry::send_telemetry_event,
super::routes::dictation::transcribe_dictation,
super::routes::dictation::get_dictation_config,
),
components(schemas(
super::routes::config_management::UpsertConfigQuery,
@@ -570,6 +572,10 @@ derive_utoipa!(Icon as IconSchema);
goose::goose_apps::CspMetadata,
goose::goose_apps::UiMetadata,
goose::goose_apps::ResourceMetadata,
super::routes::dictation::TranscribeRequest,
super::routes::dictation::TranscribeResponse,
super::routes::dictation::DictationProvider,
super::routes::dictation::DictationProviderStatus,
))
)]
pub struct ApiDoc;
crates/goose-server/src/routes/audio.rs
-512
View File
@@ -1,512 +0,0 @@
/// Audio transcription route handler
///
/// This module provides endpoints for audio transcription using OpenAI's Whisper API.
/// The OpenAI API key must be configured in the backend for this to work.
use crate::routes::errors::ErrorResponse;
use crate::state::AppState;
use axum::{
http::StatusCode,
routing::{get, post},
Json, Router,
};
use base64::{engine::general_purpose::STANDARD as BASE64, Engine};
use reqwest::Client;
use serde::{Deserialize, Serialize};
use std::sync::Arc;
use std::time::Duration;
// Constants
const MAX_AUDIO_SIZE_BYTES: usize = 25 * 1024 * 1024; // 25MB
const OPENAI_TIMEOUT_SECONDS: u64 = 30;
#[derive(Debug, Deserialize)]
struct TranscribeRequest {
audio: String, // Base64 encoded audio data
mime_type: String,
}
#[derive(Debug, Deserialize)]
struct TranscribeElevenLabsRequest {
audio: String, // Base64 encoded audio data
mime_type: String,
}
#[derive(Debug, Serialize)]
struct TranscribeResponse {
text: String,
}
#[derive(Debug, Deserialize)]
struct WhisperResponse {
text: String,
}
/// Validate audio input and return decoded bytes and file extension
fn validate_audio_input(
audio: &str,
mime_type: &str,
) -> Result<(Vec<u8>, &'static str), ErrorResponse> {
// Decode the base64 audio data
let audio_bytes = BASE64
.decode(audio)
.map_err(|_| ErrorResponse::bad_request("Invalid base64 audio data"))?;
// Check file size
if audio_bytes.len() > MAX_AUDIO_SIZE_BYTES {
return Err(ErrorResponse {
message: format!(
"Audio file too large: {} bytes (max: {} bytes)",
audio_bytes.len(),
MAX_AUDIO_SIZE_BYTES
),
status: StatusCode::PAYLOAD_TOO_LARGE,
});
}
// Determine file extension based on MIME type
let file_extension = match mime_type {
"audio/webm" => "webm",
"audio/webm;codecs=opus" => "webm",
"audio/mp4" => "mp4",
"audio/mpeg" => "mp3",
"audio/mpga" => "mpga",
"audio/m4a" => "m4a",
"audio/wav" => "wav",
"audio/x-wav" => "wav",
_ => {
return Err(ErrorResponse {
message: format!("Unsupported audio format: {}", mime_type),
status: StatusCode::UNSUPPORTED_MEDIA_TYPE,
})
}
};
Ok((audio_bytes, file_extension))
}
/// Get OpenAI configuration (API key and host)
fn get_openai_config() -> Result<(String, String), ErrorResponse> {
let config = goose::config::Config::global();
let api_key: String = config.get_secret("OPENAI_API_KEY").map_err(|e| match e {
goose::config::ConfigError::NotFound(_) => ErrorResponse {
message: "OpenAI API key not configured. Please set OPENAI_API_KEY in settings."
.to_string(),
status: StatusCode::PRECONDITION_FAILED,
},
_ => ErrorResponse::internal(format!("Failed to get OpenAI API key: {:?}", e)),
})?;
let openai_host = match config.get("OPENAI_HOST", false) {
Ok(value) => value
.as_str()
.map(|s| s.to_string())
.unwrap_or_else(|| "https://api.openai.com".to_string()),
Err(_) => "https://api.openai.com".to_string(),
};
Ok((api_key, openai_host))
}
/// Send transcription request to OpenAI Whisper API
async fn send_openai_request(
audio_bytes: Vec<u8>,
file_extension: &str,
mime_type: &str,
api_key: &str,
openai_host: &str,
) -> Result<WhisperResponse, ErrorResponse> {
tracing::info!("Using OpenAI host: {}", openai_host);
tracing::info!(
"Audio file size: {} bytes, extension: {}, mime_type: {}",
audio_bytes.len(),
file_extension,
mime_type
);
// Create a multipart form with the audio file
let part = reqwest::multipart::Part::bytes(audio_bytes)
.file_name(format!("audio.{}", file_extension))
.mime_str(mime_type)
.map_err(|e| {
ErrorResponse::internal(format!("Failed to create multipart part: {:?}", e))
})?;
let form = reqwest::multipart::Form::new()
.part("file", part)
.text("model", "whisper-1")
.text("response_format", "json");
tracing::info!("Created multipart form for OpenAI Whisper API");
// Make request to OpenAI Whisper API
let client = Client::builder()
.timeout(Duration::from_secs(OPENAI_TIMEOUT_SECONDS))
.build()
.map_err(|e| ErrorResponse::internal(format!("Failed to create HTTP client: {}", e)))?;
tracing::info!(
"Sending request to OpenAI: {}/v1/audio/transcriptions",
openai_host
);
let response = client
.post(format!("{}/v1/audio/transcriptions", openai_host))
.header("Authorization", format!("Bearer {}", api_key))
.multipart(form)
.send()
.await
.map_err(|e| {
if e.is_timeout() {
ErrorResponse {
message: format!(
"OpenAI API request timed out after {}s",
OPENAI_TIMEOUT_SECONDS
),
status: StatusCode::GATEWAY_TIMEOUT,
}
} else {
ErrorResponse {
message: format!("Failed to send request to OpenAI: {}", e),
status: StatusCode::SERVICE_UNAVAILABLE,
}
}
})?;
tracing::info!(
"Received response from OpenAI with status: {}",
response.status()
);
if !response.status().is_success() {
let status = response.status();
let error_text = response.text().await.unwrap_or_default();
tracing::error!("OpenAI API error (status: {}): {}", status, error_text);
// Check for specific error codes
if status == 401 {
return Err(ErrorResponse {
message: "OpenAI API key appears to be invalid or unauthorized".to_string(),
status: StatusCode::UNAUTHORIZED,
});
} else if status == 429 {
return Err(ErrorResponse {
message: "OpenAI API quota or rate limit exceeded".to_string(),
status: StatusCode::TOO_MANY_REQUESTS,
});
}
return Err(ErrorResponse {
message: format!("OpenAI API error: {}", error_text),
status: StatusCode::BAD_GATEWAY,
});
}
let whisper_response: WhisperResponse = response
.json()
.await
.map_err(|e| ErrorResponse::internal(format!("Failed to parse OpenAI response: {}", e)))?;
Ok(whisper_response)
}
/// Transcribe audio using OpenAI's Whisper API
///
/// # Request
/// - `audio`: Base64 encoded audio data
/// - `mime_type`: MIME type of the audio (e.g., "audio/webm", "audio/wav")
///
/// # Response
/// - `text`: Transcribed text from the audio
///
/// # Errors
/// - 401: Unauthorized (missing or invalid X-Secret-Key header)
/// - 412: Precondition Failed (OpenAI API key not configured)
/// - 400: Bad Request (invalid base64 audio data)
/// - 413: Payload Too Large (audio file exceeds 25MB limit)
/// - 415: Unsupported Media Type (unsupported audio format)
/// - 502: Bad Gateway (OpenAI API error)
/// - 503: Service Unavailable (network error)
async fn transcribe_handler(
Json(request): Json<TranscribeRequest>,
) -> Result<Json<TranscribeResponse>, ErrorResponse> {
let (audio_bytes, file_extension) = validate_audio_input(&request.audio, &request.mime_type)?;
let (api_key, openai_host) = get_openai_config()?;
let whisper_response = send_openai_request(
audio_bytes,
file_extension,
&request.mime_type,
&api_key,
&openai_host,
)
.await?;
Ok(Json(TranscribeResponse {
text: whisper_response.text,
}))
}
/// Transcribe audio using ElevenLabs Speech-to-Text API
///
/// Uses ElevenLabs' speech-to-text endpoint for transcription.
/// Requires an ElevenLabs API key with speech-to-text access.
async fn transcribe_elevenlabs_handler(
Json(request): Json<TranscribeElevenLabsRequest>,
) -> Result<Json<TranscribeResponse>, ErrorResponse> {
let (audio_bytes, file_extension) = validate_audio_input(&request.audio, &request.mime_type)?;
// Get the ElevenLabs API key from config (after input validation)
let config = goose::config::Config::global();
// First try to get it as a secret
let api_key: String = match config.get_secret::<String>("ELEVENLABS_API_KEY") {
Ok(key) => key,
Err(_) => {
// Try to get it as non-secret (for backward compatibility)
match config.get("ELEVENLABS_API_KEY", false) {
Ok(value) => {
match value.as_str() {
Some(key_str) => {
let key = key_str.to_string();
// Migrate to secret storage
if let Err(e) = config.set(
"ELEVENLABS_API_KEY",
&serde_json::Value::String(key.clone()),
true,
) {
tracing::error!("Failed to migrate ElevenLabs API key: {:?}", e);
}
// Delete the non-secret version
if let Err(e) = config.delete("ELEVENLABS_API_KEY") {
tracing::warn!(
"Failed to delete non-secret ElevenLabs API key: {:?}",
e
);
}
key
}
None => {
return Err(ErrorResponse::bad_request(format!(
"ElevenLabs API key is not a string, found: {:?}",
value
)));
}
}
}
Err(_) => {
return Err(ErrorResponse::bad_request(
"No ElevenLabs API key found in configuration",
));
}
}
}
};
// Create multipart form for ElevenLabs API
let part = reqwest::multipart::Part::bytes(audio_bytes)
.file_name(format!("audio.{}", file_extension))
.mime_str(&request.mime_type)
.map_err(|_| ErrorResponse::internal("Failed to create multipart part"))?;
let form = reqwest::multipart::Form::new()
.part("file", part) // Changed from "audio" to "file"
.text("model_id", "scribe_v1") // Use the correct model_id for speech-to-text
.text("tag_audio_events", "false")
.text("diarize", "false");
// Make request to ElevenLabs Speech-to-Text API
let client = Client::builder()
.timeout(Duration::from_secs(OPENAI_TIMEOUT_SECONDS))
.build()
.map_err(|e| ErrorResponse::internal(format!("Failed to create HTTP client: {}", e)))?;
let response = client
.post("https://api.elevenlabs.io/v1/speech-to-text")
.header("xi-api-key", &api_key)
.multipart(form)
.send()
.await
.map_err(|e| {
if e.is_timeout() {
ErrorResponse {
message: format!(
"ElevenLabs API request timed out after {}s",
OPENAI_TIMEOUT_SECONDS
),
status: StatusCode::GATEWAY_TIMEOUT,
}
} else {
ErrorResponse {
message: format!("Failed to send request to ElevenLabs: {}", e),
status: StatusCode::SERVICE_UNAVAILABLE,
}
}
})?;
if !response.status().is_success() {
let status = response.status();
let error_text = response.text().await.unwrap_or_default();
tracing::error!("ElevenLabs API error (status: {}): {}", status, error_text);
// Check for specific error codes
if error_text.contains("Unauthorized") || error_text.contains("Invalid API key") {
return Err(ErrorResponse {
message: "ElevenLabs API key is invalid or unauthorized".to_string(),
status: StatusCode::UNAUTHORIZED,
});
} else if error_text.contains("quota") || error_text.contains("limit") {
return Err(ErrorResponse {
message: "ElevenLabs API quota or rate limit exceeded".to_string(),
status: StatusCode::PAYMENT_REQUIRED,
});
}
return Err(ErrorResponse {
message: format!("ElevenLabs API error: {}", error_text),
status: StatusCode::BAD_GATEWAY,
});
}
// Parse ElevenLabs response
#[derive(Debug, Deserialize)]
struct ElevenLabsResponse {
text: String,
#[serde(rename = "chunks")]
#[allow(dead_code)]
_chunks: Option<Vec<serde_json::Value>>,
}
let elevenlabs_response: ElevenLabsResponse = response.json().await.map_err(|e| {
ErrorResponse::internal(format!("Failed to parse ElevenLabs response: {}", e))
})?;
Ok(Json(TranscribeResponse {
text: elevenlabs_response.text,
}))
}
/// Check if dictation providers are configured
///
/// Returns configuration status for dictation providers
async fn check_dictation_config() -> Result<Json<serde_json::Value>, ErrorResponse> {
let config = goose::config::Config::global();
// Check if ElevenLabs API key is configured
let has_elevenlabs = match config.get_secret::<String>("ELEVENLABS_API_KEY") {
Ok(_) => true,
Err(_) => {
// Check non-secret for backward compatibility
config.get("ELEVENLABS_API_KEY", false).is_ok()
}
};
Ok(Json(serde_json::json!({
"elevenlabs": has_elevenlabs
})))
}
pub fn routes(state: Arc<AppState>) -> Router {
Router::new()
.route("/audio/transcribe", post(transcribe_handler))
.route(
"/audio/transcribe/elevenlabs",
post(transcribe_elevenlabs_handler),
)
.route("/audio/config", get(check_dictation_config))
.with_state(state)
}
#[cfg(test)]
mod tests {
use super::*;
use axum::{body::Body, http::Request};
use tower::ServiceExt;
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
#[tokio::test(flavor = "multi_thread")]
async fn test_transcribe_endpoint_requires_auth() {
let mock_server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/audio/transcriptions"))
.respond_with(ResponseTemplate::new(401))
.mount(&mock_server)
.await;
let _guard = env_lock::lock_env([
("OPENAI_API_KEY", Some("fake-key")),
("OPENAI_HOST", Some(mock_server.uri().as_str())),
]);
let state = AppState::new().await.unwrap();
let app = routes(state);
let request = Request::builder()
.uri("/audio/transcribe")
.method("POST")
.header("content-type", "application/json")
.body(Body::from(
serde_json::to_string(&serde_json::json!({
"audio": "dGVzdA==",
"mime_type": "audio/webm"
}))
.unwrap(),
))
.unwrap();
let response = app.oneshot(request).await.unwrap();
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
}
#[tokio::test(flavor = "multi_thread")]
async fn test_transcribe_endpoint_validates_size() {
let state = AppState::new().await.unwrap();
let app = routes(state);
let request = Request::builder()
.uri("/audio/transcribe")
.method("POST")
.header("content-type", "application/json")
.header("x-secret-key", "test-secret")
.body(Body::from(
serde_json::to_string(&serde_json::json!({
"audio": "dGVzdA==",
"mime_type": "application/pdf" // Invalid MIME type
}))
.unwrap(),
))
.unwrap();
let response = app.oneshot(request).await.unwrap();
assert!(
response.status() == StatusCode::UNSUPPORTED_MEDIA_TYPE
|| response.status() == StatusCode::PRECONDITION_FAILED
);
}
#[tokio::test(flavor = "multi_thread")]
async fn test_transcribe_endpoint_validates_mime_type() {
let state = AppState::new().await.unwrap();
let app = routes(state);
let request = Request::builder()
.uri("/audio/transcribe")
.method("POST")
.header("content-type", "application/json")
.header("x-secret-key", "test-secret")
.body(Body::from(
serde_json::to_string(&serde_json::json!({
"audio": "invalid-base64-!@#$%",
"mime_type": "audio/webm"
}))
.unwrap(),
))
.unwrap();
let response = app.oneshot(request).await.unwrap();
assert!(
response.status() == StatusCode::BAD_REQUEST
|| response.status() == StatusCode::PRECONDITION_FAILED
);
}
}
crates/goose-server/src/routes/dictation.rs
+389
View File
@@ -0,0 +1,389 @@
use crate::routes::errors::ErrorResponse;
use crate::state::AppState;
use axum::{
http::StatusCode,
routing::{get, post},
Json, Router,
};
use base64::{engine::general_purpose::STANDARD as BASE64, Engine};
use goose::providers::api_client::{ApiClient, AuthMethod};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
use utoipa::ToSchema;
const MAX_AUDIO_SIZE_BYTES: usize = 25 * 1024 * 1024;
const REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
// DictationProvider definitions
struct DictationProviderDef {
config_key: &'static str,
default_url: &'static str,
host_key: Option<&'static str>,
description: &'static str,
uses_provider_config: bool,
settings_path: Option<&'static str>,
}
const PROVIDERS: &[(&str, DictationProviderDef)] = &[
(
"openai",
DictationProviderDef {
config_key: "OPENAI_API_KEY",
default_url: "https://api.openai.com/v1/audio/transcriptions",
host_key: Some("OPENAI_HOST"),
description: "Uses OpenAI Whisper API for high-quality transcription.",
uses_provider_config: true,
settings_path: Some("Settings > Models"),
},
),
(
"elevenlabs",
DictationProviderDef {
config_key: "ELEVENLABS_API_KEY",
default_url: "https://api.elevenlabs.io/v1/speech-to-text",
host_key: None,
description: "Uses ElevenLabs speech-to-text API for advanced voice processing.",
uses_provider_config: false,
settings_path: None,
},
),
];
fn get_provider_def(name: &str) -> Option<&'static DictationProviderDef> {
PROVIDERS
.iter()
.find_map(|(n, def)| if *n == name { Some(def) } else { None })
}
#[derive(Debug, Deserialize, ToSchema)]
#[serde(rename_all = "lowercase")]
pub enum DictationProvider {
OpenAI,
ElevenLabs,
}
impl DictationProvider {
fn as_str(&self) -> &'static str {
match self {
DictationProvider::OpenAI => "openai",
DictationProvider::ElevenLabs => "elevenlabs",
}
}
}
#[derive(Debug, Deserialize, ToSchema)]
pub struct TranscribeRequest {
/// Base64 encoded audio data
pub audio: String,
/// MIME type of the audio (e.g., "audio/webm", "audio/wav")
pub mime_type: String,
/// Transcription provider to use
pub provider: DictationProvider,
}
#[derive(Debug, Serialize, Deserialize, ToSchema)]
pub struct TranscribeResponse {
/// Transcribed text from the audio
pub text: String,
}
#[derive(Debug, Serialize, ToSchema)]
pub struct DictationProviderStatus {
/// Whether the provider is fully configured and ready to use
pub configured: bool,
/// Custom host URL if configured (only for providers that support it)
#[serde(skip_serializing_if = "Option::is_none")]
pub host: Option<String>,
/// Description of what this provider does
pub description: String,
/// Whether this provider uses the main provider config (true) or has its own key (false)
pub uses_provider_config: bool,
/// Path to settings if uses_provider_config is true
#[serde(skip_serializing_if = "Option::is_none")]
pub settings_path: Option<String>,
/// Config key name if uses_provider_config is false
#[serde(skip_serializing_if = "Option::is_none")]
pub config_key: Option<String>,
}
fn validate_audio(audio: &str, mime_type: &str) -> Result<(Vec<u8>, &'static str), ErrorResponse> {
let audio_bytes = BASE64
.decode(audio)
.map_err(|_| ErrorResponse::bad_request("Invalid base64 audio data"))?;
if audio_bytes.len() > MAX_AUDIO_SIZE_BYTES {
return Err(ErrorResponse {
message: format!(
"Audio file too large: {} bytes (max: {} bytes)",
audio_bytes.len(),
MAX_AUDIO_SIZE_BYTES
),
status: StatusCode::PAYLOAD_TOO_LARGE,
});
}
let extension = match mime_type {
"audio/webm" | "audio/webm;codecs=opus" => "webm",
"audio/mp4" => "mp4",
"audio/mpeg" | "audio/mpga" => "mp3",
"audio/m4a" => "m4a",
"audio/wav" | "audio/x-wav" => "wav",
_ => {
return Err(ErrorResponse {
message: format!("Unsupported audio format: {}", mime_type),
status: StatusCode::UNSUPPORTED_MEDIA_TYPE,
})
}
};
Ok((audio_bytes, extension))
}
async fn handle_response_error(response: reqwest::Response) -> ErrorResponse {
let status = response.status();
let error_text = response.text().await.unwrap_or_default();
ErrorResponse {
message: if status == 401
|| error_text.contains("Invalid API key")
|| error_text.contains("Unauthorized")
{
"Invalid API key".to_string()
} else if status == 429 || error_text.contains("quota") || error_text.contains("limit") {
"Rate limit exceeded".to_string()
} else {
format!("API error: {}", error_text)
},
status: if status.is_client_error() {
status
} else {
StatusCode::BAD_GATEWAY
},
}
}
fn build_api_client(provider: &str) -> Result<ApiClient, ErrorResponse> {
let config = goose::config::Config::global();
let def = get_provider_def(provider)
.ok_or_else(|| ErrorResponse::bad_request(format!("Unknown provider: {}", provider)))?;
let api_key = config
.get_secret(def.config_key)
.map_err(|_| ErrorResponse {
message: format!("{} not configured", def.config_key),
status: StatusCode::PRECONDITION_FAILED,
})?;
let url = if let Some(host_key) = def.host_key {
config
.get(host_key, false)
.ok()
.and_then(|v| v.as_str().map(|s| s.to_string()))
.map(|custom_host| {
let path = def
.default_url
.splitn(4, '/')
.nth(3)
.map(|p| format!("/{}", p))
.unwrap_or_default();
format!("{}{}", custom_host.trim_end_matches('/'), path)
})
.unwrap_or_else(|| def.default_url.to_string())
} else {
def.default_url.to_string()
};
let auth = match provider {
"openai" => AuthMethod::BearerToken(api_key),
"elevenlabs" => AuthMethod::ApiKey {
header_name: "xi-api-key".to_string(),
key: api_key,
},
_ => {
return Err(ErrorResponse::bad_request(format!(
"Unknown provider: {}",
provider
)))
}
};
ApiClient::with_timeout(url, auth, REQUEST_TIMEOUT)
.map_err(|e| ErrorResponse::internal(format!("Failed to create client: {}", e)))
}
async fn transcribe_openai(
audio_bytes: Vec<u8>,
extension: &str,
mime_type: &str,
client: &ApiClient,
) -> Result<String, ErrorResponse> {
let part = reqwest::multipart::Part::bytes(audio_bytes)
.file_name(format!("audio.{}", extension))
.mime_str(mime_type)
.map_err(|e| ErrorResponse::internal(format!("Failed to create multipart: {}", e)))?;
let form = reqwest::multipart::Form::new()
.part("file", part)
.text("model", "whisper-1");
let response = client
.request(None, "")
.multipart_post(form)
.await
.map_err(|e| ErrorResponse {
message: if e.to_string().contains("timeout") {
"Request timed out".to_string()
} else {
format!("Request failed: {}", e)
},
status: if e.to_string().contains("timeout") {
StatusCode::GATEWAY_TIMEOUT
} else {
StatusCode::SERVICE_UNAVAILABLE
},
})?;
if !response.status().is_success() {
return Err(handle_response_error(response).await);
}
let data: TranscribeResponse = response
.json()
.await
.map_err(|e| ErrorResponse::internal(format!("Failed to parse response: {}", e)))?;
Ok(data.text)
}
async fn transcribe_elevenlabs(
audio_bytes: Vec<u8>,
extension: &str,
mime_type: &str,
client: &ApiClient,
) -> Result<String, ErrorResponse> {
let part = reqwest::multipart::Part::bytes(audio_bytes)
.file_name(format!("audio.{}", extension))
.mime_str(mime_type)
.map_err(|_| ErrorResponse::internal("Failed to create multipart"))?;
let form = reqwest::multipart::Form::new()
.part("file", part)
.text("model_id", "scribe_v1");
let response = client
.request(None, "")
.multipart_post(form)
.await
.map_err(|e| ErrorResponse {
message: if e.to_string().contains("timeout") {
"Request timed out".to_string()
} else {
format!("Request failed: {}", e)
},
status: if e.to_string().contains("timeout") {
StatusCode::GATEWAY_TIMEOUT
} else {
StatusCode::SERVICE_UNAVAILABLE
},
})?;
if !response.status().is_success() {
return Err(handle_response_error(response).await);
}
let data: TranscribeResponse = response
.json()
.await
.map_err(|e| ErrorResponse::internal(format!("Failed to parse response: {}", e)))?;
Ok(data.text)
}
#[utoipa::path(
post,
path = "/dictation/transcribe",
request_body = TranscribeRequest,
responses(
(status = 200, description = "Audio transcribed successfully", body = TranscribeResponse),
(status = 400, description = "Invalid request (bad base64 or unsupported format)"),
(status = 401, description = "Invalid API key"),
(status = 412, description = "DictationProvider not configured"),
(status = 413, description = "Audio file too large (max 25MB)"),
(status = 429, description = "Rate limit exceeded"),
(status = 500, description = "Internal server error"),
(status = 502, description = "DictationProvider API error"),
(status = 503, description = "Service unavailable"),
(status = 504, description = "Request timeout")
)
)]
pub async fn transcribe_dictation(
Json(request): Json<TranscribeRequest>,
) -> Result<Json<TranscribeResponse>, ErrorResponse> {
let (audio_bytes, extension) = validate_audio(&request.audio, &request.mime_type)?;
let provider_name = request.provider.as_str();
let client = build_api_client(provider_name)?;
let text = match request.provider {
DictationProvider::OpenAI => {
transcribe_openai(audio_bytes, extension, &request.mime_type, &client).await?
}
DictationProvider::ElevenLabs => {
transcribe_elevenlabs(audio_bytes, extension, &request.mime_type, &client).await?
}
};
Ok(Json(TranscribeResponse { text }))
}
#[utoipa::path(
get,
path = "/dictation/config",
responses(
(status = 200, description = "Audio transcription provider configurations", body = HashMap<String, DictationProviderStatus>)
)
)]
pub async fn get_dictation_config(
) -> Result<Json<HashMap<String, DictationProviderStatus>>, ErrorResponse> {
let config = goose::config::Config::global();
let mut providers = HashMap::new();
for (name, def) in PROVIDERS.iter() {
let configured = config.get_secret::<String>(def.config_key).is_ok();
let host = if let Some(host_key) = def.host_key {
config
.get(host_key, false)
.ok()
.and_then(|v| v.as_str().map(|s| s.to_string()))
} else {
None
};
providers.insert(
name.to_string(),
DictationProviderStatus {
configured,
host,
description: def.description.to_string(),
uses_provider_config: def.uses_provider_config,
settings_path: def.settings_path.map(|s| s.to_string()),
config_key: if !def.uses_provider_config {
Some(def.config_key.to_string())
} else {
None
},
},
);
}
Ok(Json(providers))
}
pub fn routes(state: Arc<AppState>) -> Router {
Router::new()
.route("/dictation/transcribe", post(transcribe_dictation))
.route("/dictation/config", get(get_dictation_config))
.with_state(state)
}
crates/goose-server/src/routes/mod.rs
+2 -2
View File
@@ -1,7 +1,7 @@
pub mod action_required;
pub mod agent;
pub mod audio;
pub mod config_management;
pub mod dictation;
pub mod errors;
pub mod mcp_app_proxy;
pub mod mcp_ui_proxy;
@@ -28,7 +28,7 @@ pub fn configure(state: Arc<crate::state::AppState>, secret_key: String) -> Rout
.merge(reply::routes(state.clone()))
.merge(action_required::routes(state.clone()))
.merge(agent::routes(state.clone()))
.merge(audio::routes(state.clone()))
.merge(dictation::routes(state.clone()))
.merge(config_management::routes(state.clone()))
.merge(prompts::routes())
.merge(recipe::routes(state.clone()))
crates/goose/src/providers/api_client.rs
+5
View File
@@ -365,6 +365,11 @@ impl<'a> ApiRequestBuilder<'a> {
Ok(request.json(payload).send().await?)
}
pub async fn multipart_post(self, form: reqwest::multipart::Form) -> Result<Response> {
let request = self.send_request(|url, client| client.post(url)).await?;
Ok(request.multipart(form).send().await?)
}
pub async fn api_get(self) -> Result<ApiResponse> {
let response = self.response_get().await?;
ApiResponse::from_response(response).await
ui/desktop/openapi.json
+157
View File
@@ -1571,6 +1571,86 @@
}
}
},
"/dictation/config": {
"get": {
"tags": [
"super::routes::dictation"
],
"operationId": "get_dictation_config",
"responses": {
"200": {
"description": "Audio transcription provider configurations",
"content": {
"application/json": {
"schema": {
"type": "object",
"additionalProperties": {
"$ref": "#/components/schemas/DictationProviderStatus"
}
}
}
}
}
}
}
},
"/dictation/transcribe": {
"post": {
"tags": [
"super::routes::dictation"
],
"operationId": "transcribe_dictation",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/TranscribeRequest"
}
}
},
"required": true
},
"responses": {
"200": {
"description": "Audio transcribed successfully",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/TranscribeResponse"
}
}
}
},
"400": {
"description": "Invalid request (bad base64 or unsupported format)"
},
"401": {
"description": "Invalid API key"
},
"412": {
"description": "DictationProvider not configured"
},
"413": {
"description": "Audio file too large (max 25MB)"
},
"429": {
"description": "Rate limit exceeded"
},
"500": {
"description": "Internal server error"
},
"502": {
"description": "DictationProvider API error"
},
"503": {
"description": "Service unavailable"
},
"504": {
"description": "Request timeout"
}
}
}
},
"/handle_openrouter": {
"post": {
"tags": [
@@ -3593,6 +3673,50 @@
}
}
},
"DictationProvider": {
"type": "string",
"enum": [
"openai",
"elevenlabs"
]
},
"DictationProviderStatus": {
"type": "object",
"required": [
"configured",
"description",
"uses_provider_config"
],
"properties": {
"config_key": {
"type": "string",
"description": "Config key name if uses_provider_config is false",
"nullable": true
},
"configured": {
"type": "boolean",
"description": "Whether the provider is fully configured and ready to use"
},
"description": {
"type": "string",
"description": "Description of what this provider does"
},
"host": {
"type": "string",
"description": "Custom host URL if configured (only for providers that support it)",
"nullable": true
},
"settings_path": {
"type": "string",
"description": "Path to settings if uses_provider_config is true",
"nullable": true
},
"uses_provider_config": {
"type": "boolean",
"description": "Whether this provider uses the main provider config (true) or has its own key (false)"
}
}
},
"EmbeddedResource": {
"type": "object",
"required": [
@@ -6612,6 +6736,39 @@
}
}
},
"TranscribeRequest": {
"type": "object",
"required": [
"audio",
"mime_type",
"provider"
],
"properties": {
"audio": {
"type": "string",
"description": "Base64 encoded audio data"
},
"mime_type": {
"type": "string",
"description": "MIME type of the audio (e.g., \"audio/webm\", \"audio/wav\")"
},
"provider": {
"$ref": "#/components/schemas/DictationProvider"
}
}
},
"TranscribeResponse": {
"type": "object",
"required": [
"text"
],
"properties": {
"text": {
"type": "string",
"description": "Transcribed text from the audio"
}
}
},
"TunnelInfo": {
"type": "object",
"required": [
ui/desktop/src/api/index.ts
+2 -2
View File
File diff suppressed because one or more lines are too long
ui/desktop/src/api/sdk.gen.ts
+12 -1
View File
File diff suppressed because one or more lines are too long
ui/desktop/src/api/types.gen.ts
+121
View File
@@ -190,6 +190,35 @@ export type DetectProviderResponse = {
provider_name: string;
};
export type DictationProvider = 'openai' | 'elevenlabs';
export type DictationProviderStatus = {
/**
* Config key name if uses_provider_config is false
*/
config_key?: string | null;
/**
* Whether the provider is fully configured and ready to use
*/
configured: boolean;
/**
* Description of what this provider does
*/
description: string;
/**
* Custom host URL if configured (only for providers that support it)
*/
host?: string | null;
/**
* Path to settings if uses_provider_config is true
*/
settings_path?: string | null;
/**
* Whether this provider uses the main provider config (true) or has its own key (false)
*/
uses_provider_config: boolean;
};
export type EmbeddedResource = {
_meta?: {
[key: string]: unknown;
@@ -1173,6 +1202,25 @@ export type ToolResponse = {
};
};
export type TranscribeRequest = {
/**
* Base64 encoded audio data
*/
audio: string;
/**
* MIME type of the audio (e.g., "audio/webm", "audio/wav")
*/
mime_type: string;
provider: DictationProvider;
};
export type TranscribeResponse = {
/**
* Transcribed text from the audio
*/
text: string;
};
export type TunnelInfo = {
hostname: string;
secret: string;
@@ -2456,6 +2504,79 @@ export type DiagnosticsResponses = {
export type DiagnosticsResponse = DiagnosticsResponses[keyof DiagnosticsResponses];
export type GetDictationConfigData = {
body?: never;
path?: never;
query?: never;
url: '/dictation/config';
};
export type GetDictationConfigResponses = {
/**
* Audio transcription provider configurations
*/
200: {
[key: string]: DictationProviderStatus;
};
};
export type GetDictationConfigResponse = GetDictationConfigResponses[keyof GetDictationConfigResponses];
export type TranscribeDictationData = {
body: TranscribeRequest;
path?: never;
query?: never;
url: '/dictation/transcribe';
};
export type TranscribeDictationErrors = {
/**
* Invalid request (bad base64 or unsupported format)
*/
400: unknown;
/**
* Invalid API key
*/
401: unknown;
/**
* DictationProvider not configured
*/
412: unknown;
/**
* Audio file too large (max 25MB)
*/
413: unknown;
/**
* Rate limit exceeded
*/
429: unknown;
/**
* Internal server error
*/
500: unknown;
/**
* DictationProvider API error
*/
502: unknown;
/**
* Service unavailable
*/
503: unknown;
/**
* Request timeout
*/
504: unknown;
};
export type TranscribeDictationResponses = {
/**
* Audio transcribed successfully
*/
200: TranscribeResponse;
};
export type TranscribeDictationResponse = TranscribeDictationResponses[keyof TranscribeDictationResponses];
export type StartOpenrouterSetupData = {
body?: never;
path?: never;
ui/desktop/src/components/ChatInput.tsx
+19 -36
View File
@@ -16,12 +16,10 @@ import { BottomMenuExtensionSelection } from './bottom_menu/BottomMenuExtensionS
import { AlertType, useAlerts } from './alerts';
import { useConfig } from './ConfigContext';
import { useModelAndProvider } from './ModelAndProviderContext';
import { useWhisper } from '../hooks/useWhisper';
import { DICTATION_PROVIDER_ELEVENLABS } from '../hooks/dictationConstants';
import { WaveformVisualizer } from './WaveformVisualizer';
import { useAudioRecorder } from '../hooks/useAudioRecorder';
import { toastError } from '../toasts';
import MentionPopover, { DisplayItemWithMatch } from './MentionPopover';
import { COST_TRACKING_ENABLED, VOICE_DICTATION_ELEVENLABS_ENABLED } from '../updates';
import { COST_TRACKING_ENABLED } from '../updates';
import { CostTracker } from './bottom_menu/CostTracker';
import { DroppedFile, useFileDrop } from '../hooks/useFileDrop';
import { Recipe } from '../recipe';
@@ -254,19 +252,17 @@ export default function ChatInput({
selectFile: (index: number) => void;
}>(null);
// Whisper hook for voice dictation
// Audio recorder hook for voice dictation
const {
isEnabled,
dictationProvider,
isRecording,
isTranscribing,
canUseDictation,
audioContext,
analyser,
startRecording,
stopRecording,
recordingDuration,
estimatedSize,
dictationSettings,
} = useWhisper({
} = useAudioRecorder({
onTranscription: (text) => {
trackVoiceDictation('transcribed');
// Append transcribed text to the current input
@@ -275,18 +271,12 @@ export default function ChatInput({
setValue(newValue);
textAreaRef.current?.focus();
},
onError: (error) => {
const errorType = error.name || 'DictationError';
onError: (message) => {
const errorType = 'DictationError';
trackVoiceDictation('error', undefined, errorType);
toastError({
title: 'Dictation Error',
msg: error.message,
});
},
onSizeWarning: (sizeMB) => {
toastError({
title: 'Recording Size Warning',
msg: `Recording is ${sizeMB.toFixed(1)}MB. Maximum size is 25MB.`,
msg: message,
});
},
});
@@ -1236,26 +1226,25 @@ export default function ChatInput({
maxHeight: `${maxHeight}px`,
overflowY: 'auto',
opacity: isRecording ? 0 : 1,
paddingRight: dictationSettings?.enabled ? '180px' : '120px',
paddingRight: dictationProvider ? '180px' : '120px',
}}
className="w-full outline-none border-none focus:ring-0 bg-transparent px-3 pt-3 pb-1.5 text-sm resize-none text-textStandard placeholder:text-textPlaceholder"
/>
{isRecording && (
<div className="absolute inset-0 flex items-center pl-4 pr-32 pt-3 pb-1.5">
<WaveformVisualizer
audioContext={audioContext}
analyser={analyser}
isRecording={isRecording}
/>
<div className="flex items-center gap-2 text-textSubtle">
<span className="inline-block w-2 h-2 bg-red-500 rounded-full animate-pulse" />
<span>Recording...</span>
</div>
</div>
)}
{/* Inline action buttons - absolutely positioned on the right */}
<div className="absolute right-2 top-1/2 -translate-y-1/2 flex items-center gap-1">
{/* Microphone button - show only if dictation is enabled */}
{dictationSettings?.enabled && (
{/* Microphone button - show only if provider is selected */}
{dictationProvider && (
<>
{!canUseDictation ? (
{!isEnabled ? (
<Tooltip>
<TooltipTrigger asChild>
<span className="inline-flex">
@@ -1273,22 +1262,16 @@ export default function ChatInput({
</span>
</TooltipTrigger>
<TooltipContent>
{dictationSettings.provider === 'openai' ? (
{dictationProvider === 'openai' ? (
<p>
OpenAI API key is not configured. Set it up in <b>Settings</b> {'>'}{' '}
<b>Models.</b>
</p>
) : VOICE_DICTATION_ELEVENLABS_ENABLED &&
dictationSettings.provider === DICTATION_PROVIDER_ELEVENLABS ? (
) : dictationProvider === 'elevenlabs' ? (
<p>
ElevenLabs API key is not configured. Set it up in <b>Settings</b> {'>'}{' '}
<b>Chat</b> {'>'} <b>Voice Dictation.</b>
</p>
) : dictationSettings.provider === null ? (
<p>
Dictation is not configured. Configure it in <b>Settings</b> {'>'}{' '}
<b>Chat</b> {'>'} <b>Voice Dictation.</b>
</p>
) : (
<p>Dictation provider is not properly configured.</p>
)}
ui/desktop/src/components/settings/chat/ChatSettingsSection.tsx
+10 -10
View File
@@ -1,5 +1,5 @@
import { ModeSection } from '../mode/ModeSection';
import DictationSection from '../dictation/DictationSection';
import { DictationSettings } from '../dictation/DictationSettings';
import { SecurityToggle } from '../security/SecurityToggle';
import { ResponseStylesSection } from '../response_styles/ResponseStylesSection';
import { GoosehintsSection } from './GoosehintsSection';
@@ -21,7 +21,14 @@ export default function ChatSettingsSection() {
<Card className="pb-2 rounded-lg">
<CardContent className="px-2">
<SecurityToggle />
<GoosehintsSection />
</CardContent>
</Card>
<Card className="pb-2 rounded-lg">
<CardContent className="px-2">
<DictationSettings />
<SpellcheckToggle />
</CardContent>
</Card>
@@ -37,14 +44,7 @@ export default function ChatSettingsSection() {
<Card className="pb-2 rounded-lg">
<CardContent className="px-2">
<DictationSection />
<SpellcheckToggle />
</CardContent>
</Card>
<Card className="pb-2 rounded-lg">
<CardContent className="px-2">
<GoosehintsSection />
<SecurityToggle />
</CardContent>
</Card>
</div>
ui/desktop/src/components/settings/dictation/DictationSection.tsx
-5
View File
@@ -1,5 +0,0 @@
import { VoiceDictationToggle } from './VoiceDictationToggle';
export default function DictationSection() {
return <VoiceDictationToggle />;
}
ui/desktop/src/components/settings/dictation/DictationSettings.tsx
+235
View File
@@ -0,0 +1,235 @@
import { useState, useEffect } from 'react';
import { ChevronDown } from 'lucide-react';
import { DictationProvider, getDictationConfig, DictationProviderStatus } from '../../../api';
import { useConfig } from '../../ConfigContext';
import { Input } from '../../ui/input';
import { Button } from '../../ui/button';
import { trackSettingToggled } from '../../../utils/analytics';
export const DictationSettings = () => {
const [provider, setProvider] = useState<DictationProvider | null>(null);
const [showProviderDropdown, setShowProviderDropdown] = useState(false);
const [providerStatuses, setProviderStatuses] = useState<Record<string, DictationProviderStatus>>(
{}
);
const [apiKey, setApiKey] = useState('');
const [isEditingKey, setIsEditingKey] = useState(false);
const [keyValidationError, setKeyValidationError] = useState('');
const { read, upsert, remove } = useConfig();
useEffect(() => {
const loadSettings = async () => {
const providerValue = await read('voice_dictation_provider', false);
const loadedProvider: DictationProvider | null = (providerValue as DictationProvider) || null;
setProvider(loadedProvider);
const audioConfig = await getDictationConfig();
setProviderStatuses(audioConfig.data || {});
};
loadSettings();
}, [read]);
const saveProvider = async (newProvider: DictationProvider | null) => {
console.log('Saving dictation provider to backend config:', newProvider);
setProvider(newProvider);
await upsert('voice_dictation_provider', newProvider || '', false);
trackSettingToggled('voice_dictation', newProvider !== null);
};
const handleProviderChange = (newProvider: DictationProvider | null) => {
saveProvider(newProvider);
setShowProviderDropdown(false);
};
const handleDropdownToggle = async () => {
const newShowState = !showProviderDropdown;
setShowProviderDropdown(newShowState);
if (newShowState) {
const audioConfig = await getDictationConfig();
setProviderStatuses(audioConfig.data || {});
}
};
const handleSaveKey = async () => {
if (!provider) return;
const providerConfig = providerStatuses[provider];
if (!providerConfig || providerConfig.uses_provider_config) return;
const trimmedKey = apiKey.trim();
if (!trimmedKey) {
setKeyValidationError('API key is required');
return;
}
try {
const keyName = providerConfig.config_key!;
await upsert(keyName, trimmedKey, true);
setApiKey('');
setKeyValidationError('');
setIsEditingKey(false);
const audioConfig = await getDictationConfig();
setProviderStatuses(audioConfig.data || {});
} catch (error) {
console.error('Error saving API key:', error);
setKeyValidationError('Failed to save API key');
}
};
const handleRemoveKey = async () => {
if (!provider) return;
const providerConfig = providerStatuses[provider];
if (!providerConfig || providerConfig.uses_provider_config) return;
try {
const keyName = providerConfig.config_key!;
await remove(keyName, true);
setApiKey('');
setKeyValidationError('');
setIsEditingKey(false);
const audioConfig = await getDictationConfig();
setProviderStatuses(audioConfig.data || {});
} catch (error) {
console.error('Error removing API key:', error);
setKeyValidationError('Failed to remove API key');
}
};
const handleCancelEdit = () => {
setApiKey('');
setKeyValidationError('');
setIsEditingKey(false);
};
const getProviderLabel = (provider: DictationProvider | null): string => {
if (!provider) return 'Disabled';
return provider.charAt(0).toUpperCase() + provider.slice(1);
};
return (
<div className="space-y-4">
<div className="flex items-center justify-between py-2 px-2 hover:bg-background-muted rounded-lg transition-all">
<div>
<h3 className="text-text-default">Voice Dictation Provider</h3>
<p className="text-xs text-text-muted max-w-md mt-[2px]">
Choose how voice is converted to text
</p>
</div>
<div className="relative">
<button
onClick={handleDropdownToggle}
className="flex items-center gap-2 px-3 py-1.5 text-sm border border-border-subtle rounded-md hover:border-border-default transition-colors text-text-default bg-background-default"
>
{getProviderLabel(provider)}
<ChevronDown className="w-4 h-4" />
</button>
{showProviderDropdown && (
<div className="absolute right-0 mt-1 w-max min-w-[250px] max-w-[350px] bg-background-default border border-border-default rounded-md shadow-lg z-50">
<button
onClick={() => handleProviderChange(null)}
className="w-full px-3 py-2 text-left text-sm transition-colors hover:bg-background-subtle text-text-default whitespace-nowrap first:rounded-t-md"
>
<span className="flex items-center justify-between gap-2">
<span>Disabled</span>
{provider === null && <span>✓</span>}
</span>
</button>
{(Object.keys(providerStatuses) as DictationProvider[]).map((p) => (
<button
key={p}
onClick={() => handleProviderChange(p)}
className="w-full px-3 py-2 text-left text-sm transition-colors hover:bg-background-subtle text-text-default whitespace-nowrap last:rounded-b-md"
>
<span className="flex items-center justify-between gap-2">
<span>
{getProviderLabel(p)}
{!providerStatuses[p]?.configured && (
<span className="text-xs ml-1 text-text-muted">(not configured)</span>
)}
</span>
{provider === p && <span>✓</span>}
</span>
</button>
))}
</div>
)}
</div>
</div>
{provider && providerStatuses[provider] && (
<>
<div className="py-2 px-2">
<p className="text-xs text-text-muted">{providerStatuses[provider].description}</p>
</div>
{providerStatuses[provider].uses_provider_config ? (
<div className="py-2 px-2 bg-background-subtle rounded-lg">
{!providerStatuses[provider].configured ? (
<p className="text-xs text-text-muted">
Configure the API key in <b>{providerStatuses[provider].settings_path}</b>
</p>
) : (
<p className="text-xs text-green-600">
✓ Configured in {providerStatuses[provider].settings_path}
</p>
)}
</div>
) : (
<div className="py-2 px-2 bg-background-subtle rounded-lg">
<div className="mb-2">
<h4 className="text-text-default text-sm">API Key</h4>
<p className="text-xs text-text-muted mt-[2px]">
Required for transcription
{providerStatuses[provider]?.configured && (
<span className="text-green-600 ml-2">(Configured)</span>
)}
</p>
</div>
{!isEditingKey ? (
<Button variant="outline" size="sm" onClick={() => setIsEditingKey(true)}>
{providerStatuses[provider]?.configured ? 'Update API Key' : 'Add API Key'}
</Button>
) : (
<div className="space-y-2">
<Input
type="password"
value={apiKey}
onChange={(e) => {
setApiKey(e.target.value);
if (keyValidationError) setKeyValidationError('');
}}
placeholder="Enter your API key"
className="max-w-md"
autoFocus
/>
{keyValidationError && (
<p className="text-xs text-red-600 mt-1">{keyValidationError}</p>
)}
<div className="flex gap-2">
<Button size="sm" onClick={handleSaveKey}>
Save
</Button>
<Button variant="outline" size="sm" onClick={handleCancelEdit}>
Cancel
</Button>
{providerStatuses[provider]?.configured && (
<Button variant="destructive" size="sm" onClick={handleRemoveKey}>
Remove
</Button>
)}
</div>
</div>
)}
</div>
)}
</>
)}
</div>
);
};
ui/desktop/src/components/settings/dictation/ElevenLabsKeyInput.tsx
-128
View File
@@ -1,128 +0,0 @@
import { useState, useEffect, useCallback } from 'react';
import { Input } from '../../ui/input';
import { Button } from '../../ui/button';
import { useConfig } from '../../ConfigContext';
import { ELEVENLABS_API_KEY, isSecretKeyConfigured } from '../../../hooks/dictationConstants';
import { setElevenLabsKeyCache } from '../../../hooks/useDictationSettings';
export const ElevenLabsKeyInput = () => {
const [elevenLabsApiKey, setElevenLabsApiKey] = useState('');
const [isLoadingKey, setIsLoadingKey] = useState(false);
const [hasElevenLabsKey, setHasElevenLabsKey] = useState(false);
const [validationError, setValidationError] = useState('');
const [isEditing, setIsEditing] = useState(false);
const { upsert, read, remove } = useConfig();
const loadKey = useCallback(async () => {
setIsLoadingKey(true);
try {
const response = await read(ELEVENLABS_API_KEY, true);
const hasKey = isSecretKeyConfigured(response);
setHasElevenLabsKey(hasKey);
setElevenLabsKeyCache(hasKey);
} catch (error) {
console.error(error);
setElevenLabsKeyCache(false);
} finally {
setIsLoadingKey(false);
}
}, [read]);
useEffect(() => {
loadKey();
}, [loadKey]);
const handleElevenLabsKeyChange = (key: string) => {
setElevenLabsApiKey(key);
if (validationError) {
setValidationError('');
}
};
const handleSave = async () => {
try {
const trimmedKey = elevenLabsApiKey.trim();
if (!trimmedKey) {
setValidationError('API key is required');
return;
}
await upsert(ELEVENLABS_API_KEY, trimmedKey, true);
setElevenLabsApiKey('');
setValidationError('');
setIsEditing(false);
await loadKey();
} catch (error) {
console.error(error);
setValidationError('Failed to save API key');
}
};
const handleRemove = async () => {
try {
await remove(ELEVENLABS_API_KEY, true);
await loadKey();
setElevenLabsApiKey('');
setValidationError('');
setIsEditing(false);
} catch (error) {
console.error(error);
setValidationError('Failed to remove API key');
}
};
const handleCancel = () => {
setElevenLabsApiKey('');
setValidationError('');
setIsEditing(false);
};
return (
<div className="py-2 px-2 bg-background-subtle rounded-lg">
<div className="mb-2">
<h4 className="text-text-default text-sm">ElevenLabs API Key</h4>
<p className="text-xs text-text-muted mt-[2px]">
Required for ElevenLabs voice recognition
{hasElevenLabsKey && <span className="text-green-600 ml-2">(Configured)</span>}
</p>
</div>
{!isEditing ? (
<Button
variant="outline"
size="sm"
onClick={() => setIsEditing(true)}
disabled={isLoadingKey}
>
{hasElevenLabsKey ? 'Update API Key' : 'Add API Key'}
</Button>
) : (
<div className="space-y-2">
<Input
type="password"
value={elevenLabsApiKey}
onChange={(e) => handleElevenLabsKeyChange(e.target.value)}
placeholder="Enter your ElevenLabs API key"
className="max-w-md"
autoFocus
/>
{validationError && <p className="text-xs text-red-600 mt-1">{validationError}</p>}
<div className="flex gap-2">
<Button size="sm" onClick={handleSave}>
Save
</Button>
<Button variant="outline" size="sm" onClick={handleCancel}>
Cancel
</Button>
{hasElevenLabsKey && (
<Button variant="destructive" size="sm" onClick={handleRemove}>
Remove
</Button>
)}
</div>
</div>
)}
</div>
);
};
ui/desktop/src/components/settings/dictation/ProviderInfo.tsx
-41
View File
@@ -1,41 +0,0 @@
import { DictationProvider } from '../../../hooks/useDictationSettings';
import { DICTATION_PROVIDER_ELEVENLABS } from '../../../hooks/dictationConstants';
import { VOICE_DICTATION_ELEVENLABS_ENABLED } from '../../../updates';
interface ProviderInfoProps {
provider: DictationProvider;
}
export const ProviderInfo = ({ provider }: ProviderInfoProps) => {
if (!provider) return null;
return (
<div className="p-3 bg-background-subtle rounded-md">
{provider === 'openai' && (
<p className="text-xs text-text-muted">
Uses OpenAI's Whisper API for high-quality transcription. Requires an OpenAI API key
configured in the Models section.
</p>
)}
{VOICE_DICTATION_ELEVENLABS_ENABLED && provider === DICTATION_PROVIDER_ELEVENLABS && (
<div>
<p className="text-xs text-text-muted">
Uses ElevenLabs speech-to-text API for high-quality transcription.
</p>
<p className="text-xs text-text-muted mt-2">
<strong>Features:</strong>
</p>
<ul className="text-xs text-text-muted ml-4 mt-1 list-disc">
<li>Advanced voice processing</li>
<li>High accuracy transcription</li>
<li>Multiple language support</li>
<li>Fast processing</li>
</ul>
<p className="text-xs text-text-muted mt-2">
<strong>Note:</strong> Requires an ElevenLabs API key with speech-to-text access.
</p>
</div>
)}
</div>
);
};
ui/desktop/src/components/settings/dictation/ProviderSelector.tsx
-128
View File
@@ -1,128 +0,0 @@
import { useState, useEffect } from 'react';
import { ChevronDown } from 'lucide-react';
import { DictationProvider, DictationSettings } from '../../../hooks/useDictationSettings';
import {
DICTATION_PROVIDER_OPENAI,
DICTATION_PROVIDER_ELEVENLABS,
} from '../../../hooks/dictationConstants';
import { useConfig } from '../../ConfigContext';
import { ElevenLabsKeyInput } from './ElevenLabsKeyInput';
import { ProviderInfo } from './ProviderInfo';
import { VOICE_DICTATION_ELEVENLABS_ENABLED } from '../../../updates';
interface ProviderSelectorProps {
settings: DictationSettings;
onProviderChange: (provider: DictationProvider) => void;
}
export const ProviderSelector = ({ settings, onProviderChange }: ProviderSelectorProps) => {
const [hasOpenAIKey, setHasOpenAIKey] = useState(false);
const [showProviderDropdown, setShowProviderDropdown] = useState(false);
const { getProviders } = useConfig();
useEffect(() => {
const checkOpenAIKey = async () => {
try {
const providers = await getProviders(false);
const openAIProvider = providers.find((p) => p.name === 'openai');
setHasOpenAIKey(openAIProvider?.is_configured || false);
} catch (error) {
console.error('Error checking OpenAI configuration:', error);
setHasOpenAIKey(false);
}
};
checkOpenAIKey();
}, [getProviders]);
const handleDropdownToggle = async () => {
const newShowState = !showProviderDropdown;
setShowProviderDropdown(newShowState);
if (newShowState) {
try {
const providers = await getProviders(true);
const openAIProvider = providers.find((p) => p.name === 'openai');
const isConfigured = !!openAIProvider?.is_configured;
setHasOpenAIKey(isConfigured);
} catch (error) {
console.error('Error checking OpenAI configuration:', error);
setHasOpenAIKey(false);
}
}
};
const handleProviderChange = (provider: DictationProvider) => {
onProviderChange(provider);
setShowProviderDropdown(false);
};
const getProviderLabel = (provider: DictationProvider): string => {
switch (provider) {
case DICTATION_PROVIDER_OPENAI:
return 'OpenAI Whisper';
case DICTATION_PROVIDER_ELEVENLABS:
return 'ElevenLabs';
default:
return 'None (disabled)';
}
};
return (
<div className="space-y-4">
<div className="flex items-center justify-between py-2 px-2 hover:bg-background-muted rounded-lg transition-all">
<div>
<h3 className="text-text-default">Dictation Provider</h3>
<p className="text-xs text-text-muted max-w-md mt-[2px]">
Choose how voice is converted to text
</p>
</div>
<div className="relative">
<button
onClick={handleDropdownToggle}
className="flex items-center gap-2 px-3 py-1.5 text-sm border border-border-subtle rounded-md hover:border-border-default transition-colors text-text-default bg-background-default"
>
{getProviderLabel(settings.provider)}
<ChevronDown className="w-4 h-4" />
</button>
{showProviderDropdown && (
<div className="absolute right-0 mt-1 w-max min-w-[250px] max-w-[350px] bg-background-default border border-border-default rounded-md shadow-lg z-50">
<button
onClick={() => handleProviderChange(DICTATION_PROVIDER_OPENAI)}
className={`w-full px-3 py-2 text-left text-sm transition-colors hover:bg-background-subtle text-text-default whitespace-nowrap ${!VOICE_DICTATION_ELEVENLABS_ENABLED ? 'first:rounded-t-md last:rounded-b-md' : 'first:rounded-t-md'}`}
>
<span className="flex items-center justify-between gap-2">
<span>
OpenAI Whisper
{!hasOpenAIKey && (
<span className="text-xs ml-1 text-text-muted">(not configured)</span>
)}
</span>
{settings.provider === DICTATION_PROVIDER_OPENAI && <span>✓</span>}
</span>
</button>
{VOICE_DICTATION_ELEVENLABS_ENABLED && (
<button
onClick={() => handleProviderChange(DICTATION_PROVIDER_ELEVENLABS)}
className="w-full px-3 py-2 text-left text-sm hover:bg-background-subtle transition-colors text-text-default last:rounded-b-md whitespace-nowrap"
>
<span className="flex items-center justify-between gap-2">
<span>ElevenLabs</span>
{settings.provider === DICTATION_PROVIDER_ELEVENLABS && <span>✓</span>}
</span>
</button>
)}
</div>
)}
</div>
</div>
{VOICE_DICTATION_ELEVENLABS_ENABLED &&
settings.provider === DICTATION_PROVIDER_ELEVENLABS && <ElevenLabsKeyInput />}
<ProviderInfo provider={settings.provider} />
</div>
);
};
ui/desktop/src/components/settings/dictation/VoiceDictationToggle.tsx
-97
View File
@@ -1,97 +0,0 @@
import { useState, useEffect } from 'react';
import { Switch } from '../../ui/switch';
import { DictationProvider, DictationSettings } from '../../../hooks/useDictationSettings';
import {
DICTATION_SETTINGS_KEY,
DICTATION_PROVIDER_OPENAI,
DICTATION_PROVIDER_ELEVENLABS,
getDefaultDictationSettings,
} from '../../../hooks/dictationConstants';
import { useConfig } from '../../ConfigContext';
import { ProviderSelector } from './ProviderSelector';
import { VOICE_DICTATION_ELEVENLABS_ENABLED } from '../../../updates';
import { trackSettingToggled } from '../../../utils/analytics';
export const VoiceDictationToggle = () => {
const [settings, setSettings] = useState<DictationSettings>({
enabled: false,
provider: null,
});
const { getProviders } = useConfig();
useEffect(() => {
const loadSettings = async () => {
const savedSettings = localStorage.getItem(DICTATION_SETTINGS_KEY);
let loadedSettings: DictationSettings;
if (savedSettings) {
const parsed = JSON.parse(savedSettings);
loadedSettings = parsed;
// If ElevenLabs is disabled and user has it selected, reset to OpenAI
if (
!VOICE_DICTATION_ELEVENLABS_ENABLED &&
loadedSettings.provider === DICTATION_PROVIDER_ELEVENLABS
) {
loadedSettings = {
...loadedSettings,
provider: DICTATION_PROVIDER_OPENAI,
};
localStorage.setItem(DICTATION_SETTINGS_KEY, JSON.stringify(loadedSettings));
}
} else {
loadedSettings = await getDefaultDictationSettings(getProviders);
}
setSettings(loadedSettings);
};
loadSettings();
}, [getProviders]);
const saveSettings = (newSettings: DictationSettings) => {
console.log('Saving dictation settings to localStorage:', newSettings);
setSettings(newSettings);
localStorage.setItem(DICTATION_SETTINGS_KEY, JSON.stringify(newSettings));
};
const handleToggle = (enabled: boolean) => {
saveSettings({
...settings,
enabled,
provider: settings.provider === null ? DICTATION_PROVIDER_OPENAI : settings.provider,
});
trackSettingToggled('voice_dictation', enabled);
};
const handleProviderChange = (provider: DictationProvider) => {
saveSettings({ ...settings, provider });
};
return (
<div className="space-y-1">
<div className="flex items-center justify-between py-2 px-2 hover:bg-background-muted rounded-lg transition-all">
<div>
<h3 className="text-text-default">Enable Voice Dictation</h3>
<p className="text-xs text-text-muted max-w-md mt-[2px]">
Show microphone button for voice input
</p>
</div>
<div className="flex items-center">
<Switch checked={settings.enabled} onCheckedChange={handleToggle} variant="mono" />
</div>
</div>
<div
className={`overflow-visible transition-all duration-300 ease-in-out ${
settings.enabled ? 'max-h-96 opacity-100 mt-2' : 'max-h-0 opacity-0 mt-0'
}`}
>
<div className="space-y-3 pb-2">
<ProviderSelector settings={settings} onProviderChange={handleProviderChange} />
</div>
</div>
</div>
);
};
ui/desktop/src/hooks/dictationConstants.ts
-31
View File
@@ -1,31 +0,0 @@
import { DictationSettings, DictationProvider } from './useDictationSettings';
export const DICTATION_SETTINGS_KEY = 'dictation_settings';
export const ELEVENLABS_API_KEY = 'ELEVENLABS_API_KEY';
export const DICTATION_PROVIDER_OPENAI = 'openai' as const;
export const DICTATION_PROVIDER_ELEVENLABS = 'elevenlabs' as const;
export const isSecretKeyConfigured = (response: unknown): boolean =>
typeof response === 'object' &&
response !== null &&
'maskedValue' in response &&
!!(response as { maskedValue: string }).maskedValue;
export const getDefaultDictationSettings = async (
getProviders: (refresh: boolean) => Promise<Array<{ name: string; is_configured: boolean }>>
): Promise<DictationSettings> => {
const providers = await getProviders(false);
const openAIProvider = providers.find((p) => p.name === 'openai');
if (openAIProvider && openAIProvider.is_configured) {
return {
enabled: true,
provider: DICTATION_PROVIDER_OPENAI,
};
} else {
return {
enabled: false,
provider: null as DictationProvider,
};
}
};
ui/desktop/src/hooks/useAudioRecorder.ts
+219
View File
@@ -0,0 +1,219 @@
import { useState, useRef, useCallback, useEffect } from 'react';
import { transcribeDictation, getDictationConfig, DictationProvider } from '../api';
import { useConfig } from '../components/ConfigContext';
import { errorMessage } from '../utils/conversionUtils';
interface UseAudioRecorderOptions {
onTranscription: (text: string) => void;
onError: (message: string) => void;
}
const MAX_AUDIO_SIZE_MB = 25;
const MAX_RECORDING_DURATION_SECONDS = 10 * 60;
export const useAudioRecorder = ({ onTranscription, onError }: UseAudioRecorderOptions) => {
const [isRecording, setIsRecording] = useState(false);
const [isTranscribing, setIsTranscribing] = useState(false);
const [recordingDuration, setRecordingDuration] = useState(0);
const [estimatedSize, setEstimatedSize] = useState(0);
const [isEnabled, setIsEnabled] = useState(false);
const [provider, setProvider] = useState<DictationProvider | null>(null);
const { read } = useConfig();
const mediaRecorderRef = useRef<MediaRecorder | null>(null);
const audioChunksRef = useRef<Blob[]>([]);
const streamRef = useRef<MediaStream | null>(null);
const durationIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
useEffect(() => {
const checkProviderConfig = async () => {
try {
const providerValue = await read('voice_dictation_provider', false);
const preferredProvider = (providerValue as DictationProvider) || null;
if (!preferredProvider) {
setIsEnabled(false);
setProvider(null);
return;
}
const audioConfigResponse = await getDictationConfig();
const providerStatus = audioConfigResponse.data?.[preferredProvider];
setIsEnabled(!!providerStatus?.configured);
setProvider(preferredProvider);
} catch (error) {
console.error('Error checking audio config:', error);
setIsEnabled(false);
setProvider(null);
}
};
checkProviderConfig();
}, [read]);
const stopRecording = useCallback(() => {
setIsRecording(false);
if (mediaRecorderRef.current && mediaRecorderRef.current.state !== 'inactive') {
mediaRecorderRef.current.stop();
}
if (durationIntervalRef.current) {
clearInterval(durationIntervalRef.current);
durationIntervalRef.current = null;
}
if (streamRef.current) {
streamRef.current.getTracks().forEach((track) => track.stop());
streamRef.current = null;
}
}, []);
useEffect(() => {
return () => {
if (durationIntervalRef.current) {
clearInterval(durationIntervalRef.current);
}
if (streamRef.current) {
streamRef.current.getTracks().forEach((track) => track.stop());
}
};
}, []);
const transcribeAudio = useCallback(
async (audioBlob: Blob) => {
if (!provider) {
onError('No transcription provider configured');
return;
}
setIsTranscribing(true);
try {
const sizeMB = audioBlob.size / (1024 * 1024);
if (sizeMB > MAX_AUDIO_SIZE_MB) {
onError(
`Audio file too large (${sizeMB.toFixed(1)}MB). Maximum size is ${MAX_AUDIO_SIZE_MB}MB.`
);
return;
}
const reader = new FileReader();
const base64Audio = await new Promise<string>((resolve, reject) => {
reader.onloadend = () => {
const base64 = reader.result as string;
resolve(base64.split(',')[1]);
};
reader.onerror = reject;
reader.readAsDataURL(audioBlob);
});
const mimeType = audioBlob.type;
if (!mimeType) {
throw new Error('Unable to determine audio format');
}
const result = await transcribeDictation({
body: {
audio: base64Audio,
mime_type: mimeType,
provider: provider,
},
throwOnError: true,
});
if (result.data?.text) {
onTranscription(result.data.text);
}
} catch (error) {
onError(errorMessage(error));
} finally {
setIsTranscribing(false);
setRecordingDuration(0);
setEstimatedSize(0);
}
},
[provider, onTranscription, onError]
);
const startRecording = useCallback(async () => {
if (!isEnabled) {
onError('Voice dictation is not enabled');
return;
}
try {
const stream = await navigator.mediaDevices.getUserMedia({
audio: {
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true,
},
});
streamRef.current = stream;
const supportedTypes = ['audio/webm;codecs=opus', 'audio/webm', 'audio/mp4', 'audio/wav'];
const mimeType = supportedTypes.find((type) => MediaRecorder.isTypeSupported(type)) || '';
const mediaRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : {});
mediaRecorderRef.current = mediaRecorder;
audioChunksRef.current = [];
const startTime = Date.now();
durationIntervalRef.current = setInterval(() => {
const elapsed = (Date.now() - startTime) / 1000;
setRecordingDuration(elapsed);
const estimatedSizeMB = (elapsed * 128 * 1024) / (8 * 1024 * 1024);
setEstimatedSize(estimatedSizeMB);
if (elapsed >= MAX_RECORDING_DURATION_SECONDS) {
stopRecording();
onError(
`Maximum recording duration (${MAX_RECORDING_DURATION_SECONDS / 60} minutes) reached`
);
}
}, 100);
mediaRecorder.ondataavailable = (event) => {
if (event.data.size > 0) {
audioChunksRef.current.push(event.data);
}
};
mediaRecorder.onstop = async () => {
const audioBlob = new Blob(audioChunksRef.current, { type: mimeType || 'audio/webm' });
if (audioBlob.size === 0) {
onError('No audio data was recorded. Please check your microphone.');
return;
}
await transcribeAudio(audioBlob);
};
mediaRecorder.onerror = (_event) => {
onError('Recording failed');
};
mediaRecorder.start(100);
setIsRecording(true);
} catch (error) {
stopRecording();
onError(errorMessage(error));
}
}, [isEnabled, onError, transcribeAudio, stopRecording]);
return {
isEnabled,
dictationProvider: provider,
isRecording,
isTranscribing,
recordingDuration,
estimatedSize,
startRecording,
stopRecording,
};
};
ui/desktop/src/hooks/useDictationSettings.ts
-76
View File
@@ -1,76 +0,0 @@
import { useState, useEffect } from 'react';
import { useConfig } from '../components/ConfigContext';
import {
DICTATION_SETTINGS_KEY,
ELEVENLABS_API_KEY,
DICTATION_PROVIDER_OPENAI,
DICTATION_PROVIDER_ELEVENLABS,
getDefaultDictationSettings,
isSecretKeyConfigured,
} from './dictationConstants';
export type DictationProvider =
| typeof DICTATION_PROVIDER_OPENAI
| typeof DICTATION_PROVIDER_ELEVENLABS
| null;
export interface DictationSettings {
enabled: boolean;
provider: DictationProvider;
}
let elevenLabsKeyCache: boolean | null = null;
export const setElevenLabsKeyCache = (value: boolean) => {
elevenLabsKeyCache = value;
};
export const useDictationSettings = () => {
const [settings, setSettings] = useState<DictationSettings | null>(null);
const [hasElevenLabsKey, setHasElevenLabsKey] = useState<boolean>(elevenLabsKeyCache ?? false);
const { read, getProviders } = useConfig();
useEffect(() => {
const loadSettings = async () => {
const saved = localStorage.getItem(DICTATION_SETTINGS_KEY);
let currentSettings: DictationSettings;
if (saved) {
currentSettings = JSON.parse(saved);
} else {
currentSettings = await getDefaultDictationSettings(getProviders);
}
setSettings(currentSettings);
if (
currentSettings.provider === DICTATION_PROVIDER_ELEVENLABS &&
elevenLabsKeyCache === null
) {
try {
const response = await read(ELEVENLABS_API_KEY, true);
const hasKey = isSecretKeyConfigured(response);
elevenLabsKeyCache = hasKey;
setHasElevenLabsKey(hasKey);
} catch (error) {
elevenLabsKeyCache = false;
setHasElevenLabsKey(false);
console.error('[useDictationSettings] Error checking ElevenLabs API key:', error);
}
}
};
loadSettings();
// Listen for storage changes from other tabs/windows
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const handleStorageChange = (e: any) => {
if (e.key === DICTATION_SETTINGS_KEY && e.newValue) {
setSettings(JSON.parse(e.newValue));
}
};
window.addEventListener('storage', handleStorageChange);
return () => window.removeEventListener('storage', handleStorageChange);
}, [read, getProviders]);
return { settings, hasElevenLabsKey };
};
ui/desktop/src/hooks/useWhisper.ts
-378
View File
@@ -1,378 +0,0 @@
import { useState, useRef, useCallback, useEffect } from 'react';
import { useConfig } from '../components/ConfigContext';
import { getApiUrl } from '../config';
import { useDictationSettings } from './useDictationSettings';
import { DICTATION_PROVIDER_OPENAI, DICTATION_PROVIDER_ELEVENLABS } from './dictationConstants';
import { safeJsonParse, errorMessage } from '../utils/conversionUtils';
interface UseWhisperOptions {
onTranscription?: (text: string) => void;
onError?: (error: Error) => void;
onSizeWarning?: (sizeInMB: number) => void;
}
// Constants
const MAX_AUDIO_SIZE_MB = 25;
const MAX_RECORDING_DURATION_SECONDS = 600; // 10 minutes
const WARNING_SIZE_MB = 20; // Warn at 20MB
export const useWhisper = ({ onTranscription, onError, onSizeWarning }: UseWhisperOptions = {}) => {
const [isRecording, setIsRecording] = useState(false);
const [isTranscribing, setIsTranscribing] = useState(false);
const [hasOpenAIKey, setHasOpenAIKey] = useState(false);
const [canUseDictation, setCanUseDictation] = useState(false);
const [audioContext, setAudioContext] = useState<AudioContext | null>(null);
const [analyser, setAnalyser] = useState<AnalyserNode | null>(null);
const [recordingDuration, setRecordingDuration] = useState(0);
const [estimatedSize, setEstimatedSize] = useState(0);
const mediaRecorderRef = useRef<MediaRecorder | null>(null);
const audioChunksRef = useRef<Blob[]>([]);
const streamRef = useRef<MediaStream | null>(null);
const recordingStartTimeRef = useRef<number | null>(null);
const durationIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
const currentSizeRef = useRef<number>(0);
const { getProviders } = useConfig();
const { settings: dictationSettings, hasElevenLabsKey } = useDictationSettings();
// Check if OpenAI API key is configured (regardless of current provider)
useEffect(() => {
const checkOpenAIKey = async () => {
try {
// Get all configured providers
const providers = await getProviders(false);
// Find OpenAI provider
const openAIProvider = providers.find((p) => p.name === 'openai');
// Check if OpenAI is configured
if (openAIProvider && openAIProvider.is_configured) {
setHasOpenAIKey(true);
} else {
setHasOpenAIKey(false);
}
} catch (error) {
console.error('Error checking OpenAI configuration:', error);
setHasOpenAIKey(false);
}
};
checkOpenAIKey();
}, [getProviders]); // Re-check when providers change
// Check if dictation can be used based on settings
useEffect(() => {
if (!dictationSettings) {
setCanUseDictation(false);
return;
}
if (!dictationSettings.enabled) {
setCanUseDictation(false);
return;
}
// Check provider availability
switch (dictationSettings.provider) {
case DICTATION_PROVIDER_OPENAI:
setCanUseDictation(hasOpenAIKey);
break;
case DICTATION_PROVIDER_ELEVENLABS:
setCanUseDictation(hasElevenLabsKey);
break;
default:
setCanUseDictation(false);
}
}, [dictationSettings, hasOpenAIKey, hasElevenLabsKey]);
// Define stopRecording before startRecording to avoid circular dependency
const stopRecording = useCallback(() => {
setIsRecording(false);
if (mediaRecorderRef.current && mediaRecorderRef.current.state !== 'inactive') {
mediaRecorderRef.current.stop();
}
// Clear interval
if (durationIntervalRef.current) {
clearInterval(durationIntervalRef.current);
durationIntervalRef.current = null;
}
// Stop all tracks in the stream
if (streamRef.current) {
streamRef.current.getTracks().forEach((track) => track.stop());
streamRef.current = null;
}
// Close audio context
if (audioContext && audioContext.state !== 'closed') {
audioContext.close().catch(console.error);
setAudioContext(null);
setAnalyser(null);
}
}, [audioContext]);
// Cleanup effect to prevent memory leaks
useEffect(() => {
return () => {
// Cleanup on unmount
if (durationIntervalRef.current) {
clearInterval(durationIntervalRef.current);
}
if (streamRef.current) {
streamRef.current.getTracks().forEach((track) => track.stop());
}
if (audioContext && audioContext.state !== 'closed') {
audioContext.close().catch(console.error);
}
};
}, [audioContext]);
const transcribeAudio = useCallback(
async (audioBlob: Blob) => {
if (!dictationSettings) {
stopRecording();
onError?.(new Error('Dictation settings not loaded'));
return;
}
setIsTranscribing(true);
try {
// Check final size
const sizeMB = audioBlob.size / (1024 * 1024);
if (sizeMB > MAX_AUDIO_SIZE_MB) {
throw new Error(
`Audio file too large (${sizeMB.toFixed(1)}MB). Maximum size is ${MAX_AUDIO_SIZE_MB}MB.`
);
}
// Convert blob to base64 for easier transport
const reader = new FileReader();
const base64Audio = await new Promise<string>((resolve, reject) => {
reader.onloadend = () => {
const base64 = reader.result as string;
resolve(base64.split(',')[1]); // Remove data:audio/webm;base64, prefix
};
reader.onerror = reject;
reader.readAsDataURL(audioBlob);
});
const mimeType = audioBlob.type;
if (!mimeType) {
throw new Error('Unable to determine audio format. Please try again.');
}
let endpoint = '';
let headers: Record<string, string> = {
'Content-Type': 'application/json',
'X-Secret-Key': await window.electron.getSecretKey(),
};
let body: Record<string, string> = {
audio: base64Audio,
mime_type: mimeType,
};
// Choose endpoint based on provider
switch (dictationSettings.provider) {
case DICTATION_PROVIDER_OPENAI:
endpoint = '/audio/transcribe';
break;
case DICTATION_PROVIDER_ELEVENLABS:
endpoint = '/audio/transcribe/elevenlabs';
break;
default:
throw new Error(`Unsupported provider: ${dictationSettings.provider}`);
}
const response = await fetch(getApiUrl(endpoint), {
method: 'POST',
headers,
body: JSON.stringify(body),
});
if (!response.ok) {
if (response.status === 404) {
throw new Error(
`Audio transcription endpoint not found. Please implement ${endpoint} endpoint in the Goose backend.`
);
} else if (response.status === 401) {
throw new Error('Invalid API key. Please check your API key is correct.');
} else if (response.status === 402) {
throw new Error('API quota exceeded. Please check your account limits.');
}
const errorData = await safeJsonParse<{
error: { message: string };
}>(response, 'Failed to parse error response').catch(() => ({
error: { message: 'Transcription failed' },
}));
throw new Error(errorData.error?.message || 'Transcription failed');
}
const data = await safeJsonParse<{ text: string }>(
response,
'Failed to parse transcription response'
);
if (data.text) {
onTranscription?.(data.text);
}
} catch (error) {
console.error('Error transcribing audio:', error);
stopRecording();
onError?.(error as Error);
} finally {
setIsTranscribing(false);
setRecordingDuration(0);
setEstimatedSize(0);
}
},
[onTranscription, onError, dictationSettings, stopRecording]
);
const startRecording = useCallback(async () => {
if (!dictationSettings) {
stopRecording();
onError?.(new Error('Dictation settings not loaded'));
return;
}
try {
// Request microphone permission
const stream = await navigator.mediaDevices.getUserMedia({
audio: {
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true,
sampleRate: 44100,
},
});
streamRef.current = stream;
// Verify we have valid audio tracks
const audioTracks = stream.getAudioTracks();
if (audioTracks.length === 0) {
throw new Error('No audio tracks available in the microphone stream');
}
// AudioContext creation is disabled to prevent MediaRecorder conflicts
setAudioContext(null);
setAnalyser(null);
// Determine best supported MIME type
const supportedTypes = ['audio/webm;codecs=opus', 'audio/webm', 'audio/mp4', 'audio/wav'];
const mimeType = supportedTypes.find((type) => MediaRecorder.isTypeSupported(type)) || '';
const mediaRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : {});
mediaRecorderRef.current = mediaRecorder;
audioChunksRef.current = [];
currentSizeRef.current = 0;
recordingStartTimeRef.current = Date.now();
// Start duration and size tracking
durationIntervalRef.current = setInterval(() => {
const elapsed = (Date.now() - (recordingStartTimeRef.current || 0)) / 1000;
setRecordingDuration(elapsed);
// Estimate size based on typical webm bitrate (~128kbps)
const estimatedSizeMB = (elapsed * 128 * 1024) / (8 * 1024 * 1024);
setEstimatedSize(estimatedSizeMB);
// Check if we're approaching the limit
if (estimatedSizeMB > WARNING_SIZE_MB) {
onSizeWarning?.(estimatedSizeMB);
}
// Auto-stop if we hit the duration limit
if (elapsed >= MAX_RECORDING_DURATION_SECONDS) {
stopRecording();
onError?.(
new Error(
`Maximum recording duration (${MAX_RECORDING_DURATION_SECONDS / 60} minutes) reached.`
)
);
}
}, 100);
mediaRecorder.ondataavailable = (event) => {
if (event.data.size > 0) {
audioChunksRef.current.push(event.data);
currentSizeRef.current += event.data.size;
// Check actual size
const actualSizeMB = currentSizeRef.current / (1024 * 1024);
if (actualSizeMB > MAX_AUDIO_SIZE_MB) {
stopRecording();
onError?.(new Error(`Maximum file size (${MAX_AUDIO_SIZE_MB}MB) reached.`));
}
}
};
mediaRecorder.onstop = async () => {
const audioBlob = new Blob(audioChunksRef.current, { type: mimeType || 'audio/webm' });
// Check if the blob is empty
if (audioBlob.size === 0) {
onError?.(
new Error(
'No audio data was recorded. Please check your microphone permissions and try again.'
)
);
return;
}
await transcribeAudio(audioBlob);
};
// Add error handler for MediaRecorder
mediaRecorder.onerror = (event) => {
console.error('MediaRecorder error:', event);
onError?.(new Error('Recording failed: Unknown error'));
};
if (!stream.active) {
throw new Error('Audio stream became inactive before recording could start');
}
// Check audio tracks again before starting recording
if (audioTracks.length === 0) {
throw new Error('No audio tracks available in the stream');
}
const activeAudioTracks = audioTracks.filter((track) => track.readyState === 'live');
if (activeAudioTracks.length === 0) {
throw new Error('No live audio tracks available');
}
try {
mediaRecorder.start(100);
setIsRecording(true);
} catch (startError) {
console.error('Error calling mediaRecorder.start():', startError);
throw new Error(`Failed to start recording: ${errorMessage(startError)}`);
}
} catch (error) {
console.error('Error starting recording:', error);
stopRecording();
onError?.(error as Error);
}
}, [onError, onSizeWarning, transcribeAudio, stopRecording, dictationSettings]);
return {
isRecording,
isTranscribing,
hasOpenAIKey,
canUseDictation,
audioContext,
analyser,
startRecording,
stopRecording,
recordingDuration,
estimatedSize,
dictationSettings,
};
};
ui/desktop/src/updates.ts
-1
View File
@@ -2,5 +2,4 @@ export const UPDATES_ENABLED = true;
export const COST_TRACKING_ENABLED = true;
export const ANNOUNCEMENTS_ENABLED = false;
export const CONFIGURATION_ENABLED = true;
export const VOICE_DICTATION_ELEVENLABS_ENABLED = true;
export const TELEMETRY_UI_ENABLED = true;