tkmind/tkmind_go
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Careless whisper (#6877)

Browse Source 
Co-authored-by: Douwe Osinga <douwe@squareup.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
Douwe Osinga
2026-02-03 12:54:29 +01:00
committed by GitHub
parent 3d0bb3c670
commit 1373d9c5f9
25 changed files with 118541 additions and 485 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/goose-server/src/openapi.rs
+10 -1
View File
@@ -4,6 +4,7 @@ use goose::agents::ExtensionConfig;
use goose::config::permission::PermissionLevel;
use goose::config::ExtensionEntry;
use goose::conversation::Conversation;
use goose::dictation::download_manager::{DownloadProgress, DownloadStatus};
use goose::model::ModelConfig;
use goose::permission::permission_confirmation::PrincipalType;
use goose::providers::base::{ConfigKey, ModelInfo, ProviderMetadata, ProviderType};
@@ -414,6 +415,11 @@ derive_utoipa!(Icon as IconSchema);
super::routes::telemetry::send_telemetry_event,
super::routes::dictation::transcribe_dictation,
super::routes::dictation::get_dictation_config,
super::routes::dictation::list_models,
super::routes::dictation::download_model,
super::routes::dictation::get_download_progress,
super::routes::dictation::cancel_download,
super::routes::dictation::delete_model,
),
components(schemas(
super::routes::config_management::UpsertConfigQuery,
@@ -574,8 +580,11 @@ derive_utoipa!(Icon as IconSchema);
goose::goose_apps::ResourceMetadata,
super::routes::dictation::TranscribeRequest,
super::routes::dictation::TranscribeResponse,
super::routes::dictation::DictationProvider,
goose::dictation::providers::DictationProvider,
super::routes::dictation::DictationProviderStatus,
super::routes::dictation::WhisperModelResponse,
DownloadProgress,
DownloadStatus,
))
)]
pub struct ApiDoc;
crates/goose-server/src/routes/dictation.rs
+202 -240
View File
@@ -1,76 +1,31 @@
use crate::routes::errors::ErrorResponse;
use crate::state::AppState;
use axum::{
extract::{DefaultBodyLimit, Path},
http::StatusCode,
routing::{get, post},
routing::{delete, get, post},
Json, Router,
};
use base64::{engine::general_purpose::STANDARD as BASE64, Engine};
use goose::providers::api_client::{ApiClient, AuthMethod};
use goose::dictation::download_manager::{get_download_manager, DownloadProgress};
use goose::dictation::providers::{
is_configured, transcribe_local, transcribe_with_provider, DictationProvider, PROVIDERS,
};
use goose::dictation::whisper;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
use utoipa::ToSchema;
const MAX_AUDIO_SIZE_BYTES: usize = 25 * 1024 * 1024;
const REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
const MAX_AUDIO_SIZE_BYTES: usize = 50 * 1024 * 1024;
// DictationProvider definitions
struct DictationProviderDef {
config_key: &'static str,
default_url: &'static str,
host_key: Option<&'static str>,
description: &'static str,
uses_provider_config: bool,
settings_path: Option<&'static str>,
}
const PROVIDERS: &[(&str, DictationProviderDef)] = &[
(
"openai",
DictationProviderDef {
config_key: "OPENAI_API_KEY",
default_url: "https://api.openai.com/v1/audio/transcriptions",
host_key: Some("OPENAI_HOST"),
description: "Uses OpenAI Whisper API for high-quality transcription.",
uses_provider_config: true,
settings_path: Some("Settings > Models"),
},
),
(
"elevenlabs",
DictationProviderDef {
config_key: "ELEVENLABS_API_KEY",
default_url: "https://api.elevenlabs.io/v1/speech-to-text",
host_key: None,
description: "Uses ElevenLabs speech-to-text API for advanced voice processing.",
uses_provider_config: false,
settings_path: None,
},
),
];
fn get_provider_def(name: &str) -> Option<&'static DictationProviderDef> {
PROVIDERS
.iter()
.find_map(|(n, def)| if *n == name { Some(def) } else { None })
}
#[derive(Debug, Deserialize, ToSchema)]
#[serde(rename_all = "lowercase")]
pub enum DictationProvider {
OpenAI,
ElevenLabs,
}
impl DictationProvider {
fn as_str(&self) -> &'static str {
match self {
DictationProvider::OpenAI => "openai",
DictationProvider::ElevenLabs => "elevenlabs",
}
}
#[derive(Debug, Serialize, ToSchema)]
pub struct WhisperModelResponse {
#[serde(flatten)]
#[schema(inline)]
model: &'static whisper::WhisperModel,
downloaded: bool,
recommended: bool,
}
#[derive(Debug, Deserialize, ToSchema)]
@@ -113,17 +68,6 @@ fn validate_audio(audio: &str, mime_type: &str) -> Result<(Vec<u8>, &'static str
.decode(audio)
.map_err(|_| ErrorResponse::bad_request("Invalid base64 audio data"))?;
if audio_bytes.len() > MAX_AUDIO_SIZE_BYTES {
return Err(ErrorResponse {
message: format!(
"Audio file too large: {} bytes (max: {} bytes)",
audio_bytes.len(),
MAX_AUDIO_SIZE_BYTES
),
status: StatusCode::PAYLOAD_TOO_LARGE,
});
}
let extension = match mime_type {
"audio/webm" | "audio/webm;codecs=opus" => "webm",
"audio/mp4" => "mp4",
@@ -141,164 +85,37 @@ fn validate_audio(audio: &str, mime_type: &str) -> Result<(Vec<u8>, &'static str
Ok((audio_bytes, extension))
}
async fn handle_response_error(response: reqwest::Response) -> ErrorResponse {
let status = response.status();
let error_text = response.text().await.unwrap_or_default();
fn convert_error(e: anyhow::Error) -> ErrorResponse {
let error_msg = e.to_string();
ErrorResponse {
message: if status == 401
|| error_text.contains("Invalid API key")
|| error_text.contains("Unauthorized")
{
"Invalid API key".to_string()
} else if status == 429 || error_text.contains("quota") || error_text.contains("limit") {
"Rate limit exceeded".to_string()
} else {
format!("API error: {}", error_text)
},
status: if status.is_client_error() {
status
} else {
StatusCode::BAD_GATEWAY
},
}
}
fn build_api_client(provider: &str) -> Result<ApiClient, ErrorResponse> {
let config = goose::config::Config::global();
let def = get_provider_def(provider)
.ok_or_else(|| ErrorResponse::bad_request(format!("Unknown provider: {}", provider)))?;
let api_key = config
.get_secret(def.config_key)
.map_err(|_| ErrorResponse {
message: format!("{} not configured", def.config_key),
status: StatusCode::PRECONDITION_FAILED,
})?;
let url = if let Some(host_key) = def.host_key {
config
.get(host_key, false)
.ok()
.and_then(|v| v.as_str().map(|s| s.to_string()))
.map(|custom_host| {
let path = def
.default_url
.splitn(4, '/')
.nth(3)
.map(|p| format!("/{}", p))
.unwrap_or_default();
format!("{}{}", custom_host.trim_end_matches('/'), path)
})
.unwrap_or_else(|| def.default_url.to_string())
} else {
def.default_url.to_string()
};
let auth = match provider {
"openai" => AuthMethod::BearerToken(api_key),
"elevenlabs" => AuthMethod::ApiKey {
header_name: "xi-api-key".to_string(),
key: api_key,
},
_ => {
return Err(ErrorResponse::bad_request(format!(
"Unknown provider: {}",
provider
)))
if error_msg.contains("Invalid API key") {
ErrorResponse {
message: error_msg,
status: StatusCode::UNAUTHORIZED,
}
};
ApiClient::with_timeout(url, auth, REQUEST_TIMEOUT)
.map_err(|e| ErrorResponse::internal(format!("Failed to create client: {}", e)))
}
async fn transcribe_openai(
audio_bytes: Vec<u8>,
extension: &str,
mime_type: &str,
client: &ApiClient,
) -> Result<String, ErrorResponse> {
let part = reqwest::multipart::Part::bytes(audio_bytes)
.file_name(format!("audio.{}", extension))
.mime_str(mime_type)
.map_err(|e| ErrorResponse::internal(format!("Failed to create multipart: {}", e)))?;
let form = reqwest::multipart::Form::new()
.part("file", part)
.text("model", "whisper-1");
let response = client
.request(None, "")
.multipart_post(form)
.await
.map_err(|e| ErrorResponse {
message: if e.to_string().contains("timeout") {
"Request timed out".to_string()
} else {
format!("Request failed: {}", e)
},
status: if e.to_string().contains("timeout") {
StatusCode::GATEWAY_TIMEOUT
} else {
StatusCode::SERVICE_UNAVAILABLE
},
})?;
if !response.status().is_success() {
return Err(handle_response_error(response).await);
} else if error_msg.contains("Rate limit exceeded") || error_msg.contains("quota") {
ErrorResponse {
message: error_msg,
status: StatusCode::TOO_MANY_REQUESTS,
}
} else if error_msg.contains("not configured") {
ErrorResponse {
message: error_msg,
status: StatusCode::PRECONDITION_FAILED,
}
} else if error_msg.contains("timeout") {
ErrorResponse {
message: error_msg,
status: StatusCode::GATEWAY_TIMEOUT,
}
} else if error_msg.contains("API error") {
ErrorResponse {
message: error_msg,
status: StatusCode::BAD_GATEWAY,
}
} else {
ErrorResponse::internal(error_msg)
}
let data: TranscribeResponse = response
.json()
.await
.map_err(|e| ErrorResponse::internal(format!("Failed to parse response: {}", e)))?;
Ok(data.text)
}
async fn transcribe_elevenlabs(
audio_bytes: Vec<u8>,
extension: &str,
mime_type: &str,
client: &ApiClient,
) -> Result<String, ErrorResponse> {
let part = reqwest::multipart::Part::bytes(audio_bytes)
.file_name(format!("audio.{}", extension))
.mime_str(mime_type)
.map_err(|_| ErrorResponse::internal("Failed to create multipart"))?;
let form = reqwest::multipart::Form::new()
.part("file", part)
.text("model_id", "scribe_v1");
let response = client
.request(None, "")
.multipart_post(form)
.await
.map_err(|e| ErrorResponse {
message: if e.to_string().contains("timeout") {
"Request timed out".to_string()
} else {
format!("Request failed: {}", e)
},
status: if e.to_string().contains("timeout") {
StatusCode::GATEWAY_TIMEOUT
} else {
StatusCode::SERVICE_UNAVAILABLE
},
})?;
if !response.status().is_success() {
return Err(handle_response_error(response).await);
}
let data: TranscribeResponse = response
.json()
.await
.map_err(|e| ErrorResponse::internal(format!("Failed to parse response: {}", e)))?;
Ok(data.text)
}
#[utoipa::path(
@@ -309,11 +126,11 @@ async fn transcribe_elevenlabs(
(status = 200, description = "Audio transcribed successfully", body = TranscribeResponse),
(status = 400, description = "Invalid request (bad base64 or unsupported format)"),
(status = 401, description = "Invalid API key"),
(status = 412, description = "DictationProvider not configured"),
(status = 412, description = "Provider not configured"),
(status = 413, description = "Audio file too large (max 25MB)"),
(status = 429, description = "Rate limit exceeded"),
(status = 500, description = "Internal server error"),
(status = 502, description = "DictationProvider API error"),
(status = 502, description = "Provider API error"),
(status = 503, description = "Service unavailable"),
(status = 504, description = "Request timeout")
)
@@ -322,16 +139,39 @@ pub async fn transcribe_dictation(
Json(request): Json<TranscribeRequest>,
) -> Result<Json<TranscribeResponse>, ErrorResponse> {
let (audio_bytes, extension) = validate_audio(&request.audio, &request.mime_type)?;
let provider_name = request.provider.as_str();
let client = build_api_client(provider_name)?;
let text = match request.provider {
DictationProvider::OpenAI => {
transcribe_openai(audio_bytes, extension, &request.mime_type, &client).await?
}
DictationProvider::ElevenLabs => {
transcribe_elevenlabs(audio_bytes, extension, &request.mime_type, &client).await?
}
DictationProvider::OpenAI => transcribe_with_provider(
DictationProvider::OpenAI,
"model".to_string(),
"whisper-1".to_string(),
audio_bytes,
extension,
&request.mime_type,
)
.await
.map_err(convert_error)?,
DictationProvider::Groq => transcribe_with_provider(
DictationProvider::Groq,
"model".to_string(),
"whisper-large-v3-turbo".to_string(),
audio_bytes,
extension,
&request.mime_type,
)
.await
.map_err(convert_error)?,
DictationProvider::ElevenLabs => transcribe_with_provider(
DictationProvider::ElevenLabs,
"model_id".to_string(),
"scribe_v1".to_string(),
audio_bytes,
extension,
&request.mime_type,
)
.await
.map_err(convert_error)?,
DictationProvider::Local => transcribe_local(audio_bytes).await.map_err(convert_error)?,
};
Ok(Json(TranscribeResponse { text }))
@@ -345,12 +185,13 @@ pub async fn transcribe_dictation(
)
)]
pub async fn get_dictation_config(
) -> Result<Json<HashMap<String, DictationProviderStatus>>, ErrorResponse> {
) -> Result<Json<HashMap<DictationProvider, DictationProviderStatus>>, ErrorResponse> {
let config = goose::config::Config::global();
let mut providers = HashMap::new();
for (name, def) in PROVIDERS.iter() {
let configured = config.get_secret::<String>(def.config_key).is_ok();
for def in PROVIDERS {
let provider = def.provider;
let configured = is_configured(provider);
let host = if let Some(host_key) = def.host_key {
config
@@ -362,7 +203,7 @@ pub async fn get_dictation_config(
};
providers.insert(
name.to_string(),
provider,
DictationProviderStatus {
configured,
host,
@@ -381,9 +222,130 @@ pub async fn get_dictation_config(
Ok(Json(providers))
}
#[utoipa::path(
get,
path = "/dictation/models",
responses(
(status = 200, description = "List of available Whisper models", body = Vec<WhisperModelResponse>)
)
)]
pub async fn list_models() -> Result<Json<Vec<WhisperModelResponse>>, ErrorResponse> {
let recommended_id = whisper::recommend_model();
let models = whisper::available_models()
.iter()
.map(|m| WhisperModelResponse {
model: m,
downloaded: m.is_downloaded(),
recommended: m.id == recommended_id,
})
.collect();
Ok(Json(models))
}
#[utoipa::path(
post,
path = "/dictation/models/{model_id}/download",
responses(
(status = 202, description = "Download started"),
(status = 400, description = "Download already in progress"),
(status = 500, description = "Internal server error")
)
)]
pub async fn download_model(Path(model_id): Path<String>) -> Result<StatusCode, ErrorResponse> {
let model = whisper::get_model(&model_id)
.ok_or_else(|| ErrorResponse::bad_request("Model not found"))?;
let manager = get_download_manager();
manager
.download_model(
model.id.to_string(),
model.url.to_string(),
model.local_path(),
)
.await
.map_err(convert_error)?;
Ok(StatusCode::ACCEPTED)
}
#[utoipa::path(
get,
path = "/dictation/models/{model_id}/download",
responses(
(status = 200, description = "Download progress", body = DownloadProgress),
(status = 404, description = "Download not found")
)
)]
pub async fn get_download_progress(
Path(model_id): Path<String>,
) -> Result<Json<DownloadProgress>, ErrorResponse> {
let manager = get_download_manager();
let progress = manager
.get_progress(&model_id)
.ok_or_else(|| ErrorResponse::bad_request("Download not found"))?;
Ok(Json(progress))
}
#[utoipa::path(
delete,
path = "/dictation/models/{model_id}/download",
responses(
(status = 200, description = "Download cancelled"),
(status = 404, description = "Download not found")
)
)]
pub async fn cancel_download(Path(model_id): Path<String>) -> Result<StatusCode, ErrorResponse> {
let manager = get_download_manager();
manager.cancel_download(&model_id).map_err(convert_error)?;
Ok(StatusCode::OK)
}
#[utoipa::path(
delete,
path = "/dictation/models/{model_id}",
responses(
(status = 200, description = "Model deleted"),
(status = 404, description = "Model not found or not downloaded"),
(status = 500, description = "Failed to delete model")
)
)]
pub async fn delete_model(Path(model_id): Path<String>) -> Result<StatusCode, ErrorResponse> {
let model = whisper::get_model(&model_id)
.ok_or_else(|| ErrorResponse::bad_request("Model not found"))?;
let path = model.local_path();
if !path.exists() {
return Err(ErrorResponse::bad_request("Model not downloaded"));
}
tokio::fs::remove_file(&path)
.await
.map_err(|e| ErrorResponse::internal(format!("Failed to delete model: {}", e)))?;
Ok(StatusCode::OK)
}
pub fn routes(state: Arc<AppState>) -> Router {
Router::new()
.route("/dictation/transcribe", post(transcribe_dictation))
.route("/dictation/config", get(get_dictation_config))
.route("/dictation/models", get(list_models))
.route(
"/dictation/models/{model_id}/download",
post(download_model),
)
.route(
"/dictation/models/{model_id}/download",
get(get_download_progress),
)
.route(
"/dictation/models/{model_id}/download",
delete(cancel_download),
)
.route("/dictation/models/{model_id}", delete(delete_model))
.layer(DefaultBodyLimit::max(MAX_AUDIO_SIZE_BYTES))
.with_state(state)
}