tkmind/tkmind_go
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

Fix eleven labs audio transcription and added more logging (#4358)

Browse Source
This commit is contained in:
Zane
2025-08-27 12:53:05 -07:00
committed by GitHub
parent 862dd97ca4
commit 1014d9e519
7 changed files with 316 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/goose-server/src/routes/audio.rs
+182 -137
View File
@@ -42,6 +42,155 @@ struct WhisperResponse {
text: String,
}
/// Validate audio input and return decoded bytes and file extension
fn validate_audio_input(
audio: &str,
mime_type: &str,
) -> Result<(Vec<u8>, &'static str), StatusCode> {
// Decode the base64 audio data
let audio_bytes = BASE64.decode(audio).map_err(|_| StatusCode::BAD_REQUEST)?;
// Check file size
if audio_bytes.len() > MAX_AUDIO_SIZE_BYTES {
tracing::warn!(
"Audio file too large: {} bytes (max: {} bytes)",
audio_bytes.len(),
MAX_AUDIO_SIZE_BYTES
);
return Err(StatusCode::PAYLOAD_TOO_LARGE);
}
// Determine file extension based on MIME type
let file_extension = match mime_type {
"audio/webm" => "webm",
"audio/webm;codecs=opus" => "webm",
"audio/mp4" => "mp4",
"audio/mpeg" => "mp3",
"audio/mpga" => "mpga",
"audio/m4a" => "m4a",
"audio/wav" => "wav",
"audio/x-wav" => "wav",
_ => return Err(StatusCode::UNSUPPORTED_MEDIA_TYPE),
};
Ok((audio_bytes, file_extension))
}
/// Get OpenAI configuration (API key and host)
fn get_openai_config() -> Result<(String, String), StatusCode> {
let config = goose::config::Config::global();
let api_key: String = config.get_secret("OPENAI_API_KEY").map_err(|e| {
tracing::error!("Failed to get OpenAI API key: {:?}", e);
StatusCode::PRECONDITION_FAILED
})?;
let openai_host = match config.get("OPENAI_HOST", false) {
Ok(value) => value
.as_str()
.map(|s| s.to_string())
.unwrap_or_else(|| "https://api.openai.com".to_string()),
Err(_) => "https://api.openai.com".to_string(),
};
Ok((api_key, openai_host))
}
/// Send transcription request to OpenAI Whisper API
async fn send_openai_request(
audio_bytes: Vec<u8>,
file_extension: &str,
mime_type: &str,
api_key: &str,
openai_host: &str,
) -> Result<WhisperResponse, StatusCode> {
tracing::info!("Using OpenAI host: {}", openai_host);
tracing::info!(
"Audio file size: {} bytes, extension: {}, mime_type: {}",
audio_bytes.len(),
file_extension,
mime_type
);
// Create a multipart form with the audio file
let part = reqwest::multipart::Part::bytes(audio_bytes)
.file_name(format!("audio.{}", file_extension))
.mime_str(mime_type)
.map_err(|e| {
tracing::error!("Failed to create multipart part: {:?}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
let form = reqwest::multipart::Form::new()
.part("file", part)
.text("model", "whisper-1")
.text("response_format", "json");
tracing::info!("Created multipart form for OpenAI Whisper API");
// Make request to OpenAI Whisper API
let client = Client::builder()
.timeout(Duration::from_secs(OPENAI_TIMEOUT_SECONDS))
.build()
.map_err(|e| {
tracing::error!("Failed to create HTTP client: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
tracing::info!(
"Sending request to OpenAI: {}/v1/audio/transcriptions",
openai_host
);
let response = client
.post(format!("{}/v1/audio/transcriptions", openai_host))
.header("Authorization", format!("Bearer {}", api_key))
.multipart(form)
.send()
.await
.map_err(|e| {
if e.is_timeout() {
tracing::error!(
"OpenAI API request timed out after {}s",
OPENAI_TIMEOUT_SECONDS
);
StatusCode::GATEWAY_TIMEOUT
} else {
tracing::error!("Failed to send request to OpenAI: {}", e);
StatusCode::SERVICE_UNAVAILABLE
}
})?;
tracing::info!(
"Received response from OpenAI with status: {}",
response.status()
);
if !response.status().is_success() {
let status = response.status();
let error_text = response.text().await.unwrap_or_default();
tracing::error!("OpenAI API error (status: {}): {}", status, error_text);
// Check for specific error codes
if status == 401 {
tracing::error!("OpenAI API key appears to be invalid or unauthorized");
return Err(StatusCode::UNAUTHORIZED);
} else if status == 429 {
tracing::error!("OpenAI API quota or rate limit exceeded");
return Err(StatusCode::TOO_MANY_REQUESTS);
}
return Err(StatusCode::BAD_GATEWAY);
}
let whisper_response: WhisperResponse = response.json().await.map_err(|e| {
tracing::error!("Failed to parse OpenAI response: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
Ok(whisper_response)
}
/// Transcribe audio using OpenAI's Whisper API
///
/// # Request
@@ -66,100 +215,17 @@ async fn transcribe_handler(
) -> Result<Json<TranscribeResponse>, StatusCode> {
verify_secret_key(&headers, &state)?;
// Validate input first before checking API key configuration
// Decode the base64 audio data
let audio_bytes = BASE64
.decode(&request.audio)
.map_err(|_| StatusCode::BAD_REQUEST)?;
let (audio_bytes, file_extension) = validate_audio_input(&request.audio, &request.mime_type)?;
let (api_key, openai_host) = get_openai_config()?;
// Check file size
if audio_bytes.len() > MAX_AUDIO_SIZE_BYTES {
tracing::warn!(
"Audio file too large: {} bytes (max: {} bytes)",
audio_bytes.len(),
MAX_AUDIO_SIZE_BYTES
);
return Err(StatusCode::PAYLOAD_TOO_LARGE);
}
// Determine file extension based on MIME type
let file_extension = match request.mime_type.as_str() {
"audio/webm" => "webm",
"audio/mp4" => "mp4",
"audio/mpeg" => "mp3",
"audio/mpga" => "mpga",
"audio/m4a" => "m4a",
"audio/wav" => "wav",
"audio/x-wav" => "wav",
_ => return Err(StatusCode::UNSUPPORTED_MEDIA_TYPE),
};
// Get the OpenAI API key from config (after input validation)
let config = goose::config::Config::global();
let api_key: String = config
.get_secret("OPENAI_API_KEY")
.map_err(|_| StatusCode::PRECONDITION_FAILED)?;
// Get the OpenAI host from config (with default)
let openai_host = match config.get("OPENAI_HOST", false) {
Ok(value) => value
.as_str()
.map(|s| s.to_string())
.unwrap_or_else(|| "https://api.openai.com".to_string()),
Err(_) => "https://api.openai.com".to_string(),
};
tracing::debug!("Using OpenAI host: {}", openai_host);
// Create a multipart form with the audio file
let part = reqwest::multipart::Part::bytes(audio_bytes)
.file_name(format!("audio.{}", file_extension))
.mime_str(&request.mime_type)
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
let form = reqwest::multipart::Form::new()
.part("file", part)
.text("model", "whisper-1")
.text("response_format", "json");
// Make request to OpenAI Whisper API
let client = Client::builder()
.timeout(Duration::from_secs(OPENAI_TIMEOUT_SECONDS))
.build()
.map_err(|e| {
tracing::error!("Failed to create HTTP client: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
let response = client
.post(format!("{}/v1/audio/transcriptions", openai_host))
.header("Authorization", format!("Bearer {}", api_key))
.multipart(form)
.send()
.await
.map_err(|e| {
if e.is_timeout() {
tracing::error!(
"OpenAI API request timed out after {}s",
OPENAI_TIMEOUT_SECONDS
);
StatusCode::GATEWAY_TIMEOUT
} else {
tracing::error!("Failed to send request to OpenAI: {}", e);
StatusCode::SERVICE_UNAVAILABLE
}
})?;
if !response.status().is_success() {
let error_text = response.text().await.unwrap_or_default();
tracing::error!("OpenAI API error: {}", error_text);
return Err(StatusCode::BAD_GATEWAY);
}
let whisper_response: WhisperResponse = response.json().await.map_err(|e| {
tracing::error!("Failed to parse OpenAI response: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
let whisper_response = send_openai_request(
audio_bytes,
file_extension,
&request.mime_type,
&api_key,
&openai_host,
)
.await?;
Ok(Json(TranscribeResponse {
text: whisper_response.text,
@@ -177,39 +243,13 @@ async fn transcribe_elevenlabs_handler(
) -> Result<Json<TranscribeResponse>, StatusCode> {
verify_secret_key(&headers, &state)?;
// Validate input first before checking API key configuration
// Decode the base64 audio data
let audio_bytes = BASE64
.decode(&request.audio)
.map_err(|_| StatusCode::BAD_REQUEST)?;
// Check file size
if audio_bytes.len() > MAX_AUDIO_SIZE_BYTES {
tracing::warn!(
"Audio file too large: {} bytes (max: {} bytes)",
audio_bytes.len(),
MAX_AUDIO_SIZE_BYTES
);
return Err(StatusCode::PAYLOAD_TOO_LARGE);
}
// Determine file extension and content type based on MIME type
let (file_extension, content_type) = match request.mime_type.as_str() {
"audio/webm" => ("webm", "audio/webm"),
"audio/mp4" => ("mp4", "audio/mp4"),
"audio/mpeg" => ("mp3", "audio/mpeg"),
"audio/mpga" => ("mp3", "audio/mpeg"),
"audio/m4a" => ("m4a", "audio/m4a"),
"audio/wav" => ("wav", "audio/wav"),
"audio/x-wav" => ("wav", "audio/wav"),
_ => return Err(StatusCode::UNSUPPORTED_MEDIA_TYPE),
};
let (audio_bytes, file_extension) = validate_audio_input(&request.audio, &request.mime_type)?;
// Get the ElevenLabs API key from config (after input validation)
let config = goose::config::Config::global();
// First try to get it as a secret
let api_key: String = match config.get_secret("ELEVENLABS_API_KEY") {
let api_key: String = match config.get_secret::<String>("ELEVENLABS_API_KEY") {
Ok(key) => key,
Err(_) => {
// Try to get it as non-secret (for backward compatibility)
@@ -217,7 +257,6 @@ async fn transcribe_elevenlabs_handler(
Ok(value) => {
match value.as_str() {
Some(key_str) => {
tracing::info!("Migrating ElevenLabs API key to secret storage");
let key = key_str.to_string();
// Migrate to secret storage
if let Err(e) = config.set(
@@ -228,17 +267,25 @@ async fn transcribe_elevenlabs_handler(
tracing::error!("Failed to migrate ElevenLabs API key: {:?}", e);
}
// Delete the non-secret version
let _ = config.delete("ELEVENLABS_API_KEY");
if let Err(e) = config.delete("ELEVENLABS_API_KEY") {
tracing::warn!(
"Failed to delete non-secret ElevenLabs API key: {:?}",
e
);
}
key
}
None => {
tracing::error!("ElevenLabs API key is not a string");
tracing::error!(
"ElevenLabs API key is not a string, found: {:?}",
value
);
return Err(StatusCode::PRECONDITION_FAILED);
}
}
}
Err(e) => {
tracing::error!("Failed to get ElevenLabs API key from config: {:?}", e);
Err(_) => {
tracing::error!("No ElevenLabs API key found in configuration");
return Err(StatusCode::PRECONDITION_FAILED);
}
}
@@ -248,7 +295,7 @@ async fn transcribe_elevenlabs_handler(
// Create multipart form for ElevenLabs API
let part = reqwest::multipart::Part::bytes(audio_bytes)
.file_name(format!("audio.{}", file_extension))
.mime_str(content_type)
.mime_str(&request.mime_type)
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
let form = reqwest::multipart::Form::new()
@@ -286,8 +333,9 @@ async fn transcribe_elevenlabs_handler(
})?;
if !response.status().is_success() {
let status = response.status();
let error_text = response.text().await.unwrap_or_default();
tracing::error!("ElevenLabs API error: {}", error_text);
tracing::error!("ElevenLabs API error (status: {}): {}", status, error_text);
// Check for specific error codes
if error_text.contains("Unauthorized") || error_text.contains("Invalid API key") {
@@ -330,16 +378,13 @@ async fn check_dictation_config(
let config = goose::config::Config::global();
// Check if ElevenLabs API key is configured
let has_elevenlabs = config
.get_secret::<String>("ELEVENLABS_API_KEY")
.map(|_| true)
.unwrap_or_else(|_| {
let has_elevenlabs = match config.get_secret::<String>("ELEVENLABS_API_KEY") {
Ok(_) => true,
Err(_) => {
// Check non-secret for backward compatibility
config
.get("ELEVENLABS_API_KEY", false)
.map(|_| true)
.unwrap_or(false)
});
config.get("ELEVENLABS_API_KEY", false).is_ok()
}
};
Ok(Json(serde_json::json!({
"elevenlabs": has_elevenlabs
crates/goose-server/src/routes/config_management.rs
+4 -1
View File
@@ -167,6 +167,7 @@ pub async fn read_config(
}
let config = Config::global();
let response_value = match config.get(&query.key, query.is_secret) {
Ok(value) => {
if query.is_secret {
@@ -182,7 +183,9 @@ pub async fn read_config(
Value::Null
}
}
Err(_) => return Err(StatusCode::INTERNAL_SERVER_ERROR),
Err(_) => {
return Err(StatusCode::INTERNAL_SERVER_ERROR);
}
};
Ok(Json(response_value))
}