feat(goose2): voice dictation via direct-ACP pattern (#8609)
Signed-off-by: tulsi <tulsi@block.xyz> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
tulsi
@@ -14,6 +14,7 @@ path = "src/bin/generate_acp_schema.rs"
|
||||
[features]
|
||||
default = ["code-mode", "rustls-tls"]
|
||||
code-mode = ["goose/code-mode"]
|
||||
local-inference = ["goose/local-inference"]
|
||||
rustls-tls = ["goose/rustls-tls", "goose-mcp/rustls-tls"]
|
||||
native-tls = ["goose/native-tls", "goose-mcp/native-tls"]
|
||||
|
||||
@@ -48,6 +49,7 @@ uuid = { workspace = true, features = ["v7"] }
|
||||
schemars = { workspace = true, features = ["derive"] }
|
||||
goose-acp-macros = { path = "../goose-acp-macros" }
|
||||
goose-sdk = { path = "../goose-sdk" }
|
||||
base64 = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
async-trait = { workspace = true }
|
||||
|
||||
@@ -104,6 +104,46 @@
|
||||
"method": "_goose/session/unarchive",
|
||||
"requestType": "UnarchiveSessionRequest",
|
||||
"responseType": "EmptyResponse"
|
||||
},
|
||||
{
|
||||
"method": "_goose/dictation/transcribe",
|
||||
"requestType": "DictationTranscribeRequest",
|
||||
"responseType": "DictationTranscribeResponse"
|
||||
},
|
||||
{
|
||||
"method": "_goose/dictation/config",
|
||||
"requestType": "DictationConfigRequest",
|
||||
"responseType": "DictationConfigResponse"
|
||||
},
|
||||
{
|
||||
"method": "_goose/dictation/models/list",
|
||||
"requestType": "DictationModelsListRequest",
|
||||
"responseType": "DictationModelsListResponse"
|
||||
},
|
||||
{
|
||||
"method": "_goose/dictation/models/download",
|
||||
"requestType": "DictationModelDownloadRequest",
|
||||
"responseType": "EmptyResponse"
|
||||
},
|
||||
{
|
||||
"method": "_goose/dictation/models/download/progress",
|
||||
"requestType": "DictationModelDownloadProgressRequest",
|
||||
"responseType": "DictationModelDownloadProgressResponse"
|
||||
},
|
||||
{
|
||||
"method": "_goose/dictation/models/cancel",
|
||||
"requestType": "DictationModelCancelRequest",
|
||||
"responseType": "EmptyResponse"
|
||||
},
|
||||
{
|
||||
"method": "_goose/dictation/models/delete",
|
||||
"requestType": "DictationModelDeleteRequest",
|
||||
"responseType": "EmptyResponse"
|
||||
},
|
||||
{
|
||||
"method": "_goose/dictation/model/select",
|
||||
"requestType": "DictationModelSelectRequest",
|
||||
"responseType": "EmptyResponse"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -607,6 +607,329 @@
|
||||
"x-side": "agent",
|
||||
"x-method": "_goose/session/unarchive"
|
||||
},
|
||||
"DictationTranscribeRequest": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"audio": {
|
||||
"type": "string",
|
||||
"description": "Base64-encoded audio data"
|
||||
},
|
||||
"mimeType": {
|
||||
"type": "string",
|
||||
"description": "MIME type (e.g. \"audio/wav\", \"audio/webm\")"
|
||||
},
|
||||
"provider": {
|
||||
"type": "string",
|
||||
"description": "Provider to use: \"openai\", \"groq\", \"elevenlabs\", or \"local\""
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"audio",
|
||||
"mimeType",
|
||||
"provider"
|
||||
],
|
||||
"description": "Transcribe audio via a dictation provider.",
|
||||
"x-side": "agent",
|
||||
"x-method": "_goose/dictation/transcribe"
|
||||
},
|
||||
"DictationTranscribeResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"text": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"text"
|
||||
],
|
||||
"description": "Transcription result.",
|
||||
"x-side": "agent",
|
||||
"x-method": "_goose/dictation/transcribe"
|
||||
},
|
||||
"DictationConfigRequest": {
|
||||
"type": "object",
|
||||
"description": "Get the configuration status of all dictation providers.",
|
||||
"x-side": "agent",
|
||||
"x-method": "_goose/dictation/config"
|
||||
},
|
||||
"DictationConfigResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"providers": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"$ref": "#/$defs/DictationProviderStatusEntry"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"providers"
|
||||
],
|
||||
"description": "Dictation config response — map of provider name to status.",
|
||||
"x-side": "agent",
|
||||
"x-method": "_goose/dictation/config"
|
||||
},
|
||||
"DictationProviderStatusEntry": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"configured": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"host": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"description": {
|
||||
"type": "string"
|
||||
},
|
||||
"usesProviderConfig": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"settingsPath": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"configKey": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"modelConfigKey": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"defaultModel": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"selectedModel": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"availableModels": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/DictationModelOption"
|
||||
},
|
||||
"default": []
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"configured",
|
||||
"description",
|
||||
"usesProviderConfig"
|
||||
],
|
||||
"description": "Per-provider configuration status."
|
||||
},
|
||||
"DictationModelOption": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"label": {
|
||||
"type": "string"
|
||||
},
|
||||
"description": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"label",
|
||||
"description"
|
||||
]
|
||||
},
|
||||
"DictationModelsListRequest": {
|
||||
"type": "object",
|
||||
"description": "List available local Whisper models with their download status.",
|
||||
"x-side": "agent",
|
||||
"x-method": "_goose/dictation/models/list"
|
||||
},
|
||||
"DictationModelsListResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"models": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/DictationLocalModelStatus"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"models"
|
||||
],
|
||||
"x-side": "agent",
|
||||
"x-method": "_goose/dictation/models/list"
|
||||
},
|
||||
"DictationLocalModelStatus": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"label": {
|
||||
"type": "string"
|
||||
},
|
||||
"description": {
|
||||
"type": "string"
|
||||
},
|
||||
"sizeMb": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"downloaded": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"downloadInProgress": {
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"label",
|
||||
"description",
|
||||
"sizeMb",
|
||||
"downloaded",
|
||||
"downloadInProgress"
|
||||
]
|
||||
},
|
||||
"DictationModelDownloadRequest": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"modelId": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"modelId"
|
||||
],
|
||||
"description": "Kick off a background download of a local Whisper model.",
|
||||
"x-side": "agent",
|
||||
"x-method": "_goose/dictation/models/download"
|
||||
},
|
||||
"DictationModelDownloadProgressRequest": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"modelId": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"modelId"
|
||||
],
|
||||
"description": "Poll the progress of an in-flight download.",
|
||||
"x-side": "agent",
|
||||
"x-method": "_goose/dictation/models/download/progress"
|
||||
},
|
||||
"DictationModelDownloadProgressResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"progress": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/$defs/DictationDownloadProgress"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "None when no download is active for this model id."
|
||||
}
|
||||
},
|
||||
"x-side": "agent",
|
||||
"x-method": "_goose/dictation/models/download/progress"
|
||||
},
|
||||
"DictationDownloadProgress": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"bytesDownloaded": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"totalBytes": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"progressPercent": {
|
||||
"type": "number",
|
||||
"format": "float"
|
||||
},
|
||||
"status": {
|
||||
"type": "string",
|
||||
"description": "serde lowercase of DownloadStatus: \"downloading\" | \"completed\" | \"failed\" | \"cancelled\""
|
||||
},
|
||||
"error": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"bytesDownloaded",
|
||||
"totalBytes",
|
||||
"progressPercent",
|
||||
"status"
|
||||
]
|
||||
},
|
||||
"DictationModelCancelRequest": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"modelId": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"modelId"
|
||||
],
|
||||
"description": "Cancel an in-flight download.",
|
||||
"x-side": "agent",
|
||||
"x-method": "_goose/dictation/models/cancel"
|
||||
},
|
||||
"DictationModelDeleteRequest": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"modelId": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"modelId"
|
||||
],
|
||||
"description": "Delete a downloaded local Whisper model from disk.",
|
||||
"x-side": "agent",
|
||||
"x-method": "_goose/dictation/models/delete"
|
||||
},
|
||||
"DictationModelSelectRequest": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"provider": {
|
||||
"type": "string"
|
||||
},
|
||||
"modelId": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"provider",
|
||||
"modelId"
|
||||
],
|
||||
"description": "Persist the user's model selection for a given provider.",
|
||||
"x-side": "agent",
|
||||
"x-method": "_goose/dictation/model/select"
|
||||
},
|
||||
"ExtRequest": {
|
||||
"properties": {
|
||||
"id": {
|
||||
@@ -807,6 +1130,78 @@
|
||||
],
|
||||
"description": "Params for _goose/session/unarchive",
|
||||
"title": "UnarchiveSessionRequest"
|
||||
},
|
||||
{
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/$defs/DictationTranscribeRequest"
|
||||
}
|
||||
],
|
||||
"description": "Params for _goose/dictation/transcribe",
|
||||
"title": "DictationTranscribeRequest"
|
||||
},
|
||||
{
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/$defs/DictationConfigRequest"
|
||||
}
|
||||
],
|
||||
"description": "Params for _goose/dictation/config",
|
||||
"title": "DictationConfigRequest"
|
||||
},
|
||||
{
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/$defs/DictationModelsListRequest"
|
||||
}
|
||||
],
|
||||
"description": "Params for _goose/dictation/models/list",
|
||||
"title": "DictationModelsListRequest"
|
||||
},
|
||||
{
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/$defs/DictationModelDownloadRequest"
|
||||
}
|
||||
],
|
||||
"description": "Params for _goose/dictation/models/download",
|
||||
"title": "DictationModelDownloadRequest"
|
||||
},
|
||||
{
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/$defs/DictationModelDownloadProgressRequest"
|
||||
}
|
||||
],
|
||||
"description": "Params for _goose/dictation/models/download/progress",
|
||||
"title": "DictationModelDownloadProgressRequest"
|
||||
},
|
||||
{
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/$defs/DictationModelCancelRequest"
|
||||
}
|
||||
],
|
||||
"description": "Params for _goose/dictation/models/cancel",
|
||||
"title": "DictationModelCancelRequest"
|
||||
},
|
||||
{
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/$defs/DictationModelDeleteRequest"
|
||||
}
|
||||
],
|
||||
"description": "Params for _goose/dictation/models/delete",
|
||||
"title": "DictationModelDeleteRequest"
|
||||
},
|
||||
{
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/$defs/DictationModelSelectRequest"
|
||||
}
|
||||
],
|
||||
"description": "Params for _goose/dictation/model/select",
|
||||
"title": "DictationModelSelectRequest"
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -933,6 +1328,38 @@
|
||||
}
|
||||
],
|
||||
"title": "ImportSessionResponse"
|
||||
},
|
||||
{
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/$defs/DictationTranscribeResponse"
|
||||
}
|
||||
],
|
||||
"title": "DictationTranscribeResponse"
|
||||
},
|
||||
{
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/$defs/DictationConfigResponse"
|
||||
}
|
||||
],
|
||||
"title": "DictationConfigResponse"
|
||||
},
|
||||
{
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/$defs/DictationModelsListResponse"
|
||||
}
|
||||
],
|
||||
"title": "DictationModelsListResponse"
|
||||
},
|
||||
{
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/$defs/DictationModelDownloadProgressResponse"
|
||||
}
|
||||
],
|
||||
"title": "DictationModelDownloadProgressResponse"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -16,6 +16,13 @@ use goose::config::paths::Paths;
|
||||
use goose::config::permission::PermissionManager;
|
||||
use goose::config::{Config, GooseMode};
|
||||
use goose::conversation::message::{ActionRequiredData, Message, MessageContent};
|
||||
#[cfg(feature = "local-inference")]
|
||||
use goose::dictation::providers::transcribe_local;
|
||||
use goose::dictation::providers::{
|
||||
all_providers, is_configured, transcribe_with_provider, DictationProvider,
|
||||
};
|
||||
#[cfg(feature = "local-inference")]
|
||||
use goose::dictation::whisper;
|
||||
use goose::mcp_utils::ToolResult;
|
||||
use goose::permission::permission_confirmation::PrincipalType;
|
||||
use goose::permission::{Permission, PermissionConfirmation};
|
||||
@@ -68,6 +75,12 @@ pub type AcpProviderFactory = Arc<
|
||||
|
||||
const DEFAULT_PROVIDER_ID: &str = "goose";
|
||||
const DEFAULT_PROVIDER_LABEL: &str = "Goose (Default)";
|
||||
const OPENAI_TRANSCRIPTION_MODEL_CONFIG_KEY: &str = "OPENAI_TRANSCRIPTION_MODEL";
|
||||
const GROQ_TRANSCRIPTION_MODEL_CONFIG_KEY: &str = "GROQ_TRANSCRIPTION_MODEL";
|
||||
const ELEVENLABS_TRANSCRIPTION_MODEL_CONFIG_KEY: &str = "ELEVENLABS_TRANSCRIPTION_MODEL";
|
||||
const OPENAI_TRANSCRIPTION_MODEL: &str = "whisper-1";
|
||||
const GROQ_TRANSCRIPTION_MODEL: &str = "whisper-large-v3-turbo";
|
||||
const ELEVENLABS_TRANSCRIPTION_MODEL: &str = "scribe_v1";
|
||||
|
||||
/// In-memory state for an active ACP session.
|
||||
///
|
||||
@@ -2904,6 +2917,420 @@ impl GooseAcpAgent {
|
||||
.map_err(|e| sacp::Error::internal_error().data(e.to_string()))?;
|
||||
Ok(EmptyResponse {})
|
||||
}
|
||||
|
||||
#[custom_method(DictationTranscribeRequest)]
|
||||
async fn on_dictation_transcribe(
|
||||
&self,
|
||||
req: DictationTranscribeRequest,
|
||||
) -> Result<DictationTranscribeResponse, sacp::Error> {
|
||||
use base64::{engine::general_purpose::STANDARD as BASE64, Engine};
|
||||
let config = goose::config::Config::global();
|
||||
|
||||
#[cfg(not(feature = "local-inference"))]
|
||||
if req.provider == "local" {
|
||||
return Err(sacp::Error::invalid_params()
|
||||
.data("Local inference is not available in this build"));
|
||||
}
|
||||
|
||||
let provider: DictationProvider = serde_json::from_value(serde_json::Value::String(
|
||||
req.provider.clone(),
|
||||
))
|
||||
.map_err(|_| {
|
||||
sacp::Error::invalid_params().data(format!("Unknown provider: {}", req.provider))
|
||||
})?;
|
||||
|
||||
let audio_bytes = BASE64
|
||||
.decode(&req.audio)
|
||||
.map_err(|_| sacp::Error::invalid_params().data("Invalid base64 audio data"))?;
|
||||
|
||||
if audio_bytes.len() > 50 * 1024 * 1024 {
|
||||
return Err(sacp::Error::invalid_params().data("Audio too large (max 50MB)"));
|
||||
}
|
||||
|
||||
let extension = match req.mime_type.as_str() {
|
||||
"audio/webm" | "audio/webm;codecs=opus" => "webm",
|
||||
"audio/mp4" => "mp4",
|
||||
"audio/mpeg" | "audio/mpga" => "mp3",
|
||||
"audio/m4a" => "m4a",
|
||||
"audio/wav" | "audio/x-wav" => "wav",
|
||||
other => {
|
||||
return Err(
|
||||
sacp::Error::invalid_params().data(format!("Unsupported format: {other}"))
|
||||
)
|
||||
}
|
||||
};
|
||||
|
||||
let text = match provider {
|
||||
DictationProvider::OpenAI => {
|
||||
let model = dictation_selected_model(config, DictationProvider::OpenAI)
|
||||
.unwrap_or_else(|| OPENAI_TRANSCRIPTION_MODEL.to_string());
|
||||
transcribe_with_provider(
|
||||
DictationProvider::OpenAI,
|
||||
"model".to_string(),
|
||||
model,
|
||||
audio_bytes,
|
||||
extension,
|
||||
&req.mime_type,
|
||||
)
|
||||
.await
|
||||
}
|
||||
DictationProvider::Groq => {
|
||||
let model = dictation_selected_model(config, DictationProvider::Groq)
|
||||
.unwrap_or_else(|| GROQ_TRANSCRIPTION_MODEL.to_string());
|
||||
transcribe_with_provider(
|
||||
DictationProvider::Groq,
|
||||
"model".to_string(),
|
||||
model,
|
||||
audio_bytes,
|
||||
extension,
|
||||
&req.mime_type,
|
||||
)
|
||||
.await
|
||||
}
|
||||
DictationProvider::ElevenLabs => {
|
||||
let model = dictation_selected_model(config, DictationProvider::ElevenLabs)
|
||||
.unwrap_or_else(|| ELEVENLABS_TRANSCRIPTION_MODEL.to_string());
|
||||
transcribe_with_provider(
|
||||
DictationProvider::ElevenLabs,
|
||||
"model_id".to_string(),
|
||||
model,
|
||||
audio_bytes,
|
||||
extension,
|
||||
&req.mime_type,
|
||||
)
|
||||
.await
|
||||
}
|
||||
#[cfg(feature = "local-inference")]
|
||||
DictationProvider::Local => transcribe_local(audio_bytes).await,
|
||||
}
|
||||
.map_err(|e| sacp::Error::internal_error().data(e.to_string()))?;
|
||||
|
||||
Ok(DictationTranscribeResponse { text })
|
||||
}
|
||||
|
||||
#[custom_method(DictationConfigRequest)]
|
||||
async fn on_dictation_config(
|
||||
&self,
|
||||
_req: DictationConfigRequest,
|
||||
) -> Result<DictationConfigResponse, sacp::Error> {
|
||||
let config = goose::config::Config::global();
|
||||
let mut providers = std::collections::HashMap::new();
|
||||
|
||||
for def in all_providers() {
|
||||
let provider = def.provider;
|
||||
let host = if let Some(host_key) = def.host_key {
|
||||
config
|
||||
.get(host_key, false)
|
||||
.ok()
|
||||
.and_then(|v| v.as_str().map(|s| s.to_string()))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let provider_key = serde_json::to_value(provider)
|
||||
.ok()
|
||||
.and_then(|v| v.as_str().map(|s| s.to_string()))
|
||||
.unwrap_or_else(|| format!("{:?}", provider).to_lowercase());
|
||||
providers.insert(
|
||||
provider_key,
|
||||
DictationProviderStatusEntry {
|
||||
configured: is_configured(provider),
|
||||
host,
|
||||
description: def.description.to_string(),
|
||||
uses_provider_config: def.uses_provider_config,
|
||||
settings_path: def.settings_path.map(|s| s.to_string()),
|
||||
config_key: if !def.uses_provider_config {
|
||||
Some(def.config_key.to_string())
|
||||
} else {
|
||||
None
|
||||
},
|
||||
model_config_key: dictation_model_config_key(provider),
|
||||
default_model: dictation_default_model(provider),
|
||||
selected_model: dictation_selected_model(config, provider),
|
||||
available_models: dictation_available_models(provider),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
Ok(DictationConfigResponse { providers })
|
||||
}
|
||||
|
||||
#[custom_method(DictationModelsListRequest)]
|
||||
async fn on_dictation_models_list(
|
||||
&self,
|
||||
_req: DictationModelsListRequest,
|
||||
) -> Result<DictationModelsListResponse, sacp::Error> {
|
||||
#[cfg(feature = "local-inference")]
|
||||
{
|
||||
use goose::download_manager::{get_download_manager, DownloadStatus};
|
||||
|
||||
let manager = get_download_manager();
|
||||
let models = whisper::available_models()
|
||||
.iter()
|
||||
.map(|model| DictationLocalModelStatus {
|
||||
id: model.id.to_string(),
|
||||
label: model.id.to_string(),
|
||||
description: model.description.to_string(),
|
||||
size_mb: model.size_mb,
|
||||
downloaded: model.is_downloaded(),
|
||||
download_in_progress: manager
|
||||
.get_progress(model.id)
|
||||
.map(|progress| progress.status == DownloadStatus::Downloading)
|
||||
.unwrap_or(false),
|
||||
})
|
||||
.collect();
|
||||
|
||||
Ok(DictationModelsListResponse { models })
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "local-inference"))]
|
||||
Ok(DictationModelsListResponse::default())
|
||||
}
|
||||
|
||||
#[custom_method(DictationModelDownloadRequest)]
|
||||
async fn on_dictation_model_download(
|
||||
&self,
|
||||
_req: DictationModelDownloadRequest,
|
||||
) -> Result<EmptyResponse, sacp::Error> {
|
||||
#[cfg(feature = "local-inference")]
|
||||
{
|
||||
use goose::download_manager::get_download_manager;
|
||||
|
||||
let model = whisper::get_model(&_req.model_id)
|
||||
.ok_or_else(|| sacp::Error::invalid_params().data("Unknown model id"))?;
|
||||
let manager = get_download_manager();
|
||||
let model_id_for_config = model.id.to_string();
|
||||
|
||||
manager
|
||||
.download_model(
|
||||
model.id.to_string(),
|
||||
model.url.to_string(),
|
||||
model.local_path(),
|
||||
Some(Box::new(move || {
|
||||
let config = goose::config::Config::global();
|
||||
// Only auto-select this model if the user has no model
|
||||
// currently selected. This prevents silently switching
|
||||
// the active model mid-session when a user downloads an
|
||||
// additional model while one is already in use.
|
||||
let already_selected = config
|
||||
.get(whisper::LOCAL_WHISPER_MODEL_CONFIG_KEY, false)
|
||||
.ok()
|
||||
.and_then(|value| value.as_str().map(str::to_owned))
|
||||
.filter(|model_id| {
|
||||
// Treat a deleted model file as no active selection
|
||||
// so a fresh download can auto-select cleanly.
|
||||
whisper::get_model(model_id)
|
||||
.is_some_and(|model| model.is_downloaded())
|
||||
});
|
||||
if already_selected.is_none() {
|
||||
if let Err(e) = config.set_param(
|
||||
whisper::LOCAL_WHISPER_MODEL_CONFIG_KEY,
|
||||
model_id_for_config.clone(),
|
||||
) {
|
||||
error!("Failed to save LOCAL_WHISPER_MODEL after download: {}", e);
|
||||
}
|
||||
}
|
||||
})),
|
||||
)
|
||||
.await
|
||||
.map_err(|e| sacp::Error::internal_error().data(e.to_string()))?;
|
||||
|
||||
Ok(EmptyResponse {})
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "local-inference"))]
|
||||
Err(sacp::Error::invalid_params().data("Local inference not enabled"))
|
||||
}
|
||||
|
||||
#[custom_method(DictationModelDownloadProgressRequest)]
|
||||
async fn on_dictation_model_download_progress(
|
||||
&self,
|
||||
_req: DictationModelDownloadProgressRequest,
|
||||
) -> Result<DictationModelDownloadProgressResponse, sacp::Error> {
|
||||
#[cfg(feature = "local-inference")]
|
||||
{
|
||||
use goose::download_manager::get_download_manager;
|
||||
|
||||
let manager = get_download_manager();
|
||||
let progress =
|
||||
manager
|
||||
.get_progress(&_req.model_id)
|
||||
.map(|progress| DictationDownloadProgress {
|
||||
bytes_downloaded: progress.bytes_downloaded,
|
||||
total_bytes: progress.total_bytes,
|
||||
progress_percent: progress.progress_percent,
|
||||
status: serde_json::to_value(&progress.status)
|
||||
.ok()
|
||||
.and_then(|value| value.as_str().map(ToOwned::to_owned))
|
||||
.unwrap_or_else(|| "unknown".to_string()),
|
||||
error: progress.error,
|
||||
});
|
||||
|
||||
Ok(DictationModelDownloadProgressResponse { progress })
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "local-inference"))]
|
||||
Ok(DictationModelDownloadProgressResponse { progress: None })
|
||||
}
|
||||
|
||||
#[custom_method(DictationModelCancelRequest)]
|
||||
async fn on_dictation_model_cancel(
|
||||
&self,
|
||||
_req: DictationModelCancelRequest,
|
||||
) -> Result<EmptyResponse, sacp::Error> {
|
||||
#[cfg(feature = "local-inference")]
|
||||
{
|
||||
use goose::download_manager::get_download_manager;
|
||||
|
||||
let manager = get_download_manager();
|
||||
manager
|
||||
.cancel_download(&_req.model_id)
|
||||
.map_err(|e| sacp::Error::internal_error().data(e.to_string()))?;
|
||||
|
||||
Ok(EmptyResponse {})
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "local-inference"))]
|
||||
Err(sacp::Error::invalid_params().data("Local inference not enabled"))
|
||||
}
|
||||
|
||||
#[custom_method(DictationModelDeleteRequest)]
|
||||
async fn on_dictation_model_delete(
|
||||
&self,
|
||||
_req: DictationModelDeleteRequest,
|
||||
) -> Result<EmptyResponse, sacp::Error> {
|
||||
#[cfg(feature = "local-inference")]
|
||||
{
|
||||
let model = whisper::get_model(&_req.model_id)
|
||||
.ok_or_else(|| sacp::Error::invalid_params().data("Unknown model id"))?;
|
||||
let path = model.local_path();
|
||||
|
||||
if !path.exists() {
|
||||
return Err(sacp::Error::invalid_params().data("Model not downloaded"));
|
||||
}
|
||||
|
||||
std::fs::remove_file(path)
|
||||
.map_err(|e| sacp::Error::internal_error().data(e.to_string()))?;
|
||||
|
||||
Ok(EmptyResponse {})
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "local-inference"))]
|
||||
Err(sacp::Error::invalid_params().data("Local inference not enabled"))
|
||||
}
|
||||
|
||||
#[custom_method(DictationModelSelectRequest)]
|
||||
async fn on_dictation_model_select(
|
||||
&self,
|
||||
req: DictationModelSelectRequest,
|
||||
) -> Result<EmptyResponse, sacp::Error> {
|
||||
#[cfg(not(feature = "local-inference"))]
|
||||
if req.provider == "local" {
|
||||
return Err(sacp::Error::invalid_params().data("Local inference not enabled"));
|
||||
}
|
||||
|
||||
let provider: DictationProvider = serde_json::from_value(serde_json::Value::String(
|
||||
req.provider.clone(),
|
||||
))
|
||||
.map_err(|_| {
|
||||
sacp::Error::invalid_params().data(format!("Unknown provider: {}", req.provider))
|
||||
})?;
|
||||
|
||||
let key = match provider {
|
||||
DictationProvider::OpenAI => OPENAI_TRANSCRIPTION_MODEL_CONFIG_KEY,
|
||||
DictationProvider::Groq => GROQ_TRANSCRIPTION_MODEL_CONFIG_KEY,
|
||||
DictationProvider::ElevenLabs => ELEVENLABS_TRANSCRIPTION_MODEL_CONFIG_KEY,
|
||||
#[cfg(feature = "local-inference")]
|
||||
DictationProvider::Local => {
|
||||
let model = whisper::get_model(&req.model_id)
|
||||
.ok_or_else(|| sacp::Error::invalid_params().data("Unknown model id"))?;
|
||||
if !model.is_downloaded() {
|
||||
return Err(
|
||||
sacp::Error::invalid_params().data("Local Whisper model is not downloaded")
|
||||
);
|
||||
}
|
||||
whisper::LOCAL_WHISPER_MODEL_CONFIG_KEY
|
||||
}
|
||||
};
|
||||
|
||||
goose::config::Config::global()
|
||||
.set_param(key, req.model_id)
|
||||
.map_err(|e| sacp::Error::internal_error().data(e.to_string()))?;
|
||||
|
||||
Ok(EmptyResponse {})
|
||||
}
|
||||
}
|
||||
|
||||
fn dictation_model_config_key(provider: DictationProvider) -> Option<String> {
|
||||
match provider {
|
||||
DictationProvider::OpenAI => Some(OPENAI_TRANSCRIPTION_MODEL_CONFIG_KEY.to_string()),
|
||||
DictationProvider::Groq => Some(GROQ_TRANSCRIPTION_MODEL_CONFIG_KEY.to_string()),
|
||||
DictationProvider::ElevenLabs => {
|
||||
Some(ELEVENLABS_TRANSCRIPTION_MODEL_CONFIG_KEY.to_string())
|
||||
}
|
||||
#[cfg(feature = "local-inference")]
|
||||
DictationProvider::Local => Some(whisper::LOCAL_WHISPER_MODEL_CONFIG_KEY.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
fn dictation_default_model(provider: DictationProvider) -> Option<String> {
|
||||
match provider {
|
||||
DictationProvider::OpenAI => Some(OPENAI_TRANSCRIPTION_MODEL.to_string()),
|
||||
DictationProvider::Groq => Some(GROQ_TRANSCRIPTION_MODEL.to_string()),
|
||||
DictationProvider::ElevenLabs => Some(ELEVENLABS_TRANSCRIPTION_MODEL.to_string()),
|
||||
#[cfg(feature = "local-inference")]
|
||||
DictationProvider::Local => Some(whisper::recommend_model().to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
fn dictation_selected_model(config: &Config, provider: DictationProvider) -> Option<String> {
|
||||
#[cfg(feature = "local-inference")]
|
||||
if provider == DictationProvider::Local {
|
||||
return config
|
||||
.get(whisper::LOCAL_WHISPER_MODEL_CONFIG_KEY, false)
|
||||
.ok()
|
||||
.and_then(|value| value.as_str().map(str::to_owned))
|
||||
.filter(|model_id| whisper::get_model(model_id).is_some())
|
||||
.or_else(|| dictation_default_model(provider));
|
||||
}
|
||||
|
||||
dictation_model_config_key(provider)
|
||||
.and_then(|key| {
|
||||
config
|
||||
.get(&key, false)
|
||||
.ok()
|
||||
.and_then(|value| value.as_str().map(str::to_owned))
|
||||
})
|
||||
.or_else(|| dictation_default_model(provider))
|
||||
}
|
||||
|
||||
fn dictation_available_models(provider: DictationProvider) -> Vec<DictationModelOption> {
|
||||
match provider {
|
||||
DictationProvider::OpenAI => vec![DictationModelOption {
|
||||
id: OPENAI_TRANSCRIPTION_MODEL.to_string(),
|
||||
label: "Whisper-1".to_string(),
|
||||
description: "OpenAI's hosted Whisper transcription model.".to_string(),
|
||||
}],
|
||||
DictationProvider::Groq => vec![DictationModelOption {
|
||||
id: GROQ_TRANSCRIPTION_MODEL.to_string(),
|
||||
label: "Whisper Large V3 Turbo".to_string(),
|
||||
description: "Groq's fast hosted Whisper transcription model.".to_string(),
|
||||
}],
|
||||
DictationProvider::ElevenLabs => vec![DictationModelOption {
|
||||
id: ELEVENLABS_TRANSCRIPTION_MODEL.to_string(),
|
||||
label: "Scribe v1".to_string(),
|
||||
description: "ElevenLabs' hosted speech-to-text model.".to_string(),
|
||||
}],
|
||||
#[cfg(feature = "local-inference")]
|
||||
DictationProvider::Local => whisper::available_models()
|
||||
.iter()
|
||||
.map(|model| DictationModelOption {
|
||||
id: model.id.to_string(),
|
||||
label: model.id.to_string(),
|
||||
description: model.description.to_string(),
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
pub struct GooseAcpHandler {
|
||||
|
||||
@@ -71,7 +71,7 @@ winapi = { workspace = true }
|
||||
[features]
|
||||
default = ["code-mode", "local-inference", "aws-providers", "telemetry", "otel", "rustls-tls"]
|
||||
code-mode = ["goose/code-mode", "goose-acp/code-mode"]
|
||||
local-inference = ["goose/local-inference"]
|
||||
local-inference = ["goose/local-inference", "goose-acp/local-inference"]
|
||||
aws-providers = ["goose/aws-providers"]
|
||||
cuda = ["goose/cuda", "local-inference"]
|
||||
telemetry = ["goose/telemetry"]
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use sacp::{JsonRpcRequest, JsonRpcResponse};
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// Schema descriptor for a single custom method, produced by the
|
||||
/// `#[custom_methods]` macro's generated `custom_method_schemas()` function.
|
||||
@@ -309,6 +310,154 @@ pub struct ProviderConfigKey {
|
||||
pub primary: bool,
|
||||
}
|
||||
|
||||
/// Transcribe audio via a dictation provider.
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema, JsonRpcRequest)]
|
||||
#[request(method = "_goose/dictation/transcribe", response = DictationTranscribeResponse)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DictationTranscribeRequest {
|
||||
/// Base64-encoded audio data
|
||||
pub audio: String,
|
||||
/// MIME type (e.g. "audio/wav", "audio/webm")
|
||||
pub mime_type: String,
|
||||
/// Provider to use: "openai", "groq", "elevenlabs", or "local"
|
||||
pub provider: String,
|
||||
}
|
||||
|
||||
/// Transcription result.
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema, JsonRpcResponse)]
|
||||
pub struct DictationTranscribeResponse {
|
||||
pub text: String,
|
||||
}
|
||||
|
||||
/// Get the configuration status of all dictation providers.
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema, JsonRpcRequest)]
|
||||
#[request(method = "_goose/dictation/config", response = DictationConfigResponse)]
|
||||
pub struct DictationConfigRequest {}
|
||||
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct DictationModelOption {
|
||||
pub id: String,
|
||||
pub label: String,
|
||||
pub description: String,
|
||||
}
|
||||
|
||||
/// Per-provider configuration status.
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DictationProviderStatusEntry {
|
||||
pub configured: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub host: Option<String>,
|
||||
pub description: String,
|
||||
pub uses_provider_config: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub settings_path: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub config_key: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub model_config_key: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub default_model: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub selected_model: Option<String>,
|
||||
#[serde(default)]
|
||||
pub available_models: Vec<DictationModelOption>,
|
||||
}
|
||||
|
||||
/// Dictation config response — map of provider name to status.
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema, JsonRpcResponse)]
|
||||
pub struct DictationConfigResponse {
|
||||
pub providers: HashMap<String, DictationProviderStatusEntry>,
|
||||
}
|
||||
|
||||
/// Empty success response for operations that return no data.
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema, JsonRpcResponse)]
|
||||
pub struct EmptyResponse {}
|
||||
|
||||
/// List available local Whisper models with their download status.
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema, JsonRpcRequest)]
|
||||
#[request(
|
||||
method = "_goose/dictation/models/list",
|
||||
response = DictationModelsListResponse
|
||||
)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DictationModelsListRequest {}
|
||||
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema, JsonRpcResponse)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DictationModelsListResponse {
|
||||
pub models: Vec<DictationLocalModelStatus>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DictationLocalModelStatus {
|
||||
pub id: String,
|
||||
pub label: String,
|
||||
pub description: String,
|
||||
pub size_mb: u32,
|
||||
pub downloaded: bool,
|
||||
pub download_in_progress: bool,
|
||||
}
|
||||
|
||||
/// Kick off a background download of a local Whisper model.
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema, JsonRpcRequest)]
|
||||
#[request(method = "_goose/dictation/models/download", response = EmptyResponse)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DictationModelDownloadRequest {
|
||||
pub model_id: String,
|
||||
}
|
||||
|
||||
/// Poll the progress of an in-flight download.
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema, JsonRpcRequest)]
|
||||
#[request(
|
||||
method = "_goose/dictation/models/download/progress",
|
||||
response = DictationModelDownloadProgressResponse
|
||||
)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DictationModelDownloadProgressRequest {
|
||||
pub model_id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema, JsonRpcResponse)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DictationModelDownloadProgressResponse {
|
||||
/// None when no download is active for this model id.
|
||||
pub progress: Option<DictationDownloadProgress>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DictationDownloadProgress {
|
||||
pub bytes_downloaded: u64,
|
||||
pub total_bytes: u64,
|
||||
pub progress_percent: f32,
|
||||
/// serde lowercase of DownloadStatus: "downloading" | "completed" | "failed" | "cancelled"
|
||||
pub status: String,
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
/// Cancel an in-flight download.
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema, JsonRpcRequest)]
|
||||
#[request(method = "_goose/dictation/models/cancel", response = EmptyResponse)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DictationModelCancelRequest {
|
||||
pub model_id: String,
|
||||
}
|
||||
|
||||
/// Delete a downloaded local Whisper model from disk.
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema, JsonRpcRequest)]
|
||||
#[request(method = "_goose/dictation/models/delete", response = EmptyResponse)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DictationModelDeleteRequest {
|
||||
pub model_id: String,
|
||||
}
|
||||
|
||||
/// Persist the user's model selection for a given provider.
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema, JsonRpcRequest)]
|
||||
#[request(method = "_goose/dictation/model/select", response = EmptyResponse)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DictationModelSelectRequest {
|
||||
pub provider: String,
|
||||
pub model_id: String,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user