refactor: remove fast model routing (#11469)

Signed-off-by: f.sol <75c4a74c87b014467d845f4ddc1ee89c84c9ce5a106a7c595b7ecb940a5f0bb3@buzz.gdk.so>
Co-authored-by: f.sol <75c4a74c87b014467d845f4ddc1ee89c84c9ce5a106a7c595b7ecb940a5f0bb3@buzz.gdk.so>
This commit is contained in:
filip
2026-09-02 19:29:09 +00:00
committed by GitHub
parent 5f64232706
commit bbf0ddf7d0
28 changed files with 20 additions and 179 deletions
-12
View File
@@ -41,10 +41,6 @@ pub struct ProviderMetadata {
/// step-by-step instructions for set up providers eg: api key
#[serde(default)]
pub setup_steps: Vec<String>,
/// The name of a fast/cheap model to use for lightweight tasks (e.g. session naming,
/// compaction). When set, fast-path callers prefer this model over the main model.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub fast_model: Option<String>,
/// Setup information exposed to clients.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub setup: Option<ProviderSetupMetadata>,
@@ -80,7 +76,6 @@ impl ProviderMetadata {
model_doc_link: model_doc_link.to_string(),
config_keys,
setup_steps: vec![],
fast_model: None,
setup: None,
deprecated: None,
}
@@ -104,7 +99,6 @@ impl ProviderMetadata {
model_doc_link: model_doc_link.to_string(),
config_keys,
setup_steps: vec![],
fast_model: None,
setup: None,
deprecated: None,
}
@@ -120,7 +114,6 @@ impl ProviderMetadata {
model_doc_link: "".to_string(),
config_keys: vec![],
setup_steps: vec![],
fast_model: None,
setup: None,
deprecated: None,
}
@@ -131,11 +124,6 @@ impl ProviderMetadata {
self
}
pub fn with_fast_model(mut self, fast_model: &str) -> Self {
self.fast_model = Some(fast_model.to_string());
self
}
pub fn with_setup(mut self, setup: ProviderSetupMetadata) -> Self {
self.setup = Some(setup);
self
-2
View File
@@ -26,7 +26,6 @@ use crate::model::ModelConfig;
use rmcp::model::Tool;
pub const ANTHROPIC_DEFAULT_MODEL: &str = "claude-sonnet-4-5";
pub const ANTHROPIC_DEFAULT_FAST_MODEL: &str = "claude-haiku-4-5";
const ANTHROPIC_KNOWN_MODELS: &[&str] = &[
"claude-opus-5",
"claude-sonnet-5",
@@ -305,7 +304,6 @@ impl ProviderDescriptor for AnthropicProvider {
),
],
)
.with_fast_model(ANTHROPIC_DEFAULT_FAST_MODEL)
.with_setup_steps(vec![
"Go to https://platform.claude.com/settings/keys",
"Click 'Create Key'",
-2
View File
@@ -76,7 +76,6 @@ static DATABRICKS_ENDPOINT_INFO_CACHE: LazyLock<
Mutex<std::collections::HashMap<String, CachedDatabricksEndpointInfo>>,
> = LazyLock::new(|| Mutex::new(std::collections::HashMap::new()));
pub const DATABRICKS_DEFAULT_MODEL: &str = "databricks-claude-sonnet-4";
pub const DATABRICKS_DEFAULT_FAST_MODEL: &str = "databricks-claude-haiku-4-5";
pub const DATABRICKS_KNOWN_MODELS: &[&str] = &[
"databricks-claude-sonnet-4-5",
"databricks-meta-llama-3-3-70b-instruct",
@@ -532,7 +531,6 @@ impl crate::base::ProviderDescriptor for DatabricksProvider {
ConfigKey::new("DATABRICKS_TOKEN", false, true, None, true),
],
)
.with_fast_model(DATABRICKS_DEFAULT_FAST_MODEL)
}
}
@@ -182,8 +182,6 @@ pub struct DeclarativeProviderConfig {
pub model_doc_link: Option<String>,
#[serde(default)]
pub setup_steps: Vec<String>,
#[serde(default, deserialize_with = "deserialize_non_empty_string")]
pub fast_model: Option<String>,
#[serde(default)]
pub preserves_thinking: bool,
/// Enables Z.AI's `clear_thinking` field, which Anthropic does not support.
@@ -21,7 +21,6 @@
],
"dynamic_models": true,
"supports_streaming": true,
"fast_model": "anthropic/claude-haiku-4-5",
"model_doc_link": "https://opper.ai/models",
"setup_steps": [
"Sign up at https://opper.ai",
@@ -43,6 +43,5 @@
"Register at portal.salad.com and enable access to the AI Gateway.",
"Copy your API key from https://portal.salad.com/api-key and paste it above as SALAD_CLOUD_API_KEY.",
"Select a supported model such as qwen3.6-35b-a3b."
],
"fast_model": "qwen3.5-9b"
]
}
@@ -16,7 +16,6 @@
],
"catalog_provider_id": "zai",
"model_doc_link": "https://docs.z.ai/devpack/tool/goose",
"fast_model": "glm-5.3-flash",
"preserves_thinking": true,
"emit_clear_thinking": true,
"models": [
-2
View File
@@ -24,7 +24,6 @@ use tokio_util::io::StreamReader;
pub const GOOGLE_PROVIDER_NAME: &str = "google";
pub const GOOGLE_API_HOST: &str = "https://generativelanguage.googleapis.com";
pub const GOOGLE_DEFAULT_MODEL: &str = "gemini-2.5-pro";
pub const GOOGLE_DEFAULT_FAST_MODEL: &str = "gemini-2.5-flash";
pub const GOOGLE_KNOWN_MODELS: &[&str] = &[
"gemini-3.6-flash",
"gemini-3.5-flash",
@@ -128,7 +127,6 @@ impl crate::base::ProviderDescriptor for GoogleProvider {
ConfigKey::new("GOOGLE_HOST", false, false, Some(GOOGLE_API_HOST), false),
],
)
.with_fast_model(GOOGLE_DEFAULT_FAST_MODEL)
.with_setup_steps(vec![
"Go to https://aistudio.google.com and sign in with your Google account",
"Click 'Get API key' on the left sidebar",
-1
View File
@@ -588,7 +588,6 @@ mod tests {
skip_canonical_filtering: false,
model_doc_link: None,
setup_steps: vec![],
fast_model: None,
preserves_thinking: false,
emit_clear_thinking: false,
setup: None,
-2
View File
@@ -58,7 +58,6 @@ impl CachedContextLimit {
}
}
pub const OPEN_AI_DEFAULT_MODEL: &str = "gpt-4o";
pub const OPEN_AI_DEFAULT_FAST_MODEL: &str = "gpt-4o-mini";
pub const OPEN_AI_KNOWN_MODELS: &[(&str, usize)] = &[
("gpt-4o", 128_000),
("gpt-4o-mini", 128_000),
@@ -1391,7 +1390,6 @@ mod tests {
skip_canonical_filtering: false,
model_doc_link: None,
setup_steps: vec![],
fast_model: None,
preserves_thinking: false,
emit_clear_thinking: false,
setup: None,
-2
View File
@@ -23,7 +23,6 @@ use rmcp::model::Tool;
pub const OPENROUTER_PROVIDER_NAME: &str = "openrouter";
const OPENROUTER_PARAMETERS_CONFIG_KEY: &str = "OPENROUTER_PARAMETERS";
pub const OPENROUTER_DEFAULT_MODEL: &str = "anthropic/claude-sonnet-4";
pub const OPENROUTER_DEFAULT_FAST_MODEL: &str = "google/gemini-2.5-flash";
// OpenRouter can run many models, we suggest the default
pub const OPENROUTER_KNOWN_MODELS: &[&str] = &[
@@ -393,7 +392,6 @@ impl crate::base::ProviderDescriptor for OpenRouterProvider {
ConfigKey::new(OPENROUTER_PARAMETERS_CONFIG_KEY, false, false, None, false),
],
)
.with_fast_model(OPENROUTER_DEFAULT_FAST_MODEL)
}
}
@@ -355,14 +355,13 @@ impl OrchestratorClient {
));
let model_config = self.parent_model_config(provider.get_name()).await?;
let (response, _usage) = crate::model_config::complete_fast(
let (response, _usage) = crate::model_config::complete_one_shot(
provider.as_ref(),
&model_config,
session_id,
system,
&[user_message],
&[],
true,
)
.await
.map_err(|e| format!("LLM summarization failed: {}", e))?;
@@ -225,7 +225,6 @@ pub fn create_custom_provider(
skip_canonical_filtering: false,
model_doc_link: None,
setup_steps: vec![],
fast_model: None,
preserves_thinking,
emit_clear_thinking: false,
setup: None,
@@ -344,7 +343,6 @@ pub fn update_custom_provider(params: UpdateCustomProviderParams) -> Result<()>
skip_canonical_filtering: existing_config.skip_canonical_filtering,
model_doc_link: existing_config.model_doc_link,
setup_steps: existing_config.setup_steps,
fast_model: existing_config.fast_model.clone(),
preserves_thinking,
emit_clear_thinking: existing_config.emit_clear_thinking,
setup: existing_config.setup,
@@ -621,7 +619,6 @@ mod tests {
skip_canonical_filtering: false,
model_doc_link: None,
setup_steps: Vec::new(),
fast_model: None,
preserves_thinking: true,
emit_clear_thinking: false,
setup: None,
+2 -2
View File
@@ -287,7 +287,7 @@ impl goose_context_management::CompactionModel for GooseCompactionModel<'_> {
system: &str,
messages: &[Message],
) -> Result<(Message, ProviderUsage), ProviderError> {
crate::model_config::complete_compaction(
crate::model_config::complete_one_shot(
self.provider,
self.model_config,
self.session_id,
@@ -473,7 +473,7 @@ pub async fn summarize_tool_call(
if that is what it was.
"#};
let (mut response, _) = crate::model_config::complete_compaction(
let (mut response, _) = crate::model_config::complete_one_shot(
provider,
model_config,
session_id,
+13 -100
View File
@@ -81,97 +81,16 @@ fn materialize_model_config_inner(
Ok(model)
}
fn configured_fast_model_name() -> Option<String> {
Config::global()
.get_param::<String>("GOOSE_FAST_MODEL")
.ok()
.map(|v| v.trim().to_string())
.filter(|v| !v.is_empty())
}
/// Resolve the model config to use for lightweight "fast" tasks (session
/// naming, tool-call labels, orchestrator routing). Resolution order:
/// 1. `GOOSE_FAST_MODEL` (user override)
/// 2. the provider's declared default fast model
/// 3. the supplied `model_config` (i.e. the main model)
///
/// The resulting config is materialized against the same provider so it picks
/// up context limits, temperature, and other provider defaults.
pub async fn get_fast_model(
provider_name: &str,
model_config: &ModelConfig,
) -> Result<ModelConfig> {
let fast_model_name = match configured_fast_model_name() {
Some(name) => Some(name),
None => provider_default_fast_model(provider_name).await,
};
match fast_model_name {
Some(name) if name != model_config.model_name => {
model_config_from_user_config(provider_name, name)
.map(|config| config.with_request_headers(model_config.request_headers.clone()))
}
_ => Ok(model_config.clone()),
}
}
/// A one-shot task summarizes a transcript or tool result that never recurs, so
/// a prompt cache entry written for it can never be read back and only costs the
/// cache-write premium.
fn one_shot_model_config(model_config: ModelConfig) -> ModelConfig {
model_config
.with_thinking_effort(ThinkingEffort::Off)
.with_prompt_cache_disabled()
}
/// Run a completion for a lightweight "fast" task (session naming, tool-call
/// labels, orchestrator routing) using the provider's fast model.
pub async fn complete_fast(
provider: &dyn Provider,
model_config: &ModelConfig,
session_id: &str,
system: &str,
messages: &[Message],
tools: &[Tool],
fallback_to_main_model: bool,
) -> Result<(Message, ProviderUsage), ProviderError> {
let fast_model_config = one_shot_model_config(
get_fast_model(provider.get_name(), model_config)
.await
.map_err(|e| ProviderError::ExecutionError(e.to_string()))?,
);
match crate::session_context::with_session_id(
Some(session_id.to_string()),
provider.complete(&fast_model_config, system, messages, tools),
)
.await
{
Ok(response) => Ok(response),
Err(e)
if fallback_to_main_model
&& fast_model_config.model_name != model_config.model_name =>
{
tracing::warn!(
"Fast model {} failed with error: {}. Falling back to main model {}",
fast_model_config.model_name,
e,
model_config.model_name
);
let fallback_config = one_shot_model_config(model_config.clone());
crate::session_context::with_session_id(
Some(session_id.to_string()),
provider.complete(&fallback_config, system, messages, tools),
)
.await
}
Err(e) => Err(e),
}
}
/// Run a completion for compaction or tool-result summarization on the main
/// session model with one-shot semantics (thinking off, no prompt-cache writes).
pub async fn complete_compaction(
/// Run a completion for a one-shot auxiliary task on the main session model.
/// Thinking is disabled and prompt-cache writes are skipped because this prompt
/// will not recur.
pub async fn complete_one_shot(
provider: &dyn Provider,
model_config: &ModelConfig,
session_id: &str,
@@ -179,26 +98,15 @@ pub async fn complete_compaction(
messages: &[Message],
tools: &[Tool],
) -> Result<(Message, ProviderUsage), ProviderError> {
let compaction_model_config = one_shot_model_config(model_config.clone());
let one_shot_model_config = one_shot_model_config(model_config.clone());
crate::session_context::with_session_id(
Some(session_id.to_string()),
provider.complete(&compaction_model_config, system, messages, tools),
provider.complete(&one_shot_model_config, system, messages, tools),
)
.await
}
async fn provider_default_fast_model(provider_name: &str) -> Option<String> {
if provider_name == goose_providers::openai::OPEN_AI_PROVIDER_NAME {
return crate::providers::openai_def::live_fast_model();
}
crate::providers::get_from_registry(provider_name)
.await
.ok()
.and_then(|entry| entry.metadata().fast_model.clone())
}
fn apply_openai_request_params(mut model: ModelConfig) -> ModelConfig {
let config = Config::global();
if let Some(store) = config.get_openai_store() {
@@ -328,8 +236,13 @@ mod one_shot_tests {
use super::*;
#[test]
fn prompt_cache_is_disabled() {
assert!(one_shot_model_config(ModelConfig::new("claude-haiku-4-5")).prompt_cache_disabled());
fn thinking_and_prompt_cache_are_disabled() {
let config = one_shot_model_config(
ModelConfig::new("claude-haiku-4-5").with_thinking_effort(ThinkingEffort::High),
);
assert_eq!(config.thinking_effort(), Some(ThinkingEffort::Off));
assert!(config.prompt_cache_disabled());
}
}
@@ -144,7 +144,6 @@ mod tests {
skip_canonical_filtering: false,
model_doc_link: None,
setup_steps: vec![],
fast_model: None,
preserves_thinking: false,
emit_clear_thinking: false,
setup: None,
@@ -13,7 +13,6 @@ use goose_providers::model::ModelConfig;
use goose_providers::request_log::{start_log, LoggerHandleExt};
const GEMINI_OAUTH_DEFAULT_MODEL: &str = "gemini-3-flash-preview";
const GEMINI_OAUTH_DEFAULT_FAST_MODEL: &str = "gemini-2.5-flash-lite";
use crate::providers::retry::ProviderRetry;
use anyhow::{anyhow, Result};
use async_stream::try_stream;
@@ -946,7 +945,6 @@ impl goose_providers::base::ProviderDescriptor for GeminiOAuthProvider {
false,
)],
)
.with_fast_model(GEMINI_OAUTH_DEFAULT_FAST_MODEL)
}
}
@@ -581,7 +581,6 @@ mod tests {
skip_canonical_filtering: false,
model_doc_link: None,
setup_steps: vec![],
fast_model: None,
preserves_thinking: true,
emit_clear_thinking: false,
setup: None,
@@ -486,7 +486,6 @@ mod tests {
skip_canonical_filtering: false,
model_doc_link: None,
setup_steps: vec![],
fast_model: None,
preserves_thinking: true,
emit_clear_thinking: false,
setup: None,
+1 -15
View File
@@ -11,18 +11,13 @@ use crate::providers::custom_provider_config::ConfigKeyResolver;
use goose_providers::api_client::{ApiClient, AuthMethod};
use goose_providers::openai::{
parse_custom_headers, parse_openai_base_url, OpenAiProvider, OpenAiProviderBuilder,
OPEN_AI_DEFAULT_BASE_PATH, OPEN_AI_DEFAULT_FAST_MODEL, OPEN_AI_VERSIONLESS_BASE_PATH,
OPEN_AI_DEFAULT_BASE_PATH, OPEN_AI_VERSIONLESS_BASE_PATH,
};
pub struct OpenAiProviderDef;
impl ProviderDescriptor for OpenAiProviderDef {
fn metadata() -> goose_providers::base::ProviderMetadata {
// The default fast model is resolved live in `live_fast_model` rather
// than baked into metadata here: registry metadata is snapshotted at
// init time, but the OpenAI base URL can change at runtime (e.g.
// switching to an OpenAI-compatible endpoint), which would otherwise
// leave the cached fast model stale.
OpenAiProvider::metadata().with_setup(
crate::providers::catalog::ProviderSetupMetadata::new(
crate::providers::catalog::ProviderSetupCategory::Model,
@@ -40,15 +35,6 @@ impl ProviderDescriptor for OpenAiProviderDef {
}
}
pub fn live_fast_model() -> Option<String> {
match resolve_base_url(crate::config::Config::global()) {
Ok(parsed) if is_direct_openai_host(&parsed.host) => {
Some(OPEN_AI_DEFAULT_FAST_MODEL.to_string())
}
_ => None,
}
}
impl ProviderDef for OpenAiProviderDef {
type Provider = OpenAiProvider;
@@ -281,7 +281,6 @@ impl ProviderRegistry {
.unwrap_or(base_metadata.model_doc_link),
config_keys,
setup_steps: config.setup_steps.clone(),
fast_model: config.fast_model.clone(),
setup: config.setup.clone(),
deprecated: None,
};
@@ -380,7 +379,6 @@ mod tests {
skip_canonical_filtering: false,
model_doc_link: None,
setup_steps: vec![],
fast_model: None,
preserves_thinking: false,
emit_clear_thinking: false,
setup: None,
+1 -2
View File
@@ -131,14 +131,13 @@ pub(crate) async fn generate_session_name(
&[message],
)?
} else {
crate::model_config::complete_fast(
crate::model_config::complete_one_shot(
provider,
model_config,
session_id,
&system,
&[message],
&[],
true,
)
.await?
};
+1 -2
View File
@@ -194,14 +194,13 @@ async fn complete_label(
message: &Message,
) -> Option<String> {
for attempt in 0..LABEL_GENERATION_MAX_ATTEMPTS {
if let Ok((response, _)) = crate::model_config::complete_fast(
if let Ok((response, _)) = crate::model_config::complete_one_shot(
provider,
model_config,
session_id,
system_prompt,
from_ref(message),
&[],
false,
)
.await
{
-10
View File
@@ -539,7 +539,6 @@ mod tests {
model_doc_link: "".to_string(),
config_keys: vec![],
setup_steps: vec![],
fast_model: None,
setup: None,
deprecated: None,
}
@@ -713,7 +712,6 @@ mod tests {
model_doc_link: "".to_string(),
config_keys: vec![],
setup_steps: vec![],
fast_model: None,
setup: None,
deprecated: None,
}
@@ -895,7 +893,6 @@ mod tests {
model_doc_link: "".to_string(),
config_keys: vec![],
setup_steps: vec![],
fast_model: None,
setup: None,
deprecated: None,
}
@@ -1255,7 +1252,6 @@ mod tests {
model_doc_link: "".to_string(),
config_keys: vec![],
setup_steps: vec![],
fast_model: None,
setup: None,
deprecated: None,
}
@@ -1534,7 +1530,6 @@ mod tests {
model_doc_link: "".to_string(),
config_keys: vec![],
setup_steps: vec![],
fast_model: None,
setup: None,
deprecated: None,
}
@@ -1736,7 +1731,6 @@ mod tests {
model_doc_link: "".to_string(),
config_keys: vec![],
setup_steps: vec![],
fast_model: None,
setup: None,
deprecated: None,
}
@@ -1888,7 +1882,6 @@ mod tests {
model_doc_link: "".to_string(),
config_keys: vec![],
setup_steps: vec![],
fast_model: None,
setup: None,
deprecated: None,
}
@@ -2248,7 +2241,6 @@ mod tests {
model_doc_link: "".to_string(),
config_keys: vec![],
setup_steps: vec![],
fast_model: None,
setup: None,
deprecated: None,
}
@@ -3198,7 +3190,6 @@ mod tests {
model_doc_link: "".to_string(),
config_keys: vec![],
setup_steps: vec![],
fast_model: None,
setup: None,
deprecated: None,
}
@@ -3286,7 +3277,6 @@ mod tests {
model_doc_link: "".to_string(),
config_keys: vec![],
setup_steps: vec![],
fast_model: None,
setup: None,
deprecated: None,
}
-1
View File
@@ -198,7 +198,6 @@ impl goose::providers::base::ProviderDescriptor for MockCompactionProvider {
model_doc_link: "".to_string(),
config_keys: vec![],
setup_steps: vec![],
fast_model: None,
setup: None,
deprecated: None,
}
@@ -35,7 +35,6 @@ fn custom_config_with_auth(base_url: &str, auth: AuthConfig) -> DeclarativeProvi
skip_canonical_filtering: false,
model_doc_link: None,
setup_steps: vec![],
fast_model: None,
preserves_thinking: false,
emit_clear_thinking: false,
setup: None,
@@ -18,7 +18,6 @@ These are the minimum required variables to get started with goose.
|----------|---------|---------|---------|
| `GOOSE_PROVIDER` | Specifies the LLM provider to use | [See available providers](/docs/getting-started/providers#available-providers) | None (must be [configured](/docs/getting-started/providers#configure-provider-and-model)) |
| `GOOSE_MODEL` | Specifies which model to use from the provider | Model name (e.g., "gpt-4", "claude-sonnet-4-20250514") | None (must be [configured](/docs/getting-started/providers#configure-provider-and-model)) |
| `GOOSE_FAST_MODEL` | Overrides the provider's default fast model used for auxiliary calls (tool-selection, classification, session titles) | Model name (e.g., "gpt-4o-mini", "google/gemini-2.5-flash") | Provider-specific default |
| `GOOSE_TEMPERATURE` | Sets the [temperature](https://medium.com/@kelseyywang/a-comprehensive-guide-to-llm-temperature-%EF%B8%8F-363a40bbc91f) for model responses | Float between 0.0 and 1.0 | Model-specific default |
| `GOOSE_MAX_TOKENS` | Sets the maximum number of tokens for each model response (truncates longer responses) | Positive integer (e.g., 4096, 8192) | Model-specific default |
| `GOOSE_CACHE_TTL` | Sets the Anthropic prompt-cache TTL. `1h` keeps the cached prefix alive across idle gaps (e.g. stepping away mid-session) but bills cache writes at 2x input instead of 1.25x, so it only pays off for sessions that actually idle. Headless runs (`goose run`, subagents, scheduled recipes) always use `5m` | `5m`, `1h` | `5m` |
@@ -31,9 +30,6 @@ export GOOSE_PROVIDER="anthropic"
export GOOSE_MODEL="claude-sonnet-4-5-20250929"
export GOOSE_TEMPERATURE=0.7
# Override the fast model used for auxiliary calls (tool-selection, classification, etc.)
export GOOSE_FAST_MODEL="gpt-4o-mini"
# Set a lower limit for shorter interactions
export GOOSE_MAX_TOKENS=4096
-1
View File
@@ -28,7 +28,6 @@ export type ProviderMetadata = {
default_model: string;
description: string;
display_name: string;
fast_model?: string | null;
known_models: ModelInfo[];
model_doc_link: string;
name: string;