diff --git a/crates/goose-providers/src/openrouter.rs b/crates/goose-providers/src/openrouter.rs index 6cdf9fa3a..a06348339 100644 --- a/crates/goose-providers/src/openrouter.rs +++ b/crates/goose-providers/src/openrouter.rs @@ -4,6 +4,8 @@ use async_trait::async_trait; use serde_json::{json, Value}; use std::collections::{HashMap, HashSet}; use std::ops::Range; +use std::sync::Arc; +use std::time::{Duration, Instant}; use crate::api_client::ApiClient; @@ -45,6 +47,21 @@ pub const OPENROUTER_DOC_URL: &str = "https://openrouter.ai/models"; const GEMINI_SCHEMA_REF_KEY: &str = "$ref"; const GEMINI_SAFE_SCHEMA_REF_KEY_BASE: &str = "dollar_ref"; +const MODELS_DISCOVERY_TIMEOUT: Duration = Duration::from_secs(5); +const MODELS_SUCCESS_TTL: Duration = Duration::from_secs(60 * 60); +const MODELS_FAILURE_TTL: Duration = Duration::from_secs(60); + +struct OpenRouterModel { + id: String, + context_length: Option, + supports_tools: bool, +} + +struct CachedModels { + models: Result, ProviderError>, + expires_at: Instant, +} + #[derive(serde::Serialize)] pub struct OpenRouterProvider { #[serde(skip)] @@ -56,6 +73,8 @@ pub struct OpenRouterProvider { configured_parameters: Option>, #[serde(skip)] session_id_provider: Option, + #[serde(skip)] + cached_models: tokio::sync::Mutex>, } impl OpenRouterProvider { @@ -70,9 +89,97 @@ impl OpenRouterProvider { name: OPENROUTER_PROVIDER_NAME.to_string(), configured_parameters, session_id_provider, + cached_models: tokio::sync::Mutex::new(None), } } + async fn get_or_fetch_models(&self) -> Result, ProviderError> { + let mut cache = self.cached_models.lock().await; + if let Some(cached) = cache + .as_ref() + .filter(|cached| Instant::now() < cached.expires_at) + { + return cached.models.clone(); + } + + let fetched = tokio::time::timeout(MODELS_DISCOVERY_TIMEOUT, self.fetch_models_from_api()) + .await + .unwrap_or_else(|_| { + Err(ProviderError::RequestFailed( + "Timed out fetching models from OpenRouter API".to_string(), + )) + }); + let previous = cache.take().and_then(|cached| cached.models.ok()); + let (models, ttl) = match (fetched, previous) { + (Ok(models), _) => (Ok(models), MODELS_SUCCESS_TTL), + (Err(error), Some(previous)) => { + tracing::warn!(%error, "Failed to refresh OpenRouter models; keeping the previous list"); + (Ok(previous), MODELS_FAILURE_TTL) + } + (Err(error), None) => (Err(error), MODELS_FAILURE_TTL), + }; + cache + .insert(CachedModels { + models, + expires_at: Instant::now() + ttl, + }) + .models + .clone() + } + + async fn fetch_models_from_api(&self) -> Result, ProviderError> { + let response = self + .api_client + .request("api/v1/models") + .response_get() + .await + .map_err(|e| { + ProviderError::RequestFailed(format!( + "Failed to fetch models from OpenRouter API: {}", + e + )) + })?; + + let json: serde_json::Value = response.json().await.map_err(|e| { + ProviderError::RequestFailed(format!( + "Failed to parse OpenRouter API response as JSON: {}", + e + )) + })?; + + if let Some(err_obj) = json.get("error") { + let msg = err_obj + .get("message") + .and_then(|v| v.as_str()) + .unwrap_or("unknown error"); + return Err(ProviderError::RequestFailed(format!( + "OpenRouter API returned an error: {}", + msg + ))); + } + + let data = json.get("data").and_then(|v| v.as_array()).ok_or_else(|| { + ProviderError::UsageError("Missing data field in JSON response".into()) + })?; + + Ok(data + .iter() + .filter_map(|model| { + Some(OpenRouterModel { + id: model.get("id").and_then(|v| v.as_str())?.to_string(), + context_length: model + .get("context_length") + .and_then(|v| v.as_u64()) + .map(|limit| limit as usize), + supports_tools: model + .get("supported_parameters") + .and_then(|v| v.as_array()) + .is_some_and(|params| params.iter().any(|p| p.as_str() == Some("tools"))), + }) + }) + .collect()) + } + async fn post_chat_completions( &self, model_config: &ModelConfig, @@ -459,58 +566,33 @@ impl Provider for OpenRouterProvider { true } - async fn fetch_recommended_models(&self, toolshim: bool) -> Result, ProviderError> { - let response = self - .api_client - .request("api/v1/models") - .response_get() - .await - .map_err(|e| { - ProviderError::RequestFailed(format!( - "Failed to fetch models from OpenRouter API: {}", - e - )) - })?; - - let json: serde_json::Value = response.json().await.map_err(|e| { - ProviderError::RequestFailed(format!( - "Failed to parse OpenRouter API response as JSON: {}", - e - )) - })?; - - if let Some(err_obj) = json.get("error") { - let msg = err_obj - .get("message") - .and_then(|v| v.as_str()) - .unwrap_or("unknown error"); - return Err(ProviderError::RequestFailed(format!( - "OpenRouter API returned an error: {}", - msg - ))); - } - - let data = json.get("data").and_then(|v| v.as_array()).ok_or_else(|| { - ProviderError::UsageError("Missing data field in JSON response".into()) - })?; - - let mut models: Vec = data - .iter() - .filter_map(|model| { - let id = model.get("id").and_then(|v| v.as_str())?; - if toolshim { - return Some(id.to_string()); - } - let supports_tools = model - .get("supported_parameters") - .and_then(|v| v.as_array()) - .is_some_and(|params| params.iter().any(|p| p.as_str() == Some("tools"))); - if supports_tools { - Some(id.to_string()) - } else { - None + async fn get_context_limit(&self, model: &str, override_limit: Option) -> usize { + crate::context_limit::ContextLimitResolver::new(&self.name) + .resolve(model, override_limit, || async { + // The catalog outranks the reported length, so a model it knows + // never needs the model list. + if let Some(canonical) = + crate::canonical::maybe_get_canonical_model(&self.name, model) + { + return Ok(Some(canonical.limit.context)); } + Ok(self + .get_or_fetch_models() + .await? + .iter() + .find(|info| info.id == model) + .and_then(|info| info.context_length)) }) + .await + } + + async fn fetch_recommended_models(&self, toolshim: bool) -> Result, ProviderError> { + let mut models: Vec = self + .get_or_fetch_models() + .await? + .iter() + .filter(|model| toolshim || model.supports_tools) + .map(|model| model.id.clone()) .collect(); models.sort(); Ok(models) @@ -743,6 +825,246 @@ mod tests { ); } + const MODEL_NOT_IN_CATALOG: &str = "test-vendor/not-in-catalog"; + + fn models_response() -> wiremock::ResponseTemplate { + wiremock::ResponseTemplate::new(200).set_body_json(json!({ + "data": [ + { + "id": MODEL_NOT_IN_CATALOG, + "context_length": 1_048_576, + "supported_parameters": ["tools"] + }, + { + "id": OPENROUTER_DEFAULT_MODEL, + "context_length": 4_096, + "supported_parameters": ["tools"] + }, + { + "id": "test-vendor/no-tools", + "context_length": 32_768, + "supported_parameters": ["temperature"] + } + ] + })) + } + + async fn models_server( + response: wiremock::ResponseTemplate, + expected_requests: u64, + ) -> wiremock::MockServer { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer}; + + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/api/v1/models")) + .respond_with(response) + .expect(expected_requests) + .mount(&server) + .await; + server + } + + fn provider_for(server: &wiremock::MockServer) -> OpenRouterProvider { + OpenRouterProvider::new( + ApiClient::new_with_tls( + server.uri(), + crate::api_client::AuthMethod::BearerToken("test-key".to_string()), + None, + ) + .unwrap(), + None, + None, + ) + } + + #[tokio::test] + async fn context_limit_uses_reported_length_for_model_not_in_catalog() { + let server = models_server(models_response(), 1).await; + let provider = provider_for(&server); + + assert_eq!( + provider.get_context_limit(MODEL_NOT_IN_CATALOG, None).await, + 1_048_576 + ); + } + + #[tokio::test] + async fn context_limit_keeps_override_and_catalog_precedence_without_fetching() { + let server = models_server(models_response(), 0).await; + let provider = provider_for(&server); + let catalog_limit = crate::canonical::maybe_get_canonical_model( + OPENROUTER_PROVIDER_NAME, + OPENROUTER_DEFAULT_MODEL, + ) + .unwrap() + .limit + .context; + + assert_eq!( + provider + .get_context_limit(MODEL_NOT_IN_CATALOG, Some(64_000)) + .await, + 64_000 + ); + assert_eq!( + provider + .get_context_limit(OPENROUTER_DEFAULT_MODEL, None) + .await, + catalog_limit + ); + } + + #[test_case::test_case(wiremock::ResponseTemplate::new(500) ; "server error")] + #[test_case::test_case( + wiremock::ResponseTemplate::new(200).set_body_string("not json") ; "invalid json" + )] + #[test_case::test_case(models_response().set_delay(Duration::from_secs(10)) ; "timeout")] + #[tokio::test] + async fn context_limit_falls_back_and_caches_models_failure( + response: wiremock::ResponseTemplate, + ) { + let server = models_server(response, 1).await; + let provider = provider_for(&server); + + for _ in 0..2 { + assert_eq!( + provider.get_context_limit(MODEL_NOT_IN_CATALOG, None).await, + crate::model::DEFAULT_CONTEXT_LIMIT + ); + } + assert_eq!(server.received_requests().await.unwrap().len(), 1); + assert!(provider.fetch_recommended_models(false).await.is_err()); + } + + fn previous_models(context_length: usize) -> Arc<[OpenRouterModel]> { + Arc::from([OpenRouterModel { + id: MODEL_NOT_IN_CATALOG.to_string(), + context_length: Some(context_length), + supports_tools: true, + }]) + } + + async fn seed_expired_cache( + provider: &OpenRouterProvider, + models: Result, ProviderError>, + ) { + *provider.cached_models.lock().await = Some(CachedModels { + models, + expires_at: Instant::now(), + }); + } + + #[tokio::test] + async fn context_limit_refetches_models_after_failure_ttl() { + let server = models_server(models_response(), 1).await; + let provider = provider_for(&server); + seed_expired_cache( + &provider, + Err(ProviderError::RequestFailed("earlier failure".to_string())), + ) + .await; + + assert_eq!( + provider.get_context_limit(MODEL_NOT_IN_CATALOG, None).await, + 1_048_576 + ); + } + + #[tokio::test] + async fn context_limit_refetches_models_after_success_ttl() { + let server = models_server(models_response(), 1).await; + let provider = provider_for(&server); + seed_expired_cache(&provider, Ok(previous_models(131_072))).await; + + for _ in 0..2 { + assert_eq!( + provider.get_context_limit(MODEL_NOT_IN_CATALOG, None).await, + 1_048_576 + ); + } + } + + #[tokio::test] + async fn context_limit_keeps_previous_models_when_refetch_fails() { + let server = models_server(wiremock::ResponseTemplate::new(500), 2).await; + let provider = provider_for(&server); + seed_expired_cache(&provider, Ok(previous_models(131_072))).await; + + for _ in 0..2 { + assert_eq!( + provider.get_context_limit(MODEL_NOT_IN_CATALOG, None).await, + 131_072 + ); + } + assert_eq!(server.received_requests().await.unwrap().len(), 1); + let retry_at = provider + .cached_models + .lock() + .await + .as_ref() + .unwrap() + .expires_at; + assert!(retry_at <= Instant::now() + MODELS_FAILURE_TTL); + + provider + .cached_models + .lock() + .await + .as_mut() + .unwrap() + .expires_at = Instant::now(); + assert_eq!( + provider.get_context_limit(MODEL_NOT_IN_CATALOG, None).await, + 131_072 + ); + } + + #[tokio::test] + async fn context_limit_fetches_models_once() { + let server = models_server(models_response(), 1).await; + let provider = provider_for(&server); + + for _ in 0..3 { + assert_eq!( + provider.get_context_limit(MODEL_NOT_IN_CATALOG, None).await, + 1_048_576 + ); + } + assert_eq!( + provider + .get_context_limit("test-vendor/no-tools", None) + .await, + 32_768 + ); + } + + #[tokio::test] + async fn recommended_models_and_context_limit_share_one_fetch() { + let server = models_server(models_response(), 1).await; + let provider = provider_for(&server); + + let (recommended, context_limit) = tokio::join!( + provider.fetch_recommended_models(false), + provider.get_context_limit(MODEL_NOT_IN_CATALOG, None) + ); + + assert_eq!( + recommended.unwrap(), + vec![OPENROUTER_DEFAULT_MODEL, MODEL_NOT_IN_CATALOG] + ); + assert_eq!(context_limit, 1_048_576); + assert_eq!( + provider.fetch_recommended_models(true).await.unwrap(), + vec![ + OPENROUTER_DEFAULT_MODEL, + "test-vendor/no-tools", + MODEL_NOT_IN_CATALOG + ] + ); + } + #[test] fn gemini_tool_result_schema_ref_keys_are_escaped_reversibly() { let mut payload = json!({