fix(openrouter): use the context length OpenRouter reports (#12853)

Co-authored-by: Cassian433 <262026629+Cassian433@users.noreply.github.com>
This commit is contained in:
Cassian433
2026-10-16 00:35:59 +00:00
committed by GitHub
parent 0df818175b
commit 9fa665262b
+372 -50
View File
@@ -4,6 +4,8 @@ use async_trait::async_trait;
use serde_json::{json, Value};
use std::collections::{HashMap, HashSet};
use std::ops::Range;
use std::sync::Arc;
use std::time::{Duration, Instant};
use crate::api_client::ApiClient;
@@ -45,6 +47,21 @@ pub const OPENROUTER_DOC_URL: &str = "https://openrouter.ai/models";
const GEMINI_SCHEMA_REF_KEY: &str = "$ref";
const GEMINI_SAFE_SCHEMA_REF_KEY_BASE: &str = "dollar_ref";
const MODELS_DISCOVERY_TIMEOUT: Duration = Duration::from_secs(5);
const MODELS_SUCCESS_TTL: Duration = Duration::from_secs(60 * 60);
const MODELS_FAILURE_TTL: Duration = Duration::from_secs(60);
struct OpenRouterModel {
id: String,
context_length: Option<usize>,
supports_tools: bool,
}
struct CachedModels {
models: Result<Arc<[OpenRouterModel]>, ProviderError>,
expires_at: Instant,
}
#[derive(serde::Serialize)]
pub struct OpenRouterProvider {
#[serde(skip)]
@@ -56,6 +73,8 @@ pub struct OpenRouterProvider {
configured_parameters: Option<HashMap<String, Value>>,
#[serde(skip)]
session_id_provider: Option<OpenRouterSessionIdProvider>,
#[serde(skip)]
cached_models: tokio::sync::Mutex<Option<CachedModels>>,
}
impl OpenRouterProvider {
@@ -70,9 +89,97 @@ impl OpenRouterProvider {
name: OPENROUTER_PROVIDER_NAME.to_string(),
configured_parameters,
session_id_provider,
cached_models: tokio::sync::Mutex::new(None),
}
}
async fn get_or_fetch_models(&self) -> Result<Arc<[OpenRouterModel]>, ProviderError> {
let mut cache = self.cached_models.lock().await;
if let Some(cached) = cache
.as_ref()
.filter(|cached| Instant::now() < cached.expires_at)
{
return cached.models.clone();
}
let fetched = tokio::time::timeout(MODELS_DISCOVERY_TIMEOUT, self.fetch_models_from_api())
.await
.unwrap_or_else(|_| {
Err(ProviderError::RequestFailed(
"Timed out fetching models from OpenRouter API".to_string(),
))
});
let previous = cache.take().and_then(|cached| cached.models.ok());
let (models, ttl) = match (fetched, previous) {
(Ok(models), _) => (Ok(models), MODELS_SUCCESS_TTL),
(Err(error), Some(previous)) => {
tracing::warn!(%error, "Failed to refresh OpenRouter models; keeping the previous list");
(Ok(previous), MODELS_FAILURE_TTL)
}
(Err(error), None) => (Err(error), MODELS_FAILURE_TTL),
};
cache
.insert(CachedModels {
models,
expires_at: Instant::now() + ttl,
})
.models
.clone()
}
async fn fetch_models_from_api(&self) -> Result<Arc<[OpenRouterModel]>, ProviderError> {
let response = self
.api_client
.request("api/v1/models")
.response_get()
.await
.map_err(|e| {
ProviderError::RequestFailed(format!(
"Failed to fetch models from OpenRouter API: {}",
e
))
})?;
let json: serde_json::Value = response.json().await.map_err(|e| {
ProviderError::RequestFailed(format!(
"Failed to parse OpenRouter API response as JSON: {}",
e
))
})?;
if let Some(err_obj) = json.get("error") {
let msg = err_obj
.get("message")
.and_then(|v| v.as_str())
.unwrap_or("unknown error");
return Err(ProviderError::RequestFailed(format!(
"OpenRouter API returned an error: {}",
msg
)));
}
let data = json.get("data").and_then(|v| v.as_array()).ok_or_else(|| {
ProviderError::UsageError("Missing data field in JSON response".into())
})?;
Ok(data
.iter()
.filter_map(|model| {
Some(OpenRouterModel {
id: model.get("id").and_then(|v| v.as_str())?.to_string(),
context_length: model
.get("context_length")
.and_then(|v| v.as_u64())
.map(|limit| limit as usize),
supports_tools: model
.get("supported_parameters")
.and_then(|v| v.as_array())
.is_some_and(|params| params.iter().any(|p| p.as_str() == Some("tools"))),
})
})
.collect())
}
async fn post_chat_completions(
&self,
model_config: &ModelConfig,
@@ -459,58 +566,33 @@ impl Provider for OpenRouterProvider {
true
}
async fn fetch_recommended_models(&self, toolshim: bool) -> Result<Vec<String>, ProviderError> {
let response = self
.api_client
.request("api/v1/models")
.response_get()
.await
.map_err(|e| {
ProviderError::RequestFailed(format!(
"Failed to fetch models from OpenRouter API: {}",
e
))
})?;
let json: serde_json::Value = response.json().await.map_err(|e| {
ProviderError::RequestFailed(format!(
"Failed to parse OpenRouter API response as JSON: {}",
e
))
})?;
if let Some(err_obj) = json.get("error") {
let msg = err_obj
.get("message")
.and_then(|v| v.as_str())
.unwrap_or("unknown error");
return Err(ProviderError::RequestFailed(format!(
"OpenRouter API returned an error: {}",
msg
)));
}
let data = json.get("data").and_then(|v| v.as_array()).ok_or_else(|| {
ProviderError::UsageError("Missing data field in JSON response".into())
})?;
let mut models: Vec<String> = data
.iter()
.filter_map(|model| {
let id = model.get("id").and_then(|v| v.as_str())?;
if toolshim {
return Some(id.to_string());
}
let supports_tools = model
.get("supported_parameters")
.and_then(|v| v.as_array())
.is_some_and(|params| params.iter().any(|p| p.as_str() == Some("tools")));
if supports_tools {
Some(id.to_string())
} else {
None
async fn get_context_limit(&self, model: &str, override_limit: Option<usize>) -> usize {
crate::context_limit::ContextLimitResolver::new(&self.name)
.resolve(model, override_limit, || async {
// The catalog outranks the reported length, so a model it knows
// never needs the model list.
if let Some(canonical) =
crate::canonical::maybe_get_canonical_model(&self.name, model)
{
return Ok(Some(canonical.limit.context));
}
Ok(self
.get_or_fetch_models()
.await?
.iter()
.find(|info| info.id == model)
.and_then(|info| info.context_length))
})
.await
}
async fn fetch_recommended_models(&self, toolshim: bool) -> Result<Vec<String>, ProviderError> {
let mut models: Vec<String> = self
.get_or_fetch_models()
.await?
.iter()
.filter(|model| toolshim || model.supports_tools)
.map(|model| model.id.clone())
.collect();
models.sort();
Ok(models)
@@ -743,6 +825,246 @@ mod tests {
);
}
const MODEL_NOT_IN_CATALOG: &str = "test-vendor/not-in-catalog";
fn models_response() -> wiremock::ResponseTemplate {
wiremock::ResponseTemplate::new(200).set_body_json(json!({
"data": [
{
"id": MODEL_NOT_IN_CATALOG,
"context_length": 1_048_576,
"supported_parameters": ["tools"]
},
{
"id": OPENROUTER_DEFAULT_MODEL,
"context_length": 4_096,
"supported_parameters": ["tools"]
},
{
"id": "test-vendor/no-tools",
"context_length": 32_768,
"supported_parameters": ["temperature"]
}
]
}))
}
async fn models_server(
response: wiremock::ResponseTemplate,
expected_requests: u64,
) -> wiremock::MockServer {
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer};
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/api/v1/models"))
.respond_with(response)
.expect(expected_requests)
.mount(&server)
.await;
server
}
fn provider_for(server: &wiremock::MockServer) -> OpenRouterProvider {
OpenRouterProvider::new(
ApiClient::new_with_tls(
server.uri(),
crate::api_client::AuthMethod::BearerToken("test-key".to_string()),
None,
)
.unwrap(),
None,
None,
)
}
#[tokio::test]
async fn context_limit_uses_reported_length_for_model_not_in_catalog() {
let server = models_server(models_response(), 1).await;
let provider = provider_for(&server);
assert_eq!(
provider.get_context_limit(MODEL_NOT_IN_CATALOG, None).await,
1_048_576
);
}
#[tokio::test]
async fn context_limit_keeps_override_and_catalog_precedence_without_fetching() {
let server = models_server(models_response(), 0).await;
let provider = provider_for(&server);
let catalog_limit = crate::canonical::maybe_get_canonical_model(
OPENROUTER_PROVIDER_NAME,
OPENROUTER_DEFAULT_MODEL,
)
.unwrap()
.limit
.context;
assert_eq!(
provider
.get_context_limit(MODEL_NOT_IN_CATALOG, Some(64_000))
.await,
64_000
);
assert_eq!(
provider
.get_context_limit(OPENROUTER_DEFAULT_MODEL, None)
.await,
catalog_limit
);
}
#[test_case::test_case(wiremock::ResponseTemplate::new(500) ; "server error")]
#[test_case::test_case(
wiremock::ResponseTemplate::new(200).set_body_string("not json") ; "invalid json"
)]
#[test_case::test_case(models_response().set_delay(Duration::from_secs(10)) ; "timeout")]
#[tokio::test]
async fn context_limit_falls_back_and_caches_models_failure(
response: wiremock::ResponseTemplate,
) {
let server = models_server(response, 1).await;
let provider = provider_for(&server);
for _ in 0..2 {
assert_eq!(
provider.get_context_limit(MODEL_NOT_IN_CATALOG, None).await,
crate::model::DEFAULT_CONTEXT_LIMIT
);
}
assert_eq!(server.received_requests().await.unwrap().len(), 1);
assert!(provider.fetch_recommended_models(false).await.is_err());
}
fn previous_models(context_length: usize) -> Arc<[OpenRouterModel]> {
Arc::from([OpenRouterModel {
id: MODEL_NOT_IN_CATALOG.to_string(),
context_length: Some(context_length),
supports_tools: true,
}])
}
async fn seed_expired_cache(
provider: &OpenRouterProvider,
models: Result<Arc<[OpenRouterModel]>, ProviderError>,
) {
*provider.cached_models.lock().await = Some(CachedModels {
models,
expires_at: Instant::now(),
});
}
#[tokio::test]
async fn context_limit_refetches_models_after_failure_ttl() {
let server = models_server(models_response(), 1).await;
let provider = provider_for(&server);
seed_expired_cache(
&provider,
Err(ProviderError::RequestFailed("earlier failure".to_string())),
)
.await;
assert_eq!(
provider.get_context_limit(MODEL_NOT_IN_CATALOG, None).await,
1_048_576
);
}
#[tokio::test]
async fn context_limit_refetches_models_after_success_ttl() {
let server = models_server(models_response(), 1).await;
let provider = provider_for(&server);
seed_expired_cache(&provider, Ok(previous_models(131_072))).await;
for _ in 0..2 {
assert_eq!(
provider.get_context_limit(MODEL_NOT_IN_CATALOG, None).await,
1_048_576
);
}
}
#[tokio::test]
async fn context_limit_keeps_previous_models_when_refetch_fails() {
let server = models_server(wiremock::ResponseTemplate::new(500), 2).await;
let provider = provider_for(&server);
seed_expired_cache(&provider, Ok(previous_models(131_072))).await;
for _ in 0..2 {
assert_eq!(
provider.get_context_limit(MODEL_NOT_IN_CATALOG, None).await,
131_072
);
}
assert_eq!(server.received_requests().await.unwrap().len(), 1);
let retry_at = provider
.cached_models
.lock()
.await
.as_ref()
.unwrap()
.expires_at;
assert!(retry_at <= Instant::now() + MODELS_FAILURE_TTL);
provider
.cached_models
.lock()
.await
.as_mut()
.unwrap()
.expires_at = Instant::now();
assert_eq!(
provider.get_context_limit(MODEL_NOT_IN_CATALOG, None).await,
131_072
);
}
#[tokio::test]
async fn context_limit_fetches_models_once() {
let server = models_server(models_response(), 1).await;
let provider = provider_for(&server);
for _ in 0..3 {
assert_eq!(
provider.get_context_limit(MODEL_NOT_IN_CATALOG, None).await,
1_048_576
);
}
assert_eq!(
provider
.get_context_limit("test-vendor/no-tools", None)
.await,
32_768
);
}
#[tokio::test]
async fn recommended_models_and_context_limit_share_one_fetch() {
let server = models_server(models_response(), 1).await;
let provider = provider_for(&server);
let (recommended, context_limit) = tokio::join!(
provider.fetch_recommended_models(false),
provider.get_context_limit(MODEL_NOT_IN_CATALOG, None)
);
assert_eq!(
recommended.unwrap(),
vec![OPENROUTER_DEFAULT_MODEL, MODEL_NOT_IN_CATALOG]
);
assert_eq!(context_limit, 1_048_576);
assert_eq!(
provider.fetch_recommended_models(true).await.unwrap(),
vec![
OPENROUTER_DEFAULT_MODEL,
"test-vendor/no-tools",
MODEL_NOT_IN_CATALOG
]
);
}
#[test]
fn gemini_tool_result_schema_ref_keys_are_escaped_reversibly() {
let mut payload = json!({