fix(openrouter): use the context length OpenRouter reports (#12853)
Co-authored-by: Cassian433 <262026629+Cassian433@users.noreply.github.com>
This commit is contained in:
@@ -4,6 +4,8 @@ use async_trait::async_trait;
|
||||
use serde_json::{json, Value};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::ops::Range;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crate::api_client::ApiClient;
|
||||
|
||||
@@ -45,6 +47,21 @@ pub const OPENROUTER_DOC_URL: &str = "https://openrouter.ai/models";
|
||||
const GEMINI_SCHEMA_REF_KEY: &str = "$ref";
|
||||
const GEMINI_SAFE_SCHEMA_REF_KEY_BASE: &str = "dollar_ref";
|
||||
|
||||
const MODELS_DISCOVERY_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const MODELS_SUCCESS_TTL: Duration = Duration::from_secs(60 * 60);
|
||||
const MODELS_FAILURE_TTL: Duration = Duration::from_secs(60);
|
||||
|
||||
struct OpenRouterModel {
|
||||
id: String,
|
||||
context_length: Option<usize>,
|
||||
supports_tools: bool,
|
||||
}
|
||||
|
||||
struct CachedModels {
|
||||
models: Result<Arc<[OpenRouterModel]>, ProviderError>,
|
||||
expires_at: Instant,
|
||||
}
|
||||
|
||||
#[derive(serde::Serialize)]
|
||||
pub struct OpenRouterProvider {
|
||||
#[serde(skip)]
|
||||
@@ -56,6 +73,8 @@ pub struct OpenRouterProvider {
|
||||
configured_parameters: Option<HashMap<String, Value>>,
|
||||
#[serde(skip)]
|
||||
session_id_provider: Option<OpenRouterSessionIdProvider>,
|
||||
#[serde(skip)]
|
||||
cached_models: tokio::sync::Mutex<Option<CachedModels>>,
|
||||
}
|
||||
|
||||
impl OpenRouterProvider {
|
||||
@@ -70,9 +89,97 @@ impl OpenRouterProvider {
|
||||
name: OPENROUTER_PROVIDER_NAME.to_string(),
|
||||
configured_parameters,
|
||||
session_id_provider,
|
||||
cached_models: tokio::sync::Mutex::new(None),
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_or_fetch_models(&self) -> Result<Arc<[OpenRouterModel]>, ProviderError> {
|
||||
let mut cache = self.cached_models.lock().await;
|
||||
if let Some(cached) = cache
|
||||
.as_ref()
|
||||
.filter(|cached| Instant::now() < cached.expires_at)
|
||||
{
|
||||
return cached.models.clone();
|
||||
}
|
||||
|
||||
let fetched = tokio::time::timeout(MODELS_DISCOVERY_TIMEOUT, self.fetch_models_from_api())
|
||||
.await
|
||||
.unwrap_or_else(|_| {
|
||||
Err(ProviderError::RequestFailed(
|
||||
"Timed out fetching models from OpenRouter API".to_string(),
|
||||
))
|
||||
});
|
||||
let previous = cache.take().and_then(|cached| cached.models.ok());
|
||||
let (models, ttl) = match (fetched, previous) {
|
||||
(Ok(models), _) => (Ok(models), MODELS_SUCCESS_TTL),
|
||||
(Err(error), Some(previous)) => {
|
||||
tracing::warn!(%error, "Failed to refresh OpenRouter models; keeping the previous list");
|
||||
(Ok(previous), MODELS_FAILURE_TTL)
|
||||
}
|
||||
(Err(error), None) => (Err(error), MODELS_FAILURE_TTL),
|
||||
};
|
||||
cache
|
||||
.insert(CachedModels {
|
||||
models,
|
||||
expires_at: Instant::now() + ttl,
|
||||
})
|
||||
.models
|
||||
.clone()
|
||||
}
|
||||
|
||||
async fn fetch_models_from_api(&self) -> Result<Arc<[OpenRouterModel]>, ProviderError> {
|
||||
let response = self
|
||||
.api_client
|
||||
.request("api/v1/models")
|
||||
.response_get()
|
||||
.await
|
||||
.map_err(|e| {
|
||||
ProviderError::RequestFailed(format!(
|
||||
"Failed to fetch models from OpenRouter API: {}",
|
||||
e
|
||||
))
|
||||
})?;
|
||||
|
||||
let json: serde_json::Value = response.json().await.map_err(|e| {
|
||||
ProviderError::RequestFailed(format!(
|
||||
"Failed to parse OpenRouter API response as JSON: {}",
|
||||
e
|
||||
))
|
||||
})?;
|
||||
|
||||
if let Some(err_obj) = json.get("error") {
|
||||
let msg = err_obj
|
||||
.get("message")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("unknown error");
|
||||
return Err(ProviderError::RequestFailed(format!(
|
||||
"OpenRouter API returned an error: {}",
|
||||
msg
|
||||
)));
|
||||
}
|
||||
|
||||
let data = json.get("data").and_then(|v| v.as_array()).ok_or_else(|| {
|
||||
ProviderError::UsageError("Missing data field in JSON response".into())
|
||||
})?;
|
||||
|
||||
Ok(data
|
||||
.iter()
|
||||
.filter_map(|model| {
|
||||
Some(OpenRouterModel {
|
||||
id: model.get("id").and_then(|v| v.as_str())?.to_string(),
|
||||
context_length: model
|
||||
.get("context_length")
|
||||
.and_then(|v| v.as_u64())
|
||||
.map(|limit| limit as usize),
|
||||
supports_tools: model
|
||||
.get("supported_parameters")
|
||||
.and_then(|v| v.as_array())
|
||||
.is_some_and(|params| params.iter().any(|p| p.as_str() == Some("tools"))),
|
||||
})
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn post_chat_completions(
|
||||
&self,
|
||||
model_config: &ModelConfig,
|
||||
@@ -459,58 +566,33 @@ impl Provider for OpenRouterProvider {
|
||||
true
|
||||
}
|
||||
|
||||
async fn fetch_recommended_models(&self, toolshim: bool) -> Result<Vec<String>, ProviderError> {
|
||||
let response = self
|
||||
.api_client
|
||||
.request("api/v1/models")
|
||||
.response_get()
|
||||
.await
|
||||
.map_err(|e| {
|
||||
ProviderError::RequestFailed(format!(
|
||||
"Failed to fetch models from OpenRouter API: {}",
|
||||
e
|
||||
))
|
||||
})?;
|
||||
|
||||
let json: serde_json::Value = response.json().await.map_err(|e| {
|
||||
ProviderError::RequestFailed(format!(
|
||||
"Failed to parse OpenRouter API response as JSON: {}",
|
||||
e
|
||||
))
|
||||
})?;
|
||||
|
||||
if let Some(err_obj) = json.get("error") {
|
||||
let msg = err_obj
|
||||
.get("message")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("unknown error");
|
||||
return Err(ProviderError::RequestFailed(format!(
|
||||
"OpenRouter API returned an error: {}",
|
||||
msg
|
||||
)));
|
||||
}
|
||||
|
||||
let data = json.get("data").and_then(|v| v.as_array()).ok_or_else(|| {
|
||||
ProviderError::UsageError("Missing data field in JSON response".into())
|
||||
})?;
|
||||
|
||||
let mut models: Vec<String> = data
|
||||
.iter()
|
||||
.filter_map(|model| {
|
||||
let id = model.get("id").and_then(|v| v.as_str())?;
|
||||
if toolshim {
|
||||
return Some(id.to_string());
|
||||
}
|
||||
let supports_tools = model
|
||||
.get("supported_parameters")
|
||||
.and_then(|v| v.as_array())
|
||||
.is_some_and(|params| params.iter().any(|p| p.as_str() == Some("tools")));
|
||||
if supports_tools {
|
||||
Some(id.to_string())
|
||||
} else {
|
||||
None
|
||||
async fn get_context_limit(&self, model: &str, override_limit: Option<usize>) -> usize {
|
||||
crate::context_limit::ContextLimitResolver::new(&self.name)
|
||||
.resolve(model, override_limit, || async {
|
||||
// The catalog outranks the reported length, so a model it knows
|
||||
// never needs the model list.
|
||||
if let Some(canonical) =
|
||||
crate::canonical::maybe_get_canonical_model(&self.name, model)
|
||||
{
|
||||
return Ok(Some(canonical.limit.context));
|
||||
}
|
||||
Ok(self
|
||||
.get_or_fetch_models()
|
||||
.await?
|
||||
.iter()
|
||||
.find(|info| info.id == model)
|
||||
.and_then(|info| info.context_length))
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn fetch_recommended_models(&self, toolshim: bool) -> Result<Vec<String>, ProviderError> {
|
||||
let mut models: Vec<String> = self
|
||||
.get_or_fetch_models()
|
||||
.await?
|
||||
.iter()
|
||||
.filter(|model| toolshim || model.supports_tools)
|
||||
.map(|model| model.id.clone())
|
||||
.collect();
|
||||
models.sort();
|
||||
Ok(models)
|
||||
@@ -743,6 +825,246 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
const MODEL_NOT_IN_CATALOG: &str = "test-vendor/not-in-catalog";
|
||||
|
||||
fn models_response() -> wiremock::ResponseTemplate {
|
||||
wiremock::ResponseTemplate::new(200).set_body_json(json!({
|
||||
"data": [
|
||||
{
|
||||
"id": MODEL_NOT_IN_CATALOG,
|
||||
"context_length": 1_048_576,
|
||||
"supported_parameters": ["tools"]
|
||||
},
|
||||
{
|
||||
"id": OPENROUTER_DEFAULT_MODEL,
|
||||
"context_length": 4_096,
|
||||
"supported_parameters": ["tools"]
|
||||
},
|
||||
{
|
||||
"id": "test-vendor/no-tools",
|
||||
"context_length": 32_768,
|
||||
"supported_parameters": ["temperature"]
|
||||
}
|
||||
]
|
||||
}))
|
||||
}
|
||||
|
||||
async fn models_server(
|
||||
response: wiremock::ResponseTemplate,
|
||||
expected_requests: u64,
|
||||
) -> wiremock::MockServer {
|
||||
use wiremock::matchers::{method, path};
|
||||
use wiremock::{Mock, MockServer};
|
||||
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/v1/models"))
|
||||
.respond_with(response)
|
||||
.expect(expected_requests)
|
||||
.mount(&server)
|
||||
.await;
|
||||
server
|
||||
}
|
||||
|
||||
fn provider_for(server: &wiremock::MockServer) -> OpenRouterProvider {
|
||||
OpenRouterProvider::new(
|
||||
ApiClient::new_with_tls(
|
||||
server.uri(),
|
||||
crate::api_client::AuthMethod::BearerToken("test-key".to_string()),
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn context_limit_uses_reported_length_for_model_not_in_catalog() {
|
||||
let server = models_server(models_response(), 1).await;
|
||||
let provider = provider_for(&server);
|
||||
|
||||
assert_eq!(
|
||||
provider.get_context_limit(MODEL_NOT_IN_CATALOG, None).await,
|
||||
1_048_576
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn context_limit_keeps_override_and_catalog_precedence_without_fetching() {
|
||||
let server = models_server(models_response(), 0).await;
|
||||
let provider = provider_for(&server);
|
||||
let catalog_limit = crate::canonical::maybe_get_canonical_model(
|
||||
OPENROUTER_PROVIDER_NAME,
|
||||
OPENROUTER_DEFAULT_MODEL,
|
||||
)
|
||||
.unwrap()
|
||||
.limit
|
||||
.context;
|
||||
|
||||
assert_eq!(
|
||||
provider
|
||||
.get_context_limit(MODEL_NOT_IN_CATALOG, Some(64_000))
|
||||
.await,
|
||||
64_000
|
||||
);
|
||||
assert_eq!(
|
||||
provider
|
||||
.get_context_limit(OPENROUTER_DEFAULT_MODEL, None)
|
||||
.await,
|
||||
catalog_limit
|
||||
);
|
||||
}
|
||||
|
||||
#[test_case::test_case(wiremock::ResponseTemplate::new(500) ; "server error")]
|
||||
#[test_case::test_case(
|
||||
wiremock::ResponseTemplate::new(200).set_body_string("not json") ; "invalid json"
|
||||
)]
|
||||
#[test_case::test_case(models_response().set_delay(Duration::from_secs(10)) ; "timeout")]
|
||||
#[tokio::test]
|
||||
async fn context_limit_falls_back_and_caches_models_failure(
|
||||
response: wiremock::ResponseTemplate,
|
||||
) {
|
||||
let server = models_server(response, 1).await;
|
||||
let provider = provider_for(&server);
|
||||
|
||||
for _ in 0..2 {
|
||||
assert_eq!(
|
||||
provider.get_context_limit(MODEL_NOT_IN_CATALOG, None).await,
|
||||
crate::model::DEFAULT_CONTEXT_LIMIT
|
||||
);
|
||||
}
|
||||
assert_eq!(server.received_requests().await.unwrap().len(), 1);
|
||||
assert!(provider.fetch_recommended_models(false).await.is_err());
|
||||
}
|
||||
|
||||
fn previous_models(context_length: usize) -> Arc<[OpenRouterModel]> {
|
||||
Arc::from([OpenRouterModel {
|
||||
id: MODEL_NOT_IN_CATALOG.to_string(),
|
||||
context_length: Some(context_length),
|
||||
supports_tools: true,
|
||||
}])
|
||||
}
|
||||
|
||||
async fn seed_expired_cache(
|
||||
provider: &OpenRouterProvider,
|
||||
models: Result<Arc<[OpenRouterModel]>, ProviderError>,
|
||||
) {
|
||||
*provider.cached_models.lock().await = Some(CachedModels {
|
||||
models,
|
||||
expires_at: Instant::now(),
|
||||
});
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn context_limit_refetches_models_after_failure_ttl() {
|
||||
let server = models_server(models_response(), 1).await;
|
||||
let provider = provider_for(&server);
|
||||
seed_expired_cache(
|
||||
&provider,
|
||||
Err(ProviderError::RequestFailed("earlier failure".to_string())),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
provider.get_context_limit(MODEL_NOT_IN_CATALOG, None).await,
|
||||
1_048_576
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn context_limit_refetches_models_after_success_ttl() {
|
||||
let server = models_server(models_response(), 1).await;
|
||||
let provider = provider_for(&server);
|
||||
seed_expired_cache(&provider, Ok(previous_models(131_072))).await;
|
||||
|
||||
for _ in 0..2 {
|
||||
assert_eq!(
|
||||
provider.get_context_limit(MODEL_NOT_IN_CATALOG, None).await,
|
||||
1_048_576
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn context_limit_keeps_previous_models_when_refetch_fails() {
|
||||
let server = models_server(wiremock::ResponseTemplate::new(500), 2).await;
|
||||
let provider = provider_for(&server);
|
||||
seed_expired_cache(&provider, Ok(previous_models(131_072))).await;
|
||||
|
||||
for _ in 0..2 {
|
||||
assert_eq!(
|
||||
provider.get_context_limit(MODEL_NOT_IN_CATALOG, None).await,
|
||||
131_072
|
||||
);
|
||||
}
|
||||
assert_eq!(server.received_requests().await.unwrap().len(), 1);
|
||||
let retry_at = provider
|
||||
.cached_models
|
||||
.lock()
|
||||
.await
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.expires_at;
|
||||
assert!(retry_at <= Instant::now() + MODELS_FAILURE_TTL);
|
||||
|
||||
provider
|
||||
.cached_models
|
||||
.lock()
|
||||
.await
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.expires_at = Instant::now();
|
||||
assert_eq!(
|
||||
provider.get_context_limit(MODEL_NOT_IN_CATALOG, None).await,
|
||||
131_072
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn context_limit_fetches_models_once() {
|
||||
let server = models_server(models_response(), 1).await;
|
||||
let provider = provider_for(&server);
|
||||
|
||||
for _ in 0..3 {
|
||||
assert_eq!(
|
||||
provider.get_context_limit(MODEL_NOT_IN_CATALOG, None).await,
|
||||
1_048_576
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
provider
|
||||
.get_context_limit("test-vendor/no-tools", None)
|
||||
.await,
|
||||
32_768
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn recommended_models_and_context_limit_share_one_fetch() {
|
||||
let server = models_server(models_response(), 1).await;
|
||||
let provider = provider_for(&server);
|
||||
|
||||
let (recommended, context_limit) = tokio::join!(
|
||||
provider.fetch_recommended_models(false),
|
||||
provider.get_context_limit(MODEL_NOT_IN_CATALOG, None)
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
recommended.unwrap(),
|
||||
vec![OPENROUTER_DEFAULT_MODEL, MODEL_NOT_IN_CATALOG]
|
||||
);
|
||||
assert_eq!(context_limit, 1_048_576);
|
||||
assert_eq!(
|
||||
provider.fetch_recommended_models(true).await.unwrap(),
|
||||
vec![
|
||||
OPENROUTER_DEFAULT_MODEL,
|
||||
"test-vendor/no-tools",
|
||||
MODEL_NOT_IN_CATALOG
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gemini_tool_result_schema_ref_keys_are_escaped_reversibly() {
|
||||
let mut payload = json!({
|
||||
|
||||
Reference in New Issue
Block a user