feat: Add LiteLLM provider with automatic prompt caching support (#3380)

Signed-off-by: HikaruEgashira <account@egahika.dev>
This commit is contained in:
hikae
2025-07-19 19:07:01 +09:00
committed by GitHub
parent c38c7f16dc
commit 78b30cc0ca
8 changed files with 411 additions and 10 deletions
+13
View File
@@ -41,6 +41,8 @@ pub struct ModelInfo {
pub output_token_cost: Option<f64>,
/// Currency for the costs (default: "$")
pub currency: Option<String>,
/// Whether this model supports cache control
pub supports_cache_control: Option<bool>,
}
impl ModelInfo {
@@ -52,6 +54,7 @@ impl ModelInfo {
input_token_cost: None,
output_token_cost: None,
currency: None,
supports_cache_control: None,
}
}
@@ -68,6 +71,7 @@ impl ModelInfo {
input_token_cost: Some(input_cost),
output_token_cost: Some(output_cost),
currency: Some("$".to_string()),
supports_cache_control: None,
}
}
}
@@ -115,6 +119,7 @@ impl ProviderMetadata {
input_token_cost: None,
output_token_cost: None,
currency: None,
supports_cache_control: None,
})
.collect(),
model_doc_link: model_doc_link.to_string(),
@@ -290,6 +295,11 @@ pub trait Provider: Send + Sync {
false
}
/// Check if this provider supports cache control
fn supports_cache_control(&self) -> bool {
false
}
/// Create embeddings if supported. Default implementation returns an error.
async fn create_embeddings(&self, _texts: Vec<String>) -> Result<Vec<Vec<f32>>, ProviderError> {
Err(ProviderError::ExecutionError(
@@ -435,6 +445,7 @@ mod tests {
input_token_cost: None,
output_token_cost: None,
currency: None,
supports_cache_control: None,
};
assert_eq!(info.context_limit, 1000);
@@ -445,6 +456,7 @@ mod tests {
input_token_cost: None,
output_token_cost: None,
currency: None,
supports_cache_control: None,
};
assert_eq!(info, info2);
@@ -455,6 +467,7 @@ mod tests {
input_token_cost: None,
output_token_cost: None,
currency: None,
supports_cache_control: None,
};
assert_ne!(info, info3);
}