feat: Add LiteLLM provider with automatic prompt caching support (#3380)
Signed-off-by: HikaruEgashira <account@egahika.dev>
This commit is contained in:
@@ -41,6 +41,8 @@ pub struct ModelInfo {
|
||||
pub output_token_cost: Option<f64>,
|
||||
/// Currency for the costs (default: "$")
|
||||
pub currency: Option<String>,
|
||||
/// Whether this model supports cache control
|
||||
pub supports_cache_control: Option<bool>,
|
||||
}
|
||||
|
||||
impl ModelInfo {
|
||||
@@ -52,6 +54,7 @@ impl ModelInfo {
|
||||
input_token_cost: None,
|
||||
output_token_cost: None,
|
||||
currency: None,
|
||||
supports_cache_control: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -68,6 +71,7 @@ impl ModelInfo {
|
||||
input_token_cost: Some(input_cost),
|
||||
output_token_cost: Some(output_cost),
|
||||
currency: Some("$".to_string()),
|
||||
supports_cache_control: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -115,6 +119,7 @@ impl ProviderMetadata {
|
||||
input_token_cost: None,
|
||||
output_token_cost: None,
|
||||
currency: None,
|
||||
supports_cache_control: None,
|
||||
})
|
||||
.collect(),
|
||||
model_doc_link: model_doc_link.to_string(),
|
||||
@@ -290,6 +295,11 @@ pub trait Provider: Send + Sync {
|
||||
false
|
||||
}
|
||||
|
||||
/// Check if this provider supports cache control
|
||||
fn supports_cache_control(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// Create embeddings if supported. Default implementation returns an error.
|
||||
async fn create_embeddings(&self, _texts: Vec<String>) -> Result<Vec<Vec<f32>>, ProviderError> {
|
||||
Err(ProviderError::ExecutionError(
|
||||
@@ -435,6 +445,7 @@ mod tests {
|
||||
input_token_cost: None,
|
||||
output_token_cost: None,
|
||||
currency: None,
|
||||
supports_cache_control: None,
|
||||
};
|
||||
assert_eq!(info.context_limit, 1000);
|
||||
|
||||
@@ -445,6 +456,7 @@ mod tests {
|
||||
input_token_cost: None,
|
||||
output_token_cost: None,
|
||||
currency: None,
|
||||
supports_cache_control: None,
|
||||
};
|
||||
assert_eq!(info, info2);
|
||||
|
||||
@@ -455,6 +467,7 @@ mod tests {
|
||||
input_token_cost: None,
|
||||
output_token_cost: None,
|
||||
currency: None,
|
||||
supports_cache_control: None,
|
||||
};
|
||||
assert_ne!(info, info3);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user