provider: Support provider-defined cache prices (#12944)
This commit is contained in:
@@ -259,6 +259,12 @@ pub struct ModelInfo {
|
||||
pub input_token_cost: Option<f64>,
|
||||
/// Cost per token for output in USD (optional)
|
||||
pub output_token_cost: Option<f64>,
|
||||
/// Cost per cached input token read in USD
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_read_token_cost: Option<f64>,
|
||||
/// Cost per input token written to cache in USD
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_write_token_cost: Option<f64>,
|
||||
/// Currency for the costs (default: "$")
|
||||
pub currency: Option<String>,
|
||||
/// Whether this model supports cache control
|
||||
@@ -284,6 +290,8 @@ impl ModelInfo {
|
||||
context_limit: None,
|
||||
input_token_cost: None,
|
||||
output_token_cost: None,
|
||||
cache_read_token_cost: None,
|
||||
cache_write_token_cost: None,
|
||||
currency: None,
|
||||
supports_cache_control: None,
|
||||
reasoning: false,
|
||||
@@ -321,6 +329,8 @@ impl ModelInfo {
|
||||
context_limit: Some(context_limit),
|
||||
input_token_cost: Some(input_cost),
|
||||
output_token_cost: Some(output_cost),
|
||||
cache_read_token_cost: None,
|
||||
cache_write_token_cost: None,
|
||||
currency: Some("$".to_string()),
|
||||
supports_cache_control: None,
|
||||
reasoning: false,
|
||||
@@ -372,6 +382,8 @@ pub fn model_info_for_provider_model(provider_name: &str, model_name: &str) -> M
|
||||
context_limit: canonical.as_ref().map(|model| model.limit.context),
|
||||
input_token_cost: None,
|
||||
output_token_cost: None,
|
||||
cache_read_token_cost: None,
|
||||
cache_write_token_cost: None,
|
||||
currency: None,
|
||||
supports_cache_control: None,
|
||||
reasoning,
|
||||
@@ -1115,6 +1127,8 @@ mod tests {
|
||||
context_limit: Some(1000),
|
||||
input_token_cost: None,
|
||||
output_token_cost: None,
|
||||
cache_read_token_cost: None,
|
||||
cache_write_token_cost: None,
|
||||
currency: None,
|
||||
supports_cache_control: None,
|
||||
reasoning: false,
|
||||
@@ -1131,6 +1145,8 @@ mod tests {
|
||||
context_limit: Some(1000),
|
||||
input_token_cost: None,
|
||||
output_token_cost: None,
|
||||
cache_read_token_cost: None,
|
||||
cache_write_token_cost: None,
|
||||
currency: None,
|
||||
supports_cache_control: None,
|
||||
reasoning: false,
|
||||
@@ -1147,6 +1163,8 @@ mod tests {
|
||||
context_limit: Some(2000),
|
||||
input_token_cost: None,
|
||||
output_token_cost: None,
|
||||
cache_read_token_cost: None,
|
||||
cache_write_token_cost: None,
|
||||
currency: None,
|
||||
supports_cache_control: None,
|
||||
reasoning: false,
|
||||
|
||||
@@ -412,6 +412,8 @@ fn model_info_for_deployment(deployment_name: &str, model_name: &str) -> ModelIn
|
||||
context_limit: canonical.as_ref().map(|model| model.limit.context),
|
||||
input_token_cost: None,
|
||||
output_token_cost: None,
|
||||
cache_read_token_cost: None,
|
||||
cache_write_token_cost: None,
|
||||
currency: None,
|
||||
supports_cache_control: None,
|
||||
reasoning: canonical
|
||||
|
||||
@@ -517,6 +517,8 @@ impl DatabricksProvider {
|
||||
context_limit,
|
||||
input_token_cost: None,
|
||||
output_token_cost: None,
|
||||
cache_read_token_cost: None,
|
||||
cache_write_token_cost: None,
|
||||
currency: None,
|
||||
supports_cache_control: None,
|
||||
reasoning,
|
||||
|
||||
@@ -1206,9 +1206,6 @@ impl GooseAcpAgent {
|
||||
let model_info =
|
||||
crate::providers::canonical::maybe_get_canonical_model(&req.provider, &req.model)
|
||||
.map(|canonical_model| {
|
||||
// Config-declared prices outrank the registry's catalog rates;
|
||||
// registry cache rates survive, so tooltip math matches
|
||||
// estimate_model_cost exactly.
|
||||
let pricing = crate::providers::canonical_cost::resolve_pricing(
|
||||
&req.provider,
|
||||
&req.model,
|
||||
|
||||
@@ -303,6 +303,12 @@ pub fn update_custom_provider(params: UpdateCustomProviderParams) -> Result<()>
|
||||
model.input_token_cost = model.input_token_cost.or(existing.input_token_cost);
|
||||
model.output_token_cost =
|
||||
model.output_token_cost.or(existing.output_token_cost);
|
||||
model.cache_read_token_cost = model
|
||||
.cache_read_token_cost
|
||||
.or(existing.cache_read_token_cost);
|
||||
model.cache_write_token_cost = model
|
||||
.cache_write_token_cost
|
||||
.or(existing.cache_write_token_cost);
|
||||
model.currency = model.currency.or(existing.currency.clone());
|
||||
model.supports_cache_control = model
|
||||
.supports_cache_control
|
||||
@@ -609,6 +615,8 @@ mod tests {
|
||||
context_limit: Some(128_000),
|
||||
input_token_cost: None,
|
||||
output_token_cost: None,
|
||||
cache_read_token_cost: None,
|
||||
cache_write_token_cost: None,
|
||||
currency: None,
|
||||
supports_cache_control: None,
|
||||
reasoning: false,
|
||||
@@ -803,6 +811,8 @@ mod tests {
|
||||
|
||||
let mut model = ModelInfo::with_cost("large-model", 1_048_576, 0.000002, 0.000006)
|
||||
.with_vision_support(true);
|
||||
model.cache_read_token_cost = Some(0.0000002);
|
||||
model.cache_write_token_cost = Some(0.0000004);
|
||||
model.request_params = Some(HashMap::from([(
|
||||
"temperature".to_string(),
|
||||
serde_json::json!(0.25),
|
||||
@@ -859,6 +869,8 @@ mod tests {
|
||||
assert_eq!(model.context_limit, Some(2_097_152));
|
||||
assert_eq!(model.input_token_cost, Some(0.000002));
|
||||
assert_eq!(model.output_token_cost, Some(0.000006));
|
||||
assert_eq!(model.cache_read_token_cost, Some(0.0000002));
|
||||
assert_eq!(model.cache_write_token_cost, Some(0.0000004));
|
||||
assert_eq!(
|
||||
model.request_params.as_ref().unwrap()["temperature"],
|
||||
serde_json::json!(0.25)
|
||||
|
||||
@@ -69,6 +69,8 @@ pub(crate) fn resolve_pricing(provider: &str, model: &str) -> Option<Pricing> {
|
||||
if is_price_gap(canonical.output) {
|
||||
canonical.output = bundled.output;
|
||||
}
|
||||
canonical.cache_read = canonical.cache_read.or(bundled.cache_read);
|
||||
canonical.cache_write = canonical.cache_write.or(bundled.cache_write);
|
||||
Some(canonical)
|
||||
}
|
||||
(Some(canonical), None) => Some(canonical),
|
||||
@@ -114,8 +116,7 @@ fn currency_symbol(declared: &str) -> String {
|
||||
}
|
||||
|
||||
/// Merge user-declared pricing with registry/bundled pricing: declared
|
||||
/// input/output prices win; canonical cache rates fill the gaps so cached
|
||||
/// tokens are not overestimated at the full input rate.
|
||||
/// input/output and supplied cache prices win; canonical cache rates fill gaps.
|
||||
fn merge_pricing(mut declared: Pricing, base: &Pricing) -> Pricing {
|
||||
// A declared zero price means "free here": registry cache rates must not
|
||||
// reintroduce cost for cached tokens.
|
||||
@@ -137,8 +138,8 @@ fn pricing_from_model_info(info: &ModelInfo) -> Option<Pricing> {
|
||||
Some(Pricing {
|
||||
input: Some(info.input_token_cost.map(|c| c * 1_000_000.0)?),
|
||||
output: Some(info.output_token_cost.map(|c| c * 1_000_000.0)?),
|
||||
cache_read: None,
|
||||
cache_write: None,
|
||||
cache_read: info.cache_read_token_cost.map(|c| c * 1_000_000.0),
|
||||
cache_write: info.cache_write_token_cost.map(|c| c * 1_000_000.0),
|
||||
tiers: Vec::new(),
|
||||
})
|
||||
}
|
||||
@@ -196,7 +197,6 @@ fn full_pricing_model(config: &DeclarativeProviderConfig, model: &str) -> Option
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::declarative_providers::custom_providers_dir;
|
||||
|
||||
fn usage(input: Option<i32>, output: Option<i32>, cache_read: Option<i32>) -> Usage {
|
||||
Usage {
|
||||
@@ -283,87 +283,6 @@ mod tests {
|
||||
assert!(pricing_from_model_info(&info).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_pricing_is_rejected() {
|
||||
let mut info = ModelInfo::new("m").with_context_limit(1_000);
|
||||
info.input_token_cost = Some(0.000002);
|
||||
assert!(pricing_from_model_info(&info).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn declared_input_output_win_and_canonical_cache_rates_fill_the_gap() {
|
||||
let merged = merge_pricing(
|
||||
Pricing {
|
||||
input: Some(2.0),
|
||||
output: Some(6.0),
|
||||
cache_read: None,
|
||||
cache_write: None,
|
||||
..Default::default()
|
||||
},
|
||||
&Pricing {
|
||||
input: Some(3.0),
|
||||
output: Some(15.0),
|
||||
cache_read: Some(0.3),
|
||||
cache_write: Some(3.75),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
assert_eq!(merged.input, Some(2.0));
|
||||
assert_eq!(merged.output, Some(6.0));
|
||||
assert_eq!(merged.cache_read, Some(0.3));
|
||||
assert_eq!(merged.cache_write, Some(3.75));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn declared_zero_price_suppresses_registry_cache_rates() {
|
||||
// A model explicitly declared free in a custom config must not pick up
|
||||
// catalog cache pricing from a canonical name-match.
|
||||
let merged = merge_pricing(
|
||||
Pricing {
|
||||
input: Some(0.0),
|
||||
output: Some(0.0),
|
||||
cache_read: None,
|
||||
cache_write: None,
|
||||
..Default::default()
|
||||
},
|
||||
&Pricing {
|
||||
input: Some(3.0),
|
||||
output: Some(15.0),
|
||||
cache_read: Some(0.3),
|
||||
cache_write: Some(3.75),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
assert_eq!(merged.cache_read, None);
|
||||
assert_eq!(merged.cache_write, None);
|
||||
let usage = usage(Some(1_000_000), Some(0), Some(1_000_000));
|
||||
assert_eq!(merged.estimate_cost(&usage), Some(0.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cached_tokens_price_at_cache_rate_not_full_input_rate() {
|
||||
// A canonical model proxied through a gateway with declared
|
||||
// input/output prices but no cache prices.
|
||||
let merged = merge_pricing(
|
||||
Pricing {
|
||||
input: Some(3.0),
|
||||
output: Some(15.0),
|
||||
cache_read: None,
|
||||
cache_write: None,
|
||||
..Default::default()
|
||||
},
|
||||
&Pricing {
|
||||
input: Some(3.0),
|
||||
output: Some(15.0),
|
||||
cache_read: Some(0.3),
|
||||
cache_write: None,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let usage = usage(Some(1_000_000), Some(0), Some(1_000_000));
|
||||
assert!((merged.estimate_cost(&usage).unwrap() - 0.3).abs() < 1e-9);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bundled_provider_prices_are_respected() {
|
||||
let info = configured_model_info("ovhcloud", "Qwen3-32B").unwrap();
|
||||
@@ -373,42 +292,6 @@ mod tests {
|
||||
assert!((pricing.output.unwrap() - 0.25).abs() < 1e-9);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn custom_file_prices_drive_estimation() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let temp_root = temp_dir.path().display().to_string();
|
||||
let _guard = env_lock::lock_env([("GOOSE_PATH_ROOT", Some(temp_root.as_str()))]);
|
||||
|
||||
let dir = custom_providers_dir();
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
std::fs::write(
|
||||
dir.join("priced_gateway.json"),
|
||||
r#"{
|
||||
"name": "priced_gateway",
|
||||
"engine": "openai",
|
||||
"display_name": "Gateway",
|
||||
"description": null,
|
||||
"api_key_env": "",
|
||||
"base_url": "https://example.invalid/v1",
|
||||
"models": [
|
||||
{
|
||||
"name": "claude-sonnet-4-5",
|
||||
"context_limit": 200000,
|
||||
"input_token_cost": 0.000001,
|
||||
"output_token_cost": 0.000004
|
||||
}
|
||||
],
|
||||
"requires_auth": false
|
||||
}"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let used = usage(Some(10_000), Some(100), None);
|
||||
let cost = estimate_model_cost("priced_gateway", "claude-sonnet-4-5", &used).unwrap();
|
||||
let expected = (10_000.0 * 1.0 + 100.0 * 4.0) / 1_000_000.0;
|
||||
assert!((cost - expected).abs() < 1e-12);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn declared_currency_is_rendered_as_a_symbol() {
|
||||
let mut info = ModelInfo::with_cost("m", 1_000, 0.000001, 0.000004);
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
use goose::config::declarative_providers::custom_providers_dir;
|
||||
use goose::providers::canonical_cost::estimate_model_cost;
|
||||
use goose_providers::conversation::token_usage::Usage;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn custom_cache_prices_require_input_and_output_prices() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let root_path = root.path().to_str().unwrap();
|
||||
let _guard = env_lock::lock_env([("GOOSE_PATH_ROOT", Some(root_path))]);
|
||||
let dir = custom_providers_dir();
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let usage = Usage::new(Some(1_000_000), Some(1_000_000), None)
|
||||
.with_cache_tokens(Some(200_000), Some(300_000));
|
||||
|
||||
for (prices, expected) in [
|
||||
(json!({}), 12.25),
|
||||
(
|
||||
json!({"input_token_cost": 0.000001, "currency": "EUR",
|
||||
"cache_read_token_cost": 0.0000002}),
|
||||
12.25,
|
||||
),
|
||||
(json!({"output_token_cost": 0.000004}), 12.25),
|
||||
(
|
||||
json!({"input_token_cost": 0.000001, "output_token_cost": 0.000004}),
|
||||
5.05,
|
||||
),
|
||||
(
|
||||
json!({"input_token_cost": 0.000001, "output_token_cost": 0.000004,
|
||||
"cache_read_token_cost": 0.0000002}),
|
||||
4.84,
|
||||
),
|
||||
(
|
||||
json!({"input_token_cost": 0.000001, "output_token_cost": 0.000004,
|
||||
"cache_write_token_cost": 0.000003}),
|
||||
5.65,
|
||||
),
|
||||
(
|
||||
json!({"input_token_cost": 0.000001, "output_token_cost": 0.000004,
|
||||
"cache_read_token_cost": 0.0}),
|
||||
4.8,
|
||||
),
|
||||
(
|
||||
json!({"input_token_cost": 0.000001, "output_token_cost": 0.000004,
|
||||
"cache_write_token_cost": 0.0}),
|
||||
4.75,
|
||||
),
|
||||
(
|
||||
json!({"input_token_cost": 0.0, "output_token_cost": 0.0}),
|
||||
0.0,
|
||||
),
|
||||
] {
|
||||
let mut model = prices;
|
||||
model["name"] = json!("gpt-4o");
|
||||
let config = json!({
|
||||
"name": "priced_gateway",
|
||||
"engine": "openai",
|
||||
"display_name": "Gateway",
|
||||
"api_key_env": "",
|
||||
"base_url": "https://example.invalid/v1",
|
||||
"models": [model],
|
||||
"requires_auth": false
|
||||
});
|
||||
std::fs::write(dir.join("priced_gateway.json"), config.to_string()).unwrap();
|
||||
|
||||
let cost = estimate_model_cost("priced_gateway", "gpt-4o", &usage).unwrap();
|
||||
assert!(
|
||||
(cost - expected).abs() < 1e-9,
|
||||
"{config}: got {cost}, expected {expected}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn custom_cache_prices_work_without_a_catalog_entry() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let root_path = root.path().to_str().unwrap();
|
||||
let _guard = env_lock::lock_env([("GOOSE_PATH_ROOT", Some(root_path))]);
|
||||
let dir = custom_providers_dir();
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let mut config = json!({
|
||||
"name": "private_gateway",
|
||||
"engine": "openai",
|
||||
"display_name": "Gateway",
|
||||
"api_key_env": "",
|
||||
"base_url": "https://example.invalid/v1",
|
||||
"models": [{
|
||||
"name": "private-model",
|
||||
"input_token_cost": 0.00000242,
|
||||
"output_token_cost": 0.00001,
|
||||
"cache_read_token_cost": 0.000000242,
|
||||
"cache_write_token_cost": 0.000003025
|
||||
}],
|
||||
"requires_auth": false
|
||||
});
|
||||
let path = dir.join("private_gateway.json");
|
||||
std::fs::write(&path, config.to_string()).unwrap();
|
||||
|
||||
let usage =
|
||||
Usage::new(Some(100_000), Some(0), None).with_cache_tokens(Some(50_000), Some(25_000));
|
||||
let cost = estimate_model_cost("private_gateway", "private-model", &usage).unwrap();
|
||||
assert!((cost - 0.148225).abs() < 1e-9, "got {cost}");
|
||||
|
||||
config["models"][0]
|
||||
.as_object_mut()
|
||||
.unwrap()
|
||||
.remove("output_token_cost");
|
||||
std::fs::write(&path, config.to_string()).unwrap();
|
||||
let usage = Usage::new(Some(100_000), Some(100), None);
|
||||
assert!(estimate_model_cost("private_gateway", "private-model", &usage).is_none());
|
||||
}
|
||||
@@ -550,6 +550,30 @@ Custom providers must use OpenAI, Anthropic, or Ollama compatible API formats. T
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
### Custom Model Prices
|
||||
|
||||
Each model in a custom provider's JSON file can declare prices in USD **per token**:
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "my-model",
|
||||
"input_token_cost": 0.00000242,
|
||||
"output_token_cost": 0.00001,
|
||||
"cache_read_token_cost": 0.000000242,
|
||||
"cache_write_token_cost": 0.000003025
|
||||
}
|
||||
```
|
||||
|
||||
Both `input_token_cost` and `output_token_cost` must be present for goose to
|
||||
use a model's configured pricing. If either is missing, goose falls back to
|
||||
catalog pricing for the model.
|
||||
|
||||
Cache-read and cache-write prices are independently optional. Supplied cache
|
||||
prices, including `0`, override catalog cache rates. Omitted cache prices use
|
||||
catalog cache rates when available, otherwise the input price. Declaring both
|
||||
input and output prices as `0` keeps the model free unless you explicitly set
|
||||
a cache price. Provider-reported costs take precedence over estimates.
|
||||
|
||||
### Command-Based Authentication
|
||||
|
||||
Instead of a static `api_key_env`, a custom provider can be configured to run a command to obtain its credential. This is useful for short-lived credentials issued by an IdP or key vault: goose re-runs the command to refresh the credential instead of requiring a restart when it expires.
|
||||
|
||||
@@ -13,6 +13,8 @@ export type ConfigKey = {
|
||||
};
|
||||
|
||||
export type ModelInfo = {
|
||||
cache_read_token_cost?: number | null;
|
||||
cache_write_token_cost?: number | null;
|
||||
context_limit?: number | null;
|
||||
currency?: string | null;
|
||||
input_token_cost?: number | null;
|
||||
|
||||
Reference in New Issue
Block a user