fix(cost): apply long-context price tiers to cost estimates (#12787)
This commit is contained in:
@@ -4,7 +4,7 @@ pub mod models_dev;
|
||||
mod name_builder;
|
||||
mod registry;
|
||||
|
||||
pub use model::{CanonicalModel, Limit, Modalities, Modality, Pricing, ThinkingMode};
|
||||
pub use model::{CanonicalModel, Limit, Modalities, Modality, Pricing, PricingTier, ThinkingMode};
|
||||
pub use name_builder::{
|
||||
canonical_name, map_provider_name, map_to_canonical_model, strip_version_suffix,
|
||||
};
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -52,6 +52,28 @@ pub struct Pricing {
|
||||
/// Cost per million cached write tokens
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub cache_write: Option<f64>,
|
||||
|
||||
/// Long-context rates that replace the base rates for the whole request
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub tiers: Vec<PricingTier>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct PricingTier {
|
||||
/// The tier applies once the prompt exceeds this many input tokens
|
||||
pub above_input_tokens: u64,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub input: Option<f64>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub output: Option<f64>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub cache_read: Option<f64>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub cache_write: Option<f64>,
|
||||
}
|
||||
|
||||
impl Pricing {
|
||||
@@ -67,12 +89,24 @@ impl Pricing {
|
||||
}
|
||||
|
||||
pub fn estimate_cost(&self, usage: &Usage) -> Option<f64> {
|
||||
let input_price = self.input?;
|
||||
let output_price = self.output?;
|
||||
let cache_read_price = self.cache_read.unwrap_or(input_price);
|
||||
let cache_write_price = self.cache_write.unwrap_or(input_price);
|
||||
let prompt_tokens = usage.input_tokens.unwrap_or(0).max(0) as u64;
|
||||
let tier = self
|
||||
.tiers
|
||||
.iter()
|
||||
.filter(|tier| prompt_tokens > tier.above_input_tokens)
|
||||
.max_by_key(|tier| tier.above_input_tokens);
|
||||
let input_price = tier.and_then(|t| t.input).or(self.input)?;
|
||||
let output_price = tier.and_then(|t| t.output).or(self.output)?;
|
||||
let cache_read_price = tier
|
||||
.and_then(|t| t.cache_read)
|
||||
.or(self.cache_read)
|
||||
.unwrap_or(input_price);
|
||||
let cache_write_price = tier
|
||||
.and_then(|t| t.cache_write)
|
||||
.or(self.cache_write)
|
||||
.unwrap_or(input_price);
|
||||
|
||||
let input_tokens = usage.input_tokens.unwrap_or(0).max(0) as f64;
|
||||
let input_tokens = prompt_tokens as f64;
|
||||
let output_tokens = usage.output_tokens.unwrap_or(0).max(0) as f64;
|
||||
let cache_read_tokens = usage.cache_read_input_tokens.unwrap_or(0).max(0) as f64;
|
||||
let cache_write_tokens = usage.cache_write_input_tokens.unwrap_or(0).max(0) as f64;
|
||||
@@ -187,6 +221,7 @@ mod tests {
|
||||
output,
|
||||
cache_read,
|
||||
cache_write,
|
||||
tiers: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use super::{
|
||||
canonical_name, CanonicalModel, CanonicalModelRegistry, Limit, Modalities, Modality, Pricing,
|
||||
ThinkingMode,
|
||||
PricingTier, ThinkingMode,
|
||||
};
|
||||
use anyhow::{Context, Result};
|
||||
use serde_json::Value;
|
||||
@@ -62,6 +62,25 @@ fn parse_modalities(model_data: &Value, field: &str) -> Vec<Modality> {
|
||||
.unwrap_or_else(|| vec![Modality::Text])
|
||||
}
|
||||
|
||||
fn parse_context_tiers(cost: &Value) -> Vec<PricingTier> {
|
||||
let Some(tiers) = cost.get("tiers").and_then(|v| v.as_array()) else {
|
||||
return Vec::new();
|
||||
};
|
||||
tiers
|
||||
.iter()
|
||||
.filter(|t| t["tier"]["type"] == "context")
|
||||
.filter_map(|t| {
|
||||
Some(PricingTier {
|
||||
above_input_tokens: t["tier"]["size"].as_u64()?,
|
||||
input: t.get("input").and_then(|v| v.as_f64()),
|
||||
output: t.get("output").and_then(|v| v.as_f64()),
|
||||
cache_read: t.get("cache_read").and_then(|v| v.as_f64()),
|
||||
cache_write: t.get("cache_write").and_then(|v| v.as_f64()),
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn process_model(
|
||||
model_id: &str,
|
||||
model_data: &Value,
|
||||
@@ -84,13 +103,9 @@ fn process_model(
|
||||
output: c.get("output").and_then(|v| v.as_f64()),
|
||||
cache_read: c.get("cache_read").and_then(|v| v.as_f64()),
|
||||
cache_write: c.get("cache_write").and_then(|v| v.as_f64()),
|
||||
tiers: parse_context_tiers(c),
|
||||
},
|
||||
_ => Pricing {
|
||||
input: None,
|
||||
output: None,
|
||||
cache_read: None,
|
||||
cache_write: None,
|
||||
},
|
||||
_ => Pricing::default(),
|
||||
};
|
||||
|
||||
let limit = Limit {
|
||||
|
||||
@@ -139,6 +139,7 @@ fn pricing_from_model_info(info: &ModelInfo) -> Option<Pricing> {
|
||||
output: Some(info.output_token_cost.map(|c| c * 1_000_000.0)?),
|
||||
cache_read: None,
|
||||
cache_write: None,
|
||||
tiers: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -207,12 +208,13 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// Prices 1M input + 1M output tokens, so `expected` reads as the model's
|
||||
/// input-plus-output rate per million tokens.
|
||||
/// `expected` reads as the model's base input-plus-output rate per million
|
||||
/// tokens; the prompt stays below every long-context tier.
|
||||
fn assert_cost(provider: &str, model: &str, expected: f64) {
|
||||
let used = usage(Some(1_000_000), Some(1_000_000), None);
|
||||
let used = usage(Some(10_000), Some(10_000), None);
|
||||
let actual = estimate_model_cost(provider, model, &used)
|
||||
.unwrap_or_else(|| panic!("{provider}/{model} should have public pricing"));
|
||||
.unwrap_or_else(|| panic!("{provider}/{model} should have public pricing"))
|
||||
* 100.0;
|
||||
assert!((actual - expected).abs() < 1e-9, "got {actual}");
|
||||
}
|
||||
|
||||
@@ -296,12 +298,14 @@ mod tests {
|
||||
output: Some(6.0),
|
||||
cache_read: None,
|
||||
cache_write: None,
|
||||
..Default::default()
|
||||
},
|
||||
&Pricing {
|
||||
input: Some(3.0),
|
||||
output: Some(15.0),
|
||||
cache_read: Some(0.3),
|
||||
cache_write: Some(3.75),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
assert_eq!(merged.input, Some(2.0));
|
||||
@@ -320,12 +324,14 @@ mod tests {
|
||||
output: Some(0.0),
|
||||
cache_read: None,
|
||||
cache_write: None,
|
||||
..Default::default()
|
||||
},
|
||||
&Pricing {
|
||||
input: Some(3.0),
|
||||
output: Some(15.0),
|
||||
cache_read: Some(0.3),
|
||||
cache_write: Some(3.75),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
assert_eq!(merged.cache_read, None);
|
||||
@@ -344,12 +350,14 @@ mod tests {
|
||||
output: Some(15.0),
|
||||
cache_read: None,
|
||||
cache_write: None,
|
||||
..Default::default()
|
||||
},
|
||||
&Pricing {
|
||||
input: Some(3.0),
|
||||
output: Some(15.0),
|
||||
cache_read: Some(0.3),
|
||||
cache_write: None,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let usage = usage(Some(1_000_000), Some(0), Some(1_000_000));
|
||||
|
||||
Reference in New Issue
Block a user