fix(cost): apply long-context price tiers to cost estimates (#12787)

This commit is contained in:
filip
2026-10-09 03:57:54 +00:00
committed by GitHub
parent 241061468a
commit 7bdeadb916
5 changed files with 6177 additions and 642 deletions
+1 -1
View File
@@ -4,7 +4,7 @@ pub mod models_dev;
mod name_builder;
mod registry;
pub use model::{CanonicalModel, Limit, Modalities, Modality, Pricing, ThinkingMode};
pub use model::{CanonicalModel, Limit, Modalities, Modality, Pricing, PricingTier, ThinkingMode};
pub use name_builder::{
canonical_name, map_provider_name, map_to_canonical_model, strip_version_suffix,
};
File diff suppressed because it is too large Load Diff
@@ -52,6 +52,28 @@ pub struct Pricing {
/// Cost per million cached write tokens
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_write: Option<f64>,
/// Long-context rates that replace the base rates for the whole request
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub tiers: Vec<PricingTier>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PricingTier {
/// The tier applies once the prompt exceeds this many input tokens
pub above_input_tokens: u64,
#[serde(skip_serializing_if = "Option::is_none")]
pub input: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub output: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_write: Option<f64>,
}
impl Pricing {
@@ -67,12 +89,24 @@ impl Pricing {
}
pub fn estimate_cost(&self, usage: &Usage) -> Option<f64> {
let input_price = self.input?;
let output_price = self.output?;
let cache_read_price = self.cache_read.unwrap_or(input_price);
let cache_write_price = self.cache_write.unwrap_or(input_price);
let prompt_tokens = usage.input_tokens.unwrap_or(0).max(0) as u64;
let tier = self
.tiers
.iter()
.filter(|tier| prompt_tokens > tier.above_input_tokens)
.max_by_key(|tier| tier.above_input_tokens);
let input_price = tier.and_then(|t| t.input).or(self.input)?;
let output_price = tier.and_then(|t| t.output).or(self.output)?;
let cache_read_price = tier
.and_then(|t| t.cache_read)
.or(self.cache_read)
.unwrap_or(input_price);
let cache_write_price = tier
.and_then(|t| t.cache_write)
.or(self.cache_write)
.unwrap_or(input_price);
let input_tokens = usage.input_tokens.unwrap_or(0).max(0) as f64;
let input_tokens = prompt_tokens as f64;
let output_tokens = usage.output_tokens.unwrap_or(0).max(0) as f64;
let cache_read_tokens = usage.cache_read_input_tokens.unwrap_or(0).max(0) as f64;
let cache_write_tokens = usage.cache_write_input_tokens.unwrap_or(0).max(0) as f64;
@@ -187,6 +221,7 @@ mod tests {
output,
cache_read,
cache_write,
tiers: Vec::new(),
}
}
@@ -1,6 +1,6 @@
use super::{
canonical_name, CanonicalModel, CanonicalModelRegistry, Limit, Modalities, Modality, Pricing,
ThinkingMode,
PricingTier, ThinkingMode,
};
use anyhow::{Context, Result};
use serde_json::Value;
@@ -62,6 +62,25 @@ fn parse_modalities(model_data: &Value, field: &str) -> Vec<Modality> {
.unwrap_or_else(|| vec![Modality::Text])
}
fn parse_context_tiers(cost: &Value) -> Vec<PricingTier> {
let Some(tiers) = cost.get("tiers").and_then(|v| v.as_array()) else {
return Vec::new();
};
tiers
.iter()
.filter(|t| t["tier"]["type"] == "context")
.filter_map(|t| {
Some(PricingTier {
above_input_tokens: t["tier"]["size"].as_u64()?,
input: t.get("input").and_then(|v| v.as_f64()),
output: t.get("output").and_then(|v| v.as_f64()),
cache_read: t.get("cache_read").and_then(|v| v.as_f64()),
cache_write: t.get("cache_write").and_then(|v| v.as_f64()),
})
})
.collect()
}
fn process_model(
model_id: &str,
model_data: &Value,
@@ -84,13 +103,9 @@ fn process_model(
output: c.get("output").and_then(|v| v.as_f64()),
cache_read: c.get("cache_read").and_then(|v| v.as_f64()),
cache_write: c.get("cache_write").and_then(|v| v.as_f64()),
tiers: parse_context_tiers(c),
},
_ => Pricing {
input: None,
output: None,
cache_read: None,
cache_write: None,
},
_ => Pricing::default(),
};
let limit = Limit {
+12 -4
View File
@@ -139,6 +139,7 @@ fn pricing_from_model_info(info: &ModelInfo) -> Option<Pricing> {
output: Some(info.output_token_cost.map(|c| c * 1_000_000.0)?),
cache_read: None,
cache_write: None,
tiers: Vec::new(),
})
}
@@ -207,12 +208,13 @@ mod tests {
}
}
/// Prices 1M input + 1M output tokens, so `expected` reads as the model's
/// input-plus-output rate per million tokens.
/// `expected` reads as the model's base input-plus-output rate per million
/// tokens; the prompt stays below every long-context tier.
fn assert_cost(provider: &str, model: &str, expected: f64) {
let used = usage(Some(1_000_000), Some(1_000_000), None);
let used = usage(Some(10_000), Some(10_000), None);
let actual = estimate_model_cost(provider, model, &used)
.unwrap_or_else(|| panic!("{provider}/{model} should have public pricing"));
.unwrap_or_else(|| panic!("{provider}/{model} should have public pricing"))
* 100.0;
assert!((actual - expected).abs() < 1e-9, "got {actual}");
}
@@ -296,12 +298,14 @@ mod tests {
output: Some(6.0),
cache_read: None,
cache_write: None,
..Default::default()
},
&Pricing {
input: Some(3.0),
output: Some(15.0),
cache_read: Some(0.3),
cache_write: Some(3.75),
..Default::default()
},
);
assert_eq!(merged.input, Some(2.0));
@@ -320,12 +324,14 @@ mod tests {
output: Some(0.0),
cache_read: None,
cache_write: None,
..Default::default()
},
&Pricing {
input: Some(3.0),
output: Some(15.0),
cache_read: Some(0.3),
cache_write: Some(3.75),
..Default::default()
},
);
assert_eq!(merged.cache_read, None);
@@ -344,12 +350,14 @@ mod tests {
output: Some(15.0),
cache_read: None,
cache_write: None,
..Default::default()
},
&Pricing {
input: Some(3.0),
output: Some(15.0),
cache_read: Some(0.3),
cache_write: None,
..Default::default()
},
);
let usage = usage(Some(1_000_000), Some(0), Some(1_000_000));