feat: configurable Anthropic prompt-cache TTL (5m/1h) (#11576)

Signed-off-by: Michael Neale <michael.neale@gmail.com>
Co-authored-by: Galadriel <galadriel@buzz.agent>
This commit is contained in:
Michael Neale
2026-08-31 00:43:58 +00:00
committed by GitHub
parent 815bd0b69a
commit fb15d4eade
9 changed files with 425 additions and 14 deletions
+93 -2
View File
@@ -411,7 +411,7 @@ async fn resolve_provider_and_model(
process::exit(1);
});
let model_config = if session_config.resume
let mut model_config = if session_config.resume
&& saved_provider_matches
&& saved_model_config
.as_ref()
@@ -419,6 +419,10 @@ async fn resolve_provider_and_model(
{
let mut config = saved_model_config.unwrap();
config.normalize_effort_suffix();
config = goose::model_config::with_rederived_cache_ttl(config).unwrap_or_else(|e| {
output::render_error(&format!("Invalid cache TTL configuration: {}", e));
process::exit(1);
});
if let Some(temp) = recipe_settings.and_then(|s| s.temperature) {
config = config.with_temperature(Some(temp));
}
@@ -436,6 +440,10 @@ async fn resolve_provider_and_model(
config
};
if !session_config.interactive {
model_config = model_config.with_cache_ttl_clamped();
}
ResolvedProviderConfig {
provider_name,
model_name,
@@ -730,7 +738,7 @@ pub async fn build_session(session_config: SessionBuilderConfig) -> CliSession {
))
.yellow()
);
let fallback_model_config =
let mut fallback_model_config =
model_config_from_user_config(fallback_provider.as_str(), &fallback_model)
.unwrap_or_else(|e| {
output::render_error(&format!(
@@ -739,6 +747,9 @@ pub async fn build_session(session_config: SessionBuilderConfig) -> CliSession {
));
process::exit(1);
});
if !session_config.interactive {
fallback_model_config = fallback_model_config.with_cache_ttl_clamped();
}
match create(&fallback_provider, extensions_for_provider.clone()).await {
Ok(provider) => (
provider,
@@ -1433,6 +1444,86 @@ mod tests {
.is_some_and(|params| params.contains_key("anthropic_beta")));
}
#[tokio::test]
async fn resume_rederives_cache_ttl_from_config_not_session() {
let _guard = env_lock::lock_env([
("GOOSE_PROVIDER", None::<&str>),
("GOOSE_MODEL", None::<&str>),
("GOOSE_CACHE_TTL", Some("1h")),
]);
let temp_dir = TempDir::new().unwrap();
let config = test_config(&temp_dir);
let saved =
goose_providers::model::ModelConfig::new("claude-sonnet-4-6").with_cache_ttl("5m");
let resolved = resolve_provider_and_model(
&SessionBuilderConfig {
resume: true,
interactive: true,
..SessionBuilderConfig::default()
},
&config,
Some("anthropic".to_string()),
Some(saved),
)
.await;
assert_eq!(resolved.model_config.cache_ttl().as_deref(), Some("1h"));
}
#[tokio::test]
async fn resume_drops_saved_cache_ttl_when_config_absent() {
let _guard = env_lock::lock_env([
("GOOSE_PROVIDER", None::<&str>),
("GOOSE_MODEL", None::<&str>),
("GOOSE_CACHE_TTL", None::<&str>),
]);
let temp_dir = TempDir::new().unwrap();
let config = test_config(&temp_dir);
let saved =
goose_providers::model::ModelConfig::new("claude-sonnet-4-6").with_cache_ttl("1h");
let resolved = resolve_provider_and_model(
&SessionBuilderConfig {
resume: true,
interactive: true,
..SessionBuilderConfig::default()
},
&config,
Some("anthropic".to_string()),
Some(saved),
)
.await;
assert!(resolved.model_config.cache_ttl().is_none());
}
#[tokio::test]
async fn headless_resume_clamps_rederived_cache_ttl() {
let _guard = env_lock::lock_env([
("GOOSE_PROVIDER", None::<&str>),
("GOOSE_MODEL", None::<&str>),
("GOOSE_CACHE_TTL", Some("1h")),
]);
let temp_dir = TempDir::new().unwrap();
let config = test_config(&temp_dir);
let saved = goose_providers::model::ModelConfig::new("claude-sonnet-4-6");
let resolved = resolve_provider_and_model(
&SessionBuilderConfig {
resume: true,
interactive: false,
..SessionBuilderConfig::default()
},
&config,
Some("anthropic".to_string()),
Some(saved),
)
.await;
assert_eq!(resolved.model_config.cache_ttl().as_deref(), Some("5m"));
}
#[test]
fn resumed_provider_override_rejects_context_owning_provider() {
let error = validate_provider_override_context(
@@ -45,6 +45,7 @@ macro_rules! string_enum {
}
string_enum!(ThinkingType { Adaptive => "adaptive", Enabled => "enabled", Disabled => "disabled" });
string_enum!(CacheTtl { FiveMinutes => "5m", OneHour => "1h" });
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct AnthropicFormatOptions {
@@ -54,6 +55,7 @@ pub struct AnthropicFormatOptions {
pub emit_clear_thinking: bool,
pub current_model: Option<String>,
pub prompt_cache_disabled: bool,
pub cache_ttl: Option<CacheTtl>,
}
impl AnthropicFormatOptions {
@@ -70,6 +72,10 @@ impl AnthropicFormatOptions {
let emit_clear_thinking = model_config
.request_param::<bool>("emit_clear_thinking")
.unwrap_or(self.emit_clear_thinking);
let cache_ttl = model_config
.cache_ttl()
.and_then(|ttl| ttl.parse::<CacheTtl>().ok())
.or(self.cache_ttl);
Self {
preserve_unsigned_thinking,
@@ -80,6 +86,19 @@ impl AnthropicFormatOptions {
.current_model
.or_else(|| Some(model_config.model_name.clone())),
prompt_cache_disabled: model_config.prompt_cache_disabled(),
cache_ttl,
}
}
/// `{"type":"ephemeral"}` selects Anthropic's default 5m TTL; the `ttl`
/// field is only sent for an explicit 1h opt-in, since a 1h write is
/// billed at 2x input instead of 1.25x.
fn cache_control(&self) -> Value {
match self.cache_ttl {
Some(CacheTtl::OneHour) => {
json!({ TYPE_FIELD: "ephemeral", "ttl": "1h" })
}
_ => json!({ TYPE_FIELD: "ephemeral" }),
}
}
}
@@ -446,10 +465,7 @@ fn format_messages_with_options(
.and_then(|content_array| content_array.last_mut())
.and_then(|b| b.as_object_mut())
{
block.insert(
CACHE_CONTROL_FIELD.to_string(),
json!({ TYPE_FIELD: "ephemeral" }),
);
block.insert(CACHE_CONTROL_FIELD.to_string(), options.cache_control());
user_count += 1;
if user_count >= 2 {
break;
@@ -491,10 +507,10 @@ pub fn format_tools(tools: &[Tool], options: &AnthropicFormatOptions) -> Vec<Val
// Add "cache_control" to the last tool spec, if any. This means that all tool definitions,
// will be cached as a single prefix.
if let Some(last_tool) = tool_specs.last_mut() {
last_tool.as_object_mut().unwrap().insert(
CACHE_CONTROL_FIELD.to_string(),
json!({ TYPE_FIELD: "ephemeral" }),
);
last_tool
.as_object_mut()
.unwrap()
.insert(CACHE_CONTROL_FIELD.to_string(), options.cache_control());
}
tool_specs
@@ -511,7 +527,7 @@ pub fn format_system(system: &str, options: &AnthropicFormatOptions) -> Value {
json!([{
TYPE_FIELD: TEXT_TYPE,
TEXT_TYPE: system,
CACHE_CONTROL_FIELD: { TYPE_FIELD: "ephemeral" }
CACHE_CONTROL_FIELD: options.cache_control()
}])
}
@@ -2777,6 +2793,115 @@ mod tests {
assert!(!req.to_string().contains(CACHE_CONTROL_FIELD));
}
fn cache_control_values(req: &Value) -> Vec<Value> {
let mut found = Vec::new();
let tools = req["tools"].as_array().unwrap();
let system = req["system"].as_array().unwrap();
let messages = req["messages"].as_array().unwrap();
for block in tools
.iter()
.chain(system.iter())
.chain(messages.iter().flat_map(|m| {
m["content"]
.as_array()
.map(|c| c.iter())
.unwrap_or_default()
}))
{
if let Some(cc) = block.get(CACHE_CONTROL_FIELD) {
found.push(cc.clone());
}
}
found
}
#[test]
fn default_breakpoints_omit_ttl() {
let req = create_request_with_default_options(
&cfg("claude-sonnet-4-5"),
"You are a careful coding assistant.",
&[Message::user().with_text("Hello")],
&sample_tools(),
)
.unwrap();
let values = cache_control_values(&req);
assert_eq!(values.len(), 3);
for cc in values {
assert_eq!(cc, json!({ "type": "ephemeral" }));
}
}
#[test]
fn one_hour_ttl_stamps_every_breakpoint() {
let config = cfg("claude-sonnet-4-5").with_cache_ttl("1h");
let req = create_request_with_default_options(
&config,
"You are a careful coding assistant.",
&[
Message::user().with_text("Hello"),
Message::assistant().with_text("Hi."),
Message::user().with_text("Continue"),
],
&sample_tools(),
)
.unwrap();
let values = cache_control_values(&req);
assert_eq!(values.len(), 4);
for cc in values {
assert_eq!(cc, json!({ "type": "ephemeral", "ttl": "1h" }));
}
}
#[test]
fn explicit_five_minute_ttl_matches_default_wire_format() {
let config = cfg("claude-sonnet-4-5").with_cache_ttl("5m");
let req = create_request_with_default_options(
&config,
"You are a careful coding assistant.",
&[Message::user().with_text("Hello")],
&sample_tools(),
)
.unwrap();
for cc in cache_control_values(&req) {
assert_eq!(cc, json!({ "type": "ephemeral" }));
}
}
#[test]
fn unrecognized_ttl_value_falls_back_to_default() {
let config = cfg("claude-sonnet-4-5").with_cache_ttl("2h");
let req = create_request_with_default_options(
&config,
"You are a careful coding assistant.",
&[Message::user().with_text("Hello")],
&sample_tools(),
)
.unwrap();
for cc in cache_control_values(&req) {
assert_eq!(cc, json!({ "type": "ephemeral" }));
}
}
#[test]
fn disable_prompt_cache_wins_over_ttl() {
let config = cfg("claude-sonnet-4-5")
.with_cache_ttl("1h")
.with_prompt_cache_disabled();
let req = create_request_with_default_options(
&config,
"You are a summarizer.",
&[Message::user().with_text("Summarize.")],
&sample_tools(),
)
.unwrap();
assert!(!req.to_string().contains(CACHE_CONTROL_FIELD));
}
#[test]
fn breakpoints_land_on_the_last_content_block() {
let messages = vec![Message::user()
+87
View File
@@ -30,6 +30,7 @@ pub fn is_goose_internal_request_param(key: &str) -> bool {
key,
"thinking_effort"
| "disable_prompt_cache"
| "cache_ttl"
| "emit_clear_thinking"
| "preserve_thinking_context"
| "preserve_unsigned_thinking"
@@ -346,6 +347,46 @@ impl ModelConfig {
.unwrap_or(false)
}
/// Set the prompt-cache TTL requested from providers that support one
/// (currently the Anthropic message format). Valid values are "5m" and
/// "1h"; absent means the provider default (5m).
pub fn with_cache_ttl(self, ttl: &str) -> Self {
self.with_merged_request_params(HashMap::from([(
"cache_ttl".to_string(),
Value::String(ttl.to_string()),
)]))
}
/// Remove any prompt-cache TTL request parameter. The TTL is
/// configuration state, not session state: callers that resume a
/// persisted config drop the stored value and re-derive it from the
/// current configuration so a clamped run never sticks to the session.
pub fn without_cache_ttl(mut self) -> Self {
if let Some(params) = self.request_params.as_mut() {
params.remove("cache_ttl");
if params.is_empty() {
self.request_params = None;
}
}
self
}
/// Clamp the prompt-cache TTL back to the provider default (5m).
/// Burst-only surfaces (headless runs, subagents, scheduled recipes) call
/// this so a user-level 1h opt-in never pays the 2x cache-write premium on
/// workloads that finish in one burst and cannot idle.
pub fn with_cache_ttl_clamped(self) -> Self {
if self.cache_ttl().is_some_and(|ttl| ttl != "5m") {
self.with_cache_ttl("5m")
} else {
self
}
}
pub fn cache_ttl(&self) -> Option<String> {
self.request_param::<String>("cache_ttl")
}
pub fn request_param<T: for<'de> serde::Deserialize<'de>>(
&self,
request_key: &str,
@@ -361,6 +402,52 @@ impl ModelConfig {
mod tests {
use super::*;
#[test]
fn cache_ttl_round_trips_through_request_params() {
let config = ModelConfig::new("claude-sonnet-4-5").with_cache_ttl("1h");
assert_eq!(config.cache_ttl().as_deref(), Some("1h"));
assert!(ModelConfig::new("claude-sonnet-4-5").cache_ttl().is_none());
}
#[test]
fn cache_ttl_clamp_resets_one_hour_to_default() {
let config = ModelConfig::new("claude-sonnet-4-5")
.with_cache_ttl("1h")
.with_cache_ttl_clamped();
assert_eq!(config.cache_ttl().as_deref(), Some("5m"));
}
#[test]
fn without_cache_ttl_removes_the_param_and_empty_map() {
let config = ModelConfig::new("claude-sonnet-4-5")
.with_cache_ttl("1h")
.without_cache_ttl();
assert!(config.cache_ttl().is_none());
assert!(config.request_params.is_none());
}
#[test]
fn without_cache_ttl_preserves_other_request_params() {
let config = ModelConfig::new("claude-sonnet-4-5")
.with_merged_request_params(HashMap::from([(
"thinking_effort".to_string(),
serde_json::json!("high"),
)]))
.with_cache_ttl("1h")
.without_cache_ttl();
assert!(config.cache_ttl().is_none());
assert_eq!(
config.request_param::<String>("thinking_effort").as_deref(),
Some("high")
);
}
#[test]
fn cache_ttl_clamp_leaves_unset_ttl_absent() {
let config = ModelConfig::new("claude-sonnet-4-5").with_cache_ttl_clamped();
assert!(config.cache_ttl().is_none());
}
#[test]
fn request_headers_never_serialize_into_bodies() {
let config = ModelConfig::new("test-model").with_request_headers(Some(HashMap::from([(
+2 -1
View File
@@ -3567,7 +3567,8 @@ impl Agent {
.ok_or_else(|| anyhow!("Could not configure agent: missing provider"))?;
let mut model_config = match session.model_config.clone() {
Some(saved_config) => saved_config,
Some(saved_config) => crate::model_config::with_rederived_cache_ttl(saved_config)
.map_err(|e| anyhow!("Could not configure agent: {}", e))?,
None => {
let model_name = config
.get_goose_model()
@@ -41,7 +41,7 @@ impl TaskConfig {
) -> Self {
Self {
provider,
model_config,
model_config: model_config.with_cache_ttl_clamped(),
parent_session_id: parent_session_id.to_owned(),
parent_working_dir: parent_working_dir.to_owned(),
extensions,
+104
View File
@@ -68,6 +68,12 @@ fn materialize_model_config_inner(
model = model.with_default_thinking_effort(config.get_goose_thinking_effort());
}
if model.cache_ttl().is_none() {
if let Some(ttl) = get_goose_cache_ttl(config)? {
model = model.with_cache_ttl(&ttl);
}
}
if provider_name == goose_providers::openai::OPEN_AI_PROVIDER_NAME {
model = apply_openai_request_params(model);
}
@@ -227,6 +233,34 @@ fn base_model_config_from_user_config(
Ok(model)
}
/// Re-derive the prompt-cache TTL from the current configuration, discarding
/// any value stored on the model config. The TTL is configuration state, not
/// session state: a resumed session must reflect the user's current opt-in,
/// not a value persisted by an earlier (possibly clamped) run.
pub fn with_rederived_cache_ttl(model: ModelConfig) -> Result<ModelConfig> {
let mut model = model.without_cache_ttl();
if let Some(ttl) = get_goose_cache_ttl(Config::global())? {
model = model.with_cache_ttl(&ttl);
}
Ok(model)
}
fn get_goose_cache_ttl(config: &Config) -> Result<Option<String>> {
match config.get_param::<String>("GOOSE_CACHE_TTL") {
Ok(ttl) => {
let ttl = ttl.trim().to_lowercase();
match ttl.as_str() {
"5m" | "1h" => Ok(Some(ttl)),
other => Err(anyhow!(
"GOOSE_CACHE_TTL must be '5m' or '1h', got '{other}'"
)),
}
}
Err(ConfigError::NotFound(_)) => Ok(None),
Err(e) => Err(e.into()),
}
}
fn get_goose_temperature(config: &Config) -> Result<Option<f32>> {
match config.get_param::<f32>("GOOSE_TEMPERATURE") {
Ok(temp) if temp < 0.0 => Err(anyhow!(
@@ -299,6 +333,76 @@ mod one_shot_tests {
}
}
#[cfg(test)]
mod cache_ttl_tests {
use super::*;
#[test]
fn env_var_populates_cache_ttl() {
let _guard = env_lock::lock_env([("GOOSE_CACHE_TTL", Some("1h"))]);
let model = materialize_model_config_inner(
ModelConfig::new("claude-sonnet-4-5"),
"anthropic",
false,
)
.unwrap();
assert_eq!(model.cache_ttl().as_deref(), Some("1h"));
}
#[test]
fn absent_env_var_leaves_cache_ttl_unset() {
let _guard = env_lock::lock_env([("GOOSE_CACHE_TTL", None::<&str>)]);
let model = materialize_model_config_inner(
ModelConfig::new("claude-sonnet-4-5"),
"anthropic",
false,
)
.unwrap();
assert!(model.cache_ttl().is_none());
}
#[test]
fn invalid_env_var_is_rejected() {
let _guard = env_lock::lock_env([("GOOSE_CACHE_TTL", Some("2h"))]);
let result = materialize_model_config_inner(
ModelConfig::new("claude-sonnet-4-5"),
"anthropic",
false,
);
assert!(result.is_err());
}
#[test]
fn rederive_replaces_stored_ttl_with_configured_value() {
let _guard = env_lock::lock_env([("GOOSE_CACHE_TTL", Some("1h"))]);
let model =
with_rederived_cache_ttl(ModelConfig::new("claude-sonnet-4-5").with_cache_ttl("5m"))
.unwrap();
assert_eq!(model.cache_ttl().as_deref(), Some("1h"));
}
#[test]
fn rederive_drops_stored_ttl_when_config_absent() {
let _guard = env_lock::lock_env([("GOOSE_CACHE_TTL", None::<&str>)]);
let model =
with_rederived_cache_ttl(ModelConfig::new("claude-sonnet-4-5").with_cache_ttl("1h"))
.unwrap();
assert!(model.cache_ttl().is_none());
}
#[test]
fn explicit_model_ttl_wins_over_env_var() {
let _guard = env_lock::lock_env([("GOOSE_CACHE_TTL", Some("1h"))]);
let model = materialize_model_config_inner(
ModelConfig::new("claude-sonnet-4-5").with_cache_ttl("5m"),
"anthropic",
false,
)
.unwrap();
assert_eq!(model.cache_ttl().as_deref(), Some("5m"));
}
}
#[cfg(test)]
mod azure_foundry_tests {
use super::*;
+2 -1
View File
@@ -1035,7 +1035,8 @@ async fn execute_job(
let provider_name = config.get_goose_provider()?;
let model_name = config.get_goose_model()?;
let model_config =
crate::model_config::model_config_from_user_config(&provider_name, &model_name)?;
crate::model_config::model_config_from_user_config(&provider_name, &model_name)?
.with_cache_ttl_clamped();
let session = agent
.config
@@ -48,6 +48,7 @@ The following settings can be configured at the root level of your config.yaml f
|---------|---------|---------|---------|-----------|
| `GOOSE_TEMPERATURE` | Model response randomness | Float between 0.0 and 1.0 | Model-specific | No |
| `GOOSE_MAX_TOKENS` | Maximum number of tokens for each model response (truncates longer responses) | Positive integer | Model-specific | No |
| `GOOSE_CACHE_TTL` | Anthropic prompt-cache TTL; `1h` keeps the cached prefix alive across idle gaps at a higher cache-write rate. Headless runs always use `5m` | "5m", "1h" | "5m" | No |
| `GOOSE_MODE` | [Tool execution behavior](/docs/guides/managing-tools/goose-permissions) | "auto", "approve", "chat", "smart_approve" | "auto" | No |
| `GOOSE_MAX_TURNS` | [Maximum number of turns](/docs/guides/sessions/smart-context-management#maximum-turns) allowed without user input | Integer (e.g., 10, 50, 100) | 1000 | No |
| `GOOSE_PLANNER_PROVIDER` | Provider for [planning mode](/docs/guides/context-engineering/creating-plans) | Same as `GOOSE_PROVIDER` options | Falls back to `GOOSE_PROVIDER` | No |
@@ -21,6 +21,7 @@ These are the minimum required variables to get started with goose.
| `GOOSE_FAST_MODEL` | Overrides the provider's default fast model used for auxiliary calls (tool-selection, classification, session titles) | Model name (e.g., "gpt-4o-mini", "google/gemini-2.5-flash") | Provider-specific default |
| `GOOSE_TEMPERATURE` | Sets the [temperature](https://medium.com/@kelseyywang/a-comprehensive-guide-to-llm-temperature-%EF%B8%8F-363a40bbc91f) for model responses | Float between 0.0 and 1.0 | Model-specific default |
| `GOOSE_MAX_TOKENS` | Sets the maximum number of tokens for each model response (truncates longer responses) | Positive integer (e.g., 4096, 8192) | Model-specific default |
| `GOOSE_CACHE_TTL` | Sets the Anthropic prompt-cache TTL. `1h` keeps the cached prefix alive across idle gaps (e.g. stepping away mid-session) but bills cache writes at 2x input instead of 1.25x, so it only pays off for sessions that actually idle. Headless runs (`goose run`, subagents, scheduled recipes) always use `5m` | `5m`, `1h` | `5m` |
**Examples**