From fb15d4eade4ab3947b85e20852a18de269c789c9 Mon Sep 17 00:00:00 2001 From: Michael Neale Date: Mon, 31 Aug 2026 00:43:58 +0000 Subject: [PATCH] feat: configurable Anthropic prompt-cache TTL (5m/1h) (#11576) Signed-off-by: Michael Neale Co-authored-by: Galadriel --- crates/goose-cli/src/session/builder.rs | 95 +++++++++++- .../src/formats/anthropic.rs | 143 ++++++++++++++++-- crates/goose-provider-types/src/model.rs | 87 +++++++++++ crates/goose/src/agents/agent.rs | 3 +- .../goose/src/agents/subagent_task_config.rs | 2 +- crates/goose/src/model_config.rs | 104 +++++++++++++ crates/goose/src/scheduler.rs | 3 +- documentation/docs/guides/config-files.md | 1 + .../docs/guides/environment-variables.md | 1 + 9 files changed, 425 insertions(+), 14 deletions(-) diff --git a/crates/goose-cli/src/session/builder.rs b/crates/goose-cli/src/session/builder.rs index 60ce54b52..888453677 100644 --- a/crates/goose-cli/src/session/builder.rs +++ b/crates/goose-cli/src/session/builder.rs @@ -411,7 +411,7 @@ async fn resolve_provider_and_model( process::exit(1); }); - let model_config = if session_config.resume + let mut model_config = if session_config.resume && saved_provider_matches && saved_model_config .as_ref() @@ -419,6 +419,10 @@ async fn resolve_provider_and_model( { let mut config = saved_model_config.unwrap(); config.normalize_effort_suffix(); + config = goose::model_config::with_rederived_cache_ttl(config).unwrap_or_else(|e| { + output::render_error(&format!("Invalid cache TTL configuration: {}", e)); + process::exit(1); + }); if let Some(temp) = recipe_settings.and_then(|s| s.temperature) { config = config.with_temperature(Some(temp)); } @@ -436,6 +440,10 @@ async fn resolve_provider_and_model( config }; + if !session_config.interactive { + model_config = model_config.with_cache_ttl_clamped(); + } + ResolvedProviderConfig { provider_name, model_name, @@ -730,7 +738,7 @@ pub async fn build_session(session_config: SessionBuilderConfig) -> CliSession { )) .yellow() ); - let fallback_model_config = + let mut fallback_model_config = model_config_from_user_config(fallback_provider.as_str(), &fallback_model) .unwrap_or_else(|e| { output::render_error(&format!( @@ -739,6 +747,9 @@ pub async fn build_session(session_config: SessionBuilderConfig) -> CliSession { )); process::exit(1); }); + if !session_config.interactive { + fallback_model_config = fallback_model_config.with_cache_ttl_clamped(); + } match create(&fallback_provider, extensions_for_provider.clone()).await { Ok(provider) => ( provider, @@ -1433,6 +1444,86 @@ mod tests { .is_some_and(|params| params.contains_key("anthropic_beta"))); } + #[tokio::test] + async fn resume_rederives_cache_ttl_from_config_not_session() { + let _guard = env_lock::lock_env([ + ("GOOSE_PROVIDER", None::<&str>), + ("GOOSE_MODEL", None::<&str>), + ("GOOSE_CACHE_TTL", Some("1h")), + ]); + let temp_dir = TempDir::new().unwrap(); + let config = test_config(&temp_dir); + let saved = + goose_providers::model::ModelConfig::new("claude-sonnet-4-6").with_cache_ttl("5m"); + + let resolved = resolve_provider_and_model( + &SessionBuilderConfig { + resume: true, + interactive: true, + ..SessionBuilderConfig::default() + }, + &config, + Some("anthropic".to_string()), + Some(saved), + ) + .await; + + assert_eq!(resolved.model_config.cache_ttl().as_deref(), Some("1h")); + } + + #[tokio::test] + async fn resume_drops_saved_cache_ttl_when_config_absent() { + let _guard = env_lock::lock_env([ + ("GOOSE_PROVIDER", None::<&str>), + ("GOOSE_MODEL", None::<&str>), + ("GOOSE_CACHE_TTL", None::<&str>), + ]); + let temp_dir = TempDir::new().unwrap(); + let config = test_config(&temp_dir); + let saved = + goose_providers::model::ModelConfig::new("claude-sonnet-4-6").with_cache_ttl("1h"); + + let resolved = resolve_provider_and_model( + &SessionBuilderConfig { + resume: true, + interactive: true, + ..SessionBuilderConfig::default() + }, + &config, + Some("anthropic".to_string()), + Some(saved), + ) + .await; + + assert!(resolved.model_config.cache_ttl().is_none()); + } + + #[tokio::test] + async fn headless_resume_clamps_rederived_cache_ttl() { + let _guard = env_lock::lock_env([ + ("GOOSE_PROVIDER", None::<&str>), + ("GOOSE_MODEL", None::<&str>), + ("GOOSE_CACHE_TTL", Some("1h")), + ]); + let temp_dir = TempDir::new().unwrap(); + let config = test_config(&temp_dir); + let saved = goose_providers::model::ModelConfig::new("claude-sonnet-4-6"); + + let resolved = resolve_provider_and_model( + &SessionBuilderConfig { + resume: true, + interactive: false, + ..SessionBuilderConfig::default() + }, + &config, + Some("anthropic".to_string()), + Some(saved), + ) + .await; + + assert_eq!(resolved.model_config.cache_ttl().as_deref(), Some("5m")); + } + #[test] fn resumed_provider_override_rejects_context_owning_provider() { let error = validate_provider_override_context( diff --git a/crates/goose-provider-types/src/formats/anthropic.rs b/crates/goose-provider-types/src/formats/anthropic.rs index aec33e4d1..dc030fd99 100644 --- a/crates/goose-provider-types/src/formats/anthropic.rs +++ b/crates/goose-provider-types/src/formats/anthropic.rs @@ -45,6 +45,7 @@ macro_rules! string_enum { } string_enum!(ThinkingType { Adaptive => "adaptive", Enabled => "enabled", Disabled => "disabled" }); +string_enum!(CacheTtl { FiveMinutes => "5m", OneHour => "1h" }); #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct AnthropicFormatOptions { @@ -54,6 +55,7 @@ pub struct AnthropicFormatOptions { pub emit_clear_thinking: bool, pub current_model: Option, pub prompt_cache_disabled: bool, + pub cache_ttl: Option, } impl AnthropicFormatOptions { @@ -70,6 +72,10 @@ impl AnthropicFormatOptions { let emit_clear_thinking = model_config .request_param::("emit_clear_thinking") .unwrap_or(self.emit_clear_thinking); + let cache_ttl = model_config + .cache_ttl() + .and_then(|ttl| ttl.parse::().ok()) + .or(self.cache_ttl); Self { preserve_unsigned_thinking, @@ -80,6 +86,19 @@ impl AnthropicFormatOptions { .current_model .or_else(|| Some(model_config.model_name.clone())), prompt_cache_disabled: model_config.prompt_cache_disabled(), + cache_ttl, + } + } + + /// `{"type":"ephemeral"}` selects Anthropic's default 5m TTL; the `ttl` + /// field is only sent for an explicit 1h opt-in, since a 1h write is + /// billed at 2x input instead of 1.25x. + fn cache_control(&self) -> Value { + match self.cache_ttl { + Some(CacheTtl::OneHour) => { + json!({ TYPE_FIELD: "ephemeral", "ttl": "1h" }) + } + _ => json!({ TYPE_FIELD: "ephemeral" }), } } } @@ -446,10 +465,7 @@ fn format_messages_with_options( .and_then(|content_array| content_array.last_mut()) .and_then(|b| b.as_object_mut()) { - block.insert( - CACHE_CONTROL_FIELD.to_string(), - json!({ TYPE_FIELD: "ephemeral" }), - ); + block.insert(CACHE_CONTROL_FIELD.to_string(), options.cache_control()); user_count += 1; if user_count >= 2 { break; @@ -491,10 +507,10 @@ pub fn format_tools(tools: &[Tool], options: &AnthropicFormatOptions) -> Vec Value { json!([{ TYPE_FIELD: TEXT_TYPE, TEXT_TYPE: system, - CACHE_CONTROL_FIELD: { TYPE_FIELD: "ephemeral" } + CACHE_CONTROL_FIELD: options.cache_control() }]) } @@ -2777,6 +2793,115 @@ mod tests { assert!(!req.to_string().contains(CACHE_CONTROL_FIELD)); } + fn cache_control_values(req: &Value) -> Vec { + let mut found = Vec::new(); + let tools = req["tools"].as_array().unwrap(); + let system = req["system"].as_array().unwrap(); + let messages = req["messages"].as_array().unwrap(); + for block in tools + .iter() + .chain(system.iter()) + .chain(messages.iter().flat_map(|m| { + m["content"] + .as_array() + .map(|c| c.iter()) + .unwrap_or_default() + })) + { + if let Some(cc) = block.get(CACHE_CONTROL_FIELD) { + found.push(cc.clone()); + } + } + found + } + + #[test] + fn default_breakpoints_omit_ttl() { + let req = create_request_with_default_options( + &cfg("claude-sonnet-4-5"), + "You are a careful coding assistant.", + &[Message::user().with_text("Hello")], + &sample_tools(), + ) + .unwrap(); + + let values = cache_control_values(&req); + assert_eq!(values.len(), 3); + for cc in values { + assert_eq!(cc, json!({ "type": "ephemeral" })); + } + } + + #[test] + fn one_hour_ttl_stamps_every_breakpoint() { + let config = cfg("claude-sonnet-4-5").with_cache_ttl("1h"); + let req = create_request_with_default_options( + &config, + "You are a careful coding assistant.", + &[ + Message::user().with_text("Hello"), + Message::assistant().with_text("Hi."), + Message::user().with_text("Continue"), + ], + &sample_tools(), + ) + .unwrap(); + + let values = cache_control_values(&req); + assert_eq!(values.len(), 4); + for cc in values { + assert_eq!(cc, json!({ "type": "ephemeral", "ttl": "1h" })); + } + } + + #[test] + fn explicit_five_minute_ttl_matches_default_wire_format() { + let config = cfg("claude-sonnet-4-5").with_cache_ttl("5m"); + let req = create_request_with_default_options( + &config, + "You are a careful coding assistant.", + &[Message::user().with_text("Hello")], + &sample_tools(), + ) + .unwrap(); + + for cc in cache_control_values(&req) { + assert_eq!(cc, json!({ "type": "ephemeral" })); + } + } + + #[test] + fn unrecognized_ttl_value_falls_back_to_default() { + let config = cfg("claude-sonnet-4-5").with_cache_ttl("2h"); + let req = create_request_with_default_options( + &config, + "You are a careful coding assistant.", + &[Message::user().with_text("Hello")], + &sample_tools(), + ) + .unwrap(); + + for cc in cache_control_values(&req) { + assert_eq!(cc, json!({ "type": "ephemeral" })); + } + } + + #[test] + fn disable_prompt_cache_wins_over_ttl() { + let config = cfg("claude-sonnet-4-5") + .with_cache_ttl("1h") + .with_prompt_cache_disabled(); + let req = create_request_with_default_options( + &config, + "You are a summarizer.", + &[Message::user().with_text("Summarize.")], + &sample_tools(), + ) + .unwrap(); + + assert!(!req.to_string().contains(CACHE_CONTROL_FIELD)); + } + #[test] fn breakpoints_land_on_the_last_content_block() { let messages = vec![Message::user() diff --git a/crates/goose-provider-types/src/model.rs b/crates/goose-provider-types/src/model.rs index 00730a289..e47a366e2 100644 --- a/crates/goose-provider-types/src/model.rs +++ b/crates/goose-provider-types/src/model.rs @@ -30,6 +30,7 @@ pub fn is_goose_internal_request_param(key: &str) -> bool { key, "thinking_effort" | "disable_prompt_cache" + | "cache_ttl" | "emit_clear_thinking" | "preserve_thinking_context" | "preserve_unsigned_thinking" @@ -346,6 +347,46 @@ impl ModelConfig { .unwrap_or(false) } + /// Set the prompt-cache TTL requested from providers that support one + /// (currently the Anthropic message format). Valid values are "5m" and + /// "1h"; absent means the provider default (5m). + pub fn with_cache_ttl(self, ttl: &str) -> Self { + self.with_merged_request_params(HashMap::from([( + "cache_ttl".to_string(), + Value::String(ttl.to_string()), + )])) + } + + /// Remove any prompt-cache TTL request parameter. The TTL is + /// configuration state, not session state: callers that resume a + /// persisted config drop the stored value and re-derive it from the + /// current configuration so a clamped run never sticks to the session. + pub fn without_cache_ttl(mut self) -> Self { + if let Some(params) = self.request_params.as_mut() { + params.remove("cache_ttl"); + if params.is_empty() { + self.request_params = None; + } + } + self + } + + /// Clamp the prompt-cache TTL back to the provider default (5m). + /// Burst-only surfaces (headless runs, subagents, scheduled recipes) call + /// this so a user-level 1h opt-in never pays the 2x cache-write premium on + /// workloads that finish in one burst and cannot idle. + pub fn with_cache_ttl_clamped(self) -> Self { + if self.cache_ttl().is_some_and(|ttl| ttl != "5m") { + self.with_cache_ttl("5m") + } else { + self + } + } + + pub fn cache_ttl(&self) -> Option { + self.request_param::("cache_ttl") + } + pub fn request_param serde::Deserialize<'de>>( &self, request_key: &str, @@ -361,6 +402,52 @@ impl ModelConfig { mod tests { use super::*; + #[test] + fn cache_ttl_round_trips_through_request_params() { + let config = ModelConfig::new("claude-sonnet-4-5").with_cache_ttl("1h"); + assert_eq!(config.cache_ttl().as_deref(), Some("1h")); + assert!(ModelConfig::new("claude-sonnet-4-5").cache_ttl().is_none()); + } + + #[test] + fn cache_ttl_clamp_resets_one_hour_to_default() { + let config = ModelConfig::new("claude-sonnet-4-5") + .with_cache_ttl("1h") + .with_cache_ttl_clamped(); + assert_eq!(config.cache_ttl().as_deref(), Some("5m")); + } + + #[test] + fn without_cache_ttl_removes_the_param_and_empty_map() { + let config = ModelConfig::new("claude-sonnet-4-5") + .with_cache_ttl("1h") + .without_cache_ttl(); + assert!(config.cache_ttl().is_none()); + assert!(config.request_params.is_none()); + } + + #[test] + fn without_cache_ttl_preserves_other_request_params() { + let config = ModelConfig::new("claude-sonnet-4-5") + .with_merged_request_params(HashMap::from([( + "thinking_effort".to_string(), + serde_json::json!("high"), + )])) + .with_cache_ttl("1h") + .without_cache_ttl(); + assert!(config.cache_ttl().is_none()); + assert_eq!( + config.request_param::("thinking_effort").as_deref(), + Some("high") + ); + } + + #[test] + fn cache_ttl_clamp_leaves_unset_ttl_absent() { + let config = ModelConfig::new("claude-sonnet-4-5").with_cache_ttl_clamped(); + assert!(config.cache_ttl().is_none()); + } + #[test] fn request_headers_never_serialize_into_bodies() { let config = ModelConfig::new("test-model").with_request_headers(Some(HashMap::from([( diff --git a/crates/goose/src/agents/agent.rs b/crates/goose/src/agents/agent.rs index 467c815c5..80aee5079 100644 --- a/crates/goose/src/agents/agent.rs +++ b/crates/goose/src/agents/agent.rs @@ -3567,7 +3567,8 @@ impl Agent { .ok_or_else(|| anyhow!("Could not configure agent: missing provider"))?; let mut model_config = match session.model_config.clone() { - Some(saved_config) => saved_config, + Some(saved_config) => crate::model_config::with_rederived_cache_ttl(saved_config) + .map_err(|e| anyhow!("Could not configure agent: {}", e))?, None => { let model_name = config .get_goose_model() diff --git a/crates/goose/src/agents/subagent_task_config.rs b/crates/goose/src/agents/subagent_task_config.rs index 33e99ab01..f6da81a50 100644 --- a/crates/goose/src/agents/subagent_task_config.rs +++ b/crates/goose/src/agents/subagent_task_config.rs @@ -41,7 +41,7 @@ impl TaskConfig { ) -> Self { Self { provider, - model_config, + model_config: model_config.with_cache_ttl_clamped(), parent_session_id: parent_session_id.to_owned(), parent_working_dir: parent_working_dir.to_owned(), extensions, diff --git a/crates/goose/src/model_config.rs b/crates/goose/src/model_config.rs index ed5461605..95fd1f22f 100644 --- a/crates/goose/src/model_config.rs +++ b/crates/goose/src/model_config.rs @@ -68,6 +68,12 @@ fn materialize_model_config_inner( model = model.with_default_thinking_effort(config.get_goose_thinking_effort()); } + if model.cache_ttl().is_none() { + if let Some(ttl) = get_goose_cache_ttl(config)? { + model = model.with_cache_ttl(&ttl); + } + } + if provider_name == goose_providers::openai::OPEN_AI_PROVIDER_NAME { model = apply_openai_request_params(model); } @@ -227,6 +233,34 @@ fn base_model_config_from_user_config( Ok(model) } +/// Re-derive the prompt-cache TTL from the current configuration, discarding +/// any value stored on the model config. The TTL is configuration state, not +/// session state: a resumed session must reflect the user's current opt-in, +/// not a value persisted by an earlier (possibly clamped) run. +pub fn with_rederived_cache_ttl(model: ModelConfig) -> Result { + let mut model = model.without_cache_ttl(); + if let Some(ttl) = get_goose_cache_ttl(Config::global())? { + model = model.with_cache_ttl(&ttl); + } + Ok(model) +} + +fn get_goose_cache_ttl(config: &Config) -> Result> { + match config.get_param::("GOOSE_CACHE_TTL") { + Ok(ttl) => { + let ttl = ttl.trim().to_lowercase(); + match ttl.as_str() { + "5m" | "1h" => Ok(Some(ttl)), + other => Err(anyhow!( + "GOOSE_CACHE_TTL must be '5m' or '1h', got '{other}'" + )), + } + } + Err(ConfigError::NotFound(_)) => Ok(None), + Err(e) => Err(e.into()), + } +} + fn get_goose_temperature(config: &Config) -> Result> { match config.get_param::("GOOSE_TEMPERATURE") { Ok(temp) if temp < 0.0 => Err(anyhow!( @@ -299,6 +333,76 @@ mod one_shot_tests { } } +#[cfg(test)] +mod cache_ttl_tests { + use super::*; + + #[test] + fn env_var_populates_cache_ttl() { + let _guard = env_lock::lock_env([("GOOSE_CACHE_TTL", Some("1h"))]); + let model = materialize_model_config_inner( + ModelConfig::new("claude-sonnet-4-5"), + "anthropic", + false, + ) + .unwrap(); + assert_eq!(model.cache_ttl().as_deref(), Some("1h")); + } + + #[test] + fn absent_env_var_leaves_cache_ttl_unset() { + let _guard = env_lock::lock_env([("GOOSE_CACHE_TTL", None::<&str>)]); + let model = materialize_model_config_inner( + ModelConfig::new("claude-sonnet-4-5"), + "anthropic", + false, + ) + .unwrap(); + assert!(model.cache_ttl().is_none()); + } + + #[test] + fn invalid_env_var_is_rejected() { + let _guard = env_lock::lock_env([("GOOSE_CACHE_TTL", Some("2h"))]); + let result = materialize_model_config_inner( + ModelConfig::new("claude-sonnet-4-5"), + "anthropic", + false, + ); + assert!(result.is_err()); + } + + #[test] + fn rederive_replaces_stored_ttl_with_configured_value() { + let _guard = env_lock::lock_env([("GOOSE_CACHE_TTL", Some("1h"))]); + let model = + with_rederived_cache_ttl(ModelConfig::new("claude-sonnet-4-5").with_cache_ttl("5m")) + .unwrap(); + assert_eq!(model.cache_ttl().as_deref(), Some("1h")); + } + + #[test] + fn rederive_drops_stored_ttl_when_config_absent() { + let _guard = env_lock::lock_env([("GOOSE_CACHE_TTL", None::<&str>)]); + let model = + with_rederived_cache_ttl(ModelConfig::new("claude-sonnet-4-5").with_cache_ttl("1h")) + .unwrap(); + assert!(model.cache_ttl().is_none()); + } + + #[test] + fn explicit_model_ttl_wins_over_env_var() { + let _guard = env_lock::lock_env([("GOOSE_CACHE_TTL", Some("1h"))]); + let model = materialize_model_config_inner( + ModelConfig::new("claude-sonnet-4-5").with_cache_ttl("5m"), + "anthropic", + false, + ) + .unwrap(); + assert_eq!(model.cache_ttl().as_deref(), Some("5m")); + } +} + #[cfg(test)] mod azure_foundry_tests { use super::*; diff --git a/crates/goose/src/scheduler.rs b/crates/goose/src/scheduler.rs index ed9dad2f3..8ba410262 100644 --- a/crates/goose/src/scheduler.rs +++ b/crates/goose/src/scheduler.rs @@ -1035,7 +1035,8 @@ async fn execute_job( let provider_name = config.get_goose_provider()?; let model_name = config.get_goose_model()?; let model_config = - crate::model_config::model_config_from_user_config(&provider_name, &model_name)?; + crate::model_config::model_config_from_user_config(&provider_name, &model_name)? + .with_cache_ttl_clamped(); let session = agent .config diff --git a/documentation/docs/guides/config-files.md b/documentation/docs/guides/config-files.md index db4f6e143..f555669a7 100644 --- a/documentation/docs/guides/config-files.md +++ b/documentation/docs/guides/config-files.md @@ -48,6 +48,7 @@ The following settings can be configured at the root level of your config.yaml f |---------|---------|---------|---------|-----------| | `GOOSE_TEMPERATURE` | Model response randomness | Float between 0.0 and 1.0 | Model-specific | No | | `GOOSE_MAX_TOKENS` | Maximum number of tokens for each model response (truncates longer responses) | Positive integer | Model-specific | No | +| `GOOSE_CACHE_TTL` | Anthropic prompt-cache TTL; `1h` keeps the cached prefix alive across idle gaps at a higher cache-write rate. Headless runs always use `5m` | "5m", "1h" | "5m" | No | | `GOOSE_MODE` | [Tool execution behavior](/docs/guides/managing-tools/goose-permissions) | "auto", "approve", "chat", "smart_approve" | "auto" | No | | `GOOSE_MAX_TURNS` | [Maximum number of turns](/docs/guides/sessions/smart-context-management#maximum-turns) allowed without user input | Integer (e.g., 10, 50, 100) | 1000 | No | | `GOOSE_PLANNER_PROVIDER` | Provider for [planning mode](/docs/guides/context-engineering/creating-plans) | Same as `GOOSE_PROVIDER` options | Falls back to `GOOSE_PROVIDER` | No | diff --git a/documentation/docs/guides/environment-variables.md b/documentation/docs/guides/environment-variables.md index c2735f71f..7f56cbd4f 100644 --- a/documentation/docs/guides/environment-variables.md +++ b/documentation/docs/guides/environment-variables.md @@ -21,6 +21,7 @@ These are the minimum required variables to get started with goose. | `GOOSE_FAST_MODEL` | Overrides the provider's default fast model used for auxiliary calls (tool-selection, classification, session titles) | Model name (e.g., "gpt-4o-mini", "google/gemini-2.5-flash") | Provider-specific default | | `GOOSE_TEMPERATURE` | Sets the [temperature](https://medium.com/@kelseyywang/a-comprehensive-guide-to-llm-temperature-%EF%B8%8F-363a40bbc91f) for model responses | Float between 0.0 and 1.0 | Model-specific default | | `GOOSE_MAX_TOKENS` | Sets the maximum number of tokens for each model response (truncates longer responses) | Positive integer (e.g., 4096, 8192) | Model-specific default | +| `GOOSE_CACHE_TTL` | Sets the Anthropic prompt-cache TTL. `1h` keeps the cached prefix alive across idle gaps (e.g. stepping away mid-session) but bills cache writes at 2x input instead of 1.25x, so it only pays off for sessions that actually idle. Headless runs (`goose run`, subagents, scheduled recipes) always use `5m` | `5m`, `1h` | `5m` | **Examples**