fix(bedrock): cache trailing message for stable prefix across agent turns (#8916)
Signed-off-by: Carl Youngblood <carl.youngblood@auctane.com> Signed-off-by: Douwe Osinga <douwe@squareup.com> Co-authored-by: Douwe Osinga <douwe@squareup.com>
This commit is contained in:
@@ -232,15 +232,7 @@ impl BedrockProvider {
|
|||||||
let visible_messages: Vec<&Message> =
|
let visible_messages: Vec<&Message> =
|
||||||
messages.iter().filter(|m| m.is_agent_visible()).collect();
|
messages.iter().filter(|m| m.is_agent_visible()).collect();
|
||||||
|
|
||||||
// Cache the earliest messages (not most recent) because prompt caching
|
let last_idx = visible_messages.len().saturating_sub(1);
|
||||||
// requires exact prefix matching — caching recent messages would shift
|
|
||||||
// positions each turn, causing misses.
|
|
||||||
const MESSAGE_CACHE_BUDGET: usize = 3;
|
|
||||||
let cache_count = if enable_caching {
|
|
||||||
visible_messages.len().min(MESSAGE_CACHE_BUDGET)
|
|
||||||
} else {
|
|
||||||
0
|
|
||||||
};
|
|
||||||
|
|
||||||
let mut request = self
|
let mut request = self
|
||||||
.client
|
.client
|
||||||
@@ -251,7 +243,9 @@ impl BedrockProvider {
|
|||||||
visible_messages
|
visible_messages
|
||||||
.iter()
|
.iter()
|
||||||
.enumerate()
|
.enumerate()
|
||||||
.map(|(idx, m)| to_bedrock_message_with_caching(m, idx < cache_count))
|
.map(|(idx, m)| {
|
||||||
|
to_bedrock_message_with_caching(m, enable_caching && idx == last_idx)
|
||||||
|
})
|
||||||
.collect::<Result<_>>()?,
|
.collect::<Result<_>>()?,
|
||||||
));
|
));
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user