fix(agents): auto-compact at tool boundaries in the unrolled agent loop (#12444)
Signed-off-by: Filip Kujawa <filip.j.kujawa@gmail.com>
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
//! Compacts conversation history when it is too large for the configured context window.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::{anyhow, Result};
|
||||
@@ -7,13 +8,16 @@ use async_trait::async_trait;
|
||||
use tracing_futures::Instrument;
|
||||
|
||||
use crate::agents::state_machine::ops_llm::{chat_span, record_chat_usage};
|
||||
use crate::agents::state_machine::ops_recipe::RecipeOperation;
|
||||
use crate::agents::state_machine::{
|
||||
applied, last_effective_role, messages_since_kickoff, not_applicable, trailing_error, yielded,
|
||||
yielded_with, ConversationEffect, Emitter, GooseEffect, Operation, OperationResult,
|
||||
SlashCommand,
|
||||
};
|
||||
use crate::context_mgmt::compact_messages;
|
||||
use crate::conversation::message::{Message, MessageErrorKind, SystemNotificationType};
|
||||
use crate::context_mgmt::{compact_messages, count_context_tokens};
|
||||
use crate::conversation::message::{
|
||||
Message, MessageContent, MessageErrorKind, SystemNotificationType,
|
||||
};
|
||||
use crate::conversation::{Conversation, EffectiveRole};
|
||||
use crate::providers::base::Provider;
|
||||
use crate::session::Session;
|
||||
@@ -44,6 +48,36 @@ fn compaction_part(
|
||||
))
|
||||
}
|
||||
|
||||
/// Several operations answer parts of one tool batch in separate messages, so a
|
||||
/// tool tail alone does not mean the batch is complete.
|
||||
fn awaits_tool_responses(messages: &[Message]) -> bool {
|
||||
let answered: HashSet<&str> = messages
|
||||
.iter()
|
||||
.flat_map(Message::get_tool_response_ids)
|
||||
.collect();
|
||||
messages
|
||||
.iter()
|
||||
.flat_map(|message| &message.content)
|
||||
.filter_map(MessageContent::as_tool_request)
|
||||
.any(|request| {
|
||||
!request.was_executed_externally() && !answered.contains(request.id.as_str())
|
||||
})
|
||||
}
|
||||
|
||||
/// Reported usage stops at the inference that requested the tools, so the
|
||||
/// results that answered it are not counted until the next request.
|
||||
async fn unreported_tool_tokens(conversation: &Conversation) -> Result<i32> {
|
||||
let messages = conversation.messages();
|
||||
if last_effective_role(messages)? != EffectiveRole::Tool {
|
||||
return Ok(0);
|
||||
}
|
||||
let after_request = messages
|
||||
.iter()
|
||||
.rposition(Message::is_tool_call)
|
||||
.map_or(0, |index| index + 1);
|
||||
count_context_tokens(&messages[after_request..]).await
|
||||
}
|
||||
|
||||
pub struct CompactionOperation {
|
||||
provider: Arc<dyn Provider>,
|
||||
model_config: ModelConfig,
|
||||
@@ -78,8 +112,8 @@ impl CompactionOperation {
|
||||
|
||||
async fn context_tokens(&self, session: &Session, conversation: &Conversation) -> Result<i32> {
|
||||
match session.usage.total_tokens {
|
||||
Some(tokens) => Ok(tokens),
|
||||
None => crate::context_mgmt::count_context_tokens(conversation).await,
|
||||
Some(tokens) => Ok(tokens + unreported_tool_tokens(conversation).await?),
|
||||
None => count_context_tokens(conversation.messages()).await,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -244,7 +278,15 @@ impl Operation<Session, GooseEffect> for CompactionOperation {
|
||||
return not_applicable();
|
||||
}
|
||||
} else {
|
||||
if last_effective_role(messages)? != EffectiveRole::User {
|
||||
// Compact only ahead of an inference. An assistant tail ends the turn or
|
||||
// awaits tool responses, hiding an unanswered request orphans its result,
|
||||
// and RecipeOperation delivers a successful final output from a tool tail.
|
||||
let tail = last_effective_role(messages)?;
|
||||
if tail == EffectiveRole::Assistant
|
||||
|| awaits_tool_responses(messages)
|
||||
|| (tail == EffectiveRole::Tool
|
||||
&& RecipeOperation::successful_final_output(messages).is_some())
|
||||
{
|
||||
return not_applicable();
|
||||
}
|
||||
let tokens = self.context_tokens(session, conversation).await?;
|
||||
|
||||
@@ -105,7 +105,7 @@ impl RecipeOperation {
|
||||
})
|
||||
}
|
||||
|
||||
fn successful_final_output(messages: &[Message]) -> Option<String> {
|
||||
pub(super) fn successful_final_output(messages: &[Message]) -> Option<String> {
|
||||
let answered_responses: HashSet<&str> = messages
|
||||
.iter()
|
||||
.flat_map(|message| &message.content)
|
||||
|
||||
@@ -79,6 +79,6 @@ pub(super) async fn record(
|
||||
}
|
||||
|
||||
pub(super) async fn estimate_context(conversation: &Conversation) -> Result<TokenUsage> {
|
||||
let tokens = crate::context_mgmt::count_context_tokens(conversation).await?;
|
||||
let tokens = crate::context_mgmt::count_context_tokens(conversation.messages()).await?;
|
||||
Ok(TokenUsage::new(Some(tokens), None, Some(tokens)))
|
||||
}
|
||||
|
||||
@@ -191,7 +191,7 @@ pub async fn compact_messages(
|
||||
}
|
||||
|
||||
let conversation = Conversation::new_unvalidated(final_messages);
|
||||
let retained_context_tokens = match count_context_tokens(&conversation).await {
|
||||
let retained_context_tokens = match count_context_tokens(conversation.messages()).await {
|
||||
Ok(tokens) => tokens,
|
||||
Err(error) => {
|
||||
warn!("Failed to count retained context tokens, using billable output tokens: {error}");
|
||||
@@ -206,14 +206,13 @@ pub async fn compact_messages(
|
||||
})
|
||||
}
|
||||
|
||||
/// Estimate the tokens of the agent-visible conversation, counted the same way
|
||||
/// Estimate the tokens of the agent-visible messages, counted the same way
|
||||
/// as the fallback estimation in `check_if_compaction_needed`.
|
||||
pub(crate) async fn count_context_tokens(conversation: &Conversation) -> Result<i32> {
|
||||
pub(crate) async fn count_context_tokens(messages: &[Message]) -> Result<i32> {
|
||||
let counter = create_token_counter()
|
||||
.await
|
||||
.map_err(|error| anyhow::anyhow!("Failed to create token counter: {error}"))?;
|
||||
let total: usize = conversation
|
||||
.messages()
|
||||
let total: usize = messages
|
||||
.iter()
|
||||
.filter(|message| message.is_agent_visible())
|
||||
.map(|message| counter.count_chat_tokens("", std::slice::from_ref(message), &[]))
|
||||
|
||||
Reference in New Issue
Block a user