fix(agents): keep streamed thinking ahead of text and tool calls (#11837)

This commit is contained in:
filip
2026-09-14 17:30:56 +00:00
committed by GitHub
parent 5183da9d99
commit ffda0a4b7d
4 changed files with 126 additions and 199 deletions
+11 -38
View File
@@ -75,7 +75,7 @@ fn is_thinking(content: &MessageContent) -> bool {
)
}
fn normalize_tool_call_thinking(accumulator: &mut Conversation, chunk: &mut Message) {
fn drop_repeated_tool_call_thinking(accumulator: &Conversation, chunk: &mut Message) {
if !chunk
.content
.iter()
@@ -83,42 +83,15 @@ fn normalize_tool_call_thinking(accumulator: &mut Conversation, chunk: &mut Mess
{
return;
}
let has_direct_thinking = chunk.content.iter().any(is_thinking);
let mut prior_thinking = Vec::new();
for message in accumulator.messages_mut() {
if message.role != chunk.role
|| message
.content
.iter()
.any(|content| matches!(content, MessageContent::ToolRequest(_)))
{
continue;
}
prior_thinking.extend(
message
.content
.iter()
.filter(|content| is_thinking(content))
.cloned(),
);
message.content.retain(|content| !is_thinking(content));
}
accumulator
.messages_mut()
.retain(|message| !message.content.is_empty());
if !has_direct_thinking && !prior_thinking.is_empty() {
if let Some(tool_request) = chunk
.content
.iter()
.position(|content| matches!(content, MessageContent::ToolRequest(_)))
{
chunk
.content
.splice(tool_request..tool_request, prior_thinking);
}
}
let prior: Vec<&MessageContent> = accumulator
.iter()
.filter(|message| message.role == chunk.role)
.flat_map(|message| message.content.iter())
.filter(|content| is_thinking(content))
.collect();
chunk
.content
.retain(|content| !(is_thinking(content) && prior.contains(&content)));
}
pub fn chat_span(
@@ -479,7 +452,7 @@ impl<S: Sync, E: InferenceEffect> Inference<S, E> for InferenceRunner<'_, S, E>
}
_ => true,
});
normalize_tool_call_thinking(&mut accumulator, &mut chunk);
drop_repeated_tool_call_thinking(&accumulator, &mut chunk);
if chunk.content.is_empty() {
if chunk.metadata.output_token_limit_reached {
chunk = emit.message(chunk).await;
+38 -119
View File
@@ -2963,138 +2963,52 @@ impl Agent {
}
}
// Thinking/reasoning belongs on the tool-call messages, not also
// as a separate standalone message: Gemini and Kimi/DeepSeek
// require it echoed on each assistant tool-call message, and the
// provider formatters reconstruct per-provider shape from there.
// Storing it both standalone AND on the tool-call message
// duplicates it; once merge_consecutive_messages glues the adjacent
// standalone and tool-call messages together, the duplicate signed
// blocks make Anthropic reject the turn with a 400. So the thinking
// is carried onto the split request messages below and never kept
// as a redundant standalone message.
// DeepSeek and Kimi need the turn's thinking on every split
// tool-call message; fix_conversation removes the signed copies.
let is_thinking = |c: &MessageContent| {
matches!(
c,
MessageContent::Thinking(_)
| MessageContent::RedactedThinking(_)
)
};
let prior_thinking: Vec<MessageContent> = messages_to_add
.iter()
.filter(|m| m.role == response.role)
.flat_map(|m| m.content.iter())
.filter(|c| is_thinking(c))
.cloned()
.collect();
let direct_thinking: Vec<MessageContent> = response
.content
.iter()
.filter(|c| {
matches!(
c,
MessageContent::Thinking(_)
| MessageContent::RedactedThinking(_)
)
})
.filter(|c| is_thinking(c) && !prior_thinking.contains(c))
.cloned()
.collect();
// When thinking arrived in earlier stream chunks it was stored as
// standalone thinking-only messages; reuse that thinking on the
// tool-call messages and drop the standalone messages so the
// thinking isn't duplicated.
// Always accumulate ALL prior thinking — even when
// direct_thinking is non-empty (reasoning arrived on the same
// chunk as tool_calls) — because otherwise only the last chunk's
// reasoning ends up on split tool-call messages.
// Also extract thinking from mixed (thinking+text) messages,
// not just pure-thinking-only ones.
let mut accumulated_prior: Vec<MessageContent> = Vec::new();
let mut indices_to_remove: Vec<usize> = Vec::new();
for (idx, m) in messages_to_add.messages_mut().iter_mut().enumerate()
{
if m.role != response.role || m.content.is_empty() {
continue;
}
let thinking_only = m.content.iter().all(|c| {
matches!(
c,
MessageContent::Thinking(_)
| MessageContent::RedactedThinking(_)
)
});
let has_thinking = m.content.iter().any(|c| {
matches!(
c,
MessageContent::Thinking(_)
| MessageContent::RedactedThinking(_)
)
});
if has_thinking {
// Only accumulate thinking from messages that
// have not already been split into tool-call
// request_msg items — prior-split messages
// already carry their own thinking copy.
if !m.content.iter().any(|c| {
matches!(c, MessageContent::ToolRequest(_))
}) {
for c in &m.content {
if matches!(
c,
MessageContent::Thinking(_)
| MessageContent::RedactedThinking(_)
) {
accumulated_prior.push(c.clone());
}
}
}
}
if thinking_only {
indices_to_remove.push(idx);
} else if has_thinking
&& !m.content.iter().any(|c| {
matches!(c, MessageContent::ToolRequest(_))
})
{
// Strip thinking blocks from mixed text+thinking
// messages so the same signed/unsigned thinking is not
// duplicated when carried onto the tool-call request
// messages below. Messages that already contain tool
// requests are prior-split request_msg items whose
// thinking was already attached — stripping their
// thinking would leave only the last split message
// with reasoning, violating the signed-thinking
// dedup expectation that the first split message
// retains it.
m.content.retain(|c| {
!matches!(
c,
MessageContent::Thinking(_)
| MessageContent::RedactedThinking(_)
)
});
}
}
// Remove in reverse order to preserve indices
for idx in indices_to_remove.into_iter().rev() {
messages_to_add.remove(idx);
}
let response_thinking = if direct_thinking.is_empty() {
accumulated_prior
} else if accumulated_prior.is_empty() {
direct_thinking
} else {
let mut merged = accumulated_prior;
merged.extend(direct_thinking);
merged
};
let mut turn_thinking = prior_thinking;
turn_thinking.extend(direct_thinking.iter().cloned());
let response_message_id = response
.id
.as_deref()
.expect("provider stream responses have IDs");
let has_existing_message_id_carrier = messages_to_add
.iter()
.any(|message| {
message.id.as_deref() == Some(response_message_id)
});
let carrier_tool_call_id = if has_existing_message_id_carrier {
None
} else {
tool_requests
.first()
.map(|request| request.id.as_str())
let is_response_message = |message: &Message| {
message.id.as_deref() == Some(response_message_id)
};
let first_tool_call_id = tool_requests
.first()
.map(|request| request.id.as_str());
// A same-id prefix at the tail coalesces with the first request on
// push, so tool-pair hiding removes the thinking with the call.
let carrier_tool_call_id = match messages_to_add.messages().last() {
Some(last) if is_response_message(last) => first_tool_call_id,
_ if messages_to_add.iter().any(is_response_message) => None,
_ => first_tool_call_id,
};
preferred_turn_usage_message_id =
Some(response_message_id.to_owned());
for request in &tool_requests {
for (index, request) in tool_requests.iter().enumerate() {
let mut request_msg =
if carrier_tool_call_id == Some(request.id.as_str()) {
Message::assistant().with_id(response_message_id)
@@ -3102,7 +3016,12 @@ impl Agent {
Message::assistant().with_generated_id()
};
for thinking in &response_thinking {
let thinking = if index == 0 {
&direct_thinking
} else {
&turn_thinking
};
for thinking in thinking {
request_msg = request_msg.with_content(thinking.clone());
}
@@ -36,15 +36,38 @@ async fn provider_lifecycle() -> Result<()> {
.with_image(image_data, "image/png"),
)
.await?;
result.assert_message(2, Agent, "The image is suitable. I will add one.");
result.assert_message(
3,
2,
Thinking,
"I should inspect the image before calculating.",
);
result.assert_message(3, Agent, "The image is suitable. I will add one.");
result.assert_message(4, ToolCall, ADD);
result.assert_message(5, ToolResponse, "result: 1");
result.assert_message(-1, Agent, "The total is 1.");
let tool_turn = result
.conversation()
.messages()
.iter()
.find(|message| {
message
.content
.iter()
.any(|content| matches!(content, MessageContent::ToolRequest(_)))
})
.expect("tool-call assistant message");
assert!(
matches!(
tool_turn.content.as_slice(),
[
MessageContent::Thinking(_),
MessageContent::Text(_),
MessageContent::ToolRequest(_)
]
),
"thinking, text, and the tool call must stay one message in model order: {:#?}",
tool_turn.content
);
assert!(result
.conversation()
.messages()
+52 -40
View File
@@ -1915,19 +1915,23 @@ mod tests {
);
match call {
0 => {
// Chunk 1: reasoning only (no tool calls)
let thinking =
Message::assistant().with_thinking("multi-tool reasoning", "sig_0");
// Chunk 2: two tool calls, no reasoning — the multi-tool bug scenario
let thinking = Message::assistant()
.with_id("msg_multi")
.with_thinking("multi-tool reasoning", "sig_0");
let text = Message::assistant()
.with_id("msg_multi")
.with_text("Calling both tools.");
let tc1 = CallToolRequestParams::new("tool_a")
.with_arguments(object!({"p": "1"}));
let tc2 = CallToolRequestParams::new("tool_b")
.with_arguments(object!({"p": "2"}));
let tool_msg = Message::assistant()
.with_id("msg_multi")
.with_tool_request("call_1", Ok(tc1))
.with_tool_request("call_2", Ok(tc2));
let stream = futures::stream::iter(vec![
Ok((Some(thinking), None)),
Ok((Some(text), None)),
Ok((Some(tool_msg), Some(usage))),
]);
Ok(Box::pin(stream))
@@ -2096,14 +2100,11 @@ mod tests {
Ok(())
}
/// Regression for the Anthropic 400: signed thinking arriving in a
/// separate chunk before the tool calls must be stored once per
/// tool-call message and never as an extra standalone message. When the
/// Anthropic formatter serializes the persisted history, each assistant
/// turn must carry exactly one thinking block — a duplicate signed block
/// is rejected with `thinking blocks ... cannot be modified`.
#[tokio::test]
async fn test_signed_thinking_not_duplicated_for_anthropic() -> Result<()> {
async fn test_signed_thinking_leads_text_and_tool_calls_for_anthropic() -> Result<()> {
use goose::conversation::{
fix_conversation, merge_consecutive_messages_for_request, Conversation,
};
use goose_providers::formats::anthropic::format_messages as anthropic_format;
let temp_dir = tempfile::tempdir()?;
@@ -2159,38 +2160,49 @@ mod tests {
.messages()
.to_vec();
// No standalone thinking-only assistant message should be persisted —
// thinking lives on the tool-call messages.
let standalone_thinking = messages.iter().any(|m| {
m.role == rmcp::model::Role::Assistant
&& !m.content.is_empty()
&& m.content
.iter()
.all(|c| matches!(c, MessageContent::Thinking(_)))
});
let first_tool_row = messages
.iter()
.find(|message| {
message.content.iter().any(
|content| matches!(content, MessageContent::ToolRequest(r) if r.id == "call_1"),
)
})
.expect("the first tool call is persisted");
assert_eq!(first_tool_row.id.as_deref(), Some("msg_multi"));
assert!(
!standalone_thinking,
"thinking must not be persisted as a standalone message: {messages:#?}"
matches!(
first_tool_row.content.as_slice(),
[
MessageContent::Thinking(_),
MessageContent::Text(_),
MessageContent::ToolRequest(_)
]
),
"the first tool call must share the prefix's row: {:#?}",
first_tool_row.content
);
// Every serialized Anthropic assistant message must contain at most
// one thinking block; a duplicate is what triggers the 400.
let spec = anthropic_format(&messages);
for msg in &spec {
if msg.get("role") == Some(&serde_json::json!("assistant")) {
if let Some(content) = msg.get("content").and_then(|c| c.as_array()) {
let thinking_blocks = content
.iter()
.filter(|c| c.get("type") == Some(&serde_json::json!("thinking")))
.count();
assert!(
thinking_blocks <= 1,
"assistant message has {thinking_blocks} thinking blocks, \
Anthropic rejects duplicates: {msg}"
);
}
}
}
let (fixed, _) = fix_conversation(Conversation::new_unvalidated(messages));
let spec = anthropic_format(&merge_consecutive_messages_for_request(
fixed.messages().to_vec(),
));
let assistant_block_types: Vec<Vec<&str>> = spec
.iter()
.filter(|msg| msg["role"] == "assistant")
.map(|msg| {
msg["content"]
.as_array()
.unwrap()
.iter()
.filter_map(|block| block["type"].as_str())
.collect()
})
.collect();
assert_eq!(
assistant_block_types,
vec![vec!["thinking", "text", "tool_use"], vec!["tool_use"]],
"{spec:#?}"
);
Ok(())
}