fix(agents): keep streamed thinking ahead of text and tool calls (#11837)
This commit is contained in:
@@ -75,7 +75,7 @@ fn is_thinking(content: &MessageContent) -> bool {
|
||||
)
|
||||
}
|
||||
|
||||
fn normalize_tool_call_thinking(accumulator: &mut Conversation, chunk: &mut Message) {
|
||||
fn drop_repeated_tool_call_thinking(accumulator: &Conversation, chunk: &mut Message) {
|
||||
if !chunk
|
||||
.content
|
||||
.iter()
|
||||
@@ -83,42 +83,15 @@ fn normalize_tool_call_thinking(accumulator: &mut Conversation, chunk: &mut Mess
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
let has_direct_thinking = chunk.content.iter().any(is_thinking);
|
||||
let mut prior_thinking = Vec::new();
|
||||
for message in accumulator.messages_mut() {
|
||||
if message.role != chunk.role
|
||||
|| message
|
||||
.content
|
||||
.iter()
|
||||
.any(|content| matches!(content, MessageContent::ToolRequest(_)))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
prior_thinking.extend(
|
||||
message
|
||||
.content
|
||||
.iter()
|
||||
.filter(|content| is_thinking(content))
|
||||
.cloned(),
|
||||
);
|
||||
message.content.retain(|content| !is_thinking(content));
|
||||
}
|
||||
accumulator
|
||||
.messages_mut()
|
||||
.retain(|message| !message.content.is_empty());
|
||||
|
||||
if !has_direct_thinking && !prior_thinking.is_empty() {
|
||||
if let Some(tool_request) = chunk
|
||||
.content
|
||||
.iter()
|
||||
.position(|content| matches!(content, MessageContent::ToolRequest(_)))
|
||||
{
|
||||
chunk
|
||||
.content
|
||||
.splice(tool_request..tool_request, prior_thinking);
|
||||
}
|
||||
}
|
||||
let prior: Vec<&MessageContent> = accumulator
|
||||
.iter()
|
||||
.filter(|message| message.role == chunk.role)
|
||||
.flat_map(|message| message.content.iter())
|
||||
.filter(|content| is_thinking(content))
|
||||
.collect();
|
||||
chunk
|
||||
.content
|
||||
.retain(|content| !(is_thinking(content) && prior.contains(&content)));
|
||||
}
|
||||
|
||||
pub fn chat_span(
|
||||
@@ -479,7 +452,7 @@ impl<S: Sync, E: InferenceEffect> Inference<S, E> for InferenceRunner<'_, S, E>
|
||||
}
|
||||
_ => true,
|
||||
});
|
||||
normalize_tool_call_thinking(&mut accumulator, &mut chunk);
|
||||
drop_repeated_tool_call_thinking(&accumulator, &mut chunk);
|
||||
if chunk.content.is_empty() {
|
||||
if chunk.metadata.output_token_limit_reached {
|
||||
chunk = emit.message(chunk).await;
|
||||
|
||||
@@ -2963,138 +2963,52 @@ impl Agent {
|
||||
}
|
||||
}
|
||||
|
||||
// Thinking/reasoning belongs on the tool-call messages, not also
|
||||
// as a separate standalone message: Gemini and Kimi/DeepSeek
|
||||
// require it echoed on each assistant tool-call message, and the
|
||||
// provider formatters reconstruct per-provider shape from there.
|
||||
// Storing it both standalone AND on the tool-call message
|
||||
// duplicates it; once merge_consecutive_messages glues the adjacent
|
||||
// standalone and tool-call messages together, the duplicate signed
|
||||
// blocks make Anthropic reject the turn with a 400. So the thinking
|
||||
// is carried onto the split request messages below and never kept
|
||||
// as a redundant standalone message.
|
||||
// DeepSeek and Kimi need the turn's thinking on every split
|
||||
// tool-call message; fix_conversation removes the signed copies.
|
||||
let is_thinking = |c: &MessageContent| {
|
||||
matches!(
|
||||
c,
|
||||
MessageContent::Thinking(_)
|
||||
| MessageContent::RedactedThinking(_)
|
||||
)
|
||||
};
|
||||
let prior_thinking: Vec<MessageContent> = messages_to_add
|
||||
.iter()
|
||||
.filter(|m| m.role == response.role)
|
||||
.flat_map(|m| m.content.iter())
|
||||
.filter(|c| is_thinking(c))
|
||||
.cloned()
|
||||
.collect();
|
||||
let direct_thinking: Vec<MessageContent> = response
|
||||
.content
|
||||
.iter()
|
||||
.filter(|c| {
|
||||
matches!(
|
||||
c,
|
||||
MessageContent::Thinking(_)
|
||||
| MessageContent::RedactedThinking(_)
|
||||
)
|
||||
})
|
||||
.filter(|c| is_thinking(c) && !prior_thinking.contains(c))
|
||||
.cloned()
|
||||
.collect();
|
||||
// When thinking arrived in earlier stream chunks it was stored as
|
||||
// standalone thinking-only messages; reuse that thinking on the
|
||||
// tool-call messages and drop the standalone messages so the
|
||||
// thinking isn't duplicated.
|
||||
// Always accumulate ALL prior thinking — even when
|
||||
// direct_thinking is non-empty (reasoning arrived on the same
|
||||
// chunk as tool_calls) — because otherwise only the last chunk's
|
||||
// reasoning ends up on split tool-call messages.
|
||||
// Also extract thinking from mixed (thinking+text) messages,
|
||||
// not just pure-thinking-only ones.
|
||||
let mut accumulated_prior: Vec<MessageContent> = Vec::new();
|
||||
let mut indices_to_remove: Vec<usize> = Vec::new();
|
||||
for (idx, m) in messages_to_add.messages_mut().iter_mut().enumerate()
|
||||
{
|
||||
if m.role != response.role || m.content.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let thinking_only = m.content.iter().all(|c| {
|
||||
matches!(
|
||||
c,
|
||||
MessageContent::Thinking(_)
|
||||
| MessageContent::RedactedThinking(_)
|
||||
)
|
||||
});
|
||||
let has_thinking = m.content.iter().any(|c| {
|
||||
matches!(
|
||||
c,
|
||||
MessageContent::Thinking(_)
|
||||
| MessageContent::RedactedThinking(_)
|
||||
)
|
||||
});
|
||||
if has_thinking {
|
||||
// Only accumulate thinking from messages that
|
||||
// have not already been split into tool-call
|
||||
// request_msg items — prior-split messages
|
||||
// already carry their own thinking copy.
|
||||
if !m.content.iter().any(|c| {
|
||||
matches!(c, MessageContent::ToolRequest(_))
|
||||
}) {
|
||||
for c in &m.content {
|
||||
if matches!(
|
||||
c,
|
||||
MessageContent::Thinking(_)
|
||||
| MessageContent::RedactedThinking(_)
|
||||
) {
|
||||
accumulated_prior.push(c.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if thinking_only {
|
||||
indices_to_remove.push(idx);
|
||||
} else if has_thinking
|
||||
&& !m.content.iter().any(|c| {
|
||||
matches!(c, MessageContent::ToolRequest(_))
|
||||
})
|
||||
{
|
||||
// Strip thinking blocks from mixed text+thinking
|
||||
// messages so the same signed/unsigned thinking is not
|
||||
// duplicated when carried onto the tool-call request
|
||||
// messages below. Messages that already contain tool
|
||||
// requests are prior-split request_msg items whose
|
||||
// thinking was already attached — stripping their
|
||||
// thinking would leave only the last split message
|
||||
// with reasoning, violating the signed-thinking
|
||||
// dedup expectation that the first split message
|
||||
// retains it.
|
||||
m.content.retain(|c| {
|
||||
!matches!(
|
||||
c,
|
||||
MessageContent::Thinking(_)
|
||||
| MessageContent::RedactedThinking(_)
|
||||
)
|
||||
});
|
||||
}
|
||||
}
|
||||
// Remove in reverse order to preserve indices
|
||||
for idx in indices_to_remove.into_iter().rev() {
|
||||
messages_to_add.remove(idx);
|
||||
}
|
||||
let response_thinking = if direct_thinking.is_empty() {
|
||||
accumulated_prior
|
||||
} else if accumulated_prior.is_empty() {
|
||||
direct_thinking
|
||||
} else {
|
||||
let mut merged = accumulated_prior;
|
||||
merged.extend(direct_thinking);
|
||||
merged
|
||||
};
|
||||
let mut turn_thinking = prior_thinking;
|
||||
turn_thinking.extend(direct_thinking.iter().cloned());
|
||||
|
||||
let response_message_id = response
|
||||
.id
|
||||
.as_deref()
|
||||
.expect("provider stream responses have IDs");
|
||||
let has_existing_message_id_carrier = messages_to_add
|
||||
.iter()
|
||||
.any(|message| {
|
||||
message.id.as_deref() == Some(response_message_id)
|
||||
});
|
||||
let carrier_tool_call_id = if has_existing_message_id_carrier {
|
||||
None
|
||||
} else {
|
||||
tool_requests
|
||||
.first()
|
||||
.map(|request| request.id.as_str())
|
||||
let is_response_message = |message: &Message| {
|
||||
message.id.as_deref() == Some(response_message_id)
|
||||
};
|
||||
let first_tool_call_id = tool_requests
|
||||
.first()
|
||||
.map(|request| request.id.as_str());
|
||||
// A same-id prefix at the tail coalesces with the first request on
|
||||
// push, so tool-pair hiding removes the thinking with the call.
|
||||
let carrier_tool_call_id = match messages_to_add.messages().last() {
|
||||
Some(last) if is_response_message(last) => first_tool_call_id,
|
||||
_ if messages_to_add.iter().any(is_response_message) => None,
|
||||
_ => first_tool_call_id,
|
||||
};
|
||||
preferred_turn_usage_message_id =
|
||||
Some(response_message_id.to_owned());
|
||||
|
||||
for request in &tool_requests {
|
||||
for (index, request) in tool_requests.iter().enumerate() {
|
||||
let mut request_msg =
|
||||
if carrier_tool_call_id == Some(request.id.as_str()) {
|
||||
Message::assistant().with_id(response_message_id)
|
||||
@@ -3102,7 +3016,12 @@ impl Agent {
|
||||
Message::assistant().with_generated_id()
|
||||
};
|
||||
|
||||
for thinking in &response_thinking {
|
||||
let thinking = if index == 0 {
|
||||
&direct_thinking
|
||||
} else {
|
||||
&turn_thinking
|
||||
};
|
||||
for thinking in thinking {
|
||||
request_msg = request_msg.with_content(thinking.clone());
|
||||
}
|
||||
|
||||
|
||||
@@ -36,15 +36,38 @@ async fn provider_lifecycle() -> Result<()> {
|
||||
.with_image(image_data, "image/png"),
|
||||
)
|
||||
.await?;
|
||||
result.assert_message(2, Agent, "The image is suitable. I will add one.");
|
||||
result.assert_message(
|
||||
3,
|
||||
2,
|
||||
Thinking,
|
||||
"I should inspect the image before calculating.",
|
||||
);
|
||||
result.assert_message(3, Agent, "The image is suitable. I will add one.");
|
||||
result.assert_message(4, ToolCall, ADD);
|
||||
result.assert_message(5, ToolResponse, "result: 1");
|
||||
result.assert_message(-1, Agent, "The total is 1.");
|
||||
let tool_turn = result
|
||||
.conversation()
|
||||
.messages()
|
||||
.iter()
|
||||
.find(|message| {
|
||||
message
|
||||
.content
|
||||
.iter()
|
||||
.any(|content| matches!(content, MessageContent::ToolRequest(_)))
|
||||
})
|
||||
.expect("tool-call assistant message");
|
||||
assert!(
|
||||
matches!(
|
||||
tool_turn.content.as_slice(),
|
||||
[
|
||||
MessageContent::Thinking(_),
|
||||
MessageContent::Text(_),
|
||||
MessageContent::ToolRequest(_)
|
||||
]
|
||||
),
|
||||
"thinking, text, and the tool call must stay one message in model order: {:#?}",
|
||||
tool_turn.content
|
||||
);
|
||||
assert!(result
|
||||
.conversation()
|
||||
.messages()
|
||||
|
||||
+52
-40
@@ -1915,19 +1915,23 @@ mod tests {
|
||||
);
|
||||
match call {
|
||||
0 => {
|
||||
// Chunk 1: reasoning only (no tool calls)
|
||||
let thinking =
|
||||
Message::assistant().with_thinking("multi-tool reasoning", "sig_0");
|
||||
// Chunk 2: two tool calls, no reasoning — the multi-tool bug scenario
|
||||
let thinking = Message::assistant()
|
||||
.with_id("msg_multi")
|
||||
.with_thinking("multi-tool reasoning", "sig_0");
|
||||
let text = Message::assistant()
|
||||
.with_id("msg_multi")
|
||||
.with_text("Calling both tools.");
|
||||
let tc1 = CallToolRequestParams::new("tool_a")
|
||||
.with_arguments(object!({"p": "1"}));
|
||||
let tc2 = CallToolRequestParams::new("tool_b")
|
||||
.with_arguments(object!({"p": "2"}));
|
||||
let tool_msg = Message::assistant()
|
||||
.with_id("msg_multi")
|
||||
.with_tool_request("call_1", Ok(tc1))
|
||||
.with_tool_request("call_2", Ok(tc2));
|
||||
let stream = futures::stream::iter(vec![
|
||||
Ok((Some(thinking), None)),
|
||||
Ok((Some(text), None)),
|
||||
Ok((Some(tool_msg), Some(usage))),
|
||||
]);
|
||||
Ok(Box::pin(stream))
|
||||
@@ -2096,14 +2100,11 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Regression for the Anthropic 400: signed thinking arriving in a
|
||||
/// separate chunk before the tool calls must be stored once per
|
||||
/// tool-call message and never as an extra standalone message. When the
|
||||
/// Anthropic formatter serializes the persisted history, each assistant
|
||||
/// turn must carry exactly one thinking block — a duplicate signed block
|
||||
/// is rejected with `thinking blocks ... cannot be modified`.
|
||||
#[tokio::test]
|
||||
async fn test_signed_thinking_not_duplicated_for_anthropic() -> Result<()> {
|
||||
async fn test_signed_thinking_leads_text_and_tool_calls_for_anthropic() -> Result<()> {
|
||||
use goose::conversation::{
|
||||
fix_conversation, merge_consecutive_messages_for_request, Conversation,
|
||||
};
|
||||
use goose_providers::formats::anthropic::format_messages as anthropic_format;
|
||||
|
||||
let temp_dir = tempfile::tempdir()?;
|
||||
@@ -2159,38 +2160,49 @@ mod tests {
|
||||
.messages()
|
||||
.to_vec();
|
||||
|
||||
// No standalone thinking-only assistant message should be persisted —
|
||||
// thinking lives on the tool-call messages.
|
||||
let standalone_thinking = messages.iter().any(|m| {
|
||||
m.role == rmcp::model::Role::Assistant
|
||||
&& !m.content.is_empty()
|
||||
&& m.content
|
||||
.iter()
|
||||
.all(|c| matches!(c, MessageContent::Thinking(_)))
|
||||
});
|
||||
let first_tool_row = messages
|
||||
.iter()
|
||||
.find(|message| {
|
||||
message.content.iter().any(
|
||||
|content| matches!(content, MessageContent::ToolRequest(r) if r.id == "call_1"),
|
||||
)
|
||||
})
|
||||
.expect("the first tool call is persisted");
|
||||
assert_eq!(first_tool_row.id.as_deref(), Some("msg_multi"));
|
||||
assert!(
|
||||
!standalone_thinking,
|
||||
"thinking must not be persisted as a standalone message: {messages:#?}"
|
||||
matches!(
|
||||
first_tool_row.content.as_slice(),
|
||||
[
|
||||
MessageContent::Thinking(_),
|
||||
MessageContent::Text(_),
|
||||
MessageContent::ToolRequest(_)
|
||||
]
|
||||
),
|
||||
"the first tool call must share the prefix's row: {:#?}",
|
||||
first_tool_row.content
|
||||
);
|
||||
|
||||
// Every serialized Anthropic assistant message must contain at most
|
||||
// one thinking block; a duplicate is what triggers the 400.
|
||||
let spec = anthropic_format(&messages);
|
||||
for msg in &spec {
|
||||
if msg.get("role") == Some(&serde_json::json!("assistant")) {
|
||||
if let Some(content) = msg.get("content").and_then(|c| c.as_array()) {
|
||||
let thinking_blocks = content
|
||||
.iter()
|
||||
.filter(|c| c.get("type") == Some(&serde_json::json!("thinking")))
|
||||
.count();
|
||||
assert!(
|
||||
thinking_blocks <= 1,
|
||||
"assistant message has {thinking_blocks} thinking blocks, \
|
||||
Anthropic rejects duplicates: {msg}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
let (fixed, _) = fix_conversation(Conversation::new_unvalidated(messages));
|
||||
let spec = anthropic_format(&merge_consecutive_messages_for_request(
|
||||
fixed.messages().to_vec(),
|
||||
));
|
||||
let assistant_block_types: Vec<Vec<&str>> = spec
|
||||
.iter()
|
||||
.filter(|msg| msg["role"] == "assistant")
|
||||
.map(|msg| {
|
||||
msg["content"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.filter_map(|block| block["type"].as_str())
|
||||
.collect()
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(
|
||||
assistant_block_types,
|
||||
vec![vec!["thinking", "text", "tool_use"], vec!["tool_use"]],
|
||||
"{spec:#?}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user