agent-loop: not infer or steer while a tool approval is pending (#12808)
This commit is contained in:
@@ -1,12 +1,15 @@
|
||||
//! Provider inference operation for the unrolled agent loop.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::sync::{Arc, Mutex, MutexGuard};
|
||||
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use futures::StreamExt;
|
||||
use goose_provider_types::base::Provider;
|
||||
use goose_provider_types::conversation::message::{InferenceMetadata, Message, MessageContent};
|
||||
use goose_provider_types::conversation::message::{
|
||||
ActionRequiredData, InferenceMetadata, Message, MessageContent,
|
||||
};
|
||||
use goose_provider_types::conversation::token_usage::ProviderUsage;
|
||||
use goose_provider_types::conversation::{
|
||||
effective_role, fix_conversation, merge_consecutive_messages_for_request, Conversation,
|
||||
@@ -222,7 +225,7 @@ fn messages_for_provider(
|
||||
turn: &[Message],
|
||||
keep_empty_messages: bool,
|
||||
) -> Vec<Message> {
|
||||
let answered: std::collections::HashSet<&str> = conversation
|
||||
let answered: HashSet<&str> = conversation
|
||||
.messages()
|
||||
.iter()
|
||||
.flat_map(|message| message.get_tool_response_ids())
|
||||
@@ -269,7 +272,35 @@ fn ends_with_provider_turn(messages: &[Message]) -> bool {
|
||||
})
|
||||
}
|
||||
|
||||
fn has_open_tool_approval(turn: &[Message]) -> bool {
|
||||
let mut goose_calls = HashSet::new();
|
||||
let mut answered = HashSet::new();
|
||||
let mut prompted = Vec::new();
|
||||
for content in turn.iter().flat_map(|message| &message.content) {
|
||||
match content {
|
||||
MessageContent::ToolRequest(request) if !request.was_executed_externally() => {
|
||||
goose_calls.insert(request.id.as_str());
|
||||
}
|
||||
MessageContent::ToolResponse(response) => {
|
||||
answered.insert(response.id.as_str());
|
||||
}
|
||||
MessageContent::ActionRequired(action) => {
|
||||
if let ActionRequiredData::ToolConfirmation { id, .. } = &action.data {
|
||||
prompted.push(id.as_str());
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
prompted
|
||||
.into_iter()
|
||||
.any(|id| goose_calls.contains(id) && !answered.contains(id))
|
||||
}
|
||||
|
||||
fn should_infer(conversation: &Conversation, turn: &[Message]) -> bool {
|
||||
if has_open_tool_approval(turn) {
|
||||
return false;
|
||||
}
|
||||
let projected = messages_for_provider(conversation, turn, true);
|
||||
if projected
|
||||
.last()
|
||||
@@ -466,7 +497,7 @@ impl<S: MaybeSync, E: InferenceEffect> Inference<S, E> for InferenceRunner<'_, S
|
||||
provider_session_id,
|
||||
});
|
||||
|
||||
let mut tool_request_ids = std::collections::HashSet::new();
|
||||
let mut tool_request_ids = HashSet::new();
|
||||
while let Some(result) = stream.next().await {
|
||||
let (msg_opt, usage_opt) = match result {
|
||||
Ok(chunk) => chunk,
|
||||
@@ -554,6 +585,9 @@ impl<S: MaybeSync, E: InferenceEffect> Inference<S, E> for InferenceRunner<'_, S
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use goose_provider_types::conversation::message::TOOL_META_EXTERNAL_DISPATCH_KEY;
|
||||
use goose_provider_types::permission::Permission;
|
||||
use rmcp::model::{CallToolRequestParams, CallToolResult};
|
||||
|
||||
#[test]
|
||||
fn provider_session_id_comes_only_from_latest_inference() {
|
||||
@@ -592,6 +626,70 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
fn call(id: &str) -> Message {
|
||||
Message::assistant().with_tool_request(id, Ok(CallToolRequestParams::new("write")))
|
||||
}
|
||||
|
||||
fn prompt(id: &str) -> Message {
|
||||
Message::assistant()
|
||||
.with_action_required(id, "write".to_string(), Default::default(), None)
|
||||
.user_only()
|
||||
}
|
||||
|
||||
fn answer(id: &str) -> Message {
|
||||
Message::user().with_tool_response(id, Ok(CallToolResult::success(Vec::new())))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_tool_approval_stays_open_until_the_call_is_answered() {
|
||||
assert!(has_open_tool_approval(&[call("write"), prompt("write")]));
|
||||
|
||||
let decided = Message::user().with_content(
|
||||
MessageContent::action_required_tool_confirmation_response(
|
||||
"write",
|
||||
Permission::AllowOnce,
|
||||
),
|
||||
);
|
||||
assert!(has_open_tool_approval(&[
|
||||
call("write"),
|
||||
prompt("write"),
|
||||
decided
|
||||
]));
|
||||
|
||||
assert!(!has_open_tool_approval(&[
|
||||
call("write"),
|
||||
prompt("write"),
|
||||
answer("write")
|
||||
]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prompts_without_a_goose_call_are_not_open() {
|
||||
assert!(!has_open_tool_approval(&[prompt("cli-request")]));
|
||||
|
||||
let external = Message::assistant().with_tool_request_with_metadata(
|
||||
"external",
|
||||
Ok(CallToolRequestParams::new("write")),
|
||||
None,
|
||||
Some(serde_json::json!({ TOOL_META_EXTERNAL_DISPATCH_KEY: true })),
|
||||
);
|
||||
assert!(!has_open_tool_approval(&[external, prompt("external")]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_inference_while_a_tool_approval_is_open() {
|
||||
let conversation = Conversation::new_unvalidated([
|
||||
Message::user().with_text("add and write"),
|
||||
Message::assistant()
|
||||
.with_tool_request("free", Ok(CallToolRequestParams::new("add")))
|
||||
.with_tool_request("gated", Ok(CallToolRequestParams::new("write"))),
|
||||
answer("free"),
|
||||
prompt("gated"),
|
||||
]);
|
||||
let turn = messages_since_kickoff(&conversation).unwrap();
|
||||
assert!(!should_infer(&conversation, turn));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn whitespace_only_text_is_an_empty_response() {
|
||||
assert!(is_empty_response(&Message::assistant().with_text(" \n\t ")));
|
||||
|
||||
@@ -9,8 +9,8 @@ use tokio::sync::Mutex;
|
||||
|
||||
use crate::agents::state_machine::effects::GooseEffect;
|
||||
use crate::agents::state_machine::{
|
||||
applied, ends_turn, last_effective_role, messages_since_kickoff, not_applicable, Emitter,
|
||||
Operation, OperationResult,
|
||||
applied, awaits_tool_responses, ends_turn, last_effective_role, messages_since_kickoff,
|
||||
not_applicable, Emitter, Operation, OperationResult,
|
||||
};
|
||||
use crate::conversation::message::Message;
|
||||
use crate::conversation::{Conversation, EffectiveRole};
|
||||
@@ -66,7 +66,7 @@ impl Operation<Session, GooseEffect> for SteerOperation {
|
||||
let messages = messages_since_kickoff(conversation)?;
|
||||
let between_turns =
|
||||
ends_turn(messages) || last_effective_role(messages)? == EffectiveRole::Tool;
|
||||
if !between_turns {
|
||||
if !between_turns || awaits_tool_responses(messages) {
|
||||
return not_applicable();
|
||||
}
|
||||
|
||||
|
||||
@@ -156,6 +156,28 @@ impl ApiCall {
|
||||
self.body["model"].as_str() == Some(model)
|
||||
}
|
||||
|
||||
pub(super) fn answers_every_tool_call_next(&self) -> bool {
|
||||
let messages: Vec<_> = self.body["messages"]
|
||||
.as_array()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect();
|
||||
messages.iter().enumerate().all(|(index, message)| {
|
||||
let requested = message["tool_calls"]
|
||||
.as_array()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.filter_map(|call| call["id"].as_str())
|
||||
.collect::<std::collections::HashSet<_>>();
|
||||
let answered = messages[index + 1..]
|
||||
.iter()
|
||||
.take_while(|next| next["role"] == "tool")
|
||||
.filter_map(|next| next["tool_call_id"].as_str())
|
||||
.collect::<std::collections::HashSet<_>>();
|
||||
requested.is_subset(&answered)
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn input_occurrences(&self, needle: &str) -> usize {
|
||||
request_input(&self.body).matches(needle).count()
|
||||
}
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::Result;
|
||||
use rmcp::model::ElicitationAction;
|
||||
use serde_json::json;
|
||||
@@ -5,7 +7,9 @@ use serde_json::json;
|
||||
use super::calculator_extension::{
|
||||
delayed_value, value, ADD, ADD_WITH_AUDIENCE, APP_ONLY, DIVIDE, REQUEST_VALUE,
|
||||
};
|
||||
use super::dummy_api::DummyApi;
|
||||
use super::pipeline::MessageKind::{Agent, Confirmation, ToolCall, ToolResponse};
|
||||
use super::pipeline::TestPipeline;
|
||||
use super::pipeline::MAX_TURNS;
|
||||
use super::test_pipeline;
|
||||
use crate::agents::tool_execution::{CHAT_MODE_TOOL_SKIPPED_RESPONSE, DECLINED_RESPONSE};
|
||||
@@ -15,6 +19,70 @@ use crate::config::GooseMode;
|
||||
use crate::conversation::message::{Message, MessageContent};
|
||||
use crate::permission::Permission;
|
||||
|
||||
async fn pipeline_with_gated_divide() -> Result<(TestPipeline, Arc<DummyApi>)> {
|
||||
let (pipeline, api) = test_pipeline().await?;
|
||||
let pipeline = pipeline.with_goose_mode(GooseMode::Approve).await;
|
||||
pipeline.set_permission(ADD, PermissionLevel::AlwaysAllow);
|
||||
api.on("add four and halve it")
|
||||
.calls([("free", ADD, value(4)), ("gated", DIVIDE, value(2))]);
|
||||
Ok((pipeline, api))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn no_inference_while_a_tool_confirmation_is_open() -> Result<()> {
|
||||
let (pipeline, api) = pipeline_with_gated_divide().await?;
|
||||
|
||||
let awaiting_confirmation = pipeline.run(["add four and halve it"]).await?;
|
||||
assert_eq!(api.call_count(), 1);
|
||||
assert_eq!(pipeline.calculator_total(), 4);
|
||||
awaiting_confirmation.assert_message(-1, ToolResponse, "result: 4");
|
||||
|
||||
api.on("result: 2").reply("halved");
|
||||
pipeline.confirm("gated", Permission::AllowOnce).await?;
|
||||
let result = pipeline.resume().await?;
|
||||
result.assert_message(-1, Agent, "halved");
|
||||
assert_eq!(pipeline.calculator_total(), 2);
|
||||
assert_eq!(api.call_count(), 2);
|
||||
assert!(api.calls().last().unwrap().answers_every_tool_call_next());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn steer_waits_for_an_open_tool_confirmation() -> Result<()> {
|
||||
let (pipeline, api) = pipeline_with_gated_divide().await?;
|
||||
pipeline
|
||||
.steer(Message::user().with_text("then say done"))
|
||||
.await;
|
||||
|
||||
let awaiting_confirmation = pipeline.run(["add four and halve it"]).await?;
|
||||
assert_eq!(api.call_count(), 1);
|
||||
assert!(pipeline.has_pending_steers().await);
|
||||
assert!(!awaiting_confirmation
|
||||
.conversation()
|
||||
.messages()
|
||||
.iter()
|
||||
.any(|message| message.as_concat_text() == "then say done"));
|
||||
|
||||
api.on("then say done").reply("done");
|
||||
pipeline.confirm("gated", Permission::AllowOnce).await?;
|
||||
let result = pipeline.resume().await?;
|
||||
result.assert_message(-1, Agent, "done");
|
||||
assert_eq!(pipeline.calculator_total(), 2);
|
||||
assert!(!pipeline.has_pending_steers().await);
|
||||
let messages = result.conversation().messages();
|
||||
let steer = messages
|
||||
.iter()
|
||||
.position(|message| message.as_concat_text() == "then say done")
|
||||
.expect("steer was persisted");
|
||||
let gated_answer = messages
|
||||
.iter()
|
||||
.position(|message| message.get_tool_response_ids().contains("gated"))
|
||||
.expect("gated call was answered");
|
||||
assert!(gated_answer < steer);
|
||||
assert!(api.calls().last().unwrap().answers_every_tool_call_next());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn basic_tool_calling() -> Result<()> {
|
||||
let (pipeline, api) = test_pipeline().await?;
|
||||
|
||||
Reference in New Issue
Block a user