diff --git a/crates/goose-agent/src/inference.rs b/crates/goose-agent/src/inference.rs index 92abd1405..d24aaee86 100644 --- a/crates/goose-agent/src/inference.rs +++ b/crates/goose-agent/src/inference.rs @@ -1,12 +1,15 @@ //! Provider inference operation for the unrolled agent loop. +use std::collections::HashSet; use std::sync::{Arc, Mutex, MutexGuard}; use anyhow::Result; use async_trait::async_trait; use futures::StreamExt; use goose_provider_types::base::Provider; -use goose_provider_types::conversation::message::{InferenceMetadata, Message, MessageContent}; +use goose_provider_types::conversation::message::{ + ActionRequiredData, InferenceMetadata, Message, MessageContent, +}; use goose_provider_types::conversation::token_usage::ProviderUsage; use goose_provider_types::conversation::{ effective_role, fix_conversation, merge_consecutive_messages_for_request, Conversation, @@ -222,7 +225,7 @@ fn messages_for_provider( turn: &[Message], keep_empty_messages: bool, ) -> Vec { - let answered: std::collections::HashSet<&str> = conversation + let answered: HashSet<&str> = conversation .messages() .iter() .flat_map(|message| message.get_tool_response_ids()) @@ -269,7 +272,35 @@ fn ends_with_provider_turn(messages: &[Message]) -> bool { }) } +fn has_open_tool_approval(turn: &[Message]) -> bool { + let mut goose_calls = HashSet::new(); + let mut answered = HashSet::new(); + let mut prompted = Vec::new(); + for content in turn.iter().flat_map(|message| &message.content) { + match content { + MessageContent::ToolRequest(request) if !request.was_executed_externally() => { + goose_calls.insert(request.id.as_str()); + } + MessageContent::ToolResponse(response) => { + answered.insert(response.id.as_str()); + } + MessageContent::ActionRequired(action) => { + if let ActionRequiredData::ToolConfirmation { id, .. } = &action.data { + prompted.push(id.as_str()); + } + } + _ => {} + } + } + prompted + .into_iter() + .any(|id| goose_calls.contains(id) && !answered.contains(id)) +} + fn should_infer(conversation: &Conversation, turn: &[Message]) -> bool { + if has_open_tool_approval(turn) { + return false; + } let projected = messages_for_provider(conversation, turn, true); if projected .last() @@ -466,7 +497,7 @@ impl Inference for InferenceRunner<'_, S provider_session_id, }); - let mut tool_request_ids = std::collections::HashSet::new(); + let mut tool_request_ids = HashSet::new(); while let Some(result) = stream.next().await { let (msg_opt, usage_opt) = match result { Ok(chunk) => chunk, @@ -554,6 +585,9 @@ impl Inference for InferenceRunner<'_, S #[cfg(test)] mod tests { use super::*; + use goose_provider_types::conversation::message::TOOL_META_EXTERNAL_DISPATCH_KEY; + use goose_provider_types::permission::Permission; + use rmcp::model::{CallToolRequestParams, CallToolResult}; #[test] fn provider_session_id_comes_only_from_latest_inference() { @@ -592,6 +626,70 @@ mod tests { )); } + fn call(id: &str) -> Message { + Message::assistant().with_tool_request(id, Ok(CallToolRequestParams::new("write"))) + } + + fn prompt(id: &str) -> Message { + Message::assistant() + .with_action_required(id, "write".to_string(), Default::default(), None) + .user_only() + } + + fn answer(id: &str) -> Message { + Message::user().with_tool_response(id, Ok(CallToolResult::success(Vec::new()))) + } + + #[test] + fn a_tool_approval_stays_open_until_the_call_is_answered() { + assert!(has_open_tool_approval(&[call("write"), prompt("write")])); + + let decided = Message::user().with_content( + MessageContent::action_required_tool_confirmation_response( + "write", + Permission::AllowOnce, + ), + ); + assert!(has_open_tool_approval(&[ + call("write"), + prompt("write"), + decided + ])); + + assert!(!has_open_tool_approval(&[ + call("write"), + prompt("write"), + answer("write") + ])); + } + + #[test] + fn prompts_without_a_goose_call_are_not_open() { + assert!(!has_open_tool_approval(&[prompt("cli-request")])); + + let external = Message::assistant().with_tool_request_with_metadata( + "external", + Ok(CallToolRequestParams::new("write")), + None, + Some(serde_json::json!({ TOOL_META_EXTERNAL_DISPATCH_KEY: true })), + ); + assert!(!has_open_tool_approval(&[external, prompt("external")])); + } + + #[test] + fn no_inference_while_a_tool_approval_is_open() { + let conversation = Conversation::new_unvalidated([ + Message::user().with_text("add and write"), + Message::assistant() + .with_tool_request("free", Ok(CallToolRequestParams::new("add"))) + .with_tool_request("gated", Ok(CallToolRequestParams::new("write"))), + answer("free"), + prompt("gated"), + ]); + let turn = messages_since_kickoff(&conversation).unwrap(); + assert!(!should_infer(&conversation, turn)); + } + #[test] fn whitespace_only_text_is_an_empty_response() { assert!(is_empty_response(&Message::assistant().with_text(" \n\t "))); diff --git a/crates/goose/src/agents/state_machine/ops_steer.rs b/crates/goose/src/agents/state_machine/ops_steer.rs index 51513132a..4310ccf12 100644 --- a/crates/goose/src/agents/state_machine/ops_steer.rs +++ b/crates/goose/src/agents/state_machine/ops_steer.rs @@ -9,8 +9,8 @@ use tokio::sync::Mutex; use crate::agents::state_machine::effects::GooseEffect; use crate::agents::state_machine::{ - applied, ends_turn, last_effective_role, messages_since_kickoff, not_applicable, Emitter, - Operation, OperationResult, + applied, awaits_tool_responses, ends_turn, last_effective_role, messages_since_kickoff, + not_applicable, Emitter, Operation, OperationResult, }; use crate::conversation::message::Message; use crate::conversation::{Conversation, EffectiveRole}; @@ -66,7 +66,7 @@ impl Operation for SteerOperation { let messages = messages_since_kickoff(conversation)?; let between_turns = ends_turn(messages) || last_effective_role(messages)? == EffectiveRole::Tool; - if !between_turns { + if !between_turns || awaits_tool_responses(messages) { return not_applicable(); } diff --git a/crates/goose/src/agents/state_machine/tests/dummy_api.rs b/crates/goose/src/agents/state_machine/tests/dummy_api.rs index 48bcfb13f..d20de1d41 100644 --- a/crates/goose/src/agents/state_machine/tests/dummy_api.rs +++ b/crates/goose/src/agents/state_machine/tests/dummy_api.rs @@ -156,6 +156,28 @@ impl ApiCall { self.body["model"].as_str() == Some(model) } + pub(super) fn answers_every_tool_call_next(&self) -> bool { + let messages: Vec<_> = self.body["messages"] + .as_array() + .into_iter() + .flatten() + .collect(); + messages.iter().enumerate().all(|(index, message)| { + let requested = message["tool_calls"] + .as_array() + .into_iter() + .flatten() + .filter_map(|call| call["id"].as_str()) + .collect::>(); + let answered = messages[index + 1..] + .iter() + .take_while(|next| next["role"] == "tool") + .filter_map(|next| next["tool_call_id"].as_str()) + .collect::>(); + requested.is_subset(&answered) + }) + } + pub(super) fn input_occurrences(&self, needle: &str) -> usize { request_input(&self.body).matches(needle).count() } diff --git a/crates/goose/src/agents/state_machine/tests/tool_lifecycle.rs b/crates/goose/src/agents/state_machine/tests/tool_lifecycle.rs index e6de4ee12..f5329ace6 100644 --- a/crates/goose/src/agents/state_machine/tests/tool_lifecycle.rs +++ b/crates/goose/src/agents/state_machine/tests/tool_lifecycle.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use anyhow::Result; use rmcp::model::ElicitationAction; use serde_json::json; @@ -5,7 +7,9 @@ use serde_json::json; use super::calculator_extension::{ delayed_value, value, ADD, ADD_WITH_AUDIENCE, APP_ONLY, DIVIDE, REQUEST_VALUE, }; +use super::dummy_api::DummyApi; use super::pipeline::MessageKind::{Agent, Confirmation, ToolCall, ToolResponse}; +use super::pipeline::TestPipeline; use super::pipeline::MAX_TURNS; use super::test_pipeline; use crate::agents::tool_execution::{CHAT_MODE_TOOL_SKIPPED_RESPONSE, DECLINED_RESPONSE}; @@ -15,6 +19,70 @@ use crate::config::GooseMode; use crate::conversation::message::{Message, MessageContent}; use crate::permission::Permission; +async fn pipeline_with_gated_divide() -> Result<(TestPipeline, Arc)> { + let (pipeline, api) = test_pipeline().await?; + let pipeline = pipeline.with_goose_mode(GooseMode::Approve).await; + pipeline.set_permission(ADD, PermissionLevel::AlwaysAllow); + api.on("add four and halve it") + .calls([("free", ADD, value(4)), ("gated", DIVIDE, value(2))]); + Ok((pipeline, api)) +} + +#[tokio::test] +async fn no_inference_while_a_tool_confirmation_is_open() -> Result<()> { + let (pipeline, api) = pipeline_with_gated_divide().await?; + + let awaiting_confirmation = pipeline.run(["add four and halve it"]).await?; + assert_eq!(api.call_count(), 1); + assert_eq!(pipeline.calculator_total(), 4); + awaiting_confirmation.assert_message(-1, ToolResponse, "result: 4"); + + api.on("result: 2").reply("halved"); + pipeline.confirm("gated", Permission::AllowOnce).await?; + let result = pipeline.resume().await?; + result.assert_message(-1, Agent, "halved"); + assert_eq!(pipeline.calculator_total(), 2); + assert_eq!(api.call_count(), 2); + assert!(api.calls().last().unwrap().answers_every_tool_call_next()); + Ok(()) +} + +#[tokio::test] +async fn steer_waits_for_an_open_tool_confirmation() -> Result<()> { + let (pipeline, api) = pipeline_with_gated_divide().await?; + pipeline + .steer(Message::user().with_text("then say done")) + .await; + + let awaiting_confirmation = pipeline.run(["add four and halve it"]).await?; + assert_eq!(api.call_count(), 1); + assert!(pipeline.has_pending_steers().await); + assert!(!awaiting_confirmation + .conversation() + .messages() + .iter() + .any(|message| message.as_concat_text() == "then say done")); + + api.on("then say done").reply("done"); + pipeline.confirm("gated", Permission::AllowOnce).await?; + let result = pipeline.resume().await?; + result.assert_message(-1, Agent, "done"); + assert_eq!(pipeline.calculator_total(), 2); + assert!(!pipeline.has_pending_steers().await); + let messages = result.conversation().messages(); + let steer = messages + .iter() + .position(|message| message.as_concat_text() == "then say done") + .expect("steer was persisted"); + let gated_answer = messages + .iter() + .position(|message| message.get_tool_response_ids().contains("gated")) + .expect("gated call was answered"); + assert!(gated_answer < steer); + assert!(api.calls().last().unwrap().answers_every_tool_call_next()); + Ok(()) +} + #[tokio::test] async fn basic_tool_calling() -> Result<()> { let (pipeline, api) = test_pipeline().await?;