From c7195186836f83d2f878c0caf5447257c0496d35 Mon Sep 17 00:00:00 2001 From: Michael Neale Date: Mon, 7 Apr 2025 08:48:30 +1000 Subject: [PATCH] chore: add to the benchmark suite core developer a git project scenario (#2032) --- .../src/eval_suites/core/developer/mod.rs | 1 + .../core/developer/simple_repo_clone_test.rs | 214 ++++++++++++++++++ 2 files changed, 215 insertions(+) create mode 100644 crates/goose-bench/src/eval_suites/core/developer/simple_repo_clone_test.rs diff --git a/crates/goose-bench/src/eval_suites/core/developer/mod.rs b/crates/goose-bench/src/eval_suites/core/developer/mod.rs index 1b1fc7bc..d2d83f65 100644 --- a/crates/goose-bench/src/eval_suites/core/developer/mod.rs +++ b/crates/goose-bench/src/eval_suites/core/developer/mod.rs @@ -1,3 +1,4 @@ // developer extension evals mod create_file; mod list_files; +mod simple_repo_clone_test; diff --git a/crates/goose-bench/src/eval_suites/core/developer/simple_repo_clone_test.rs b/crates/goose-bench/src/eval_suites/core/developer/simple_repo_clone_test.rs new file mode 100644 index 00000000..b1af460d --- /dev/null +++ b/crates/goose-bench/src/eval_suites/core/developer/simple_repo_clone_test.rs @@ -0,0 +1,214 @@ +use crate::bench_work_dir::BenchmarkWorkDir; +use crate::eval_suites::{ + collect_baseline_metrics, metrics_hashmap_to_vec, BenchAgent, Evaluation, EvaluationMetric, + ExtensionRequirements, +}; +use crate::register_evaluation; +use async_trait::async_trait; +use goose::message::MessageContent; +use mcp_core::role::Role; +use serde_json::{self, Value}; + +#[derive(Debug)] +pub struct SimpleRepoCloneTest {} + +impl SimpleRepoCloneTest { + pub fn new() -> Self { + SimpleRepoCloneTest {} + } +} + +#[async_trait] +impl Evaluation for SimpleRepoCloneTest { + async fn run( + &self, + mut agent: Box, + _work_dir: &mut BenchmarkWorkDir, + ) -> anyhow::Result> { + // Send the prompt to clone the repo and add a test + let (messages, perf_metrics) = collect_baseline_metrics( + &mut agent, + "Clone the Git repository https://github.com/michaelneale/mcp-read-pdf to a temporary location. \ + Then add a new test file that verifies the PDF reading functionality. The test should \ + check if the PDF content can be read and processed correctly.".to_string() + ).await; + + // Convert HashMap to Vec for our metrics + let mut metrics = metrics_hashmap_to_vec(perf_metrics); + + // Check for git clone operation + let git_clone_executed = messages.iter().any(|msg| { + msg.role == Role::Assistant + && msg.content.iter().any(|content| { + if let MessageContent::ToolRequest(tool_req) = content { + if let Ok(tool_call) = tool_req.tool_call.as_ref() { + if tool_call.name != "developer__shell" { + return false; + } + + if let Ok(args) = + serde_json::from_value::(tool_call.arguments.clone()) + { + let command = args.get("command").and_then(Value::as_str); + command.is_some_and(|cmd| { + cmd.contains("git clone") + && cmd.contains("michaelneale/mcp-read-pdf") + }) + } else { + false + } + } else { + false + } + } else { + false + } + }) + }); + + // Check for exploring the repository structure + let repo_explored = messages.iter().any(|msg| { + msg.role == Role::Assistant + && msg.content.iter().any(|content| { + if let MessageContent::ToolRequest(tool_req) = content { + if let Ok(tool_call) = tool_req.tool_call.as_ref() { + if tool_call.name != "developer__shell" { + return false; + } + + if let Ok(args) = + serde_json::from_value::(tool_call.arguments.clone()) + { + let command = args.get("command").and_then(Value::as_str); + command.is_some_and(|cmd| { + (cmd.contains("ls") + || cmd.contains("find") + || cmd.contains("rg")) + && cmd.contains("mcp-read-pdf") + }) + } else { + false + } + } else { + false + } + } else { + false + } + }) + }); + + // Check for file creation to add a test + let test_added = messages.iter().any(|msg| { + msg.role == Role::Assistant + && msg.content.iter().any(|content| { + if let MessageContent::ToolRequest(tool_req) = content { + if let Ok(tool_call) = tool_req.tool_call.as_ref() { + if tool_call.name != "developer__text_editor" { + return false; + } + + if let Ok(args) = + serde_json::from_value::(tool_call.arguments.clone()) + { + let command = args.get("command").and_then(Value::as_str); + let file_text = args.get("file_text").and_then(Value::as_str); + let path = args.get("path").and_then(Value::as_str); + + command == Some("write") + && path.is_some_and(|p| { + p.contains("test") + || p.ends_with(".py") + || p.ends_with(".js") + || p.ends_with(".ts") + }) + && file_text.is_some_and(|text| { + text.contains("test") + || text.contains("assert") + || text.contains("expect") + || text.contains("should") + }) + } else { + false + } + } else { + false + } + } else { + false + } + }) + }); + + // Check if the agent ran the test + let test_executed = messages.iter().any(|msg| { + msg.role == Role::Assistant + && msg.content.iter().any(|content| { + if let MessageContent::ToolRequest(tool_req) = content { + if let Ok(tool_call) = tool_req.tool_call.as_ref() { + if tool_call.name != "developer__shell" { + return false; + } + + if let Ok(args) = + serde_json::from_value::(tool_call.arguments.clone()) + { + let command = args.get("command").and_then(Value::as_str); + command.is_some_and(|cmd| { + cmd.contains("test") + || cmd.contains("pytest") + || cmd.contains("jest") + || cmd.contains("mocha") + || (cmd.contains("node") && cmd.contains("test")) + || (cmd.contains("python") && cmd.contains("test")) + }) + } else { + false + } + } else { + false + } + } else { + false + } + }) + }); + + // Add metrics + metrics.push(( + "Git repo cloned".to_string(), + EvaluationMetric::Boolean(git_clone_executed), + )); + metrics.push(( + "Repository explored".to_string(), + EvaluationMetric::Boolean(repo_explored), + )); + metrics.push(( + "Test file added".to_string(), + EvaluationMetric::Boolean(test_added), + )); + metrics.push(( + "Test executed".to_string(), + EvaluationMetric::Boolean(test_executed), + )); + metrics.push(( + "Complete task".to_string(), + EvaluationMetric::Boolean(git_clone_executed && test_added), + )); + + Ok(metrics) + } + + fn name(&self) -> &str { + "simple_repo_clone_test" + } + + fn required_extensions(&self) -> ExtensionRequirements { + ExtensionRequirements { + builtin: vec!["developer".to_string()], + external: Vec::new(), + } + } +} + +register_evaluation!(SimpleRepoCloneTest);