From a9fefd0e43385568be6515dff3562d000d7cabdc Mon Sep 17 00:00:00 2001 From: Zaki Ali Date: Fri, 14 Mar 2025 18:11:31 -0700 Subject: [PATCH] feat: add default metrics for core evals (#1602) --- .../src/eval_suites/core/create_file.rs | 15 +++++++++++---- .../goose-bench/src/eval_suites/core/image.rs | 18 ++++++++++++------ .../src/eval_suites/core/list_files.rs | 19 ++++++++++++------- .../src/eval_suites/core/save_fact.rs | 17 ++++++++++++----- .../src/eval_suites/core/script.rs | 14 +++++++++----- .../src/eval_suites/core/search_replace.rs | 15 +++++++++++---- .../src/eval_suites/core/web_scrape.rs | 17 +++++++++++------ 7 files changed, 78 insertions(+), 37 deletions(-) diff --git a/crates/goose-bench/src/eval_suites/core/create_file.rs b/crates/goose-bench/src/eval_suites/core/create_file.rs index bb177438..ce493146 100644 --- a/crates/goose-bench/src/eval_suites/core/create_file.rs +++ b/crates/goose-bench/src/eval_suites/core/create_file.rs @@ -1,7 +1,10 @@ // Create a new file called test.txt with the content 'Hello, World! use crate::bench_work_dir::BenchmarkWorkDir; -use crate::eval_suites::{BenchAgent, Evaluation, EvaluationMetric, ExtensionRequirements}; +use crate::eval_suites::{ + collect_baseline_metrics, metrics_hashmap_to_vec, BenchAgent, Evaluation, EvaluationMetric, + ExtensionRequirements, +}; use crate::register_evaluation; use async_trait::async_trait; use goose::message::MessageContent; @@ -24,10 +27,14 @@ impl Evaluation for DeveloperCreateFile { mut agent: Box, _work_dir: &mut BenchmarkWorkDir, ) -> anyhow::Result> { - let mut metrics = Vec::new(); + // Send the prompt to create and read + let (messages, perf_metrics) = collect_baseline_metrics( + &mut agent, + "Create a new file called test.txt in the current directory with the content 'Hello, World!'. Then read the contents of the new file to confirm.".to_string() + ).await; - // Send the prompt to create and read file - let messages = agent.prompt("Create a new file called test.txt in the current directory with the content 'Hello, World!'. Then read the contents of the new file to confirm.".to_string()).await?; + // Convert HashMap to Vec for our metrics + let mut metrics = metrics_hashmap_to_vec(perf_metrics); // Check for write operation let write_tool_call = messages.iter().any(|msg| { diff --git a/crates/goose-bench/src/eval_suites/core/image.rs b/crates/goose-bench/src/eval_suites/core/image.rs index c80b3d3f..81cfda58 100644 --- a/crates/goose-bench/src/eval_suites/core/image.rs +++ b/crates/goose-bench/src/eval_suites/core/image.rs @@ -1,5 +1,8 @@ use crate::bench_work_dir::BenchmarkWorkDir; -use crate::eval_suites::{BenchAgent, Evaluation, EvaluationMetric, ExtensionRequirements}; +use crate::eval_suites::{ + collect_baseline_metrics, metrics_hashmap_to_vec, BenchAgent, Evaluation, EvaluationMetric, + ExtensionRequirements, +}; use crate::register_evaluation; use async_trait::async_trait; use goose::message::MessageContent; @@ -23,12 +26,15 @@ impl Evaluation for DeveloperImage { mut agent: Box, _work_dir: &mut BenchmarkWorkDir, ) -> anyhow::Result> { - let mut metrics = Vec::new(); - // Send the prompt to list files - let messages = agent - .prompt("Take a screenshot of the display 0 and describe what you see.".to_string()) - .await?; + let (messages, perf_metrics) = collect_baseline_metrics( + &mut agent, + "Take a screenshot of the display 0 and describe what you see.".to_string(), + ) + .await; + + // Convert HashMap to Vec for our metrics + let mut metrics = metrics_hashmap_to_vec(perf_metrics); // Check if the assistant makes appropriate tool calls and gets valid responses let mut valid_tool_call = false; diff --git a/crates/goose-bench/src/eval_suites/core/list_files.rs b/crates/goose-bench/src/eval_suites/core/list_files.rs index e419c300..76015217 100644 --- a/crates/goose-bench/src/eval_suites/core/list_files.rs +++ b/crates/goose-bench/src/eval_suites/core/list_files.rs @@ -1,5 +1,8 @@ use crate::bench_work_dir::BenchmarkWorkDir; -use crate::eval_suites::{BenchAgent, Evaluation, EvaluationMetric, ExtensionRequirements}; +use crate::eval_suites::{ + collect_baseline_metrics, metrics_hashmap_to_vec, BenchAgent, Evaluation, EvaluationMetric, + ExtensionRequirements, +}; use crate::register_evaluation; use async_trait::async_trait; use goose::message::MessageContent; @@ -22,13 +25,15 @@ impl Evaluation for DeveloperListFiles { mut agent: Box, _work_dir: &mut BenchmarkWorkDir, ) -> anyhow::Result> { - let mut metrics = Vec::new(); - // Send the prompt to list files - let messages = agent - .prompt("list the files in the current directory".to_string()) - .await?; - // println!("asdhflkahjsdflkasdfl"); + let (messages, perf_metrics) = collect_baseline_metrics( + &mut agent, + "list the files in the current directory".to_string(), + ) + .await; + + // Convert HashMap to Vec for our metrics + let mut metrics = metrics_hashmap_to_vec(perf_metrics); // Check if the assistant makes appropriate tool calls let valid_tool_call = messages.iter().any(|msg| { diff --git a/crates/goose-bench/src/eval_suites/core/save_fact.rs b/crates/goose-bench/src/eval_suites/core/save_fact.rs index 6446817e..39f4d154 100644 --- a/crates/goose-bench/src/eval_suites/core/save_fact.rs +++ b/crates/goose-bench/src/eval_suites/core/save_fact.rs @@ -1,7 +1,10 @@ // Create a new file called test.txt with the content 'Hello, World! use crate::bench_work_dir::BenchmarkWorkDir; -use crate::eval_suites::{BenchAgent, Evaluation, EvaluationMetric, ExtensionRequirements}; +use crate::eval_suites::{ + collect_baseline_metrics, metrics_hashmap_to_vec, BenchAgent, Evaluation, EvaluationMetric, + ExtensionRequirements, +}; use crate::register_evaluation; use async_trait::async_trait; use goose::message::MessageContent; @@ -24,11 +27,15 @@ impl Evaluation for MemoryRememberMemory { mut agent: Box, _work_dir: &mut BenchmarkWorkDir, ) -> anyhow::Result> { - let mut metrics = Vec::new(); - // Send the prompt to list files - let messages = agent.prompt("Save this fact: The capital of France is Paris.".to_string()); - let messages = messages.await?; + let (messages, perf_metrics) = collect_baseline_metrics( + &mut agent, + "Save this fact: The capital of France is Paris.".to_string(), + ) + .await; + + // Convert HashMap to Vec for our metrics + let mut metrics = metrics_hashmap_to_vec(perf_metrics); let valid_tool_call = messages.iter().any(|msg| { // Check if it's an assistant message diff --git a/crates/goose-bench/src/eval_suites/core/script.rs b/crates/goose-bench/src/eval_suites/core/script.rs index 19362ffb..0c905273 100644 --- a/crates/goose-bench/src/eval_suites/core/script.rs +++ b/crates/goose-bench/src/eval_suites/core/script.rs @@ -1,7 +1,10 @@ // Create a new file called test.txt with the content 'Hello, World! use crate::bench_work_dir::BenchmarkWorkDir; -use crate::eval_suites::{BenchAgent, Evaluation, EvaluationMetric, ExtensionRequirements}; +use crate::eval_suites::{ + collect_baseline_metrics, metrics_hashmap_to_vec, BenchAgent, Evaluation, EvaluationMetric, + ExtensionRequirements, +}; use crate::register_evaluation; use async_trait::async_trait; use goose::message::MessageContent; @@ -24,11 +27,12 @@ impl Evaluation for ComputerControllerScript { mut agent: Box, _work_dir: &mut BenchmarkWorkDir, ) -> anyhow::Result> { - let mut metrics = Vec::new(); - // Send the prompt to list files - let messages = agent.prompt("Make a beep sound".to_string()); - let messages = messages.await?; + let (messages, perf_metrics) = + collect_baseline_metrics(&mut agent, "Make a beep sound".to_string()).await; + + // Convert HashMap to Vec for our metrics + let mut metrics = metrics_hashmap_to_vec(perf_metrics); let valid_tool_call = messages.iter().any(|msg| { // Check if it's an assistant message diff --git a/crates/goose-bench/src/eval_suites/core/search_replace.rs b/crates/goose-bench/src/eval_suites/core/search_replace.rs index 285a7f69..92daccfc 100644 --- a/crates/goose-bench/src/eval_suites/core/search_replace.rs +++ b/crates/goose-bench/src/eval_suites/core/search_replace.rs @@ -1,5 +1,8 @@ use crate::bench_work_dir::BenchmarkWorkDir; -use crate::eval_suites::{BenchAgent, Evaluation, EvaluationMetric, ExtensionRequirements}; +use crate::eval_suites::{ + collect_baseline_metrics, metrics_hashmap_to_vec, BenchAgent, Evaluation, EvaluationMetric, + ExtensionRequirements, +}; use crate::register_evaluation; use async_trait::async_trait; use std::fs; @@ -20,8 +23,6 @@ impl Evaluation for DeveloperSearchReplace { mut agent: Box, work_dir: &mut BenchmarkWorkDir, ) -> anyhow::Result> { - let mut metrics = Vec::new(); - let _target_file = match work_dir.fs_get("./assets/kubernetes_swagger.json".to_string()) { Ok(file) => file, Err(_) => { @@ -34,7 +35,13 @@ impl Evaluation for DeveloperSearchReplace { source_file.push("assets/kubernetes_swagger.json"); // Send the prompt to modify the file - let _messages = agent.prompt("Remove the io.k8s.api.admissionregistration.v1.ServiceReference definition block and replace with a new definition for io.k8s.api.admissionregistration.v1.FakeServiceReference. Update the fields in the definition as well to be consistent. Don't change the property names. Don't update any references to the old definition. Only modify the definition and it's description to 'FakeServiceReference simulates a reference to a fake service for testing purposes.'.The file to modify is kubernetes_swagger.json.".to_string()).await?; + let (_messages, perf_metrics) = collect_baseline_metrics( + &mut agent, + "Remove the io.k8s.api.admissionregistration.v1.ServiceReference definition block and replace with a new definition for io.k8s.api.admissionregistration.v1.FakeServiceReference. Update the fields in the definition as well to be consistent. Don't change the property names. Don't update any references to the old definition. Only modify the definition and it's description to 'FakeServiceReference simulates a reference to a fake service for testing purposes.'.The file to modify is kubernetes_swagger.json.".to_string() + ).await; + + // Convert HashMap to Vec for our metrics + let mut metrics = metrics_hashmap_to_vec(perf_metrics); // Get the path to the modified file let modified_file_path = std::env::current_dir() diff --git a/crates/goose-bench/src/eval_suites/core/web_scrape.rs b/crates/goose-bench/src/eval_suites/core/web_scrape.rs index 2c3d25db..8b996729 100644 --- a/crates/goose-bench/src/eval_suites/core/web_scrape.rs +++ b/crates/goose-bench/src/eval_suites/core/web_scrape.rs @@ -1,7 +1,10 @@ // Create a new file called test.txt with the content 'Hello, World! use crate::bench_work_dir::BenchmarkWorkDir; -use crate::eval_suites::{BenchAgent, Evaluation, EvaluationMetric, ExtensionRequirements}; +use crate::eval_suites::{ + collect_baseline_metrics, metrics_hashmap_to_vec, BenchAgent, Evaluation, EvaluationMetric, + ExtensionRequirements, +}; use crate::register_evaluation; use async_trait::async_trait; use goose::message::MessageContent; @@ -24,13 +27,15 @@ impl Evaluation for ComputerControllerWebScrape { mut agent: Box, _work_dir: &mut BenchmarkWorkDir, ) -> anyhow::Result> { - let mut metrics = Vec::new(); - // Send the prompt to list files - let messages = agent.prompt( + let (messages, perf_metrics) = collect_baseline_metrics( + &mut agent, "What are the headlines on hackernews? Organize the list into categories.".to_string(), - ); - let messages = messages.await?; + ) + .await; + + // Convert HashMap to Vec for our metrics + let mut metrics = metrics_hashmap_to_vec(perf_metrics); let valid_tool_call = messages.iter().any(|msg| { // Check if it's an assistant message