tkmind/tkmind_go
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

feat: add default metrics for core evals (#1602)

Browse Source
This commit is contained in:
Zaki Ali
2025-03-14 18:11:31 -07:00
committed by GitHub
parent 7f445b1d6b
commit a9fefd0e43
7 changed files with 78 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/goose-bench/src/eval_suites/core/create_file.rs
+11 -4
View File
@@ -1,7 +1,10 @@
// Create a new file called test.txt with the content 'Hello, World!
use crate::bench_work_dir::BenchmarkWorkDir;
use crate::eval_suites::{BenchAgent, Evaluation, EvaluationMetric, ExtensionRequirements};
use crate::eval_suites::{
collect_baseline_metrics, metrics_hashmap_to_vec, BenchAgent, Evaluation, EvaluationMetric,
ExtensionRequirements,
};
use crate::register_evaluation;
use async_trait::async_trait;
use goose::message::MessageContent;
@@ -24,10 +27,14 @@ impl Evaluation for DeveloperCreateFile {
mut agent: Box<dyn BenchAgent>,
_work_dir: &mut BenchmarkWorkDir,
) -> anyhow::Result<Vec<(String, EvaluationMetric)>> {
let mut metrics = Vec::new();
// Send the prompt to create and read
let (messages, perf_metrics) = collect_baseline_metrics(
&mut agent,
"Create a new file called test.txt in the current directory with the content 'Hello, World!'. Then read the contents of the new file to confirm.".to_string()
).await;
// Send the prompt to create and read file
let messages = agent.prompt("Create a new file called test.txt in the current directory with the content 'Hello, World!'. Then read the contents of the new file to confirm.".to_string()).await?;
// Convert HashMap to Vec for our metrics
let mut metrics = metrics_hashmap_to_vec(perf_metrics);
// Check for write operation
let write_tool_call = messages.iter().any(|msg| {
crates/goose-bench/src/eval_suites/core/image.rs
+12 -6
View File
@@ -1,5 +1,8 @@
use crate::bench_work_dir::BenchmarkWorkDir;
use crate::eval_suites::{BenchAgent, Evaluation, EvaluationMetric, ExtensionRequirements};
use crate::eval_suites::{
collect_baseline_metrics, metrics_hashmap_to_vec, BenchAgent, Evaluation, EvaluationMetric,
ExtensionRequirements,
};
use crate::register_evaluation;
use async_trait::async_trait;
use goose::message::MessageContent;
@@ -23,12 +26,15 @@ impl Evaluation for DeveloperImage {
mut agent: Box<dyn BenchAgent>,
_work_dir: &mut BenchmarkWorkDir,
) -> anyhow::Result<Vec<(String, EvaluationMetric)>> {
let mut metrics = Vec::new();
// Send the prompt to list files
let messages = agent
.prompt("Take a screenshot of the display 0 and describe what you see.".to_string())
.await?;
let (messages, perf_metrics) = collect_baseline_metrics(
&mut agent,
"Take a screenshot of the display 0 and describe what you see.".to_string(),
)
.await;
// Convert HashMap to Vec for our metrics
let mut metrics = metrics_hashmap_to_vec(perf_metrics);
// Check if the assistant makes appropriate tool calls and gets valid responses
let mut valid_tool_call = false;
crates/goose-bench/src/eval_suites/core/list_files.rs
+12 -7
View File
@@ -1,5 +1,8 @@
use crate::bench_work_dir::BenchmarkWorkDir;
use crate::eval_suites::{BenchAgent, Evaluation, EvaluationMetric, ExtensionRequirements};
use crate::eval_suites::{
collect_baseline_metrics, metrics_hashmap_to_vec, BenchAgent, Evaluation, EvaluationMetric,
ExtensionRequirements,
};
use crate::register_evaluation;
use async_trait::async_trait;
use goose::message::MessageContent;
@@ -22,13 +25,15 @@ impl Evaluation for DeveloperListFiles {
mut agent: Box<dyn BenchAgent>,
_work_dir: &mut BenchmarkWorkDir,
) -> anyhow::Result<Vec<(String, EvaluationMetric)>> {
let mut metrics = Vec::new();
// Send the prompt to list files
let messages = agent
.prompt("list the files in the current directory".to_string())
.await?;
// println!("asdhflkahjsdflkasdfl");
let (messages, perf_metrics) = collect_baseline_metrics(
&mut agent,
"list the files in the current directory".to_string(),
)
.await;
// Convert HashMap to Vec for our metrics
let mut metrics = metrics_hashmap_to_vec(perf_metrics);
// Check if the assistant makes appropriate tool calls
let valid_tool_call = messages.iter().any(|msg| {
crates/goose-bench/src/eval_suites/core/save_fact.rs
+12 -5
View File
@@ -1,7 +1,10 @@
// Create a new file called test.txt with the content 'Hello, World!
use crate::bench_work_dir::BenchmarkWorkDir;
use crate::eval_suites::{BenchAgent, Evaluation, EvaluationMetric, ExtensionRequirements};
use crate::eval_suites::{
collect_baseline_metrics, metrics_hashmap_to_vec, BenchAgent, Evaluation, EvaluationMetric,
ExtensionRequirements,
};
use crate::register_evaluation;
use async_trait::async_trait;
use goose::message::MessageContent;
@@ -24,11 +27,15 @@ impl Evaluation for MemoryRememberMemory {
mut agent: Box<dyn BenchAgent>,
_work_dir: &mut BenchmarkWorkDir,
) -> anyhow::Result<Vec<(String, EvaluationMetric)>> {
let mut metrics = Vec::new();
// Send the prompt to list files
let messages = agent.prompt("Save this fact: The capital of France is Paris.".to_string());
let messages = messages.await?;
let (messages, perf_metrics) = collect_baseline_metrics(
&mut agent,
"Save this fact: The capital of France is Paris.".to_string(),
)
.await;
// Convert HashMap to Vec for our metrics
let mut metrics = metrics_hashmap_to_vec(perf_metrics);
let valid_tool_call = messages.iter().any(|msg| {
// Check if it's an assistant message
crates/goose-bench/src/eval_suites/core/script.rs
+9 -5
View File
@@ -1,7 +1,10 @@
// Create a new file called test.txt with the content 'Hello, World!
use crate::bench_work_dir::BenchmarkWorkDir;
use crate::eval_suites::{BenchAgent, Evaluation, EvaluationMetric, ExtensionRequirements};
use crate::eval_suites::{
collect_baseline_metrics, metrics_hashmap_to_vec, BenchAgent, Evaluation, EvaluationMetric,
ExtensionRequirements,
};
use crate::register_evaluation;
use async_trait::async_trait;
use goose::message::MessageContent;
@@ -24,11 +27,12 @@ impl Evaluation for ComputerControllerScript {
mut agent: Box<dyn BenchAgent>,
_work_dir: &mut BenchmarkWorkDir,
) -> anyhow::Result<Vec<(String, EvaluationMetric)>> {
let mut metrics = Vec::new();
// Send the prompt to list files
let messages = agent.prompt("Make a beep sound".to_string());
let messages = messages.await?;
let (messages, perf_metrics) =
collect_baseline_metrics(&mut agent, "Make a beep sound".to_string()).await;
// Convert HashMap to Vec for our metrics
let mut metrics = metrics_hashmap_to_vec(perf_metrics);
let valid_tool_call = messages.iter().any(|msg| {
// Check if it's an assistant message
crates/goose-bench/src/eval_suites/core/search_replace.rs
+11 -4
View File
@@ -1,5 +1,8 @@
use crate::bench_work_dir::BenchmarkWorkDir;
use crate::eval_suites::{BenchAgent, Evaluation, EvaluationMetric, ExtensionRequirements};
use crate::eval_suites::{
collect_baseline_metrics, metrics_hashmap_to_vec, BenchAgent, Evaluation, EvaluationMetric,
ExtensionRequirements,
};
use crate::register_evaluation;
use async_trait::async_trait;
use std::fs;
@@ -20,8 +23,6 @@ impl Evaluation for DeveloperSearchReplace {
mut agent: Box<dyn BenchAgent>,
work_dir: &mut BenchmarkWorkDir,
) -> anyhow::Result<Vec<(String, EvaluationMetric)>> {
let mut metrics = Vec::new();
let _target_file = match work_dir.fs_get("./assets/kubernetes_swagger.json".to_string()) {
Ok(file) => file,
Err(_) => {
@@ -34,7 +35,13 @@ impl Evaluation for DeveloperSearchReplace {
source_file.push("assets/kubernetes_swagger.json");
// Send the prompt to modify the file
let _messages = agent.prompt("Remove the io.k8s.api.admissionregistration.v1.ServiceReference definition block and replace with a new definition for io.k8s.api.admissionregistration.v1.FakeServiceReference. Update the fields in the definition as well to be consistent. Don't change the property names. Don't update any references to the old definition. Only modify the definition and it's description to 'FakeServiceReference simulates a reference to a fake service for testing purposes.'.The file to modify is kubernetes_swagger.json.".to_string()).await?;
let (_messages, perf_metrics) = collect_baseline_metrics(
&mut agent,
"Remove the io.k8s.api.admissionregistration.v1.ServiceReference definition block and replace with a new definition for io.k8s.api.admissionregistration.v1.FakeServiceReference. Update the fields in the definition as well to be consistent. Don't change the property names. Don't update any references to the old definition. Only modify the definition and it's description to 'FakeServiceReference simulates a reference to a fake service for testing purposes.'.The file to modify is kubernetes_swagger.json.".to_string()
).await;
// Convert HashMap to Vec for our metrics
let mut metrics = metrics_hashmap_to_vec(perf_metrics);
// Get the path to the modified file
let modified_file_path = std::env::current_dir()
crates/goose-bench/src/eval_suites/core/web_scrape.rs
+11 -6
View File
@@ -1,7 +1,10 @@
// Create a new file called test.txt with the content 'Hello, World!
use crate::bench_work_dir::BenchmarkWorkDir;
use crate::eval_suites::{BenchAgent, Evaluation, EvaluationMetric, ExtensionRequirements};
use crate::eval_suites::{
collect_baseline_metrics, metrics_hashmap_to_vec, BenchAgent, Evaluation, EvaluationMetric,
ExtensionRequirements,
};
use crate::register_evaluation;
use async_trait::async_trait;
use goose::message::MessageContent;
@@ -24,13 +27,15 @@ impl Evaluation for ComputerControllerWebScrape {
mut agent: Box<dyn BenchAgent>,
_work_dir: &mut BenchmarkWorkDir,
) -> anyhow::Result<Vec<(String, EvaluationMetric)>> {
let mut metrics = Vec::new();
// Send the prompt to list files
let messages = agent.prompt(
let (messages, perf_metrics) = collect_baseline_metrics(
&mut agent,
"What are the headlines on hackernews? Organize the list into categories.".to_string(),
);
let messages = messages.await?;
)
.await;
// Convert HashMap to Vec for our metrics
let mut metrics = metrics_hashmap_to_vec(perf_metrics);
let valid_tool_call = messages.iter().any(|msg| {
// Check if it's an assistant message