chore: use a Conversation type (#3735)

This commit is contained in:
Jack Amadeo
2025-08-06 22:06:13 -04:00
committed by GitHub
parent 17ea675def
commit 5eb01004a4
90 changed files with 994 additions and 790 deletions
+3 -3
View File
@@ -1,6 +1,6 @@
use async_trait::async_trait;
use chrono::{DateTime, Utc};
use goose::message::Message;
use goose::conversation::Conversation;
use serde::{Deserialize, Serialize};
use std::path::PathBuf;
@@ -19,7 +19,7 @@ pub struct BenchAgentError {
pub trait BenchBaseSession: Send + Sync {
async fn headless(&mut self, message: String) -> anyhow::Result<()>;
fn session_file(&self) -> Option<PathBuf>;
fn message_history(&self) -> Vec<Message>;
fn message_history(&self) -> Conversation;
fn get_total_token_usage(&self) -> anyhow::Result<Option<i32>>;
}
// struct for managing agent-session-access. to be passed to evals for benchmarking
@@ -34,7 +34,7 @@ impl BenchAgent {
Self { session, errors }
}
pub(crate) async fn prompt(&mut self, p: String) -> anyhow::Result<Vec<Message>> {
pub(crate) async fn prompt(&mut self, p: String) -> anyhow::Result<Conversation> {
// Clear previous errors
{
let mut errors = self.errors.lock().await;
@@ -8,7 +8,7 @@ use crate::eval_suites::{
};
use crate::register_evaluation;
use async_trait::async_trait;
use goose::message::MessageContent;
use goose::conversation::message::MessageContent;
use rmcp::model::Role;
use serde_json::{self, Value};
@@ -8,7 +8,7 @@ use crate::eval_suites::{
};
use crate::register_evaluation;
use async_trait::async_trait;
use goose::message::MessageContent;
use goose::conversation::message::MessageContent;
use rmcp::model::Role;
use serde_json::{self, Value};
@@ -8,7 +8,7 @@ use crate::eval_suites::{
};
use crate::register_evaluation;
use async_trait::async_trait;
use goose::message::MessageContent;
use goose::conversation::message::MessageContent;
use rmcp::model::Role;
use serde_json::{self, Value};
@@ -6,7 +6,7 @@ use crate::eval_suites::{
};
use crate::register_evaluation;
use async_trait::async_trait;
use goose::message::MessageContent;
use goose::conversation::message::MessageContent;
use rmcp::model::Role;
use serde_json::{self, Value};
@@ -6,7 +6,7 @@ use crate::eval_suites::{
};
use crate::register_evaluation;
use async_trait::async_trait;
use goose::message::MessageContent;
use goose::conversation::message::MessageContent;
use rmcp::model::Role;
use serde_json::{self, Value};
@@ -6,7 +6,7 @@ use crate::eval_suites::{
};
use crate::register_evaluation;
use async_trait::async_trait;
use goose::message::MessageContent;
use goose::conversation::message::MessageContent;
use rmcp::model::Role;
use serde_json::{self, Value};
@@ -8,7 +8,7 @@ use crate::eval_suites::{
};
use crate::register_evaluation;
use async_trait::async_trait;
use goose::message::MessageContent;
use goose::conversation::message::MessageContent;
use rmcp::model::Role;
use serde_json::{self, Value};
@@ -1,6 +1,7 @@
use crate::bench_session::BenchAgent;
use crate::eval_suites::EvalMetricValue;
use goose::message::{Message, MessageContent};
use goose::conversation::message::{Message, MessageContent};
use goose::conversation::Conversation;
use std::collections::HashMap;
use std::time::Instant;
@@ -8,7 +9,7 @@ use std::time::Instant;
pub async fn collect_baseline_metrics(
agent: &mut BenchAgent,
prompt: String,
) -> (Vec<Message>, HashMap<String, EvalMetricValue>) {
) -> (Conversation, HashMap<String, EvalMetricValue>) {
// Initialize metrics map
let mut metrics = HashMap::new();
@@ -23,7 +24,7 @@ pub async fn collect_baseline_metrics(
"prompt_error".to_string(),
EvalMetricValue::String(format!("Error: {}", e)),
);
Vec::new()
Conversation::new_unvalidated(Vec::new())
}
};
@@ -35,7 +36,7 @@ pub async fn collect_baseline_metrics(
);
// Count tool calls
let (total_tool_calls, tool_calls_by_name) = count_tool_calls(&messages);
let (total_tool_calls, tool_calls_by_name) = count_tool_calls(messages.messages());
metrics.insert(
"total_tool_calls".to_string(),
EvalMetricValue::Integer(total_tool_calls),
+1 -1
View File
@@ -1,6 +1,6 @@
use crate::bench_work_dir::BenchmarkWorkDir;
use anyhow::{Context, Result};
use goose::message::Message;
use goose::conversation::message::Message;
use std::fs::File;
use std::io::Write;
use std::path::PathBuf;
@@ -37,7 +37,7 @@ impl Evaluation for BlogSummary {
// Write response to file and get the text content
let response_text =
match write_response_to_file(&response, run_loc, "blog_summary_output.txt") {
match write_response_to_file(response.messages(), run_loc, "blog_summary_output.txt") {
Ok(text) => text,
Err(e) => {
println!("Warning: Failed to write blog summary output: {}", e);
@@ -59,7 +59,7 @@ impl Evaluation for BlogSummary {
));
// Check if the fetch tool was used
let used_fetch_tool = crate::eval_suites::used_tool(&response, "fetch");
let used_fetch_tool = crate::eval_suites::used_tool(response.messages(), "fetch");
metrics.push((
"used_fetch_tool".to_string(),
EvalMetricValue::Boolean(used_fetch_tool),
@@ -6,7 +6,7 @@ use crate::eval_suites::{
};
use crate::register_evaluation;
use async_trait::async_trait;
use goose::message::MessageContent;
use goose::conversation::message::MessageContent;
use rmcp::model::Role;
use serde_json::{self, Value};
use std::fs;
@@ -6,7 +6,7 @@ use crate::eval_suites::{
};
use crate::register_evaluation;
use async_trait::async_trait;
use goose::message::MessageContent;
use goose::conversation::message::MessageContent;
use rmcp::model::Role;
use serde_json::{self, Value};
use std::fs;
@@ -50,17 +50,20 @@ Present the information in order of significance or quality. Focus specifically
).await;
// Write response to file and get the text content
let response_text =
match write_response_to_file(&response, run_loc, "restaurant_research_output.txt") {
Ok(text) => text,
Err(e) => {
println!("Warning: Failed to write restaurant research output: {}", e);
// If file write fails, still continue with the evaluation
response
.last()
.map_or_else(String::new, |msg| msg.as_concat_text())
}
};
let response_text = match write_response_to_file(
response.messages(),
run_loc,
"restaurant_research_output.txt",
) {
Ok(text) => text,
Err(e) => {
println!("Warning: Failed to write restaurant research output: {}", e);
// If file write fails, still continue with the evaluation
response
.last()
.map_or_else(String::new, |msg| msg.as_concat_text())
}
};
// Convert HashMap to Vec for our metrics
let mut metrics = metrics_hashmap_to_vec(perf_metrics);
@@ -79,7 +82,7 @@ Present the information in order of significance or quality. Focus specifically
));
// Check if the fetch tool was used
let used_fetch_tool = crate::eval_suites::used_tool(&response, "fetch");
let used_fetch_tool = crate::eval_suites::used_tool(response.messages(), "fetch");
metrics.push((
"used_fetch_tool".to_string(),
EvalMetricValue::Boolean(used_fetch_tool),
@@ -6,7 +6,7 @@ use crate::eval_suites::{
};
use crate::register_evaluation;
use async_trait::async_trait;
use goose::message::MessageContent;
use goose::conversation::message::MessageContent;
use rmcp::model::Role;
use serde_json::{self, Value};