chore: use a Conversation type (#3735)
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use goose::message::Message;
|
||||
use goose::conversation::Conversation;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::path::PathBuf;
|
||||
@@ -19,7 +19,7 @@ pub struct BenchAgentError {
|
||||
pub trait BenchBaseSession: Send + Sync {
|
||||
async fn headless(&mut self, message: String) -> anyhow::Result<()>;
|
||||
fn session_file(&self) -> Option<PathBuf>;
|
||||
fn message_history(&self) -> Vec<Message>;
|
||||
fn message_history(&self) -> Conversation;
|
||||
fn get_total_token_usage(&self) -> anyhow::Result<Option<i32>>;
|
||||
}
|
||||
// struct for managing agent-session-access. to be passed to evals for benchmarking
|
||||
@@ -34,7 +34,7 @@ impl BenchAgent {
|
||||
Self { session, errors }
|
||||
}
|
||||
|
||||
pub(crate) async fn prompt(&mut self, p: String) -> anyhow::Result<Vec<Message>> {
|
||||
pub(crate) async fn prompt(&mut self, p: String) -> anyhow::Result<Conversation> {
|
||||
// Clear previous errors
|
||||
{
|
||||
let mut errors = self.errors.lock().await;
|
||||
|
||||
@@ -8,7 +8,7 @@ use crate::eval_suites::{
|
||||
};
|
||||
use crate::register_evaluation;
|
||||
use async_trait::async_trait;
|
||||
use goose::message::MessageContent;
|
||||
use goose::conversation::message::MessageContent;
|
||||
use rmcp::model::Role;
|
||||
use serde_json::{self, Value};
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ use crate::eval_suites::{
|
||||
};
|
||||
use crate::register_evaluation;
|
||||
use async_trait::async_trait;
|
||||
use goose::message::MessageContent;
|
||||
use goose::conversation::message::MessageContent;
|
||||
use rmcp::model::Role;
|
||||
use serde_json::{self, Value};
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ use crate::eval_suites::{
|
||||
};
|
||||
use crate::register_evaluation;
|
||||
use async_trait::async_trait;
|
||||
use goose::message::MessageContent;
|
||||
use goose::conversation::message::MessageContent;
|
||||
use rmcp::model::Role;
|
||||
use serde_json::{self, Value};
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ use crate::eval_suites::{
|
||||
};
|
||||
use crate::register_evaluation;
|
||||
use async_trait::async_trait;
|
||||
use goose::message::MessageContent;
|
||||
use goose::conversation::message::MessageContent;
|
||||
use rmcp::model::Role;
|
||||
use serde_json::{self, Value};
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ use crate::eval_suites::{
|
||||
};
|
||||
use crate::register_evaluation;
|
||||
use async_trait::async_trait;
|
||||
use goose::message::MessageContent;
|
||||
use goose::conversation::message::MessageContent;
|
||||
use rmcp::model::Role;
|
||||
use serde_json::{self, Value};
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ use crate::eval_suites::{
|
||||
};
|
||||
use crate::register_evaluation;
|
||||
use async_trait::async_trait;
|
||||
use goose::message::MessageContent;
|
||||
use goose::conversation::message::MessageContent;
|
||||
use rmcp::model::Role;
|
||||
use serde_json::{self, Value};
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ use crate::eval_suites::{
|
||||
};
|
||||
use crate::register_evaluation;
|
||||
use async_trait::async_trait;
|
||||
use goose::message::MessageContent;
|
||||
use goose::conversation::message::MessageContent;
|
||||
use rmcp::model::Role;
|
||||
use serde_json::{self, Value};
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use crate::bench_session::BenchAgent;
|
||||
use crate::eval_suites::EvalMetricValue;
|
||||
use goose::message::{Message, MessageContent};
|
||||
use goose::conversation::message::{Message, MessageContent};
|
||||
use goose::conversation::Conversation;
|
||||
use std::collections::HashMap;
|
||||
use std::time::Instant;
|
||||
|
||||
@@ -8,7 +9,7 @@ use std::time::Instant;
|
||||
pub async fn collect_baseline_metrics(
|
||||
agent: &mut BenchAgent,
|
||||
prompt: String,
|
||||
) -> (Vec<Message>, HashMap<String, EvalMetricValue>) {
|
||||
) -> (Conversation, HashMap<String, EvalMetricValue>) {
|
||||
// Initialize metrics map
|
||||
let mut metrics = HashMap::new();
|
||||
|
||||
@@ -23,7 +24,7 @@ pub async fn collect_baseline_metrics(
|
||||
"prompt_error".to_string(),
|
||||
EvalMetricValue::String(format!("Error: {}", e)),
|
||||
);
|
||||
Vec::new()
|
||||
Conversation::new_unvalidated(Vec::new())
|
||||
}
|
||||
};
|
||||
|
||||
@@ -35,7 +36,7 @@ pub async fn collect_baseline_metrics(
|
||||
);
|
||||
|
||||
// Count tool calls
|
||||
let (total_tool_calls, tool_calls_by_name) = count_tool_calls(&messages);
|
||||
let (total_tool_calls, tool_calls_by_name) = count_tool_calls(messages.messages());
|
||||
metrics.insert(
|
||||
"total_tool_calls".to_string(),
|
||||
EvalMetricValue::Integer(total_tool_calls),
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use crate::bench_work_dir::BenchmarkWorkDir;
|
||||
use anyhow::{Context, Result};
|
||||
use goose::message::Message;
|
||||
use goose::conversation::message::Message;
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
|
||||
@@ -37,7 +37,7 @@ impl Evaluation for BlogSummary {
|
||||
|
||||
// Write response to file and get the text content
|
||||
let response_text =
|
||||
match write_response_to_file(&response, run_loc, "blog_summary_output.txt") {
|
||||
match write_response_to_file(response.messages(), run_loc, "blog_summary_output.txt") {
|
||||
Ok(text) => text,
|
||||
Err(e) => {
|
||||
println!("Warning: Failed to write blog summary output: {}", e);
|
||||
@@ -59,7 +59,7 @@ impl Evaluation for BlogSummary {
|
||||
));
|
||||
|
||||
// Check if the fetch tool was used
|
||||
let used_fetch_tool = crate::eval_suites::used_tool(&response, "fetch");
|
||||
let used_fetch_tool = crate::eval_suites::used_tool(response.messages(), "fetch");
|
||||
metrics.push((
|
||||
"used_fetch_tool".to_string(),
|
||||
EvalMetricValue::Boolean(used_fetch_tool),
|
||||
|
||||
@@ -6,7 +6,7 @@ use crate::eval_suites::{
|
||||
};
|
||||
use crate::register_evaluation;
|
||||
use async_trait::async_trait;
|
||||
use goose::message::MessageContent;
|
||||
use goose::conversation::message::MessageContent;
|
||||
use rmcp::model::Role;
|
||||
use serde_json::{self, Value};
|
||||
use std::fs;
|
||||
|
||||
@@ -6,7 +6,7 @@ use crate::eval_suites::{
|
||||
};
|
||||
use crate::register_evaluation;
|
||||
use async_trait::async_trait;
|
||||
use goose::message::MessageContent;
|
||||
use goose::conversation::message::MessageContent;
|
||||
use rmcp::model::Role;
|
||||
use serde_json::{self, Value};
|
||||
use std::fs;
|
||||
|
||||
@@ -50,17 +50,20 @@ Present the information in order of significance or quality. Focus specifically
|
||||
).await;
|
||||
|
||||
// Write response to file and get the text content
|
||||
let response_text =
|
||||
match write_response_to_file(&response, run_loc, "restaurant_research_output.txt") {
|
||||
Ok(text) => text,
|
||||
Err(e) => {
|
||||
println!("Warning: Failed to write restaurant research output: {}", e);
|
||||
// If file write fails, still continue with the evaluation
|
||||
response
|
||||
.last()
|
||||
.map_or_else(String::new, |msg| msg.as_concat_text())
|
||||
}
|
||||
};
|
||||
let response_text = match write_response_to_file(
|
||||
response.messages(),
|
||||
run_loc,
|
||||
"restaurant_research_output.txt",
|
||||
) {
|
||||
Ok(text) => text,
|
||||
Err(e) => {
|
||||
println!("Warning: Failed to write restaurant research output: {}", e);
|
||||
// If file write fails, still continue with the evaluation
|
||||
response
|
||||
.last()
|
||||
.map_or_else(String::new, |msg| msg.as_concat_text())
|
||||
}
|
||||
};
|
||||
|
||||
// Convert HashMap to Vec for our metrics
|
||||
let mut metrics = metrics_hashmap_to_vec(perf_metrics);
|
||||
@@ -79,7 +82,7 @@ Present the information in order of significance or quality. Focus specifically
|
||||
));
|
||||
|
||||
// Check if the fetch tool was used
|
||||
let used_fetch_tool = crate::eval_suites::used_tool(&response, "fetch");
|
||||
let used_fetch_tool = crate::eval_suites::used_tool(response.messages(), "fetch");
|
||||
metrics.push((
|
||||
"used_fetch_tool".to_string(),
|
||||
EvalMetricValue::Boolean(used_fetch_tool),
|
||||
|
||||
@@ -6,7 +6,7 @@ use crate::eval_suites::{
|
||||
};
|
||||
use crate::register_evaluation;
|
||||
use async_trait::async_trait;
|
||||
use goose::message::MessageContent;
|
||||
use goose::conversation::message::MessageContent;
|
||||
use rmcp::model::Role;
|
||||
use serde_json::{self, Value};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user