tkmind/tkmind_go
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

tidy: clean up old benchmark and add gym (#7081)

Browse Source
This commit is contained in:
Michael Neale
2026-02-09 17:08:46 +11:00
committed by GitHub
parent d4865ae9fe
commit a3ba124178
76 changed files with 6448 additions and 91262 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/goose-cli/src/cli.rs
+1 -89
View File
@@ -10,7 +10,6 @@ use goose_mcp::{
AutoVisualiserRouter, ComputerControllerServer, DeveloperServer, MemoryServer, TutorialServer,
};
use crate::commands::bench::agent_generator;
use crate::commands::configure::{configure_telemetry_consent_dialog, handle_configure};
use crate::commands::info::handle_info;
use crate::commands::project::{handle_project_default, handle_projects_interactive};
@@ -31,11 +30,6 @@ use crate::session::{build_session, SessionBuilderConfig};
use goose::agents::Container;
use goose::session::session_manager::SessionType;
use goose::session::SessionManager;
use goose_bench::bench_config::BenchRunConfig;
use goose_bench::runners::bench_runner::BenchRunner;
use goose_bench::runners::eval_runner::EvalRunner;
use goose_bench::runners::metric_aggregator::MetricAggregator;
use goose_bench::runners::model_runner::ModelRunner;
use std::io::Read;
use std::path::PathBuf;
use tracing::warn;
@@ -598,60 +592,6 @@ enum SchedulerCommand {
CronHelp {},
}
#[derive(Subcommand)]
pub enum BenchCommand {
#[command(name = "init-config", about = "Create a new starter-config")]
InitConfig {
#[arg(short, long, help = "filename with extension for generated config")]
name: String,
},
#[command(about = "Run all benchmarks from a config")]
Run {
#[arg(
short,
long,
help = "A config file generated by the config-init command"
)]
config: PathBuf,
},
#[command(about = "List all available selectors")]
Selectors {
#[arg(
short,
long,
help = "A config file generated by the config-init command"
)]
config: Option<PathBuf>,
},
#[command(name = "eval-model", about = "Run an eval of model")]
EvalModel {
#[arg(short, long, help = "A serialized config file for the model only.")]
config: String,
},
#[command(name = "exec-eval", about = "run a single eval")]
ExecEval {
#[arg(short, long, help = "A serialized config file for the eval only.")]
config: String,
},
#[command(
name = "generate-leaderboard",
about = "Generate a leaderboard CSV from benchmark results"
)]
GenerateLeaderboard {
#[arg(
short,
long,
help = "Path to the benchmark directory containing model evaluation results"
)]
benchmark_dir: PathBuf,
},
}
#[derive(Subcommand)]
enum RecipeCommand {
/// Validate a recipe file
@@ -862,13 +802,6 @@ enum Command {
reconfigure: bool,
},
/// Evaluate system configuration across a range of practical tasks
#[command(about = "Evaluate system configuration across a range of practical tasks")]
Bench {
#[command(subcommand)]
cmd: BenchCommand,
},
/// Start a web server with a chat interface
#[command(about = "Experimental: Start a web server with a chat interface")]
Web {
@@ -1018,7 +951,6 @@ fn get_command_name(command: &Option<Command>) -> &'static str {
Some(Command::Run { .. }) => "run",
Some(Command::Schedule { .. }) => "schedule",
Some(Command::Update { .. }) => "update",
Some(Command::Bench { .. }) => "bench",
Some(Command::Recipe { .. }) => "recipe",
Some(Command::Web { .. }) => "web",
Some(Command::Term { .. }) => "term",
@@ -1029,7 +961,7 @@ fn get_command_name(command: &Option<Command>) -> &'static str {
async fn handle_mcp_command(server: McpCommand) -> Result<()> {
let name = server.name();
crate::logging::setup_logging(Some(&format!("mcp-{name}")), None)?;
let _ = crate::logging::setup_logging(Some(&format!("mcp-{name}")));
match server {
McpCommand::AutoVisualiser => serve(AutoVisualiserRouter::new()).await?,
McpCommand::ComputerController => serve(ComputerControllerServer::new()).await?,
@@ -1426,25 +1358,6 @@ async fn handle_schedule_command(command: SchedulerCommand) -> Result<()> {
}
}
async fn handle_bench_command(cmd: BenchCommand) -> Result<()> {
match cmd {
BenchCommand::Selectors { config } => BenchRunner::list_selectors(config)?,
BenchCommand::InitConfig { name } => {
let mut config = BenchRunConfig::default();
let cwd = std::env::current_dir()?;
config.output_dir = Some(cwd);
config.save(name);
}
BenchCommand::Run { config } => BenchRunner::new(config)?.run()?,
BenchCommand::EvalModel { config } => ModelRunner::from(config)?.run()?,
BenchCommand::ExecEval { config } => EvalRunner::from(config)?.run(agent_generator).await?,
BenchCommand::GenerateLeaderboard { benchmark_dir } => {
MetricAggregator::generate_csv_from_benchmark_dir(&benchmark_dir)?
}
}
Ok(())
}
fn handle_recipe_subcommand(command: RecipeCommand) -> Result<()> {
match command {
RecipeCommand::Validate { recipe_name } => handle_validate(&recipe_name),
@@ -1597,7 +1510,6 @@ pub async fn cli() -> anyhow::Result<()> {
crate::commands::update::update(canary, reconfigure)?;
Ok(())
}
Some(Command::Bench { cmd }) => handle_bench_command(cmd).await,
Some(Command::Recipe { command }) => handle_recipe_subcommand(command),
Some(Command::Web {
port,
crates/goose-cli/src/commands/bench.rs
-76
View File
@@ -1,76 +0,0 @@
use crate::cli::StreamableHttpOptions;
use crate::session::build_session;
use crate::session::SessionBuilderConfig;
use crate::{logging, CliSession};
use async_trait::async_trait;
use goose::conversation::Conversation;
use goose::session::session_manager::Session;
use goose_bench::bench_session::{BenchAgent, BenchBaseSession};
use goose_bench::eval_suites::ExtensionRequirements;
use std::sync::Arc;
use tokio::sync::Mutex;
// allow session obj to be used in benchmarking
#[async_trait]
impl BenchBaseSession for CliSession {
async fn headless(&mut self, message: String) -> anyhow::Result<()> {
self.headless(message).await
}
fn message_history(&self) -> Conversation {
self.message_history()
}
fn get_total_token_usage(&self) -> anyhow::Result<Option<i32>> {
// Since the trait requires sync but the session method is async,
// we need to block on the async call
tokio::task::block_in_place(|| {
tokio::runtime::Handle::current().block_on(self.get_total_token_usage())
})
}
async fn get_session(&self) -> anyhow::Result<Session> {
self.get_session().await
}
}
pub async fn agent_generator(
requirements: ExtensionRequirements,
session_id: String,
) -> BenchAgent {
let streamable_http_extensions: Vec<StreamableHttpOptions> = requirements
.streamable_http
.iter()
.map(|s| StreamableHttpOptions {
url: s.clone(),
timeout: goose::config::DEFAULT_EXTENSION_TIMEOUT,
})
.collect();
let base_session = build_session(SessionBuilderConfig {
session_id: Some(session_id),
resume: false,
fork: false,
no_session: false,
extensions: requirements.external,
streamable_http_extensions,
builtins: requirements.builtin,
no_profile: true,
recipe: None,
additional_system_prompt: None,
provider: None,
model: None,
debug: false,
max_tool_repetitions: None,
interactive: false, // Benchmarking is non-interactive
scheduled_job_id: None,
max_turns: None,
quiet: false,
output_format: "text".to_string(),
container: None,
})
.await;
let bench_agent = BenchAgent::new(Box::new(base_session));
let errors = Some(Arc::new(Mutex::new(bench_agent.get_errors().await)));
logging::setup_logging(Some("bench"), errors).expect("Failed to initialize logging");
bench_agent
}
crates/goose-cli/src/commands/mod.rs
-1
View File
@@ -1,4 +1,3 @@
pub mod bench;
pub mod configure;
pub mod info;
pub mod project;
crates/goose-cli/src/commands/web.rs
+1 -1
View File
@@ -238,7 +238,7 @@ pub async fn handle_web(
no_auth: bool,
) -> Result<()> {
validate_network_auth(&host, &auth_token, no_auth);
crate::logging::setup_logging(Some("goose-web"), None)?;
crate::logging::setup_logging(Some("goose-web"))?;
let (provider_name, model) = get_provider_and_model();
let agent = create_agent(&provider_name, &model).await?;
crates/goose-cli/src/logging.rs
+3 -25
View File
@@ -1,7 +1,5 @@
use anyhow::{Context, Result};
use std::sync::Arc;
use std::sync::Once;
use tokio::sync::Mutex;
use tracing_appender::rolling::Rotation;
use tracing_subscriber::{
filter::LevelFilter, fmt, layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Layer,
@@ -9,8 +7,6 @@ use tracing_subscriber::{
};
use goose::tracing::{langfuse_layer, otlp_layer};
use goose_bench::bench_session::BenchAgentError;
use goose_bench::error_capture::ErrorCaptureLayer;
// Used to ensure we only set up tracing once
static INIT: Once = Once::new();
@@ -20,27 +16,14 @@ static INIT: Once = Once::new();
/// - File-based logging with JSON formatting (DEBUG level)
/// - No console output (all logs go to files only)
/// - Optional Langfuse integration (DEBUG level)
/// - Optional error capture layer for benchmarking
pub fn setup_logging(
name: Option<&str>,
error_capture: Option<Arc<Mutex<Vec<BenchAgentError>>>>,
) -> Result<()> {
setup_logging_internal(name, error_capture, false)
pub fn setup_logging(name: Option<&str>) -> Result<()> {
setup_logging_internal(name, false)
}
/// Internal function that allows bypassing the Once check for testing
fn setup_logging_internal(
name: Option<&str>,
error_capture: Option<Arc<Mutex<Vec<BenchAgentError>>>>,
force: bool,
) -> Result<()> {
fn setup_logging_internal(name: Option<&str>, force: bool) -> Result<()> {
let mut result = Ok(());
// Register the error vector if provided
if let Some(errors) = error_capture {
ErrorCaptureLayer::register_error_vector(errors);
}
let mut setup = || {
result = (|| {
let log_dir = goose::logging::prepare_log_directory("cli", true)?;
@@ -84,11 +67,6 @@ fn setup_logging_internal(
// Console logging disabled for CLI - all logs go to files only
];
// Only add ErrorCaptureLayer if not in test mode
if !force {
layers.push(ErrorCaptureLayer::new().boxed());
}
if !force {
if let Ok((otlp_tracing_layer, otlp_metrics_layer, otlp_logs_layer)) =
otlp_layer::init_otlp()
crates/goose-cli/src/main.rs
+1 -1
View File
@@ -3,7 +3,7 @@ use goose_cli::cli::cli;
#[tokio::main]
async fn main() -> Result<()> {
if let Err(e) = goose_cli::logging::setup_logging(None, None) {
if let Err(e) = goose_cli::logging::setup_logging(None) {
eprintln!("Warning: Failed to initialize logging: {}", e);
}