fix: ensure repeating benches return to initial run-dir (#1617)
This commit is contained in:
@@ -11,6 +11,7 @@ pub static BUILTIN_EVAL_ASSETS: Dir = include_dir!("$CARGO_MANIFEST_DIR/src/asse
|
||||
|
||||
pub struct BenchmarkWorkDir {
|
||||
pub base_path: PathBuf,
|
||||
run_dir: PathBuf,
|
||||
cwd: PathBuf,
|
||||
run_name: String,
|
||||
suite: Option<String>,
|
||||
@@ -24,6 +25,7 @@ impl Default for BenchmarkWorkDir {
|
||||
}
|
||||
impl BenchmarkWorkDir {
|
||||
pub fn new(work_dir_name: String, include_dirs: Vec<PathBuf>) -> Self {
|
||||
let run_dir = std::env::current_dir().unwrap().canonicalize().unwrap();
|
||||
let base_path = PathBuf::from(format!("./benchmark-{}", work_dir_name));
|
||||
fs::create_dir_all(&base_path).unwrap();
|
||||
|
||||
@@ -54,6 +56,7 @@ impl BenchmarkWorkDir {
|
||||
|
||||
BenchmarkWorkDir {
|
||||
base_path: base_path.clone(),
|
||||
run_dir,
|
||||
cwd: base_path.clone(),
|
||||
run_name,
|
||||
suite: None,
|
||||
@@ -178,3 +181,9 @@ impl BenchmarkWorkDir {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for BenchmarkWorkDir {
|
||||
fn drop(&mut self) {
|
||||
std::env::set_current_dir(&self.run_dir).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,15 +20,17 @@ impl Evaluation for ExampleEval {
|
||||
_work_dir: &mut BenchmarkWorkDir,
|
||||
) -> anyhow::Result<Vec<(String, EvaluationMetric)>> {
|
||||
println!("ExampleEval - run");
|
||||
// let f = work_dir.fs_get(String::from("./arbitrary_dir/arbitrary_file.txt"))?;
|
||||
// let _contents = fs::read_to_string(f)?;
|
||||
let mut metrics = Vec::new();
|
||||
|
||||
let _ = agent.prompt("What can you do?".to_string()).await;
|
||||
|
||||
metrics.push((
|
||||
"example_metric".to_string(),
|
||||
EvaluationMetric::Boolean(true),
|
||||
));
|
||||
|
||||
metrics.push(("example_count".to_string(), EvaluationMetric::Integer(42)));
|
||||
|
||||
Ok(metrics)
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user