@@ -48,7 +48,7 @@ impl Evaluation for ComputerControllerScript {
|
||||
}
|
||||
|
||||
// Parse the arguments as JSON
|
||||
if let Ok(args) = serde_json::from_value::<Value>(serde_json::Value::Object(tool_call.arguments.clone().unwrap_or_default())) {
|
||||
if let Ok(args) = serde_json::from_value::<Value>(tool_call.arguments.clone()) {
|
||||
// Check all required parameters match exactly
|
||||
args.get("script").and_then(Value::as_str).is_some_and(|s| s.contains("beep"))
|
||||
} else {
|
||||
|
||||
@@ -51,7 +51,7 @@ impl Evaluation for ComputerControllerWebScrape {
|
||||
}
|
||||
|
||||
// Parse the arguments as JSON
|
||||
if let Ok(args) = serde_json::from_value::<Value>(serde_json::Value::Object(tool_call.arguments.clone().unwrap_or_default())) {
|
||||
if let Ok(args) = serde_json::from_value::<Value>(tool_call.arguments.clone()) {
|
||||
// Check all required parameters match exactly
|
||||
args.get("url").and_then(Value::as_str).map(|s| s.trim_end_matches('/')) == Some("https://news.ycombinator.com")
|
||||
} else {
|
||||
|
||||
@@ -51,7 +51,7 @@ impl Evaluation for DeveloperCreateFile {
|
||||
}
|
||||
|
||||
// Parse the arguments as JSON
|
||||
if let Ok(args) = serde_json::from_value::<Value>(serde_json::Value::Object(tool_call.arguments.clone().unwrap_or_default())) {
|
||||
if let Ok(args) = serde_json::from_value::<Value>(tool_call.arguments.clone()) {
|
||||
// Check all required parameters match exactly
|
||||
args.get("command").and_then(Value::as_str) == Some("write") &&
|
||||
args.get("path").and_then(Value::as_str).is_some_and(|s| s.contains("test.txt")) &&
|
||||
@@ -82,7 +82,7 @@ impl Evaluation for DeveloperCreateFile {
|
||||
}
|
||||
|
||||
// Parse the arguments as JSON
|
||||
if let Ok(args) = serde_json::from_value::<Value>(serde_json::Value::Object(tool_call.arguments.clone().unwrap_or_default())) {
|
||||
if let Ok(args) = serde_json::from_value::<Value>(tool_call.arguments.clone()) {
|
||||
// Check all required parameters match exactly
|
||||
args.get("command").and_then(Value::as_str) == Some("view") &&
|
||||
args.get("path").and_then(Value::as_str).is_some_and(|s| s.contains("test.txt"))
|
||||
|
||||
@@ -44,7 +44,7 @@ impl Evaluation for DeveloperListFiles {
|
||||
// Check if the tool call is for shell with ls or rg --files
|
||||
if let Ok(tool_call) = tool_req.tool_call.as_ref() {
|
||||
// Parse arguments as JSON Value first
|
||||
if let Ok(args) = serde_json::from_value::<Value>(serde_json::Value::Object(tool_call.arguments.clone().unwrap_or_default())) {
|
||||
if let Ok(args) = serde_json::from_value::<Value>(tool_call.arguments.clone()) {
|
||||
tool_call.name == "developer__shell" &&
|
||||
args.get("command")
|
||||
.and_then(Value::as_str).is_some_and(|cmd| {
|
||||
|
||||
@@ -48,9 +48,7 @@ impl Evaluation for SimpleRepoCloneTest {
|
||||
}
|
||||
|
||||
if let Ok(args) =
|
||||
serde_json::from_value::<Value>(serde_json::Value::Object(
|
||||
tool_call.arguments.clone().unwrap_or_default(),
|
||||
))
|
||||
serde_json::from_value::<Value>(tool_call.arguments.clone())
|
||||
{
|
||||
let command = args.get("command").and_then(Value::as_str);
|
||||
command.is_some_and(|cmd| {
|
||||
@@ -80,9 +78,7 @@ impl Evaluation for SimpleRepoCloneTest {
|
||||
}
|
||||
|
||||
if let Ok(args) =
|
||||
serde_json::from_value::<Value>(serde_json::Value::Object(
|
||||
tool_call.arguments.clone().unwrap_or_default(),
|
||||
))
|
||||
serde_json::from_value::<Value>(tool_call.arguments.clone())
|
||||
{
|
||||
let command = args.get("command").and_then(Value::as_str);
|
||||
command.is_some_and(|cmd| {
|
||||
@@ -114,9 +110,7 @@ impl Evaluation for SimpleRepoCloneTest {
|
||||
}
|
||||
|
||||
if let Ok(args) =
|
||||
serde_json::from_value::<Value>(serde_json::Value::Object(
|
||||
tool_call.arguments.clone().unwrap_or_default(),
|
||||
))
|
||||
serde_json::from_value::<Value>(tool_call.arguments.clone())
|
||||
{
|
||||
let command = args.get("command").and_then(Value::as_str);
|
||||
let file_text = args.get("file_text").and_then(Value::as_str);
|
||||
@@ -158,9 +152,7 @@ impl Evaluation for SimpleRepoCloneTest {
|
||||
}
|
||||
|
||||
if let Ok(args) =
|
||||
serde_json::from_value::<Value>(serde_json::Value::Object(
|
||||
tool_call.arguments.clone().unwrap_or_default(),
|
||||
))
|
||||
serde_json::from_value::<Value>(tool_call.arguments.clone())
|
||||
{
|
||||
let command = args.get("command").and_then(Value::as_str);
|
||||
command.is_some_and(|cmd| {
|
||||
|
||||
@@ -47,9 +47,7 @@ impl Evaluation for DeveloperImage {
|
||||
if let MessageContent::ToolRequest(tool_req) = content {
|
||||
if let Ok(tool_call) = tool_req.tool_call.as_ref() {
|
||||
if let Ok(args) =
|
||||
serde_json::from_value::<Value>(serde_json::Value::Object(
|
||||
tool_call.arguments.clone().unwrap_or_default(),
|
||||
))
|
||||
serde_json::from_value::<Value>(tool_call.arguments.clone())
|
||||
{
|
||||
if tool_call.name == "developer__screen_capture"
|
||||
&& (args.get("display").and_then(Value::as_i64) == Some(0))
|
||||
|
||||
@@ -51,7 +51,7 @@ impl Evaluation for MemoryRememberMemory {
|
||||
}
|
||||
|
||||
// Parse the arguments as JSON
|
||||
if let Ok(args) = serde_json::from_value::<Value>(serde_json::Value::Object(tool_call.arguments.clone().unwrap_or_default())) {
|
||||
if let Ok(args) = serde_json::from_value::<Value>(tool_call.arguments.clone()) {
|
||||
// Check all required parameters match exactly
|
||||
args.get("category").and_then(Value::as_str).is_some_and(|s| s.contains("fact")) &&
|
||||
args.get("data").and_then(Value::as_str) == Some("The capital of France is Paris.") &&
|
||||
|
||||
@@ -73,9 +73,7 @@ fn count_tool_calls(messages: &[Message]) -> (i64, HashMap<String, i64>) {
|
||||
total_count += 1;
|
||||
|
||||
// Count by name
|
||||
*counts_by_name
|
||||
.entry(tool_call.name.to_string())
|
||||
.or_insert(0) += 1;
|
||||
*counts_by_name.entry(tool_call.name.clone()).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,9 +59,7 @@ impl Evaluation for FlappyBird {
|
||||
|
||||
// Parse the arguments as JSON
|
||||
if let Ok(args) =
|
||||
serde_json::from_value::<Value>(serde_json::Value::Object(
|
||||
tool_call.arguments.clone().unwrap_or_default(),
|
||||
))
|
||||
serde_json::from_value::<Value>(tool_call.arguments.clone())
|
||||
{
|
||||
// Only check command is write and correct filename
|
||||
args.get("command").and_then(Value::as_str) == Some("write")
|
||||
|
||||
@@ -69,9 +69,7 @@ impl Evaluation for GooseWiki {
|
||||
|
||||
// Parse the arguments as JSON
|
||||
if let Ok(args) =
|
||||
serde_json::from_value::<Value>(serde_json::Value::Object(
|
||||
tool_call.arguments.clone().unwrap_or_default(),
|
||||
))
|
||||
serde_json::from_value::<Value>(tool_call.arguments.clone())
|
||||
{
|
||||
// Only check command is write and correct filename
|
||||
args.get("command").and_then(Value::as_str) == Some("write")
|
||||
|
||||
@@ -29,7 +29,6 @@ impl SquirrelCensus {
|
||||
|
||||
#[async_trait]
|
||||
impl Evaluation for SquirrelCensus {
|
||||
#[allow(clippy::too_many_lines)]
|
||||
async fn run(
|
||||
&self,
|
||||
agent: &mut BenchAgent,
|
||||
@@ -82,9 +81,7 @@ After writing the script, run it using python3 and show the results. Do not ask
|
||||
}
|
||||
|
||||
if let Ok(args) =
|
||||
serde_json::from_value::<Value>(serde_json::Value::Object(
|
||||
tool_call.arguments.clone().unwrap_or_default(),
|
||||
))
|
||||
serde_json::from_value::<Value>(tool_call.arguments.clone())
|
||||
{
|
||||
args.get("command").and_then(Value::as_str) == Some("write")
|
||||
&& args
|
||||
@@ -114,9 +111,7 @@ After writing the script, run it using python3 and show the results. Do not ask
|
||||
}
|
||||
|
||||
if let Ok(args) =
|
||||
serde_json::from_value::<Value>(serde_json::Value::Object(
|
||||
tool_call.arguments.clone().unwrap_or_default(),
|
||||
))
|
||||
serde_json::from_value::<Value>(tool_call.arguments.clone())
|
||||
{
|
||||
args.get("command")
|
||||
.and_then(Value::as_str)
|
||||
|
||||
Reference in New Issue
Block a user