tidy: clean up old benchmark and add gym (#7081)

evals/open-model-gym/suite/src/types.ts

@@ -0,0 +1,74 @@
+export interface AgentConfig {
+  model: string;
+  provider: string;
+  /** Extensions (runner knows which are platform vs builtin) */
+  extensions?: string[];
+  /** Stdio extension commands (for custom MCP servers) */
+  stdio?: string[];
+  /** Path to goose binary (default: "goose") */
+  "goose-bin"?: string;
+  temperature?: number;
+  maxTokens?: number;
+}
+
+export interface Scenario {
+  name: string;
+  description: string;
+  prompt?: string;
+  /** Files to create before running (relative paths) */
+  setup?: Record<string, string>;
+  /** Validation rules to check after agent completes (single-turn) */
+  validate?: ValidationRule[];
+  /** Multi-turn conversation (alternative to single prompt+validate) */
+  turns?: Turn[];
+  /** Tags for filtering scenarios */
+  tags?: string[];
+}
+
+/** A single turn in a multi-turn conversation */
+export interface Turn {
+  /** The prompt for this turn */
+  prompt: string;
+  /** Validation rules to check after this turn completes */
+  validate: ValidationRule[];
+}
+
+export type ValidationRule =
+  | { type: "file_exists"; path: string; name?: string }
+  | { type: "file_contains"; path: string; pattern: string; name?: string }
+  | { type: "file_matches"; path: string; regex: string; name?: string }
+  | { type: "file_not_matches"; path: string; regex: string; name?: string }
+  | { type: "file_not_empty"; path: string; name?: string }
+  | { type: "command_succeeds"; command: string; name?: string }
+  | { type: "tool_called"; tool: string; args?: Record<string, string | RegExp>; name?: string }
+  | { type: "custom"; fn: string; name?: string };
+
+export interface TestRun {
+  scenario: Scenario;
+  config: AgentConfig;
+  workdir: string;
+  startTime: Date;
+  endTime?: Date;
+  status: "pending" | "running" | "passed" | "failed";
+  errors?: string[];
+}
+
+export interface TestResult {
+  run: TestRun;
+  validations: Array<{
+    rule: ValidationRule;
+    passed: boolean;
+    message?: string;
+  }>;
+}
+
+export interface SuiteConfig {
+  /** Agent configurations to permute */
+  agents: AgentConfig[];
+  /** Scenarios to run */
+  scenarios: string[];
+  /** Base directory for test workspaces */
+  workdir: string;
+  /** Parallel execution count */
+  parallel?: number;
+}

evals/open-model-gym/suite/src/validator.ts

@@ -0,0 +1,184 @@
+import { existsSync, readFileSync, statSync } from "node:fs";
+import { execSync } from "node:child_process";
+import { join } from "node:path";
+import type { ValidationRule } from "./types.js";
+
+export interface ValidationResult {
+  passed: boolean;
+  message?: string;
+}
+
+export function validateRule(
+  rule: ValidationRule,
+  workdir: string
+): ValidationResult {
+  switch (rule.type) {
+    case "file_exists": {
+      const fullPath = join(workdir, rule.path);
+      const exists = existsSync(fullPath);
+      return {
+        passed: exists,
+        message: exists ? undefined : `File not found: ${rule.path}`,
+      };
+    }
+
+    case "file_not_empty": {
+      const fullPath = join(workdir, rule.path);
+      if (!existsSync(fullPath)) {
+        return { passed: false, message: `File not found: ${rule.path}` };
+      }
+      const stat = statSync(fullPath);
+      return {
+        passed: stat.size > 0,
+        message: stat.size > 0 ? undefined : `File is empty: ${rule.path}`,
+      };
+    }
+
+    case "file_contains": {
+      const fullPath = join(workdir, rule.path);
+      if (!existsSync(fullPath)) {
+        return { passed: false, message: `File not found: ${rule.path}` };
+      }
+      const content = readFileSync(fullPath, "utf-8");
+      const contains = content.includes(rule.pattern);
+      return {
+        passed: contains,
+        message: contains
+          ? undefined
+          : `File ${rule.path} does not contain: ${rule.pattern}`,
+      };
+    }
+
+    case "file_matches": {
+      const fullPath = join(workdir, rule.path);
+      if (!existsSync(fullPath)) {
+        return { passed: false, message: `File not found: ${rule.path}` };
+      }
+      const content = readFileSync(fullPath, "utf-8");
+      const regex = new RegExp(rule.regex);
+      const matches = regex.test(content);
+      return {
+        passed: matches,
+        message: matches
+          ? undefined
+          : `File ${rule.path} does not match regex: ${rule.regex}`,
+      };
+    }
+
+    case "file_not_matches": {
+      const fullPath = join(workdir, rule.path);
+      if (!existsSync(fullPath)) {
+        return { passed: false, message: `File not found: ${rule.path}` };
+      }
+      const content = readFileSync(fullPath, "utf-8");
+      const regex = new RegExp(rule.regex);
+      const matches = regex.test(content);
+      return {
+        passed: !matches,
+        message: !matches
+          ? undefined
+          : `File ${rule.path} should not match regex: ${rule.regex}`,
+      };
+    }
+
+    case "command_succeeds": {
+      try {
+        execSync(rule.command, { cwd: workdir, stdio: "pipe" });
+        return { passed: true };
+      } catch (err) {
+        return {
+          passed: false,
+          message: `Command failed: ${rule.command}`,
+        };
+      }
+    }
+
+    case "tool_called": {
+      const logPath = join(workdir, "tool-calls.log");
+      if (!existsSync(logPath)) {
+        return { passed: false, message: "tool-calls.log not found" };
+      }
+
+      const content = readFileSync(logPath, "utf-8");
+      const lines = content.trim().split("\n").filter(Boolean);
+
+      // Find all calls to the specified tool
+      const matchingCalls = lines
+        .map((line) => {
+          try {
+            return JSON.parse(line);
+          } catch {
+            return null;
+          }
+        })
+        .filter((entry) => entry?.tool === rule.tool);
+
+      if (matchingCalls.length === 0) {
+        return { passed: false, message: `Tool not called: ${rule.tool}` };
+      }
+
+      // If no arg requirements, just check tool was called
+      if (!rule.args) {
+        return { passed: true };
+      }
+
+      // Check if any call matches the arg requirements
+      for (const call of matchingCalls) {
+        const args = call.arguments || {};
+        let allMatch = true;
+
+        for (const [key, expected] of Object.entries(rule.args)) {
+          const actual = args[key];
+          if (actual === undefined) {
+            allMatch = false;
+            break;
+          }
+
+          // If expected starts/ends with /, treat as regex pattern
+          if (typeof expected === "string" && expected.startsWith("/") && expected.endsWith("/")) {
+            const pattern = new RegExp(expected.slice(1, -1), "i");
+            if (!pattern.test(String(actual))) {
+              allMatch = false;
+              break;
+            }
+          } else {
+            // Exact match (case-insensitive for strings)
+            const actualStr = String(actual).toLowerCase();
+            const expectedStr = String(expected).toLowerCase();
+            if (!actualStr.includes(expectedStr)) {
+              allMatch = false;
+              break;
+            }
+          }
+        }
+
+        if (allMatch) {
+          return { passed: true };
+        }
+      }
+
+      return {
+        passed: false,
+        message: `Tool ${rule.tool} called but args didn't match: expected ${JSON.stringify(rule.args)}`,
+      };
+    }
+
+    case "custom": {
+      // Custom validators loaded dynamically
+      return { passed: false, message: "Custom validators not yet implemented" };
+    }
+
+    default:
+      return { passed: false, message: `Unknown rule type` };
+  }
+}
+
+export function validateAll(
+  rules: ValidationRule[],
+  workdir: string
+): Array<{ rule: ValidationRule; result: ValidationResult }> {
+  return rules.map((rule) => ({
+    rule,
+    result: validateRule(rule, workdir),
+  }));
+}