tkmind/tkmind_go
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

feat(gym): make Open Model Gym output dir configurable (#9789)

Browse Source 
Signed-off-by: Kyle De Freitas <kdefreitas@squareup.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Douwe M Osinga <douwe@sidewalklabs.com>
This commit is contained in:
Kyle E DeFreitas
2026-06-17 13:20:56 -04:00
committed by GitHub
parent 1a500d8f48
commit d2090b1de5
3 changed files with 98 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
evals/open-model-gym/Justfile
+13 -2
View File
@@ -7,6 +7,14 @@ default: run
run: _install
cd suite && npm run test
# Full run with artifacts isolated under ~/.goose/gym-runs/<YYYYDDMMHHMMSS> (keeps the repo clean)
run-clean: _install
#!/usr/bin/env bash
set -euo pipefail
export GYM_OUTPUT_DIR="$HOME/.goose/gym-runs/$(date +%Y%d%m%H%M%S)"
echo "Artifacts → $GYM_OUTPUT_DIR"
cd suite && npm run test
# Quick test - file-editing + everyday-app-automation, single run each (no repetition)
test: _install
cd suite && npx tsx src/runner.ts --scenario=file-editing,everyday-app-automation --run-count=1
@@ -19,9 +27,12 @@ scenario name: _install
agent name: _install
cd suite && npx tsx src/runner.ts --agent={{name}}
# Open report in browser
# Open report in browser (honors GYM_OUTPUT_DIR if set)
report:
open report.html
#!/usr/bin/env bash
set -euo pipefail
dir="${GYM_OUTPUT_DIR:-.}"
open "${dir/#\~/$HOME}/report.html"
# Install all dependencies
install:
evals/open-model-gym/README.md
+28
View File
@@ -267,6 +267,7 @@ Each tool returns realistic mock data. Tool calls are logged to `tool-calls.log`
| Command | Description |
|---------|-------------|
| `just run` | Full test run (3 reps each, worst kept) |
| `just run-clean` | Full run with artifacts isolated under `~/.goose/gym-runs/<YYYYDDMMHHMMSS>` |
| `just test` | Quick run (1 rep each) |
| `just scenario <name>` | Run specific scenario |
| `just agent <name>` | Run specific agent |
@@ -284,6 +285,9 @@ npx tsx src/runner.ts --run-count=5
# Don't auto-open browser
npx tsx src/runner.ts --no-open
# Redirect all run artifacts outside the repo (see Output below)
npx tsx src/runner.ts --output-dir=~/.goose/gym-runs/latest
# Raise the per-agent timeout (seconds) for slow local models on heavy
# scenarios. Default 300s; also settable via GYM_AGENT_TIMEOUT.
npx tsx src/runner.ts --agent-timeout=1200
@@ -293,3 +297,27 @@ npx tsx src/runner.ts --agent-timeout=1200
- `report.html` — Live-updating HTML matrix showing pass/fail status, duration, and validation details
- `logs/` — Full agent output logs for each run
By default these (plus the cache, scratch workdir, and isolated agent config
roots `.goose-root/` / `.opencode-root/` / `.pi-root/`) are written inside the
gym directory. They're gitignored, but still pile up in your checkout — awkward
if you want to run the bench regularly or from a worktree.
To keep the repo clean, redirect **all** run artifacts to a single base
directory with the `GYM_OUTPUT_DIR` env var (or the `--output-dir=` flag).
`config.yaml` and `scenarios/` are still read from the repo.
```bash
# Everything lands under a timestamped dir outside the repo (YYYYDDMMHHMMSS)
GYM_OUTPUT_DIR=~/.goose/gym-runs/$(date +%Y%d%m%H%M%S) just run
# Convenience recipe that does the timestamping for you
just run-clean
# View the report from a redirected run
GYM_OUTPUT_DIR=~/.goose/gym-runs/20261406101500 just report
```
> Note: the run cache lives under the output dir too, so a fresh timestamped
> dir means a fresh (cold) cache. Point `GYM_OUTPUT_DIR` at a stable directory
> if you want cache reuse across runs.
evals/open-model-gym/suite/src/runner.ts
+57 -17
View File
@@ -1,8 +1,8 @@
#!/usr/bin/env node
import { mkdirSync, writeFileSync, rmSync, readdirSync, existsSync, copyFileSync } from "node:fs";
import { join, basename, dirname } from "node:path";
import { join, basename, dirname, resolve } from "node:path";
import { homedir } from "node:os";
import { execSync } from "node:child_process";
import { execSync, execFileSync } from "node:child_process";
import { parse, stringify } from "yaml";
import { readFileSync } from "node:fs";
import { createHash } from "node:crypto";
@@ -89,6 +89,41 @@ interface CacheIndex {
}
// =============================================================================
// Output directory resolution
// =============================================================================
// All run artifacts (cache, isolated agent config roots, scratch workdir, logs,
// and the HTML report) live under a single base directory. By default this is
// the in-repo gym directory, so existing behavior is unchanged. Set the
// GYM_OUTPUT_DIR env var or pass --output-dir=<path> to redirect everything
// outside the repo and keep your checkout clean, e.g.:
//
// GYM_OUTPUT_DIR=~/.goose/gym-runs/$(date +%Y%d%m%H%M%S) just run
//
// config.yaml and scenarios/ are inputs and always read from the repo.
const SUITE_DIR = join(import.meta.dirname, ".."); // .../open-model-gym/suite
const GYM_DIR = join(import.meta.dirname, "../.."); // .../open-model-gym
function expandHome(p: string): string {
return p === "~" || p.startsWith("~/") ? join(homedir(), p.slice(1)) : p;
}
// Resolved output base, or null to fall back to the legacy in-repo locations.
const OUTPUT_DIR: string | null = (() => {
const flag = process.argv
.find((a) => a.startsWith("--output-dir="))
?.split("=")[1];
const base = flag ?? process.env.GYM_OUTPUT_DIR;
// Resolve to an absolute path: runners exec with cwd set to the workdir, so a
// relative base would make prompt/log paths resolve against the wrong dir.
return base ? resolve(expandHome(base)) : null;
})();
// Resolve an artifact path under OUTPUT_DIR when set, else its legacy anchor.
function artifactPath(name: string, legacyAnchor: string): string {
return join(OUTPUT_DIR ?? legacyAnchor, name);
}
// Agent timeout
// =============================================================================
// Per-invocation timeout for an agent run, in milliseconds. Larger local models
@@ -107,7 +142,7 @@ const AGENT_TIMEOUT_MS = (() => {
// Cache Utilities
// =============================================================================
const CACHE_DIR = join(import.meta.dirname, "../.cache");
const CACHE_DIR = artifactPath(".cache", SUITE_DIR);
const CACHE_INDEX_PATH = join(CACHE_DIR, "index.json");
const CACHE_LOGS_DIR = join(CACHE_DIR, "logs");
const CACHE_VERSION = 1;
@@ -317,7 +352,7 @@ const PLATFORM_EXTENSIONS = new Set([
]);
// Isolated goose config directory
const GOOSE_ROOT = join(import.meta.dirname, "../.goose-root");
const GOOSE_ROOT = artifactPath(".goose-root", SUITE_DIR);
const GOOSE_CONFIG_DIR = join(GOOSE_ROOT, "config");
function generateGooseConfig(model: ModelConfig, runner: RunnerConfig): object {
@@ -418,7 +453,7 @@ async function runGooseAgent(
// =============================================================================
// Isolated opencode config directory
const OPENCODE_ROOT = join(import.meta.dirname, "../.opencode-root");
const OPENCODE_ROOT = artifactPath(".opencode-root", SUITE_DIR);
function generateOpenCodeConfig(model: ModelConfig, runner: RunnerConfig, workdir: string): object {
const mcp: Record<string, object> = {};
@@ -517,7 +552,7 @@ async function runOpenCodeAgent(
// MCP support via pi-mcp-adapter: `pi install npm:pi-mcp-adapter`
// Isolated Pi config directory (like Goose/OpenCode)
const PI_CONFIG_DIR = join(import.meta.dirname, "../.pi-root");
const PI_CONFIG_DIR = artifactPath(".pi-root", SUITE_DIR);
// User's real Pi config (for copying auth.json)
const PI_USER_CONFIG = join(homedir(), ".pi", "agent");
@@ -1000,11 +1035,15 @@ function generateHtmlReport(
): void {
const { isRunning = false, allPairs = [] } = options;
// Read and embed gym.png as base64
const rootDir = join(outputPath, "..");
// Read and embed gym.png as base64. Prefer one sitting next to the report
// (legacy in-repo layout); otherwise fall back to the copy in the source tree
// so the image still embeds when output is redirected via GYM_OUTPUT_DIR.
let gymBase64 = "";
try {
const gymPath = join(rootDir, "gym.png");
const adjacent = join(outputPath, "..", "gym.png");
const gymPath = existsSync(adjacent)
? adjacent
: join(import.meta.dirname, "gym.png");
gymBase64 = readFileSync(gymPath).toString("base64");
} catch (e) {
// gym.png not found, will use external reference
@@ -1344,6 +1383,7 @@ function generateHtmlReport(
</body>
</html>`;
mkdirSync(dirname(outputPath), { recursive: true });
writeFileSync(outputPath, html);
console.log(`\n📊 Report saved to: ${outputPath}`);
}
@@ -1382,12 +1422,11 @@ async function main() {
return;
}
const rootDir = join(import.meta.dirname, "../..");
const configPath = join(rootDir, "config.yaml");
const configPath = join(GYM_DIR, "config.yaml");
const scenariosDir = join(import.meta.dirname, "../scenarios");
const workdir = join(import.meta.dirname, "../.workdir");
const logsDir = join(rootDir, "logs");
const reportPath = join(rootDir, "report.html");
const workdir = artifactPath(".workdir", SUITE_DIR);
const logsDir = artifactPath("logs", GYM_DIR);
const reportPath = artifactPath("report.html", GYM_DIR);
const config = loadConfig(configPath);
let scenarios = loadAllScenarios(scenariosDir);
@@ -1444,6 +1483,7 @@ async function main() {
}
const mcpHarnessHash = getMcpHarnessHash();
console.log(`Output: ${OUTPUT_DIR ?? GYM_DIR}${OUTPUT_DIR ? "" : " (in-repo; set GYM_OUTPUT_DIR to redirect)"}`);
console.log(`Models: ${config.models.map((m) => m.name).join(", ")}`);
console.log(`Runners: ${config.runners.map((r) => r.name).join(", ")}`);
console.log(`Running ${pairs.length} test pairs (${RUN_COUNT}x each, worst result kept)`);
@@ -1476,7 +1516,7 @@ async function main() {
if (!browserOpened) {
generateHtmlReport(results, reportPath, { isRunning: true, allPairs: pairs });
if (!noOpen) {
execSync(`open "${reportPath}"`);
execFileSync("open", [reportPath]);
}
browserOpened = true;
}
@@ -1511,10 +1551,10 @@ async function main() {
}
generateHtmlReport(results, reportPath, { isRunning: false, allPairs: pairs });
// If everything was cached, open browser now with final report
if (!browserOpened && !noOpen) {
execSync(`open "${reportPath}"`);
execFileSync("open", [reportPath]);
}
printResults(results);