diff --git a/AGENTS.md b/AGENTS.md index e3ab069bd..0f01fcde5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -48,7 +48,6 @@ crates/ ├── goose-test # test utilities └── goose-test-support # test helpers -evals/open-model-gym/ # benchmarking / evals ui/desktop/ # Electron app ``` diff --git a/evals/open-model-gym/.gitignore b/evals/open-model-gym/.gitignore deleted file mode 100644 index b95b7690d..000000000 --- a/evals/open-model-gym/.gitignore +++ /dev/null @@ -1,72 +0,0 @@ -# Dependencies -node_modules/ -.pnpm-store/ -.workdir/ -report.html - -# Build outputs -dist/ -build/ -out/ -.next/ -.nuxt/ -.output/ -.opencode-root -suite/.pi-root - -# TypeScript -*.tsbuildinfo -*.d.ts.map -.g3/ -# Logs -logs/ -*.log -npm-debug.log* -yarn-debug.log* -yarn-error.log* -pnpm-debug.log* - -# Runtime data -pids/ -*.pid -*.seed -*.pid.lock - -# Coverage & testing -coverage/ -.nyc_output/ -.jest/ - -# Caches -.cache/ -.parcel-cache/ -.turbo/ -.eslintcache -.stylelintcache -*.swp -*.swo - -# IDE & editors -.idea/ -.vscode/ -*.sublime-project -*.sublime-workspace - -# OS files -.DS_Store -Thumbs.db - -# Environment variables -.env -.env.local -.env.*.local - -# Lock files (optional - uncomment if you don't want to track) -# package-lock.json -# yarn.lock -# pnpm-lock.yaml - -# Temporary files -tmp/ -temp/ -*.tmp diff --git a/evals/open-model-gym/Justfile b/evals/open-model-gym/Justfile deleted file mode 100644 index 09ccdf62d..000000000 --- a/evals/open-model-gym/Justfile +++ /dev/null @@ -1,71 +0,0 @@ -# Agent Runner - Test Suite (supports goose and opencode) - -# Default recipe -default: run - -# Full test run - all scenarios, all agents, 3 repetitions (worst kept) -run: _install - cd suite && npm run test - -# Full run with artifacts isolated under ~/.goose/gym-runs/ (keeps the repo clean) -run-clean: _install - #!/usr/bin/env bash - set -euo pipefail - export GYM_OUTPUT_DIR="$HOME/.goose/gym-runs/$(date +%Y%d%m%H%M%S)" - echo "Artifacts → $GYM_OUTPUT_DIR" - cd suite && npm run test - -# Quick test - file-editing + everyday-app-automation, single run each (no repetition) -test: _install - cd suite && npx tsx src/runner.ts --scenario=file-editing,everyday-app-automation --run-count=1 - -# Run a specific scenario (all agents, 3 reps) -scenario name: _install - cd suite && npx tsx src/runner.ts --scenario={{name}} - -# Run against a specific agent (all scenarios, 3 reps) -agent name: _install - cd suite && npx tsx src/runner.ts --agent={{name}} - -# Open report in browser (honors GYM_OUTPUT_DIR if set) -report: - #!/usr/bin/env bash - set -euo pipefail - dir="${GYM_OUTPUT_DIR:-.}" - open "${dir/#\~/$HOME}/report.html" - -# Install all dependencies -install: - cd suite && npm install - cd mcp-harness && npm install && npm run build - @# Install pi-mcp-adapter for Pi runner MCP support - @pi list 2>/dev/null | grep -q "pi-mcp-adapter" || pi install npm:pi-mcp-adapter - -# Build TypeScript -build: _install - cd suite && npm run build - -# Clear the test cache -clear-cache: - cd suite && npx tsx src/runner.ts --clear-cache - -# Run tests ignoring cache (force fresh runs) -run-fresh: _install - cd suite && npx tsx src/runner.ts --no-cache - -# Show cache stats -cache-stats: - @if [ -f suite/.cache/index.json ]; then \ - echo "Cache entries: $$(cat suite/.cache/index.json | grep -o '"[a-f0-9]\{16\}":' | wc -l | tr -d ' ')"; \ - echo "Cache size: $$(du -sh suite/.cache 2>/dev/null | cut -f1 || echo '0')"; \ - else \ - echo "No cache found"; \ - fi - -# Internal: install if node_modules missing, always rebuild mcp-harness -_install: - @[ -d suite/node_modules ] || (cd suite && npm install) - @[ -d mcp-harness/node_modules ] || (cd mcp-harness && npm install) - @cd mcp-harness && npm run build - @# Ensure pi-mcp-adapter is installed for Pi runner - @pi list 2>/dev/null | grep -q "pi-mcp-adapter" || pi install npm:pi-mcp-adapter diff --git a/evals/open-model-gym/README.md b/evals/open-model-gym/README.md deleted file mode 100644 index 7aa8899a7..000000000 --- a/evals/open-model-gym/README.md +++ /dev/null @@ -1,323 +0,0 @@ -# Open Model Gym - -Run agent tests across a matrix of **models × runners × scenarios**. - -It isn't hard for any agent to do ok with opus, but lets scale things in the other direction. What do we have to break things down to. - -image - -## Quick Start - -```bash -just install # one-time setup -just run # run full matrix (3 reps each) -just report # view results -``` - -## How It Works - -The test harness runs every combination of models, runners, and scenarios defined in your matrix. Each test runs multiple times (default 3) and keeps the **worst result** — if a test fails even once, it's marked failed. This catches flaky passes. - -## Configuration - -Edit `config.yaml` to define your test matrix: - -### Models - -LLMs to test against. Supports any provider (Anthropic, OpenAI, Ollama, etc.): - -```yaml -models: - - name: opus - provider: anthropic - model: claude-opus-4-5-20251101 - - - name: qwen3-coder - provider: ollama - model: qwen3-coder:64k - - - name: gpt4 - provider: openai - model: gpt-4-turbo -``` - -### Runners - -Agent frameworks that execute the tests. Each runner has its own binary, type, and configuration: - -```yaml -runners: - # Goose agent with extensions - - name: goose-full - type: goose - bin: goose # path to binary (can be absolute) - extensions: [developer, todo, skills] - stdio: - - node mcp-harness/dist/index.js - - # OpenCode agent - - name: opencode - type: opencode - bin: opencode # path to binary - stdio: - - node mcp-harness/dist/index.js - - # Custom goose binary path - - name: goose-dev - type: goose - bin: /path/to/my/goose-dev - extensions: [developer] -``` - -**Supported runner types:** -- `goose` — [Goose](https://github.com/aaif-goose/goose) agent framework -- `opencode` — [OpenCode](https://opencode.ai) agent framework -- `pi` — [Pi](https://github.com/badlogic/pi-mono) coding agent - -## Runner Details - -Each runner has different setup requirements, MCP integration methods, and session handling. - -### Goose - -[Goose](https://github.com/aaif-goose/goose) is an open-source coding agent with built-in MCP support. - -**Setup:** Install via `brew install goose` or from source. - -**MCP Integration:** Native support. The harness writes a `config.yaml` to an isolated `.goose-root/` directory with extensions and MCP servers: - -```yaml -extensions: - developer: - enabled: true - mcp_harness: - type: stdio - enabled: true - cmd: node - args: [mcp-harness/dist/index.js] -``` - -**Session Handling:** Uses `--name ` for named sessions, `--resume` to continue: -- Turn 1: `goose run -i --name ` -- Turn 2+: `goose run -i --name --resume` -- Single-turn: `goose run -i --no-session` - -### OpenCode - -[OpenCode](https://opencode.ai) is a terminal-based coding agent. - -**Setup:** Install via their website or package manager. - -**MCP Integration:** Native support. The harness writes an `opencode.json` config to the workdir: - -```json -{ - "mcp": { - "harness": { - "type": "local", - "command": ["node", "mcp-harness/dist/index.js"], - "enabled": true - } - }, - "model": "anthropic/claude-opus-4-5-20251101" -} -``` - -**Session Handling:** Uses `--continue` to resume the last session in the working directory: -- Turn 1: `opencode run ""` -- Turn 2+: `opencode run --continue ""` - -⚠️ OpenCode doesn't support named sessions, so multi-turn scenarios exclude it. - -### Pi - -[Pi](https://github.com/badlogic/pi-mono) is a lightweight coding agent that requires an adapter for MCP support. - -**Setup:** -```bash -# Install Pi -npm install -g @anthropic/pi # or from source - -# Install the MCP adapter (required for MCP tools) -pi install npm:pi-mcp-adapter -``` - -The `just install` recipe auto-installs pi-mcp-adapter if missing. - -**MCP Integration:** Via [pi-mcp-adapter](https://github.com/nicobailon/pi-mcp-adapter). The harness dynamically writes a `.pi-mcp.json` config to the workdir: - -```json -{ - "mcpServers": { - "harness": { - "command": "node", - "args": ["mcp-harness/dist/index.js"], - "lifecycle": "eager", - "env": { "MCP_HARNESS_LOG": "/tool-calls.log" } - } - }, - "settings": { "directTools": true } -} -``` - -Key settings: -- `directTools: true` — Registers MCP tools directly in Pi's tool list (no wrapper) -- `lifecycle: "eager"` — Connects to MCP servers at startup - -**Model Configuration:** Pi requires custom models (like Ollama) to be defined in `models.json`. The harness automatically generates this config in an isolated `.pi-root/` directory and sets `PI_CODING_AGENT_DIR` to use it: - -```json -{ - "providers": { - "ollama": { - "baseUrl": "http://localhost:11434/v1", - "api": "openai-completions", - "apiKey": "ollama", - "models": [{ "id": "model-name", "name": "Model Name", ... }] - } - } -} -``` - -The harness copies `auth.json` from your real Pi config (`~/.pi/agent/`) so API keys work. - -**Session Handling:** Uses `--session ` for file-based sessions, `--continue` to resume: -- Turn 1: `pi -p --session ""` -- Turn 2+: `pi -p --continue --session ""` -- Single-turn: `pi -p --no-session ""` - -The `-p` flag runs Pi in non-interactive "print" mode for automation - -### Matrix - -Define which scenarios run against which models/runners: - -```yaml -matrix: - - scenario: file-editing - models: [opus, qwen3-coder] # omit to run all models - runners: [goose-full, opencode] # omit to run all runners - - - scenario: everyday-app-automation - # runs against ALL models and ALL runners -``` - -## Scenarios - -Scenarios live in `suite/scenarios/` as YAML files: - -```yaml -name: file-editing -description: Create and edit files -prompt: | - 1. Create joke.md containing a short joke - 2. Edit hello.rs to add a debug function - -setup: - hello.rs: | - fn main() { println!("Hello!"); } - -validate: - - type: file_exists - path: joke.md - - type: file_matches - path: hello.rs - regex: "fn\\s+debug" -``` - -### Validation Rules - -| Rule | Description | -|------|-------------| -| `file_exists` | File exists at path | -| `file_not_empty` | File exists and has content | -| `file_contains` | File contains literal string | -| `file_matches` | File matches regex pattern | -| `command_succeeds` | Shell command exits 0 | -| `tool_called` | MCP tool was called with matching args (regex supported) | - -**Tool call validation example:** -```yaml -validate: - - type: tool_called - tool: slack_search_messages - args: - query: /quarterly.?review/ # regex pattern - - type: tool_called - tool: jira_create_issue - args: - summary: /Q1.*Review/ - description: /David Brown/ -``` - -## MCP Harness - -Mock MCP server providing simulated tools for testing agent tool-use without hitting real APIs. - -```bash -cd mcp-harness && npm install && npm run build -``` - -**Available tools:** gdrive, sheets, salesforce, slack, calendar, gmail, jira, github - -Each tool returns realistic mock data. Tool calls are logged to `tool-calls.log` in the workdir for validation. - -## Commands - -| Command | Description | -|---------|-------------| -| `just run` | Full test run (3 reps each, worst kept) | -| `just run-clean` | Full run with artifacts isolated under `~/.goose/gym-runs/` | -| `just test` | Quick run (1 rep each) | -| `just scenario ` | Run specific scenario | -| `just agent ` | Run specific agent | -| `just report` | Open HTML results | - -### CLI Flags - -```bash -# Filter by scenario, model, or runner -npx tsx src/runner.ts --scenario=file-editing --model=opus --runner=goose - -# Control repetition count -npx tsx src/runner.ts --run-count=5 - -# Don't auto-open browser -npx tsx src/runner.ts --no-open - -# Redirect all run artifacts outside the repo (see Output below) -npx tsx src/runner.ts --output-dir=~/.goose/gym-runs/latest - -# Raise the per-agent timeout (seconds) for slow local models on heavy -# scenarios. Default 300s; also settable via GYM_AGENT_TIMEOUT. -npx tsx src/runner.ts --agent-timeout=1200 -``` - -## Output - -- `report.html` — Live-updating HTML matrix showing pass/fail status, duration, and validation details -- `logs/` — Full agent output logs for each run - -By default these (plus the cache, scratch workdir, and isolated agent config -roots `.goose-root/` / `.opencode-root/` / `.pi-root/`) are written inside the -gym directory. They're gitignored, but still pile up in your checkout — awkward -if you want to run the bench regularly or from a worktree. - -To keep the repo clean, redirect **all** run artifacts to a single base -directory with the `GYM_OUTPUT_DIR` env var (or the `--output-dir=` flag). -`config.yaml` and `scenarios/` are still read from the repo. - -```bash -# Everything lands under a timestamped dir outside the repo (YYYYDDMMHHMMSS) -GYM_OUTPUT_DIR=~/.goose/gym-runs/$(date +%Y%d%m%H%M%S) just run - -# Convenience recipe that does the timestamping for you -just run-clean - -# View the report from a redirected run -GYM_OUTPUT_DIR=~/.goose/gym-runs/20261406101500 just report -``` - -> Note: the run cache lives under the output dir too, so a fresh timestamped -> dir means a fresh (cold) cache. Point `GYM_OUTPUT_DIR` at a stable directory -> if you want cache reuse across runs. diff --git a/evals/open-model-gym/config.yaml b/evals/open-model-gym/config.yaml deleted file mode 100644 index 64aede15d..000000000 --- a/evals/open-model-gym/config.yaml +++ /dev/null @@ -1,85 +0,0 @@ -# ============================================================================= -# Models - the LLMs to test -# ============================================================================= -models: - - name: opus - provider: anthropic - model: claude-opus-4-5-20251101 - - - name: glm-4.7-flash - provider: ollama - model: glm-4.7-flash:latest - - # too slow on 64g: - #- name: frob/qwen3-coder-next:latest - # provider: ollama - # model: frob/qwen3-coder-next:latest - - - name: kimi-k2.5 - provider: ollama - model: kimi-k2.5:cloud - - - name: gpt-oss-120b - provider: ollama - model: gpt-oss:120b-cloud - - - name: gpt-oss-20b - provider: ollama - model: gpt-oss:20b - - - name: qwen3-coder:latest - provider: ollama - model: qwen3-coder:latest - - # good but too slow on 64G - #- name: nemotron-3-nano - # provider: ollama - # model: nemotron-3-nano:latest - -# ============================================================================= -# Runners - agent frameworks with their specific configurations -# ============================================================================= -# Each runner has its own binary, extensions/config, and isolated config directory -runners: - # - name: goose - # type: goose - # bin: goose - # extensions: [developer] - # stdio: - # - node mcp-harness/dist/index.js - - - name: goose-full - type: goose - bin: goose - extensions: [developer, todo, skills, code_execution, extensionmanager] - stdio: - - node mcp-harness/dist/index.js - - - name: opencode - type: opencode - bin: opencode - stdio: - - node mcp-harness/dist/index.js - - - name: pi - type: pi - bin: pi - # Pi takes provider/model from the test matrix, not config - # MCP support via pi-mcp-adapter: `pi install npm:pi-mcp-adapter` - stdio: - - node mcp-harness/dist/index.js - -# ============================================================================= -# Test Matrix -# ============================================================================= -# scenarios × models × runners -# - Omit 'models' to run against ALL models -# - Omit 'runners' to run against ALL runners -matrix: - # Single-turn scenarios: all models × all runners - - scenario: everyday-app-automation - - scenario: file-editing - - # Multi-turn: goose and pi only (opencode doesn't support session continuation) - - scenario: multi-turn-edit - runners: [goose-full] diff --git a/evals/open-model-gym/gym.png b/evals/open-model-gym/gym.png deleted file mode 100644 index 242cf3be7..000000000 Binary files a/evals/open-model-gym/gym.png and /dev/null differ diff --git a/evals/open-model-gym/mcp-harness/README.md b/evals/open-model-gym/mcp-harness/README.md deleted file mode 100644 index 726ea1189..000000000 --- a/evals/open-model-gym/mcp-harness/README.md +++ /dev/null @@ -1,87 +0,0 @@ -# MCP Harness - -A simulated MCP server with realistic fake tools for testing. Provides mock implementations of common business integrations without requiring actual API credentials. - -## Tools Included (35 tools) - -### Google Drive -- `gdrive_search` - Search files by name, content, or type -- `gdrive_read_file` - Read file contents -- `gdrive_create_file` - Create new files -- `gdrive_share_file` - Share files with users - -### Google Sheets -- `sheets_read` - Read spreadsheet data -- `sheets_write` - Write/update cells -- `sheets_append` - Append rows -- `sheets_create` - Create new spreadsheets - -### Salesforce -- `salesforce_query` - Execute SOQL queries -- `salesforce_get_record` - Get record by ID -- `salesforce_create_record` - Create records -- `salesforce_update_record` - Update records -- `salesforce_search` - SOSL search - -### Slack -- `slack_send_message` - Send messages -- `slack_get_messages` - Get channel messages -- `slack_search_messages` - Search messages -- `slack_list_channels` - List channels -- `slack_get_user_info` - Get user info -- `slack_set_status` - Set user status - -### Google Calendar -- `calendar_list_events` - List events -- `calendar_create_event` - Create events -- `calendar_update_event` - Update events -- `calendar_delete_event` - Delete events - -### Gmail -- `gmail_search` - Search emails -- `gmail_read_message` - Read email content -- `gmail_send` - Send emails -- `gmail_create_draft` - Create drafts - -### Jira -- `jira_search_issues` - Search with JQL -- `jira_get_issue` - Get issue details -- `jira_create_issue` - Create issues -- `jira_update_issue` - Update issues -- `jira_add_comment` - Add comments - -### GitHub -- `github_search_repos` - Search repositories -- `github_list_issues` - List issues -- `github_create_issue` - Create issues -- `github_list_prs` - List pull requests - -## Setup - -```bash -npm install -npm run build -``` - -## Run - -```bash -npm run start -# or -./run.sh -``` - -## MCP Config - -Add to your MCP client config: - -```json -{ - "mcpServers": { - "harness": { - "command": "node", - "args": ["/path/to/mcp-harness/dist/index.js"] - } - } -} -``` diff --git a/evals/open-model-gym/mcp-harness/package-lock.json b/evals/open-model-gym/mcp-harness/package-lock.json deleted file mode 100644 index b8dfaf41e..000000000 --- a/evals/open-model-gym/mcp-harness/package-lock.json +++ /dev/null @@ -1,1173 +0,0 @@ -{ - "name": "mcp-harness", - "version": "1.0.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "mcp-harness", - "version": "1.0.0", - "dependencies": { - "@modelcontextprotocol/sdk": "^1.26.0" - }, - "devDependencies": { - "@types/node": "^25.2.0", - "typescript": "^5.6.3" - } - }, - "node_modules/@hono/node-server": { - "version": "1.19.13", - "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.13.tgz", - "integrity": "sha512-TsQLe4i2gvoTtrHje625ngThGBySOgSK3Xo2XRYOdqGN1teR8+I7vchQC46uLJi8OF62YTYA3AhSpumtkhsaKQ==", - "license": "MIT", - "engines": { - "node": ">=18.14.1" - }, - "peerDependencies": { - "hono": "^4" - } - }, - "node_modules/@modelcontextprotocol/sdk": { - "version": "1.26.0", - "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.26.0.tgz", - "integrity": "sha512-Y5RmPncpiDtTXDbLKswIJzTqu2hyBKxTNsgKqKclDbhIgg1wgtf1fRuvxgTnRfcnxtvvgbIEcqUOzZrJ6iSReg==", - "license": "MIT", - "dependencies": { - "@hono/node-server": "^1.19.9", - "ajv": "^8.17.1", - "ajv-formats": "^3.0.1", - "content-type": "^1.0.5", - "cors": "^2.8.5", - "cross-spawn": "^7.0.5", - "eventsource": "^3.0.2", - "eventsource-parser": "^3.0.0", - "express": "^5.2.1", - "express-rate-limit": "^8.2.1", - "hono": "^4.11.4", - "jose": "^6.1.3", - "json-schema-typed": "^8.0.2", - "pkce-challenge": "^5.0.0", - "raw-body": "^3.0.0", - "zod": "^3.25 || ^4.0", - "zod-to-json-schema": "^3.25.1" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@cfworker/json-schema": "^4.1.1", - "zod": "^3.25 || ^4.0" - }, - "peerDependenciesMeta": { - "@cfworker/json-schema": { - "optional": true - }, - "zod": { - "optional": false - } - } - }, - "node_modules/@types/node": { - "version": "25.2.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-25.2.0.tgz", - "integrity": "sha512-DZ8VwRFUNzuqJ5khrvwMXHmvPe+zGayJhr2CDNiKB1WBE1ST8Djl00D0IC4vvNmHMdj6DlbYRIaFE7WHjlDl5w==", - "dev": true, - "license": "MIT", - "dependencies": { - "undici-types": "~7.16.0" - } - }, - "node_modules/accepts": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", - "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", - "license": "MIT", - "dependencies": { - "mime-types": "^3.0.0", - "negotiator": "^1.0.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/ajv": { - "version": "8.18.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", - "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.3", - "fast-uri": "^3.0.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/ajv-formats": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", - "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", - "license": "MIT", - "dependencies": { - "ajv": "^8.0.0" - }, - "peerDependencies": { - "ajv": "^8.0.0" - }, - "peerDependenciesMeta": { - "ajv": { - "optional": true - } - } - }, - "node_modules/body-parser": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.2.tgz", - "integrity": "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA==", - "license": "MIT", - "dependencies": { - "bytes": "^3.1.2", - "content-type": "^1.0.5", - "debug": "^4.4.3", - "http-errors": "^2.0.0", - "iconv-lite": "^0.7.0", - "on-finished": "^2.4.1", - "qs": "^6.14.1", - "raw-body": "^3.0.1", - "type-is": "^2.0.1" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/bytes": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", - "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/call-bind-apply-helpers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", - "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/call-bound": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", - "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "get-intrinsic": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/content-disposition": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.1.tgz", - "integrity": "sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q==", - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/content-type": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", - "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/cookie": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", - "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/cookie-signature": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", - "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", - "license": "MIT", - "engines": { - "node": ">=6.6.0" - } - }, - "node_modules/cors": { - "version": "2.8.6", - "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.6.tgz", - "integrity": "sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw==", - "license": "MIT", - "dependencies": { - "object-assign": "^4", - "vary": "^1" - }, - "engines": { - "node": ">= 0.10" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/cross-spawn": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", - "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", - "license": "MIT", - "dependencies": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/depd": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", - "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/dunder-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", - "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.1", - "es-errors": "^1.3.0", - "gopd": "^1.2.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/ee-first": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", - "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", - "license": "MIT" - }, - "node_modules/encodeurl": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", - "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/es-define-property": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", - "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-errors": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-object-atoms": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", - "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/escape-html": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", - "license": "MIT" - }, - "node_modules/etag": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", - "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/eventsource": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", - "integrity": "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==", - "license": "MIT", - "dependencies": { - "eventsource-parser": "^3.0.1" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/eventsource-parser": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.0.6.tgz", - "integrity": "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==", - "license": "MIT", - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/express": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", - "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", - "license": "MIT", - "dependencies": { - "accepts": "^2.0.0", - "body-parser": "^2.2.1", - "content-disposition": "^1.0.0", - "content-type": "^1.0.5", - "cookie": "^0.7.1", - "cookie-signature": "^1.2.1", - "debug": "^4.4.0", - "depd": "^2.0.0", - "encodeurl": "^2.0.0", - "escape-html": "^1.0.3", - "etag": "^1.8.1", - "finalhandler": "^2.1.0", - "fresh": "^2.0.0", - "http-errors": "^2.0.0", - "merge-descriptors": "^2.0.0", - "mime-types": "^3.0.0", - "on-finished": "^2.4.1", - "once": "^1.4.0", - "parseurl": "^1.3.3", - "proxy-addr": "^2.0.7", - "qs": "^6.14.0", - "range-parser": "^1.2.1", - "router": "^2.2.0", - "send": "^1.1.0", - "serve-static": "^2.2.0", - "statuses": "^2.0.1", - "type-is": "^2.0.1", - "vary": "^1.1.2" - }, - "engines": { - "node": ">= 18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/express-rate-limit": { - "version": "8.5.1", - "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.5.1.tgz", - "integrity": "sha512-5O6KYmyJEpuPJV5hNTXKbAHWRqrzyu+OI3vUnSd2kXFubIVpG7ezpgxQy76Zo5GQZtrQBg86hF+CM/NX+cioiQ==", - "license": "MIT", - "dependencies": { - "ip-address": "^10.2.0" - }, - "engines": { - "node": ">= 16" - }, - "funding": { - "url": "https://github.com/sponsors/express-rate-limit" - }, - "peerDependencies": { - "express": ">= 4.11" - } - }, - "node_modules/fast-deep-equal": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "license": "MIT" - }, - "node_modules/fast-uri": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.2.tgz", - "integrity": "sha512-rVjf7ArG3LTk+FS6Yw81V1DLuZl1bRbNrev6Tmd/9RaroeeRRJhAt7jg/6YFxbvAQXUCavSoZhPPj6oOx+5KjQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "BSD-3-Clause" - }, - "node_modules/finalhandler": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", - "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", - "license": "MIT", - "dependencies": { - "debug": "^4.4.0", - "encodeurl": "^2.0.0", - "escape-html": "^1.0.3", - "on-finished": "^2.4.1", - "parseurl": "^1.3.3", - "statuses": "^2.0.1" - }, - "engines": { - "node": ">= 18.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/forwarded": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", - "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/fresh": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", - "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-intrinsic": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", - "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "es-define-property": "^1.0.1", - "es-errors": "^1.3.0", - "es-object-atoms": "^1.1.1", - "function-bind": "^1.1.2", - "get-proto": "^1.0.1", - "gopd": "^1.2.0", - "has-symbols": "^1.1.0", - "hasown": "^2.0.2", - "math-intrinsics": "^1.1.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", - "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", - "license": "MIT", - "dependencies": { - "dunder-proto": "^1.0.1", - "es-object-atoms": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/gopd": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", - "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-symbols": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", - "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/hasown": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", - "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/hono": { - "version": "4.12.23", - "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.23.tgz", - "integrity": "sha512-eIaZ9qDgu7XV0pxOCrg7/WhnQ6Ivm22UcxhXx/A3dcbqbbYgBEkc6e/J/s7j2tS96zoB0S9VBdLwQNCWwUo4LA==", - "license": "MIT", - "engines": { - "node": ">=16.9.0" - } - }, - "node_modules/http-errors": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", - "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", - "license": "MIT", - "dependencies": { - "depd": "~2.0.0", - "inherits": "~2.0.4", - "setprototypeof": "~1.2.0", - "statuses": "~2.0.2", - "toidentifier": "~1.0.1" - }, - "engines": { - "node": ">= 0.8" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/iconv-lite": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", - "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", - "license": "MIT", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3.0.0" - }, - "engines": { - "node": ">=0.10.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "license": "ISC" - }, - "node_modules/ip-address": { - "version": "10.2.0", - "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.0.tgz", - "integrity": "sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==", - "license": "MIT", - "engines": { - "node": ">= 12" - } - }, - "node_modules/ipaddr.js": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", - "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", - "license": "MIT", - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/is-promise": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", - "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", - "license": "MIT" - }, - "node_modules/isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "license": "ISC" - }, - "node_modules/jose": { - "version": "6.1.3", - "resolved": "https://registry.npmjs.org/jose/-/jose-6.1.3.tgz", - "integrity": "sha512-0TpaTfihd4QMNwrz/ob2Bp7X04yuxJkjRGi4aKmOqwhov54i6u79oCv7T+C7lo70MKH6BesI3vscD1yb/yzKXQ==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/panva" - } - }, - "node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "license": "MIT" - }, - "node_modules/json-schema-typed": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/json-schema-typed/-/json-schema-typed-8.0.2.tgz", - "integrity": "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==", - "license": "BSD-2-Clause" - }, - "node_modules/math-intrinsics": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", - "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/media-typer": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", - "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/merge-descriptors": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", - "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/mime-db": { - "version": "1.54.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", - "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", - "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", - "license": "MIT", - "dependencies": { - "mime-db": "^1.54.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/negotiator": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", - "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/object-assign": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", - "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/object-inspect": { - "version": "1.13.4", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", - "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/on-finished": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", - "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", - "license": "MIT", - "dependencies": { - "ee-first": "1.1.1" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "license": "ISC", - "dependencies": { - "wrappy": "1" - } - }, - "node_modules/parseurl": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", - "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/path-key": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/path-to-regexp": { - "version": "8.4.0", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.0.tgz", - "integrity": "sha512-PuseHIvAnz3bjrM2rGJtSgo1zjgxapTLZ7x2pjhzWwlp4SJQgK3f3iZIQwkpEnBaKz6seKBADpM4B4ySkuYypg==", - "license": "MIT", - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/pkce-challenge": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", - "integrity": "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==", - "license": "MIT", - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/proxy-addr": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", - "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", - "license": "MIT", - "dependencies": { - "forwarded": "0.2.0", - "ipaddr.js": "1.9.1" - }, - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/qs": { - "version": "6.15.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.2.tgz", - "integrity": "sha512-Rzq0KEyX/w/tEybncDgdkZrJgVUsUMk3xjh3t5bv3S1HTAtg+uOYt72+ZfwiQwKdysThkTBdL/rTi6HDmX9Ddw==", - "license": "BSD-3-Clause", - "dependencies": { - "side-channel": "^1.1.0" - }, - "engines": { - "node": ">=0.6" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/range-parser": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", - "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/raw-body": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", - "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", - "license": "MIT", - "dependencies": { - "bytes": "~3.1.2", - "http-errors": "~2.0.1", - "iconv-lite": "~0.7.0", - "unpipe": "~1.0.0" - }, - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/require-from-string": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", - "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/router": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", - "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", - "license": "MIT", - "dependencies": { - "debug": "^4.4.0", - "depd": "^2.0.0", - "is-promise": "^4.0.0", - "parseurl": "^1.3.3", - "path-to-regexp": "^8.0.0" - }, - "engines": { - "node": ">= 18" - } - }, - "node_modules/safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "license": "MIT" - }, - "node_modules/send": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", - "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", - "license": "MIT", - "dependencies": { - "debug": "^4.4.3", - "encodeurl": "^2.0.0", - "escape-html": "^1.0.3", - "etag": "^1.8.1", - "fresh": "^2.0.0", - "http-errors": "^2.0.1", - "mime-types": "^3.0.2", - "ms": "^2.1.3", - "on-finished": "^2.4.1", - "range-parser": "^1.2.1", - "statuses": "^2.0.2" - }, - "engines": { - "node": ">= 18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/serve-static": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", - "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", - "license": "MIT", - "dependencies": { - "encodeurl": "^2.0.0", - "escape-html": "^1.0.3", - "parseurl": "^1.3.3", - "send": "^1.2.0" - }, - "engines": { - "node": ">= 18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/setprototypeof": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", - "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", - "license": "ISC" - }, - "node_modules/shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "license": "MIT", - "dependencies": { - "shebang-regex": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/side-channel": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", - "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "object-inspect": "^1.13.3", - "side-channel-list": "^1.0.0", - "side-channel-map": "^1.0.1", - "side-channel-weakmap": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/side-channel-list": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", - "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "object-inspect": "^1.13.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/side-channel-map": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", - "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.5", - "object-inspect": "^1.13.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/side-channel-weakmap": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", - "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.5", - "object-inspect": "^1.13.3", - "side-channel-map": "^1.0.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/statuses": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", - "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/toidentifier": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", - "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", - "license": "MIT", - "engines": { - "node": ">=0.6" - } - }, - "node_modules/type-is": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.0.1.tgz", - "integrity": "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==", - "license": "MIT", - "dependencies": { - "content-type": "^1.0.5", - "media-typer": "^1.1.0", - "mime-types": "^3.0.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/typescript": { - "version": "5.9.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", - "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "dev": true, - "license": "Apache-2.0", - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=14.17" - } - }, - "node_modules/undici-types": { - "version": "7.16.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", - "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", - "dev": true, - "license": "MIT" - }, - "node_modules/unpipe": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", - "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/vary": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", - "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "license": "ISC", - "dependencies": { - "isexe": "^2.0.0" - }, - "bin": { - "node-which": "bin/node-which" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "license": "ISC" - }, - "node_modules/zod": { - "version": "4.3.6", - "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", - "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - }, - "node_modules/zod-to-json-schema": { - "version": "3.25.1", - "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.1.tgz", - "integrity": "sha512-pM/SU9d3YAggzi6MtR4h7ruuQlqKtad8e9S0fmxcMi+ueAK5Korys/aWcV9LIIHTVbj01NdzxcnXSN+O74ZIVA==", - "license": "ISC", - "peerDependencies": { - "zod": "^3.25 || ^4" - } - } - } -} diff --git a/evals/open-model-gym/mcp-harness/package.json b/evals/open-model-gym/mcp-harness/package.json deleted file mode 100644 index 76c14dcb5..000000000 --- a/evals/open-model-gym/mcp-harness/package.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "mcp-harness", - "version": "1.0.0", - "description": "Simulated real-world MCP tools for testing - Google Drive, Sheets, Salesforce, Slack, and more", - "private": true, - "type": "module", - "scripts": { - "build": "tsc -p tsconfig.json", - "start": "node dist/index.js" - }, - "dependencies": { - "@modelcontextprotocol/sdk": "^1.26.0" - }, - "devDependencies": { - "@types/node": "^25.2.0", - "typescript": "^5.6.3" - } -} diff --git a/evals/open-model-gym/mcp-harness/run.sh b/evals/open-model-gym/mcp-harness/run.sh deleted file mode 100755 index 02497dd6e..000000000 --- a/evals/open-model-gym/mcp-harness/run.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -cd "$(dirname "$0")" -npm run build && npm run start diff --git a/evals/open-model-gym/mcp-harness/src/index.ts b/evals/open-model-gym/mcp-harness/src/index.ts deleted file mode 100644 index 57f198041..000000000 --- a/evals/open-model-gym/mcp-harness/src/index.ts +++ /dev/null @@ -1,1040 +0,0 @@ -import { Server } from '@modelcontextprotocol/sdk/server/index.js'; -import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'; -import { CallToolRequestSchema, ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js'; -import * as fs from 'fs'; -import * as path from 'path'; - -// Logging configuration -const LOG_FILE = process.env.MCP_HARNESS_LOG || path.join(process.cwd(), 'tool-calls.log'); - -function logToolCall(toolName: string, args: Record, result: any) { - const entry = { - timestamp: new Date().toISOString(), - tool: toolName, - arguments: args, - result: result, - }; - const line = JSON.stringify(entry) + '\n'; - fs.appendFileSync(LOG_FILE, line); -} - -// Fake data generators -const fakeUsers = [ - { id: 'U001', name: 'Alice Johnson', email: 'alice@company.com', department: 'Engineering' }, - { id: 'U002', name: 'Bob Smith', email: 'bob@company.com', department: 'Sales' }, - { id: 'U003', name: 'Carol Williams', email: 'carol@company.com', department: 'Marketing' }, - { id: 'U004', name: 'David Brown', email: 'david@company.com', department: 'Finance' }, - { id: 'U005', name: 'Emma Davis', email: 'emma@company.com', department: 'Engineering' }, -]; - -const fakeCompanies = [ - { id: 'ACC001', name: 'Acme Corp', industry: 'Technology', revenue: 5000000 }, - { id: 'ACC002', name: 'GlobalTech Inc', industry: 'Manufacturing', revenue: 12000000 }, - { id: 'ACC003', name: 'StartupXYZ', industry: 'SaaS', revenue: 800000 }, - { id: 'ACC004', name: 'MegaCorp Ltd', industry: 'Retail', revenue: 45000000 }, - { id: 'ACC005', name: 'InnovateCo', industry: 'Healthcare', revenue: 3200000 }, -]; - -const fakeOpportunities = [ - { id: 'OPP001', name: 'Enterprise Deal - Acme', accountId: 'ACC001', stage: 'Negotiation', amount: 150000, closeDate: '2026-03-15' }, - { id: 'OPP002', name: 'Expansion - GlobalTech', accountId: 'ACC002', stage: 'Proposal', amount: 75000, closeDate: '2026-02-28' }, - { id: 'OPP003', name: 'New Business - StartupXYZ', accountId: 'ACC003', stage: 'Discovery', amount: 25000, closeDate: '2026-04-10' }, - { id: 'OPP004', name: 'Renewal - MegaCorp', accountId: 'ACC004', stage: 'Closed Won', amount: 200000, closeDate: '2026-01-20' }, -]; - -const fakeFiles = [ - { id: 'FILE001', name: 'Q4 Report.docx', mimeType: 'application/vnd.google-apps.document', size: 245000, modifiedTime: '2026-01-28T14:30:00Z', owner: 'alice@company.com' }, - { id: 'FILE002', name: 'Sales Forecast.xlsx', mimeType: 'application/vnd.google-apps.spreadsheet', size: 128000, modifiedTime: '2026-02-01T09:15:00Z', owner: 'bob@company.com' }, - { id: 'FILE003', name: 'Marketing Plan 2026.pdf', mimeType: 'application/pdf', size: 1520000, modifiedTime: '2026-01-25T16:45:00Z', owner: 'carol@company.com' }, - { id: 'FILE004', name: 'Budget Template.xlsx', mimeType: 'application/vnd.google-apps.spreadsheet', size: 89000, modifiedTime: '2026-01-30T11:00:00Z', owner: 'david@company.com' }, - { id: 'FILE005', name: 'Architecture Diagram.png', mimeType: 'image/png', size: 456000, modifiedTime: '2026-02-02T08:20:00Z', owner: 'emma@company.com' }, -]; - -const fakeSpreadsheets: Record = { - 'SHEET001': { - title: 'Sales Pipeline Q1 2026', - sheets: [ - { - name: 'Deals', - data: [ - ['Deal Name', 'Company', 'Amount', 'Stage', 'Close Date'], - ['Enterprise License', 'Acme Corp', '$150,000', 'Negotiation', '2026-03-15'], - ['Platform Upgrade', 'GlobalTech', '$75,000', 'Proposal', '2026-02-28'], - ['Starter Package', 'StartupXYZ', '$25,000', 'Discovery', '2026-04-10'], - ] - }, - { - name: 'Summary', - data: [ - ['Metric', 'Value'], - ['Total Pipeline', '$250,000'], - ['Deals in Negotiation', '1'], - ['Expected Close Rate', '65%'], - ] - } - ] - }, - 'SHEET002': { - title: 'Employee Directory', - sheets: [ - { - name: 'Employees', - data: [ - ['Name', 'Email', 'Department', 'Start Date'], - ['Alice Johnson', 'alice@company.com', 'Engineering', '2022-03-01'], - ['Bob Smith', 'bob@company.com', 'Sales', '2021-08-15'], - ['Carol Williams', 'carol@company.com', 'Marketing', '2023-01-10'], - ] - } - ] - } -}; - -const fakeSlackChannels = [ - { id: 'C001', name: 'general', memberCount: 150, topic: 'Company-wide announcements' }, - { id: 'C002', name: 'engineering', memberCount: 45, topic: 'Engineering discussions' }, - { id: 'C003', name: 'sales', memberCount: 28, topic: 'Sales team coordination' }, - { id: 'C004', name: 'random', memberCount: 142, topic: 'Non-work banter' }, -]; - -const fakeSlackMessages = [ - { channel: 'C001', user: 'U001', text: 'Reminder: All-hands meeting tomorrow at 2pm', ts: '1706886000.000100' }, - { channel: 'C001', user: 'U003', text: 'Thanks for the reminder!', ts: '1706886060.000200' }, - { channel: 'C002', user: 'U005', text: 'Just merged the new auth PR', ts: '1706885400.000300' }, - { channel: 'C002', user: 'U001', text: 'Great work! Any breaking changes?', ts: '1706885460.000400' }, - { channel: 'C003', user: 'U002', text: 'Closed the MegaCorp deal! 🎉', ts: '1706884800.000500' }, - { channel: 'C001', user: 'U004', text: 'Please review the quarterly review document I shared. Key metrics show 15% growth.', ts: '1706886100.000600' }, -]; - -const fakeCalendarEvents = [ - { id: 'EVT001', summary: 'Weekly Standup', start: '2026-02-03T09:00:00Z', end: '2026-02-03T09:30:00Z', attendees: ['alice@company.com', 'emma@company.com'] }, - { id: 'EVT002', summary: 'Client Call - Acme Corp', start: '2026-02-03T14:00:00Z', end: '2026-02-03T15:00:00Z', attendees: ['bob@company.com', 'alice@company.com'] }, - { id: 'EVT003', summary: 'Product Review', start: '2026-02-04T11:00:00Z', end: '2026-02-04T12:00:00Z', attendees: ['carol@company.com', 'david@company.com', 'emma@company.com'] }, -]; - -const fakeEmails = [ - { id: 'MSG001', from: 'client@acme.com', to: 'bob@company.com', subject: 'Re: Proposal Follow-up', snippet: 'Thanks for sending over the revised proposal...', date: '2026-02-02T10:30:00Z' }, - { id: 'MSG002', from: 'hr@company.com', to: 'all@company.com', subject: 'February Benefits Update', snippet: 'Please review the updated benefits information...', date: '2026-02-01T08:00:00Z' }, - { id: 'MSG003', from: 'alice@company.com', to: 'emma@company.com', subject: 'Code Review Request', snippet: 'Could you take a look at PR #423...', date: '2026-02-02T14:15:00Z' }, -]; - -// Utility functions -function generateId(prefix: string): string { - return `${prefix}${Date.now().toString(36)}${Math.random().toString(36).substr(2, 5)}`; -} - -function now(): string { - return new Date().toISOString(); -} - -function randomDelay(): number { - return Math.floor(Math.random() * 200) + 50; -} - -// Tool definitions -const tools = [ - // === Google Drive Tools === - { - name: 'gdrive_search', - description: 'Search for files in Google Drive by name, content, or type. Returns matching files with metadata.', - inputSchema: { - type: 'object', - properties: { - query: { type: 'string', description: 'Search query (supports name:, type:, owner: prefixes)' }, - limit: { type: 'integer', minimum: 1, maximum: 50, default: 10, description: 'Maximum results to return' }, - includeShared: { type: 'boolean', default: true, description: 'Include files shared with you' }, - }, - required: ['query'], - }, - }, - { - name: 'gdrive_read_file', - description: 'Read the contents of a file from Google Drive. Supports documents, text files, and exports spreadsheets as CSV.', - inputSchema: { - type: 'object', - properties: { - fileId: { type: 'string', description: 'The Google Drive file ID' }, - exportFormat: { type: 'string', enum: ['text', 'html', 'csv', 'pdf'], default: 'text', description: 'Export format for Google Docs' }, - }, - required: ['fileId'], - }, - }, - { - name: 'gdrive_create_file', - description: 'Create a new file in Google Drive with the specified content.', - inputSchema: { - type: 'object', - properties: { - name: { type: 'string', description: 'File name including extension' }, - content: { type: 'string', description: 'File content' }, - mimeType: { type: 'string', description: 'MIME type of the file' }, - folderId: { type: 'string', description: 'Parent folder ID (optional)' }, - }, - required: ['name', 'content'], - }, - }, - { - name: 'gdrive_share_file', - description: 'Share a file with specific users or make it publicly accessible.', - inputSchema: { - type: 'object', - properties: { - fileId: { type: 'string', description: 'The Google Drive file ID' }, - email: { type: 'string', description: 'Email address to share with' }, - role: { type: 'string', enum: ['reader', 'commenter', 'writer'], default: 'reader', description: 'Permission level' }, - sendNotification: { type: 'boolean', default: true, description: 'Send email notification' }, - }, - required: ['fileId', 'email'], - }, - }, - - // === Google Sheets Tools === - { - name: 'sheets_read', - description: 'Read data from a Google Sheets spreadsheet. Returns cell values from the specified range.', - inputSchema: { - type: 'object', - properties: { - spreadsheetId: { type: 'string', description: 'The spreadsheet ID' }, - range: { type: 'string', description: 'A1 notation range (e.g., "Sheet1!A1:D10")' }, - valueRenderOption: { type: 'string', enum: ['FORMATTED_VALUE', 'UNFORMATTED_VALUE', 'FORMULA'], default: 'FORMATTED_VALUE' }, - }, - required: ['spreadsheetId', 'range'], - }, - }, - { - name: 'sheets_write', - description: 'Write data to a Google Sheets spreadsheet. Overwrites existing data in the specified range.', - inputSchema: { - type: 'object', - properties: { - spreadsheetId: { type: 'string', description: 'The spreadsheet ID' }, - range: { type: 'string', description: 'A1 notation range (e.g., "Sheet1!A1")' }, - values: { type: 'array', items: { type: 'array', items: { type: 'string' } }, description: '2D array of values to write' }, - }, - required: ['spreadsheetId', 'range', 'values'], - }, - }, - { - name: 'sheets_append', - description: 'Append rows to a Google Sheets spreadsheet. Adds data after the last row with content.', - inputSchema: { - type: 'object', - properties: { - spreadsheetId: { type: 'string', description: 'The spreadsheet ID' }, - range: { type: 'string', description: 'A1 notation range indicating the table (e.g., "Sheet1!A:D")' }, - values: { type: 'array', items: { type: 'array', items: { type: 'string' } }, description: '2D array of rows to append' }, - }, - required: ['spreadsheetId', 'range', 'values'], - }, - }, - { - name: 'sheets_create', - description: 'Create a new Google Sheets spreadsheet with optional initial data.', - inputSchema: { - type: 'object', - properties: { - title: { type: 'string', description: 'Spreadsheet title' }, - sheetNames: { type: 'array', items: { type: 'string' }, description: 'Names of sheets to create' }, - initialData: { type: 'object', description: 'Map of sheet name to 2D array of initial values' }, - }, - required: ['title'], - }, - }, - - // === Salesforce Tools === - { - name: 'salesforce_query', - description: 'Execute a SOQL query against Salesforce. Returns matching records with pagination support.', - inputSchema: { - type: 'object', - properties: { - soql: { type: 'string', description: 'SOQL query (e.g., "SELECT Id, Name FROM Account WHERE Industry = \'Technology\'")' }, - limit: { type: 'integer', minimum: 1, maximum: 2000, default: 100, description: 'Maximum records to return' }, - }, - required: ['soql'], - }, - }, - { - name: 'salesforce_get_record', - description: 'Get a single Salesforce record by ID with all or specified fields.', - inputSchema: { - type: 'object', - properties: { - objectType: { type: 'string', description: 'Salesforce object type (e.g., Account, Contact, Opportunity)' }, - recordId: { type: 'string', description: 'The record ID' }, - fields: { type: 'array', items: { type: 'string' }, description: 'Fields to retrieve (optional, returns all if not specified)' }, - }, - required: ['objectType', 'recordId'], - }, - }, - { - name: 'salesforce_create_record', - description: 'Create a new record in Salesforce.', - inputSchema: { - type: 'object', - properties: { - objectType: { type: 'string', description: 'Salesforce object type' }, - data: { type: 'object', description: 'Field values for the new record' }, - }, - required: ['objectType', 'data'], - }, - }, - { - name: 'salesforce_update_record', - description: 'Update an existing Salesforce record.', - inputSchema: { - type: 'object', - properties: { - objectType: { type: 'string', description: 'Salesforce object type' }, - recordId: { type: 'string', description: 'The record ID to update' }, - data: { type: 'object', description: 'Field values to update' }, - }, - required: ['objectType', 'recordId', 'data'], - }, - }, - { - name: 'salesforce_search', - description: 'Execute a SOSL search across multiple Salesforce objects.', - inputSchema: { - type: 'object', - properties: { - searchTerm: { type: 'string', description: 'Search term' }, - objects: { type: 'array', items: { type: 'string' }, description: 'Objects to search (e.g., ["Account", "Contact"])' }, - limit: { type: 'integer', minimum: 1, maximum: 200, default: 20 }, - }, - required: ['searchTerm'], - }, - }, - - // === Slack Tools === - { - name: 'slack_send_message', - description: 'Send a message to a Slack channel or direct message.', - inputSchema: { - type: 'object', - properties: { - channel: { type: 'string', description: 'Channel ID or name (e.g., "#general" or "C001")' }, - text: { type: 'string', description: 'Message text (supports Slack markdown)' }, - threadTs: { type: 'string', description: 'Thread timestamp to reply to (optional)' }, - }, - required: ['channel', 'text'], - }, - }, - { - name: 'slack_get_messages', - description: 'Retrieve recent messages from a Slack channel.', - inputSchema: { - type: 'object', - properties: { - channel: { type: 'string', description: 'Channel ID or name' }, - limit: { type: 'integer', minimum: 1, maximum: 100, default: 20 }, - oldest: { type: 'string', description: 'Only messages after this timestamp' }, - latest: { type: 'string', description: 'Only messages before this timestamp' }, - }, - required: ['channel'], - }, - }, - { - name: 'slack_search_messages', - description: 'Search for messages across Slack channels.', - inputSchema: { - type: 'object', - properties: { - query: { type: 'string', description: 'Search query (supports from:, in:, has: modifiers)' }, - sort: { type: 'string', enum: ['score', 'timestamp'], default: 'score' }, - limit: { type: 'integer', minimum: 1, maximum: 100, default: 20 }, - }, - required: ['query'], - }, - }, - { - name: 'slack_list_channels', - description: 'List available Slack channels the user has access to.', - inputSchema: { - type: 'object', - properties: { - types: { type: 'string', enum: ['public', 'private', 'mpim', 'im', 'all'], default: 'public' }, - limit: { type: 'integer', minimum: 1, maximum: 200, default: 50 }, - }, - }, - }, - { - name: 'slack_get_user_info', - description: 'Get information about a Slack user.', - inputSchema: { - type: 'object', - properties: { - userId: { type: 'string', description: 'User ID' }, - }, - required: ['userId'], - }, - }, - { - name: 'slack_set_status', - description: 'Set your Slack status message and emoji.', - inputSchema: { - type: 'object', - properties: { - statusText: { type: 'string', description: 'Status message' }, - statusEmoji: { type: 'string', description: 'Status emoji (e.g., ":calendar:")' }, - expirationMinutes: { type: 'integer', description: 'Minutes until status expires (optional)' }, - }, - required: ['statusText'], - }, - }, - - // === Google Calendar Tools === - { - name: 'calendar_list_events', - description: 'List upcoming calendar events.', - inputSchema: { - type: 'object', - properties: { - calendarId: { type: 'string', default: 'primary', description: 'Calendar ID' }, - timeMin: { type: 'string', description: 'Start time (ISO 8601)' }, - timeMax: { type: 'string', description: 'End time (ISO 8601)' }, - maxResults: { type: 'integer', minimum: 1, maximum: 250, default: 10 }, - }, - }, - }, - { - name: 'calendar_create_event', - description: 'Create a new calendar event.', - inputSchema: { - type: 'object', - properties: { - summary: { type: 'string', description: 'Event title' }, - description: { type: 'string', description: 'Event description' }, - start: { type: 'string', description: 'Start time (ISO 8601)' }, - end: { type: 'string', description: 'End time (ISO 8601)' }, - attendees: { type: 'array', items: { type: 'string' }, description: 'Attendee email addresses' }, - location: { type: 'string', description: 'Event location' }, - }, - required: ['summary', 'start', 'end'], - }, - }, - { - name: 'calendar_update_event', - description: 'Update an existing calendar event.', - inputSchema: { - type: 'object', - properties: { - eventId: { type: 'string', description: 'Event ID' }, - summary: { type: 'string', description: 'New event title' }, - description: { type: 'string', description: 'New description' }, - start: { type: 'string', description: 'New start time' }, - end: { type: 'string', description: 'New end time' }, - }, - required: ['eventId'], - }, - }, - { - name: 'calendar_delete_event', - description: 'Delete a calendar event.', - inputSchema: { - type: 'object', - properties: { - eventId: { type: 'string', description: 'Event ID to delete' }, - sendNotifications: { type: 'boolean', default: true, description: 'Notify attendees' }, - }, - required: ['eventId'], - }, - }, - - // === Gmail Tools === - { - name: 'gmail_search', - description: 'Search emails in Gmail.', - inputSchema: { - type: 'object', - properties: { - query: { type: 'string', description: 'Gmail search query (supports from:, to:, subject:, has:attachment, etc.)' }, - maxResults: { type: 'integer', minimum: 1, maximum: 100, default: 10 }, - labelIds: { type: 'array', items: { type: 'string' }, description: 'Filter by label IDs' }, - }, - required: ['query'], - }, - }, - { - name: 'gmail_read_message', - description: 'Read a specific email message.', - inputSchema: { - type: 'object', - properties: { - messageId: { type: 'string', description: 'Message ID' }, - format: { type: 'string', enum: ['full', 'metadata', 'minimal'], default: 'full' }, - }, - required: ['messageId'], - }, - }, - { - name: 'gmail_send', - description: 'Send an email.', - inputSchema: { - type: 'object', - properties: { - to: { type: 'array', items: { type: 'string' }, description: 'Recipient email addresses' }, - cc: { type: 'array', items: { type: 'string' }, description: 'CC recipients' }, - bcc: { type: 'array', items: { type: 'string' }, description: 'BCC recipients' }, - subject: { type: 'string', description: 'Email subject' }, - body: { type: 'string', description: 'Email body (plain text or HTML)' }, - isHtml: { type: 'boolean', default: false, description: 'Whether body is HTML' }, - replyToMessageId: { type: 'string', description: 'Message ID to reply to' }, - }, - required: ['to', 'subject', 'body'], - }, - }, - { - name: 'gmail_create_draft', - description: 'Create an email draft.', - inputSchema: { - type: 'object', - properties: { - to: { type: 'array', items: { type: 'string' }, description: 'Recipient email addresses' }, - subject: { type: 'string', description: 'Email subject' }, - body: { type: 'string', description: 'Email body' }, - }, - required: ['to', 'subject', 'body'], - }, - }, - - // === Jira Tools === - { - name: 'jira_search_issues', - description: 'Search for Jira issues using JQL.', - inputSchema: { - type: 'object', - properties: { - jql: { type: 'string', description: 'JQL query (e.g., "project = PROJ AND status = Open")' }, - maxResults: { type: 'integer', minimum: 1, maximum: 100, default: 50 }, - fields: { type: 'array', items: { type: 'string' }, description: 'Fields to return' }, - }, - required: ['jql'], - }, - }, - { - name: 'jira_get_issue', - description: 'Get details of a specific Jira issue.', - inputSchema: { - type: 'object', - properties: { - issueKey: { type: 'string', description: 'Issue key (e.g., "PROJ-123")' }, - expand: { type: 'array', items: { type: 'string' }, description: 'Fields to expand (e.g., ["changelog", "comments"])' }, - }, - required: ['issueKey'], - }, - }, - { - name: 'jira_create_issue', - description: 'Create a new Jira issue.', - inputSchema: { - type: 'object', - properties: { - projectKey: { type: 'string', description: 'Project key' }, - issueType: { type: 'string', description: 'Issue type (Bug, Task, Story, Epic)' }, - summary: { type: 'string', description: 'Issue summary/title' }, - description: { type: 'string', description: 'Issue description' }, - priority: { type: 'string', enum: ['Highest', 'High', 'Medium', 'Low', 'Lowest'], default: 'Medium' }, - assignee: { type: 'string', description: 'Assignee username' }, - labels: { type: 'array', items: { type: 'string' }, description: 'Issue labels' }, - }, - required: ['projectKey', 'issueType', 'summary'], - }, - }, - { - name: 'jira_update_issue', - description: 'Update an existing Jira issue.', - inputSchema: { - type: 'object', - properties: { - issueKey: { type: 'string', description: 'Issue key' }, - fields: { type: 'object', description: 'Fields to update' }, - transition: { type: 'string', description: 'Transition to apply (e.g., "Done", "In Progress")' }, - }, - required: ['issueKey'], - }, - }, - { - name: 'jira_add_comment', - description: 'Add a comment to a Jira issue.', - inputSchema: { - type: 'object', - properties: { - issueKey: { type: 'string', description: 'Issue key' }, - body: { type: 'string', description: 'Comment text' }, - }, - required: ['issueKey', 'body'], - }, - }, - - // === GitHub Tools === - { - name: 'github_search_repos', - description: 'Search GitHub repositories.', - inputSchema: { - type: 'object', - properties: { - query: { type: 'string', description: 'Search query' }, - sort: { type: 'string', enum: ['stars', 'forks', 'updated'], default: 'stars' }, - limit: { type: 'integer', minimum: 1, maximum: 100, default: 10 }, - }, - required: ['query'], - }, - }, - { - name: 'github_list_issues', - description: 'List issues in a GitHub repository.', - inputSchema: { - type: 'object', - properties: { - owner: { type: 'string', description: 'Repository owner' }, - repo: { type: 'string', description: 'Repository name' }, - state: { type: 'string', enum: ['open', 'closed', 'all'], default: 'open' }, - labels: { type: 'array', items: { type: 'string' }, description: 'Filter by labels' }, - limit: { type: 'integer', minimum: 1, maximum: 100, default: 30 }, - }, - required: ['owner', 'repo'], - }, - }, - { - name: 'github_create_issue', - description: 'Create a new GitHub issue.', - inputSchema: { - type: 'object', - properties: { - owner: { type: 'string', description: 'Repository owner' }, - repo: { type: 'string', description: 'Repository name' }, - title: { type: 'string', description: 'Issue title' }, - body: { type: 'string', description: 'Issue body' }, - labels: { type: 'array', items: { type: 'string' }, description: 'Labels to apply' }, - assignees: { type: 'array', items: { type: 'string' }, description: 'Assignee usernames' }, - }, - required: ['owner', 'repo', 'title'], - }, - }, - { - name: 'github_list_prs', - description: 'List pull requests in a GitHub repository.', - inputSchema: { - type: 'object', - properties: { - owner: { type: 'string', description: 'Repository owner' }, - repo: { type: 'string', description: 'Repository name' }, - state: { type: 'string', enum: ['open', 'closed', 'all'], default: 'open' }, - limit: { type: 'integer', minimum: 1, maximum: 100, default: 30 }, - }, - required: ['owner', 'repo'], - }, - }, -]; - -// Tool handlers -async function handleTool(name: string, args: Record): Promise { - const timestamp = now(); - - switch (name) { - // Google Drive - case 'gdrive_search': { - const query = (args.query || '').toLowerCase(); - const limit = args.limit || 10; - const results = fakeFiles.filter(f => - f.name.toLowerCase().includes(query) || - f.owner.toLowerCase().includes(query) - ).slice(0, limit); - return { success: true, files: results, totalResults: results.length, query: args.query }; - } - - case 'gdrive_read_file': { - const file = fakeFiles.find(f => f.id === args.fileId); - if (!file) return { success: false, error: `File not found: ${args.fileId}` }; - return { - success: true, - file: file, - content: `[Simulated content for ${file.name}]\n\nThis is placeholder content representing the file "${file.name}".\nIn a real implementation, this would contain the actual file contents.`, - }; - } - - case 'gdrive_create_file': { - const newFile = { - id: generateId('FILE'), - name: args.name, - mimeType: args.mimeType || 'text/plain', - size: (args.content || '').length, - modifiedTime: timestamp, - owner: 'you@company.com', - }; - return { success: true, file: newFile, message: 'File created successfully' }; - } - - case 'gdrive_share_file': { - return { - success: true, - fileId: args.fileId, - sharedWith: args.email, - role: args.role || 'reader', - permissionId: generateId('PERM'), - message: `File shared with ${args.email} as ${args.role || 'reader'}`, - }; - } - - // Google Sheets - case 'sheets_read': { - const sheet = fakeSpreadsheets[args.spreadsheetId]; - if (!sheet) return { success: false, error: `Spreadsheet not found: ${args.spreadsheetId}` }; - const sheetData = sheet.sheets[0]; - return { - success: true, - spreadsheetId: args.spreadsheetId, - range: args.range, - values: sheetData.data, - majorDimension: 'ROWS', - }; - } - - case 'sheets_write': { - return { - success: true, - spreadsheetId: args.spreadsheetId, - updatedRange: args.range, - updatedRows: (args.values || []).length, - updatedColumns: (args.values?.[0] || []).length, - updatedCells: (args.values || []).flat().length, - }; - } - - case 'sheets_append': { - return { - success: true, - spreadsheetId: args.spreadsheetId, - tableRange: args.range, - updates: { - updatedRange: `${args.range.split('!')[0]}!A${Math.floor(Math.random() * 100) + 10}`, - updatedRows: (args.values || []).length, - updatedCells: (args.values || []).flat().length, - }, - }; - } - - case 'sheets_create': { - const newId = generateId('SHEET'); - return { - success: true, - spreadsheetId: newId, - spreadsheetUrl: `https://docs.google.com/spreadsheets/d/${newId}`, - title: args.title, - sheets: (args.sheetNames || ['Sheet1']).map((name: string, i: number) => ({ - sheetId: i, - title: name, - })), - }; - } - - // Salesforce - case 'salesforce_query': { - const soql = (args.soql || '').toLowerCase(); - let records: any[] = []; - - if (soql.includes('account')) { - records = fakeCompanies.map(c => ({ Id: c.id, Name: c.name, Industry: c.industry, AnnualRevenue: c.revenue })); - } else if (soql.includes('opportunity')) { - records = fakeOpportunities.map(o => ({ Id: o.id, Name: o.name, StageName: o.stage, Amount: o.amount, CloseDate: o.closeDate })); - } else if (soql.includes('contact') || soql.includes('user')) { - records = fakeUsers.map(u => ({ Id: u.id, Name: u.name, Email: u.email, Department: u.department })); - } - - return { - success: true, - totalSize: records.length, - done: true, - records: records.slice(0, args.limit || 100), - }; - } - - case 'salesforce_get_record': { - let record: any = null; - if (args.objectType === 'Account') { - record = fakeCompanies.find(c => c.id === args.recordId); - } else if (args.objectType === 'Opportunity') { - record = fakeOpportunities.find(o => o.id === args.recordId); - } - if (!record) return { success: false, error: `Record not found: ${args.recordId}` }; - return { success: true, record }; - } - - case 'salesforce_create_record': { - const newId = generateId(args.objectType?.substring(0, 3).toUpperCase() || 'REC'); - return { - success: true, - id: newId, - objectType: args.objectType, - message: `${args.objectType} created successfully`, - }; - } - - case 'salesforce_update_record': { - return { - success: true, - id: args.recordId, - objectType: args.objectType, - updatedFields: Object.keys(args.data || {}), - message: `${args.objectType} updated successfully`, - }; - } - - case 'salesforce_search': { - const term = (args.searchTerm || '').toLowerCase(); - const results: any[] = []; - fakeCompanies.filter(c => c.name.toLowerCase().includes(term)).forEach(c => results.push({ type: 'Account', ...c })); - fakeUsers.filter(u => u.name.toLowerCase().includes(term)).forEach(u => results.push({ type: 'Contact', ...u })); - return { success: true, searchRecords: results.slice(0, args.limit || 20) }; - } - - // Slack - case 'slack_send_message': { - return { - success: true, - ok: true, - channel: args.channel, - ts: `${Date.now() / 1000}.000100`, - message: { text: args.text, user: 'U001', ts: `${Date.now() / 1000}.000100` }, - }; - } - - case 'slack_get_messages': { - const channelId = args.channel.startsWith('#') ? fakeSlackChannels.find(c => c.name === args.channel.slice(1))?.id : args.channel; - const messages = fakeSlackMessages.filter(m => m.channel === channelId).slice(0, args.limit || 20); - return { success: true, ok: true, messages, hasMore: false }; - } - - case 'slack_search_messages': { - const query = (args.query || '').toLowerCase(); - const matches = fakeSlackMessages.filter(m => m.text.toLowerCase().includes(query)); - return { - success: true, - ok: true, - query: args.query, - messages: { total: matches.length, matches: matches.slice(0, args.limit || 20) }, - }; - } - - case 'slack_list_channels': { - return { success: true, ok: true, channels: fakeSlackChannels }; - } - - case 'slack_get_user_info': { - const user = fakeUsers.find(u => u.id === args.userId); - if (!user) return { success: false, ok: false, error: 'user_not_found' }; - return { success: true, ok: true, user: { ...user, realName: user.name, displayName: user.name.split(' ')[0] } }; - } - - case 'slack_set_status': { - return { - success: true, - ok: true, - profile: { - statusText: args.statusText, - statusEmoji: args.statusEmoji || ':speech_balloon:', - statusExpiration: args.expirationMinutes ? Date.now() + args.expirationMinutes * 60000 : 0, - }, - }; - } - - // Calendar - case 'calendar_list_events': { - return { success: true, items: fakeCalendarEvents }; - } - - case 'calendar_create_event': { - const newEvent = { - id: generateId('EVT'), - summary: args.summary, - description: args.description, - start: args.start, - end: args.end, - attendees: args.attendees || [], - htmlLink: `https://calendar.google.com/event?eid=${generateId('E')}`, - }; - return { success: true, event: newEvent }; - } - - case 'calendar_update_event': { - return { - success: true, - event: { - id: args.eventId, - ...(args.summary && { summary: args.summary }), - ...(args.description && { description: args.description }), - ...(args.start && { start: args.start }), - ...(args.end && { end: args.end }), - updated: timestamp, - }, - }; - } - - case 'calendar_delete_event': { - return { success: true, deleted: true, eventId: args.eventId }; - } - - // Gmail - case 'gmail_search': { - const query = (args.query || '').toLowerCase(); - const results = fakeEmails.filter(e => - e.subject.toLowerCase().includes(query) || - e.from.toLowerCase().includes(query) || - e.snippet.toLowerCase().includes(query) - ); - return { success: true, messages: results.slice(0, args.maxResults || 10), resultSizeEstimate: results.length }; - } - - case 'gmail_read_message': { - const email = fakeEmails.find(e => e.id === args.messageId); - if (!email) return { success: false, error: `Message not found: ${args.messageId}` }; - return { - success: true, - message: { - ...email, - body: `Full body of email: "${email.subject}"\n\n${email.snippet}\n\n[Additional content would appear here in a real implementation]`, - }, - }; - } - - case 'gmail_send': { - return { - success: true, - id: generateId('MSG'), - threadId: generateId('THR'), - labelIds: ['SENT'], - message: `Email sent to ${(args.to || []).join(', ')}`, - }; - } - - case 'gmail_create_draft': { - return { - success: true, - id: generateId('DRF'), - message: { id: generateId('MSG'), threadId: generateId('THR') }, - }; - } - - // Jira - case 'jira_search_issues': { - const issues = [ - { key: 'PROJ-101', summary: 'Implement user authentication', status: 'In Progress', priority: 'High', assignee: 'alice' }, - { key: 'PROJ-102', summary: 'Fix login page CSS', status: 'Open', priority: 'Medium', assignee: 'emma' }, - { key: 'PROJ-103', summary: 'Add API rate limiting', status: 'Done', priority: 'High', assignee: 'alice' }, - ]; - return { success: true, issues, total: issues.length, maxResults: args.maxResults || 50 }; - } - - case 'jira_get_issue': { - return { - success: true, - key: args.issueKey, - fields: { - summary: `Issue ${args.issueKey}`, - status: { name: 'In Progress' }, - priority: { name: 'High' }, - assignee: { displayName: 'Alice Johnson' }, - description: 'Detailed description of the issue...', - created: '2026-01-15T10:00:00Z', - updated: timestamp, - }, - }; - } - - case 'jira_create_issue': { - const issueKey = `${args.projectKey}-${Math.floor(Math.random() * 900) + 100}`; - return { - success: true, - id: generateId(''), - key: issueKey, - self: `https://your-domain.atlassian.net/rest/api/2/issue/${issueKey}`, - }; - } - - case 'jira_update_issue': { - return { success: true, key: args.issueKey, updated: true }; - } - - case 'jira_add_comment': { - return { - success: true, - id: generateId('CMT'), - issueKey: args.issueKey, - body: args.body, - created: timestamp, - }; - } - - // GitHub - case 'github_search_repos': { - const repos = [ - { fullName: 'facebook/react', description: 'A declarative UI library', stars: 220000, language: 'JavaScript' }, - { fullName: 'microsoft/vscode', description: 'Visual Studio Code', stars: 155000, language: 'TypeScript' }, - { fullName: 'torvalds/linux', description: 'Linux kernel source tree', stars: 165000, language: 'C' }, - ]; - return { success: true, totalCount: repos.length, items: repos.slice(0, args.limit || 10) }; - } - - case 'github_list_issues': { - const issues = [ - { number: 1234, title: 'Bug in component rendering', state: 'open', labels: ['bug'], user: 'contributor1' }, - { number: 1235, title: 'Feature request: dark mode', state: 'open', labels: ['enhancement'], user: 'contributor2' }, - ]; - return { success: true, issues }; - } - - case 'github_create_issue': { - return { - success: true, - number: Math.floor(Math.random() * 9000) + 1000, - title: args.title, - htmlUrl: `https://github.com/${args.owner}/${args.repo}/issues/${Math.floor(Math.random() * 9000) + 1000}`, - }; - } - - case 'github_list_prs': { - const prs = [ - { number: 567, title: 'Fix memory leak in worker', state: 'open', user: 'dev1', draft: false }, - { number: 568, title: 'Add TypeScript support', state: 'open', user: 'dev2', draft: true }, - ]; - return { success: true, pullRequests: prs }; - } - - default: - return { error: `Unknown tool: ${name}` }; - } -} - -// Server setup -const server = new Server( - { name: 'mcp-harness', version: '1.0.0' }, - { capabilities: { tools: {} } } -); - -server.setRequestHandler(ListToolsRequestSchema, async () => { - return { tools }; -}); - -server.setRequestHandler(CallToolRequestSchema, async (request) => { - const toolName = request.params.name; - const args = (request.params.arguments || {}) as Record; - - const result = await handleTool(toolName, args); - - // Log the tool call - logToolCall(toolName, args, result); - - return { - content: [ - { - type: 'text', - text: JSON.stringify(result, null, 2), - }, - ], - }; -}); - -const transport = new StdioServerTransport(); -await server.connect(transport); diff --git a/evals/open-model-gym/mcp-harness/tsconfig.json b/evals/open-model-gym/mcp-harness/tsconfig.json deleted file mode 100644 index 486fed81c..000000000 --- a/evals/open-model-gym/mcp-harness/tsconfig.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2022", - "module": "NodeNext", - "moduleResolution": "NodeNext", - "outDir": "dist", - "rootDir": "src", - "strict": true, - "esModuleInterop": true, - "skipLibCheck": true, - "declaration": true - }, - "include": ["src/**/*"] -} diff --git a/evals/open-model-gym/suite/.gitignore b/evals/open-model-gym/suite/.gitignore deleted file mode 100644 index 5dc418c7c..000000000 --- a/evals/open-model-gym/suite/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -node_modules/ -.workdir/ -.goose-root/ -.cache/ diff --git a/evals/open-model-gym/suite/package-lock.json b/evals/open-model-gym/suite/package-lock.json deleted file mode 100644 index 90443a050..000000000 --- a/evals/open-model-gym/suite/package-lock.json +++ /dev/null @@ -1,1059 +0,0 @@ -{ - "name": "agent-runner", - "version": "0.1.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "agent-runner", - "version": "0.1.0", - "dependencies": { - "glob": "^11.0.0", - "yaml": "^2.8.3" - }, - "devDependencies": { - "@types/node": "^22.0.0", - "tsx": "^4.22.4", - "typescript": "^5.5.0" - } - }, - "node_modules/@esbuild/aix-ppc64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", - "integrity": "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "aix" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.1.tgz", - "integrity": "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.1.tgz", - "integrity": "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-x64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.1.tgz", - "integrity": "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-arm64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.1.tgz", - "integrity": "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-x64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.1.tgz", - "integrity": "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-arm64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.1.tgz", - "integrity": "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-x64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.1.tgz", - "integrity": "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.1.tgz", - "integrity": "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.1.tgz", - "integrity": "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ia32": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.1.tgz", - "integrity": "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-loong64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.1.tgz", - "integrity": "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==", - "cpu": [ - "loong64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-mips64el": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.1.tgz", - "integrity": "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==", - "cpu": [ - "mips64el" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ppc64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.1.tgz", - "integrity": "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-riscv64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.1.tgz", - "integrity": "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-s390x": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.1.tgz", - "integrity": "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-x64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.1.tgz", - "integrity": "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-arm64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.1.tgz", - "integrity": "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-x64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.1.tgz", - "integrity": "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-arm64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.1.tgz", - "integrity": "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-x64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.1.tgz", - "integrity": "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openharmony-arm64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.1.tgz", - "integrity": "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/sunos-x64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.1.tgz", - "integrity": "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-arm64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.1.tgz", - "integrity": "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-ia32": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.1.tgz", - "integrity": "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-x64": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.1.tgz", - "integrity": "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@isaacs/cliui": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", - "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", - "license": "ISC", - "dependencies": { - "string-width": "^5.1.2", - "string-width-cjs": "npm:string-width@^4.2.0", - "strip-ansi": "^7.0.1", - "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", - "wrap-ansi": "^8.1.0", - "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/@types/node": { - "version": "22.19.7", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.7.tgz", - "integrity": "sha512-MciR4AKGHWl7xwxkBa6xUGxQJ4VBOmPTF7sL+iGzuahOFaO0jHCsuEfS80pan1ef4gWId1oWOweIhrDEYLuaOw==", - "dev": true, - "license": "MIT", - "dependencies": { - "undici-types": "~6.21.0" - } - }, - "node_modules/ansi-regex": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", - "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-regex?sponsor=1" - } - }, - "node_modules/ansi-styles": { - "version": "6.2.3", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", - "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/balanced-match": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", - "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", - "license": "MIT", - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/brace-expansion": { - "version": "5.0.6", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.6.tgz", - "integrity": "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==", - "license": "MIT", - "dependencies": { - "balanced-match": "^4.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "license": "MIT", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "license": "MIT" - }, - "node_modules/cross-spawn": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", - "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", - "license": "MIT", - "dependencies": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/eastasianwidth": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", - "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", - "license": "MIT" - }, - "node_modules/emoji-regex": { - "version": "9.2.2", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", - "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", - "license": "MIT" - }, - "node_modules/esbuild": { - "version": "0.28.1", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", - "integrity": "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "bin": { - "esbuild": "bin/esbuild" - }, - "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "@esbuild/aix-ppc64": "0.28.1", - "@esbuild/android-arm": "0.28.1", - "@esbuild/android-arm64": "0.28.1", - "@esbuild/android-x64": "0.28.1", - "@esbuild/darwin-arm64": "0.28.1", - "@esbuild/darwin-x64": "0.28.1", - "@esbuild/freebsd-arm64": "0.28.1", - "@esbuild/freebsd-x64": "0.28.1", - "@esbuild/linux-arm": "0.28.1", - "@esbuild/linux-arm64": "0.28.1", - "@esbuild/linux-ia32": "0.28.1", - "@esbuild/linux-loong64": "0.28.1", - "@esbuild/linux-mips64el": "0.28.1", - "@esbuild/linux-ppc64": "0.28.1", - "@esbuild/linux-riscv64": "0.28.1", - "@esbuild/linux-s390x": "0.28.1", - "@esbuild/linux-x64": "0.28.1", - "@esbuild/netbsd-arm64": "0.28.1", - "@esbuild/netbsd-x64": "0.28.1", - "@esbuild/openbsd-arm64": "0.28.1", - "@esbuild/openbsd-x64": "0.28.1", - "@esbuild/openharmony-arm64": "0.28.1", - "@esbuild/sunos-x64": "0.28.1", - "@esbuild/win32-arm64": "0.28.1", - "@esbuild/win32-ia32": "0.28.1", - "@esbuild/win32-x64": "0.28.1" - } - }, - "node_modules/foreground-child": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", - "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", - "license": "ISC", - "dependencies": { - "cross-spawn": "^7.0.6", - "signal-exit": "^4.0.1" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/glob": { - "version": "11.1.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-11.1.0.tgz", - "integrity": "sha512-vuNwKSaKiqm7g0THUBu2x7ckSs3XJLXE+2ssL7/MfTGPLLcrJQ/4Uq1CjPTtO5cCIiRxqvN6Twy1qOwhL0Xjcw==", - "license": "BlueOak-1.0.0", - "dependencies": { - "foreground-child": "^3.3.1", - "jackspeak": "^4.1.1", - "minimatch": "^10.1.1", - "minipass": "^7.1.2", - "package-json-from-dist": "^1.0.0", - "path-scurry": "^2.0.0" - }, - "bin": { - "glob": "dist/esm/bin.mjs" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "license": "ISC" - }, - "node_modules/jackspeak": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.1.1.tgz", - "integrity": "sha512-zptv57P3GpL+O0I7VdMJNBZCu+BPHVQUk55Ft8/QCJjTVxrnJHuVuX/0Bl2A6/+2oyR/ZMEuFKwmzqqZ/U5nPQ==", - "license": "BlueOak-1.0.0", - "dependencies": { - "@isaacs/cliui": "^8.0.2" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/lru-cache": { - "version": "11.2.5", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.5.tgz", - "integrity": "sha512-vFrFJkWtJvJnD5hg+hJvVE8Lh/TcMzKnTgCWmtBipwI5yLX/iX+5UB2tfuyODF5E7k9xEzMdYgGqaSb1c0c5Yw==", - "license": "BlueOak-1.0.0", - "engines": { - "node": "20 || >=22" - } - }, - "node_modules/minimatch": { - "version": "10.2.3", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.3.tgz", - "integrity": "sha512-Rwi3pnapEqirPSbWbrZaa6N3nmqq4Xer/2XooiOKyV3q12ML06f7MOuc5DVH8ONZIFhwIYQ3yzPH4nt7iWHaTg==", - "license": "BlueOak-1.0.0", - "dependencies": { - "brace-expansion": "^5.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "license": "ISC", - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, - "node_modules/package-json-from-dist": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", - "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", - "license": "BlueOak-1.0.0" - }, - "node_modules/path-key": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/path-scurry": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.1.tgz", - "integrity": "sha512-oWyT4gICAu+kaA7QWk/jvCHWarMKNs6pXOGWKDTr7cw4IGcUbW+PeTfbaQiLGheFRpjo6O9J0PmyMfQPjH71oA==", - "license": "BlueOak-1.0.0", - "dependencies": { - "lru-cache": "^11.0.0", - "minipass": "^7.1.2" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "license": "MIT", - "dependencies": { - "shebang-regex": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/signal-exit": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", - "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", - "license": "ISC", - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/string-width": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", - "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", - "license": "MIT", - "dependencies": { - "eastasianwidth": "^0.2.0", - "emoji-regex": "^9.2.2", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/string-width-cjs": { - "name": "string-width", - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/string-width-cjs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/string-width-cjs/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "license": "MIT" - }, - "node_modules/string-width-cjs/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.2.tgz", - "integrity": "sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA==", - "license": "MIT", - "dependencies": { - "ansi-regex": "^6.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/strip-ansi?sponsor=1" - } - }, - "node_modules/strip-ansi-cjs": { - "name": "strip-ansi", - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/tsx": { - "version": "4.22.4", - "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.22.4.tgz", - "integrity": "sha512-X8EX+XV4QR5xCsrgxaED954zTDfY8KqlDtskKEL0cHhyS/P8b4IFOvGDQpsC9Q1XnLq915wEfwwY/zzskCtmhg==", - "dev": true, - "license": "MIT", - "dependencies": { - "esbuild": "~0.28.0" - }, - "bin": { - "tsx": "dist/cli.mjs" - }, - "engines": { - "node": ">=18.0.0" - }, - "optionalDependencies": { - "fsevents": "~2.3.3" - } - }, - "node_modules/typescript": { - "version": "5.9.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", - "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "dev": true, - "license": "Apache-2.0", - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=14.17" - } - }, - "node_modules/undici-types": { - "version": "6.21.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", - "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "license": "ISC", - "dependencies": { - "isexe": "^2.0.0" - }, - "bin": { - "node-which": "bin/node-which" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/wrap-ansi": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", - "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^6.1.0", - "string-width": "^5.0.1", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/wrap-ansi-cjs": { - "name": "wrap-ansi", - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "license": "MIT" - }, - "node_modules/wrap-ansi-cjs/node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/yaml": { - "version": "2.8.3", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.3.tgz", - "integrity": "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg==", - "license": "ISC", - "bin": { - "yaml": "bin.mjs" - }, - "engines": { - "node": ">= 14.6" - }, - "funding": { - "url": "https://github.com/sponsors/eemeli" - } - } - } -} diff --git a/evals/open-model-gym/suite/package.json b/evals/open-model-gym/suite/package.json deleted file mode 100644 index f6771dc4c..000000000 --- a/evals/open-model-gym/suite/package.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "name": "agent-runner", - "version": "0.1.0", - "type": "module", - "scripts": { - "build": "tsc", - "test": "tsx src/runner.ts", - "test:scenario": "tsx src/runner.ts --scenario" - }, - "dependencies": { - "glob": "^11.0.0", - "yaml": "^2.8.3" - }, - "devDependencies": { - "@types/node": "^22.0.0", - "tsx": "^4.22.4", - "typescript": "^5.5.0" - } -} diff --git a/evals/open-model-gym/suite/scenarios/everyday-app-automation.yaml b/evals/open-model-gym/suite/scenarios/everyday-app-automation.yaml deleted file mode 100644 index 8e919a5e8..000000000 --- a/evals/open-model-gym/suite/scenarios/everyday-app-automation.yaml +++ /dev/null @@ -1,47 +0,0 @@ -name: everyday-app-automation -description: Multi-step workflow using everyday app tools (Slack, Jira, Calendar) with data dependencies -prompt: | - Using the available tools, complete these tasks: - 1. Search Slack for messages mentioning "quarterly review" - 2. Look up the user who posted the message about quarterly review to get their full name - 3. Create a Jira issue titled "Q1 Review Follow-ups" with a description that includes the name of the person who posted the Slack message - 4. Create a calendar event for next Monday at 2pm called "Review Discussion" - 5. Write a summary of what you did to a file called workflow-log.md - -tags: - - complex - - multi-step - - mcp-harness - - data-flow - -validate: - # Check workflow summary was written - - type: file_exists - path: workflow-log.md - - type: file_not_empty - path: workflow-log.md - - # Check Slack search was called with the right query - - type: tool_called - tool: slack_search_messages - args: - query: /quarterly.?review/ - - # Check that the agent looked up the user info (data dependency: requires reading user ID from search results) - - type: tool_called - tool: slack_get_user_info - args: - userId: /U004/ - - # Check Jira issue was created with expected title and includes the user's name (data dependency: requires reading name from user info) - - type: tool_called - tool: jira_create_issue - args: - summary: /q1.?review|follow.?up/ - description: /David.?Brown/ - - # Check calendar event was created with expected title - - type: tool_called - tool: calendar_create_event - args: - summary: /review.?discussion/ diff --git a/evals/open-model-gym/suite/scenarios/file-editing.yaml b/evals/open-model-gym/suite/scenarios/file-editing.yaml deleted file mode 100644 index 5eb1d818e..000000000 --- a/evals/open-model-gym/suite/scenarios/file-editing.yaml +++ /dev/null @@ -1,110 +0,0 @@ -name: file-editing -description: Navigate a small codebase and make a targeted edit -prompt: | - The User struct in user.rs is missing a display_name() method. - Add a method that returns the full name formatted as "first_name last_name". - -tags: - - file-editing - - code-navigation - -setup: - Cargo.toml: | - [package] - name = "user-service" - version = "0.1.0" - edition = "2021" - - [workspace] - - src/main.rs: | - mod models; - mod utils; - - use models::user::User; - - fn main() { - let user = User::new("Alice", "Smith", "alice@example.com"); - println!("Created user: {}", user.email()); - } - - src/models/mod.rs: | - pub mod user; - - src/models/user.rs: | - pub struct User { - first_name: String, - last_name: String, - email: String, - } - - impl User { - pub fn new(first_name: &str, last_name: &str, email: &str) -> Self { - Self { - first_name: first_name.to_string(), - last_name: last_name.to_string(), - email: email.to_string(), - } - } - - pub fn email(&self) -> &str { - &self.email - } - - pub fn first_name(&self) -> &str { - &self.first_name - } - - pub fn last_name(&self) -> &str { - &self.last_name - } - } - - src/utils/mod.rs: | - pub mod formatting; - - src/utils/formatting.rs: | - pub fn capitalize(s: &str) -> String { - let mut chars = s.chars(); - match chars.next() { - None => String::new(), - Some(c) => c.to_uppercase().collect::() + chars.as_str(), - } - } - -validate: - # The edit was made to the correct file - - type: file_exists - path: src/models/user.rs - name: user.rs exists - # Method was added - - type: file_matches - path: src/models/user.rs - regex: "fn\\s+display_name" - name: display_name() added - # Method returns a String or &str - - type: file_matches - path: src/models/user.rs - regex: "display_name.*->.*String|display_name.*->.*str" - name: has return type - # Original code preserved - - type: file_contains - path: src/models/user.rs - pattern: "pub fn email" - name: email() preserved - - type: file_contains - path: src/models/user.rs - pattern: "pub fn first_name" - name: first_name() preserved - # Other files untouched - - type: file_exists - path: src/main.rs - name: main.rs exists - - type: file_contains - path: src/main.rs - pattern: "mod models" - name: main.rs unchanged - # Code compiles - - type: command_succeeds - command: "cargo build" - name: cargo build diff --git a/evals/open-model-gym/suite/scenarios/multi-turn-edit.yaml b/evals/open-model-gym/suite/scenarios/multi-turn-edit.yaml deleted file mode 100644 index 03798a85c..000000000 --- a/evals/open-model-gym/suite/scenarios/multi-turn-edit.yaml +++ /dev/null @@ -1,80 +0,0 @@ -name: multi-turn-edit -description: Multi-turn conversation - add a method, then rename it - -tags: - - file-editing - - multi-turn - -setup: - Cargo.toml: | - [package] - name = "user-service" - version = "0.1.0" - edition = "2021" - - [workspace] - - src/main.rs: | - mod models; - - use models::user::User; - - fn main() { - let user = User::new("Alice", "Smith"); - println!("User: {} {}", user.first_name(), user.last_name()); - } - - src/models/mod.rs: | - pub mod user; - - src/models/user.rs: | - pub struct User { - first_name: String, - last_name: String, - } - - impl User { - pub fn new(first_name: &str, last_name: &str) -> Self { - Self { - first_name: first_name.to_string(), - last_name: last_name.to_string(), - } - } - - pub fn first_name(&self) -> &str { - &self.first_name - } - - pub fn last_name(&self) -> &str { - &self.last_name - } - } - -turns: - - prompt: | - Add an email() method to the User struct in src/models/user.rs. - It should return a generated email in the format "first_name.last_name@example.com" (lowercase). - validate: - - type: file_matches - path: src/models/user.rs - regex: "fn\\s+email" - name: email() added - - type: command_succeeds - command: "cargo build" - name: compiles after turn 1 - - - prompt: | - Actually, can you rename the email() method to generated_email() instead? - Make sure to update any references. - validate: - - type: file_matches - path: src/models/user.rs - regex: "fn\\s+generated_email" - name: renamed to generated_email() - - type: file_not_matches - path: src/models/user.rs - regex: "fn\\s+email\\s*\\(" - name: old email() removed - - type: command_succeeds - command: "cargo build" - name: compiles after turn 2 diff --git a/evals/open-model-gym/suite/scenarios/remove-feature.yaml b/evals/open-model-gym/suite/scenarios/remove-feature.yaml deleted file mode 100644 index 292815a98..000000000 --- a/evals/open-model-gym/suite/scenarios/remove-feature.yaml +++ /dev/null @@ -1,926 +0,0 @@ -name: remove-feature -description: Remove a feature (insert_image) from a document editor tool module -prompt: | - Remove the `insert_image` feature entirely from this codebase. - It should be removed from docs.py and any references in doc_utils.py. - Make sure no traces remain: the operation literal, the ImageParams class, - the image_params parameter, the insert_image handler function, and any - validation/error handling specific to insert_image should all be removed. - -tags: - - file-editing - - feature-removal - - python - -setup: - docs.py: | - """Tools for interacting with documents.""" - - import json - from dataclasses import dataclass - from functools import partial - from typing import Annotated, Any, Literal - - from pydantic import Field - - from doc_utils import execute_request, execute_requests - - DOC_OPERATIONS = Literal[ - "get_document", - "insert_text", - "append_text", - "replace_text", - "delete_content", - "insert_table", - "update_table_cell", - "insert_table_row", - "insert_table_column", - "delete_table_row", - "delete_table_column", - "insert_image", - "format_existing_text", - ] - - RESPONSE_CHAR_LIMIT = 400000 - - - @dataclass - class FormatTextParams: - """Parameters for format_existing_text operation.""" - - search_text: str - foreground_color: str | None = None - background_color: str | None = None - bold: bool | None = None - italic: bool | None = None - underline: bool | None = None - strikethrough: bool | None = None - font_size: int | None = None - font_family: str | None = None - heading_level: int | None = None - link_url: str | None = None - list_type: str | None = None - - - @dataclass - class TableParams: - """Parameters for table operations. - - Used by operations: insert_table, update_table_cell, insert_table_row, - insert_table_column, delete_table_row, delete_table_column. - """ - - rows: int | None = Field(None, description="Number of rows for insert_table operation") - columns: int | None = Field(None, description="Number of columns for insert_table operation") - row_index: int | None = Field( - None, description="Row index (0-based) for update_table_cell, insert_table_row, and delete_table_row" - ) - column_index: int | None = Field( - None, description="Column index (0-based) for update_table_cell, insert_table_column, and delete_table_column" - ) - insert_below: bool = Field( - False, description="For insert_table_row: True to insert below the specified row, False to insert above" - ) - insert_right: bool = Field( - False, description="For insert_table_column: True to insert right of column, False to insert left" - ) - - - @dataclass - class ImageParams: - """Parameters for image insertion.""" - - image_url: str = Field(..., description="URL of the image to insert (must be publicly accessible)") - width: int | None = Field(None, description="Width of the image in points (PT)") - height: int | None = Field(None, description="Height of the image in points (PT)") - - - async def doc_tool( - document_id: str, - operation: DOC_OPERATIONS = "get_document", - text: Annotated[ - str, - Field( - description=( - "Text content to insert, append, or match for replacement. " - "For insert_text and append_text, Markdown formatting is supported. " - "For replace_text, this should be unformatted plain text." - ) - ), - ] = "", - replace_text: Annotated[ - str, - Field( - description=( - "New plain text that will replace all occurrences of the original text. " - "Only used for the replace_text operation." - ) - ), - ] = "", - start_position: Annotated[ - int | None, - Field( - description="Document index (1-based) for insert or delete operations.", - ge=1, - ), - ] = None, - end_position: Annotated[ - int | None, - Field( - description="Document index (1-based, exclusive) for delete_content", - ge=1, - ), - ] = None, - table_params: Annotated[ - TableParams | None, - Field( - None, - description="Parameters for table operations", - ), - ] = None, - image_params: ImageParams | None = None, - format_params: FormatTextParams | None = None, - ) -> str: - """Perform operations on an existing document. - - Supported operations: - - get_document: Returns document content - - insert_text: Inserts text at a specific position - - append_text: Appends text at the end of the document - - replace_text: Replaces all instances of text with replace_text - - delete_content: Deletes content between two positions - - insert_table: Creates a table with specified rows and columns - - update_table_cell: Updates content in a specific table cell - - insert_table_row: Inserts a row above or below the specified row - - insert_table_column: Inserts a column left or right of the specified column - - delete_table_row: Deletes the specified row from a table - - delete_table_column: Deletes the specified column from a table - - insert_image: Inserts an image from a URL at the specified position - - format_existing_text: Finds and applies formatting to text - """ - if not text and operation in ["insert_text", "append_text", "replace_text"]: - raise ValueError(f"text is required for {operation} operation") - - table_operations = [ - "insert_table", - "update_table_cell", - "insert_table_row", - "insert_table_column", - "delete_table_row", - "delete_table_column", - ] - if operation in table_operations and not table_params: - raise ValueError(f"table_params is required for {operation} operation") - - if operation == "insert_image" and not image_params: - raise ValueError("image_params is required for insert_image operation") - - if operation == "format_existing_text" and not format_params: - raise ValueError("format_params is required for format_existing_text operation") - - operation_handlers = { - "get_document": partial(read_document, document_id), - "insert_text": partial(insert_text, document_id, text, start_position), - "append_text": partial(append_text, document_id, text), - "replace_text": partial(replace_all_text, document_id, text, replace_text), - "delete_content": partial(delete_content, document_id, start_position, end_position), - "insert_table": partial( - insert_table, - document_id, - table_params.rows if table_params else None, - table_params.columns if table_params else None, - start_position, - ), - "update_table_cell": partial( - update_table_cell, - document_id, - table_params.row_index if table_params else None, - table_params.column_index if table_params else None, - text, - start_position, - ), - "insert_table_row": partial( - modify_table_structure, - document_id, - "insert_row", - table_params.row_index if table_params else None, - None, - table_params.insert_below if table_params else False, - False, - start_position, - ), - "insert_table_column": partial( - modify_table_structure, - document_id, - "insert_column", - None, - table_params.column_index if table_params else None, - False, - table_params.insert_right if table_params else False, - start_position, - ), - "delete_table_row": partial( - modify_table_structure, - document_id, - "delete_row", - table_params.row_index if table_params else None, - None, - False, - False, - start_position, - ), - "delete_table_column": partial( - modify_table_structure, - document_id, - "delete_column", - None, - table_params.column_index if table_params else None, - False, - False, - start_position, - ), - "insert_image": partial( - insert_image, - document_id, - image_params.image_url if image_params else None, - start_position, - image_params.width if image_params else None, - image_params.height if image_params else None, - ), - "format_existing_text": partial( - format_existing_text, - document_id, - format_params.search_text if format_params else None, - format_params.foreground_color if format_params else None, - format_params.background_color if format_params else None, - format_params.font_size if format_params else None, - format_params.font_family if format_params else None, - format_params.bold if format_params else None, - format_params.italic if format_params else None, - format_params.underline if format_params else None, - format_params.strikethrough if format_params else None, - format_params.heading_level if format_params else None, - format_params.link_url if format_params else None, - format_params.list_type if format_params else None, - ), - } - - if operation not in operation_handlers: - raise ValueError(f"Invalid operation: {operation}") - - response = await operation_handlers[operation]() - return json.dumps(response, indent=2) - - - def _extract_text_from_element(element: dict) -> str: - """Recursively pull text from paragraphs, tables, etc.""" - text_parts = "" - if "paragraph" in element: - paragraph_elements = element["paragraph"].get("elements", []) - for el in paragraph_elements: - if "textRun" in el: - content = el["textRun"]["content"] - url = el["textRun"].get("textStyle", {}).get("link", {}).get("url") - if url: - text_parts += f"[{content}]({url})" - else: - text_parts += content - elif "table" in element: - table_rows = element["table"].get("tableRows", []) - for row in table_rows: - for cell in row["tableCells"]: - for cell_content in cell["content"]: - text_parts += _extract_text_from_element(cell_content) - text_parts += "\n" - return text_parts - - - async def read_document(document_id: str) -> dict[str, Any]: - """Returns document content.""" - document = await execute_request("get", document_id, {}) - content = document.get("body", {}).get("content", []) - text = "".join(_extract_text_from_element(e) for e in content) - - result = {"content": text, "document_id": document_id} - - if len(json.dumps(result)) > RESPONSE_CHAR_LIMIT: - raise ValueError(f"Document {document_id} is too large to read.") - - return result - - - async def insert_text( - document_id: str, text: str, start_position: int | None - ) -> dict[str, Any]: - """Insert text at a specific index in a document.""" - if start_position is None: - raise ValueError("start_position is required for insert_text operation") - - request = {"insertText": {"location": {"index": start_position}, "text": text}} - await execute_request("update", document_id, request) - - from doc_utils import calculate_utf16_length - - inserted_length = calculate_utf16_length(text) - - return { - "message": f"Inserted text at position {start_position}", - "text": text, - "start_position": start_position, - "end_position": start_position + inserted_length, - } - - - async def append_text(document_id: str, text: str) -> dict[str, Any]: - """Append text to the end of a document.""" - end_index = await get_document_last_index(document_id) - if text[0] != "\n": - text = "\n" + text - response = await insert_text(document_id, text, end_index - 1) - response["message"] = "Appended text to the end of the document" - return response - - - async def replace_all_text( - document_id: str, text: str, replace_text: str - ) -> dict[str, str]: - """Replace all instances of a string in a document.""" - if not replace_text: - raise ValueError("replace_text parameter is required for replace_text operation") - - request = { - "replaceAllText": { - "containsText": {"text": text, "matchCase": True}, - "replaceText": replace_text, - } - } - - response = await execute_request("update", document_id, request) - occurrences = response.get("occurrencesChanged", 0) - if occurrences == 0: - return { - "message": f"No occurrences of text '{text}' found in the document.", - "text": text, - "replace_text": replace_text, - } - - return { - "message": f"Replaced {occurrences} occurrences of '{text}' with '{replace_text}'", - "text": text, - "replace_text": replace_text, - } - - - async def delete_content( - document_id: str, start_index: int | None, end_index: int | None - ) -> dict[str, Any]: - """Delete content between two positions.""" - if start_index is None or end_index is None: - raise ValueError("Both start_index and end_index are required for delete_content") - - request = { - "deleteContentRange": { - "range": {"startIndex": start_index, "endIndex": end_index} - } - } - await execute_request("update", document_id, request) - return { - "message": f"Deleted content between index {start_index} and {end_index}", - "start_index": start_index, - "end_index": end_index, - } - - - async def get_document_last_index(document_id: str) -> int: - """Get the last index of the document.""" - document = await execute_request("get", document_id, {}) - return document.get("body", {}).get("content", [])[-1].get("endIndex", 1) - - - async def insert_table( - document_id: str, rows: int | None, columns: int | None, start_position: int | None - ) -> str: - """Insert a table at a specific position.""" - if not rows or not columns: - raise ValueError("rows and columns are required for insert_table operation") - if start_position is None: - raise ValueError("start_position is required for insert_table operation") - - request = { - "insertTable": { - "rows": rows, - "columns": columns, - "location": {"index": start_position}, - } - } - await execute_request("update", document_id, request) - return f"Inserted {rows}x{columns} table at position {start_position}" - - - def _table_matches_position(element: dict, start_position: int | None) -> bool: - """Check if a table element contains the specified position.""" - if start_position is None: - return True - table_start = element.get("startIndex") - table_end = element.get("endIndex") - return table_start <= start_position < table_end - - - def _find_table_cell_range( - document: dict, row_index: int, column_index: int, start_position: int | None = None - ) -> tuple[int, int] | None: - """Find the content range within a table cell.""" - content = document.get("body", {}).get("content", []) - for element in content: - if "table" in element: - if not _table_matches_position(element, start_position): - continue - table = element["table"] - if row_index < len(table.get("tableRows", [])): - row = table["tableRows"][row_index] - if column_index < len(row.get("tableCells", [])): - cell = row["tableCells"][column_index] - cell_content = cell.get("content", []) - for cell_element in cell_content: - if "paragraph" in cell_element: - para_start = cell_element.get("startIndex") - para_end = cell_element.get("endIndex") - return (para_start, para_end) - cell_start = cell.get("startIndex") - cell_end = cell.get("endIndex") - if cell_start is not None and cell_end is not None: - return (cell_start + 1, cell_end - 1) - return None - - - def _find_table_start_index( - document: dict, row_index: int, column_index: int, start_position: int | None = None - ) -> int | None: - """Find the start index of a table element.""" - content = document.get("body", {}).get("content", []) - for element in content: - if "table" in element: - if not _table_matches_position(element, start_position): - continue - table = element["table"] - if row_index < len(table.get("tableRows", [])): - row = table["tableRows"][row_index] - if column_index < len(row.get("tableCells", [])): - return element.get("startIndex") - return None - - - async def update_table_cell( - document_id: str, - row_index: int | None, - column_index: int | None, - text: str, - start_position: int | None = None, - ) -> str: - """Update content in a specific table cell.""" - if row_index is None or column_index is None: - raise ValueError("row_index and column_index are required for update_table_cell") - - document = await execute_request("get", document_id, {}) - cell_range = _find_table_cell_range(document, row_index, column_index, start_position) - if not cell_range: - raise ValueError(f"Table cell at row {row_index}, col {column_index} not found") - - cell_start, cell_end = cell_range - - requests = [] - if cell_end > cell_start + 1: - requests.append( - {"deleteContentRange": {"range": {"startIndex": cell_start, "endIndex": cell_end - 1}}} - ) - - requests.append({"insertText": {"location": {"index": cell_start}, "text": text}}) - - await execute_requests(document_id, requests) - return f"Updated cell at row {row_index}, column {column_index}" - - - async def modify_table_structure( - document_id: str, - operation: str, - row_index: int | None = None, - column_index: int | None = None, - insert_below: bool = False, - insert_right: bool = False, - start_position: int | None = None, - ) -> str: - """Modify table structure by inserting or deleting rows/columns.""" - if operation in ["insert_row", "delete_row"] and row_index is None: - raise ValueError(f"row_index is required for {operation} operation") - if operation in ["insert_column", "delete_column"] and column_index is None: - raise ValueError(f"column_index is required for {operation} operation") - - document = await execute_request("get", document_id, {}) - - if operation in ["insert_row", "delete_row"]: - table_start = _find_table_start_index(document, row_index, 0, start_position) - if not table_start: - raise ValueError(f"Table row {row_index} not found") - location = {"tableStartLocation": {"index": table_start}, "rowIndex": row_index} - else: - table_start = _find_table_start_index(document, 0, column_index, start_position) - if not table_start: - raise ValueError(f"Table column {column_index} not found") - location = {"tableStartLocation": {"index": table_start}, "columnIndex": column_index} - - operation_map = { - "insert_row": "insertTableRow", - "insert_column": "insertTableColumn", - "delete_row": "deleteTableRow", - "delete_column": "deleteTableColumn", - } - request_key = operation_map[operation] - - if operation == "insert_row": - request = {request_key: {"tableCellLocation": location, "insertBelow": insert_below}} - message = f"Inserted row {'below' if insert_below else 'above'} row {row_index}" - elif operation == "insert_column": - request = {request_key: {"tableCellLocation": location, "insertRight": insert_right}} - message = f"Inserted column {'right of' if insert_right else 'left of'} column {column_index}" - elif operation == "delete_row": - request = {request_key: {"tableCellLocation": location}} - message = f"Deleted row {row_index}" - else: - request = {request_key: {"tableCellLocation": location}} - message = f"Deleted column {column_index}" - - await execute_request("update", document_id, request) - return message - - - async def insert_image( - document_id: str, - image_url: str | None, - start_position: int | None, - width: int | None, - height: int | None, - ) -> str: - """Insert an image from a URL at the specified position.""" - if not image_url: - raise ValueError("image_url is required for insert_image operation") - if start_position is None: - raise ValueError("start_position is required for insert_image operation") - - request = { - "insertInlineImage": { - "uri": image_url, - "location": {"index": start_position}, - } - } - - if width or height: - object_size = {} - if width: - object_size["width"] = {"magnitude": width, "unit": "PT"} - if height: - object_size["height"] = {"magnitude": height, "unit": "PT"} - request["insertInlineImage"]["objectSize"] = object_size - - await execute_request("update", document_id, request) - return f"Inserted image from {image_url} at position {start_position}" - - - async def format_existing_text( - document_id: str, - search_text: str | None, - foreground_color: str | None, - background_color: str | None, - font_size: int | None, - font_family: str | None, - bold: bool | None, - italic: bool | None, - underline: bool | None, - strikethrough: bool | None, - heading_level: int | None, - link_url: str | None, - list_type: str | None, - ) -> str: - """Find and apply formatting to all occurrences of text.""" - if not search_text: - raise ValueError("search_text is required for format_existing_text operation") - - document = await execute_request("get", document_id, {}) - - from doc_utils import build_text_style, find_text_positions - - positions = find_text_positions(document, search_text) - - if not positions: - return f"No occurrences of '{search_text}' found" - - text_style, fields = build_text_style( - foreground_color=foreground_color, - background_color=background_color, - font_size=font_size, - font_family=font_family, - bold=bold, - italic=italic, - underline=underline, - strikethrough=strikethrough, - link_url=link_url, - ) - - if not fields and not heading_level and not list_type: - raise ValueError("At least one formatting option must be specified") - - positions.sort(reverse=True) - - requests = [] - for start, end in positions: - range_dict = {"startIndex": start, "endIndex": end} - - if fields: - requests.append( - {"updateTextStyle": {"range": range_dict, "textStyle": text_style, "fields": fields}} - ) - - if heading_level: - if not (1 <= heading_level <= 6): - raise ValueError("heading_level must be between 1 and 6") - requests.append( - { - "updateParagraphStyle": { - "range": range_dict, - "paragraphStyle": {"namedStyleType": f"HEADING_{heading_level}"}, - "fields": "namedStyleType", - } - } - ) - - if list_type: - if list_type == "bullet": - preset = "BULLET_DISC_CIRCLE_SQUARE" - elif list_type == "numbered": - preset = "NUMBERED_DECIMAL_ALPHA_ROMAN" - else: - raise ValueError("list_type must be 'bullet' or 'numbered'") - requests.append({"createParagraphBullets": {"range": range_dict, "bulletPreset": preset}}) - - if not requests: - raise ValueError(f"No formatting requests generated for '{search_text}'.") - - await execute_requests(document_id, requests) - return f"Formatted {len(positions)} occurrences of '{search_text}'" - - doc_utils.py: | - """Utility functions for document operations.""" - - import re - from typing import Any - - - async def execute_request( - method: str, document_id: str, request: dict, **kwargs - ) -> dict[str, Any]: - """Execute a single document API request.""" - raise NotImplementedError("Stub: would call document backend") - - - async def execute_requests( - document_id: str, requests: list[dict], **kwargs - ) -> dict[str, Any]: - """Execute a batch of document API requests.""" - raise NotImplementedError("Stub: would call document backend") - - - def calculate_utf16_length(text: str) -> int: - """Calculate text length in UTF-16 code units.""" - return len(text.encode("utf-16-le")) // 2 - - - def hex_to_rgb(hex_color: str) -> dict: - """Convert hex color to RGB format (0.0-1.0 range).""" - hex_color = hex_color.lstrip("#") - - if len(hex_color) == 3: - hex_color = "".join([c * 2 for c in hex_color]) - - if not re.match(r"^[0-9A-Fa-f]{6}$", hex_color): - raise ValueError(f"Invalid hex color: {hex_color}") - - return { - "color": { - "rgbColor": { - "red": int(hex_color[0:2], 16) / 255.0, - "green": int(hex_color[2:4], 16) / 255.0, - "blue": int(hex_color[4:6], 16) / 255.0, - } - } - } - - - def build_text_style( - foreground_color: str | None = None, - background_color: str | None = None, - font_size: int | None = None, - font_family: str | None = None, - bold: bool | None = None, - italic: bool | None = None, - underline: bool | None = None, - strikethrough: bool | None = None, - link_url: str | None = None, - ) -> tuple[dict, str]: - """Build text style object and field mask from provided formatting options.""" - style = {} - fields = [] - - if foreground_color: - style["foregroundColor"] = hex_to_rgb(foreground_color) - fields.append("foregroundColor") - - if background_color: - style["backgroundColor"] = hex_to_rgb(background_color) - fields.append("backgroundColor") - - if font_size is not None: - style["fontSize"] = {"magnitude": font_size, "unit": "PT"} - fields.append("fontSize") - - if font_family: - style["weightedFontFamily"] = {"fontFamily": font_family} - fields.append("weightedFontFamily") - - if bold is not None: - style["bold"] = bold - fields.append("bold") - - if italic is not None: - style["italic"] = italic - fields.append("italic") - - if underline is not None: - style["underline"] = underline - fields.append("underline") - - if strikethrough is not None: - style["strikethrough"] = strikethrough - fields.append("strikethrough") - - if link_url: - style["link"] = {"url": link_url} - fields.append("link") - - return style, ",".join(fields) - - - def find_text_positions( - document: dict, search_text: str - ) -> list[tuple[int, int]]: - """Find all occurrences of text in a document with UTF-16 positions.""" - positions = [] - search_len = calculate_utf16_length(search_text) - - content = document.get("body", {}).get("content", []) - _find_text_in_elements(content, search_text, search_len, positions) - - return positions - - - def _find_text_in_elements( - elements: list[dict], search_text: str, search_len: int, positions: list - ): - """Recursively find text in document elements.""" - for element in elements: - if "paragraph" in element: - para_elements = element["paragraph"].get("elements", []) - for elem in para_elements: - if "textRun" in elem: - text_run = elem["textRun"] - content = text_run.get("content", "") - start_index = elem.get("startIndex", 0) - - idx = 0 - while True: - pos = content.find(search_text, idx) - if pos == -1: - break - prefix_len = calculate_utf16_length(content[:pos]) - match_start = start_index + prefix_len - match_end = match_start + search_len - positions.append((match_start, match_end)) - idx = pos + len(search_text) - - elif "table" in element: - table_rows = element["table"].get("tableRows", []) - for row in table_rows: - for cell in row.get("tableCells", []): - cell_content = cell.get("content", []) - _find_text_in_elements(cell_content, search_text, search_len, positions) - -validate: - # docs.py must still exist - - type: file_exists - path: docs.py - name: docs.py exists - - # doc_utils.py must still exist - - type: file_exists - path: doc_utils.py - name: doc_utils.py exists - - # insert_image removed from the operations literal - - type: file_not_matches - path: docs.py - regex: "insert_image" - name: insert_image removed from operations literal - - # ImageParams class removed - - type: file_not_matches - path: docs.py - regex: "ImageParams" - name: ImageParams class removed - - # image_params parameter removed - - type: file_not_matches - path: docs.py - regex: "image_params" - name: image_params parameter removed - - # insert_image function removed - - type: file_not_matches - path: docs.py - regex: "async def insert_image" - name: insert_image function removed - - # image_url reference removed - - type: file_not_matches - path: docs.py - regex: "image_url" - name: image_url references removed - - # insertInlineImage reference removed - - type: file_not_matches - path: docs.py - regex: "insertInlineImage" - name: insertInlineImage reference removed - - # Other operations still present - - type: file_contains - path: docs.py - pattern: "get_document" - name: get_document preserved - - - type: file_contains - path: docs.py - pattern: "insert_text" - name: insert_text preserved - - - type: file_contains - path: docs.py - pattern: "replace_text" - name: replace_text preserved - - - type: file_contains - path: docs.py - pattern: "insert_table" - name: insert_table preserved - - - type: file_contains - path: docs.py - pattern: "format_existing_text" - name: format_existing_text preserved - - - type: file_contains - path: docs.py - pattern: "async def doc_tool" - name: doc_tool function preserved - - - type: file_contains - path: docs.py - pattern: "FormatTextParams" - name: FormatTextParams preserved - - - type: file_contains - path: docs.py - pattern: "TableParams" - name: TableParams preserved - - # doc_utils.py should be untouched (no image references existed there) - - type: file_contains - path: doc_utils.py - pattern: "def build_text_style" - name: doc_utils build_text_style preserved - - - type: file_contains - path: doc_utils.py - pattern: "def find_text_positions" - name: doc_utils find_text_positions preserved - - - type: file_contains - path: doc_utils.py - pattern: "def calculate_utf16_length" - name: doc_utils calculate_utf16_length preserved - - # Python syntax check - - type: command_succeeds - command: "python3 -c \"import ast; ast.parse(open('docs.py').read())\"" - name: docs.py valid python syntax - - - type: command_succeeds - command: "python3 -c \"import ast; ast.parse(open('doc_utils.py').read())\"" - name: doc_utils.py valid python syntax diff --git a/evals/open-model-gym/suite/src/gym.png b/evals/open-model-gym/suite/src/gym.png deleted file mode 100644 index 242cf3be7..000000000 Binary files a/evals/open-model-gym/suite/src/gym.png and /dev/null differ diff --git a/evals/open-model-gym/suite/src/runner.ts b/evals/open-model-gym/suite/src/runner.ts deleted file mode 100644 index 62ebff6f9..000000000 --- a/evals/open-model-gym/suite/src/runner.ts +++ /dev/null @@ -1,1565 +0,0 @@ -#!/usr/bin/env node -import { mkdirSync, writeFileSync, rmSync, readdirSync, existsSync, copyFileSync } from "node:fs"; -import { join, basename, dirname, resolve } from "node:path"; -import { homedir } from "node:os"; -import { execSync, execFileSync } from "node:child_process"; -import { parse, stringify } from "yaml"; -import { readFileSync } from "node:fs"; -import { createHash } from "node:crypto"; -import type { Scenario, TestResult, TestRun, Turn } from "./types.js"; -import { validateAll } from "./validator.js"; - -// ============================================================================= -// Types -// ============================================================================= - -type RunnerType = "goose" | "opencode" | "pi"; - -interface ModelConfig { - name: string; - provider: string; - model: string; -} - -interface RunnerConfig { - name: string; - type: RunnerType; - bin: string; - extensions?: string[]; // goose-specific - stdio?: string[]; // MCP servers -} - -interface MatrixEntry { - scenario: string; - models?: string[]; // omit = all models - runners?: string[]; // omit = all runners -} - -interface SuiteConfig { - models: ModelConfig[]; - runners: RunnerConfig[]; - matrix?: MatrixEntry[]; -} - -// A test pair: scenario × model × runner -interface TestPair { - scenario: Scenario; - model: ModelConfig; - runner: RunnerConfig; -} - -interface TestResultWithLog extends TestResult { - logFile: string; - runnerName: string; - toolCalls: number; - turns: number; - cached?: boolean; -} - -// ============================================================================= -// Cache Types -// ============================================================================= - -interface CacheInputs { - scenarioHash: string; - modelKey: string; - runnerHash: string; - binaryHash: string; - mcpHarnessHash: string; - timeoutMs: number; -} - -interface CacheEntry { - timestamp: string; - inputs: CacheInputs; - result: { - status: "passed" | "failed"; - validations: Array<{ rule: any; passed: boolean; message?: string }>; - duration: number; - toolCalls: number; - turns: number; - errors?: string[]; - }; - logFile: string; -} - -interface CacheIndex { - version: number; - entries: Record; -} - -// ============================================================================= -// Output directory resolution -// ============================================================================= -// All run artifacts (cache, isolated agent config roots, scratch workdir, logs, -// and the HTML report) live under a single base directory. By default this is -// the in-repo gym directory, so existing behavior is unchanged. Set the -// GYM_OUTPUT_DIR env var or pass --output-dir= to redirect everything -// outside the repo and keep your checkout clean, e.g.: -// -// GYM_OUTPUT_DIR=~/.goose/gym-runs/$(date +%Y%d%m%H%M%S) just run -// -// config.yaml and scenarios/ are inputs and always read from the repo. - -const SUITE_DIR = join(import.meta.dirname, ".."); // .../open-model-gym/suite -const GYM_DIR = join(import.meta.dirname, "../.."); // .../open-model-gym - -function expandHome(p: string): string { - return p === "~" || p.startsWith("~/") ? join(homedir(), p.slice(1)) : p; -} - -// Resolved output base, or null to fall back to the legacy in-repo locations. -const OUTPUT_DIR: string | null = (() => { - const flag = process.argv - .find((a) => a.startsWith("--output-dir=")) - ?.split("=")[1]; - const base = flag ?? process.env.GYM_OUTPUT_DIR; - // Resolve to an absolute path: runners exec with cwd set to the workdir, so a - // relative base would make prompt/log paths resolve against the wrong dir. - return base ? resolve(expandHome(base)) : null; -})(); - -// Resolve an artifact path under OUTPUT_DIR when set, else its legacy anchor. -function artifactPath(name: string, legacyAnchor: string): string { - return join(OUTPUT_DIR ?? legacyAnchor, name); -} - -// Agent timeout -// ============================================================================= -// Per-invocation timeout for an agent run, in milliseconds. Larger local models -// can exceed the old fixed 5-minute cap on heavier scenarios and get killed -// mid-task (recorded as a failure rather than a timeout). Override the cap with -// GYM_AGENT_TIMEOUT (seconds) or --agent-timeout=; default 300s. -const AGENT_TIMEOUT_MS = (() => { - const flag = process.argv - .find((a) => a.startsWith("--agent-timeout=")) - ?.split("=")[1]; - const secs = parseInt(flag ?? process.env.GYM_AGENT_TIMEOUT ?? "", 10); - return (Number.isFinite(secs) && secs > 0 ? secs : 300) * 1000; -})(); - -// ============================================================================= -// Cache Utilities -// ============================================================================= - -const CACHE_DIR = artifactPath(".cache", SUITE_DIR); -const CACHE_INDEX_PATH = join(CACHE_DIR, "index.json"); -const CACHE_LOGS_DIR = join(CACHE_DIR, "logs"); -const CACHE_VERSION = 1; - -function sha256(data: string | Buffer): string { - return createHash("sha256").update(data).digest("hex").slice(0, 16); -} - -function loadCache(): CacheIndex { - try { - if (existsSync(CACHE_INDEX_PATH)) { - const data = JSON.parse(readFileSync(CACHE_INDEX_PATH, "utf-8")); - if (data.version === CACHE_VERSION) { - return data; - } - console.log("Cache version mismatch, starting fresh"); - } - } catch (e) { - console.log("Cache corrupted, starting fresh"); - } - return { version: CACHE_VERSION, entries: {} }; -} - -function saveCache(cache: CacheIndex): void { - mkdirSync(CACHE_DIR, { recursive: true }); - writeFileSync(CACHE_INDEX_PATH, JSON.stringify(cache, null, 2)); -} - -function getBinaryHash(binName: string): string { - try { - const binaryPath = execSync(`which ${binName}`, { encoding: "utf-8" }).trim(); - const binaryContent = readFileSync(binaryPath); - return sha256(binaryContent); - } catch (e) { - // Fallback to version string if we can't read the binary - try { - const version = execSync(`${binName} --version 2>/dev/null || echo "unknown"`, { encoding: "utf-8" }).trim(); - return sha256(version); - } catch { - return "unknown"; - } - } -} - -function getMcpHarnessHash(): string { - const mcpHarnessPath = join(import.meta.dirname, "../../mcp-harness/dist/index.js"); - try { - if (existsSync(mcpHarnessPath)) { - return sha256(readFileSync(mcpHarnessPath)); - } - } catch (e) { - // Ignore - } - return "no-mcp-harness"; -} - -function computeCacheKey(pair: TestPair, binaryHashes: Map, mcpHarnessHash: string): { key: string; inputs: CacheInputs } { - // Hash scenario content (name + prompt/turns + setup + validate) - const scenarioContent = stringify({ - name: pair.scenario.name, - prompt: pair.scenario.prompt, - turns: pair.scenario.turns, - setup: pair.scenario.setup, - validate: pair.scenario.validate, - }); - const scenarioHash = sha256(scenarioContent); - - // Model key - const modelKey = `${pair.model.provider}/${pair.model.model}`; - - // Hash runner config - const runnerContent = JSON.stringify({ - name: pair.runner.name, - type: pair.runner.type, - extensions: pair.runner.extensions ?? [], - stdio: pair.runner.stdio ?? [], - }); - const runnerHash = sha256(runnerContent); - - // Binary hash (cached per binary name) - const binaryHash = binaryHashes.get(pair.runner.bin) ?? "unknown"; - - const inputs: CacheInputs = { - scenarioHash, - modelKey, - runnerHash, - binaryHash, - mcpHarnessHash, - timeoutMs: AGENT_TIMEOUT_MS, - }; - - // Combine all into single key. The timeout is included so a result cached - // under a short timeout (e.g. a 300s ETIMEDOUT) isn't reused when the run is - // retried with a larger GYM_AGENT_TIMEOUT. - const key = sha256( - scenarioHash + modelKey + runnerHash + binaryHash + mcpHarnessHash + AGENT_TIMEOUT_MS, - ); - - return { key, inputs }; -} - -function getCachedResult( - cache: CacheIndex, - cacheKey: string, - pair: TestPair, - logsDir: string -): TestResultWithLog | null { - const entry = cache.entries[cacheKey]; - if (!entry) return null; - - // Verify the cached log file exists - const cachedLogPath = join(CACHE_LOGS_DIR, entry.logFile); - if (!existsSync(cachedLogPath)) { - console.log(` Cache log missing, will re-run`); - delete cache.entries[cacheKey]; - return null; - } - - // Copy cached log to current logs directory - const testId = `${pair.scenario.name}_${pair.model.name}_${pair.runner.name}`.replace(/[\/\\:]/g, "_"); - const logFile = join(logsDir, `${testId}_cached.log`); - mkdirSync(logsDir, { recursive: true }); - copyFileSync(cachedLogPath, logFile); - - // Reconstruct result - const config = { - provider: pair.model.provider, - model: pair.model.model, - extensions: pair.runner.extensions, - stdio: pair.runner.stdio, - }; - - const run: TestRun = { - scenario: pair.scenario, - config, - workdir: "", // Not relevant for cached results - startTime: new Date(entry.timestamp), - endTime: new Date(new Date(entry.timestamp).getTime() + entry.result.duration), - status: entry.result.status, - errors: entry.result.errors, - }; - - return { - run, - validations: entry.result.validations, - logFile, - runnerName: pair.runner.name, - toolCalls: entry.result.toolCalls, - turns: entry.result.turns, - cached: true, - }; -} - -function storeCacheResult( - cache: CacheIndex, - cacheKey: string, - inputs: CacheInputs, - result: TestResultWithLog -): void { - // Copy log to cache directory - const logFileName = `${cacheKey}.log`; - const cachedLogPath = join(CACHE_LOGS_DIR, logFileName); - mkdirSync(CACHE_LOGS_DIR, { recursive: true }); - - try { - copyFileSync(result.logFile, cachedLogPath); - } catch (e) { - console.log(` Warning: Could not cache log file`); - return; - } - - cache.entries[cacheKey] = { - timestamp: new Date().toISOString(), - inputs, - result: { - status: result.run.status as "passed" | "failed", - validations: result.validations, - duration: result.run.endTime && result.run.startTime - ? result.run.endTime.getTime() - result.run.startTime.getTime() - : 0, - toolCalls: result.toolCalls, - turns: result.turns, - errors: result.run.errors, - }, - logFile: logFileName, - }; - - saveCache(cache); -} - -function clearCache(): void { - if (existsSync(CACHE_DIR)) { - rmSync(CACHE_DIR, { recursive: true, force: true }); - console.log("Cache cleared"); - } else { - console.log("No cache to clear"); - } -} - -// ============================================================================= -// Goose Runner -// ============================================================================= - -const PLATFORM_EXTENSIONS = new Set([ - "todo", "skills", "code_execution", "extensionmanager", - "chatrecall", "apps", "imagegenerator" -]); - -// Isolated goose config directory -const GOOSE_ROOT = artifactPath(".goose-root", SUITE_DIR); -const GOOSE_CONFIG_DIR = join(GOOSE_ROOT, "config"); - -function generateGooseConfig(model: ModelConfig, runner: RunnerConfig): object { - const extensions: Record = {}; - - // Add extensions (detect platform vs builtin) - for (const ext of runner.extensions ?? []) { - if (PLATFORM_EXTENSIONS.has(ext)) { - extensions[ext] = { - enabled: true, - type: "platform", - name: ext, - bundled: true, - }; - } else { - extensions[ext] = { - enabled: true, - type: "builtin", - name: ext, - timeout: 300, - bundled: true, - }; - } - } - - // Add stdio MCP servers - for (const extCmd of runner.stdio ?? []) { - const parts = extCmd.split(" "); - const cmd = parts[0]; - const args = parts.slice(1); - const name = basename(args[args.length - 1] || cmd).replace(/\.[^.]+$/, ""); - - extensions[name] = { - enabled: true, - type: "stdio", - name, - cmd, - args, - timeout: 300, - }; - } - - return { - extensions, - GOOSE_PROVIDER: model.provider, - GOOSE_MODEL: model.model, - GOOSE_TELEMETRY_ENABLED: false, - }; -} - -async function runGooseAgent( - model: ModelConfig, - runner: RunnerConfig, - prompt: string, - workdir: string, - sessionName?: string, // If provided, use/continue this session - resume: boolean = false // If true, resume existing session (for turn 2+) -): Promise { - const promptFile = join(workdir, ".goose-prompt.txt"); - writeFileSync(promptFile, prompt); - - // Write goose config - mkdirSync(GOOSE_CONFIG_DIR, { recursive: true }); - const gooseConfig = generateGooseConfig(model, runner); - writeFileSync(join(GOOSE_CONFIG_DIR, "config.yaml"), stringify(gooseConfig)); - - let cmd: string; - if (sessionName) { - if (resume) { - cmd = `${runner.bin} run -i "${promptFile}" --name "${sessionName}" --resume`; - console.log(` Running: ${runner.bin} run -i --name "${sessionName}" --resume`); - } else { - // First turn: create new session with this name - cmd = `${runner.bin} run -i "${promptFile}" --name "${sessionName}"`; - console.log(` Running: ${runner.bin} run -i --name "${sessionName}"`); - } - } else { - cmd = `${runner.bin} run -i "${promptFile}" --no-session`; - console.log(` Running: ${runner.bin} run -i --no-session`); - } - - const output = execSync(cmd, { - cwd: workdir, - env: { - ...process.env, - GOOSE_PATH_ROOT: GOOSE_ROOT, - MCP_HARNESS_LOG: join(workdir, "tool-calls.log"), - }, - timeout: AGENT_TIMEOUT_MS, - encoding: "utf-8", - }); - - return output; -} - -// ============================================================================= -// OpenCode Runner -// ============================================================================= - -// Isolated opencode config directory -const OPENCODE_ROOT = artifactPath(".opencode-root", SUITE_DIR); - -function generateOpenCodeConfig(model: ModelConfig, runner: RunnerConfig, workdir: string): object { - const mcp: Record = {}; - - // Add stdio MCP servers - for (const extCmd of runner.stdio ?? []) { - const parts = extCmd.split(" "); - const cmd = parts[0]; - const args = parts.slice(1); - const name = basename(args[args.length - 1] || cmd).replace(/\.[^.]+$/, ""); - - mcp[name] = { - type: "local", - command: [cmd, ...args], - enabled: true, - environment: { - MCP_HARNESS_LOG: join(workdir, "tool-calls.log"), - }, - }; - } - - const config: Record = { - $schema: "https://opencode.ai/config.json", - mcp, - }; - - // Handle ollama as a custom provider (OpenCode doesn't have built-in ollama support) - if (model.provider === "ollama") { - config.model = `ollama/${model.model}`; - config.provider = { - ollama: { - npm: "@ai-sdk/openai-compatible", - name: "Ollama (local)", - options: { - baseURL: "http://localhost:11434/v1", - }, - models: { - [model.model]: { - name: model.name, - }, - }, - }, - }; - } else { - // Standard providers (anthropic, openai, etc.) - config.model = `${model.provider}/${model.model}`; - } - - return config; -} - -async function runOpenCodeAgent( - model: ModelConfig, - runner: RunnerConfig, - prompt: string, - workdir: string, - resume: boolean = false -): Promise { - // Write opencode.json config to workdir - const openCodeConfig = generateOpenCodeConfig(model, runner, workdir); - writeFileSync(join(workdir, "opencode.json"), JSON.stringify(openCodeConfig, null, 2)); - - // Write prompt to file (use cat to avoid shell escaping issues) - const promptFile = join(workdir, ".opencode-prompt.txt"); - writeFileSync(promptFile, prompt); - - // Ensure isolated config directory exists - mkdirSync(OPENCODE_ROOT, { recursive: true }); - - // Use --continue on turn 2+ to continue last session - const continueFlag = resume ? "--continue " : ""; - const cmd = `${runner.bin} run ${continueFlag}"$(cat "${promptFile}")"`; - console.log(` Running: ${runner.bin} run ${continueFlag}""`); - - const output = execSync(cmd, { - cwd: workdir, - env: { - ...process.env, - XDG_CONFIG_HOME: OPENCODE_ROOT, - XDG_DATA_HOME: OPENCODE_ROOT, - }, - timeout: AGENT_TIMEOUT_MS, - encoding: "utf-8", - shell: "/bin/bash", - }); - - return output; -} - - -// ============================================================================= -// Pi Runner -// ============================================================================= - -// Pi takes --provider and --model as CLI arguments -// MCP support via pi-mcp-adapter: `pi install npm:pi-mcp-adapter` - -// Isolated Pi config directory (like Goose/OpenCode) -const PI_CONFIG_DIR = artifactPath(".pi-root", SUITE_DIR); - -// User's real Pi config (for copying auth.json) -const PI_USER_CONFIG = join(homedir(), ".pi", "agent"); - -/** - * Generate models.json for Pi with the test model. - * For ollama models, we need to define them since Pi doesn't have built-in ollama support. - */ -function generatePiModelsConfig(model: ModelConfig): object { - // Only generate config for ollama provider (others are built-in) - if (model.provider !== "ollama") { - return { providers: {} }; - } - - return { - providers: { - ollama: { - baseUrl: "http://localhost:11434/v1", - api: "openai-completions", - apiKey: "ollama", // Ollama doesn't need a real key - models: [ - { - id: model.model, - name: model.name, - reasoning: false, - input: ["text"], - contextWindow: 128000, - maxTokens: 32768, - compat: { - supportsUsageInStreaming: false, - maxTokensField: "max_tokens", - supportsDeveloperRole: false - } - } - ] - } - } - }; -} - -async function runPiAgent( - model: ModelConfig, - runner: RunnerConfig, - prompt: string, - workdir: string, - sessionName?: string, // If provided, use/continue this session (for multi-turn) - resume: boolean = false // If true, continue existing session (for turn 2+) -): Promise { - // Write prompt to file (use cat to avoid shell escaping issues) - const promptFile = join(workdir, ".pi-prompt.txt"); - writeFileSync(promptFile, prompt); - - // Set up isolated Pi config directory - mkdirSync(PI_CONFIG_DIR, { recursive: true }); - - // Generate models.json with the test model (for ollama) - const modelsConfig = generatePiModelsConfig(model); - writeFileSync(join(PI_CONFIG_DIR, "models.json"), JSON.stringify(modelsConfig, null, 2)); - - // Copy auth.json from user's config (for API keys) - const userAuthPath = join(PI_USER_CONFIG, "auth.json"); - if (existsSync(userAuthPath)) { - copyFileSync(userAuthPath, join(PI_CONFIG_DIR, "auth.json")); - } - - // Copy settings.json from user's config (for installed packages like pi-mcp-adapter) - const userSettingsPath = join(PI_USER_CONFIG, "settings.json"); - if (existsSync(userSettingsPath)) { - copyFileSync(userSettingsPath, join(PI_CONFIG_DIR, "settings.json")); - } - - // If runner has stdio MCP servers, write .pi/mcp.json to the workdir (project config) - // pi-mcp-adapter checks for .pi/mcp.json in cwd, which overrides global config - let hasMcp = false; - if (runner.stdio?.length) { - const mcpConfig: { - mcpServers: Record; - }>; - settings: { toolPrefix: string }; - } = { - mcpServers: {}, - settings: { - toolPrefix: "none" // No prefix - use raw tool names - } - // Proxy mode: LLM uses mcp({ search: "..." }) to discover tools on-demand - // This scales better with many MCP tools vs directTools which burns context - }; - - // Add each stdio server from runner config - runner.stdio.forEach((extCmd, i) => { - const parts = extCmd.split(" "); - const serverName = `harness${i > 0 ? i : ''}`; - mcpConfig.mcpServers[serverName] = { - command: parts[0], - args: parts.slice(1), - lifecycle: "eager", // Connect at startup for tests - env: { - MCP_HARNESS_LOG: join(workdir, "tool-calls.log") - } - }; - }); - - // Write .pi/mcp.json to workdir (project-local config that pi-mcp-adapter finds) - const piConfigDir = join(workdir, ".pi"); - mkdirSync(piConfigDir, { recursive: true }); - writeFileSync(join(piConfigDir, "mcp.json"), JSON.stringify(mcpConfig, null, 2)); - hasMcp = true; - } - - // Build base command with provider/model - // -p = non-interactive (print mode) - let cmd = `${runner.bin} -p --provider ${model.provider} --model "${model.model}"`; - - // Session handling for multi-turn - if (sessionName) { - const sessionPath = join(workdir, `.pi-session-${sessionName}.jsonl`); - if (resume) { - // Turn 2+: continue the existing session - cmd += ` --continue --session "${sessionPath}"`; - } else { - // Turn 1: create a new session file - cmd += ` --session "${sessionPath}"`; - } - } else { - // Single-turn: don't save session - cmd += ` --no-session`; - } - - cmd += ` "$(cat "${promptFile}")"`; - - // Build log message - const sessionInfo = sessionName - ? (resume ? ` --continue --session ` : ` --session `) - : ` --no-session`; - console.log(` Running: ${runner.bin} -p${sessionInfo} --provider ${model.provider} --model "${model.model}"${hasMcp ? ' (mcp)' : ''} ""`); - - const output = execSync(cmd, { - cwd: workdir, - env: { - ...process.env, - PI_CODING_AGENT_DIR: PI_CONFIG_DIR, // Use isolated config dir - MCP_HARNESS_LOG: join(workdir, "tool-calls.log"), - }, - timeout: AGENT_TIMEOUT_MS, - encoding: "utf-8", - shell: "/bin/bash", - }); - - return output; -} - -// ============================================================================= -// Unified Runner -// ============================================================================= - -interface AgentResult { - output: string; - sessionId?: string; // For multi-turn (goose, pi) -} - -async function runAgent( - model: ModelConfig, - runner: RunnerConfig, - prompt: string, - workdir: string, - sessionId?: string, // For multi-turn (goose, pi) - resume: boolean = false // For multi-turn: true on turn 2+ -): Promise { - if (runner.type === "opencode") { - const output = await runOpenCodeAgent(model, runner, prompt, workdir, resume); - return { output }; - } - if (runner.type === "pi") { - const output = await runPiAgent(model, runner, prompt, workdir, sessionId, resume); - return { output, sessionId }; - } - const output = await runGooseAgent(model, runner, prompt, workdir, sessionId, resume); - return { output, sessionId }; -} - -// ============================================================================= -// Scenario & Config Loading -// ============================================================================= - -function loadScenario(path: string): Scenario { - const content = readFileSync(path, "utf-8"); - return parse(content) as Scenario; -} - -function loadAllScenarios(dir: string): Scenario[] { - const files = readdirSync(dir).filter((f) => f.endsWith(".yaml")); - return files.map((f) => loadScenario(join(dir, f))); -} - -function loadConfig(configPath: string): SuiteConfig { - const content = readFileSync(configPath, "utf-8"); - const config = parse(content) as SuiteConfig; - const configDir = join(configPath, ".."); - - // Resolve relative paths in stdio for all runners - for (const runner of config.runners) { - if (runner.stdio) { - runner.stdio = runner.stdio.map((ext) => { - const parts = ext.split(" "); - const cmd = parts[0]; - const args = parts.slice(1).map((arg) => { - if (!arg.startsWith("/") && (arg.includes("/") || arg.startsWith("."))) { - return join(configDir, arg); - } - return arg; - }); - return [cmd, ...args].join(" "); - }); - } - } - - return config; -} - -function setupWorkdir(scenario: Scenario, workdir: string): void { - rmSync(workdir, { recursive: true, force: true }); - mkdirSync(workdir, { recursive: true }); - - if (scenario.setup) { - for (const [path, content] of Object.entries(scenario.setup)) { - const fullPath = join(workdir, path); - mkdirSync(join(fullPath, ".."), { recursive: true }); - writeFileSync(fullPath, content); - } - } -} - -// ============================================================================= -// Log Metrics Parsing -// ============================================================================= - -function parseLogMetrics(logContent: string, workdir?: string): { toolCalls: number; turns: number } { - // First, try to read tool-calls.log from MCP harness (most accurate) - let mcpToolCalls = 0; - if (workdir) { - try { - const toolCallsLog = readFileSync(join(workdir, "tool-calls.log"), "utf-8"); - // Each line is a JSON object representing one tool call - mcpToolCalls = toolCallsLog.trim().split("\n").filter(line => line.trim()).length; - } catch (e) { - // tool-calls.log doesn't exist, fall back to log parsing - } - } - - // Goose format: ─── tool_name | extension ─── - const gooseToolCalls = (logContent.match(/─── .+ \| .+ ───/g) || []).length; - - // OpenCode format: TURN N - const opencodeTurns = (logContent.match(/^TURN \d+$/gm) || []).length; - - // Total tool calls = MCP harness calls + Goose built-in tool calls - const toolCalls = mcpToolCalls + gooseToolCalls; - - // For OpenCode, use explicit TURN markers - const turns = opencodeTurns > 0 ? opencodeTurns : Math.ceil(toolCalls / 3); // Estimate ~3 tool calls per turn - - return { toolCalls, turns }; -} - -// ============================================================================= -// Test Execution -// ============================================================================= - -function buildTestPairs(config: SuiteConfig, scenarios: Scenario[]): TestPair[] { - const modelsByName = new Map(config.models.map((m) => [m.name, m])); - const runnersByName = new Map(config.runners.map((r) => [r.name, r])); - const scenariosByName = new Map(scenarios.map((s) => [s.name, s])); - - const pairs: TestPair[] = []; - - if (config.matrix?.length) { - for (const entry of config.matrix) { - // Validate scenario name - const scenario = scenariosByName.get(entry.scenario); - if (!scenario) { - throw new Error(`Unknown scenario "${entry.scenario}" in matrix. Available: ${[...scenariosByName.keys()].join(", ")}`); - } - - // Validate model names - if (entry.models) { - for (const name of entry.models) { - if (!modelsByName.has(name)) { - throw new Error(`Unknown model "${name}" in matrix entry for scenario "${entry.scenario}". Available: ${[...modelsByName.keys()].join(", ")}`); - } - } - } - - // Validate runner names - if (entry.runners) { - for (const name of entry.runners) { - if (!runnersByName.has(name)) { - throw new Error(`Unknown runner "${name}" in matrix entry for scenario "${entry.scenario}". Available: ${[...runnersByName.keys()].join(", ")}`); - } - } - } - - const models = entry.models - ? entry.models.map((n) => modelsByName.get(n)).filter(Boolean) as ModelConfig[] - : config.models; - - const runners = entry.runners - ? entry.runners.map((n) => runnersByName.get(n)).filter(Boolean) as RunnerConfig[] - : config.runners; - - for (const model of models) { - for (const runner of runners) { - pairs.push({ scenario, model, runner }); - } - } - } - return pairs; - } - - // No matrix: all scenarios × all models × all runners - for (const scenario of scenarios) { - for (const model of config.models) { - for (const runner of config.runners) { - pairs.push({ scenario, model, runner }); - } - } - } - return pairs; -} - -function scoreResult(result: TestResultWithLog): number { - if (result.run.status === "failed" && result.run.errors?.length) { - return -1; - } - const passedCount = result.validations.filter((v) => v.passed).length; - const statusBonus = result.run.status === "passed" ? 1000 : 0; - return statusBonus + passedCount; -} - -async function runScenario( - pair: TestPair, - baseWorkdir: string, - logsDir: string, - attempt: number = 1 -): Promise { - const { scenario, model, runner } = pair; - const testId = `${scenario.name}_${model.name}_${runner.name}`.replace(/[\/\\:]/g, "_"); - const workdir = join(baseWorkdir, testId); - const logFile = join(logsDir, `${testId}_attempt${attempt}.log`); - - console.log(`\n▶ ${scenario.name} [${model.provider}/${model.model}] (${runner.name})`); - - setupWorkdir(scenario, workdir); - mkdirSync(logsDir, { recursive: true }); - - // Create a minimal config for TestRun compatibility - const config = { - provider: model.provider, - model: model.model, - extensions: runner.extensions, - stdio: runner.stdio, - }; - - const run: TestRun = { - scenario, - config, - workdir, - startTime: new Date(), - status: "running", - }; - - // Determine if this is a multi-turn or single-turn scenario - const turns = scenario.turns ?? [ - { prompt: scenario.prompt!, validate: scenario.validate ?? [] } - ]; - const isMultiTurn = turns.length > 1; - - // For goose/pi: generate session ID upfront - // For opencode: capture session ID from first turn's output - let sessionId: string | undefined = isMultiTurn && (runner.type === "goose" || runner.type === "pi") - ? `test_${testId}_${Date.now()}` - : undefined; - - let output = ""; - const allValidations: Array<{ rule: any; passed: boolean; message?: string }> = []; - - try { - for (let turnIndex = 0; turnIndex < turns.length; turnIndex++) { - const turn = turns[turnIndex]; - const turnLabel = isMultiTurn ? ` [turn ${turnIndex + 1}/${turns.length}]` : ""; - console.log(` Running${turnLabel}...`); - - // Run the agent (with session for multi-turn) - const resume = turnIndex > 0; // Resume session on turn 2+ - const result = await runAgent(model, runner, turn.prompt, workdir, sessionId, resume); - - // Capture session ID from first turn (for opencode) - if (turnIndex === 0 && result.sessionId) { - sessionId = result.sessionId; - } - - output += `\n${'='.repeat(60)}\nTURN ${turnIndex + 1}\n${'='.repeat(60)}\n${result.output}`; - - // Validate this turn - const turnValidations = validateAll(turn.validate, workdir); - for (const v of turnValidations) { - allValidations.push({ - rule: v.rule, - passed: v.result.passed, - message: v.result.message, - }); - } - - // If any validation failed, stop early - const turnPassed = turnValidations.every((v) => v.result.passed); - if (!turnPassed) { - console.log(` Turn ${turnIndex + 1} failed validation`); - break; - } - } - - run.endTime = new Date(); - const allPassed = allValidations.every((v) => v.passed); - - writeFileSync(logFile, output); - - const metrics = parseLogMetrics(output, workdir); - return { - run: { ...run, status: allPassed ? "passed" : "failed" }, - validations: allValidations, - logFile, - runnerName: runner.name, - toolCalls: metrics.toolCalls, - turns: metrics.turns, - }; - } catch (err) { - const errorOutput = output + "\n\nERROR:\n" + String(err); - writeFileSync(logFile, errorOutput); - - return { - run: { - ...run, - status: "failed", - endTime: new Date(), - errors: [String(err)], - }, - validations: allValidations, - logFile, - runnerName: runner.name, - toolCalls: parseLogMetrics(errorOutput, workdir).toolCalls, - turns: parseLogMetrics(errorOutput, workdir).turns, - }; - } -} - -// ============================================================================= -// Reporting -// ============================================================================= - -function pairKey(pair: TestPair): string { - return `${pair.model.name}::${pair.runner.name}`; -} - -function resultKey(result: TestResultWithLog): string { - return `${result.run.config.provider}/${result.run.config.model}::${result.runnerName}`; -} - -interface ReportOptions { - isRunning?: boolean; - allPairs?: TestPair[]; -} - -function generateHtmlReport( - results: TestResultWithLog[], - outputPath: string, - options: ReportOptions = {} -): void { - const { isRunning = false, allPairs = [] } = options; - - // Read and embed gym.png as base64. Prefer one sitting next to the report - // (legacy in-repo layout); otherwise fall back to the copy in the source tree - // so the image still embeds when output is redirected via GYM_OUTPUT_DIR. - let gymBase64 = ""; - try { - const adjacent = join(outputPath, "..", "gym.png"); - const gymPath = existsSync(adjacent) - ? adjacent - : join(import.meta.dirname, "gym.png"); - gymBase64 = readFileSync(gymPath).toString("base64"); - } catch (e) { - // gym.png not found, will use external reference - } - - // Collect all logs for embedding - const logsData: Record = {}; - for (const r of results) { - if (r.logFile) { - try { - logsData[basename(r.logFile)] = readFileSync(r.logFile, "utf-8"); - } catch (e) { /* ignore missing logs */ } - } - } - - // Calculate max duration for scaling bars - const maxDuration = Math.max(...results.map(r => { - if (!r.run.endTime || !r.run.startTime) return 0; - return (r.run.endTime.getTime() - r.run.startTime.getTime()) / 1000; - }), 1); - - const maxToolCalls = Math.max(...results.map(r => r.toolCalls || 0), 1); - - // Get all scenarios (columns) - const scenarios = allPairs.length - ? [...new Set(allPairs.map((p) => p.scenario.name))] - : [...new Set(results.map((r) => r.run.scenario.name))]; - - // Rows are model × runner combinations - const rowKeys = allPairs.length - ? [...new Set(allPairs.map(pairKey))] - : [...new Set(results.map((r) => `${r.run.config.provider}/${r.run.config.model}::${r.runnerName}`))]; - - // Map row key -> pair info - const rowsByKey = new Map(); - for (const pair of allPairs) { - rowsByKey.set(pairKey(pair), { model: pair.model, runner: pair.runner }); - } - - // Group rows by model for rowspan display - const modelKey = (m: ModelConfig) => `${m.provider}/${m.model}`; - const modelGroups = new Map(); // modelKey -> rowKeys[] - for (const key of rowKeys) { - const row = rowsByKey.get(key); - if (!row) continue; - const mk = modelKey(row.model); - if (!modelGroups.has(mk)) modelGroups.set(mk, []); - modelGroups.get(mk)!.push(key); - } - - // Build set of valid (scenario, rowKey) combinations from the matrix - const validCells = new Set(); - for (const pair of allPairs) { - validCells.add(`${pair.scenario.name}::${pairKey(pair)}`); - } - - const getResult = (scenario: string, rowKey: string) => { - const [modelPart, runnerName] = rowKey.split("::"); - return results.find( - (r) => - r.run.scenario.name === scenario && - `${r.run.config.provider}/${r.run.config.model}` === `${rowsByKey.get(rowKey)?.model.provider}/${rowsByKey.get(rowKey)?.model.model}` && - r.runnerName === runnerName - ); - }; - - const passed = results.filter((r) => r.run.status === "passed").length; - const failed = results.filter((r) => r.run.status === "failed").length; - const total = allPairs.length || results.length; - const pending = total - results.length; - - const runnerNames = [...new Set(allPairs.map((p) => p.runner.name))]; - - const html = ` - - - - - - ${isRunning ? "Running..." : "Results"} - Agent Gym Workout - - - -
Agent Gym

Agent Gym Workout${isRunning ? " (Running...)" : ""}

${!isRunning ? '' : ''}
-

- ${passed} passed / - ${failed} failed${pending > 0 ? ` / ${pending} pending` : ""} / - ${total} total -

-

Agent Configurations: ${runnerNames.map(n => `${n}`).join(", ")}

- - - - - - - ${scenarios.map((s) => ``).join("")} - - - - ${[...modelGroups.entries()].map(([mk, keys]) => { - return keys.map((key, idx) => { - const row = rowsByKey.get(key); - if (!row) return ""; - const { model, runner } = row; - const isFirst = idx === 0; - const rowspan = keys.length; - return ` - - ${isFirst ? `` : ''} - - ${scenarios.map((scenario) => { - const r = getResult(scenario, key); - if (!r) { - // Check if this combination is in the matrix - const cellKey = `${scenario}::${key}`; - const isInMatrix = validCells.has(cellKey); - if (!isInMatrix) return ``; - return ``; - } - if (r.run.status === "running") { - return ``; - } - const duration = r.run.endTime - ? ((r.run.endTime.getTime() - r.run.startTime.getTime()) / 1000).toFixed(1) - : "-"; - const logPath = r.logFile ? `logs/${basename(r.logFile)}` : ""; - const validationHtml = r.validations.map((v) => { - const icon = v.passed ? "✓" : "✗"; - const cls = v.passed ? "pass" : "fail"; - const ruleLabel = (v.rule as any).name - ? (v.rule as any).name - : v.rule.type === "tool_called" - ? `tool_called: ${(v.rule as any).tool}` - : v.rule.type + (("path" in v.rule) ? `: ${(v.rule as any).path}` : ""); - return `
${icon} ${ruleLabel}
`; - }).join(""); - return ``; - }).join("")} - `; - }).join(""); - }).join("")} - -
ModelAgent Configuration${s}
- ${model.provider}/${model.model} -
- ${runner.name} - (${runner.type}) -
—
⋯
...
 -
- ${r.run.status === "passed" ? "✓" : "✗"} - ${r.cached ? 'cached' : ''} - ${duration}s - ${logPath ? `log` : ""} -
-
-
-
- 🔧 ${r.toolCalls || 0} - ↻ ${r.turns || 0} -
-
${validationHtml}
-
- -

Generated: ${new Date().toISOString()}

- - - - - - -`; - - mkdirSync(dirname(outputPath), { recursive: true }); - writeFileSync(outputPath, html); - console.log(`\n📊 Report saved to: ${outputPath}`); -} - -function printResults(results: TestResultWithLog[]): void { - console.log("\n" + "=".repeat(60)); - console.log("RESULTS"); - console.log("=".repeat(60)); - - for (const result of results) { - const icon = result.run.status === "passed" ? "✓" : "✗"; - const { scenario, config } = result.run; - console.log( - `${icon} ${scenario.name} [${config.provider}/${config.model}] (${result.runnerName}) - ${result.run.status.toUpperCase()}` - ); - - for (const v of result.validations) { - if (!v.passed) { - console.log(` ✗ ${v.message}`); - } - } - } - - const passed = results.filter((r) => r.run.status === "passed").length; - console.log(`\n${passed}/${results.length} tests passed`); -} - -// ============================================================================= -// Main -// ============================================================================= - -async function main() { - // CLI --clear-cache: clear cache and exit - if (process.argv.includes("--clear-cache")) { - clearCache(); - return; - } - - const configPath = join(GYM_DIR, "config.yaml"); - const scenariosDir = join(import.meta.dirname, "../scenarios"); - const workdir = artifactPath(".workdir", SUITE_DIR); - const logsDir = artifactPath("logs", GYM_DIR); - const reportPath = artifactPath("report.html", GYM_DIR); - - const config = loadConfig(configPath); - let scenarios = loadAllScenarios(scenariosDir); - - // CLI --scenario= filter - const scenarioFilter = process.argv.find((a) => a.startsWith("--scenario="))?.split("=")[1]; - if (scenarioFilter) { - const filters = scenarioFilter.split(","); - scenarios = scenarios.filter((s) => filters.some((f) => s.name.includes(f))); - } - - // CLI --model= filter - const modelFilter = process.argv.find((a) => a.startsWith("--model="))?.split("=")[1]; - if (modelFilter) { - const filters = modelFilter.split(","); - config.models = config.models.filter((m) => filters.some((f) => m.name.includes(f))); - } - - // CLI --runner= filter - const runnerFilter = process.argv.find((a) => a.startsWith("--runner="))?.split("=")[1]; - if (runnerFilter) { - const filters = runnerFilter.split(","); - config.runners = config.runners.filter((r) => filters.some((f) => r.name.includes(f))); - } - - const pairs = buildTestPairs(config, scenarios); - - // Sort pairs by model name so same models run together (keeps model loaded in memory) - pairs.sort((a, b) => a.model.name.localeCompare(b.model.name)); - - // Show model grouping - const modelOrder = [...new Set(pairs.map(p => p.model.name))]; - console.log(`\nExecution order (grouped by model for efficiency):`); - for (const m of modelOrder) { - const count = pairs.filter(p => p.model.name === m).length; - console.log(` ${m}: ${count} tests`); - } - - // CLI --run-count=N (default 1) - const runCountArg = process.argv.find((a) => a.startsWith("--run-count="))?.split("=")[1]; - const RUN_COUNT = runCountArg ? parseInt(runCountArg, 10) : 1; - - // CLI --no-cache: skip cache lookup (still stores results) - const noCache = process.argv.includes("--no-cache"); - - // Load cache and precompute hashes - const cache = loadCache(); - const binaryHashes = new Map(); - for (const runner of config.runners) { - if (!binaryHashes.has(runner.bin)) { - console.log(`Computing hash for ${runner.bin}...`); - binaryHashes.set(runner.bin, getBinaryHash(runner.bin)); - } - } - const mcpHarnessHash = getMcpHarnessHash(); - - console.log(`Output: ${OUTPUT_DIR ?? GYM_DIR}${OUTPUT_DIR ? "" : " (in-repo; set GYM_OUTPUT_DIR to redirect)"}`); - console.log(`Models: ${config.models.map((m) => m.name).join(", ")}`); - console.log(`Runners: ${config.runners.map((r) => r.name).join(", ")}`); - console.log(`Running ${pairs.length} test pairs (${RUN_COUNT}x each, worst result kept)`); - console.log(`Cache: ${noCache ? "disabled" : "enabled"} (${Object.keys(cache.entries).length} entries)`); - - const results: TestResultWithLog[] = []; - - // CLI --no-open to skip opening browser - const noOpen = process.argv.includes("--no-open"); - - let cacheHits = 0; - let cacheMisses = 0; - let browserOpened = false; - - for (const pair of pairs) { - // Check cache first - const { key: cacheKey, inputs: cacheInputs } = computeCacheKey(pair, binaryHashes, mcpHarnessHash); - - if (!noCache) { - const cachedResult = getCachedResult(cache, cacheKey, pair, logsDir); - if (cachedResult) { - console.log(`\n${cachedResult.run.status === "passed" ? "✓" : "✗"} ${pair.scenario.name} [${pair.model.name}] (${pair.runner.name}) [CACHED]`); - results.push(cachedResult); - cacheHits++; - continue; - } - } - - // First cache miss - generate report with cached results so far and open browser - if (!browserOpened) { - generateHtmlReport(results, reportPath, { isRunning: true, allPairs: pairs }); - if (!noOpen) { - execFileSync("open", [reportPath]); - } - browserOpened = true; - } - - cacheMisses++; - let worstResult: TestResultWithLog | null = null; - - for (let attempt = 1; attempt <= RUN_COUNT; attempt++) { - console.log(` Attempt ${attempt}/${RUN_COUNT} [${pair.runner.name}]`); - const result = await runScenario(pair, workdir, logsDir, attempt); - - if (!worstResult) { - worstResult = result; - } else { - const prevScore = scoreResult(worstResult); - const currScore = scoreResult(result); - if (currScore < prevScore) { - worstResult = result; - } - } - - if (result.run.status === "failed") { - break; - } - } - - // Store in cache - storeCacheResult(cache, cacheKey, cacheInputs, worstResult!); - - results.push(worstResult!); - generateHtmlReport(results, reportPath, { isRunning: true, allPairs: pairs }); - } - - generateHtmlReport(results, reportPath, { isRunning: false, allPairs: pairs }); - - // If everything was cached, open browser now with final report - if (!browserOpened && !noOpen) { - execFileSync("open", [reportPath]); - } - - printResults(results); - - console.log(`\nCache summary: ${cacheHits} hits, ${cacheMisses} misses`); -} - -main().catch(console.error); diff --git a/evals/open-model-gym/suite/src/types.ts b/evals/open-model-gym/suite/src/types.ts deleted file mode 100644 index f8b44bc29..000000000 --- a/evals/open-model-gym/suite/src/types.ts +++ /dev/null @@ -1,74 +0,0 @@ -export interface AgentConfig { - model: string; - provider: string; - /** Extensions (runner knows which are platform vs builtin) */ - extensions?: string[]; - /** Stdio extension commands (for custom MCP servers) */ - stdio?: string[]; - /** Path to goose binary (default: "goose") */ - "goose-bin"?: string; - temperature?: number; - maxTokens?: number; -} - -export interface Scenario { - name: string; - description: string; - prompt?: string; - /** Files to create before running (relative paths) */ - setup?: Record; - /** Validation rules to check after agent completes (single-turn) */ - validate?: ValidationRule[]; - /** Multi-turn conversation (alternative to single prompt+validate) */ - turns?: Turn[]; - /** Tags for filtering scenarios */ - tags?: string[]; -} - -/** A single turn in a multi-turn conversation */ -export interface Turn { - /** The prompt for this turn */ - prompt: string; - /** Validation rules to check after this turn completes */ - validate: ValidationRule[]; -} - -export type ValidationRule = - | { type: "file_exists"; path: string; name?: string } - | { type: "file_contains"; path: string; pattern: string; name?: string } - | { type: "file_matches"; path: string; regex: string; name?: string } - | { type: "file_not_matches"; path: string; regex: string; name?: string } - | { type: "file_not_empty"; path: string; name?: string } - | { type: "command_succeeds"; command: string; name?: string } - | { type: "tool_called"; tool: string; args?: Record; name?: string } - | { type: "custom"; fn: string; name?: string }; - -export interface TestRun { - scenario: Scenario; - config: AgentConfig; - workdir: string; - startTime: Date; - endTime?: Date; - status: "pending" | "running" | "passed" | "failed"; - errors?: string[]; -} - -export interface TestResult { - run: TestRun; - validations: Array<{ - rule: ValidationRule; - passed: boolean; - message?: string; - }>; -} - -export interface SuiteConfig { - /** Agent configurations to permute */ - agents: AgentConfig[]; - /** Scenarios to run */ - scenarios: string[]; - /** Base directory for test workspaces */ - workdir: string; - /** Parallel execution count */ - parallel?: number; -} diff --git a/evals/open-model-gym/suite/src/validator.ts b/evals/open-model-gym/suite/src/validator.ts deleted file mode 100644 index 399a5ca2d..000000000 --- a/evals/open-model-gym/suite/src/validator.ts +++ /dev/null @@ -1,184 +0,0 @@ -import { existsSync, readFileSync, statSync } from "node:fs"; -import { execSync } from "node:child_process"; -import { join } from "node:path"; -import type { ValidationRule } from "./types.js"; - -export interface ValidationResult { - passed: boolean; - message?: string; -} - -export function validateRule( - rule: ValidationRule, - workdir: string -): ValidationResult { - switch (rule.type) { - case "file_exists": { - const fullPath = join(workdir, rule.path); - const exists = existsSync(fullPath); - return { - passed: exists, - message: exists ? undefined : `File not found: ${rule.path}`, - }; - } - - case "file_not_empty": { - const fullPath = join(workdir, rule.path); - if (!existsSync(fullPath)) { - return { passed: false, message: `File not found: ${rule.path}` }; - } - const stat = statSync(fullPath); - return { - passed: stat.size > 0, - message: stat.size > 0 ? undefined : `File is empty: ${rule.path}`, - }; - } - - case "file_contains": { - const fullPath = join(workdir, rule.path); - if (!existsSync(fullPath)) { - return { passed: false, message: `File not found: ${rule.path}` }; - } - const content = readFileSync(fullPath, "utf-8"); - const contains = content.includes(rule.pattern); - return { - passed: contains, - message: contains - ? undefined - : `File ${rule.path} does not contain: ${rule.pattern}`, - }; - } - - case "file_matches": { - const fullPath = join(workdir, rule.path); - if (!existsSync(fullPath)) { - return { passed: false, message: `File not found: ${rule.path}` }; - } - const content = readFileSync(fullPath, "utf-8"); - const regex = new RegExp(rule.regex); - const matches = regex.test(content); - return { - passed: matches, - message: matches - ? undefined - : `File ${rule.path} does not match regex: ${rule.regex}`, - }; - } - - case "file_not_matches": { - const fullPath = join(workdir, rule.path); - if (!existsSync(fullPath)) { - return { passed: false, message: `File not found: ${rule.path}` }; - } - const content = readFileSync(fullPath, "utf-8"); - const regex = new RegExp(rule.regex); - const matches = regex.test(content); - return { - passed: !matches, - message: !matches - ? undefined - : `File ${rule.path} should not match regex: ${rule.regex}`, - }; - } - - case "command_succeeds": { - try { - execSync(rule.command, { cwd: workdir, stdio: "pipe" }); - return { passed: true }; - } catch (err) { - return { - passed: false, - message: `Command failed: ${rule.command}`, - }; - } - } - - case "tool_called": { - const logPath = join(workdir, "tool-calls.log"); - if (!existsSync(logPath)) { - return { passed: false, message: "tool-calls.log not found" }; - } - - const content = readFileSync(logPath, "utf-8"); - const lines = content.trim().split("\n").filter(Boolean); - - // Find all calls to the specified tool - const matchingCalls = lines - .map((line) => { - try { - return JSON.parse(line); - } catch { - return null; - } - }) - .filter((entry) => entry?.tool === rule.tool); - - if (matchingCalls.length === 0) { - return { passed: false, message: `Tool not called: ${rule.tool}` }; - } - - // If no arg requirements, just check tool was called - if (!rule.args) { - return { passed: true }; - } - - // Check if any call matches the arg requirements - for (const call of matchingCalls) { - const args = call.arguments || {}; - let allMatch = true; - - for (const [key, expected] of Object.entries(rule.args)) { - const actual = args[key]; - if (actual === undefined) { - allMatch = false; - break; - } - - // If expected starts/ends with /, treat as regex pattern - if (typeof expected === "string" && expected.startsWith("/") && expected.endsWith("/")) { - const pattern = new RegExp(expected.slice(1, -1), "i"); - if (!pattern.test(String(actual))) { - allMatch = false; - break; - } - } else { - // Exact match (case-insensitive for strings) - const actualStr = String(actual).toLowerCase(); - const expectedStr = String(expected).toLowerCase(); - if (!actualStr.includes(expectedStr)) { - allMatch = false; - break; - } - } - } - - if (allMatch) { - return { passed: true }; - } - } - - return { - passed: false, - message: `Tool ${rule.tool} called but args didn't match: expected ${JSON.stringify(rule.args)}`, - }; - } - - case "custom": { - // Custom validators loaded dynamically - return { passed: false, message: "Custom validators not yet implemented" }; - } - - default: - return { passed: false, message: `Unknown rule type` }; - } -} - -export function validateAll( - rules: ValidationRule[], - workdir: string -): Array<{ rule: ValidationRule; result: ValidationResult }> { - return rules.map((rule) => ({ - rule, - result: validateRule(rule, workdir), - })); -} diff --git a/evals/open-model-gym/suite/tsconfig.json b/evals/open-model-gym/suite/tsconfig.json deleted file mode 100644 index 406f69d8f..000000000 --- a/evals/open-model-gym/suite/tsconfig.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2022", - "module": "NodeNext", - "moduleResolution": "NodeNext", - "strict": true, - "esModuleInterop": true, - "skipLibCheck": true, - "outDir": "dist" - }, - "include": ["src"] -}