tidy: clean up old benchmark and add gym (#7081)
This commit is contained in:
Michael Neale
@@ -0,0 +1,72 @@
|
||||
# Dependencies
|
||||
node_modules/
|
||||
.pnpm-store/
|
||||
.workdir/
|
||||
report.html
|
||||
|
||||
# Build outputs
|
||||
dist/
|
||||
build/
|
||||
out/
|
||||
.next/
|
||||
.nuxt/
|
||||
.output/
|
||||
.opencode-root
|
||||
suite/.pi-root
|
||||
|
||||
# TypeScript
|
||||
*.tsbuildinfo
|
||||
*.d.ts.map
|
||||
.g3/
|
||||
# Logs
|
||||
logs/
|
||||
*.log
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
pnpm-debug.log*
|
||||
|
||||
# Runtime data
|
||||
pids/
|
||||
*.pid
|
||||
*.seed
|
||||
*.pid.lock
|
||||
|
||||
# Coverage & testing
|
||||
coverage/
|
||||
.nyc_output/
|
||||
.jest/
|
||||
|
||||
# Caches
|
||||
.cache/
|
||||
.parcel-cache/
|
||||
.turbo/
|
||||
.eslintcache
|
||||
.stylelintcache
|
||||
*.swp
|
||||
*.swo
|
||||
|
||||
# IDE & editors
|
||||
.idea/
|
||||
.vscode/
|
||||
*.sublime-project
|
||||
*.sublime-workspace
|
||||
|
||||
# OS files
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Environment variables
|
||||
.env
|
||||
.env.local
|
||||
.env.*.local
|
||||
|
||||
# Lock files (optional - uncomment if you don't want to track)
|
||||
# package-lock.json
|
||||
# yarn.lock
|
||||
# pnpm-lock.yaml
|
||||
|
||||
# Temporary files
|
||||
tmp/
|
||||
temp/
|
||||
*.tmp
|
||||
@@ -0,0 +1,60 @@
|
||||
# Agent Runner - Test Suite (supports goose and opencode)
|
||||
|
||||
# Default recipe
|
||||
default: run
|
||||
|
||||
# Full test run - all scenarios, all agents, 3 repetitions (worst kept)
|
||||
run: _install
|
||||
cd suite && npm run test
|
||||
|
||||
# Quick test - file-editing + everyday-app-automation, single run each (no repetition)
|
||||
test: _install
|
||||
cd suite && npx tsx src/runner.ts --scenario=file-editing,everyday-app-automation --run-count=1
|
||||
|
||||
# Run a specific scenario (all agents, 3 reps)
|
||||
scenario name: _install
|
||||
cd suite && npx tsx src/runner.ts --scenario={{name}}
|
||||
|
||||
# Run against a specific agent (all scenarios, 3 reps)
|
||||
agent name: _install
|
||||
cd suite && npx tsx src/runner.ts --agent={{name}}
|
||||
|
||||
# Open report in browser
|
||||
report:
|
||||
open report.html
|
||||
|
||||
# Install all dependencies
|
||||
install:
|
||||
cd suite && npm install
|
||||
cd mcp-harness && npm install && npm run build
|
||||
@# Install pi-mcp-adapter for Pi runner MCP support
|
||||
@pi list 2>/dev/null | grep -q "pi-mcp-adapter" || pi install npm:pi-mcp-adapter
|
||||
|
||||
# Build TypeScript
|
||||
build: _install
|
||||
cd suite && npm run build
|
||||
|
||||
# Clear the test cache
|
||||
clear-cache:
|
||||
cd suite && npx tsx src/runner.ts --clear-cache
|
||||
|
||||
# Run tests ignoring cache (force fresh runs)
|
||||
run-fresh: _install
|
||||
cd suite && npx tsx src/runner.ts --no-cache
|
||||
|
||||
# Show cache stats
|
||||
cache-stats:
|
||||
@if [ -f suite/.cache/index.json ]; then \
|
||||
echo "Cache entries: $$(cat suite/.cache/index.json | grep -o '"[a-f0-9]\{16\}":' | wc -l | tr -d ' ')"; \
|
||||
echo "Cache size: $$(du -sh suite/.cache 2>/dev/null | cut -f1 || echo '0')"; \
|
||||
else \
|
||||
echo "No cache found"; \
|
||||
fi
|
||||
|
||||
# Internal: install if node_modules missing, always rebuild mcp-harness
|
||||
_install:
|
||||
@[ -d suite/node_modules ] || (cd suite && npm install)
|
||||
@[ -d mcp-harness/node_modules ] || (cd mcp-harness && npm install)
|
||||
@cd mcp-harness && npm run build
|
||||
@# Ensure pi-mcp-adapter is installed for Pi runner
|
||||
@pi list 2>/dev/null | grep -q "pi-mcp-adapter" || pi install npm:pi-mcp-adapter
|
||||
@@ -0,0 +1,291 @@
|
||||
# Open Model Gym
|
||||
|
||||
Run agent tests across a matrix of **models × runners × scenarios**.
|
||||
|
||||
It isn't hard for any agent to do ok with opus, but lets scale things in the other direction. What do we have to break things down to.
|
||||
|
||||
<img width="1768" height="1133" alt="image" src="https://github.com/user-attachments/assets/29915659-ee6b-4a8b-ba5e-58420b168b43" />
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
just install # one-time setup
|
||||
just run # run full matrix (3 reps each)
|
||||
just report # view results
|
||||
```
|
||||
|
||||
## How It Works
|
||||
|
||||
The test harness runs every combination of models, runners, and scenarios defined in your matrix. Each test runs multiple times (default 3) and keeps the **worst result** — if a test fails even once, it's marked failed. This catches flaky passes.
|
||||
|
||||
## Configuration
|
||||
|
||||
Edit `config.yaml` to define your test matrix:
|
||||
|
||||
### Models
|
||||
|
||||
LLMs to test against. Supports any provider (Anthropic, OpenAI, Ollama, etc.):
|
||||
|
||||
```yaml
|
||||
models:
|
||||
- name: opus
|
||||
provider: anthropic
|
||||
model: claude-opus-4-5-20251101
|
||||
|
||||
- name: qwen3-coder
|
||||
provider: ollama
|
||||
model: qwen3-coder:64k
|
||||
|
||||
- name: gpt4
|
||||
provider: openai
|
||||
model: gpt-4-turbo
|
||||
```
|
||||
|
||||
### Runners
|
||||
|
||||
Agent frameworks that execute the tests. Each runner has its own binary, type, and configuration:
|
||||
|
||||
```yaml
|
||||
runners:
|
||||
# Goose agent with extensions
|
||||
- name: goose-full
|
||||
type: goose
|
||||
bin: goose # path to binary (can be absolute)
|
||||
extensions: [developer, todo, skills]
|
||||
stdio:
|
||||
- node mcp-harness/dist/index.js
|
||||
|
||||
# OpenCode agent
|
||||
- name: opencode
|
||||
type: opencode
|
||||
bin: opencode # path to binary
|
||||
stdio:
|
||||
- node mcp-harness/dist/index.js
|
||||
|
||||
# Custom goose binary path
|
||||
- name: goose-dev
|
||||
type: goose
|
||||
bin: /path/to/my/goose-dev
|
||||
extensions: [developer]
|
||||
```
|
||||
|
||||
**Supported runner types:**
|
||||
- `goose` — [Goose](https://github.com/block/goose) agent framework
|
||||
- `opencode` — [OpenCode](https://opencode.ai) agent framework
|
||||
- `pi` — [Pi](https://github.com/badlogic/pi-mono) coding agent
|
||||
|
||||
## Runner Details
|
||||
|
||||
Each runner has different setup requirements, MCP integration methods, and session handling.
|
||||
|
||||
### Goose
|
||||
|
||||
[Goose](https://github.com/block/goose) is Block's open-source coding agent with built-in MCP support.
|
||||
|
||||
**Setup:** Install via `brew install goose` or from source.
|
||||
|
||||
**MCP Integration:** Native support. The harness writes a `config.yaml` to an isolated `.goose-root/` directory with extensions and MCP servers:
|
||||
|
||||
```yaml
|
||||
extensions:
|
||||
developer:
|
||||
enabled: true
|
||||
mcp_harness:
|
||||
type: stdio
|
||||
enabled: true
|
||||
cmd: node
|
||||
args: [mcp-harness/dist/index.js]
|
||||
```
|
||||
|
||||
**Session Handling:** Uses `--name <session>` for named sessions, `--resume` to continue:
|
||||
- Turn 1: `goose run -i <prompt> --name <session>`
|
||||
- Turn 2+: `goose run -i <prompt> --name <session> --resume`
|
||||
- Single-turn: `goose run -i <prompt> --no-session`
|
||||
|
||||
### OpenCode
|
||||
|
||||
[OpenCode](https://opencode.ai) is a terminal-based coding agent.
|
||||
|
||||
**Setup:** Install via their website or package manager.
|
||||
|
||||
**MCP Integration:** Native support. The harness writes an `opencode.json` config to the workdir:
|
||||
|
||||
```json
|
||||
{
|
||||
"mcp": {
|
||||
"harness": {
|
||||
"type": "local",
|
||||
"command": ["node", "mcp-harness/dist/index.js"],
|
||||
"enabled": true
|
||||
}
|
||||
},
|
||||
"model": "anthropic/claude-opus-4-5-20251101"
|
||||
}
|
||||
```
|
||||
|
||||
**Session Handling:** Uses `--continue` to resume the last session in the working directory:
|
||||
- Turn 1: `opencode run "<prompt>"`
|
||||
- Turn 2+: `opencode run --continue "<prompt>"`
|
||||
|
||||
⚠️ OpenCode doesn't support named sessions, so multi-turn scenarios exclude it.
|
||||
|
||||
### Pi
|
||||
|
||||
[Pi](https://github.com/badlogic/pi-mono) is a lightweight coding agent that requires an adapter for MCP support.
|
||||
|
||||
**Setup:**
|
||||
```bash
|
||||
# Install Pi
|
||||
npm install -g @anthropic/pi # or from source
|
||||
|
||||
# Install the MCP adapter (required for MCP tools)
|
||||
pi install npm:pi-mcp-adapter
|
||||
```
|
||||
|
||||
The `just install` recipe auto-installs pi-mcp-adapter if missing.
|
||||
|
||||
**MCP Integration:** Via [pi-mcp-adapter](https://github.com/nicobailon/pi-mcp-adapter). The harness dynamically writes a `.pi-mcp.json` config to the workdir:
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"harness": {
|
||||
"command": "node",
|
||||
"args": ["mcp-harness/dist/index.js"],
|
||||
"lifecycle": "eager",
|
||||
"env": { "MCP_HARNESS_LOG": "<workdir>/tool-calls.log" }
|
||||
}
|
||||
},
|
||||
"settings": { "directTools": true }
|
||||
}
|
||||
```
|
||||
|
||||
Key settings:
|
||||
- `directTools: true` — Registers MCP tools directly in Pi's tool list (no wrapper)
|
||||
- `lifecycle: "eager"` — Connects to MCP servers at startup
|
||||
|
||||
**Model Configuration:** Pi requires custom models (like Ollama) to be defined in `models.json`. The harness automatically generates this config in an isolated `.pi-root/` directory and sets `PI_CODING_AGENT_DIR` to use it:
|
||||
|
||||
```json
|
||||
{
|
||||
"providers": {
|
||||
"ollama": {
|
||||
"baseUrl": "http://localhost:11434/v1",
|
||||
"api": "openai-completions",
|
||||
"apiKey": "ollama",
|
||||
"models": [{ "id": "model-name", "name": "Model Name", ... }]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The harness copies `auth.json` from your real Pi config (`~/.pi/agent/`) so API keys work.
|
||||
|
||||
**Session Handling:** Uses `--session <path>` for file-based sessions, `--continue` to resume:
|
||||
- Turn 1: `pi -p --session <path> "<prompt>"`
|
||||
- Turn 2+: `pi -p --continue --session <path> "<prompt>"`
|
||||
- Single-turn: `pi -p --no-session "<prompt>"`
|
||||
|
||||
The `-p` flag runs Pi in non-interactive "print" mode for automation
|
||||
|
||||
### Matrix
|
||||
|
||||
Define which scenarios run against which models/runners:
|
||||
|
||||
```yaml
|
||||
matrix:
|
||||
- scenario: file-editing
|
||||
models: [opus, qwen3-coder] # omit to run all models
|
||||
runners: [goose-full, opencode] # omit to run all runners
|
||||
|
||||
- scenario: everyday-app-automation
|
||||
# runs against ALL models and ALL runners
|
||||
```
|
||||
|
||||
## Scenarios
|
||||
|
||||
Scenarios live in `suite/scenarios/` as YAML files:
|
||||
|
||||
```yaml
|
||||
name: file-editing
|
||||
description: Create and edit files
|
||||
prompt: |
|
||||
1. Create joke.md containing a short joke
|
||||
2. Edit hello.rs to add a debug function
|
||||
|
||||
setup:
|
||||
hello.rs: |
|
||||
fn main() { println!("Hello!"); }
|
||||
|
||||
validate:
|
||||
- type: file_exists
|
||||
path: joke.md
|
||||
- type: file_matches
|
||||
path: hello.rs
|
||||
regex: "fn\\s+debug"
|
||||
```
|
||||
|
||||
### Validation Rules
|
||||
|
||||
| Rule | Description |
|
||||
|------|-------------|
|
||||
| `file_exists` | File exists at path |
|
||||
| `file_not_empty` | File exists and has content |
|
||||
| `file_contains` | File contains literal string |
|
||||
| `file_matches` | File matches regex pattern |
|
||||
| `command_succeeds` | Shell command exits 0 |
|
||||
| `tool_called` | MCP tool was called with matching args (regex supported) |
|
||||
|
||||
**Tool call validation example:**
|
||||
```yaml
|
||||
validate:
|
||||
- type: tool_called
|
||||
tool: slack_search_messages
|
||||
args:
|
||||
query: /quarterly.?review/ # regex pattern
|
||||
- type: tool_called
|
||||
tool: jira_create_issue
|
||||
args:
|
||||
summary: /Q1.*Review/
|
||||
description: /David Brown/
|
||||
```
|
||||
|
||||
## MCP Harness
|
||||
|
||||
Mock MCP server providing simulated tools for testing agent tool-use without hitting real APIs.
|
||||
|
||||
```bash
|
||||
cd mcp-harness && npm install && npm run build
|
||||
```
|
||||
|
||||
**Available tools:** gdrive, sheets, salesforce, slack, calendar, gmail, jira, github
|
||||
|
||||
Each tool returns realistic mock data. Tool calls are logged to `tool-calls.log` in the workdir for validation.
|
||||
|
||||
## Commands
|
||||
|
||||
| Command | Description |
|
||||
|---------|-------------|
|
||||
| `just run` | Full test run (3 reps each, worst kept) |
|
||||
| `just test` | Quick run (1 rep each) |
|
||||
| `just scenario <name>` | Run specific scenario |
|
||||
| `just agent <name>` | Run specific agent |
|
||||
| `just report` | Open HTML results |
|
||||
|
||||
### CLI Flags
|
||||
|
||||
```bash
|
||||
# Filter by scenario, model, or runner
|
||||
npx tsx src/runner.ts --scenario=file-editing --model=opus --runner=goose
|
||||
|
||||
# Control repetition count
|
||||
npx tsx src/runner.ts --run-count=5
|
||||
|
||||
# Don't auto-open browser
|
||||
npx tsx src/runner.ts --no-open
|
||||
```
|
||||
|
||||
## Output
|
||||
|
||||
- `report.html` — Live-updating HTML matrix showing pass/fail status, duration, and validation details
|
||||
- `logs/` — Full agent output logs for each run
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,85 @@
|
||||
# =============================================================================
|
||||
# Models - the LLMs to test
|
||||
# =============================================================================
|
||||
models:
|
||||
- name: opus
|
||||
provider: anthropic
|
||||
model: claude-opus-4-5-20251101
|
||||
|
||||
- name: glm-4.7-flash
|
||||
provider: ollama
|
||||
model: glm-4.7-flash:latest
|
||||
|
||||
# too slow on 64g:
|
||||
#- name: frob/qwen3-coder-next:latest
|
||||
# provider: ollama
|
||||
# model: frob/qwen3-coder-next:latest
|
||||
|
||||
- name: kimi-k2.5
|
||||
provider: ollama
|
||||
model: kimi-k2.5:cloud
|
||||
|
||||
- name: gpt-oss-120b
|
||||
provider: ollama
|
||||
model: gpt-oss:120b-cloud
|
||||
|
||||
- name: gpt-oss-20b
|
||||
provider: ollama
|
||||
model: gpt-oss:20b
|
||||
|
||||
- name: qwen3-coder:latest
|
||||
provider: ollama
|
||||
model: qwen3-coder:latest
|
||||
|
||||
# good but too slow on 64G
|
||||
#- name: nemotron-3-nano
|
||||
# provider: ollama
|
||||
# model: nemotron-3-nano:latest
|
||||
|
||||
# =============================================================================
|
||||
# Runners - agent frameworks with their specific configurations
|
||||
# =============================================================================
|
||||
# Each runner has its own binary, extensions/config, and isolated config directory
|
||||
runners:
|
||||
# - name: goose
|
||||
# type: goose
|
||||
# bin: goose
|
||||
# extensions: [developer]
|
||||
# stdio:
|
||||
# - node mcp-harness/dist/index.js
|
||||
|
||||
- name: goose-full
|
||||
type: goose
|
||||
bin: goose
|
||||
extensions: [developer, todo, skills, code_execution, extensionmanager]
|
||||
stdio:
|
||||
- node mcp-harness/dist/index.js
|
||||
|
||||
- name: opencode
|
||||
type: opencode
|
||||
bin: opencode
|
||||
stdio:
|
||||
- node mcp-harness/dist/index.js
|
||||
|
||||
- name: pi
|
||||
type: pi
|
||||
bin: pi
|
||||
# Pi takes provider/model from the test matrix, not config
|
||||
# MCP support via pi-mcp-adapter: `pi install npm:pi-mcp-adapter`
|
||||
stdio:
|
||||
- node mcp-harness/dist/index.js
|
||||
|
||||
# =============================================================================
|
||||
# Test Matrix
|
||||
# =============================================================================
|
||||
# scenarios × models × runners
|
||||
# - Omit 'models' to run against ALL models
|
||||
# - Omit 'runners' to run against ALL runners
|
||||
matrix:
|
||||
# Single-turn scenarios: all models × all runners
|
||||
- scenario: everyday-app-automation
|
||||
- scenario: file-editing
|
||||
|
||||
# Multi-turn: goose and pi only (opencode doesn't support session continuation)
|
||||
- scenario: multi-turn-edit
|
||||
runners: [pi, goose-full]
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 2.3 MiB |
@@ -0,0 +1,87 @@
|
||||
# MCP Harness
|
||||
|
||||
A simulated MCP server with realistic fake tools for testing. Provides mock implementations of common business integrations without requiring actual API credentials.
|
||||
|
||||
## Tools Included (35 tools)
|
||||
|
||||
### Google Drive
|
||||
- `gdrive_search` - Search files by name, content, or type
|
||||
- `gdrive_read_file` - Read file contents
|
||||
- `gdrive_create_file` - Create new files
|
||||
- `gdrive_share_file` - Share files with users
|
||||
|
||||
### Google Sheets
|
||||
- `sheets_read` - Read spreadsheet data
|
||||
- `sheets_write` - Write/update cells
|
||||
- `sheets_append` - Append rows
|
||||
- `sheets_create` - Create new spreadsheets
|
||||
|
||||
### Salesforce
|
||||
- `salesforce_query` - Execute SOQL queries
|
||||
- `salesforce_get_record` - Get record by ID
|
||||
- `salesforce_create_record` - Create records
|
||||
- `salesforce_update_record` - Update records
|
||||
- `salesforce_search` - SOSL search
|
||||
|
||||
### Slack
|
||||
- `slack_send_message` - Send messages
|
||||
- `slack_get_messages` - Get channel messages
|
||||
- `slack_search_messages` - Search messages
|
||||
- `slack_list_channels` - List channels
|
||||
- `slack_get_user_info` - Get user info
|
||||
- `slack_set_status` - Set user status
|
||||
|
||||
### Google Calendar
|
||||
- `calendar_list_events` - List events
|
||||
- `calendar_create_event` - Create events
|
||||
- `calendar_update_event` - Update events
|
||||
- `calendar_delete_event` - Delete events
|
||||
|
||||
### Gmail
|
||||
- `gmail_search` - Search emails
|
||||
- `gmail_read_message` - Read email content
|
||||
- `gmail_send` - Send emails
|
||||
- `gmail_create_draft` - Create drafts
|
||||
|
||||
### Jira
|
||||
- `jira_search_issues` - Search with JQL
|
||||
- `jira_get_issue` - Get issue details
|
||||
- `jira_create_issue` - Create issues
|
||||
- `jira_update_issue` - Update issues
|
||||
- `jira_add_comment` - Add comments
|
||||
|
||||
### GitHub
|
||||
- `github_search_repos` - Search repositories
|
||||
- `github_list_issues` - List issues
|
||||
- `github_create_issue` - Create issues
|
||||
- `github_list_prs` - List pull requests
|
||||
|
||||
## Setup
|
||||
|
||||
```bash
|
||||
npm install
|
||||
npm run build
|
||||
```
|
||||
|
||||
## Run
|
||||
|
||||
```bash
|
||||
npm run start
|
||||
# or
|
||||
./run.sh
|
||||
```
|
||||
|
||||
## MCP Config
|
||||
|
||||
Add to your MCP client config:
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"harness": {
|
||||
"command": "node",
|
||||
"args": ["/path/to/mcp-harness/dist/index.js"]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
+1161
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"name": "mcp-harness",
|
||||
"version": "1.0.0",
|
||||
"description": "Simulated real-world MCP tools for testing - Google Drive, Sheets, Salesforce, Slack, and more",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"build": "tsc -p tsconfig.json",
|
||||
"start": "node dist/index.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"@modelcontextprotocol/sdk": "^1.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^25.2.0",
|
||||
"typescript": "^5.6.3"
|
||||
}
|
||||
}
|
||||
Executable
+3
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
cd "$(dirname "$0")"
|
||||
npm run build && npm run start
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"outDir": "dist",
|
||||
"rootDir": "src",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"declaration": true
|
||||
},
|
||||
"include": ["src/**/*"]
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
node_modules/
|
||||
.workdir/
|
||||
.goose-root/
|
||||
.cache/
|
||||
+1083
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"name": "agent-runner",
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"test": "tsx src/runner.ts",
|
||||
"test:scenario": "tsx src/runner.ts --scenario"
|
||||
},
|
||||
"dependencies": {
|
||||
"glob": "^11.0.0",
|
||||
"yaml": "^2.4.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.0.0",
|
||||
"tsx": "^4.19.0",
|
||||
"typescript": "^5.5.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
name: everyday-app-automation
|
||||
description: Multi-step workflow using everyday app tools (Slack, Jira, Calendar) with data dependencies
|
||||
prompt: |
|
||||
Using the available tools, complete these tasks:
|
||||
1. Search Slack for messages mentioning "quarterly review"
|
||||
2. Look up the user who posted the message about quarterly review to get their full name
|
||||
3. Create a Jira issue titled "Q1 Review Follow-ups" with a description that includes the name of the person who posted the Slack message
|
||||
4. Create a calendar event for next Monday at 2pm called "Review Discussion"
|
||||
5. Write a summary of what you did to a file called workflow-log.md
|
||||
|
||||
tags:
|
||||
- complex
|
||||
- multi-step
|
||||
- mcp-harness
|
||||
- data-flow
|
||||
|
||||
validate:
|
||||
# Check workflow summary was written
|
||||
- type: file_exists
|
||||
path: workflow-log.md
|
||||
- type: file_not_empty
|
||||
path: workflow-log.md
|
||||
|
||||
# Check Slack search was called with the right query
|
||||
- type: tool_called
|
||||
tool: slack_search_messages
|
||||
args:
|
||||
query: /quarterly.?review/
|
||||
|
||||
# Check that the agent looked up the user info (data dependency: requires reading user ID from search results)
|
||||
- type: tool_called
|
||||
tool: slack_get_user_info
|
||||
args:
|
||||
userId: /U004/
|
||||
|
||||
# Check Jira issue was created with expected title and includes the user's name (data dependency: requires reading name from user info)
|
||||
- type: tool_called
|
||||
tool: jira_create_issue
|
||||
args:
|
||||
summary: /q1.?review|follow.?up/
|
||||
description: /David.?Brown/
|
||||
|
||||
# Check calendar event was created with expected title
|
||||
- type: tool_called
|
||||
tool: calendar_create_event
|
||||
args:
|
||||
summary: /review.?discussion/
|
||||
@@ -0,0 +1,108 @@
|
||||
name: file-editing
|
||||
description: Navigate a small codebase and make a targeted edit
|
||||
prompt: |
|
||||
The User struct in user.rs is missing a display_name() method.
|
||||
Add a method that returns the full name formatted as "first_name last_name".
|
||||
|
||||
tags:
|
||||
- file-editing
|
||||
- code-navigation
|
||||
|
||||
setup:
|
||||
Cargo.toml: |
|
||||
[package]
|
||||
name = "user-service"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
src/main.rs: |
|
||||
mod models;
|
||||
mod utils;
|
||||
|
||||
use models::user::User;
|
||||
|
||||
fn main() {
|
||||
let user = User::new("Alice", "Smith", "alice@example.com");
|
||||
println!("Created user: {}", user.email());
|
||||
}
|
||||
|
||||
src/models/mod.rs: |
|
||||
pub mod user;
|
||||
|
||||
src/models/user.rs: |
|
||||
pub struct User {
|
||||
first_name: String,
|
||||
last_name: String,
|
||||
email: String,
|
||||
}
|
||||
|
||||
impl User {
|
||||
pub fn new(first_name: &str, last_name: &str, email: &str) -> Self {
|
||||
Self {
|
||||
first_name: first_name.to_string(),
|
||||
last_name: last_name.to_string(),
|
||||
email: email.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn email(&self) -> &str {
|
||||
&self.email
|
||||
}
|
||||
|
||||
pub fn first_name(&self) -> &str {
|
||||
&self.first_name
|
||||
}
|
||||
|
||||
pub fn last_name(&self) -> &str {
|
||||
&self.last_name
|
||||
}
|
||||
}
|
||||
|
||||
src/utils/mod.rs: |
|
||||
pub mod formatting;
|
||||
|
||||
src/utils/formatting.rs: |
|
||||
pub fn capitalize(s: &str) -> String {
|
||||
let mut chars = s.chars();
|
||||
match chars.next() {
|
||||
None => String::new(),
|
||||
Some(c) => c.to_uppercase().collect::<String>() + chars.as_str(),
|
||||
}
|
||||
}
|
||||
|
||||
validate:
|
||||
# The edit was made to the correct file
|
||||
- type: file_exists
|
||||
path: src/models/user.rs
|
||||
name: user.rs exists
|
||||
# Method was added
|
||||
- type: file_matches
|
||||
path: src/models/user.rs
|
||||
regex: "fn\\s+display_name"
|
||||
name: display_name() added
|
||||
# Method returns a String or &str
|
||||
- type: file_matches
|
||||
path: src/models/user.rs
|
||||
regex: "display_name.*->.*String|display_name.*->.*str"
|
||||
name: has return type
|
||||
# Original code preserved
|
||||
- type: file_contains
|
||||
path: src/models/user.rs
|
||||
pattern: "pub fn email"
|
||||
name: email() preserved
|
||||
- type: file_contains
|
||||
path: src/models/user.rs
|
||||
pattern: "pub fn first_name"
|
||||
name: first_name() preserved
|
||||
# Other files untouched
|
||||
- type: file_exists
|
||||
path: src/main.rs
|
||||
name: main.rs exists
|
||||
- type: file_contains
|
||||
path: src/main.rs
|
||||
pattern: "mod models"
|
||||
name: main.rs unchanged
|
||||
# Code compiles
|
||||
- type: command_succeeds
|
||||
command: "cargo build"
|
||||
name: cargo build
|
||||
@@ -0,0 +1,78 @@
|
||||
name: multi-turn-edit
|
||||
description: Multi-turn conversation - add a method, then rename it
|
||||
|
||||
tags:
|
||||
- file-editing
|
||||
- multi-turn
|
||||
|
||||
setup:
|
||||
Cargo.toml: |
|
||||
[package]
|
||||
name = "user-service"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
src/main.rs: |
|
||||
mod models;
|
||||
|
||||
use models::user::User;
|
||||
|
||||
fn main() {
|
||||
let user = User::new("Alice", "Smith");
|
||||
println!("User: {} {}", user.first_name(), user.last_name());
|
||||
}
|
||||
|
||||
src/models/mod.rs: |
|
||||
pub mod user;
|
||||
|
||||
src/models/user.rs: |
|
||||
pub struct User {
|
||||
first_name: String,
|
||||
last_name: String,
|
||||
}
|
||||
|
||||
impl User {
|
||||
pub fn new(first_name: &str, last_name: &str) -> Self {
|
||||
Self {
|
||||
first_name: first_name.to_string(),
|
||||
last_name: last_name.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn first_name(&self) -> &str {
|
||||
&self.first_name
|
||||
}
|
||||
|
||||
pub fn last_name(&self) -> &str {
|
||||
&self.last_name
|
||||
}
|
||||
}
|
||||
|
||||
turns:
|
||||
- prompt: |
|
||||
Add an email() method to the User struct in src/models/user.rs.
|
||||
It should return a generated email in the format "first_name.last_name@example.com" (lowercase).
|
||||
validate:
|
||||
- type: file_matches
|
||||
path: src/models/user.rs
|
||||
regex: "fn\\s+email"
|
||||
name: email() added
|
||||
- type: command_succeeds
|
||||
command: "cargo build"
|
||||
name: compiles after turn 1
|
||||
|
||||
- prompt: |
|
||||
Actually, can you rename the email() method to generated_email() instead?
|
||||
Make sure to update any references.
|
||||
validate:
|
||||
- type: file_matches
|
||||
path: src/models/user.rs
|
||||
regex: "fn\\s+generated_email"
|
||||
name: renamed to generated_email()
|
||||
- type: file_not_matches
|
||||
path: src/models/user.rs
|
||||
regex: "fn\\s+email\\s*\\("
|
||||
name: old email() removed
|
||||
- type: command_succeeds
|
||||
command: "cargo build"
|
||||
name: compiles after turn 2
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 2.3 MiB |
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,74 @@
|
||||
export interface AgentConfig {
|
||||
model: string;
|
||||
provider: string;
|
||||
/** Extensions (runner knows which are platform vs builtin) */
|
||||
extensions?: string[];
|
||||
/** Stdio extension commands (for custom MCP servers) */
|
||||
stdio?: string[];
|
||||
/** Path to goose binary (default: "goose") */
|
||||
"goose-bin"?: string;
|
||||
temperature?: number;
|
||||
maxTokens?: number;
|
||||
}
|
||||
|
||||
export interface Scenario {
|
||||
name: string;
|
||||
description: string;
|
||||
prompt?: string;
|
||||
/** Files to create before running (relative paths) */
|
||||
setup?: Record<string, string>;
|
||||
/** Validation rules to check after agent completes (single-turn) */
|
||||
validate?: ValidationRule[];
|
||||
/** Multi-turn conversation (alternative to single prompt+validate) */
|
||||
turns?: Turn[];
|
||||
/** Tags for filtering scenarios */
|
||||
tags?: string[];
|
||||
}
|
||||
|
||||
/** A single turn in a multi-turn conversation */
|
||||
export interface Turn {
|
||||
/** The prompt for this turn */
|
||||
prompt: string;
|
||||
/** Validation rules to check after this turn completes */
|
||||
validate: ValidationRule[];
|
||||
}
|
||||
|
||||
export type ValidationRule =
|
||||
| { type: "file_exists"; path: string; name?: string }
|
||||
| { type: "file_contains"; path: string; pattern: string; name?: string }
|
||||
| { type: "file_matches"; path: string; regex: string; name?: string }
|
||||
| { type: "file_not_matches"; path: string; regex: string; name?: string }
|
||||
| { type: "file_not_empty"; path: string; name?: string }
|
||||
| { type: "command_succeeds"; command: string; name?: string }
|
||||
| { type: "tool_called"; tool: string; args?: Record<string, string | RegExp>; name?: string }
|
||||
| { type: "custom"; fn: string; name?: string };
|
||||
|
||||
export interface TestRun {
|
||||
scenario: Scenario;
|
||||
config: AgentConfig;
|
||||
workdir: string;
|
||||
startTime: Date;
|
||||
endTime?: Date;
|
||||
status: "pending" | "running" | "passed" | "failed";
|
||||
errors?: string[];
|
||||
}
|
||||
|
||||
export interface TestResult {
|
||||
run: TestRun;
|
||||
validations: Array<{
|
||||
rule: ValidationRule;
|
||||
passed: boolean;
|
||||
message?: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface SuiteConfig {
|
||||
/** Agent configurations to permute */
|
||||
agents: AgentConfig[];
|
||||
/** Scenarios to run */
|
||||
scenarios: string[];
|
||||
/** Base directory for test workspaces */
|
||||
workdir: string;
|
||||
/** Parallel execution count */
|
||||
parallel?: number;
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
import { existsSync, readFileSync, statSync } from "node:fs";
|
||||
import { execSync } from "node:child_process";
|
||||
import { join } from "node:path";
|
||||
import type { ValidationRule } from "./types.js";
|
||||
|
||||
export interface ValidationResult {
|
||||
passed: boolean;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
export function validateRule(
|
||||
rule: ValidationRule,
|
||||
workdir: string
|
||||
): ValidationResult {
|
||||
switch (rule.type) {
|
||||
case "file_exists": {
|
||||
const fullPath = join(workdir, rule.path);
|
||||
const exists = existsSync(fullPath);
|
||||
return {
|
||||
passed: exists,
|
||||
message: exists ? undefined : `File not found: ${rule.path}`,
|
||||
};
|
||||
}
|
||||
|
||||
case "file_not_empty": {
|
||||
const fullPath = join(workdir, rule.path);
|
||||
if (!existsSync(fullPath)) {
|
||||
return { passed: false, message: `File not found: ${rule.path}` };
|
||||
}
|
||||
const stat = statSync(fullPath);
|
||||
return {
|
||||
passed: stat.size > 0,
|
||||
message: stat.size > 0 ? undefined : `File is empty: ${rule.path}`,
|
||||
};
|
||||
}
|
||||
|
||||
case "file_contains": {
|
||||
const fullPath = join(workdir, rule.path);
|
||||
if (!existsSync(fullPath)) {
|
||||
return { passed: false, message: `File not found: ${rule.path}` };
|
||||
}
|
||||
const content = readFileSync(fullPath, "utf-8");
|
||||
const contains = content.includes(rule.pattern);
|
||||
return {
|
||||
passed: contains,
|
||||
message: contains
|
||||
? undefined
|
||||
: `File ${rule.path} does not contain: ${rule.pattern}`,
|
||||
};
|
||||
}
|
||||
|
||||
case "file_matches": {
|
||||
const fullPath = join(workdir, rule.path);
|
||||
if (!existsSync(fullPath)) {
|
||||
return { passed: false, message: `File not found: ${rule.path}` };
|
||||
}
|
||||
const content = readFileSync(fullPath, "utf-8");
|
||||
const regex = new RegExp(rule.regex);
|
||||
const matches = regex.test(content);
|
||||
return {
|
||||
passed: matches,
|
||||
message: matches
|
||||
? undefined
|
||||
: `File ${rule.path} does not match regex: ${rule.regex}`,
|
||||
};
|
||||
}
|
||||
|
||||
case "file_not_matches": {
|
||||
const fullPath = join(workdir, rule.path);
|
||||
if (!existsSync(fullPath)) {
|
||||
return { passed: false, message: `File not found: ${rule.path}` };
|
||||
}
|
||||
const content = readFileSync(fullPath, "utf-8");
|
||||
const regex = new RegExp(rule.regex);
|
||||
const matches = regex.test(content);
|
||||
return {
|
||||
passed: !matches,
|
||||
message: !matches
|
||||
? undefined
|
||||
: `File ${rule.path} should not match regex: ${rule.regex}`,
|
||||
};
|
||||
}
|
||||
|
||||
case "command_succeeds": {
|
||||
try {
|
||||
execSync(rule.command, { cwd: workdir, stdio: "pipe" });
|
||||
return { passed: true };
|
||||
} catch (err) {
|
||||
return {
|
||||
passed: false,
|
||||
message: `Command failed: ${rule.command}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
case "tool_called": {
|
||||
const logPath = join(workdir, "tool-calls.log");
|
||||
if (!existsSync(logPath)) {
|
||||
return { passed: false, message: "tool-calls.log not found" };
|
||||
}
|
||||
|
||||
const content = readFileSync(logPath, "utf-8");
|
||||
const lines = content.trim().split("\n").filter(Boolean);
|
||||
|
||||
// Find all calls to the specified tool
|
||||
const matchingCalls = lines
|
||||
.map((line) => {
|
||||
try {
|
||||
return JSON.parse(line);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
})
|
||||
.filter((entry) => entry?.tool === rule.tool);
|
||||
|
||||
if (matchingCalls.length === 0) {
|
||||
return { passed: false, message: `Tool not called: ${rule.tool}` };
|
||||
}
|
||||
|
||||
// If no arg requirements, just check tool was called
|
||||
if (!rule.args) {
|
||||
return { passed: true };
|
||||
}
|
||||
|
||||
// Check if any call matches the arg requirements
|
||||
for (const call of matchingCalls) {
|
||||
const args = call.arguments || {};
|
||||
let allMatch = true;
|
||||
|
||||
for (const [key, expected] of Object.entries(rule.args)) {
|
||||
const actual = args[key];
|
||||
if (actual === undefined) {
|
||||
allMatch = false;
|
||||
break;
|
||||
}
|
||||
|
||||
// If expected starts/ends with /, treat as regex pattern
|
||||
if (typeof expected === "string" && expected.startsWith("/") && expected.endsWith("/")) {
|
||||
const pattern = new RegExp(expected.slice(1, -1), "i");
|
||||
if (!pattern.test(String(actual))) {
|
||||
allMatch = false;
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
// Exact match (case-insensitive for strings)
|
||||
const actualStr = String(actual).toLowerCase();
|
||||
const expectedStr = String(expected).toLowerCase();
|
||||
if (!actualStr.includes(expectedStr)) {
|
||||
allMatch = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (allMatch) {
|
||||
return { passed: true };
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
passed: false,
|
||||
message: `Tool ${rule.tool} called but args didn't match: expected ${JSON.stringify(rule.args)}`,
|
||||
};
|
||||
}
|
||||
|
||||
case "custom": {
|
||||
// Custom validators loaded dynamically
|
||||
return { passed: false, message: "Custom validators not yet implemented" };
|
||||
}
|
||||
|
||||
default:
|
||||
return { passed: false, message: `Unknown rule type` };
|
||||
}
|
||||
}
|
||||
|
||||
export function validateAll(
|
||||
rules: ValidationRule[],
|
||||
workdir: string
|
||||
): Array<{ rule: ValidationRule; result: ValidationResult }> {
|
||||
return rules.map((rule) => ({
|
||||
rule,
|
||||
result: validateRule(rule, workdir),
|
||||
}));
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"outDir": "dist"
|
||||
},
|
||||
"include": ["src"]
|
||||
}
|
||||
Reference in New Issue
Block a user