Michael Neale
100 lines
2.7 KiB
YAML
100 lines
2.7 KiB
YAML
# =============================================================================
|
||
# Models - the LLMs to test
|
||
# =============================================================================
|
||
models:
|
||
- name: opus
|
||
provider: anthropic
|
||
model: claude-opus-4-5-20251101
|
||
|
||
- name: glm-4.7-flash
|
||
provider: ollama
|
||
model: glm-4.7-flash:latest
|
||
|
||
# too slow on 64g:
|
||
#- name: frob/qwen3-coder-next:latest
|
||
# provider: ollama
|
||
# model: frob/qwen3-coder-next:latest
|
||
|
||
- name: kimi-k2.5
|
||
provider: ollama
|
||
model: kimi-k2.5:cloud
|
||
|
||
- name: gpt-oss-120b
|
||
provider: ollama
|
||
model: gpt-oss:120b-cloud
|
||
|
||
- name: gpt-oss-20b
|
||
provider: ollama
|
||
model: gpt-oss:20b
|
||
|
||
- name: qwen3-coder:latest
|
||
provider: ollama
|
||
model: qwen3-coder:latest
|
||
|
||
# good but too slow on 64G
|
||
#- name: nemotron-3-nano
|
||
# provider: ollama
|
||
# model: nemotron-3-nano:latest
|
||
|
||
# =============================================================================
|
||
# Runners - agent frameworks with their specific configurations
|
||
# =============================================================================
|
||
# Each runner has its own binary, extensions/config, and isolated config directory
|
||
runners:
|
||
# - name: goose
|
||
# type: goose
|
||
# bin: goose
|
||
# extensions: [developer]
|
||
# stdio:
|
||
# - node mcp-harness/dist/index.js
|
||
|
||
- name: goose
|
||
type: goose
|
||
bin: goose
|
||
extensions: [developer, todo, skills, code_execution, extensionmanager]
|
||
stdio:
|
||
- node mcp-harness/dist/index.js
|
||
|
||
- name: goose-diet
|
||
type: goose
|
||
bin: ~/Downloads/goose-diet
|
||
extensions: [developer]
|
||
|
||
- name: opencode
|
||
type: opencode
|
||
bin: opencode
|
||
stdio:
|
||
- node mcp-harness/dist/index.js
|
||
|
||
- name: pi
|
||
type: pi
|
||
bin: pi
|
||
# Pi takes provider/model from the test matrix, not config
|
||
# MCP support via pi-mcp-adapter: `pi install npm:pi-mcp-adapter`
|
||
stdio:
|
||
- node mcp-harness/dist/index.js
|
||
|
||
- name: pi-lean
|
||
type: pi
|
||
bin: pi
|
||
# Pi takes provider/model from the test matrix, not config
|
||
# MCP support via pi-mcp-adapter: `pi install npm:pi-mcp-adapter`
|
||
|
||
# =============================================================================
|
||
# Test Matrix
|
||
# =============================================================================
|
||
# scenarios × models × runners
|
||
# - Omit 'models' to run against ALL models
|
||
# - Omit 'runners' to run against ALL runners
|
||
matrix:
|
||
# Single-turn scenarios: all models × all runners
|
||
- scenario: everyday-app-automation
|
||
- scenario: file-editing
|
||
|
||
# Feature removal: all runners
|
||
- scenario: remove-feature
|
||
|
||
# Multi-turn: goose and pi only (opencode doesn't support session continuation)
|
||
- scenario: multi-turn-edit
|
||
runners: [pi, goose]
|