diff --git a/evals/harbor/.agents/skills/compare_tasks/SKILL.md b/evals/harbor/.agents/skills/compare_tasks/SKILL.md new file mode 100644 index 00000000..15e186d9 --- /dev/null +++ b/evals/harbor/.agents/skills/compare_tasks/SKILL.md @@ -0,0 +1,179 @@ +--- +name: compare_tasks +description: Compare how two harbor benchmark runs performed on a single shared task +--- + +# Compare two harbor runs on one task + +Use when given two harbor run names and a task name, and the goal is to understand +*why* the two runs differ on that task — not just *that* they differ. + +## Inputs + +- `RUN_A`: harbor run name (e.g. `sonnet46-full`) +- `RUN_B`: harbor run name (e.g. `pi-sonnet46-full`) +- `TASK`: bare task name (e.g. `extract-elf`, not `terminal-bench/extract-elf`) +- `RUNS_DIR`: defaults to `evals/harbor/runs/` relative to the repo root + +## Procedure + +### 1. Find each run's trial directory for the task + +Harbor 0.8 names trial dirs `__` (e.g. +`extract-elf__bU3GHs4`), **not** `.1`. The suffix is unique per trial, +so don't guess it — discover it from disk: + +```bash +TRIAL_A_DIR=$(ls -d "$RUNS_DIR/$RUN_A/${TASK}__"*/ 2>/dev/null | head -1) +TRIAL_B_DIR=$(ls -d "$RUNS_DIR/$RUN_B/${TASK}__"*/ 2>/dev/null | head -1) +``` + +If either is empty, that run didn't include this task — stop and say so. +(`ls "$RUNS_DIR/$RUN_A/"` shows what's there.) + +If you want to confirm the match, every `result.json` carries `task_name` +and `trial_name`: + +```bash +jq '{task_name, trial_name}' "$TRIAL_A_DIR/result.json" +``` + +### 2. Headline facts + +Pull these fields from each trial's `result.json`. The actual shape (harbor +0.8 `TrialResult`): + +```bash +jq '{ + reward: (.verifier_result.rewards.reward // null), + rewards_all: .verifier_result.rewards, + duration_seconds: ((.finished_at | fromdateiso8601) - (.started_at | fromdateiso8601)), + input_tokens: .agent_result.n_input_tokens, + cache_tokens: .agent_result.n_cache_tokens, + output_tokens: .agent_result.n_output_tokens, + cost_usd: .agent_result.cost_usd, + error_type: .exception_info.exception_type, + error_message: (.exception_info.exception_message // "" | split("\n")[0]) +}' "$TRIAL_A_DIR/result.json" +``` + +Derive status from those: + +- `pass` if `reward >= 1.0` +- `partial` if `reward > 0` (and < 1) +- `fail` if `reward == 0` +- `timeout` if reward is 0/null **and** `error_type` contains "timeout" +- `error` if reward is 0/null **and** `error_type` is set (non-timeout) +- `no-reward` if neither `verifier_result.rewards` nor `exception_info` is set + +Reward wins over errors: harbor can record an `AgentTimeoutError` *after* the +verifier already scored a pass (the agent finished the work then the harness +timed out during teardown, or it timed out after writing the correct answer). +If we got points, count them. See `reporter.trial_status` for the canonical +rule. + +Several `agent_result` fields are commonly `null` for older `GooseBinaryAgent` +runs (notably `n_cache_tokens`, `n_output_tokens`, `cost_usd`). Don't treat +that as a failure — just omit those facts from the comparison if missing on +either side. The reporter has fallbacks that read goose's `complete` event +from `agent/goose.txt`; you don't normally need to replicate them here. + +### 3. Read the task spec + +The task definitions are NOT in the harbor Python package. They are plain +text files on disk, in harbor's dataset cache. Do not run `find /` or +`pip show harbor` — that is the wrong direction. + +Find the task directory (works on Linux and macOS): + +```bash +TASK_DIR=$( + ls -d ~/.cache/harbor/datasets/terminal-bench__terminal-bench-2__*/tasks/"$TASK"/ 2>/dev/null \ + || ls -d ~/Library/Caches/harbor/datasets/terminal-bench__terminal-bench-2__*/tasks/"$TASK"/ 2>/dev/null +) +echo "$TASK_DIR" +ls "$TASK_DIR" +``` + +If both lookups return empty, the dataset hasn't been downloaded yet — bail +out and report that, rather than guessing. + +Inside, you care about three files: + +- `instruction.md` — exactly what the agent was asked to do +- `tests/test_outputs.py` (or sometimes `run-tests.sh`) — what the verifier + actually checks, line by line +- `solution/solution.sh` — the reference correct answer + +Without all three you can't tell whether a wrong answer was a misread, a +shallow bug, or a verifier surprise. **Quote the assertion that failed** +when you describe a failure — paraphrasing is how wrong conclusions sneak in. + +### 4. Read each agent's trajectory + +Two sources, prefer the first when present: + +- `$TRIAL_DIR/agent/trajectory.json` — harbor's ATIF format, one entry per + agent step. `jq '.steps[] | {step_id, source, message, tool_calls: [.tool_calls[]?.function_name]}'` + gives a compact view. Recent goose runs (after the populate_context_post_run + fix) have this; older `GooseBinaryAgent` runs may not. +- `$TRIAL_DIR/agent/.txt` — raw stream-json or log. The filename + matches the harness: `goose.txt`, `pi.txt`, `opencode.txt`, + `claude-code.txt`. `ls "$TRIAL_DIR/agent/"` to find it. + +Skim, don't quote in full. For each agent identify: + +- the approach it took (e.g. "wrote a Python script that walks the ELF section + headers") +- the final artifacts it left in the container (file paths it created or + modified) +- for losers, the **failure mode** — one of: + - misread the spec (wrong assumption about input/output) + - right approach, shallow bug (off-by-one, wrong encoding, wrong base address) + - ran out of clock (timeout) — note whether it was still making progress or + had gone in circles + - diverged into an unproductive thread (e.g. debugging a non-issue) + - the verifier expected something the spec didn't telegraph + +### 5. Read the verifier output + +`$TRIAL_DIR/verifier/` typically contains: + +- `test-stdout.txt` — the verifier's full stdout (assertion failures, pytest + output, etc.). This is usually the most diagnostic file. +- `reward.txt` — the scalar reward as a string. +- `ctrf.json` — structured test results in CTRF format, useful if you want + per-assertion pass/fail without grepping stdout. + +```bash +tail -50 "$TRIAL_DIR/verifier/test-stdout.txt" +``` + +This is often more diagnostic than the agent log — it tells you exactly which +assertion failed and what the agent's output was at that point. + +### 6. Produce the comparison + +Output markdown with these sections in order: + +- **Headline** (1 line): who won, by how much (reward + cost / duration if + meaningful, omitting fields that are null on either side). +- **What A did** (2-4 sentences): plan, final artifact, verifier outcome. +- **What B did** (2-4 sentences): same shape as A. +- **Why outcomes differ** (2-4 sentences): the actual mechanism. Not "B was + smarter" but "B's script used `nm -n` so its addresses matched the verifier's + ground truth, A's script used PIE-relocated virtual addresses which the + verifier doesn't normalize". +- **Generalizable lesson** (optional, 1-2 sentences): is this a pattern that + probably affects other tasks, or a one-off accident of this verifier? Skip + if unclear from one task. + +## Tools you'll need + +- `ls -d` to discover the `__` trial directories +- `jq` for `result.json` +- file reads against `$TRIAL_DIR/agent/` and `$TRIAL_DIR/verifier/` +- file reads against the dataset cache (`~/.cache/harbor/datasets/...`) + +No Python imports, no `harbor` package required. Everything you need is on +disk as JSON / text files. diff --git a/evals/harbor/.gitignore b/evals/harbor/.gitignore index 29d2d56f..8455fae2 100644 --- a/evals/harbor/.gitignore +++ b/evals/harbor/.gitignore @@ -1,4 +1,5 @@ -.runs/ +runs/ +.env .venv/ -.pytest_cache/ -uv.lock +__pycache__/ +*.pyc diff --git a/evals/harbor/README.md b/evals/harbor/README.md index 3dd37a0a..e307fc2f 100644 --- a/evals/harbor/README.md +++ b/evals/harbor/README.md @@ -1,141 +1,238 @@ -# Harbor +# Harbor benchmark tooling for Goose -This directory contains a developer tool for running Harbor benchmark datasets -with Goose. +A small command-line tool for running and comparing terminal-bench-style +benchmarks against different agent harnesses, models, and goose builds. -The runner takes a prebuilt Goose executable, writes a Harbor job config, and -runs Harbor with the local `goose_harbor` adapter. +## Current results -## Requirements +Latest `cmd.py list` snapshot across the runs in `runs/`. All `*-full` runs +cover the full `terminal-bench/terminal-bench-2` dataset (89 tasks). +`pass/fail/err/tout` is the per-status breakdown. `compute` is the sum of +per-trial durations (parallelism unrolled), not wall clock — it's a stable +measure of how much agent time a run cost regardless of host concurrency. +`turns` is the total number of agent turns across all trials (one per +assistant message / harness step). -- `uv` -- `harbor` -- Docker, for Docker-backed Harbor datasets -- A Goose executable compatible with the benchmark task environment +``` +job_name model rate compute in out turns cost pass/fail/err/tout +----------------------------------------------------------------------------------------------------------------------------------- +claude-sonnet46-full claude-sonnet-4-6 55.1% 20.2h 102.3M 1.2M 3k $42.83 49/23/1/16 +goose-1.30-sonnet46-full claude-sonnet-4-6 50.6% 23.7h 2.4M - 3k - 45/24/2/18 +goose-sonnet46-full-code-mode claude-sonnet-4-6 57.3% 22.0h 63.3M 1.1M 3k $206.43 51/20/2/16 +nemotron-full nemotron-3-nano-30b-a3b 1.1% 21.8h 9.5M 2.2M 1k - 1/64/2/22 +opencode-sonnet46-full claude-sonnet-4-6 52.8% 22.2h 111.5M 1.6M 3k $70.30 47/23/0/19 +pi-sonnet46-full claude-sonnet-4-6 47.2% 24.4h 114.4M 1.8M 3k $74.82 42/25/1/21 +sonnet46-dev-only claude-sonnet-4-6 48.3% 23.2h 70.6M 1.2M 3k $229.19 43/25/2/19 +sonnet46-full claude-sonnet-4-6 50.6% 22.5h 62.4M - 3k - 45/21/3/20 +sonnet46-sum_codem claude-sonnet-4-6 57.3% 21.9h 78.1M 1.4M 3k $254.53 51/23/2/13 +sonnet46-summon-full claude-sonnet-4-6 55.1% 23.5h 67.2M 1.0M 3k $217.28 49/19/3/18 +``` -Dependencies are declared in `pyproject.toml`. `uv` resolves them from the -developer's configured package index. +Quick read: -## Run A Task +- `goose-sonnet46-full-code-mode` and `sonnet46-sum_codem` (both run codemode, + the latter also enabling summon) lead at **57.3%**. +- Stock goose (`sonnet46-full`, `developer,todo`) lands at **50.6%**, roughly + on par with `opencode` (52.8%) and ahead of `pi` (47.2%) on the same model. + Notably, `pi` also burned the most compute (24.4h) — slowest *and* lowest + scoring of the sonnet runs. +- `claude-sonnet46-full` at **55.1%** is harbor's vanilla `Goose` harness + (curl-installed) — useful sanity check that our `GooseBinaryAgent` adapter + isn't leaving points on the floor. +- `nemotron-full` solves 1 task using roughly the same compute budget but + only ~1k turns (vs 3k for sonnet runs) — the small model gives up or + loses tool-call structure earlier, so it doesn't even reach the + 100-turn cap on most trials. + +## Setup + +Requires `uv`, Docker, and `rsync` on the host. `cmd.py` is a +[PEP 723 inline-uv script](https://peps.python.org/pep-0723/), so `uv` installs +its Python deps (just `harbor` and `PyYAML`) on first run. + +Secrets live in a `.env` file. `cmd.py` looks for one in the current working +directory first, then in this script's directory. Only the keys for the +provider you're using need to be set: + +``` +ANTHROPIC_API_KEY=sk-ant-... +OPENROUTER_API_KEY=sk-or-... +DATABRICKS_HOST=https://... +DATABRICKS_TOKEN=... +OPENAI_API_KEY=sk-... +``` + +alternatively, you can just export them in the session where you run the benchmark + +## Running a goose benchmark + +The `run` subcommand builds a harbor config that uses our `GooseBinaryAgent` +adapter — it uploads your local goose binary into each task container, +generates a `config.yaml` from the template with the requested extensions +flipped on, runs the recipe, and streams JSON output. ```bash -uv run --project evals/harbor evals/harbor/run \ - --goose-binary ./target/x86_64-unknown-linux-gnu/release/goose \ - --goose-profile ~/.config/goose-benchmark \ - --dataset terminal-bench/terminal-bench-2 \ - --model databricks/ \ - --task terminal-bench/fix-git \ - --trials 1 \ - --concurrency 1 +# Pin a specific binary, default everything else +./evals/harbor/cmd.py run /path/to/goose --job-name my-run + +# Different model +./evals/harbor/cmd.py run /path/to/goose \ + --model anthropic/claude-opus-4-5 --job-name opus-run + +# OpenRouter +./evals/harbor/cmd.py run /path/to/goose \ + --model openrouter/nvidia/nemotron-3-nano-30b-a3b \ + --job-name nemotron-smoke + +# Subset of tasks (note: harbor wants the qualified form) +./evals/harbor/cmd.py run /path/to/goose \ + --tasks terminal-bench/fix-git,terminal-bench/extract-elf \ + --job-name smoke + +# Toggle which extensions are enabled in config.yaml +./evals/harbor/cmd.py run /path/to/goose \ + --extensions developer,todo,codemode --job-name codemode-run + +# Double the per-task timeout (useful for rerunning AgentTimeoutError trials) +./evals/harbor/cmd.py run /path/to/goose \ + --timeout-multiplier 2.0 \ + --tasks terminal-bench/oom,terminal-bench/compile-vim \ + --job-name oom-retry-2x ``` -Use `--dry-run` to write the Harbor config without starting the benchmark: +Defaults: +- dataset: `terminal-bench/terminal-bench-2` +- model: `anthropic/claude-sonnet-4-6` +- extensions: `developer,todo` +- concurrency: 4 +- max turns: 100 +- trials: 1 +- installs `libgomp1` in each container (disable with `--no-install-goose-runtime-deps`) -```bash -uv run --project evals/harbor evals/harbor/run \ - --goose-binary ./target/x86_64-unknown-linux-gnu/release/goose \ - --goose-profile ~/.config/goose-benchmark \ - --dataset terminal-bench/terminal-bench-2 \ - --model databricks/ \ - --task terminal-bench/fix-git \ - --dry-run -``` +Use `--dry-run` to print the generated harbor config without launching. -Outputs default to: +## Running a non-goose harness -```text -evals/harbor/.runs/configs/ -evals/harbor/.runs/jobs/ -``` - -Override them with `--config-dir` and `--jobs-dir`. - -## Goose Executable - -`--goose-binary` must point to a Goose executable that can run inside the -benchmark task container. The runner does not build Goose for you; it uploads -the executable you provide into each task container and runs that copy. - -For Terminal-Bench 2.0, use a Linux amd64 Goose binary. - -On Linux: - -```bash -cargo build --release -p goose-cli --bin goose -uv run --project evals/harbor evals/harbor/run --goose-binary ./target/release/goose ... -``` - -On macOS or Windows, use a cross-compiled Linux amd64 binary. Prefer a binary -built for benchmark/container use. In particular, a Goose CLI binary without -local inference is usually the best fit for Harbor runs because local inference -pulls in runtime dependencies that may not exist in benchmark task images. - -When using a GitHub release binary for Terminal-Bench, use the standard Linux -amd64 artifact, not the Vulkan artifact. - -Some Linux release binaries still require GCC's OpenMP runtime, packaged as -`libgomp1` on Debian and Ubuntu. If the binary fails to start with a missing -`libgomp.so.1` error, rerun with: - -```bash -uv run --project evals/harbor evals/harbor/run \ - --goose-binary ./goose \ - --goose-profile ~/.config/goose-benchmark \ - --dataset terminal-bench/terminal-bench-2 \ - --model databricks/ \ - --install-goose-runtime-deps -``` - -This installs only the minimal known Goose runtime dependency, currently -`libgomp1`, inside each Debian/Ubuntu task container before Goose starts. Leave -it off when the provided Goose executable can start in the task container -without extra OS packages. - -For local models, prefer running Ollama or llama.cpp outside the task container -and configuring Goose to call that server through its normal provider/profile -configuration. Avoid running local inference inside each benchmark task -container unless you have specifically built and verified a compatible Goose -binary for that environment. - -## Goose Profile - -Pass `--goose-profile` to copy an explicit Goose profile into each benchmark -task container. The path can be either: - -- a `GOOSE_PATH_ROOT` directory with `config/`, `data/`, and `state/` -- a Goose config directory containing `config.yaml` - -The adapter sets `GOOSE_PATH_ROOT` inside the container after copying the -profile. `--model provider/model` still selects the provider and model for the -benchmark run. - -If the profile contains `secrets.yaml`, that file will be copied into arbitrary -benchmark task containers. Prefer benchmark-scoped or disposable credentials. - -## Local Models - -For local models, prefer running the model server on the host and configuring -the benchmark profile to reach it from the task container. This keeps model -loading and hardware acceleration outside Docker while Goose runs inside the -benchmark environment. - -For example, an Ollama profile can set: +Stock harnesses that harbor ships with (opencode, pi, aider, claude-code, ...) +don't need our adapter — they install themselves in the container and read +secrets from env. Write a harbor YAML config directly and call `harbor run`: ```yaml -GOOSE_PROVIDER: ollama -GOOSE_MODEL: qwen3.6:27b -OLLAMA_HOST: http://host.docker.internal:11434 +# opencode-sonnet46-full.yaml +job_name: opencode-sonnet46-full +jobs_dir: /path/to/goose/evals/harbor/runs # so cmd.py picks it up +n_attempts: 1 +n_concurrent_trials: 4 +environment: + type: docker + force_build: false + delete: true + env: + - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} +agents: + - import_path: harbor.agents.installed.opencode:OpenCode + model_name: anthropic/claude-sonnet-4-6 +datasets: + - name: terminal-bench/terminal-bench-2 ``` -Then run with `--goose-profile` pointing at that profile and `--model -ollama/qwen3.6:27b`. - -Running Goose's built-in local inference inside the benchmark container is less -portable: the model file, CPU/GPU support, target architecture, and container -runtime all have to line up. - -## Tests - ```bash -uv run --project evals/harbor pytest evals/harbor/tests +export ANTHROPIC_API_KEY=... +uv tool install harbor +harbor run -c opencode-sonnet46-full.yaml ``` + +The output lands under `evals/harbor/runs/opencode-sonnet46-full/`, alongside +goose runs. `cmd.py list / show / compare` treats them identically — they're +all harbor `TrialResult` JSON under the hood. + +For pi specifically you can lift the existing config we used: + +```yaml +agents: + - import_path: harbor.agents.installed.pi:Pi + model_name: anthropic/claude-sonnet-4-6 + kwargs: + thinking: "off" +``` + +## Inspecting results + +`cmd.py list` shows every run under `runs/` with one line per job: + +```bash +./evals/harbor/cmd.py list +``` + +Drill into a specific run: + +```bash +./evals/harbor/cmd.py show # all tasks +./evals/harbor/cmd.py show --status error # filter by outcome +./evals/harbor/cmd.py show --status timeout +``` + +Drill into a single task in a single run: + +```bash +./evals/harbor/cmd.py task +./evals/harbor/cmd.py task --tail 50 # tail agent log +``` + +Compare two runs head-to-head: + +```bash +./evals/harbor/cmd.py compare # summary +./evals/harbor/cmd.py compare -v # plus per-task diffs +``` + +Delete runs: + +```bash +./evals/harbor/cmd.py rm [ ...] # confirms by default +./evals/harbor/cmd.py rm -y # skip the prompt +``` + +## Syncing runs between machines + +If you run benchmarks on a remote box and want to inspect them locally: + +```bash +# Pull everything +./evals/harbor/cmd.py pull tbench@douwe.com:/home/tbench/work/goose + +# Just specific jobs +./evals/harbor/cmd.py pull tbench@douwe.com:/home/tbench/work/goose \ + --jobs sonnet46-full pi-sonnet46-full + +# Mirror exactly (delete local runs that aren't on the remote) +./evals/harbor/cmd.py pull tbench@douwe.com:/home/tbench/work/goose --delete +``` + +The remote argument is `user@host:/path/to/goose` — `pull` appends +`evals/harbor/runs/` to it and rsyncs into the local `runs/`. + +## A typical comparison workflow + +```bash +# Run two configurations on the remote (in screen / mosh / tmux) +ssh tbench@douwe.com +cd /home/tbench/work/goose +./evals/harbor/cmd.py run ./target/release/goose --job-name baseline +./evals/harbor/cmd.py run ./target/release/goose \ + --extensions developer,todo,codemode --job-name codemode + +# Pull results locally +./evals/harbor/cmd.py pull tbench@douwe.com:/home/tbench/work/goose \ + --jobs baseline codemode + +# Diff +./evals/harbor/cmd.py compare baseline codemode -v +``` + +For deeper per-task understanding (why did A pass and B fail on this one +task?), see the `compare_tasks` skill under `.agents/skills/`. Delegate to +it with the two job names and a task name and it will read both +trajectories, the task spec, and the verifier output, then explain the +mechanism behind the divergence. + diff --git a/evals/harbor/goose_harbor/goose_binary.py b/evals/harbor/agent.py similarity index 52% rename from evals/harbor/goose_harbor/goose_binary.py rename to evals/harbor/agent.py index 868bca06..b86af574 100644 --- a/evals/harbor/goose_harbor/goose_binary.py +++ b/evals/harbor/agent.py @@ -1,34 +1,61 @@ +"""Harbor agent that runs a caller-provided Goose binary inside the task container.""" + from __future__ import annotations +import json import os import shlex from pathlib import Path from tempfile import TemporaryDirectory -from harbor.agents.installed.base import with_prompt_template +import yaml + +from harbor.agents.installed.base import NonZeroAgentExitCodeError, with_prompt_template from harbor.agents.installed.goose import Goose from harbor.environments.base import BaseEnvironment from harbor.models.agent.context import AgentContext + +PROVIDER_SECRETS = { + "anthropic": ["ANTHROPIC_API_KEY"], + "openai": ["OPENAI_API_KEY"], + "databricks": ["DATABRICKS_HOST", "DATABRICKS_TOKEN"], + "google": ["GOOGLE_API_KEY"], + "gemini": ["GEMINI_API_KEY"], + "openrouter": ["OPENROUTER_API_KEY"], +} + CONTAINER_GOOSE_PATH_ROOT = "/installed-agent/goose-profile" +CONTAINER_CONFIG_PATH = f"{CONTAINER_GOOSE_PATH_ROOT}/config/config.yaml" CONTAINER_RECIPE_PATH = "/installed-agent/harbor-recipe.yaml" CONTAINER_CA_BUNDLE_PATH = "/installed-agent/ca-certificates.crt" +FATAL_GOOSE_NOTIFICATIONS = ("creditsExhausted",) + class GooseBinaryAgent(Goose): - """Run a caller-provided Goose binary in the benchmark environment.""" + """Run a caller-provided Goose binary in the benchmark environment. + + Differs from harbor's vanilla ``Goose``: + * Uses a pre-built binary uploaded into the container (no curl install). + * Generates ``config.yaml`` from ``config_template.yaml`` with a + caller-specified set of enabled extensions. + * Reads provider secrets from the harbor host env, not from a profile file. + """ def __init__( self, *args, goose_binary: str, - goose_profile: str, + config_yaml: str, + extension_entries: list[dict[str, str]], install_goose_runtime_deps: bool = False, **kwargs, ): super().__init__(*args, **kwargs) self.goose_binary = Path(goose_binary).expanduser().resolve() - self.goose_profile = Path(goose_profile).expanduser().resolve() + self.config_yaml = config_yaml + self.extension_entries = extension_entries self.install_goose_runtime_deps = install_goose_runtime_deps self.ca_bundle_env_path: str | None = None @@ -39,15 +66,6 @@ class GooseBinaryAgent(Goose): def get_version_command(self) -> str | None: return "/installed-agent/goose --version" - def _profile_source_target(self) -> tuple[Path, str]: - if not self.goose_profile.is_dir(): - raise FileNotFoundError(f"Goose profile does not exist: {self.goose_profile}") - - if (self.goose_profile / "config.yaml").is_file(): - return self.goose_profile, f"{CONTAINER_GOOSE_PATH_ROOT}/config" - - return self.goose_profile, CONTAINER_GOOSE_PATH_ROOT - def _run_env(self) -> dict[str, str]: if not self.model_name or "/" not in self.model_name: raise ValueError("Model name must be in the format provider/model_name") @@ -62,29 +80,26 @@ class GooseBinaryAgent(Goose): "GOOSE_PATH_ROOT": CONTAINER_GOOSE_PATH_ROOT, "GOOSE_DISABLE_KEYRING": "true", } + for key in PROVIDER_SECRETS.get(provider, []): + value = os.environ.get(key) + if value: + env[key] = value if self.ca_bundle_env_path: env["SSL_CERT_FILE"] = self.ca_bundle_env_path return env def _host_ca_bundle(self) -> Path: - candidates = [ - "SSL_CERT_FILE", - "REQUESTS_CA_BUNDLE", - "CURL_CA_BUNDLE", - ] - for env_var in candidates: + for env_var in ("SSL_CERT_FILE", "REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE"): value = os.environ.get(env_var) if value and Path(value).expanduser().is_file(): return Path(value).expanduser().resolve() - - for path in [ + for path in ( Path("/etc/ssl/certs/ca-certificates.crt"), Path("/etc/ssl/cert.pem"), Path("/opt/homebrew/etc/ca-certificates/cert.pem"), - ]: + ): if path.is_file(): return path.resolve() - raise FileNotFoundError("Could not find a host CA bundle to copy into the task container") async def _ensure_ca_bundle(self, environment: BaseEnvironment) -> None: @@ -98,7 +113,6 @@ class GooseBinaryAgent(Goose): ) if result.stdout.strip() != "missing": return - await environment.upload_file(self._host_ca_bundle(), CONTAINER_CA_BUNDLE_PATH) await self.exec_as_root( environment, @@ -119,43 +133,21 @@ class GooseBinaryAgent(Goose): timeout_sec=300, ) - def _build_register_skills_command(self) -> str | None: - if not self.skills_dir: - return None - skills_target = f"{CONTAINER_GOOSE_PATH_ROOT}/config/skills" - return ( - f"mkdir -p {shlex.quote(skills_target)} && " - f"cp -r {shlex.quote(self.skills_dir)}/* " - f"{shlex.quote(skills_target)}/ 2>/dev/null || true" - ) - async def _agent_uid_gid(self, environment: BaseEnvironment) -> tuple[str, str]: - result = await self.exec_as_agent( - environment, - command="id -u && id -g", - timeout_sec=10, - ) + result = await self.exec_as_agent(environment, command="id -u && id -g", timeout_sec=10) ids = [line.strip() for line in result.stdout.splitlines() if line.strip()] if len(ids) < 2: raise RuntimeError(f"Could not determine agent uid/gid: {result.stdout!r}") - return ids[0], ids[1] async def _chown_to_agent_user( - self, - environment: BaseEnvironment, - path: str, - *, - recursive: bool = False, + self, environment: BaseEnvironment, path: str, *, recursive: bool = False ) -> None: uid, gid = await self._agent_uid_gid(environment) - recursive_flag = "-R " if recursive else "" + flag = "-R " if recursive else "" await self.exec_as_root( environment, - command=( - f"chown {recursive_flag}{shlex.quote(uid)}:{shlex.quote(gid)} " - f"{shlex.quote(path)}" - ), + command=f"chown {flag}{shlex.quote(uid)}:{shlex.quote(gid)} {shlex.quote(path)}", ) async def install(self, environment: BaseEnvironment) -> None: @@ -168,12 +160,15 @@ class GooseBinaryAgent(Goose): await self._install_goose_runtime_deps(environment) await self._ensure_ca_bundle(environment) - source, target = self._profile_source_target() - await self.exec_as_root(environment, command=f"mkdir -p {shlex.quote(target)}") - await environment.upload_dir(source, target) - await self._chown_to_agent_user( - environment, CONTAINER_GOOSE_PATH_ROOT, recursive=True + config_dir = f"{CONTAINER_GOOSE_PATH_ROOT}/config" + await self.exec_as_root( + environment, command=f"mkdir -p {shlex.quote(config_dir)}" ) + with TemporaryDirectory() as tmp: + config_path = Path(tmp) / "config.yaml" + config_path.write_text(self.config_yaml) + await environment.upload_file(config_path, CONTAINER_CONFIG_PATH) + await self._chown_to_agent_user(environment, CONTAINER_GOOSE_PATH_ROOT, recursive=True) await self.exec_as_agent( environment, @@ -191,6 +186,24 @@ class GooseBinaryAgent(Goose): timeout_sec=30, ) + def _create_recipe_yaml(self, instruction: str) -> str: + return yaml.dump( + { + "version": "1.0.0", + "title": "harbor-task", + "description": "harbor task recipe", + "instructions": ( + "You are given a task and you need to complete it. " + "You are currently executing in a docker container where you are " + "being evaluated on a benchmark for LLM agents. Act autonomously. " + "You will not receive any feedback on your progress, so you must " + "use your own tools to complete the task without any intervention." + ), + "prompt": instruction, + "extensions": self.extension_entries, + } + ) + @with_prompt_template async def run( self, @@ -201,20 +214,10 @@ class GooseBinaryAgent(Goose): env = self._run_env() recipe_yaml = self._create_recipe_yaml(instruction) - skills_command = self._build_register_skills_command() - if skills_command: - await self.exec_as_agent( - environment, - command=skills_command, - env=env, - timeout_sec=10, - ) - - with TemporaryDirectory() as tmp_dir: - recipe_path = Path(tmp_dir) / "harbor-recipe.yaml" + with TemporaryDirectory() as tmp: + recipe_path = Path(tmp) / "harbor-recipe.yaml" recipe_path.write_text(recipe_yaml) await environment.upload_file(recipe_path, CONTAINER_RECIPE_PATH) - await self._chown_to_agent_user(environment, CONTAINER_RECIPE_PATH) cli_flags = self.build_cli_flags() @@ -229,3 +232,69 @@ class GooseBinaryAgent(Goose): ), env=env, ) + self._raise_on_fatal_goose_notification() + + def _raise_on_fatal_goose_notification(self) -> None: + log_path = self.logs_dir / "goose.txt" + if not log_path.is_file(): + return + log_text = log_path.read_text(errors="replace") + for notification in FATAL_GOOSE_NOTIFICATIONS: + if f'"notificationType":"{notification}"' in log_text: + raise NonZeroAgentExitCodeError( + f"Goose exited without running the task: {notification}. " + f"See {log_path} for details." + ) + + @staticmethod + def _extract_complete_event_tokens( + log_text: str, + ) -> tuple[int | None, int | None, int | None]: + total = inp = out = None + for line in log_text.strip().split("\n"): + line = line.strip() + if not line or '"complete"' not in line: + continue + event = json.loads(line) + if event.get("type") != "complete": + continue + total = event.get("total_tokens") + inp = event.get("input_tokens") + out = event.get("output_tokens") + return total, inp, out + + def _compute_cost_from_pricing( + self, prompt_tokens: int | None, completion_tokens: int | None + ) -> float | None: + if not self.model_name or not (prompt_tokens or completion_tokens): + return None + try: + import litellm + except ImportError: + return None + pricing = None + for key in (self.model_name, self.model_name.split("/", 1)[-1]): + entry = litellm.model_cost.get(key) + if entry: + pricing = entry + break + if pricing is None: + return None + return (prompt_tokens or 0) * (pricing.get("input_cost_per_token") or 0.0) + ( + completion_tokens or 0 + ) * (pricing.get("output_cost_per_token") or 0.0) + + def populate_context_post_run(self, context: AgentContext) -> None: + super().populate_context_post_run(context) + txt_path = self.logs_dir / "goose.txt" + if not txt_path.exists(): + return + log_text = txt_path.read_text() + _total, inp, out = self._extract_complete_event_tokens(log_text) + if inp is not None: + context.n_input_tokens = inp + if out is not None: + context.n_output_tokens = out + cost = self._compute_cost_from_pricing(inp, out) + if cost is not None: + context.cost_usd = cost diff --git a/evals/harbor/cmd.py b/evals/harbor/cmd.py new file mode 100755 index 00000000..b1fd95b8 --- /dev/null +++ b/evals/harbor/cmd.py @@ -0,0 +1,132 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.12" +# dependencies = ["harbor==0.8.0", "PyYAML>=6.0"] +# /// +"""Harbor benchmark runner and reporter for Goose. + +Subcommands: + run run a benchmark job + list list all runs in the runs/ directory + show per-task results for one run + task full detail for one task in one run + compare compare two runs task-by-task + rm remove one or more runs +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +from reporter import cmd_compare, cmd_list, cmd_pull, cmd_rm, cmd_show, cmd_task +from runner import ( + DEFAULT_CONCURRENCY, + DEFAULT_DATASET, + DEFAULT_EXTENSIONS, + DEFAULT_MAX_TURNS, + DEFAULT_MODEL, + cmd_run, + parse_csv, +) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + sub = parser.add_subparsers(dest="cmd", required=True) + + p_run = sub.add_parser("run", help="run a benchmark job") + p_run.add_argument("goose_binary", type=Path, help="path to the goose binary to test") + p_run.add_argument("--dataset", default=DEFAULT_DATASET) + p_run.add_argument("--model", default=DEFAULT_MODEL) + p_run.add_argument( + "--tasks", + type=parse_csv, + default=[], + help="comma-separated task names (default: all tasks in dataset)", + ) + p_run.add_argument( + "--extensions", + type=parse_csv, + default=DEFAULT_EXTENSIONS, + help=f"comma-separated extension names (default: {','.join(DEFAULT_EXTENSIONS)})", + ) + p_run.add_argument("--trials", type=int, default=1) + p_run.add_argument("--concurrency", type=int, default=DEFAULT_CONCURRENCY) + p_run.add_argument("--max-turns", type=int, default=DEFAULT_MAX_TURNS) + p_run.add_argument("--timeout-multiplier", type=float, default=1.0) + p_run.add_argument("--job-name") + p_run.add_argument( + "--no-install-goose-runtime-deps", + dest="install_goose_runtime_deps", + action="store_false", + default=True, + help="skip apt-get install libgomp1 inside the task container", + ) + p_run.add_argument("--dry-run", action="store_true") + + sub.add_parser("list", help="list all runs with summary stats") + + p_show = sub.add_parser("show", help="per-task results for one run") + p_show.add_argument("job_name") + p_show.add_argument( + "--status", + choices=["pass", "partial", "fail", "timeout", "error", "no-reward"], + ) + + p_task = sub.add_parser("task", help="full detail for one task in one run") + p_task.add_argument("job_name") + p_task.add_argument("task_name") + p_task.add_argument("--tail", type=int, default=0, help="tail N lines of the agent log") + + p_cmp = sub.add_parser("compare", help="compare two runs task-by-task") + p_cmp.add_argument("job_a") + p_cmp.add_argument("job_b") + p_cmp.add_argument("-v", "--verbose", action="store_true") + + p_rm = sub.add_parser("rm", help="remove one or more runs") + p_rm.add_argument("job_names", nargs="+", help="job names under runs/") + p_rm.add_argument("-y", "--yes", action="store_true", help="skip confirmation prompt") + + p_pull = sub.add_parser("pull", help="rsync runs from a remote machine") + p_pull.add_argument( + "remote", + help="user@host:/path/to/goose (we append evals/harbor/runs/)", + ) + p_pull.add_argument( + "--jobs", + nargs="*", + help="restrict to specific job names (default: all runs)", + ) + p_pull.add_argument( + "--delete", + action="store_true", + help="remove local runs that no longer exist on the remote", + ) + + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if args.cmd == "run": + return cmd_run(args) + if args.cmd == "list": + return cmd_list(args) + if args.cmd == "show": + return cmd_show(args) + if args.cmd == "task": + return cmd_task(args) + if args.cmd == "compare": + return cmd_compare(args) + if args.cmd == "rm": + return cmd_rm(args) + if args.cmd == "pull": + return cmd_pull(args) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/harbor/config_template.yaml b/evals/harbor/config_template.yaml new file mode 100644 index 00000000..d786d76f --- /dev/null +++ b/evals/harbor/config_template.yaml @@ -0,0 +1,39 @@ +GOOSE_PROVIDER: ${GOOSE_PROVIDER} +GOOSE_MODEL: ${GOOSE_MODEL} +GOOSE_THINKING_EFFORT: "off" +GOOSE_CLI_MIN_PRIORITY: 0.1 +extensions: + developer: + bundled: true + enabled: false + name: developer + type: builtin + timeout: 300 + todo: + bundled: true + enabled: false + name: todo + type: platform + computercontroller: + bundled: true + enabled: false + name: computercontroller + type: builtin + timeout: 300 + memory: + bundled: true + enabled: false + name: memory + type: builtin + timeout: 300 + summon: + bundled: true + enabled: false + name: summon + type: platform + codemode: + bundled: true + enabled: false + name: codemode + type: builtin + timeout: 300 diff --git a/evals/harbor/goose_harbor/__init__.py b/evals/harbor/goose_harbor/__init__.py deleted file mode 100644 index 8b137891..00000000 --- a/evals/harbor/goose_harbor/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/evals/harbor/goose_harbor/runner.py b/evals/harbor/goose_harbor/runner.py deleted file mode 100644 index efdcaf41..00000000 --- a/evals/harbor/goose_harbor/runner.py +++ /dev/null @@ -1,207 +0,0 @@ -from __future__ import annotations - -import argparse -import json -import os -import re -import subprocess -import sys -from datetime import datetime -from pathlib import Path -from typing import Any - -HARBOR_AGENT_IMPORT_PATH = "goose_harbor.goose_binary:GooseBinaryAgent" - - -def harbor_dir() -> Path: - return Path(__file__).resolve().parents[1] - - -def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - description="Run a Harbor dataset with a caller-provided Goose binary.", - ) - parser.add_argument("--goose-binary", required=True, type=Path) - parser.add_argument( - "--goose-profile", - required=True, - type=Path, - help=( - "Goose profile directory to copy into the benchmark container. " - "Accepts either a GOOSE_PATH_ROOT-style directory or a config directory " - "containing config.yaml." - ), - ) - parser.add_argument("--dataset", required=True) - parser.add_argument("--model", required=True) - parser.add_argument("--task", action="append", default=[], dest="tasks") - parser.add_argument("--trials", type=int, default=1) - parser.add_argument("--concurrency", type=int, default=1) - parser.add_argument("--max-turns", type=int) - parser.add_argument("--jobs-dir", type=Path, default=harbor_dir() / ".runs" / "jobs") - parser.add_argument( - "--config-dir", type=Path, default=harbor_dir() / ".runs" / "configs" - ) - parser.add_argument("--job-name") - parser.add_argument("--force-build", action="store_true") - parser.add_argument( - "--install-goose-runtime-deps", - action="store_true", - help=( - "Install minimal OS runtime dependencies required by some Goose release " - "binaries inside Debian/Ubuntu task containers." - ), - ) - parser.add_argument("--dry-run", action="store_true") - return parser - - -def pythonpath_with_harbor() -> str: - existing = os.environ.get("PYTHONPATH", "") - return f"{harbor_dir()}{os.pathsep}{existing}" if existing else str(harbor_dir()) - - -def dataset_config(dataset_ref: str, tasks: list[str]) -> dict[str, Any]: - name, sep, ref = dataset_ref.rpartition("@") - dataset: dict[str, Any] = {"name": name if sep else dataset_ref} - if sep: - dataset["ref" if "/" in name else "version"] = ref - if tasks: - dataset["task_names"] = tasks - return dataset - - -def package_index_env() -> dict[str, str]: - index_url = next( - ( - os.environ[key] - for key in ("UV_DEFAULT_INDEX", "PIP_INDEX_URL", "UV_INDEX_URL") - if os.environ.get(key) - ), - None, - ) - if index_url is None: - return {} - return { - "PIP_INDEX_URL": index_url, - "UV_DEFAULT_INDEX": index_url, - "UV_INDEX_URL": index_url, - } - - -def default_job_name(model: str, dataset: str) -> str: - safe_model = re.sub(r"[^A-Za-z0-9._-]+", "-", model).strip("-") - safe_dataset = re.sub(r"[^A-Za-z0-9._-]+", "-", dataset).strip("-") - timestamp = datetime.now().strftime("%Y-%m-%d__%H-%M-%S") - return f"goose-{safe_dataset}-{safe_model}-{timestamp}" - - -def validate_job_name(job_name: str) -> str: - if not re.match(r"^[A-Za-z0-9][A-Za-z0-9._-]*$", job_name): - raise ValueError( - "Job name must start with a letter or number and contain only " - "letters, numbers, dots, underscores, and hyphens" - ) - return job_name - - -def build_harbor_config(args: argparse.Namespace) -> dict[str, Any]: - goose_binary = args.goose_binary.expanduser().resolve() - goose_profile = args.goose_profile.expanduser().resolve() - - if "/" not in args.model: - raise ValueError( - "Model must be in provider/model form, for example databricks/my-model" - ) - if args.trials < 1: - raise ValueError("--trials must be at least 1") - if args.concurrency < 1: - raise ValueError("--concurrency must be at least 1") - if not goose_binary.is_file(): - raise ValueError( - f"--goose-binary does not exist or is not a file: {args.goose_binary}" - ) - if not goose_profile.is_dir(): - raise ValueError( - "--goose-profile does not exist or is not a directory: " - f"{args.goose_profile}" - ) - - agent_kwargs: dict[str, Any] = { - "goose_binary": str(goose_binary), - "goose_profile": str(goose_profile), - } - if args.install_goose_runtime_deps: - agent_kwargs["install_goose_runtime_deps"] = True - if args.max_turns is not None: - agent_kwargs["max_turns"] = args.max_turns - - index_env = package_index_env() - job_name = ( - validate_job_name(args.job_name) - if args.job_name - else default_job_name(args.model, args.dataset) - ) - - return { - "job_name": job_name, - "jobs_dir": str(args.jobs_dir.expanduser()), - "n_attempts": args.trials, - "n_concurrent_trials": args.concurrency, - "environment": { - "type": "docker", - "force_build": args.force_build, - "delete": True, - "env": index_env, - }, - "verifier": {"env": index_env}, - "agents": [ - { - "import_path": HARBOR_AGENT_IMPORT_PATH, - "model_name": args.model, - "kwargs": agent_kwargs, - } - ], - "datasets": [dataset_config(args.dataset, args.tasks)], - } - - -def run_harbor(command: list[str]) -> int: - env = os.environ.copy() - env["PYTHONPATH"] = pythonpath_with_harbor() - completed = subprocess.run(command, env=env, check=False) - return completed.returncode - - -def main(argv: list[str] | None = None) -> int: - parser = build_parser() - args = parser.parse_args(argv) - - try: - config = build_harbor_config(args) - config_dir = args.config_dir.expanduser() - config_dir.mkdir(parents=True, exist_ok=True) - config_path = config_dir / f"{config['job_name']}.json" - config_path.write_text(json.dumps(config, indent=2) + "\n") - command = ["harbor", "run", "-c", str(config_path)] - except Exception as error: - print(f"error: {error}", file=sys.stderr) - return 2 - - print(f"Wrote Harbor config: {config_path}") - print(f"Jobs directory: {config['jobs_dir']}") - print(f"PYTHONPATH: {pythonpath_with_harbor()}") - print(f"Command: {' '.join(command)}") - - if args.dry_run: - return 0 - - try: - return run_harbor(command) - except FileNotFoundError: - print("error: `harbor` was not found on PATH", file=sys.stderr) - return 127 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/evals/harbor/pyproject.toml b/evals/harbor/pyproject.toml deleted file mode 100644 index 13a986d5..00000000 --- a/evals/harbor/pyproject.toml +++ /dev/null @@ -1,13 +0,0 @@ -[project] -name = "goose-harbor-eval" -version = "0.1.0" -description = "Goose eval tooling for Harbor benchmark datasets" -requires-python = ">=3.12" -dependencies = [ - "harbor==0.6.4", -] - -[dependency-groups] -dev = [ - "pytest>=8.4.0", -] diff --git a/evals/harbor/reporter.py b/evals/harbor/reporter.py new file mode 100644 index 00000000..5013621b --- /dev/null +++ b/evals/harbor/reporter.py @@ -0,0 +1,532 @@ +"""Load harbor 0.8.0 job/trial results and render list/show/task/compare reports.""" + +from __future__ import annotations + +import argparse +import json +import shutil +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +from harbor.models.job.result import JobResult +from harbor.models.trial.result import TrialResult + + +RUNS_DIR = Path(__file__).resolve().parent / "runs" + + +@dataclass +class LoadedJob: + summary: JobResult + results: list[TrialResult] + job_dir: Path + + @property + def job_name(self) -> str: + return self.job_dir.name + + @property + def started_at(self): + return self.summary.started_at + + +def load_job(job_dir: Path) -> LoadedJob: + summary = JobResult.model_validate_json((job_dir / "result.json").read_text()) + results: list[TrialResult] = [] + for child in sorted(job_dir.iterdir()): + if not child.is_dir(): + continue + trial_result = child / "result.json" + if not trial_result.is_file(): + continue + results.append(TrialResult.model_validate_json(trial_result.read_text())) + return LoadedJob(summary=summary, results=results, job_dir=job_dir) + + +def trial_reward(trial: TrialResult) -> float | None: + if trial.verifier_result is None or not trial.verifier_result.rewards: + return None + rewards = trial.verifier_result.rewards + value = rewards.get("reward", next(iter(rewards.values()))) + return float(value) + + +def trial_error(trial: TrialResult) -> tuple[str, str] | None: + if trial.exception_info is None: + return None + return trial.exception_info.exception_type, trial.exception_info.exception_message + + +def trial_duration(trial: TrialResult) -> float | None: + if trial.started_at is None or trial.finished_at is None: + return None + return (trial.finished_at - trial.started_at).total_seconds() + + +def trial_token_totals(trial: TrialResult) -> tuple[int | None, int | None, float | None]: + n_in, _n_cache, n_out, cost = trial.compute_token_cost_totals() + return n_in, n_out, cost + + +def _trial_dir(trial: TrialResult, job_dir: Path) -> Path: + return job_dir / trial.trial_name + + +def trial_turns(trial: TrialResult, job_dir: Path) -> int | None: + """Number of agent turns in a trial. + + Preferred source is ``agent/trajectory.json`` (harbor's standard format, + one entry per agent step). Falls back to parsing harness-specific logs + when the trajectory isn't present: + + * goose stream-json: count messages with role=assistant + * pi log: count "turn_start" events + """ + trial_dir = _trial_dir(trial, job_dir) + trajectory = trial_dir / "agent" / "trajectory.json" + if trajectory.is_file(): + try: + data = json.loads(trajectory.read_text()) + except json.JSONDecodeError: + data = None + steps = data.get("steps") if isinstance(data, dict) else None + if isinstance(steps, list): + return sum(1 for s in steps if isinstance(s, dict) and s.get("source") == "agent") + + goose_log = trial_dir / "agent" / "goose.txt" + if goose_log.is_file(): + # stream-json emits one `message` event per streamed chunk (sharing the + # same message.id for a single assistant turn). Dedupe by id so a turn + # that streamed 2000 tokens counts as 1, not 2000. + seen_ids: set[str] = set() + anon_chunks = 0 + for line in goose_log.read_text(errors="replace").splitlines(): + line = line.strip() + if not line.startswith("{"): + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + if obj.get("type") != "message": + continue + msg = obj.get("message", {}) + if msg.get("role") != "assistant": + continue + mid = msg.get("id") + if mid: + seen_ids.add(mid) + else: + anon_chunks += 1 + count = len(seen_ids) + anon_chunks + return count if count else None + + pi_log = trial_dir / "agent" / "pi.txt" + if pi_log.is_file(): + count = 0 + for line in pi_log.read_text(errors="replace").splitlines(): + line = line.strip() + if not line.startswith("{"): + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + if obj.get("type") == "turn_start": + count += 1 + return count if count else None + + return None + + +def job_turn_totals(job: LoadedJob) -> int: + return sum((trial_turns(t, job.job_dir) or 0) for t in job.results) + + +def job_token_totals(job: LoadedJob) -> tuple[int, int, float]: + totals = [trial_token_totals(t) for t in job.results] + return ( + sum((n_in or 0) for n_in, _, _ in totals), + sum((n_out or 0) for _, n_out, _ in totals), + sum((c or 0.0) for _, _, c in totals), + ) + + +def trial_status(trial: TrialResult) -> str: + """Classify a trial as pass / partial / fail / timeout / error / no-reward. + + Reward wins over exception_info: harbor can record an AgentTimeoutError or + other post-run exception even when the verifier already scored the trial as + a pass (e.g. the agent finished the work then the harness crashed during + teardown, or the agent timed out after writing the correct answer). If we + got points, we got points — count them. + """ + reward = trial_reward(trial) + if reward is not None and reward > 0: + return "pass" if reward >= 1.0 else "partial" + err = trial_error(trial) + if err is not None: + error_type, _ = err + if "timeout" in error_type.lower(): + return "timeout" + return "error" + if reward is None: + return "no-reward" + return "fail" + + +def job_duration(job: LoadedJob) -> float | None: + """Total trial time, summed across all trials. + + This unrolls parallelism: a 4-hour run with 4 concurrent workers reports + ~16h. We deliberately don't use elapsed job wall clock (min start → max + finish) because that conflates "how long the benchmark took" with "how + much concurrency I had on the host", making cross-run comparisons noisy. + The sum is a stable measure of total compute. + """ + durations = [d for d in (trial_duration(t) for t in job.results) if d is not None] + return sum(durations) if durations else None + + +def job_model(job: LoadedJob) -> str: + for trial in job.results: + info = trial.agent_info + if info and info.model_info and info.model_info.name: + return info.model_info.name.rsplit("/", 1)[-1] + return "?" + + +def task_name(trial: TrialResult) -> str: + return trial.task_id.get_name() + + +def fmt_duration(sec: float | None) -> str: + if sec is None: + return "-" + if sec < 60: + return f"{sec:.0f}s" + if sec < 3600: + return f"{sec / 60:.1f}m" + return f"{sec / 3600:.1f}h" + + +def fmt_tokens(n: int | None) -> str: + if n is None or n == 0: + return "-" + if n >= 1_000_000: + return f"{n / 1_000_000:.1f}M" + if n >= 1_000: + return f"{n / 1_000:.0f}k" + return str(n) + + +def fmt_cost(usd: float | None) -> str: + if usd is None or usd == 0: + return "-" + return f"${usd:.2f}" + + +def status_counts(trials: list[TrialResult]) -> dict[str, int]: + counts = {"pass": 0, "partial": 0, "fail": 0, "timeout": 0, "error": 0, "no-reward": 0} + for trial in trials: + counts[trial_status(trial)] += 1 + return counts + + +def cmd_list(args: argparse.Namespace) -> int: + if not RUNS_DIR.is_dir(): + print(f"No runs directory at {RUNS_DIR}", file=sys.stderr) + return 1 + + rows = [] + for child in sorted(RUNS_DIR.iterdir()): + if not child.is_dir(): + continue + if not (child / "result.json").is_file(): + continue + job = load_job(child) + counts = status_counts(job.results) + total = len(job.results) + rate = f"{100 * counts['pass'] / total:.1f}%" if total else "-" + tok_in, tok_out, cost = job_token_totals(job) + breakdown = f"{counts['pass']}/{counts['fail']}/{counts['error']}/{counts['timeout']}" + rows.append( + ( + child.name, + job_model(job), + rate, + fmt_duration(job_duration(job)), + fmt_tokens(tok_in), + fmt_tokens(tok_out), + fmt_tokens(job_turn_totals(job)), + fmt_cost(cost), + breakdown, + ) + ) + + if not rows: + print(f"No jobs found in {RUNS_DIR}") + return 0 + print( + f"{'job_name':<40} {'model':<25} {'rate':>7} {'compute':>8} " + f"{'in':>7} {'out':>7} {'turns':>6} {'cost':>8} {'pass/fail/err/tout':>18}" + ) + print("-" * 131) + for row in rows: + print( + f"{row[0]:<40} {row[1]:<25} {row[2]:>7} {row[3]:>8} " + f"{row[4]:>7} {row[5]:>7} {row[6]:>6} {row[7]:>8} {row[8]:>18}" + ) + return 0 + + +def cmd_show(args: argparse.Namespace) -> int: + job = load_job(RUNS_DIR / args.job_name) + counts = status_counts(job.results) + total = len(job.results) + + print(f"Job: {job.job_name}") + print(f"Model: {job_model(job)}") + print(f"Started: {job.started_at}") + print(f"Compute time: {fmt_duration(job_duration(job))} (sum of trial durations)") + print(f"Trials: {total}") + print( + f" pass={counts['pass']} partial={counts['partial']} fail={counts['fail']} " + f"timeout={counts['timeout']} error={counts['error']} no-reward={counts['no-reward']}" + ) + if total: + print(f"Pass rate: {100 * counts['pass'] / total:.1f}%") + total_in, total_out, total_cost = job_token_totals(job) + print(f"Tokens: in={fmt_tokens(total_in)} out={fmt_tokens(total_out)}") + print(f"Turns: {fmt_tokens(job_turn_totals(job))}") + print(f"Cost: {fmt_cost(total_cost)}") + print() + print( + f"{'task':<45} {'status':<10} {'reward':>7} {'dur':>7} " + f"{'in':>7} {'out':>7} {'turns':>6} {'cost':>7} error" + ) + print("-" * 137) + for trial in sorted(job.results, key=task_name): + status = trial_status(trial) + if args.status and status != args.status: + continue + reward = trial_reward(trial) + reward_str = f"{reward:.2f}" if reward is not None else "-" + error = trial_error(trial) + if error is not None: + exception_class, message = error + msg_first_line = (message or "").splitlines()[0] if message else "" + err_str = f"{exception_class}: {msg_first_line}" if msg_first_line else exception_class + else: + err_str = "" + if len(err_str) > 50: + err_str = err_str[:47] + "..." + n_in, n_out, cost = trial_token_totals(trial) + turns = trial_turns(trial, job.job_dir) + turns_str = str(turns) if turns is not None else "-" + print( + f"{task_name(trial):<45} {status:<10} {reward_str:>7} " + f"{fmt_duration(trial_duration(trial)):>7} " + f"{fmt_tokens(n_in):>7} " + f"{fmt_tokens(n_out):>7} " + f"{turns_str:>6} " + f"{fmt_cost(cost):>7} {err_str}" + ) + return 0 + + +def cmd_task(args: argparse.Namespace) -> int: + job_dir = RUNS_DIR / args.job_name + job = load_job(job_dir) + matches = [t for t in job.results if task_name(t) == args.task_name] + if not matches: + names = sorted({task_name(t) for t in job.results}) + print(f"No task '{args.task_name}' in {args.job_name}.", file=sys.stderr) + print(f"Available: {', '.join(names[:10])}{'...' if len(names) > 10 else ''}", file=sys.stderr) + return 1 + + for trial in matches: + print(f"=== {trial.trial_name} ===") + print(f"Status: {trial_status(trial)}") + print(f"Reward: {trial_reward(trial)}") + print(f"Duration: {fmt_duration(trial_duration(trial))}") + print(f"Started: {trial.started_at}") + print(f"Ended: {trial.finished_at}") + n_in, n_out, cost = trial_token_totals(trial) + print(f"Tokens: in={fmt_tokens(n_in)} out={fmt_tokens(n_out)}") + turns = trial_turns(trial, job_dir) + print(f"Turns: {turns if turns is not None else '-'}") + print(f"Cost: {fmt_cost(cost)}") + error = trial_error(trial) + if error is not None: + exception_class, message = error + print(f"Error class: {exception_class}") + for line in (message or "").splitlines()[:10]: + print(f" {line}") + if trial.verifier_result and trial.verifier_result.rewards: + rewards_str = ", ".join(f"{k}={v}" for k, v in trial.verifier_result.rewards.items()) + print(f"Verifier: {rewards_str}") + + trial_dir = job_dir / trial.trial_name + if trial_dir.is_dir(): + stdout_file = trial_dir / "verifier" / "test-stdout.txt" + if stdout_file.is_file(): + lines = stdout_file.read_text(errors="replace").splitlines() + if lines: + print(" verifier output (last 15 lines):") + for line in lines[-15:]: + print(f" {line}") + print(f"\nArtifacts in: {trial_dir}") + agent_log = trial_dir / "agent" / "goose.txt" + if not agent_log.is_file(): + agent_log = trial_dir / "agent" / "pi.txt" + if agent_log.is_file(): + size = agent_log.stat().st_size + print(f" agent log: {agent_log.name} ({size:,} bytes)") + if args.tail and size: + print(f"\n--- last {args.tail} lines of {agent_log.name} ---") + lines = agent_log.read_text(errors="replace").splitlines() + for line in lines[-args.tail:]: + print(line) + print() + return 0 + + +def cmd_rm(args: argparse.Namespace) -> int: + runs_dir = RUNS_DIR.resolve() + targets: list[Path] = [] + for name in args.job_names: + target = (RUNS_DIR / name).resolve() + if runs_dir not in target.parents: + print(f"refusing to remove path outside runs dir: {name}", file=sys.stderr) + return 2 + if not target.is_dir(): + print(f"not a run directory: {target}", file=sys.stderr) + return 1 + targets.append(target) + + for target in targets: + size_kb = sum(p.stat().st_size for p in target.rglob("*") if p.is_file()) // 1024 + print(f" {target.name} ({size_kb:,} KB)") + + if not args.yes: + prompt = f"Remove {len(targets)} run{'s' if len(targets) > 1 else ''}? [y/N] " + if input(prompt).strip().lower() not in ("y", "yes"): + print("aborted") + return 1 + + for target in targets: + shutil.rmtree(target) + print(f"removed {target.name}") + return 0 + + +def cmd_pull(args: argparse.Namespace) -> int: + """Rsync runs from a remote into the local runs directory. + + ``remote`` should be ``user@host:/path/to/goose`` — we append + ``/evals/harbor/runs/`` and pull into our own runs/. + """ + remote = args.remote.rstrip("/") + if ":" not in remote: + print("remote must include host:path, e.g. tbench@douwe.com:/home/tbench/work/goose", file=sys.stderr) + return 2 + source = f"{remote}/evals/harbor/runs/" + RUNS_DIR.mkdir(parents=True, exist_ok=True) + cmd = ["rsync", "-az", "--stats"] + if args.delete: + cmd.append("--delete") + if args.jobs: + for name in args.jobs: + cmd.extend(["--include", f"{name}/", "--include", f"{name}/**"]) + cmd.extend(["--exclude", "*"]) + cmd.extend([source, str(RUNS_DIR) + "/"]) + print(" ".join(cmd)) + return subprocess.run(cmd, check=False).returncode + + +def cmd_compare(args: argparse.Namespace) -> int: + job_a = load_job(RUNS_DIR / args.job_a) + job_b = load_job(RUNS_DIR / args.job_b) + a_by_task = {task_name(t): t for t in job_a.results} + b_by_task = {task_name(t): t for t in job_b.results} + only_a = sorted(set(a_by_task) - set(b_by_task)) + only_b = sorted(set(b_by_task) - set(a_by_task)) + common = sorted(set(a_by_task) & set(b_by_task)) + + ca = status_counts(job_a.results) + cb = status_counts(job_b.results) + na, nb = len(job_a.results), len(job_b.results) + + print(f"A: {args.job_a} ({job_model(job_a)})") + print(f"B: {args.job_b} ({job_model(job_b)})") + print() + print(f"{'metric':<18} {'A':>10} {'B':>10} {'diff':>8}") + print("-" * 50) + + def row(label: str, a: float | int, b: float | int, fmt: str = "{:.0f}") -> None: + diff = b - a + diff_fmt = fmt.replace("{:", "{:+", 1) + print(f"{label:<18} {fmt.format(a):>10} {fmt.format(b):>10} {diff_fmt.format(diff):>8}") + + row("trials", na, nb) + row("pass", ca["pass"], cb["pass"]) + row("partial", ca["partial"], cb["partial"]) + row("fail", ca["fail"], cb["fail"]) + row("timeout", ca["timeout"], cb["timeout"]) + row("error", ca["error"], cb["error"]) + if na and nb: + row("pass rate %", 100 * ca["pass"] / na, 100 * cb["pass"] / nb, "{:.1f}") + + a_in, a_out, a_cost = job_token_totals(job_a) + b_in, b_out, b_cost = job_token_totals(job_b) + print(f"{'tokens in':<18} {fmt_tokens(a_in):>10} {fmt_tokens(b_in):>10}") + print(f"{'tokens out':<18} {fmt_tokens(a_out):>10} {fmt_tokens(b_out):>10}") + print(f"{'turns':<18} {fmt_tokens(job_turn_totals(job_a)):>10} " + f"{fmt_tokens(job_turn_totals(job_b)):>10}") + print(f"{'cost':<18} {fmt_cost(a_cost):>10} {fmt_cost(b_cost):>10}") + print(f"{'compute time':<18} {fmt_duration(job_duration(job_a)):>10} " + f"{fmt_duration(job_duration(job_b)):>10}") + + if only_a or only_b: + print() + if only_a: + print(f"Only in A ({len(only_a)}): {', '.join(only_a)}") + if only_b: + print(f"Only in B ({len(only_b)}): {', '.join(only_b)}") + + transitions: dict[tuple[str, str], list[str]] = {} + for name in common: + sa = trial_status(a_by_task[name]) + sb = trial_status(b_by_task[name]) + transitions.setdefault((sa, sb), []).append(name) + + same_pass = transitions.get(("pass", "pass"), []) + same_not = [ + name + for (sa, sb), names in transitions.items() + if sa != "pass" and sb != "pass" + for name in names + ] + a_only = [n for (sa, sb), ns in transitions.items() if sa == "pass" and sb != "pass" for n in ns] + b_only = [n for (sa, sb), ns in transitions.items() if sa != "pass" and sb == "pass" for n in ns] + + print() + print(f"Task-level comparison ({len(common)} shared tasks):") + print(f" both pass: {len(same_pass)}") + print(f" both not-pass: {len(same_not)}") + print(f" only A passes: {len(a_only)}") + print(f" only B passes: {len(b_only)}") + + if args.verbose: + if a_only: + print(f"\nOnly A ({args.job_a}) solved:") + for name in sorted(a_only): + print(f" {name:<40} B={trial_status(b_by_task[name])}") + if b_only: + print(f"\nOnly B ({args.job_b}) solved:") + for name in sorted(b_only): + print(f" {name:<40} A={trial_status(a_by_task[name])}") + return 0 diff --git a/evals/harbor/run b/evals/harbor/run deleted file mode 100755 index 02572a4c..00000000 --- a/evals/harbor/run +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env sh -set -eu - -SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) - -if [ -n "${PYTHONPATH:-}" ]; then - export PYTHONPATH="$SCRIPT_DIR:$PYTHONPATH" -else - export PYTHONPATH="$SCRIPT_DIR" -fi - -exec python3 "$SCRIPT_DIR/goose_harbor/runner.py" "$@" diff --git a/evals/harbor/runner.py b/evals/harbor/runner.py new file mode 100644 index 00000000..7603e094 --- /dev/null +++ b/evals/harbor/runner.py @@ -0,0 +1,225 @@ +"""Build the harbor config and launch a benchmark job.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +from datetime import datetime +from pathlib import Path +from typing import Any + +import yaml + +from agent import PROVIDER_SECRETS + + +HARBOR_DIR = Path(__file__).resolve().parent +RUNS_DIR = HARBOR_DIR / "runs" +CONFIG_TEMPLATE_PATH = HARBOR_DIR / "config_template.yaml" + +DEFAULT_DATASET = "terminal-bench/terminal-bench-2" +DEFAULT_MODEL = "anthropic/claude-sonnet-4-6" +DEFAULT_EXTENSIONS = ["developer", "todo"] +DEFAULT_CONCURRENCY = 4 +DEFAULT_MAX_TURNS = 100 + + +def find_dotenv() -> Path | None: + cwd_env = Path.cwd() / ".env" + if cwd_env.is_file(): + return cwd_env + script_env = HARBOR_DIR / ".env" + if script_env.is_file(): + return script_env + return None + + +def load_dotenv() -> None: + env_path = find_dotenv() + if env_path is None: + return + for line in env_path.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, value = line.partition("=") + key = key.strip() + value = value.strip().strip('"').strip("'") + os.environ.setdefault(key, value) + + +def render_goose_config(extensions: list[str]) -> tuple[str, list[dict[str, str]]]: + """Render config.yaml from the template, enabling the given extensions. + + Returns (config_yaml_text, recipe_extension_entries). + Raises ValueError for any extension not found in the template. + """ + if not CONFIG_TEMPLATE_PATH.is_file(): + raise FileNotFoundError(f"Missing template: {CONFIG_TEMPLATE_PATH}") + template = yaml.safe_load(CONFIG_TEMPLATE_PATH.read_text()) + available = template.get("extensions") or {} + + unknown = [name for name in extensions if name not in available] + if unknown: + raise ValueError( + f"Unknown extensions: {', '.join(unknown)}. " + f"Known: {', '.join(sorted(available))}" + ) + + for name, entry in available.items(): + entry["enabled"] = name in extensions + + recipe_entries = [ + {"type": available[name]["type"], "name": name} for name in extensions + ] + return yaml.dump(template, sort_keys=False), recipe_entries + + +def default_job_name(model: str, dataset: str) -> str: + safe_model = re.sub(r"[^A-Za-z0-9._-]+", "-", model).strip("-") + safe_dataset = re.sub(r"[^A-Za-z0-9._-]+", "-", dataset).strip("-") + timestamp = datetime.now().strftime("%Y-%m-%d__%H-%M-%S") + return f"goose-{safe_dataset}-{safe_model}-{timestamp}" + + +def validate_job_name(job_name: str) -> str: + if not re.match(r"^[A-Za-z0-9][A-Za-z0-9._-]*$", job_name): + raise ValueError( + "Job name must start with a letter or number and contain only " + "letters, numbers, dots, underscores, and hyphens" + ) + return job_name + + +def parse_csv(value: str) -> list[str]: + return [item.strip() for item in value.split(",") if item.strip()] + + +PACKAGE_INDEX_ENV_VARS = ("UV_DEFAULT_INDEX", "PIP_INDEX_URL", "UV_INDEX_URL") + + +def package_index_env() -> dict[str, str]: + index_url = next( + (os.environ[key] for key in PACKAGE_INDEX_ENV_VARS if os.environ.get(key)), + None, + ) + if index_url is None: + return {} + return {key: index_url for key in PACKAGE_INDEX_ENV_VARS} + + +def dataset_config(dataset_ref: str, tasks: list[str]) -> dict[str, Any]: + name, sep, ref = dataset_ref.rpartition("@") + dataset_name = name if sep else dataset_ref + dataset: dict[str, Any] = {"name": dataset_name} + if sep: + dataset["ref" if "/" in name else "version"] = ref + if tasks: + dataset["task_names"] = tasks + return dataset + + +def build_harbor_config(args: argparse.Namespace) -> dict[str, Any]: + if "/" not in args.model: + raise ValueError("--model must be in provider/model form, e.g. anthropic/claude-sonnet-4-6") + if args.trials < 1: + raise ValueError("--trials must be at least 1") + if args.concurrency < 1: + raise ValueError("--concurrency must be at least 1") + if args.timeout_multiplier <= 0: + raise ValueError("--timeout-multiplier must be positive") + + goose_binary = args.goose_binary.expanduser().resolve() + if not goose_binary.is_file(): + raise ValueError(f"--goose-binary does not exist or is not a file: {args.goose_binary}") + + config_yaml, extension_entries = render_goose_config(args.extensions) + + provider = args.model.split("/", 1)[0] + missing_secrets = [ + key for key in PROVIDER_SECRETS.get(provider, []) if not os.environ.get(key) + ] + if missing_secrets: + raise ValueError( + f"Missing env vars for provider '{provider}': {', '.join(missing_secrets)}. " + f"Set them in a .env file (cwd or {HARBOR_DIR}) or your shell." + ) + + agent_kwargs: dict[str, Any] = { + "goose_binary": str(goose_binary), + "config_yaml": config_yaml, + "extension_entries": extension_entries, + "install_goose_runtime_deps": args.install_goose_runtime_deps, + } + if args.max_turns is not None: + agent_kwargs["max_turns"] = args.max_turns + + job_name = ( + validate_job_name(args.job_name) + if args.job_name + else default_job_name(args.model, args.dataset) + ) + + index_env = package_index_env() + container_env_passthrough = [ + f"{key}=${{{key}}}" + for key in PROVIDER_SECRETS.get(provider, []) + if os.environ.get(key) + ] + [f"{key}={value}" for key, value in index_env.items()] + + config: dict[str, Any] = { + "job_name": job_name, + "jobs_dir": str(RUNS_DIR), + "n_attempts": args.trials, + "n_concurrent_trials": args.concurrency, + "environment": { + "type": "docker", + "force_build": False, + "delete": True, + "env": container_env_passthrough, + }, + "agents": [ + { + "import_path": "agent:GooseBinaryAgent", + "model_name": args.model, + "kwargs": agent_kwargs, + } + ], + "datasets": [dataset_config(args.dataset, args.tasks)], + } + if index_env: + config["verifier"] = {"env": index_env} + if args.timeout_multiplier != 1.0: + config["timeout_multiplier"] = args.timeout_multiplier + return config + + +def cmd_run(args: argparse.Namespace) -> int: + load_dotenv() + try: + config = build_harbor_config(args) + except Exception as error: + print(f"error: {error}", file=sys.stderr) + return 2 + + RUNS_DIR.mkdir(parents=True, exist_ok=True) + job_dir = RUNS_DIR / config["job_name"] + job_dir.mkdir(parents=True, exist_ok=True) + config_path = job_dir / "_generated_config.json" + config_path.write_text(json.dumps(config, indent=2) + "\n") + + command = ["harbor", "run", "-c", str(config_path)] + print(f"Job: {config['job_name']}") + print(f"Config: {config_path}") + print(f"Runs: {RUNS_DIR}") + if args.dry_run: + return 0 + + env = os.environ.copy() + env["PYTHONPATH"] = f"{HARBOR_DIR}{os.pathsep}{env.get('PYTHONPATH', '')}".rstrip(os.pathsep) + completed = subprocess.run(command, env=env, check=False) + return completed.returncode diff --git a/evals/harbor/tests/conftest.py b/evals/harbor/tests/conftest.py deleted file mode 100644 index c06546d7..00000000 --- a/evals/harbor/tests/conftest.py +++ /dev/null @@ -1,7 +0,0 @@ -from __future__ import annotations - -import sys -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT)) diff --git a/evals/harbor/tests/test_goose_binary.py b/evals/harbor/tests/test_goose_binary.py deleted file mode 100644 index 316b1fca..00000000 --- a/evals/harbor/tests/test_goose_binary.py +++ /dev/null @@ -1,315 +0,0 @@ -from __future__ import annotations - -import asyncio -from pathlib import Path - -import pytest - -from goose_harbor.goose_binary import GooseBinaryAgent -from goose_harbor.goose_binary import CONTAINER_CA_BUNDLE_PATH -from goose_harbor.goose_binary import CONTAINER_RECIPE_PATH -from goose_harbor.goose_binary import CONTAINER_GOOSE_PATH_ROOT - - -class ExecResult: - def __init__(self, stdout: str = "goose 1.0.0") -> None: - self.return_code = 0 - self.stdout = stdout - self.stderr = "" - - -class FakeEnvironment: - def __init__(self) -> None: - self.uploads: list[tuple[Path, str]] = [] - self.dir_uploads: list[tuple[Path, str]] = [] - self.commands: list[dict[str, object]] = [] - self.default_user: str | int | None = None - self.has_system_ca = True - - async def upload_file(self, source_path: Path | str, target_path: str) -> None: - self.uploads.append((Path(source_path), target_path)) - - async def upload_dir(self, source_dir: Path | str, target_dir: str) -> None: - self.dir_uploads.append((Path(source_dir), target_dir)) - - async def exec( - self, - command: str, - cwd: str | None = None, - env: dict[str, str] | None = None, - timeout_sec: int | None = None, - user: str | int | None = None, - ) -> ExecResult: - self.commands.append( - { - "command": command, - "cwd": cwd, - "env": env, - "timeout_sec": timeout_sec, - "user": user, - } - ) - if "id -u && id -g" in command: - return ExecResult("1000\n1000\n") - if "ca-certificates.crt" in command and "echo present" in command: - return ExecResult("present\n" if self.has_system_ca else "missing\n") - return ExecResult() - - -@pytest.fixture -def goose_binary(tmp_path: Path) -> Path: - path = tmp_path / "goose" - path.write_text("#!/bin/sh\n") - return path - - -@pytest.fixture -def goose_profile(tmp_path: Path) -> Path: - path = tmp_path / "profile" - (path / "config").mkdir(parents=True) - (path / "config" / "config.yaml").write_text("GOOSE_PROVIDER: databricks\n") - return path - - -def test_install_uploads_binary_and_profile( - goose_binary: Path, - goose_profile: Path, - tmp_path: Path, -) -> None: - async def run_test() -> FakeEnvironment: - agent = GooseBinaryAgent( - logs_dir=tmp_path, - model_name="databricks/model", - goose_binary=str(goose_binary), - goose_profile=str(goose_profile), - ) - environment = FakeEnvironment() - - await agent.install(environment) - return environment - - environment = asyncio.run(run_test()) - - assert environment.uploads == [(goose_binary.resolve(), "/installed-agent/goose")] - commands = "\n".join(str(item["command"]) for item in environment.commands) - assert "chmod 755 /installed-agent/goose" in commands - assert "ln -sf /installed-agent/goose ~/.local/bin/goose" in commands - assert environment.dir_uploads == [(goose_profile.resolve(), "/installed-agent/goose-profile")] - - -def test_install_uploads_config_directory_profile( - goose_binary: Path, - tmp_path: Path, -) -> None: - async def run_test() -> FakeEnvironment: - config_dir = tmp_path / "config" - config_dir.mkdir() - (config_dir / "config.yaml").write_text("GOOSE_PROVIDER: databricks\n") - agent = GooseBinaryAgent( - logs_dir=tmp_path, - model_name="databricks/model", - goose_binary=str(goose_binary), - goose_profile=str(config_dir), - ) - environment = FakeEnvironment() - - await agent.install(environment) - return environment - - environment = asyncio.run(run_test()) - - assert environment.dir_uploads == [(tmp_path / "config", "/installed-agent/goose-profile/config")] - - -def test_install_chowns_uploaded_profile_when_agent_user_is_image_default( - goose_binary: Path, - goose_profile: Path, - tmp_path: Path, -) -> None: - async def run_test() -> FakeEnvironment: - agent = GooseBinaryAgent( - logs_dir=tmp_path, - model_name="databricks/model", - goose_binary=str(goose_binary), - goose_profile=str(goose_profile), - ) - environment = FakeEnvironment() - - await agent.install(environment) - return environment - - environment = asyncio.run(run_test()) - - commands = [str(item["command"]) for item in environment.commands] - assert any("id -u && id -g" in command for command in commands) - assert any( - "chown -R 1000:1000 /installed-agent/goose-profile" in command - for command in commands - ) - - -def test_install_can_install_goose_runtime_deps( - goose_binary: Path, - goose_profile: Path, - tmp_path: Path, -) -> None: - async def run_test() -> FakeEnvironment: - agent = GooseBinaryAgent( - logs_dir=tmp_path, - model_name="databricks/model", - goose_binary=str(goose_binary), - goose_profile=str(goose_profile), - install_goose_runtime_deps=True, - ) - environment = FakeEnvironment() - - await agent.install(environment) - return environment - - environment = asyncio.run(run_test()) - - commands = [str(item["command"]) for item in environment.commands] - assert any("apt-get install -y libgomp1" in command for command in commands) - - -def test_missing_container_ca_bundle_is_uploaded_and_used( - goose_binary: Path, - goose_profile: Path, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - async def run_test() -> FakeEnvironment: - host_ca_bundle = tmp_path / "cert.pem" - host_ca_bundle.write_text("test cert\n") - monkeypatch.setenv("SSL_CERT_FILE", str(host_ca_bundle)) - agent = GooseBinaryAgent( - logs_dir=tmp_path, - model_name="databricks/model", - goose_binary=str(goose_binary), - goose_profile=str(goose_profile), - ) - environment = FakeEnvironment() - environment.has_system_ca = False - - await agent.install(environment) - await agent.run("fix the repo", environment, object()) - return environment - - environment = asyncio.run(run_test()) - - assert any(target == CONTAINER_CA_BUNDLE_PATH for _, target in environment.uploads) - assert environment.commands[-1]["env"]["SSL_CERT_FILE"] == CONTAINER_CA_BUNDLE_PATH - - -def test_run_uses_profile_without_keyring_or_provider_env_forwarding( - goose_binary: Path, - tmp_path: Path, -) -> None: - async def run_test() -> FakeEnvironment: - profile_root = tmp_path / "profile" - (profile_root / "config").mkdir(parents=True) - (profile_root / "config" / "config.yaml").write_text("GOOSE_PROVIDER: databricks\n") - agent = GooseBinaryAgent( - logs_dir=tmp_path, - model_name="databricks/model", - goose_binary=str(goose_binary), - goose_profile=str(profile_root), - ) - environment = FakeEnvironment() - - await agent.run("fix the repo", environment, object()) - return environment - - environment = asyncio.run(run_test()) - - run_command = environment.commands[-1] - env = run_command["env"] - assert isinstance(env, dict) - assert env["GOOSE_PATH_ROOT"] == "/installed-agent/goose-profile" - assert env["GOOSE_DISABLE_KEYRING"] == "true" - assert "DATABRICKS_TOKEN" not in env - - -def test_run_uploads_recipe_file_instead_of_heredoc( - goose_binary: Path, - goose_profile: Path, - tmp_path: Path, -) -> None: - async def run_test() -> FakeEnvironment: - agent = GooseBinaryAgent( - logs_dir=tmp_path, - model_name="databricks/model", - goose_binary=str(goose_binary), - goose_profile=str(goose_profile), - ) - environment = FakeEnvironment() - - await agent.run("line before\nEOF\nline after", environment, object()) - return environment - - environment = asyncio.run(run_test()) - - commands = [str(item["command"]) for item in environment.commands] - assert all("<< 'EOF'" not in command for command in commands) - assert any(target == CONTAINER_RECIPE_PATH for _, target in environment.uploads) - assert any( - f"goose run --recipe {CONTAINER_RECIPE_PATH}" in command - for command in commands - ) - - -def test_run_copies_skills_into_isolated_profile( - goose_binary: Path, - goose_profile: Path, - tmp_path: Path, -) -> None: - async def run_test() -> FakeEnvironment: - skills_dir = tmp_path / "skills" - skills_dir.mkdir() - agent = GooseBinaryAgent( - logs_dir=tmp_path, - model_name="databricks/model", - goose_binary=str(goose_binary), - goose_profile=str(goose_profile), - skills_dir=str(skills_dir), - ) - environment = FakeEnvironment() - - await agent.run("fix the repo", environment, object()) - return environment - - environment = asyncio.run(run_test()) - - commands = [str(item["command"]) for item in environment.commands] - assert any( - f"{CONTAINER_GOOSE_PATH_ROOT}/config/skills" in command - and "~/.config/goose/skills" not in command - for command in commands - ) - - -def test_run_chowns_uploaded_recipe_for_image_default_agent_user( - goose_binary: Path, - goose_profile: Path, - tmp_path: Path, -) -> None: - async def run_test() -> FakeEnvironment: - agent = GooseBinaryAgent( - logs_dir=tmp_path, - model_name="databricks/model", - goose_binary=str(goose_binary), - goose_profile=str(goose_profile), - ) - environment = FakeEnvironment() - - await agent.run("fix the repo", environment, object()) - return environment - - environment = asyncio.run(run_test()) - - commands = [str(item["command"]) for item in environment.commands] - assert any("id -u && id -g" in command for command in commands) - assert any( - f"chown 1000:1000 {CONTAINER_RECIPE_PATH}" in command - for command in commands - ) diff --git a/evals/harbor/tests/test_runner.py b/evals/harbor/tests/test_runner.py deleted file mode 100644 index 7ab5f13f..00000000 --- a/evals/harbor/tests/test_runner.py +++ /dev/null @@ -1,145 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from goose_harbor import runner - - -@pytest.fixture(autouse=True) -def clear_package_index_env(monkeypatch: pytest.MonkeyPatch) -> None: - for key in ("UV_DEFAULT_INDEX", "PIP_INDEX_URL", "UV_INDEX_URL"): - monkeypatch.delenv(key, raising=False) - - -def test_dry_run_writes_config_without_running_harbor(tmp_path: Path) -> None: - goose_binary = tmp_path / "goose" - goose_binary.write_text("#!/bin/sh\n") - goose_profile = tmp_path / "goose-profile" - goose_profile.mkdir() - config_dir = tmp_path / "configs" - - result = runner.main( - [ - "--goose-binary", - str(goose_binary), - "--goose-profile", - str(goose_profile), - "--dataset", - "terminal-bench/terminal-bench-2", - "--model", - "databricks/model", - "--task", - "terminal-bench/fix-git", - "--install-goose-runtime-deps", - "--config-dir", - str(config_dir), - "--dry-run", - ] - ) - - assert result == 0 - config_path = next(config_dir.glob("*.json")) - config = json.loads(config_path.read_text()) - assert config["datasets"] == [ - { - "name": "terminal-bench/terminal-bench-2", - "task_names": ["terminal-bench/fix-git"], - } - ] - assert config["agents"][0]["kwargs"]["install_goose_runtime_deps"] is True - - -def test_package_dataset_suffix_uses_ref(tmp_path: Path) -> None: - goose_binary = tmp_path / "goose" - goose_binary.write_text("#!/bin/sh\n") - goose_profile = tmp_path / "goose-profile" - goose_profile.mkdir() - config_dir = tmp_path / "configs" - - result = runner.main( - [ - "--goose-binary", - str(goose_binary), - "--goose-profile", - str(goose_profile), - "--dataset", - "terminal-bench/terminal-bench-2@v1", - "--model", - "databricks/model", - "--config-dir", - str(config_dir), - "--dry-run", - ] - ) - - assert result == 0 - config = json.loads(next(config_dir.glob("*.json")).read_text()) - assert config["datasets"] == [ - {"name": "terminal-bench/terminal-bench-2", "ref": "v1"} - ] - - -def test_registry_dataset_suffix_uses_version(tmp_path: Path) -> None: - goose_binary = tmp_path / "goose" - goose_binary.write_text("#!/bin/sh\n") - goose_profile = tmp_path / "goose-profile" - goose_profile.mkdir() - config_dir = tmp_path / "configs" - - result = runner.main( - [ - "--goose-binary", - str(goose_binary), - "--goose-profile", - str(goose_profile), - "--dataset", - "terminal-bench@2.0", - "--model", - "databricks/model", - "--config-dir", - str(config_dir), - "--dry-run", - ] - ) - - assert result == 0 - config = json.loads(next(config_dir.glob("*.json")).read_text()) - assert config["datasets"] == [{"name": "terminal-bench", "version": "2.0"}] - - -def test_dry_run_accepts_unexpanded_home_paths( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - home = tmp_path / "home" - goose_binary = home / "bin" / "goose" - goose_binary.parent.mkdir(parents=True) - goose_binary.write_text("#!/bin/sh\n") - goose_profile = home / "goose-profile" - goose_profile.mkdir() - config_dir = tmp_path / "configs" - monkeypatch.setenv("HOME", str(home)) - - result = runner.main( - [ - "--goose-binary", - "~/bin/goose", - "--goose-profile", - "~/goose-profile", - "--dataset", - "terminal-bench/terminal-bench-2", - "--model", - "databricks/model", - "--config-dir", - str(config_dir), - "--dry-run", - ] - ) - - assert result == 0 - config = json.loads(next(config_dir.glob("*.json")).read_text()) - assert config["agents"][0]["kwargs"]["goose_binary"] == str(goose_binary) - assert config["agents"][0]["kwargs"]["goose_profile"] == str(goose_profile)