From 7f8685a6f865b546a4458bd15b53fb5ebca3b663 Mon Sep 17 00:00:00 2001 From: Douwe Osinga Date: Fri, 21 Aug 2026 05:31:46 +0000 Subject: [PATCH] More provider test scripts (#10515) Signed-off-by: Michael Neale <14976+michaelneale@users.noreply.github.com> Co-authored-by: Douwe M Osinga Co-authored-by: Michael Neale <14976+michaelneale@users.noreply.github.com> Co-authored-by: Lifei Zhou --- .../workflows/model-toolcall-conformance.yml | 77 +++ crates/goose-cli/src/cli.rs | 300 ++++++++++- scripts/test_local_model_smoke.sh | 465 ++++++++++++++++++ scripts/test_openrouter_toolcalls.sh | 281 +++++++++++ 4 files changed, 1109 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/model-toolcall-conformance.yml create mode 100755 scripts/test_local_model_smoke.sh create mode 100755 scripts/test_openrouter_toolcalls.sh diff --git a/.github/workflows/model-toolcall-conformance.yml b/.github/workflows/model-toolcall-conformance.yml new file mode 100644 index 000000000..0b0b4fd89 --- /dev/null +++ b/.github/workflows/model-toolcall-conformance.yml @@ -0,0 +1,77 @@ +name: Model Tool Call Conformance + +# Sweeps many models and reports which ones can actually drive an MCP tool call +# through goose. This is deliberately NOT a PR gate: it exercises third-party +# models over the network, so a failure here usually means a provider or model +# regressed, not that a goose change broke. Running it on a schedule keeps the +# signal without turning unrelated PRs red. +on: + schedule: + # 03:00 UTC daily + - cron: '0 3 * * *' + workflow_dispatch: + inputs: + model_count: + description: 'Number of top OpenRouter tool-capable models to test' + required: false + default: '10' + type: string + models: + description: 'Comma-separated explicit model list (overrides model_count)' + required: false + default: '' + type: string + +permissions: + contents: read + +jobs: + openrouter-toolcalls: + name: OpenRouter Tool Calls + runs-on: ubuntu-latest + # Only meaningful on the upstream repo, which holds the provider secrets. + if: github.repository == 'aaif-goose/goose' + timeout-minutes: 60 + steps: + - name: Checkout Code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 + + - name: Install Dependencies + run: | + sudo apt update -y + sudo apt install -y libdbus-1-dev gnome-keyring libxcb1-dev jq + + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + + - name: Build Binary + run: cargo build --bin goose + + - name: Run OpenRouter Tool Call Sweep + env: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + HOME: /tmp/goose-home + GOOSE_DISABLE_KEYRING: 1 + SKIP_BUILD: 1 + GOOSE_BIN: ${{ github.workspace }}/target/debug/goose + run: | + mkdir -p /tmp/goose-home + if [ -n "${{ inputs.models }}" ]; then + bash scripts/test_openrouter_toolcalls.sh \ + --models "${{ inputs.models }}" \ + --output-dir openrouter-toolcall-results + else + bash scripts/test_openrouter_toolcalls.sh \ + --count "${{ inputs.model_count || '10' }}" \ + --output-dir openrouter-toolcall-results + fi + + - name: Upload Per-Model Logs + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: openrouter-toolcall-logs + path: openrouter-toolcall-results + retention-days: 14 diff --git a/crates/goose-cli/src/cli.rs b/crates/goose-cli/src/cli.rs index e16ce315a..d03657c9c 100644 --- a/crates/goose-cli/src/cli.rs +++ b/crates/goose-cli/src/cli.rs @@ -1220,11 +1220,31 @@ enum LocalModelsCommand { #[command(about = "Search HuggingFace for local GGUF and MLX models")] Search { /// Search query - query: String, + query: Option, /// Maximum number of results #[arg(short, long, default_value = "10")] limit: usize, + + /// Only include repos whose id starts with this prefix + #[arg(long)] + repo_prefix: Option, + + /// Only include repos whose id ends with this suffix + #[arg(long)] + repo_suffix: Option, + + /// Only include variants whose quantization contains this text + #[arg(long)] + quant: Option, + + /// Override available memory used for recommendations, in GB + #[arg(long)] + ram_gb: Option, + + /// Print results as JSON + #[arg(long)] + json: bool, }, /// Download a model from HuggingFace @@ -2272,6 +2292,104 @@ fn print_download_progress(manager: &goose::download_manager::DownloadManager) { std::io::stdout().flush().ok(); } +#[cfg(feature = "local-inference")] +fn gb_to_bytes(gb: f64) -> Result { + if !gb.is_finite() || gb <= 0.0 { + anyhow::bail!("--ram-gb must be a positive number"); + } + Ok((gb * 1024.0 * 1024.0 * 1024.0) as u64) +} + +#[cfg(feature = "local-inference")] +fn search_query_from_filters( + query: Option, + repo_prefix: Option<&str>, + repo_suffix: Option<&str>, +) -> String { + if let Some(query) = query { + return query; + } + + if let Some(prefix) = repo_prefix { + let term = search_term_from_repo_filter(prefix); + if !term.is_empty() { + return term; + } + } + + if let Some(suffix) = repo_suffix { + let term = search_term_from_repo_filter(suffix); + if !term.is_empty() { + return term; + } + } + + String::new() +} + +#[cfg(feature = "local-inference")] +fn search_term_from_repo_filter(value: &str) -> String { + value + .trim_matches('/') + .rsplit('/') + .next() + .unwrap_or_default() + .trim_matches(|c| matches!(c, '-' | '_' | '.')) + .to_string() +} + +#[cfg(feature = "local-inference")] +fn local_search_memory_limit(ram_gb: Option) -> Result { + if let Some(gb) = ram_gb { + return gb_to_bytes(gb); + } + + match goose::providers::local_inference::InferenceRuntime::get_or_init() { + Ok(runtime) => Ok( + goose::providers::local_inference::available_inference_memory_bytes(runtime.as_ref()), + ), + Err(_) => gb_to_bytes(16.0), + } +} + +#[cfg(feature = "local-inference")] +fn format_size(bytes: u64) -> String { + if bytes == 0 { + "unknown".to_string() + } else { + format!("{:.1}GB", bytes as f64 / (1024.0 * 1024.0 * 1024.0)) + } +} + +#[cfg(feature = "local-inference")] +fn recommended_variant( + model: &goose::providers::local_inference::hf_models::HfModelInfo, + available_memory: u64, +) -> Option<&goose::providers::local_inference::hf_models::HfModelVariant> { + use goose::providers::local_inference::hf_models::{recommend_variant, HfQuantVariant}; + + let mut variant_indexes = Vec::new(); + let mut gguf_variants = Vec::new(); + for (index, variant) in model.variants.iter().enumerate() { + if variant.backend_id != "llamacpp" || !variant.supported { + continue; + } + variant_indexes.push(index); + gguf_variants.push(HfQuantVariant { + quantization: variant.variant_id.clone(), + size_bytes: variant.size_bytes, + filename: variant.filename.clone().unwrap_or_default(), + download_url: variant.download_url.clone().unwrap_or_default(), + description: "", + quality_rank: variant.quality_rank, + sharded: variant.sharded, + }); + } + + recommend_variant(&gguf_variants, available_memory) + .map(|index| &model.variants[variant_indexes[index]]) +} + #[cfg(feature = "local-inference")] async fn handle_local_models_command(command: LocalModelsCommand) -> Result<()> { use goose::providers::local_inference::hf_models; @@ -2280,12 +2398,91 @@ async fn handle_local_models_command(command: LocalModelsCommand) -> Result<()> goose::providers::local_inference::configure_huggingface_auth(); match command { - LocalModelsCommand::Search { query, limit } => { - println!("Searching HuggingFace for '{}'...", query); - let results = hf_models::search_local_models(&query, limit).await?; + LocalModelsCommand::Search { + query, + limit, + repo_prefix, + repo_suffix, + quant, + ram_gb, + json, + } => { + let query = + search_query_from_filters(query, repo_prefix.as_deref(), repo_suffix.as_deref()); + if !json { + if query.is_empty() { + println!("Searching HuggingFace for local models..."); + } else { + println!("Searching HuggingFace for '{}'...", query); + } + } + let has_local_filter = + repo_prefix.is_some() || repo_suffix.is_some() || quant.is_some(); + let fetch_limit = if has_local_filter { + limit.saturating_mul(5).max(limit).max(50) + } else { + limit + }; + let mut results = hf_models::search_local_models(&query, fetch_limit).await?; + let quant = quant.map(|value| value.to_lowercase()); + results.retain_mut(|model| { + if repo_prefix + .as_deref() + .is_some_and(|prefix| !model.repo_id.starts_with(prefix)) + { + return false; + } + if repo_suffix + .as_deref() + .is_some_and(|suffix| !model.repo_id.ends_with(suffix)) + { + return false; + } + + if let Some(quant) = &quant { + model + .variants + .retain(|variant| variant.variant_id.to_lowercase().contains(quant)); + } + + !model.variants.is_empty() + }); + results.truncate(limit); if results.is_empty() { - println!("No compatible local models found."); + if json { + println!("[]"); + } else { + println!("No compatible local models found."); + } + return Ok(()); + } + + let available_memory = local_search_memory_limit(ram_gb)?; + + if json { + let output = results + .iter() + .map(|model| { + let recommended_variant = + recommended_variant(model, available_memory).map(|variant| { + serde_json::json!({ + "model_id": variant.model_id, + "download_id": variant.download_id, + "label": variant.label, + "size_bytes": variant.size_bytes, + }) + }); + serde_json::json!({ + "repo_id": model.repo_id, + "author": model.author, + "model_name": model.model_name, + "downloads": model.downloads, + "recommended_variant": recommended_variant, + }) + }) + .collect::>(); + println!("{}", serde_json::to_string_pretty(&output)?); return Ok(()); } @@ -2294,15 +2491,23 @@ async fn handle_local_models_command(command: LocalModelsCommand) -> Result<()> "\n{} (by {}) — {} downloads", model.model_name, model.author, model.downloads ); + if let Some(variant) = recommended_variant(model, available_memory) { + println!( + " Recommended: {} — {}", + variant.label, + format_size(variant.size_bytes) + ); + println!( + " Download: goose local-models download '{}'", + variant.download_id + ); + } else { + println!( + " Recommended: none fits in {}", + format_size(available_memory) + ); + } for variant in &model.variants { - let size = if variant.size_bytes > 0 { - format!( - "{:.1}GB", - variant.size_bytes as f64 / (1024.0 * 1024.0 * 1024.0) - ) - } else { - "unknown".to_string() - }; let support = if variant.supported { String::new() } else { @@ -2316,7 +2521,11 @@ async fn handle_local_models_command(command: LocalModelsCommand) -> Result<()> }; println!( " [{}] {} — {} — {}{}", - variant.format, variant.label, size, variant.description, support + variant.format, + variant.label, + format_size(variant.size_bytes), + variant.description, + support ); if variant.supported { println!( @@ -2857,4 +3066,67 @@ mod tests { _ => panic!("expected tui command"), } } + + #[cfg(feature = "local-inference")] + mod local_search { + use super::super::{ + format_size, gb_to_bytes, search_query_from_filters, search_term_from_repo_filter, + }; + + #[test] + fn gb_to_bytes_converts_and_rejects_nonpositive() { + assert_eq!(gb_to_bytes(1.0).unwrap(), 1024 * 1024 * 1024); + assert_eq!(gb_to_bytes(0.5).unwrap(), 512 * 1024 * 1024); + assert!(gb_to_bytes(0.0).is_err()); + assert!(gb_to_bytes(-4.0).is_err()); + assert!(gb_to_bytes(f64::NAN).is_err()); + assert!(gb_to_bytes(f64::INFINITY).is_err()); + } + + #[test] + fn explicit_query_wins_over_repo_filters() { + let query = search_query_from_filters( + Some("qwen".to_string()), + Some("unsloth/Llama-3.2"), + Some("-GGUF"), + ); + assert_eq!(query, "qwen"); + } + + #[test] + fn repo_prefix_is_used_when_query_is_absent() { + let query = search_query_from_filters(None, Some("unsloth/Llama-3.2"), None); + assert_eq!(query, "Llama-3.2"); + } + + #[test] + fn repo_suffix_is_used_when_prefix_yields_nothing() { + let query = search_query_from_filters(None, Some("///"), Some("-Qwen3-GGUF")); + assert_eq!(query, "Qwen3-GGUF"); + } + + #[test] + fn empty_when_nothing_is_provided() { + assert_eq!(search_query_from_filters(None, None, None), ""); + } + + #[test] + fn repo_filter_takes_last_path_segment_and_trims_separators() { + assert_eq!( + search_term_from_repo_filter("unsloth/Llama-3.2"), + "Llama-3.2" + ); + assert_eq!(search_term_from_repo_filter("-GGUF"), "GGUF"); + assert_eq!(search_term_from_repo_filter("/bartowski/"), "bartowski"); + assert_eq!(search_term_from_repo_filter("_model_."), "model"); + assert_eq!(search_term_from_repo_filter(""), ""); + } + + #[test] + fn format_size_reports_unknown_for_zero() { + assert_eq!(format_size(0), "unknown"); + assert_eq!(format_size(1024 * 1024 * 1024), "1.0GB"); + assert_eq!(format_size(3 * 1024 * 1024 * 1024 / 2), "1.5GB"); + } + } } diff --git a/scripts/test_local_model_smoke.sh b/scripts/test_local_model_smoke.sh new file mode 100755 index 000000000..ece024c93 --- /dev/null +++ b/scripts/test_local_model_smoke.sh @@ -0,0 +1,465 @@ +#!/usr/bin/env bash +set -euo pipefail + +show_usage() { + echo "Usage: $0 [options]" + echo "" + echo "Options:" + echo " -n, --top-n NUM Number of recommended models to test (default: 3)" + echo " -m, --models MODELS Comma-separated download ids. Skips search." + echo " -o, --output-dir DIR Directory for logs (default: ./local-model-smoke-results)" + echo " --ram-gb NUM Override RAM passed to goose lm search" + echo " --instruction TEXT Prompt to send to each model" + echo " --repo-prefix TEXT Forwarded to goose lm search" + echo " --repo-suffix TEXT Forwarded to goose lm search" + echo " --quant TEXT Forwarded to goose lm search" + echo " --download-retries N Retry model downloads after HF rate limits (default: 3)" + echo " --retry-delay SEC Initial retry delay for HF rate limits (default: 60)" + echo " --run-timeout SEC Kill a model run after this many seconds (default: 600, 0 disables)" + echo " --keep-downloads Do not delete models after testing" + echo " -h, --help Show this help message" + echo "" + echo "Environment:" + echo " GOOSE_BIN Optional goose binary path" + echo " SKIP_BUILD Skip cargo build when set" +} + +TOP_N=3 +OUTPUT_DIR="./local-model-smoke-results" +MODEL_LIST="" +RAM_GB="" +INSTRUCTION="Say hello in one short sentence. Do not use tools." +REPO_PREFIX="unsloth/" +REPO_SUFFIX="" +QUANT="Q4" +DOWNLOAD_RETRIES=3 +RETRY_DELAY=60 +RUN_TIMEOUT=600 +KEEP_DOWNLOADS=false + +while [[ $# -gt 0 ]]; do + case "$1" in + -n|--top-n) + TOP_N="$2" + shift 2 + ;; + -o|--output-dir) + OUTPUT_DIR="$2" + shift 2 + ;; + -m|--models) + MODEL_LIST="$2" + shift 2 + ;; + --ram-gb) + RAM_GB="$2" + shift 2 + ;; + --instruction) + INSTRUCTION="$2" + shift 2 + ;; + --repo-prefix) + REPO_PREFIX="$2" + shift 2 + ;; + --repo-suffix) + REPO_SUFFIX="$2" + shift 2 + ;; + --quant) + QUANT="$2" + shift 2 + ;; + --download-retries) + DOWNLOAD_RETRIES="$2" + shift 2 + ;; + --retry-delay) + RETRY_DELAY="$2" + shift 2 + ;; + --run-timeout) + RUN_TIMEOUT="$2" + shift 2 + ;; + --keep-downloads) + KEEP_DOWNLOADS=true + shift + ;; + -h|--help) + show_usage + exit 0 + ;; + *) + echo "Error: Unknown option: $1" + show_usage + exit 1 + ;; + esac +done + +if ! [[ "$TOP_N" =~ ^[0-9]+$ ]] || [[ "$TOP_N" -eq 0 ]]; then + echo "Error: --top-n must be a positive integer" + exit 1 +fi + +if ! [[ "$DOWNLOAD_RETRIES" =~ ^[0-9]+$ ]]; then + echo "Error: --download-retries must be a non-negative integer" + exit 1 +fi + +if ! [[ "$RETRY_DELAY" =~ ^[0-9]+$ ]]; then + echo "Error: --retry-delay must be a non-negative integer" + exit 1 +fi + +if ! [[ "$RUN_TIMEOUT" =~ ^[0-9]+$ ]]; then + echo "Error: --run-timeout must be a non-negative integer" + exit 1 +fi + +if ! command -v jq >/dev/null 2>&1; then + echo "Error: jq is required" + exit 1 +fi + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +if [[ -z "${SKIP_BUILD:-}" && -z "${GOOSE_BIN:-}" ]]; then + echo "Building goose..." + (cd "$REPO_ROOT" && cargo build -p goose-cli --features local-inference --bin goose) + echo "" +fi + +GOOSE_BIN="${GOOSE_BIN:-$REPO_ROOT/target/debug/goose}" +if [[ ! -x "$GOOSE_BIN" ]]; then + echo "Error: goose binary not found or not executable: $GOOSE_BIN" + exit 1 +fi + +mkdir -p "$OUTPUT_DIR" + +EXISTING_MODELS_FILE="$OUTPUT_DIR/existing-models.txt" +RESULTS_FILE="$OUTPUT_DIR/results.tsv" +"$GOOSE_BIN" lm list | awk 'NR > 2 && $4 == "✓" { print $1 }' > "$EXISTING_MODELS_FILE" +printf "status\tmodel_id\tdetail\n" > "$RESULTS_FILE" + +TEMP_HF_CACHE_ROOT="" +TEMP_MODELS=() + +cleanup_temp_models() { + local model_id + local cleanup_failed=false + + for model_id in "${TEMP_MODELS[@]}"; do + if ! "$GOOSE_BIN" lm delete "$model_id" >/dev/null 2>&1; then + cleanup_failed=true + fi + done + + if [[ "$cleanup_failed" = true ]]; then + echo "Warning: could not unregister all temporary models; cache retained at $TEMP_HF_CACHE_ROOT" >&2 + return + fi + + if [[ -n "$TEMP_HF_CACHE_ROOT" && -d "$TEMP_HF_CACHE_ROOT" ]]; then + rm -rf -- "$TEMP_HF_CACHE_ROOT" + fi +} + +untrack_temp_model() { + local target="$1" + local model_id + local remaining=() + + for model_id in "${TEMP_MODELS[@]}"; do + if [[ "$model_id" != "$target" ]]; then + remaining+=("$model_id") + fi + done + TEMP_MODELS=("${remaining[@]}") +} + +if [[ "$KEEP_DOWNLOADS" = false ]]; then + TEMP_HF_CACHE_ROOT=$(mktemp -d) +fi +trap cleanup_temp_models EXIT + +MODELS=() +if [[ -n "$MODEL_LIST" ]]; then + IFS=',' read -ra REQUESTED_MODELS <<< "$MODEL_LIST" + for model in "${REQUESTED_MODELS[@]}"; do + repo="${model%%:*}" + variant="${model#*:}" + if [[ "$variant" = "$model" ]]; then + variant="manual" + fi + MODELS+=("$repo"$'\t'"$model"$'\t'"$model"$'\t'"$variant"$'\t'"0") + done +else + search_query="${REPO_PREFIX%/}" + SEARCH_ARGS=(lm search) + if [[ -n "$search_query" ]]; then + SEARCH_ARGS+=("$search_query") + fi + SEARCH_ARGS+=(--limit "$TOP_N" --json) + if [[ -n "$RAM_GB" ]]; then + SEARCH_ARGS+=(--ram-gb "$RAM_GB") + fi + if [[ -n "$REPO_PREFIX" ]]; then + SEARCH_ARGS+=(--repo-prefix "$REPO_PREFIX") + fi + if [[ -n "$REPO_SUFFIX" ]]; then + SEARCH_ARGS+=(--repo-suffix "$REPO_SUFFIX") + fi + if [[ -n "$QUANT" ]]; then + SEARCH_ARGS+=(--quant "$QUANT") + fi + + SEARCH_JSON="$OUTPUT_DIR/search.json" + echo "Finding recommended local models..." + "$GOOSE_BIN" "${SEARCH_ARGS[@]}" > "$SEARCH_JSON" + + while IFS= read -r model_row; do + MODELS+=("$model_row") + done < <( + jq -r --argjson limit "$TOP_N" ' + [.[] | select(.recommended_variant != null)] + | .[:$limit][] + | [ + .repo_id, + .recommended_variant.model_id, + .recommended_variant.download_id, + .recommended_variant.label, + (.recommended_variant.size_bytes | tostring) + ] + | @tsv + ' "$SEARCH_JSON" + ) +fi + +if [[ ${#MODELS[@]} -eq 0 ]]; then + echo "No recommended models found." + exit 1 +fi + +RESULTS=() +OVERALL_SUCCESS=true + +record_result() { + local status="$1" + local model_id="$2" + local detail="$3" + RESULTS+=("$status $model_id${detail:+ - $detail}") + printf "%s\t%s\t%s\n" "$status" "$model_id" "$detail" >> "$RESULTS_FILE" +} + +summarize_goose_error() { + awk ' + /Ran into this error:/ { + sub(/^.*Ran into this error: /, "") + print + found = 1 + exit + } + /Request failed:/ { + sub(/^.*Request failed: /, "Request failed: ") + print + found = 1 + exit + } + /Provider error:/ { + sub(/^.*Provider error: /, "Provider error: ") + print + found = 1 + exit + } + END { if (!found) exit 1 } + ' "$1" +} + +download_once() { + local download_id="$1" + local cache_root="$2" + + if [[ -n "$cache_root" ]]; then + HF_HUB_CACHE="$cache_root/hub" \ + HF_XET_CACHE="$cache_root/xet" \ + "$GOOSE_BIN" lm download "$download_id" + else + "$GOOSE_BIN" lm download "$download_id" + fi +} + +download_model() { + local download_id="$1" + local log_file="$2" + local cache_root="$3" + local attempt=1 + local delay="$RETRY_DELAY" + + while true; do + : > "$log_file" + if download_once "$download_id" "$cache_root" 2>&1 | tee "$log_file"; then + return 0 + fi + + if ! grep -q "429 Too Many Requests" "$log_file"; then + return 1 + fi + + if [[ "$attempt" -gt "$DOWNLOAD_RETRIES" ]]; then + return 2 + fi + + echo "Hugging Face rate limit hit. Retrying in ${delay}s ($attempt/$DOWNLOAD_RETRIES)..." + sleep "$delay" + attempt=$((attempt + 1)) + if [[ "$delay" -gt 0 ]]; then + delay=$((delay * 2)) + fi + done +} + +run_model() { + local model_id="$1" + local log_file="$2" + + if [[ "$RUN_TIMEOUT" -eq 0 ]]; then + GOOSE_MODE=auto GOOSE_PROVIDER=local GOOSE_MODEL="$model_id" \ + "$GOOSE_BIN" run --no-profile --text "$INSTRUCTION" 2>&1 | tee "$log_file" + return "${PIPESTATUS[0]}" + fi + + perl -e ' + my $timeout = shift; + my $pid = fork(); + die "fork failed: $!" unless defined $pid; + if ($pid == 0) { + exec @ARGV; + die "exec failed: $!"; + } + local $SIG{ALRM} = sub { + kill "TERM", $pid; + sleep 2; + kill "KILL", $pid; + exit 124; + }; + alarm $timeout; + waitpid($pid, 0); + my $status = $?; + alarm 0; + exit($status & 127 ? 128 + ($status & 127) : $status >> 8); + ' \ + "$RUN_TIMEOUT" \ + env GOOSE_MODE=auto GOOSE_PROVIDER=local GOOSE_MODEL="$model_id" \ + "$GOOSE_BIN" run --no-profile --text "$INSTRUCTION" 2>&1 | tee "$log_file" + return "${PIPESTATUS[0]}" +} + +echo "Testing ${#MODELS[@]} model(s)" +echo "" + +for row in "${MODELS[@]}"; do + IFS=$'\t' read -r repo_id model_id download_id label size_bytes <<< "$row" + safe_model=$(echo "$model_id" | tr '/:' '__' | tr -cd '[:alnum:]_.-') + download_log="$OUTPUT_DIR/$safe_model.download.log" + run_log="$OUTPUT_DIR/$safe_model.run.log" + delete_log="$OUTPUT_DIR/$safe_model.delete.log" + size_gb=$(awk "BEGIN { printf \"%.1f\", $size_bytes / 1024 / 1024 / 1024 }") + + echo "==========================================================" + echo "Model: $model_id" + echo "Repo: $repo_id" + echo "Variant: $label (${size_gb}GB)" + echo "==========================================================" + + existed_before=false + if grep -Fxq "$model_id" "$EXISTING_MODELS_FILE"; then + existed_before=true + fi + + downloaded=false + if [[ "$existed_before" = true ]]; then + echo "Using pre-existing download for $model_id" + downloaded=true + else + if [[ -n "$TEMP_HF_CACHE_ROOT" ]]; then + TEMP_MODELS+=("$model_id") + fi + set +e + download_model "$download_id" "$download_log" "$TEMP_HF_CACHE_ROOT" + download_status=$? + set -e + if [[ "$download_status" -eq 0 ]]; then + downloaded=true + elif [[ "$download_status" -eq 2 ]]; then + echo "Download rate limited for $model_id" + record_result "FAIL" "$model_id" "Hugging Face rate limited" + OVERALL_SUCCESS=false + else + echo "Download failed for $model_id" + record_result "FAIL" "$model_id" "download failed" + OVERALL_SUCCESS=false + fi + fi + + if [[ "$downloaded" = true ]]; then + set +e + run_model "$model_id" "$run_log" + run_status=$? + set -e + + if [[ ! -s "$run_log" ]]; then + echo "Run produced no output for $model_id" + record_result "FAIL" "$model_id" "empty output" + OVERALL_SUCCESS=false + elif [[ "$run_status" -eq 124 || "$run_status" -eq 142 ]]; then + echo "Run timed out after ${RUN_TIMEOUT}s for $model_id" + record_result "FAIL" "$model_id" "run timed out" + OVERALL_SUCCESS=false + elif error_summary=$(summarize_goose_error "$run_log"); then + echo "Goose reported an error for $model_id" + echo " $error_summary" + record_result "FAIL" "$model_id" "$error_summary" + OVERALL_SUCCESS=false + elif [[ "$run_status" -eq 0 ]]; then + echo "Run passed for $model_id" + record_result "PASS" "$model_id" "" + else + echo "Run replied but exited with status $run_status for $model_id" + record_result "FAIL" "$model_id" "replied but exited $run_status" + OVERALL_SUCCESS=false + fi + fi + + if [[ "$KEEP_DOWNLOADS" = false && "$downloaded" = true && "$existed_before" = false ]]; then + if "$GOOSE_BIN" lm delete "$model_id" 2>&1 | tee "$delete_log"; then + untrack_temp_model "$model_id" + echo "Unregistered $model_id; its temporary cache will be removed at exit" + else + echo "Delete failed for $model_id" + record_result "FAIL" "$model_id" "delete failed" + OVERALL_SUCCESS=false + fi + elif [[ "$KEEP_DOWNLOADS" = false && "$downloaded" = true ]]; then + echo "Keeping $model_id because it existed before this run" + fi + + echo "" +done + +echo "=== Test Summary ===" +for result in "${RESULTS[@]}"; do + echo "$result" +done + +if [[ "$OVERALL_SUCCESS" = false ]]; then + echo "" + echo "Some local model smoke tests failed." + exit 1 +fi + +echo "" +echo "All local model smoke tests passed." diff --git a/scripts/test_openrouter_toolcalls.sh b/scripts/test_openrouter_toolcalls.sh new file mode 100755 index 000000000..c4507b6d8 --- /dev/null +++ b/scripts/test_openrouter_toolcalls.sh @@ -0,0 +1,281 @@ +#!/usr/bin/env bash +set -euo pipefail + +show_usage() { + echo "Usage: $0 [options]" + echo "" + echo "Options:" + echo " -n, --count NUM Number of OpenRouter models to test (default: 10)" + echo " -m, --models MODELS Comma-separated model list. Skips fetching models." + echo " -o, --output-dir DIR Directory for logs (default: ./openrouter-toolcall-results)" + echo " -s, --sort SORT OpenRouter model sort (default: top-weekly)" + echo " --run-timeout SEC Kill a model run after this many seconds (default: 180, 0 disables)" + echo " -h, --help Show this help message" + echo "" + echo "Environment:" + echo " OPENROUTER_API_KEY Required by goose's OpenRouter provider" + echo " OPENROUTER_HOST Optional OpenRouter host (default: https://openrouter.ai)" + echo " GOOSE_BIN Optional goose binary path" + echo " SKIP_BUILD Skip cargo build when set" + echo "" + echo "Examples:" + echo " $0 --count 5" + echo " $0 --models 'anthropic/claude-sonnet-4.5,google/gemini-2.5-flash'" +} + +MODEL_COUNT=10 +MODEL_LIST="" +OUTPUT_DIR="./openrouter-toolcall-results" +MODEL_SORT="top-weekly" +RUN_TIMEOUT=180 + +while [[ $# -gt 0 ]]; do + case "$1" in + -n|--count) + MODEL_COUNT="$2" + shift 2 + ;; + -m|--models) + MODEL_LIST="$2" + shift 2 + ;; + -o|--output-dir) + OUTPUT_DIR="$2" + shift 2 + ;; + -s|--sort) + MODEL_SORT="$2" + shift 2 + ;; + --run-timeout) + RUN_TIMEOUT="$2" + shift 2 + ;; + -h|--help) + show_usage + exit 0 + ;; + *) + echo "Error: Unknown option: $1" + show_usage + exit 1 + ;; + esac +done + +if [[ -z "${OPENROUTER_API_KEY:-}" ]]; then + echo "Error: OPENROUTER_API_KEY must be set" + exit 1 +fi + +if ! command -v jq >/dev/null 2>&1; then + echo "Error: jq is required" + exit 1 +fi + +if ! command -v curl >/dev/null 2>&1; then + echo "Error: curl is required" + exit 1 +fi + +if ! command -v uv >/dev/null 2>&1; then + echo "Error: uv is required to run the temporary FastMCP server" + exit 1 +fi + +if ! [[ "$RUN_TIMEOUT" =~ ^[0-9]+$ ]]; then + echo "Error: --run-timeout must be a non-negative integer" + exit 1 +fi + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +if [[ -z "${SKIP_BUILD:-}" && -z "${GOOSE_BIN:-}" ]]; then + echo "Building goose..." + (cd "$REPO_ROOT" && cargo build --bin goose) + echo "" +fi + +GOOSE_BIN="${GOOSE_BIN:-$REPO_ROOT/target/debug/goose}" +if [[ ! -x "$GOOSE_BIN" ]]; then + echo "Error: goose binary not found or not executable: $GOOSE_BIN" + exit 1 +fi + +mkdir -p "$OUTPUT_DIR" + +MODELS=() +if [[ -n "$MODEL_LIST" ]]; then + IFS=',' read -ra MODELS <<< "$MODEL_LIST" +else + OPENROUTER_HOST="${OPENROUTER_HOST:-https://openrouter.ai}" + MODELS_URL="$OPENROUTER_HOST/api/v1/models?supported_parameters=tools&sort=$MODEL_SORT" + + echo "Fetching OpenRouter models: $MODELS_URL" + MODELS_JSON=$(curl --fail --silent --show-error --max-time 30 "$MODELS_URL") + while IFS= read -r model; do + MODELS+=("$model") + done < <(jq -r --argjson limit "$MODEL_COUNT" '.data[:$limit][] | .id' <<< "$MODELS_JSON") +fi + +if [[ ${#MODELS[@]} -eq 0 ]]; then + echo "Error: no models found" + exit 1 +fi + +TESTDIR=$(mktemp -d) +trap 'rm -rf "$TESTDIR"' EXIT + +cat > "$TESTDIR/weather.py" << 'EOF' +from typing import Annotated +from fastmcp import FastMCP + +mcp = FastMCP("weather") + +@mcp.tool +def get_weather( + location: Annotated[str, "City or place to check"], +) -> Annotated[str, "Weather report"]: + """Get the current weather for a location.""" + return f"GOOSE_TOOL_CALL_OK: The weather in {location} is 68 F and clear." +EOF + +cat > "$TESTDIR/recipe.yaml" << 'EOF' +title: OpenRouter Tool Call Test +description: Test a model can call a simple MCP tool through goose +prompt: Use the get_weather tool to check the weather in San Francisco. Do not answer from memory. +extensions: + - name: weather + cmd: uv + args: + - run + - --with + - fastmcp==2.14.4 + - fastmcp + - run + - weather.py + type: stdio +EOF + +RESULTS=() +OVERALL_SUCCESS=true + +summarize_error() { + awk ' + /Ran into this error:/ { + sub(/^.*Ran into this error: /, "") + print + found = 1 + exit + } + /Request failed:/ { + sub(/^.*Request failed: /, "Request failed: ") + print + found = 1 + exit + } + /Provider error:/ { + sub(/^.*Provider error: /, "Provider error: ") + print + found = 1 + exit + } + END { if (!found) exit 1 } + ' "$1" +} + +run_model() { + local model="$1" + + if [[ "$RUN_TIMEOUT" -eq 0 ]]; then + GOOSE_MODE=auto GOOSE_PROVIDER=openrouter GOOSE_MODEL="$model" \ + "$GOOSE_BIN" run --no-profile --max-turns 4 --recipe recipe.yaml + return $? + fi + + perl -e ' + my $timeout = shift; + my $pid = fork(); + die "fork failed: $!" unless defined $pid; + if ($pid == 0) { + exec @ARGV; + die "exec failed: $!"; + } + local $SIG{ALRM} = sub { + kill "TERM", $pid; + sleep 2; + kill "KILL", $pid; + exit 124; + }; + alarm $timeout; + waitpid($pid, 0); + my $status = $?; + alarm 0; + exit($status & 127 ? 128 + ($status & 127) : $status >> 8); + ' \ + "$RUN_TIMEOUT" \ + env GOOSE_MODE=auto GOOSE_PROVIDER=openrouter GOOSE_MODEL="$model" \ + "$GOOSE_BIN" run --no-profile --max-turns 4 --recipe recipe.yaml +} + +echo "Testing ${#MODELS[@]} OpenRouter model(s)" +echo "" + +for model in "${MODELS[@]}"; do + safe_model=$(echo "$model" | tr '/:' '__' | tr -cd '[:alnum:]_.-') + log_file="$OUTPUT_DIR/$safe_model.log" + + echo "==========================================================" + echo "Model: $model" + echo "Log: $log_file" + echo "==========================================================" + + if (cd "$TESTDIR" && run_model "$model" 2>&1) | tee "$log_file"; then + if error_summary=$(summarize_error "$log_file"); then + echo "✗ Goose reported an error for $model" + echo " $error_summary" + RESULTS+=("✗ $model - $error_summary") + OVERALL_SUCCESS=false + elif grep -qE "(get_weather \| weather)|(▸.*get_weather.*weather)" "$log_file" && \ + grep -Fq "GOOSE_TOOL_CALL_OK:" "$log_file"; then + echo "✓ Tool call passed for $model" + RESULTS+=("✓ $model") + elif grep -qE "(get_weather \| weather)|(▸.*get_weather.*weather)" "$log_file"; then + echo "✗ Tool call did not return a successful result for $model" + RESULTS+=("✗ $model - no successful get_weather result found") + OVERALL_SUCCESS=false + else + echo "✗ Tool call not found for $model" + RESULTS+=("✗ $model - no get_weather call found") + OVERALL_SUCCESS=false + fi + else + run_status=${PIPESTATUS[0]} + echo "✗ Goose run failed for $model" + if [[ "$run_status" -eq 124 ]]; then + RESULTS+=("✗ $model - run timed out") + elif error_summary=$(summarize_error "$log_file"); then + echo " $error_summary" + RESULTS+=("✗ $model - $error_summary") + else + RESULTS+=("✗ $model - goose run failed") + fi + OVERALL_SUCCESS=false + fi + + echo "" +done + +echo "=== Test Summary ===" +for result in "${RESULTS[@]}"; do + echo "$result" +done + +if [[ "$OVERALL_SUCCESS" = false ]]; then + echo "" + echo "Some OpenRouter tool call tests failed." + exit 1 +fi + +echo "" +echo "All OpenRouter tool call tests passed."