More provider test scripts (#10515)
Signed-off-by: Michael Neale <14976+michaelneale@users.noreply.github.com> Co-authored-by: Douwe M Osinga <douwe@sidewalklabs.com> Co-authored-by: Michael Neale <14976+michaelneale@users.noreply.github.com> Co-authored-by: Lifei Zhou <lifei@squareup.com>
This commit is contained in:
Douwe Osinga
@@ -0,0 +1,77 @@
|
||||
name: Model Tool Call Conformance
|
||||
|
||||
# Sweeps many models and reports which ones can actually drive an MCP tool call
|
||||
# through goose. This is deliberately NOT a PR gate: it exercises third-party
|
||||
# models over the network, so a failure here usually means a provider or model
|
||||
# regressed, not that a goose change broke. Running it on a schedule keeps the
|
||||
# signal without turning unrelated PRs red.
|
||||
on:
|
||||
schedule:
|
||||
# 03:00 UTC daily
|
||||
- cron: '0 3 * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
model_count:
|
||||
description: 'Number of top OpenRouter tool-capable models to test'
|
||||
required: false
|
||||
default: '10'
|
||||
type: string
|
||||
models:
|
||||
description: 'Comma-separated explicit model list (overrides model_count)'
|
||||
required: false
|
||||
default: ''
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
openrouter-toolcalls:
|
||||
name: OpenRouter Tool Calls
|
||||
runs-on: ubuntu-latest
|
||||
# Only meaningful on the upstream repo, which holds the provider secrets.
|
||||
if: github.repository == 'aaif-goose/goose'
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
|
||||
- uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0
|
||||
|
||||
- name: Install Dependencies
|
||||
run: |
|
||||
sudo apt update -y
|
||||
sudo apt install -y libdbus-1-dev gnome-keyring libxcb1-dev jq
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
||||
|
||||
- name: Build Binary
|
||||
run: cargo build --bin goose
|
||||
|
||||
- name: Run OpenRouter Tool Call Sweep
|
||||
env:
|
||||
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
|
||||
HOME: /tmp/goose-home
|
||||
GOOSE_DISABLE_KEYRING: 1
|
||||
SKIP_BUILD: 1
|
||||
GOOSE_BIN: ${{ github.workspace }}/target/debug/goose
|
||||
run: |
|
||||
mkdir -p /tmp/goose-home
|
||||
if [ -n "${{ inputs.models }}" ]; then
|
||||
bash scripts/test_openrouter_toolcalls.sh \
|
||||
--models "${{ inputs.models }}" \
|
||||
--output-dir openrouter-toolcall-results
|
||||
else
|
||||
bash scripts/test_openrouter_toolcalls.sh \
|
||||
--count "${{ inputs.model_count || '10' }}" \
|
||||
--output-dir openrouter-toolcall-results
|
||||
fi
|
||||
|
||||
- name: Upload Per-Model Logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: openrouter-toolcall-logs
|
||||
path: openrouter-toolcall-results
|
||||
retention-days: 14
|
||||
+286
-14
@@ -1220,11 +1220,31 @@ enum LocalModelsCommand {
|
||||
#[command(about = "Search HuggingFace for local GGUF and MLX models")]
|
||||
Search {
|
||||
/// Search query
|
||||
query: String,
|
||||
query: Option<String>,
|
||||
|
||||
/// Maximum number of results
|
||||
#[arg(short, long, default_value = "10")]
|
||||
limit: usize,
|
||||
|
||||
/// Only include repos whose id starts with this prefix
|
||||
#[arg(long)]
|
||||
repo_prefix: Option<String>,
|
||||
|
||||
/// Only include repos whose id ends with this suffix
|
||||
#[arg(long)]
|
||||
repo_suffix: Option<String>,
|
||||
|
||||
/// Only include variants whose quantization contains this text
|
||||
#[arg(long)]
|
||||
quant: Option<String>,
|
||||
|
||||
/// Override available memory used for recommendations, in GB
|
||||
#[arg(long)]
|
||||
ram_gb: Option<f64>,
|
||||
|
||||
/// Print results as JSON
|
||||
#[arg(long)]
|
||||
json: bool,
|
||||
},
|
||||
|
||||
/// Download a model from HuggingFace
|
||||
@@ -2272,6 +2292,104 @@ fn print_download_progress(manager: &goose::download_manager::DownloadManager) {
|
||||
std::io::stdout().flush().ok();
|
||||
}
|
||||
|
||||
#[cfg(feature = "local-inference")]
|
||||
fn gb_to_bytes(gb: f64) -> Result<u64> {
|
||||
if !gb.is_finite() || gb <= 0.0 {
|
||||
anyhow::bail!("--ram-gb must be a positive number");
|
||||
}
|
||||
Ok((gb * 1024.0 * 1024.0 * 1024.0) as u64)
|
||||
}
|
||||
|
||||
#[cfg(feature = "local-inference")]
|
||||
fn search_query_from_filters(
|
||||
query: Option<String>,
|
||||
repo_prefix: Option<&str>,
|
||||
repo_suffix: Option<&str>,
|
||||
) -> String {
|
||||
if let Some(query) = query {
|
||||
return query;
|
||||
}
|
||||
|
||||
if let Some(prefix) = repo_prefix {
|
||||
let term = search_term_from_repo_filter(prefix);
|
||||
if !term.is_empty() {
|
||||
return term;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(suffix) = repo_suffix {
|
||||
let term = search_term_from_repo_filter(suffix);
|
||||
if !term.is_empty() {
|
||||
return term;
|
||||
}
|
||||
}
|
||||
|
||||
String::new()
|
||||
}
|
||||
|
||||
#[cfg(feature = "local-inference")]
|
||||
fn search_term_from_repo_filter(value: &str) -> String {
|
||||
value
|
||||
.trim_matches('/')
|
||||
.rsplit('/')
|
||||
.next()
|
||||
.unwrap_or_default()
|
||||
.trim_matches(|c| matches!(c, '-' | '_' | '.'))
|
||||
.to_string()
|
||||
}
|
||||
|
||||
#[cfg(feature = "local-inference")]
|
||||
fn local_search_memory_limit(ram_gb: Option<f64>) -> Result<u64> {
|
||||
if let Some(gb) = ram_gb {
|
||||
return gb_to_bytes(gb);
|
||||
}
|
||||
|
||||
match goose::providers::local_inference::InferenceRuntime::get_or_init() {
|
||||
Ok(runtime) => Ok(
|
||||
goose::providers::local_inference::available_inference_memory_bytes(runtime.as_ref()),
|
||||
),
|
||||
Err(_) => gb_to_bytes(16.0),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "local-inference")]
|
||||
fn format_size(bytes: u64) -> String {
|
||||
if bytes == 0 {
|
||||
"unknown".to_string()
|
||||
} else {
|
||||
format!("{:.1}GB", bytes as f64 / (1024.0 * 1024.0 * 1024.0))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "local-inference")]
|
||||
fn recommended_variant(
|
||||
model: &goose::providers::local_inference::hf_models::HfModelInfo,
|
||||
available_memory: u64,
|
||||
) -> Option<&goose::providers::local_inference::hf_models::HfModelVariant> {
|
||||
use goose::providers::local_inference::hf_models::{recommend_variant, HfQuantVariant};
|
||||
|
||||
let mut variant_indexes = Vec::new();
|
||||
let mut gguf_variants = Vec::new();
|
||||
for (index, variant) in model.variants.iter().enumerate() {
|
||||
if variant.backend_id != "llamacpp" || !variant.supported {
|
||||
continue;
|
||||
}
|
||||
variant_indexes.push(index);
|
||||
gguf_variants.push(HfQuantVariant {
|
||||
quantization: variant.variant_id.clone(),
|
||||
size_bytes: variant.size_bytes,
|
||||
filename: variant.filename.clone().unwrap_or_default(),
|
||||
download_url: variant.download_url.clone().unwrap_or_default(),
|
||||
description: "",
|
||||
quality_rank: variant.quality_rank,
|
||||
sharded: variant.sharded,
|
||||
});
|
||||
}
|
||||
|
||||
recommend_variant(&gguf_variants, available_memory)
|
||||
.map(|index| &model.variants[variant_indexes[index]])
|
||||
}
|
||||
|
||||
#[cfg(feature = "local-inference")]
|
||||
async fn handle_local_models_command(command: LocalModelsCommand) -> Result<()> {
|
||||
use goose::providers::local_inference::hf_models;
|
||||
@@ -2280,12 +2398,91 @@ async fn handle_local_models_command(command: LocalModelsCommand) -> Result<()>
|
||||
goose::providers::local_inference::configure_huggingface_auth();
|
||||
|
||||
match command {
|
||||
LocalModelsCommand::Search { query, limit } => {
|
||||
println!("Searching HuggingFace for '{}'...", query);
|
||||
let results = hf_models::search_local_models(&query, limit).await?;
|
||||
LocalModelsCommand::Search {
|
||||
query,
|
||||
limit,
|
||||
repo_prefix,
|
||||
repo_suffix,
|
||||
quant,
|
||||
ram_gb,
|
||||
json,
|
||||
} => {
|
||||
let query =
|
||||
search_query_from_filters(query, repo_prefix.as_deref(), repo_suffix.as_deref());
|
||||
if !json {
|
||||
if query.is_empty() {
|
||||
println!("Searching HuggingFace for local models...");
|
||||
} else {
|
||||
println!("Searching HuggingFace for '{}'...", query);
|
||||
}
|
||||
}
|
||||
let has_local_filter =
|
||||
repo_prefix.is_some() || repo_suffix.is_some() || quant.is_some();
|
||||
let fetch_limit = if has_local_filter {
|
||||
limit.saturating_mul(5).max(limit).max(50)
|
||||
} else {
|
||||
limit
|
||||
};
|
||||
let mut results = hf_models::search_local_models(&query, fetch_limit).await?;
|
||||
let quant = quant.map(|value| value.to_lowercase());
|
||||
results.retain_mut(|model| {
|
||||
if repo_prefix
|
||||
.as_deref()
|
||||
.is_some_and(|prefix| !model.repo_id.starts_with(prefix))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if repo_suffix
|
||||
.as_deref()
|
||||
.is_some_and(|suffix| !model.repo_id.ends_with(suffix))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if let Some(quant) = &quant {
|
||||
model
|
||||
.variants
|
||||
.retain(|variant| variant.variant_id.to_lowercase().contains(quant));
|
||||
}
|
||||
|
||||
!model.variants.is_empty()
|
||||
});
|
||||
results.truncate(limit);
|
||||
|
||||
if results.is_empty() {
|
||||
println!("No compatible local models found.");
|
||||
if json {
|
||||
println!("[]");
|
||||
} else {
|
||||
println!("No compatible local models found.");
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let available_memory = local_search_memory_limit(ram_gb)?;
|
||||
|
||||
if json {
|
||||
let output = results
|
||||
.iter()
|
||||
.map(|model| {
|
||||
let recommended_variant =
|
||||
recommended_variant(model, available_memory).map(|variant| {
|
||||
serde_json::json!({
|
||||
"model_id": variant.model_id,
|
||||
"download_id": variant.download_id,
|
||||
"label": variant.label,
|
||||
"size_bytes": variant.size_bytes,
|
||||
})
|
||||
});
|
||||
serde_json::json!({
|
||||
"repo_id": model.repo_id,
|
||||
"author": model.author,
|
||||
"model_name": model.model_name,
|
||||
"downloads": model.downloads,
|
||||
"recommended_variant": recommended_variant,
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
println!("{}", serde_json::to_string_pretty(&output)?);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -2294,15 +2491,23 @@ async fn handle_local_models_command(command: LocalModelsCommand) -> Result<()>
|
||||
"\n{} (by {}) — {} downloads",
|
||||
model.model_name, model.author, model.downloads
|
||||
);
|
||||
if let Some(variant) = recommended_variant(model, available_memory) {
|
||||
println!(
|
||||
" Recommended: {} — {}",
|
||||
variant.label,
|
||||
format_size(variant.size_bytes)
|
||||
);
|
||||
println!(
|
||||
" Download: goose local-models download '{}'",
|
||||
variant.download_id
|
||||
);
|
||||
} else {
|
||||
println!(
|
||||
" Recommended: none fits in {}",
|
||||
format_size(available_memory)
|
||||
);
|
||||
}
|
||||
for variant in &model.variants {
|
||||
let size = if variant.size_bytes > 0 {
|
||||
format!(
|
||||
"{:.1}GB",
|
||||
variant.size_bytes as f64 / (1024.0 * 1024.0 * 1024.0)
|
||||
)
|
||||
} else {
|
||||
"unknown".to_string()
|
||||
};
|
||||
let support = if variant.supported {
|
||||
String::new()
|
||||
} else {
|
||||
@@ -2316,7 +2521,11 @@ async fn handle_local_models_command(command: LocalModelsCommand) -> Result<()>
|
||||
};
|
||||
println!(
|
||||
" [{}] {} — {} — {}{}",
|
||||
variant.format, variant.label, size, variant.description, support
|
||||
variant.format,
|
||||
variant.label,
|
||||
format_size(variant.size_bytes),
|
||||
variant.description,
|
||||
support
|
||||
);
|
||||
if variant.supported {
|
||||
println!(
|
||||
@@ -2857,4 +3066,67 @@ mod tests {
|
||||
_ => panic!("expected tui command"),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "local-inference")]
|
||||
mod local_search {
|
||||
use super::super::{
|
||||
format_size, gb_to_bytes, search_query_from_filters, search_term_from_repo_filter,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn gb_to_bytes_converts_and_rejects_nonpositive() {
|
||||
assert_eq!(gb_to_bytes(1.0).unwrap(), 1024 * 1024 * 1024);
|
||||
assert_eq!(gb_to_bytes(0.5).unwrap(), 512 * 1024 * 1024);
|
||||
assert!(gb_to_bytes(0.0).is_err());
|
||||
assert!(gb_to_bytes(-4.0).is_err());
|
||||
assert!(gb_to_bytes(f64::NAN).is_err());
|
||||
assert!(gb_to_bytes(f64::INFINITY).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_query_wins_over_repo_filters() {
|
||||
let query = search_query_from_filters(
|
||||
Some("qwen".to_string()),
|
||||
Some("unsloth/Llama-3.2"),
|
||||
Some("-GGUF"),
|
||||
);
|
||||
assert_eq!(query, "qwen");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repo_prefix_is_used_when_query_is_absent() {
|
||||
let query = search_query_from_filters(None, Some("unsloth/Llama-3.2"), None);
|
||||
assert_eq!(query, "Llama-3.2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repo_suffix_is_used_when_prefix_yields_nothing() {
|
||||
let query = search_query_from_filters(None, Some("///"), Some("-Qwen3-GGUF"));
|
||||
assert_eq!(query, "Qwen3-GGUF");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_when_nothing_is_provided() {
|
||||
assert_eq!(search_query_from_filters(None, None, None), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repo_filter_takes_last_path_segment_and_trims_separators() {
|
||||
assert_eq!(
|
||||
search_term_from_repo_filter("unsloth/Llama-3.2"),
|
||||
"Llama-3.2"
|
||||
);
|
||||
assert_eq!(search_term_from_repo_filter("-GGUF"), "GGUF");
|
||||
assert_eq!(search_term_from_repo_filter("/bartowski/"), "bartowski");
|
||||
assert_eq!(search_term_from_repo_filter("_model_."), "model");
|
||||
assert_eq!(search_term_from_repo_filter(""), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn format_size_reports_unknown_for_zero() {
|
||||
assert_eq!(format_size(0), "unknown");
|
||||
assert_eq!(format_size(1024 * 1024 * 1024), "1.0GB");
|
||||
assert_eq!(format_size(3 * 1024 * 1024 * 1024 / 2), "1.5GB");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Executable
+465
@@ -0,0 +1,465 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
show_usage() {
|
||||
echo "Usage: $0 [options]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " -n, --top-n NUM Number of recommended models to test (default: 3)"
|
||||
echo " -m, --models MODELS Comma-separated download ids. Skips search."
|
||||
echo " -o, --output-dir DIR Directory for logs (default: ./local-model-smoke-results)"
|
||||
echo " --ram-gb NUM Override RAM passed to goose lm search"
|
||||
echo " --instruction TEXT Prompt to send to each model"
|
||||
echo " --repo-prefix TEXT Forwarded to goose lm search"
|
||||
echo " --repo-suffix TEXT Forwarded to goose lm search"
|
||||
echo " --quant TEXT Forwarded to goose lm search"
|
||||
echo " --download-retries N Retry model downloads after HF rate limits (default: 3)"
|
||||
echo " --retry-delay SEC Initial retry delay for HF rate limits (default: 60)"
|
||||
echo " --run-timeout SEC Kill a model run after this many seconds (default: 600, 0 disables)"
|
||||
echo " --keep-downloads Do not delete models after testing"
|
||||
echo " -h, --help Show this help message"
|
||||
echo ""
|
||||
echo "Environment:"
|
||||
echo " GOOSE_BIN Optional goose binary path"
|
||||
echo " SKIP_BUILD Skip cargo build when set"
|
||||
}
|
||||
|
||||
TOP_N=3
|
||||
OUTPUT_DIR="./local-model-smoke-results"
|
||||
MODEL_LIST=""
|
||||
RAM_GB=""
|
||||
INSTRUCTION="Say hello in one short sentence. Do not use tools."
|
||||
REPO_PREFIX="unsloth/"
|
||||
REPO_SUFFIX=""
|
||||
QUANT="Q4"
|
||||
DOWNLOAD_RETRIES=3
|
||||
RETRY_DELAY=60
|
||||
RUN_TIMEOUT=600
|
||||
KEEP_DOWNLOADS=false
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
-n|--top-n)
|
||||
TOP_N="$2"
|
||||
shift 2
|
||||
;;
|
||||
-o|--output-dir)
|
||||
OUTPUT_DIR="$2"
|
||||
shift 2
|
||||
;;
|
||||
-m|--models)
|
||||
MODEL_LIST="$2"
|
||||
shift 2
|
||||
;;
|
||||
--ram-gb)
|
||||
RAM_GB="$2"
|
||||
shift 2
|
||||
;;
|
||||
--instruction)
|
||||
INSTRUCTION="$2"
|
||||
shift 2
|
||||
;;
|
||||
--repo-prefix)
|
||||
REPO_PREFIX="$2"
|
||||
shift 2
|
||||
;;
|
||||
--repo-suffix)
|
||||
REPO_SUFFIX="$2"
|
||||
shift 2
|
||||
;;
|
||||
--quant)
|
||||
QUANT="$2"
|
||||
shift 2
|
||||
;;
|
||||
--download-retries)
|
||||
DOWNLOAD_RETRIES="$2"
|
||||
shift 2
|
||||
;;
|
||||
--retry-delay)
|
||||
RETRY_DELAY="$2"
|
||||
shift 2
|
||||
;;
|
||||
--run-timeout)
|
||||
RUN_TIMEOUT="$2"
|
||||
shift 2
|
||||
;;
|
||||
--keep-downloads)
|
||||
KEEP_DOWNLOADS=true
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
show_usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Error: Unknown option: $1"
|
||||
show_usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if ! [[ "$TOP_N" =~ ^[0-9]+$ ]] || [[ "$TOP_N" -eq 0 ]]; then
|
||||
echo "Error: --top-n must be a positive integer"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! [[ "$DOWNLOAD_RETRIES" =~ ^[0-9]+$ ]]; then
|
||||
echo "Error: --download-retries must be a non-negative integer"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! [[ "$RETRY_DELAY" =~ ^[0-9]+$ ]]; then
|
||||
echo "Error: --retry-delay must be a non-negative integer"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! [[ "$RUN_TIMEOUT" =~ ^[0-9]+$ ]]; then
|
||||
echo "Error: --run-timeout must be a non-negative integer"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v jq >/dev/null 2>&1; then
|
||||
echo "Error: jq is required"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
|
||||
if [[ -z "${SKIP_BUILD:-}" && -z "${GOOSE_BIN:-}" ]]; then
|
||||
echo "Building goose..."
|
||||
(cd "$REPO_ROOT" && cargo build -p goose-cli --features local-inference --bin goose)
|
||||
echo ""
|
||||
fi
|
||||
|
||||
GOOSE_BIN="${GOOSE_BIN:-$REPO_ROOT/target/debug/goose}"
|
||||
if [[ ! -x "$GOOSE_BIN" ]]; then
|
||||
echo "Error: goose binary not found or not executable: $GOOSE_BIN"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
EXISTING_MODELS_FILE="$OUTPUT_DIR/existing-models.txt"
|
||||
RESULTS_FILE="$OUTPUT_DIR/results.tsv"
|
||||
"$GOOSE_BIN" lm list | awk 'NR > 2 && $4 == "✓" { print $1 }' > "$EXISTING_MODELS_FILE"
|
||||
printf "status\tmodel_id\tdetail\n" > "$RESULTS_FILE"
|
||||
|
||||
TEMP_HF_CACHE_ROOT=""
|
||||
TEMP_MODELS=()
|
||||
|
||||
cleanup_temp_models() {
|
||||
local model_id
|
||||
local cleanup_failed=false
|
||||
|
||||
for model_id in "${TEMP_MODELS[@]}"; do
|
||||
if ! "$GOOSE_BIN" lm delete "$model_id" >/dev/null 2>&1; then
|
||||
cleanup_failed=true
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "$cleanup_failed" = true ]]; then
|
||||
echo "Warning: could not unregister all temporary models; cache retained at $TEMP_HF_CACHE_ROOT" >&2
|
||||
return
|
||||
fi
|
||||
|
||||
if [[ -n "$TEMP_HF_CACHE_ROOT" && -d "$TEMP_HF_CACHE_ROOT" ]]; then
|
||||
rm -rf -- "$TEMP_HF_CACHE_ROOT"
|
||||
fi
|
||||
}
|
||||
|
||||
untrack_temp_model() {
|
||||
local target="$1"
|
||||
local model_id
|
||||
local remaining=()
|
||||
|
||||
for model_id in "${TEMP_MODELS[@]}"; do
|
||||
if [[ "$model_id" != "$target" ]]; then
|
||||
remaining+=("$model_id")
|
||||
fi
|
||||
done
|
||||
TEMP_MODELS=("${remaining[@]}")
|
||||
}
|
||||
|
||||
if [[ "$KEEP_DOWNLOADS" = false ]]; then
|
||||
TEMP_HF_CACHE_ROOT=$(mktemp -d)
|
||||
fi
|
||||
trap cleanup_temp_models EXIT
|
||||
|
||||
MODELS=()
|
||||
if [[ -n "$MODEL_LIST" ]]; then
|
||||
IFS=',' read -ra REQUESTED_MODELS <<< "$MODEL_LIST"
|
||||
for model in "${REQUESTED_MODELS[@]}"; do
|
||||
repo="${model%%:*}"
|
||||
variant="${model#*:}"
|
||||
if [[ "$variant" = "$model" ]]; then
|
||||
variant="manual"
|
||||
fi
|
||||
MODELS+=("$repo"$'\t'"$model"$'\t'"$model"$'\t'"$variant"$'\t'"0")
|
||||
done
|
||||
else
|
||||
search_query="${REPO_PREFIX%/}"
|
||||
SEARCH_ARGS=(lm search)
|
||||
if [[ -n "$search_query" ]]; then
|
||||
SEARCH_ARGS+=("$search_query")
|
||||
fi
|
||||
SEARCH_ARGS+=(--limit "$TOP_N" --json)
|
||||
if [[ -n "$RAM_GB" ]]; then
|
||||
SEARCH_ARGS+=(--ram-gb "$RAM_GB")
|
||||
fi
|
||||
if [[ -n "$REPO_PREFIX" ]]; then
|
||||
SEARCH_ARGS+=(--repo-prefix "$REPO_PREFIX")
|
||||
fi
|
||||
if [[ -n "$REPO_SUFFIX" ]]; then
|
||||
SEARCH_ARGS+=(--repo-suffix "$REPO_SUFFIX")
|
||||
fi
|
||||
if [[ -n "$QUANT" ]]; then
|
||||
SEARCH_ARGS+=(--quant "$QUANT")
|
||||
fi
|
||||
|
||||
SEARCH_JSON="$OUTPUT_DIR/search.json"
|
||||
echo "Finding recommended local models..."
|
||||
"$GOOSE_BIN" "${SEARCH_ARGS[@]}" > "$SEARCH_JSON"
|
||||
|
||||
while IFS= read -r model_row; do
|
||||
MODELS+=("$model_row")
|
||||
done < <(
|
||||
jq -r --argjson limit "$TOP_N" '
|
||||
[.[] | select(.recommended_variant != null)]
|
||||
| .[:$limit][]
|
||||
| [
|
||||
.repo_id,
|
||||
.recommended_variant.model_id,
|
||||
.recommended_variant.download_id,
|
||||
.recommended_variant.label,
|
||||
(.recommended_variant.size_bytes | tostring)
|
||||
]
|
||||
| @tsv
|
||||
' "$SEARCH_JSON"
|
||||
)
|
||||
fi
|
||||
|
||||
if [[ ${#MODELS[@]} -eq 0 ]]; then
|
||||
echo "No recommended models found."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
RESULTS=()
|
||||
OVERALL_SUCCESS=true
|
||||
|
||||
record_result() {
|
||||
local status="$1"
|
||||
local model_id="$2"
|
||||
local detail="$3"
|
||||
RESULTS+=("$status $model_id${detail:+ - $detail}")
|
||||
printf "%s\t%s\t%s\n" "$status" "$model_id" "$detail" >> "$RESULTS_FILE"
|
||||
}
|
||||
|
||||
summarize_goose_error() {
|
||||
awk '
|
||||
/Ran into this error:/ {
|
||||
sub(/^.*Ran into this error: /, "")
|
||||
print
|
||||
found = 1
|
||||
exit
|
||||
}
|
||||
/Request failed:/ {
|
||||
sub(/^.*Request failed: /, "Request failed: ")
|
||||
print
|
||||
found = 1
|
||||
exit
|
||||
}
|
||||
/Provider error:/ {
|
||||
sub(/^.*Provider error: /, "Provider error: ")
|
||||
print
|
||||
found = 1
|
||||
exit
|
||||
}
|
||||
END { if (!found) exit 1 }
|
||||
' "$1"
|
||||
}
|
||||
|
||||
download_once() {
|
||||
local download_id="$1"
|
||||
local cache_root="$2"
|
||||
|
||||
if [[ -n "$cache_root" ]]; then
|
||||
HF_HUB_CACHE="$cache_root/hub" \
|
||||
HF_XET_CACHE="$cache_root/xet" \
|
||||
"$GOOSE_BIN" lm download "$download_id"
|
||||
else
|
||||
"$GOOSE_BIN" lm download "$download_id"
|
||||
fi
|
||||
}
|
||||
|
||||
download_model() {
|
||||
local download_id="$1"
|
||||
local log_file="$2"
|
||||
local cache_root="$3"
|
||||
local attempt=1
|
||||
local delay="$RETRY_DELAY"
|
||||
|
||||
while true; do
|
||||
: > "$log_file"
|
||||
if download_once "$download_id" "$cache_root" 2>&1 | tee "$log_file"; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
if ! grep -q "429 Too Many Requests" "$log_file"; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ "$attempt" -gt "$DOWNLOAD_RETRIES" ]]; then
|
||||
return 2
|
||||
fi
|
||||
|
||||
echo "Hugging Face rate limit hit. Retrying in ${delay}s ($attempt/$DOWNLOAD_RETRIES)..."
|
||||
sleep "$delay"
|
||||
attempt=$((attempt + 1))
|
||||
if [[ "$delay" -gt 0 ]]; then
|
||||
delay=$((delay * 2))
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
run_model() {
|
||||
local model_id="$1"
|
||||
local log_file="$2"
|
||||
|
||||
if [[ "$RUN_TIMEOUT" -eq 0 ]]; then
|
||||
GOOSE_MODE=auto GOOSE_PROVIDER=local GOOSE_MODEL="$model_id" \
|
||||
"$GOOSE_BIN" run --no-profile --text "$INSTRUCTION" 2>&1 | tee "$log_file"
|
||||
return "${PIPESTATUS[0]}"
|
||||
fi
|
||||
|
||||
perl -e '
|
||||
my $timeout = shift;
|
||||
my $pid = fork();
|
||||
die "fork failed: $!" unless defined $pid;
|
||||
if ($pid == 0) {
|
||||
exec @ARGV;
|
||||
die "exec failed: $!";
|
||||
}
|
||||
local $SIG{ALRM} = sub {
|
||||
kill "TERM", $pid;
|
||||
sleep 2;
|
||||
kill "KILL", $pid;
|
||||
exit 124;
|
||||
};
|
||||
alarm $timeout;
|
||||
waitpid($pid, 0);
|
||||
my $status = $?;
|
||||
alarm 0;
|
||||
exit($status & 127 ? 128 + ($status & 127) : $status >> 8);
|
||||
' \
|
||||
"$RUN_TIMEOUT" \
|
||||
env GOOSE_MODE=auto GOOSE_PROVIDER=local GOOSE_MODEL="$model_id" \
|
||||
"$GOOSE_BIN" run --no-profile --text "$INSTRUCTION" 2>&1 | tee "$log_file"
|
||||
return "${PIPESTATUS[0]}"
|
||||
}
|
||||
|
||||
echo "Testing ${#MODELS[@]} model(s)"
|
||||
echo ""
|
||||
|
||||
for row in "${MODELS[@]}"; do
|
||||
IFS=$'\t' read -r repo_id model_id download_id label size_bytes <<< "$row"
|
||||
safe_model=$(echo "$model_id" | tr '/:' '__' | tr -cd '[:alnum:]_.-')
|
||||
download_log="$OUTPUT_DIR/$safe_model.download.log"
|
||||
run_log="$OUTPUT_DIR/$safe_model.run.log"
|
||||
delete_log="$OUTPUT_DIR/$safe_model.delete.log"
|
||||
size_gb=$(awk "BEGIN { printf \"%.1f\", $size_bytes / 1024 / 1024 / 1024 }")
|
||||
|
||||
echo "=========================================================="
|
||||
echo "Model: $model_id"
|
||||
echo "Repo: $repo_id"
|
||||
echo "Variant: $label (${size_gb}GB)"
|
||||
echo "=========================================================="
|
||||
|
||||
existed_before=false
|
||||
if grep -Fxq "$model_id" "$EXISTING_MODELS_FILE"; then
|
||||
existed_before=true
|
||||
fi
|
||||
|
||||
downloaded=false
|
||||
if [[ "$existed_before" = true ]]; then
|
||||
echo "Using pre-existing download for $model_id"
|
||||
downloaded=true
|
||||
else
|
||||
if [[ -n "$TEMP_HF_CACHE_ROOT" ]]; then
|
||||
TEMP_MODELS+=("$model_id")
|
||||
fi
|
||||
set +e
|
||||
download_model "$download_id" "$download_log" "$TEMP_HF_CACHE_ROOT"
|
||||
download_status=$?
|
||||
set -e
|
||||
if [[ "$download_status" -eq 0 ]]; then
|
||||
downloaded=true
|
||||
elif [[ "$download_status" -eq 2 ]]; then
|
||||
echo "Download rate limited for $model_id"
|
||||
record_result "FAIL" "$model_id" "Hugging Face rate limited"
|
||||
OVERALL_SUCCESS=false
|
||||
else
|
||||
echo "Download failed for $model_id"
|
||||
record_result "FAIL" "$model_id" "download failed"
|
||||
OVERALL_SUCCESS=false
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$downloaded" = true ]]; then
|
||||
set +e
|
||||
run_model "$model_id" "$run_log"
|
||||
run_status=$?
|
||||
set -e
|
||||
|
||||
if [[ ! -s "$run_log" ]]; then
|
||||
echo "Run produced no output for $model_id"
|
||||
record_result "FAIL" "$model_id" "empty output"
|
||||
OVERALL_SUCCESS=false
|
||||
elif [[ "$run_status" -eq 124 || "$run_status" -eq 142 ]]; then
|
||||
echo "Run timed out after ${RUN_TIMEOUT}s for $model_id"
|
||||
record_result "FAIL" "$model_id" "run timed out"
|
||||
OVERALL_SUCCESS=false
|
||||
elif error_summary=$(summarize_goose_error "$run_log"); then
|
||||
echo "Goose reported an error for $model_id"
|
||||
echo " $error_summary"
|
||||
record_result "FAIL" "$model_id" "$error_summary"
|
||||
OVERALL_SUCCESS=false
|
||||
elif [[ "$run_status" -eq 0 ]]; then
|
||||
echo "Run passed for $model_id"
|
||||
record_result "PASS" "$model_id" ""
|
||||
else
|
||||
echo "Run replied but exited with status $run_status for $model_id"
|
||||
record_result "FAIL" "$model_id" "replied but exited $run_status"
|
||||
OVERALL_SUCCESS=false
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$KEEP_DOWNLOADS" = false && "$downloaded" = true && "$existed_before" = false ]]; then
|
||||
if "$GOOSE_BIN" lm delete "$model_id" 2>&1 | tee "$delete_log"; then
|
||||
untrack_temp_model "$model_id"
|
||||
echo "Unregistered $model_id; its temporary cache will be removed at exit"
|
||||
else
|
||||
echo "Delete failed for $model_id"
|
||||
record_result "FAIL" "$model_id" "delete failed"
|
||||
OVERALL_SUCCESS=false
|
||||
fi
|
||||
elif [[ "$KEEP_DOWNLOADS" = false && "$downloaded" = true ]]; then
|
||||
echo "Keeping $model_id because it existed before this run"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
done
|
||||
|
||||
echo "=== Test Summary ==="
|
||||
for result in "${RESULTS[@]}"; do
|
||||
echo "$result"
|
||||
done
|
||||
|
||||
if [[ "$OVERALL_SUCCESS" = false ]]; then
|
||||
echo ""
|
||||
echo "Some local model smoke tests failed."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "All local model smoke tests passed."
|
||||
Executable
+281
@@ -0,0 +1,281 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
show_usage() {
|
||||
echo "Usage: $0 [options]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " -n, --count NUM Number of OpenRouter models to test (default: 10)"
|
||||
echo " -m, --models MODELS Comma-separated model list. Skips fetching models."
|
||||
echo " -o, --output-dir DIR Directory for logs (default: ./openrouter-toolcall-results)"
|
||||
echo " -s, --sort SORT OpenRouter model sort (default: top-weekly)"
|
||||
echo " --run-timeout SEC Kill a model run after this many seconds (default: 180, 0 disables)"
|
||||
echo " -h, --help Show this help message"
|
||||
echo ""
|
||||
echo "Environment:"
|
||||
echo " OPENROUTER_API_KEY Required by goose's OpenRouter provider"
|
||||
echo " OPENROUTER_HOST Optional OpenRouter host (default: https://openrouter.ai)"
|
||||
echo " GOOSE_BIN Optional goose binary path"
|
||||
echo " SKIP_BUILD Skip cargo build when set"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 --count 5"
|
||||
echo " $0 --models 'anthropic/claude-sonnet-4.5,google/gemini-2.5-flash'"
|
||||
}
|
||||
|
||||
MODEL_COUNT=10
|
||||
MODEL_LIST=""
|
||||
OUTPUT_DIR="./openrouter-toolcall-results"
|
||||
MODEL_SORT="top-weekly"
|
||||
RUN_TIMEOUT=180
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
-n|--count)
|
||||
MODEL_COUNT="$2"
|
||||
shift 2
|
||||
;;
|
||||
-m|--models)
|
||||
MODEL_LIST="$2"
|
||||
shift 2
|
||||
;;
|
||||
-o|--output-dir)
|
||||
OUTPUT_DIR="$2"
|
||||
shift 2
|
||||
;;
|
||||
-s|--sort)
|
||||
MODEL_SORT="$2"
|
||||
shift 2
|
||||
;;
|
||||
--run-timeout)
|
||||
RUN_TIMEOUT="$2"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
show_usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Error: Unknown option: $1"
|
||||
show_usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -z "${OPENROUTER_API_KEY:-}" ]]; then
|
||||
echo "Error: OPENROUTER_API_KEY must be set"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v jq >/dev/null 2>&1; then
|
||||
echo "Error: jq is required"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v curl >/dev/null 2>&1; then
|
||||
echo "Error: curl is required"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v uv >/dev/null 2>&1; then
|
||||
echo "Error: uv is required to run the temporary FastMCP server"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! [[ "$RUN_TIMEOUT" =~ ^[0-9]+$ ]]; then
|
||||
echo "Error: --run-timeout must be a non-negative integer"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
|
||||
if [[ -z "${SKIP_BUILD:-}" && -z "${GOOSE_BIN:-}" ]]; then
|
||||
echo "Building goose..."
|
||||
(cd "$REPO_ROOT" && cargo build --bin goose)
|
||||
echo ""
|
||||
fi
|
||||
|
||||
GOOSE_BIN="${GOOSE_BIN:-$REPO_ROOT/target/debug/goose}"
|
||||
if [[ ! -x "$GOOSE_BIN" ]]; then
|
||||
echo "Error: goose binary not found or not executable: $GOOSE_BIN"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
MODELS=()
|
||||
if [[ -n "$MODEL_LIST" ]]; then
|
||||
IFS=',' read -ra MODELS <<< "$MODEL_LIST"
|
||||
else
|
||||
OPENROUTER_HOST="${OPENROUTER_HOST:-https://openrouter.ai}"
|
||||
MODELS_URL="$OPENROUTER_HOST/api/v1/models?supported_parameters=tools&sort=$MODEL_SORT"
|
||||
|
||||
echo "Fetching OpenRouter models: $MODELS_URL"
|
||||
MODELS_JSON=$(curl --fail --silent --show-error --max-time 30 "$MODELS_URL")
|
||||
while IFS= read -r model; do
|
||||
MODELS+=("$model")
|
||||
done < <(jq -r --argjson limit "$MODEL_COUNT" '.data[:$limit][] | .id' <<< "$MODELS_JSON")
|
||||
fi
|
||||
|
||||
if [[ ${#MODELS[@]} -eq 0 ]]; then
|
||||
echo "Error: no models found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TESTDIR=$(mktemp -d)
|
||||
trap 'rm -rf "$TESTDIR"' EXIT
|
||||
|
||||
cat > "$TESTDIR/weather.py" << 'EOF'
|
||||
from typing import Annotated
|
||||
from fastmcp import FastMCP
|
||||
|
||||
mcp = FastMCP("weather")
|
||||
|
||||
@mcp.tool
|
||||
def get_weather(
|
||||
location: Annotated[str, "City or place to check"],
|
||||
) -> Annotated[str, "Weather report"]:
|
||||
"""Get the current weather for a location."""
|
||||
return f"GOOSE_TOOL_CALL_OK: The weather in {location} is 68 F and clear."
|
||||
EOF
|
||||
|
||||
cat > "$TESTDIR/recipe.yaml" << 'EOF'
|
||||
title: OpenRouter Tool Call Test
|
||||
description: Test a model can call a simple MCP tool through goose
|
||||
prompt: Use the get_weather tool to check the weather in San Francisco. Do not answer from memory.
|
||||
extensions:
|
||||
- name: weather
|
||||
cmd: uv
|
||||
args:
|
||||
- run
|
||||
- --with
|
||||
- fastmcp==2.14.4
|
||||
- fastmcp
|
||||
- run
|
||||
- weather.py
|
||||
type: stdio
|
||||
EOF
|
||||
|
||||
RESULTS=()
|
||||
OVERALL_SUCCESS=true
|
||||
|
||||
summarize_error() {
|
||||
awk '
|
||||
/Ran into this error:/ {
|
||||
sub(/^.*Ran into this error: /, "")
|
||||
print
|
||||
found = 1
|
||||
exit
|
||||
}
|
||||
/Request failed:/ {
|
||||
sub(/^.*Request failed: /, "Request failed: ")
|
||||
print
|
||||
found = 1
|
||||
exit
|
||||
}
|
||||
/Provider error:/ {
|
||||
sub(/^.*Provider error: /, "Provider error: ")
|
||||
print
|
||||
found = 1
|
||||
exit
|
||||
}
|
||||
END { if (!found) exit 1 }
|
||||
' "$1"
|
||||
}
|
||||
|
||||
run_model() {
|
||||
local model="$1"
|
||||
|
||||
if [[ "$RUN_TIMEOUT" -eq 0 ]]; then
|
||||
GOOSE_MODE=auto GOOSE_PROVIDER=openrouter GOOSE_MODEL="$model" \
|
||||
"$GOOSE_BIN" run --no-profile --max-turns 4 --recipe recipe.yaml
|
||||
return $?
|
||||
fi
|
||||
|
||||
perl -e '
|
||||
my $timeout = shift;
|
||||
my $pid = fork();
|
||||
die "fork failed: $!" unless defined $pid;
|
||||
if ($pid == 0) {
|
||||
exec @ARGV;
|
||||
die "exec failed: $!";
|
||||
}
|
||||
local $SIG{ALRM} = sub {
|
||||
kill "TERM", $pid;
|
||||
sleep 2;
|
||||
kill "KILL", $pid;
|
||||
exit 124;
|
||||
};
|
||||
alarm $timeout;
|
||||
waitpid($pid, 0);
|
||||
my $status = $?;
|
||||
alarm 0;
|
||||
exit($status & 127 ? 128 + ($status & 127) : $status >> 8);
|
||||
' \
|
||||
"$RUN_TIMEOUT" \
|
||||
env GOOSE_MODE=auto GOOSE_PROVIDER=openrouter GOOSE_MODEL="$model" \
|
||||
"$GOOSE_BIN" run --no-profile --max-turns 4 --recipe recipe.yaml
|
||||
}
|
||||
|
||||
echo "Testing ${#MODELS[@]} OpenRouter model(s)"
|
||||
echo ""
|
||||
|
||||
for model in "${MODELS[@]}"; do
|
||||
safe_model=$(echo "$model" | tr '/:' '__' | tr -cd '[:alnum:]_.-')
|
||||
log_file="$OUTPUT_DIR/$safe_model.log"
|
||||
|
||||
echo "=========================================================="
|
||||
echo "Model: $model"
|
||||
echo "Log: $log_file"
|
||||
echo "=========================================================="
|
||||
|
||||
if (cd "$TESTDIR" && run_model "$model" 2>&1) | tee "$log_file"; then
|
||||
if error_summary=$(summarize_error "$log_file"); then
|
||||
echo "✗ Goose reported an error for $model"
|
||||
echo " $error_summary"
|
||||
RESULTS+=("✗ $model - $error_summary")
|
||||
OVERALL_SUCCESS=false
|
||||
elif grep -qE "(get_weather \| weather)|(▸.*get_weather.*weather)" "$log_file" && \
|
||||
grep -Fq "GOOSE_TOOL_CALL_OK:" "$log_file"; then
|
||||
echo "✓ Tool call passed for $model"
|
||||
RESULTS+=("✓ $model")
|
||||
elif grep -qE "(get_weather \| weather)|(▸.*get_weather.*weather)" "$log_file"; then
|
||||
echo "✗ Tool call did not return a successful result for $model"
|
||||
RESULTS+=("✗ $model - no successful get_weather result found")
|
||||
OVERALL_SUCCESS=false
|
||||
else
|
||||
echo "✗ Tool call not found for $model"
|
||||
RESULTS+=("✗ $model - no get_weather call found")
|
||||
OVERALL_SUCCESS=false
|
||||
fi
|
||||
else
|
||||
run_status=${PIPESTATUS[0]}
|
||||
echo "✗ Goose run failed for $model"
|
||||
if [[ "$run_status" -eq 124 ]]; then
|
||||
RESULTS+=("✗ $model - run timed out")
|
||||
elif error_summary=$(summarize_error "$log_file"); then
|
||||
echo " $error_summary"
|
||||
RESULTS+=("✗ $model - $error_summary")
|
||||
else
|
||||
RESULTS+=("✗ $model - goose run failed")
|
||||
fi
|
||||
OVERALL_SUCCESS=false
|
||||
fi
|
||||
|
||||
echo ""
|
||||
done
|
||||
|
||||
echo "=== Test Summary ==="
|
||||
for result in "${RESULTS[@]}"; do
|
||||
echo "$result"
|
||||
done
|
||||
|
||||
if [[ "$OVERALL_SUCCESS" = false ]]; then
|
||||
echo ""
|
||||
echo "Some OpenRouter tool call tests failed."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "All OpenRouter tool call tests passed."
|
||||
Reference in New Issue
Block a user