tkmind/tkmind_go
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

test(mcp): add image tool test and consolidate MCP test fixtures (#7019)

Browse Source 
Signed-off-by: Adrian Cole <adrian@tetrate.io>
This commit is contained in:
Adrian Cole
2026-02-09 09:12:11 +08:00
committed by GitHub
parent 08b89ca66b
commit 1371f5df4a
26 changed files with 1129 additions and 1070 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/goose/Cargo.toml
+1
View File
@@ -148,6 +148,7 @@ ctor = "0.2.9"
test-case = { workspace = true }
env-lock = { workspace = true }
rmcp = { workspace = true, features = ["transport-streamable-http-server"] }
goose-test-support = { path = "../goose-test-support" }
[[example]]
name = "agent"
crates/goose/src/providers/chatgpt_codex.rs
+3 -5
View File
@@ -997,6 +997,7 @@ impl Provider for ChatGptCodexProvider {
mod tests {
use super::*;
use crate::conversation::message::Message;
use goose_test_support::TEST_IMAGE_B64;
use jsonwebtoken::{Algorithm, EncodingKey, Header};
use rmcp::model::{CallToolRequestParams, CallToolResult, Content, ErrorCode, ErrorData};
use rmcp::object;
@@ -1004,9 +1005,6 @@ mod tests {
use wiremock::matchers::{body_string_contains, method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
/// 1x1 transparent PNG, base64-encoded.
const TINY_PNG_B64: &str = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR4nGNgYAAAAAMAASsJTYQAAAAASUVORK5CYII=";
fn input_kinds(payload: &Value) -> Vec<String> {
payload["input"]
.as_array()
@@ -1106,7 +1104,7 @@ mod tests {
vec![
Message::user()
.with_text("describe this")
.with_image(TINY_PNG_B64, "image/png"),
.with_image(TEST_IMAGE_B64, "image/png"),
],
vec![
"message:user".to_string(),
@@ -1122,7 +1120,7 @@ mod tests {
#[test]
fn test_image_url_format() {
let messages = vec![Message::user().with_image(TINY_PNG_B64, "image/png")];
let messages = vec![Message::user().with_image(TEST_IMAGE_B64, "image/png")];
let items = build_input_items(&messages).unwrap();
// The image is inside the content array of the user message
let content = items[0]["content"].as_array().unwrap();
crates/goose/src/providers/codex.rs
+3 -5
View File
@@ -670,11 +670,9 @@ impl Provider for CodexProvider {
#[cfg(test)]
mod tests {
use super::*;
use goose_test_support::TEST_IMAGE_B64;
use test_case::test_case;
/// 1x1 transparent PNG, base64-encoded.
const TINY_PNG_B64: &str = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR4nGNgYAAAAAMAASsJTYQAAAAASUVORK5CYII=";
#[test]
fn test_codex_metadata() {
let metadata = CodexProvider::metadata();
@@ -694,7 +692,7 @@ mod tests {
let dir = tempfile::tempdir().unwrap();
let messages = vec![Message::user()
.with_text("Describe")
.with_image(TINY_PNG_B64, mime)];
.with_image(TEST_IMAGE_B64, mime)];
let (_prompt, temp_files) = prepare_input("", &messages, dir.path()).unwrap();
assert_eq!(temp_files.len(), 1);
let path = temp_files[0].path();
@@ -712,7 +710,7 @@ mod tests {
let dir = tempfile::tempdir().unwrap();
let messages = vec![Message::user()
.with_text("Describe")
.with_image(TINY_PNG_B64, mime)];
.with_image(TEST_IMAGE_B64, mime)];
let err = prepare_input("", &messages, dir.path()).unwrap_err();
assert!(
err.to_string().contains("Unsupported image MIME type"),
crates/goose/src/providers/formats/bedrock.rs
+5 -7
View File
@@ -386,11 +386,9 @@ pub fn from_bedrock_json(document: &Document) -> Result<Value> {
mod tests {
use super::*;
use anyhow::Result;
use goose_test_support::TEST_IMAGE_B64;
use rmcp::model::{AnnotateAble, RawImageContent};
// Base64 encoded 1x1 PNG image for testing
const TEST_IMAGE_BASE64: &str = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==";
#[test]
fn test_to_bedrock_image_supported_formats() -> Result<()> {
let supported_formats = [
@@ -403,7 +401,7 @@ mod tests {
for mime_type in supported_formats {
let image = RawImageContent {
data: TEST_IMAGE_BASE64.to_string(),
data: TEST_IMAGE_B64.to_string(),
mime_type: mime_type.to_string(),
meta: None,
}
@@ -419,7 +417,7 @@ mod tests {
#[test]
fn test_to_bedrock_image_unsupported_format() {
let image = RawImageContent {
data: TEST_IMAGE_BASE64.to_string(),
data: TEST_IMAGE_B64.to_string(),
mime_type: "image/bmp".to_string(),
meta: None,
}
@@ -450,7 +448,7 @@ mod tests {
#[test]
fn test_to_bedrock_message_content_image() -> Result<()> {
let image = RawImageContent {
data: TEST_IMAGE_BASE64.to_string(),
data: TEST_IMAGE_B64.to_string(),
mime_type: "image/png".to_string(),
meta: None,
}
@@ -467,7 +465,7 @@ mod tests {
#[test]
fn test_to_bedrock_tool_result_content_block_image() -> Result<()> {
let content = Content::image(TEST_IMAGE_BASE64.to_string(), "image/png".to_string());
let content = Content::image(TEST_IMAGE_B64.to_string(), "image/png".to_string());
let result = to_bedrock_tool_result_content_block("test_id", content)?;
// Verify the wrapper correctly converts Content::Image to ToolResultContentBlock::Image
crates/goose/src/providers/ollama.rs
+1
View File
@@ -38,6 +38,7 @@ pub const OLLAMA_DEFAULT_PORT: u16 = 11434;
pub const OLLAMA_DEFAULT_MODEL: &str = "qwen3";
pub const OLLAMA_KNOWN_MODELS: &[&str] = &[
OLLAMA_DEFAULT_MODEL,
"qwen3-vl",
"qwen3-coder:30b",
"qwen3-coder:480b-cloud",
];
crates/goose/tests/providers.rs
+114 -193
View File
@@ -1,5 +1,7 @@
use anyhow::Result;
use dotenvy::dotenv;
use goose::agents::{ExtensionManager, PromptManager};
use goose::config::ExtensionConfig;
use goose::conversation::message::{Message, MessageContent};
use goose::providers::anthropic::ANTHROPIC_DEFAULT_MODEL;
use goose::providers::azure::AZURE_DEFAULT_MODEL;
@@ -10,17 +12,15 @@ use goose::providers::databricks::DATABRICKS_DEFAULT_MODEL;
use goose::providers::errors::ProviderError;
use goose::providers::google::GOOGLE_DEFAULT_MODEL;
use goose::providers::litellm::LITELLM_DEFAULT_MODEL;
use goose::providers::ollama::OLLAMA_DEFAULT_MODEL;
use goose::providers::openai::OPEN_AI_DEFAULT_MODEL;
use goose::providers::sagemaker_tgi::SAGEMAKER_TGI_DEFAULT_MODEL;
use goose::providers::snowflake::SNOWFLAKE_DEFAULT_MODEL;
use goose::providers::xai::XAI_DEFAULT_MODEL;
use rmcp::model::{AnnotateAble, Content, RawImageContent};
use rmcp::model::{CallToolRequestParams, Tool};
use rmcp::object;
use goose::session::SessionManager;
use goose_test_support::{ExpectedSessionId, McpFixture, FAKE_CODE, TEST_SESSION_ID};
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::{Arc, Mutex};
use tokio_util::sync::CancellationToken;
#[derive(Debug, Clone, Copy)]
enum TestStatus {
@@ -88,11 +88,77 @@ lazy_static::lazy_static! {
struct ProviderTester {
provider: Arc<dyn Provider>,
name: String,
extension_manager: Arc<ExtensionManager>,
}
impl ProviderTester {
fn new(provider: Arc<dyn Provider>, name: String) -> Self {
Self { provider, name }
fn new(
provider: Arc<dyn Provider>,
name: String,
extension_manager: Arc<ExtensionManager>,
) -> Self {
Self {
provider,
name,
extension_manager,
}
}
async fn tool_roundtrip(&self, prompt: &str) -> Result<Message> {
let tools = self
.extension_manager
.get_prefixed_tools(TEST_SESSION_ID, None)
.await
.expect("get_prefixed_tools failed");
let info = self.extension_manager.get_extensions_info().await;
let system = PromptManager::new()
.builder()
.with_extensions(info.into_iter())
.build();
let message = Message::user().with_text(prompt);
let (response1, _) = self
.provider
.complete(
TEST_SESSION_ID,
&system,
std::slice::from_ref(&message),
&tools,
)
.await?;
let tool_req = response1
.content
.iter()
.filter_map(|c| c.as_tool_request())
.next_back()
.expect("Expected provider to return a tool request");
let params = tool_req
.tool_call
.as_ref()
.expect("tool_call should be Ok")
.clone();
let result = self
.extension_manager
.dispatch_tool_call(TEST_SESSION_ID, params, None, CancellationToken::new())
.await
.expect("dispatch failed")
.result
.await
.expect("tool call failed");
let tool_response = Message::user().with_tool_response(&tool_req.id, Ok(result));
let (response2, _) = self
.provider
.complete(
TEST_SESSION_ID,
&system,
&[message, response1, tool_response],
&tools,
)
.await?;
Ok(response2)
}
async fn test_basic_response(&self) -> Result<()> {
@@ -101,7 +167,7 @@ impl ProviderTester {
let (response, _) = self
.provider
.complete(
"test-session-id",
TEST_SESSION_ID,
"You are a helpful assistant.",
&[message],
&[],
@@ -123,94 +189,13 @@ impl ProviderTester {
}
async fn test_tool_usage(&self) -> Result<()> {
let weather_tool = Tool::new(
"get_weather",
"Get the weather for a location",
object!({
"type": "object",
"required": ["location"],
"properties": {
"location": {
"type": "string",
"description": "The city and state, e.g. San Francisco, CA"
}
}
}),
);
let message = Message::user().with_text("What's the weather like in San Francisco?");
let (response1, _) = self
.provider
.complete(
"test-session-id",
"You are a helpful weather assistant.",
std::slice::from_ref(&message),
std::slice::from_ref(&weather_tool),
)
let response = self
.tool_roundtrip("Use the get_code tool and output only its result.")
.await?;
println!("=== {}::reponse1 ===", self.name);
dbg!(&response1);
println!("===================");
assert!(
response1
.content
.iter()
.any(|content| matches!(content, MessageContent::ToolRequest(_))),
"Expected tool request in response"
response.as_concat_text().contains(FAKE_CODE),
"Expected lookup code in final response"
);
let id = &response1
.content
.iter()
.filter_map(|message| message.as_tool_request())
.next_back()
.expect("got tool request")
.id;
let weather = Message::user().with_tool_response(
id,
Ok(rmcp::model::CallToolResult {
content: vec![Content::text(
"
50°F°C
Precipitation: 0%
Humidity: 84%
Wind: 2 mph
Weather
Saturday 9:00 PM
Clear",
)],
structured_content: None,
is_error: Some(false),
meta: None,
}),
);
let (response2, _) = self
.provider
.complete(
"test-session-id",
"You are a helpful weather assistant.",
&[message, response1, weather],
&[weather_tool],
)
.await?;
println!("=== {}::reponse2 ===", self.name);
dbg!(&response2);
println!("===================");
assert!(
response2
.content
.iter()
.any(|content| matches!(content, MessageContent::Text(_))),
"Expected text for final response"
);
Ok(())
}
@@ -236,7 +221,7 @@ impl ProviderTester {
let result = self
.provider
.complete(
"test-session-id",
TEST_SESSION_ID,
"You are a helpful assistant.",
&messages,
&[],
@@ -268,102 +253,15 @@ impl ProviderTester {
}
async fn test_image_content_support(&self) -> Result<()> {
use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
use goose::conversation::message::Message;
use std::fs;
let image_path = "crates/goose/examples/test_assets/test_image.png";
let image_data = match fs::read(image_path) {
Ok(data) => data,
Err(_) => {
println!(
"Test image not found at {}, skipping image test",
image_path
);
return Ok(());
}
};
let base64_image = BASE64.encode(image_data);
let image_content = RawImageContent {
data: base64_image,
mime_type: "image/png".to_string(),
meta: None,
}
.no_annotation();
let message_with_image =
Message::user().with_image(image_content.data.clone(), image_content.mime_type.clone());
let result = self
.provider
.complete(
"test-session-id",
"You are a helpful assistant. Describe what you see in the image briefly.",
&[message_with_image],
&[],
)
.await;
println!("=== {}::image_content_support ===", self.name);
let (response, _) = result?;
println!("Image response: {:?}", response);
let response = self
.tool_roundtrip("Use the get_image tool and describe what you see in its result.")
.await?;
let text = response.as_concat_text().to_lowercase();
assert!(
response
.content
.iter()
.any(|content| matches!(content, MessageContent::Text(_))),
"Expected text response for image"
text.contains("hello goose") || text.contains("test image"),
"Expected response to describe the test image, got: {}",
text
);
println!("===================");
let screenshot_tool = Tool::new(
"get_screenshot",
"Get a screenshot of the current screen",
object!({
"type": "object",
"properties": {}
}),
);
let user_message = Message::user().with_text("Take a screenshot please");
let tool_request = Message::assistant().with_tool_request(
"test_id",
Ok(CallToolRequestParams {
meta: None,
task: None,
name: "get_screenshot".into(),
arguments: Some(object!({})),
}),
);
let tool_response = Message::user().with_tool_response(
"test_id",
Ok(rmcp::model::CallToolResult {
content: vec![Content::image(
image_content.data.clone(),
image_content.mime_type.clone(),
)],
structured_content: None,
is_error: Some(false),
meta: None,
}),
);
let result2 = self
.provider
.complete(
"test-session-id",
"You are a helpful assistant.",
&[user_message, tool_request, tool_response],
&[screenshot_tool],
)
.await;
println!("=== {}::tool_image_response ===", self.name);
let (response, _) = result2?;
println!("Tool image response: {:?}", response);
println!("===================");
Ok(())
}
@@ -452,7 +350,12 @@ async fn test_provider(
original_env
};
let provider = match create_with_named_model(&name.to_lowercase(), model_name).await {
let expected_session_id = ExpectedSessionId::default();
let provider_name = name.to_lowercase();
let mcp = McpFixture::new(Some(expected_session_id.clone())).await;
expected_session_id.set(TEST_SESSION_ID);
let provider = match create_with_named_model(&provider_name, model_name).await {
Ok(p) => p,
Err(e) => {
println!("Skipping {} tests - failed to create provider: {}", name, e);
@@ -475,8 +378,25 @@ async fn test_provider(
}
}
let tester = ProviderTester::new(provider, name.to_string());
match tester.run_test_suite().await {
let temp_dir = tempfile::tempdir()?;
let shared_provider = Arc::new(tokio::sync::Mutex::new(Some(provider.clone())));
let session_manager = Arc::new(SessionManager::new(temp_dir.path().to_path_buf()));
let extension_manager = Arc::new(ExtensionManager::new(shared_provider, session_manager));
extension_manager
.add_extension(
ExtensionConfig::streamable_http("mcp-fixture", &mcp.url, "MCP fixture", 30_u64),
None,
None,
None,
)
.await
.expect("failed to add extension");
let tester = ProviderTester::new(provider, name.to_string(), extension_manager);
let _mcp = mcp;
let result = tester.run_test_suite().await;
match result {
Ok(_) => {
TEST_REPORT.record_pass(name);
Ok(())
@@ -565,7 +485,8 @@ async fn test_databricks_provider() -> Result<()> {
#[tokio::test]
async fn test_ollama_provider() -> Result<()> {
test_provider("Ollama", OLLAMA_DEFAULT_MODEL, &["OLLAMA_HOST"], None).await
// qwen3-vl supports text, tools, and vision (needed for image test)
test_provider("Ollama", "qwen3-vl", &["OLLAMA_HOST"], None).await
}
#[tokio::test]