diff --git a/crates/goose-provider-types/src/formats/anthropic.rs b/crates/goose-provider-types/src/formats/anthropic.rs index a496665b2..49484dea7 100644 --- a/crates/goose-provider-types/src/formats/anthropic.rs +++ b/crates/goose-provider-types/src/formats/anthropic.rs @@ -8,7 +8,10 @@ use crate::mcp_utils::extract_text_from_resource; use crate::model::ModelConfig; use crate::thinking::ThinkingEffort; use anyhow::{anyhow, Result}; -use rmcp::model::{object, CallToolRequestParams, ErrorCode, ErrorData, JsonObject, Role, Tool}; +use rmcp::model::{ + object, CallToolRequestParams, ContentBlock, ErrorCode, ErrorData, JsonObject, + ResourceContents, Role, Tool, +}; use rmcp::object as json_object; use serde_json::{json, Value}; use std::collections::HashSet; @@ -138,6 +141,16 @@ const TOOL_USE_ID_FIELD: &str = "tool_use_id"; const IS_ERROR_FIELD: &str = "is_error"; const SIGNATURE_FIELD: &str = "signature"; const DATA_FIELD: &str = "data"; +const IMAGE_TYPE: &str = "image"; +const DOCUMENT_TYPE: &str = "document"; +const SOURCE_FIELD: &str = "source"; +const BASE64_TYPE: &str = "base64"; +const MEDIA_TYPE_FIELD: &str = "media_type"; +// Claude vision only accepts these image media types; other image/* blobs fall +// through to the text/binary-marker path so an unsupported type (e.g. +// image/svg+xml) doesn't turn the next request into a provider rejection. +const ANTHROPIC_IMAGE_MEDIA_TYPES: [&str; 4] = + ["image/jpeg", "image/png", "image/gif", "image/webp"]; const EVENT_MESSAGE_START: &str = "message_start"; const EVENT_MESSAGE_DELTA: &str = "message_delta"; const EVENT_MESSAGE_STOP: &str = "message_stop"; @@ -217,28 +230,97 @@ fn format_messages_with_options( MessageContentBlock::ToolResponse(tool_response) => { match &tool_response.tool_result { Ok(result) => { - let text = result - .content - .iter() - .filter_map(|c| { - if let Some(t) = c.as_text() { - return Some(t.text.clone()); + let mut blocks: Vec = Vec::new(); + let mut text_parts: Vec = Vec::new(); + let mut has_media = false; + + for c in result.content.iter() { + if let Some(t) = c.as_text() { + text_parts.push(t.text.clone()); + if !t.text.is_empty() { + blocks.push(json!({ + TYPE_FIELD: TEXT_TYPE, + TEXT_TYPE: t.text.clone() + })); } - if let Some(r) = c.as_resource() { - let text = extract_text_from_resource(&r.resource); - if !text.is_empty() { - return Some(text); + continue; + } + if let Some(r) = c.as_resource() { + // Claude only accepts a fixed set of media types, so + // unsupported blobs fall back to text below rather than + // being rejected by the provider. + if let ResourceContents::BlobResourceContents { + blob, + mime_type, + .. + } = &r.resource + { + let mime = mime_type.as_deref().unwrap_or(""); + if ANTHROPIC_IMAGE_MEDIA_TYPES.contains(&mime) { + has_media = true; + blocks.push(json!({ + TYPE_FIELD: IMAGE_TYPE, + SOURCE_FIELD: { + TYPE_FIELD: BASE64_TYPE, + MEDIA_TYPE_FIELD: mime, + DATA_FIELD: blob, + } + })); + continue; + } + if mime == "application/pdf" { + has_media = true; + blocks.push(json!({ + TYPE_FIELD: DOCUMENT_TYPE, + SOURCE_FIELD: { + TYPE_FIELD: BASE64_TYPE, + MEDIA_TYPE_FIELD: mime, + DATA_FIELD: blob, + } + })); + continue; } } - None - }) - .collect::>() - .join("\n"); + let text = extract_text_from_resource(&r.resource); + if !text.is_empty() { + text_parts.push(text.clone()); + blocks.push(json!({ + TYPE_FIELD: TEXT_TYPE, + TEXT_TYPE: text + })); + } + continue; + } + if let ContentBlock::Image(image) = c { + if ANTHROPIC_IMAGE_MEDIA_TYPES + .contains(&image.mime_type.as_str()) + { + has_media = true; + blocks.push(convert_image( + &image.clone(), + &ImageFormat::Anthropic, + )); + } else { + let marker = format!("[Image: {}]", image.mime_type); + text_parts.push(marker.clone()); + blocks.push(json!({ + TYPE_FIELD: TEXT_TYPE, + TEXT_TYPE: marker + })); + } + } + } + + let content_value = if has_media { + Value::Array(blocks) + } else { + Value::String(text_parts.join("\n")) + }; content.push(json!({ TYPE_FIELD: TOOL_RESULT_TYPE, TOOL_USE_ID_FIELD: tool_response.id, - CONTENT_FIELD: text + CONTENT_FIELD: content_value })); } Err(tool_error) => { @@ -1672,6 +1754,99 @@ mod tests { ); } + #[test] + fn test_tool_response_forwards_image_resource_as_image_block() { + use rmcp::model::CallToolResult; + + let image = ContentBlock::resource(ResourceContents::BlobResourceContents { + uri: "file:///shot.png".to_string(), + mime_type: Some("image/png".to_string()), + blob: "aGVsbG8=".to_string(), + meta: None, + }); + + let messages = vec![ + Message::assistant() + .with_tool_request("tool_1", Ok(CallToolRequestParams::new("screenshot"))), + Message::user().with_tool_response("tool_1", Ok(CallToolResult::success(vec![image]))), + ]; + + let spec = format_messages(&messages); + + let block = &spec[1]["content"][0]["content"][0]; + assert_eq!(block["type"], "image"); + assert_eq!(block["source"]["type"], "base64"); + assert_eq!(block["source"]["media_type"], "image/png"); + assert_eq!(block["source"]["data"], "aGVsbG8="); + } + + #[test] + fn test_tool_response_unsupported_image_mime_falls_back_to_text() { + use rmcp::model::CallToolResult; + + // image/svg+xml is not a Claude-supported image type, so it must fall + // through to text rather than an image block. + let svg = ContentBlock::resource(ResourceContents::BlobResourceContents { + uri: "file:///diagram.svg".to_string(), + mime_type: Some("image/svg+xml".to_string()), + blob: "aGVsbG8=".to_string(), + meta: None, + }); + + let messages = vec![ + Message::assistant() + .with_tool_request("tool_1", Ok(CallToolRequestParams::new("render"))), + Message::user().with_tool_response("tool_1", Ok(CallToolResult::success(vec![svg]))), + ]; + + let spec = format_messages(&messages); + + // Serializer contract: an unsupported image type is not emitted as an + // image block — the content collapses to a text string. + assert!(spec[1]["content"][0]["content"].is_string()); + } + + #[test] + fn test_tool_response_forwards_raw_image_as_image_block() { + use rmcp::model::CallToolResult; + + let image = ContentBlock::image("aGVsbG8=", "image/png"); + + let messages = vec![ + Message::assistant() + .with_tool_request("tool_1", Ok(CallToolRequestParams::new("screenshot"))), + Message::user().with_tool_response("tool_1", Ok(CallToolResult::success(vec![image]))), + ]; + + let spec = format_messages(&messages); + + let block = &spec[1]["content"][0]["content"][0]; + assert_eq!(block["type"], "image"); + assert_eq!(block["source"]["type"], "base64"); + assert_eq!(block["source"]["media_type"], "image/png"); + assert_eq!(block["source"]["data"], "aGVsbG8="); + } + + #[test] + fn test_tool_response_unsupported_raw_image_mime_falls_back_to_text() { + use rmcp::model::CallToolResult; + + // image/svg+xml is not a Claude-supported image type, so a raw image with + // that mime must fall back to a text marker rather than an image block + // (which the provider would reject). + let image = ContentBlock::image("aGVsbG8=", "image/svg+xml"); + + let messages = vec![ + Message::assistant() + .with_tool_request("tool_1", Ok(CallToolRequestParams::new("render"))), + Message::user().with_tool_response("tool_1", Ok(CallToolResult::success(vec![image]))), + ]; + + let spec = format_messages(&messages); + + assert_eq!(spec[1]["content"][0]["content"], "[Image: image/svg+xml]"); + } + #[test] fn test_args_to_input_value_returns_empty_object_for_none() { let value = args_to_input_value(None); diff --git a/crates/goose-provider-types/src/formats/google.rs b/crates/goose-provider-types/src/formats/google.rs index 4962e8833..806e42e7f 100644 --- a/crates/goose-provider-types/src/formats/google.rs +++ b/crates/goose-provider-types/src/formats/google.rs @@ -1,10 +1,13 @@ use crate::conversation::token_usage::{ProviderUsage, Usage}; use crate::errors::ProviderError; use crate::formats::openai::{is_valid_function_name, sanitize_function_name}; +use crate::mcp_utils::extract_text_from_resource; use crate::model::ModelConfig; use crate::thinking::ThinkingEffort; use anyhow::Result; -use rmcp::model::{object, CallToolRequestParams, ContentBlock, ErrorCode, ErrorData, Role, Tool}; +use rmcp::model::{ + object, CallToolRequestParams, ContentBlock, ErrorCode, ErrorData, ResourceContents, Role, Tool, +}; use serde::Serialize; use std::borrow::Cow; use uuid::Uuid; @@ -169,36 +172,44 @@ pub fn format_messages(messages: &[Message], nested_function_response_media: boo let mut tool_content = Vec::new(); let mut media = Vec::new(); for content in result.content.iter().cloned() { - match content { + let inline = match &content { ContentBlock::Image(image) => { - if nested_function_response_media { - media.push(json!({ - "inlineData": { - "mimeType": image.mime_type, - "data": image.data, - } - })); - } else { - parts.push(json!({ - "inline_data": { - "mime_type": image.mime_type, - "data": image.data, - } - })); - } + Some((image.mime_type.clone(), image.data.clone())) } - _ => { - tool_content.push(content); + ContentBlock::Resource(embedded) => match &embedded.resource { + ResourceContents::BlobResourceContents { + blob, + mime_type, + .. + } => mime_type + .clone() + .filter(|m| !m.is_empty()) + .map(|mime| (mime, blob.clone())), + _ => None, + }, + _ => None, + }; + match inline { + Some((mime, data)) if nested_function_response_media => { + media.push(json!({ + "inlineData": {"mimeType": mime, "data": data} + })); } + Some((mime, data)) => { + parts.push(json!({ + "inline_data": {"mime_type": mime, "data": data} + })); + } + None => tool_content.push(content), } } let mut text = tool_content .iter() .filter_map(|c| match c { ContentBlock::Text(t) => Some(t.text.clone()), - ContentBlock::Resource(raw_embedded_resource) => { - Some(raw_embedded_resource.clone().get_text()) - } + ContentBlock::Resource(raw_embedded_resource) => Some( + extract_text_from_resource(&raw_embedded_resource.resource), + ), _ => None, }) .collect::>() @@ -957,6 +968,27 @@ mod tests { ); } + #[test] + fn test_blob_resource_tool_result_is_forwarded_as_media() { + let blob = ContentBlock::resource(ResourceContents::BlobResourceContents { + uri: "file:///shot.png".to_string(), + mime_type: Some("image/png".to_string()), + blob: "aGVsbG8=".to_string(), + meta: None, + }); + let messages = vec![ + set_up_tool_request_message("call_123", CallToolRequestParams::new("screenshot")), + set_up_tool_response_message("call_123", vec![blob]), + ]; + + let payload = format_messages(&messages, true); + + assert_eq!( + payload[1]["parts"][0]["functionResponse"]["parts"], + json!([{"inlineData": {"mimeType": "image/png", "data": "aGVsbG8="}}]) + ); + } + #[test] fn test_message_to_google_spec_tool_result_multiple_texts() { let tool_result: Vec = vec![