tkmind/tkmind_go
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

feat(providers): forward images and MCP embedded-resource blobs in Anthropic and Google formats (#10340)

Browse Source
This commit is contained in:
thinkmore-eric
2026-07-29 14:55:36 -06:00
committed by GitHub
parent dd45906cb8
commit d71213d2ad
2 changed files with 245 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/goose-provider-types/src/formats/anthropic.rs
+191 -16
View File
@@ -8,7 +8,10 @@ use crate::mcp_utils::extract_text_from_resource;
use crate::model::ModelConfig;
use crate::thinking::ThinkingEffort;
use anyhow::{anyhow, Result};
use rmcp::model::{object, CallToolRequestParams, ErrorCode, ErrorData, JsonObject, Role, Tool};
use rmcp::model::{
object, CallToolRequestParams, ContentBlock, ErrorCode, ErrorData, JsonObject,
ResourceContents, Role, Tool,
};
use rmcp::object as json_object;
use serde_json::{json, Value};
use std::collections::HashSet;
@@ -138,6 +141,16 @@ const TOOL_USE_ID_FIELD: &str = "tool_use_id";
const IS_ERROR_FIELD: &str = "is_error";
const SIGNATURE_FIELD: &str = "signature";
const DATA_FIELD: &str = "data";
const IMAGE_TYPE: &str = "image";
const DOCUMENT_TYPE: &str = "document";
const SOURCE_FIELD: &str = "source";
const BASE64_TYPE: &str = "base64";
const MEDIA_TYPE_FIELD: &str = "media_type";
// Claude vision only accepts these image media types; other image/* blobs fall
// through to the text/binary-marker path so an unsupported type (e.g.
// image/svg+xml) doesn't turn the next request into a provider rejection.
const ANTHROPIC_IMAGE_MEDIA_TYPES: [&str; 4] =
["image/jpeg", "image/png", "image/gif", "image/webp"];
const EVENT_MESSAGE_START: &str = "message_start";
const EVENT_MESSAGE_DELTA: &str = "message_delta";
const EVENT_MESSAGE_STOP: &str = "message_stop";
@@ -217,28 +230,97 @@ fn format_messages_with_options(
MessageContentBlock::ToolResponse(tool_response) => {
match &tool_response.tool_result {
Ok(result) => {
let text = result
.content
.iter()
.filter_map(|c| {
if let Some(t) = c.as_text() {
return Some(t.text.clone());
let mut blocks: Vec<Value> = Vec::new();
let mut text_parts: Vec<String> = Vec::new();
let mut has_media = false;
for c in result.content.iter() {
if let Some(t) = c.as_text() {
text_parts.push(t.text.clone());
if !t.text.is_empty() {
blocks.push(json!({
TYPE_FIELD: TEXT_TYPE,
TEXT_TYPE: t.text.clone()
}));
}
if let Some(r) = c.as_resource() {
let text = extract_text_from_resource(&r.resource);
if !text.is_empty() {
return Some(text);
continue;
}
if let Some(r) = c.as_resource() {
// Claude only accepts a fixed set of media types, so
// unsupported blobs fall back to text below rather than
// being rejected by the provider.
if let ResourceContents::BlobResourceContents {
blob,
mime_type,
..
} = &r.resource
{
let mime = mime_type.as_deref().unwrap_or("");
if ANTHROPIC_IMAGE_MEDIA_TYPES.contains(&mime) {
has_media = true;
blocks.push(json!({
TYPE_FIELD: IMAGE_TYPE,
SOURCE_FIELD: {
TYPE_FIELD: BASE64_TYPE,
MEDIA_TYPE_FIELD: mime,
DATA_FIELD: blob,
}
}));
continue;
}
if mime == "application/pdf" {
has_media = true;
blocks.push(json!({
TYPE_FIELD: DOCUMENT_TYPE,
SOURCE_FIELD: {
TYPE_FIELD: BASE64_TYPE,
MEDIA_TYPE_FIELD: mime,
DATA_FIELD: blob,
}
}));
continue;
}
}
None
})
.collect::<Vec<_>>()
.join("\n");
let text = extract_text_from_resource(&r.resource);
if !text.is_empty() {
text_parts.push(text.clone());
blocks.push(json!({
TYPE_FIELD: TEXT_TYPE,
TEXT_TYPE: text
}));
}
continue;
}
if let ContentBlock::Image(image) = c {
if ANTHROPIC_IMAGE_MEDIA_TYPES
.contains(&image.mime_type.as_str())
{
has_media = true;
blocks.push(convert_image(
&image.clone(),
&ImageFormat::Anthropic,
));
} else {
let marker = format!("[Image: {}]", image.mime_type);
text_parts.push(marker.clone());
blocks.push(json!({
TYPE_FIELD: TEXT_TYPE,
TEXT_TYPE: marker
}));
}
}
}
let content_value = if has_media {
Value::Array(blocks)
} else {
Value::String(text_parts.join("\n"))
};
content.push(json!({
TYPE_FIELD: TOOL_RESULT_TYPE,
TOOL_USE_ID_FIELD: tool_response.id,
CONTENT_FIELD: text
CONTENT_FIELD: content_value
}));
}
Err(tool_error) => {
@@ -1672,6 +1754,99 @@ mod tests {
);
}
#[test]
fn test_tool_response_forwards_image_resource_as_image_block() {
use rmcp::model::CallToolResult;
let image = ContentBlock::resource(ResourceContents::BlobResourceContents {
uri: "file:///shot.png".to_string(),
mime_type: Some("image/png".to_string()),
blob: "aGVsbG8=".to_string(),
meta: None,
});
let messages = vec![
Message::assistant()
.with_tool_request("tool_1", Ok(CallToolRequestParams::new("screenshot"))),
Message::user().with_tool_response("tool_1", Ok(CallToolResult::success(vec![image]))),
];
let spec = format_messages(&messages);
let block = &spec[1]["content"][0]["content"][0];
assert_eq!(block["type"], "image");
assert_eq!(block["source"]["type"], "base64");
assert_eq!(block["source"]["media_type"], "image/png");
assert_eq!(block["source"]["data"], "aGVsbG8=");
}
#[test]
fn test_tool_response_unsupported_image_mime_falls_back_to_text() {
use rmcp::model::CallToolResult;
// image/svg+xml is not a Claude-supported image type, so it must fall
// through to text rather than an image block.
let svg = ContentBlock::resource(ResourceContents::BlobResourceContents {
uri: "file:///diagram.svg".to_string(),
mime_type: Some("image/svg+xml".to_string()),
blob: "aGVsbG8=".to_string(),
meta: None,
});
let messages = vec![
Message::assistant()
.with_tool_request("tool_1", Ok(CallToolRequestParams::new("render"))),
Message::user().with_tool_response("tool_1", Ok(CallToolResult::success(vec![svg]))),
];
let spec = format_messages(&messages);
// Serializer contract: an unsupported image type is not emitted as an
// image block — the content collapses to a text string.
assert!(spec[1]["content"][0]["content"].is_string());
}
#[test]
fn test_tool_response_forwards_raw_image_as_image_block() {
use rmcp::model::CallToolResult;
let image = ContentBlock::image("aGVsbG8=", "image/png");
let messages = vec![
Message::assistant()
.with_tool_request("tool_1", Ok(CallToolRequestParams::new("screenshot"))),
Message::user().with_tool_response("tool_1", Ok(CallToolResult::success(vec![image]))),
];
let spec = format_messages(&messages);
let block = &spec[1]["content"][0]["content"][0];
assert_eq!(block["type"], "image");
assert_eq!(block["source"]["type"], "base64");
assert_eq!(block["source"]["media_type"], "image/png");
assert_eq!(block["source"]["data"], "aGVsbG8=");
}
#[test]
fn test_tool_response_unsupported_raw_image_mime_falls_back_to_text() {
use rmcp::model::CallToolResult;
// image/svg+xml is not a Claude-supported image type, so a raw image with
// that mime must fall back to a text marker rather than an image block
// (which the provider would reject).
let image = ContentBlock::image("aGVsbG8=", "image/svg+xml");
let messages = vec![
Message::assistant()
.with_tool_request("tool_1", Ok(CallToolRequestParams::new("render"))),
Message::user().with_tool_response("tool_1", Ok(CallToolResult::success(vec![image]))),
];
let spec = format_messages(&messages);
assert_eq!(spec[1]["content"][0]["content"], "[Image: image/svg+xml]");
}
#[test]
fn test_args_to_input_value_returns_empty_object_for_none() {
let value = args_to_input_value(None);
crates/goose-provider-types/src/formats/google.rs
+54 -22
View File
@@ -1,10 +1,13 @@
use crate::conversation::token_usage::{ProviderUsage, Usage};
use crate::errors::ProviderError;
use crate::formats::openai::{is_valid_function_name, sanitize_function_name};
use crate::mcp_utils::extract_text_from_resource;
use crate::model::ModelConfig;
use crate::thinking::ThinkingEffort;
use anyhow::Result;
use rmcp::model::{object, CallToolRequestParams, ContentBlock, ErrorCode, ErrorData, Role, Tool};
use rmcp::model::{
object, CallToolRequestParams, ContentBlock, ErrorCode, ErrorData, ResourceContents, Role, Tool,
};
use serde::Serialize;
use std::borrow::Cow;
use uuid::Uuid;
@@ -169,36 +172,44 @@ pub fn format_messages(messages: &[Message], nested_function_response_media: boo
let mut tool_content = Vec::new();
let mut media = Vec::new();
for content in result.content.iter().cloned() {
match content {
let inline = match &content {
ContentBlock::Image(image) => {
if nested_function_response_media {
media.push(json!({
"inlineData": {
"mimeType": image.mime_type,
"data": image.data,
}
}));
} else {
parts.push(json!({
"inline_data": {
"mime_type": image.mime_type,
"data": image.data,
}
}));
}
Some((image.mime_type.clone(), image.data.clone()))
}
_ => {
tool_content.push(content);
ContentBlock::Resource(embedded) => match &embedded.resource {
ResourceContents::BlobResourceContents {
blob,
mime_type,
..
} => mime_type
.clone()
.filter(|m| !m.is_empty())
.map(|mime| (mime, blob.clone())),
_ => None,
},
_ => None,
};
match inline {
Some((mime, data)) if nested_function_response_media => {
media.push(json!({
"inlineData": {"mimeType": mime, "data": data}
}));
}
Some((mime, data)) => {
parts.push(json!({
"inline_data": {"mime_type": mime, "data": data}
}));
}
None => tool_content.push(content),
}
}
let mut text = tool_content
.iter()
.filter_map(|c| match c {
ContentBlock::Text(t) => Some(t.text.clone()),
ContentBlock::Resource(raw_embedded_resource) => {
Some(raw_embedded_resource.clone().get_text())
}
ContentBlock::Resource(raw_embedded_resource) => Some(
extract_text_from_resource(&raw_embedded_resource.resource),
),
_ => None,
})
.collect::<Vec<_>>()
@@ -957,6 +968,27 @@ mod tests {
);
}
#[test]
fn test_blob_resource_tool_result_is_forwarded_as_media() {
let blob = ContentBlock::resource(ResourceContents::BlobResourceContents {
uri: "file:///shot.png".to_string(),
mime_type: Some("image/png".to_string()),
blob: "aGVsbG8=".to_string(),
meta: None,
});
let messages = vec![
set_up_tool_request_message("call_123", CallToolRequestParams::new("screenshot")),
set_up_tool_response_message("call_123", vec![blob]),
];
let payload = format_messages(&messages, true);
assert_eq!(
payload[1]["parts"][0]["functionResponse"]["parts"],
json!([{"inlineData": {"mimeType": "image/png", "data": "aGVsbG8="}}])
);
}
#[test]
fn test_message_to_google_spec_tool_result_multiple_texts() {
let tool_result: Vec<ContentBlock> = vec![