From 7259c05174d25b67f218b6d9ee5f6992c89c0835 Mon Sep 17 00:00:00 2001 From: Alex Hancock Date: Tue, 1 Sep 2026 18:28:11 +0000 Subject: [PATCH] feat(gdk): add first-class document content support (#11629) --- crates/goose-context-management/src/format.rs | 4 + .../src/conversation/message.rs | 100 ++++++++++++++++++ crates/goose-provider-types/src/documents.rs | 80 ++++++++++++++ .../src/formats/anthropic.rs | 89 ++++++++++++++++ .../src/formats/databricks.rs | 66 ++++++++++++ .../src/formats/google.rs | 42 ++++++++ .../src/formats/openai.rs | 87 +++++++++++++++ .../src/formats/openai_responses.rs | 72 +++++++++++++ .../src/formats/snowflake.rs | 10 ++ crates/goose-provider-types/src/lib.rs | 1 + crates/goose-providers/src/lib.rs | 4 +- crates/goose-sdk/src/bindings.rs | 95 +++++++++++++++++ crates/goose/src/agents/gen_ai_telemetry.rs | 7 ++ crates/goose/src/providers/formats/bedrock.rs | 4 + documentation/src/data/gdk-api.json | 27 +++++ 15 files changed, 686 insertions(+), 2 deletions(-) create mode 100644 crates/goose-provider-types/src/documents.rs diff --git a/crates/goose-context-management/src/format.rs b/crates/goose-context-management/src/format.rs index cda1f1d3f..c7358b2bf 100644 --- a/crates/goose-context-management/src/format.rs +++ b/crates/goose-context-management/src/format.rs @@ -8,6 +8,10 @@ pub fn format_message_for_compacting(msg: &Message) -> String { .filter_map(|content| match content { MessageContent::Text(text) => Some(text.text.clone()), MessageContent::Image(img) => Some(format!("[image: {}]", img.mime_type)), + MessageContent::Document(doc) => Some(match &doc.name { + Some(name) => format!("[document: {} ({})]", name, doc.mime_type), + None => format!("[document: {}]", doc.mime_type), + }), MessageContent::ToolRequest(req) => { if let Ok(call) = &req.tool_call { Some(format!( diff --git a/crates/goose-provider-types/src/conversation/message.rs b/crates/goose-provider-types/src/conversation/message.rs index 55bae8044..d3bf3bb30 100644 --- a/crates/goose-provider-types/src/conversation/message.rs +++ b/crates/goose-provider-types/src/conversation/message.rs @@ -286,6 +286,30 @@ pub struct ErrorContent { pub message: String, } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DocumentContent { + pub data: String, + pub mime_type: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option, +} + +impl DocumentContent { + pub fn new, T: Into>(data: S, mime_type: T) -> Self { + Self { + data: data.into(), + mime_type: mime_type.into(), + name: None, + } + } + + pub fn with_name>(mut self, name: S) -> Self { + self.name = Some(name.into()); + self + } +} + pub type MessageContent = MessageContentBlock; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -294,6 +318,7 @@ pub type MessageContent = MessageContentBlock; pub enum MessageContentBlock { Text(TextContent), Image(ImageContent), + Document(DocumentContent), ToolRequest(ToolRequest), ToolResponse(ToolResponse), ToolConfirmationRequest(ToolConfirmationRequest), @@ -309,6 +334,10 @@ impl fmt::Display for MessageContentBlock { match self { MessageContentBlock::Text(t) => write!(f, "{}", t.text), MessageContentBlock::Image(i) => write!(f, "[Image: {}]", i.mime_type), + MessageContentBlock::Document(d) => match &d.name { + Some(name) => write!(f, "[Document: {} ({})]", name, d.mime_type), + None => write!(f, "[Document: {}]", d.mime_type), + }, MessageContentBlock::ToolRequest(r) => { write!(f, "[ToolRequest: {}]", r.to_readable_string()) } @@ -441,6 +470,18 @@ impl MessageContentBlock { MessageContentBlock::Image(ImageContent::new(data, mime_type)) } + pub fn document, T: Into>( + data: S, + mime_type: T, + name: Option, + ) -> Self { + let document = DocumentContent::new(data, mime_type); + MessageContentBlock::Document(match name { + Some(name) => document.with_name(name), + None => document, + }) + } + pub fn tool_request>( id: S, tool_call: ToolResult, @@ -1044,6 +1085,15 @@ impl Message { self.with_content(MessageContentBlock::image(data, mime_type)) } + pub fn with_document, T: Into>( + self, + data: S, + mime_type: T, + name: Option, + ) -> Self { + self.with_content(MessageContentBlock::document(data, mime_type, name)) + } + /// Add a tool request to the message pub fn with_tool_request>( self, @@ -1313,6 +1363,56 @@ pub struct TokenState { pub accumulated_cost: Option, } +#[cfg(test)] +mod document_tests { + use super::*; + + #[test] + fn document_content_carries_media_type_and_name() { + let message = Message::user().with_document( + "cGRmLWJ5dGVz", + "application/pdf", + Some("q3-report.pdf".to_string()), + ); + + let MessageContent::Document(document) = &message.content[0] else { + panic!("expected document content"); + }; + assert_eq!(document.data, "cGRmLWJ5dGVz"); + assert_eq!(document.mime_type, "application/pdf"); + assert_eq!(document.name.as_deref(), Some("q3-report.pdf")); + assert_eq!( + message.content[0].to_string(), + "[Document: q3-report.pdf (application/pdf)]" + ); + } + + #[test] + fn document_content_without_name_is_allowed() { + let content = MessageContent::document("cGRmLWJ5dGVz", "application/pdf", None); + + assert!(matches!(&content, MessageContent::Document(document) if document.name.is_none())); + assert_eq!(content.to_string(), "[Document: application/pdf]"); + } + + #[test] + fn document_content_round_trips_through_serde() { + let message = Message::user().with_document( + "cGRmLWJ5dGVz", + "application/pdf", + Some("q3-report.pdf".to_string()), + ); + + let json = serde_json::to_value(&message).unwrap(); + assert_eq!(json["content"][0]["type"], "document"); + assert_eq!(json["content"][0]["mimeType"], "application/pdf"); + assert_eq!(json["content"][0]["name"], "q3-report.pdf"); + + let restored: Message = serde_json::from_value(json).unwrap(); + assert_eq!(restored, message); + } +} + #[cfg(test)] mod tests { use crate::conversation::message::{ diff --git a/crates/goose-provider-types/src/documents.rs b/crates/goose-provider-types/src/documents.rs new file mode 100644 index 000000000..8ae5fb66f --- /dev/null +++ b/crates/goose-provider-types/src/documents.rs @@ -0,0 +1,80 @@ +use serde_json::{json, Value}; + +use crate::conversation::message::DocumentContent; + +#[derive(Debug, Copy, Clone)] +pub enum DocumentFormat { + OpenAi, + Anthropic, +} + +/// Media types that providers accept as native document input. Anything else is +/// reported back to the caller rather than being sent as an unusable blob. +pub const SUPPORTED_DOCUMENT_MEDIA_TYPES: [&str; 1] = ["application/pdf"]; + +pub fn document_media_type_is_supported(mime_type: &str) -> bool { + SUPPORTED_DOCUMENT_MEDIA_TYPES.contains(&mime_type) +} + +pub fn convert_document(document: &DocumentContent, format: &DocumentFormat) -> Value { + match format { + DocumentFormat::OpenAi => json!({ + "type": "file", + "file": { + "filename": document.name.clone().unwrap_or_else(|| "document.pdf".to_string()), + "file_data": format!("data:{};base64,{}", document.mime_type, document.data), + } + }), + DocumentFormat::Anthropic => { + let mut block = json!({ + "type": "document", + "source": { + "type": "base64", + "media_type": document.mime_type, + "data": document.data, + } + }); + if let Some(name) = &document.name { + block["title"] = json!(name); + } + block + } + } +} + +/// Explains why a document was dropped so the model, and the caller reading the +/// request, can act on it instead of silently losing the attachment. +pub fn unsupported_document_text(document: &DocumentContent, reason: &str) -> String { + match &document.name { + Some(name) => format!( + "[document \"{}\" ({}) not sent: {}]", + name, document.mime_type, reason + ), + None => format!("[document ({}) not sent: {}]", document.mime_type, reason), + } +} + +pub const UNSUPPORTED_MEDIA_TYPE_REASON: &str = + "only application/pdf documents can be sent to this provider"; +pub const UNSUPPORTED_PROVIDER_REASON: &str = "this provider does not accept document input"; +pub const ASSISTANT_ROLE_REASON: &str = "documents can only be sent in user messages"; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unsupported_document_text_names_the_document_and_reason() { + let named = DocumentContent::new("data", "text/csv").with_name("rows.csv"); + assert_eq!( + unsupported_document_text(&named, UNSUPPORTED_MEDIA_TYPE_REASON), + "[document \"rows.csv\" (text/csv) not sent: only application/pdf documents can be sent to this provider]" + ); + + let unnamed = DocumentContent::new("data", "text/csv"); + assert_eq!( + unsupported_document_text(&unnamed, UNSUPPORTED_PROVIDER_REASON), + "[document (text/csv) not sent: this provider does not accept document input]" + ); + } +} diff --git a/crates/goose-provider-types/src/formats/anthropic.rs b/crates/goose-provider-types/src/formats/anthropic.rs index bc714ce12..60736f0c2 100644 --- a/crates/goose-provider-types/src/formats/anthropic.rs +++ b/crates/goose-provider-types/src/formats/anthropic.rs @@ -2,6 +2,10 @@ use crate::canonical::maybe_get_canonical_model; use crate::canonical::ThinkingMode; use crate::conversation::message::{Message, MessageContentBlock}; use crate::conversation::token_usage::{CostSource, ProviderUsage, Usage}; +use crate::documents::{ + convert_document, document_media_type_is_supported, unsupported_document_text, DocumentFormat, + ASSISTANT_ROLE_REASON, UNSUPPORTED_MEDIA_TYPE_REASON, +}; use crate::errors::ProviderError; use crate::images::{convert_image, ImageFormat}; use crate::mcp_utils::extract_text_from_resource; @@ -427,6 +431,24 @@ fn format_messages_with_options( MessageContentBlock::Image(image) => { content.push(convert_image(image, &ImageFormat::Anthropic)); } + MessageContentBlock::Document(document) => { + if message.role != Role::User { + content.push(json!({ + TYPE_FIELD: TEXT_TYPE, + TEXT_TYPE: unsupported_document_text(document, ASSISTANT_ROLE_REASON) + })); + } else if document_media_type_is_supported(&document.mime_type) { + content.push(convert_document(document, &DocumentFormat::Anthropic)); + } else { + content.push(json!({ + TYPE_FIELD: TEXT_TYPE, + TEXT_TYPE: unsupported_document_text( + document, + UNSUPPORTED_MEDIA_TYPE_REASON, + ) + })); + } + } } } @@ -1224,6 +1246,73 @@ where } } +#[cfg(test)] +mod document_tests { + use super::*; + + #[test] + fn user_document_becomes_a_base64_document_block() { + let messages = vec![Message::user().with_document( + "cGRmLWJ5dGVz", + "application/pdf", + Some("q3-report.pdf".to_string()), + )]; + + let spec = format_messages(&messages); + + assert_eq!(spec.len(), 1); + assert_eq!(spec[0]["role"], "user"); + let block = &spec[0]["content"][0]; + assert_eq!(block["type"], DOCUMENT_TYPE); + assert_eq!(block["title"], "q3-report.pdf"); + assert_eq!( + block[SOURCE_FIELD], + json!({ + "type": "base64", + "media_type": "application/pdf", + "data": "cGRmLWJ5dGVz", + }) + ); + } + + #[test] + fn assistant_document_becomes_a_text_block() { + let messages = vec![Message::assistant().with_document( + "cGRmLWJ5dGVz", + "application/pdf", + Some("q3-report.pdf".to_string()), + )]; + + let spec = format_messages(&messages); + + assert_eq!(spec.len(), 1); + assert_eq!(spec[0]["role"], "assistant"); + assert_eq!(spec[0]["content"][0]["type"], "text"); + let text = spec[0]["content"][0]["text"].as_str().unwrap(); + assert!(text.contains("q3-report.pdf"), "{text}"); + assert!(text.contains("user messages"), "{text}"); + assert!(!text.contains("cGRmLWJ5dGVz"), "{text}"); + } + + #[test] + fn unsupported_document_media_type_becomes_an_explicit_text_block() { + let messages = vec![Message::user().with_document( + "cm93cw==", + "text/csv", + Some("rows.csv".to_string()), + )]; + + let spec = format_messages(&messages); + + assert_eq!(spec[0]["content"][0]["type"], "text"); + let text = spec[0]["content"][0]["text"].as_str().unwrap(); + assert!(text.contains("rows.csv"), "{text}"); + assert!(text.contains("text/csv"), "{text}"); + assert!(text.contains("application/pdf"), "{text}"); + assert!(!text.contains("cm93cw=="), "{text}"); + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/goose-provider-types/src/formats/databricks.rs b/crates/goose-provider-types/src/formats/databricks.rs index d45b456ab..12a37ed4c 100644 --- a/crates/goose-provider-types/src/formats/databricks.rs +++ b/crates/goose-provider-types/src/formats/databricks.rs @@ -6,6 +6,10 @@ use crate::formats::anthropic::{ }; use crate::model::{is_goose_internal_request_param, ModelConfig}; +use crate::documents::{ + convert_document, document_media_type_is_supported, unsupported_document_text, DocumentFormat, + ASSISTANT_ROLE_REASON, UNSUPPORTED_MEDIA_TYPE_REASON, +}; use crate::formats::openai::{ extract_reasoning_effort, is_openai_responses_model, is_valid_function_name, openai_reasoning_effort_for_thinking, sanitize_function_name, validate_tool_schemas, @@ -244,6 +248,21 @@ fn format_messages( })); } } + MessageContentBlock::Document(document) => { + if message.role != Role::User { + content_array.push(json!({ + "type": "text", + "text": unsupported_document_text(document, ASSISTANT_ROLE_REASON) + })); + } else if document_media_type_is_supported(&document.mime_type) { + content_array.push(convert_document(document, &DocumentFormat::OpenAi)); + } else { + content_array.push(json!({ + "type": "text", + "text": unsupported_document_text(document, UNSUPPORTED_MEDIA_TYPE_REASON) + })); + } + } MessageContentBlock::SystemNotification(_) | MessageContentBlock::Error(_) | MessageContentBlock::ToolConfirmationRequest(_) @@ -626,6 +645,53 @@ pub fn create_request_for_provider( Ok(payload) } +#[cfg(test)] +mod document_tests { + use super::*; + use crate::conversation::message::Message; + + fn format(messages: &[Message]) -> Vec { + format_messages(messages, &ImageFormat::OpenAi, None, true) + } + + #[test] + fn user_document_becomes_a_file_content_part() { + let spec = format(&[Message::user().with_document( + "cGRmLWJ5dGVz", + "application/pdf", + Some("q3-report.pdf".to_string()), + )]); + + assert_eq!(spec.len(), 1); + assert_eq!( + spec[0].content[0], + json!({ + "type": "file", + "file": { + "filename": "q3-report.pdf", + "file_data": "data:application/pdf;base64,cGRmLWJ5dGVz", + } + }) + ); + } + + #[test] + fn assistant_document_becomes_a_text_part() { + let spec = format(&[Message::assistant().with_document( + "cGRmLWJ5dGVz", + "application/pdf", + Some("q3-report.pdf".to_string()), + )]); + + assert_eq!(spec.len(), 1); + assert_eq!(spec[0].role, "assistant"); + let text = spec[0].content.as_str().unwrap(); + assert!(text.contains("q3-report.pdf"), "{text}"); + assert!(text.contains("user messages"), "{text}"); + assert!(!text.contains("cGRmLWJ5dGVz"), "{text}"); + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/goose-provider-types/src/formats/google.rs b/crates/goose-provider-types/src/formats/google.rs index 704946000..bd892c64c 100644 --- a/crates/goose-provider-types/src/formats/google.rs +++ b/crates/goose-provider-types/src/formats/google.rs @@ -1,4 +1,7 @@ use crate::conversation::token_usage::{ProviderUsage, Usage}; +use crate::documents::{ + document_media_type_is_supported, unsupported_document_text, UNSUPPORTED_MEDIA_TYPE_REASON, +}; use crate::errors::ProviderError; use crate::formats::openai::{is_valid_function_name, sanitize_function_name}; use crate::mcp_utils::extract_text_from_resource; @@ -255,6 +258,20 @@ pub fn format_messages(messages: &[Message], nested_function_response_media: boo } })); } + MessageContentBlock::Document(document) => { + if document_media_type_is_supported(&document.mime_type) { + parts.push(json!({ + "inline_data": { + "mime_type": document.mime_type, + "data": document.data, + } + })); + } else { + parts.push(json!({ + "text": unsupported_document_text(document, UNSUPPORTED_MEDIA_TYPE_REASON) + })); + } + } _ => {} } @@ -935,6 +952,31 @@ mod tests { ); } + #[test] + fn test_message_to_google_spec_document_only_message() { + let messages = vec![Message::new( + Role::User, + 0, + vec![MessageContentBlock::document( + "base64pdfdata".to_string(), + "application/pdf".to_string(), + Some("report.pdf".to_string()), + )], + )]; + let payload = format_messages(&messages, false); + + assert_eq!(payload.len(), 1); + assert_eq!(payload[0]["role"], "user"); + assert_eq!( + payload[0]["parts"][0]["inline_data"]["mime_type"], + "application/pdf" + ); + assert_eq!( + payload[0]["parts"][0]["inline_data"]["data"], + "base64pdfdata" + ); + } + #[test] fn test_message_to_google_spec_tool_request_message() { let arguments = json!({ diff --git a/crates/goose-provider-types/src/formats/openai.rs b/crates/goose-provider-types/src/formats/openai.rs index c78676d62..7c15df36f 100644 --- a/crates/goose-provider-types/src/formats/openai.rs +++ b/crates/goose-provider-types/src/formats/openai.rs @@ -1,6 +1,10 @@ use crate::base::ThinkingPreservationFormat; use crate::conversation::message::{Message, MessageContentBlock, ProviderMetadata}; use crate::conversation::token_usage::{CostSource, ProviderUsage, Usage}; +use crate::documents::{ + convert_document, document_media_type_is_supported, unsupported_document_text, DocumentFormat, + ASSISTANT_ROLE_REASON, UNSUPPORTED_MEDIA_TYPE_REASON, +}; use crate::errors::ProviderError; use crate::images::{convert_image, detect_image_path, load_image_file, ImageFormat}; use crate::json::{parse_tool_arguments, truncation_error_message}; @@ -426,6 +430,22 @@ pub fn format_messages_with_options( })); } } + MessageContentBlock::Document(document) => { + if message.role != Role::User { + content_array.push(json!({ + "type": "text", + "text": unsupported_document_text(document, ASSISTANT_ROLE_REASON) + })); + } else if document_media_type_is_supported(&document.mime_type) { + has_non_text_content = true; + content_array.push(convert_document(document, &DocumentFormat::OpenAi)); + } else { + content_array.push(json!({ + "type": "text", + "text": unsupported_document_text(document, UNSUPPORTED_MEDIA_TYPE_REASON) + })); + } + } } } @@ -1930,6 +1950,73 @@ pub fn is_valid_function_name(name: &str) -> bool { re.is_match(name) } +#[cfg(test)] +mod document_tests { + use super::*; + + fn format(messages: &[Message]) -> Vec { + format_messages_with_options( + messages, + &ImageFormat::OpenAi, + OpenAiFormatOptions { + supports_vision: true, + ..Default::default() + }, + ) + } + + #[test] + fn user_document_becomes_a_file_content_part() { + let spec = format(&[Message::user().with_document( + "cGRmLWJ5dGVz", + "application/pdf", + Some("q3-report.pdf".to_string()), + )]); + + assert_eq!(spec.len(), 1); + assert_eq!( + spec[0]["content"][0], + json!({ + "type": "file", + "file": { + "filename": "q3-report.pdf", + "file_data": "data:application/pdf;base64,cGRmLWJ5dGVz", + } + }) + ); + } + + #[test] + fn assistant_document_becomes_a_text_part() { + let spec = format(&[Message::assistant().with_document( + "cGRmLWJ5dGVz", + "application/pdf", + Some("q3-report.pdf".to_string()), + )]); + + assert_eq!(spec.len(), 1); + assert_eq!(spec[0]["role"], "assistant"); + let text = spec[0]["content"].as_str().unwrap(); + assert!(text.contains("q3-report.pdf"), "{text}"); + assert!(text.contains("user messages"), "{text}"); + assert!(!text.contains("cGRmLWJ5dGVz"), "{text}"); + } + + #[test] + fn unsupported_document_media_type_becomes_an_explicit_text_part() { + let spec = format(&[Message::user().with_document( + "cm93cw==", + "text/csv", + Some("rows.csv".to_string()), + )]); + + let text = spec[0]["content"].as_str().unwrap(); + assert!(text.contains("rows.csv"), "{text}"); + assert!(text.contains("application/pdf"), "{text}"); + assert!(!text.contains("cm93cw=="), "{text}"); + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/goose-provider-types/src/formats/openai_responses.rs b/crates/goose-provider-types/src/formats/openai_responses.rs index 821c8f7cb..799dd9806 100644 --- a/crates/goose-provider-types/src/formats/openai_responses.rs +++ b/crates/goose-provider-types/src/formats/openai_responses.rs @@ -1,5 +1,9 @@ use crate::conversation::message::{Message, MessageContentBlock}; use crate::conversation::token_usage::{ProviderUsage, Usage}; +use crate::documents::{ + convert_document, document_media_type_is_supported, unsupported_document_text, DocumentFormat, + ASSISTANT_ROLE_REASON, UNSUPPORTED_MEDIA_TYPE_REASON, +}; use crate::errors::ProviderError; use crate::formats::openai::{ extract_reasoning_effort, is_openai_responses_model, openai_reasoning_effort_for_thinking, @@ -480,6 +484,25 @@ fn add_message_items(input_items: &mut Vec, messages: &[Message], support })); } } + MessageContentBlock::Document(document) => { + if message.role != Role::User { + text_items.push(json!({ + "type": "output_text", + "text": unsupported_document_text(document, ASSISTANT_ROLE_REASON), + "annotations": [] + })); + } else if document_media_type_is_supported(&document.mime_type) { + let mut converted = convert_document(document, &DocumentFormat::OpenAi); + let mut file = converted["file"].take(); + file["type"] = json!("input_file"); + text_items.push(file); + } else { + text_items.push(json!({ + "type": "input_text", + "text": unsupported_document_text(document, UNSUPPORTED_MEDIA_TYPE_REASON) + })); + } + } MessageContentBlock::ToolResponse(response) => { if !text_items.is_empty() { input_items.push(json!({ @@ -1194,6 +1217,55 @@ where } } +#[cfg(test)] +mod document_tests { + use super::*; + + fn format(messages: &[Message]) -> Vec { + let mut items = Vec::new(); + add_message_items(&mut items, messages, true); + items + } + + #[test] + fn user_document_becomes_an_input_file_item() { + let items = format(&[Message::user().with_document( + "cGRmLWJ5dGVz", + "application/pdf", + Some("q3-report.pdf".to_string()), + )]); + + assert_eq!(items.len(), 1); + assert_eq!(items[0]["role"], "user"); + assert_eq!( + items[0]["content"][0], + json!({ + "type": "input_file", + "filename": "q3-report.pdf", + "file_data": "data:application/pdf;base64,cGRmLWJ5dGVz", + }) + ); + } + + #[test] + fn assistant_document_becomes_an_output_text_item() { + let items = format(&[Message::assistant().with_document( + "cGRmLWJ5dGVz", + "application/pdf", + Some("q3-report.pdf".to_string()), + )]); + + assert_eq!(items.len(), 1); + assert_eq!(items[0]["role"], "assistant"); + let part = &items[0]["content"][0]; + assert_eq!(part["type"], "output_text"); + let text = part["text"].as_str().unwrap(); + assert!(text.contains("q3-report.pdf"), "{text}"); + assert!(text.contains("user messages"), "{text}"); + assert!(!text.contains("cGRmLWJ5dGVz"), "{text}"); + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/goose-provider-types/src/formats/snowflake.rs b/crates/goose-provider-types/src/formats/snowflake.rs index 5913a8c17..b4e34650c 100644 --- a/crates/goose-provider-types/src/formats/snowflake.rs +++ b/crates/goose-provider-types/src/formats/snowflake.rs @@ -1,5 +1,6 @@ use crate::conversation::message::{Message, MessageContentBlock}; use crate::conversation::token_usage::Usage; +use crate::documents::{unsupported_document_text, UNSUPPORTED_PROVIDER_REASON}; use crate::errors::ProviderError; use crate::mcp_utils::extract_text_from_resource; use crate::model::ModelConfig; @@ -74,6 +75,15 @@ pub fn format_messages(messages: &[Message]) -> Vec { // Skip redacted thinking for now } MessageContentBlock::Image(_) => continue, // Snowflake doesn't support image content yet + MessageContentBlock::Document(document) => { + if !text_content.is_empty() { + text_content.push('\n'); + } + text_content.push_str(&unsupported_document_text( + document, + UNSUPPORTED_PROVIDER_REASON, + )); + } } } diff --git a/crates/goose-provider-types/src/lib.rs b/crates/goose-provider-types/src/lib.rs index e9b73cb97..d959ad707 100644 --- a/crates/goose-provider-types/src/lib.rs +++ b/crates/goose-provider-types/src/lib.rs @@ -3,6 +3,7 @@ pub mod cache_semantics; pub mod canonical; pub mod context_limit; pub mod conversation; +pub mod documents; pub mod errors; pub mod formats; pub mod goose_mode; diff --git a/crates/goose-providers/src/lib.rs b/crates/goose-providers/src/lib.rs index 287a3cf0b..10089ed88 100644 --- a/crates/goose-providers/src/lib.rs +++ b/crates/goose-providers/src/lib.rs @@ -6,8 +6,8 @@ pub mod databricks_auth; pub mod databricks_v2; pub mod google; pub use goose_provider_types::{ - base, cache_semantics, canonical, context_limit, conversation, errors, formats, goose_mode, - images, json, model, permission, request_log, retry, thinking, utils, + base, cache_semantics, canonical, context_limit, conversation, documents, errors, formats, + goose_mode, images, json, model, permission, request_log, retry, thinking, utils, }; pub mod declarative; pub mod http_status; diff --git a/crates/goose-sdk/src/bindings.rs b/crates/goose-sdk/src/bindings.rs index 967b7b94c..3b0ef5e7c 100644 --- a/crates/goose-sdk/src/bindings.rs +++ b/crates/goose-sdk/src/bindings.rs @@ -20,6 +20,7 @@ use goose_providers::{ databricks_auth::DatabricksAuth, databricks_v2::DatabricksV2Provider as GooseDatabricksV2Provider, declarative::{DeclarativeProviderConfig, EnvKeyResolver}, + documents::{document_media_type_is_supported, SUPPORTED_DOCUMENT_MEDIA_TYPES}, model::ModelConfig, openai::OpenAiProviderBuilder, utils::sanitize_unicode_tags, @@ -202,6 +203,11 @@ pub enum MessageContent { mime_type: String, data: Vec, }, + Document { + mime_type: String, + data: Vec, + name: Option, + }, ToolRequest { id: String, name: String, @@ -249,6 +255,23 @@ impl MessageContent { base64::engine::general_purpose::STANDARD.encode(data), mime_type.clone(), )), + MessageContent::Document { + mime_type, + data, + name, + } => { + if !document_media_type_is_supported(mime_type) { + return Err(GooseError::generic(format!( + "unsupported document media type {mime_type}: supported types are {}", + SUPPORTED_DOCUMENT_MEDIA_TYPES.join(", ") + ))); + } + Ok(GooseMessageContent::document( + base64::engine::general_purpose::STANDARD.encode(data), + mime_type.clone(), + name.clone(), + )) + } MessageContent::ToolRequest { id, name, @@ -631,6 +654,7 @@ pub enum Feature { Tools, Streaming, Images, + Documents, JsonSchema, Reasoning, } @@ -867,6 +891,12 @@ impl Provider { ) { features.push(Feature::Images); } + if matches!( + name.as_str(), + "openai" | "anthropic" | "databricks" | "databricks_v2" | "google" + ) { + features.push(Feature::Documents); + } if matches!( name.as_str(), "openai" | "anthropic" | "databricks" | "databricks_v2" @@ -1362,6 +1392,71 @@ mod tests { assert_eq!(message.as_concat_text(), "what is the capital of France?"); } + #[test] + fn provider_message_converts_document_to_base64_content() { + let message = ProviderMessage { + role: MessageRole::User, + content: vec![MessageContent::Document { + mime_type: "application/pdf".to_string(), + data: b"pdf-bytes".to_vec(), + name: Some("q3-report.pdf".to_string()), + }], + } + .to_goose_message() + .unwrap() + .unwrap(); + + let GooseMessageContent::Document(document) = &message.content[0] else { + panic!("expected document content"); + }; + assert_eq!(document.mime_type, "application/pdf"); + assert_eq!(document.name.as_deref(), Some("q3-report.pdf")); + assert_eq!( + base64::engine::general_purpose::STANDARD + .decode(&document.data) + .unwrap(), + b"pdf-bytes" + ); + } + + #[test] + fn provider_message_converts_document_serializes_for_anthropic() { + let message = ProviderMessage { + role: MessageRole::User, + content: vec![MessageContent::Document { + mime_type: "application/pdf".to_string(), + data: b"pdf-bytes".to_vec(), + name: Some("q3-report.pdf".to_string()), + }], + } + .to_goose_message() + .unwrap() + .unwrap(); + + let spec = goose_providers::formats::anthropic::format_messages(&[message]); + let block = &spec[0]["content"][0]; + + assert_eq!(block["type"], "document"); + assert_eq!(block["title"], "q3-report.pdf"); + assert_eq!(block["source"]["type"], "base64"); + assert_eq!(block["source"]["media_type"], "application/pdf"); + assert_eq!(block["source"]["data"], "cGRmLWJ5dGVz"); + } + + #[test] + fn provider_message_rejects_unsupported_document_media_type() { + let error = MessageContent::Document { + mime_type: "text/csv".to_string(), + data: b"rows".to_vec(), + name: Some("rows.csv".to_string()), + } + .to_goose_content() + .unwrap_err(); + + assert!(error.to_string().contains("text/csv"), "{error}"); + assert!(error.to_string().contains("application/pdf"), "{error}"); + } + #[test] fn tool_config_converts_to_rmcp_tool() { let tool = ProviderTool { diff --git a/crates/goose/src/agents/gen_ai_telemetry.rs b/crates/goose/src/agents/gen_ai_telemetry.rs index 5bf40474c..eb0ecefbd 100644 --- a/crates/goose/src/agents/gen_ai_telemetry.rs +++ b/crates/goose/src/agents/gen_ai_telemetry.rs @@ -220,6 +220,13 @@ fn message_part_json(content: &MessageContent) -> Value { "mime_type": image.mime_type, "content": image.data, }), + MessageContent::Document(document) => json!({ + "type": "blob", + "modality": "document", + "mime_type": document.mime_type, + "name": document.name, + "content": document.data, + }), MessageContent::ToolRequest(request) => tool_call_part(&request.id, &request.tool_call), MessageContent::ToolResponse(response) => json!({ "type": "tool_call_response", diff --git a/crates/goose/src/providers/formats/bedrock.rs b/crates/goose/src/providers/formats/bedrock.rs index 084eb9d3a..584a37d3b 100644 --- a/crates/goose/src/providers/formats/bedrock.rs +++ b/crates/goose/src/providers/formats/bedrock.rs @@ -23,6 +23,7 @@ use crate::providers::formats::anthropic::{ }; use crate::utils::{sanitize_unicode_tags, strip_unicode_tags}; use goose_providers::conversation::token_usage::Usage; +use goose_providers::documents::{unsupported_document_text, UNSUPPORTED_PROVIDER_REASON}; use goose_providers::model::ModelConfig; use once_cell::sync::Lazy; use regex::Regex; @@ -204,6 +205,9 @@ pub fn to_bedrock_message_content(content: &MessageContent) -> Result { bedrock::ContentBlock::Image(to_bedrock_image(&image.data, &image.mime_type)?) } + MessageContent::Document(document) => bedrock::ContentBlock::Text( + unsupported_document_text(document, UNSUPPORTED_PROVIDER_REASON), + ), MessageContent::Thinking(thinking) => { let mut builder = bedrock::ReasoningTextBlock::builder().text(&thinking.thinking); if !thinking.signature.is_empty() { diff --git a/documentation/src/data/gdk-api.json b/documentation/src/data/gdk-api.json index f373936e9..29eef54e1 100644 --- a/documentation/src/data/gdk-api.json +++ b/documentation/src/data/gdk-api.json @@ -390,6 +390,29 @@ } ] }, + { + "name": "Document", + "fields": [ + { + "name": "mime_type", + "type": "String", + "default": null, + "docs": "" + }, + { + "name": "data", + "type": "Vec", + "default": null, + "docs": "" + }, + { + "name": "name", + "type": "Option", + "default": null, + "docs": "" + } + ] + }, { "name": "ToolRequest", "fields": [ @@ -864,6 +887,10 @@ "name": "Images", "fields": [] }, + { + "name": "Documents", + "fields": [] + }, { "name": "JsonSchema", "fields": []