feat(gdk): add first-class document content support (#11629)

This commit is contained in:
Alex Hancock
2026-09-01 18:28:11 +00:00
committed by GitHub
parent e1283c4d20
commit 7259c05174
15 changed files with 686 additions and 2 deletions
@@ -8,6 +8,10 @@ pub fn format_message_for_compacting(msg: &Message) -> String {
.filter_map(|content| match content {
MessageContent::Text(text) => Some(text.text.clone()),
MessageContent::Image(img) => Some(format!("[image: {}]", img.mime_type)),
MessageContent::Document(doc) => Some(match &doc.name {
Some(name) => format!("[document: {} ({})]", name, doc.mime_type),
None => format!("[document: {}]", doc.mime_type),
}),
MessageContent::ToolRequest(req) => {
if let Ok(call) = &req.tool_call {
Some(format!(
@@ -286,6 +286,30 @@ pub struct ErrorContent {
pub message: String,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct DocumentContent {
pub data: String,
pub mime_type: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
}
impl DocumentContent {
pub fn new<S: Into<String>, T: Into<String>>(data: S, mime_type: T) -> Self {
Self {
data: data.into(),
mime_type: mime_type.into(),
name: None,
}
}
pub fn with_name<S: Into<String>>(mut self, name: S) -> Self {
self.name = Some(name.into());
self
}
}
pub type MessageContent = MessageContentBlock;
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
@@ -294,6 +318,7 @@ pub type MessageContent = MessageContentBlock;
pub enum MessageContentBlock {
Text(TextContent),
Image(ImageContent),
Document(DocumentContent),
ToolRequest(ToolRequest),
ToolResponse(ToolResponse),
ToolConfirmationRequest(ToolConfirmationRequest),
@@ -309,6 +334,10 @@ impl fmt::Display for MessageContentBlock {
match self {
MessageContentBlock::Text(t) => write!(f, "{}", t.text),
MessageContentBlock::Image(i) => write!(f, "[Image: {}]", i.mime_type),
MessageContentBlock::Document(d) => match &d.name {
Some(name) => write!(f, "[Document: {} ({})]", name, d.mime_type),
None => write!(f, "[Document: {}]", d.mime_type),
},
MessageContentBlock::ToolRequest(r) => {
write!(f, "[ToolRequest: {}]", r.to_readable_string())
}
@@ -441,6 +470,18 @@ impl MessageContentBlock {
MessageContentBlock::Image(ImageContent::new(data, mime_type))
}
pub fn document<S: Into<String>, T: Into<String>>(
data: S,
mime_type: T,
name: Option<String>,
) -> Self {
let document = DocumentContent::new(data, mime_type);
MessageContentBlock::Document(match name {
Some(name) => document.with_name(name),
None => document,
})
}
pub fn tool_request<S: Into<String>>(
id: S,
tool_call: ToolResult<CallToolRequestParams>,
@@ -1044,6 +1085,15 @@ impl Message {
self.with_content(MessageContentBlock::image(data, mime_type))
}
pub fn with_document<S: Into<String>, T: Into<String>>(
self,
data: S,
mime_type: T,
name: Option<String>,
) -> Self {
self.with_content(MessageContentBlock::document(data, mime_type, name))
}
/// Add a tool request to the message
pub fn with_tool_request<S: Into<String>>(
self,
@@ -1313,6 +1363,56 @@ pub struct TokenState {
pub accumulated_cost: Option<f64>,
}
#[cfg(test)]
mod document_tests {
use super::*;
#[test]
fn document_content_carries_media_type_and_name() {
let message = Message::user().with_document(
"cGRmLWJ5dGVz",
"application/pdf",
Some("q3-report.pdf".to_string()),
);
let MessageContent::Document(document) = &message.content[0] else {
panic!("expected document content");
};
assert_eq!(document.data, "cGRmLWJ5dGVz");
assert_eq!(document.mime_type, "application/pdf");
assert_eq!(document.name.as_deref(), Some("q3-report.pdf"));
assert_eq!(
message.content[0].to_string(),
"[Document: q3-report.pdf (application/pdf)]"
);
}
#[test]
fn document_content_without_name_is_allowed() {
let content = MessageContent::document("cGRmLWJ5dGVz", "application/pdf", None);
assert!(matches!(&content, MessageContent::Document(document) if document.name.is_none()));
assert_eq!(content.to_string(), "[Document: application/pdf]");
}
#[test]
fn document_content_round_trips_through_serde() {
let message = Message::user().with_document(
"cGRmLWJ5dGVz",
"application/pdf",
Some("q3-report.pdf".to_string()),
);
let json = serde_json::to_value(&message).unwrap();
assert_eq!(json["content"][0]["type"], "document");
assert_eq!(json["content"][0]["mimeType"], "application/pdf");
assert_eq!(json["content"][0]["name"], "q3-report.pdf");
let restored: Message = serde_json::from_value(json).unwrap();
assert_eq!(restored, message);
}
}
#[cfg(test)]
mod tests {
use crate::conversation::message::{
@@ -0,0 +1,80 @@
use serde_json::{json, Value};
use crate::conversation::message::DocumentContent;
#[derive(Debug, Copy, Clone)]
pub enum DocumentFormat {
OpenAi,
Anthropic,
}
/// Media types that providers accept as native document input. Anything else is
/// reported back to the caller rather than being sent as an unusable blob.
pub const SUPPORTED_DOCUMENT_MEDIA_TYPES: [&str; 1] = ["application/pdf"];
pub fn document_media_type_is_supported(mime_type: &str) -> bool {
SUPPORTED_DOCUMENT_MEDIA_TYPES.contains(&mime_type)
}
pub fn convert_document(document: &DocumentContent, format: &DocumentFormat) -> Value {
match format {
DocumentFormat::OpenAi => json!({
"type": "file",
"file": {
"filename": document.name.clone().unwrap_or_else(|| "document.pdf".to_string()),
"file_data": format!("data:{};base64,{}", document.mime_type, document.data),
}
}),
DocumentFormat::Anthropic => {
let mut block = json!({
"type": "document",
"source": {
"type": "base64",
"media_type": document.mime_type,
"data": document.data,
}
});
if let Some(name) = &document.name {
block["title"] = json!(name);
}
block
}
}
}
/// Explains why a document was dropped so the model, and the caller reading the
/// request, can act on it instead of silently losing the attachment.
pub fn unsupported_document_text(document: &DocumentContent, reason: &str) -> String {
match &document.name {
Some(name) => format!(
"[document \"{}\" ({}) not sent: {}]",
name, document.mime_type, reason
),
None => format!("[document ({}) not sent: {}]", document.mime_type, reason),
}
}
pub const UNSUPPORTED_MEDIA_TYPE_REASON: &str =
"only application/pdf documents can be sent to this provider";
pub const UNSUPPORTED_PROVIDER_REASON: &str = "this provider does not accept document input";
pub const ASSISTANT_ROLE_REASON: &str = "documents can only be sent in user messages";
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn unsupported_document_text_names_the_document_and_reason() {
let named = DocumentContent::new("data", "text/csv").with_name("rows.csv");
assert_eq!(
unsupported_document_text(&named, UNSUPPORTED_MEDIA_TYPE_REASON),
"[document \"rows.csv\" (text/csv) not sent: only application/pdf documents can be sent to this provider]"
);
let unnamed = DocumentContent::new("data", "text/csv");
assert_eq!(
unsupported_document_text(&unnamed, UNSUPPORTED_PROVIDER_REASON),
"[document (text/csv) not sent: this provider does not accept document input]"
);
}
}
@@ -2,6 +2,10 @@ use crate::canonical::maybe_get_canonical_model;
use crate::canonical::ThinkingMode;
use crate::conversation::message::{Message, MessageContentBlock};
use crate::conversation::token_usage::{CostSource, ProviderUsage, Usage};
use crate::documents::{
convert_document, document_media_type_is_supported, unsupported_document_text, DocumentFormat,
ASSISTANT_ROLE_REASON, UNSUPPORTED_MEDIA_TYPE_REASON,
};
use crate::errors::ProviderError;
use crate::images::{convert_image, ImageFormat};
use crate::mcp_utils::extract_text_from_resource;
@@ -427,6 +431,24 @@ fn format_messages_with_options(
MessageContentBlock::Image(image) => {
content.push(convert_image(image, &ImageFormat::Anthropic));
}
MessageContentBlock::Document(document) => {
if message.role != Role::User {
content.push(json!({
TYPE_FIELD: TEXT_TYPE,
TEXT_TYPE: unsupported_document_text(document, ASSISTANT_ROLE_REASON)
}));
} else if document_media_type_is_supported(&document.mime_type) {
content.push(convert_document(document, &DocumentFormat::Anthropic));
} else {
content.push(json!({
TYPE_FIELD: TEXT_TYPE,
TEXT_TYPE: unsupported_document_text(
document,
UNSUPPORTED_MEDIA_TYPE_REASON,
)
}));
}
}
}
}
@@ -1224,6 +1246,73 @@ where
}
}
#[cfg(test)]
mod document_tests {
use super::*;
#[test]
fn user_document_becomes_a_base64_document_block() {
let messages = vec![Message::user().with_document(
"cGRmLWJ5dGVz",
"application/pdf",
Some("q3-report.pdf".to_string()),
)];
let spec = format_messages(&messages);
assert_eq!(spec.len(), 1);
assert_eq!(spec[0]["role"], "user");
let block = &spec[0]["content"][0];
assert_eq!(block["type"], DOCUMENT_TYPE);
assert_eq!(block["title"], "q3-report.pdf");
assert_eq!(
block[SOURCE_FIELD],
json!({
"type": "base64",
"media_type": "application/pdf",
"data": "cGRmLWJ5dGVz",
})
);
}
#[test]
fn assistant_document_becomes_a_text_block() {
let messages = vec![Message::assistant().with_document(
"cGRmLWJ5dGVz",
"application/pdf",
Some("q3-report.pdf".to_string()),
)];
let spec = format_messages(&messages);
assert_eq!(spec.len(), 1);
assert_eq!(spec[0]["role"], "assistant");
assert_eq!(spec[0]["content"][0]["type"], "text");
let text = spec[0]["content"][0]["text"].as_str().unwrap();
assert!(text.contains("q3-report.pdf"), "{text}");
assert!(text.contains("user messages"), "{text}");
assert!(!text.contains("cGRmLWJ5dGVz"), "{text}");
}
#[test]
fn unsupported_document_media_type_becomes_an_explicit_text_block() {
let messages = vec![Message::user().with_document(
"cm93cw==",
"text/csv",
Some("rows.csv".to_string()),
)];
let spec = format_messages(&messages);
assert_eq!(spec[0]["content"][0]["type"], "text");
let text = spec[0]["content"][0]["text"].as_str().unwrap();
assert!(text.contains("rows.csv"), "{text}");
assert!(text.contains("text/csv"), "{text}");
assert!(text.contains("application/pdf"), "{text}");
assert!(!text.contains("cm93cw=="), "{text}");
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -6,6 +6,10 @@ use crate::formats::anthropic::{
};
use crate::model::{is_goose_internal_request_param, ModelConfig};
use crate::documents::{
convert_document, document_media_type_is_supported, unsupported_document_text, DocumentFormat,
ASSISTANT_ROLE_REASON, UNSUPPORTED_MEDIA_TYPE_REASON,
};
use crate::formats::openai::{
extract_reasoning_effort, is_openai_responses_model, is_valid_function_name,
openai_reasoning_effort_for_thinking, sanitize_function_name, validate_tool_schemas,
@@ -244,6 +248,21 @@ fn format_messages(
}));
}
}
MessageContentBlock::Document(document) => {
if message.role != Role::User {
content_array.push(json!({
"type": "text",
"text": unsupported_document_text(document, ASSISTANT_ROLE_REASON)
}));
} else if document_media_type_is_supported(&document.mime_type) {
content_array.push(convert_document(document, &DocumentFormat::OpenAi));
} else {
content_array.push(json!({
"type": "text",
"text": unsupported_document_text(document, UNSUPPORTED_MEDIA_TYPE_REASON)
}));
}
}
MessageContentBlock::SystemNotification(_)
| MessageContentBlock::Error(_)
| MessageContentBlock::ToolConfirmationRequest(_)
@@ -626,6 +645,53 @@ pub fn create_request_for_provider(
Ok(payload)
}
#[cfg(test)]
mod document_tests {
use super::*;
use crate::conversation::message::Message;
fn format(messages: &[Message]) -> Vec<DatabricksMessage> {
format_messages(messages, &ImageFormat::OpenAi, None, true)
}
#[test]
fn user_document_becomes_a_file_content_part() {
let spec = format(&[Message::user().with_document(
"cGRmLWJ5dGVz",
"application/pdf",
Some("q3-report.pdf".to_string()),
)]);
assert_eq!(spec.len(), 1);
assert_eq!(
spec[0].content[0],
json!({
"type": "file",
"file": {
"filename": "q3-report.pdf",
"file_data": "data:application/pdf;base64,cGRmLWJ5dGVz",
}
})
);
}
#[test]
fn assistant_document_becomes_a_text_part() {
let spec = format(&[Message::assistant().with_document(
"cGRmLWJ5dGVz",
"application/pdf",
Some("q3-report.pdf".to_string()),
)]);
assert_eq!(spec.len(), 1);
assert_eq!(spec[0].role, "assistant");
let text = spec[0].content.as_str().unwrap();
assert!(text.contains("q3-report.pdf"), "{text}");
assert!(text.contains("user messages"), "{text}");
assert!(!text.contains("cGRmLWJ5dGVz"), "{text}");
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -1,4 +1,7 @@
use crate::conversation::token_usage::{ProviderUsage, Usage};
use crate::documents::{
document_media_type_is_supported, unsupported_document_text, UNSUPPORTED_MEDIA_TYPE_REASON,
};
use crate::errors::ProviderError;
use crate::formats::openai::{is_valid_function_name, sanitize_function_name};
use crate::mcp_utils::extract_text_from_resource;
@@ -255,6 +258,20 @@ pub fn format_messages(messages: &[Message], nested_function_response_media: boo
}
}));
}
MessageContentBlock::Document(document) => {
if document_media_type_is_supported(&document.mime_type) {
parts.push(json!({
"inline_data": {
"mime_type": document.mime_type,
"data": document.data,
}
}));
} else {
parts.push(json!({
"text": unsupported_document_text(document, UNSUPPORTED_MEDIA_TYPE_REASON)
}));
}
}
_ => {}
}
@@ -935,6 +952,31 @@ mod tests {
);
}
#[test]
fn test_message_to_google_spec_document_only_message() {
let messages = vec![Message::new(
Role::User,
0,
vec![MessageContentBlock::document(
"base64pdfdata".to_string(),
"application/pdf".to_string(),
Some("report.pdf".to_string()),
)],
)];
let payload = format_messages(&messages, false);
assert_eq!(payload.len(), 1);
assert_eq!(payload[0]["role"], "user");
assert_eq!(
payload[0]["parts"][0]["inline_data"]["mime_type"],
"application/pdf"
);
assert_eq!(
payload[0]["parts"][0]["inline_data"]["data"],
"base64pdfdata"
);
}
#[test]
fn test_message_to_google_spec_tool_request_message() {
let arguments = json!({
@@ -1,6 +1,10 @@
use crate::base::ThinkingPreservationFormat;
use crate::conversation::message::{Message, MessageContentBlock, ProviderMetadata};
use crate::conversation::token_usage::{CostSource, ProviderUsage, Usage};
use crate::documents::{
convert_document, document_media_type_is_supported, unsupported_document_text, DocumentFormat,
ASSISTANT_ROLE_REASON, UNSUPPORTED_MEDIA_TYPE_REASON,
};
use crate::errors::ProviderError;
use crate::images::{convert_image, detect_image_path, load_image_file, ImageFormat};
use crate::json::{parse_tool_arguments, truncation_error_message};
@@ -426,6 +430,22 @@ pub fn format_messages_with_options(
}));
}
}
MessageContentBlock::Document(document) => {
if message.role != Role::User {
content_array.push(json!({
"type": "text",
"text": unsupported_document_text(document, ASSISTANT_ROLE_REASON)
}));
} else if document_media_type_is_supported(&document.mime_type) {
has_non_text_content = true;
content_array.push(convert_document(document, &DocumentFormat::OpenAi));
} else {
content_array.push(json!({
"type": "text",
"text": unsupported_document_text(document, UNSUPPORTED_MEDIA_TYPE_REASON)
}));
}
}
}
}
@@ -1930,6 +1950,73 @@ pub fn is_valid_function_name(name: &str) -> bool {
re.is_match(name)
}
#[cfg(test)]
mod document_tests {
use super::*;
fn format(messages: &[Message]) -> Vec<Value> {
format_messages_with_options(
messages,
&ImageFormat::OpenAi,
OpenAiFormatOptions {
supports_vision: true,
..Default::default()
},
)
}
#[test]
fn user_document_becomes_a_file_content_part() {
let spec = format(&[Message::user().with_document(
"cGRmLWJ5dGVz",
"application/pdf",
Some("q3-report.pdf".to_string()),
)]);
assert_eq!(spec.len(), 1);
assert_eq!(
spec[0]["content"][0],
json!({
"type": "file",
"file": {
"filename": "q3-report.pdf",
"file_data": "data:application/pdf;base64,cGRmLWJ5dGVz",
}
})
);
}
#[test]
fn assistant_document_becomes_a_text_part() {
let spec = format(&[Message::assistant().with_document(
"cGRmLWJ5dGVz",
"application/pdf",
Some("q3-report.pdf".to_string()),
)]);
assert_eq!(spec.len(), 1);
assert_eq!(spec[0]["role"], "assistant");
let text = spec[0]["content"].as_str().unwrap();
assert!(text.contains("q3-report.pdf"), "{text}");
assert!(text.contains("user messages"), "{text}");
assert!(!text.contains("cGRmLWJ5dGVz"), "{text}");
}
#[test]
fn unsupported_document_media_type_becomes_an_explicit_text_part() {
let spec = format(&[Message::user().with_document(
"cm93cw==",
"text/csv",
Some("rows.csv".to_string()),
)]);
let text = spec[0]["content"].as_str().unwrap();
assert!(text.contains("rows.csv"), "{text}");
assert!(text.contains("application/pdf"), "{text}");
assert!(!text.contains("cm93cw=="), "{text}");
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -1,5 +1,9 @@
use crate::conversation::message::{Message, MessageContentBlock};
use crate::conversation::token_usage::{ProviderUsage, Usage};
use crate::documents::{
convert_document, document_media_type_is_supported, unsupported_document_text, DocumentFormat,
ASSISTANT_ROLE_REASON, UNSUPPORTED_MEDIA_TYPE_REASON,
};
use crate::errors::ProviderError;
use crate::formats::openai::{
extract_reasoning_effort, is_openai_responses_model, openai_reasoning_effort_for_thinking,
@@ -480,6 +484,25 @@ fn add_message_items(input_items: &mut Vec<Value>, messages: &[Message], support
}));
}
}
MessageContentBlock::Document(document) => {
if message.role != Role::User {
text_items.push(json!({
"type": "output_text",
"text": unsupported_document_text(document, ASSISTANT_ROLE_REASON),
"annotations": []
}));
} else if document_media_type_is_supported(&document.mime_type) {
let mut converted = convert_document(document, &DocumentFormat::OpenAi);
let mut file = converted["file"].take();
file["type"] = json!("input_file");
text_items.push(file);
} else {
text_items.push(json!({
"type": "input_text",
"text": unsupported_document_text(document, UNSUPPORTED_MEDIA_TYPE_REASON)
}));
}
}
MessageContentBlock::ToolResponse(response) => {
if !text_items.is_empty() {
input_items.push(json!({
@@ -1194,6 +1217,55 @@ where
}
}
#[cfg(test)]
mod document_tests {
use super::*;
fn format(messages: &[Message]) -> Vec<Value> {
let mut items = Vec::new();
add_message_items(&mut items, messages, true);
items
}
#[test]
fn user_document_becomes_an_input_file_item() {
let items = format(&[Message::user().with_document(
"cGRmLWJ5dGVz",
"application/pdf",
Some("q3-report.pdf".to_string()),
)]);
assert_eq!(items.len(), 1);
assert_eq!(items[0]["role"], "user");
assert_eq!(
items[0]["content"][0],
json!({
"type": "input_file",
"filename": "q3-report.pdf",
"file_data": "data:application/pdf;base64,cGRmLWJ5dGVz",
})
);
}
#[test]
fn assistant_document_becomes_an_output_text_item() {
let items = format(&[Message::assistant().with_document(
"cGRmLWJ5dGVz",
"application/pdf",
Some("q3-report.pdf".to_string()),
)]);
assert_eq!(items.len(), 1);
assert_eq!(items[0]["role"], "assistant");
let part = &items[0]["content"][0];
assert_eq!(part["type"], "output_text");
let text = part["text"].as_str().unwrap();
assert!(text.contains("q3-report.pdf"), "{text}");
assert!(text.contains("user messages"), "{text}");
assert!(!text.contains("cGRmLWJ5dGVz"), "{text}");
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -1,5 +1,6 @@
use crate::conversation::message::{Message, MessageContentBlock};
use crate::conversation::token_usage::Usage;
use crate::documents::{unsupported_document_text, UNSUPPORTED_PROVIDER_REASON};
use crate::errors::ProviderError;
use crate::mcp_utils::extract_text_from_resource;
use crate::model::ModelConfig;
@@ -74,6 +75,15 @@ pub fn format_messages(messages: &[Message]) -> Vec<Value> {
// Skip redacted thinking for now
}
MessageContentBlock::Image(_) => continue, // Snowflake doesn't support image content yet
MessageContentBlock::Document(document) => {
if !text_content.is_empty() {
text_content.push('\n');
}
text_content.push_str(&unsupported_document_text(
document,
UNSUPPORTED_PROVIDER_REASON,
));
}
}
}
+1
View File
@@ -3,6 +3,7 @@ pub mod cache_semantics;
pub mod canonical;
pub mod context_limit;
pub mod conversation;
pub mod documents;
pub mod errors;
pub mod formats;
pub mod goose_mode;
+2 -2
View File
@@ -6,8 +6,8 @@ pub mod databricks_auth;
pub mod databricks_v2;
pub mod google;
pub use goose_provider_types::{
base, cache_semantics, canonical, context_limit, conversation, errors, formats, goose_mode,
images, json, model, permission, request_log, retry, thinking, utils,
base, cache_semantics, canonical, context_limit, conversation, documents, errors, formats,
goose_mode, images, json, model, permission, request_log, retry, thinking, utils,
};
pub mod declarative;
pub mod http_status;
+95
View File
@@ -20,6 +20,7 @@ use goose_providers::{
databricks_auth::DatabricksAuth,
databricks_v2::DatabricksV2Provider as GooseDatabricksV2Provider,
declarative::{DeclarativeProviderConfig, EnvKeyResolver},
documents::{document_media_type_is_supported, SUPPORTED_DOCUMENT_MEDIA_TYPES},
model::ModelConfig,
openai::OpenAiProviderBuilder,
utils::sanitize_unicode_tags,
@@ -202,6 +203,11 @@ pub enum MessageContent {
mime_type: String,
data: Vec<u8>,
},
Document {
mime_type: String,
data: Vec<u8>,
name: Option<String>,
},
ToolRequest {
id: String,
name: String,
@@ -249,6 +255,23 @@ impl MessageContent {
base64::engine::general_purpose::STANDARD.encode(data),
mime_type.clone(),
)),
MessageContent::Document {
mime_type,
data,
name,
} => {
if !document_media_type_is_supported(mime_type) {
return Err(GooseError::generic(format!(
"unsupported document media type {mime_type}: supported types are {}",
SUPPORTED_DOCUMENT_MEDIA_TYPES.join(", ")
)));
}
Ok(GooseMessageContent::document(
base64::engine::general_purpose::STANDARD.encode(data),
mime_type.clone(),
name.clone(),
))
}
MessageContent::ToolRequest {
id,
name,
@@ -631,6 +654,7 @@ pub enum Feature {
Tools,
Streaming,
Images,
Documents,
JsonSchema,
Reasoning,
}
@@ -867,6 +891,12 @@ impl Provider {
) {
features.push(Feature::Images);
}
if matches!(
name.as_str(),
"openai" | "anthropic" | "databricks" | "databricks_v2" | "google"
) {
features.push(Feature::Documents);
}
if matches!(
name.as_str(),
"openai" | "anthropic" | "databricks" | "databricks_v2"
@@ -1362,6 +1392,71 @@ mod tests {
assert_eq!(message.as_concat_text(), "what is the capital of France?");
}
#[test]
fn provider_message_converts_document_to_base64_content() {
let message = ProviderMessage {
role: MessageRole::User,
content: vec![MessageContent::Document {
mime_type: "application/pdf".to_string(),
data: b"pdf-bytes".to_vec(),
name: Some("q3-report.pdf".to_string()),
}],
}
.to_goose_message()
.unwrap()
.unwrap();
let GooseMessageContent::Document(document) = &message.content[0] else {
panic!("expected document content");
};
assert_eq!(document.mime_type, "application/pdf");
assert_eq!(document.name.as_deref(), Some("q3-report.pdf"));
assert_eq!(
base64::engine::general_purpose::STANDARD
.decode(&document.data)
.unwrap(),
b"pdf-bytes"
);
}
#[test]
fn provider_message_converts_document_serializes_for_anthropic() {
let message = ProviderMessage {
role: MessageRole::User,
content: vec![MessageContent::Document {
mime_type: "application/pdf".to_string(),
data: b"pdf-bytes".to_vec(),
name: Some("q3-report.pdf".to_string()),
}],
}
.to_goose_message()
.unwrap()
.unwrap();
let spec = goose_providers::formats::anthropic::format_messages(&[message]);
let block = &spec[0]["content"][0];
assert_eq!(block["type"], "document");
assert_eq!(block["title"], "q3-report.pdf");
assert_eq!(block["source"]["type"], "base64");
assert_eq!(block["source"]["media_type"], "application/pdf");
assert_eq!(block["source"]["data"], "cGRmLWJ5dGVz");
}
#[test]
fn provider_message_rejects_unsupported_document_media_type() {
let error = MessageContent::Document {
mime_type: "text/csv".to_string(),
data: b"rows".to_vec(),
name: Some("rows.csv".to_string()),
}
.to_goose_content()
.unwrap_err();
assert!(error.to_string().contains("text/csv"), "{error}");
assert!(error.to_string().contains("application/pdf"), "{error}");
}
#[test]
fn tool_config_converts_to_rmcp_tool() {
let tool = ProviderTool {
@@ -220,6 +220,13 @@ fn message_part_json(content: &MessageContent) -> Value {
"mime_type": image.mime_type,
"content": image.data,
}),
MessageContent::Document(document) => json!({
"type": "blob",
"modality": "document",
"mime_type": document.mime_type,
"name": document.name,
"content": document.data,
}),
MessageContent::ToolRequest(request) => tool_call_part(&request.id, &request.tool_call),
MessageContent::ToolResponse(response) => json!({
"type": "tool_call_response",
@@ -23,6 +23,7 @@ use crate::providers::formats::anthropic::{
};
use crate::utils::{sanitize_unicode_tags, strip_unicode_tags};
use goose_providers::conversation::token_usage::Usage;
use goose_providers::documents::{unsupported_document_text, UNSUPPORTED_PROVIDER_REASON};
use goose_providers::model::ModelConfig;
use once_cell::sync::Lazy;
use regex::Regex;
@@ -204,6 +205,9 @@ pub fn to_bedrock_message_content(content: &MessageContent) -> Result<bedrock::C
MessageContent::Image(image) => {
bedrock::ContentBlock::Image(to_bedrock_image(&image.data, &image.mime_type)?)
}
MessageContent::Document(document) => bedrock::ContentBlock::Text(
unsupported_document_text(document, UNSUPPORTED_PROVIDER_REASON),
),
MessageContent::Thinking(thinking) => {
let mut builder = bedrock::ReasoningTextBlock::builder().text(&thinking.thinking);
if !thinking.signature.is_empty() {
+27
View File
@@ -390,6 +390,29 @@
}
]
},
{
"name": "Document",
"fields": [
{
"name": "mime_type",
"type": "String",
"default": null,
"docs": ""
},
{
"name": "data",
"type": "Vec<u8>",
"default": null,
"docs": ""
},
{
"name": "name",
"type": "Option<String>",
"default": null,
"docs": ""
}
]
},
{
"name": "ToolRequest",
"fields": [
@@ -864,6 +887,10 @@
"name": "Images",
"fields": []
},
{
"name": "Documents",
"fields": []
},
{
"name": "JsonSchema",
"fields": []