feat(gdk): add first-class document content support (#11629)
This commit is contained in:
@@ -8,6 +8,10 @@ pub fn format_message_for_compacting(msg: &Message) -> String {
|
||||
.filter_map(|content| match content {
|
||||
MessageContent::Text(text) => Some(text.text.clone()),
|
||||
MessageContent::Image(img) => Some(format!("[image: {}]", img.mime_type)),
|
||||
MessageContent::Document(doc) => Some(match &doc.name {
|
||||
Some(name) => format!("[document: {} ({})]", name, doc.mime_type),
|
||||
None => format!("[document: {}]", doc.mime_type),
|
||||
}),
|
||||
MessageContent::ToolRequest(req) => {
|
||||
if let Ok(call) = &req.tool_call {
|
||||
Some(format!(
|
||||
|
||||
@@ -286,6 +286,30 @@ pub struct ErrorContent {
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DocumentContent {
|
||||
pub data: String,
|
||||
pub mime_type: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub name: Option<String>,
|
||||
}
|
||||
|
||||
impl DocumentContent {
|
||||
pub fn new<S: Into<String>, T: Into<String>>(data: S, mime_type: T) -> Self {
|
||||
Self {
|
||||
data: data.into(),
|
||||
mime_type: mime_type.into(),
|
||||
name: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_name<S: Into<String>>(mut self, name: S) -> Self {
|
||||
self.name = Some(name.into());
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
pub type MessageContent = MessageContentBlock;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
@@ -294,6 +318,7 @@ pub type MessageContent = MessageContentBlock;
|
||||
pub enum MessageContentBlock {
|
||||
Text(TextContent),
|
||||
Image(ImageContent),
|
||||
Document(DocumentContent),
|
||||
ToolRequest(ToolRequest),
|
||||
ToolResponse(ToolResponse),
|
||||
ToolConfirmationRequest(ToolConfirmationRequest),
|
||||
@@ -309,6 +334,10 @@ impl fmt::Display for MessageContentBlock {
|
||||
match self {
|
||||
MessageContentBlock::Text(t) => write!(f, "{}", t.text),
|
||||
MessageContentBlock::Image(i) => write!(f, "[Image: {}]", i.mime_type),
|
||||
MessageContentBlock::Document(d) => match &d.name {
|
||||
Some(name) => write!(f, "[Document: {} ({})]", name, d.mime_type),
|
||||
None => write!(f, "[Document: {}]", d.mime_type),
|
||||
},
|
||||
MessageContentBlock::ToolRequest(r) => {
|
||||
write!(f, "[ToolRequest: {}]", r.to_readable_string())
|
||||
}
|
||||
@@ -441,6 +470,18 @@ impl MessageContentBlock {
|
||||
MessageContentBlock::Image(ImageContent::new(data, mime_type))
|
||||
}
|
||||
|
||||
pub fn document<S: Into<String>, T: Into<String>>(
|
||||
data: S,
|
||||
mime_type: T,
|
||||
name: Option<String>,
|
||||
) -> Self {
|
||||
let document = DocumentContent::new(data, mime_type);
|
||||
MessageContentBlock::Document(match name {
|
||||
Some(name) => document.with_name(name),
|
||||
None => document,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn tool_request<S: Into<String>>(
|
||||
id: S,
|
||||
tool_call: ToolResult<CallToolRequestParams>,
|
||||
@@ -1044,6 +1085,15 @@ impl Message {
|
||||
self.with_content(MessageContentBlock::image(data, mime_type))
|
||||
}
|
||||
|
||||
pub fn with_document<S: Into<String>, T: Into<String>>(
|
||||
self,
|
||||
data: S,
|
||||
mime_type: T,
|
||||
name: Option<String>,
|
||||
) -> Self {
|
||||
self.with_content(MessageContentBlock::document(data, mime_type, name))
|
||||
}
|
||||
|
||||
/// Add a tool request to the message
|
||||
pub fn with_tool_request<S: Into<String>>(
|
||||
self,
|
||||
@@ -1313,6 +1363,56 @@ pub struct TokenState {
|
||||
pub accumulated_cost: Option<f64>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod document_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn document_content_carries_media_type_and_name() {
|
||||
let message = Message::user().with_document(
|
||||
"cGRmLWJ5dGVz",
|
||||
"application/pdf",
|
||||
Some("q3-report.pdf".to_string()),
|
||||
);
|
||||
|
||||
let MessageContent::Document(document) = &message.content[0] else {
|
||||
panic!("expected document content");
|
||||
};
|
||||
assert_eq!(document.data, "cGRmLWJ5dGVz");
|
||||
assert_eq!(document.mime_type, "application/pdf");
|
||||
assert_eq!(document.name.as_deref(), Some("q3-report.pdf"));
|
||||
assert_eq!(
|
||||
message.content[0].to_string(),
|
||||
"[Document: q3-report.pdf (application/pdf)]"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn document_content_without_name_is_allowed() {
|
||||
let content = MessageContent::document("cGRmLWJ5dGVz", "application/pdf", None);
|
||||
|
||||
assert!(matches!(&content, MessageContent::Document(document) if document.name.is_none()));
|
||||
assert_eq!(content.to_string(), "[Document: application/pdf]");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn document_content_round_trips_through_serde() {
|
||||
let message = Message::user().with_document(
|
||||
"cGRmLWJ5dGVz",
|
||||
"application/pdf",
|
||||
Some("q3-report.pdf".to_string()),
|
||||
);
|
||||
|
||||
let json = serde_json::to_value(&message).unwrap();
|
||||
assert_eq!(json["content"][0]["type"], "document");
|
||||
assert_eq!(json["content"][0]["mimeType"], "application/pdf");
|
||||
assert_eq!(json["content"][0]["name"], "q3-report.pdf");
|
||||
|
||||
let restored: Message = serde_json::from_value(json).unwrap();
|
||||
assert_eq!(restored, message);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::conversation::message::{
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::conversation::message::DocumentContent;
|
||||
|
||||
#[derive(Debug, Copy, Clone)]
|
||||
pub enum DocumentFormat {
|
||||
OpenAi,
|
||||
Anthropic,
|
||||
}
|
||||
|
||||
/// Media types that providers accept as native document input. Anything else is
|
||||
/// reported back to the caller rather than being sent as an unusable blob.
|
||||
pub const SUPPORTED_DOCUMENT_MEDIA_TYPES: [&str; 1] = ["application/pdf"];
|
||||
|
||||
pub fn document_media_type_is_supported(mime_type: &str) -> bool {
|
||||
SUPPORTED_DOCUMENT_MEDIA_TYPES.contains(&mime_type)
|
||||
}
|
||||
|
||||
pub fn convert_document(document: &DocumentContent, format: &DocumentFormat) -> Value {
|
||||
match format {
|
||||
DocumentFormat::OpenAi => json!({
|
||||
"type": "file",
|
||||
"file": {
|
||||
"filename": document.name.clone().unwrap_or_else(|| "document.pdf".to_string()),
|
||||
"file_data": format!("data:{};base64,{}", document.mime_type, document.data),
|
||||
}
|
||||
}),
|
||||
DocumentFormat::Anthropic => {
|
||||
let mut block = json!({
|
||||
"type": "document",
|
||||
"source": {
|
||||
"type": "base64",
|
||||
"media_type": document.mime_type,
|
||||
"data": document.data,
|
||||
}
|
||||
});
|
||||
if let Some(name) = &document.name {
|
||||
block["title"] = json!(name);
|
||||
}
|
||||
block
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Explains why a document was dropped so the model, and the caller reading the
|
||||
/// request, can act on it instead of silently losing the attachment.
|
||||
pub fn unsupported_document_text(document: &DocumentContent, reason: &str) -> String {
|
||||
match &document.name {
|
||||
Some(name) => format!(
|
||||
"[document \"{}\" ({}) not sent: {}]",
|
||||
name, document.mime_type, reason
|
||||
),
|
||||
None => format!("[document ({}) not sent: {}]", document.mime_type, reason),
|
||||
}
|
||||
}
|
||||
|
||||
pub const UNSUPPORTED_MEDIA_TYPE_REASON: &str =
|
||||
"only application/pdf documents can be sent to this provider";
|
||||
pub const UNSUPPORTED_PROVIDER_REASON: &str = "this provider does not accept document input";
|
||||
pub const ASSISTANT_ROLE_REASON: &str = "documents can only be sent in user messages";
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn unsupported_document_text_names_the_document_and_reason() {
|
||||
let named = DocumentContent::new("data", "text/csv").with_name("rows.csv");
|
||||
assert_eq!(
|
||||
unsupported_document_text(&named, UNSUPPORTED_MEDIA_TYPE_REASON),
|
||||
"[document \"rows.csv\" (text/csv) not sent: only application/pdf documents can be sent to this provider]"
|
||||
);
|
||||
|
||||
let unnamed = DocumentContent::new("data", "text/csv");
|
||||
assert_eq!(
|
||||
unsupported_document_text(&unnamed, UNSUPPORTED_PROVIDER_REASON),
|
||||
"[document (text/csv) not sent: this provider does not accept document input]"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,10 @@ use crate::canonical::maybe_get_canonical_model;
|
||||
use crate::canonical::ThinkingMode;
|
||||
use crate::conversation::message::{Message, MessageContentBlock};
|
||||
use crate::conversation::token_usage::{CostSource, ProviderUsage, Usage};
|
||||
use crate::documents::{
|
||||
convert_document, document_media_type_is_supported, unsupported_document_text, DocumentFormat,
|
||||
ASSISTANT_ROLE_REASON, UNSUPPORTED_MEDIA_TYPE_REASON,
|
||||
};
|
||||
use crate::errors::ProviderError;
|
||||
use crate::images::{convert_image, ImageFormat};
|
||||
use crate::mcp_utils::extract_text_from_resource;
|
||||
@@ -427,6 +431,24 @@ fn format_messages_with_options(
|
||||
MessageContentBlock::Image(image) => {
|
||||
content.push(convert_image(image, &ImageFormat::Anthropic));
|
||||
}
|
||||
MessageContentBlock::Document(document) => {
|
||||
if message.role != Role::User {
|
||||
content.push(json!({
|
||||
TYPE_FIELD: TEXT_TYPE,
|
||||
TEXT_TYPE: unsupported_document_text(document, ASSISTANT_ROLE_REASON)
|
||||
}));
|
||||
} else if document_media_type_is_supported(&document.mime_type) {
|
||||
content.push(convert_document(document, &DocumentFormat::Anthropic));
|
||||
} else {
|
||||
content.push(json!({
|
||||
TYPE_FIELD: TEXT_TYPE,
|
||||
TEXT_TYPE: unsupported_document_text(
|
||||
document,
|
||||
UNSUPPORTED_MEDIA_TYPE_REASON,
|
||||
)
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1224,6 +1246,73 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod document_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn user_document_becomes_a_base64_document_block() {
|
||||
let messages = vec![Message::user().with_document(
|
||||
"cGRmLWJ5dGVz",
|
||||
"application/pdf",
|
||||
Some("q3-report.pdf".to_string()),
|
||||
)];
|
||||
|
||||
let spec = format_messages(&messages);
|
||||
|
||||
assert_eq!(spec.len(), 1);
|
||||
assert_eq!(spec[0]["role"], "user");
|
||||
let block = &spec[0]["content"][0];
|
||||
assert_eq!(block["type"], DOCUMENT_TYPE);
|
||||
assert_eq!(block["title"], "q3-report.pdf");
|
||||
assert_eq!(
|
||||
block[SOURCE_FIELD],
|
||||
json!({
|
||||
"type": "base64",
|
||||
"media_type": "application/pdf",
|
||||
"data": "cGRmLWJ5dGVz",
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assistant_document_becomes_a_text_block() {
|
||||
let messages = vec![Message::assistant().with_document(
|
||||
"cGRmLWJ5dGVz",
|
||||
"application/pdf",
|
||||
Some("q3-report.pdf".to_string()),
|
||||
)];
|
||||
|
||||
let spec = format_messages(&messages);
|
||||
|
||||
assert_eq!(spec.len(), 1);
|
||||
assert_eq!(spec[0]["role"], "assistant");
|
||||
assert_eq!(spec[0]["content"][0]["type"], "text");
|
||||
let text = spec[0]["content"][0]["text"].as_str().unwrap();
|
||||
assert!(text.contains("q3-report.pdf"), "{text}");
|
||||
assert!(text.contains("user messages"), "{text}");
|
||||
assert!(!text.contains("cGRmLWJ5dGVz"), "{text}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsupported_document_media_type_becomes_an_explicit_text_block() {
|
||||
let messages = vec![Message::user().with_document(
|
||||
"cm93cw==",
|
||||
"text/csv",
|
||||
Some("rows.csv".to_string()),
|
||||
)];
|
||||
|
||||
let spec = format_messages(&messages);
|
||||
|
||||
assert_eq!(spec[0]["content"][0]["type"], "text");
|
||||
let text = spec[0]["content"][0]["text"].as_str().unwrap();
|
||||
assert!(text.contains("rows.csv"), "{text}");
|
||||
assert!(text.contains("text/csv"), "{text}");
|
||||
assert!(text.contains("application/pdf"), "{text}");
|
||||
assert!(!text.contains("cm93cw=="), "{text}");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -6,6 +6,10 @@ use crate::formats::anthropic::{
|
||||
};
|
||||
use crate::model::{is_goose_internal_request_param, ModelConfig};
|
||||
|
||||
use crate::documents::{
|
||||
convert_document, document_media_type_is_supported, unsupported_document_text, DocumentFormat,
|
||||
ASSISTANT_ROLE_REASON, UNSUPPORTED_MEDIA_TYPE_REASON,
|
||||
};
|
||||
use crate::formats::openai::{
|
||||
extract_reasoning_effort, is_openai_responses_model, is_valid_function_name,
|
||||
openai_reasoning_effort_for_thinking, sanitize_function_name, validate_tool_schemas,
|
||||
@@ -244,6 +248,21 @@ fn format_messages(
|
||||
}));
|
||||
}
|
||||
}
|
||||
MessageContentBlock::Document(document) => {
|
||||
if message.role != Role::User {
|
||||
content_array.push(json!({
|
||||
"type": "text",
|
||||
"text": unsupported_document_text(document, ASSISTANT_ROLE_REASON)
|
||||
}));
|
||||
} else if document_media_type_is_supported(&document.mime_type) {
|
||||
content_array.push(convert_document(document, &DocumentFormat::OpenAi));
|
||||
} else {
|
||||
content_array.push(json!({
|
||||
"type": "text",
|
||||
"text": unsupported_document_text(document, UNSUPPORTED_MEDIA_TYPE_REASON)
|
||||
}));
|
||||
}
|
||||
}
|
||||
MessageContentBlock::SystemNotification(_)
|
||||
| MessageContentBlock::Error(_)
|
||||
| MessageContentBlock::ToolConfirmationRequest(_)
|
||||
@@ -626,6 +645,53 @@ pub fn create_request_for_provider(
|
||||
Ok(payload)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod document_tests {
|
||||
use super::*;
|
||||
use crate::conversation::message::Message;
|
||||
|
||||
fn format(messages: &[Message]) -> Vec<DatabricksMessage> {
|
||||
format_messages(messages, &ImageFormat::OpenAi, None, true)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn user_document_becomes_a_file_content_part() {
|
||||
let spec = format(&[Message::user().with_document(
|
||||
"cGRmLWJ5dGVz",
|
||||
"application/pdf",
|
||||
Some("q3-report.pdf".to_string()),
|
||||
)]);
|
||||
|
||||
assert_eq!(spec.len(), 1);
|
||||
assert_eq!(
|
||||
spec[0].content[0],
|
||||
json!({
|
||||
"type": "file",
|
||||
"file": {
|
||||
"filename": "q3-report.pdf",
|
||||
"file_data": "data:application/pdf;base64,cGRmLWJ5dGVz",
|
||||
}
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assistant_document_becomes_a_text_part() {
|
||||
let spec = format(&[Message::assistant().with_document(
|
||||
"cGRmLWJ5dGVz",
|
||||
"application/pdf",
|
||||
Some("q3-report.pdf".to_string()),
|
||||
)]);
|
||||
|
||||
assert_eq!(spec.len(), 1);
|
||||
assert_eq!(spec[0].role, "assistant");
|
||||
let text = spec[0].content.as_str().unwrap();
|
||||
assert!(text.contains("q3-report.pdf"), "{text}");
|
||||
assert!(text.contains("user messages"), "{text}");
|
||||
assert!(!text.contains("cGRmLWJ5dGVz"), "{text}");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
use crate::conversation::token_usage::{ProviderUsage, Usage};
|
||||
use crate::documents::{
|
||||
document_media_type_is_supported, unsupported_document_text, UNSUPPORTED_MEDIA_TYPE_REASON,
|
||||
};
|
||||
use crate::errors::ProviderError;
|
||||
use crate::formats::openai::{is_valid_function_name, sanitize_function_name};
|
||||
use crate::mcp_utils::extract_text_from_resource;
|
||||
@@ -255,6 +258,20 @@ pub fn format_messages(messages: &[Message], nested_function_response_media: boo
|
||||
}
|
||||
}));
|
||||
}
|
||||
MessageContentBlock::Document(document) => {
|
||||
if document_media_type_is_supported(&document.mime_type) {
|
||||
parts.push(json!({
|
||||
"inline_data": {
|
||||
"mime_type": document.mime_type,
|
||||
"data": document.data,
|
||||
}
|
||||
}));
|
||||
} else {
|
||||
parts.push(json!({
|
||||
"text": unsupported_document_text(document, UNSUPPORTED_MEDIA_TYPE_REASON)
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
_ => {}
|
||||
}
|
||||
@@ -935,6 +952,31 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_message_to_google_spec_document_only_message() {
|
||||
let messages = vec![Message::new(
|
||||
Role::User,
|
||||
0,
|
||||
vec![MessageContentBlock::document(
|
||||
"base64pdfdata".to_string(),
|
||||
"application/pdf".to_string(),
|
||||
Some("report.pdf".to_string()),
|
||||
)],
|
||||
)];
|
||||
let payload = format_messages(&messages, false);
|
||||
|
||||
assert_eq!(payload.len(), 1);
|
||||
assert_eq!(payload[0]["role"], "user");
|
||||
assert_eq!(
|
||||
payload[0]["parts"][0]["inline_data"]["mime_type"],
|
||||
"application/pdf"
|
||||
);
|
||||
assert_eq!(
|
||||
payload[0]["parts"][0]["inline_data"]["data"],
|
||||
"base64pdfdata"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_message_to_google_spec_tool_request_message() {
|
||||
let arguments = json!({
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
use crate::base::ThinkingPreservationFormat;
|
||||
use crate::conversation::message::{Message, MessageContentBlock, ProviderMetadata};
|
||||
use crate::conversation::token_usage::{CostSource, ProviderUsage, Usage};
|
||||
use crate::documents::{
|
||||
convert_document, document_media_type_is_supported, unsupported_document_text, DocumentFormat,
|
||||
ASSISTANT_ROLE_REASON, UNSUPPORTED_MEDIA_TYPE_REASON,
|
||||
};
|
||||
use crate::errors::ProviderError;
|
||||
use crate::images::{convert_image, detect_image_path, load_image_file, ImageFormat};
|
||||
use crate::json::{parse_tool_arguments, truncation_error_message};
|
||||
@@ -426,6 +430,22 @@ pub fn format_messages_with_options(
|
||||
}));
|
||||
}
|
||||
}
|
||||
MessageContentBlock::Document(document) => {
|
||||
if message.role != Role::User {
|
||||
content_array.push(json!({
|
||||
"type": "text",
|
||||
"text": unsupported_document_text(document, ASSISTANT_ROLE_REASON)
|
||||
}));
|
||||
} else if document_media_type_is_supported(&document.mime_type) {
|
||||
has_non_text_content = true;
|
||||
content_array.push(convert_document(document, &DocumentFormat::OpenAi));
|
||||
} else {
|
||||
content_array.push(json!({
|
||||
"type": "text",
|
||||
"text": unsupported_document_text(document, UNSUPPORTED_MEDIA_TYPE_REASON)
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1930,6 +1950,73 @@ pub fn is_valid_function_name(name: &str) -> bool {
|
||||
re.is_match(name)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod document_tests {
|
||||
use super::*;
|
||||
|
||||
fn format(messages: &[Message]) -> Vec<Value> {
|
||||
format_messages_with_options(
|
||||
messages,
|
||||
&ImageFormat::OpenAi,
|
||||
OpenAiFormatOptions {
|
||||
supports_vision: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn user_document_becomes_a_file_content_part() {
|
||||
let spec = format(&[Message::user().with_document(
|
||||
"cGRmLWJ5dGVz",
|
||||
"application/pdf",
|
||||
Some("q3-report.pdf".to_string()),
|
||||
)]);
|
||||
|
||||
assert_eq!(spec.len(), 1);
|
||||
assert_eq!(
|
||||
spec[0]["content"][0],
|
||||
json!({
|
||||
"type": "file",
|
||||
"file": {
|
||||
"filename": "q3-report.pdf",
|
||||
"file_data": "data:application/pdf;base64,cGRmLWJ5dGVz",
|
||||
}
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assistant_document_becomes_a_text_part() {
|
||||
let spec = format(&[Message::assistant().with_document(
|
||||
"cGRmLWJ5dGVz",
|
||||
"application/pdf",
|
||||
Some("q3-report.pdf".to_string()),
|
||||
)]);
|
||||
|
||||
assert_eq!(spec.len(), 1);
|
||||
assert_eq!(spec[0]["role"], "assistant");
|
||||
let text = spec[0]["content"].as_str().unwrap();
|
||||
assert!(text.contains("q3-report.pdf"), "{text}");
|
||||
assert!(text.contains("user messages"), "{text}");
|
||||
assert!(!text.contains("cGRmLWJ5dGVz"), "{text}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsupported_document_media_type_becomes_an_explicit_text_part() {
|
||||
let spec = format(&[Message::user().with_document(
|
||||
"cm93cw==",
|
||||
"text/csv",
|
||||
Some("rows.csv".to_string()),
|
||||
)]);
|
||||
|
||||
let text = spec[0]["content"].as_str().unwrap();
|
||||
assert!(text.contains("rows.csv"), "{text}");
|
||||
assert!(text.contains("application/pdf"), "{text}");
|
||||
assert!(!text.contains("cm93cw=="), "{text}");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
use crate::conversation::message::{Message, MessageContentBlock};
|
||||
use crate::conversation::token_usage::{ProviderUsage, Usage};
|
||||
use crate::documents::{
|
||||
convert_document, document_media_type_is_supported, unsupported_document_text, DocumentFormat,
|
||||
ASSISTANT_ROLE_REASON, UNSUPPORTED_MEDIA_TYPE_REASON,
|
||||
};
|
||||
use crate::errors::ProviderError;
|
||||
use crate::formats::openai::{
|
||||
extract_reasoning_effort, is_openai_responses_model, openai_reasoning_effort_for_thinking,
|
||||
@@ -480,6 +484,25 @@ fn add_message_items(input_items: &mut Vec<Value>, messages: &[Message], support
|
||||
}));
|
||||
}
|
||||
}
|
||||
MessageContentBlock::Document(document) => {
|
||||
if message.role != Role::User {
|
||||
text_items.push(json!({
|
||||
"type": "output_text",
|
||||
"text": unsupported_document_text(document, ASSISTANT_ROLE_REASON),
|
||||
"annotations": []
|
||||
}));
|
||||
} else if document_media_type_is_supported(&document.mime_type) {
|
||||
let mut converted = convert_document(document, &DocumentFormat::OpenAi);
|
||||
let mut file = converted["file"].take();
|
||||
file["type"] = json!("input_file");
|
||||
text_items.push(file);
|
||||
} else {
|
||||
text_items.push(json!({
|
||||
"type": "input_text",
|
||||
"text": unsupported_document_text(document, UNSUPPORTED_MEDIA_TYPE_REASON)
|
||||
}));
|
||||
}
|
||||
}
|
||||
MessageContentBlock::ToolResponse(response) => {
|
||||
if !text_items.is_empty() {
|
||||
input_items.push(json!({
|
||||
@@ -1194,6 +1217,55 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod document_tests {
|
||||
use super::*;
|
||||
|
||||
fn format(messages: &[Message]) -> Vec<Value> {
|
||||
let mut items = Vec::new();
|
||||
add_message_items(&mut items, messages, true);
|
||||
items
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn user_document_becomes_an_input_file_item() {
|
||||
let items = format(&[Message::user().with_document(
|
||||
"cGRmLWJ5dGVz",
|
||||
"application/pdf",
|
||||
Some("q3-report.pdf".to_string()),
|
||||
)]);
|
||||
|
||||
assert_eq!(items.len(), 1);
|
||||
assert_eq!(items[0]["role"], "user");
|
||||
assert_eq!(
|
||||
items[0]["content"][0],
|
||||
json!({
|
||||
"type": "input_file",
|
||||
"filename": "q3-report.pdf",
|
||||
"file_data": "data:application/pdf;base64,cGRmLWJ5dGVz",
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assistant_document_becomes_an_output_text_item() {
|
||||
let items = format(&[Message::assistant().with_document(
|
||||
"cGRmLWJ5dGVz",
|
||||
"application/pdf",
|
||||
Some("q3-report.pdf".to_string()),
|
||||
)]);
|
||||
|
||||
assert_eq!(items.len(), 1);
|
||||
assert_eq!(items[0]["role"], "assistant");
|
||||
let part = &items[0]["content"][0];
|
||||
assert_eq!(part["type"], "output_text");
|
||||
let text = part["text"].as_str().unwrap();
|
||||
assert!(text.contains("q3-report.pdf"), "{text}");
|
||||
assert!(text.contains("user messages"), "{text}");
|
||||
assert!(!text.contains("cGRmLWJ5dGVz"), "{text}");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use crate::conversation::message::{Message, MessageContentBlock};
|
||||
use crate::conversation::token_usage::Usage;
|
||||
use crate::documents::{unsupported_document_text, UNSUPPORTED_PROVIDER_REASON};
|
||||
use crate::errors::ProviderError;
|
||||
use crate::mcp_utils::extract_text_from_resource;
|
||||
use crate::model::ModelConfig;
|
||||
@@ -74,6 +75,15 @@ pub fn format_messages(messages: &[Message]) -> Vec<Value> {
|
||||
// Skip redacted thinking for now
|
||||
}
|
||||
MessageContentBlock::Image(_) => continue, // Snowflake doesn't support image content yet
|
||||
MessageContentBlock::Document(document) => {
|
||||
if !text_content.is_empty() {
|
||||
text_content.push('\n');
|
||||
}
|
||||
text_content.push_str(&unsupported_document_text(
|
||||
document,
|
||||
UNSUPPORTED_PROVIDER_REASON,
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ pub mod cache_semantics;
|
||||
pub mod canonical;
|
||||
pub mod context_limit;
|
||||
pub mod conversation;
|
||||
pub mod documents;
|
||||
pub mod errors;
|
||||
pub mod formats;
|
||||
pub mod goose_mode;
|
||||
|
||||
@@ -6,8 +6,8 @@ pub mod databricks_auth;
|
||||
pub mod databricks_v2;
|
||||
pub mod google;
|
||||
pub use goose_provider_types::{
|
||||
base, cache_semantics, canonical, context_limit, conversation, errors, formats, goose_mode,
|
||||
images, json, model, permission, request_log, retry, thinking, utils,
|
||||
base, cache_semantics, canonical, context_limit, conversation, documents, errors, formats,
|
||||
goose_mode, images, json, model, permission, request_log, retry, thinking, utils,
|
||||
};
|
||||
pub mod declarative;
|
||||
pub mod http_status;
|
||||
|
||||
@@ -20,6 +20,7 @@ use goose_providers::{
|
||||
databricks_auth::DatabricksAuth,
|
||||
databricks_v2::DatabricksV2Provider as GooseDatabricksV2Provider,
|
||||
declarative::{DeclarativeProviderConfig, EnvKeyResolver},
|
||||
documents::{document_media_type_is_supported, SUPPORTED_DOCUMENT_MEDIA_TYPES},
|
||||
model::ModelConfig,
|
||||
openai::OpenAiProviderBuilder,
|
||||
utils::sanitize_unicode_tags,
|
||||
@@ -202,6 +203,11 @@ pub enum MessageContent {
|
||||
mime_type: String,
|
||||
data: Vec<u8>,
|
||||
},
|
||||
Document {
|
||||
mime_type: String,
|
||||
data: Vec<u8>,
|
||||
name: Option<String>,
|
||||
},
|
||||
ToolRequest {
|
||||
id: String,
|
||||
name: String,
|
||||
@@ -249,6 +255,23 @@ impl MessageContent {
|
||||
base64::engine::general_purpose::STANDARD.encode(data),
|
||||
mime_type.clone(),
|
||||
)),
|
||||
MessageContent::Document {
|
||||
mime_type,
|
||||
data,
|
||||
name,
|
||||
} => {
|
||||
if !document_media_type_is_supported(mime_type) {
|
||||
return Err(GooseError::generic(format!(
|
||||
"unsupported document media type {mime_type}: supported types are {}",
|
||||
SUPPORTED_DOCUMENT_MEDIA_TYPES.join(", ")
|
||||
)));
|
||||
}
|
||||
Ok(GooseMessageContent::document(
|
||||
base64::engine::general_purpose::STANDARD.encode(data),
|
||||
mime_type.clone(),
|
||||
name.clone(),
|
||||
))
|
||||
}
|
||||
MessageContent::ToolRequest {
|
||||
id,
|
||||
name,
|
||||
@@ -631,6 +654,7 @@ pub enum Feature {
|
||||
Tools,
|
||||
Streaming,
|
||||
Images,
|
||||
Documents,
|
||||
JsonSchema,
|
||||
Reasoning,
|
||||
}
|
||||
@@ -867,6 +891,12 @@ impl Provider {
|
||||
) {
|
||||
features.push(Feature::Images);
|
||||
}
|
||||
if matches!(
|
||||
name.as_str(),
|
||||
"openai" | "anthropic" | "databricks" | "databricks_v2" | "google"
|
||||
) {
|
||||
features.push(Feature::Documents);
|
||||
}
|
||||
if matches!(
|
||||
name.as_str(),
|
||||
"openai" | "anthropic" | "databricks" | "databricks_v2"
|
||||
@@ -1362,6 +1392,71 @@ mod tests {
|
||||
assert_eq!(message.as_concat_text(), "what is the capital of France?");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_message_converts_document_to_base64_content() {
|
||||
let message = ProviderMessage {
|
||||
role: MessageRole::User,
|
||||
content: vec![MessageContent::Document {
|
||||
mime_type: "application/pdf".to_string(),
|
||||
data: b"pdf-bytes".to_vec(),
|
||||
name: Some("q3-report.pdf".to_string()),
|
||||
}],
|
||||
}
|
||||
.to_goose_message()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
let GooseMessageContent::Document(document) = &message.content[0] else {
|
||||
panic!("expected document content");
|
||||
};
|
||||
assert_eq!(document.mime_type, "application/pdf");
|
||||
assert_eq!(document.name.as_deref(), Some("q3-report.pdf"));
|
||||
assert_eq!(
|
||||
base64::engine::general_purpose::STANDARD
|
||||
.decode(&document.data)
|
||||
.unwrap(),
|
||||
b"pdf-bytes"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_message_converts_document_serializes_for_anthropic() {
|
||||
let message = ProviderMessage {
|
||||
role: MessageRole::User,
|
||||
content: vec![MessageContent::Document {
|
||||
mime_type: "application/pdf".to_string(),
|
||||
data: b"pdf-bytes".to_vec(),
|
||||
name: Some("q3-report.pdf".to_string()),
|
||||
}],
|
||||
}
|
||||
.to_goose_message()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
let spec = goose_providers::formats::anthropic::format_messages(&[message]);
|
||||
let block = &spec[0]["content"][0];
|
||||
|
||||
assert_eq!(block["type"], "document");
|
||||
assert_eq!(block["title"], "q3-report.pdf");
|
||||
assert_eq!(block["source"]["type"], "base64");
|
||||
assert_eq!(block["source"]["media_type"], "application/pdf");
|
||||
assert_eq!(block["source"]["data"], "cGRmLWJ5dGVz");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_message_rejects_unsupported_document_media_type() {
|
||||
let error = MessageContent::Document {
|
||||
mime_type: "text/csv".to_string(),
|
||||
data: b"rows".to_vec(),
|
||||
name: Some("rows.csv".to_string()),
|
||||
}
|
||||
.to_goose_content()
|
||||
.unwrap_err();
|
||||
|
||||
assert!(error.to_string().contains("text/csv"), "{error}");
|
||||
assert!(error.to_string().contains("application/pdf"), "{error}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tool_config_converts_to_rmcp_tool() {
|
||||
let tool = ProviderTool {
|
||||
|
||||
@@ -220,6 +220,13 @@ fn message_part_json(content: &MessageContent) -> Value {
|
||||
"mime_type": image.mime_type,
|
||||
"content": image.data,
|
||||
}),
|
||||
MessageContent::Document(document) => json!({
|
||||
"type": "blob",
|
||||
"modality": "document",
|
||||
"mime_type": document.mime_type,
|
||||
"name": document.name,
|
||||
"content": document.data,
|
||||
}),
|
||||
MessageContent::ToolRequest(request) => tool_call_part(&request.id, &request.tool_call),
|
||||
MessageContent::ToolResponse(response) => json!({
|
||||
"type": "tool_call_response",
|
||||
|
||||
@@ -23,6 +23,7 @@ use crate::providers::formats::anthropic::{
|
||||
};
|
||||
use crate::utils::{sanitize_unicode_tags, strip_unicode_tags};
|
||||
use goose_providers::conversation::token_usage::Usage;
|
||||
use goose_providers::documents::{unsupported_document_text, UNSUPPORTED_PROVIDER_REASON};
|
||||
use goose_providers::model::ModelConfig;
|
||||
use once_cell::sync::Lazy;
|
||||
use regex::Regex;
|
||||
@@ -204,6 +205,9 @@ pub fn to_bedrock_message_content(content: &MessageContent) -> Result<bedrock::C
|
||||
MessageContent::Image(image) => {
|
||||
bedrock::ContentBlock::Image(to_bedrock_image(&image.data, &image.mime_type)?)
|
||||
}
|
||||
MessageContent::Document(document) => bedrock::ContentBlock::Text(
|
||||
unsupported_document_text(document, UNSUPPORTED_PROVIDER_REASON),
|
||||
),
|
||||
MessageContent::Thinking(thinking) => {
|
||||
let mut builder = bedrock::ReasoningTextBlock::builder().text(&thinking.thinking);
|
||||
if !thinking.signature.is_empty() {
|
||||
|
||||
@@ -390,6 +390,29 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Document",
|
||||
"fields": [
|
||||
{
|
||||
"name": "mime_type",
|
||||
"type": "String",
|
||||
"default": null,
|
||||
"docs": ""
|
||||
},
|
||||
{
|
||||
"name": "data",
|
||||
"type": "Vec<u8>",
|
||||
"default": null,
|
||||
"docs": ""
|
||||
},
|
||||
{
|
||||
"name": "name",
|
||||
"type": "Option<String>",
|
||||
"default": null,
|
||||
"docs": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "ToolRequest",
|
||||
"fields": [
|
||||
@@ -864,6 +887,10 @@
|
||||
"name": "Images",
|
||||
"fields": []
|
||||
},
|
||||
{
|
||||
"name": "Documents",
|
||||
"fields": []
|
||||
},
|
||||
{
|
||||
"name": "JsonSchema",
|
||||
"fields": []
|
||||
|
||||
Reference in New Issue
Block a user