tkmind/tkmind_go
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix user message text silently dropped when message contains both text and image (#8071)

Browse Source 
Signed-off-by: Vincenzo Palazzo <vincenzopalazzodev@gmail.com>
Signed-off-by: Douwe Osinga <douwe@squareup.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: Douwe Osinga <douwe@squareup.com>
This commit is contained in:
Vincenzo Palazzo
2026-03-25 00:25:53 +01:00
committed by GitHub
parent 18bc030e48
commit 9849671413
1 changed files with 49 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/goose/src/providers/formats/openai.rs
+49 -7
View File
@@ -126,7 +126,7 @@ pub fn format_messages(messages: &[Message], image_format: &ImageFormat) -> Vec<
let mut output = Vec::new();
let mut content_array = Vec::new();
let mut text_array = Vec::new();
let mut has_non_text_content = false;
let mut reasoning_text = String::new();
for content in &message.content {
@@ -136,16 +136,17 @@ pub fn format_messages(messages: &[Message], image_format: &ImageFormat) -> Vec<
if message.role == Role::User {
if let Some(image_path) = detect_image_path(&text.text) {
if let Ok(image) = load_image_file(image_path) {
has_non_text_content = true;
content_array.push(json!({"type": "text", "text": text.text}));
content_array.push(convert_image(&image, image_format));
} else {
text_array.push(text.text.clone());
content_array.push(json!({"type": "text", "text": text.text}));
}
} else {
text_array.push(text.text.clone());
content_array.push(json!({"type": "text", "text": text.text}));
}
} else {
text_array.push(text.text.clone());
content_array.push(json!({"type": "text", "text": text.text}));
}
}
}
@@ -259,6 +260,7 @@ pub fn format_messages(messages: &[Message], image_format: &ImageFormat) -> Vec<
MessageContent::ActionRequired(_) => {}
MessageContent::Image(image) => {
if message.role == Role::User {
has_non_text_content = true;
content_array.push(convert_image(image, image_format));
} else {
content_array.push(json!({
@@ -304,9 +306,15 @@ pub fn format_messages(messages: &[Message], image_format: &ImageFormat) -> Vec<
}
if !content_array.is_empty() {
converted["content"] = json!(content_array);
} else if !text_array.is_empty() {
converted["content"] = json!(text_array.join("\n"));
if has_non_text_content {
converted["content"] = json!(content_array);
} else {
let texts: Vec<String> = content_array
.iter()
.filter_map(|v| v["text"].as_str().map(|s| s.to_string()))
.collect();
converted["content"] = json!(texts.join("\n"));
}
}
// Some strict OpenAI-compatible providers require "content" to be present
@@ -1210,6 +1218,40 @@ mod tests {
Ok(())
}
#[test]
fn test_format_messages_with_text_and_image_preserves_order() {
// Text before image: order should be [text, image]
let msg_text_first = Message::user()
.with_text("Describe this image")
.with_image("aW1hZ2VkYXRh", "image/png");
let spec = format_messages(&[msg_text_first], &ImageFormat::OpenAi);
assert_eq!(spec.len(), 1);
assert_eq!(spec[0]["role"], "user");
let content = spec[0]["content"]
.as_array()
.expect("content should be an array");
assert_eq!(content.len(), 2, "expected text + image entries");
assert_eq!(content[0]["type"], "text");
assert_eq!(content[0]["text"], "Describe this image");
assert_eq!(content[1]["type"], "image_url");
// Image before text: order should be [image, text]
let msg_image_first = Message::user()
.with_image("aW1hZ2VkYXRh", "image/png")
.with_text("What do you see?");
let spec2 = format_messages(&[msg_image_first], &ImageFormat::OpenAi);
let content2 = spec2[0]["content"]
.as_array()
.expect("content should be an array");
assert_eq!(content2.len(), 2, "expected image + text entries");
assert_eq!(content2[0]["type"], "image_url");
assert_eq!(content2[1]["type"], "text");
assert_eq!(content2[1]["text"], "What do you see?");
}
#[test]
fn test_response_to_message_text() -> anyhow::Result<()> {
let response = json!({