tkmind/tkmind_go
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

fix(#5626 #5832): handle multiple content chunks & images better (#5839)

Browse Source 
Co-authored-by: Pradeepta Dash <pradeepta@exceeds.ai>
This commit is contained in:
Alex Hancock
2025-11-24 10:52:01 -05:00
committed by GitHub
parent 4cc95d4202
commit c82a0dd8fc
3 changed files with 65 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/goose/src/providers/formats/anthropic.rs
+4 -1
View File
@@ -2,6 +2,7 @@ use crate::conversation::message::{Message, MessageContent};
use crate::model::ModelConfig;
use crate::providers::base::Usage;
use crate::providers::errors::ProviderError;
use crate::providers::utils::{convert_image, ImageFormat};
use anyhow::{anyhow, Result};
use rmcp::model::{object, CallToolRequestParam, ErrorCode, ErrorData, JsonObject, Role, Tool};
use rmcp::object as json_object;
@@ -106,7 +107,9 @@ pub fn format_messages(messages: &[Message]) -> Vec<Value> {
DATA_FIELD: redacted.data
}));
}
MessageContent::Image(_) => continue, // Anthropic doesn't support image content yet
MessageContent::Image(image) => {
content.push(convert_image(image, &ImageFormat::Anthropic));
}
MessageContent::FrontendToolRequest(tool_request) => {
if let Ok(tool_call) = &tool_request.tool_call {
content.push(json!({
crates/goose/src/providers/formats/openai.rs
+31 -11
View File
@@ -63,25 +63,22 @@ pub fn format_messages(messages: &[Message], image_format: &ImageFormat) -> Vec<
});
let mut output = Vec::new();
let mut content_array = Vec::new();
let mut text_array = Vec::new();
for content in &message.content {
match content {
MessageContent::Text(text) => {
if !text.text.is_empty() {
// Check for image paths in the text
if let Some(image_path) = detect_image_path(&text.text) {
// Try to load and convert the image
if let Ok(image) = load_image_file(image_path) {
converted["content"] = json!([
{"type": "text", "text": text.text},
convert_image(&image, image_format)
]);
content_array.push(json!({"type": "text", "text": text.text}));
content_array.push(convert_image(&image, image_format));
} else {
// If image loading fails, just use the text
converted["content"] = json!(text.text);
text_array.push(text.text.clone());
}
} else {
converted["content"] = json!(text.text);
text_array.push(text.text.clone());
}
}
}
@@ -205,8 +202,7 @@ pub fn format_messages(messages: &[Message], image_format: &ImageFormat) -> Vec<
// Skip tool confirmation requests
}
MessageContent::Image(image) => {
// Handle direct image content
converted["content"] = json!([convert_image(image, image_format)]);
content_array.push(convert_image(image, image_format));
}
MessageContent::FrontendToolRequest(request) => match &request.tool_call {
Ok(tool_call) => {
@@ -244,9 +240,16 @@ pub fn format_messages(messages: &[Message], image_format: &ImageFormat) -> Vec<
}
}
if !content_array.is_empty() {
converted["content"] = json!(content_array);
} else if !text_array.is_empty() {
converted["content"] = json!(text_array.join("\n"));
}
if converted.get("content").is_some() || converted.get("tool_calls").is_some() {
output.insert(0, converted);
}
messages_spec.extend(output);
}
@@ -1230,6 +1233,23 @@ mod tests {
Ok(())
}
#[test]
fn test_format_messages_multiple_text_blocks() -> anyhow::Result<()> {
let message = Message::user()
.with_text("--- Resource: file:///test.md ---\n# Test\n\n---\n")
.with_text(" What is in the file?");
let spec = format_messages(&[message], &ImageFormat::OpenAi);
assert_eq!(spec.len(), 1);
assert_eq!(spec[0]["role"], "user");
assert_eq!(
spec[0]["content"],
"--- Resource: file:///test.md ---\n# Test\n\n---\n\n What is in the file?"
);
Ok(())
}
#[test]
fn test_create_request_gpt_4o() -> anyhow::Result<()> {
// Test default medium reasoning effort for O3 model
ui/desktop/package-lock.json
Generated
+30 -10
View File
@@ -331,6 +331,7 @@
"integrity": "sha512-2BCOP7TN8M+gVDj7/ht3hsaO/B/n5oDbiAyyvnRlNOs+u1o+JWNYTQrmpuNp1/Wq2gcFrI01JAW+paEKDMx/CA==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@babel/code-frame": "^7.27.1",
"@babel/generator": "^7.28.3",
@@ -700,6 +701,7 @@
}
],
"license": "MIT",
"peer": true,
"engines": {
"node": ">=18"
},
@@ -746,6 +748,7 @@
}
],
"license": "MIT",
"peer": true,
"engines": {
"node": ">=18"
}
@@ -1185,6 +1188,7 @@
"integrity": "sha512-zx0EIq78WlY/lBb1uXlziZmDZI4ubcCXIMJ4uGjXzZW0nS19TjSPeXPAjzzTmKQlJUZm0SbmZhPKP7tuQ1SsEw==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"chalk": "^4.1.1",
"fs-extra": "^9.0.1",
@@ -2734,6 +2738,7 @@
"integrity": "sha512-yl43JD/86CIj3Mz5mvvLJqAOfIup7ncxfJ0Btnl0/v5TouVUyeEdcpknfgc+yMevS/48oH9WAkkw93m7otLb/A==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@inquirer/checkbox": "^3.0.1",
"@inquirer/confirm": "^4.0.1",
@@ -5963,8 +5968,7 @@
"resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.4.tgz",
"integrity": "sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw==",
"dev": true,
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/@types/babel__core": {
"version": "7.20.5",
@@ -6259,6 +6263,7 @@
"integrity": "sha512-QoiaXANRkSXK6p0Duvt56W208du4P9Uye9hWLWgGMDTEoKPhuenzNcC4vGUmrNkiOKTlIrBoyNQYNpSwfEZXSg==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"undici-types": "~7.16.0"
}
@@ -6300,6 +6305,7 @@
"resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.2.tgz",
"integrity": "sha512-6mDvHUFSjyT2B2yeNx2nUgMxh9LtOWvkhIU3uePn2I2oyNymUAX1NIsdgviM4CH+JSrp2D2hsMvJOkxY+0wNRA==",
"license": "MIT",
"peer": true,
"dependencies": {
"csstype": "^3.0.2"
}
@@ -6310,6 +6316,7 @@
"integrity": "sha512-9KQPoO6mZCi7jcIStSnlOWn2nEF3mNmyr3rIAsGnAbQKYbRLyqmeSc39EVgtxXVia+LMT8j3knZLAZAh+xLmrw==",
"devOptional": true,
"license": "MIT",
"peer": true,
"peerDependencies": {
"@types/react": "^19.2.0"
}
@@ -6456,6 +6463,7 @@
"integrity": "sha512-BnOroVl1SgrPLywqxyqdJ4l3S2MsKVLDVxZvjI1Eoe8ev2r3kGDo+PcMihNmDE+6/KjkTubSJnmqGZZjQSBq/g==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@typescript-eslint/scope-manager": "8.46.2",
"@typescript-eslint/types": "8.46.2",
@@ -6857,6 +6865,7 @@
"integrity": "sha512-hGISOaP18plkzbWEcP/QvtRW1xDXF2+96HbEX6byqQhAUbiS5oH6/9JwW+QsQCIYON2bI6QZBF+2PvOmrRZ9wA==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@vitest/utils": "3.2.4",
"fflate": "^0.8.2",
@@ -7106,6 +7115,7 @@
"integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==",
"dev": true,
"license": "MIT",
"peer": true,
"bin": {
"acorn": "bin/acorn"
},
@@ -7745,6 +7755,7 @@
}
],
"license": "MIT",
"peer": true,
"dependencies": {
"baseline-browser-mapping": "^2.8.9",
"caniuse-lite": "^1.0.30001746",
@@ -9080,8 +9091,7 @@
"resolved": "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.5.16.tgz",
"integrity": "sha512-X7BJ2yElsnOJ30pZF4uIIDfBEVgF4XEBxL9Bxhy6dnrm5hkzqmsWHGTiHqRiITNhMyFLyAiWndIJP7Z1NTteDg==",
"dev": true,
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/dom-helpers": {
"version": "5.2.1",
@@ -9139,6 +9149,7 @@
"dev": true,
"hasInstallScript": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@electron/get": "^2.0.0",
"@types/node": "^22.7.7",
@@ -10153,6 +10164,7 @@
"integrity": "sha512-t5aPOpmtJcZcz5UJyY2GbvpDlsK5E8JqRqoKtfiKE3cNh437KIqfJr3A3AKf5k64NPx6d0G3dno6XDY05PqPtw==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@eslint-community/eslint-utils": "^4.8.0",
"@eslint-community/regexpp": "^4.12.1",
@@ -10612,6 +10624,7 @@
"resolved": "https://registry.npmjs.org/express/-/express-5.1.0.tgz",
"integrity": "sha512-DT9ck5YIRU+8GYzzU5kT3eHGA5iL+1Zd0EutOmTE9Dtk+Tvuzd23VBU+ec7HPNSTxXYO55gPV/hq4pSBJDjFpA==",
"license": "MIT",
"peer": true,
"dependencies": {
"accepts": "^2.0.0",
"body-parser": "^2.2.0",
@@ -12937,6 +12950,7 @@
"integrity": "sha512-SNSQteBL1IlV2zqhwwolaG9CwhIhTvVHWg3kTss/cLE7H/X4644mtPQqYvCfsSrGQWt9hSZcgOXX8bOZaMN+kA==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@asamuzakjp/dom-selector": "^6.7.2",
"cssstyle": "^5.3.1",
@@ -14186,7 +14200,6 @@
"integrity": "sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==",
"dev": true,
"license": "MIT",
"peer": true,
"bin": {
"lz-string": "bin/bin.js"
}
@@ -14207,6 +14220,7 @@
"integrity": "sha512-L0WhttDl+2BOsybvEOLK7fW3UA0OQ0IQ2d6Zl2x/a6vVRs3bAY0ECOSHHeL5jD+SbOpOCUEi0y1DgHEn9Qn1AQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@babel/parser": "^7.25.4",
"@babel/types": "^7.25.4",
@@ -16619,6 +16633,7 @@
}
],
"license": "MIT",
"peer": true,
"dependencies": {
"nanoid": "^3.3.11",
"picocolors": "^1.1.1",
@@ -16707,7 +16722,6 @@
"integrity": "sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"ansi-regex": "^5.0.1",
"ansi-styles": "^5.0.0",
@@ -16723,7 +16737,6 @@
"integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=10"
},
@@ -17040,6 +17053,7 @@
"resolved": "https://registry.npmjs.org/react/-/react-19.2.0.tgz",
"integrity": "sha512-tmbWg6W31tQLeB5cdIBOicJDJRR2KzXsV7uSK9iNfLWQ5bIZfxuPEHp7M8wiHyHnn0DD1i7w3Zmin0FtkrwoCQ==",
"license": "MIT",
"peer": true,
"engines": {
"node": ">=0.10.0"
}
@@ -17049,6 +17063,7 @@
"resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.0.tgz",
"integrity": "sha512-UlbRu4cAiGaIewkPyiRGJk0imDN2T3JjieT6spoL2UeSf5od4n5LB/mQ4ejmxhCFT1tYe8IvaFulzynWovsEFQ==",
"license": "MIT",
"peer": true,
"dependencies": {
"scheduler": "^0.27.0"
},
@@ -17070,8 +17085,7 @@
"resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz",
"integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==",
"dev": true,
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/react-markdown": {
"version": "10.1.0",
@@ -18055,6 +18069,7 @@
"integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"fast-deep-equal": "^3.1.3",
"fast-uri": "^3.0.1",
@@ -19048,7 +19063,8 @@
"version": "4.1.15",
"resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.1.15.tgz",
"integrity": "sha512-k2WLnWkYFkdpRv+Oby3EBXIyQC8/s1HOFMBUViwtAh6Z5uAozeUSMQlIsn/c6Q2iJzqG6aJT3wdPaRNj70iYxQ==",
"license": "MIT"
"license": "MIT",
"peer": true
},
"node_modules/tailwindcss-animate": {
"version": "1.0.7",
@@ -19667,6 +19683,7 @@
"integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
"dev": true,
"license": "Apache-2.0",
"peer": true,
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
@@ -20094,6 +20111,7 @@
"integrity": "sha512-uzcxnSDVjAopEUjljkWh8EIrg6tlzrjFUfMcR1EVsRDGwf/ccef0qQPRyOrROwhrTDaApueq+ja+KLPlzR/zdg==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"esbuild": "^0.25.0",
"fdir": "^6.5.0",
@@ -20207,6 +20225,7 @@
"integrity": "sha512-LUCP5ev3GURDysTWiP47wRRUpLKMOfPh+yKTx3kVIEiu5KOMeqzpnYNsKyOoVrULivR8tLcks4+lga33Whn90A==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@types/chai": "^5.2.2",
"@vitest/expect": "3.2.4",
@@ -20922,6 +20941,7 @@
"resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz",
"integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==",
"license": "MIT",
"peer": true,
"funding": {
"url": "https://github.com/sponsors/colinhacks"
}