tkmind/tkmind_go
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

Better image compression (#6746)

Browse Source 
Co-authored-by: Douwe Osinga <douwe@squareup.com>
This commit is contained in:
Douwe Osinga
2026-01-27 17:08:22 -05:00
committed by GitHub
parent 3fcf022236
commit 2a4ed84e94
2 changed files with 58 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/goose-mcp/Cargo.toml
+1 -1
View File
@@ -36,7 +36,7 @@ once_cell = "1.20.2"
ignore = { workspace = true }
lopdf = "0.36.0"
docx-rs = "0.4.7"
image = "0.24.9"
image = { version = "0.24.9", features = ["jpeg"] }
umya-spreadsheet = "2.2.3"
which = {workspace = true}
lru = "0.16"
crates/goose-mcp/src/developer/rmcp_developer.rs
+57 -55
View File
@@ -639,7 +639,7 @@ impl DeveloperServer {
) -> Result<CallToolResult, ErrorData> {
let params = params.0;
let mut image = if let Some(window_title) = &params.window_title {
let image = if let Some(window_title) = &params.window_title {
// Try to find and capture the specified window
let windows = Window::all().map_err(|_| {
ErrorData::new(
@@ -700,29 +700,8 @@ impl DeveloperServer {
})?
};
// Resize the image to a reasonable width while maintaining aspect ratio
let max_width = 768;
if image.width() > max_width {
let scale = max_width as f32 / image.width() as f32;
let new_height = (image.height() as f32 * scale) as u32;
image = xcap::image::imageops::resize(
&image,
max_width,
new_height,
xcap::image::imageops::FilterType::Lanczos3,
);
}
let mut bytes: Vec<u8> = Vec::new();
image
.write_to(&mut Cursor::new(&mut bytes), xcap::image::ImageFormat::Png)
.map_err(|e| {
ErrorData::new(
ErrorCode::INTERNAL_ERROR,
format!("Failed to write image buffer {}", e),
None,
)
})?;
let dynamic_image = xcap::image::DynamicImage::ImageRgba8(image);
let (bytes, mime_type) = Self::prepare_image_for_llm(dynamic_image)?;
// Convert to base64
let data = base64::prelude::BASE64_STANDARD.encode(bytes);
@@ -731,7 +710,7 @@ impl DeveloperServer {
// one text for Assistant, one image with priority 0.0
Ok(CallToolResult::success(vec![
Content::text("Screenshot captured").with_audience(vec![Role::Assistant]),
Content::image(data, "image/png").with_priority(0.0),
Content::image(data, &mime_type).with_priority(0.0),
]))
}
@@ -1176,14 +1155,14 @@ impl DeveloperServer {
/// Process an image file from disk.
///
/// The image will be:
/// 1. Resized if larger than max width while maintaining aspect ratio
/// 2. Converted to PNG format
/// 1. Resized to max 1024px on either dimension while maintaining aspect ratio
/// 2. Converted to JPEG format (85% quality)
/// 3. Returned as base64 encoded data
///
/// This allows processing image files for use in the conversation.
/// This allows processing image files for use in the conversation with optimized file sizes.
#[tool(
name = "image_processor",
description = "Process an image file from disk. Resizes if needed, converts to PNG, and returns as base64 data."
description = "Process an image file from disk. Resizes to max 1024px, converts to JPEG (85% quality), and returns as base64 data for optimized LLM consumption."
)]
pub async fn image_processor(
&self,
@@ -1255,31 +1234,7 @@ impl DeveloperServer {
)
})?;
// Resize if necessary (same logic as screen_capture)
let mut processed_image = image;
let max_width = 768;
if processed_image.width() > max_width {
let scale = max_width as f32 / processed_image.width() as f32;
let new_height = (processed_image.height() as f32 * scale) as u32;
processed_image = xcap::image::DynamicImage::ImageRgba8(xcap::image::imageops::resize(
&processed_image,
max_width,
new_height,
xcap::image::imageops::FilterType::Lanczos3,
));
}
// Convert to PNG and encode as base64
let mut bytes: Vec<u8> = Vec::new();
processed_image
.write_to(&mut Cursor::new(&mut bytes), xcap::image::ImageFormat::Png)
.map_err(|e| {
ErrorData::new(
ErrorCode::INTERNAL_ERROR,
format!("Failed to write image buffer: {}", e),
None,
)
})?;
let (bytes, mime_type) = Self::prepare_image_for_llm(image)?;
let data = base64::prelude::BASE64_STANDARD.encode(bytes);
@@ -1289,10 +1244,57 @@ impl DeveloperServer {
path.display()
))
.with_audience(vec![Role::Assistant]),
Content::image(data, "image/png").with_priority(0.0),
Content::image(data, &mime_type).with_priority(0.0),
]))
}
fn prepare_image_for_llm(
mut image: xcap::image::DynamicImage,
) -> Result<(Vec<u8>, String), ErrorData> {
let max_dimension = 1024;
let (width, height) = (image.width(), image.height());
if width > max_dimension || height > max_dimension {
let (new_width, new_height) = if width > height {
let scale = max_dimension as f32 / width as f32;
(max_dimension, (height as f32 * scale) as u32)
} else {
let scale = max_dimension as f32 / height as f32;
((width as f32 * scale) as u32, max_dimension)
};
image = xcap::image::DynamicImage::ImageRgba8(xcap::image::imageops::resize(
&image,
new_width,
new_height,
xcap::image::imageops::FilterType::Lanczos3,
));
}
let rgb_image = image.to_rgb8();
let (img_width, img_height) = rgb_image.dimensions();
let mut bytes: Vec<u8> = Vec::new();
let mut cursor = Cursor::new(&mut bytes);
image::codecs::jpeg::JpegEncoder::new_with_quality(&mut cursor, 85)
.encode(
rgb_image.as_raw(),
img_width,
img_height,
image::ColorType::Rgb8,
)
.map_err(|e| {
ErrorData::new(
ErrorCode::INTERNAL_ERROR,
format!("Failed to encode image as JPEG: {}", e),
None,
)
})?;
Ok((bytes, "image/jpeg".to_string()))
}
// Helper method to resolve and validate file paths
fn resolve_path(&self, path_str: &str) -> Result<PathBuf, ErrorData> {
let cwd = std::env::current_dir().expect("should have a current working dir");