diff --git a/crates/goose-mcp/Cargo.toml b/crates/goose-mcp/Cargo.toml index b1ccb06d..1fd2ac7f 100644 --- a/crates/goose-mcp/Cargo.toml +++ b/crates/goose-mcp/Cargo.toml @@ -36,7 +36,7 @@ once_cell = "1.20.2" ignore = { workspace = true } lopdf = "0.36.0" docx-rs = "0.4.7" -image = "0.24.9" +image = { version = "0.24.9", features = ["jpeg"] } umya-spreadsheet = "2.2.3" which = {workspace = true} lru = "0.16" diff --git a/crates/goose-mcp/src/developer/rmcp_developer.rs b/crates/goose-mcp/src/developer/rmcp_developer.rs index 65e9384e..bd75f32e 100644 --- a/crates/goose-mcp/src/developer/rmcp_developer.rs +++ b/crates/goose-mcp/src/developer/rmcp_developer.rs @@ -639,7 +639,7 @@ impl DeveloperServer { ) -> Result { let params = params.0; - let mut image = if let Some(window_title) = ¶ms.window_title { + let image = if let Some(window_title) = ¶ms.window_title { // Try to find and capture the specified window let windows = Window::all().map_err(|_| { ErrorData::new( @@ -700,29 +700,8 @@ impl DeveloperServer { })? }; - // Resize the image to a reasonable width while maintaining aspect ratio - let max_width = 768; - if image.width() > max_width { - let scale = max_width as f32 / image.width() as f32; - let new_height = (image.height() as f32 * scale) as u32; - image = xcap::image::imageops::resize( - &image, - max_width, - new_height, - xcap::image::imageops::FilterType::Lanczos3, - ); - } - - let mut bytes: Vec = Vec::new(); - image - .write_to(&mut Cursor::new(&mut bytes), xcap::image::ImageFormat::Png) - .map_err(|e| { - ErrorData::new( - ErrorCode::INTERNAL_ERROR, - format!("Failed to write image buffer {}", e), - None, - ) - })?; + let dynamic_image = xcap::image::DynamicImage::ImageRgba8(image); + let (bytes, mime_type) = Self::prepare_image_for_llm(dynamic_image)?; // Convert to base64 let data = base64::prelude::BASE64_STANDARD.encode(bytes); @@ -731,7 +710,7 @@ impl DeveloperServer { // one text for Assistant, one image with priority 0.0 Ok(CallToolResult::success(vec![ Content::text("Screenshot captured").with_audience(vec![Role::Assistant]), - Content::image(data, "image/png").with_priority(0.0), + Content::image(data, &mime_type).with_priority(0.0), ])) } @@ -1176,14 +1155,14 @@ impl DeveloperServer { /// Process an image file from disk. /// /// The image will be: - /// 1. Resized if larger than max width while maintaining aspect ratio - /// 2. Converted to PNG format + /// 1. Resized to max 1024px on either dimension while maintaining aspect ratio + /// 2. Converted to JPEG format (85% quality) /// 3. Returned as base64 encoded data /// - /// This allows processing image files for use in the conversation. + /// This allows processing image files for use in the conversation with optimized file sizes. #[tool( name = "image_processor", - description = "Process an image file from disk. Resizes if needed, converts to PNG, and returns as base64 data." + description = "Process an image file from disk. Resizes to max 1024px, converts to JPEG (85% quality), and returns as base64 data for optimized LLM consumption." )] pub async fn image_processor( &self, @@ -1255,31 +1234,7 @@ impl DeveloperServer { ) })?; - // Resize if necessary (same logic as screen_capture) - let mut processed_image = image; - let max_width = 768; - if processed_image.width() > max_width { - let scale = max_width as f32 / processed_image.width() as f32; - let new_height = (processed_image.height() as f32 * scale) as u32; - processed_image = xcap::image::DynamicImage::ImageRgba8(xcap::image::imageops::resize( - &processed_image, - max_width, - new_height, - xcap::image::imageops::FilterType::Lanczos3, - )); - } - - // Convert to PNG and encode as base64 - let mut bytes: Vec = Vec::new(); - processed_image - .write_to(&mut Cursor::new(&mut bytes), xcap::image::ImageFormat::Png) - .map_err(|e| { - ErrorData::new( - ErrorCode::INTERNAL_ERROR, - format!("Failed to write image buffer: {}", e), - None, - ) - })?; + let (bytes, mime_type) = Self::prepare_image_for_llm(image)?; let data = base64::prelude::BASE64_STANDARD.encode(bytes); @@ -1289,10 +1244,57 @@ impl DeveloperServer { path.display() )) .with_audience(vec![Role::Assistant]), - Content::image(data, "image/png").with_priority(0.0), + Content::image(data, &mime_type).with_priority(0.0), ])) } + fn prepare_image_for_llm( + mut image: xcap::image::DynamicImage, + ) -> Result<(Vec, String), ErrorData> { + let max_dimension = 1024; + let (width, height) = (image.width(), image.height()); + + if width > max_dimension || height > max_dimension { + let (new_width, new_height) = if width > height { + let scale = max_dimension as f32 / width as f32; + (max_dimension, (height as f32 * scale) as u32) + } else { + let scale = max_dimension as f32 / height as f32; + ((width as f32 * scale) as u32, max_dimension) + }; + + image = xcap::image::DynamicImage::ImageRgba8(xcap::image::imageops::resize( + &image, + new_width, + new_height, + xcap::image::imageops::FilterType::Lanczos3, + )); + } + + let rgb_image = image.to_rgb8(); + let (img_width, img_height) = rgb_image.dimensions(); + + let mut bytes: Vec = Vec::new(); + let mut cursor = Cursor::new(&mut bytes); + + image::codecs::jpeg::JpegEncoder::new_with_quality(&mut cursor, 85) + .encode( + rgb_image.as_raw(), + img_width, + img_height, + image::ColorType::Rgb8, + ) + .map_err(|e| { + ErrorData::new( + ErrorCode::INTERNAL_ERROR, + format!("Failed to encode image as JPEG: {}", e), + None, + ) + })?; + + Ok((bytes, "image/jpeg".to_string())) + } + // Helper method to resolve and validate file paths fn resolve_path(&self, path_str: &str) -> Result { let cwd = std::env::current_dir().expect("should have a current working dir");