diff --git a/Cargo.lock b/Cargo.lock index db116e3228..3fcc3ee56f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4582,6 +4582,7 @@ dependencies = [ "icu_calendar", "icu_locale", "ignore", + "image 0.24.9", "include_dir", "indexmap 2.14.0", "indoc", diff --git a/crates/goose/Cargo.toml b/crates/goose/Cargo.toml index 4f3b684293..af827e4304 100644 --- a/crates/goose/Cargo.toml +++ b/crates/goose/Cargo.toml @@ -214,6 +214,7 @@ pctx_code_mode = { version = "0.3", default-features = false, optional = true } icu_calendar = { version = "=2.1.1", default-features = false } icu_locale = { version = "=2.1.1", default-features = false } llama-cpp-sys-2 = { workspace = true, optional = true } +image = { version = "0.24.9", default-features = false, features = ["png", "jpeg", "gif", "webp"] } [target.'cfg(target_os = "windows")'.dependencies] winapi = { workspace = true } diff --git a/crates/goose/src/agents/platform_extensions/developer/image.rs b/crates/goose/src/agents/platform_extensions/developer/image.rs new file mode 100644 index 0000000000..83ea7b39be --- /dev/null +++ b/crates/goose/src/agents/platform_extensions/developer/image.rs @@ -0,0 +1,255 @@ +use std::io::Cursor; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use base64::Engine; +use image::GenericImageView; +use rmcp::model::{CallToolResult, Content}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::json; + +use super::edit::resolve_path; + +const MAX_IMAGE_BYTES: u64 = 20 * 1024 * 1024; + +#[derive(Debug, Deserialize, JsonSchema)] +pub struct ImageReadParams { + /// Local file path or http(s) URL of the image to load. + pub source: String, + /// Optional crop rectangle in pixels. Coordinates are measured from the top-left corner. + /// use to zoom in and get more details. + #[serde(default)] + pub crop: Option, +} + +#[derive(Debug, Deserialize, Serialize, JsonSchema)] +pub struct CropParams { + /// Left edge of the crop rectangle in pixels. + pub x: u32, + /// Top edge of the crop rectangle in pixels. + pub y: u32, + /// Width of the crop rectangle in pixels. + pub width: u32, + /// Height of the crop rectangle in pixels. + pub height: u32, +} + +pub struct ImageTool; + +impl ImageTool { + pub fn new() -> Self { + Self + } + + pub async fn image_read_with_cwd( + &self, + params: ImageReadParams, + working_dir: Option<&Path>, + ) -> CallToolResult { + match load_image(¶ms, working_dir).await { + Ok(loaded) => { + let mut result = CallToolResult::success(vec![ + Content::text(loaded.summary(¶ms.source)).with_priority(0.0), + Content::image(loaded.data, loaded.mime_type.clone()).with_priority(0.0), + ]); + result.structured_content = Some(json!({ + "source": params.source, + "mimeType": loaded.mime_type, + "width": loaded.width, + "height": loaded.height, + "bytes": loaded.bytes_len, + "originalWidth": loaded.original_width, + "originalHeight": loaded.original_height, + "crop": params.crop, + })); + result + } + Err(error) => CallToolResult::error(vec![ + Content::text(format!("Error: {error}")).with_priority(0.0) + ]), + } + } +} + +impl Default for ImageTool { + fn default() -> Self { + Self::new() + } +} + +#[derive(Debug)] +struct LoadedImage { + data: String, + mime_type: String, + bytes_len: usize, + width: u32, + height: u32, + original_width: u32, + original_height: u32, + cropped: bool, +} + +impl LoadedImage { + fn summary(&self, source: &str) -> String { + let crop_note = if self.cropped { + format!( + " Cropped from {}x{} to {}x{}.", + self.original_width, self.original_height, self.width, self.height + ) + } else { + String::new() + }; + + format!( + "Loaded image from {source} ({} bytes, {}, {}x{}).{crop_note}", + self.bytes_len, self.mime_type, self.width, self.height + ) + } +} + +async fn load_image( + params: &ImageReadParams, + working_dir: Option<&Path>, +) -> Result { + if params.source.trim().is_empty() { + return Err("source cannot be empty".to_string()); + } + + let bytes = load_image_bytes(¶ms.source, working_dir).await?; + ensure_image_size(bytes.len() as u64)?; + + let format = image::guess_format(&bytes).map_err(|_| { + "unsupported image format; supported formats are png, jpeg, gif, and webp".to_string() + })?; + let mime_type = mime_type(format)?; + let image = image::load_from_memory_with_format(&bytes, format) + .map_err(|error| format!("failed to decode image: {error}"))?; + let (original_width, original_height) = image.dimensions(); + + let Some(crop) = ¶ms.crop else { + return Ok(LoadedImage { + data: base64::prelude::BASE64_STANDARD.encode(&bytes), + mime_type: mime_type.to_string(), + bytes_len: bytes.len(), + width: original_width, + height: original_height, + original_width, + original_height, + cropped: false, + }); + }; + + validate_crop(crop, original_width, original_height)?; + let cropped = image.crop_imm(crop.x, crop.y, crop.width, crop.height); + let mut cropped_bytes = Cursor::new(Vec::new()); + cropped + .write_to(&mut cropped_bytes, image::ImageFormat::Png) + .map_err(|error| format!("failed to encode cropped image: {error}"))?; + let cropped_bytes = cropped_bytes.into_inner(); + ensure_image_size(cropped_bytes.len() as u64)?; + + Ok(LoadedImage { + data: base64::prelude::BASE64_STANDARD.encode(&cropped_bytes), + mime_type: "image/png".to_string(), + bytes_len: cropped_bytes.len(), + width: crop.width, + height: crop.height, + original_width, + original_height, + cropped: true, + }) +} + +async fn load_image_bytes(source: &str, working_dir: Option<&Path>) -> Result, String> { + if let Ok(url) = url::Url::parse(source) { + match url.scheme() { + "http" | "https" => load_url_bytes(url).await, + "file" => { + let path = url + .to_file_path() + .map_err(|_| "invalid file URL".to_string())?; + load_file_bytes(path) + } + _ => load_file_bytes(resolve_path(source, working_dir)), + } + } else { + load_file_bytes(resolve_path(source, working_dir)) + } +} + +async fn load_url_bytes(url: url::Url) -> Result, String> { + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(30)) + .build() + .map_err(|error| format!("failed to create HTTP client: {error}"))?; + + let response = client + .get(url) + .send() + .await + .map_err(|error| format!("failed to download image: {error}"))? + .error_for_status() + .map_err(|error| format!("failed to download image: {error}"))?; + + if let Some(len) = response.content_length() { + ensure_image_size(len)?; + } + + let bytes = response + .bytes() + .await + .map_err(|error| format!("failed to read image response: {error}"))?; + + Ok(bytes.to_vec()) +} + +fn load_file_bytes(path: PathBuf) -> Result, String> { + std::fs::read(path).map_err(|error| format!("failed to read image file: {error}")) +} + +fn validate_crop(crop: &CropParams, image_width: u32, image_height: u32) -> Result<(), String> { + if crop.width == 0 || crop.height == 0 { + return Err("crop width and height must be greater than zero".to_string()); + } + + let right = crop + .x + .checked_add(crop.width) + .ok_or_else(|| "crop rectangle is out of bounds".to_string())?; + let bottom = crop + .y + .checked_add(crop.height) + .ok_or_else(|| "crop rectangle is out of bounds".to_string())?; + + if right > image_width || bottom > image_height { + return Err(format!( + "crop rectangle {}x{} at {},{} exceeds image bounds {}x{}", + crop.width, crop.height, crop.x, crop.y, image_width, image_height + )); + } + + Ok(()) +} + +fn ensure_image_size(len: u64) -> Result<(), String> { + if len > MAX_IMAGE_BYTES { + Err(format!( + "image is too large: {len} bytes exceeds {MAX_IMAGE_BYTES} byte limit" + )) + } else { + Ok(()) + } +} + +fn mime_type(format: image::ImageFormat) -> Result<&'static str, String> { + match format { + image::ImageFormat::Png => Ok("image/png"), + image::ImageFormat::Jpeg => Ok("image/jpeg"), + image::ImageFormat::Gif => Ok("image/gif"), + image::ImageFormat::WebP => Ok("image/webp"), + _ => Err( + "unsupported image format; supported formats are png, jpeg, gif, and webp".to_string(), + ), + } +} diff --git a/crates/goose/src/agents/platform_extensions/developer/mod.rs b/crates/goose/src/agents/platform_extensions/developer/mod.rs index 2a171fca7c..71607aa385 100644 --- a/crates/goose/src/agents/platform_extensions/developer/mod.rs +++ b/crates/goose/src/agents/platform_extensions/developer/mod.rs @@ -1,4 +1,5 @@ pub mod edit; +pub mod image; pub mod shell; pub mod tree; @@ -8,6 +9,7 @@ use crate::agents::ToolCallContext; use anyhow::Result; use async_trait::async_trait; use edit::{EditTools, FileEditParams, FileWriteParams}; +use image::{ImageReadParams, ImageTool}; use indoc::indoc; use rmcp::model::{ CallToolResult, Content, Implementation, InitializeResult, JsonObject, ListToolsResult, @@ -27,6 +29,7 @@ pub struct DeveloperClient { shell_tool: Arc, edit_tools: Arc, tree_tool: Arc, + image_tool: Arc, } fn developer_instructions() -> &'static str { @@ -74,6 +77,7 @@ impl DeveloperClient { shell_tool: Arc::new(ShellTool::new(context.use_login_shell_path)?), edit_tools: Arc::new(EditTools::new()), tree_tool: Arc::new(TreeTool::new()), + image_tool: Arc::new(ImageTool::new()), }) } @@ -152,6 +156,18 @@ impl DeveloperClient { Some(true), Some(false), )), + Tool::new( + "read_image".to_string(), + "Read an image from a local file path or http(s) URL and return it as image content for the model to inspect. Supports png, jpeg, gif, and webp.".to_string(), + Self::schema::(), + ) + .annotate(ToolAnnotations::from_raw( + Some("Read Image".to_string()), + Some(true), + Some(false), + Some(true), + Some(false), + )), ] } } @@ -205,6 +221,16 @@ impl McpClientTrait for DeveloperClient { )) .with_priority(0.0)])), }, + "read_image" => match Self::parse_args::(arguments) { + Ok(params) => Ok(self + .image_tool + .image_read_with_cwd(params, working_dir) + .await), + Err(error) => Ok(CallToolResult::error(vec![Content::text(format!( + "Error: {error}" + )) + .with_priority(0.0)])), + }, _ => Ok(CallToolResult::error(vec![Content::text(format!( "Error: Unknown tool: {name}" )) @@ -232,7 +258,7 @@ mod tests { .map(|t| t.name.to_string()) .collect(); - assert_eq!(names, vec!["write", "edit", "shell", "tree"]); + assert_eq!(names, vec!["write", "edit", "shell", "tree", "read_image"]); } fn test_context(data_dir: std::path::PathBuf) -> PlatformExtensionContext {