Image read tool (#9607)

Co-authored-by: Douwe Osinga <douwe@squareup.com>

Cargo.lock

@@ -4582,6 +4582,7 @@ dependencies = [
  "icu_calendar",
  "icu_locale",
  "ignore",
+ "image 0.24.9",
  "include_dir",
  "indexmap 2.14.0",
  "indoc",

crates/goose/Cargo.toml

@@ -214,6 +214,7 @@ pctx_code_mode = { version = "0.3", default-features = false, optional = true }
 icu_calendar = { version = "=2.1.1", default-features = false }
 icu_locale = { version = "=2.1.1", default-features = false }
 llama-cpp-sys-2 = { workspace = true, optional = true }
+image = { version = "0.24.9", default-features = false, features = ["png", "jpeg", "gif", "webp"] }
 
 [target.'cfg(target_os = "windows")'.dependencies]
 winapi = { workspace = true }

crates/goose/src/agents/platform_extensions/developer/image.rs

@@ -0,0 +1,255 @@
+use std::io::Cursor;
+use std::path::{Path, PathBuf};
+use std::time::Duration;
+
+use base64::Engine;
+use image::GenericImageView;
+use rmcp::model::{CallToolResult, Content};
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+use serde_json::json;
+
+use super::edit::resolve_path;
+
+const MAX_IMAGE_BYTES: u64 = 20 * 1024 * 1024;
+
+#[derive(Debug, Deserialize, JsonSchema)]
+pub struct ImageReadParams {
+    /// Local file path or http(s) URL of the image to load.
+    pub source: String,
+    /// Optional crop rectangle in pixels. Coordinates are measured from the top-left corner.
+    /// use to zoom in and get more details.
+    #[serde(default)]
+    pub crop: Option<CropParams>,
+}
+
+#[derive(Debug, Deserialize, Serialize, JsonSchema)]
+pub struct CropParams {
+    /// Left edge of the crop rectangle in pixels.
+    pub x: u32,
+    /// Top edge of the crop rectangle in pixels.
+    pub y: u32,
+    /// Width of the crop rectangle in pixels.
+    pub width: u32,
+    /// Height of the crop rectangle in pixels.
+    pub height: u32,
+}
+
+pub struct ImageTool;
+
+impl ImageTool {
+    pub fn new() -> Self {
+        Self
+    }
+
+    pub async fn image_read_with_cwd(
+        &self,
+        params: ImageReadParams,
+        working_dir: Option<&Path>,
+    ) -> CallToolResult {
+        match load_image(&params, working_dir).await {
+            Ok(loaded) => {
+                let mut result = CallToolResult::success(vec![
+                    Content::text(loaded.summary(&params.source)).with_priority(0.0),
+                    Content::image(loaded.data, loaded.mime_type.clone()).with_priority(0.0),
+                ]);
+                result.structured_content = Some(json!({
+                    "source": params.source,
+                    "mimeType": loaded.mime_type,
+                    "width": loaded.width,
+                    "height": loaded.height,
+                    "bytes": loaded.bytes_len,
+                    "originalWidth": loaded.original_width,
+                    "originalHeight": loaded.original_height,
+                    "crop": params.crop,
+                }));
+                result
+            }
+            Err(error) => CallToolResult::error(vec![
+                Content::text(format!("Error: {error}")).with_priority(0.0)
+            ]),
+        }
+    }
+}
+
+impl Default for ImageTool {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[derive(Debug)]
+struct LoadedImage {
+    data: String,
+    mime_type: String,
+    bytes_len: usize,
+    width: u32,
+    height: u32,
+    original_width: u32,
+    original_height: u32,
+    cropped: bool,
+}
+
+impl LoadedImage {
+    fn summary(&self, source: &str) -> String {
+        let crop_note = if self.cropped {
+            format!(
+                " Cropped from {}x{} to {}x{}.",
+                self.original_width, self.original_height, self.width, self.height
+            )
+        } else {
+            String::new()
+        };
+
+        format!(
+            "Loaded image from {source} ({} bytes, {}, {}x{}).{crop_note}",
+            self.bytes_len, self.mime_type, self.width, self.height
+        )
+    }
+}
+
+async fn load_image(
+    params: &ImageReadParams,
+    working_dir: Option<&Path>,
+) -> Result<LoadedImage, String> {
+    if params.source.trim().is_empty() {
+        return Err("source cannot be empty".to_string());
+    }
+
+    let bytes = load_image_bytes(&params.source, working_dir).await?;
+    ensure_image_size(bytes.len() as u64)?;
+
+    let format = image::guess_format(&bytes).map_err(|_| {
+        "unsupported image format; supported formats are png, jpeg, gif, and webp".to_string()
+    })?;
+    let mime_type = mime_type(format)?;
+    let image = image::load_from_memory_with_format(&bytes, format)
+        .map_err(|error| format!("failed to decode image: {error}"))?;
+    let (original_width, original_height) = image.dimensions();
+
+    let Some(crop) = &params.crop else {
+        return Ok(LoadedImage {
+            data: base64::prelude::BASE64_STANDARD.encode(&bytes),
+            mime_type: mime_type.to_string(),
+            bytes_len: bytes.len(),
+            width: original_width,
+            height: original_height,
+            original_width,
+            original_height,
+            cropped: false,
+        });
+    };
+
+    validate_crop(crop, original_width, original_height)?;
+    let cropped = image.crop_imm(crop.x, crop.y, crop.width, crop.height);
+    let mut cropped_bytes = Cursor::new(Vec::new());
+    cropped
+        .write_to(&mut cropped_bytes, image::ImageFormat::Png)
+        .map_err(|error| format!("failed to encode cropped image: {error}"))?;
+    let cropped_bytes = cropped_bytes.into_inner();
+    ensure_image_size(cropped_bytes.len() as u64)?;
+
+    Ok(LoadedImage {
+        data: base64::prelude::BASE64_STANDARD.encode(&cropped_bytes),
+        mime_type: "image/png".to_string(),
+        bytes_len: cropped_bytes.len(),
+        width: crop.width,
+        height: crop.height,
+        original_width,
+        original_height,
+        cropped: true,
+    })
+}
+
+async fn load_image_bytes(source: &str, working_dir: Option<&Path>) -> Result<Vec<u8>, String> {
+    if let Ok(url) = url::Url::parse(source) {
+        match url.scheme() {
+            "http" | "https" => load_url_bytes(url).await,
+            "file" => {
+                let path = url
+                    .to_file_path()
+                    .map_err(|_| "invalid file URL".to_string())?;
+                load_file_bytes(path)
+            }
+            _ => load_file_bytes(resolve_path(source, working_dir)),
+        }
+    } else {
+        load_file_bytes(resolve_path(source, working_dir))
+    }
+}
+
+async fn load_url_bytes(url: url::Url) -> Result<Vec<u8>, String> {
+    let client = reqwest::Client::builder()
+        .timeout(Duration::from_secs(30))
+        .build()
+        .map_err(|error| format!("failed to create HTTP client: {error}"))?;
+
+    let response = client
+        .get(url)
+        .send()
+        .await
+        .map_err(|error| format!("failed to download image: {error}"))?
+        .error_for_status()
+        .map_err(|error| format!("failed to download image: {error}"))?;
+
+    if let Some(len) = response.content_length() {
+        ensure_image_size(len)?;
+    }
+
+    let bytes = response
+        .bytes()
+        .await
+        .map_err(|error| format!("failed to read image response: {error}"))?;
+
+    Ok(bytes.to_vec())
+}
+
+fn load_file_bytes(path: PathBuf) -> Result<Vec<u8>, String> {
+    std::fs::read(path).map_err(|error| format!("failed to read image file: {error}"))
+}
+
+fn validate_crop(crop: &CropParams, image_width: u32, image_height: u32) -> Result<(), String> {
+    if crop.width == 0 || crop.height == 0 {
+        return Err("crop width and height must be greater than zero".to_string());
+    }
+
+    let right = crop
+        .x
+        .checked_add(crop.width)
+        .ok_or_else(|| "crop rectangle is out of bounds".to_string())?;
+    let bottom = crop
+        .y
+        .checked_add(crop.height)
+        .ok_or_else(|| "crop rectangle is out of bounds".to_string())?;
+
+    if right > image_width || bottom > image_height {
+        return Err(format!(
+            "crop rectangle {}x{} at {},{} exceeds image bounds {}x{}",
+            crop.width, crop.height, crop.x, crop.y, image_width, image_height
+        ));
+    }
+
+    Ok(())
+}
+
+fn ensure_image_size(len: u64) -> Result<(), String> {
+    if len > MAX_IMAGE_BYTES {
+        Err(format!(
+            "image is too large: {len} bytes exceeds {MAX_IMAGE_BYTES} byte limit"
+        ))
+    } else {
+        Ok(())
+    }
+}
+
+fn mime_type(format: image::ImageFormat) -> Result<&'static str, String> {
+    match format {
+        image::ImageFormat::Png => Ok("image/png"),
+        image::ImageFormat::Jpeg => Ok("image/jpeg"),
+        image::ImageFormat::Gif => Ok("image/gif"),
+        image::ImageFormat::WebP => Ok("image/webp"),
+        _ => Err(
+            "unsupported image format; supported formats are png, jpeg, gif, and webp".to_string(),
+        ),
+    }
+}

crates/goose/src/agents/platform_extensions/developer/mod.rs

@@ -1,4 +1,5 @@
 pub mod edit;
+pub mod image;
 pub mod shell;
 pub mod tree;
 
@@ -8,6 +9,7 @@ use crate::agents::ToolCallContext;
 use anyhow::Result;
 use async_trait::async_trait;
 use edit::{EditTools, FileEditParams, FileWriteParams};
+use image::{ImageReadParams, ImageTool};
 use indoc::indoc;
 use rmcp::model::{
     CallToolResult, Content, Implementation, InitializeResult, JsonObject, ListToolsResult,
@@ -27,6 +29,7 @@ pub struct DeveloperClient {
     shell_tool: Arc<ShellTool>,
     edit_tools: Arc<EditTools>,
     tree_tool: Arc<TreeTool>,
+    image_tool: Arc<ImageTool>,
 }
 
 fn developer_instructions() -> &'static str {
@@ -74,6 +77,7 @@ impl DeveloperClient {
             shell_tool: Arc::new(ShellTool::new(context.use_login_shell_path)?),
             edit_tools: Arc::new(EditTools::new()),
             tree_tool: Arc::new(TreeTool::new()),
+            image_tool: Arc::new(ImageTool::new()),
         })
     }
 
@@ -152,6 +156,18 @@ impl DeveloperClient {
                 Some(true),
                 Some(false),
             )),
+            Tool::new(
+                "read_image".to_string(),
+                "Read an image from a local file path or http(s) URL and return it as image content for the model to inspect. Supports png, jpeg, gif, and webp.".to_string(),
+                Self::schema::<ImageReadParams>(),
+            )
+            .annotate(ToolAnnotations::from_raw(
+                Some("Read Image".to_string()),
+                Some(true),
+                Some(false),
+                Some(true),
+                Some(false),
+            )),
         ]
     }
 }
@@ -205,6 +221,16 @@ impl McpClientTrait for DeveloperClient {
                 ))
                 .with_priority(0.0)])),
             },
+            "read_image" => match Self::parse_args::<ImageReadParams>(arguments) {
+                Ok(params) => Ok(self
+                    .image_tool
+                    .image_read_with_cwd(params, working_dir)
+                    .await),
+                Err(error) => Ok(CallToolResult::error(vec![Content::text(format!(
+                    "Error: {error}"
+                ))
+                .with_priority(0.0)])),
+            },
             _ => Ok(CallToolResult::error(vec![Content::text(format!(
                 "Error: Unknown tool: {name}"
             ))
@@ -232,7 +258,7 @@ mod tests {
             .map(|t| t.name.to_string())
             .collect();
 
-        assert_eq!(names, vec!["write", "edit", "shell", "tree"]);
+        assert_eq!(names, vec!["write", "edit", "shell", "tree", "read_image"]);
     }
 
     fn test_context(data_dir: std::path::PathBuf) -> PlatformExtensionContext {