From 62d2aa8bb5174185a4f9f268300b6369edcdcc72 Mon Sep 17 00:00:00 2001 From: Michael Neale Date: Fri, 20 Feb 2026 12:38:41 +1100 Subject: [PATCH] feat: computer controller overhaul, adding peekaboo (#7342) --- Cargo.lock | 7 + crates/goose-mcp/Cargo.toml | 1 + .../goose-mcp/src/computercontroller/mod.rs | 369 ++++++++++++++++-- .../src/computercontroller/platform/mod.rs | 1 + crates/goose-mcp/src/lib.rs | 2 + crates/goose-mcp/src/peekaboo/mod.rs | 85 ++++ 6 files changed, 423 insertions(+), 42 deletions(-) create mode 100644 crates/goose-mcp/src/peekaboo/mod.rs diff --git a/Cargo.lock b/Cargo.lock index c4252b9ea5..c5409c4e14 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4451,6 +4451,7 @@ dependencies = [ "serde", "serde_json", "serial_test", + "shell-words", "shellexpand", "tempfile", "tokio", @@ -8934,6 +8935,12 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shell-words" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" + [[package]] name = "shellexpand" version = "3.1.1" diff --git a/crates/goose-mcp/Cargo.toml b/crates/goose-mcp/Cargo.toml index a54c337e72..c928ce00cd 100644 --- a/crates/goose-mcp/Cargo.toml +++ b/crates/goose-mcp/Cargo.toml @@ -66,6 +66,7 @@ libc = "0.2" # ~1000 downloads). Pinned to exact version to prevent supply chain attacks. mpatch = "=0.2.0" tokio-util = { workspace = true } +shell-words = "1.1.1" [dev-dependencies] serial_test = { workspace = true } diff --git a/crates/goose-mcp/src/computercontroller/mod.rs b/crates/goose-mcp/src/computercontroller/mod.rs index 785d612578..d957e73138 100644 --- a/crates/goose-mcp/src/computercontroller/mod.rs +++ b/crates/goose-mcp/src/computercontroller/mod.rs @@ -1,4 +1,6 @@ use crate::subprocess::SubprocessExt; +#[cfg(target_os = "macos")] +use base64::Engine; use etcetera::{choose_app_strategy, AppStrategy}; use indoc::{formatdoc, indoc}; use reqwest::{Client, Url}; @@ -17,6 +19,11 @@ use serde::{Deserialize, Serialize}; use std::{collections::HashMap, fs, path::PathBuf, sync::Arc, sync::Mutex}; use tokio::process::Command; +#[cfg(target_os = "macos")] +use rmcp::model::Role; +#[cfg(target_os = "macos")] +use std::sync::atomic::{AtomicBool, Ordering}; + #[cfg(unix)] use std::os::unix::fs::PermissionsExt; @@ -91,16 +98,38 @@ pub struct AutomationScriptParams { pub save_output: bool, } -/// Parameters for the computer_control tool +/// Parameters for the computer_control tool (Windows, Linux) +#[cfg(not(target_os = "macos"))] #[derive(Debug, Serialize, Deserialize, JsonSchema)] pub struct ComputerControlParams { - /// The automation script content (PowerShell for Windows, AppleScript for macOS) + /// The automation script content (PowerShell for Windows, shell for Linux) pub script: String, /// Whether to save the script output to a file #[serde(default)] pub save_output: bool, } +/// Parameters for the computer_control tool (macOS — Peekaboo CLI passthrough) +#[cfg(target_os = "macos")] +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +pub struct ComputerControlParams { + /// The peekaboo subcommand and arguments as a single string. + /// Examples: + /// "see --app Safari --annotate" + /// "click --on B1" + /// "type --text \"hello\" --return" + /// "hotkey --keys cmd,c" + /// "app launch Safari --open https://example.com" + /// "window list --app Safari --json" + /// "press tab --count 3" + /// "clipboard --action get" + pub command: String, + /// Whether to capture and return a screenshot as part of the result. + /// Useful after click/type actions to see the updated UI state. + #[serde(default)] + pub capture_screenshot: bool, +} + /// Parameters for the cache tool #[derive(Debug, Serialize, Deserialize, JsonSchema)] pub struct CacheParams { @@ -285,6 +314,8 @@ pub struct ComputerControllerServer { http_client: Client, instructions: String, system_automation: Arc>, + #[cfg(target_os = "macos")] + peekaboo_installed: Arc, } impl Default for ComputerControllerServer { @@ -334,25 +365,96 @@ impl ComputerControllerServer { "macos" => indoc! {r#" Here are some extra tools: automation_script - - Create and run Shell and Ruby scripts - - Shell (bash) is recommended for most tasks + - Create and run Shell, Ruby, or AppleScript scripts - Scripts can save their output to files - - macOS-specific features: - - AppleScript for system and UI control - - Integration with macOS apps and services - - Use the screenshot tool if needed to help with tasks - computer_control - - System automation using AppleScript - - Consider the screenshot tool to work out what is on screen and what to do to help with the control task. + computer_control — Peekaboo CLI for macOS UI automation (auto-installed via Homebrew). + Peekaboo captures/inspects screens, targets UI elements, drives input, and manages + apps/windows/menus. Pass a peekaboo subcommand string as the `command` parameter. + Set `capture_screenshot: true` to capture the screen after actions (click, type, etc.). + Commands support `--json`/`-j` for structured output. Run `peekaboo --help` for + full flags if needed. - When you need to interact with websites or web applications, consider using the computer_control tool with AppleScript, which can automate Safari or other browsers to: - - Open specific URLs - - Fill in forms - - Click buttons - - Extract content - - Handle web-based workflows - This is often more reliable than web scraping for modern web applications. + Quickstart (most reliable flow): + 1. command: "see --app Safari --annotate" — get annotated screenshot with element IDs + 2. command: "click --on B3 --app Safari" — click element B3 + 3. command: "type \"user@example.com\" --app Safari" — type text + 4. command: "press tab --count 1 --app Safari" — press tab + 5. command: "type \"password\" --app Safari --return" — type and press enter + + Vision: + - see — annotated UI map with element IDs and optional AI analysis + `see --app Safari --annotate`, `see --mode screen --screen-index 0` + `see --app Notes --analyze "describe what's on screen"` + - image — capture screenshots without annotation + `image --mode frontmost`, `image --mode screen --screen-index 1 --retina` + `image --app Safari --window-title "Dashboard" --analyze "Summarize KPIs"` + - capture — live motion-aware capture + `capture live --mode region --region 100,100,800,600 --duration 30` + + Interaction: + - click — by element ID, query, or coordinates with smart waits + `click --on B1`, `click --coords 100,200`, `click --on B1 --double`, `click --on B1 --right` + - type — text input with optional control keys + `type "hello" --return`, `type "text" --clear --app Notes`, `type "slow" --wpm 80` + - press — special key sequences with repeats + `press tab --count 3`, `press escape`, `press return`, `press space` + - hotkey — modifier key combos (comma-separated) + `hotkey --keys cmd,c`, `hotkey --keys cmd,shift,t`, `hotkey --keys cmd,a` + - paste — set clipboard then paste (more reliable than type for long text) + `paste --text "long multi-line content"` + - scroll — directional scrolling with optional targeting + `scroll --direction down --amount 5 --smooth`, `scroll --direction up --amount 3` + - drag — drag between elements or coordinates + `drag --from B1 --to T2`, `drag --from-coords 100,100 --to-coords 500,300` + - swipe — gesture-style drags + `swipe --from-coords 100,500 --to-coords 100,200 --duration 800` + - move — cursor positioning + `move 500,300 --smooth` + + Apps & Windows: + - app — launch, quit, switch, list applications + `app launch Safari --open https://example.com`, `app quit Safari` + `app switch Safari`, `app list`, `app hide Safari`, `app unhide Safari` + - window — manage window position, size, focus, list + `window list --app Safari --json`, `window focus --app Safari` + `window set-bounds --app Safari --x 50 --y 50 --width 1200 --height 800` + `window close --app Safari`, `window minimize --app Safari` + - list — enumerate apps, windows, screens + `list apps --json`, `list windows --json`, `list screens --json` + - space — macOS Spaces (virtual desktops) + `space list`, `space switch --index 2` + + Menus & System: + - menu — click application menu items + `menu click --app Safari --item "New Window"` + `menu click --app TextEdit --path "Format > Font > Show Fonts"` + - menubar — status bar / menu extras + `menubar list --json`, `menubar click --title "WiFi"` + - dock — Dock items + `dock launch Safari`, `dock list --json` + - dialog — system dialogs and alerts + `dialog click --button "OK"`, `dialog list` + - clipboard — read/write clipboard + `clipboard --action get`, `clipboard --action set --text "content"` + - open — open URLs or files with app targeting + `open https://example.com --app Safari` + - permissions — check Screen Recording / Accessibility status + `permissions status` + + Common targeting parameters (work across most commands): + - App/window: `--app Name`, `--pid 1234`, `--window-title "title"`, `--window-id 5678`, `--window-index 0` + - Elements: `--on B1` (element ID from see), `--coords 100,200` + - Snapshot reuse: `--snapshot ` (reuse a previous see result without re-capturing) + - Focus: `--no-auto-focus`, `--space-switch`, `--bring-to-current-space` + + Tips: + - Always `see --annotate` first to identify element IDs before clicking + - Use `--json` for structured output on list/query commands + - Use `paste` over `type` for long or multi-line text + - Use `--screen-index` for multi-monitor setups + - If something fails, check `permissions status` for missing permissions + - Use `capture_screenshot: true` on click/type/press actions to verify the result "#}, _ => indoc! {r#" Here are some extra tools: @@ -421,6 +523,8 @@ impl ComputerControllerServer { http_client: Client::builder().user_agent("goose/1.0").build().unwrap(), instructions, system_automation, + #[cfg(target_os = "macos")] + peekaboo_installed: Arc::new(AtomicBool::new(crate::peekaboo::is_peekaboo_installed())), } } @@ -592,20 +696,36 @@ impl ComputerControllerServer { } /// Create and run small scripts for automation tasks - #[cfg(not(target_os = "windows"))] + #[cfg(target_os = "macos")] #[tool( name = "automation_script", description = " - Create and run small scripts for automation tasks. - Supports Shell and Ruby (on macOS). + Create and run Shell, Ruby, or AppleScript (via osascript) scripts. + Use shell (bash) for most tasks. AppleScript for app scripting and system settings. + Examples: + - sort file.txt | uniq + - awk -F ',' '{ print $2}' file.csv + - osascript -e 'tell app \"Finder\" to get name of every window' + " + )] + pub async fn automation_script( + &self, + params: Parameters, + ) -> Result { + self.automation_script_impl(params).await + } - The script is saved to a temporary file and executed. + /// Create and run small scripts for automation tasks + #[cfg(not(any(target_os = "windows", target_os = "macos")))] + #[tool( + name = "automation_script", + description = " + Create and run Shell scripts for automation tasks. Consider using shell script (bash) for most simple tasks first. - Ruby is useful for text processing or when you need more sophisticated scripting capabilities. - Some examples of shell: - - create a sorted list of unique lines: sort file.txt | uniq - - extract 2nd column in csv: awk -F ',' '{ print $2}' - - pattern matching: grep pattern file.txt + Examples: + - sort file.txt | uniq + - awk -F ',' '{ print $2}' file.csv + - grep pattern file.txt " )] pub async fn automation_script( @@ -788,33 +908,33 @@ impl ComputerControllerServer { self.computer_control_impl(params).await } - /// Control the computer using system automation + /// Control the computer using Peekaboo CLI for macOS GUI automation. + /// Auto-installs via Homebrew on first use. #[cfg(target_os = "macos")] #[tool( name = "computer_control", description = " - Control the computer using AppleScript (macOS only). Automate applications and system features. + macOS UI automation via Peekaboo CLI. Pass a subcommand string as `command`. - Key capabilities: - - Control Applications: Launch, quit, manage apps (Mail, Safari, iTunes, etc) - - Interact with app-specific feature: (e.g, edit documents, process photos) - - Perform tasks in third-party apps that support AppleScript - - UI Automation: Simulate user interactions like, clicking buttons, select menus, type text, filling out forms - - System Control: Manage settings (volume, brightness, wifi), shutdown/restart, monitor events - - Web & Email: Open URLs, web automation, send/organize emails, handle attachments - - Media: Manage music libraries, photo collections, playlists - - File Operations: Organize files/folders - - Integration: Calendar, reminders, messages - - Data: Interact with spreadsheets and documents + Core workflow: see → click → type + 1. see --app Safari --annotate (get annotated screenshot with element IDs) + 2. click --on B3 (click element by ID) + 3. type \"hello\" --return (type text, press enter) - Can be combined with screenshot tool for visual task assistance. + Key commands: see, image, click, type, press, hotkey, paste, scroll, drag, + swipe, move, app, window, list, menu, menubar, dock, dialog, clipboard, + space, open, permissions. + + Targeting: --app Name, --window-title, --window-id, --on ID, --coords x,y + Set capture_screenshot=true to verify UI state after actions. + See extension instructions for full command reference and examples. " )] pub async fn computer_control( &self, params: Parameters, ) -> Result { - self.computer_control_impl(params).await + self.peekaboo_impl(params).await } /// Control the computer using system automation @@ -856,6 +976,7 @@ impl ComputerControllerServer { self.computer_control_impl(params).await } + #[cfg(not(target_os = "macos"))] async fn computer_control_impl( &self, params: Parameters, @@ -892,6 +1013,170 @@ impl ComputerControllerServer { Ok(CallToolResult::success(vec![Content::text(result)])) } + #[cfg(target_os = "macos")] + fn ensure_peekaboo(&self) -> Result<(), ErrorData> { + if self.peekaboo_installed.load(Ordering::Relaxed) { + return Ok(()); + } + if crate::peekaboo::is_peekaboo_installed() { + self.peekaboo_installed.store(true, Ordering::Relaxed); + return Ok(()); + } + tracing::info!("Peekaboo not found, attempting auto-install via brew"); + match crate::peekaboo::auto_install_peekaboo() { + Ok(()) => { + self.peekaboo_installed.store(true, Ordering::Relaxed); + tracing::info!("Peekaboo installed successfully"); + Ok(()) + } + Err(msg) => Err(ErrorData::new( + ErrorCode::INTERNAL_ERROR, + format!( + "Peekaboo is not installed and auto-install failed: {}\n\ + Install manually with: brew install steipete/tap/peekaboo\n\ + Peekaboo requires macOS 15+ (Sequoia) with Screen Recording and Accessibility permissions.", + msg + ), + None, + )), + } + } + + #[cfg(target_os = "macos")] + fn run_peekaboo_cmd(&self, args: &[&str]) -> Result { + let output = std::process::Command::new("peekaboo") + .args(args) + .output() + .map_err(|e| { + ErrorData::new( + ErrorCode::INTERNAL_ERROR, + format!("Failed to run peekaboo: {}", e), + None, + ) + })?; + let stdout = String::from_utf8_lossy(&output.stdout).into_owned(); + let stderr = String::from_utf8_lossy(&output.stderr).into_owned(); + if !output.status.success() { + return Err(ErrorData::new( + ErrorCode::INTERNAL_ERROR, + format!( + "peekaboo {} failed (exit {}):\n{}\n{}", + args.first().unwrap_or(&""), + output.status, + stderr.trim(), + stdout.trim() + ), + None, + )); + } + Ok(stdout) + } + + #[cfg(target_os = "macos")] + async fn peekaboo_impl( + &self, + params: Parameters, + ) -> Result { + self.ensure_peekaboo()?; + let params = params.0; + + let args = shell_words::split(¶ms.command).map_err(|e| { + ErrorData::new( + ErrorCode::INVALID_PARAMS, + format!("Failed to parse command: {}", e), + None, + ) + })?; + if args.is_empty() { + return Err(ErrorData::new( + ErrorCode::INVALID_PARAMS, + "Command cannot be empty".to_string(), + None, + )); + } + + let is_see = args[0] == "see"; + let is_image = args[0] == "image"; + let screenshot_path = if is_see || is_image { + Some(self.get_cache_path(&args[0], "png")) + } else { + None + }; + + let mut full_args: Vec = args.clone(); + + if let Some(ref path) = screenshot_path { + if !full_args.iter().any(|a| a == "--path") { + full_args.push("--path".to_string()); + full_args.push(path.to_string_lossy().to_string()); + } + } + if is_see && !full_args.iter().any(|a| a == "--json-output") { + full_args.push("--json-output".to_string()); + } + + let wants_json = matches!( + args[0].as_str(), + "list" | "window" | "menubar" | "permissions" | "clipboard" + ); + if wants_json + && !full_args.iter().any(|a| a == "--json" || a == "-j") + && !full_args.iter().any(|a| a == "--json-output") + { + full_args.push("--json".to_string()); + } + + let arg_refs: Vec<&str> = full_args.iter().map(|s| s.as_str()).collect(); + let stdout = self.run_peekaboo_cmd(&arg_refs)?; + + let mut contents = Vec::new(); + + if let Some(ref path) = screenshot_path { + let annotated = path.to_string_lossy().replace(".png", "_annotated.png"); + let image_path = if is_see && std::path::Path::new(&annotated).exists() { + PathBuf::from(&annotated) + } else { + path.clone() + }; + if image_path.exists() { + if let Ok(bytes) = fs::read(&image_path) { + let data = base64::prelude::BASE64_STANDARD.encode(&bytes); + contents.push(Content::image(data, "image/png").with_priority(0.0)); + } + } + } + + if params.capture_screenshot && screenshot_path.is_none() { + let cap_path = self.get_cache_path("peekaboo_capture", "png"); + let cap_path_str = cap_path.to_string_lossy().to_string(); + if self + .run_peekaboo_cmd(&["image", "--mode", "frontmost", "--path", &cap_path_str]) + .is_ok() + && cap_path.exists() + { + if let Ok(bytes) = fs::read(&cap_path) { + let data = base64::prelude::BASE64_STANDARD.encode(&bytes); + contents.push(Content::image(data, "image/png").with_priority(0.0)); + } + } + } + + let text = if stdout.len() > 12000 { + let truncated: String = stdout.chars().take(12000).collect(); + format!( + "{}\n\n[Output truncated. {} total chars.]", + truncated, + stdout.len() + ) + } else { + stdout + }; + + contents.insert(0, Content::text(&text).with_audience(vec![Role::Assistant])); + + Ok(CallToolResult::success(contents)) + } + /// Process Excel (XLSX) files to read and manipulate spreadsheet data #[tool( name = "xlsx_tool", diff --git a/crates/goose-mcp/src/computercontroller/platform/mod.rs b/crates/goose-mcp/src/computercontroller/platform/mod.rs index 5a456ca896..ea1ea8f108 100644 --- a/crates/goose-mcp/src/computercontroller/platform/mod.rs +++ b/crates/goose-mcp/src/computercontroller/platform/mod.rs @@ -14,6 +14,7 @@ pub use self::macos::MacOSAutomation; #[cfg(any(target_os = "linux", target_os = "android"))] pub use self::linux::LinuxAutomation; +#[allow(dead_code)] pub trait SystemAutomation: Send + Sync { fn execute_system_script(&self, script: &str) -> std::io::Result; fn get_shell_command(&self) -> (&'static str, &'static str); // (shell, arg) diff --git a/crates/goose-mcp/src/lib.rs b/crates/goose-mcp/src/lib.rs index e1077683c9..530c30dfb9 100644 --- a/crates/goose-mcp/src/lib.rs +++ b/crates/goose-mcp/src/lib.rs @@ -14,6 +14,8 @@ pub mod computercontroller; pub mod developer; pub mod mcp_server_runner; mod memory; +#[cfg(target_os = "macos")] +pub mod peekaboo; pub mod subprocess; pub mod tutorial; diff --git a/crates/goose-mcp/src/peekaboo/mod.rs b/crates/goose-mcp/src/peekaboo/mod.rs new file mode 100644 index 0000000000..2754f63222 --- /dev/null +++ b/crates/goose-mcp/src/peekaboo/mod.rs @@ -0,0 +1,85 @@ +//! Peekaboo helper functions for macOS GUI automation via the Peekaboo CLI. +//! +//! These are used by `ComputerControllerServer` on macOS to auto-install +//! and invoke peekaboo. This module does not expose its own MCP server — +//! peekaboo is accessed through the `computer_control` tool on macOS. + +const BREW_FORMULA: &str = "steipete/tap/peekaboo"; + +pub fn is_peekaboo_installed() -> bool { + std::process::Command::new("which") + .arg("peekaboo") + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +pub fn resolve_brew() -> Option { + if let Ok(output) = std::process::Command::new("which").arg("brew").output() { + if output.status.success() { + let path = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if !path.is_empty() { + return Some(path); + } + } + } + + for candidate in &["/opt/homebrew/bin/brew", "/usr/local/bin/brew"] { + if std::path::Path::new(candidate).exists() { + return Some(candidate.to_string()); + } + } + + None +} + +pub fn auto_install_peekaboo() -> Result<(), String> { + let brew = resolve_brew().ok_or_else(|| { + "Homebrew is not installed. Install Homebrew first (https://brew.sh), then run: brew install steipete/tap/peekaboo".to_string() + })?; + + tracing::info!("Running: {} install {}", brew, BREW_FORMULA); + + let output = std::process::Command::new(&brew) + .args(["install", BREW_FORMULA]) + .output() + .map_err(|e| format!("Failed to run brew: {}", e))?; + + if output.status.success() { + if is_peekaboo_installed() { + return Ok(()); + } + // brew succeeded but binary not on PATH — try adding brew bin + if let Ok(prefix_output) = std::process::Command::new(&brew) + .args(["--prefix"]) + .output() + { + let prefix = String::from_utf8_lossy(&prefix_output.stdout) + .trim() + .to_string(); + let bin_path = format!("{}/bin/peekaboo", prefix); + if std::path::Path::new(&bin_path).exists() { + if let Ok(current_path) = std::env::var("PATH") { + std::env::set_var("PATH", format!("{}/bin:{}", prefix, current_path)); + if is_peekaboo_installed() { + return Ok(()); + } + } + } + } + Err("brew install succeeded but peekaboo binary not found on PATH".to_string()) + } else { + let stderr = String::from_utf8_lossy(&output.stderr); + let stdout = String::from_utf8_lossy(&output.stdout); + Err(format!( + "brew install failed (exit {}):\n{}{}", + output.status, + stderr.trim(), + if stdout.trim().is_empty() { + String::new() + } else { + format!("\n{}", stdout.trim()) + } + )) + } +}