diff --git a/Cargo.lock b/Cargo.lock index 5aad2d6df5..61beaa6f1d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3343,6 +3343,7 @@ dependencies = [ "tracing", "tracing-opentelemetry", "tracing-subscriber", + "unicode-normalization", "url", "urlencoding", "utoipa", @@ -8856,6 +8857,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" +[[package]] +name = "unicode-normalization" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-segmentation" version = "1.12.0" diff --git a/crates/goose/Cargo.toml b/crates/goose/Cargo.toml index b8fad75b5e..2016871c47 100644 --- a/crates/goose/Cargo.toml +++ b/crates/goose/Cargo.toml @@ -95,6 +95,7 @@ tempfile = "3.15.0" dashmap = "6.1" ahash = "0.8" tokio-util = "0.7.15" +unicode-normalization = "0.1" # Vector database for tool selection lancedb = "0.13" diff --git a/crates/goose/src/conversation/message.rs b/crates/goose/src/conversation/message.rs index 2206a75f60..20b703c795 100644 --- a/crates/goose/src/conversation/message.rs +++ b/crates/goose/src/conversation/message.rs @@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; use std::collections::HashSet; use std::fmt; +use unicode_normalization::UnicodeNormalization; use utoipa::ToSchema; use crate::conversation::tool_result_serde; @@ -409,9 +410,27 @@ impl Message { self } + fn sanitize_unicode_tags(text: &str) -> String { + let normalized: String = text.nfc().collect(); + + // Remove Unicode Tags Block characters only + normalized + .chars() + .filter(|&c| !matches!(c, '\u{E0000}'..='\u{E007F}')) + .collect() + } + /// Add text content to the message pub fn with_text>(self, text: S) -> Self { - self.with_content(MessageContent::text(text)) + let raw_text = text.into(); + let sanitized_text = Self::sanitize_unicode_tags(&raw_text); + + self.with_content(MessageContent::Text( + RawTextContent { + text: sanitized_text, + } + .no_annotation(), + )) } /// Add image content to the message @@ -565,6 +584,34 @@ mod tests { }; use serde_json::{json, Value}; + #[test] + fn test_sanitize_unicode_tags() { + let malicious = "Hello\u{E0041}\u{E0042}\u{E0043}world"; // Invisible "ABC" + let cleaned = Message::sanitize_unicode_tags(malicious); + assert_eq!(cleaned, "Helloworld"); + } + + #[test] + fn test_no_sanitize_unicode_tags() { + let clean_text = "Hello world δΈ–η•Œ 🌍"; + let cleaned = Message::sanitize_unicode_tags(clean_text); + assert_eq!(cleaned, clean_text); + } + + #[test] + fn test_sanitize_with_text() { + let malicious = "Hello\u{E0041}\u{E0042}\u{E0043}world"; // Invisible "ABC" + let message = Message::user().with_text(malicious); + assert_eq!(message.as_concat_text(), "Helloworld"); + } + + #[test] + fn test_no_sanitize_with_text() { + let clean_text = "Hello world δΈ–η•Œ 🌍"; + let message = Message::user().with_text(clean_text); + assert_eq!(message.as_concat_text(), clean_text); + } + #[test] fn test_message_serialization() { let message = Message::assistant()