From ad5cd81b6a2ce5238024dadc9397ea0fc20170e5 Mon Sep 17 00:00:00 2001 From: YoSiem <95193135+YoSiem@users.noreply.github.com> Date: Tue, 3 Nov 2026 14:50:05 +0100 Subject: [PATCH] Paste pictures into the prompt box with Ctrl+V and show attachments as picture tiles (#631) Pictures can be pasted into the prompt box with Ctrl+V (Cmd+V, Shift+Insert): a copied image is saved as a PNG in the person's own cache folder and attached, copied picture files are attached as they are, and attachments show as picture tiles. --- Cargo.lock | 1 + Cargo.toml | 2 + crates/engine/src/tools/web_common.rs | 66 +++++++ crates/engine/src/workspace.rs | 156 ++++++++++++++- crates/ui-egui/Cargo.toml | 2 + crates/ui-egui/src/chat.rs | 1 + crates/ui-egui/src/fr.tsv | 10 +- crates/ui-egui/src/i18n/ru.tsv | 10 +- crates/ui-egui/src/i18n/zh-hans.tsv | 10 +- crates/ui-egui/src/i18n/zh-hant.tsv | 10 +- crates/ui-egui/src/ja.tsv | 10 +- crates/ui-egui/src/lib.rs | 34 +++- crates/ui-egui/src/make.rs | 5 + crates/ui-egui/src/paste.rs | 246 ++++++++++++++++++++++++ crates/ui-egui/src/project.rs | 190 +++++++++++++++++- crates/ui-egui/src/prompt_box.rs | 129 +++++++++++-- crates/ui-egui/tests/headless_parity.rs | 2 + docs/ROADMAP.md | 6 + docs/STATUS.md | 2 +- 19 files changed, 866 insertions(+), 26 deletions(-) create mode 100644 crates/ui-egui/src/paste.rs diff --git a/Cargo.lock b/Cargo.lock index af6b0d62..f29e9b0b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1378,6 +1378,7 @@ dependencies = [ name = "craft-anything-ui-egui" version = "0.4.0" dependencies = [ + "arboard", "craft-anything-ai", "craft-anything-engine", "craft-anything-engines", diff --git a/Cargo.toml b/Cargo.toml index be1fcbab..856a84ba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,8 @@ egui = "0.36" egui_extras = { version = "0.36", default-features = false } egui_kittest = "0.36" rfd = "0.17" +# Pictures pasted from the clipboard (egui only receives pasted text); egui-winit already uses it. +arboard = { version = "3.6", default-features = false, features = ["image-data"] } rayon = "1" proptest = "1" # JavaScript (pure Rust, wasm-safe) diff --git a/crates/engine/src/tools/web_common.rs b/crates/engine/src/tools/web_common.rs index c28ce2ba..dd567a82 100644 --- a/crates/engine/src/tools/web_common.rs +++ b/crates/engine/src/tools/web_common.rs @@ -863,6 +863,47 @@ pub fn encode_png(img: &Image) -> Result, ToolError> { Ok(out) } +/// `img` as an 8-bit RGBA PNG of at most `max` bytes, or `None` when it would be bigger. It's +/// compressed a chunk at a time, so a picture that won't fit stops as soon as it passes `max` +/// instead of being held whole first. +pub fn encode_png_within(img: &Image, max: usize) -> Result>, ToolError> { + /// Takes bytes until `max`, then refuses them and says so. + struct Capped { + out: Vec, + max: usize, + over: bool, + } + impl std::io::Write for Capped { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + if self.out.len().saturating_add(buf.len()) > self.max { + self.over = true; + return Err(std::io::Error::other("the PNG passed its size limit")); + } + self.out.extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let mut capped = Capped { out: Vec::new(), max, over: false }; + let written = (|| { + let mut enc = png::Encoder::new(&mut capped, img.width, img.height); + enc.set_color(png::ColorType::Rgba); + enc.set_depth(png::BitDepth::Eight); + let mut w = enc.write_header().map_err(|e| format!("PNG header: {e}"))?; + let mut stream = w.stream_writer().map_err(|e| format!("PNG data: {e}"))?; + std::io::Write::write_all(&mut stream, &img.rgba).map_err(|e| format!("PNG data: {e}"))?; + stream.finish().map_err(|e| format!("PNG data: {e}"))?; + w.finish().map_err(|e| format!("PNG end: {e}")) + })(); + match written { + _ if capped.over => Ok(None), + Ok(()) => Ok(Some(capped.out)), + Err(e) => Err(ToolError::internal(e)), + } +} + /// `img` scaled to `w`×`h` (box sampling: each output pixel is the mean of the input pixels it /// covers). Aspect ratio isn't kept: comparisons bring both pictures to one size. pub fn resize(img: &Image, w: u32, h: u32) -> Image { @@ -997,6 +1038,31 @@ pub fn png_artifact(bytes: &[u8]) -> Result { #[cfg(test)] mod tests { + /// The capped encoder gives a PNG of the same pixels when it fits, and stops without one when + /// it doesn't: noise hardly compresses, so 64×64 of it can't fit in 4 KB. + #[test] + fn a_png_over_its_limit_is_refused_while_encoding() { + let small = Image { width: 3, height: 2, rgba: (0..24).collect() }; + let png = encode_png_within(&small, 1 << 20).unwrap().unwrap(); + let back = decode_png(&png).unwrap(); + assert_eq!((back.width, back.height, back.rgba), (3, 2, small.rgba.clone())); + assert_eq!(encode_png_within(&small, png.len()).unwrap().as_deref(), Some(png.as_slice()), "exactly the limit fits"); + assert_eq!(encode_png_within(&small, png.len() - 1).unwrap(), None); + let mut seed = 0x2545_f491_u32; + let noise: Vec = (0..64 * 64 * 4) + .map(|_| { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + seed as u8 + }) + .collect(); + let noisy = Image { width: 64, height: 64, rgba: noise }; + assert_eq!(encode_png_within(&noisy, 4096).unwrap(), None); + let fits = encode_png_within(&noisy, 1 << 20).unwrap().unwrap(); + assert_eq!(png_dimensions(&fits).unwrap(), (64, 64)); + } + use super::*; #[test] diff --git a/crates/engine/src/workspace.rs b/crates/engine/src/workspace.rs index 1b3cbbfb..9ce7d6de 100644 --- a/crates/engine/src/workspace.rs +++ b/crates/engine/src/workspace.rs @@ -1063,12 +1063,11 @@ fn pictures_in(dir: &Path) -> Vec { /// Copy pictures into `dir` as picture-N.ext, numbering on from what's there. Files that aren't /// PNG, JPEG, WebP or GIF, or are over 20 MB, are skipped. fn copy_pictures(dir: &Path, files: &[PathBuf]) -> Result, ToolError> { - const MAX: u64 = 20 * 1024 * 1024; std::fs::create_dir_all(dir).map_err(|e| ToolError::io(dir.display(), e))?; let mut next = pictures_in(dir).iter().map(|p| picture_number(p)).max().unwrap_or(0); let mut added = Vec::new(); for f in files { - if !is_picture(f) || f.metadata().map_or(true, |m| m.len() > MAX) { + if !attachable_picture(f) { continue; } next += 1; @@ -1085,6 +1084,93 @@ fn picture_number(p: &Path) -> u32 { p.file_stem().and_then(|s| s.to_str()).and_then(|s| s.strip_prefix("picture-")).and_then(|n| n.parse().ok()).unwrap_or(0) } +/// The most pixels a pasted picture may have (a 16K screen): more is refused before encoding. +const MAX_PASTED_PIXELS: u64 = 15360 * 8640; + +/// The largest picture a message sends ([`copy_pictures`] skips bigger files). +const MAX_PICTURE_BYTES: u64 = 20 * 1024 * 1024; + +/// Whether the file starts the way a PNG, JPEG, GIF or WebP picture does, whatever its name says. +/// Reads its first bytes, so not while drawing. +pub fn picture_content(p: &Path) -> bool { + use std::io::Read as _; + let mut head = Vec::with_capacity(12); + if std::fs::File::open(p).and_then(|f| f.take(12).read_to_end(&mut head)).is_err() { + return false; + } + head.starts_with(b"\x89PNG\r\n\x1a\n") + || head.starts_with(&[0xFF, 0xD8, 0xFF]) + || head.starts_with(b"GIF87a") + || head.starts_with(b"GIF89a") + || (head.starts_with(b"RIFF") && head.get(8..12) == Some(b"WEBP".as_slice())) +} + +/// A picture file a message can take: a picture ([`is_picture`]) of at most 20 MB. Reads its +/// size, so not while drawing. +pub fn attachable_picture(p: &Path) -> bool { + is_picture(p) && p.metadata().is_ok_and(|m| m.is_file() && m.len() <= MAX_PICTURE_BYTES) +} + +/// Where pictures pasted from the clipboard wait until their message is sent: this person's own +/// cache folder (never a temp folder other users can read). +pub fn pasted_dir() -> PathBuf { + dirs::cache_dir().or_else(|| dirs::home_dir().map(|h| h.join(".cache"))).unwrap_or_else(std::env::temp_dir).join("Craft Anything").join("pasted") +} + +/// Save a picture pasted from the clipboard (8-bit RGBA, `width`×`height`) as `pasted-