diff --git a/src/core/render.ts b/src/core/render.ts index f9ee9ff..9a42d32 100644 --- a/src/core/render.ts +++ b/src/core/render.ts @@ -445,6 +445,61 @@ export function dereflow(reflowed: string): string { return reflowed.split(NL_SENTINEL).join('\n'); } +// --- atlas-miss escaping ----------------------------------------------------- +// +// Codepoints absent from the atlas used to render as blank cells (droppedChars +// telemetry). Emoji and other astral glyphs can't be DRAWN at 5Γ—8 px, but they +// can be PRESERVED: substitute a deterministic textual escape (`[U+HEX]`) before +// wrap, so the model reads the codepoint instead of a gap. Applied at the same layer +// as tab expansion (wrapLines + measureContentCols), so wrap math and canvas +// measurement stay consistent. Pure stringβ†’string β‡’ renders stay deterministic. + +/** Pure-ASCII delimiters: `πŸ”₯` β†’ `[U+1F525]`. ASCII is the one range guaranteed + * present in BOTH atlases (mathematical white brackets U+27E6/7 were tried first + * and are absent from each β€” the "full-bmp" note on NL_SENTINEL overstates real + * coverage). ASCII output also makes idempotency structural: an escaped line + * contains no atlas misses, so a second pass is a no-op. `[U+…]` can collide + * with literal source text discussing codepoints; the escape is for model + * legibility, not machine round-trip, so the ambiguity is acceptable. */ +export const GLYPH_ESCAPE_OPEN = '[U+'; +export const GLYPH_ESCAPE_CLOSE = ']'; + +/** Codepoints that stay DROPPED (blank cell) rather than escaped. Escaping these + * would add noise, not information β€” they're modifiers/invisibles, not content. + * C0 additionally MUST keep width-1: the parallel slot string carries \x01/\x02 + * role markers and \x03 neutral (see SLOT_MARK_*), and escaping them would + * desync slot/text alignment and smear role hues. */ +function isEscapeExempt(cp: number): boolean { + if (cp < 0x20) return true; // C0 controls (slot markers; \t is expanded before we run) + if (cp >= 0x7f && cp <= 0x9f) return true; // DEL + C1 controls + if (cp >= 0x0300 && cp <= 0x036f) return true; // combining diacritics + if (cp === 0x200b || cp === 0x200c || cp === 0x200d || cp === 0x2060 || cp === 0xfeff) + return true; // zero-width / word-joiner / BOM + if (cp >= 0xfe00 && cp <= 0xfe0f) return true; // variation selectors (emoji presentation) + if (cp >= 0xe0100 && cp <= 0xe01ef) return true; // variation selectors supplement + return false; +} + +/** Replace atlas-missing codepoints with `[U+HEX]` (uppercase hex β€” e.g. + * πŸ”₯ β†’ `[U+1F525]`). Lossless for non-exempt misses (hex β†’ codepoint) and + * idempotent: the escape spells only atlas-present chars, so a second pass is + * a no-op. Fast path allocates nothing when every codepoint is in the atlas. */ +export function escapeMissingGlyphs(line: string): string { + let out: string | null = null; // lazily materialized on first miss + let i = 0; + for (const ch of line) { + const cp = ch.codePointAt(0)!; + if (atlasRank(cp) < 0 && !isEscapeExempt(cp)) { + if (out === null) out = line.slice(0, i); + out += GLYPH_ESCAPE_OPEN + cp.toString(16).toUpperCase() + GLYPH_ESCAPE_CLOSE; + } else if (out !== null) { + out += ch; + } + i += ch.length; + } + return out ?? line; +} + /** Expand \t to U+2192 β†’ + padding to the next TAB_WIDTH stop. Visible marker lets the * model distinguish indent-spaces from intentional-spaces. Wide CJK chars count as 2 cols. * U+0009 is absent from the atlas (control codepoint), so without this every tab was a drop. */ @@ -512,7 +567,7 @@ export function measureContentCols( let start = 0; for (let i = 0; i <= text.length; i++) { if (i === text.length || text[i] === '\n') { - const w = measureLineCols(expandTabsInLine(text.slice(start, i)), markerScale, font); + const w = measureLineCols(escapeMissingGlyphs(expandTabsInLine(text.slice(start, i))), markerScale, font); if (w > widest) widest = w; if (widest >= cap) return cap; start = i + 1; @@ -530,7 +585,7 @@ export function wrapLines( const out: string[] = []; const minified = minifyForRender(text); for (const rawWithTabs of minified.split('\n')) { - const raw = expandTabsInLine(rawWithTabs); + const raw = escapeMissingGlyphs(expandTabsInLine(rawWithTabs)); if (raw.length === 0) { out.push(''); continue; diff --git a/tests/glyph-escape.test.ts b/tests/glyph-escape.test.ts new file mode 100644 index 0000000..3c7c655 --- /dev/null +++ b/tests/glyph-escape.test.ts @@ -0,0 +1,115 @@ +/** + * tests/glyph-escape.test.ts + * + * Atlas-miss escaping: emoji and other unrenderable codepoints are preserved + * as ASCII `[U+HEX]` escapes instead of blank cells. + * + * CONTRACT: + * β€’ escapeMissingGlyphs is pure, idempotent, and lossless for non-exempt + * misses (hex payload β†’ codepoint) + * β€’ rendering text containing emoji yields droppedChars === 0 on both the + * 1-bit and the AA (production dense) paths + * β€’ exempt codepoints (C0 slot markers, variation selectors, ZWJ, zero-width) + * are left untouched β€” still dropped β€” so slot-string width alignment and + * role coloring are unchanged + */ + +import { describe, expect, it } from 'vitest'; +import { + escapeMissingGlyphs, + GLYPH_ESCAPE_OPEN, + GLYPH_ESCAPE_CLOSE, + renderChunkToPng, + wrapLines, + measureContentCols, + DENSE_RENDER_STYLE, + SLOT_MARK_USER, + SLOT_MARK_ASSISTANT, +} from '../src/core/render.js'; + +// --------------------------------------------------------------------------- +// 1. escapeMissingGlyphs unit contract +// --------------------------------------------------------------------------- + +describe('escapeMissingGlyphs', () => { + it('escapes an astral emoji as [U+HEX]', () => { + expect(escapeMissingGlyphs('a\u{1F525}b')).toBe( + `a${GLYPH_ESCAPE_OPEN}1F525${GLYPH_ESCAPE_CLOSE}b`, + ); + }); + + it('fast path: returns the SAME reference when nothing misses', () => { + const s = 'plain ascii β€” no atlas misses, incl. ↡ and CJK ζΌ’ε­—'; + expect(escapeMissingGlyphs(s)).toBe(s); + }); + + it('is idempotent (escape output contains only atlas-present chars)', () => { + const once = escapeMissingGlyphs('\u{1F4CA} chart \u{1F680}'); + expect(escapeMissingGlyphs(once)).toBe(once); + }); + + it('is lossless: hex payload round-trips to the source codepoint', () => { + const escaped = escapeMissingGlyphs('\u{1F680}'); + const m = /\[U\+([0-9A-F]+)\]/.exec(escaped); + expect(m).not.toBeNull(); + expect(String.fromCodePoint(parseInt(m![1]!, 16))).toBe('\u{1F680}'); + }); + + it('escapes every miss in a multi-emoji line, preserving order', () => { + expect(escapeMissingGlyphs('\u{1F525}x\u{1F680}')).toBe( + `${GLYPH_ESCAPE_OPEN}1F525${GLYPH_ESCAPE_CLOSE}x${GLYPH_ESCAPE_OPEN}1F680${GLYPH_ESCAPE_CLOSE}`, + ); + }); + + it('leaves exempt codepoints untouched: VS16, ZWJ, C0 slot markers', () => { + const s = '️‍' + SLOT_MARK_USER + SLOT_MARK_ASSISTANT + 'Μ‚'; + expect(escapeMissingGlyphs(s)).toBe(s); + }); +}); + +// --------------------------------------------------------------------------- +// 2. Render integration: emoji no longer drop +// --------------------------------------------------------------------------- + +describe('rendering emoji', () => { + it('1-bit path: droppedChars === 0 and escape survives into wrapped lines', async () => { + const text = 'deploy \u{1F680} done, fire \u{1F525} out'; + const img = await renderChunkToPng(text, 80); + expect(img.droppedChars).toBe(0); + expect(img.droppedCodepoints.size).toBe(0); + expect(wrapLines(text, 80).join('')).toContain( + `${GLYPH_ESCAPE_OPEN}1F680${GLYPH_ESCAPE_CLOSE}`, + ); + }); + + it('AA dense path (production DENSE_RENDER_STYLE): droppedChars === 0', async () => { + const img = await renderChunkToPng( + 'metrics \u{1F4CA} look good \u{1F389}', + 80, + DENSE_RENDER_STYLE, + ); + expect(img.droppedChars).toBe(0); + }); + + it('exempt invisibles still count as drops (behavior unchanged)', async () => { + const img = await renderChunkToPng('x️y', 80); + expect(img.droppedCodepoints.get(0xfe0f)).toBe(1); + }); + + it('measureContentCols sees the ESCAPED width, matching what wrapLines lays out', () => { + const text = '\u{1F525}'; // 1 source cell naively; 9 cells escaped + const measured = measureContentCols(text, 312); + const laidOut = wrapLines(text, 312)[0]!.length; + expect(measured).toBe(laidOut); + expect(measured).toBe(9); // [U+1F525] + }); + + it('slot-string alignment: body slot copy wraps identically to text', () => { + // Slot bodies are verbatim codepoint copies of the text body (slotCopyBody), + // so both sides pass through the same escape and wrap to identical shapes. + const body = 'alert \u{1F6A8} now, and again \u{1F6A8} later'; + expect(wrapLines(body, 10)).toEqual(wrapLines(body, 10)); + // Marker chars themselves are exempt β†’ width-1 on the slot side is preserved. + expect(escapeMissingGlyphs(SLOT_MARK_USER).length).toBe(1); + }); +}); diff --git a/tests/render.test.ts b/tests/render.test.ts index 249ddf2..127c7a9 100644 --- a/tests/render.test.ts +++ b/tests/render.test.ts @@ -364,17 +364,26 @@ describe('renderer', () => { expect(img.droppedChars).toBe(0); }); - it('treats codepoints outside the atlas as dropped (e.g. emoji)', async () => { + it('escapes codepoints outside the atlas instead of dropping them (e.g. emoji)', async () => { // πŸ˜€ is U+1F600 β€” Supplementary Plane, not in BMP. Even `full-bmp` profile - // wouldn't cover it. Renderer must advance by 1 cell and bump the counter, - // not crash on the surrogate pair. + // wouldn't cover it. It renders as the lossless ASCII escape [U+1F600] + // (see escapeMissingGlyphs), not as a blank cell β€” and must not crash on + // the surrogate pair. const img = await renderChunkToPng('hi πŸ˜€ world'); - expect(img.droppedChars).toBe(1); - // charsRendered counts codepoints, NOT UTF-16 units β€” the emoji is one - // codepoint even though it occupies two UTF-16 units. + expect(img.droppedChars).toBe(0); + // charsRendered counts SOURCE codepoints, NOT UTF-16 units β€” the emoji is + // one codepoint even though it occupies two UTF-16 units. expect(img.charsRendered).toBe(10); // 'hi ' (3) + πŸ˜€ (1) + ' world' (6) = 10 }); + it('treats escape-exempt invisibles as dropped with 1-cell advance (e.g. VS16)', async () => { + // U+FE0F (variation selector-16) is deliberately NOT escaped β€” it's an + // emoji presentation modifier, noise if spelled out. Renderer must advance + // by 1 cell and bump the counter. + const img = await renderChunkToPng('hi ️ world'); + expect(img.droppedChars).toBe(1); + }); + it('CJK characters advance two cells; mixed lines wrap correctly', async () => { // 100 cols, mixed Latin + CJK. 30 Latin chars + 40 CJK chars = 30 + 80 = // 110 visual columns β†’ must wrap to 2 lines. @@ -497,21 +506,21 @@ describe('renderer', () => { }); it('droppedCodepoints map is populated correctly when drops occur', async () => { - // πŸ˜€ is supplementary-plane (not in atlas regardless of profile). The - // codepoint should appear in the map with count 1; charsRendered counts - // it as a single codepoint. - const img = await renderChunkToPng('hi πŸ˜€ there'); + // Emoji now escape to [U+HEX] instead of dropping, so the drop path is + // exercised via an escape-EXEMPT codepoint: U+FE0F (variation selector). + // The codepoint should appear in the map with count 1. + const img = await renderChunkToPng('hi ️ there'); expect(img.droppedChars).toBe(1); expect(img.droppedCodepoints.size).toBe(1); - expect(img.droppedCodepoints.get(0x1f600)).toBe(1); + expect(img.droppedCodepoints.get(0xfe0f)).toBe(1); }); it('droppedCodepoints tallies repeat drops correctly', async () => { - // Three occurrences of the same dropped codepoint β†’ count 3. - const img = await renderChunkToPng('πŸ˜€πŸ˜€πŸ˜€'); + // Three occurrences of the same dropped (exempt) codepoint β†’ count 3. + const img = await renderChunkToPng('a️b️c️'); expect(img.droppedChars).toBe(3); expect(img.droppedCodepoints.size).toBe(1); - expect(img.droppedCodepoints.get(0x1f600)).toBe(3); + expect(img.droppedCodepoints.get(0xfe0f)).toBe(3); }); // --- Whitespace minify (HANDOFF R1) --------------------------------------- @@ -1832,17 +1841,19 @@ describe('transform', () => { // capture & inspect the request body. it('populates droppedCodepointsTop when drops occur, sorted by count', async () => { - // System slab forces compression. The slab contains drops for two distinct - // supplementary-plane codepoints at different rates so we can verify the - // sort order. - const cpA = String.fromCodePoint(0x1f600); // πŸ˜€ - const cpB = String.fromCodePoint(0x1f604); // πŸ˜„ - const cpC = String.fromCodePoint(0x1f60a); // 😊 + // System slab forces compression. Emoji now escape to [U+HEX] instead of + // dropping, so the drop path is exercised via escape-EXEMPT codepoints: + // plane-14 variation selectors (U+E01xx β€” astral, guaranteed absent from + // the BMP atlas, and deliberately never escaped). Three distinct + // codepoints at different rates so we can verify the sort order. + const cpA = String.fromCodePoint(0xe0100); + const cpB = String.fromCodePoint(0xe0104); + const cpC = String.fromCodePoint(0xe010a); const sys = 'x'.repeat(150000) + // bulk to force compression - '\n' + cpA.repeat(10) + // 10 drops of U+1F600 - '\n' + cpB.repeat(3) + // 3 drops of U+1F604 - '\n' + cpC.repeat(1); // 1 drop of U+1F60A + '\n' + cpA.repeat(10) + // 10 drops of U+E0100 + '\n' + cpB.repeat(3) + // 3 drops of U+E0104 + '\n' + cpC.repeat(1); // 1 drop of U+E010A const req = JSON.stringify({ model: 'claude-3-5-sonnet', messages: [{ role: 'user', content: 'hi' }], @@ -1853,9 +1864,9 @@ describe('transform', () => { expect(info.droppedChars).toBeGreaterThanOrEqual(14); expect(info.droppedCodepointsTop).toBeDefined(); const top = info.droppedCodepointsTop!; - expect(top['U+1F600']).toBe(10); - expect(top['U+1F604']).toBe(3); - expect(top['U+1F60A']).toBe(1); + expect(top['U+E0100']).toBe(10); + expect(top['U+E0104']).toBe(3); + expect(top['U+E010A']).toBe(1); // Ensure key format is the expected U+HHHH uppercase with no surprises. for (const k of Object.keys(top)) { expect(k).toMatch(/^U\+[0-9A-F]{4,}$/); @@ -1863,7 +1874,7 @@ describe('transform', () => { // Sorted by count desc: iteration of object keys preserves insertion order // in V8/JSC, so the first key is the highest-count drop. const keys = Object.keys(top); - expect(keys[0]).toBe('U+1F600'); + expect(keys[0]).toBe('U+E0100'); }); it('omits droppedCodepointsTop entirely when no drops occur', async () => { @@ -1880,13 +1891,15 @@ describe('transform', () => { }); it('caps droppedCodepointsTop at 20 entries', async () => { - // 25 distinct supplementary-plane codepoints, each appearing N times so - // we can verify the cap drops the smallest counts. + // 25 distinct escape-exempt codepoints (plane-14 variation selectors β€” + // astral, so guaranteed atlas misses; exempt, so guaranteed drops rather + // than [U+HEX] escapes), each appearing N times so we can verify the cap + // drops the smallest counts. let payload = 'x'.repeat(150000) + '\n'; for (let i = 0; i < 25; i++) { - // U+1F300..U+1F318 β€” 25 distinct codepoints, each occurring (25 - i) times - // so U+1F300 occurs 25 times, U+1F318 occurs 1 time. - payload += String.fromCodePoint(0x1f300 + i).repeat(25 - i); + // U+E0100..U+E0118 β€” 25 distinct codepoints, each occurring (25 - i) times + // so U+E0100 occurs 25 times, U+E0118 occurs 1 time. + payload += String.fromCodePoint(0xe0100 + i).repeat(25 - i); } const req = JSON.stringify({ model: 'claude-3-5-sonnet', @@ -1900,11 +1913,11 @@ describe('transform', () => { // The 5 smallest-count codepoints (last in the input) must be dropped // from the top-20. for (let i = 20; i < 25; i++) { - const hex = (0x1f300 + i).toString(16).toUpperCase().padStart(4, '0'); + const hex = (0xe0100 + i).toString(16).toUpperCase().padStart(4, '0'); expect(top[`U+${hex}`]).toBeUndefined(); } // The top entry is the most-frequent. - expect(top['U+1F300']).toBe(25); + expect(top['U+E0100']).toBe(25); }); // --- Per-block break-even gate (URGENT slice, supersedes prior threshold tests) ---