mirror of
https://github.com/teamchong/pxpipe.git
synced 2026-07-22 02:02:51 +02:00
This commit is contained in:
Michael Coelho
+57
-2
@@ -445,6 +445,61 @@ export function dereflow(reflowed: string): string {
|
||||
return reflowed.split(NL_SENTINEL).join('\n');
|
||||
}
|
||||
|
||||
// --- atlas-miss escaping -----------------------------------------------------
|
||||
//
|
||||
// Codepoints absent from the atlas used to render as blank cells (droppedChars
|
||||
// telemetry). Emoji and other astral glyphs can't be DRAWN at 5×8 px, but they
|
||||
// can be PRESERVED: substitute a deterministic textual escape (`[U+HEX]`) before
|
||||
// wrap, so the model reads the codepoint instead of a gap. Applied at the same layer
|
||||
// as tab expansion (wrapLines + measureContentCols), so wrap math and canvas
|
||||
// measurement stay consistent. Pure string→string ⇒ renders stay deterministic.
|
||||
|
||||
/** Pure-ASCII delimiters: `🔥` → `[U+1F525]`. ASCII is the one range guaranteed
|
||||
* present in BOTH atlases (mathematical white brackets U+27E6/7 were tried first
|
||||
* and are absent from each — the "full-bmp" note on NL_SENTINEL overstates real
|
||||
* coverage). ASCII output also makes idempotency structural: an escaped line
|
||||
* contains no atlas misses, so a second pass is a no-op. `[U+…]` can collide
|
||||
* with literal source text discussing codepoints; the escape is for model
|
||||
* legibility, not machine round-trip, so the ambiguity is acceptable. */
|
||||
export const GLYPH_ESCAPE_OPEN = '[U+';
|
||||
export const GLYPH_ESCAPE_CLOSE = ']';
|
||||
|
||||
/** Codepoints that stay DROPPED (blank cell) rather than escaped. Escaping these
|
||||
* would add noise, not information — they're modifiers/invisibles, not content.
|
||||
* C0 additionally MUST keep width-1: the parallel slot string carries \x01/\x02
|
||||
* role markers and \x03 neutral (see SLOT_MARK_*), and escaping them would
|
||||
* desync slot/text alignment and smear role hues. */
|
||||
function isEscapeExempt(cp: number): boolean {
|
||||
if (cp < 0x20) return true; // C0 controls (slot markers; \t is expanded before we run)
|
||||
if (cp >= 0x7f && cp <= 0x9f) return true; // DEL + C1 controls
|
||||
if (cp >= 0x0300 && cp <= 0x036f) return true; // combining diacritics
|
||||
if (cp === 0x200b || cp === 0x200c || cp === 0x200d || cp === 0x2060 || cp === 0xfeff)
|
||||
return true; // zero-width / word-joiner / BOM
|
||||
if (cp >= 0xfe00 && cp <= 0xfe0f) return true; // variation selectors (emoji presentation)
|
||||
if (cp >= 0xe0100 && cp <= 0xe01ef) return true; // variation selectors supplement
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Replace atlas-missing codepoints with `[U+HEX]` (uppercase hex — e.g.
|
||||
* 🔥 → `[U+1F525]`). Lossless for non-exempt misses (hex → codepoint) and
|
||||
* idempotent: the escape spells only atlas-present chars, so a second pass is
|
||||
* a no-op. Fast path allocates nothing when every codepoint is in the atlas. */
|
||||
export function escapeMissingGlyphs(line: string): string {
|
||||
let out: string | null = null; // lazily materialized on first miss
|
||||
let i = 0;
|
||||
for (const ch of line) {
|
||||
const cp = ch.codePointAt(0)!;
|
||||
if (atlasRank(cp) < 0 && !isEscapeExempt(cp)) {
|
||||
if (out === null) out = line.slice(0, i);
|
||||
out += GLYPH_ESCAPE_OPEN + cp.toString(16).toUpperCase() + GLYPH_ESCAPE_CLOSE;
|
||||
} else if (out !== null) {
|
||||
out += ch;
|
||||
}
|
||||
i += ch.length;
|
||||
}
|
||||
return out ?? line;
|
||||
}
|
||||
|
||||
/** Expand \t to U+2192 → + padding to the next TAB_WIDTH stop. Visible marker lets the
|
||||
* model distinguish indent-spaces from intentional-spaces. Wide CJK chars count as 2 cols.
|
||||
* U+0009 is absent from the atlas (control codepoint), so without this every tab was a drop. */
|
||||
@@ -512,7 +567,7 @@ export function measureContentCols(
|
||||
let start = 0;
|
||||
for (let i = 0; i <= text.length; i++) {
|
||||
if (i === text.length || text[i] === '\n') {
|
||||
const w = measureLineCols(expandTabsInLine(text.slice(start, i)), markerScale, font);
|
||||
const w = measureLineCols(escapeMissingGlyphs(expandTabsInLine(text.slice(start, i))), markerScale, font);
|
||||
if (w > widest) widest = w;
|
||||
if (widest >= cap) return cap;
|
||||
start = i + 1;
|
||||
@@ -530,7 +585,7 @@ export function wrapLines(
|
||||
const out: string[] = [];
|
||||
const minified = minifyForRender(text);
|
||||
for (const rawWithTabs of minified.split('\n')) {
|
||||
const raw = expandTabsInLine(rawWithTabs);
|
||||
const raw = escapeMissingGlyphs(expandTabsInLine(rawWithTabs));
|
||||
if (raw.length === 0) {
|
||||
out.push('');
|
||||
continue;
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
/**
|
||||
* tests/glyph-escape.test.ts
|
||||
*
|
||||
* Atlas-miss escaping: emoji and other unrenderable codepoints are preserved
|
||||
* as ASCII `[U+HEX]` escapes instead of blank cells.
|
||||
*
|
||||
* CONTRACT:
|
||||
* • escapeMissingGlyphs is pure, idempotent, and lossless for non-exempt
|
||||
* misses (hex payload → codepoint)
|
||||
* • rendering text containing emoji yields droppedChars === 0 on both the
|
||||
* 1-bit and the AA (production dense) paths
|
||||
* • exempt codepoints (C0 slot markers, variation selectors, ZWJ, zero-width)
|
||||
* are left untouched — still dropped — so slot-string width alignment and
|
||||
* role coloring are unchanged
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
escapeMissingGlyphs,
|
||||
GLYPH_ESCAPE_OPEN,
|
||||
GLYPH_ESCAPE_CLOSE,
|
||||
renderChunkToPng,
|
||||
wrapLines,
|
||||
measureContentCols,
|
||||
DENSE_RENDER_STYLE,
|
||||
SLOT_MARK_USER,
|
||||
SLOT_MARK_ASSISTANT,
|
||||
} from '../src/core/render.js';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. escapeMissingGlyphs unit contract
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('escapeMissingGlyphs', () => {
|
||||
it('escapes an astral emoji as [U+HEX]', () => {
|
||||
expect(escapeMissingGlyphs('a\u{1F525}b')).toBe(
|
||||
`a${GLYPH_ESCAPE_OPEN}1F525${GLYPH_ESCAPE_CLOSE}b`,
|
||||
);
|
||||
});
|
||||
|
||||
it('fast path: returns the SAME reference when nothing misses', () => {
|
||||
const s = 'plain ascii — no atlas misses, incl. ↵ and CJK 漢字';
|
||||
expect(escapeMissingGlyphs(s)).toBe(s);
|
||||
});
|
||||
|
||||
it('is idempotent (escape output contains only atlas-present chars)', () => {
|
||||
const once = escapeMissingGlyphs('\u{1F4CA} chart \u{1F680}');
|
||||
expect(escapeMissingGlyphs(once)).toBe(once);
|
||||
});
|
||||
|
||||
it('is lossless: hex payload round-trips to the source codepoint', () => {
|
||||
const escaped = escapeMissingGlyphs('\u{1F680}');
|
||||
const m = /\[U\+([0-9A-F]+)\]/.exec(escaped);
|
||||
expect(m).not.toBeNull();
|
||||
expect(String.fromCodePoint(parseInt(m![1]!, 16))).toBe('\u{1F680}');
|
||||
});
|
||||
|
||||
it('escapes every miss in a multi-emoji line, preserving order', () => {
|
||||
expect(escapeMissingGlyphs('\u{1F525}x\u{1F680}')).toBe(
|
||||
`${GLYPH_ESCAPE_OPEN}1F525${GLYPH_ESCAPE_CLOSE}x${GLYPH_ESCAPE_OPEN}1F680${GLYPH_ESCAPE_CLOSE}`,
|
||||
);
|
||||
});
|
||||
|
||||
it('leaves exempt codepoints untouched: VS16, ZWJ, C0 slot markers', () => {
|
||||
const s = '️' + SLOT_MARK_USER + SLOT_MARK_ASSISTANT + '̂';
|
||||
expect(escapeMissingGlyphs(s)).toBe(s);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 2. Render integration: emoji no longer drop
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('rendering emoji', () => {
|
||||
it('1-bit path: droppedChars === 0 and escape survives into wrapped lines', async () => {
|
||||
const text = 'deploy \u{1F680} done, fire \u{1F525} out';
|
||||
const img = await renderChunkToPng(text, 80);
|
||||
expect(img.droppedChars).toBe(0);
|
||||
expect(img.droppedCodepoints.size).toBe(0);
|
||||
expect(wrapLines(text, 80).join('')).toContain(
|
||||
`${GLYPH_ESCAPE_OPEN}1F680${GLYPH_ESCAPE_CLOSE}`,
|
||||
);
|
||||
});
|
||||
|
||||
it('AA dense path (production DENSE_RENDER_STYLE): droppedChars === 0', async () => {
|
||||
const img = await renderChunkToPng(
|
||||
'metrics \u{1F4CA} look good \u{1F389}',
|
||||
80,
|
||||
DENSE_RENDER_STYLE,
|
||||
);
|
||||
expect(img.droppedChars).toBe(0);
|
||||
});
|
||||
|
||||
it('exempt invisibles still count as drops (behavior unchanged)', async () => {
|
||||
const img = await renderChunkToPng('x️y', 80);
|
||||
expect(img.droppedCodepoints.get(0xfe0f)).toBe(1);
|
||||
});
|
||||
|
||||
it('measureContentCols sees the ESCAPED width, matching what wrapLines lays out', () => {
|
||||
const text = '\u{1F525}'; // 1 source cell naively; 9 cells escaped
|
||||
const measured = measureContentCols(text, 312);
|
||||
const laidOut = wrapLines(text, 312)[0]!.length;
|
||||
expect(measured).toBe(laidOut);
|
||||
expect(measured).toBe(9); // [U+1F525]
|
||||
});
|
||||
|
||||
it('slot-string alignment: body slot copy wraps identically to text', () => {
|
||||
// Slot bodies are verbatim codepoint copies of the text body (slotCopyBody),
|
||||
// so both sides pass through the same escape and wrap to identical shapes.
|
||||
const body = 'alert \u{1F6A8} now, and again \u{1F6A8} later';
|
||||
expect(wrapLines(body, 10)).toEqual(wrapLines(body, 10));
|
||||
// Marker chars themselves are exempt → width-1 on the slot side is preserved.
|
||||
expect(escapeMissingGlyphs(SLOT_MARK_USER).length).toBe(1);
|
||||
});
|
||||
});
|
||||
+47
-34
@@ -364,17 +364,26 @@ describe('renderer', () => {
|
||||
expect(img.droppedChars).toBe(0);
|
||||
});
|
||||
|
||||
it('treats codepoints outside the atlas as dropped (e.g. emoji)', async () => {
|
||||
it('escapes codepoints outside the atlas instead of dropping them (e.g. emoji)', async () => {
|
||||
// 😀 is U+1F600 — Supplementary Plane, not in BMP. Even `full-bmp` profile
|
||||
// wouldn't cover it. Renderer must advance by 1 cell and bump the counter,
|
||||
// not crash on the surrogate pair.
|
||||
// wouldn't cover it. It renders as the lossless ASCII escape [U+1F600]
|
||||
// (see escapeMissingGlyphs), not as a blank cell — and must not crash on
|
||||
// the surrogate pair.
|
||||
const img = await renderChunkToPng('hi 😀 world');
|
||||
expect(img.droppedChars).toBe(1);
|
||||
// charsRendered counts codepoints, NOT UTF-16 units — the emoji is one
|
||||
// codepoint even though it occupies two UTF-16 units.
|
||||
expect(img.droppedChars).toBe(0);
|
||||
// charsRendered counts SOURCE codepoints, NOT UTF-16 units — the emoji is
|
||||
// one codepoint even though it occupies two UTF-16 units.
|
||||
expect(img.charsRendered).toBe(10); // 'hi ' (3) + 😀 (1) + ' world' (6) = 10
|
||||
});
|
||||
|
||||
it('treats escape-exempt invisibles as dropped with 1-cell advance (e.g. VS16)', async () => {
|
||||
// U+FE0F (variation selector-16) is deliberately NOT escaped — it's an
|
||||
// emoji presentation modifier, noise if spelled out. Renderer must advance
|
||||
// by 1 cell and bump the counter.
|
||||
const img = await renderChunkToPng('hi ️ world');
|
||||
expect(img.droppedChars).toBe(1);
|
||||
});
|
||||
|
||||
it('CJK characters advance two cells; mixed lines wrap correctly', async () => {
|
||||
// 100 cols, mixed Latin + CJK. 30 Latin chars + 40 CJK chars = 30 + 80 =
|
||||
// 110 visual columns → must wrap to 2 lines.
|
||||
@@ -497,21 +506,21 @@ describe('renderer', () => {
|
||||
});
|
||||
|
||||
it('droppedCodepoints map is populated correctly when drops occur', async () => {
|
||||
// 😀 is supplementary-plane (not in atlas regardless of profile). The
|
||||
// codepoint should appear in the map with count 1; charsRendered counts
|
||||
// it as a single codepoint.
|
||||
const img = await renderChunkToPng('hi 😀 there');
|
||||
// Emoji now escape to [U+HEX] instead of dropping, so the drop path is
|
||||
// exercised via an escape-EXEMPT codepoint: U+FE0F (variation selector).
|
||||
// The codepoint should appear in the map with count 1.
|
||||
const img = await renderChunkToPng('hi ️ there');
|
||||
expect(img.droppedChars).toBe(1);
|
||||
expect(img.droppedCodepoints.size).toBe(1);
|
||||
expect(img.droppedCodepoints.get(0x1f600)).toBe(1);
|
||||
expect(img.droppedCodepoints.get(0xfe0f)).toBe(1);
|
||||
});
|
||||
|
||||
it('droppedCodepoints tallies repeat drops correctly', async () => {
|
||||
// Three occurrences of the same dropped codepoint → count 3.
|
||||
const img = await renderChunkToPng('😀😀😀');
|
||||
// Three occurrences of the same dropped (exempt) codepoint → count 3.
|
||||
const img = await renderChunkToPng('a️b️c️');
|
||||
expect(img.droppedChars).toBe(3);
|
||||
expect(img.droppedCodepoints.size).toBe(1);
|
||||
expect(img.droppedCodepoints.get(0x1f600)).toBe(3);
|
||||
expect(img.droppedCodepoints.get(0xfe0f)).toBe(3);
|
||||
});
|
||||
|
||||
// --- Whitespace minify (HANDOFF R1) ---------------------------------------
|
||||
@@ -1832,17 +1841,19 @@ describe('transform', () => {
|
||||
// capture & inspect the request body.
|
||||
|
||||
it('populates droppedCodepointsTop when drops occur, sorted by count', async () => {
|
||||
// System slab forces compression. The slab contains drops for two distinct
|
||||
// supplementary-plane codepoints at different rates so we can verify the
|
||||
// sort order.
|
||||
const cpA = String.fromCodePoint(0x1f600); // 😀
|
||||
const cpB = String.fromCodePoint(0x1f604); // 😄
|
||||
const cpC = String.fromCodePoint(0x1f60a); // 😊
|
||||
// System slab forces compression. Emoji now escape to [U+HEX] instead of
|
||||
// dropping, so the drop path is exercised via escape-EXEMPT codepoints:
|
||||
// plane-14 variation selectors (U+E01xx — astral, guaranteed absent from
|
||||
// the BMP atlas, and deliberately never escaped). Three distinct
|
||||
// codepoints at different rates so we can verify the sort order.
|
||||
const cpA = String.fromCodePoint(0xe0100);
|
||||
const cpB = String.fromCodePoint(0xe0104);
|
||||
const cpC = String.fromCodePoint(0xe010a);
|
||||
const sys =
|
||||
'x'.repeat(150000) + // bulk to force compression
|
||||
'\n' + cpA.repeat(10) + // 10 drops of U+1F600
|
||||
'\n' + cpB.repeat(3) + // 3 drops of U+1F604
|
||||
'\n' + cpC.repeat(1); // 1 drop of U+1F60A
|
||||
'\n' + cpA.repeat(10) + // 10 drops of U+E0100
|
||||
'\n' + cpB.repeat(3) + // 3 drops of U+E0104
|
||||
'\n' + cpC.repeat(1); // 1 drop of U+E010A
|
||||
const req = JSON.stringify({
|
||||
model: 'claude-3-5-sonnet',
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
@@ -1853,9 +1864,9 @@ describe('transform', () => {
|
||||
expect(info.droppedChars).toBeGreaterThanOrEqual(14);
|
||||
expect(info.droppedCodepointsTop).toBeDefined();
|
||||
const top = info.droppedCodepointsTop!;
|
||||
expect(top['U+1F600']).toBe(10);
|
||||
expect(top['U+1F604']).toBe(3);
|
||||
expect(top['U+1F60A']).toBe(1);
|
||||
expect(top['U+E0100']).toBe(10);
|
||||
expect(top['U+E0104']).toBe(3);
|
||||
expect(top['U+E010A']).toBe(1);
|
||||
// Ensure key format is the expected U+HHHH uppercase with no surprises.
|
||||
for (const k of Object.keys(top)) {
|
||||
expect(k).toMatch(/^U\+[0-9A-F]{4,}$/);
|
||||
@@ -1863,7 +1874,7 @@ describe('transform', () => {
|
||||
// Sorted by count desc: iteration of object keys preserves insertion order
|
||||
// in V8/JSC, so the first key is the highest-count drop.
|
||||
const keys = Object.keys(top);
|
||||
expect(keys[0]).toBe('U+1F600');
|
||||
expect(keys[0]).toBe('U+E0100');
|
||||
});
|
||||
|
||||
it('omits droppedCodepointsTop entirely when no drops occur', async () => {
|
||||
@@ -1880,13 +1891,15 @@ describe('transform', () => {
|
||||
});
|
||||
|
||||
it('caps droppedCodepointsTop at 20 entries', async () => {
|
||||
// 25 distinct supplementary-plane codepoints, each appearing N times so
|
||||
// we can verify the cap drops the smallest counts.
|
||||
// 25 distinct escape-exempt codepoints (plane-14 variation selectors —
|
||||
// astral, so guaranteed atlas misses; exempt, so guaranteed drops rather
|
||||
// than [U+HEX] escapes), each appearing N times so we can verify the cap
|
||||
// drops the smallest counts.
|
||||
let payload = 'x'.repeat(150000) + '\n';
|
||||
for (let i = 0; i < 25; i++) {
|
||||
// U+1F300..U+1F318 — 25 distinct codepoints, each occurring (25 - i) times
|
||||
// so U+1F300 occurs 25 times, U+1F318 occurs 1 time.
|
||||
payload += String.fromCodePoint(0x1f300 + i).repeat(25 - i);
|
||||
// U+E0100..U+E0118 — 25 distinct codepoints, each occurring (25 - i) times
|
||||
// so U+E0100 occurs 25 times, U+E0118 occurs 1 time.
|
||||
payload += String.fromCodePoint(0xe0100 + i).repeat(25 - i);
|
||||
}
|
||||
const req = JSON.stringify({
|
||||
model: 'claude-3-5-sonnet',
|
||||
@@ -1900,11 +1913,11 @@ describe('transform', () => {
|
||||
// The 5 smallest-count codepoints (last in the input) must be dropped
|
||||
// from the top-20.
|
||||
for (let i = 20; i < 25; i++) {
|
||||
const hex = (0x1f300 + i).toString(16).toUpperCase().padStart(4, '0');
|
||||
const hex = (0xe0100 + i).toString(16).toUpperCase().padStart(4, '0');
|
||||
expect(top[`U+${hex}`]).toBeUndefined();
|
||||
}
|
||||
// The top entry is the most-frequent.
|
||||
expect(top['U+1F300']).toBe(25);
|
||||
expect(top['U+E0100']).toBe(25);
|
||||
});
|
||||
|
||||
// --- Per-block break-even gate (URGENT slice, supersedes prior threshold tests) ---
|
||||
|
||||
Reference in New Issue
Block a user