From a346ec7ac5743c8f0333170e5c4feaf47b887d10 Mon Sep 17 00:00:00 2001 From: teamchong <25894545+teamchong@users.noreply.github.com> Date: Wed, 1 Jul 2026 16:41:00 -0400 Subject: [PATCH] feat(factsheet): carry occurrence counts, match ticket-style codes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - extractFactSheetEntries() returns {token, count}; the kept-token set and order are byte-identical to the old extractFactSheetTokens(), which now delegates. Counts render as a ×N suffix, so tally questions over imaged content are answerable from the text sheet instead of by counting 5x8 px glyph rows. - Same-token spans matched by overlapping patterns dedupe by offset (token + match index) so nothing double-counts. - New tier-0 shape: uppercase hyphenated codes with a digit (PROJ-1482, CVE-2024-30078); digit lookahead is bounded. - export.ts consumes entries across all pages; 14 new/extended tests. --- src/core/export.ts | 7 ++-- src/core/factsheet.ts | 83 +++++++++++++++++++++++++++++++++-------- tests/factsheet.test.ts | 65 +++++++++++++++++++++++++++++++- 3 files changed, 136 insertions(+), 19 deletions(-) diff --git a/src/core/export.ts b/src/core/export.ts index 1778b66..8ce1e6c 100644 --- a/src/core/export.ts +++ b/src/core/export.ts @@ -21,8 +21,9 @@ import { renderTextToImages } from './library.js'; import { estimateImageCount, ANTHROPIC_PIXELS_PER_TOKEN, IMAGE_COST_SAFETY_MARGIN, REPORT_CHARS_PER_TOKEN } from './transform.js'; import { openAIVisionTokens } from './openai.js'; import { - factSheetTextFromTokens, + factSheetTextFromEntries, extractFactSheetTokensAllPages, + extractFactSheetEntriesAllPages, } from './factsheet.js'; // --------------------------------------------------------------------------- @@ -450,11 +451,11 @@ export async function runExportCore( : 0; // Extract factsheet tokens across ALL pages so identifiers from page 3+ are covered. - const { kept: fsKept, dropped: fsDropped } = extractFactSheetTokensAllPages( + const { kept: fsKept, dropped: fsDropped } = extractFactSheetEntriesAllPages( sourceText, DENSE_CONTENT_CHARS_PER_IMAGE, ); - const fsText = factSheetTextFromTokens(fsKept); + const fsText = factSheetTextFromEntries(fsKept); const tokenReport: ExportTokenReport = { textTokens, imageTokens, diff --git a/src/core/factsheet.ts b/src/core/factsheet.ts index 02cb806..9a09508 100644 --- a/src/core/factsheet.ts +++ b/src/core/factsheet.ts @@ -27,6 +27,9 @@ const PATTERNS: readonly RegExp[] = [ /\b\d[\d,_]{3,}\b/g, // large / separated number /\b\d+\.\d+\b/g, // decimal /\b[A-Z][A-Z0-9]{2,}(?:_[A-Z0-9]+)+\b/g, // CONST_IDS / env var names + // Ticket/advisory-style codes: uppercase hyphenated with ≥1 digit (PROJ-1482, + // CVE-2024-30078, AUDIT-ZX9). Digit lookahead is bounded → no backtracking blowup. + /\b(?=[A-Z0-9-]{0,119}\d)[A-Z][A-Z0-9]+(?:-[A-Z0-9]+)+\b/g, ]; const MIN_LEN = 3; @@ -48,6 +51,7 @@ const MAX_CHUNK = 512; // whitespace-free chunks longer than this are blobs (bas const SHAPE_UUID = /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$/; const SHAPE_HEX = /^(?=[0-9a-f]*\d)[0-9a-f]{7,40}$/; // git sha / opaque hex const SHAPE_CONST = /^[A-Z][A-Z0-9]{2,}(?:_[A-Z0-9]+)+$/; // CONST_IDS / env vars +const SHAPE_TICKET = /^(?=[A-Z0-9-]*\d)[A-Z][A-Z0-9]+(?:-[A-Z0-9]+)+$/; // PROJ-1482 / CVE-2024-30078 const SHAPE_FLAG = /^--?[A-Za-z][\w-]+$/; // CLI flag const SHAPE_NUM = /^\d[\d,_]*$|^\d+\.\d+$/; // port / large or separated number / decimal const SHAPE_URL = /^https?:\/\//; @@ -58,6 +62,7 @@ function priorityTier(tok: string): 0 | 1 | 2 { SHAPE_HEX.test(tok) || SHAPE_UUID.test(tok) || SHAPE_CONST.test(tok) || + SHAPE_TICKET.test(tok) || SHAPE_FLAG.test(tok) || SHAPE_NUM.test(tok) ) { @@ -80,24 +85,51 @@ function priorityTier(tok: string): 0 | 1 | 2 { * minified bundles (which embed `/` and would otherwise make the path patterns blow up). */ export function extractFactSheetTokens(text: string): string[] { + return extractFactSheetEntries(text).map((e) => e.token); +} + +/** A kept fact-sheet token plus how many times it occurs in the scanned text. + * Counts are advisory (occurrences, not lines) but deterministic → cache-stable. */ +export interface FactSheetEntry { + readonly token: string; + readonly count: number; +} + +/** + * Like `extractFactSheetTokens`, but each kept token carries its occurrence count. + * Counts make the fact sheet a *quantitative* index: tally questions over imaged + * content ("how many lines mention CODE-X?") become answerable from text instead + * of from counting rows of 5×8 px glyphs — the one operation page images are worst + * at. The kept-token SET and its order are byte-identical to the pre-count + * behaviour; only counts are new. Same-token spans matched by two patterns are + * deduped by offset so a token is never double-counted. + */ +export function extractFactSheetEntries(text: string): FactSheetEntry[] { const scan = text.length > MAX_SCAN ? text.slice(0, MAX_SCAN) : text; - const seen = new Set(); + const counts = new Map(); for (const chunk of scan.split(/\s+/)) { if (chunk.length < MIN_LEN || chunk.length > MAX_CHUNK) continue; + // Offset-level dedup WITHIN this chunk: two patterns matching the identical + // token at the same position must count once. Keyed by token+start offset. + const spanSeen = new Set(); for (const re of PATTERNS) { for (const m of chunk.matchAll(re)) { // Strip trailing sentence punctuation pulled in from prose (`pull/93.` → `pull/93`); // no real identifier we extract ends in these. const tok = (m[1] ?? m[0]).trim().replace(/[.,;:!?]+$/, ''); - if (tok.length >= MIN_LEN && tok.length <= MAX_LEN) seen.add(tok); + if (tok.length < MIN_LEN || tok.length > MAX_LEN) continue; + const key = `${m.index ?? 0}\x00${tok}`; + if (spanSeen.has(key)) continue; + spanSeen.add(key); + counts.set(tok, (counts.get(tok) ?? 0) + 1); } } - if (seen.size >= MAX_SEEN) break; + if (counts.size >= MAX_SEEN) break; } // Phase 1 — substring collapse (length-desc): keep the most-specific form, folding e.g. // a URL's path-portion into the full URL. Cross-tier on purpose. Total order (length, // then lexical) so the result is independent of Set iteration order. - const ordered = [...seen].sort((a, b) => b.length - a.length || (a < b ? -1 : a > b ? 1 : 0)); + const ordered = [...counts.keys()].sort((a, b) => b.length - a.length || (a < b ? -1 : a > b ? 1 : 0)); const specific: string[] = []; for (const t of ordered) { if (!specific.some((k) => k.includes(t))) specific.push(t); @@ -115,7 +147,7 @@ export function extractFactSheetTokens(text: string): string[] { if (tier === 2 && urls++ >= MAX_URLS) continue; kept.push(t); } - return kept; + return kept.map((t) => ({ token: t, count: counts.get(t) ?? 1 })); } /** @@ -136,19 +168,27 @@ export function extractFactSheetTokensAllPages( text: string, charsPerPage: number, ): { kept: string[]; dropped: number } { - const seen = new Set(); + const { kept, dropped } = extractFactSheetEntriesAllPages(text, charsPerPage); + return { kept: kept.map((e) => e.token), dropped }; +} + +/** Entry-carrying variant of `extractFactSheetTokensAllPages`: same kept set and + * order, with per-token occurrence counts summed across all pages. */ +export function extractFactSheetEntriesAllPages( + text: string, + charsPerPage: number, +): { kept: FactSheetEntry[]; dropped: number } { + const counts = new Map(); const all: string[] = []; // Walk the text in page-sized chunks. Each chunk is ≤ charsPerPage < MAX_SCAN, - // so extractFactSheetTokens will not truncate within a chunk. + // so extractFactSheetEntries will not truncate within a chunk. const pageCount = Math.max(1, Math.ceil(text.length / charsPerPage)); for (let i = 0; i < pageCount; i++) { const chunk = text.slice(i * charsPerPage, (i + 1) * charsPerPage); - for (const tok of extractFactSheetTokens(chunk)) { - if (!seen.has(tok)) { - seen.add(tok); - all.push(tok); - } + for (const { token, count } of extractFactSheetEntries(chunk)) { + if (!counts.has(token)) all.push(token); + counts.set(token, (counts.get(token) ?? 0) + count); } } @@ -157,12 +197,12 @@ export function extractFactSheetTokensAllPages( const ranked = all .map((t) => ({ t, tier: priorityTier(t) })) .sort((a, b) => a.tier - b.tier || b.t.length - a.t.length || (a.t < b.t ? -1 : a.t > b.t ? 1 : 0)); - const kept: string[] = []; + const kept: FactSheetEntry[] = []; let urls = 0; for (const { t, tier } of ranked) { if (kept.length >= MAX_TOKENS) break; if (tier === 2 && urls++ >= MAX_URLS) continue; - kept.push(t); + kept.push({ token: t, count: counts.get(t) ?? 1 }); } return { kept, dropped: all.length - kept.length }; @@ -170,13 +210,26 @@ export function extractFactSheetTokensAllPages( const OPEN = '[Exact identifiers from the rendered context above (paths, ids, versions, numbers) — quote these verbatim instead of transcribing them from the image: '; +/** Variant used when at least one token repeats — explains the ×N annotation so the + * model can answer tally questions from the sheet instead of counting glyph rows. */ +const OPEN_COUNTS = + '[Exact identifiers from the rendered context above (paths, ids, versions, numbers) — quote these verbatim instead of transcribing them from the image; ×N marks a token that occurs N times within the imaged content: '; /** Build the one-line fact-sheet string from a pre-extracted token list. */ export function factSheetTextFromTokens(tokens: string[]): string { return tokens.length > 0 ? OPEN + tokens.join(' · ') + ']' : ''; } +/** Build the one-line fact-sheet string from token+count entries. Byte-identical to + * `factSheetTextFromTokens` when no token repeats, so existing sheets stay cache-stable. */ +export function factSheetTextFromEntries(entries: readonly FactSheetEntry[]): string { + if (entries.length === 0) return ''; + const anyRepeat = entries.some((e) => e.count >= 2); + const body = entries.map((e) => (e.count >= 2 ? `${e.token} ×${e.count}` : e.token)).join(' · '); + return (anyRepeat ? OPEN_COUNTS : OPEN) + body + ']'; +} + /** One-line fact-sheet string for `text`, or `''` when nothing notable was found. */ export function factSheetText(text: string): string { - return factSheetTextFromTokens(extractFactSheetTokens(text)); + return factSheetTextFromEntries(extractFactSheetEntries(text)); } diff --git a/tests/factsheet.test.ts b/tests/factsheet.test.ts index 792676c..4539733 100644 --- a/tests/factsheet.test.ts +++ b/tests/factsheet.test.ts @@ -1,5 +1,10 @@ import { describe, it, expect } from 'vitest'; -import { extractFactSheetTokens, factSheetText } from '../src/core/factsheet.js'; +import { + extractFactSheetTokens, + extractFactSheetEntries, + extractFactSheetEntriesAllPages, + factSheetText, +} from '../src/core/factsheet.js'; describe('factsheet extraction', () => { it('captures precision-critical, hard-to-OCR tokens', () => { @@ -57,3 +62,61 @@ describe('factsheet extraction', () => { expect(toks.filter((t) => t.startsWith('http')).length).toBeLessThanOrEqual(8); }); }); + +describe('ticket-style codes and occurrence counts', () => { + it('captures uppercase hyphenated codes that contain a digit', () => { + const toks = extractFactSheetTokens( + 'audit marker AUDIT-ZX9 tracked as PROJ-1482, see CVE-2024-30078 for details', + ); + expect(toks).toContain('AUDIT-ZX9'); + expect(toks).toContain('PROJ-1482'); + expect(toks).toContain('CVE-2024-30078'); + }); + + it('does not flag digit-free hyphenated prose (READ-ONLY, NON-NULL)', () => { + const toks = extractFactSheetTokens('column is READ-ONLY and NON-NULL by default'); + expect(toks).not.toContain('READ-ONLY'); + expect(toks).not.toContain('NON-NULL'); + }); + + it('annotates repeated tokens with ×N and explains the notation', () => { + const text = 'retry DEPLOY-77 failed\nretry DEPLOY-77 ok\nfinal DEPLOY-77 done\nsha 9d121ac'; + const sheet = factSheetText(text); + expect(sheet).toContain('DEPLOY-77 ×3'); + expect(sheet).toContain('×N marks a token that occurs N times'); + expect(sheet).not.toContain('9d121ac ×'); + }); + + it('emits byte-identical sheets to the pre-count format when nothing repeats', () => { + const text = 'commit 9d121ac on port 47821'; + expect(factSheetText(text)).toContain('from the image: '); + expect(factSheetText(text)).not.toContain('×'); + }); + + it('never double-counts one span matched by two patterns', () => { + // 1.2.3 is hit by the version pattern; its 1.2 substring by decimal — offset dedup + // plus substring-collapse must leave a single un-annotated v1.2.3-style entry. + const sheet = factSheetText('release v1.2.3 shipped'); + expect(sheet).not.toMatch(/×\d/); + }); + + it('keeps a rare ticket code over a flood of per-line hex ids (log-file shape)', () => { + const lines = Array.from({ length: 300 }, (_, i) => + `2026-07-26T09:40:41Z WARN svc=ingest req=${(0x10000000 + i * 7919).toString(16)} shard=12 msg=processed batch ${10000 + i} ok`, + ); + lines[137] += ' AUDIT-ZX9'; + lines[201] += ' AUDIT-ZX9'; + const entries = extractFactSheetEntries(lines.join('\n')); + const hit = entries.find((e) => e.token === 'AUDIT-ZX9'); + expect(hit).toBeDefined(); + expect(hit!.count).toBe(2); + }); + + it('sums counts across pages in the all-pages variant', () => { + const page = 'x'.repeat(90) + ' TICK-42 '; + const { kept } = extractFactSheetEntriesAllPages(page.repeat(5), 100); + const hit = kept.find((e) => e.token === 'TICK-42'); + expect(hit).toBeDefined(); + expect(hit!.count).toBe(5); + }); +});