From 27e60d0c17b0cfe58e85ce889d3a841a62efccbf Mon Sep 17 00:00:00 2001 From: Danil Silantyev Date: Sat, 4 Jul 2026 12:39:44 +0700 Subject: [PATCH] fix(export): apply include/exclude and size limit to --git untracked files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In --git mode, untracked files (git ls-files --others) were read straight to memory with only a binary check — unlike directory and single-file modes, which also apply --include/--exclude globs and the 1 MiB MAX_FILE_BYTES cap. So `pxpipe export --git --include '*.ts'` still slurped every untracked file regardless of the filter, and an untracked multi-MB file was read whole (a resource-safety hazard). - extract the shared gate into src/export-collect.ts: readExportTextFile() applies include/exclude → size → binary, in that order, and is import-safe (src/core/export.ts is contractually fs-free, so it can't live there) - route all three collection paths (walkDir, single-file target, --git untracked) through it; untracked skips now warn like explicit targets do - tests/export-collect.test.ts covers include, exclude, oversized, the size boundary, binary, and inaccessible Verified end-to-end: `export --git --include '*.ts'` in a temp repo excludes an untracked .md, skips an oversized .ts (with a warning), and keeps the .ts. --- src/export-collect.ts | 67 ++++++++++++++++++++++++++++++++ src/node.ts | 51 ++++++++----------------- tests/export-collect.test.ts | 74 ++++++++++++++++++++++++++++++++++++ 3 files changed, 157 insertions(+), 35 deletions(-) create mode 100644 src/export-collect.ts create mode 100644 tests/export-collect.test.ts diff --git a/src/export-collect.ts b/src/export-collect.ts new file mode 100644 index 0000000..d4bbc38 --- /dev/null +++ b/src/export-collect.ts @@ -0,0 +1,67 @@ +/** + * Shared file-reading gate for `pxpipe export`. Every collection mode + * (directory walk, explicit single-file target, and `--git` untracked files) + * must apply the SAME three checks — include/exclude globs, a max size, and a + * binary sniff — so the untracked path can no longer read gigabyte or filtered + * files that directory mode would have skipped. + * + * Kept in its own import-safe module (no top-level side effects, unlike + * src/node.ts which starts the server on import) so it can be unit-tested + * directly. src/core/export.ts is deliberately fs-free, so the fs-touching gate + * lives here rather than there. + */ + +import * as fs from 'node:fs'; +import { shouldIncludeFile } from './core/export.js'; + +/** Files larger than this are skipped by export (1 MiB). A single export bundle + * is meant to be paste-sized; a multi-MB file is never intentional context and + * reading it fully into memory is a resource-safety hazard. */ +export const MAX_FILE_BYTES = 1_000_000; + +/** Returns true if `buf` looks like binary (contains a null byte in the first 512 bytes). */ +export function looksLikeBinary(buf: Buffer): boolean { + const check = Math.min(buf.byteLength, 512); + for (let i = 0; i < check; i++) { + if (buf[i] === 0) return true; + } + return false; +} + +/** Outcome of trying to read one file for export. `excluded` is a normal filter + * hit (callers stay quiet); the other non-ok kinds are worth a warning. */ +export type ExportReadResult = + | { readonly kind: 'ok'; readonly content: string } + | { readonly kind: 'excluded' | 'oversized' | 'binary' | 'inaccessible' }; + +/** + * Read a single text file for export if it passes every gate, in this order: + * 1. include/exclude globs (relative path) + * 2. size <= MAX_FILE_BYTES + * 3. not binary + * Returns the utf8 content on success, or the reason it was skipped. Pure of + * logging so each caller can decide whether a skip is noteworthy. + */ +export function readExportTextFile( + fullPath: string, + relPath: string, + include: string[], + exclude: string[], +): ExportReadResult { + if (!shouldIncludeFile(relPath, include, exclude)) return { kind: 'excluded' }; + let stat: fs.Stats; + try { + stat = fs.statSync(fullPath); + } catch { + return { kind: 'inaccessible' }; + } + if (stat.size > MAX_FILE_BYTES) return { kind: 'oversized' }; + let buf: Buffer; + try { + buf = fs.readFileSync(fullPath); + } catch { + return { kind: 'inaccessible' }; + } + if (looksLikeBinary(buf)) return { kind: 'binary' }; + return { kind: 'ok', content: buf.toString('utf8') }; +} diff --git a/src/node.ts b/src/node.ts index 3ccba4c..b29c02b 100644 --- a/src/node.ts +++ b/src/node.ts @@ -16,10 +16,10 @@ import { createProxy, parseGatewayHeaders, resolveUpstreams, type ProxyConfig } import { parseExportArgv, runExportCore, - shouldIncludeFile, type ExportParsed, type ExportResult, } from './core/export.js'; +import { readExportTextFile } from './export-collect.js'; import { toTrackEvent, TRACK_BODY_INLINE_MAX, @@ -630,17 +630,6 @@ const WALK_SKIP_DIRS = new Set([ '__pycache__', '.cache', '.next', '.nuxt', '.turbo', ]); -const MAX_FILE_BYTES = 1_000_000; // 1 MiB - -/** Returns true if `buf` looks like binary (contains a null byte in the first 512 bytes). */ -function looksLikeBinary(buf: Buffer): boolean { - const check = Math.min(buf.byteLength, 512); - for (let i = 0; i < check; i++) { - if (buf[i] === 0) return true; - } - return false; -} - interface CollectedFile { relPath: string; content: string; @@ -667,14 +656,10 @@ function walkDir( if (WALK_SKIP_DIRS.has(entry.name)) continue; walkDir(full, rootDir, include, exclude, out); } else if (entry.isFile()) { - if (!shouldIncludeFile(rel, include, exclude)) continue; - let stat: fs.Stats; - try { stat = fs.statSync(full); } catch { continue; } - if (stat.size > MAX_FILE_BYTES) continue; - let buf: Buffer; - try { buf = fs.readFileSync(full); } catch { continue; } - if (looksLikeBinary(buf)) continue; - out.push({ relPath: rel, content: buf.toString('utf8') }); + // Bulk directory walk: skip silently on any gate miss (per-file warnings + // would be noise across a whole tree). + const r = readExportTextFile(full, rel, include, exclude); + if (r.kind === 'ok') out.push({ relPath: rel, content: r.content }); } } } @@ -696,18 +681,11 @@ function collectFilesFromTargets( walkDir(target, target, include, exclude, files); } else if (st.isFile()) { const rel = path.basename(target); - if (!shouldIncludeFile(rel, include, exclude)) continue; - if (st.size > MAX_FILE_BYTES) { - console.warn(`[pxpipe export] skipping oversized file: ${target}`); - continue; + const r = readExportTextFile(target, rel, include, exclude); + if (r.kind === 'ok') files.push({ relPath: rel, content: r.content }); + else if (r.kind !== 'excluded') { + console.warn(`[pxpipe export] skipping ${r.kind} file: ${target}`); } - let buf: Buffer; - try { buf = fs.readFileSync(target); } catch { continue; } - if (looksLikeBinary(buf)) { - console.warn(`[pxpipe export] skipping binary file: ${target}`); - continue; - } - files.push({ relPath: rel, content: buf.toString('utf8') }); } } return files; @@ -756,10 +734,13 @@ async function collectSource(opts: ExportParsed): Promise<[string, string[]]> { let untracked = ''; for (const rel of untrackedFiles) { const full = path.join(cwd, rel); - let buf: Buffer; - try { buf = fs.readFileSync(full); } catch { continue; } - if (!looksLikeBinary(buf)) { - untracked += `\n===== ${rel} =====\n` + buf.toString('utf8'); + // Same include/exclude + size + binary gate as directory mode. Untracked + // files previously bypassed all of it: --include/--exclude were ignored + // and an oversized file was read fully into memory. + const r = readExportTextFile(full, rel, opts.include, opts.exclude); + if (r.kind === 'ok') untracked += `\n===== ${rel} =====\n` + r.content; + else if (r.kind !== 'excluded') { + console.warn(`[pxpipe export] skipping ${r.kind} untracked file: ${rel}`); } } const sourceText = diff + untracked; diff --git a/tests/export-collect.test.ts b/tests/export-collect.test.ts new file mode 100644 index 0000000..3a0de42 --- /dev/null +++ b/tests/export-collect.test.ts @@ -0,0 +1,74 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as os from 'node:os'; +import { readExportTextFile, looksLikeBinary, MAX_FILE_BYTES } from '../src/export-collect.js'; + +// readExportTextFile is the single gate now shared by every `pxpipe export` +// collection mode (directory walk, single-file target, and --git untracked). +// Before this, the --git untracked path applied none of these checks — it +// ignored --include/--exclude and read files of any size fully into memory. +describe('readExportTextFile — shared export file gate', () => { + let tmpDir: string; + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pxpipe-collect-test-')); + }); + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + const write = (name: string, content: string | Buffer): string => { + const p = path.join(tmpDir, name); + fs.writeFileSync(p, content); + return p; + }; + + it('reads an included text file as utf8', () => { + const p = write('a.ts', 'export const x = 1;\n'); + const r = readExportTextFile(p, 'a.ts', [], []); + expect(r.kind).toBe('ok'); + if (r.kind === 'ok') expect(r.content).toBe('export const x = 1;\n'); + }); + + it('respects --include: a .md is excluded when include is *.ts, the .ts sibling passes', () => { + const md = write('b.md', '# readme\n'); + const ts = write('a.ts', 'const a = 1;\n'); + expect(readExportTextFile(md, 'b.md', ['*.ts'], []).kind).toBe('excluded'); + expect(readExportTextFile(ts, 'a.ts', ['*.ts'], []).kind).toBe('ok'); + }); + + it('respects --exclude', () => { + const p = write('secret.ts', 'const s = 1;\n'); + expect(readExportTextFile(p, 'secret.ts', [], ['secret.*']).kind).toBe('excluded'); + }); + + it('skips an oversized file instead of reading it into memory', () => { + const p = write('big.ts', 'a'.repeat(MAX_FILE_BYTES + 1)); + expect(readExportTextFile(p, 'big.ts', [], []).kind).toBe('oversized'); + }); + + it('accepts a file exactly at the size limit', () => { + const p = write('edge.ts', 'a'.repeat(MAX_FILE_BYTES)); + expect(readExportTextFile(p, 'edge.ts', [], []).kind).toBe('ok'); + }); + + it('skips a binary file (null byte in first 512 bytes)', () => { + const p = write('bin.dat', Buffer.from([0x41, 0x00, 0x42])); + expect(readExportTextFile(p, 'bin.dat', [], []).kind).toBe('binary'); + }); + + it('reports a missing/inaccessible file', () => { + expect(readExportTextFile(path.join(tmpDir, 'nope.ts'), 'nope.ts', [], []).kind).toBe('inaccessible'); + }); + + it('applies the include/exclude filter before touching the filesystem', () => { + // A path that does not exist but is filtered out reports 'excluded', not + // 'inaccessible' — the glob gate short-circuits before any stat. + expect(readExportTextFile(path.join(tmpDir, 'ghost.md'), 'ghost.md', ['*.ts'], []).kind).toBe('excluded'); + }); + + it('looksLikeBinary flags a null byte but passes plain text', () => { + expect(looksLikeBinary(Buffer.from('plain text'))).toBe(false); + expect(looksLikeBinary(Buffer.from([0x00]))).toBe(true); + }); +});