fix(export): apply include/exclude and size limit to --git untracked files

In --git mode, untracked files (git ls-files --others) were read straight to
memory with only a binary check — unlike directory and single-file modes, which
also apply --include/--exclude globs and the 1 MiB MAX_FILE_BYTES cap. So
`pxpipe export --git --include '*.ts'` still slurped every untracked file
regardless of the filter, and an untracked multi-MB file was read whole (a
resource-safety hazard).

- extract the shared gate into src/export-collect.ts: readExportTextFile()
  applies include/exclude → size → binary, in that order, and is import-safe
  (src/core/export.ts is contractually fs-free, so it can't live there)
- route all three collection paths (walkDir, single-file target, --git
  untracked) through it; untracked skips now warn like explicit targets do
- tests/export-collect.test.ts covers include, exclude, oversized, the size
  boundary, binary, and inaccessible

Verified end-to-end: `export --git --include '*.ts'` in a temp repo excludes an
untracked .md, skips an oversized .ts (with a warning), and keeps the .ts.
This commit is contained in:
Danil Silantyev
2026-07-04 12:39:44 +07:00
committed by Steven
parent 69c9f2e83c
commit 27e60d0c17
3 changed files with 157 additions and 35 deletions
+67
View File
@@ -0,0 +1,67 @@
/**
* Shared file-reading gate for `pxpipe export`. Every collection mode
* (directory walk, explicit single-file target, and `--git` untracked files)
* must apply the SAME three checks — include/exclude globs, a max size, and a
* binary sniff — so the untracked path can no longer read gigabyte or filtered
* files that directory mode would have skipped.
*
* Kept in its own import-safe module (no top-level side effects, unlike
* src/node.ts which starts the server on import) so it can be unit-tested
* directly. src/core/export.ts is deliberately fs-free, so the fs-touching gate
* lives here rather than there.
*/
import * as fs from 'node:fs';
import { shouldIncludeFile } from './core/export.js';
/** Files larger than this are skipped by export (1 MiB). A single export bundle
* is meant to be paste-sized; a multi-MB file is never intentional context and
* reading it fully into memory is a resource-safety hazard. */
export const MAX_FILE_BYTES = 1_000_000;
/** Returns true if `buf` looks like binary (contains a null byte in the first 512 bytes). */
export function looksLikeBinary(buf: Buffer): boolean {
const check = Math.min(buf.byteLength, 512);
for (let i = 0; i < check; i++) {
if (buf[i] === 0) return true;
}
return false;
}
/** Outcome of trying to read one file for export. `excluded` is a normal filter
* hit (callers stay quiet); the other non-ok kinds are worth a warning. */
export type ExportReadResult =
| { readonly kind: 'ok'; readonly content: string }
| { readonly kind: 'excluded' | 'oversized' | 'binary' | 'inaccessible' };
/**
* Read a single text file for export if it passes every gate, in this order:
* 1. include/exclude globs (relative path)
* 2. size <= MAX_FILE_BYTES
* 3. not binary
* Returns the utf8 content on success, or the reason it was skipped. Pure of
* logging so each caller can decide whether a skip is noteworthy.
*/
export function readExportTextFile(
fullPath: string,
relPath: string,
include: string[],
exclude: string[],
): ExportReadResult {
if (!shouldIncludeFile(relPath, include, exclude)) return { kind: 'excluded' };
let stat: fs.Stats;
try {
stat = fs.statSync(fullPath);
} catch {
return { kind: 'inaccessible' };
}
if (stat.size > MAX_FILE_BYTES) return { kind: 'oversized' };
let buf: Buffer;
try {
buf = fs.readFileSync(fullPath);
} catch {
return { kind: 'inaccessible' };
}
if (looksLikeBinary(buf)) return { kind: 'binary' };
return { kind: 'ok', content: buf.toString('utf8') };
}
+16 -35
View File
@@ -16,10 +16,10 @@ import { createProxy, parseGatewayHeaders, resolveUpstreams, type ProxyConfig }
import {
parseExportArgv,
runExportCore,
shouldIncludeFile,
type ExportParsed,
type ExportResult,
} from './core/export.js';
import { readExportTextFile } from './export-collect.js';
import {
toTrackEvent,
TRACK_BODY_INLINE_MAX,
@@ -630,17 +630,6 @@ const WALK_SKIP_DIRS = new Set([
'__pycache__', '.cache', '.next', '.nuxt', '.turbo',
]);
const MAX_FILE_BYTES = 1_000_000; // 1 MiB
/** Returns true if `buf` looks like binary (contains a null byte in the first 512 bytes). */
function looksLikeBinary(buf: Buffer): boolean {
const check = Math.min(buf.byteLength, 512);
for (let i = 0; i < check; i++) {
if (buf[i] === 0) return true;
}
return false;
}
interface CollectedFile {
relPath: string;
content: string;
@@ -667,14 +656,10 @@ function walkDir(
if (WALK_SKIP_DIRS.has(entry.name)) continue;
walkDir(full, rootDir, include, exclude, out);
} else if (entry.isFile()) {
if (!shouldIncludeFile(rel, include, exclude)) continue;
let stat: fs.Stats;
try { stat = fs.statSync(full); } catch { continue; }
if (stat.size > MAX_FILE_BYTES) continue;
let buf: Buffer;
try { buf = fs.readFileSync(full); } catch { continue; }
if (looksLikeBinary(buf)) continue;
out.push({ relPath: rel, content: buf.toString('utf8') });
// Bulk directory walk: skip silently on any gate miss (per-file warnings
// would be noise across a whole tree).
const r = readExportTextFile(full, rel, include, exclude);
if (r.kind === 'ok') out.push({ relPath: rel, content: r.content });
}
}
}
@@ -696,18 +681,11 @@ function collectFilesFromTargets(
walkDir(target, target, include, exclude, files);
} else if (st.isFile()) {
const rel = path.basename(target);
if (!shouldIncludeFile(rel, include, exclude)) continue;
if (st.size > MAX_FILE_BYTES) {
console.warn(`[pxpipe export] skipping oversized file: ${target}`);
continue;
const r = readExportTextFile(target, rel, include, exclude);
if (r.kind === 'ok') files.push({ relPath: rel, content: r.content });
else if (r.kind !== 'excluded') {
console.warn(`[pxpipe export] skipping ${r.kind} file: ${target}`);
}
let buf: Buffer;
try { buf = fs.readFileSync(target); } catch { continue; }
if (looksLikeBinary(buf)) {
console.warn(`[pxpipe export] skipping binary file: ${target}`);
continue;
}
files.push({ relPath: rel, content: buf.toString('utf8') });
}
}
return files;
@@ -756,10 +734,13 @@ async function collectSource(opts: ExportParsed): Promise<[string, string[]]> {
let untracked = '';
for (const rel of untrackedFiles) {
const full = path.join(cwd, rel);
let buf: Buffer;
try { buf = fs.readFileSync(full); } catch { continue; }
if (!looksLikeBinary(buf)) {
untracked += `\n===== ${rel} =====\n` + buf.toString('utf8');
// Same include/exclude + size + binary gate as directory mode. Untracked
// files previously bypassed all of it: --include/--exclude were ignored
// and an oversized file was read fully into memory.
const r = readExportTextFile(full, rel, opts.include, opts.exclude);
if (r.kind === 'ok') untracked += `\n===== ${rel} =====\n` + r.content;
else if (r.kind !== 'excluded') {
console.warn(`[pxpipe export] skipping ${r.kind} untracked file: ${rel}`);
}
}
const sourceText = diff + untracked;
+74
View File
@@ -0,0 +1,74 @@
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import * as fs from 'node:fs';
import * as path from 'node:path';
import * as os from 'node:os';
import { readExportTextFile, looksLikeBinary, MAX_FILE_BYTES } from '../src/export-collect.js';
// readExportTextFile is the single gate now shared by every `pxpipe export`
// collection mode (directory walk, single-file target, and --git untracked).
// Before this, the --git untracked path applied none of these checks — it
// ignored --include/--exclude and read files of any size fully into memory.
describe('readExportTextFile — shared export file gate', () => {
let tmpDir: string;
beforeEach(() => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pxpipe-collect-test-'));
});
afterEach(() => {
fs.rmSync(tmpDir, { recursive: true, force: true });
});
const write = (name: string, content: string | Buffer): string => {
const p = path.join(tmpDir, name);
fs.writeFileSync(p, content);
return p;
};
it('reads an included text file as utf8', () => {
const p = write('a.ts', 'export const x = 1;\n');
const r = readExportTextFile(p, 'a.ts', [], []);
expect(r.kind).toBe('ok');
if (r.kind === 'ok') expect(r.content).toBe('export const x = 1;\n');
});
it('respects --include: a .md is excluded when include is *.ts, the .ts sibling passes', () => {
const md = write('b.md', '# readme\n');
const ts = write('a.ts', 'const a = 1;\n');
expect(readExportTextFile(md, 'b.md', ['*.ts'], []).kind).toBe('excluded');
expect(readExportTextFile(ts, 'a.ts', ['*.ts'], []).kind).toBe('ok');
});
it('respects --exclude', () => {
const p = write('secret.ts', 'const s = 1;\n');
expect(readExportTextFile(p, 'secret.ts', [], ['secret.*']).kind).toBe('excluded');
});
it('skips an oversized file instead of reading it into memory', () => {
const p = write('big.ts', 'a'.repeat(MAX_FILE_BYTES + 1));
expect(readExportTextFile(p, 'big.ts', [], []).kind).toBe('oversized');
});
it('accepts a file exactly at the size limit', () => {
const p = write('edge.ts', 'a'.repeat(MAX_FILE_BYTES));
expect(readExportTextFile(p, 'edge.ts', [], []).kind).toBe('ok');
});
it('skips a binary file (null byte in first 512 bytes)', () => {
const p = write('bin.dat', Buffer.from([0x41, 0x00, 0x42]));
expect(readExportTextFile(p, 'bin.dat', [], []).kind).toBe('binary');
});
it('reports a missing/inaccessible file', () => {
expect(readExportTextFile(path.join(tmpDir, 'nope.ts'), 'nope.ts', [], []).kind).toBe('inaccessible');
});
it('applies the include/exclude filter before touching the filesystem', () => {
// A path that does not exist but is filtered out reports 'excluded', not
// 'inaccessible' — the glob gate short-circuits before any stat.
expect(readExportTextFile(path.join(tmpDir, 'ghost.md'), 'ghost.md', ['*.ts'], []).kind).toBe('excluded');
});
it('looksLikeBinary flags a null byte but passes plain text', () => {
expect(looksLikeBinary(Buffer.from('plain text'))).toBe(false);
expect(looksLikeBinary(Buffer.from([0x00]))).toBe(true);
});
});