mirror of
https://github.com/teamchong/pxpipe.git
synced 2026-07-22 02:02:51 +02:00
fix(export): apply include/exclude and size limit to --git untracked files
In --git mode, untracked files (git ls-files --others) were read straight to memory with only a binary check — unlike directory and single-file modes, which also apply --include/--exclude globs and the 1 MiB MAX_FILE_BYTES cap. So `pxpipe export --git --include '*.ts'` still slurped every untracked file regardless of the filter, and an untracked multi-MB file was read whole (a resource-safety hazard). - extract the shared gate into src/export-collect.ts: readExportTextFile() applies include/exclude → size → binary, in that order, and is import-safe (src/core/export.ts is contractually fs-free, so it can't live there) - route all three collection paths (walkDir, single-file target, --git untracked) through it; untracked skips now warn like explicit targets do - tests/export-collect.test.ts covers include, exclude, oversized, the size boundary, binary, and inaccessible Verified end-to-end: `export --git --include '*.ts'` in a temp repo excludes an untracked .md, skips an oversized .ts (with a warning), and keeps the .ts.
This commit is contained in:
@@ -0,0 +1,67 @@
|
||||
/**
|
||||
* Shared file-reading gate for `pxpipe export`. Every collection mode
|
||||
* (directory walk, explicit single-file target, and `--git` untracked files)
|
||||
* must apply the SAME three checks — include/exclude globs, a max size, and a
|
||||
* binary sniff — so the untracked path can no longer read gigabyte or filtered
|
||||
* files that directory mode would have skipped.
|
||||
*
|
||||
* Kept in its own import-safe module (no top-level side effects, unlike
|
||||
* src/node.ts which starts the server on import) so it can be unit-tested
|
||||
* directly. src/core/export.ts is deliberately fs-free, so the fs-touching gate
|
||||
* lives here rather than there.
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import { shouldIncludeFile } from './core/export.js';
|
||||
|
||||
/** Files larger than this are skipped by export (1 MiB). A single export bundle
|
||||
* is meant to be paste-sized; a multi-MB file is never intentional context and
|
||||
* reading it fully into memory is a resource-safety hazard. */
|
||||
export const MAX_FILE_BYTES = 1_000_000;
|
||||
|
||||
/** Returns true if `buf` looks like binary (contains a null byte in the first 512 bytes). */
|
||||
export function looksLikeBinary(buf: Buffer): boolean {
|
||||
const check = Math.min(buf.byteLength, 512);
|
||||
for (let i = 0; i < check; i++) {
|
||||
if (buf[i] === 0) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Outcome of trying to read one file for export. `excluded` is a normal filter
|
||||
* hit (callers stay quiet); the other non-ok kinds are worth a warning. */
|
||||
export type ExportReadResult =
|
||||
| { readonly kind: 'ok'; readonly content: string }
|
||||
| { readonly kind: 'excluded' | 'oversized' | 'binary' | 'inaccessible' };
|
||||
|
||||
/**
|
||||
* Read a single text file for export if it passes every gate, in this order:
|
||||
* 1. include/exclude globs (relative path)
|
||||
* 2. size <= MAX_FILE_BYTES
|
||||
* 3. not binary
|
||||
* Returns the utf8 content on success, or the reason it was skipped. Pure of
|
||||
* logging so each caller can decide whether a skip is noteworthy.
|
||||
*/
|
||||
export function readExportTextFile(
|
||||
fullPath: string,
|
||||
relPath: string,
|
||||
include: string[],
|
||||
exclude: string[],
|
||||
): ExportReadResult {
|
||||
if (!shouldIncludeFile(relPath, include, exclude)) return { kind: 'excluded' };
|
||||
let stat: fs.Stats;
|
||||
try {
|
||||
stat = fs.statSync(fullPath);
|
||||
} catch {
|
||||
return { kind: 'inaccessible' };
|
||||
}
|
||||
if (stat.size > MAX_FILE_BYTES) return { kind: 'oversized' };
|
||||
let buf: Buffer;
|
||||
try {
|
||||
buf = fs.readFileSync(fullPath);
|
||||
} catch {
|
||||
return { kind: 'inaccessible' };
|
||||
}
|
||||
if (looksLikeBinary(buf)) return { kind: 'binary' };
|
||||
return { kind: 'ok', content: buf.toString('utf8') };
|
||||
}
|
||||
+16
-35
@@ -16,10 +16,10 @@ import { createProxy, parseGatewayHeaders, resolveUpstreams, type ProxyConfig }
|
||||
import {
|
||||
parseExportArgv,
|
||||
runExportCore,
|
||||
shouldIncludeFile,
|
||||
type ExportParsed,
|
||||
type ExportResult,
|
||||
} from './core/export.js';
|
||||
import { readExportTextFile } from './export-collect.js';
|
||||
import {
|
||||
toTrackEvent,
|
||||
TRACK_BODY_INLINE_MAX,
|
||||
@@ -630,17 +630,6 @@ const WALK_SKIP_DIRS = new Set([
|
||||
'__pycache__', '.cache', '.next', '.nuxt', '.turbo',
|
||||
]);
|
||||
|
||||
const MAX_FILE_BYTES = 1_000_000; // 1 MiB
|
||||
|
||||
/** Returns true if `buf` looks like binary (contains a null byte in the first 512 bytes). */
|
||||
function looksLikeBinary(buf: Buffer): boolean {
|
||||
const check = Math.min(buf.byteLength, 512);
|
||||
for (let i = 0; i < check; i++) {
|
||||
if (buf[i] === 0) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
interface CollectedFile {
|
||||
relPath: string;
|
||||
content: string;
|
||||
@@ -667,14 +656,10 @@ function walkDir(
|
||||
if (WALK_SKIP_DIRS.has(entry.name)) continue;
|
||||
walkDir(full, rootDir, include, exclude, out);
|
||||
} else if (entry.isFile()) {
|
||||
if (!shouldIncludeFile(rel, include, exclude)) continue;
|
||||
let stat: fs.Stats;
|
||||
try { stat = fs.statSync(full); } catch { continue; }
|
||||
if (stat.size > MAX_FILE_BYTES) continue;
|
||||
let buf: Buffer;
|
||||
try { buf = fs.readFileSync(full); } catch { continue; }
|
||||
if (looksLikeBinary(buf)) continue;
|
||||
out.push({ relPath: rel, content: buf.toString('utf8') });
|
||||
// Bulk directory walk: skip silently on any gate miss (per-file warnings
|
||||
// would be noise across a whole tree).
|
||||
const r = readExportTextFile(full, rel, include, exclude);
|
||||
if (r.kind === 'ok') out.push({ relPath: rel, content: r.content });
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -696,18 +681,11 @@ function collectFilesFromTargets(
|
||||
walkDir(target, target, include, exclude, files);
|
||||
} else if (st.isFile()) {
|
||||
const rel = path.basename(target);
|
||||
if (!shouldIncludeFile(rel, include, exclude)) continue;
|
||||
if (st.size > MAX_FILE_BYTES) {
|
||||
console.warn(`[pxpipe export] skipping oversized file: ${target}`);
|
||||
continue;
|
||||
const r = readExportTextFile(target, rel, include, exclude);
|
||||
if (r.kind === 'ok') files.push({ relPath: rel, content: r.content });
|
||||
else if (r.kind !== 'excluded') {
|
||||
console.warn(`[pxpipe export] skipping ${r.kind} file: ${target}`);
|
||||
}
|
||||
let buf: Buffer;
|
||||
try { buf = fs.readFileSync(target); } catch { continue; }
|
||||
if (looksLikeBinary(buf)) {
|
||||
console.warn(`[pxpipe export] skipping binary file: ${target}`);
|
||||
continue;
|
||||
}
|
||||
files.push({ relPath: rel, content: buf.toString('utf8') });
|
||||
}
|
||||
}
|
||||
return files;
|
||||
@@ -756,10 +734,13 @@ async function collectSource(opts: ExportParsed): Promise<[string, string[]]> {
|
||||
let untracked = '';
|
||||
for (const rel of untrackedFiles) {
|
||||
const full = path.join(cwd, rel);
|
||||
let buf: Buffer;
|
||||
try { buf = fs.readFileSync(full); } catch { continue; }
|
||||
if (!looksLikeBinary(buf)) {
|
||||
untracked += `\n===== ${rel} =====\n` + buf.toString('utf8');
|
||||
// Same include/exclude + size + binary gate as directory mode. Untracked
|
||||
// files previously bypassed all of it: --include/--exclude were ignored
|
||||
// and an oversized file was read fully into memory.
|
||||
const r = readExportTextFile(full, rel, opts.include, opts.exclude);
|
||||
if (r.kind === 'ok') untracked += `\n===== ${rel} =====\n` + r.content;
|
||||
else if (r.kind !== 'excluded') {
|
||||
console.warn(`[pxpipe export] skipping ${r.kind} untracked file: ${rel}`);
|
||||
}
|
||||
}
|
||||
const sourceText = diff + untracked;
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import * as os from 'node:os';
|
||||
import { readExportTextFile, looksLikeBinary, MAX_FILE_BYTES } from '../src/export-collect.js';
|
||||
|
||||
// readExportTextFile is the single gate now shared by every `pxpipe export`
|
||||
// collection mode (directory walk, single-file target, and --git untracked).
|
||||
// Before this, the --git untracked path applied none of these checks — it
|
||||
// ignored --include/--exclude and read files of any size fully into memory.
|
||||
describe('readExportTextFile — shared export file gate', () => {
|
||||
let tmpDir: string;
|
||||
beforeEach(() => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pxpipe-collect-test-'));
|
||||
});
|
||||
afterEach(() => {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const write = (name: string, content: string | Buffer): string => {
|
||||
const p = path.join(tmpDir, name);
|
||||
fs.writeFileSync(p, content);
|
||||
return p;
|
||||
};
|
||||
|
||||
it('reads an included text file as utf8', () => {
|
||||
const p = write('a.ts', 'export const x = 1;\n');
|
||||
const r = readExportTextFile(p, 'a.ts', [], []);
|
||||
expect(r.kind).toBe('ok');
|
||||
if (r.kind === 'ok') expect(r.content).toBe('export const x = 1;\n');
|
||||
});
|
||||
|
||||
it('respects --include: a .md is excluded when include is *.ts, the .ts sibling passes', () => {
|
||||
const md = write('b.md', '# readme\n');
|
||||
const ts = write('a.ts', 'const a = 1;\n');
|
||||
expect(readExportTextFile(md, 'b.md', ['*.ts'], []).kind).toBe('excluded');
|
||||
expect(readExportTextFile(ts, 'a.ts', ['*.ts'], []).kind).toBe('ok');
|
||||
});
|
||||
|
||||
it('respects --exclude', () => {
|
||||
const p = write('secret.ts', 'const s = 1;\n');
|
||||
expect(readExportTextFile(p, 'secret.ts', [], ['secret.*']).kind).toBe('excluded');
|
||||
});
|
||||
|
||||
it('skips an oversized file instead of reading it into memory', () => {
|
||||
const p = write('big.ts', 'a'.repeat(MAX_FILE_BYTES + 1));
|
||||
expect(readExportTextFile(p, 'big.ts', [], []).kind).toBe('oversized');
|
||||
});
|
||||
|
||||
it('accepts a file exactly at the size limit', () => {
|
||||
const p = write('edge.ts', 'a'.repeat(MAX_FILE_BYTES));
|
||||
expect(readExportTextFile(p, 'edge.ts', [], []).kind).toBe('ok');
|
||||
});
|
||||
|
||||
it('skips a binary file (null byte in first 512 bytes)', () => {
|
||||
const p = write('bin.dat', Buffer.from([0x41, 0x00, 0x42]));
|
||||
expect(readExportTextFile(p, 'bin.dat', [], []).kind).toBe('binary');
|
||||
});
|
||||
|
||||
it('reports a missing/inaccessible file', () => {
|
||||
expect(readExportTextFile(path.join(tmpDir, 'nope.ts'), 'nope.ts', [], []).kind).toBe('inaccessible');
|
||||
});
|
||||
|
||||
it('applies the include/exclude filter before touching the filesystem', () => {
|
||||
// A path that does not exist but is filtered out reports 'excluded', not
|
||||
// 'inaccessible' — the glob gate short-circuits before any stat.
|
||||
expect(readExportTextFile(path.join(tmpDir, 'ghost.md'), 'ghost.md', ['*.ts'], []).kind).toBe('excluded');
|
||||
});
|
||||
|
||||
it('looksLikeBinary flags a null byte but passes plain text', () => {
|
||||
expect(looksLikeBinary(Buffer.from('plain text'))).toBe(false);
|
||||
expect(looksLikeBinary(Buffer.from([0x00]))).toBe(true);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user