zedcode/bs-manager
1
0
Fork 0
mirror of https://github.com/Zagrios/bs-manager.git synced 2026-07-03 14:08:25 +02:00
Code Issues Packages Projects Releases Wiki Activity

[bugfix] use YauzlZip class wrapper for extracting files

Browse Source 
* fixed issue where yauzl just closes after reading all files
This commit is contained in:
silentrald
2024-11-14 14:24:45 +08:00
parent 8c6f3b0aa8
commit bfb6d28a34
6 changed files with 90 additions and 511 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/__tests__/unit/zip.test.ts
+9 -99
View File
@@ -1,6 +1,5 @@
import path from "path";
import { mkdir, pathExistsSync, readFile, rm } from "fs-extra";
import { extractZip, getFilesFromZip } from "main/helpers/zip.helpers";
import { YauzlZip } from "main/models/yauzl-zip.class";
const TEST_FOLDER = path.resolve(__dirname, "../../..", "assets", "tests");
@@ -10,99 +9,8 @@ const SUBFOLDERS_ZIP = path.join(TEST_FOLDER, "subfolders.zip");
const MANIFEST_ZIP = path.join(TEST_FOLDER, "manifest.zip");
const DESTINATION_FOLDER = path.join(TEST_FOLDER, "out");
describe("Zip Server Service Test", () => {
beforeAll(async () => {
if (pathExistsSync(DESTINATION_FOLDER)) {
await rm(DESTINATION_FOLDER, { recursive: true, force: true });
}
await mkdir(DESTINATION_FOLDER);
});
// Uses '/'
it("Extract standard zip", async () => {
await extractZip(STANDARD_ZIP, DESTINATION_FOLDER);
for (const file of [
"file_1.2.txt",
"file_1.3.txt",
]) {
expect(pathExistsSync(path.join(DESTINATION_FOLDER, file)))
.toBe(true);
}
const SUBFOLDER_PATH = path.join(DESTINATION_FOLDER, "folder_1.1");
for (const file of [
"file_2.1.txt",
"file_2.2.txt",
]) {
expect(pathExistsSync(path.join(SUBFOLDER_PATH, file)))
.toBe(true);
}
});
// Uses '\\'
it("Extract map zips using forward slashes", async () => {
const beforeExtractedFolders: string[] = [];
const afterExtractedFolders: string[] = [];
await extractZip(WINDOWS_LEGACY_MAP_ZIP, DESTINATION_FOLDER, {
beforeFolderExtracted: (folder) => beforeExtractedFolders.push(folder),
afterFolderExtracted: (folder) => afterExtractedFolders.push(folder),
});
// Expect all the files to exists
for (const file of [
"BPMInfo.dat",
"cover.jpg",
"ExpertLegacy.dat",
"ExpertPlusLawless.dat",
"ExpertPlusLegacy.dat",
"ExpertPlusStandard.dat",
"ExpertStandard.dat",
"Info.dat",
"song.egg",
]) {
expect(pathExistsSync(path.join(DESTINATION_FOLDER, file)))
.toBe(true);
}
const SUBFOLDER_PATH = path.join(DESTINATION_FOLDER, "pfp");
for (const file of [
"galaxyCompressed.jpg",
"gojiCompressed.jpg",
]) {
expect(pathExistsSync(path.join(SUBFOLDER_PATH, file)))
.toBe(true);
}
expect(beforeExtractedFolders).toContain(".");
expect(beforeExtractedFolders).toContain("pfp");
expect(beforeExtractedFolders.length).toBe(2);
expect(afterExtractedFolders).toContain(".");
expect(afterExtractedFolders).toContain("pfp");
expect(afterExtractedFolders.length).toBe(2);
});
it("Extract manifest.json from zip file", async () => {
const zipBuffer = await readFile(MANIFEST_ZIP);
const buffers = await getFilesFromZip(zipBuffer, ["manifest.json"]);
const manifest = JSON.parse(buffers["manifest.json"].toString());
expect(manifest).toBeTruthy();
// Check some of the fields if they are correct
expect(manifest.appId).toBe("some-app-id");
expect(manifest.canonicalName).toBe("some-canonical-name");
expect(manifest.isCore).toBe(true);
});
afterAll(async () => {
if (pathExistsSync(DESTINATION_FOLDER)) {
await rm(DESTINATION_FOLDER, { recursive: true, force: true });
}
});
});
describe("Test YauzlZip class", () => {
let zip: YauzlZip;
beforeAll(async () => {
if (pathExistsSync(DESTINATION_FOLDER)) {
@@ -111,11 +19,9 @@ describe("Test YauzlZip class", () => {
await mkdir(DESTINATION_FOLDER);
});
it("Extract standard zip", async () => {
const zip = await YauzlZip.fromPath(STANDARD_ZIP);
zip = await YauzlZip.fromPath(STANDARD_ZIP);
const res = await zip.extract(DESTINATION_FOLDER);
expect(res.sort()).toEqual([
@@ -147,7 +53,7 @@ describe("Test YauzlZip class", () => {
it("Extract map zips using back slashes", async () => {
const zip = await YauzlZip.fromPath(WINDOWS_LEGACY_MAP_ZIP);
zip = await YauzlZip.fromPath(WINDOWS_LEGACY_MAP_ZIP);
const res = await zip.extract(DESTINATION_FOLDER);
expect(res.sort()).toEqual([
@@ -223,7 +129,7 @@ describe("Test YauzlZip class", () => {
it("Read manifest.json from zip file", async () => {
const zipBuffer = await readFile(MANIFEST_ZIP);
const zip = await YauzlZip.fromBuffer(zipBuffer);
zip = await YauzlZip.fromBuffer(zipBuffer);
const buffer = await (await zip.findEntry(entry => entry.fileName === "manifest.json")).read();
const manifest = JSON.parse(buffer.toString());
expect(manifest).toBeTruthy();
@@ -234,7 +140,11 @@ describe("Test YauzlZip class", () => {
expect(manifest.isCore).toBe(true);
});
afterEach(() => {
if (zip) {
zip.close();
}
})
afterAll(async () => {
if (pathExistsSync(DESTINATION_FOLDER)) {
src/main/helpers/zip.helpers.ts
-331
View File
@@ -1,331 +0,0 @@
import fs from "fs-extra";
import path from "path";
import yauzl from "yauzl";
import { ensureFolderExist } from "./fs.helpers";
// NOTE: yauzl needs to be reopened once readEntry is done
export interface ZipEntry {
name: string;
directory: boolean;
buffer?: Buffer;
};
export interface ZipExtractOptions {
// Stops the extraction immediately when set to true
terminate?: (entry: ZipEntry) => boolean;
// Whether or not to extract the file
condition?: (entry: ZipEntry) => boolean;
// Called before extracting the folder
beforeFolderExtracted?: (folder: string) => void;
// Called when the folder contents is fully extracted
afterFolderExtracted?: (folder: string) => void;
};
export interface ZipProcessOptions {
// Stops the extraction immediately when set to true
terminate?: (entry: ZipEntry) => boolean;
getBuffer?: boolean;
};
const YAUZL_OPTIONS: yauzl.Options = {
lazyEntries: true,
decodeStrings: true,
};
function openZip(data: string | Buffer, callback: (error: Error, zip: yauzl.ZipFile) => void) {
if (typeof data === "string") {
yauzl.open(data, YAUZL_OPTIONS, callback);
} else {
yauzl.fromBuffer(data, YAUZL_OPTIONS, callback);
}
}
export async function extractZip(
data: string | Buffer,
destination: string,
options?: Readonly<ZipExtractOptions>
): Promise<string[]> {
await ensureFolderExist(destination);
return new Promise((resolve, reject) => {
openZip(data, (openError: Error, zip: yauzl.ZipFile) => {
if (openError) return reject(openError);
handleExtractZip(zip, destination, options, resolve, reject);
});
});
}
function handleExtractZip(
zip: yauzl.ZipFile,
destination: string,
options: Readonly<ZipExtractOptions> | undefined,
resolve: (result: string[]) => void, reject: (error: Error) => void
) {
const files: string[] = [];
const zipEntry: ZipEntry = {
name: "",
directory: false,
};
let indexes: number[] = [];
let dirname = "";
options?.beforeFolderExtracted?.(".");
const folderExtractEvents = options?.beforeFolderExtracted
|| options?.afterFolderExtracted
? (filename: string) => {
let newDirname = path.dirname(filename);
if (newDirname === ".") newDirname = "";
const comparison = compareStringIndex(dirname, newDirname);
if (comparison === -1) { return; }
// Push after extraction
if (options?.afterFolderExtracted) {
for (let i = indexes.length - 1; i > -1 && indexes[i] > comparison; --i) {
options.afterFolderExtracted(dirname.substring(0, indexes[i]));
}
}
// Push before extraction
indexes = getSlashIndexes(newDirname);
if (options?.beforeFolderExtracted) {
for (let i = 0; i < indexes.length; ++i) {
if (indexes[i] <= comparison) { continue; }
options.beforeFolderExtracted(newDirname.substring(0, indexes[i]));
}
}
dirname = newDirname;
} : () => {};
zip.readEntry();
zip.on("entry", (entry: yauzl.Entry) => {
const absolutePath = path.join(destination, entry.fileName);
// Entry is a directory / folder
if (entry.fileName.endsWith("/")) {
if (!fs.pathExistsSync(absolutePath)) {
fs.mkdirSync(absolutePath, { recursive: true });
}
return zip.readEntry();
}
// Entry is a file
zip.openReadStream(entry, (readError, readStream) => {
if (readError) return reject(readError);
folderExtractEvents(entry.fileName);
zipEntry.name = entry.fileName;
zipEntry.directory = false;
zipEntry.buffer = undefined;
if (options?.terminate?.(zipEntry)) {
readStream.destroy();
zip.close();
return resolve(files);
}
if (options?.condition) {
if (!options.condition(zipEntry)) {
readStream.destroy();
return zip.readEntry();
}
}
// For forward slashes since they don't create directories
const directory = path.dirname(absolutePath);
if (!fs.pathExistsSync(directory)) {
fs.mkdirSync(directory, { recursive: true });
}
const writeStream = fs.createWriteStream(absolutePath);
readStream.pipe(writeStream);
readStream.on("end", () => {
zip.readEntry()
});
});
});
zip.once("end", () => {
if (options?.afterFolderExtracted) {
// Push after extraction
for (let i = indexes.length - 1; i > -1; --i) {
options.afterFolderExtracted(dirname.substring(0, indexes[i]));
}
options.afterFolderExtracted(".");
}
zip.close();
resolve(files);
});
}
export function getFilesFromZip(
data: string | Buffer,
files: string[]
): Promise<Record<string, Buffer>> {
return new Promise((resolve, reject) => {
openZip(data, (openError: Error, zip: yauzl.ZipFile) => {
if (openError) return reject(openError);
handleGetFilesFromZip(zip, files, resolve, reject);
});
});
}
function handleGetFilesFromZip(
zip: yauzl.ZipFile,
files: string[],
resolve: (buffers: Record<string, Buffer>) => void,
reject: (error: Error) => void
) {
const buffers: Record<string, Buffer> = {};
let count = 0;
zip.readEntry();
zip.on("entry", (entry: yauzl.Entry) => {
// Entry is a directory / folder
if (entry.fileName.endsWith("/") || !files.includes(entry.fileName)) {
return zip.readEntry();
}
// Entry is in the files list
zip.openReadStream(entry, (readError, readStream) => {
if (readError) return reject(readError);
const chunks: Buffer[] = [];
readStream.on("data", data => {
chunks.push(data);
});
readStream.on("end", () => {
buffers[entry.fileName] = Buffer.concat(chunks);
++count;
if (count === files.length) {
zip.close();
return resolve(buffers);
}
zip.readEntry();
});
});
});
zip.once("end", () => {
zip.close();
resolve(buffers);
});
}
// @params loop - this should not throw an error
export function processZip<T>(
data: string | Buffer,
options: Readonly<ZipProcessOptions>,
loop: (accumulator: T, entry: ZipEntry) => T,
initValue: T
): Promise<T> {
return new Promise((resolve, reject) => {
openZip(data, (openError: Error, zip: yauzl.ZipFile) => {
if (openError) return reject(openError);
handleProcessZip<T>(
zip, options, loop, initValue,
resolve, reject
);
});
});
}
function handleProcessZip<T>(
zip: yauzl.ZipFile,
options: Readonly<ZipProcessOptions>,
loop: (accumulator: T, entry: ZipEntry) => T,
accumulator: T,
resolve: (result: T) => void, reject: (error: Error) => void
): void {
const zipEntry: ZipEntry = {
name: "",
directory: false,
};
zip.readEntry();
zip.on("entry", (entry: yauzl.Entry) => {
const isDirectory = entry.fileName.endsWith("/");
if (isDirectory || !options.getBuffer) {
zipEntry.name = entry.fileName;
zipEntry.directory = isDirectory;
zipEntry.buffer = undefined;
accumulator = loop(accumulator, zipEntry);
return zip.readEntry();
}
// Entry is a file
zip.openReadStream(entry, (readError, readStream) => {
if (readError) return reject(readError);
zipEntry.name = entry.fileName;
zipEntry.directory = false;
zipEntry.buffer = undefined;
if (options?.terminate?.(zipEntry)) {
readStream.destroy();
zip.close();
return resolve(accumulator);
}
const chunks: Buffer[] = [];
readStream.on("data", data => {
chunks.push(data);
});
readStream.on("end", () => {
zipEntry.buffer = Buffer.concat(chunks);
accumulator = loop(accumulator, zipEntry);
zip.readEntry();
});
});
});
zip.once("end", () => {
zip.close();
resolve(accumulator);
});
}
/**
* @returns(number)
* -1 = string are the same
* > 0 = index where they don't equal the same
*/
function compareStringIndex(str1: string, str2: string) {
const N = Math.min(str1.length, str2.length);
let i = 0;
for (; i < N; ++i) {
if (str1[i] !== str2[i]) { break; }
}
return i === str1.length && i === str2.length
? -1 : i;
}
/**
* @returns(number[]) - int array of the positions of the slashes
*/
function getSlashIndexes(str: string) {
if (str === "") {
return [];
}
const indexes: number[] = [];
for (let i = 0; i < str.length; ++i) {
if (str[i] === "/") {
indexes.push(i);
}
}
indexes.push(str.length);
return indexes;
}
src/main/models/oculus-downloader.class.ts
+9 -5
View File
@@ -6,7 +6,7 @@ import { inflate } from "pako"
import { EMPTY, Observable, ReplaySubject, Subscriber, catchError, filter, from, lastValueFrom, mergeMap, scan, share, tap } from "rxjs";
import { Progression, hashFile } from "../helpers/fs.helpers";
import { OculusDownloaderErrorCodes } from "../../shared/models/bs-version-download/oculus-download.model";
import { getFilesFromZip } from "main/helpers/zip.helpers";
import { YauzlZip } from "./yauzl-zip.class";
export class OculusDownloader {
@@ -35,14 +35,18 @@ export class OculusDownloader {
.catch(err => CustomError.throw(err, "DOWNLOAD_MANIFEST_FAILED"));
const manifestName = "manifest.json";
const buffers = await getFilesFromZip(buffer, [manifestName]);
const manifest = buffers[manifestName];
if(!manifest) {
const zip = await YauzlZip.fromBuffer(buffer);
const entry = await zip.findEntry((entry) => entry.fileName === manifestName);
if(!entry) {
throw new CustomError("Manifest file not found", "MANIFEST_FILE_NOT_FOUND");
}
const manifest = await entry.read();
return JSON.parse(manifest.toString())
.catch((err: Error) => CustomError.throw(err, "PARSE_MANIFEST_FILE_FAILED"));
.catch((err: Error) => CustomError.throw(err, "PARSE_MANIFEST_FILE_FAILED"))
.finally(() => {
zip.close();
});
}
private downloadManifestFile(file: OculusManifestFile, destination: string): Observable<Progression<OculusManifestFile>> {
src/main/models/yauzl-zip.class.ts
+1 -1
View File
@@ -7,6 +7,7 @@ export class YauzlZip {
private static readonly YAUZL_OPEN_OPTIONS: Options = {
lazyEntries: true,
decodeStrings: true,
autoClose: false,
};
public static fromPath(path: string): Promise<YauzlZip> {
@@ -36,7 +37,6 @@ export class YauzlZip {
private async readEntry(): Promise<YauzlZipEntry> {
return new Promise((resolve, reject) => {
const onEntry = (entry: Entry) => {
cleanup();
resolve(new YauzlZipEntry({ entry, zip: this.zip }));
src/main/services/additional-content/maps/local-maps-manager.service.ts
+61 -65
View File
@@ -13,7 +13,7 @@ import sanitize from "sanitize-filename";
import { DeepLinkService } from "../../deep-link.service";
import log from "electron-log";
import { WindowManagerService } from "../../window-manager.service";
import { Observable, Subject, lastValueFrom, tap } from "rxjs";
import { Observable, Subject, lastValueFrom } from "rxjs";
import { Archive } from "../../../models/archive.class";
import { Progression, deleteFolder, ensureFolderExist, getFilesInFolder, getFoldersInFolder, pathExist } from "../../../helpers/fs.helpers";
import { readFile } from "fs/promises";
@@ -29,7 +29,7 @@ import { MapInfo } from "shared/models/maps/info/map-info.model";
import { parseMapInfoDat } from "shared/parsers/maps/map-info.parser";
import { CustomError } from "shared/models/exceptions/custom-error.class";
import { tryit } from "shared/helpers/error.helpers";
import { extractZip, processZip } from "main/helpers/zip.helpers";
import { YauzlZip } from "main/models/yauzl-zip.class";
export class LocalMapsManagerService {
private static instance: LocalMapsManagerService;
@@ -324,35 +324,15 @@ export class LocalMapsManagerService {
public importMaps(zipPaths: string[], version?: BSVersion): Observable<Progression<BsmLocalMap>> {
return new Observable<Progression<BsmLocalMap>>(obs => {
let unsubscribed = false;
let progress: Progression<BsmLocalMap> = { total: 0, current: 0 };
let nbImportedMaps = 0;
const abortController = new AbortController();
let zip: YauzlZip;
(async () => {
const terminate = () => unsubscribed;
let completeNewFolder: () => void;
const newFolderTap = tap((folder: string) => {
this.loadMapInfoFromPath(folder)
.then(map => {
progress.data = map;
})
.catch(error => {
log.error("Could not load map info", error);
progress.data = null;
})
.finally(() => {
++nbImportedMaps;
++progress.current;
obs.next(progress);
if (progress.current === progress.total) {
completeNewFolder();
}
});
});
const mapsPath = await this.getMapsFolderPath(version);
for(const zipPath of zipPaths) {
if(unsubscribed) {
if(abortController.signal?.aborted) {
log.info("Maps import from zip has been cancelled");
return;
}
@@ -362,59 +342,68 @@ export class LocalMapsManagerService {
if(!pathExistsSync(zipPath)) { continue; }
const { result: mapsFolders, error } = await tryit(() => processZip(
zipPath, {},
(set, entry) => {
if (!entry.directory && /(^|\/)[Ii]nfo\.dat$/.test(entry.name)) {
set.add(path.dirname(entry.name));
}
return set;
},
new Set<string>())
);
const mapsFolders: string[] = [];
zip = await YauzlZip.fromPath(zipPath);
for await (const entry of zip.entries()) {
if (/(^|\/)[Ii]nfo\.dat$/.test(entry.fileName)) {
mapsFolders.push(path.dirname(entry.fileName));
}
}
if (error) {
log.error("Could not read zip entries", zipPath, error);
if (mapsFolders.length === 0) {
log.warn("No maps \"info.dat\" found in zip", zipPath);
progress.total = 1;
progress.current = 1;
obs.next(progress);
continue;
}
if (mapsFolders.size === 0) {
log.warn("No maps \"info.dat\" found in zip", zipPath);
}
progress.total = mapsFolders.size;
progress.total = mapsFolders.length;
obs.next(progress);
const newFolder = new Subject<string>();
const promise = lastValueFrom(newFolder.pipe(newFolderTap));
completeNewFolder = () => newFolder.complete();
const isRoot = mapsFolders.size === 1 && mapsFolders.has(".");
const isRoot = mapsFolders.length === 1 && mapsFolders[0] === ".";
const destination = isRoot
? path.join(mapsPath, path.basename(zipPath, ".zip"))
: mapsPath;
log.info("Extracting", `"${zipPath}"`, "into", `"${mapsPath}"`);
await extractZip(zipPath, destination, {
terminate,
condition: !isRoot && ((entry) => mapsFolders.has(path.dirname(entry.name))),
beforeFolderExtracted: (folder) => {
if (!mapsFolders.has(folder)) return;
log.info("*", `"${folder}"`, "before");
},
afterFolderExtracted: (folder) => {
if (!mapsFolders.has(folder)) return;
newFolder.next(path.resolve(destination, folder));
}
});
for (const folder of mapsFolders) {
log.info("*", folder);
const regex = new RegExp(`^${folder
.replaceAll(".", "\\.")
.replaceAll("+", "\\+")
.replaceAll("(", "\\(")
.replaceAll(")", "\\)")
.replaceAll("[", "\\[")
.replaceAll("]", "\\]")
}\/`);
if (unsubscribed) {
await zip.extract(destination, {
entriesNames: [regex],
abortToken: abortController
});
if (abortController.signal?.aborted) {
break;
}
++nbImportedMaps;
++progress.current;
progress.data = await this.loadMapInfoFromPath(path.join(destination, folder));
obs.next(progress);
if (abortController.signal?.aborted) {
break;
}
}
zip.close();
if (abortController.signal?.aborted) {
log.info("Maps import from zip has been cancelled");
return;
}
await promise;
}
})()
.then(() => {
@@ -424,10 +413,15 @@ export class LocalMapsManagerService {
return log.info("Successfully imported", nbImportedMaps, "maps from", zipPaths.length, "zips");
})
.catch(e => obs.error(e))
.finally(() => obs.complete());
.finally(() => {
if (zip) {
zip.close();
}
obs.complete()
});
return () => {
unsubscribed = true;
abortController.abort();
};
});
}
@@ -455,7 +449,9 @@ export class LocalMapsManagerService {
}
const zipPath = await this.downloadMapZip(zipUrl);
await extractZip(zipPath, mapPath);
const zip = await YauzlZip.fromPath(zipPath);
await zip.extract(mapPath);
zip.close();
await unlink(zipPath);
const localMap = await this.loadMapInfoFromPath(mapPath);
src/main/services/mods/bs-mods-manager.service.ts
+10 -10
View File
@@ -10,7 +10,6 @@ import { BS_EXECUTABLE } from "../../constants";
import log from "electron-log";
import { deleteFolder, ensureFolderExist, pathExist, Progression, unlinkPath } from "../../helpers/fs.helpers";
import { lastValueFrom, Observable } from "rxjs";
import { extractZip, processZip } from "../../helpers/zip.helpers";
import recursiveReadDir from "recursive-readdir";
import { sToMs } from "../../../shared/helpers/time.helpers";
import { pathExistsSync, unlinkSync } from "fs-extra";
@@ -20,6 +19,7 @@ import { LinuxService } from "../linux.service";
import { tryit } from "shared/helpers/error.helpers";
import { UtilsService } from "../utils.service";
import crypto from "crypto";
import { YauzlZip } from "main/models/yauzl-zip.class";
export class BsModsManagerService {
private static instance: BsModsManagerService;
@@ -205,16 +205,15 @@ export class BsModsManagerService {
return false;
}
const hashCount = await processZip(zipPath, {
getBuffer: true
}, (acc, entry) => {
if (entry.directory) return acc;
const zip = await YauzlZip.fromPath(zipPath);
let hashCount = 0;
for await (const entry of zip.entries()) {
const buffer = await entry.read();
const md5Hash = crypto.createHash("md5")
.update(entry.buffer)
.update(buffer)
.digest("hex");
return acc + +download.hashMd5.some(md5 => md5.hash === md5Hash);
}, 0);
hashCount += +download.hashMd5.some(md5 => md5.hash === md5Hash);
}
if (hashCount !== download.hashMd5.length) {
return false;
@@ -225,13 +224,14 @@ export class BsModsManagerService {
const destDir = isBSIPA ? versionPath : path.join(versionPath, ModsInstallFolder.PENDING);
log.info("Start extracting mod zip", mod.name, "to", destDir);
const extracted = await extractZip(zipPath, destDir)
const extracted = await zip.extract(destDir)
.then(() => true)
.catch(e => {
log.error("Error while extracting mod zip", e);
return false;
})
.finally(() => {
zip.close();
if (pathExistsSync(zipPath)) {
unlinkSync(zipPath);
}