zedcode/bs-manager
1
0
Fork 0
mirror of https://github.com/Zagrios/bs-manager.git synced 2026-07-03 14:08:25 +02:00
Code Issues Packages Projects Releases Wiki Activity

[bugfix] use yauzl for extracting map files for importing maps

Browse Source
This commit is contained in:
silentrald
2024-11-09 00:55:25 +08:00
parent 68559add5d
commit 2b37972dd4
5 changed files with 199 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/__tests__/unit/zip.test.ts
+17
View File
@@ -5,6 +5,7 @@ import { extractZip } from "main/helpers/zip.helpers";
const TEST_FOLDER = path.resolve(__dirname, "../../..", "assets", "tests");
const STANDARD_ZIP = path.join(TEST_FOLDER, "standard.zip");
const WINDOWS_LEGACY_MAP_ZIP = path.join(TEST_FOLDER, "windows_legacy.zip");
const SPECIAL_ZIP = path.join(TEST_FOLDER, "special.zip");
const DESTINATION_FOLDER = path.join(TEST_FOLDER, "out");
describe("Zip Server Service Test", () => {
@@ -67,6 +68,22 @@ describe("Zip Server Service Test", () => {
}
});
it("Extract special zip file", async () => {
const beforeExtracted: string[] = [];
const afterExtracted: string[] = [];
await extractZip(SPECIAL_ZIP, DESTINATION_FOLDER, {
beforeFolderExtracted: (folder) => beforeExtracted.push(folder),
afterFolderExtracted: (folder) => afterExtracted.push(folder),
});
const expected = [
"1/a", "1/aa", "1/aaa",
"2/a", "2/aa", "2/aaa",
]
expect(beforeExtracted).toEqual(expected);
expect(afterExtracted).toEqual(expected);
})
afterAll(async () => {
if (pathExistsSync(DESTINATION_FOLDER)) {
await rm(DESTINATION_FOLDER, { recursive: true, force: true });
src/main/helpers/zip.helpers.ts
+105 -23
View File
@@ -1,13 +1,36 @@
import crypto from "crypto";
import fs from "fs-extra";
import path from "path";
import yauzl from "yauzl";
import { FileHashes } from "shared/models/mods";
import { ensureFolderExist } from "./fs.helpers";
// NOTE: yauzl needs to be reopened when it is read
// NOTE: yauzl needs to be reopened once readEntry is done
export async function extractZip(zipPath: string, destination: string): Promise<string[]> {
export interface ZipEntry {
name: string;
directory: boolean;
buffer?: Buffer;
};
export interface ZipExtractOptions {
// Stops the extraction immediately when set to true
terminate?: (entry: ZipEntry) => boolean;
// Whether or not to extract the file
condition?: (entry: ZipEntry) => boolean;
// Called before extracting the folder
beforeFolderExtracted?: (folder: string) => void;
// Called when the folder contents is fully extracted
afterFolderExtracted?: (folder: string) => void;
};
export interface ZipProcessOptions {
getBuffer?: boolean;
};
export async function extractZip(
zipPath: string,
destination: string,
options?: Readonly<ZipExtractOptions>
): Promise<string[]> {
await ensureFolderExist(destination);
return new Promise((resolve, reject) => {
@@ -17,16 +40,23 @@ export async function extractZip(zipPath: string, destination: string): Promise<
},
(openError, zip) => {
if (openError) return reject(openError);
handleExtractZip(zip, destination, resolve, reject);
handleExtractZip(zip, destination, options, resolve, reject);
});
});
}
function handleExtractZip(
zip: yauzl.ZipFile, destination: string,
resolve: (result: string[]) => void, reject: (error: any) => void
zip: yauzl.ZipFile,
destination: string,
options: Readonly<ZipExtractOptions> | undefined,
resolve: (result: string[]) => void, reject: (error: Error) => void
) {
const files: string[] = [];
const zipEntry: ZipEntry = {
name: "",
directory: false,
};
let currentDirname = "";
zip.readEntry();
@@ -45,6 +75,37 @@ function handleExtractZip(
zip.openReadStream(entry, (readError, readStream) => {
if (readError) return reject(readError);
const dirname = path.dirname(entry.fileName);
if (dirname !== currentDirname) {
if (path.dirname(dirname) === path.dirname(currentDirname)) {
options?.afterFolderExtracted?.(currentDirname);
options?.beforeFolderExtracted?.(dirname);
} else if (currentDirname.startsWith(dirname)) {
options?.afterFolderExtracted?.(currentDirname);
} else if (dirname.startsWith(currentDirname)) {
options?.beforeFolderExtracted?.(dirname);
} else {
options?.afterFolderExtracted?.(currentDirname);
options?.beforeFolderExtracted?.(dirname);
}
currentDirname = dirname;
}
zipEntry.name = entry.fileName;
zipEntry.directory = false;
zipEntry.buffer = undefined;
if (options?.terminate?.(zipEntry)) {
return readStream.destroy();
}
if (options?.condition) {
if (!options.condition(zipEntry)) {
readStream.destroy();
return zip.readEntry();
}
}
// For forward slashes since they don't create directories
const directory = path.dirname(absolutePath);
if (!fs.pathExistsSync(directory)) {
@@ -53,18 +114,26 @@ function handleExtractZip(
const writeStream = fs.createWriteStream(absolutePath);
readStream.pipe(writeStream);
readStream.on("end", () => zip.readEntry());
readStream.on("end", () => {
zip.readEntry()
});
});
});
zip.once("end", () => {
options?.afterFolderExtracted?.(currentDirname);
zip.close();
resolve(files);
});
}
export function validateZip(zipPath: string, hashes: FileHashes[]): Promise<boolean> {
// @params loop - this should not throw an error
export function processZip<T>(
zipPath: string,
options: Readonly<ZipProcessOptions>,
loop: (accumulator: T, entry: ZipEntry) => T,
initValue: T
): Promise<T> {
return new Promise((resolve, reject) => {
yauzl.open(zipPath, {
lazyEntries: true,
@@ -72,22 +141,35 @@ export function validateZip(zipPath: string, hashes: FileHashes[]): Promise<bool
},
(openError, zip) => {
if (openError) return reject(openError);
handleValidateZip(zip, hashes, resolve, reject);
handleProcessZip<T>(
zip, options, loop, initValue,
resolve, reject
);
});
});
}
function handleValidateZip(
zip: yauzl.ZipFile, hashes: FileHashes[],
resolve: (result: boolean) => void, reject: (error: any) => void
) {
let hashCount = 0;
async function handleProcessZip<T>(
zip: yauzl.ZipFile,
options: Readonly<ZipProcessOptions>,
loop: (accumulator: T, entry: ZipEntry) => T,
accumulator: T,
resolve: (result: T) => void, reject: (error: Error) => void
): Promise<void> {
const zipEntry: ZipEntry = {
name: "",
directory: false,
};
zip.readEntry();
zip.on("entry", (entry: yauzl.Entry) => {
// Entry is a directory / folder
if (entry.fileName.endsWith("/")) {
const isDirectory = entry.fileName.endsWith("/");
if (isDirectory || !options.getBuffer) {
zipEntry.name = entry.fileName;
zipEntry.directory = isDirectory;
zipEntry.buffer = undefined;
accumulator = loop(accumulator, zipEntry);
return zip.readEntry();
}
@@ -101,10 +183,10 @@ function handleValidateZip(
});
readStream.on("end", () => {
const md5Hash = crypto.createHash("md5")
.update(Buffer.concat(chunks))
.digest("hex");
hashCount += +hashes.some(md5 => md5.hash === md5Hash);
zipEntry.name = entry.fileName;
zipEntry.directory = false;
zipEntry.buffer = Buffer.concat(chunks);
accumulator = loop(accumulator, zipEntry);
zip.readEntry();
});
});
@@ -112,7 +194,7 @@ function handleValidateZip(
zip.once("end", () => {
zip.close();
resolve(hashCount === hashes.length);
resolve(accumulator);
});
}
src/main/services/additional-content/maps/local-maps-manager.service.ts
+64 -60
View File
@@ -8,13 +8,12 @@ import { UtilsService } from "../../utils.service";
import crypto, { BinaryLike } from "crypto";
import { lstatSync } from "fs";
import { copy, createReadStream, ensureDir, pathExists, pathExistsSync, realpath, unlink } from "fs-extra";
import StreamZip from "node-stream-zip";
import { RequestService } from "../../request.service";
import sanitize from "sanitize-filename";
import { DeepLinkService } from "../../deep-link.service";
import log from "electron-log";
import { WindowManagerService } from "../../window-manager.service";
import { Observable, Subject, lastValueFrom } from "rxjs";
import { Observable, Subject, lastValueFrom, tap } from "rxjs";
import { Archive } from "../../../models/archive.class";
import { Progression, deleteFolder, ensureFolderExist, getFilesInFolder, getFoldersInFolder, pathExist } from "../../../helpers/fs.helpers";
import { readFile } from "fs/promises";
@@ -30,7 +29,7 @@ import { MapInfo } from "shared/models/maps/info/map-info.model";
import { parseMapInfoDat } from "shared/parsers/maps/map-info.parser";
import { CustomError } from "shared/models/exceptions/custom-error.class";
import { tryit } from "shared/helpers/error.helpers";
import { extractZip } from "main/helpers/zip.helpers";
import { extractZip, processZip } from "main/helpers/zip.helpers";
export class LocalMapsManagerService {
private static instance: LocalMapsManagerService;
@@ -330,11 +329,32 @@ export class LocalMapsManagerService {
let nbImportedMaps = 0;
(async () => {
// BUG: UI bug where the map is not showing even if it was extracted properly
const terminate = () => unsubscribed;
let completeNewFolder: () => void;
const newFolderTap = tap((folder: string) => {
this.loadMapInfoFromPath(folder)
.then(map => {
progress.data = map;
})
.catch(error => {
log.error("Could not load map info", error);
progress.data = null;
})
.finally(() => {
++nbImportedMaps;
++progress.current;
obs.next(progress);
if (progress.current === progress.total) {
completeNewFolder();
}
});
});
const mapsPath = await this.getMapsFolderPath(version);
for(const zipPath of zipPaths) {
if(unsubscribed) {
log.info("Maps importation from zip has been cancelled");
log.info("Maps import from zip has been cancelled");
return;
}
@@ -343,75 +363,59 @@ export class LocalMapsManagerService {
if(!pathExistsSync(zipPath)) { continue; }
const zip = new StreamZip.async({ file: zipPath });
const { result: zipEntries, error } = await tryit(() => zip.entries());
const { result: mapsFolders, error } = await tryit(() => processZip(
zipPath, {},
(set, entry) => {
if (!entry.directory && /(^|\/)[Ii]nfo\.dat$/.test(entry.name)) {
set.add(path.dirname(entry.name));
}
return set;
},
new Set<string>())
);
if(error) {
const res = await tryit(() => zip.close());
log.error("Could not read zip entries", zipPath, error, res?.error);
if (error) {
log.error("Could not read zip entries", zipPath, error);
continue;
}
const zipEntriesValues = Object.values(zipEntries);
const mapsFolders = zipEntriesValues.reduce((acc, entry) => {
if(!/(^|\/)[Ii]nfo\.dat$/.test(entry.name)){ return acc; }
acc.push(path.dirname(entry.name));
return acc;
}, []);
if(mapsFolders.length === 0) {
if (mapsFolders.size === 0) {
log.warn("No maps \"info.dat\" found in zip", zipPath);
}
progress.total = mapsFolders.length;
progress.total = mapsFolders.size;
obs.next(progress);
for(const folder of mapsFolders) {
const newFolder = new Subject<string>();
const promise = lastValueFrom(newFolder.pipe(newFolderTap));
completeNewFolder = () => newFolder.complete();
if(unsubscribed) {
log.info("Maps importation from zip has been cancelled");
await zip.close();
return;
const isRoot = mapsFolders.size === 1 && mapsFolders.has(".");
const destination = isRoot
? path.join(mapsPath, path.basename(zipPath, ".zip"))
: mapsPath;
log.info("Extracting", `"${zipPath}"`, "into", `"${mapsPath}"`);
await extractZip(zipPath, destination, {
terminate,
condition: !isRoot && ((entry) => mapsFolders.has(path.dirname(entry.name))),
beforeFolderExtracted: (folder) => {
if (!mapsFolders.has(folder)) return;
log.info("*", `"${folder}"`);
},
afterFolderExtracted: (folder) => {
if (!mapsFolders.has(folder)) return;
newFolder.next(path.join(destination, folder));
}
});
const isRoot = folder === ".";
const dest = isRoot ? path.join(mapsPath, path.basename(zipPath, ".zip")) : path.join(mapsPath, folder);
let extract: () => Promise<BsmLocalMap>;
if(isRoot){
const entries = zipEntriesValues.filter(entry => entry.isFile && path.dirname(entry.name) === ".");
extract = async () => {
await Promise.all(entries.map(entry => {
log.info("Extracting", `"${entry.name}"`, "from", `"${zipPath}"`, "into", `"${path.join(dest, path.basename(entry.name))}"`);
return zip.extract(entry.name, path.join(dest, path.basename(entry.name)));
}));
return this.loadMapInfoFromPath(dest);
};
} else {
extract = async () => {
log.info("Extracting", `"${folder}"`, "from", `"${zipPath}"`, "into", `"${mapsPath}"`);
await zip.extract(folder, dest);
return this.loadMapInfoFromPath(dest);
}
}
await ensureDir(dest);
const { result: bsmMap, error } = await tryit(extract);
if(error) {
log.error("Could not extract map", zipPath, folder, mapsPath, error);
continue;
}
nbImportedMaps++;
progress.current++;
progress.data = bsmMap;
obs.next(progress);
if (unsubscribed) {
log.info("Maps import from zip has been cancelled");
return;
}
await zip.close();
await promise;
}
})()
.then(() => {
src/main/services/mods/bs-mods-manager.service.ts
+13 -2
View File
@@ -10,7 +10,7 @@ import { BS_EXECUTABLE } from "../../constants";
import log from "electron-log";
import { deleteFolder, ensureFolderExist, pathExist, Progression, unlinkPath } from "../../helpers/fs.helpers";
import { lastValueFrom, Observable } from "rxjs";
import { extractZip, validateZip } from "../../helpers/zip.helpers";
import { extractZip, processZip } from "../../helpers/zip.helpers";
import recursiveReadDir from "recursive-readdir";
import { sToMs } from "../../../shared/helpers/time.helpers";
import { pathExistsSync, unlinkSync } from "fs-extra";
@@ -205,7 +205,18 @@ export class BsModsManagerService {
return false;
}
if (!(await validateZip(zipPath, download.hashMd5))) {
const hashCount = await processZip(zipPath, {
getBuffer: true
}, (acc, entry) => {
if (entry.directory) return acc;
const md5Hash = crypto.createHash("md5")
.update(entry.buffer)
.digest("hex");
return acc + +download.hashMd5.some(md5 => md5.hash === md5Hash);
}, 0);
if (hashCount !== download.hashMd5.length) {
return false;
}