diff --git a/assets/tests/special.zip b/assets/tests/special.zip new file mode 100644 index 00000000..5e8a0fba Binary files /dev/null and b/assets/tests/special.zip differ diff --git a/src/__tests__/unit/zip.test.ts b/src/__tests__/unit/zip.test.ts index 4cbd304c..9f2f382d 100644 --- a/src/__tests__/unit/zip.test.ts +++ b/src/__tests__/unit/zip.test.ts @@ -5,6 +5,7 @@ import { extractZip } from "main/helpers/zip.helpers"; const TEST_FOLDER = path.resolve(__dirname, "../../..", "assets", "tests"); const STANDARD_ZIP = path.join(TEST_FOLDER, "standard.zip"); const WINDOWS_LEGACY_MAP_ZIP = path.join(TEST_FOLDER, "windows_legacy.zip"); +const SPECIAL_ZIP = path.join(TEST_FOLDER, "special.zip"); const DESTINATION_FOLDER = path.join(TEST_FOLDER, "out"); describe("Zip Server Service Test", () => { @@ -67,6 +68,22 @@ describe("Zip Server Service Test", () => { } }); + it("Extract special zip file", async () => { + const beforeExtracted: string[] = []; + const afterExtracted: string[] = []; + await extractZip(SPECIAL_ZIP, DESTINATION_FOLDER, { + beforeFolderExtracted: (folder) => beforeExtracted.push(folder), + afterFolderExtracted: (folder) => afterExtracted.push(folder), + }); + + const expected = [ + "1/a", "1/aa", "1/aaa", + "2/a", "2/aa", "2/aaa", + ] + expect(beforeExtracted).toEqual(expected); + expect(afterExtracted).toEqual(expected); + }) + afterAll(async () => { if (pathExistsSync(DESTINATION_FOLDER)) { await rm(DESTINATION_FOLDER, { recursive: true, force: true }); diff --git a/src/main/helpers/zip.helpers.ts b/src/main/helpers/zip.helpers.ts index 0c9337da..463f8e53 100644 --- a/src/main/helpers/zip.helpers.ts +++ b/src/main/helpers/zip.helpers.ts @@ -1,13 +1,36 @@ -import crypto from "crypto"; import fs from "fs-extra"; import path from "path"; import yauzl from "yauzl"; -import { FileHashes } from "shared/models/mods"; import { ensureFolderExist } from "./fs.helpers"; -// NOTE: yauzl needs to be reopened when it is read +// NOTE: yauzl needs to be reopened once readEntry is done -export async function extractZip(zipPath: string, destination: string): Promise { +export interface ZipEntry { + name: string; + directory: boolean; + buffer?: Buffer; +}; + +export interface ZipExtractOptions { + // Stops the extraction immediately when set to true + terminate?: (entry: ZipEntry) => boolean; + // Whether or not to extract the file + condition?: (entry: ZipEntry) => boolean; + // Called before extracting the folder + beforeFolderExtracted?: (folder: string) => void; + // Called when the folder contents is fully extracted + afterFolderExtracted?: (folder: string) => void; +}; + +export interface ZipProcessOptions { + getBuffer?: boolean; +}; + +export async function extractZip( + zipPath: string, + destination: string, + options?: Readonly +): Promise { await ensureFolderExist(destination); return new Promise((resolve, reject) => { @@ -17,16 +40,23 @@ export async function extractZip(zipPath: string, destination: string): Promise< }, (openError, zip) => { if (openError) return reject(openError); - handleExtractZip(zip, destination, resolve, reject); + handleExtractZip(zip, destination, options, resolve, reject); }); }); } function handleExtractZip( - zip: yauzl.ZipFile, destination: string, - resolve: (result: string[]) => void, reject: (error: any) => void + zip: yauzl.ZipFile, + destination: string, + options: Readonly | undefined, + resolve: (result: string[]) => void, reject: (error: Error) => void ) { const files: string[] = []; + const zipEntry: ZipEntry = { + name: "", + directory: false, + }; + let currentDirname = ""; zip.readEntry(); @@ -45,6 +75,37 @@ function handleExtractZip( zip.openReadStream(entry, (readError, readStream) => { if (readError) return reject(readError); + const dirname = path.dirname(entry.fileName); + if (dirname !== currentDirname) { + if (path.dirname(dirname) === path.dirname(currentDirname)) { + options?.afterFolderExtracted?.(currentDirname); + options?.beforeFolderExtracted?.(dirname); + } else if (currentDirname.startsWith(dirname)) { + options?.afterFolderExtracted?.(currentDirname); + } else if (dirname.startsWith(currentDirname)) { + options?.beforeFolderExtracted?.(dirname); + } else { + options?.afterFolderExtracted?.(currentDirname); + options?.beforeFolderExtracted?.(dirname); + } + currentDirname = dirname; + } + + zipEntry.name = entry.fileName; + zipEntry.directory = false; + zipEntry.buffer = undefined; + + if (options?.terminate?.(zipEntry)) { + return readStream.destroy(); + } + + if (options?.condition) { + if (!options.condition(zipEntry)) { + readStream.destroy(); + return zip.readEntry(); + } + } + // For forward slashes since they don't create directories const directory = path.dirname(absolutePath); if (!fs.pathExistsSync(directory)) { @@ -53,18 +114,26 @@ function handleExtractZip( const writeStream = fs.createWriteStream(absolutePath); readStream.pipe(writeStream); - - readStream.on("end", () => zip.readEntry()); + readStream.on("end", () => { + zip.readEntry() + }); }); }); zip.once("end", () => { + options?.afterFolderExtracted?.(currentDirname); zip.close(); resolve(files); }); } -export function validateZip(zipPath: string, hashes: FileHashes[]): Promise { +// @params loop - this should not throw an error +export function processZip( + zipPath: string, + options: Readonly, + loop: (accumulator: T, entry: ZipEntry) => T, + initValue: T +): Promise { return new Promise((resolve, reject) => { yauzl.open(zipPath, { lazyEntries: true, @@ -72,22 +141,35 @@ export function validateZip(zipPath: string, hashes: FileHashes[]): Promise { if (openError) return reject(openError); - handleValidateZip(zip, hashes, resolve, reject); + handleProcessZip( + zip, options, loop, initValue, + resolve, reject + ); }); }); } -function handleValidateZip( - zip: yauzl.ZipFile, hashes: FileHashes[], - resolve: (result: boolean) => void, reject: (error: any) => void -) { - let hashCount = 0; +async function handleProcessZip( + zip: yauzl.ZipFile, + options: Readonly, + loop: (accumulator: T, entry: ZipEntry) => T, + accumulator: T, + resolve: (result: T) => void, reject: (error: Error) => void +): Promise { + const zipEntry: ZipEntry = { + name: "", + directory: false, + }; zip.readEntry(); zip.on("entry", (entry: yauzl.Entry) => { - // Entry is a directory / folder - if (entry.fileName.endsWith("/")) { + const isDirectory = entry.fileName.endsWith("/"); + if (isDirectory || !options.getBuffer) { + zipEntry.name = entry.fileName; + zipEntry.directory = isDirectory; + zipEntry.buffer = undefined; + accumulator = loop(accumulator, zipEntry); return zip.readEntry(); } @@ -101,10 +183,10 @@ function handleValidateZip( }); readStream.on("end", () => { - const md5Hash = crypto.createHash("md5") - .update(Buffer.concat(chunks)) - .digest("hex"); - hashCount += +hashes.some(md5 => md5.hash === md5Hash); + zipEntry.name = entry.fileName; + zipEntry.directory = false; + zipEntry.buffer = Buffer.concat(chunks); + accumulator = loop(accumulator, zipEntry); zip.readEntry(); }); }); @@ -112,7 +194,7 @@ function handleValidateZip( zip.once("end", () => { zip.close(); - resolve(hashCount === hashes.length); + resolve(accumulator); }); } diff --git a/src/main/services/additional-content/maps/local-maps-manager.service.ts b/src/main/services/additional-content/maps/local-maps-manager.service.ts index b20808ee..a737e82b 100644 --- a/src/main/services/additional-content/maps/local-maps-manager.service.ts +++ b/src/main/services/additional-content/maps/local-maps-manager.service.ts @@ -8,13 +8,12 @@ import { UtilsService } from "../../utils.service"; import crypto, { BinaryLike } from "crypto"; import { lstatSync } from "fs"; import { copy, createReadStream, ensureDir, pathExists, pathExistsSync, realpath, unlink } from "fs-extra"; -import StreamZip from "node-stream-zip"; import { RequestService } from "../../request.service"; import sanitize from "sanitize-filename"; import { DeepLinkService } from "../../deep-link.service"; import log from "electron-log"; import { WindowManagerService } from "../../window-manager.service"; -import { Observable, Subject, lastValueFrom } from "rxjs"; +import { Observable, Subject, lastValueFrom, tap } from "rxjs"; import { Archive } from "../../../models/archive.class"; import { Progression, deleteFolder, ensureFolderExist, getFilesInFolder, getFoldersInFolder, pathExist } from "../../../helpers/fs.helpers"; import { readFile } from "fs/promises"; @@ -30,7 +29,7 @@ import { MapInfo } from "shared/models/maps/info/map-info.model"; import { parseMapInfoDat } from "shared/parsers/maps/map-info.parser"; import { CustomError } from "shared/models/exceptions/custom-error.class"; import { tryit } from "shared/helpers/error.helpers"; -import { extractZip } from "main/helpers/zip.helpers"; +import { extractZip, processZip } from "main/helpers/zip.helpers"; export class LocalMapsManagerService { private static instance: LocalMapsManagerService; @@ -330,11 +329,32 @@ export class LocalMapsManagerService { let nbImportedMaps = 0; (async () => { + // BUG: UI bug where the map is not showing even if it was extracted properly + const terminate = () => unsubscribed; + let completeNewFolder: () => void; + const newFolderTap = tap((folder: string) => { + this.loadMapInfoFromPath(folder) + .then(map => { + progress.data = map; + }) + .catch(error => { + log.error("Could not load map info", error); + progress.data = null; + }) + .finally(() => { + ++nbImportedMaps; + ++progress.current; + obs.next(progress); + if (progress.current === progress.total) { + completeNewFolder(); + } + }); + }); + const mapsPath = await this.getMapsFolderPath(version); for(const zipPath of zipPaths) { - if(unsubscribed) { - log.info("Maps importation from zip has been cancelled"); + log.info("Maps import from zip has been cancelled"); return; } @@ -343,75 +363,59 @@ export class LocalMapsManagerService { if(!pathExistsSync(zipPath)) { continue; } - const zip = new StreamZip.async({ file: zipPath }); - const { result: zipEntries, error } = await tryit(() => zip.entries()); + const { result: mapsFolders, error } = await tryit(() => processZip( + zipPath, {}, + (set, entry) => { + if (!entry.directory && /(^|\/)[Ii]nfo\.dat$/.test(entry.name)) { + set.add(path.dirname(entry.name)); + } + return set; + }, + new Set()) + ); - if(error) { - const res = await tryit(() => zip.close()); - log.error("Could not read zip entries", zipPath, error, res?.error); + + if (error) { + log.error("Could not read zip entries", zipPath, error); continue; } - const zipEntriesValues = Object.values(zipEntries); - - const mapsFolders = zipEntriesValues.reduce((acc, entry) => { - if(!/(^|\/)[Ii]nfo\.dat$/.test(entry.name)){ return acc; } - acc.push(path.dirname(entry.name)); - return acc; - }, []); - - if(mapsFolders.length === 0) { + if (mapsFolders.size === 0) { log.warn("No maps \"info.dat\" found in zip", zipPath); } - progress.total = mapsFolders.length; + progress.total = mapsFolders.size; obs.next(progress); - for(const folder of mapsFolders) { + const newFolder = new Subject(); + const promise = lastValueFrom(newFolder.pipe(newFolderTap)); + completeNewFolder = () => newFolder.complete(); - if(unsubscribed) { - log.info("Maps importation from zip has been cancelled"); - await zip.close(); - return; + const isRoot = mapsFolders.size === 1 && mapsFolders.has("."); + const destination = isRoot + ? path.join(mapsPath, path.basename(zipPath, ".zip")) + : mapsPath; + + log.info("Extracting", `"${zipPath}"`, "into", `"${mapsPath}"`); + await extractZip(zipPath, destination, { + terminate, + condition: !isRoot && ((entry) => mapsFolders.has(path.dirname(entry.name))), + beforeFolderExtracted: (folder) => { + if (!mapsFolders.has(folder)) return; + log.info("*", `"${folder}"`); + }, + afterFolderExtracted: (folder) => { + if (!mapsFolders.has(folder)) return; + newFolder.next(path.join(destination, folder)); } + }); - const isRoot = folder === "."; - const dest = isRoot ? path.join(mapsPath, path.basename(zipPath, ".zip")) : path.join(mapsPath, folder); - - let extract: () => Promise; - - if(isRoot){ - const entries = zipEntriesValues.filter(entry => entry.isFile && path.dirname(entry.name) === "."); - extract = async () => { - await Promise.all(entries.map(entry => { - log.info("Extracting", `"${entry.name}"`, "from", `"${zipPath}"`, "into", `"${path.join(dest, path.basename(entry.name))}"`); - return zip.extract(entry.name, path.join(dest, path.basename(entry.name))); - })); - return this.loadMapInfoFromPath(dest); - }; - } else { - extract = async () => { - log.info("Extracting", `"${folder}"`, "from", `"${zipPath}"`, "into", `"${mapsPath}"`); - await zip.extract(folder, dest); - return this.loadMapInfoFromPath(dest); - } - } - - await ensureDir(dest); - const { result: bsmMap, error } = await tryit(extract); - - if(error) { - log.error("Could not extract map", zipPath, folder, mapsPath, error); - continue; - } - - nbImportedMaps++; - progress.current++; - progress.data = bsmMap; - obs.next(progress); + if (unsubscribed) { + log.info("Maps import from zip has been cancelled"); + return; } - await zip.close(); + await promise; } })() .then(() => { diff --git a/src/main/services/mods/bs-mods-manager.service.ts b/src/main/services/mods/bs-mods-manager.service.ts index 48d5f8f5..a2535099 100644 --- a/src/main/services/mods/bs-mods-manager.service.ts +++ b/src/main/services/mods/bs-mods-manager.service.ts @@ -10,7 +10,7 @@ import { BS_EXECUTABLE } from "../../constants"; import log from "electron-log"; import { deleteFolder, ensureFolderExist, pathExist, Progression, unlinkPath } from "../../helpers/fs.helpers"; import { lastValueFrom, Observable } from "rxjs"; -import { extractZip, validateZip } from "../../helpers/zip.helpers"; +import { extractZip, processZip } from "../../helpers/zip.helpers"; import recursiveReadDir from "recursive-readdir"; import { sToMs } from "../../../shared/helpers/time.helpers"; import { pathExistsSync, unlinkSync } from "fs-extra"; @@ -205,7 +205,18 @@ export class BsModsManagerService { return false; } - if (!(await validateZip(zipPath, download.hashMd5))) { + const hashCount = await processZip(zipPath, { + getBuffer: true + }, (acc, entry) => { + if (entry.directory) return acc; + + const md5Hash = crypto.createHash("md5") + .update(entry.buffer) + .digest("hex"); + return acc + +download.hashMd5.some(md5 => md5.hash === md5Hash); + }, 0); + + if (hashCount !== download.hashMd5.length) { return false; }