Merge pull request #649 from silentrald/bugfix/non-standard-zip-extraction

[bugfix] handle extracting non-conformant zip files

(cherry picked from commit f7f5f768fc)

src/main/helpers/zip.helpers.ts

@@ -1,28 +0,0 @@
-import JSZip from "jszip";
-import { pathExist } from "./fs.helpers";
-import path from "path";
-import { mkdir, writeFile } from "fs/promises";
-
-export async function extractZip(zip: JSZip, dest: string): Promise<string[]> {
-    if (!(await pathExist(dest))) {
-        throw new Error(`Path ${dest} does not exist`);
-    }
-    const files: string[] = [];
-
-    for (const [relativePath, entry] of Object.entries(zip.files)) {
-        if (entry.dir) {
-            continue;
-        }
-
-        const content = await entry.async("nodebuffer");
-        const outPath = path.join(dest, relativePath);
-        const outDir = path.dirname(outPath);
-
-        await mkdir(outDir, { recursive: true });
-        await writeFile(outPath, content);
-
-        files.push(outPath);
-    }
-
-    return files;
-}

src/main/models/bsm-zip-extractor.class.ts

@@ -0,0 +1,218 @@
+import { createWriteStream, ensureDir } from "fs-extra";
+import path from "path";
+import { Readable } from "stream";
+import yauzl, { ZipFile, Options, Entry } from "yauzl"
+
+export class BsmZipExtractor {
+
+    private static readonly YAUZL_OPEN_OPTIONS: Options = {
+        lazyEntries: true,
+        decodeStrings: true,
+        autoClose: false,
+    };
+
+    public static fromPath(path: string): Promise<BsmZipExtractor> {
+        return new Promise((resolve, reject) => {
+            yauzl.open(path, BsmZipExtractor.YAUZL_OPEN_OPTIONS, (error: Error, zip: yauzl.ZipFile) => {
+                if (error) return reject(error);
+                resolve(new BsmZipExtractor(zip));
+            });
+        });
+    }
+
+    public static fromBuffer(buffer: Buffer): Promise<BsmZipExtractor> {
+        return new Promise((resolve, reject) => {
+            yauzl.fromBuffer(buffer, BsmZipExtractor.YAUZL_OPEN_OPTIONS, (error: Error, zip: yauzl.ZipFile) => {
+                if (error) return reject(error);
+                resolve(new BsmZipExtractor(zip));
+            });
+        });
+    }
+
+    private readonly zip: ZipFile;
+    private readonly entriesMap = new Map<string, BsmZipExtractorEntry>();
+
+    private constructor(zip: ZipFile) {
+        this.zip = zip;
+    }
+
+    private async readEntry(): Promise<BsmZipExtractorEntry> {
+        return new Promise((resolve, reject) => {
+            const onEntry = (entry: Entry) => {
+                cleanup();
+                resolve(new BsmZipExtractorEntry({ entry, zip: this.zip }));
+            };
+
+            const onEnd = () => {
+                cleanup();
+                resolve(null);
+            };
+
+            const onError = (err: Error) => {
+                cleanup();
+                reject(err);
+            };
+
+            const cleanup = () => {
+                this.zip.removeListener("entry", onEntry);
+                this.zip.removeListener("end", onEnd);
+                this.zip.removeListener("error", onError);
+            };
+
+            this.zip.once("entry", onEntry);
+            this.zip.once("end", onEnd);
+            this.zip.once("error", onError);
+            this.zip.readEntry();
+        });
+    }
+
+    // The first time this method is called, it will read all the entries and store them in a map
+    // The next times it will return the entries from the map to avoid reopening the zip file again
+    public async *entries(): AsyncGenerator<BsmZipExtractorEntry> {
+
+        for (const entry of Array.from(this.entriesMap.values())) {
+            yield entry;
+        }
+
+        let entry: BsmZipExtractorEntry = await this.readEntry();
+        while (entry) {
+            this.entriesMap.set(entry.fileName, entry);
+            yield entry;
+            entry = await this.readEntry();
+        }
+    }
+
+    public async findEntry(func: (entry: BsmZipExtractorEntry) => boolean): Promise<BsmZipExtractorEntry|null> {
+        for await (const entry of this.entries()) {
+            if (func(entry)) {
+                return entry;
+            }
+        }
+        return null;
+    }
+
+    public async filterEntries(func: (entry: BsmZipExtractorEntry) => boolean): Promise<BsmZipExtractorEntry[]> {
+        const filtered: BsmZipExtractorEntry[] = [];
+        for await (const entry of this.entries()) {
+            if (func(entry)) {
+                filtered.push(entry);
+            }
+        }
+        return filtered;
+    }
+
+    public async getEntry(fileName: string): Promise<BsmZipExtractorEntry> {
+        return this.entriesMap.get(fileName) ?? this.findEntry(entry => entry.fileName === fileName);
+    }
+
+    /**
+     * Extracts all entries from the zip file to the destination folder
+     * @param destination
+     * @param opt - { entriesNames?: string[], abortToken: AbortController }
+     * @returns {Promise<string[]>} The list of extracted files (relative paths in the destination folder)
+     *
+     * `opt` object:
+     *   - `entriesNames` - The list of entries to extract (can be regexs or glob pattern). If not provided, all entries will be extracted
+     *   - `abortToken` - The AbortController instance to abort the extraction
+     *
+     */
+    public async extract(destination: string, opt?: { entriesNames?: (string|RegExp)[], abortToken?: AbortController }): Promise<string[]> {
+        const entriesNames = opt?.entriesNames;
+        const abortToken = opt?.abortToken;
+
+        if (abortToken?.signal?.aborted) {
+            return [];
+        }
+
+        await ensureDir(destination);
+
+        const extracted = new Set<string>()
+        for await (const entry of this.entries()) {
+
+            if (abortToken?.signal?.aborted) {
+                break;
+            }
+
+            if (entriesNames && !entriesNames.some(name => typeof name === "string" ? entry.fileName === name : name.test(entry.fileName))){
+                continue;
+            }
+
+            const extractedFile = await entry.extract(destination);
+            const dirname = path.dirname(extractedFile);
+
+            // Make zip that use backslash as separator have the same behavior as zip that use forward slash
+            if(dirname !== "."){
+                const split = dirname.split(path.posix.sep);
+                for (let i = 1; i <= split.length; i++) {
+                    const folder = split.slice(0, i).join(path.posix.sep);
+                    extracted.add(folder + path.posix.sep);
+                }
+            }
+
+
+            extracted.add(extractedFile);
+        }
+
+        return Array.from(extracted);
+    }
+
+
+    public close(): void {
+        this.zip.close();
+    }
+
+}
+
+class BsmZipExtractorEntry {
+
+    private readonly entry: Entry;
+    private readonly zip: ZipFile;
+
+    constructor(opt: { entry: Entry, zip: ZipFile }) {
+        this.entry = opt.entry;
+        this.zip = opt.zip;
+    }
+
+    public async read(): Promise<Buffer> {
+        return new Promise((resolve, reject) => {
+            this.zip.openReadStream(this.entry, (error: Error, stream: Readable) => {
+                if (error) return reject(error);
+                const buffers: Buffer[] = [];
+                stream.on("data", (data) => buffers.push(data as Buffer));
+                stream.on("end", () => resolve(Buffer.concat(buffers)));
+                stream.on("error", reject);
+            });
+        });
+    }
+
+    public async extract(destination: string): Promise<string> {
+
+        const destPath = path.join(destination, this.entry.fileName);
+
+        if (this.isDirectory) {
+            return ensureDir(destPath).then(() => this.entry.fileName);
+        }
+
+        return new Promise((resolve, reject) => {
+            this.zip.openReadStream(this.entry, async (error: Error, stream: Readable) => {
+                if (error) return reject(error);
+
+                await ensureDir(path.dirname(destPath));
+
+                const writeStream = createWriteStream(destPath);
+                stream.pipe(writeStream);
+                writeStream.on("finish", () => resolve(this.entry.fileName));
+                writeStream.on("error", reject);
+            });
+        });
+    }
+
+    public get isDirectory(): boolean {
+        return this.entry.fileName.endsWith("/");
+    }
+
+    public get fileName(): string {
+        return this.entry.fileName;
+    }
+
+}

src/main/models/oculus-downloader.class.ts

@@ -1,4 +1,3 @@
-import JSZip from "jszip";
 import fetch from "node-fetch";
 import { CustomError } from "../../shared/models/exceptions/custom-error.class";
 import { mkdirs, createWriteStream, pathExists, WriteStream } from "fs-extra";
@@ -7,6 +6,8 @@ import { inflate } from "pako"
 import { EMPTY, Observable, ReplaySubject, Subscriber, catchError, filter, from, lastValueFrom, mergeMap, scan, share, tap } from "rxjs";
 import { Progression, hashFile } from "../helpers/fs.helpers";
 import { OculusDownloaderErrorCodes } from "../../shared/models/bs-version-download/oculus-download.model";
+import { BsmZipExtractor } from "./bsm-zip-extractor.class";
+import { tryit } from "../../shared/helpers/error.helpers";
 
 export class OculusDownloader {
 
@@ -23,26 +24,38 @@ export class OculusDownloader {
         return `https://securecdn.oculus.com/binaries/segment/?access_token=${token}&binary_id=${binaryId}&segment_sha256=${segmentSha256}`;
     }
 
-    private async downloadManifestZip(manifestUrl: string): Promise<JSZip> {
+    private async downloadManifestZip(manifestUrl: string): Promise<Buffer> {
         const response = await fetch(manifestUrl);
         const arrBuffer = await response.arrayBuffer();
-        return JSZip.loadAsync(arrBuffer);
+        return Buffer.from(arrBuffer);
     }
 
     private async getManifest(): Promise<OculusManifest> {
         const downloadUrl = this.getDownloadManifestUrl(this.options.accessToken, this.options.binaryId);
-        const manifestZip = await this.downloadManifestZip(downloadUrl).catch(err => CustomError.throw(err, "DOWNLOAD_MANIFEST_FAILED"));
-        const manifestFile = manifestZip.file("manifest.json");
+        const buffer = await this.downloadManifestZip(downloadUrl)
+            .catch(err => CustomError.throw(err, "DOWNLOAD_MANIFEST_FAILED"));
 
-        if(!manifestFile){
+        const zip = await BsmZipExtractor.fromBuffer(buffer);
+        const entry = await zip.getEntry("manifest.json");
+
+        if(!entry) {
             throw new CustomError("Manifest file not found", "MANIFEST_FILE_NOT_FOUND");
         }
 
-        return manifestFile.async("text").then(JSON.parse).catch(err => CustomError.throw(err, "PARSE_MANIFEST_FILE_FAILED"));
+        const manifest = await entry.read();
+        zip.close();
+
+        const { result, error } = tryit(() => JSON.parse(manifest.toString()) as OculusManifest);
+
+        if(error){
+            throw CustomError.throw(error, "PARSE_MANIFEST_FILE_FAILED")
+        }
+
+        return result;
     }
 
     private downloadManifestFile(file: OculusManifestFile, destination: string): Observable<Progression<OculusManifestFile>> {
-            
+
             const downloadSegment = async (segment: OculusManifestFileSegment): Promise<ArrayBuffer> => {
                 const segmentUrl = this.getDownloadSegmentUrl(this.options.accessToken, this.options.binaryId, segment[1]);
                 const response = await fetch(segmentUrl);
@@ -66,11 +79,11 @@ export class OculusDownloader {
 
                         const arrBuffer = await downloadSegment(segment);
                         const inflated = inflate(arrBuffer);
-                        await writeStream.write(inflated);
+                        writeStream.write(inflated);
 
                         progress.current += inflated.byteLength;
                         progress.diff = inflated.byteLength;
-                        
+
                         sub.next(progress);
                     }
 
@@ -86,7 +99,7 @@ export class OculusDownloader {
             });
     }
 
-    private isFileIntegrityValid(file: OculusFileWithName, folder: string): Promise<boolean> {
+    private async isFileIntegrityValid(file: OculusFileWithName, folder: string): Promise<boolean> {
         const [fileName, fileData] = file;
         const destination = path.join(folder, fileName);
 
@@ -138,7 +151,7 @@ export class OculusDownloader {
             if(this.isDownloading){
                 throw new CustomError("Already downloading", "ALREADY_DOWNLOADING");
             }
-    
+
             this.options = options;
             this.isDownloading = true;
 
@@ -182,7 +195,7 @@ export class OculusDownloader {
                 })));
 
                 const integrity = await lastValueFrom(this.verifyIntegrity(manifest, options.destination)).catch(err => CustomError.throw(err, "VERIFY_INTEGRITY_FAILED"));
-                
+
                 if(integrity.data.length > 0){
                     throw new CustomError("Some files failed to download", "SOME_FILES_FAILED_TO_DOWNLOAD", integrity.data);
                 }
@@ -242,4 +255,4 @@ interface Logger {
     info: (...args: unknown[]) => void;
     warn: (...args: unknown[]) => void;
     error: (...args: unknown[]) => void;
-}
+}

src/main/services/additional-content/local-maps-manager.service.ts

@@ -8,7 +8,6 @@ import { UtilsService } from "../utils.service";
 import crypto from "crypto";
 import { lstatSync } from "fs";
 import { copy, createReadStream, ensureDir, pathExists, realpath, unlink } from "fs-extra";
-import StreamZip from "node-stream-zip";
 import { RequestService } from "../request.service";
 import sanitize from "sanitize-filename";
 import { DeepLinkService } from "../deep-link.service";
@@ -23,6 +22,7 @@ import { allSettled } from "../../../shared/helpers/promise.helpers";
 import { splitIntoChunk } from "../../../shared/helpers/array.helpers";
 import { IpcService } from "../ipc.service";
 import { parseMapInfoDat } from "../../../shared/parsers/maps/map-info.parser";
+import { BsmZipExtractor } from "../../models/bsm-zip-extractor.class";
 
 export class LocalMapsManagerService {
     private static instance: LocalMapsManagerService;
@@ -137,18 +137,15 @@ export class LocalMapsManagerService {
         return { mapInfo, coverUrl, songUrl, hash, path: mapPath };
     }
 
-    private async downloadMapZip(zipUrl: string): Promise<{ zip: StreamZip.StreamZipAsync; zipPath: string }> {
+    private async downloadMapZip(zipUrl: string): Promise<string> {
         const fileName = `${path.basename(zipUrl, ".zip")}-${crypto.randomUUID()}.zip`;
         const tempPath = this.utils.getTempPath();
         await ensureFolderExist(this.utils.getTempPath());
 
-        const zipPath = (await lastValueFrom(this.reqService.downloadFile(zipUrl, {
+        return (await lastValueFrom(this.reqService.downloadFile(zipUrl, {
             destFolder: tempPath,
             filename: fileName
         }))).data;
-        const zip = new StreamZip.async({ file: zipPath });
-
-        return { zip, zipPath };
     }
 
     private openOneClickDownloadMapWindow(mapId: string, isHash = false): void {
@@ -279,16 +276,15 @@ export class LocalMapsManagerService {
             return installedMap;
         }
 
-        const { zip, zipPath } = await this.downloadMapZip(zipUrl);
+        const zipPath = await this.downloadMapZip(zipUrl);
+        const zip = await BsmZipExtractor.fromPath(zipPath);
 
         if (!zip) {
             throw `Cannot download ${zipUrl}`;
         }
 
-        await ensureFolderExist(mapPath);
-
-        await zip.extract(null, mapPath);
-        await zip.close();
+        await zip.extract(mapPath);
+        zip.close();
         await unlink(zipPath);
 
         const localMap = await this.loadMapInfoFromPath(mapPath);

src/main/services/mods/bs-mods-manager.service.ts

@@ -11,11 +11,11 @@ import { BS_EXECUTABLE } from "../../constants";
 import log from "electron-log";
 import { deleteFolder, pathExist, unlinkPath } from "../../helpers/fs.helpers";
 import { lastValueFrom } from "rxjs";
-import JSZip from "jszip";
-import { extractZip } from "../../helpers/zip.helpers";
 import recursiveReadDir from "recursive-readdir";
 import { sToMs } from "../../../shared/helpers/time.helpers";
-import { ensureDir, pathExistsSync } from "fs-extra";
+import { pathExistsSync } from "fs-extra";
+import { BsmZipExtractor } from "../../models/bsm-zip-extractor.class";
+import crypto from "crypto";
 
 export class BsModsManagerService {
     private static instance: BsModsManagerService;
@@ -45,6 +45,7 @@ export class BsModsManagerService {
         this.bsLocalService = BSLocalVersionService.getInstance();
         this.utilsService = UtilsService.getInstance();
         this.requestService = RequestService.getInstance();
+        this.utilsService = UtilsService.getInstance();
     }
 
     private async getModFromHash(hash: string): Promise<Mod> {
@@ -123,7 +124,7 @@ export class BsModsManagerService {
         return this.getIpaFromHash(injectorMd5);
     }
 
-    private async downloadZip(zipUrl: string): Promise<JSZip> {
+    private async downloadZip(zipUrl: string): Promise<BsmZipExtractor> {
         zipUrl = path.join(this.beatModsApi.BEAT_MODS_URL, zipUrl);
 
         log.info("Download mod zip", zipUrl);
@@ -139,10 +140,7 @@ export class BsModsManagerService {
             return null;
         }
 
-        return JSZip.loadAsync(buffer).catch(e => {
-            log.error("ZIP", "Error while loading zip", e);
-            return null;
-        });
+        return BsmZipExtractor.fromBuffer(buffer);
     }
 
     private async executeBSIPA(version: BSVersion, args: string[]): Promise<boolean> {
@@ -209,43 +207,38 @@ export class BsModsManagerService {
 
         log.info("Start download mod zip", mod.name, download.url);
         const zip = await this.downloadZip(download.url);
-        log.info("Mod zip download end", mod.name, download.url, !!zip);
+        log.info("Mod zip download end", mod.name, download.url);
 
         if (!zip) {
             return false;
         }
 
-        const crypto = require("crypto");
-        const files = await zip.files;
+        let hashCount = 0;
+        for await (const entry of zip.entries()) {
+            const buffer = await entry.read();
+            const md5Hash = crypto.createHash("md5")
+                .update(buffer)
+                .digest("hex");
+            hashCount += +download.hashMd5.some(md5 => md5.hash === md5Hash);
+        }
 
-        const checkedEntries = (
-            await Promise.all(
-                Object.values(files).map(async entry => {
-                    const data = await entry.async("nodebuffer");
-                    const entryMd5 = crypto.createHash("md5").update(data).digest("hex");
-                    return download.hashMd5.some(md5 => md5.hash === entryMd5) ? entry : undefined;
-                })
-            ).catch(e => {
-                log.error("Error while checking mod zip entries", mod.name, e);
-                throw e;
-            })
-        ).filter(entry => !!entry);
-
-        if (checkedEntries.length !== download.hashMd5.length) {
+        if (hashCount !== download.hashMd5.length) {
             return false;
         }
 
-        const verionPath = await this.bsLocalService.getVersionPath(version);
+        const versionPath = await this.bsLocalService.getVersionPath(version);
         const isBSIPA = mod.name.toLowerCase() === "bsipa";
-        const destDir = isBSIPA ? verionPath : path.join(verionPath, ModsInstallFolder.PENDING);
+        const destDir = isBSIPA ? versionPath : path.join(versionPath, ModsInstallFolder.PENDING);
 
-        await ensureDir(destDir);
         log.info("Start extracting mod zip", mod.name, "to", destDir);
-        const extracted = await extractZip(zip, destDir)
+        const extracted = await zip.extract(destDir)
             .then(() => true)
             .catch(e => {
                 log.error("Error while extracting mod zip", e);
                 return false;
+            })
+            .finally(() => {
+                zip.close();
             });
 
         log.info("Mod zip extraction end", mod.name, "to", destDir, "success:", extracted);