diff --git a/.gitattributes b/.gitattributes index c26f0c38..7c651287 100644 --- a/.gitattributes +++ b/.gitattributes @@ -16,3 +16,4 @@ *.woff binary *.woff2 binary assets/scripts/* binary +assets/tests/*.zip binary diff --git a/.github/workflows/node.js.yaml b/.github/workflows/node.js.yaml index b666d9bb..3a7b1b25 100644 --- a/.github/workflows/node.js.yaml +++ b/.github/workflows/node.js.yaml @@ -13,7 +13,7 @@ jobs: build: strategy: matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-latest, windows-latest] runs-on: ${{ matrix.os }} @@ -26,6 +26,7 @@ jobs: cache: "npm" - run: npm ci - run: npm run build - - run: npm test - # Currently typescript, eslint, and prettier are unhappy - continue-on-error: true + + - name: Unit tests + run: npm run test:unit + diff --git a/assets/tests/standard.zip b/assets/tests/standard.zip new file mode 100644 index 00000000..96b81a8e Binary files /dev/null and b/assets/tests/standard.zip differ diff --git a/assets/tests/windows_legacy.zip b/assets/tests/windows_legacy.zip new file mode 100644 index 00000000..3ac92eae Binary files /dev/null and b/assets/tests/windows_legacy.zip differ diff --git a/jest.config.ts b/jest.config.ts new file mode 100644 index 00000000..30d53b30 --- /dev/null +++ b/jest.config.ts @@ -0,0 +1,19 @@ +import type { Config } from "jest"; + +const config: Config = { + // testURL: "http://localhost/", + // testEnvironment: "jsdom", + transform: { + "\\.(ts|tsx|js|jsx)$": "ts-jest", + }, + moduleNameMapper: { + "\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$": "/.erb/mocks/fileMock.js", + "\\.(css|less|sass|scss)$": "identity-obj-proxy", + }, + moduleFileExtensions: ["js", "jsx", "ts", "tsx", "json"], + moduleDirectories: ["node_modules", "src"], + testPathIgnorePatterns: ["release/app/dist"], + setupFiles: ["./.erb/scripts/check-build-exists.ts"], +}; + +export default config; diff --git a/package-lock.json b/package-lock.json index 009acc5b..cfc5e2ed 100644 --- a/package-lock.json +++ b/package-lock.json @@ -61,7 +61,8 @@ "tippy.js": "^6.3.7", "to-ico": "^1.1.5", "use-double-click": "^1.0.5", - "use-fit-text": "^2.4.0" + "use-fit-text": "^2.4.0", + "yauzl": "^3.2.0" }, "devDependencies": { "@electron/fuses": "^1.7.0", @@ -8555,7 +8556,6 @@ }, "node_modules/buffer-crc32": { "version": "0.2.13", - "dev": true, "license": "MIT", "engines": { "node": "*" @@ -12585,6 +12585,17 @@ "@types/yauzl": "^2.9.1" } }, + "node_modules/extract-zip/node_modules/yauzl": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", + "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer-crc32": "~0.2.3", + "fd-slicer": "~1.1.0" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "license": "MIT" @@ -12662,6 +12673,8 @@ }, "node_modules/fd-slicer": { "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", + "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", "dev": true, "license": "MIT", "dependencies": { @@ -17801,7 +17814,8 @@ }, "node_modules/pend": { "version": "1.2.0", - "dev": true, + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", "license": "MIT" }, "node_modules/performance-now": { @@ -22655,12 +22669,16 @@ } }, "node_modules/yauzl": { - "version": "2.10.0", - "dev": true, + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-3.2.0.tgz", + "integrity": "sha512-Ow9nuGZE+qp1u4JIPvg+uCiUr7xGQWdff7JQSk5VGYTAZMDe2q8lxJ10ygv10qmSj031Ty/6FNJpLO4o1Sgc+w==", "license": "MIT", "dependencies": { "buffer-crc32": "~0.2.3", - "fd-slicer": "~1.1.0" + "pend": "~1.2.0" + }, + "engines": { + "node": ">=12" } }, "node_modules/yn": { diff --git a/package.json b/package.json index f1998258..6e87a389 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "start:preload": "cross-env NODE_ENV=development TS_NODE_TRANSPILE_ONLY=true webpack --config ./.erb/configs/webpack.config.preload.dev.ts", "start:renderer": "cross-env NODE_ENV=development TS_NODE_TRANSPILE_ONLY=true webpack serve --config ./.erb/configs/webpack.config.renderer.dev.ts", "test": "jest", + "test:unit": "jest ./src/__tests__/unit", "publish": "npm run build && electron-builder -c.win.certificateSha1=2164d6a7d641ecf6ad57852f665a518ca2bf960f --publish always --win --x64", "publish:linux": "npm run build && electron-builder --publish always --linux --x64" }, @@ -116,34 +117,6 @@ "beat-saber" ], "homepage": "https://github.com/Zagrios/bs-manager#readme", - "jest": { - "testURL": "http://localhost/", - "testEnvironment": "jsdom", - "transform": { - "\\.(ts|tsx|js|jsx)$": "ts-jest" - }, - "moduleNameMapper": { - "\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$": "/.erb/mocks/fileMock.js", - "\\.(css|less|sass|scss)$": "identity-obj-proxy" - }, - "moduleFileExtensions": [ - "js", - "jsx", - "ts", - "tsx", - "json" - ], - "moduleDirectories": [ - "node_modules", - "src" - ], - "testPathIgnorePatterns": [ - "release/app/dist" - ], - "setupFiles": [ - "./.erb/scripts/check-build-exists.ts" - ] - }, "devDependencies": { "@electron/fuses": "^1.7.0", "@electron/notarize": "^2.3.0", @@ -281,7 +254,8 @@ "tippy.js": "^6.3.7", "to-ico": "^1.1.5", "use-double-click": "^1.0.5", - "use-fit-text": "^2.4.0" + "use-fit-text": "^2.4.0", + "yauzl": "^3.2.0" }, "engines": { "node": ">=20.0.0" diff --git a/src/__tests__/unit/zip.test.ts b/src/__tests__/unit/zip.test.ts new file mode 100644 index 00000000..4cbd304c --- /dev/null +++ b/src/__tests__/unit/zip.test.ts @@ -0,0 +1,75 @@ +import path from "path"; +import { mkdir, pathExistsSync, rm } from "fs-extra"; +import { extractZip } from "main/helpers/zip.helpers"; + +const TEST_FOLDER = path.resolve(__dirname, "../../..", "assets", "tests"); +const STANDARD_ZIP = path.join(TEST_FOLDER, "standard.zip"); +const WINDOWS_LEGACY_MAP_ZIP = path.join(TEST_FOLDER, "windows_legacy.zip"); +const DESTINATION_FOLDER = path.join(TEST_FOLDER, "out"); + +describe("Zip Server Service Test", () => { + beforeAll(async () => { + if (pathExistsSync(DESTINATION_FOLDER)) { + await rm(DESTINATION_FOLDER, { recursive: true, force: true }); + } + await mkdir(DESTINATION_FOLDER); + }); + + // Uses '/' + it("Extract standard zip", async () => { + await extractZip(STANDARD_ZIP, DESTINATION_FOLDER); + + for (const file of [ + "file_1.2.txt", + "file_1.3.txt", + ]) { + expect(pathExistsSync(path.join(DESTINATION_FOLDER, file))) + .toBe(true); + } + + const SUBFOLDER_PATH = path.join(DESTINATION_FOLDER, "folder_1.1"); + for (const file of [ + "file_2.1.txt", + "file_2.2.txt", + ]) { + expect(pathExistsSync(path.join(SUBFOLDER_PATH, file))) + .toBe(true); + } + }); + + // Uses '\\' + it("Extract map zips using forward slashes", async () => { + await extractZip(WINDOWS_LEGACY_MAP_ZIP, DESTINATION_FOLDER); + + // Expect all the files to exists + for (const file of [ + "BPMInfo.dat", + "cover.jpg", + "ExpertLegacy.dat", + "ExpertPlusLawless.dat", + "ExpertPlusLegacy.dat", + "ExpertPlusStandard.dat", + "ExpertStandard.dat", + "Info.dat", + "song.egg", + ]) { + expect(pathExistsSync(path.join(DESTINATION_FOLDER, file))) + .toBe(true); + } + + const SUBFOLDER_PATH = path.join(DESTINATION_FOLDER, "pfp"); + for (const file of [ + "galaxyCompressed.jpg", + "gojiCompressed.jpg", + ]) { + expect(pathExistsSync(path.join(SUBFOLDER_PATH, file))) + .toBe(true); + } + }); + + afterAll(async () => { + if (pathExistsSync(DESTINATION_FOLDER)) { + await rm(DESTINATION_FOLDER, { recursive: true, force: true }); + } + }); +}); diff --git a/src/main/helpers/zip.helpers.ts b/src/main/helpers/zip.helpers.ts index f1587133..0c9337da 100644 --- a/src/main/helpers/zip.helpers.ts +++ b/src/main/helpers/zip.helpers.ts @@ -1,28 +1,118 @@ -import JSZip from "jszip"; -import { pathExist } from "./fs.helpers"; +import crypto from "crypto"; +import fs from "fs-extra"; import path from "path"; -import { mkdir, writeFile } from "fs/promises"; +import yauzl from "yauzl"; +import { FileHashes } from "shared/models/mods"; +import { ensureFolderExist } from "./fs.helpers"; -export async function extractZip(zip: JSZip, dest: string): Promise { - if (!(await pathExist(dest))) { - throw new Error(`Path ${dest} does not exist`); - } +// NOTE: yauzl needs to be reopened when it is read + +export async function extractZip(zipPath: string, destination: string): Promise { + await ensureFolderExist(destination); + + return new Promise((resolve, reject) => { + yauzl.open(zipPath, { + lazyEntries: true, + decodeStrings: true, + }, + (openError, zip) => { + if (openError) return reject(openError); + handleExtractZip(zip, destination, resolve, reject); + }); + }); +} + +function handleExtractZip( + zip: yauzl.ZipFile, destination: string, + resolve: (result: string[]) => void, reject: (error: any) => void +) { const files: string[] = []; - for (const [relativePath, entry] of Object.entries(zip.files)) { - if (entry.dir) { - continue; + zip.readEntry(); + + zip.on("entry", (entry: yauzl.Entry) => { + const absolutePath = path.join(destination, entry.fileName); + + // Entry is a directory / folder + if (entry.fileName.endsWith("/")) { + if (!fs.pathExistsSync(absolutePath)) { + fs.mkdirSync(absolutePath, { recursive: true }); + } + return zip.readEntry(); } - const content = await entry.async("nodebuffer"); - const outPath = path.join(dest, relativePath); - const outDir = path.dirname(outPath); + // Entry is a file + zip.openReadStream(entry, (readError, readStream) => { + if (readError) return reject(readError); - await mkdir(outDir, { recursive: true }); - await writeFile(outPath, content); + // For forward slashes since they don't create directories + const directory = path.dirname(absolutePath); + if (!fs.pathExistsSync(directory)) { + fs.mkdirSync(directory, { recursive: true }); + } - files.push(outPath); - } + const writeStream = fs.createWriteStream(absolutePath); + readStream.pipe(writeStream); - return files; + readStream.on("end", () => zip.readEntry()); + }); + }); + + zip.once("end", () => { + zip.close(); + resolve(files); + }); } + +export function validateZip(zipPath: string, hashes: FileHashes[]): Promise { + return new Promise((resolve, reject) => { + yauzl.open(zipPath, { + lazyEntries: true, + decodeStrings: true, + }, + (openError, zip) => { + if (openError) return reject(openError); + handleValidateZip(zip, hashes, resolve, reject); + }); + }); +} + +function handleValidateZip( + zip: yauzl.ZipFile, hashes: FileHashes[], + resolve: (result: boolean) => void, reject: (error: any) => void +) { + let hashCount = 0; + + zip.readEntry(); + + zip.on("entry", (entry: yauzl.Entry) => { + // Entry is a directory / folder + if (entry.fileName.endsWith("/")) { + return zip.readEntry(); + } + + // Entry is a file + zip.openReadStream(entry, (readError, readStream) => { + if (readError) return reject(readError); + + const chunks: Buffer[] = []; + readStream.on("data", data => { + chunks.push(data); + }); + + readStream.on("end", () => { + const md5Hash = crypto.createHash("md5") + .update(Buffer.concat(chunks)) + .digest("hex"); + hashCount += +hashes.some(md5 => md5.hash === md5Hash); + zip.readEntry(); + }); + }); + }); + + zip.once("end", () => { + zip.close(); + resolve(hashCount === hashes.length); + }); +} + diff --git a/src/main/services/additional-content/maps/local-maps-manager.service.ts b/src/main/services/additional-content/maps/local-maps-manager.service.ts index e2c7508a..b20808ee 100644 --- a/src/main/services/additional-content/maps/local-maps-manager.service.ts +++ b/src/main/services/additional-content/maps/local-maps-manager.service.ts @@ -30,6 +30,7 @@ import { MapInfo } from "shared/models/maps/info/map-info.model"; import { parseMapInfoDat } from "shared/parsers/maps/map-info.parser"; import { CustomError } from "shared/models/exceptions/custom-error.class"; import { tryit } from "shared/helpers/error.helpers"; +import { extractZip } from "main/helpers/zip.helpers"; export class LocalMapsManagerService { private static instance: LocalMapsManagerService; @@ -169,17 +170,13 @@ export class LocalMapsManagerService { return getUrlsAndReturn(mapInfo, hash, mapPath); } - private async downloadMapZip(zipUrl: string): Promise<{ zip: StreamZip.StreamZipAsync; zipPath: string }> { + private async downloadMapZip(zipUrl: string): Promise { const fileName = `${path.basename(zipUrl, ".zip")}-${crypto.randomUUID()}.zip`; const tempPath = this.utils.getTempPath(); await ensureFolderExist(this.utils.getTempPath()); const dest = path.join(tempPath, fileName); - - const zipPath = (await lastValueFrom(this.reqService.downloadFile(zipUrl, dest))).data; - const zip = new StreamZip.async({ file: zipPath }); - - return { zip, zipPath }; + return (await lastValueFrom(this.reqService.downloadFile(zipUrl, dest))).data; } public getMaps(version?: BSVersion): Observable { @@ -454,16 +451,8 @@ export class LocalMapsManagerService { return installedMap; } - const { zip, zipPath } = await this.downloadMapZip(zipUrl); - - if (!zip) { - throw new Error(`Cannot download ${zipUrl}`); - } - - await ensureFolderExist(mapPath); - - await zip.extract(null, mapPath); - await zip.close(); + const zipPath = await this.downloadMapZip(zipUrl); + await extractZip(zipPath, mapPath); await unlink(zipPath); const localMap = await this.loadMapInfoFromPath(mapPath); diff --git a/src/main/services/mods/bs-mods-manager.service.ts b/src/main/services/mods/bs-mods-manager.service.ts index 91a849e4..48d5f8f5 100644 --- a/src/main/services/mods/bs-mods-manager.service.ts +++ b/src/main/services/mods/bs-mods-manager.service.ts @@ -8,17 +8,18 @@ import { RequestService } from "../request.service"; import { spawn } from "child_process"; import { BS_EXECUTABLE } from "../../constants"; import log from "electron-log"; -import { deleteFolder, pathExist, Progression, unlinkPath } from "../../helpers/fs.helpers"; +import { deleteFolder, ensureFolderExist, pathExist, Progression, unlinkPath } from "../../helpers/fs.helpers"; import { lastValueFrom, Observable } from "rxjs"; -import JSZip from "jszip"; -import { extractZip } from "../../helpers/zip.helpers"; +import { extractZip, validateZip } from "../../helpers/zip.helpers"; import recursiveReadDir from "recursive-readdir"; import { sToMs } from "../../../shared/helpers/time.helpers"; -import { ensureDir, pathExistsSync } from "fs-extra"; +import { pathExistsSync, unlinkSync } from "fs-extra"; import { CustomError } from "shared/models/exceptions/custom-error.class"; import { popElement } from "shared/helpers/array.helpers"; import { LinuxService } from "../linux.service"; import { tryit } from "shared/helpers/error.helpers"; +import { UtilsService } from "../utils.service"; +import crypto from "crypto"; export class BsModsManagerService { private static instance: BsModsManagerService; @@ -27,6 +28,7 @@ export class BsModsManagerService { private readonly bsLocalService: BSLocalVersionService; private readonly linuxService: LinuxService; private readonly requestService: RequestService; + private readonly utilsService: UtilsService; private manifestMatches: Mod[]; @@ -42,6 +44,7 @@ export class BsModsManagerService { this.bsLocalService = BSLocalVersionService.getInstance(); this.linuxService = LinuxService.getInstance(); this.requestService = RequestService.getInstance(); + this.utilsService = UtilsService.getInstance(); } private async getModFromHash(hash: string): Promise { @@ -110,26 +113,22 @@ export class BsModsManagerService { return this.beatModsApi.getModByHash(injectorMd5); } - private async downloadZip(zipUrl: string): Promise { + private async downloadZip(zipUrl: string): Promise { zipUrl = path.join(this.beatModsApi.BEAT_MODS_URL, zipUrl); log.info("Download mod zip", zipUrl); - const buffer = await lastValueFrom(this.requestService.downloadBuffer(zipUrl)) - .then(progress => progress.data) - .catch(e => { - log.error("ZIP", "Error while downloading zip", e); - return undefined; - }); + const fileName = `${path.basename(zipUrl, ".zip")}-${crypto.randomUUID()}.zip`; + const tempPath = this.utilsService.getTempPath(); + const destination = path.join(tempPath, fileName); - if (!buffer) { + try { + await ensureFolderExist(tempPath); + return (await lastValueFrom(this.requestService.downloadFile(zipUrl, destination))).data; + } catch (error) { + log.error("Could not download zip file at", zipUrl); return null; } - - return JSZip.loadAsync(buffer).catch(e => { - log.error("ZIP", "Error while loading zip", e); - return null; - }); } private async executeBSIPA(version: BSVersion, args: string[]): Promise { @@ -200,44 +199,31 @@ export class BsModsManagerService { } log.info("Start download mod zip", mod.name, download.url); - const zip = await this.downloadZip(download.url); - log.info("Mod zip download end", mod.name, download.url, !!zip); - - if (!zip) { + const zipPath = await this.downloadZip(download.url); + log.info("Mod zip download end", mod.name, download.url, !!zipPath); + if (!zipPath) { return false; } - const crypto = require("crypto"); - const { files } = zip; - - const checkedEntries = ( - await Promise.all( - Object.values(files).map(async entry => { - const data = await entry.async("nodebuffer"); - const entryMd5 = crypto.createHash("md5").update(data).digest("hex"); - return download.hashMd5.some(md5 => md5.hash === entryMd5) ? entry : undefined; - }) - ).catch(e => { - log.error("Error while checking mod zip entries", mod.name, e); - throw e; - }) - ).filter(entry => !!entry); - - if (checkedEntries.length !== download.hashMd5.length) { + if (!(await validateZip(zipPath, download.hashMd5))) { return false; } - const verionPath = await this.bsLocalService.getVersionPath(version); + const versionPath = await this.bsLocalService.getVersionPath(version); const isBSIPA = mod.name.toLowerCase() === "bsipa"; - const destDir = isBSIPA ? verionPath : path.join(verionPath, ModsInstallFolder.PENDING); + const destDir = isBSIPA ? versionPath : path.join(versionPath, ModsInstallFolder.PENDING); - await ensureDir(destDir); log.info("Start extracting mod zip", mod.name, "to", destDir); - const extracted = await extractZip(zip, destDir) + const extracted = await extractZip(zipPath, destDir) .then(() => true) .catch(e => { log.error("Error while extracting mod zip", e); return false; + }) + .finally(() => { + if (pathExistsSync(zipPath)) { + unlinkSync(zipPath); + } }); log.info("Mod zip extraction end", mod.name, "to", destDir, "success:", extracted);