diff --git a/.eslintrc.js b/.eslintrc.js index 9577afe0..c120f3c1 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -59,7 +59,7 @@ module.exports = { "react/function-component-definition": "off", "jsx-a11y/control-has-associated-label": "off", "react/button-has-type": "off", - "no-labels": ["error", { "allowLoop": true }] + "max-classes-per-file": "off", }, parserOptions: { ecmaVersion: 2020, diff --git a/.gitattributes b/.gitattributes index c26f0c38..29cc1930 100644 --- a/.gitattributes +++ b/.gitattributes @@ -16,3 +16,4 @@ *.woff binary *.woff2 binary assets/scripts/* binary +*.zip binary diff --git a/.github/workflows/node.js.yaml b/.github/workflows/node.js.yaml index b666d9bb..3a7b1b25 100644 --- a/.github/workflows/node.js.yaml +++ b/.github/workflows/node.js.yaml @@ -13,7 +13,7 @@ jobs: build: strategy: matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-latest, windows-latest] runs-on: ${{ matrix.os }} @@ -26,6 +26,7 @@ jobs: cache: "npm" - run: npm ci - run: npm run build - - run: npm test - # Currently typescript, eslint, and prettier are unhappy - continue-on-error: true + + - name: Unit tests + run: npm run test:unit + diff --git a/jest.config.ts b/jest.config.ts new file mode 100644 index 00000000..30d53b30 --- /dev/null +++ b/jest.config.ts @@ -0,0 +1,19 @@ +import type { Config } from "jest"; + +const config: Config = { + // testURL: "http://localhost/", + // testEnvironment: "jsdom", + transform: { + "\\.(ts|tsx|js|jsx)$": "ts-jest", + }, + moduleNameMapper: { + "\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$": "/.erb/mocks/fileMock.js", + "\\.(css|less|sass|scss)$": "identity-obj-proxy", + }, + moduleFileExtensions: ["js", "jsx", "ts", "tsx", "json"], + moduleDirectories: ["node_modules", "src"], + testPathIgnorePatterns: ["release/app/dist"], + setupFiles: ["./.erb/scripts/check-build-exists.ts"], +}; + +export default config; diff --git a/package-lock.json b/package-lock.json index 009acc5b..d8574f6d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -32,11 +32,9 @@ "got": "^14.4.4", "history": "^5.3.0", "is-elevated": "^4.0.0", - "jszip": "^3.10.1", "md5-file": "^5.0.0", "node-abi": "^3.65.0", "node-fetch": "^3.3.2", - "node-stream-zip": "^1.15.0", "pako": "^2.1.0", "protobufjs": "^7.4.0", "qrcode.react": "^4.0.1", @@ -61,7 +59,8 @@ "tippy.js": "^6.3.7", "to-ico": "^1.1.5", "use-double-click": "^1.0.5", - "use-fit-text": "^2.4.0" + "use-fit-text": "^2.4.0", + "yauzl": "^3.2.0" }, "devDependencies": { "@electron/fuses": "^1.7.0", @@ -8555,7 +8554,6 @@ }, "node_modules/buffer-crc32": { "version": "0.2.13", - "dev": true, "license": "MIT", "engines": { "node": "*" @@ -12585,6 +12583,17 @@ "@types/yauzl": "^2.9.1" } }, + "node_modules/extract-zip/node_modules/yauzl": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", + "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer-crc32": "~0.2.3", + "fd-slicer": "~1.1.0" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "license": "MIT" @@ -12662,6 +12671,8 @@ }, "node_modules/fd-slicer": { "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", + "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", "dev": true, "license": "MIT", "dependencies": { @@ -13884,6 +13895,7 @@ }, "node_modules/immediate": { "version": "3.0.6", + "dev": true, "license": "MIT" }, "node_modules/immutable": { @@ -15800,6 +15812,7 @@ }, "node_modules/jszip": { "version": "3.10.1", + "dev": true, "license": "(MIT OR GPL-3.0-or-later)", "dependencies": { "lie": "~3.3.0", @@ -15810,10 +15823,12 @@ }, "node_modules/jszip/node_modules/pako": { "version": "1.0.11", + "dev": true, "license": "(MIT AND Zlib)" }, "node_modules/jszip/node_modules/readable-stream": { "version": "2.3.7", + "dev": true, "license": "MIT", "dependencies": { "core-util-is": "~1.0.0", @@ -15931,6 +15946,7 @@ }, "node_modules/lie": { "version": "3.3.0", + "dev": true, "license": "MIT", "dependencies": { "immediate": "~3.0.5" @@ -17217,17 +17233,6 @@ "dev": true, "license": "MIT" }, - "node_modules/node-stream-zip": { - "version": "1.15.0", - "license": "MIT", - "engines": { - "node": ">=0.12.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/antelle" - } - }, "node_modules/nopt": { "version": "6.0.0", "dev": true, @@ -17801,7 +17806,8 @@ }, "node_modules/pend": { "version": "1.2.0", - "dev": true, + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", "license": "MIT" }, "node_modules/performance-now": { @@ -20033,6 +20039,7 @@ }, "node_modules/setimmediate": { "version": "1.0.5", + "dev": true, "license": "MIT" }, "node_modules/setprototypeof": { @@ -22655,12 +22662,16 @@ } }, "node_modules/yauzl": { - "version": "2.10.0", - "dev": true, + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-3.2.0.tgz", + "integrity": "sha512-Ow9nuGZE+qp1u4JIPvg+uCiUr7xGQWdff7JQSk5VGYTAZMDe2q8lxJ10ygv10qmSj031Ty/6FNJpLO4o1Sgc+w==", "license": "MIT", "dependencies": { "buffer-crc32": "~0.2.3", - "fd-slicer": "~1.1.0" + "pend": "~1.2.0" + }, + "engines": { + "node": ">=12" } }, "node_modules/yn": { diff --git a/package.json b/package.json index f1998258..f9dfce73 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "start:preload": "cross-env NODE_ENV=development TS_NODE_TRANSPILE_ONLY=true webpack --config ./.erb/configs/webpack.config.preload.dev.ts", "start:renderer": "cross-env NODE_ENV=development TS_NODE_TRANSPILE_ONLY=true webpack serve --config ./.erb/configs/webpack.config.renderer.dev.ts", "test": "jest", + "test:unit": "jest ./src/__tests__/unit", "publish": "npm run build && electron-builder -c.win.certificateSha1=2164d6a7d641ecf6ad57852f665a518ca2bf960f --publish always --win --x64", "publish:linux": "npm run build && electron-builder --publish always --linux --x64" }, @@ -116,34 +117,6 @@ "beat-saber" ], "homepage": "https://github.com/Zagrios/bs-manager#readme", - "jest": { - "testURL": "http://localhost/", - "testEnvironment": "jsdom", - "transform": { - "\\.(ts|tsx|js|jsx)$": "ts-jest" - }, - "moduleNameMapper": { - "\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$": "/.erb/mocks/fileMock.js", - "\\.(css|less|sass|scss)$": "identity-obj-proxy" - }, - "moduleFileExtensions": [ - "js", - "jsx", - "ts", - "tsx", - "json" - ], - "moduleDirectories": [ - "node_modules", - "src" - ], - "testPathIgnorePatterns": [ - "release/app/dist" - ], - "setupFiles": [ - "./.erb/scripts/check-build-exists.ts" - ] - }, "devDependencies": { "@electron/fuses": "^1.7.0", "@electron/notarize": "^2.3.0", @@ -252,11 +225,9 @@ "got": "^14.4.4", "history": "^5.3.0", "is-elevated": "^4.0.0", - "jszip": "^3.10.1", "md5-file": "^5.0.0", "node-abi": "^3.65.0", "node-fetch": "^3.3.2", - "node-stream-zip": "^1.15.0", "pako": "^2.1.0", "protobufjs": "^7.4.0", "qrcode.react": "^4.0.1", @@ -281,7 +252,8 @@ "tippy.js": "^6.3.7", "to-ico": "^1.1.5", "use-double-click": "^1.0.5", - "use-fit-text": "^2.4.0" + "use-fit-text": "^2.4.0", + "yauzl": "^3.2.0" }, "engines": { "node": ">=20.0.0" diff --git a/src/__tests__/App.test.tsx b/src/__tests__/App.test.tsx deleted file mode 100644 index 9e8b6e43..00000000 --- a/src/__tests__/App.test.tsx +++ /dev/null @@ -1,9 +0,0 @@ -import "@testing-library/jest-dom"; -import { render } from "@testing-library/react"; -import App from "../renderer/windows/App"; - -describe("App", () => { - it("should render", () => { - expect(render()).toBeTruthy(); - }); -}); diff --git a/src/__tests__/assets/zip/corrupted.zip b/src/__tests__/assets/zip/corrupted.zip new file mode 100644 index 00000000..4db9efcc Binary files /dev/null and b/src/__tests__/assets/zip/corrupted.zip differ diff --git a/src/__tests__/assets/zip/empty.zip b/src/__tests__/assets/zip/empty.zip new file mode 100644 index 00000000..15cb0ecb Binary files /dev/null and b/src/__tests__/assets/zip/empty.zip differ diff --git a/src/__tests__/assets/zip/manifest.zip b/src/__tests__/assets/zip/manifest.zip new file mode 100644 index 00000000..3af4d77b Binary files /dev/null and b/src/__tests__/assets/zip/manifest.zip differ diff --git a/src/__tests__/assets/zip/standard.zip b/src/__tests__/assets/zip/standard.zip new file mode 100644 index 00000000..106adc5c Binary files /dev/null and b/src/__tests__/assets/zip/standard.zip differ diff --git a/src/__tests__/assets/zip/subfolders.zip b/src/__tests__/assets/zip/subfolders.zip new file mode 100644 index 00000000..a7c44d38 Binary files /dev/null and b/src/__tests__/assets/zip/subfolders.zip differ diff --git a/src/__tests__/assets/zip/unicode.zip b/src/__tests__/assets/zip/unicode.zip new file mode 100644 index 00000000..a0a1854f Binary files /dev/null and b/src/__tests__/assets/zip/unicode.zip differ diff --git a/src/__tests__/assets/zip/windows_legacy.zip b/src/__tests__/assets/zip/windows_legacy.zip new file mode 100644 index 00000000..3ac92eae Binary files /dev/null and b/src/__tests__/assets/zip/windows_legacy.zip differ diff --git a/src/__tests__/assets/zip/windows_legacy_subfolders.zip b/src/__tests__/assets/zip/windows_legacy_subfolders.zip new file mode 100644 index 00000000..dd5c1aec Binary files /dev/null and b/src/__tests__/assets/zip/windows_legacy_subfolders.zip differ diff --git a/src/__tests__/unit/zip.test.ts b/src/__tests__/unit/zip.test.ts new file mode 100644 index 00000000..459c4242 --- /dev/null +++ b/src/__tests__/unit/zip.test.ts @@ -0,0 +1,329 @@ +import path from "path"; +import { mkdir, pathExistsSync, readFile, rm } from "fs-extra"; +import { BsmZipExtractor } from "main/models/bsm-zip-extractor.class"; + +const TEST_FOLDER = path.resolve(__dirname, "..", "assets", "zip"); +const STANDARD_ZIP = path.join(TEST_FOLDER, "standard.zip"); +const WINDOWS_LEGACY_MAP_ZIP = path.join(TEST_FOLDER, "windows_legacy.zip"); +const SUBFOLDERS_ZIP = path.join(TEST_FOLDER, "subfolders.zip"); +const MANIFEST_ZIP = path.join(TEST_FOLDER, "manifest.zip"); +const EMPTY_ZIP = path.join(TEST_FOLDER, "empty.zip"); +const UNICODE_ZIP = path.join(TEST_FOLDER, "unicode.zip"); +const DESTINATION_FOLDER = path.join(TEST_FOLDER, "out"); + +describe("Test BsmZipExtractor class", () => { + let zip: BsmZipExtractor; + + beforeAll(async () => { + if (pathExistsSync(DESTINATION_FOLDER)) { + await rm(DESTINATION_FOLDER, { recursive: true, force: true }); + } + await mkdir(DESTINATION_FOLDER); + }); + + afterEach(async () => { + if (zip) { + zip.close(); + } + if (pathExistsSync(DESTINATION_FOLDER)) { + await rm(DESTINATION_FOLDER, { recursive: true, force: true }); + } + await mkdir(DESTINATION_FOLDER); + }); + + afterAll(async () => { + if (pathExistsSync(DESTINATION_FOLDER)) { + await rm(DESTINATION_FOLDER, { recursive: true, force: true }); + } + }); + + it("Extract standard zip", async () => { + + zip = await BsmZipExtractor.fromPath(STANDARD_ZIP); + const res = await zip.extract(DESTINATION_FOLDER); + + expect(res.sort()).toEqual([ + "file_1.2.txt", + "file_1.3.txt", + "folder_1.1/", + "folder_1.1/file_2.1.txt", + "folder_1.1/file_2.2.txt", + ].sort()) + + for (const file of [ + "file_1.2.txt", + "file_1.3.txt", + ]) { + expect(pathExistsSync(path.join(DESTINATION_FOLDER, file))) + .toBe(true); + } + + const SUBFOLDER_PATH = path.join(DESTINATION_FOLDER, "folder_1.1"); + for (const file of [ + "file_2.1.txt", + "file_2.2.txt", + ]) { + expect(pathExistsSync(path.join(SUBFOLDER_PATH, file))) + .toBe(true); + } + + }); + + it("Extract map zips using back slashes", async () => { + + zip = await BsmZipExtractor.fromPath(WINDOWS_LEGACY_MAP_ZIP); + const res = await zip.extract(DESTINATION_FOLDER); + + expect(res.sort()).toEqual([ + "BPMInfo.dat", + "cover.jpg", + "ExpertLegacy.dat", + "ExpertPlusLawless.dat", + "ExpertPlusLegacy.dat", + "ExpertPlusStandard.dat", + "ExpertStandard.dat", + "Info.dat", + "song.egg", + "pfp/", + "pfp/galaxyCompressed.jpg", + "pfp/gojiCompressed.jpg", + ].sort()); + + // Expect all the files to exists + for (const file of [ + "BPMInfo.dat", + "cover.jpg", + "ExpertLegacy.dat", + "ExpertPlusLawless.dat", + "ExpertPlusLegacy.dat", + "ExpertPlusStandard.dat", + "ExpertStandard.dat", + "Info.dat", + "song.egg", + ]) { + expect(pathExistsSync(path.join(DESTINATION_FOLDER, file))) + .toBe(true); + } + + const SUBFOLDER_PATH = path.join(DESTINATION_FOLDER, "pfp"); + for (const file of [ + "galaxyCompressed.jpg", + "gojiCompressed.jpg", + ]) { + expect(pathExistsSync(path.join(SUBFOLDER_PATH, file))) + .toBe(true); + } + + }); + + it("Read Zip with multiple subfolders", async () => { + + const zip = await BsmZipExtractor.fromPath(SUBFOLDERS_ZIP); + const res = await zip.extract(DESTINATION_FOLDER); + + expect(res.sort()).toEqual([ + "1/", + "1/2/", + "1/2/3/", + "1/2/3/4/", + "1/2/3/4/5.txt", + ].sort()); + + for (const folder of [ + "1/", + "1/2/", + "1/2/3/", + "1/2/3/4/", + ]) { + expect(pathExistsSync(path.join(DESTINATION_FOLDER, folder))) + .toBe(true); + } + + const file = "1/2/3/4/5.txt"; + expect(pathExistsSync(path.join(DESTINATION_FOLDER, file))) + .toBe(true); + }); + + it("Read manifest.json from zip file", async () => { + + const zipBuffer = await readFile(MANIFEST_ZIP); + zip = await BsmZipExtractor.fromBuffer(zipBuffer); + const buffer = await (await zip.findEntry(entry => entry.fileName === "manifest.json")).read(); + const manifest = JSON.parse(buffer.toString()); + expect(manifest).toBeTruthy(); + + // Check some of the fields if they are correct + expect(manifest.appId).toBe("some-app-id"); + expect(manifest.canonicalName).toBe("some-canonical-name"); + expect(manifest.isCore).toBe(true); + }); + + + it("Should find an entry using findEntry", async () => { + zip = await BsmZipExtractor.fromPath(STANDARD_ZIP); + const entry = await zip.findEntry(entry => entry.fileName === "file_1.2.txt"); + + expect(entry).toBeTruthy(); + expect(entry.fileName).toBe("file_1.2.txt"); + + const contentBuffer = await entry.read(); + const content = contentBuffer.toString(); + + expect(content).toContain("This is file 1.2"); // Assurez-vous que le contenu correspond + }); + + it("Should filter entries using filterEntries", async () => { + zip = await BsmZipExtractor.fromPath(STANDARD_ZIP); + const entries = await zip.filterEntries(entry => !entry.isDirectory && entry.fileName.startsWith("folder_1.1/")); + + expect(entries.length).toBe(2); + const fileNames = entries.map(entry => entry.fileName).sort(); + + expect(fileNames).toEqual([ + "folder_1.1/file_2.1.txt", + "folder_1.1/file_2.2.txt", + ]); + }); + + it("Should get an entry by file name using getEntry", async () => { + zip = await BsmZipExtractor.fromPath(STANDARD_ZIP); + const entry = await zip.getEntry("file_1.3.txt"); + + expect(entry).toBeTruthy(); + expect(entry.fileName).toBe("file_1.3.txt"); + + const contentBuffer = await entry.read(); + const content = contentBuffer.toString(); + + expect(content).toContain("This is file 1.3"); // Assurez-vous que le contenu correspond + }); + + it("Should extract only specified entries using entriesNames option", async () => { + zip = await BsmZipExtractor.fromPath(STANDARD_ZIP); + const res = await zip.extract(DESTINATION_FOLDER, { entriesNames: ["file_1.2.txt", "folder_1.1/file_2.1.txt"] }); + + expect(res.sort()).toEqual([ + "file_1.2.txt", + "folder_1.1/", + "folder_1.1/file_2.1.txt", + ].sort()); + + for (const file of [ + "file_1.2.txt", + "folder_1.1/file_2.1.txt", + ]) { + expect(pathExistsSync(path.join(DESTINATION_FOLDER, file))) + .toBe(true); + } + + // Vérifiez que les autres fichiers ne sont pas extraits + expect(pathExistsSync(path.join(DESTINATION_FOLDER, "file_1.3.txt"))) + .toBe(false); + expect(pathExistsSync(path.join(DESTINATION_FOLDER, "folder_1.1/file_2.2.txt"))) + .toBe(false); + }); + + it("Should not extract any files if abortToken is already aborted", async () => { + zip = await BsmZipExtractor.fromPath(STANDARD_ZIP); + const abortController = new AbortController(); + + // Abortez avant d'appeler extract + abortController.abort(); + + const res = await zip.extract(DESTINATION_FOLDER, { abortToken: abortController }); + + expect(res.length).toBe(0); + + // Vérifiez qu'aucun fichier n'a été extrait + expect(pathExistsSync(path.join(DESTINATION_FOLDER, "file_1.2.txt"))) + .toBe(false); + }); + + + it("Should throw an error when opening a non-existent zip file", async () => { + await expect(BsmZipExtractor.fromPath("non_existent.zip")) + .rejects + .toThrow(); + }); + + it("Should handle empty zip file correctly", async () => { + zip = await BsmZipExtractor.fromPath(EMPTY_ZIP); + const entries = []; + + for await (const entry of zip.entries()) { + entries.push(entry); + } + + expect(entries.length).toBe(0); + + const res = await zip.extract(DESTINATION_FOLDER); + expect(res.length).toBe(0); + }); + + it("Should return the same entries on multiple calls to entries()", async () => { + zip = await BsmZipExtractor.fromPath(STANDARD_ZIP); + + const entries1 = []; + for await (const entry of zip.entries()) { + entries1.push(entry); + } + + expect(entries1.length).toBe(5); // Basé sur le contenu de STANDARD_ZIP + + const entries2 = []; + for await (const entry of zip.entries()) { + entries2.push(entry); + } + + expect(entries2.length).toBe(5); + expect(entries2.map(e => e.fileName).sort()).toEqual(entries1.map(e => e.fileName).sort()); + }); + + it("Should not be able to read entries after zip is closed", async () => { + zip = await BsmZipExtractor.fromPath(STANDARD_ZIP); + + zip.close(); + + await expect(zip.entries().next()).rejects.toThrow(); + }); + + it("Should handle zip file from buffer", async () => { + const zipBuffer = await readFile(STANDARD_ZIP); + zip = await BsmZipExtractor.fromBuffer(zipBuffer); + + const entries = []; + for await (const entry of zip.entries()) { + entries.push(entry.fileName); + } + + expect(entries.sort()).toEqual([ + "file_1.2.txt", + "file_1.3.txt", + "folder_1.1/", + "folder_1.1/file_2.1.txt", + "folder_1.1/file_2.2.txt", + ].sort()); + }); + + it("Should read and extract files with Unicode filenames", async () => { + zip = await BsmZipExtractor.fromPath(UNICODE_ZIP); + + const entries = []; + for await (const entry of zip.entries()) { + entries.push(entry); + } + + expect(entries.map(e => e.fileName)).toContain("こんにちは.txt"); // "Bonjour" en japonais + + const res = await zip.extract(DESTINATION_FOLDER); + expect(res).toContain("こんにちは.txt"); + expect(pathExistsSync(path.join(DESTINATION_FOLDER, "こんにちは.txt"))) + .toBe(true); + + }); + + it("Should throw an error when reading a corrupted zip file", async () => { + const CORRUPTED_ZIP = path.join(TEST_FOLDER, "corrupted.zip"); + await expect(BsmZipExtractor.fromPath(CORRUPTED_ZIP)).rejects.toThrow(/.*[Error: Invalid comment length].*/) + }); +}); diff --git a/src/main/helpers/zip.helpers.ts b/src/main/helpers/zip.helpers.ts deleted file mode 100644 index f1587133..00000000 --- a/src/main/helpers/zip.helpers.ts +++ /dev/null @@ -1,28 +0,0 @@ -import JSZip from "jszip"; -import { pathExist } from "./fs.helpers"; -import path from "path"; -import { mkdir, writeFile } from "fs/promises"; - -export async function extractZip(zip: JSZip, dest: string): Promise { - if (!(await pathExist(dest))) { - throw new Error(`Path ${dest} does not exist`); - } - const files: string[] = []; - - for (const [relativePath, entry] of Object.entries(zip.files)) { - if (entry.dir) { - continue; - } - - const content = await entry.async("nodebuffer"); - const outPath = path.join(dest, relativePath); - const outDir = path.dirname(outPath); - - await mkdir(outDir, { recursive: true }); - await writeFile(outPath, content); - - files.push(outPath); - } - - return files; -} diff --git a/src/main/models/bsm-zip-extractor.class.ts b/src/main/models/bsm-zip-extractor.class.ts new file mode 100644 index 00000000..915c0e65 --- /dev/null +++ b/src/main/models/bsm-zip-extractor.class.ts @@ -0,0 +1,218 @@ +import { createWriteStream, ensureDir } from "fs-extra"; +import path from "path"; +import { Readable } from "stream"; +import yauzl, { ZipFile, Options, Entry } from "yauzl" + +export class BsmZipExtractor { + + private static readonly YAUZL_OPEN_OPTIONS: Options = { + lazyEntries: true, + decodeStrings: true, + autoClose: false, + }; + + public static fromPath(path: string): Promise { + return new Promise((resolve, reject) => { + yauzl.open(path, BsmZipExtractor.YAUZL_OPEN_OPTIONS, (error: Error, zip: yauzl.ZipFile) => { + if (error) return reject(error); + resolve(new BsmZipExtractor(zip)); + }); + }); + } + + public static fromBuffer(buffer: Buffer): Promise { + return new Promise((resolve, reject) => { + yauzl.fromBuffer(buffer, BsmZipExtractor.YAUZL_OPEN_OPTIONS, (error: Error, zip: yauzl.ZipFile) => { + if (error) return reject(error); + resolve(new BsmZipExtractor(zip)); + }); + }); + } + + private readonly zip: ZipFile; + private readonly entriesMap = new Map(); + + private constructor(zip: ZipFile) { + this.zip = zip; + } + + private async readEntry(): Promise { + return new Promise((resolve, reject) => { + const onEntry = (entry: Entry) => { + cleanup(); + resolve(new BsmZipExtractorEntry({ entry, zip: this.zip })); + }; + + const onEnd = () => { + cleanup(); + resolve(null); + }; + + const onError = (err: Error) => { + cleanup(); + reject(err); + }; + + const cleanup = () => { + this.zip.removeListener("entry", onEntry); + this.zip.removeListener("end", onEnd); + this.zip.removeListener("error", onError); + }; + + this.zip.once("entry", onEntry); + this.zip.once("end", onEnd); + this.zip.once("error", onError); + this.zip.readEntry(); + }); + } + + // The first time this method is called, it will read all the entries and store them in a map + // The next times it will return the entries from the map to avoid reopening the zip file again + public async *entries(): AsyncGenerator { + + for (const entry of Array.from(this.entriesMap.values())) { + yield entry; + } + + let entry: BsmZipExtractorEntry = await this.readEntry(); + while (entry) { + this.entriesMap.set(entry.fileName, entry); + yield entry; + entry = await this.readEntry(); + } + } + + public async findEntry(func: (entry: BsmZipExtractorEntry) => boolean): Promise { + for await (const entry of this.entries()) { + if (func(entry)) { + return entry; + } + } + return null; + } + + public async filterEntries(func: (entry: BsmZipExtractorEntry) => boolean): Promise { + const filtered: BsmZipExtractorEntry[] = []; + for await (const entry of this.entries()) { + if (func(entry)) { + filtered.push(entry); + } + } + return filtered; + } + + public async getEntry(fileName: string): Promise { + return this.entriesMap.get(fileName) ?? this.findEntry(entry => entry.fileName === fileName); + } + + /** + * Extracts all entries from the zip file to the destination folder + * @param destination + * @param opt - { entriesNames?: string[], abortToken: AbortController } + * @returns {Promise} The list of extracted files (relative paths in the destination folder) + * + * `opt` object: + * - `entriesNames` - The list of entries to extract (can be regexs or glob pattern). If not provided, all entries will be extracted + * - `abortToken` - The AbortController instance to abort the extraction + * + */ + public async extract(destination: string, opt?: { entriesNames?: (string|RegExp)[], abortToken?: AbortController }): Promise { + const entriesNames = opt?.entriesNames; + const abortToken = opt?.abortToken; + + if (abortToken?.signal?.aborted) { + return []; + } + + await ensureDir(destination); + + const extracted = new Set() + for await (const entry of this.entries()) { + + if (abortToken?.signal?.aborted) { + break; + } + + if (entriesNames && !entriesNames.some(name => typeof name === "string" ? entry.fileName === name : name.test(entry.fileName))){ + continue; + } + + const extractedFile = await entry.extract(destination); + const dirname = path.dirname(extractedFile); + + // Make zip that use backslash as separator have the same behavior as zip that use forward slash + if(dirname !== "."){ + const split = dirname.split(path.posix.sep); + for (let i = 1; i <= split.length; i++) { + const folder = split.slice(0, i).join(path.posix.sep); + extracted.add(folder + path.posix.sep); + } + } + + + extracted.add(extractedFile); + } + + return Array.from(extracted); + } + + + public close(): void { + this.zip.close(); + } + +} + +class BsmZipExtractorEntry { + + private readonly entry: Entry; + private readonly zip: ZipFile; + + constructor(opt: { entry: Entry, zip: ZipFile }) { + this.entry = opt.entry; + this.zip = opt.zip; + } + + public async read(): Promise { + return new Promise((resolve, reject) => { + this.zip.openReadStream(this.entry, (error: Error, stream: Readable) => { + if (error) return reject(error); + const buffers: Buffer[] = []; + stream.on("data", (data) => buffers.push(data as Buffer)); + stream.on("end", () => resolve(Buffer.concat(buffers))); + stream.on("error", reject); + }); + }); + } + + public async extract(destination: string): Promise { + + const destPath = path.join(destination, this.entry.fileName); + + if (this.isDirectory) { + return ensureDir(destPath).then(() => this.entry.fileName); + } + + return new Promise((resolve, reject) => { + this.zip.openReadStream(this.entry, async (error: Error, stream: Readable) => { + if (error) return reject(error); + + await ensureDir(path.dirname(destPath)); + + const writeStream = createWriteStream(destPath); + stream.pipe(writeStream); + writeStream.on("finish", () => resolve(this.entry.fileName)); + writeStream.on("error", reject); + }); + }); + } + + public get isDirectory(): boolean { + return this.entry.fileName.endsWith("/"); + } + + public get fileName(): string { + return this.entry.fileName; + } + +} diff --git a/src/main/models/oculus-downloader.class.ts b/src/main/models/oculus-downloader.class.ts index 47cc9067..47587e73 100644 --- a/src/main/models/oculus-downloader.class.ts +++ b/src/main/models/oculus-downloader.class.ts @@ -1,4 +1,3 @@ -import JSZip from "jszip"; import fetch from "node-fetch"; import { CustomError } from "../../shared/models/exceptions/custom-error.class"; import { mkdirs, createWriteStream, pathExists, WriteStream } from "fs-extra"; @@ -7,6 +6,8 @@ import { inflate } from "pako" import { EMPTY, Observable, ReplaySubject, Subscriber, catchError, filter, from, lastValueFrom, mergeMap, scan, share, tap } from "rxjs"; import { Progression, hashFile } from "../helpers/fs.helpers"; import { OculusDownloaderErrorCodes } from "../../shared/models/bs-version-download/oculus-download.model"; +import { BsmZipExtractor } from "./bsm-zip-extractor.class"; +import { tryit } from "shared/helpers/error.helpers"; export class OculusDownloader { @@ -23,26 +24,38 @@ export class OculusDownloader { return `https://securecdn.oculus.com/binaries/segment/?access_token=${token}&binary_id=${binaryId}&segment_sha256=${segmentSha256}`; } - private async downloadManifestZip(manifestUrl: string): Promise { + private async downloadManifestZip(manifestUrl: string): Promise { const response = await fetch(manifestUrl); const arrBuffer = await response.arrayBuffer(); - return JSZip.loadAsync(arrBuffer); + return Buffer.from(arrBuffer); } private async getManifest(): Promise { const downloadUrl = this.getDownloadManifestUrl(this.options.accessToken, this.options.binaryId); - const manifestZip = await this.downloadManifestZip(downloadUrl).catch(err => CustomError.throw(err, "DOWNLOAD_MANIFEST_FAILED")); - const manifestFile = manifestZip.file("manifest.json"); + const buffer = await this.downloadManifestZip(downloadUrl) + .catch(err => CustomError.throw(err, "DOWNLOAD_MANIFEST_FAILED")); - if(!manifestFile){ + const zip = await BsmZipExtractor.fromBuffer(buffer); + const entry = await zip.getEntry("manifest.json"); + + if(!entry) { throw new CustomError("Manifest file not found", "MANIFEST_FILE_NOT_FOUND"); } - return manifestFile.async("text").then(JSON.parse).catch(err => CustomError.throw(err, "PARSE_MANIFEST_FILE_FAILED")); + const manifest = await entry.read(); + zip.close(); + + const { result, error } = tryit(() => JSON.parse(manifest.toString()) as OculusManifest); + + if(error){ + throw CustomError.throw(error, "PARSE_MANIFEST_FILE_FAILED") + } + + return result; } private downloadManifestFile(file: OculusManifestFile, destination: string): Observable> { - + const downloadSegment = async (segment: OculusManifestFileSegment): Promise => { const segmentUrl = this.getDownloadSegmentUrl(this.options.accessToken, this.options.binaryId, segment[1]); const response = await fetch(segmentUrl); @@ -66,11 +79,11 @@ export class OculusDownloader { const arrBuffer = await downloadSegment(segment); const inflated = inflate(arrBuffer); - await writeStream.write(inflated); + writeStream.write(inflated); progress.current += inflated.byteLength; progress.diff = inflated.byteLength; - + sub.next(progress); } @@ -86,7 +99,7 @@ export class OculusDownloader { }); } - private isFileIntegrityValid(file: OculusFileWithName, folder: string): Promise { + private async isFileIntegrityValid(file: OculusFileWithName, folder: string): Promise { const [fileName, fileData] = file; const destination = path.join(folder, fileName); @@ -138,7 +151,7 @@ export class OculusDownloader { if(this.isDownloading){ throw new CustomError("Already downloading", "ALREADY_DOWNLOADING"); } - + this.options = options; this.isDownloading = true; @@ -182,7 +195,7 @@ export class OculusDownloader { }))); const integrity = await lastValueFrom(this.verifyIntegrity(manifest, options.destination)).catch(err => CustomError.throw(err, "VERIFY_INTEGRITY_FAILED")); - + if(integrity.data.length > 0){ throw new CustomError("Some files failed to download", "SOME_FILES_FAILED_TO_DOWNLOAD", integrity.data); } @@ -242,4 +255,4 @@ interface Logger { info: (...args: unknown[]) => void; warn: (...args: unknown[]) => void; error: (...args: unknown[]) => void; -} \ No newline at end of file +} diff --git a/src/main/services/additional-content/maps/local-maps-manager.service.ts b/src/main/services/additional-content/maps/local-maps-manager.service.ts index e2c7508a..d4a6558d 100644 --- a/src/main/services/additional-content/maps/local-maps-manager.service.ts +++ b/src/main/services/additional-content/maps/local-maps-manager.service.ts @@ -8,7 +8,6 @@ import { UtilsService } from "../../utils.service"; import crypto, { BinaryLike } from "crypto"; import { lstatSync } from "fs"; import { copy, createReadStream, ensureDir, pathExists, pathExistsSync, realpath, unlink } from "fs-extra"; -import StreamZip from "node-stream-zip"; import { RequestService } from "../../request.service"; import sanitize from "sanitize-filename"; import { DeepLinkService } from "../../deep-link.service"; @@ -30,6 +29,8 @@ import { MapInfo } from "shared/models/maps/info/map-info.model"; import { parseMapInfoDat } from "shared/parsers/maps/map-info.parser"; import { CustomError } from "shared/models/exceptions/custom-error.class"; import { tryit } from "shared/helpers/error.helpers"; +import { BsmZipExtractor } from "main/models/bsm-zip-extractor.class"; +import { escapeRegExp } from "../../../../shared/helpers/string.helpers"; export class LocalMapsManagerService { private static instance: LocalMapsManagerService; @@ -169,17 +170,13 @@ export class LocalMapsManagerService { return getUrlsAndReturn(mapInfo, hash, mapPath); } - private async downloadMapZip(zipUrl: string): Promise<{ zip: StreamZip.StreamZipAsync; zipPath: string }> { + private async downloadMapZip(zipUrl: string): Promise { const fileName = `${path.basename(zipUrl, ".zip")}-${crypto.randomUUID()}.zip`; const tempPath = this.utils.getTempPath(); await ensureFolderExist(this.utils.getTempPath()); const dest = path.join(tempPath, fileName); - - const zipPath = (await lastValueFrom(this.reqService.downloadFile(zipUrl, dest))).data; - const zip = new StreamZip.async({ file: zipPath }); - - return { zip, zipPath }; + return (await lastValueFrom(this.reqService.downloadFile(zipUrl, dest))).data; } public getMaps(version?: BSVersion): Observable { @@ -328,16 +325,16 @@ export class LocalMapsManagerService { public importMaps(zipPaths: string[], version?: BSVersion): Observable> { return new Observable>(obs => { - let unsubscribed = false; let progress: Progression = { total: 0, current: 0 }; let nbImportedMaps = 0; + const abortController = new AbortController(); + let zip: BsmZipExtractor; (async () => { const mapsPath = await this.getMapsFolderPath(version); for(const zipPath of zipPaths) { - - if(unsubscribed) { - log.info("Maps importation from zip has been cancelled"); + if(abortController.signal?.aborted) { + log.info("Maps import from zip has been cancelled"); return; } @@ -346,75 +343,62 @@ export class LocalMapsManagerService { if(!pathExistsSync(zipPath)) { continue; } - const zip = new StreamZip.async({ file: zipPath }); - const { result: zipEntries, error } = await tryit(() => zip.entries()); + zip = await BsmZipExtractor.fromPath(zipPath); + const mapsFolders = (await zip.filterEntries(entry => /(^|\/)[Ii]nfo\.dat$/.test(entry.fileName))) + .map(entry => path.dirname(entry.fileName)); - if(error) { - const res = await tryit(() => zip.close()); - log.error("Could not read zip entries", zipPath, error, res?.error); - continue; - } - - const zipEntriesValues = Object.values(zipEntries); - - const mapsFolders = zipEntriesValues.reduce((acc, entry) => { - if(!/(^|\/)[Ii]nfo\.dat$/.test(entry.name)){ return acc; } - acc.push(path.dirname(entry.name)); - return acc; - }, []); - - if(mapsFolders.length === 0) { + if (mapsFolders.length === 0) { log.warn("No maps \"info.dat\" found in zip", zipPath); + progress.total = 1; + progress.current = 1; + obs.next(progress); + continue; } progress.total = mapsFolders.length; obs.next(progress); - for(const folder of mapsFolders) { + const isRoot = mapsFolders.length === 1 && mapsFolders[0] === "."; + const destination = isRoot + ? path.join(mapsPath, path.basename(zipPath, ".zip")) + : mapsPath; - if(unsubscribed) { - log.info("Maps importation from zip has been cancelled"); - await zip.close(); - return; - } + log.info("Extracting", `"${zipPath}"`, "into", `"${mapsPath}"`); + for (const folder of mapsFolders) { + log.info(">", folder); - const isRoot = folder === "."; - const dest = isRoot ? path.join(mapsPath, path.basename(zipPath, ".zip")) : path.join(mapsPath, folder); + const regex = new RegExp(`^${escapeRegExp(folder)}\\/`); - let extract: () => Promise; + const exported = await zip.extract(destination, { + entriesNames: [regex], + abortToken: abortController + }); - if(isRoot){ - const entries = zipEntriesValues.filter(entry => entry.isFile && path.dirname(entry.name) === "."); - extract = async () => { - await Promise.all(entries.map(entry => { - log.info("Extracting", `"${entry.name}"`, "from", `"${zipPath}"`, "into", `"${path.join(dest, path.basename(entry.name))}"`); - return zip.extract(entry.name, path.join(dest, path.basename(entry.name))); - })); - return this.loadMapInfoFromPath(dest); - }; - } else { - extract = async () => { - log.info("Extracting", `"${folder}"`, "from", `"${zipPath}"`, "into", `"${mapsPath}"`); - await zip.extract(folder, dest); - return this.loadMapInfoFromPath(dest); - } - } - - await ensureDir(dest); - const { result: bsmMap, error } = await tryit(extract); - - if(error) { - log.error("Could not extract map", zipPath, folder, mapsPath, error); + if(exported.length === 0) { + log.warn("No files extracted from", folder); continue; } - nbImportedMaps++; - progress.current++; - progress.data = bsmMap; + if (abortController.signal?.aborted) { + break; + } + + ++nbImportedMaps; + ++progress.current; + progress.data = await this.loadMapInfoFromPath(path.join(destination, folder)); obs.next(progress); + + if (abortController.signal?.aborted) { + break; + } } - await zip.close(); + zip.close(); + + if (abortController.signal?.aborted) { + log.info("Maps import from zip has been cancelled"); + return; + } } })() .then(() => { @@ -424,10 +408,15 @@ export class LocalMapsManagerService { return log.info("Successfully imported", nbImportedMaps, "maps from", zipPaths.length, "zips"); }) .catch(e => obs.error(e)) - .finally(() => obs.complete()); + .finally(() => { + if (zip) { + zip.close(); + } + obs.complete() + }); return () => { - unsubscribed = true; + abortController.abort(); }; }); } @@ -454,16 +443,10 @@ export class LocalMapsManagerService { return installedMap; } - const { zip, zipPath } = await this.downloadMapZip(zipUrl); - - if (!zip) { - throw new Error(`Cannot download ${zipUrl}`); - } - - await ensureFolderExist(mapPath); - - await zip.extract(null, mapPath); - await zip.close(); + const zipPath = await this.downloadMapZip(zipUrl); + const zip = await BsmZipExtractor.fromPath(zipPath); + await zip.extract(mapPath); + zip.close(); await unlink(zipPath); const localMap = await this.loadMapInfoFromPath(mapPath); diff --git a/src/main/services/mods/bs-mods-manager.service.ts b/src/main/services/mods/bs-mods-manager.service.ts index 91a849e4..78e969bb 100644 --- a/src/main/services/mods/bs-mods-manager.service.ts +++ b/src/main/services/mods/bs-mods-manager.service.ts @@ -10,15 +10,16 @@ import { BS_EXECUTABLE } from "../../constants"; import log from "electron-log"; import { deleteFolder, pathExist, Progression, unlinkPath } from "../../helpers/fs.helpers"; import { lastValueFrom, Observable } from "rxjs"; -import JSZip from "jszip"; -import { extractZip } from "../../helpers/zip.helpers"; import recursiveReadDir from "recursive-readdir"; import { sToMs } from "../../../shared/helpers/time.helpers"; -import { ensureDir, pathExistsSync } from "fs-extra"; +import { pathExistsSync } from "fs-extra"; import { CustomError } from "shared/models/exceptions/custom-error.class"; import { popElement } from "shared/helpers/array.helpers"; import { LinuxService } from "../linux.service"; import { tryit } from "shared/helpers/error.helpers"; +import { UtilsService } from "../utils.service"; +import crypto from "crypto"; +import { BsmZipExtractor } from "main/models/bsm-zip-extractor.class"; export class BsModsManagerService { private static instance: BsModsManagerService; @@ -27,6 +28,7 @@ export class BsModsManagerService { private readonly bsLocalService: BSLocalVersionService; private readonly linuxService: LinuxService; private readonly requestService: RequestService; + private readonly utilsService: UtilsService; private manifestMatches: Mod[]; @@ -42,6 +44,7 @@ export class BsModsManagerService { this.bsLocalService = BSLocalVersionService.getInstance(); this.linuxService = LinuxService.getInstance(); this.requestService = RequestService.getInstance(); + this.utilsService = UtilsService.getInstance(); } private async getModFromHash(hash: string): Promise { @@ -110,7 +113,7 @@ export class BsModsManagerService { return this.beatModsApi.getModByHash(injectorMd5); } - private async downloadZip(zipUrl: string): Promise { + private async downloadZip(zipUrl: string): Promise { zipUrl = path.join(this.beatModsApi.BEAT_MODS_URL, zipUrl); log.info("Download mod zip", zipUrl); @@ -126,10 +129,7 @@ export class BsModsManagerService { return null; } - return JSZip.loadAsync(buffer).catch(e => { - log.error("ZIP", "Error while loading zip", e); - return null; - }); + return BsmZipExtractor.fromBuffer(buffer); } private async executeBSIPA(version: BSVersion, args: string[]): Promise { @@ -201,43 +201,38 @@ export class BsModsManagerService { log.info("Start download mod zip", mod.name, download.url); const zip = await this.downloadZip(download.url); - log.info("Mod zip download end", mod.name, download.url, !!zip); + log.info("Mod zip download end", mod.name, download.url); if (!zip) { return false; } - const crypto = require("crypto"); - const { files } = zip; + let hashCount = 0; + for await (const entry of zip.entries()) { + const buffer = await entry.read(); + const md5Hash = crypto.createHash("md5") + .update(buffer) + .digest("hex"); + hashCount += +download.hashMd5.some(md5 => md5.hash === md5Hash); + } - const checkedEntries = ( - await Promise.all( - Object.values(files).map(async entry => { - const data = await entry.async("nodebuffer"); - const entryMd5 = crypto.createHash("md5").update(data).digest("hex"); - return download.hashMd5.some(md5 => md5.hash === entryMd5) ? entry : undefined; - }) - ).catch(e => { - log.error("Error while checking mod zip entries", mod.name, e); - throw e; - }) - ).filter(entry => !!entry); - - if (checkedEntries.length !== download.hashMd5.length) { + if (hashCount !== download.hashMd5.length) { return false; } - const verionPath = await this.bsLocalService.getVersionPath(version); + const versionPath = await this.bsLocalService.getVersionPath(version); const isBSIPA = mod.name.toLowerCase() === "bsipa"; - const destDir = isBSIPA ? verionPath : path.join(verionPath, ModsInstallFolder.PENDING); + const destDir = isBSIPA ? versionPath : path.join(versionPath, ModsInstallFolder.PENDING); - await ensureDir(destDir); log.info("Start extracting mod zip", mod.name, "to", destDir); - const extracted = await extractZip(zip, destDir) + const extracted = await zip.extract(destDir) .then(() => true) .catch(e => { log.error("Error while extracting mod zip", e); return false; + }) + .finally(() => { + zip.close(); }); log.info("Mod zip extraction end", mod.name, "to", destDir, "success:", extracted); diff --git a/src/shared/helpers/string.helpers.ts b/src/shared/helpers/string.helpers.ts index 56649c4c..584168db 100644 --- a/src/shared/helpers/string.helpers.ts +++ b/src/shared/helpers/string.helpers.ts @@ -11,3 +11,8 @@ export function findHashInString(str: string, algorithm: keyof typeof HashAlgori const match = regex.exec(str); return match ? match[0] : undefined; } + +export function escapeRegExp(str: string): string { + // Regex taken from lodash escapeRegExp function + return str.replace(/[\\^$.*+?()[\]{}|]/g, '\\$&'); +}