zedcode/bs-manager
1
0
Fork 0
mirror of https://github.com/Zagrios/bs-manager.git synced 2026-07-03 14:08:25 +02:00
Code Issues Packages Projects Releases Wiki Activity

[bugfix] use yauzl for extracting zips

Browse Source
This commit is contained in:
silentrald
2024-11-08 00:16:35 +08:00
parent 7c6c6147a6
commit 68559add5d
11 changed files with 268 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitattributes
+1
View File
@@ -16,3 +16,4 @@
*.woff binary
*.woff2 binary
assets/scripts/* binary
assets/tests/*.zip binary
.github/workflows/node.js.yaml
+5 -4
View File
@@ -13,7 +13,7 @@ jobs:
build:
strategy:
matrix:
os: [ubuntu-latest, windows-latest, macos-latest]
os: [ubuntu-latest, windows-latest]
runs-on: ${{ matrix.os }}
@@ -26,6 +26,7 @@ jobs:
cache: "npm"
- run: npm ci
- run: npm run build
- run: npm test
# Currently typescript, eslint, and prettier are unhappy
continue-on-error: true
- name: Unit tests
run: npm run test:unit
assets/tests/standard.zip
BIN
View File
Binary file not shown.
assets/tests/windows_legacy.zip
BIN
View File
Binary file not shown.
jest.config.ts
+19
View File
@@ -0,0 +1,19 @@
import type { Config } from "jest";
const config: Config = {
// testURL: "http://localhost/",
// testEnvironment: "jsdom",
transform: {
"\\.(ts|tsx|js|jsx)$": "ts-jest",
},
moduleNameMapper: {
"\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$": "<rootDir>/.erb/mocks/fileMock.js",
"\\.(css|less|sass|scss)$": "identity-obj-proxy",
},
moduleFileExtensions: ["js", "jsx", "ts", "tsx", "json"],
moduleDirectories: ["node_modules", "src"],
testPathIgnorePatterns: ["release/app/dist"],
setupFiles: ["./.erb/scripts/check-build-exists.ts"],
};
export default config;
package-lock.json
Generated
+24 -6
View File
@@ -61,7 +61,8 @@
"tippy.js": "^6.3.7",
"to-ico": "^1.1.5",
"use-double-click": "^1.0.5",
"use-fit-text": "^2.4.0"
"use-fit-text": "^2.4.0",
"yauzl": "^3.2.0"
},
"devDependencies": {
"@electron/fuses": "^1.7.0",
@@ -8555,7 +8556,6 @@
},
"node_modules/buffer-crc32": {
"version": "0.2.13",
"dev": true,
"license": "MIT",
"engines": {
"node": "*"
@@ -12585,6 +12585,17 @@
"@types/yauzl": "^2.9.1"
}
},
"node_modules/extract-zip/node_modules/yauzl": {
"version": "2.10.0",
"resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz",
"integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==",
"dev": true,
"license": "MIT",
"dependencies": {
"buffer-crc32": "~0.2.3",
"fd-slicer": "~1.1.0"
}
},
"node_modules/fast-deep-equal": {
"version": "3.1.3",
"license": "MIT"
@@ -12662,6 +12673,8 @@
},
"node_modules/fd-slicer": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz",
"integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -17801,7 +17814,8 @@
},
"node_modules/pend": {
"version": "1.2.0",
"dev": true,
"resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz",
"integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==",
"license": "MIT"
},
"node_modules/performance-now": {
@@ -22655,12 +22669,16 @@
}
},
"node_modules/yauzl": {
"version": "2.10.0",
"dev": true,
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/yauzl/-/yauzl-3.2.0.tgz",
"integrity": "sha512-Ow9nuGZE+qp1u4JIPvg+uCiUr7xGQWdff7JQSk5VGYTAZMDe2q8lxJ10ygv10qmSj031Ty/6FNJpLO4o1Sgc+w==",
"license": "MIT",
"dependencies": {
"buffer-crc32": "~0.2.3",
"fd-slicer": "~1.1.0"
"pend": "~1.2.0"
},
"engines": {
"node": ">=12"
}
},
"node_modules/yn": {
package.json
+3 -29
View File
@@ -19,6 +19,7 @@
"start:preload": "cross-env NODE_ENV=development TS_NODE_TRANSPILE_ONLY=true webpack --config ./.erb/configs/webpack.config.preload.dev.ts",
"start:renderer": "cross-env NODE_ENV=development TS_NODE_TRANSPILE_ONLY=true webpack serve --config ./.erb/configs/webpack.config.renderer.dev.ts",
"test": "jest",
"test:unit": "jest ./src/__tests__/unit",
"publish": "npm run build && electron-builder -c.win.certificateSha1=2164d6a7d641ecf6ad57852f665a518ca2bf960f --publish always --win --x64",
"publish:linux": "npm run build && electron-builder --publish always --linux --x64"
},
@@ -116,34 +117,6 @@
"beat-saber"
],
"homepage": "https://github.com/Zagrios/bs-manager#readme",
"jest": {
"testURL": "http://localhost/",
"testEnvironment": "jsdom",
"transform": {
"\\.(ts|tsx|js|jsx)$": "ts-jest"
},
"moduleNameMapper": {
"\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$": "<rootDir>/.erb/mocks/fileMock.js",
"\\.(css|less|sass|scss)$": "identity-obj-proxy"
},
"moduleFileExtensions": [
"js",
"jsx",
"ts",
"tsx",
"json"
],
"moduleDirectories": [
"node_modules",
"src"
],
"testPathIgnorePatterns": [
"release/app/dist"
],
"setupFiles": [
"./.erb/scripts/check-build-exists.ts"
]
},
"devDependencies": {
"@electron/fuses": "^1.7.0",
"@electron/notarize": "^2.3.0",
@@ -281,7 +254,8 @@
"tippy.js": "^6.3.7",
"to-ico": "^1.1.5",
"use-double-click": "^1.0.5",
"use-fit-text": "^2.4.0"
"use-fit-text": "^2.4.0",
"yauzl": "^3.2.0"
},
"engines": {
"node": ">=20.0.0"
src/__tests__/unit/zip.test.ts
+75
View File
@@ -0,0 +1,75 @@
import path from "path";
import { mkdir, pathExistsSync, rm } from "fs-extra";
import { extractZip } from "main/helpers/zip.helpers";
const TEST_FOLDER = path.resolve(__dirname, "../../..", "assets", "tests");
const STANDARD_ZIP = path.join(TEST_FOLDER, "standard.zip");
const WINDOWS_LEGACY_MAP_ZIP = path.join(TEST_FOLDER, "windows_legacy.zip");
const DESTINATION_FOLDER = path.join(TEST_FOLDER, "out");
describe("Zip Server Service Test", () => {
beforeAll(async () => {
if (pathExistsSync(DESTINATION_FOLDER)) {
await rm(DESTINATION_FOLDER, { recursive: true, force: true });
}
await mkdir(DESTINATION_FOLDER);
});
// Uses '/'
it("Extract standard zip", async () => {
await extractZip(STANDARD_ZIP, DESTINATION_FOLDER);
for (const file of [
"file_1.2.txt",
"file_1.3.txt",
]) {
expect(pathExistsSync(path.join(DESTINATION_FOLDER, file)))
.toBe(true);
}
const SUBFOLDER_PATH = path.join(DESTINATION_FOLDER, "folder_1.1");
for (const file of [
"file_2.1.txt",
"file_2.2.txt",
]) {
expect(pathExistsSync(path.join(SUBFOLDER_PATH, file)))
.toBe(true);
}
});
// Uses '\\'
it("Extract map zips using forward slashes", async () => {
await extractZip(WINDOWS_LEGACY_MAP_ZIP, DESTINATION_FOLDER);
// Expect all the files to exists
for (const file of [
"BPMInfo.dat",
"cover.jpg",
"ExpertLegacy.dat",
"ExpertPlusLawless.dat",
"ExpertPlusLegacy.dat",
"ExpertPlusStandard.dat",
"ExpertStandard.dat",
"Info.dat",
"song.egg",
]) {
expect(pathExistsSync(path.join(DESTINATION_FOLDER, file)))
.toBe(true);
}
const SUBFOLDER_PATH = path.join(DESTINATION_FOLDER, "pfp");
for (const file of [
"galaxyCompressed.jpg",
"gojiCompressed.jpg",
]) {
expect(pathExistsSync(path.join(SUBFOLDER_PATH, file)))
.toBe(true);
}
});
afterAll(async () => {
if (pathExistsSync(DESTINATION_FOLDER)) {
await rm(DESTINATION_FOLDER, { recursive: true, force: true });
}
});
});
src/main/helpers/zip.helpers.ts
+108 -18
View File
@@ -1,28 +1,118 @@
import JSZip from "jszip";
import { pathExist } from "./fs.helpers";
import crypto from "crypto";
import fs from "fs-extra";
import path from "path";
import { mkdir, writeFile } from "fs/promises";
import yauzl from "yauzl";
import { FileHashes } from "shared/models/mods";
import { ensureFolderExist } from "./fs.helpers";
export async function extractZip(zip: JSZip, dest: string): Promise<string[]> {
if (!(await pathExist(dest))) {
throw new Error(`Path ${dest} does not exist`);
}
// NOTE: yauzl needs to be reopened when it is read
export async function extractZip(zipPath: string, destination: string): Promise<string[]> {
await ensureFolderExist(destination);
return new Promise((resolve, reject) => {
yauzl.open(zipPath, {
lazyEntries: true,
decodeStrings: true,
},
(openError, zip) => {
if (openError) return reject(openError);
handleExtractZip(zip, destination, resolve, reject);
});
});
}
function handleExtractZip(
zip: yauzl.ZipFile, destination: string,
resolve: (result: string[]) => void, reject: (error: any) => void
) {
const files: string[] = [];
for (const [relativePath, entry] of Object.entries(zip.files)) {
if (entry.dir) {
continue;
zip.readEntry();
zip.on("entry", (entry: yauzl.Entry) => {
const absolutePath = path.join(destination, entry.fileName);
// Entry is a directory / folder
if (entry.fileName.endsWith("/")) {
if (!fs.pathExistsSync(absolutePath)) {
fs.mkdirSync(absolutePath, { recursive: true });
}
return zip.readEntry();
}
const content = await entry.async("nodebuffer");
const outPath = path.join(dest, relativePath);
const outDir = path.dirname(outPath);
// Entry is a file
zip.openReadStream(entry, (readError, readStream) => {
if (readError) return reject(readError);
await mkdir(outDir, { recursive: true });
await writeFile(outPath, content);
// For forward slashes since they don't create directories
const directory = path.dirname(absolutePath);
if (!fs.pathExistsSync(directory)) {
fs.mkdirSync(directory, { recursive: true });
}
files.push(outPath);
}
const writeStream = fs.createWriteStream(absolutePath);
readStream.pipe(writeStream);
return files;
readStream.on("end", () => zip.readEntry());
});
});
zip.once("end", () => {
zip.close();
resolve(files);
});
}
export function validateZip(zipPath: string, hashes: FileHashes[]): Promise<boolean> {
return new Promise((resolve, reject) => {
yauzl.open(zipPath, {
lazyEntries: true,
decodeStrings: true,
},
(openError, zip) => {
if (openError) return reject(openError);
handleValidateZip(zip, hashes, resolve, reject);
});
});
}
function handleValidateZip(
zip: yauzl.ZipFile, hashes: FileHashes[],
resolve: (result: boolean) => void, reject: (error: any) => void
) {
let hashCount = 0;
zip.readEntry();
zip.on("entry", (entry: yauzl.Entry) => {
// Entry is a directory / folder
if (entry.fileName.endsWith("/")) {
return zip.readEntry();
}
// Entry is a file
zip.openReadStream(entry, (readError, readStream) => {
if (readError) return reject(readError);
const chunks: Buffer[] = [];
readStream.on("data", data => {
chunks.push(data);
});
readStream.on("end", () => {
const md5Hash = crypto.createHash("md5")
.update(Buffer.concat(chunks))
.digest("hex");
hashCount += +hashes.some(md5 => md5.hash === md5Hash);
zip.readEntry();
});
});
});
zip.once("end", () => {
zip.close();
resolve(hashCount === hashes.length);
});
}
src/main/services/additional-content/maps/local-maps-manager.service.ts
+5 -16
View File
@@ -30,6 +30,7 @@ import { MapInfo } from "shared/models/maps/info/map-info.model";
import { parseMapInfoDat } from "shared/parsers/maps/map-info.parser";
import { CustomError } from "shared/models/exceptions/custom-error.class";
import { tryit } from "shared/helpers/error.helpers";
import { extractZip } from "main/helpers/zip.helpers";
export class LocalMapsManagerService {
private static instance: LocalMapsManagerService;
@@ -169,17 +170,13 @@ export class LocalMapsManagerService {
return getUrlsAndReturn(mapInfo, hash, mapPath);
}
private async downloadMapZip(zipUrl: string): Promise<{ zip: StreamZip.StreamZipAsync; zipPath: string }> {
private async downloadMapZip(zipUrl: string): Promise<string> {
const fileName = `${path.basename(zipUrl, ".zip")}-${crypto.randomUUID()}.zip`;
const tempPath = this.utils.getTempPath();
await ensureFolderExist(this.utils.getTempPath());
const dest = path.join(tempPath, fileName);
const zipPath = (await lastValueFrom(this.reqService.downloadFile(zipUrl, dest))).data;
const zip = new StreamZip.async({ file: zipPath });
return { zip, zipPath };
return (await lastValueFrom(this.reqService.downloadFile(zipUrl, dest))).data;
}
public getMaps(version?: BSVersion): Observable<BsmLocalMapsProgress> {
@@ -454,16 +451,8 @@ export class LocalMapsManagerService {
return installedMap;
}
const { zip, zipPath } = await this.downloadMapZip(zipUrl);
if (!zip) {
throw new Error(`Cannot download ${zipUrl}`);
}
await ensureFolderExist(mapPath);
await zip.extract(null, mapPath);
await zip.close();
const zipPath = await this.downloadMapZip(zipUrl);
await extractZip(zipPath, mapPath);
await unlink(zipPath);
const localMap = await this.loadMapInfoFromPath(mapPath);
src/main/services/mods/bs-mods-manager.service.ts
+28 -42
View File
@@ -8,17 +8,18 @@ import { RequestService } from "../request.service";
import { spawn } from "child_process";
import { BS_EXECUTABLE } from "../../constants";
import log from "electron-log";
import { deleteFolder, pathExist, Progression, unlinkPath } from "../../helpers/fs.helpers";
import { deleteFolder, ensureFolderExist, pathExist, Progression, unlinkPath } from "../../helpers/fs.helpers";
import { lastValueFrom, Observable } from "rxjs";
import JSZip from "jszip";
import { extractZip } from "../../helpers/zip.helpers";
import { extractZip, validateZip } from "../../helpers/zip.helpers";
import recursiveReadDir from "recursive-readdir";
import { sToMs } from "../../../shared/helpers/time.helpers";
import { ensureDir, pathExistsSync } from "fs-extra";
import { pathExistsSync, unlinkSync } from "fs-extra";
import { CustomError } from "shared/models/exceptions/custom-error.class";
import { popElement } from "shared/helpers/array.helpers";
import { LinuxService } from "../linux.service";
import { tryit } from "shared/helpers/error.helpers";
import { UtilsService } from "../utils.service";
import crypto from "crypto";
export class BsModsManagerService {
private static instance: BsModsManagerService;
@@ -27,6 +28,7 @@ export class BsModsManagerService {
private readonly bsLocalService: BSLocalVersionService;
private readonly linuxService: LinuxService;
private readonly requestService: RequestService;
private readonly utilsService: UtilsService;
private manifestMatches: Mod[];
@@ -42,6 +44,7 @@ export class BsModsManagerService {
this.bsLocalService = BSLocalVersionService.getInstance();
this.linuxService = LinuxService.getInstance();
this.requestService = RequestService.getInstance();
this.utilsService = UtilsService.getInstance();
}
private async getModFromHash(hash: string): Promise<Mod> {
@@ -110,26 +113,22 @@ export class BsModsManagerService {
return this.beatModsApi.getModByHash(injectorMd5);
}
private async downloadZip(zipUrl: string): Promise<JSZip> {
private async downloadZip(zipUrl: string): Promise<string> {
zipUrl = path.join(this.beatModsApi.BEAT_MODS_URL, zipUrl);
log.info("Download mod zip", zipUrl);
const buffer = await lastValueFrom(this.requestService.downloadBuffer(zipUrl))
.then(progress => progress.data)
.catch(e => {
log.error("ZIP", "Error while downloading zip", e);
return undefined;
});
const fileName = `${path.basename(zipUrl, ".zip")}-${crypto.randomUUID()}.zip`;
const tempPath = this.utilsService.getTempPath();
const destination = path.join(tempPath, fileName);
if (!buffer) {
try {
await ensureFolderExist(tempPath);
return (await lastValueFrom(this.requestService.downloadFile(zipUrl, destination))).data;
} catch (error) {
log.error("Could not download zip file at", zipUrl);
return null;
}
return JSZip.loadAsync(buffer).catch(e => {
log.error("ZIP", "Error while loading zip", e);
return null;
});
}
private async executeBSIPA(version: BSVersion, args: string[]): Promise<boolean> {
@@ -200,44 +199,31 @@ export class BsModsManagerService {
}
log.info("Start download mod zip", mod.name, download.url);
const zip = await this.downloadZip(download.url);
log.info("Mod zip download end", mod.name, download.url, !!zip);
if (!zip) {
const zipPath = await this.downloadZip(download.url);
log.info("Mod zip download end", mod.name, download.url, !!zipPath);
if (!zipPath) {
return false;
}
const crypto = require("crypto");
const { files } = zip;
const checkedEntries = (
await Promise.all(
Object.values(files).map(async entry => {
const data = await entry.async("nodebuffer");
const entryMd5 = crypto.createHash("md5").update(data).digest("hex");
return download.hashMd5.some(md5 => md5.hash === entryMd5) ? entry : undefined;
})
).catch(e => {
log.error("Error while checking mod zip entries", mod.name, e);
throw e;
})
).filter(entry => !!entry);
if (checkedEntries.length !== download.hashMd5.length) {
if (!(await validateZip(zipPath, download.hashMd5))) {
return false;
}
const verionPath = await this.bsLocalService.getVersionPath(version);
const versionPath = await this.bsLocalService.getVersionPath(version);
const isBSIPA = mod.name.toLowerCase() === "bsipa";
const destDir = isBSIPA ? verionPath : path.join(verionPath, ModsInstallFolder.PENDING);
const destDir = isBSIPA ? versionPath : path.join(versionPath, ModsInstallFolder.PENDING);
await ensureDir(destDir);
log.info("Start extracting mod zip", mod.name, "to", destDir);
const extracted = await extractZip(zip, destDir)
const extracted = await extractZip(zipPath, destDir)
.then(() => true)
.catch(e => {
log.error("Error while extracting mod zip", e);
return false;
})
.finally(() => {
if (pathExistsSync(zipPath)) {
unlinkSync(zipPath);
}
});
log.info("Mod zip extraction end", mod.name, "to", destDir, "success:", extracted);