From 8c6f3b0aa8ba3fd506852b0c8da796f1b4a3f1cb Mon Sep 17 00:00:00 2001 From: MathieuG-P <40181755+Zagrios@users.noreply.github.com> Date: Thu, 14 Nov 2024 00:52:52 +0100 Subject: [PATCH] [bugfix] Create YauzlZip class to manager zip extraction (not finished) --- .eslintrc.js | 3 +- assets/tests/subfolders.zip | Bin 0 -> 653 bytes assets/tests/windows_legacy_subfolders.zip | Bin 0 -> 806 bytes src/__tests__/App.test.tsx | 9 - src/__tests__/unit/zip.test.ts | 144 +++++++++++++++ src/main/models/yauzl-zip.class.ts | 203 +++++++++++++++++++++ 6 files changed, 349 insertions(+), 10 deletions(-) create mode 100644 assets/tests/subfolders.zip create mode 100644 assets/tests/windows_legacy_subfolders.zip delete mode 100644 src/__tests__/App.test.tsx create mode 100644 src/main/models/yauzl-zip.class.ts diff --git a/.eslintrc.js b/.eslintrc.js index 9577afe0..a4975e8f 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -59,7 +59,8 @@ module.exports = { "react/function-component-definition": "off", "jsx-a11y/control-has-associated-label": "off", "react/button-has-type": "off", - "no-labels": ["error", { "allowLoop": true }] + "no-labels": ["error", { "allowLoop": true }], + "max-classes-per-file": "off", }, parserOptions: { ecmaVersion: 2020, diff --git a/assets/tests/subfolders.zip b/assets/tests/subfolders.zip new file mode 100644 index 0000000000000000000000000000000000000000..a7c44d3830eefe5beab9b44ad6e9768d9e7ffa08 GIT binary patch literal 653 zcmWIWW@h1H0D)tBb0fhFD8U4z4fO-y@+Z*bS-|o}NGi^vsbGVuFh){y9Zd}fT#X5m zsz*RoZOa7ASb?%2%!{PTRIj9>BqKE^CqKZOkx8Bbm+Mu4dO$z`LIM54#h?IYGcrgp zoIRUyUDoV2BRY+2CZ>;(%>gQOp(kA^HGh zE;@~DE@sFgo0|qV*967niB_?%K_;Wq$R=aP2Fzqo81x{S42~P*V7_De=PSr~7|qS9 a{Sm|u@MZ<24q)W6GVEq#VAuoVFaQ9_O>MaV literal 0 HcmV?d00001 diff --git a/assets/tests/windows_legacy_subfolders.zip b/assets/tests/windows_legacy_subfolders.zip new file mode 100644 index 0000000000000000000000000000000000000000..dd5c1aec4bfc8e9d7643f0b15853fe76d7845943 GIT binary patch literal 806 zcmWIWW@Zs#0D-doxshN7lwe{|U@+7V4dG>APt-R}F9G5ZATF)oW?*CisR0uKaQy{n z`dPsGjUXm~bdqcW8`K123e4bun_)tvDI5$O43=nq;YBjVRIj9>1RN~y^dLckC0Ljl z0=(HdK%vA0B7h*kn~_O`8Fx@aR5398cLcGB33!Bg*n%Emo&(rCVj=>!iJ+)}f&Y$8 zP!oxX7~E!pq6P;3J063ZNlZjROht|=SU7{C3I?_{@-TwJ(n#M}-$b9Nhzsy$Wdmtv O1;XV(&6z;VzyJWW*J=#_ literal 0 HcmV?d00001 diff --git a/src/__tests__/App.test.tsx b/src/__tests__/App.test.tsx deleted file mode 100644 index 9e8b6e43..00000000 --- a/src/__tests__/App.test.tsx +++ /dev/null @@ -1,9 +0,0 @@ -import "@testing-library/jest-dom"; -import { render } from "@testing-library/react"; -import App from "../renderer/windows/App"; - -describe("App", () => { - it("should render", () => { - expect(render()).toBeTruthy(); - }); -}); diff --git a/src/__tests__/unit/zip.test.ts b/src/__tests__/unit/zip.test.ts index 18dd0cba..0eefaa16 100644 --- a/src/__tests__/unit/zip.test.ts +++ b/src/__tests__/unit/zip.test.ts @@ -1,10 +1,12 @@ import path from "path"; import { mkdir, pathExistsSync, readFile, rm } from "fs-extra"; import { extractZip, getFilesFromZip } from "main/helpers/zip.helpers"; +import { YauzlZip } from "main/models/yauzl-zip.class"; const TEST_FOLDER = path.resolve(__dirname, "../../..", "assets", "tests"); const STANDARD_ZIP = path.join(TEST_FOLDER, "standard.zip"); const WINDOWS_LEGACY_MAP_ZIP = path.join(TEST_FOLDER, "windows_legacy.zip"); +const SUBFOLDERS_ZIP = path.join(TEST_FOLDER, "subfolders.zip"); const MANIFEST_ZIP = path.join(TEST_FOLDER, "manifest.zip"); const DESTINATION_FOLDER = path.join(TEST_FOLDER, "out"); @@ -99,3 +101,145 @@ describe("Zip Server Service Test", () => { } }); }); + +describe("Test YauzlZip class", () => { + + beforeAll(async () => { + if (pathExistsSync(DESTINATION_FOLDER)) { + await rm(DESTINATION_FOLDER, { recursive: true, force: true }); + } + await mkdir(DESTINATION_FOLDER); + }); + + + + it("Extract standard zip", async () => { + + const zip = await YauzlZip.fromPath(STANDARD_ZIP); + const res = await zip.extract(DESTINATION_FOLDER); + + expect(res.sort()).toEqual([ + "file_1.2.txt", + "file_1.3.txt", + "folder_1.1/", + "folder_1.1/file_2.1.txt", + "folder_1.1/file_2.2.txt", + ].sort()) + + for (const file of [ + "file_1.2.txt", + "file_1.3.txt", + ]) { + expect(pathExistsSync(path.join(DESTINATION_FOLDER, file))) + .toBe(true); + } + + const SUBFOLDER_PATH = path.join(DESTINATION_FOLDER, "folder_1.1"); + for (const file of [ + "file_2.1.txt", + "file_2.2.txt", + ]) { + expect(pathExistsSync(path.join(SUBFOLDER_PATH, file))) + .toBe(true); + } + + }); + + it("Extract map zips using back slashes", async () => { + + const zip = await YauzlZip.fromPath(WINDOWS_LEGACY_MAP_ZIP); + const res = await zip.extract(DESTINATION_FOLDER); + + expect(res.sort()).toEqual([ + "BPMInfo.dat", + "cover.jpg", + "ExpertLegacy.dat", + "ExpertPlusLawless.dat", + "ExpertPlusLegacy.dat", + "ExpertPlusStandard.dat", + "ExpertStandard.dat", + "Info.dat", + "song.egg", + "pfp/", + "pfp/galaxyCompressed.jpg", + "pfp/gojiCompressed.jpg", + ].sort()); + + // Expect all the files to exists + for (const file of [ + "BPMInfo.dat", + "cover.jpg", + "ExpertLegacy.dat", + "ExpertPlusLawless.dat", + "ExpertPlusLegacy.dat", + "ExpertPlusStandard.dat", + "ExpertStandard.dat", + "Info.dat", + "song.egg", + ]) { + expect(pathExistsSync(path.join(DESTINATION_FOLDER, file))) + .toBe(true); + } + + const SUBFOLDER_PATH = path.join(DESTINATION_FOLDER, "pfp"); + for (const file of [ + "galaxyCompressed.jpg", + "gojiCompressed.jpg", + ]) { + expect(pathExistsSync(path.join(SUBFOLDER_PATH, file))) + .toBe(true); + } + + }); + + it("Read Zip with multiple subfolders", async () => { + + const zip = await YauzlZip.fromPath(SUBFOLDERS_ZIP); + const res = await zip.extract(DESTINATION_FOLDER); + + expect(res.sort()).toEqual([ + "1/", + "1/2/", + "1/2/3/", + "1/2/3/4/", + "1/2/3/4/5.txt", + ].sort()); + + for (const folder of [ + "1/", + "1/2/", + "1/2/3/", + "1/2/3/4/", + ]) { + expect(pathExistsSync(path.join(DESTINATION_FOLDER, folder))) + .toBe(true); + } + + const file = "1/2/3/4/5.txt"; + expect(pathExistsSync(path.join(DESTINATION_FOLDER, file))) + .toBe(true); + }); + + it("Read manifest.json from zip file", async () => { + + const zipBuffer = await readFile(MANIFEST_ZIP); + const zip = await YauzlZip.fromBuffer(zipBuffer); + const buffer = await (await zip.findEntry(entry => entry.fileName === "manifest.json")).read(); + const manifest = JSON.parse(buffer.toString()); + expect(manifest).toBeTruthy(); + + // Check some of the fields if they are correct + expect(manifest.appId).toBe("some-app-id"); + expect(manifest.canonicalName).toBe("some-canonical-name"); + expect(manifest.isCore).toBe(true); + }); + + + + afterAll(async () => { + if (pathExistsSync(DESTINATION_FOLDER)) { + await rm(DESTINATION_FOLDER, { recursive: true, force: true }); + } + }); + +}) diff --git a/src/main/models/yauzl-zip.class.ts b/src/main/models/yauzl-zip.class.ts new file mode 100644 index 00000000..f2c4cc31 --- /dev/null +++ b/src/main/models/yauzl-zip.class.ts @@ -0,0 +1,203 @@ +import { createWriteStream, ensureDir } from "fs-extra"; +import path from "path"; +import yauzl, { ZipFile, Options, Entry } from "yauzl" + +export class YauzlZip { + + private static readonly YAUZL_OPEN_OPTIONS: Options = { + lazyEntries: true, + decodeStrings: true, + }; + + public static fromPath(path: string): Promise { + return new Promise((resolve, reject) => { + yauzl.open(path, YauzlZip.YAUZL_OPEN_OPTIONS, (error, zip) => { + if (error) return reject(error); + resolve(new YauzlZip(zip)); + }); + }); + } + + public static fromBuffer(buffer: Buffer): Promise { + return new Promise((resolve, reject) => { + yauzl.fromBuffer(buffer, YauzlZip.YAUZL_OPEN_OPTIONS, (error, zip) => { + if (error) return reject(error); + resolve(new YauzlZip(zip)); + }); + }); + } + + private readonly zip: ZipFile; + private readonly entriesMap = new Map(); + + constructor(zip: ZipFile){ + this.zip = zip; + } + + private async readEntry(): Promise { + return new Promise((resolve, reject) => { + + const onEntry = (entry: Entry) => { + cleanup(); + resolve(new YauzlZipEntry({ entry, zip: this.zip })); + }; + + const onEnd = () => { + cleanup(); + resolve(null); + }; + + const onError = (err: Error) => { + cleanup(); + reject(err); + }; + + const cleanup = () => { + this.zip.removeListener("entry", onEntry); + this.zip.removeListener("end", onEnd); + this.zip.removeListener("error", onError); + }; + + this.zip.once("entry", onEntry); + this.zip.once("end", onEnd); + this.zip.once("error", onError); + this.zip.readEntry(); + }); + } + + // The first time this method is called, it will read all the entries and store them in a map + // The next times it will return the entries from the map to avoid reopening the zip file again + public async *entries(): AsyncGenerator { + + for (const entry of Array.from(this.entriesMap.values())) { + yield entry; + } + + let entry: YauzlZipEntry = await this.readEntry(); + while (entry) { + this.entriesMap.set(entry.fileName, entry); + yield entry; + entry = await this.readEntry(); + } + } + + public async findEntry(func: (entry: YauzlZipEntry) => boolean): Promise { + for await (const entry of this.entries()) { + if (func(entry)) { + return entry; + } + } + return null; + } + + /** + * Extracts all entries from the zip file to the destination folder + * @param destination + * @param opt - { entriesNames?: string[], abortToken: AbortController } + * @returns {Promise} The list of extracted files (relative paths in the destination folder) + * + * `opt` object: + * - `entriesNames` - The list of entries to extract (can be regexs or glob pattern). If not provided, all entries will be extracted + * - `abortToken` - The AbortController instance to abort the extraction + * + */ + public async extract(destination: string, opt?: { entriesNames?: (string|RegExp)[], abortToken?: AbortController }): Promise { + const entriesNames = opt?.entriesNames; + const abortToken = opt?.abortToken; + + if (abortToken?.signal?.aborted) { + return []; + } + + await ensureDir(destination); + + const extracted = new Set() + for await (const entry of this.entries()) { + + if (abortToken?.signal?.aborted) { + break; + } + + if (entriesNames && !entriesNames.some(name => typeof name === "string" ? (entry.fileName === name || path.matchesGlob(entry.fileName, name)) : name.test(entry.fileName))){ + continue; + } + + const extractedFile = await entry.extract(destination); + const dirname = path.dirname(extractedFile); + + // Make zip that use backslash as separator have the same behavior as zip that use forward slash + if(dirname !== "."){ + const split = dirname.split(path.posix.sep); + for (let i = 1; i <= split.length; i++) { + const folder = split.slice(0, i).join(path.posix.sep); + extracted.add(folder + path.posix.sep); + } + } + + + extracted.add(extractedFile); + } + + return Array.from(extracted); + } + + + public close(): void { + this.zip.close(); + } + +} + +class YauzlZipEntry { + + private readonly entry: Entry; + private readonly zip: ZipFile; + + constructor(opt: { entry: Entry, zip: ZipFile }) { + this.entry = opt.entry; + this.zip = opt.zip; + } + + public async read(): Promise { + return new Promise((resolve, reject) => { + this.zip.openReadStream(this.entry, (error, stream) => { + if (error) return reject(error); + const buffers: Buffer[] = []; + stream.on("data", (data) => buffers.push(data as Buffer)); + stream.on("end", () => resolve(Buffer.concat(buffers))); + stream.on("error", reject); + }); + }); + } + + public async extract(destination: string): Promise { + + const destPath = path.join(destination, this.entry.fileName); + + if (this.isDirectory) { + return ensureDir(destPath).then(() => this.entry.fileName); + } + + return new Promise((resolve, reject) => { + this.zip.openReadStream(this.entry, async (error, stream) => { + if (error) return reject(error); + + await ensureDir(path.dirname(destPath)); + + const writeStream = createWriteStream(destPath); + stream.pipe(writeStream); + writeStream.on("finish", () => resolve(this.entry.fileName)); + writeStream.on("error", reject); + }); + }); + } + + public get isDirectory(): boolean { + return this.entry.fileName.endsWith("/"); + } + + public get fileName(): string { + return this.entry.fileName; + } + +}