import { createWriteStream, WriteStream } from 'fs'; import { deleteFileSync, Progression } from 'main/helpers/fs.helpers'; import { Observable } from 'rxjs'; import { shareReplay, tap } from 'rxjs/operators'; import log from 'electron-log'; import got from 'got'; import { IncomingHttpHeaders, IncomingMessage } from 'http'; import { tryit } from 'shared/helpers/error.helpers'; import path from 'path'; import { pipeline } from 'stream/promises'; import sanitize from 'sanitize-filename'; import internal from 'stream'; import { app, net } from 'electron'; import { CookieJar } from 'tough-cookie'; export class RequestService { private static instance: RequestService; private readonly baseHeaders = { 'User-Agent': `BSManager/${app.getVersion()} (Electron/${process.versions.electron} Chrome/${process.versions.chrome} Node/${process.versions.node})`, } private readonly PREFERRED_FAMILY_TESTS = [4, 6]; private preferredFamilyCache: Record = {}; public static getInstance(): RequestService { if (!RequestService.instance) { RequestService.instance = new RequestService(); } return RequestService.instance; } private constructor() {} private isBeatmodsUrl(url: string): boolean { const { hostname } = new URL(url); return hostname === 'beatmods.com' || hostname.endsWith('.beatmods.com'); } /** * Uses Electron's Chromium network stack instead of Node's HTTP stack * to avoid Cloudflare timeout issues that occur with beatmods.com */ private async requestWithElectronNet(url: string): Promise<{ data: T; headers: IncomingHttpHeaders }> { return new Promise<{ data: T; headers: IncomingHttpHeaders }>((resolve, reject) => { const request = net.request({ method: 'GET', url, headers: this.baseHeaders, }); let responseBody = Buffer.alloc(0); let responseHeaders: IncomingHttpHeaders = {}; let isResolved = false; const timeoutId = setTimeout(() => { if (!isResolved) { isResolved = true; request.abort(); reject(new Error(`Request timeout for ${url}`)); } }, 15000); const cleanup = () => { clearTimeout(timeoutId); }; request.on('response', (response) => { responseHeaders = response.headers as IncomingHttpHeaders; // Validate HTTP status code (got throws on non-2xx by default) const { statusCode } = response; if (statusCode < 200 || statusCode >= 300) { isResolved = true; cleanup(); reject(new Error(`Request failed with status ${statusCode} for ${url}`)); return; } response.on('data', (chunk: Buffer) => { responseBody = Buffer.concat([responseBody, chunk]); }); response.on('end', () => { if (isResolved) return; isResolved = true; cleanup(); try { const bodyText = responseBody.toString('utf-8'); const data = JSON.parse(bodyText) as T; resolve({ data, headers: responseHeaders }); } catch (parseError) { reject(new Error(`Failed to parse JSON response from ${url}: ${parseError}`)); } }); response.on('error', (error) => { if (isResolved) return; isResolved = true; cleanup(); reject(new Error(`Response stream error for ${url}: ${error.message}`)); }); }); request.on('error', (error) => { if (isResolved) return; isResolved = true; cleanup(); reject(new Error(`Network error requesting ${url}: ${error.message}`)); }); request.end(); }); } public async getJSON(url: string): Promise<{ data: T; headers: IncomingHttpHeaders }> { // Node's HTTP stack has Cloudflare compatibility issues with beatmods.com if (this.isBeatmodsUrl(url)) { return this.requestWithElectronNet(url); } const domain = (new URL(url)).hostname; const cachedFamily = this.preferredFamilyCache[domain]; if (cachedFamily) { try { return await this.requestData(url, cachedFamily); } catch (error: any) { throw new Error(`Request failed: ${url}`, error); } } // Try on each IPv4/6 families on first request to a domain/website for (const family of this.PREFERRED_FAMILY_TESTS) { try { const response = await this.requestData(url, family); log.info(`Caching "${domain}" with IPv${family}`); this.preferredFamilyCache[domain] = family; return response; } catch (err) { log.warn(`IPv${family} request failed, trying next one... URL: ${url}`, err); } } throw new Error(`IPv4 and IPv6 requests failed for URL: ${url}`); } private async requestData(url: string, family: number): Promise<{ data: T; headers: IncomingHttpHeaders }> { const cookieJar = new CookieJar(); const first = await got(url, { // @ts-ignore (ESM is not well supported in this project, We need to move out electron-react-boilerplate, and use Vite) dnsLookupIpVersion: family, cookieJar, headers: this.baseHeaders }); // Follow script redirect to get the JSON if (first.headers['content-type']?.includes('text/html')) { const cookieMatch = first.body.match(/document\.cookie="([^"]+)"/); if (!cookieMatch) { throw new Error("Cookie not found in JS"); } const cookieString = cookieMatch[1]; const redirectMatch = first.body.match(/location\.href="([^"]+)"/); if (!redirectMatch) { throw new Error("Redirect URL not found"); } const jsonUrl = redirectMatch[1]; await cookieJar.setCookie(cookieString, jsonUrl); const second = await got(jsonUrl, { // @ts-ignore (ESM is not well supported in this project, We need to move out electron-react-boilerplate, and use Vite) dnsLookupIpVersion: family, responseType: "json", cookieJar, headers: this.baseHeaders }); return { data: second.body as T, headers: second.headers }; } return { data: JSON.parse(first.body) as T, headers: first.headers }; } /** * Uses Electron's Chromium network stack instead of Node's HTTP stack * to avoid Cloudflare timeout issues that occur with beatmods.com */ private downloadFileWithElectronNet( url: string, dest: string, opt?: { preferContentDisposition?: boolean } ): Observable> { return new Observable>((subscriber) => { const progress: Progression = { current: 0, total: 0 }; let file: WriteStream | undefined; let isCompleted = false; const request = net.request({ method: 'GET', url, headers: this.baseHeaders, }); const cleanup = () => { if (file) { file.destroy(); } }; request.on('response', (response) => { // Validate HTTP status code (got throws on non-2xx by default) const { statusCode } = response; if (statusCode < 200 || statusCode >= 300) { isCompleted = true; cleanup(); subscriber.error(new Error(`Download failed with status ${statusCode} for ${url}`)); return; } const contentLength = response.headers['content-length']; if (contentLength) { const length = Array.isArray(contentLength) ? contentLength[0] : contentLength; progress.total = parseInt(length, 10); } const filename = opt?.preferContentDisposition ? this.getFilenameFromContentDisposition(response.headers['content-disposition'] as string) : null; if (filename) { dest = path.join(path.dirname(dest), sanitize(filename)); } progress.data = dest; file = createWriteStream(dest); file.on('error', (error) => { cleanup(); tryit(() => deleteFileSync(dest)); if (!isCompleted) { isCompleted = true; subscriber.error(new Error(`File write error for ${dest}: ${error.message}`)); } }); response.on('data', (chunk: Buffer) => { if (file && !file.destroyed) { progress.current += chunk.length; subscriber.next(progress); file.write(chunk); } }); response.on('end', () => { if (isCompleted) return; if (file && !file.destroyed) { file.once('finish', () => { if (isCompleted) return; isCompleted = true; subscriber.next(progress); subscriber.complete(); }); file.end(); } else { isCompleted = true; subscriber.next(progress); subscriber.complete(); } }); response.on('error', (error) => { if (isCompleted) return; isCompleted = true; cleanup(); tryit(() => deleteFileSync(dest)); subscriber.error(error); }); }); request.on('error', (error) => { if (isCompleted) return; isCompleted = true; cleanup(); tryit(() => deleteFileSync(dest)); subscriber.error(error); }); request.end(); return () => { request.abort(); cleanup(); }; }).pipe( tap({ error: (e) => log.error(e, url, dest) }), shareReplay(1) ); } public downloadFile( url: string, dest: string, opt?: { preferContentDisposition?: boolean } ): Observable> { // Node's HTTP stack has Cloudflare compatibility issues with beatmods.com if (this.isBeatmodsUrl(url)) { return this.downloadFileWithElectronNet(url, dest, opt); } return new Observable>((subscriber) => { const progress: Progression = { current: 0, total: 0 }; let attempt = 0; let stream: got.GotEmitter & internal.Duplex; const domain = (new URL(url)).hostname; const cachedFamily = this.preferredFamilyCache[domain]; const familiesToTry = cachedFamily ? [ cachedFamily ] : this.PREFERRED_FAMILY_TESTS; const tryNextFamily = () => { if (attempt >= familiesToTry.length) { subscriber.error(new Error(`Download failed over IPv4 and IPv6 for URL: ${url}`)); return; } const family = familiesToTry[attempt++]; let file: WriteStream | undefined; // @ts-ignore (ESM is not well supported in this project, We need to move out electron-react-boilerplate, and use Vite) stream = got.stream(url, { dnsLookupIpVersion: family, headers: this.baseHeaders }); stream.on('response', (response) => { if (!cachedFamily) { log.info(`Caching "${domain}" with IPv${family}`); this.preferredFamilyCache[domain] = family; } const filename = opt?.preferContentDisposition ? this.getFilenameFromContentDisposition(response.headers['content-disposition']) : null; if (filename) { dest = path.join(path.dirname(dest), sanitize(filename)); } progress.data = dest; file = createWriteStream(dest); pipeline(stream, file).catch(err => { file?.destroy(); tryit(() => deleteFileSync(dest)); subscriber.error(err); }); }); stream.on('downloadProgress', ({ transferred, total }) => { progress.current = transferred; progress.total = total; subscriber.next(progress); }); stream.on('error', err => { log.warn(`Download failed over IPv${family} for URL: ${url}`, err); stream.destroy(); file?.destroy(); tryNextFamily(); }); stream.on('end', () => { file?.end(); subscriber.next(progress); subscriber.complete(); }); }; tryNextFamily(); return () => { stream?.destroy(); }; }).pipe( tap({ error: (e) => log.error(e, url, dest) }), shareReplay(1) ); } /** * Uses Electron's Chromium network stack instead of Node's HTTP stack * to avoid Cloudflare timeout issues that occur with beatmods.com */ private downloadBufferWithElectronNet( url: string, options?: got.GotOptions ): Observable> { return new Observable>((subscriber) => { const progress: Progression = { current: 0, total: 0, data: null, }; // Convert headers to the format expected by electron.net (string | string[]) const electronHeaders: Record = { ...this.baseHeaders }; if (options?.headers) { for (const [key, value] of Object.entries(options.headers)) { if (typeof value === 'string' || Array.isArray(value)) { electronHeaders[key] = value; } else if (value != null) { electronHeaders[key] = String(value); } } } let data = Buffer.alloc(0); let responseHeaders: IncomingHttpHeaders = {}; let isCompleted = false; const request = net.request({ method: 'GET', url, headers: electronHeaders, }); request.on('response', (response) => { // Validate HTTP status code (got throws on non-2xx by default) const { statusCode } = response; if (statusCode < 200 || statusCode >= 300) { isCompleted = true; subscriber.error(new Error(`Download failed with status ${statusCode} for ${url}`)); return; } const contentLength = response.headers['content-length']; if (contentLength) { const length = Array.isArray(contentLength) ? contentLength[0] : contentLength; progress.total = parseInt(length, 10); } responseHeaders = response.headers as IncomingHttpHeaders; response.on('data', (chunk: Buffer) => { data = Buffer.concat([data, chunk]); progress.current = data.length; subscriber.next(progress); }); response.on('end', () => { if (isCompleted) return; isCompleted = true; progress.data = data; // Required to maintain API compatibility with got-based implementation const mockResponse = { headers: responseHeaders, } as IncomingMessage; progress.extra = mockResponse; subscriber.next(progress); subscriber.complete(); }); response.on('error', (error) => { if (isCompleted) return; isCompleted = true; subscriber.error(error); }); }); request.on('error', (error) => { if (isCompleted) return; isCompleted = true; subscriber.error(error); }); request.end(); return () => { request.abort(); }; }).pipe( tap({ error: (e) => log.error(e, url) }), shareReplay(1) ); } public downloadBuffer( url: string, options?: got.GotOptions ): Observable> { // Node's HTTP stack has Cloudflare compatibility issues with beatmods.com if (this.isBeatmodsUrl(url)) { return this.downloadBufferWithElectronNet(url, options); } return new Observable>((subscriber) => { const progress: Progression = { current: 0, total: 0, data: null, }; const headers = { ...this.baseHeaders, ...(options?.headers ?? {}) }; let attempt = 0; let stream: got.GotEmitter & internal.Duplex; const domain = (new URL(url)).hostname; const cachedFamily = this.preferredFamilyCache[domain]; const familiesToTry = cachedFamily ? [ cachedFamily ] : this.PREFERRED_FAMILY_TESTS; const tryNextFamily = () => { if (attempt >= familiesToTry.length) { subscriber.error(new Error(`Download failed over IPv4 and IPv6 for URL: ${url}`)); return; } const family = familiesToTry[attempt++]; // @ts-ignore (ESM is not well supported in this project, We need to move out electron-react-boilerplate, and use Vite) stream = got.stream(url, { dnsLookupIpVersion: family, ...(options ?? {}), headers }); let data = Buffer.alloc(0); let response: IncomingMessage; stream.once('response', (res) => { if (!cachedFamily) { log.info(`Caching "${domain}" with IPv${family}`); this.preferredFamilyCache[domain] = family; } response = res; }); stream.on('data', (chunk: Buffer) => { data = Buffer.concat([data, chunk]); }); stream.on('downloadProgress', ({ transferred, total }) => { progress.current = transferred; progress.total = total; subscriber.next(progress); }); stream.on('error', err => { log.warn(`Download failed over IPv${family} for URL: ${url}`, err); stream.destroy(); tryNextFamily(); }); stream.on('end', () => { progress.data = data; progress.extra = response; subscriber.next(progress); subscriber.complete(); }); }; tryNextFamily(); return () => { stream?.destroy(); }; }).pipe( tap({ error: (e) => log.error(e, url) }), shareReplay(1) ); } public getFilenameFromContentDisposition(disposition: string): string | undefined { if(!disposition) { return undefined; } const utf8FilenameRegex = /filename\*=UTF-8''([\w%\-\.]+)(?:; ?|$)/i; const asciiFilenameRegex = /^filename=(["']?)(.*?[^\\])\1(?:; ?|$)/i; const utf8Match = utf8FilenameRegex.exec(disposition); if (utf8Match?.[1]) { return decodeURIComponent(utf8Match[1]); } const filenameStart = disposition.toLowerCase().indexOf('filename='); if (filenameStart < 0) { return undefined; } const partialDisposition = disposition.slice(filenameStart); return asciiFilenameRegex.exec(partialDisposition)?.[2]; } }