diff --git a/firecrawl_workflow_test.json b/firecrawl_workflow_test.json new file mode 100644 index 0000000..1d9c697 --- /dev/null +++ b/firecrawl_workflow_test.json @@ -0,0 +1,141 @@ +{ + "id": "b2020ae4-e010-49af-8673-36c128438ec4", + "name": "Workflow 1760727761212", + "nodes": [ + { + "id": "4747e8b4-c1cb-4c56-9ca8-cab73543c909", + "type": "dataInput", + "position": { + "x": 353, + "y": 63 + }, + "data": { + "id": "5fa3cdfe-9c5a-4b78-95d8-dc1e74b627bb", + "type": "dataInput", + "label": "dataInput Node", + "status": "success", + "config": { + "dataType": "url", + "defaultValue": "https://www.example.com" + }, + "inputs": [], + "outputs": [ + { + "output": "https://vesterai.com/" + } + ] + } + }, + { + "id": "3d26cd62-8f6b-449e-beaa-864683d5ccb4", + "type": "webScraping", + "position": { + "x": 425.8264547480898, + "y": 123.15899777101305 + }, + "data": { + "id": "38dcf0c1-a1e8-432b-ac38-479cc2e60150", + "type": "webScraping", + "label": "webScraping Node", + "status": "success", + "config": { + "formats": [ + "text", + "html" + ], + "url": "{{input-1.output}}", + "onlyMainContent": false + }, + "inputs": [], + "outputs": [ + { + "output": "Error scraping https://vesterai.com/: Bad Request. Please check your Firecrawl API key and try again.", + "url": "https://vesterai.com/", + "error": "Bad Request" + } + ] + } + }, + { + "id": "d33f8686-55c4-44e0-86a9-45290d6f6d2f", + "type": "llmTask", + "position": { + "x": 614.6608564428151, + "y": 226.6299028092187 + }, + "data": { + "id": "7666152a-79d7-44e3-b5c5-c312b9adf82d", + "type": "llmTask", + "label": "llmTask Node", + "status": "success", + "config": { + "prompt": "Summarize: {{scraper-1.output}}", + "model": "deepseek-chat", + "temperature": 0.7 + }, + "inputs": [], + "outputs": [ + { + "output": "Based on the error message, here is a summary:\n\nAn attempt to automatically extract data (scrape) from the website `vesterai.com` failed because the request was invalid (a \"Bad Request\" error).\n\nThe most likely cause is an issue with the **Firecrawl API key**, such as:\n* The key is missing, incorrect, or invalid.\n* The key has expired.\n* The associated account lacks the necessary permissions or credits.\n\n**The suggested solution is to verify that the Firecrawl API key is correct and properly configured, then try the operation again.**", + "prompt": "Summarize: Error scraping https://vesterai.com/: Bad Request. Please check your Firecrawl API key and try again.", + "model": "deepseek-chat", + "temperature": 0.7, + "maxTokens": 1000, + "usage": { + "promptTokens": 30, + "completionTokens": 119, + "totalTokens": 149 + } + } + ] + } + }, + { + "id": "b6fcec2c-e976-4aa6-bca1-adbb61cfc9b9", + "type": "dataOutput", + "position": { + "x": 712.9582162291103, + "y": 265.4314921985458 + }, + "data": { + "id": "94e9b104-f049-40c9-8433-b153ab32065c", + "type": "dataOutput", + "label": "dataOutput Node", + "status": "success", + "config": { + "format": "json", + "filename": "analysis.json" + }, + "inputs": [], + "outputs": [ + { + "output": "{\n \"data\": \"Based on the error message, here is a summary:\\n\\nAn attempt to automatically extract data (scrape) from the website `vesterai.com` failed because the request was invalid (a \\\"Bad Request\\\" error).\\n\\nThe most likely cause is an issue with the **Firecrawl API key**, such as:\\n* The key is missing, incorrect, or invalid.\\n* The key has expired.\\n* The associated account lacks the necessary permissions or credits.\\n\\n**The suggested solution is to verify that the Firecrawl API key is correct and properly configured, then try the operation again.**\",\n \"timestamp\": \"2025-10-17T19:19:51.704Z\"\n}", + "format": "json", + "filename": "analysis.json" + } + ] + } + } + ], + "edges": [ + { + "id": "94b044ce-0f92-4f47-981e-d56c348c015b", + "source": "4747e8b4-c1cb-4c56-9ca8-cab73543c909", + "target": "3d26cd62-8f6b-449e-beaa-864683d5ccb4" + }, + { + "id": "1f522c38-18a8-4497-ad35-034e9147c255", + "source": "3d26cd62-8f6b-449e-beaa-864683d5ccb4", + "target": "d33f8686-55c4-44e0-86a9-45290d6f6d2f" + }, + { + "id": "e3c4d985-dcdd-4486-a3ee-325013816d7f", + "source": "d33f8686-55c4-44e0-86a9-45290d6f6d2f", + "target": "b6fcec2c-e976-4aa6-bca1-adbb61cfc9b9" + } + ], + "createdAt": "2025-10-17T19:02:41.212Z", + "updatedAt": "2025-10-17T19:19:51.704Z", + "exportedAt": "2025-10-17T19:20:30.000Z", + "version": "1.0.0" +} \ No newline at end of file diff --git a/src/services/firecrawlService.ts b/src/services/firecrawlService.ts index f60d6a5..70b4331 100644 --- a/src/services/firecrawlService.ts +++ b/src/services/firecrawlService.ts @@ -84,72 +84,115 @@ export const scrapeWithFirecrawl = async ( // Initialize Firecrawl client with the current API key const firecrawl = new Firecrawl({ apiKey: apiKey, + // Add base URL if needed + // baseURL: 'https://api.firecrawl.dev/v0' }); - // Prepare the scrape options according to Firecrawl v2 API + // Validate URL + const url = (config.url || "").trim(); + if (!/^https?:\/\//i.test(url)) { + throw new Error( + `Invalid URL provided to Firecrawl: "${config.url}". URL must start with http:// or https://` + ); + } + + // Prepare a minimal, whitelisted options object per Firecrawl v2 + // Some undocumented keys can cause 400 errors; keep this strict. const scrapeOptions: any = { - formats: config.formats || ["markdown", "html"], - onlyMainContent: config.onlyMainContent !== false, // Default to true + formats: + config.formats && config.formats.length > 0 + ? config.formats + : ["markdown"], + onlyMainContent: config.onlyMainContent !== false, + ...(config.timeout !== undefined ? { timeout: config.timeout } : {}), + ...(config.location ? { location: config.location } : {}), }; - // Add optional parameters - if (config.includeTags && config.includeTags.length > 0) { - scrapeOptions.includeTags = config.includeTags; - } - - if (config.excludeTags && config.excludeTags.length > 0) { - scrapeOptions.excludeTags = config.excludeTags; - } - - if (config.waitFor !== undefined) { - scrapeOptions.waitFor = config.waitFor; - } - - if (config.timeout !== undefined) { - scrapeOptions.timeout = config.timeout; - } - - if (config.maxAge !== undefined) { - scrapeOptions.maxAge = config.maxAge; - } - - if (config.storeInCache !== undefined) { - scrapeOptions.storeInCache = config.storeInCache; - } - - if (config.actions) { - scrapeOptions.actions = config.actions; - } - - if (config.location) { - scrapeOptions.location = config.location; - } - // Perform the scrape - SDK returns data directly - const data = await firecrawl.scrape(config.url, scrapeOptions); + console.log("Firecrawl API Call - URL:", config.url); + console.log("Firecrawl API Call - Options:", scrapeOptions); + console.log( + "Firecrawl API Call - API Key:", + apiKey.substring(0, 10) + "..." + ); - // Apply max length to markdown if specified - if (config.maxLength && data.markdown) { - data.markdown = data.markdown.substring(0, config.maxLength) + "..."; + try { + // First attempt with provided options + const data = await firecrawl.scrape(url, scrapeOptions); + console.log("Firecrawl API Response:", data); + + if (!data) { + throw new Error("No data returned from Firecrawl API"); + } + + return { + success: true, + data: { + markdown: data.markdown, + html: data.html, + rawHtml: data.rawHtml, + summary: data.summary, + links: data.links, + images: data.images, + screenshot: data.screenshot, + json: data.json, + // metadata: data.metadata, + actions: data.actions, + }, + }; + } catch (apiError) { + // Extract as much detail as possible from the error + const anyErr: any = apiError as any; + const status = anyErr?.status || anyErr?.response?.status; + const body = anyErr?.response?.data || anyErr?.data || anyErr?.message; + console.error("Firecrawl API Call Error:", { + status, + body, + }); + + // If it's a 400 Bad Request, retry once with ultra-minimal options + if (status === 400) { + const minimalOptions = { + formats: ["markdown"], + onlyMainContent: true, + } as const; + console.warn( + "Retrying Firecrawl.scrape with minimal options:", + minimalOptions + ); + const retryData = await firecrawl.scrape(url, minimalOptions as any); + if (!retryData) { + throw new Error("No data returned from Firecrawl API (after retry)"); + } + return { + success: true, + data: { + markdown: retryData.markdown, + html: retryData.html, + rawHtml: retryData.rawHtml, + summary: retryData.summary, + links: retryData.links, + images: retryData.images, + screenshot: retryData.screenshot, + json: retryData.json, + actions: retryData.actions, + }, + }; + } + + throw new Error( + `Firecrawl request failed${status ? ` (status ${status})` : ""}: ${ + typeof body === "string" ? body : JSON.stringify(body) + }` + ); } - - return { - success: true, - data: { - markdown: data.markdown, - html: data.html, - rawHtml: data.rawHtml, - summary: data.summary, - links: data.links, - images: data.images, - screenshot: data.screenshot, - json: data.json, - // metadata: data.metadata, - actions: data.actions, - }, - }; } catch (error) { console.error("Firecrawl API Error:", error); + console.error("Error details:", { + message: error instanceof Error ? error.message : "Unknown error", + stack: error instanceof Error ? error.stack : undefined, + name: error instanceof Error ? error.name : undefined, + }); // Return a fallback response if API fails return {