ai-space/workflow-builder
3
0
Fork 0
mirror of https://github.com/Nikhil-Doye/workflow-builder.git synced 2026-07-22 02:01:56 +02:00
Code Issues Packages Projects Releases Wiki Activity

Enhance firecrawlService with improved error handling and URL validation

Browse Source 
- Added URL validation to ensure the provided URL starts with http:// or https://.
- Refactored scrape options to create a minimal, whitelisted options object to prevent 400 errors.
- Improved error logging to capture detailed information on API call failures.
- Implemented a retry mechanism for 400 Bad Request errors with minimal options for better resilience.
This commit is contained in:
Nikhil-Doye
2025-10-17 15:55:33 -04:00
parent ba52bd6eeb
commit 0a0fe82d98
2 changed files with 240 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
firecrawl_workflow_test.json
+141
View File
@@ -0,0 +1,141 @@
{
"id": "b2020ae4-e010-49af-8673-36c128438ec4",
"name": "Workflow 1760727761212",
"nodes": [
{
"id": "4747e8b4-c1cb-4c56-9ca8-cab73543c909",
"type": "dataInput",
"position": {
"x": 353,
"y": 63
},
"data": {
"id": "5fa3cdfe-9c5a-4b78-95d8-dc1e74b627bb",
"type": "dataInput",
"label": "dataInput Node",
"status": "success",
"config": {
"dataType": "url",
"defaultValue": "https://www.example.com"
},
"inputs": [],
"outputs": [
{
"output": "https://vesterai.com/"
}
]
}
},
{
"id": "3d26cd62-8f6b-449e-beaa-864683d5ccb4",
"type": "webScraping",
"position": {
"x": 425.8264547480898,
"y": 123.15899777101305
},
"data": {
"id": "38dcf0c1-a1e8-432b-ac38-479cc2e60150",
"type": "webScraping",
"label": "webScraping Node",
"status": "success",
"config": {
"formats": [
"text",
"html"
],
"url": "{{input-1.output}}",
"onlyMainContent": false
},
"inputs": [],
"outputs": [
{
"output": "Error scraping https://vesterai.com/: Bad Request. Please check your Firecrawl API key and try again.",
"url": "https://vesterai.com/",
"error": "Bad Request"
}
]
}
},
{
"id": "d33f8686-55c4-44e0-86a9-45290d6f6d2f",
"type": "llmTask",
"position": {
"x": 614.6608564428151,
"y": 226.6299028092187
},
"data": {
"id": "7666152a-79d7-44e3-b5c5-c312b9adf82d",
"type": "llmTask",
"label": "llmTask Node",
"status": "success",
"config": {
"prompt": "Summarize: {{scraper-1.output}}",
"model": "deepseek-chat",
"temperature": 0.7
},
"inputs": [],
"outputs": [
{
"output": "Based on the error message, here is a summary:\n\nAn attempt to automatically extract data (scrape) from the website `vesterai.com` failed because the request was invalid (a \"Bad Request\" error).\n\nThe most likely cause is an issue with the **Firecrawl API key**, such as:\n* The key is missing, incorrect, or invalid.\n* The key has expired.\n* The associated account lacks the necessary permissions or credits.\n\n**The suggested solution is to verify that the Firecrawl API key is correct and properly configured, then try the operation again.**",
"prompt": "Summarize: Error scraping https://vesterai.com/: Bad Request. Please check your Firecrawl API key and try again.",
"model": "deepseek-chat",
"temperature": 0.7,
"maxTokens": 1000,
"usage": {
"promptTokens": 30,
"completionTokens": 119,
"totalTokens": 149
}
}
]
}
},
{
"id": "b6fcec2c-e976-4aa6-bca1-adbb61cfc9b9",
"type": "dataOutput",
"position": {
"x": 712.9582162291103,
"y": 265.4314921985458
},
"data": {
"id": "94e9b104-f049-40c9-8433-b153ab32065c",
"type": "dataOutput",
"label": "dataOutput Node",
"status": "success",
"config": {
"format": "json",
"filename": "analysis.json"
},
"inputs": [],
"outputs": [
{
"output": "{\n \"data\": \"Based on the error message, here is a summary:\\n\\nAn attempt to automatically extract data (scrape) from the website `vesterai.com` failed because the request was invalid (a \\\"Bad Request\\\" error).\\n\\nThe most likely cause is an issue with the **Firecrawl API key**, such as:\\n* The key is missing, incorrect, or invalid.\\n* The key has expired.\\n* The associated account lacks the necessary permissions or credits.\\n\\n**The suggested solution is to verify that the Firecrawl API key is correct and properly configured, then try the operation again.**\",\n \"timestamp\": \"2025-10-17T19:19:51.704Z\"\n}",
"format": "json",
"filename": "analysis.json"
}
]
}
}
],
"edges": [
{
"id": "94b044ce-0f92-4f47-981e-d56c348c015b",
"source": "4747e8b4-c1cb-4c56-9ca8-cab73543c909",
"target": "3d26cd62-8f6b-449e-beaa-864683d5ccb4"
},
{
"id": "1f522c38-18a8-4497-ad35-034e9147c255",
"source": "3d26cd62-8f6b-449e-beaa-864683d5ccb4",
"target": "d33f8686-55c4-44e0-86a9-45290d6f6d2f"
},
{
"id": "e3c4d985-dcdd-4486-a3ee-325013816d7f",
"source": "d33f8686-55c4-44e0-86a9-45290d6f6d2f",
"target": "b6fcec2c-e976-4aa6-bca1-adbb61cfc9b9"
}
],
"createdAt": "2025-10-17T19:02:41.212Z",
"updatedAt": "2025-10-17T19:19:51.704Z",
"exportedAt": "2025-10-17T19:20:30.000Z",
"version": "1.0.0"
}
src/services/firecrawlService.ts
+99 -56
View File
@@ -84,72 +84,115 @@ export const scrapeWithFirecrawl = async (
// Initialize Firecrawl client with the current API key
const firecrawl = new Firecrawl({
apiKey: apiKey,
// Add base URL if needed
// baseURL: 'https://api.firecrawl.dev/v0'
});
// Prepare the scrape options according to Firecrawl v2 API
// Validate URL
const url = (config.url || "").trim();
if (!/^https?:\/\//i.test(url)) {
throw new Error(
`Invalid URL provided to Firecrawl: "${config.url}". URL must start with http:// or https://`
);
}
// Prepare a minimal, whitelisted options object per Firecrawl v2
// Some undocumented keys can cause 400 errors; keep this strict.
const scrapeOptions: any = {
formats: config.formats || ["markdown", "html"],
onlyMainContent: config.onlyMainContent !== false, // Default to true
formats:
config.formats && config.formats.length > 0
? config.formats
: ["markdown"],
onlyMainContent: config.onlyMainContent !== false,
...(config.timeout !== undefined ? { timeout: config.timeout } : {}),
...(config.location ? { location: config.location } : {}),
};
// Add optional parameters
if (config.includeTags && config.includeTags.length > 0) {
scrapeOptions.includeTags = config.includeTags;
}
if (config.excludeTags && config.excludeTags.length > 0) {
scrapeOptions.excludeTags = config.excludeTags;
}
if (config.waitFor !== undefined) {
scrapeOptions.waitFor = config.waitFor;
}
if (config.timeout !== undefined) {
scrapeOptions.timeout = config.timeout;
}
if (config.maxAge !== undefined) {
scrapeOptions.maxAge = config.maxAge;
}
if (config.storeInCache !== undefined) {
scrapeOptions.storeInCache = config.storeInCache;
}
if (config.actions) {
scrapeOptions.actions = config.actions;
}
if (config.location) {
scrapeOptions.location = config.location;
}
// Perform the scrape - SDK returns data directly
const data = await firecrawl.scrape(config.url, scrapeOptions);
console.log("Firecrawl API Call - URL:", config.url);
console.log("Firecrawl API Call - Options:", scrapeOptions);
console.log(
"Firecrawl API Call - API Key:",
apiKey.substring(0, 10) + "..."
);
// Apply max length to markdown if specified
if (config.maxLength && data.markdown) {
data.markdown = data.markdown.substring(0, config.maxLength) + "...";
try {
// First attempt with provided options
const data = await firecrawl.scrape(url, scrapeOptions);
console.log("Firecrawl API Response:", data);
if (!data) {
throw new Error("No data returned from Firecrawl API");
}
return {
success: true,
data: {
markdown: data.markdown,
html: data.html,
rawHtml: data.rawHtml,
summary: data.summary,
links: data.links,
images: data.images,
screenshot: data.screenshot,
json: data.json,
// metadata: data.metadata,
actions: data.actions,
},
};
} catch (apiError) {
// Extract as much detail as possible from the error
const anyErr: any = apiError as any;
const status = anyErr?.status || anyErr?.response?.status;
const body = anyErr?.response?.data || anyErr?.data || anyErr?.message;
console.error("Firecrawl API Call Error:", {
status,
body,
});
// If it's a 400 Bad Request, retry once with ultra-minimal options
if (status === 400) {
const minimalOptions = {
formats: ["markdown"],
onlyMainContent: true,
} as const;
console.warn(
"Retrying Firecrawl.scrape with minimal options:",
minimalOptions
);
const retryData = await firecrawl.scrape(url, minimalOptions as any);
if (!retryData) {
throw new Error("No data returned from Firecrawl API (after retry)");
}
return {
success: true,
data: {
markdown: retryData.markdown,
html: retryData.html,
rawHtml: retryData.rawHtml,
summary: retryData.summary,
links: retryData.links,
images: retryData.images,
screenshot: retryData.screenshot,
json: retryData.json,
actions: retryData.actions,
},
};
}
throw new Error(
`Firecrawl request failed${status ? ` (status ${status})` : ""}: ${
typeof body === "string" ? body : JSON.stringify(body)
}`
);
}
return {
success: true,
data: {
markdown: data.markdown,
html: data.html,
rawHtml: data.rawHtml,
summary: data.summary,
links: data.links,
images: data.images,
screenshot: data.screenshot,
json: data.json,
// metadata: data.metadata,
actions: data.actions,
},
};
} catch (error) {
console.error("Firecrawl API Error:", error);
console.error("Error details:", {
message: error instanceof Error ? error.message : "Unknown error",
stack: error instanceof Error ? error.stack : undefined,
name: error instanceof Error ? error.name : undefined,
});
// Return a fallback response if API fails
return {