Refactor workflow execution to support variable substitution and enhance node processing

- Updated workflowStore to implement variable substitution for node outputs, allowing dynamic input handling across different node types.
- Modified processing functions for LLM, data output, web scraping, embedding, similarity search, and structured output nodes to utilize the new variable substitution mechanism.
- Adjusted demo-workflows.json to reflect changes in node configurations, including format updates for web scraping and model changes for LLM tasks.

public/demo-workflows.json

@@ -33,7 +33,7 @@
             "status": "idle",
             "config": {
               "url": "{{input-1.output}}",
-              "selector": "body",
+              "formats": ["markdown", "html"],
               "maxLength": 2000
             },
             "inputs": [],
@@ -51,7 +51,7 @@
             "status": "idle",
             "config": {
               "prompt": "Analyze the following web content and provide a summary:\n\n{{scraper-1.output}}",
-              "model": "gpt-3.5-turbo",
+              "model": "deepseek-chat",
               "temperature": 0.7,
               "maxTokens": 500
             },

src/examples/variableSubstitutionExample.md

@@ -0,0 +1,145 @@
+# Variable Substitution Example
+
+This document demonstrates how the `{{input-1.output}}` variable substitution works in the workflow system.
+
+## How It Works
+
+### 1. **Data Flow Process**
+
+```
+Testing Panel Input → Data Input Node → Web Scraping Node → LLM Node → Data Output Node
+```
+
+### 2. **Variable Substitution Examples**
+
+#### Example 1: Web Scraping Node
+
+```json
+{
+  "config": {
+    "url": "{{input-1.output}}",
+    "formats": ["markdown", "html"]
+  }
+}
+```
+
+**What happens:**
+
+1. User enters `"https://github.com"` in testing panel
+2. Data Input Node stores: `{ output: "https://github.com" }`
+3. Web Scraping Node processes: `"{{input-1.output}}"` → `"https://github.com"`
+4. Firecrawl scrapes the actual GitHub URL
+
+#### Example 2: LLM Task Node
+
+```json
+{
+  "config": {
+    "prompt": "Analyze this content: {{scraper-1.output}}",
+    "model": "deepseek-chat"
+  }
+}
+```
+
+**What happens:**
+
+1. Web Scraping Node outputs: `{ output: "[scraped content]" }`
+2. LLM Node processes: `"{{scraper-1.output}}"` → `"[scraped content]"`
+3. AI analyzes the actual scraped content
+
+### 3. **Supported Variable Patterns**
+
+| Pattern             | Description               | Example               |
+| ------------------- | ------------------------- | --------------------- |
+| `{{nodeId}}`        | Entire output of the node | `{{input-1}}`         |
+| `{{nodeId.output}}` | Output property           | `{{input-1.output}}`  |
+| `{{nodeId.data}}`   | Data property             | `{{scraper-1.data}}`  |
+| `{{nodeId.status}}` | Status property           | `{{llm-1.status}}`    |
+| `{{nodeId.error}}`  | Error property            | `{{scraper-1.error}}` |
+
+### 4. **Complete Workflow Example**
+
+```json
+{
+  "nodes": [
+    {
+      "id": "input-1",
+      "type": "dataInput",
+      "config": {
+        "dataType": "url",
+        "defaultValue": "https://example.com"
+      }
+    },
+    {
+      "id": "scraper-1",
+      "type": "webScraping",
+      "config": {
+        "url": "{{input-1.output}}",
+        "formats": ["markdown", "html"]
+      }
+    },
+    {
+      "id": "llm-1",
+      "type": "llmTask",
+      "config": {
+        "prompt": "Summarize: {{scraper-1.output}}",
+        "model": "deepseek-chat"
+      }
+    },
+    {
+      "id": "output-1",
+      "type": "dataOutput",
+      "config": {
+        "format": "json",
+        "filename": "analysis.json"
+      }
+    }
+  ]
+}
+```
+
+### 5. **Execution Flow**
+
+1. **Testing Panel**: User enters `"https://github.com"`
+2. **Data Input Node** (`input-1`):
+
+   - Receives: `"https://github.com"`
+   - Stores: `{ output: "https://github.com" }`
+   - Status: `"success"`
+
+3. **Web Scraping Node** (`scraper-1`):
+
+   - Config URL: `"{{input-1.output}}"`
+   - Variable substitution: `"{{input-1.output}}"` → `"https://github.com"`
+   - Calls Firecrawl with: `"https://github.com"`
+   - Returns: `{ output: "[GitHub page content]", markdown: "...", html: "..." }`
+
+4. **LLM Task Node** (`llm-1`):
+
+   - Config prompt: `"Summarize: {{scraper-1.output}}"`
+   - Variable substitution: `"{{scraper-1.output}}"` → `"[GitHub page content]"`
+   - Calls AI with: `"Summarize: [GitHub page content]"`
+   - Returns: `{ output: "[AI summary of GitHub page]" }`
+
+5. **Data Output Node** (`output-1`):
+   - Receives: `"[AI summary of GitHub page]"`
+   - Formats as JSON and saves to `analysis.json`
+
+### 6. **Error Handling**
+
+If a variable reference doesn't exist:
+
+- `{{nonexistent-node.output}}` → `{{nonexistent-node.output}}` (unchanged)
+- Warning logged to console
+- Node continues with original string
+
+### 7. **Testing the Feature**
+
+1. Create a new workflow
+2. Add a Data Input node with URL as default value
+3. Add a Web Scraping node with URL set to `{{input-1.output}}`
+4. Add an LLM Task node with prompt containing `{{scraper-1.output}}`
+5. Run the workflow with a test URL
+6. Observe how variables are substituted with actual data
+
+This variable substitution system enables powerful data flow between nodes, making workflows dynamic and flexible!

src/store/workflowStore.ts

@@ -12,6 +12,7 @@ import {
   scrapeWithFirecrawl,
   FirecrawlConfig,
 } from "../services/firecrawlService";
+import { substituteVariables, NodeOutput } from "../utils/variableSubstitution";
 
 // localStorage key for workflows
 const WORKFLOWS_STORAGE_KEY = "agent-workflow-builder-workflows";
@@ -300,6 +301,30 @@ export const useWorkflowStore = create<WorkflowStore>((set, get) => ({
 
     set({ isExecuting: true, executionResults: {} });
 
+    // Map to track node outputs for variable substitution
+    const nodeOutputs = new Map<string, NodeOutput>();
+
+    // Create a mapping from node labels to node IDs for easier variable substitution
+    const nodeLabelToId = new Map<string, string>();
+    currentWorkflow.nodes.forEach((node) => {
+      const label = node.data.label.toLowerCase().replace(/\s+/g, "-");
+      nodeLabelToId.set(label, node.id);
+      // Also map common patterns
+      if (node.data.type === "dataInput") {
+        nodeLabelToId.set("input-1", node.id);
+        nodeLabelToId.set("data-input", node.id);
+      } else if (node.data.type === "webScraping") {
+        nodeLabelToId.set("scraper-1", node.id);
+        nodeLabelToId.set("web-scraper", node.id);
+      } else if (node.data.type === "llmTask") {
+        nodeLabelToId.set("llm-1", node.id);
+        nodeLabelToId.set("llm-task", node.id);
+      } else if (node.data.type === "dataOutput") {
+        nodeLabelToId.set("output-1", node.id);
+        nodeLabelToId.set("data-output", node.id);
+      }
+    });
+
     try {
       // Find the first data input node to start with
       const dataInputNode = currentWorkflow.nodes.find(
@@ -321,6 +346,14 @@ export const useWorkflowStore = create<WorkflowStore>((set, get) => ({
       const dataInputResult = { output: inputData };
       get().updateNodeStatus(dataInputNode.id, "success", dataInputResult);
 
+      // Store the data input node's output for variable substitution
+      nodeOutputs.set(dataInputNode.id, {
+        nodeId: dataInputNode.id,
+        output: inputData,
+        data: dataInputResult,
+        status: "success",
+      });
+
       // Process remaining nodes in order
       const remainingNodes = currentWorkflow.nodes.filter(
         (node) => node.id !== dataInputNode.id
@@ -330,32 +363,60 @@ export const useWorkflowStore = create<WorkflowStore>((set, get) => ({
         get().updateNodeStatus(node.id, "running");
         await new Promise((resolve) => setTimeout(resolve, 500));
 
-        // Process node based on type
+        // Process node based on type with variable substitution
         let result;
         switch (node.data.type) {
           case "llmTask":
-            result = await processLLMNode(node, inputData);
+            result = await processLLMNode(node, nodeOutputs, nodeLabelToId);
             break;
           case "dataOutput":
-            result = await processDataOutputNode(node, inputData);
+            result = await processDataOutputNode(
+              node,
+              nodeOutputs,
+              nodeLabelToId
+            );
             break;
           case "webScraping":
-            result = await processWebScrapingNode(node, inputData);
+            result = await processWebScrapingNode(
+              node,
+              nodeOutputs,
+              nodeLabelToId
+            );
             break;
           case "embeddingGenerator":
-            result = await processEmbeddingNode(node, inputData);
+            result = await processEmbeddingNode(
+              node,
+              nodeOutputs,
+              nodeLabelToId
+            );
             break;
           case "similaritySearch":
-            result = await processSimilaritySearchNode(node, inputData);
+            result = await processSimilaritySearchNode(
+              node,
+              nodeOutputs,
+              nodeLabelToId
+            );
             break;
           case "structuredOutput":
-            result = await processStructuredOutputNode(node, inputData);
+            result = await processStructuredOutputNode(
+              node,
+              nodeOutputs,
+              nodeLabelToId
+            );
             break;
           default:
             result = { output: `Processed by ${node.data.type} node` };
         }
 
         get().updateNodeStatus(node.id, "success", result);
+
+        // Store the node's output for future variable substitution
+        nodeOutputs.set(node.id, {
+          nodeId: node.id,
+          output: result.output,
+          data: result,
+          status: "success",
+        });
       }
     } catch (error) {
       console.error("Workflow execution failed:", error);
@@ -390,15 +451,23 @@ export const useWorkflowStore = create<WorkflowStore>((set, get) => ({
 }));
 
 // Node processing functions
-const processLLMNode = async (node: WorkflowNode, inputData: string) => {
+const processLLMNode = async (
+  node: WorkflowNode,
+  nodeOutputs: Map<string, NodeOutput>,
+  nodeLabelToId?: Map<string, string>
+) => {
   const config = node.data.config;
   const prompt = config.prompt || "Process the following input: {{input}}";
   const model = config.model || "gpt-3.5-turbo";
   const temperature = config.temperature || 0.7;
   const maxTokens = config.maxTokens || 1000;
 
-  // Replace {{input}} placeholder with actual input data
-  const processedPrompt = prompt.replace(/\{\{input\}\}/g, inputData);
+  // Apply variable substitution to the prompt
+  const processedPrompt = substituteVariables(
+    prompt,
+    nodeOutputs,
+    nodeLabelToId
+  );
 
   try {
     // Call OpenAI API
@@ -437,11 +506,18 @@ const processLLMNode = async (node: WorkflowNode, inputData: string) => {
   }
 };
 
-const processDataOutputNode = async (node: WorkflowNode, inputData: string) => {
+const processDataOutputNode = async (
+  node: WorkflowNode,
+  nodeOutputs: Map<string, NodeOutput>,
+  nodeLabelToId?: Map<string, string>
+) => {
   const config = node.data.config;
   const format = config.format || "text";
   const filename = config.filename || "output.txt";
 
+  // Get the most recent input data from the node outputs
+  const inputData = Array.from(nodeOutputs.values()).pop()?.output || "";
+
   let output;
   switch (format) {
     case "json":
@@ -463,10 +539,17 @@ const processDataOutputNode = async (node: WorkflowNode, inputData: string) => {
 
 const processWebScrapingNode = async (
   node: WorkflowNode,
-  inputData: string
+  nodeOutputs: Map<string, NodeOutput>,
+  nodeLabelToId?: Map<string, string>
 ) => {
   const config = node.data.config;
-  const url = config.url || inputData;
+
+  // Apply variable substitution to the URL
+  const urlTemplate = config.url || "";
+  const url =
+    substituteVariables(urlTemplate, nodeOutputs, nodeLabelToId) ||
+    Array.from(nodeOutputs.values()).pop()?.output ||
+    "";
 
   // Prepare Firecrawl configuration
   const firecrawlConfig: FirecrawlConfig = {
@@ -526,26 +609,37 @@ const processWebScrapingNode = async (
   }
 };
 
-const processEmbeddingNode = async (node: WorkflowNode, inputData: string) => {
+const processEmbeddingNode = async (
+  node: WorkflowNode,
+  nodeOutputs: Map<string, NodeOutput>,
+  nodeLabelToId?: Map<string, string>
+) => {
   const config = node.data.config;
   const model = config.model || "text-embedding-ada-002";
   const dimensions = config.dimensions || 1536;
 
+  // Get the most recent input data from the node outputs
+  const inputData = Array.from(nodeOutputs.values()).pop()?.output || "";
+
   // In a real app, this would generate actual embeddings
   const mockEmbedding = Array.from({ length: dimensions }, () => Math.random());
 
-  return { output: mockEmbedding, model, dimensions };
+  return { output: mockEmbedding, model, dimensions, inputData };
 };
 
 const processSimilaritySearchNode = async (
   node: WorkflowNode,
-  inputData: string
+  nodeOutputs: Map<string, NodeOutput>,
+  nodeLabelToId?: Map<string, string>
 ) => {
   const config = node.data.config;
   const vectorStore = config.vectorStore || "pinecone";
   const topK = config.topK || 5;
   const threshold = config.threshold || 0.8;
 
+  // Get the most recent input data from the node outputs
+  const inputData = Array.from(nodeOutputs.values()).pop()?.output || "";
+
   // In a real app, this would perform actual similarity search
   const mockResults = Array.from({ length: topK }, (_, i) => ({
     id: `result_${i + 1}`,
@@ -558,12 +652,16 @@ const processSimilaritySearchNode = async (
 
 const processStructuredOutputNode = async (
   node: WorkflowNode,
-  inputData: string
+  nodeOutputs: Map<string, NodeOutput>,
+  nodeLabelToId?: Map<string, string>
 ) => {
   const config = node.data.config;
   const schema = config.schema || '{"type": "object"}';
   const model = config.model || "gpt-3.5-turbo";
 
+  // Get the most recent input data from the node outputs
+  const inputData = Array.from(nodeOutputs.values()).pop()?.output || "";
+
   // In a real app, this would use an LLM to structure the data according to the schema
   const mockStructuredOutput = {
     input: inputData,

src/utils/variableSubstitution.ts

@@ -0,0 +1,170 @@
+/**
+ * Variable substitution utility for workflow nodes
+ * Handles patterns like {{nodeId.output}}, {{nodeId.data}}, etc.
+ */
+
+export interface NodeOutput {
+  nodeId: string;
+  output?: any;
+  data?: any;
+  error?: string;
+  status?: string;
+}
+
+/**
+ * Substitute variables in a string with actual node outputs
+ * @param template - String containing variables like {{nodeId.output}}
+ * @param nodeOutputs - Map of node outputs keyed by node ID
+ * @param nodeLabelToId - Optional mapping from friendly names to node IDs
+ * @returns String with variables substituted
+ */
+export const substituteVariables = (
+  template: string,
+  nodeOutputs: Map<string, NodeOutput>,
+  nodeLabelToId?: Map<string, string>
+): string => {
+  if (!template || typeof template !== "string") {
+    return template;
+  }
+
+  // Pattern to match {{nodeId.property}} or {{nodeId}}
+  const variablePattern = /\{\{([^}]+)\}\}/g;
+
+  return template.replace(variablePattern, (match, variablePath) => {
+    const trimmedPath = variablePath.trim();
+
+    // Handle different variable patterns
+    if (trimmedPath.includes(".")) {
+      // Pattern: {{nodeId.property}}
+      const [nodeIdOrLabel, property] = trimmedPath.split(".", 2);
+
+      // Try to resolve the node ID from the label mapping first
+      let actualNodeId = nodeIdOrLabel;
+      if (nodeLabelToId && nodeLabelToId.has(nodeIdOrLabel)) {
+        actualNodeId = nodeLabelToId.get(nodeIdOrLabel)!;
+      }
+
+      const nodeOutput = nodeOutputs.get(actualNodeId);
+
+      if (!nodeOutput) {
+        console.warn(
+          `Node ${nodeIdOrLabel} (${actualNodeId}) not found for variable ${match}`
+        );
+        return match; // Return original if node not found
+      }
+
+      // Handle different property types
+      switch (property) {
+        case "output":
+          return nodeOutput.output ? String(nodeOutput.output) : "";
+        case "data":
+          return nodeOutput.data ? String(nodeOutput.data) : "";
+        case "error":
+          return nodeOutput.error ? String(nodeOutput.error) : "";
+        case "status":
+          return nodeOutput.status ? String(nodeOutput.status) : "";
+        default:
+          // Try to access nested property
+          try {
+            const value = getNestedProperty(nodeOutput, property);
+            return value ? String(value) : "";
+          } catch (error) {
+            console.warn(
+              `Property ${property} not found in node ${nodeIdOrLabel}`
+            );
+            return match;
+          }
+      }
+    } else {
+      // Pattern: {{nodeId}} - return the entire output
+      // Try to resolve the node ID from the label mapping first
+      let actualNodeId = trimmedPath;
+      if (nodeLabelToId && nodeLabelToId.has(trimmedPath)) {
+        actualNodeId = nodeLabelToId.get(trimmedPath)!;
+      }
+
+      const nodeOutput = nodeOutputs.get(actualNodeId);
+      if (!nodeOutput) {
+        console.warn(
+          `Node ${trimmedPath} (${actualNodeId}) not found for variable ${match}`
+        );
+        return match;
+      }
+      return nodeOutput.output ? String(nodeOutput.output) : "";
+    }
+  });
+};
+
+/**
+ * Get nested property from an object using dot notation
+ * @param obj - Object to get property from
+ * @param path - Dot-separated property path
+ * @returns Property value or undefined
+ */
+const getNestedProperty = (obj: any, path: string): any => {
+  return path.split(".").reduce((current, key) => {
+    return current && current[key] !== undefined ? current[key] : undefined;
+  }, obj);
+};
+
+/**
+ * Extract all variable references from a string
+ * @param template - String to analyze
+ * @returns Array of variable references found
+ */
+export const extractVariables = (template: string): string[] => {
+  if (!template || typeof template !== "string") {
+    return [];
+  }
+
+  const variablePattern = /\{\{([^}]+)\}\}/g;
+  const matches = [];
+  let match;
+
+  while ((match = variablePattern.exec(template)) !== null) {
+    matches.push(match[1].trim());
+  }
+
+  return matches;
+};
+
+/**
+ * Check if a string contains variables
+ * @param template - String to check
+ * @returns True if string contains variables
+ */
+export const hasVariables = (template: string): boolean => {
+  return /\{\{[^}]+\}\}/.test(template);
+};
+
+/**
+ * Validate that all variables in a template have corresponding node outputs
+ * @param template - String containing variables
+ * @param nodeOutputs - Map of available node outputs
+ * @returns Object with validation results
+ */
+export const validateVariables = (
+  template: string,
+  nodeOutputs: Map<string, NodeOutput>
+): {
+  isValid: boolean;
+  missingNodes: string[];
+  availableNodes: string[];
+} => {
+  const variables = extractVariables(template);
+  const missingNodes: string[] = [];
+  const availableNodes = Array.from(nodeOutputs.keys());
+
+  variables.forEach((variable) => {
+    const nodeId = variable.includes(".") ? variable.split(".")[0] : variable;
+    if (!nodeOutputs.has(nodeId)) {
+      missingNodes.push(nodeId);
+    }
+  });
+
+  return {
+    isValid: missingNodes.length === 0,
+    missingNodes,
+    availableNodes,
+  };
+};