Refactor NodeConfiguration and enhance concurrency handling

- Moved the retrieval of the latest node data to a more appropriate location in NodeConfiguration, ensuring the most up-to-date configuration is used.
- Updated the ConcurrencyLimiter to correctly handle the queue resolution by passing `undefined` to the next function, improving clarity and functionality.
- Enhanced the DatabaseConnectionManager to securely merge connection updates while excluding sensitive fields, ensuring better security practices.
- Made executionId optional in ExecutionEngine methods, providing more flexibility in execution handling.
- Improved JSON parsing validation to allow extra fields, enhancing the robustness of the parsing process.
- Updated circular dependency detection in WorkflowValidationEngine to utilize a Set for improved performance and clarity.

src/components/NodeConfiguration.tsx

@@ -405,6 +405,11 @@ export const NodeConfiguration: React.FC<NodeConfigurationProps> = ({
   const [optimizationResult, setOptimizationResult] = useState<string>("");
   const [isPreviewingOptimization, setIsPreviewingOptimization] =
     useState(false);
+
+  // Get the latest node data from the store to ensure we have the most up-to-date config
+  const currentNode = currentWorkflow?.nodes.find((node) => node.id === nodeId);
+  const currentData = currentNode?.data || data;
+
   const [originalPrompt, setOriginalPrompt] = useState<string>(
     (currentData.config && currentData.config.prompt) || ""
   );
@@ -414,10 +419,6 @@ export const NodeConfiguration: React.FC<NodeConfigurationProps> = ({
   const [showLabelDialog, setShowLabelDialog] = useState(false);
   const [pendingLabel, setPendingLabel] = useState<string>("");
 
-  // Get the latest node data from the store to ensure we have the most up-to-date config
-  const currentNode = currentWorkflow?.nodes.find((node) => node.id === nodeId);
-  const currentData = currentNode?.data || data;
-
   const handleConfigChange = (key: string, value: any) => {
     updateNode(nodeId, {
       config: {

src/services/concurrencyLimiter.ts

@@ -6,7 +6,7 @@
 export class ConcurrencyLimiter {
   private maxConcurrency: number;
   private running: number = 0;
-  private queue: Array<() => void> = [];
+  private queue: Array<(value: unknown) => void> = [];
 
   constructor(maxConcurrency: number = 5) {
     this.maxConcurrency = Math.max(1, maxConcurrency);
@@ -31,7 +31,7 @@ export class ConcurrencyLimiter {
       // Process next item in queue if any
       const nextResolve = this.queue.shift();
       if (nextResolve) {
-        nextResolve();
+        nextResolve(undefined);
       }
     }
   }

src/services/databaseConnectionManager.ts

@@ -186,9 +186,21 @@ class DatabaseConnectionManager {
     const connection = this.connections.get(id);
     if (!connection) return false;
 
+    // Merge connection with updates first
+    const mergedConnection = { ...connection, ...updates };
+    // Remove id and status as required by secureCredentials
+    const {
+      id: _,
+      status: __,
+      ...connectionWithoutIdStatus
+    } = mergedConnection;
+
     // SECURITY: Handle credential updates securely
-    const secureUpdates = await this.secureCredentials(updates, id);
-    const updatedConnection = { ...connection, ...secureUpdates };
+    const secureUpdates = await this.secureCredentials(
+      connectionWithoutIdStatus,
+      id
+    );
+    const updatedConnection = { ...connection, ...updates, ...secureUpdates };
 
     // Validate credentials if connection details are being updated
     if (this.hasConnectionDetailsChanged(updates)) {

src/services/executionEngine.ts

@@ -521,7 +521,7 @@ export class ExecutionEngine {
       data?: any,
       error?: string
     ) => void,
-    executionId: string
+    executionId: string = ""
   ): Promise<void> {
     plan.status = "running";
     plan.startTime = new Date();
@@ -550,7 +550,7 @@ export class ExecutionEngine {
       data?: any,
       error?: string
     ) => void,
-    executionId: string
+    executionId: string = ""
   ): Promise<void> {
     const executionOrder = this.getExecutionOrder(plan.nodes, plan.edges);
 
@@ -602,7 +602,7 @@ export class ExecutionEngine {
       data?: any,
       error?: string
     ) => void,
-    executionId: string
+    executionId: string = ""
   ): Promise<void> {
     const parallelGroups = this.createParallelGroups(
       plan.nodes,
@@ -637,7 +637,7 @@ export class ExecutionEngine {
       data?: any,
       error?: string
     ) => void,
-    executionId: string
+    executionId: string = ""
   ): Promise<void> {
     const maxConcurrency =
       group.maxConcurrency ||
@@ -687,7 +687,7 @@ export class ExecutionEngine {
       data?: any,
       error?: string
     ) => void,
-    executionId: string
+    executionId: string = ""
   ): Promise<void> {
     const executionOrder = this.getExecutionOrder(plan.nodes, plan.edges);
     const visited = new Set<string>();
@@ -762,7 +762,7 @@ export class ExecutionEngine {
       data?: any,
       error?: string
     ) => void,
-    executionId: string
+    executionId: string = ""
   ): Promise<void> {
     for (const branch of branches) {
       const shouldExecuteTrue = await this.evaluateCondition(
@@ -924,7 +924,7 @@ export class ExecutionEngine {
       data?: any,
       error?: string
     ) => void,
-    executionId: string
+    executionId: string = ""
   ): Promise<void> {
     context.status = "running";
     context.startTime = new Date();

src/services/robustJsonParser.ts

@@ -277,6 +277,7 @@ export function parseAndValidate<T = any>(
     strictMode?: boolean;
     fallbackValue?: T;
     logAttempts?: boolean;
+    allowExtra?: boolean;
   } = {}
 ): ParseResult<T> & { validationErrors?: string[] } {
   const parseResult = parseRobustJson<T>(input, options);
@@ -285,11 +286,9 @@ export function parseAndValidate<T = any>(
     return parseResult;
   }
 
-  const validation = validateJsonStructure(
-    parseResult.data,
-    requiredFields,
-    options
-  );
+  const validation = validateJsonStructure(parseResult.data, requiredFields, {
+    allowExtra: options.allowExtra,
+  });
 
   if (!validation.isValid) {
     return {

src/services/workflowValidationEngine.ts

@@ -199,14 +199,16 @@ export class WorkflowValidationEngine {
 
     // Check for circular dependencies
     const circularDeps = this.detectCircularDependencies(nodeIds, edges);
-    if (circularDeps.length > 0) {
+    if (circularDeps.size > 0) {
       result.isValid = false;
       result.errors.push({
         id: "circular_dependency",
         severity: "error",
         type: "circular_dependency",
         nodeIds: Array.from(circularDeps),
-        message: `Circular dependencies detected: ${circularDeps.join(", ")}`,
+        message: `Circular dependencies detected: ${Array.from(
+          circularDeps
+        ).join(", ")}`,
         suggestion:
           "Remove the circular connections between nodes to create a valid execution path",
       });