feat(eko): add human interaction support and fix session issue

Implement complete human interaction system for AI agent:
- Add four interaction callbacks (confirm/input/select/help) in EkoService
- Create HumanInteractionCard component for user interactions
- Implement toolId to requestId mapping to resolve response matching
- Add human-response IPC channel for frontend-backend communication
- Integrate interaction cards into agent execution flow

Fix session execution bug:
- Clear ekoRequest state in finally block after task completion
- Clear ekoRequest on task termination to prevent "Error: cancle"
- Enable multiple consecutive sessions without restart

Dependencies:
- Upgrade @jarvis-agent/core to 0.1.4
- Upgrade @jarvis-agent/electron to 0.1.8

The human interaction feature allows AI to request user assistance
for tasks like login, captcha solving, or manual operations. Cards
are embedded in the message flow for seamless user experience.

Author: lsustc

electron/main/ipc/eko-handlers.ts

@@ -1,5 +1,6 @@
 import { ipcMain } from "electron";
 import { windowContextManager } from "../services/window-context-manager";
+import type { HumanResponseMessage } from "../../../src/models/human-interaction";
 
 /**
  * Register all Eko service related IPC handlers
@@ -76,5 +77,21 @@ export function registerEkoHandlers() {
     }
   });
 
+  // Handle human interaction response
+  ipcMain.handle('eko:human-response', async (event, response: HumanResponseMessage) => {
+    try {
+      console.log('IPC eko:human-response received:', response);
+      const context = windowContextManager.getContext(event.sender.id);
+      if (!context || !context.ekoService) {
+        throw new Error('EkoService not found for this window');
+      }
+      const result = context.ekoService.handleHumanResponse(response);
+      return { success: result };
+    } catch (error: any) {
+      console.error('IPC eko:human-response error:', error);
+      throw error;
+    }
+  });
+
   console.log('[IPC] Eko service handlers registered');
 }

electron/main/services/eko-service.ts

@@ -1,9 +1,11 @@
-import { Eko, Log, SimpleSseMcpClient, type LLMs, type StreamCallbackMessage } from "@jarvis-agent/core";
+import { Eko, Log, SimpleSseMcpClient, type LLMs, type StreamCallbackMessage, type AgentContext } from "@jarvis-agent/core";
 import { BrowserAgent, FileAgent } from "@jarvis-agent/electron";
 import type { EkoResult } from "@jarvis-agent/core/types";
 import { BrowserWindow, WebContentsView, app } from "electron";
 import path from "node:path";
+import { randomUUID } from "node:crypto";
 import { ConfigManager } from "../utils/config-manager";
+import type { HumanRequestMessage, HumanResponseMessage, HumanInteractionContext } from "../../../src/models/human-interaction";
 
 export class EkoService {
   private eko: Eko | null = null;
@@ -12,6 +14,18 @@ export class EkoService {
   private mcpClient!: SimpleSseMcpClient;
   private agents!: any[];
 
+  // Store pending human interaction requests
+  private pendingHumanRequests = new Map<string, {
+    resolve: (value: any) => void;
+    reject: (reason?: any) => void;
+  }>();
+
+  // Map toolId to requestId for human interactions
+  private toolIdToRequestId = new Map<string, string>();
+
+  // Store current human_interact toolId
+  private currentHumanInteractToolId: string | null = null;
+
   constructor(mainWindow: BrowserWindow, detailView: WebContentsView) {
     this.mainWindow = mainWindow;
     this.detailView = detailView;
@@ -32,6 +46,12 @@ export class EkoService {
           return Promise.resolve();
         }
 
+        // Capture human_interact tool's toolId when tool is being used
+        if (message.type === 'tool_use' && message.toolName === 'human_interact' && message.toolId) {
+          this.currentHumanInteractToolId = message.toolId;
+          Log.info('Captured human_interact toolId:', message.toolId);
+        }
+
         return new Promise((resolve) => {
            // Send stream message to renderer process via IPC
         this.mainWindow.webContents.send('eko-stream-message', message);
@@ -72,10 +92,68 @@ export class EkoService {
         } else {
           resolve();
         }
-        })  
+        })
+      },
+
+      // Human interaction callbacks
+      onHumanConfirm: async (agentContext: AgentContext, prompt: string): Promise<boolean> => {
+        const result = await this.requestHumanInteraction(agentContext, {
+          interactType: 'confirm',
+          prompt
+        });
+        return Boolean(result);
+      },
+
+      onHumanInput: async (agentContext: AgentContext, prompt: string): Promise<string> => {
+        const result = await this.requestHumanInteraction(agentContext, {
+          interactType: 'input',
+          prompt
+        });
+        return String(result ?? '');
+      },
+
+      onHumanSelect: async (
+        agentContext: AgentContext,
+        prompt: string,
+        options: string[],
+        multiple: boolean
+      ): Promise<string[]> => {
+        const result = await this.requestHumanInteraction(agentContext, {
+          interactType: 'select',
+          prompt,
+          selectOptions: options,
+          selectMultiple: multiple
+        });
+        return Array.isArray(result) ? result : [];
       },
-      onHuman: (message: any) => {
-        console.log('EkoService human callback:', message);
+
+      onHumanHelp: async (
+        agentContext: AgentContext,
+        helpType: 'request_login' | 'request_assistance',
+        prompt: string
+      ): Promise<boolean> => {
+        // Get current page information for context
+        let context: HumanInteractionContext | undefined;
+        try {
+          const url = this.detailView.webContents.getURL();
+          if (url && url.startsWith('http')) {
+            const hostname = new URL(url).hostname;
+            context = {
+              siteName: hostname,
+              actionUrl: url
+            };
+          }
+        } catch (error) {
+          Log.error('Failed to get URL for human help context:', error);
+        }
+
+        const result = await this.requestHumanInteraction(agentContext, {
+          interactType: 'request_help',
+          prompt,
+          helpType,
+          context
+        });
+        return Boolean(result);
       }
     };
   }
@@ -142,7 +220,7 @@ export class EkoService {
     // Abort all running tasks before reloading
     if (this.eko) {
       const allTaskIds = this.eko.getAllTaskId();
-      allTaskIds.forEach(taskId => {
+      allTaskIds.forEach((taskId: any) => {
         try {
           this.eko!.abortTask(taskId, 'config-reload');
         } catch (error) {
@@ -151,6 +229,9 @@ export class EkoService {
       });
     }
 
+    // Reject all pending human interactions
+    this.rejectAllHumanRequests(new Error('EkoService configuration reloaded'));
+
     // Get new LLMs configuration
     const configManager = ConfigManager.getInstance();
     const llms: LLMs = configManager.getLLMsConfig();
@@ -318,17 +399,153 @@ export class EkoService {
     }
 
     const allTaskIds = this.eko.getAllTaskId();
-    const abortPromises = allTaskIds.map(taskId => this.eko!.abortTask(taskId, 'window-closing'));
+    const abortPromises = allTaskIds.map((taskId: any) => this.eko!.abortTask(taskId, 'window-closing'));
 
     await Promise.all(abortPromises);
+
+    // Reject all pending human interactions (also clears toolId mappings)
+    this.rejectAllHumanRequests(new Error('All tasks aborted'));
+
     Log.info('All tasks aborted');
   }
 
+  /**
+   * Request human interaction
+   * Sends interaction request to frontend and waits for user response
+   */
+  private requestHumanInteraction(
+    agentContext: AgentContext,
+    payload: Omit<HumanRequestMessage, 'type' | 'requestId' | 'timestamp'>
+  ): Promise<any> {
+    const requestId = randomUUID();
+    const message: HumanRequestMessage = {
+      type: 'human_interaction',
+      requestId,
+      taskId: agentContext?.context?.taskId,
+      agentName: agentContext?.agent?.name,
+      timestamp: new Date(),
+      ...payload
+    };
+
+    return new Promise((resolve, reject) => {
+      // Store promise resolver/rejector
+      this.pendingHumanRequests.set(requestId, { resolve, reject });
+
+      // Map toolId to requestId for frontend response matching
+      if (this.currentHumanInteractToolId) {
+        this.toolIdToRequestId.set(this.currentHumanInteractToolId, requestId);
+        Log.info('Mapped toolId to requestId:', {
+          toolId: this.currentHumanInteractToolId,
+          requestId
+        });
+        // Clear after mapping
+        this.currentHumanInteractToolId = null;
+      }
+
+      // Listen for task abort signal
+      const controllerSignal = agentContext?.context?.controller?.signal;
+      if (controllerSignal) {
+        controllerSignal.addEventListener('abort', () => {
+          this.pendingHumanRequests.delete(requestId);
+          reject(new Error('Task aborted during human interaction'));
+        });
+      }
+
+      // Send request to frontend as a special message
+      if (!this.mainWindow || this.mainWindow.isDestroyed()) {
+        this.pendingHumanRequests.delete(requestId);
+        reject(new Error('Main window destroyed, cannot request human interaction'));
+        return;
+      }
+
+      Log.info('Requesting human interaction:', { requestId, interactType: payload.interactType, prompt: payload.prompt });
+      this.mainWindow.webContents.send('eko-stream-message', message);
+    });
+  }
+
+  /**
+   * Handle human response from frontend
+   * Called via IPC when user completes interaction
+   */
+  public handleHumanResponse(response: HumanResponseMessage): boolean {
+    Log.info('Received human response:', response);
+
+    // First try direct requestId match
+    let pending = this.pendingHumanRequests.get(response.requestId);
+    let actualRequestId = response.requestId;
+
+    // If not found, try to find via toolId mapping (frontend might send toolId as requestId)
+    if (!pending) {
+      const mappedRequestId = this.toolIdToRequestId.get(response.requestId);
+      if (mappedRequestId) {
+        pending = this.pendingHumanRequests.get(mappedRequestId);
+        actualRequestId = mappedRequestId;
+        Log.info('Found requestId via toolId mapping:', {
+          toolId: response.requestId,
+          actualRequestId: mappedRequestId
+        });
+      }
+    }
+
+    if (!pending) {
+      Log.error(`Human interaction request ${response.requestId} not found or already processed`);
+      return false;
+    }
+
+    // Clean up both maps
+    this.pendingHumanRequests.delete(actualRequestId);
+    this.toolIdToRequestId.delete(response.requestId);
+
+    if (response.success) {
+      // Resolve promise, AI continues execution
+      pending.resolve(response.result);
+
+      // Send result message to frontend to update card state
+      if (!this.mainWindow || this.mainWindow.isDestroyed()) {
+        Log.warn('Main window destroyed, cannot send interaction result');
+        return true;
+      }
+
+      this.mainWindow.webContents.send('eko-stream-message', {
+        type: 'human_interaction_result',
+        requestId: response.requestId,
+        result: response.result,
+        timestamp: new Date()
+      });
+    } else {
+      // Reject promise, AI handles error
+      pending.reject(new Error(response.error || 'Human interaction cancelled'));
+    }
+
+    return true;
+  }
+
+  /**
+   * Reject all pending human interaction requests
+   * Used when config reloads or service shuts down
+   */
+  private rejectAllHumanRequests(error: Error): void {
+    if (this.pendingHumanRequests.size === 0) {
+      return;
+    }
+
+    Log.info(`Rejecting ${this.pendingHumanRequests.size} pending human interaction requests`);
+
+    for (const pending of this.pendingHumanRequests.values()) {
+      pending.reject(error);
+    }
+
+    this.pendingHumanRequests.clear();
+    this.toolIdToRequestId.clear();
+    this.currentHumanInteractToolId = null;
+  }
+
   /**
    * Destroy service
    */
   destroy() {
-    console.log('EkoService destroyed');
+    Log.info('EkoService destroyed');
+    this.rejectAllHumanRequests(new Error('EkoService destroyed'));
     this.eko = null;
   }
 }

electron/main/vite.config.ts

@@ -30,6 +30,7 @@ export default defineConfig({
         'node:child_process',
         'node:process',
         'node:buffer',
+        'node:crypto',
         'fs',
         'fs/promises',
         'path',

electron/preload/index.ts

@@ -40,6 +40,9 @@ const api = {
   ekoCancelTask: (taskId: string) => ipcRenderer.invoke('eko:cancel-task', taskId),
   onEkoStreamMessage: (callback: (message: any) => void) => ipcRenderer.on('eko-stream-message', (_, message) => callback(message)),
 
+  // Human interaction APIs
+  sendHumanResponse: (response: any) => ipcRenderer.invoke('eko:human-response', response),
+
   // Model configuration APIs
   getUserModelConfigs: () => ipcRenderer.invoke('config:get-user-configs'),
   saveUserModelConfigs: (configs: any) => ipcRenderer.invoke('config:save-user-configs', configs),

package.json

@@ -29,8 +29,8 @@
   "dependencies": {
     "@ant-design/cssinjs": "^1.23.0",
     "@ant-design/icons": "5.x",
-    "@jarvis-agent/core": "^0.1.3",
-    "@jarvis-agent/electron": "^0.1.7",
+    "@jarvis-agent/core": "^0.1.4",
+    "@jarvis-agent/electron": "^0.1.8",
     "@jest/globals": "^30.1.2",
     "@react-spring/web": "^10.0.1",
     "antd": "^5.26.5",