New BrowserOS server based Agent - Fixes (#158)

Author: shivammittal274

src/lib/agent/TeachWebSocketAgent.ts

@@ -1,9 +1,16 @@
-import { ExecutionContext } from "@/lib/runtime/ExecutionContext";
+import { ExecutionContext, WS_AGENT_CONFIG, WS_CONNECTION_TIMEOUT } from "@/lib/runtime/ExecutionContext";
 import { PubSub } from "@/lib/pubsub";
 import { AbortError } from "@/lib/utils/Abortable";
 import { ExecutionMetadata } from "@/lib/types/messaging";
 import { Logging } from "@/lib/utils/Logging";
-import { WS_AGENT_CONFIG } from "@/lib/agent/websocket/config";
+
+
+interface PredefinedPlan {
+  agentId: string;
+  name?: string;  // Optional to match ExecutionMetadata schema
+  goal: string;
+  steps: string[];
+}
 
 /**
  * WebSocket-based agent that connects to remote server
@@ -12,7 +19,6 @@ import { WS_AGENT_CONFIG } from "@/lib/agent/websocket/config";
  */
 export class WebSocketAgent {
   private readonly executionContext: ExecutionContext;
-  private readonly wsUrl: string;
 
   // WebSocket state
   private ws: WebSocket | null = null;
@@ -21,12 +27,8 @@ export class WebSocketAgent {
   private isCompleted = false;
   private lastEventTime = 0;  // Track last event for timeout
 
-  constructor(
-    executionContext: ExecutionContext,
-    wsUrl: string = WS_AGENT_CONFIG.url
-  ) {
+  constructor(executionContext: ExecutionContext) {
     this.executionContext = executionContext;
-    this.wsUrl = wsUrl;
     Logging.log("WebSocketAgent", "Agent instance created", "info");
   }
 
@@ -40,12 +42,76 @@ export class WebSocketAgent {
     }
   }
 
+  /**
+   * Check if task is a special predefined task and return its metadata
+   * @param task - The original task string
+   * @returns Metadata with predefined plan or null if not a special task
+   */
+  private _getSpecialTaskMetadata(task: string): {task: string, metadata: ExecutionMetadata} | null {
+    const taskLower = task.toLowerCase();
+
+    // BrowserOS Launch Upvote Task
+    if (taskLower === "read about our vision and upvote ❤️") {
+      return {
+        task: "Read about our vision and upvote",
+        metadata: {
+          executionMode: 'predefined' as const,
+          predefinedPlan: {
+            agentId: 'browseros-launch-upvoter',
+            name: "BrowserOS Launch Upvoter",
+            goal: "Navigate to BrowserOS launch page and upvote it",
+            steps: [
+              "Navigate to https://dub.sh/browseros-launch",
+              "Find and click the upvote button on the page using visual_click",
+              "Use celebration tool to show confetti animation"
+            ]
+          }
+        }
+      };
+    }
+
+    // GitHub Star Task
+    if (taskLower === "support browseros on github ⭐") {
+      return {
+        task: "Support BrowserOS on GitHub",
+        metadata: {
+          executionMode: 'predefined' as const,
+          predefinedPlan: {
+            agentId: 'github-star-browseros',
+            name: "GitHub Repository Star",
+            goal: "Navigate to BrowserOS GitHub repo and star it",
+            steps: [
+              "Navigate to https://git.new/browserOS",
+              "Check if the star button indicates already starred (filled star icon)",
+              "If not starred (outline star icon), click the star button to star the repository",
+              "Use celebration_tool to show confetti animation"
+            ]
+          }
+        }
+      };
+    }
+
+    return null;
+  }
+
   /**
    * Main execution entry point
    */
   async execute(task: string, metadata?: ExecutionMetadata): Promise<void> {
+    // Check for special tasks and get their predefined plans
+    const specialTaskMetadata = this._getSpecialTaskMetadata(task);
+
+    let _task = task;
+    let _metadata = metadata;
+
+    if (specialTaskMetadata) {
+      _task = specialTaskMetadata.task;
+      _metadata = { ...metadata, ...specialTaskMetadata.metadata };
+      Logging.log("WebSocketAgent", `Special task detected: ${specialTaskMetadata.metadata.predefinedPlan?.name}`, "info");
+    }
+
     try {
-      this.executionContext.setCurrentTask(task);
+      this.executionContext.setCurrentTask(_task);
       this.executionContext.setExecutionMetrics({
         ...this.executionContext.getExecutionMetrics(),
         startTime: Date.now(),
@@ -56,8 +122,11 @@ export class WebSocketAgent {
       // Connect to WebSocket server
       await this._connect();
 
-      // Send query with browser context
-      await this._sendQuery(task);
+      // Send query with browser context and predefined plan if available
+      await this._sendQuery(
+        _task,
+        _metadata?.predefinedPlan
+      );
 
       // Wait for completion with abort and timeout checks
       await this._waitForCompletion();
@@ -79,15 +148,18 @@ export class WebSocketAgent {
    * Connect to WebSocket server and wait for connection event
    */
   private async _connect(): Promise<void> {
-    return new Promise((resolve, reject) => {
-      this.checkIfAborted();
+    this.checkIfAborted();
 
+    // Get WebSocket URL from ExecutionContext
+    const wsUrl = await this.executionContext.getAgentServerUrl();
+
+    return new Promise((resolve, reject) => {
       this._publishMessage('🔗 Connecting to reasoning server...', 'thinking');
-      Logging.log("WebSocketAgent", `Connecting to ${this.wsUrl}`, "info");
+      Logging.log("WebSocketAgent", `Connecting to ${wsUrl}`, "info");
 
       // Create WebSocket
       try {
-        this.ws = new WebSocket(this.wsUrl);
+        this.ws = new WebSocket(wsUrl);
       } catch (error) {
         Logging.log("WebSocketAgent", `Failed to create WebSocket: ${error}`, "error");
         reject(error);
@@ -96,9 +168,9 @@ export class WebSocketAgent {
 
       // Connection timeout - don't publish, let _handleExecutionError do it
       const timeout = setTimeout(() => {
-        reject(new Error(`Connection timeout after ${WS_AGENT_CONFIG.connectionTimeout}ms`));
+        reject(new Error(`Connection timeout after ${WS_CONNECTION_TIMEOUT}ms`));
         this.ws?.close();
-      }, WS_AGENT_CONFIG.connectionTimeout);
+      }, WS_CONNECTION_TIMEOUT);
 
       // WebSocket opened
       this.ws.onopen = () => {
@@ -140,14 +212,14 @@ export class WebSocketAgent {
       };
 
       // WebSocket error - don't publish, let _handleExecutionError do it
-      this.ws.onerror = (error) => {
+      this.ws.onerror = (_error) => {
         clearTimeout(timeout);
         Logging.log("WebSocketAgent", "WebSocket error", "error");
         reject(new Error('WebSocket connection failed'));
       };
 
       // WebSocket closed
-      this.ws.onclose = (event) => {
+      this.ws.onclose = (_event) => {
         Logging.log("WebSocketAgent", "WebSocket connection closed", "info");
 
         // Only publish if we were actually connected (not a connection failure)
@@ -171,7 +243,10 @@ export class WebSocketAgent {
   /**
    * Send query to server with browser context
    */
-  private async _sendQuery(task: string): Promise<void> {
+  private async _sendQuery(
+    task: string,
+    predefinedPlan?: PredefinedPlan
+  ): Promise<void> {
     this.checkIfAborted();
 
     if (!this.ws || !this.isConnected) {
@@ -181,18 +256,38 @@ export class WebSocketAgent {
     // Add user message to history (UI already showed it optimistically)
     this.executionContext.messageManager.addHuman(task);
 
-    // Gather browser context and append to task
+    // Build message content starting with task
+    let messageContent = task;
+
+    // If predefined plan exists, format steps into message
+    if (predefinedPlan) {
+      const formattedSteps = predefinedPlan.steps
+        .map((step, i) => `${i + 1}. ${step}`)
+        .join('\n');
+
+      messageContent += `
+
+PREDEFINED PLAN: ${predefinedPlan.name}
+Goal: ${predefinedPlan.goal}
+
+Steps to execute:
+${formattedSteps}`;
+
+      Logging.log("WebSocketAgent", `Sending predefined plan: ${predefinedPlan.name}`, "info");
+    }
+
+    // Gather browser context and append
     const browserContext = await this._getBrowserContext();
     const tabInfoStr = browserContext && browserContext.url
-      ? `\nContext: Current user's open tab: Title: ${browserContext.title} URL: ${browserContext.url}`
+      ? `\n\nContext: Current user's open tab: Title: ${browserContext.title} URL: ${browserContext.url}`
       : '';
 
-    const taskWithContext = task + tabInfoStr;
+    messageContent += tabInfoStr;
 
     // Send message to server
     const message = {
       type: 'message',
-      content: taskWithContext
+      content: messageContent
     };
 
     try {
@@ -212,7 +307,7 @@ export class WebSocketAgent {
   private async _getBrowserContext(): Promise<any> {
     try {
       const currentPage = await this.executionContext.browserContext.getCurrentPage();
-      const url = await currentPage.url();
+      const url = currentPage.url();
       const title = await currentPage.title();
       const selectedTabIds = this.executionContext.getSelectedTabIds();
 

src/lib/agent/WebSocketAgent.ts

@@ -6,6 +6,7 @@ import { BrowserAgent } from "@/lib/agent/BrowserAgent";
 import { LocalAgent } from "@/lib/agent/LocalAgent";
 import { TeachAgent } from "@/lib/agent/TeachAgent";
 import { WebSocketAgent } from "@/lib/agent/WebSocketAgent";
+import { TeachWebSocketAgent } from "@/lib/agent/TeachWebSocketAgent";
 import { ChatAgent } from "@/lib/agent/ChatAgent";
 import { langChainProvider } from "@/lib/llm/LangChainProvider";
 import { Logging } from "@/lib/utils/Logging";
@@ -209,14 +210,36 @@ export class Execution {
           throw new Error("Teach mode requires a workflow to execute");
         }
 
-        agentType = 'TeachAgent';
-        Logging.logMetric('execution.agent_start', {
-          mode: this.options.mode,
-          agent_type: agentType
-        });
+        // Check if BrowserOS provider is selected
+        const providerType = await langChainProvider.getCurrentProviderType() || '';
 
-        const teachAgent = new TeachAgent(executionContext);
-        await teachAgent.execute(this.options.workflow);
+        if (providerType === 'browseros') {
+          // Use TeachWebSocketAgent for teach mode with BrowserOS provider
+          agentType = 'TeachWebSocketAgent';
+          Logging.logMetric('execution.agent_start', {
+            mode: this.options.mode,
+            agent_type: agentType,
+            provider_type: providerType,
+          });
+
+          const teachWsAgent = new TeachWebSocketAgent(executionContext);
+          // Pass workflow through metadata for teach mode
+          const teachMetadata = {
+            workflow: this.options.workflow,
+            ...(metadata || this.options.metadata)
+          };
+          await teachWsAgent.execute(query, teachMetadata);
+        } else {
+          // Use local TeachAgent for other providers
+          agentType = 'TeachAgent';
+          Logging.logMetric('execution.agent_start', {
+            mode: this.options.mode,
+            agent_type: agentType
+          });
+
+          const teachAgent = new TeachAgent(executionContext);
+          await teachAgent.execute(this.options.workflow);
+        }
       } else if (this.options.mode === "chat") {
 
         agentType = 'ChatAgent';
@@ -241,6 +264,7 @@ export class Execution {
           });
 
           const wsAgent = new WebSocketAgent(executionContext);
+          // Workflow only comes from explicit metadata, not options (options.workflow is for teach mode)
           await wsAgent.execute(query, metadata || this.options.metadata);
         } else {
           // Use LocalAgent for small models, BrowserAgent for others