From 964b2afc0ffec6e1e4946d11e7830b1db4d3cc52 Mon Sep 17 00:00:00 2001 From: liangxinbing <1580466765@qq.com> Date: Mon, 31 Mar 2025 00:30:01 +0800 Subject: [PATCH] update Manus, BrowserAgent and ToolCallAgent --- app/agent/browser.py | 171 ++++++++++++++++++++---------------------- app/agent/manus.py | 46 ++++++------ app/agent/toolcall.py | 7 ++ 3 files changed, 113 insertions(+), 111 deletions(-) diff --git a/app/agent/browser.py b/app/agent/browser.py index ae1df97..92d8ea6 100644 --- a/app/agent/browser.py +++ b/app/agent/browser.py @@ -1,7 +1,7 @@ import json -from typing import Any, Optional +from typing import TYPE_CHECKING, Optional -from pydantic import Field +from pydantic import Field, model_validator from app.agent.toolcall import ToolCallAgent from app.logger import logger @@ -10,6 +10,75 @@ from app.schema import Message, ToolChoice from app.tool import BrowserUseTool, Terminate, ToolCollection +# Avoid circular import if BrowserAgent needs BrowserContextHelper +if TYPE_CHECKING: + from app.agent.base import BaseAgent # Or wherever memory is defined + + +class BrowserContextHelper: + def __init__(self, agent: "BaseAgent"): + self.agent = agent + self._current_base64_image: Optional[str] = None + + async def get_browser_state(self) -> Optional[dict]: + browser_tool = self.agent.available_tools.get_tool(BrowserUseTool().name) + if not browser_tool or not hasattr(browser_tool, "get_current_state"): + logger.warning("BrowserUseTool not found or doesn't have get_current_state") + return None + try: + result = await browser_tool.get_current_state() + if result.error: + logger.debug(f"Browser state error: {result.error}") + return None + if hasattr(result, "base64_image") and result.base64_image: + self._current_base64_image = result.base64_image + else: + self._current_base64_image = None + return json.loads(result.output) + except Exception as e: + logger.debug(f"Failed to get browser state: {str(e)}") + return None + + async def format_next_step_prompt(self) -> str: + """Gets browser state and formats the browser prompt.""" + browser_state = await self.get_browser_state() + url_info, tabs_info, content_above_info, content_below_info = "", "", "", "" + results_info = "" # Or get from agent if needed elsewhere + + if browser_state and not browser_state.get("error"): + url_info = f"\n URL: {browser_state.get('url', 'N/A')}\n Title: {browser_state.get('title', 'N/A')}" + tabs = browser_state.get("tabs", []) + if tabs: + tabs_info = f"\n {len(tabs)} tab(s) available" + pixels_above = browser_state.get("pixels_above", 0) + pixels_below = browser_state.get("pixels_below", 0) + if pixels_above > 0: + content_above_info = f" ({pixels_above} pixels)" + if pixels_below > 0: + content_below_info = f" ({pixels_below} pixels)" + + if self._current_base64_image: + image_message = Message.user_message( + content="Current browser screenshot:", + base64_image=self._current_base64_image, + ) + self.agent.memory.add_message(image_message) + self._current_base64_image = None # Consume the image after adding + + return NEXT_STEP_PROMPT.format( + url_placeholder=url_info, + tabs_placeholder=tabs_info, + content_above_placeholder=content_above_info, + content_below_placeholder=content_below_info, + results_placeholder=results_info, + ) + + async def cleanup_browser(self): + browser_tool = self.agent.available_tools.get_tool(BrowserUseTool().name) + if browser_tool and hasattr(browser_tool, "cleanup"): + await browser_tool.cleanup() + + class BrowserAgent(ToolCallAgent): """ A browser agent that uses the browser_use library to control a browser. @@ -36,98 +105,20 @@ class BrowserAgent(ToolCallAgent): tool_choices: ToolChoice = ToolChoice.AUTO special_tool_names: list[str] = Field(default_factory=lambda: [Terminate().name]) - _current_base64_image: Optional[str] = None + browser_context_helper: Optional[BrowserContextHelper] = None - async def _handle_special_tool(self, name: str, result: Any, **kwargs): - if not self._is_special_tool(name): - return - else: - await self.available_tools.get_tool(BrowserUseTool().name).cleanup() - await super()._handle_special_tool(name, result, **kwargs) - - async def get_browser_state(self) -> Optional[dict]: - """Get the current browser state for context in next steps.""" - browser_tool = self.available_tools.get_tool(BrowserUseTool().name) - if not browser_tool: - return None - - try: - # Get browser state directly from the tool - result = await browser_tool.get_current_state() - - if result.error: - logger.debug(f"Browser state error: {result.error}") - return None - - # Store screenshot if available - if hasattr(result, "base64_image") and result.base64_image: - self._current_base64_image = result.base64_image - - # Parse the state info - return json.loads(result.output) - - except Exception as e: - logger.debug(f"Failed to get browser state: {str(e)}") - return None + @model_validator(mode="after") + def initialize_helper(self) -> "BrowserAgent": + self.browser_context_helper = BrowserContextHelper(self) + return self async def think(self) -> bool: """Process current state and decide next actions using tools, with browser state info added""" - # Add browser state to the context - browser_state = await self.get_browser_state() - - # Initialize placeholder values - url_info = "" - tabs_info = "" - content_above_info = "" - content_below_info = "" - results_info = "" - - if browser_state and not browser_state.get("error"): - # URL and title info - url_info = f"\n URL: {browser_state.get('url', 'N/A')}\n Title: {browser_state.get('title', 'N/A')}" - - # Tab information - if "tabs" in browser_state: - tabs = browser_state.get("tabs", []) - if tabs: - tabs_info = f"\n {len(tabs)} tab(s) available" - - # Content above/below viewport - pixels_above = browser_state.get("pixels_above", 0) - pixels_below = browser_state.get("pixels_below", 0) - - if pixels_above > 0: - content_above_info = f" ({pixels_above} pixels)" - - if pixels_below > 0: - content_below_info = f" ({pixels_below} pixels)" - - # Add screenshot as base64 if available - if self._current_base64_image: - # Create a message with image attachment - image_message = Message.user_message( - content="Current browser screenshot:", - base64_image=self._current_base64_image, - ) - self.memory.add_message(image_message) - - # Replace placeholders with actual browser state info - self.next_step_prompt = NEXT_STEP_PROMPT.format( - url_placeholder=url_info, - tabs_placeholder=tabs_info, - content_above_placeholder=content_above_info, - content_below_placeholder=content_below_info, - results_placeholder=results_info, - ) - - # Call parent implementation - result = await super().think() - - # Reset the next_step_prompt to its original state - self.next_step_prompt = NEXT_STEP_PROMPT - - return result + self.next_step_prompt = ( + await self.browser_context_helper.format_next_step_prompt() + ) + return await super().think() async def cleanup(self): """Clean up browser agent resources by calling parent cleanup.""" - await super().cleanup() + await self.browser_context_helper.cleanup_browser() diff --git a/app/agent/manus.py b/app/agent/manus.py index 980ab7f..253e040 100644 --- a/app/agent/manus.py +++ b/app/agent/manus.py @@ -1,8 +1,10 @@ -from pydantic import Field +from typing import Optional -from app.agent.browser import BrowserAgent +from pydantic import Field, model_validator + +from app.agent.browser import BrowserContextHelper +from app.agent.toolcall import ToolCallAgent from app.config import config -from app.prompt.browser import NEXT_STEP_PROMPT as BROWSER_NEXT_STEP_PROMPT from app.prompt.manus import NEXT_STEP_PROMPT, SYSTEM_PROMPT from app.tool import Terminate, ToolCollection from app.tool.browser_use_tool import BrowserUseTool @@ -10,14 +12,8 @@ from app.tool.python_execute import PythonExecute from app.tool.str_replace_editor import StrReplaceEditor -class Manus(BrowserAgent): - """ - A versatile general-purpose agent that uses planning to solve various tasks. - - This agent extends BrowserAgent with a comprehensive set of tools and capabilities, - including Python execution, web browsing, file operations, and information retrieval - to handle a wide range of user requests. - """ +class Manus(ToolCallAgent): + """A versatile general-purpose agent.""" name: str = "Manus" description: str = ( @@ -37,24 +33,31 @@ class Manus(BrowserAgent): ) ) + special_tool_names: list[str] = Field(default_factory=lambda: [Terminate().name]) + + browser_context_helper: Optional[BrowserContextHelper] = None + + @model_validator(mode="after") + def initialize_helper(self) -> "Manus": + self.browser_context_helper = BrowserContextHelper(self) + return self + async def think(self) -> bool: """Process current state and decide next actions with appropriate context.""" - # Store original prompt original_prompt = self.next_step_prompt - - # Only check recent messages (last 3) for browser activity recent_messages = self.memory.messages[-3:] if self.memory.messages else [] browser_in_use = any( - "browser_use" in msg.content.lower() + tc.function.name == BrowserUseTool().name for msg in recent_messages - if hasattr(msg, "content") and isinstance(msg.content, str) + if msg.tool_calls + for tc in msg.tool_calls ) if browser_in_use: - # Override with browser-specific prompt temporarily to get browser context - self.next_step_prompt = BROWSER_NEXT_STEP_PROMPT + self.next_step_prompt = ( + await self.browser_context_helper.format_next_step_prompt() + ) - # Call parent's think method result = await super().think() # Restore original prompt @@ -63,5 +66,6 @@ class Manus(BrowserAgent): return result async def cleanup(self): - """Clean up Manus agent resources by calling parent cleanup.""" - await super().cleanup() + """Clean up Manus agent resources.""" + if self.browser_context_helper: + await self.browser_context_helper.cleanup_browser() diff --git a/app/agent/toolcall.py b/app/agent/toolcall.py index ad1ed2f..7db14f1 100644 --- a/app/agent/toolcall.py +++ b/app/agent/toolcall.py @@ -249,3 +249,10 @@ class ToolCallAgent(ReActAgent): f"🚨 Error cleaning up tool '{tool_name}': {e}", exc_info=True ) logger.info(f"✨ Cleanup complete for agent '{self.name}'.") + + async def run(self, request: Optional[str] = None) -> str: + """Run the agent with cleanup when done.""" + try: + return await super().run(request) + finally: + await self.cleanup()