update Manus, BrowserAgent and ToolCallAgent
This commit is contained in:
liangxinbing
+81
-90
@@ -1,7 +1,7 @@
|
||||
import json
|
||||
from typing import Any, Optional
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from pydantic import Field
|
||||
from pydantic import Field, model_validator
|
||||
|
||||
from app.agent.toolcall import ToolCallAgent
|
||||
from app.logger import logger
|
||||
@@ -10,6 +10,75 @@ from app.schema import Message, ToolChoice
|
||||
from app.tool import BrowserUseTool, Terminate, ToolCollection
|
||||
|
||||
|
||||
# Avoid circular import if BrowserAgent needs BrowserContextHelper
|
||||
if TYPE_CHECKING:
|
||||
from app.agent.base import BaseAgent # Or wherever memory is defined
|
||||
|
||||
|
||||
class BrowserContextHelper:
|
||||
def __init__(self, agent: "BaseAgent"):
|
||||
self.agent = agent
|
||||
self._current_base64_image: Optional[str] = None
|
||||
|
||||
async def get_browser_state(self) -> Optional[dict]:
|
||||
browser_tool = self.agent.available_tools.get_tool(BrowserUseTool().name)
|
||||
if not browser_tool or not hasattr(browser_tool, "get_current_state"):
|
||||
logger.warning("BrowserUseTool not found or doesn't have get_current_state")
|
||||
return None
|
||||
try:
|
||||
result = await browser_tool.get_current_state()
|
||||
if result.error:
|
||||
logger.debug(f"Browser state error: {result.error}")
|
||||
return None
|
||||
if hasattr(result, "base64_image") and result.base64_image:
|
||||
self._current_base64_image = result.base64_image
|
||||
else:
|
||||
self._current_base64_image = None
|
||||
return json.loads(result.output)
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to get browser state: {str(e)}")
|
||||
return None
|
||||
|
||||
async def format_next_step_prompt(self) -> str:
|
||||
"""Gets browser state and formats the browser prompt."""
|
||||
browser_state = await self.get_browser_state()
|
||||
url_info, tabs_info, content_above_info, content_below_info = "", "", "", ""
|
||||
results_info = "" # Or get from agent if needed elsewhere
|
||||
|
||||
if browser_state and not browser_state.get("error"):
|
||||
url_info = f"\n URL: {browser_state.get('url', 'N/A')}\n Title: {browser_state.get('title', 'N/A')}"
|
||||
tabs = browser_state.get("tabs", [])
|
||||
if tabs:
|
||||
tabs_info = f"\n {len(tabs)} tab(s) available"
|
||||
pixels_above = browser_state.get("pixels_above", 0)
|
||||
pixels_below = browser_state.get("pixels_below", 0)
|
||||
if pixels_above > 0:
|
||||
content_above_info = f" ({pixels_above} pixels)"
|
||||
if pixels_below > 0:
|
||||
content_below_info = f" ({pixels_below} pixels)"
|
||||
|
||||
if self._current_base64_image:
|
||||
image_message = Message.user_message(
|
||||
content="Current browser screenshot:",
|
||||
base64_image=self._current_base64_image,
|
||||
)
|
||||
self.agent.memory.add_message(image_message)
|
||||
self._current_base64_image = None # Consume the image after adding
|
||||
|
||||
return NEXT_STEP_PROMPT.format(
|
||||
url_placeholder=url_info,
|
||||
tabs_placeholder=tabs_info,
|
||||
content_above_placeholder=content_above_info,
|
||||
content_below_placeholder=content_below_info,
|
||||
results_placeholder=results_info,
|
||||
)
|
||||
|
||||
async def cleanup_browser(self):
|
||||
browser_tool = self.agent.available_tools.get_tool(BrowserUseTool().name)
|
||||
if browser_tool and hasattr(browser_tool, "cleanup"):
|
||||
await browser_tool.cleanup()
|
||||
|
||||
|
||||
class BrowserAgent(ToolCallAgent):
|
||||
"""
|
||||
A browser agent that uses the browser_use library to control a browser.
|
||||
@@ -36,98 +105,20 @@ class BrowserAgent(ToolCallAgent):
|
||||
tool_choices: ToolChoice = ToolChoice.AUTO
|
||||
special_tool_names: list[str] = Field(default_factory=lambda: [Terminate().name])
|
||||
|
||||
_current_base64_image: Optional[str] = None
|
||||
browser_context_helper: Optional[BrowserContextHelper] = None
|
||||
|
||||
async def _handle_special_tool(self, name: str, result: Any, **kwargs):
|
||||
if not self._is_special_tool(name):
|
||||
return
|
||||
else:
|
||||
await self.available_tools.get_tool(BrowserUseTool().name).cleanup()
|
||||
await super()._handle_special_tool(name, result, **kwargs)
|
||||
|
||||
async def get_browser_state(self) -> Optional[dict]:
|
||||
"""Get the current browser state for context in next steps."""
|
||||
browser_tool = self.available_tools.get_tool(BrowserUseTool().name)
|
||||
if not browser_tool:
|
||||
return None
|
||||
|
||||
try:
|
||||
# Get browser state directly from the tool
|
||||
result = await browser_tool.get_current_state()
|
||||
|
||||
if result.error:
|
||||
logger.debug(f"Browser state error: {result.error}")
|
||||
return None
|
||||
|
||||
# Store screenshot if available
|
||||
if hasattr(result, "base64_image") and result.base64_image:
|
||||
self._current_base64_image = result.base64_image
|
||||
|
||||
# Parse the state info
|
||||
return json.loads(result.output)
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to get browser state: {str(e)}")
|
||||
return None
|
||||
@model_validator(mode="after")
|
||||
def initialize_helper(self) -> "BrowserAgent":
|
||||
self.browser_context_helper = BrowserContextHelper(self)
|
||||
return self
|
||||
|
||||
async def think(self) -> bool:
|
||||
"""Process current state and decide next actions using tools, with browser state info added"""
|
||||
# Add browser state to the context
|
||||
browser_state = await self.get_browser_state()
|
||||
|
||||
# Initialize placeholder values
|
||||
url_info = ""
|
||||
tabs_info = ""
|
||||
content_above_info = ""
|
||||
content_below_info = ""
|
||||
results_info = ""
|
||||
|
||||
if browser_state and not browser_state.get("error"):
|
||||
# URL and title info
|
||||
url_info = f"\n URL: {browser_state.get('url', 'N/A')}\n Title: {browser_state.get('title', 'N/A')}"
|
||||
|
||||
# Tab information
|
||||
if "tabs" in browser_state:
|
||||
tabs = browser_state.get("tabs", [])
|
||||
if tabs:
|
||||
tabs_info = f"\n {len(tabs)} tab(s) available"
|
||||
|
||||
# Content above/below viewport
|
||||
pixels_above = browser_state.get("pixels_above", 0)
|
||||
pixels_below = browser_state.get("pixels_below", 0)
|
||||
|
||||
if pixels_above > 0:
|
||||
content_above_info = f" ({pixels_above} pixels)"
|
||||
|
||||
if pixels_below > 0:
|
||||
content_below_info = f" ({pixels_below} pixels)"
|
||||
|
||||
# Add screenshot as base64 if available
|
||||
if self._current_base64_image:
|
||||
# Create a message with image attachment
|
||||
image_message = Message.user_message(
|
||||
content="Current browser screenshot:",
|
||||
base64_image=self._current_base64_image,
|
||||
)
|
||||
self.memory.add_message(image_message)
|
||||
|
||||
# Replace placeholders with actual browser state info
|
||||
self.next_step_prompt = NEXT_STEP_PROMPT.format(
|
||||
url_placeholder=url_info,
|
||||
tabs_placeholder=tabs_info,
|
||||
content_above_placeholder=content_above_info,
|
||||
content_below_placeholder=content_below_info,
|
||||
results_placeholder=results_info,
|
||||
)
|
||||
|
||||
# Call parent implementation
|
||||
result = await super().think()
|
||||
|
||||
# Reset the next_step_prompt to its original state
|
||||
self.next_step_prompt = NEXT_STEP_PROMPT
|
||||
|
||||
return result
|
||||
self.next_step_prompt = (
|
||||
await self.browser_context_helper.format_next_step_prompt()
|
||||
)
|
||||
return await super().think()
|
||||
|
||||
async def cleanup(self):
|
||||
"""Clean up browser agent resources by calling parent cleanup."""
|
||||
await super().cleanup()
|
||||
await self.browser_context_helper.cleanup_browser()
|
||||
|
||||
+25
-21
@@ -1,8 +1,10 @@
|
||||
from pydantic import Field
|
||||
from typing import Optional
|
||||
|
||||
from app.agent.browser import BrowserAgent
|
||||
from pydantic import Field, model_validator
|
||||
|
||||
from app.agent.browser import BrowserContextHelper
|
||||
from app.agent.toolcall import ToolCallAgent
|
||||
from app.config import config
|
||||
from app.prompt.browser import NEXT_STEP_PROMPT as BROWSER_NEXT_STEP_PROMPT
|
||||
from app.prompt.manus import NEXT_STEP_PROMPT, SYSTEM_PROMPT
|
||||
from app.tool import Terminate, ToolCollection
|
||||
from app.tool.browser_use_tool import BrowserUseTool
|
||||
@@ -10,14 +12,8 @@ from app.tool.python_execute import PythonExecute
|
||||
from app.tool.str_replace_editor import StrReplaceEditor
|
||||
|
||||
|
||||
class Manus(BrowserAgent):
|
||||
"""
|
||||
A versatile general-purpose agent that uses planning to solve various tasks.
|
||||
|
||||
This agent extends BrowserAgent with a comprehensive set of tools and capabilities,
|
||||
including Python execution, web browsing, file operations, and information retrieval
|
||||
to handle a wide range of user requests.
|
||||
"""
|
||||
class Manus(ToolCallAgent):
|
||||
"""A versatile general-purpose agent."""
|
||||
|
||||
name: str = "Manus"
|
||||
description: str = (
|
||||
@@ -37,24 +33,31 @@ class Manus(BrowserAgent):
|
||||
)
|
||||
)
|
||||
|
||||
special_tool_names: list[str] = Field(default_factory=lambda: [Terminate().name])
|
||||
|
||||
browser_context_helper: Optional[BrowserContextHelper] = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def initialize_helper(self) -> "Manus":
|
||||
self.browser_context_helper = BrowserContextHelper(self)
|
||||
return self
|
||||
|
||||
async def think(self) -> bool:
|
||||
"""Process current state and decide next actions with appropriate context."""
|
||||
# Store original prompt
|
||||
original_prompt = self.next_step_prompt
|
||||
|
||||
# Only check recent messages (last 3) for browser activity
|
||||
recent_messages = self.memory.messages[-3:] if self.memory.messages else []
|
||||
browser_in_use = any(
|
||||
"browser_use" in msg.content.lower()
|
||||
tc.function.name == BrowserUseTool().name
|
||||
for msg in recent_messages
|
||||
if hasattr(msg, "content") and isinstance(msg.content, str)
|
||||
if msg.tool_calls
|
||||
for tc in msg.tool_calls
|
||||
)
|
||||
|
||||
if browser_in_use:
|
||||
# Override with browser-specific prompt temporarily to get browser context
|
||||
self.next_step_prompt = BROWSER_NEXT_STEP_PROMPT
|
||||
self.next_step_prompt = (
|
||||
await self.browser_context_helper.format_next_step_prompt()
|
||||
)
|
||||
|
||||
# Call parent's think method
|
||||
result = await super().think()
|
||||
|
||||
# Restore original prompt
|
||||
@@ -63,5 +66,6 @@ class Manus(BrowserAgent):
|
||||
return result
|
||||
|
||||
async def cleanup(self):
|
||||
"""Clean up Manus agent resources by calling parent cleanup."""
|
||||
await super().cleanup()
|
||||
"""Clean up Manus agent resources."""
|
||||
if self.browser_context_helper:
|
||||
await self.browser_context_helper.cleanup_browser()
|
||||
|
||||
@@ -249,3 +249,10 @@ class ToolCallAgent(ReActAgent):
|
||||
f"🚨 Error cleaning up tool '{tool_name}': {e}", exc_info=True
|
||||
)
|
||||
logger.info(f"✨ Cleanup complete for agent '{self.name}'.")
|
||||
|
||||
async def run(self, request: Optional[str] = None) -> str:
|
||||
"""Run the agent with cleanup when done."""
|
||||
try:
|
||||
return await super().run(request)
|
||||
finally:
|
||||
await self.cleanup()
|
||||
|
||||
Reference in New Issue
Block a user