update WebSearch Tool
This commit is contained in:
liangxinbing
@@ -267,32 +267,20 @@ class BrowserUseTool(BaseTool, Generic[Context]):
|
||||
return ToolResult(
|
||||
error="Query is required for 'web_search' action"
|
||||
)
|
||||
search_results = await self.web_search_tool.execute(query)
|
||||
# Execute the web search and return results directly without browser navigation
|
||||
search_response = await self.web_search_tool.execute(
|
||||
query=query, fetch_content=True, num_results=1
|
||||
)
|
||||
# Navigate to the first search result
|
||||
first_search_result = search_response.results[0]
|
||||
url_to_navigate = first_search_result.url
|
||||
|
||||
if search_results:
|
||||
# Navigate to the first search result
|
||||
first_result = search_results[0]
|
||||
if isinstance(first_result, dict) and "url" in first_result:
|
||||
url_to_navigate = first_result["url"]
|
||||
elif isinstance(first_result, str):
|
||||
url_to_navigate = first_result
|
||||
else:
|
||||
return ToolResult(
|
||||
error=f"Invalid search result format: {first_result}"
|
||||
)
|
||||
page = await context.get_current_page()
|
||||
await page.goto(url_to_navigate)
|
||||
await page.wait_for_load_state()
|
||||
|
||||
page = await context.get_current_page()
|
||||
await page.goto(url_to_navigate)
|
||||
await page.wait_for_load_state()
|
||||
|
||||
return ToolResult(
|
||||
output=f"Searched for '{query}' and navigated to first result: {url_to_navigate}\nAll results:"
|
||||
+ "\n".join([str(r) for r in search_results])
|
||||
)
|
||||
else:
|
||||
return ToolResult(
|
||||
error=f"No search results found for '{query}'"
|
||||
)
|
||||
# Use the to_tool_result method to get consistent formatting
|
||||
return search_response.to_tool_result()
|
||||
|
||||
# Element interaction actions
|
||||
elif action == "click_element":
|
||||
|
||||
@@ -1,9 +1,50 @@
|
||||
from baidusearch.baidusearch import search
|
||||
|
||||
from app.tool.search.base import WebSearchEngine
|
||||
from app.tool.search.base import SearchItem, WebSearchEngine
|
||||
|
||||
|
||||
class BaiduSearchEngine(WebSearchEngine):
|
||||
def perform_search(self, query, num_results=10, *args, **kwargs):
|
||||
"""Baidu search engine."""
|
||||
return search(query, num_results=num_results)
|
||||
"""
|
||||
Baidu search engine.
|
||||
|
||||
Returns results formatted according to SearchItem model.
|
||||
"""
|
||||
raw_results = search(query, num_results=num_results)
|
||||
|
||||
# Convert raw results to SearchItem format
|
||||
results = []
|
||||
for i, item in enumerate(raw_results):
|
||||
if isinstance(item, str):
|
||||
# If it's just a URL
|
||||
results.append(
|
||||
SearchItem(title=f"Baidu Result {i+1}", url=item, description=None)
|
||||
)
|
||||
elif isinstance(item, dict):
|
||||
# If it's a dictionary with details
|
||||
results.append(
|
||||
SearchItem(
|
||||
title=item.get("title", f"Baidu Result {i+1}"),
|
||||
url=item.get("url", ""),
|
||||
description=item.get("abstract", None),
|
||||
)
|
||||
)
|
||||
else:
|
||||
# Try to get attributes directly
|
||||
try:
|
||||
results.append(
|
||||
SearchItem(
|
||||
title=getattr(item, "title", f"Baidu Result {i+1}"),
|
||||
url=getattr(item, "url", ""),
|
||||
description=getattr(item, "abstract", None),
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
# Fallback to a basic result
|
||||
results.append(
|
||||
SearchItem(
|
||||
title=f"Baidu Result {i+1}", url=str(item), description=None
|
||||
)
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
+22
-3
@@ -1,9 +1,28 @@
|
||||
from typing import List, Optional
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class SearchItem(BaseModel):
|
||||
"""Represents a single search result item"""
|
||||
|
||||
title: str = Field(description="The title of the search result")
|
||||
url: str = Field(description="The URL of the search result")
|
||||
description: Optional[str] = Field(
|
||||
default=None, description="A description or snippet of the search result"
|
||||
)
|
||||
|
||||
def __str__(self) -> str:
|
||||
"""String representation of a search result item."""
|
||||
return f"{self.title} - {self.url}"
|
||||
|
||||
|
||||
class WebSearchEngine(object):
|
||||
def perform_search(
|
||||
self, query: str, num_results: int = 10, *args, **kwargs
|
||||
) -> list[dict]:
|
||||
) -> List[SearchItem]:
|
||||
"""
|
||||
Perform a web search and return a list of URLs.
|
||||
Perform a web search and return a list of search items.
|
||||
|
||||
Args:
|
||||
query (str): The search query to submit to the search engine.
|
||||
@@ -12,6 +31,6 @@ class WebSearchEngine(object):
|
||||
kwargs: Additional keyword arguments.
|
||||
|
||||
Returns:
|
||||
List: A list of dict matching the search query.
|
||||
List[SearchItem]: A list of SearchItem objects matching the search query.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
from typing import List
|
||||
from typing import List, Tuple
|
||||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from app.logger import logger
|
||||
from app.tool.search.base import WebSearchEngine
|
||||
from app.tool.search.base import SearchItem, WebSearchEngine
|
||||
|
||||
|
||||
ABSTRACT_MAX_LENGTH = 300
|
||||
@@ -44,21 +44,16 @@ class BingSearchEngine(WebSearchEngine):
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update(HEADERS)
|
||||
|
||||
def _search_sync(self, query: str, num_results: int = 10) -> List[str]:
|
||||
def _search_sync(self, query: str, num_results: int = 10) -> List[SearchItem]:
|
||||
"""
|
||||
Synchronous Bing search implementation to retrieve a list of URLs matching a query.
|
||||
Synchronous Bing search implementation to retrieve search results.
|
||||
|
||||
Args:
|
||||
query (str): The search query to submit to Bing. Must not be empty.
|
||||
num_results (int, optional): The maximum number of URLs to return. Defaults to 10.
|
||||
query (str): The search query to submit to Bing.
|
||||
num_results (int, optional): Maximum number of results to return. Defaults to 10.
|
||||
|
||||
Returns:
|
||||
List[str]: A list of URLs from the search results, capped at `num_results`.
|
||||
Returns an empty list if the query is empty or no results are found.
|
||||
|
||||
Notes:
|
||||
- Pagination is handled by incrementing the `first` parameter and following `next_url` links.
|
||||
- If fewer results than `num_results` are available, all found URLs are returned.
|
||||
List[SearchItem]: A list of search items with title, URL, and description.
|
||||
"""
|
||||
if not query:
|
||||
return []
|
||||
@@ -72,25 +67,21 @@ class BingSearchEngine(WebSearchEngine):
|
||||
next_url, rank_start=len(list_result), first=first
|
||||
)
|
||||
if data:
|
||||
list_result.extend([item["url"] for item in data])
|
||||
list_result.extend(data)
|
||||
if not next_url:
|
||||
break
|
||||
first += 10
|
||||
|
||||
return list_result[:num_results]
|
||||
|
||||
def _parse_html(self, url: str, rank_start: int = 0, first: int = 1) -> tuple:
|
||||
def _parse_html(
|
||||
self, url: str, rank_start: int = 0, first: int = 1
|
||||
) -> Tuple[List[SearchItem], str]:
|
||||
"""
|
||||
Parse Bing search result HTML synchronously to extract search results and the next page URL.
|
||||
Parse Bing search result HTML to extract search results and the next page URL.
|
||||
|
||||
Args:
|
||||
url (str): The URL of the Bing search results page to parse.
|
||||
rank_start (int, optional): The starting rank for numbering the search results. Defaults to 0.
|
||||
first (int, optional): Unused parameter (possibly legacy). Defaults to 1.
|
||||
Returns:
|
||||
tuple: A tuple containing:
|
||||
- list: A list of dictionaries with keys 'title', 'abstract', 'url', and 'rank' for each result.
|
||||
- str or None: The URL of the next results page, or None if there is no next page.
|
||||
tuple: (List of SearchItem objects, next page URL or None)
|
||||
"""
|
||||
try:
|
||||
res = self.session.get(url=url)
|
||||
@@ -120,13 +111,14 @@ class BingSearchEngine(WebSearchEngine):
|
||||
abstract = abstract[:ABSTRACT_MAX_LENGTH]
|
||||
|
||||
rank_start += 1
|
||||
|
||||
# Create a SearchItem object
|
||||
list_data.append(
|
||||
{
|
||||
"title": title,
|
||||
"abstract": abstract,
|
||||
"url": url,
|
||||
"rank": rank_start,
|
||||
}
|
||||
SearchItem(
|
||||
title=title or f"Bing Result {rank_start}",
|
||||
url=url,
|
||||
description=abstract,
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
continue
|
||||
@@ -142,5 +134,9 @@ class BingSearchEngine(WebSearchEngine):
|
||||
return [], None
|
||||
|
||||
def perform_search(self, query, num_results=10, *args, **kwargs):
|
||||
"""Bing search engine."""
|
||||
"""
|
||||
Bing search engine.
|
||||
|
||||
Returns results formatted according to SearchItem model.
|
||||
"""
|
||||
return self._search_sync(query, num_results=num_results)
|
||||
|
||||
@@ -1,9 +1,53 @@
|
||||
from duckduckgo_search import DDGS
|
||||
|
||||
from app.tool.search.base import WebSearchEngine
|
||||
from app.tool.search.base import SearchItem, WebSearchEngine
|
||||
|
||||
|
||||
class DuckDuckGoSearchEngine(WebSearchEngine):
|
||||
async def perform_search(self, query, num_results=10, *args, **kwargs):
|
||||
"""DuckDuckGo search engine."""
|
||||
return DDGS.text(query, num_results=num_results)
|
||||
def perform_search(self, query, num_results=10, *args, **kwargs):
|
||||
"""
|
||||
DuckDuckGo search engine.
|
||||
|
||||
Returns results formatted according to SearchItem model.
|
||||
"""
|
||||
raw_results = DDGS().text(query, max_results=num_results)
|
||||
|
||||
results = []
|
||||
for i, item in enumerate(raw_results):
|
||||
if isinstance(item, str):
|
||||
# If it's just a URL
|
||||
results.append(
|
||||
SearchItem(
|
||||
title=f"DuckDuckGo Result {i+1}", url=item, description=None
|
||||
)
|
||||
)
|
||||
elif isinstance(item, dict):
|
||||
# Extract data from the dictionary
|
||||
results.append(
|
||||
SearchItem(
|
||||
title=item.get("title", f"DuckDuckGo Result {i+1}"),
|
||||
url=item.get("href", ""),
|
||||
description=item.get("body", None),
|
||||
)
|
||||
)
|
||||
else:
|
||||
# Try to extract attributes directly
|
||||
try:
|
||||
results.append(
|
||||
SearchItem(
|
||||
title=getattr(item, "title", f"DuckDuckGo Result {i+1}"),
|
||||
url=getattr(item, "href", ""),
|
||||
description=getattr(item, "body", None),
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
# Fallback
|
||||
results.append(
|
||||
SearchItem(
|
||||
title=f"DuckDuckGo Result {i+1}",
|
||||
url=str(item),
|
||||
description=None,
|
||||
)
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
@@ -1,9 +1,29 @@
|
||||
from googlesearch import search
|
||||
|
||||
from app.tool.search.base import WebSearchEngine
|
||||
from app.tool.search.base import SearchItem, WebSearchEngine
|
||||
|
||||
|
||||
class GoogleSearchEngine(WebSearchEngine):
|
||||
def perform_search(self, query, num_results=10, *args, **kwargs):
|
||||
"""Google search engine."""
|
||||
return search(query, num_results=num_results)
|
||||
"""
|
||||
Google search engine.
|
||||
|
||||
Returns results formatted according to SearchItem model.
|
||||
"""
|
||||
raw_results = search(query, num_results=num_results, advanced=True)
|
||||
|
||||
results = []
|
||||
for i, item in enumerate(raw_results):
|
||||
if isinstance(item, str):
|
||||
# If it's just a URL
|
||||
results.append(
|
||||
{"title": f"Google Result {i+1}", "url": item, "description": ""}
|
||||
)
|
||||
else:
|
||||
results.append(
|
||||
SearchItem(
|
||||
title=item.title, url=item.url, description=item.description
|
||||
)
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
+325
-93
@@ -1,11 +1,14 @@
|
||||
import asyncio
|
||||
from typing import List
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from tenacity import retry, stop_after_attempt, wait_exponential
|
||||
|
||||
from app.config import config
|
||||
from app.logger import logger
|
||||
from app.tool.base import BaseTool
|
||||
from app.tool.base import BaseTool, ToolResult
|
||||
from app.tool.search import (
|
||||
BaiduSearchEngine,
|
||||
BingSearchEngine,
|
||||
@@ -15,11 +18,153 @@ from app.tool.search import (
|
||||
)
|
||||
|
||||
|
||||
class SearchResult(BaseModel):
|
||||
"""Represents a single search result returned by a search engine."""
|
||||
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
position: int = Field(description="Position in search results")
|
||||
url: str = Field(description="URL of the search result")
|
||||
title: str = Field(default="", description="Title of the search result")
|
||||
description: str = Field(
|
||||
default="", description="Description or snippet of the search result"
|
||||
)
|
||||
source: str = Field(description="The search engine that provided this result")
|
||||
raw_content: Optional[str] = Field(
|
||||
default=None, description="Raw content from the search result page if available"
|
||||
)
|
||||
|
||||
def __str__(self) -> str:
|
||||
"""String representation of a search result."""
|
||||
return f"{self.title} ({self.url})"
|
||||
|
||||
|
||||
class SearchMetadata(BaseModel):
|
||||
"""Metadata about the search operation."""
|
||||
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
total_results: int = Field(description="Total number of results found")
|
||||
language: str = Field(description="Language code used for the search")
|
||||
country: str = Field(description="Country code used for the search")
|
||||
|
||||
|
||||
class SearchResponse(BaseModel):
|
||||
"""Structured response from the web search tool."""
|
||||
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
status: str = Field(
|
||||
description="Status of the search operation ('success' or 'error')"
|
||||
)
|
||||
query: str = Field(description="The search query that was executed")
|
||||
results: List[SearchResult] = Field(
|
||||
default_factory=list, description="List of search results"
|
||||
)
|
||||
metadata: Optional[SearchMetadata] = Field(
|
||||
default=None, description="Metadata about the search"
|
||||
)
|
||||
message: Optional[str] = Field(
|
||||
default=None, description="Error or status message if applicable"
|
||||
)
|
||||
|
||||
def to_tool_result(self) -> ToolResult:
|
||||
"""Convert search response to a ToolResult for consistent API usage."""
|
||||
if self.status == "error":
|
||||
return ToolResult(error=self.message or "Search failed")
|
||||
|
||||
result_text = [f"Search results for '{self.query}':"]
|
||||
|
||||
for i, result in enumerate(self.results, 1):
|
||||
# Add title with position number
|
||||
title = result.title.strip() or "No title"
|
||||
result_text.append(f"\n{i}. {title}")
|
||||
|
||||
# Add URL with proper indentation
|
||||
result_text.append(f" URL: {result.url}")
|
||||
|
||||
# Add description if available
|
||||
if result.description.strip():
|
||||
result_text.append(f" Description: {result.description}")
|
||||
|
||||
# Add content preview if available
|
||||
if result.raw_content:
|
||||
content_preview = result.raw_content[:1000].replace("\n", " ").strip()
|
||||
if len(result.raw_content) > 1000:
|
||||
content_preview += "..."
|
||||
result_text.append(f" Content: {content_preview}")
|
||||
|
||||
# Add metadata at the bottom if available
|
||||
if self.metadata:
|
||||
result_text.extend(
|
||||
[
|
||||
f"\nMetadata:",
|
||||
f"- Total results: {self.metadata.total_results}",
|
||||
f"- Language: {self.metadata.language}",
|
||||
f"- Country: {self.metadata.country}",
|
||||
]
|
||||
)
|
||||
|
||||
return ToolResult(output="\n".join(result_text))
|
||||
|
||||
|
||||
class WebContentFetcher:
|
||||
"""Utility class for fetching web content."""
|
||||
|
||||
@staticmethod
|
||||
async def fetch_content(url: str, timeout: int = 10) -> Optional[str]:
|
||||
"""
|
||||
Fetch and extract the main content from a webpage.
|
||||
|
||||
Args:
|
||||
url: The URL to fetch content from
|
||||
timeout: Request timeout in seconds
|
||||
|
||||
Returns:
|
||||
Extracted text content or None if fetching fails
|
||||
"""
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
|
||||
}
|
||||
|
||||
try:
|
||||
# Use asyncio to run requests in a thread pool
|
||||
response = await asyncio.get_event_loop().run_in_executor(
|
||||
None, lambda: requests.get(url, headers=headers, timeout=timeout)
|
||||
)
|
||||
|
||||
if response.status_code != 200:
|
||||
logger.warning(
|
||||
f"Failed to fetch content from {url}: HTTP {response.status_code}"
|
||||
)
|
||||
return None
|
||||
|
||||
# Parse HTML with BeautifulSoup
|
||||
soup = BeautifulSoup(response.text, "html.parser")
|
||||
|
||||
# Remove script and style elements
|
||||
for script in soup(["script", "style", "header", "footer", "nav"]):
|
||||
script.extract()
|
||||
|
||||
# Get text content
|
||||
text = soup.get_text(separator="\n", strip=True)
|
||||
|
||||
# Clean up whitespace and limit size (100KB max)
|
||||
text = " ".join(text.split())
|
||||
return text[:10000] if text else None
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Error fetching content from {url}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
class WebSearch(BaseTool):
|
||||
"""Search the web for information using various search engines."""
|
||||
|
||||
name: str = "web_search"
|
||||
description: str = """Perform a web search and return a list of relevant links.
|
||||
This function attempts to use the primary search engine API to get up-to-date results.
|
||||
If an error occurs, it falls back to an alternative search engine."""
|
||||
description: str = """Search the web for real-time information about any topic.
|
||||
This tool returns comprehensive search results with relevant information, URLs, titles, and descriptions.
|
||||
If the primary search engine fails, it automatically falls back to alternative engines."""
|
||||
parameters: dict = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -29,8 +174,23 @@ class WebSearch(BaseTool):
|
||||
},
|
||||
"num_results": {
|
||||
"type": "integer",
|
||||
"description": "(optional) The number of search results to return. Default is 10.",
|
||||
"default": 10,
|
||||
"description": "(optional) The number of search results to return. Default is 5.",
|
||||
"default": 5,
|
||||
},
|
||||
"lang": {
|
||||
"type": "string",
|
||||
"description": "(optional) Language code for search results (default: en).",
|
||||
"default": "en",
|
||||
},
|
||||
"country": {
|
||||
"type": "string",
|
||||
"description": "(optional) Country code for search results (default: us).",
|
||||
"default": "us",
|
||||
},
|
||||
"fetch_content": {
|
||||
"type": "boolean",
|
||||
"description": "(optional) Whether to fetch full content from result pages. Default is false.",
|
||||
"default": False,
|
||||
},
|
||||
},
|
||||
"required": ["query"],
|
||||
@@ -41,35 +201,62 @@ class WebSearch(BaseTool):
|
||||
"duckduckgo": DuckDuckGoSearchEngine(),
|
||||
"bing": BingSearchEngine(),
|
||||
}
|
||||
content_fetcher: WebContentFetcher = WebContentFetcher()
|
||||
|
||||
async def execute(self, query: str, num_results: int = 10) -> List[str]:
|
||||
async def execute(
|
||||
self,
|
||||
query: str,
|
||||
num_results: int = 5,
|
||||
lang: str = "en",
|
||||
country: str = "us",
|
||||
fetch_content: bool = False,
|
||||
) -> SearchResponse:
|
||||
"""
|
||||
Execute a Web search and return a list of URLs.
|
||||
Tries engines in order based on configuration, falling back if an engine fails with errors.
|
||||
If all engines fail, it will wait and retry up to the configured number of times.
|
||||
Execute a Web search and return detailed search results.
|
||||
|
||||
Args:
|
||||
query (str): The search query to submit to the search engine.
|
||||
num_results (int, optional): The number of search results to return. Default is 10.
|
||||
query: The search query to submit to the search engine
|
||||
num_results: The number of search results to return (default: 5)
|
||||
lang: Language code for search results (default: en)
|
||||
country: Country code for search results (default: us)
|
||||
fetch_content: Whether to fetch content from result pages (default: False)
|
||||
|
||||
Returns:
|
||||
List[str]: A list of URLs matching the search query.
|
||||
A structured response containing search results and metadata
|
||||
"""
|
||||
# Get retry settings from config
|
||||
retry_delay = 60 # Default to 60 seconds
|
||||
max_retries = 3 # Default to 3 retries
|
||||
|
||||
if config.search_config:
|
||||
retry_delay = getattr(config.search_config, "retry_delay", 60)
|
||||
max_retries = getattr(config.search_config, "max_retries", 3)
|
||||
retry_delay = (
|
||||
getattr(config.search_config, "retry_delay", 60)
|
||||
if config.search_config
|
||||
else 60
|
||||
)
|
||||
max_retries = (
|
||||
getattr(config.search_config, "max_retries", 3)
|
||||
if config.search_config
|
||||
else 3
|
||||
)
|
||||
search_params = {"lang": lang, "country": country}
|
||||
|
||||
# Try searching with retries when all engines fail
|
||||
for retry_count in range(
|
||||
max_retries + 1
|
||||
): # +1 because first try is not a retry
|
||||
links = await self._try_all_engines(query, num_results)
|
||||
if links:
|
||||
return links
|
||||
for retry_count in range(max_retries + 1):
|
||||
results = await self._try_all_engines(query, num_results, search_params)
|
||||
|
||||
if results:
|
||||
# Fetch content if requested
|
||||
if fetch_content:
|
||||
results = await self._fetch_content_for_results(results)
|
||||
|
||||
# Return a successful structured response
|
||||
return SearchResponse(
|
||||
status="success",
|
||||
query=query,
|
||||
results=results,
|
||||
metadata=SearchMetadata(
|
||||
total_results=len(results),
|
||||
language=lang,
|
||||
country=country,
|
||||
),
|
||||
)
|
||||
|
||||
if retry_count < max_retries:
|
||||
# All engines failed, wait and retry
|
||||
@@ -82,95 +269,140 @@ class WebSearch(BaseTool):
|
||||
f"All search engines failed after {max_retries} retries. Giving up."
|
||||
)
|
||||
|
||||
return []
|
||||
# Return an error response
|
||||
return SearchResponse(
|
||||
status="error",
|
||||
query=query,
|
||||
message="All search engines failed to return results",
|
||||
results=[],
|
||||
)
|
||||
|
||||
async def _try_all_engines(self, query: str, num_results: int) -> List[str]:
|
||||
"""
|
||||
Try all search engines in the configured order.
|
||||
|
||||
Args:
|
||||
query (str): The search query to submit to the search engine.
|
||||
num_results (int): The number of search results to return.
|
||||
|
||||
Returns:
|
||||
List[str]: A list of URLs matching the search query, or empty list if all engines fail.
|
||||
"""
|
||||
async def _try_all_engines(
|
||||
self, query: str, num_results: int, search_params: Dict[str, Any]
|
||||
) -> List[SearchResult]:
|
||||
"""Try all search engines in the configured order."""
|
||||
engine_order = self._get_engine_order()
|
||||
failed_engines = []
|
||||
|
||||
for engine_name in engine_order:
|
||||
engine = self._search_engine[engine_name]
|
||||
try:
|
||||
logger.info(f"🔎 Attempting search with {engine_name.capitalize()}...")
|
||||
links = await self._perform_search_with_engine(
|
||||
engine, query, num_results
|
||||
)
|
||||
if links:
|
||||
if failed_engines:
|
||||
logger.info(
|
||||
f"Search successful with {engine_name.capitalize()} after trying: {', '.join(failed_engines)}"
|
||||
)
|
||||
return links
|
||||
except Exception as e:
|
||||
failed_engines.append(engine_name.capitalize())
|
||||
is_rate_limit = "429" in str(e) or "Too Many Requests" in str(e)
|
||||
logger.info(f"🔎 Attempting search with {engine_name.capitalize()}...")
|
||||
search_items = await self._perform_search_with_engine(
|
||||
engine, query, num_results, search_params
|
||||
)
|
||||
|
||||
if is_rate_limit:
|
||||
logger.warning(
|
||||
f"⚠️ {engine_name.capitalize()} search engine rate limit exceeded, trying next engine..."
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
f"⚠️ {engine_name.capitalize()} search failed with error: {e}"
|
||||
)
|
||||
if not search_items:
|
||||
continue
|
||||
|
||||
if failed_engines:
|
||||
logger.info(
|
||||
f"Search successful with {engine_name.capitalize()} after trying: {', '.join(failed_engines)}"
|
||||
)
|
||||
|
||||
# Transform search items into structured results
|
||||
return [
|
||||
SearchResult(
|
||||
position=i + 1,
|
||||
url=item.url,
|
||||
title=item.title
|
||||
or f"Result {i+1}", # Ensure we always have a title
|
||||
description=item.description or "",
|
||||
source=engine_name,
|
||||
)
|
||||
for i, item in enumerate(search_items)
|
||||
]
|
||||
|
||||
if failed_engines:
|
||||
logger.error(f"All search engines failed: {', '.join(failed_engines)}")
|
||||
return []
|
||||
|
||||
async def _fetch_content_for_results(
|
||||
self, results: List[SearchResult]
|
||||
) -> List[SearchResult]:
|
||||
"""Fetch and add web content to search results."""
|
||||
if not results:
|
||||
return []
|
||||
|
||||
# Create tasks for each result
|
||||
tasks = [self._fetch_single_result_content(result) for result in results]
|
||||
|
||||
# Type annotation to help type checker
|
||||
fetched_results: List[SearchResult] = await asyncio.gather(*tasks)
|
||||
|
||||
# Explicit validation of return type
|
||||
return [
|
||||
(
|
||||
result
|
||||
if isinstance(result, SearchResult)
|
||||
else SearchResult(**result.dict())
|
||||
)
|
||||
for result in fetched_results
|
||||
]
|
||||
|
||||
async def _fetch_single_result_content(self, result: SearchResult) -> SearchResult:
|
||||
"""Fetch content for a single search result."""
|
||||
if result.url:
|
||||
content = await self.content_fetcher.fetch_content(result.url)
|
||||
if content:
|
||||
result.raw_content = content
|
||||
return result
|
||||
|
||||
def _get_engine_order(self) -> List[str]:
|
||||
"""
|
||||
Determines the order in which to try search engines.
|
||||
Preferred engine is first (based on configuration), followed by fallback engines,
|
||||
and then the remaining engines.
|
||||
"""Determines the order in which to try search engines."""
|
||||
preferred = (
|
||||
getattr(config.search_config, "engine", "google").lower()
|
||||
if config.search_config
|
||||
else "google"
|
||||
)
|
||||
fallbacks = (
|
||||
[engine.lower() for engine in config.search_config.fallback_engines]
|
||||
if config.search_config
|
||||
and hasattr(config.search_config, "fallback_engines")
|
||||
else []
|
||||
)
|
||||
|
||||
Returns:
|
||||
List[str]: Ordered list of search engine names.
|
||||
"""
|
||||
preferred = "google"
|
||||
fallbacks = []
|
||||
|
||||
if config.search_config:
|
||||
if config.search_config.engine:
|
||||
preferred = config.search_config.engine.lower()
|
||||
if config.search_config.fallback_engines:
|
||||
fallbacks = [
|
||||
engine.lower() for engine in config.search_config.fallback_engines
|
||||
]
|
||||
|
||||
engine_order = []
|
||||
# Add preferred engine first
|
||||
if preferred in self._search_engine:
|
||||
engine_order.append(preferred)
|
||||
|
||||
# Add configured fallback engines in order
|
||||
for fallback in fallbacks:
|
||||
if fallback in self._search_engine and fallback not in engine_order:
|
||||
engine_order.append(fallback)
|
||||
# Start with preferred engine, then fallbacks, then remaining engines
|
||||
engine_order = [preferred] if preferred in self._search_engine else []
|
||||
engine_order.extend(
|
||||
[
|
||||
fb
|
||||
for fb in fallbacks
|
||||
if fb in self._search_engine and fb not in engine_order
|
||||
]
|
||||
)
|
||||
engine_order.extend([e for e in self._search_engine if e not in engine_order])
|
||||
|
||||
return engine_order
|
||||
|
||||
@retry(
|
||||
stop=stop_after_attempt(3),
|
||||
wait=wait_exponential(multiplier=1, min=1, max=10),
|
||||
stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=1, max=10)
|
||||
)
|
||||
async def _perform_search_with_engine(
|
||||
self,
|
||||
engine: WebSearchEngine,
|
||||
query: str,
|
||||
num_results: int,
|
||||
) -> List[str]:
|
||||
loop = asyncio.get_event_loop()
|
||||
return await loop.run_in_executor(
|
||||
None, lambda: list(engine.perform_search(query, num_results=num_results))
|
||||
search_params: Dict[str, Any],
|
||||
) -> List[Any]:
|
||||
"""Execute search with the given engine and parameters."""
|
||||
return await asyncio.get_event_loop().run_in_executor(
|
||||
None,
|
||||
lambda: list(
|
||||
engine.perform_search(
|
||||
query,
|
||||
num_results=num_results,
|
||||
lang=search_params.get("lang"),
|
||||
country=search_params.get("country"),
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
web_search = WebSearch()
|
||||
search_response = asyncio.run(
|
||||
web_search.execute(
|
||||
query="Python programming", fetch_content=True, num_results=1
|
||||
)
|
||||
)
|
||||
print(search_response.to_tool_result())
|
||||
|
||||
@@ -71,10 +71,10 @@ temperature = 0.0 # Controls randomness for vision mode
|
||||
|
||||
# Optional configuration, Search settings.
|
||||
# [search]
|
||||
# Search engine for agent to use. Default is "Google", can be set to "Baidu" or "DuckDuckGo".
|
||||
# Search engine for agent to use. Default is "Google", can be set to "Baidu" or "DuckDuckGo" or "Bing".
|
||||
#engine = "Google"
|
||||
# Fallback engine order. Default is ["DuckDuckGo", "Baidu"] - will try in this order after primary engine fails.
|
||||
#fallback_engines = ["DuckDuckGo", "Baidu"]
|
||||
# Fallback engine order. Default is ["DuckDuckGo", "Baidu", "Bing"] - will try in this order after primary engine fails.
|
||||
#fallback_engines = ["DuckDuckGo", "Baidu", "Bing"]
|
||||
# Seconds to wait before retrying all engines again when they all fail due to rate limits. Default is 60.
|
||||
#retry_delay = 60
|
||||
# Maximum number of times to retry all engines when all fail. Default is 3.
|
||||
|
||||
Reference in New Issue
Block a user