diff --git a/examples/capability/setup.md b/examples/capability/setup.md index e3fedf3d3..f39dc741a 100644 --- a/examples/capability/setup.md +++ b/examples/capability/setup.md @@ -162,6 +162,7 @@ Since all loads are `override=False`, the **effective priority** (highest to low | `EXA_API_KEY` | web_search (exa engine) | Only needed for `engine_type='exa'` | | `SERPAPI_API_KEY` | web_search (serpapi / google / bing / baidu) | Only needed for SerpAPI-backed engines | | `TAVILY_API_KEY` | web_search (tavily engine) | Only needed for `engine_type='tavily'` | +| `OUTCRAWL_API_KEY` | web_search (outcrawl engine) | Only needed for `engine_type='outcrawl'` | | `MS_AGENT_OUTPUT_DIR` | replace_file_*, lsp_check_* | Workspace root for file operations (defaults to cwd) | diff --git a/ms-agent-skills/SKILL.md b/ms-agent-skills/SKILL.md index 4bc76a622..7bd63c15f 100644 --- a/ms-agent-skills/SKILL.md +++ b/ms-agent-skills/SKILL.md @@ -4,7 +4,7 @@ version: 1.0.0 description: >- Access ms-agent's advanced AI capabilities via MCP tools: deep research, document research, financial research, code generation, video generation, - web search (arxiv/exa/serpapi/tavily), LSP code validation (TypeScript/Python/Java), + web search (arxiv/exa/serpapi/tavily/outcrawl), LSP code validation (TypeScript/Python/Java), concurrent-safe file editing, and agent delegation. All project-level capabilities support async submit/check/get patterns. Use when the user asks to research a topic, analyze documents, generate code or videos, @@ -207,7 +207,7 @@ User wants to... │ └── submit_fin_research_task → check_fin_research_progress → get_fin_research_report │ ├── Search the web for quick info -│ └── web_search(query="...", engine_type="arxiv|exa|serpapi|tavily") +│ └── web_search(query="...", engine_type="arxiv|exa|serpapi|tavily|outcrawl") │ ├── Generate a software project from requirements (10-30 min) │ └── submit_code_genesis_task → check_code_genesis_progress → get_code_genesis_result diff --git a/ms-agent-skills/references/web-search.md b/ms-agent-skills/references/web-search.md index df500cc38..9e111fca2 100644 --- a/ms-agent-skills/references/web-search.md +++ b/ms-agent-skills/references/web-search.md @@ -33,6 +33,7 @@ search engines and optional full-page content fetching. | `exa` | — | General web, semantic search | `EXA_API_KEY` or `EXA_API_KEYS` (comma-separated key pool) | | `serpapi` | `serp`, `google`, `bing`, `baidu` | Google/Bing/Baidu results via SerpAPI | `SERPAPI_API_KEY` | | `tavily` | — | AI-oriented web search | `TAVILY_API_KEY` | +| `outcrawl` | - | Web search and scrape API for AI agents | `OUTCRAWL_API_KEY` | ### Examples @@ -72,7 +73,7 @@ field with the page text (truncated to 10,000 characters). ### Step 1: Choose the Right Engine - For academic/research questions → `arxiv` -- For general web info → `exa`, `tavily`, or `serpapi` (and aliases) +- For general web info → `exa`, `tavily`, `outcrawl`, or `serpapi` (and aliases) - If unsure → start with `arxiv`, fall back to `exa` or `tavily` ### Step 2: Execute the Search diff --git a/ms_agent/capabilities/wrappers/web_search.py b/ms_agent/capabilities/wrappers/web_search.py index 9ea579f4e..ad0970708 100644 --- a/ms_agent/capabilities/wrappers/web_search.py +++ b/ms_agent/capabilities/wrappers/web_search.py @@ -24,6 +24,7 @@ 'bing': ['SERPAPI_API_KEY'], 'baidu': ['SERPAPI_API_KEY'], 'tavily': ['TAVILY_API_KEY'], + 'outcrawl': ['OUTCRAWL_API_KEY'], } @@ -50,14 +51,16 @@ def _get_fetcher() -> Any: granularity='tool', summary=( 'Search the web using multiple engines ' - '(arxiv / exa / serpapi / tavily, plus google/bing/baidu aliases) ' + '(arxiv / exa / serpapi / tavily / outcrawl, plus google/bing/baidu ' + 'aliases) ' 'and optionally fetch full page content.'), description=( 'Performs a web search and returns structured results including ' 'title, URL, and summary for each hit. Supported engines: ' 'arxiv (default, no key), exa (EXA_API_KEY or EXA_API_KEYS), ' 'serpapi / serp / ' - 'google / bing / baidu (SERPAPI_API_KEY), tavily (TAVILY_API_KEY). ' + 'google / bing / baidu (SERPAPI_API_KEY), tavily (TAVILY_API_KEY), ' + 'outcrawl (OUTCRAWL_API_KEY). ' 'Set fetch_content=true to additionally retrieve and return page ' 'text (truncated to 10 000 chars).'), input_schema={ @@ -77,7 +80,8 @@ def _get_fetcher() -> Any: 'string', 'description': ("Search engine: 'arxiv' (default, no key), 'exa', " - "'serpapi'/'serp'/'google'/'bing'/'baidu', or 'tavily'"), + "'serpapi'/'serp'/'google'/'bing'/'baidu', 'tavily' or " + "'outcrawl'"), 'default': 'arxiv', }, diff --git a/ms_agent/tools/search/outcrawl/__init__.py b/ms_agent/tools/search/outcrawl/__init__.py new file mode 100644 index 000000000..d90762b1e --- /dev/null +++ b/ms_agent/tools/search/outcrawl/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +from ms_agent.tools.search.outcrawl.search import OutcrawlSearch + +__all__ = ['OutcrawlSearch'] diff --git a/ms_agent/tools/search/outcrawl/fetcher.py b/ms_agent/tools/search/outcrawl/fetcher.py new file mode 100644 index 000000000..6530d78b6 --- /dev/null +++ b/ms_agent/tools/search/outcrawl/fetcher.py @@ -0,0 +1,55 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Outcrawl Scrape API as ContentFetcher (for fetch_page / URL fetch).""" +import os +import time +from typing import Any, Dict, Optional, Tuple + +from ms_agent.tools.search.outcrawl.http import post_json +from ms_agent.utils.logger import get_logger + +logger = get_logger() + + +class OutcrawlScrapeFetcher: + """Fetch page text as markdown via Outcrawl POST /v1/scrape.""" + + def __init__(self, + api_key: Optional[str] = None, + *, + only_main_content: bool = True, + timeout: float = 45.0): + self._api_key = api_key or os.getenv('OUTCRAWL_API_KEY') or '' + self._only_main_content = bool(only_main_content) + self._timeout = max(1.0, float(timeout)) + + def fetch(self, url: str) -> Tuple[str, Dict[str, Any]]: + meta: Dict[str, Any] = { + 'fetcher': 'outcrawl_scrape', + 'fetched_at': time.strftime('%Y-%m-%dT%H:%M:%S'), + } + if not self._api_key: + meta['error'] = 'OUTCRAWL_API_KEY is not set' + return '', meta + body = { + 'url': url, + 'formats': ['markdown'], + 'onlyMainContent': self._only_main_content, + } + try: + data = post_json( + '/scrape', body, api_key=self._api_key, timeout=self._timeout) + except Exception as e: + logger.warning(f'Outcrawl scrape failed for {url[:80]}: {e}') + meta['error'] = str(e) + return '', meta + + text = (data.get('markdown') or '').strip() + metadata = data.get('metadata') or {} + meta.update({ + 'status_code': data.get('statusCode'), + 'title': metadata.get('title'), + 'outcrawl_usage': data.get('usage'), + }) + if not text: + meta['error'] = 'Outcrawl returned no markdown' + return text, meta diff --git a/ms_agent/tools/search/outcrawl/http.py b/ms_agent/tools/search/outcrawl/http.py new file mode 100644 index 000000000..c0eb0e37c --- /dev/null +++ b/ms_agent/tools/search/outcrawl/http.py @@ -0,0 +1,116 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Minimal HTTP JSON client for the Outcrawl REST API (stdlib only).""" +import json +from typing import Any, Dict, Optional +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + +OUTCRAWL_API_BASE = 'https://api.outcrawl.ai/v1' + + +class OutcrawlHTTPError(RuntimeError): + """An Outcrawl call that failed, with the fields a caller can act on. + + ``status`` is the HTTP code (None for transport failures), ``code`` is + Outcrawl's machine-readable ``code`` (e.g. ``unauthorized``) and + ``retry_after`` is set when the response carried a Retry-After header. + The ``is_quota`` / ``is_auth`` properties match the ones + ``WebSearchTool._search_error_payload`` reads. + """ + + def __init__(self, + message: str, + *, + status: Optional[int] = None, + code: str = '', + retry_after: Optional[int] = None, + detail: Any = None): + super().__init__(message) + self.status = status + self.code = code + self.retry_after = retry_after + self.detail = detail + + @property + def is_quota(self) -> bool: + """Rate limited or out of credits.""" + return self.status in (402, 429) + + @property + def is_auth(self) -> bool: + return self.status in (401, 403) + + +def _ssl_context(): + """A verifying TLS context backed by certifi when it is installed.""" + try: + import certifi + import ssl + return ssl.create_default_context(cafile=certifi.where()) + except Exception: + return None + + +def post_json(path: str, + body: Dict[str, Any], + *, + api_key: str, + timeout: float = 60.0) -> Dict[str, Any]: + """ + POST JSON to ``OUTCRAWL_API_BASE + path`` and parse the JSON response. + + Raises: + OutcrawlHTTPError: on HTTP errors, network errors or invalid JSON. + """ + data = json.dumps(body, ensure_ascii=False).encode('utf-8') + req = Request( + f'{OUTCRAWL_API_BASE}{path}', + data=data, + method='POST', + headers={ + 'Authorization': f'Bearer {api_key}', + 'Content-Type': 'application/json', + 'Accept': 'application/json', + # The default Python-urllib agent is refused at the API edge. + 'User-Agent': 'ms-agent', + }) + try: + with urlopen(req, timeout=timeout, context=_ssl_context()) as resp: + raw = resp.read().decode('utf-8', errors='replace') + except HTTPError as e: + err_body = '' + try: + err_body = e.read().decode('utf-8', errors='replace') + except Exception: + pass + try: + detail = json.loads(err_body) if err_body else {} + except json.JSONDecodeError: + detail = {'raw': err_body} + code, message = '', '' + if isinstance(detail, dict): + code = str(detail.get('code') or '') + message = str(detail.get('message') or '') + retry_after = None + try: + header_value = e.headers.get('retry-after') + if header_value: + retry_after = int(float(header_value)) + except (AttributeError, TypeError, ValueError): + retry_after = None + raise OutcrawlHTTPError( + f'Outcrawl HTTP {e.code}: {message or detail}', + status=e.code, + code=code, + retry_after=retry_after, + detail=detail) from e + except URLError as e: + raise OutcrawlHTTPError(f'Outcrawl network error: {e}') from e + + if not raw.strip(): + return {} + try: + return json.loads(raw) + except json.JSONDecodeError as e: + raise OutcrawlHTTPError( + f'Outcrawl returned invalid JSON: {raw[:200]}') from e diff --git a/ms_agent/tools/search/outcrawl/schema.py b/ms_agent/tools/search/outcrawl/schema.py new file mode 100644 index 000000000..6f1a0a0b6 --- /dev/null +++ b/ms_agent/tools/search/outcrawl/schema.py @@ -0,0 +1,56 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +from dataclasses import dataclass +from typing import Any, Dict, List + + +@dataclass +class OutcrawlSearchRequest: + """Outcrawl POST /v1/search body. See https://outcrawl.ai/docs/search/overview""" + + query: str + limit: int = 5 + # When true, Outcrawl also scrapes each hit and returns its markdown. + scrape: bool = False + + def to_api_body(self) -> Dict[str, Any]: + body: Dict[str, Any] = { + 'query': self.query, + 'limit': max(1, min(20, int(self.limit))), + } + if self.scrape: + body['scrape'] = True + body['formats'] = ['markdown'] + return body + + +@dataclass +class OutcrawlSearchResult: + """Parsed Outcrawl /v1/search JSON.""" + + query: str + arguments: Dict[str, Any] + response: Dict[str, Any] + + def to_list(self) -> List[Dict[str, Any]]: + """Normalize to WebSearchTool pipeline dicts (prefill content when markdown is present).""" + if not self.response: + return [] + rows: List[Dict[str, Any]] = [] + for hit in self.response.get('hits') or []: + url = hit.get('url') or '' + snippet = (hit.get('snippet') or '').strip() + markdown = (hit.get('markdown') or '').strip() + rows.append({ + 'url': url, + 'id': url, + 'title': hit.get('title') or '', + 'highlights': None, + 'highlight_scores': None, + 'summary': snippet, + 'markdown': markdown or None, + # Pipeline uses these keys: + 'content': markdown or snippet, + 'fetch_success': bool(markdown), + 'rank': hit.get('rank'), + }) + return rows diff --git a/ms_agent/tools/search/outcrawl/search.py b/ms_agent/tools/search/outcrawl/search.py new file mode 100644 index 000000000..1f7753863 --- /dev/null +++ b/ms_agent/tools/search/outcrawl/search.py @@ -0,0 +1,109 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +import os +from typing import TYPE_CHECKING, Any, Optional + +from ms_agent.tools.search.outcrawl.http import post_json +from ms_agent.tools.search.outcrawl.schema import (OutcrawlSearchRequest, + OutcrawlSearchResult) +from ms_agent.tools.search.search_base import SearchEngine, SearchEngineType +from ms_agent.utils.logger import get_logger + +if TYPE_CHECKING: + from ms_agent.llm.utils import Tool + +logger = get_logger() + + +class OutcrawlSearch(SearchEngine): + """ + Outcrawl (https://outcrawl.ai) is a scrape and search API for AI agents. + + Uses POST /v1/search. With ``scrape=true`` each hit also carries the page + markdown, so WebSearchTool can skip the separate fetch step. + """ + + engine_type = SearchEngineType.OUTCRAWL + + def __init__( + self, + api_key: Optional[str] = None, + request_timeout: float = 60.0, + ): + self._api_key = api_key or os.getenv('OUTCRAWL_API_KEY') or '' + if not self._api_key: + raise ValueError( + 'OUTCRAWL_API_KEY must be set either as an argument ' + 'or as an environment variable') + self._request_timeout = float(request_timeout) + + def search(self, + search_request: OutcrawlSearchRequest) -> OutcrawlSearchResult: + body = search_request.to_api_body() + # Unguarded on purpose: OutcrawlHTTPError carries the status/code the + # tool layer uses to tell the agent why a search failed. + data = post_json( + '/search', + body, + api_key=self._api_key, + timeout=self._request_timeout) + return OutcrawlSearchResult( + query=search_request.query, + arguments=body, + response=data or {}, + ) + + @classmethod + def get_tool_definition(cls, server_name: str = 'web_search') -> 'Tool': + from ms_agent.llm.utils import Tool + return Tool( + tool_name=cls.get_tool_name(), + server_name=server_name, + description=( + 'Search the web using Outcrawl, a scrape and search API for ' + 'AI agents. Returns ranked results with title, URL and ' + 'snippet. Set `scrape` to true to also get each page as ' + 'markdown in the same call.'), + parameters={ + 'type': 'object', + 'properties': { + 'query': { + 'type': 'string', + 'description': 'Search query.', + }, + 'num_results': { + 'type': + 'integer', + 'minimum': + 1, + 'maximum': + 20, + 'description': + 'Max results (maps to Outcrawl limit). Default 5.', + }, + 'scrape': { + 'type': + 'boolean', + 'description': + ('Also scrape each result and return its markdown. ' + 'Uses more credits. Default false.'), + }, + }, + 'required': ['query'], + }, + ) + + @classmethod + def build_request_from_args(cls, **kwargs: Any) -> OutcrawlSearchRequest: + num = kwargs.get('num_results', kwargs.get('limit', 5)) + try: + num = int(num) + except (TypeError, ValueError): + num = 5 + scrape = kwargs.get('scrape', False) + if isinstance(scrape, str): + scrape = scrape.lower() == 'true' + return OutcrawlSearchRequest( + query=kwargs['query'], + limit=num, + scrape=bool(scrape), + ) diff --git a/ms_agent/tools/search/search_base.py b/ms_agent/tools/search/search_base.py index 3c125f11d..d2dd189bd 100644 --- a/ms_agent/tools/search/search_base.py +++ b/ms_agent/tools/search/search_base.py @@ -17,6 +17,7 @@ class SearchEngineType(enum.Enum): SERPAPI = 'serpapi' ARXIV = 'arxiv' TAVILY = 'tavily' + OUTCRAWL = 'outcrawl' # Mapping from engine type to tool name @@ -25,6 +26,7 @@ class SearchEngineType(enum.Enum): 'serpapi': 'serpapi_search', 'arxiv': 'arxiv_search', 'tavily': 'tavily_search', + 'outcrawl': 'outcrawl_search', } diff --git a/ms_agent/tools/search/websearch_tool.py b/ms_agent/tools/search/websearch_tool.py index a0f2e3e4e..c31eadb7a 100644 --- a/ms_agent/tools/search/websearch_tool.py +++ b/ms_agent/tools/search/websearch_tool.py @@ -240,6 +240,13 @@ def get_content_fetcher(fetcher_type: str = 'jina_reader', include_usage=bool( kwargs.get('tavily_extract_include_usage', False)), ) + if fetcher_type == 'outcrawl_scrape': + from ms_agent.tools.search.outcrawl.fetcher import \ + OutcrawlScrapeFetcher + return OutcrawlScrapeFetcher( + api_key=kwargs.get('outcrawl_api_key'), + timeout=float(kwargs.get('timeout', 45.0)), + ) # Future: add more fetchers # elif fetcher_type == 'docling': # return DoclingContentFetcher(**kwargs) @@ -271,7 +278,7 @@ def get_search_engine_class(engine_type: str) -> Type[SearchEngine]: Get search engine class by type. Args: - engine_type: One of 'exa', 'serpapi', 'arxiv' + engine_type: One of 'exa', 'serpapi', 'arxiv', 'tavily', 'outcrawl' Returns: SearchEngine class (not instance) @@ -290,6 +297,9 @@ def get_search_engine_class(engine_type: str) -> Type[SearchEngine]: elif engine_type == 'tavily': from ms_agent.tools.search.tavily import TavilySearch return TavilySearch + elif engine_type == 'outcrawl': + from ms_agent.tools.search.outcrawl import OutcrawlSearch + return OutcrawlSearch else: logger.warning( f"Unknown search engine '{engine_type}', falling back to arxiv") @@ -304,7 +314,7 @@ def get_search_engine(engine_type: str, Get search engine instance by type. Args: - engine_type: One of 'exa', 'serpapi', 'arxiv' + engine_type: One of 'exa', 'serpapi', 'arxiv', 'tavily', 'outcrawl' api_key: API key for the search engine (if required). For Exa, this can be a comma-separated string of multiple keys to enable automatic key rotation on credit exhaustion. @@ -337,6 +347,12 @@ def get_search_engine(engine_type: str, api_key=api_key or os.getenv('TAVILY_API_KEY'), request_timeout=float(kwargs.get('request_timeout', 120.0)), ) + elif engine_type == 'outcrawl': + from ms_agent.tools.search.outcrawl import OutcrawlSearch + return OutcrawlSearch( + api_key=api_key or os.getenv('OUTCRAWL_API_KEY'), + request_timeout=float(kwargs.get('request_timeout', 60.0)), + ) else: logger.warning( f"Unknown search engine '{engine_type}', falling back to arxiv") @@ -361,7 +377,7 @@ def build_search_request(engine_type: str, class WebSearchTool(ToolBase): """ Unified web search tool for agents. It can search the web and fetch page content. - - Search via multiple engines (Exa, SerpAPI, Arxiv, Tavily) + - Search via multiple engines (Exa, SerpAPI, Arxiv, Tavily, Outcrawl) - Dynamic tool definitions based on configured engines - Auto-fetch and parse page content - Configurable content fetcher (jina_reader, docling, etc.) @@ -372,7 +388,7 @@ class WebSearchTool(ToolBase): # Single engine mode: tools: web_search: - engine: exa # or 'serpapi', 'arxiv' + engine: exa # or 'serpapi', 'arxiv', 'tavily', 'outcrawl' api_key: xxxxxxxx fetcher: jina_reader fetch_content: true @@ -386,6 +402,7 @@ class WebSearchTool(ToolBase): - arxiv # Provides arxiv_search tool exa_api_key: $EXA_API_KEY serpapi_api_key: $SERPAPI_API_KEY + outcrawl_api_key: $OUTCRAWL_API_KEY fetch_content: true # Optional: asyncio deadline per URL (omit = auto from fetch_timeout/retries). # per_url_fetch_timeout: 0 @@ -394,7 +411,7 @@ class WebSearchTool(ToolBase): SERVER_NAME = 'web_search' # Registry of supported search engines - SUPPORTED_ENGINES = ('exa', 'serpapi', 'arxiv', 'tavily') + SUPPORTED_ENGINES = ('exa', 'serpapi', 'arxiv', 'tavily', 'outcrawl') # Process-wide (class-level) usage tracking for summarization calls. # This is intentionally separate from LLMAgent usage totals. @@ -510,6 +527,9 @@ def __init__(self, config, **kwargs): 'tavily': (getattr(tool_cfg, 'tavily_api_key', None) or os.getenv('TAVILY_API_KEY')) if tool_cfg else os.getenv('TAVILY_API_KEY'), + 'outcrawl': (getattr(tool_cfg, 'outcrawl_api_key', None) + or os.getenv('OUTCRAWL_API_KEY')) + if tool_cfg else os.getenv('OUTCRAWL_API_KEY'), } # Tavily search defaults from optional `tavily:` sub-block in YAML @@ -532,6 +552,10 @@ def __init__(self, config, **kwargs): getattr(tool_cfg, 'tavily_request_timeout', 120.0) or 120.0) if tool_cfg else 120.0 + self._outcrawl_request_timeout = float( + getattr(tool_cfg, 'outcrawl_request_timeout', 60.0) + or 60.0) if tool_cfg else 60.0 + # SerpApi provider (google, bing, baidu) self._serpapi_provider = getattr(tool_cfg, 'serpapi_provider', 'google') if tool_cfg else 'google' @@ -767,6 +791,11 @@ async def connect(self) -> None: api_key=self._api_keys.get('tavily'), request_timeout=self._tavily_request_timeout, ) + elif engine_type == 'outcrawl': + self._engines[engine_type] = engine_cls( + api_key=self._api_keys.get('outcrawl'), + request_timeout=self._outcrawl_request_timeout, + ) else: # arxiv self._engines[engine_type] = engine_cls() @@ -783,6 +812,7 @@ async def connect(self) -> None: 'timeout': self._fetch_timeout, 'retries': self._fetch_retries, 'tavily_api_key': self._api_keys.get('tavily'), + 'outcrawl_api_key': self._api_keys.get('outcrawl'), 'direct_fetch_fallback': self._jina_direct_fetch_fallback, 'direct_fetch_timeout': self._jina_direct_fetch_timeout, 'playwright_fetch_fallback': self._jina_playwright_fetch_fallback, @@ -1629,3 +1659,7 @@ async def serpapi_search(self, **kwargs) -> str: async def tavily_search(self, **kwargs) -> str: """Search using Tavily engine.""" return await self._execute_search('tavily', kwargs) + + async def outcrawl_search(self, **kwargs) -> str: + """Search using Outcrawl engine.""" + return await self._execute_search('outcrawl', kwargs) diff --git a/tests/capability/test_gateway.py b/tests/capability/test_gateway.py index 8c4438adf..864cecc0c 100644 --- a/tests/capability/test_gateway.py +++ b/tests/capability/test_gateway.py @@ -2,7 +2,7 @@ """Capability Gateway tests — registry, --check, MCP list_tools, real tools. Heavy / long-running project capabilities are not exercised end-to-end here. -Keyed engines (exa/tavily) are skipped when the corresponding env var is absent. +Keyed engines (exa/tavily/outcrawl) are skipped when the corresponding env var is absent. """ from __future__ import annotations @@ -198,6 +198,21 @@ async def _run(): self.assertIn('error', result) self.assertIn('TAVILY_API_KEY', result['error']) + def test_outcrawl_missing_key_actionable(self): + if os.environ.get('OUTCRAWL_API_KEY'): + self.skipTest('OUTCRAWL_API_KEY is set') + registry = create_registry() + + async def _run(): + return await registry.invoke( + 'web_search', + {'query': 'test', 'engine_type': 'outcrawl', 'num_results': 1}, + ) + + result = asyncio.run(_run()) + self.assertIn('error', result) + self.assertIn('OUTCRAWL_API_KEY', result['error']) + @pytest.mark.asyncio async def test_mcp_list_tools_count(): diff --git a/tests/search/test_outcrawl.py b/tests/search/test_outcrawl.py new file mode 100644 index 000000000..b92f41c81 --- /dev/null +++ b/tests/search/test_outcrawl.py @@ -0,0 +1,313 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Outcrawl search engine and scrape fetcher. + +Offline tests patch the HTTP layer. The two ``*_real`` tests call +api.outcrawl.ai and are skipped unless OUTCRAWL_API_KEY is set. +""" +import json +import os +from io import BytesIO +from urllib.error import HTTPError + +import pytest +from ms_agent.tools.search.outcrawl import http as outcrawl_http +from ms_agent.tools.search.outcrawl.fetcher import OutcrawlScrapeFetcher +from ms_agent.tools.search.outcrawl.schema import (OutcrawlSearchRequest, + OutcrawlSearchResult) +from ms_agent.tools.search.outcrawl.search import OutcrawlSearch +from ms_agent.tools.search.search_base import (ENGINE_TOOL_NAMES, + SearchEngineType) +from ms_agent.tools.search.websearch_tool import (WebSearchTool, + get_content_fetcher, + get_search_engine, + get_search_engine_class) + +SEARCH_RESPONSE = { + 'hits': [ + { + 'url': 'https://example.com/a', + 'title': 'A', + 'snippet': 'first', + 'rank': 1 + }, + { + 'url': 'https://example.com/b', + 'title': 'B', + 'snippet': 'second', + 'rank': 2, + 'markdown': '# B page' + }, + ], + 'usage': { + 'credits': '1' + }, +} + +SCRAPE_RESPONSE = { + 'url': 'https://example.com/', + 'statusCode': 200, + 'markdown': '# Example Domain', + 'metadata': { + 'title': 'Example Domain', + 'language': 'en' + }, + 'usage': { + 'credits': '1' + }, +} + +AUTH_BODY = { + 'code': 'unauthorized', + 'message': 'authentication failed: no api key presented', + 'details': { + 'reason': 'missing_token' + }, +} + + +class _Hdrs(dict): + + def get(self, key, default=None): + return dict.get(self, key.lower(), default) + + +def _raise_http(status, body, headers=None): + + def boom(*args, **kwargs): + raise HTTPError('https://api.outcrawl.ai/v1/search', status, 'err', + _Hdrs(headers or {}), + BytesIO(json.dumps(body).encode())) + + return boom + + +@pytest.fixture() +def no_env_key(monkeypatch): + monkeypatch.delenv('OUTCRAWL_API_KEY', raising=False) + + +# ------------------------------------------------------------ registration + + +def test_engine_is_registered(): + assert SearchEngineType.OUTCRAWL.value == 'outcrawl' + assert ENGINE_TOOL_NAMES['outcrawl'] == 'outcrawl_search' + assert 'outcrawl' in WebSearchTool.SUPPORTED_ENGINES + assert get_search_engine_class('outcrawl') is OutcrawlSearch + assert OutcrawlSearch.get_tool_name() == 'outcrawl_search' + + +def test_factories_pass_the_key(no_env_key): + engine = get_search_engine('outcrawl', api_key='oc-K') + assert isinstance(engine, OutcrawlSearch) + fetcher = get_content_fetcher('outcrawl_scrape', outcrawl_api_key='oc-K') + assert isinstance(fetcher, OutcrawlScrapeFetcher) + + +def test_missing_key_raises(no_env_key): + with pytest.raises(ValueError, match='OUTCRAWL_API_KEY'): + OutcrawlSearch() + + +def test_env_key_is_used(monkeypatch): + monkeypatch.setenv('OUTCRAWL_API_KEY', 'oc-FROM-ENV') + assert OutcrawlSearch()._api_key == 'oc-FROM-ENV' + + +# --------------------------------------------------------------- request + + +def test_request_body_shape(): + assert OutcrawlSearchRequest(query='q', limit=3).to_api_body() == { + 'query': 'q', + 'limit': 3 + } + body = OutcrawlSearchRequest(query='q', limit=50, scrape=True).to_api_body() + assert body == { + 'query': 'q', + 'limit': 20, + 'scrape': True, + 'formats': ['markdown'] + } + + +def test_build_request_from_tool_args(): + req = OutcrawlSearch.build_request_from_args( + query='q', num_results='4', scrape='true') + assert (req.query, req.limit, req.scrape) == ('q', 4, True) + assert OutcrawlSearch.build_request_from_args(query='q').limit == 5 + + +def test_tool_definition(): + tool = OutcrawlSearch.get_tool_definition() + assert tool['tool_name'] == 'outcrawl_search' + assert tool['parameters']['required'] == ['query'] + assert 'scrape' in tool['parameters']['properties'] + + +# ---------------------------------------------------------------- search + + +def test_search_posts_with_bearer_and_parses_hits(no_env_key, monkeypatch): + seen = {} + + def fake_post(path, body, *, api_key, timeout=60.0): + seen.update(path=path, body=body, api_key=api_key) + return SEARCH_RESPONSE + + monkeypatch.setattr('ms_agent.tools.search.outcrawl.search.post_json', + fake_post) + result = OutcrawlSearch(api_key='oc-K').search( + OutcrawlSearchRequest(query='q', limit=2)) + assert seen == { + 'path': '/search', + 'body': { + 'query': 'q', + 'limit': 2 + }, + 'api_key': 'oc-K' + } + rows = result.to_list() + assert [r['url'] for r in rows] == [ + 'https://example.com/a', 'https://example.com/b' + ] + assert rows[0]['summary'] == 'first' + assert rows[0]['fetch_success'] is False + # Scraped hits prefill content so WebSearchTool skips the fetch step. + assert rows[1]['content'] == '# B page' + assert rows[1]['fetch_success'] is True + + +def test_empty_response_gives_no_rows(): + assert OutcrawlSearchResult(query='q', arguments={}, + response={}).to_list() == [] + + +def test_http_sends_authorization_header(monkeypatch): + seen = {} + + class _Resp: + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def read(self): + return json.dumps(SEARCH_RESPONSE).encode() + + def fake_urlopen(req, timeout=None, context=None): + seen['url'] = req.full_url + seen['auth'] = req.get_header('Authorization') + seen['user_agent'] = req.get_header('User-agent') + seen['body'] = json.loads(req.data) + return _Resp() + + monkeypatch.setattr(outcrawl_http, 'urlopen', fake_urlopen) + data = outcrawl_http.post_json('/search', {'query': 'q'}, api_key='oc-K') + assert data == SEARCH_RESPONSE + assert seen == { + 'url': 'https://api.outcrawl.ai/v1/search', + 'auth': 'Bearer oc-K', + 'user_agent': 'ms-agent', + 'body': { + 'query': 'q' + } + } + + +# ------------------------------------------------------- error handling + + +def test_auth_error_is_parsed(monkeypatch): + monkeypatch.setattr(outcrawl_http, 'urlopen', _raise_http(401, AUTH_BODY)) + with pytest.raises(outcrawl_http.OutcrawlHTTPError) as ctx: + outcrawl_http.post_json('/search', {'query': 'q'}, api_key='bad') + err = ctx.value + assert err.status == 401 + assert err.code == 'unauthorized' + assert err.is_auth and not err.is_quota + p = WebSearchTool._search_error_payload('outcrawl', err) + assert p['kind'] == 'auth_failed' + + +def test_rate_limit_is_quota_with_retry_after(monkeypatch): + monkeypatch.setattr( + outcrawl_http, 'urlopen', + _raise_http(429, {'code': 'rate_limited'}, {'retry-after': '30'})) + with pytest.raises(outcrawl_http.OutcrawlHTTPError) as ctx: + outcrawl_http.post_json('/search', {'query': 'q'}, api_key='oc-K') + assert ctx.value.is_quota + assert ctx.value.retry_after == 30 + p = WebSearchTool._search_error_payload('outcrawl', ctx.value) + assert p['kind'] == 'quota_exceeded' + + +# --------------------------------------------------------------- fetcher + + +def test_fetcher_returns_markdown_and_metadata(monkeypatch): + seen = {} + + def fake_post(path, body, *, api_key, timeout=60.0): + seen.update(path=path, body=body) + return SCRAPE_RESPONSE + + monkeypatch.setattr('ms_agent.tools.search.outcrawl.fetcher.post_json', + fake_post) + text, meta = OutcrawlScrapeFetcher(api_key='oc-K').fetch( + 'https://example.com/') + assert seen == { + 'path': '/scrape', + 'body': { + 'url': 'https://example.com/', + 'formats': ['markdown'], + 'onlyMainContent': True + } + } + assert text == '# Example Domain' + assert meta['fetcher'] == 'outcrawl_scrape' + assert meta['status_code'] == 200 + assert meta['title'] == 'Example Domain' + assert 'error' not in meta + + +def test_fetcher_reports_errors_instead_of_raising(monkeypatch): + monkeypatch.setattr(outcrawl_http, 'urlopen', _raise_http(401, AUTH_BODY)) + text, meta = OutcrawlScrapeFetcher(api_key='bad').fetch( + 'https://example.com/') + assert text == '' + assert 'Outcrawl HTTP 401' in meta['error'] + + +def test_fetcher_without_key_reports_error(no_env_key): + text, meta = OutcrawlScrapeFetcher().fetch('https://example.com/') + assert text == '' + assert 'OUTCRAWL_API_KEY' in meta['error'] + + +# ------------------------------------------------------- real network + + +@pytest.mark.skipif( + not os.getenv('OUTCRAWL_API_KEY'), + reason='OUTCRAWL_API_KEY is required for Outcrawl real-network test.') +def test_outcrawl_search_real(): + engine = get_search_engine('outcrawl') + request = OutcrawlSearch.build_request_from_args( + query='ModelScope MS-Agent', num_results=3) + rows = engine.search(request).to_list() + assert rows + assert all(r['url'].startswith('http') for r in rows) + + +@pytest.mark.skipif( + not os.getenv('OUTCRAWL_API_KEY'), + reason='OUTCRAWL_API_KEY is required for Outcrawl real-network test.') +def test_outcrawl_scrape_real(): + fetcher = get_content_fetcher('outcrawl_scrape') + text, meta = fetcher.fetch('https://example.com/') + assert 'documentation examples' in text + assert meta['status_code'] == 200 + assert meta['title'] == 'Example Domain'