feat(search): add Outcrawl search engine and scrape fetcher

examples/capability/setup.md

@@ -162,6 +162,7 @@ Since all loads are `override=False`, the **effective priority** (highest to low
 | `EXA_API_KEY`         | web_search (exa engine)                          | Only needed for `engine_type='exa'`                  |
 | `SERPAPI_API_KEY`     | web_search (serpapi / google / bing / baidu)     | Only needed for SerpAPI-backed engines               |
 | `TAVILY_API_KEY`      | web_search (tavily engine)                       | Only needed for `engine_type='tavily'`               |
+| `OUTCRAWL_API_KEY`    | web_search (outcrawl engine)                     | Only needed for `engine_type='outcrawl'`             |
 | `MS_AGENT_OUTPUT_DIR` | replace_file_*, lsp_check_*                      | Workspace root for file operations (defaults to cwd) |
 
 

ms-agent-skills/SKILL.md

@@ -4,7 +4,7 @@ version: 1.0.0
 description: >-
   Access ms-agent's advanced AI capabilities via MCP tools: deep research,
   document research, financial research, code generation, video generation,
-  web search (arxiv/exa/serpapi/tavily), LSP code validation (TypeScript/Python/Java),
+  web search (arxiv/exa/serpapi/tavily/outcrawl), LSP code validation (TypeScript/Python/Java),
   concurrent-safe file editing, and agent delegation. All project-level
   capabilities support async submit/check/get patterns. Use when the user
   asks to research a topic, analyze documents, generate code or videos,
@@ -207,7 +207,7 @@ User wants to...
 │   └── submit_fin_research_task → check_fin_research_progress → get_fin_research_report
 │
 ├── Search the web for quick info
-│   └── web_search(query="...", engine_type="arxiv|exa|serpapi|tavily")
+│   └── web_search(query="...", engine_type="arxiv|exa|serpapi|tavily|outcrawl")
 │
 ├── Generate a software project from requirements (10-30 min)
 │   └── submit_code_genesis_task → check_code_genesis_progress → get_code_genesis_result

ms-agent-skills/references/web-search.md

@@ -33,6 +33,7 @@ search engines and optional full-page content fetching.
 | `exa` | — | General web, semantic search | `EXA_API_KEY` or `EXA_API_KEYS` (comma-separated key pool) |
 | `serpapi` | `serp`, `google`, `bing`, `baidu` | Google/Bing/Baidu results via SerpAPI | `SERPAPI_API_KEY` |
 | `tavily` | — | AI-oriented web search | `TAVILY_API_KEY` |
+| `outcrawl` | - | Web search and scrape API for AI agents | `OUTCRAWL_API_KEY` |
 
 ### Examples
 
@@ -72,7 +73,7 @@ field with the page text (truncated to 10,000 characters).
 ### Step 1: Choose the Right Engine
 
 - For academic/research questions → `arxiv`
-- For general web info → `exa`, `tavily`, or `serpapi` (and aliases)
+- For general web info → `exa`, `tavily`, `outcrawl`, or `serpapi` (and aliases)
 - If unsure → start with `arxiv`, fall back to `exa` or `tavily`
 
 ### Step 2: Execute the Search

ms_agent/capabilities/wrappers/web_search.py

@@ -24,6 +24,7 @@
     'bing': ['SERPAPI_API_KEY'],
     'baidu': ['SERPAPI_API_KEY'],
     'tavily': ['TAVILY_API_KEY'],
+    'outcrawl': ['OUTCRAWL_API_KEY'],
 }
 
 
@@ -50,14 +51,16 @@ def _get_fetcher() -> Any:
     granularity='tool',
     summary=(
         'Search the web using multiple engines '
-        '(arxiv / exa / serpapi / tavily, plus google/bing/baidu aliases) '
+        '(arxiv / exa / serpapi / tavily / outcrawl, plus google/bing/baidu '
+        'aliases) '
         'and optionally fetch full page content.'),
     description=(
         'Performs a web search and returns structured results including '
         'title, URL, and summary for each hit.  Supported engines: '
         'arxiv (default, no key), exa (EXA_API_KEY or EXA_API_KEYS), '
         'serpapi / serp / '
-        'google / bing / baidu (SERPAPI_API_KEY), tavily (TAVILY_API_KEY). '
+        'google / bing / baidu (SERPAPI_API_KEY), tavily (TAVILY_API_KEY), '
+        'outcrawl (OUTCRAWL_API_KEY). '
         'Set fetch_content=true to additionally retrieve and return page '
         'text (truncated to 10 000 chars).'),
     input_schema={
@@ -77,7 +80,8 @@ def _get_fetcher() -> Any:
                 'string',
                 'description':
                 ("Search engine: 'arxiv' (default, no key), 'exa', "
-                 "'serpapi'/'serp'/'google'/'bing'/'baidu', or 'tavily'"),
+                 "'serpapi'/'serp'/'google'/'bing'/'baidu', 'tavily' or "
+                 "'outcrawl'"),
                 'default':
                 'arxiv',
             },

ms_agent/tools/search/outcrawl/__init__.py

@@ -0,0 +1,4 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+from ms_agent.tools.search.outcrawl.search import OutcrawlSearch
+
+__all__ = ['OutcrawlSearch']

ms_agent/tools/search/outcrawl/fetcher.py

@@ -0,0 +1,55 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+"""Outcrawl Scrape API as ContentFetcher (for fetch_page / URL fetch)."""
+import os
+import time
+from typing import Any, Dict, Optional, Tuple
+
+from ms_agent.tools.search.outcrawl.http import post_json
+from ms_agent.utils.logger import get_logger
+
+logger = get_logger()
+
+
+class OutcrawlScrapeFetcher:
+    """Fetch page text as markdown via Outcrawl POST /v1/scrape."""
+
+    def __init__(self,
+                 api_key: Optional[str] = None,
+                 *,
+                 only_main_content: bool = True,
+                 timeout: float = 45.0):
+        self._api_key = api_key or os.getenv('OUTCRAWL_API_KEY') or ''
+        self._only_main_content = bool(only_main_content)
+        self._timeout = max(1.0, float(timeout))
+
+    def fetch(self, url: str) -> Tuple[str, Dict[str, Any]]:
+        meta: Dict[str, Any] = {
+            'fetcher': 'outcrawl_scrape',
+            'fetched_at': time.strftime('%Y-%m-%dT%H:%M:%S'),
+        }
+        if not self._api_key:
+            meta['error'] = 'OUTCRAWL_API_KEY is not set'
+            return '', meta
+        body = {
+            'url': url,
+            'formats': ['markdown'],
+            'onlyMainContent': self._only_main_content,
+        }
+        try:
+            data = post_json(
+                '/scrape', body, api_key=self._api_key, timeout=self._timeout)
+        except Exception as e:
+            logger.warning(f'Outcrawl scrape failed for {url[:80]}: {e}')
+            meta['error'] = str(e)
+            return '', meta
+
+        text = (data.get('markdown') or '').strip()
+        metadata = data.get('metadata') or {}
+        meta.update({
+            'status_code': data.get('statusCode'),
+            'title': metadata.get('title'),
+            'outcrawl_usage': data.get('usage'),
+        })
+        if not text:
+            meta['error'] = 'Outcrawl returned no markdown'
+        return text, meta

ms_agent/tools/search/outcrawl/http.py

@@ -0,0 +1,116 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+"""Minimal HTTP JSON client for the Outcrawl REST API (stdlib only)."""
+import json
+from typing import Any, Dict, Optional
+from urllib.error import HTTPError, URLError
+from urllib.request import Request, urlopen
+
+OUTCRAWL_API_BASE = 'https://api.outcrawl.ai/v1'
+
+
+class OutcrawlHTTPError(RuntimeError):
+    """An Outcrawl call that failed, with the fields a caller can act on.
+
+    ``status`` is the HTTP code (None for transport failures), ``code`` is
+    Outcrawl's machine-readable ``code`` (e.g. ``unauthorized``) and
+    ``retry_after`` is set when the response carried a Retry-After header.
+    The ``is_quota`` / ``is_auth`` properties match the ones
+    ``WebSearchTool._search_error_payload`` reads.
+    """
+
+    def __init__(self,
+                 message: str,
+                 *,
+                 status: Optional[int] = None,
+                 code: str = '',
+                 retry_after: Optional[int] = None,
+                 detail: Any = None):
+        super().__init__(message)
+        self.status = status
+        self.code = code
+        self.retry_after = retry_after
+        self.detail = detail
+
+    @property
+    def is_quota(self) -> bool:
+        """Rate limited or out of credits."""
+        return self.status in (402, 429)
+
+    @property
+    def is_auth(self) -> bool:
+        return self.status in (401, 403)
+
+
+def _ssl_context():
+    """A verifying TLS context backed by certifi when it is installed."""
+    try:
+        import certifi
+        import ssl
+        return ssl.create_default_context(cafile=certifi.where())
+    except Exception:
+        return None
+
+
+def post_json(path: str,
+              body: Dict[str, Any],
+              *,
+              api_key: str,
+              timeout: float = 60.0) -> Dict[str, Any]:
+    """
+    POST JSON to ``OUTCRAWL_API_BASE + path`` and parse the JSON response.
+
+    Raises:
+        OutcrawlHTTPError: on HTTP errors, network errors or invalid JSON.
+    """
+    data = json.dumps(body, ensure_ascii=False).encode('utf-8')
+    req = Request(
+        f'{OUTCRAWL_API_BASE}{path}',
+        data=data,
+        method='POST',
+        headers={
+            'Authorization': f'Bearer {api_key}',
+            'Content-Type': 'application/json',
+            'Accept': 'application/json',
+            # The default Python-urllib agent is refused at the API edge.
+            'User-Agent': 'ms-agent',
+        })
+    try:
+        with urlopen(req, timeout=timeout, context=_ssl_context()) as resp:
+            raw = resp.read().decode('utf-8', errors='replace')
+    except HTTPError as e:
+        err_body = ''
+        try:
+            err_body = e.read().decode('utf-8', errors='replace')
+        except Exception:
+            pass
+        try:
+            detail = json.loads(err_body) if err_body else {}
+        except json.JSONDecodeError:
+            detail = {'raw': err_body}
+        code, message = '', ''
+        if isinstance(detail, dict):
+            code = str(detail.get('code') or '')
+            message = str(detail.get('message') or '')
+        retry_after = None
+        try:
+            header_value = e.headers.get('retry-after')
+            if header_value:
+                retry_after = int(float(header_value))
+        except (AttributeError, TypeError, ValueError):
+            retry_after = None
+        raise OutcrawlHTTPError(
+            f'Outcrawl HTTP {e.code}: {message or detail}',
+            status=e.code,
+            code=code,
+            retry_after=retry_after,
+            detail=detail) from e
+    except URLError as e:
+        raise OutcrawlHTTPError(f'Outcrawl network error: {e}') from e
+
+    if not raw.strip():
+        return {}
+    try:
+        return json.loads(raw)
+    except json.JSONDecodeError as e:
+        raise OutcrawlHTTPError(
+            f'Outcrawl returned invalid JSON: {raw[:200]}') from e

ms_agent/tools/search/outcrawl/schema.py

@@ -0,0 +1,56 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+from dataclasses import dataclass
+from typing import Any, Dict, List
+
+
+@dataclass
+class OutcrawlSearchRequest:
+    """Outcrawl POST /v1/search body. See https://outcrawl.ai/docs/search/overview"""
+
+    query: str
+    limit: int = 5
+    # When true, Outcrawl also scrapes each hit and returns its markdown.
+    scrape: bool = False
+
+    def to_api_body(self) -> Dict[str, Any]:
+        body: Dict[str, Any] = {
+            'query': self.query,
+            'limit': max(1, min(20, int(self.limit))),
+        }
+        if self.scrape:
+            body['scrape'] = True
+            body['formats'] = ['markdown']
+        return body
+
+
+@dataclass
+class OutcrawlSearchResult:
+    """Parsed Outcrawl /v1/search JSON."""
+
+    query: str
+    arguments: Dict[str, Any]
+    response: Dict[str, Any]
+
+    def to_list(self) -> List[Dict[str, Any]]:
+        """Normalize to WebSearchTool pipeline dicts (prefill content when markdown is present)."""
+        if not self.response:
+            return []
+        rows: List[Dict[str, Any]] = []
+        for hit in self.response.get('hits') or []:
+            url = hit.get('url') or ''
+            snippet = (hit.get('snippet') or '').strip()
+            markdown = (hit.get('markdown') or '').strip()
+            rows.append({
+                'url': url,
+                'id': url,
+                'title': hit.get('title') or '',
+                'highlights': None,
+                'highlight_scores': None,
+                'summary': snippet,
+                'markdown': markdown or None,
+                # Pipeline uses these keys:
+                'content': markdown or snippet,
+                'fetch_success': bool(markdown),
+                'rank': hit.get('rank'),
+            })
+        return rows