From 36c3087b9581c4e94373d006952a0643dc697f4c Mon Sep 17 00:00:00 2001
From: Tavily PR Agent <tavily-pr-agent@noreply.com>
Date: Tue, 14 Jul 2026 12:04:43 +0000
Subject: [PATCH] feat: add Tavily Extract as parallel WebFetch backend

---
 .env.example                 |  2 ++
 agent_base/tools/tool_web.py | 52 +++++++++++++++++++++++++++++++-----
 requirements.txt             |  1 +
 3 files changed, 48 insertions(+), 7 deletions(-)

diff --git a/.env.example b/.env.example
index 170dd4f..245d1bf 100644
--- a/.env.example
+++ b/.env.example
@@ -31,3 +31,5 @@ DEBUG_AGENT=false                                        # Print verbose agent-l
 DEBUG_SEARCH=false                                       # Print verbose WebSearch debug logs.
 DEBUG_SCHOLAR=false                                      # Print verbose ScholarSearch debug logs.
 DEBUG_VISIT=false                                        # Print verbose WebFetch debug logs.
+WEBFETCH_PROVIDER=jina                                   # WebFetch backend: "jina" (default) or "tavily".
+TAVILY_API_KEY="your_tavily_key"                         # https://tavily.com/ — required when WEBFETCH_PROVIDER=tavily.
diff --git a/agent_base/tools/tool_web.py b/agent_base/tools/tool_web.py
index 8496e7d..ddb903f 100644
--- a/agent_base/tools/tool_web.py
+++ b/agent_base/tools/tool_web.py
@@ -355,13 +355,22 @@ def call(self, params: Union[str, dict], **kwargs) -> str:
         except ValueError as exc:
             return f"[WebFetch] {exc}"
 
-        response = self.readpage_jina(
-            url,
-            start_line=start_line,
-            end_line=end_line,
-            max_chars=max_chars,
-            runtime_deadline=runtime_deadline,
-        )
+        provider = os.getenv("WEBFETCH_PROVIDER", "jina").strip().lower()
+        if provider == "tavily":
+            response = self.readpage_tavily(
+                url,
+                start_line=start_line,
+                end_line=end_line,
+                max_chars=max_chars,
+            )
+        else:
+            response = self.readpage_jina(
+                url,
+                start_line=start_line,
+                end_line=end_line,
+                max_chars=max_chars,
+                runtime_deadline=runtime_deadline,
+            )
 
         if visit_debug_enabled():
             print(f"WebFetch Length {len(response)}")
@@ -435,6 +444,35 @@ def readpage_jina(
             max_chars=max_chars,
         )
 
+    def readpage_tavily(
+        self,
+        url: str,
+        *,
+        start_line: int = 1,
+        end_line: Optional[int] = None,
+        max_chars: int = DEFAULT_WEBFETCH_MAX_CHARS,
+    ) -> str:
+        tavily_api_key = os.getenv("TAVILY_API_KEY", "").strip()
+        if not tavily_api_key:
+            return "[WebFetch] TAVILY_API_KEY is not set."
+        try:
+            from tavily import TavilyClient
+            client = TavilyClient(api_key=tavily_api_key)
+            result = client.extract(urls=[url])
+        except Exception as exc:
+            return f"[WebFetch] Failed to read page: {exc}"
+        results = result.get("results", [])
+        if not results or not results[0].get("raw_content"):
+            return "[WebFetch] Failed to read page: the provided webpage content could not be accessed. Please check the URL or file format."
+        content = results[0]["raw_content"]
+        return self._format_page_content(
+            url=url,
+            content=content,
+            start_line=start_line,
+            end_line=end_line,
+            max_chars=max_chars,
+        )
+
 
 def main(argv: Optional[list[str]] = None) -> int:
     parser = argparse.ArgumentParser(description="Run web tools directly.")
diff --git a/requirements.txt b/requirements.txt
index 214730c..1b3c15f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,5 +4,6 @@ openai>=2.3.0
 Pillow>=11.3.0
 requests>=2.32.5
 structai>=0.1.23
+tavily-python>=0.5.0
 tiktoken>=0.12.0
 uvicorn>=0.34.0