fix(brightdata): align integration with live API docs, add markdown/mode/error-report support (#5470)

* fix(brightdata): align integration with live API docs, add markdown/mode/error-report support

- fix Discover numResults docs (max 20, not 1000) and contentFormat value ("md" not "markdown", which the API always rejected)
- add Discover mode param (standard/deep/fast/zeroRanking)
- add markdown output support to Web Unlocker scrape_url via data_format
- add include_errors support to scrape_dataset, matching sync_scrape
- add .trim() on datasetId in scrape_dataset/sync_scrape URLs
- add wandConfig on complex fields, tighten block output descriptions

* fix(brightdata): default mode field to empty so it's opt-in like other advanced params

Greptile flagged that mode shipped with a non-empty default ('standard'),
causing it to be sent on every Discover call unlike the sibling dataFormat
field which uses an empty/None default and only sends when the user opts in.
Aligns mode with that same pattern.

Author: waleedlatif1

apps/sim/blocks/blocks/brightdata.ts

@@ -59,6 +59,18 @@ export const BrightDataBlock: BlockConfig<BrightDataResponse> = {
       value: () => 'raw',
       condition: { field: 'operation', value: 'scrape_url' },
     },
+    {
+      id: 'dataFormat',
+      title: 'Convert To',
+      type: 'dropdown',
+      options: [
+        { label: 'None', id: '' },
+        { label: 'Markdown', id: 'markdown' },
+      ],
+      value: () => '',
+      mode: 'advanced',
+      condition: { field: 'operation', value: 'scrape_url' },
+    },
     {
       id: 'country',
       title: 'Country',
@@ -119,6 +131,26 @@ export const BrightDataBlock: BlockConfig<BrightDataResponse> = {
       placeholder:
         'Describe what you are looking for (e.g., "find official pricing pages and change notes")',
       condition: { field: 'operation', value: 'discover' },
+      wandConfig: {
+        enabled: true,
+        prompt:
+          'Generate a concise description of what the agent is trying to accomplish, to help rank web-discovery results by relevance (e.g., "find official pricing pages and recent change notes"). Return ONLY the intent description - no explanations, no extra text.',
+        placeholder: 'Describe what you are trying to accomplish...',
+      },
+    },
+    {
+      id: 'mode',
+      title: 'Search Mode',
+      type: 'dropdown',
+      options: [
+        { label: 'Standard (Default)', id: '' },
+        { label: 'Deep', id: 'deep' },
+        { label: 'Fast', id: 'fast' },
+        { label: 'Zero Ranking', id: 'zeroRanking' },
+      ],
+      value: () => '',
+      mode: 'advanced',
+      condition: { field: 'operation', value: 'discover' },
     },
     {
       id: 'includeContent',
@@ -133,7 +165,7 @@ export const BrightDataBlock: BlockConfig<BrightDataResponse> = {
       type: 'dropdown',
       options: [
         { label: 'JSON', id: 'json' },
-        { label: 'Markdown', id: 'markdown' },
+        { label: 'Markdown', id: 'md' },
       ],
       value: () => 'json',
       mode: 'advanced',
@@ -154,6 +186,12 @@ export const BrightDataBlock: BlockConfig<BrightDataResponse> = {
       placeholder: '[{"url": "https://example.com/product"}]',
       condition: { field: 'operation', value: 'sync_scrape' },
       required: { field: 'operation', value: 'sync_scrape' },
+      wandConfig: {
+        enabled: true,
+        prompt:
+          'Generate a JSON array of URL objects to scrape based on the user\'s description, in the form [{"url": "https://example.com/product"}]. Return ONLY the JSON array - no explanations, no extra text.',
+        placeholder: 'Describe the URLs to scrape...',
+      },
     },
     {
       id: 'syncFormat',
@@ -167,6 +205,13 @@ export const BrightDataBlock: BlockConfig<BrightDataResponse> = {
       value: () => 'json',
       condition: { field: 'operation', value: 'sync_scrape' },
     },
+    {
+      id: 'syncIncludeErrors',
+      title: 'Include Errors',
+      type: 'switch',
+      mode: 'advanced',
+      condition: { field: 'operation', value: 'sync_scrape' },
+    },
     {
       id: 'datasetId',
       title: 'Dataset ID',
@@ -182,6 +227,12 @@ export const BrightDataBlock: BlockConfig<BrightDataResponse> = {
       placeholder: '[{"url": "https://example.com/product"}]',
       condition: { field: 'operation', value: 'scrape_dataset' },
       required: { field: 'operation', value: 'scrape_dataset' },
+      wandConfig: {
+        enabled: true,
+        prompt:
+          'Generate a JSON array of URL objects to scrape based on the user\'s description, in the form [{"url": "https://example.com/product"}]. Return ONLY the JSON array - no explanations, no extra text.',
+        placeholder: 'Describe the URLs to scrape...',
+      },
     },
     {
       id: 'datasetFormat',
@@ -194,6 +245,13 @@ export const BrightDataBlock: BlockConfig<BrightDataResponse> = {
       value: () => 'json',
       condition: { field: 'operation', value: 'scrape_dataset' },
     },
+    {
+      id: 'datasetIncludeErrors',
+      title: 'Include Errors',
+      type: 'switch',
+      mode: 'advanced',
+      condition: { field: 'operation', value: 'scrape_dataset' },
+    },
     {
       id: 'snapshotId',
       title: 'Snapshot ID',
@@ -251,6 +309,7 @@ export const BrightDataBlock: BlockConfig<BrightDataResponse> = {
             result.url = params.url
             if (params.format) result.format = params.format
             if (params.country) result.country = params.country
+            if (params.dataFormat) result.dataFormat = params.dataFormat
             break
 
           case 'serp_search':
@@ -265,6 +324,7 @@ export const BrightDataBlock: BlockConfig<BrightDataResponse> = {
           case 'discover':
             result.query = params.discoverQuery
             if (params.numResults) result.numResults = Number(params.numResults)
+            if (params.mode) result.mode = params.mode
             if (params.intent) result.intent = params.intent
             if (params.includeContent != null) result.includeContent = params.includeContent
             if (params.contentFormat) result.format = params.contentFormat
@@ -276,12 +336,15 @@ export const BrightDataBlock: BlockConfig<BrightDataResponse> = {
             result.datasetId = params.syncDatasetId
             result.urls = params.syncUrls
             if (params.syncFormat) result.format = params.syncFormat
+            if (params.syncIncludeErrors != null) result.includeErrors = params.syncIncludeErrors
             break
 
           case 'scrape_dataset':
             result.datasetId = params.datasetId
             result.urls = params.urls
             if (params.datasetFormat) result.format = params.datasetFormat
+            if (params.datasetIncludeErrors != null)
+              result.includeErrors = params.datasetIncludeErrors
             break
 
           case 'snapshot_status':
@@ -308,33 +371,50 @@ export const BrightDataBlock: BlockConfig<BrightDataResponse> = {
     zone: { type: 'string', description: 'Bright Data zone name' },
     url: { type: 'string', description: 'URL to scrape' },
     format: { type: 'string', description: 'Response format' },
+    dataFormat: { type: 'string', description: 'Convert scraped content to markdown' },
     country: { type: 'string', description: 'Country code for geo-targeting' },
     query: { type: 'string', description: 'Search query' },
     searchEngine: { type: 'string', description: 'Search engine to use' },
     language: { type: 'string', description: 'Language code' },
     numResults: { type: 'number', description: 'Number of results' },
     discoverQuery: { type: 'string', description: 'Discover search query' },
     intent: { type: 'string', description: 'Intent for ranking results' },
+    mode: { type: 'string', description: 'Search depth and ranking mode for discover' },
     includeContent: { type: 'boolean', description: 'Include page content in discover results' },
     contentFormat: { type: 'string', description: 'Content format for discover results' },
     syncDatasetId: { type: 'string', description: 'Dataset scraper ID for sync scrape' },
     syncUrls: { type: 'string', description: 'JSON array of URL objects for sync scrape' },
     syncFormat: { type: 'string', description: 'Output format for sync scrape' },
+    syncIncludeErrors: {
+      type: 'boolean',
+      description: 'Include error reports in sync scrape results',
+    },
     datasetId: { type: 'string', description: 'Dataset scraper ID' },
     urls: { type: 'string', description: 'JSON array of URL objects to scrape' },
     datasetFormat: { type: 'string', description: 'Dataset output format' },
+    datasetIncludeErrors: {
+      type: 'boolean',
+      description: 'Include error reports in scrape dataset results',
+    },
     snapshotId: { type: 'string', description: 'Snapshot ID for status/download/cancel' },
     downloadFormat: { type: 'string', description: 'Download output format' },
   },
   outputs: {
     content: { type: 'string', description: 'Scraped page content' },
     url: { type: 'string', description: 'URL that was scraped' },
     statusCode: { type: 'number', description: 'HTTP status code' },
-    results: { type: 'json', description: 'Search or discover results array' },
+    results: {
+      type: 'json',
+      description:
+        'Search or discover results array: [{title, url, description, rank}] for SERP search, or [{url, title, description, relevanceScore, content}] for discover',
+    },
     query: { type: 'string', description: 'Search query executed' },
     searchEngine: { type: 'string', description: 'Search engine used' },
     totalResults: { type: 'number', description: 'Total number of discover results' },
-    data: { type: 'json', description: 'Scraped data records' },
+    data: {
+      type: 'json',
+      description: 'Array of scraped result records with dataset-specific fields',
+    },
     snapshotId: { type: 'string', description: 'Snapshot ID' },
     isAsync: { type: 'boolean', description: 'Whether sync scrape fell back to async' },
     status: { type: 'string', description: 'Job status' },

apps/sim/tools/brightdata/discover.ts

@@ -17,7 +17,7 @@ export const brightDataDiscoverTool: ToolConfig<
   id: 'brightdata_discover',
   name: 'Bright Data Discover',
   description:
-    'AI-powered web discovery that finds and ranks results by intent. Returns up to 1,000 results with optional cleaned page content for RAG and verification.',
+    'AI-powered web discovery that finds and ranks results by intent. Returns up to 20 results with optional cleaned page content for RAG and verification.',
   version: '1.0.0',
 
   params: {
@@ -37,7 +37,14 @@ export const brightDataDiscoverTool: ToolConfig<
       type: 'number',
       required: false,
       visibility: 'user-or-llm',
-      description: 'Number of results to return, up to 1000. Defaults to 10',
+      description: 'Number of results to return (1-20). Defaults to 10',
+    },
+    mode: {
+      type: 'string',
+      required: false,
+      visibility: 'user-or-llm',
+      description:
+        'Search depth and ranking mode: "standard" (balanced), "deep" (exhaustive, broader search), "fast" (optimized for speed), or "zeroRanking" (raw volume without AI filtering). Defaults to "standard"',
     },
     intent: {
       type: 'string',
@@ -56,7 +63,7 @@ export const brightDataDiscoverTool: ToolConfig<
       type: 'string',
       required: false,
       visibility: 'user-or-llm',
-      description: 'Response format: "json" or "markdown". Defaults to "json"',
+      description: 'Response format: "json" or "md". Defaults to "json"',
     },
     language: {
       type: 'string',
@@ -84,6 +91,7 @@ export const brightDataDiscoverTool: ToolConfig<
         query: params.query,
       }
       if (params.numResults) body.num_results = params.numResults
+      if (params.mode) body.mode = params.mode
       if (params.intent) body.intent = params.intent
       if (params.includeContent != null) body.include_content = params.includeContent
       if (params.format) body.format = params.format

apps/sim/tools/brightdata/scrape_dataset.ts

@@ -41,14 +41,21 @@ export const brightDataScrapeDatasetTool: ToolConfig<
       visibility: 'user-or-llm',
       description: 'Output format: "json" or "csv". Defaults to "json"',
     },
+    includeErrors: {
+      type: 'boolean',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'Whether to include a per-input error report in the results',
+    },
   },
 
   request: {
     method: 'POST',
     url: (params) => {
       const queryParams = new URLSearchParams()
-      queryParams.set('dataset_id', params.datasetId)
+      queryParams.set('dataset_id', params.datasetId.trim())
       queryParams.set('format', params.format || 'json')
+      if (params.includeErrors) queryParams.set('include_errors', 'true')
       return `https://api.brightdata.com/datasets/v3/trigger?${queryParams.toString()}`
     },
     headers: (params) => ({

apps/sim/tools/brightdata/scrape_url.ts

@@ -46,6 +46,13 @@ export const brightDataScrapeUrlTool: ToolConfig<
       visibility: 'user-or-llm',
       description: 'Two-letter country code for geo-targeting (e.g., "us", "gb")',
     },
+    dataFormat: {
+      type: 'string',
+      required: false,
+      visibility: 'user-or-llm',
+      description:
+        'Convert the response to "markdown" instead of raw HTML, useful for feeding page content to an LLM. Omit for the default HTML/JSON response',
+    },
   },
 
   request: {
@@ -62,6 +69,7 @@ export const brightDataScrapeUrlTool: ToolConfig<
         format: params.format || 'raw',
       }
       if (params.country) body.country = params.country
+      if (params.dataFormat) body.data_format = params.dataFormat
       return body
     },
   },

apps/sim/tools/brightdata/sync_scrape.ts

@@ -53,7 +53,7 @@ export const brightDataSyncScrapeTool: ToolConfig<
     method: 'POST',
     url: (params) => {
       const queryParams = new URLSearchParams()
-      queryParams.set('dataset_id', params.datasetId)
+      queryParams.set('dataset_id', params.datasetId.trim())
       queryParams.set('format', params.format || 'json')
       if (params.includeErrors) queryParams.set('include_errors', 'true')
       return `https://api.brightdata.com/datasets/v3/scrape?${queryParams.toString()}`

apps/sim/tools/brightdata/types.ts

@@ -6,6 +6,7 @@ export interface BrightDataScrapeUrlParams {
   url: string
   format?: string
   country?: string
+  dataFormat?: string
 }
 
 export interface BrightDataScrapeUrlResponse extends ToolResponse {
@@ -44,6 +45,7 @@ export interface BrightDataScrapeDatasetParams {
   datasetId: string
   urls: string
   format?: string
+  includeErrors?: boolean
 }
 
 export interface BrightDataScrapeDatasetResponse extends ToolResponse {
@@ -113,6 +115,7 @@ export interface BrightDataDiscoverParams {
   apiKey: string
   query: string
   numResults?: number
+  mode?: string
   intent?: string
   includeContent?: boolean
   format?: string