diff --git a/package.json b/package.json index d09c62a..1b8379b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@brightdata/cli", - "version": "0.3.6", + "version": "0.3.7", "description": "Command-line interface for Bright Data. Scrape, search, extract structured data, and automate browsers directly from your terminal.", "main": "dist/index.js", "bin": { @@ -58,6 +58,8 @@ "@clack/prompts": "^1.1.0", "@inquirer/prompts": "^8.2.1", "commander": "^14.0.2", + "csv-parse": "^7.0.2", + "csv-stringify": "^6.8.3", "open": "^11.0.0", "picocolors": "^1.1.1", "playwright-core": "^1.58.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 767c86a..d9bfac4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -17,6 +17,12 @@ importers: commander: specifier: ^14.0.2 version: 14.0.3 + csv-parse: + specifier: ^7.0.2 + version: 7.0.2 + csv-stringify: + specifier: ^6.8.3 + version: 6.8.3 open: specifier: ^11.0.0 version: 11.0.0 @@ -533,6 +539,12 @@ packages: resolution: {integrity: sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==} engines: {node: '>=20'} + csv-parse@7.0.2: + resolution: {integrity: sha512-uKZghv9UmPkMVLYy//KZ9HFAIJsl7wkhoEdIL0+rhuSY9pZQlhaeGEDPIe+/w7eh81MOql8Q/9+inAGWG6ZHYA==} + + csv-stringify@6.8.3: + resolution: {integrity: sha512-gIeSCvq5F4VtXV3naV3VAewLhBkiZBz+PPhTOA8H3Y8h/ELa+R1ml0GZck/4/Nzo9ep2lvOluilJ6MJlbZsKMA==} + default-browser-id@5.0.1: resolution: {integrity: sha512-x1VCxdX4t+8wVfd1so/9w+vQ4vx7lKd2Qp5tDRutErwmR85OgmfX7RlLRMWafRMY7hbEiXIbudNrjOAPa/hL8Q==} engines: {node: '>=18'} @@ -1142,6 +1154,10 @@ snapshots: commander@14.0.3: {} + csv-parse@7.0.2: {} + + csv-stringify@6.8.3: {} + default-browser-id@5.0.1: {} default-browser@5.5.0: diff --git a/src/__tests__/commands/config.test.ts b/src/__tests__/commands/config.test.ts new file mode 100644 index 0000000..fa4b7df --- /dev/null +++ b/src/__tests__/commands/config.test.ts @@ -0,0 +1,63 @@ +import {describe, it, expect, vi, beforeEach} from 'vitest'; + +const mocks = vi.hoisted(()=>({ + set_config: vi.fn(), + fail: vi.fn(), + success: vi.fn(), +})); + +vi.mock('../../utils/config', ()=>({ + load: vi.fn(), + get: vi.fn(), + set: mocks.set_config, +})); + +vi.mock('../../utils/output', ()=>({ + print: vi.fn(), + fail: mocks.fail, + success: mocks.success, +})); + +import {handle_set_config} from '../../commands/config'; + +describe('commands/config', ()=>{ + beforeEach(()=>{ + vi.clearAllMocks(); + }); + + it('stores sanitize_csv false as boolean false', ()=>{ + handle_set_config('sanitize_csv', 'false'); + + expect(mocks.set_config).toHaveBeenCalledWith( + 'sanitize_csv', + false + ); + expect(mocks.fail).not.toHaveBeenCalled(); + expect(mocks.success).toHaveBeenCalledWith( + 'Config updated: sanitize_csv=false' + ); + }); + + it('stores sanitize_csv true as boolean true', ()=>{ + handle_set_config('sanitize_csv', 'true'); + + expect(mocks.set_config).toHaveBeenCalledWith( + 'sanitize_csv', + true + ); + expect(mocks.fail).not.toHaveBeenCalled(); + expect(mocks.success).toHaveBeenCalledWith( + 'Config updated: sanitize_csv=true' + ); + }); + + it('rejects invalid sanitize_csv value without modifying config', ()=>{ + handle_set_config('sanitize_csv', 'maybe'); + + expect(mocks.fail).toHaveBeenCalledWith( + 'sanitize_csv must be true or false' + ); + expect(mocks.set_config).not.toHaveBeenCalled(); + expect(mocks.success).not.toHaveBeenCalled(); + }); +}); \ No newline at end of file diff --git a/src/__tests__/commands/dataset.test.ts b/src/__tests__/commands/dataset.test.ts index fa9ce08..b574232 100644 --- a/src/__tests__/commands/dataset.test.ts +++ b/src/__tests__/commands/dataset.test.ts @@ -1,11 +1,52 @@ -import {describe, it, expect, vi, afterEach} from 'vitest'; +import {describe, it, expect, vi, afterEach, beforeEach} from 'vitest'; +import fs from 'fs'; +import path from 'path'; +import os from 'os'; + +const mocks = vi.hoisted(()=>({ + post: vi.fn(), + poll_until: vi.fn(), + ensure_authenticated: vi.fn(), + spinner_stop: vi.fn(), + get_config: vi.fn(), +})); +vi.mock('../../utils/auth', ()=>({ + ensure_authenticated: mocks.ensure_authenticated, +})); +vi.mock('../../utils/client', ()=>({ + post: mocks.post, + get: vi.fn(), +})); +vi.mock('../../utils/polling', async import_original=>{ + const actual = await import_original(); + return { + ...actual, + poll_until: mocks.poll_until, + }; +}); +vi.mock('../../utils/spinner', ()=>({ + start: ()=>({ + stop: mocks.spinner_stop, + }), +})); +vi.mock('../../utils/config', ()=>({ + get: mocks.get_config, +})); + import {handle_pipelines} from '../../commands/dataset'; describe('commands/pipelines list', ()=>{ + beforeEach(()=>{ + mocks.ensure_authenticated.mockReturnValue('test-api-key'); + mocks.get_config.mockReturnValue(true); + mocks.post.mockResolvedValue({ + snapshot_id: 'snapshot-1', + }); + }); afterEach(()=>{ vi.restoreAllMocks(); + vi.clearAllMocks(); }); - it('prints available pipeline dataset types', async()=>{ let output = ''; const write = vi.spyOn(process.stdout, 'write').mockImplementation( @@ -20,4 +61,45 @@ describe('commands/pipelines list', ()=>{ expect(output.includes('linkedin_person_profile')).toBe(true); expect(output.includes('youtube_comments')).toBe(true); }); -}); + it('sanitizes pipeline CSV written to stdout', async()=>{ + mocks.poll_until.mockResolvedValue({ + result: 'name,value\nfoo,"=SUM(1,2)"\n', + attempts: 1, + }); + let output = ''; + vi.spyOn(process.stdout, 'write').mockImplementation(text=>{ + output += String(text); + return true; + }); + vi.spyOn(console, 'error').mockImplementation(()=>{}); + await handle_pipelines( + 'amazon_product', + ['https://example.com/product'], + {format: 'csv'} + ); + expect(output).toContain("'=SUM(1,2)"); + }); + it('sanitizes pipeline CSV written to a file', async()=>{ + mocks.poll_until.mockResolvedValue({ + result: 'name,value\nfoo,"=SUM(1,2)"\n', + attempts: 1, + }); + vi.spyOn(console, 'error').mockImplementation(()=>{}); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brightdata-')); + const output_path = path.join(dir, 'result.csv'); + try { + await handle_pipelines( + 'amazon_product', + ['https://example.com/product'], + { + format: 'csv', + output: output_path, + } + ); + const output = fs.readFileSync(output_path, 'utf8'); + expect(output).toContain("'=SUM(1,2)"); + } finally { + fs.rmSync(dir, {recursive: true, force: true}); + } + }); +}); diff --git a/src/__tests__/utils/config.test.ts b/src/__tests__/utils/config.test.ts index fee8614..eac3dbd 100644 --- a/src/__tests__/utils/config.test.ts +++ b/src/__tests__/utils/config.test.ts @@ -10,6 +10,7 @@ import { resolve_api_key, DEFAULTS, } from '../../utils/config'; +import {get_config_dir} from '../../utils/credentials'; const mk_tmp_home = ()=>{ const stamp = `${Date.now()}-${Math.random()}`; @@ -49,6 +50,49 @@ describe('utils/config', ()=>{ expect(get('default_format')).toBe('json'); }); + it('persists sanitize_csv false as JSON boolean', ()=>{ + set('sanitize_csv', false); + const config_path = path.join(get_config_dir(), 'config.json'); + const persisted = JSON.parse(fs.readFileSync(config_path, 'utf8')); + expect(persisted.sanitize_csv).toBe(false); + expect(typeof persisted.sanitize_csv).toBe('boolean'); + }); + + it('persists sanitize_csv true as JSON boolean', ()=>{ + set('sanitize_csv', true); + const config_path = path.join(get_config_dir(), 'config.json'); + const persisted = JSON.parse(fs.readFileSync(config_path, 'utf8')); + expect(persisted.sanitize_csv).toBe(true); + expect(typeof persisted.sanitize_csv).toBe('boolean'); + }); + + it('normalizes persisted sanitize_csv string false', ()=>{ + set('sanitize_csv', true); + const config_path = path.join(get_config_dir(), 'config.json'); + fs.writeFileSync(config_path, JSON.stringify({ + sanitize_csv: 'false', + })); + expect(load().sanitize_csv).toBe(false); + }); + + it('normalizes persisted sanitize_csv string true', ()=>{ + set('sanitize_csv', false); + const config_path = path.join(get_config_dir(), 'config.json'); + fs.writeFileSync(config_path, JSON.stringify({ + sanitize_csv: 'true', + })); + expect(load().sanitize_csv).toBe(true); + }); + + it('rejects invalid persisted sanitize_csv value', ()=>{ + set('sanitize_csv', true); + const config_path = path.join(get_config_dir(), 'config.json'); + fs.writeFileSync(config_path, JSON.stringify({ + sanitize_csv: 'maybe', + })); + expect(()=>load()).toThrow('sanitize_csv must be true or false'); + }); + it('resolves value by cli then env then config', ()=>{ set('default_zone_unlocker', 'from_config'); process.env['TEST_ZONE_ENV'] = 'from_env'; diff --git a/src/__tests__/utils/output.test.ts b/src/__tests__/utils/output.test.ts index a863679..c5b0fe0 100644 --- a/src/__tests__/utils/output.test.ts +++ b/src/__tests__/utils/output.test.ts @@ -2,9 +2,28 @@ import {describe, it, expect, vi, beforeEach, afterEach} from 'vitest'; import fs from 'fs'; import path from 'path'; import os from 'os'; -import {serialize, format_from_ext, print, print_table} from '../../utils/output'; +import { + serialize, + format_from_ext, + print, + print_table, + sanitize_serialized_csv, +} from '../../utils/output'; + +const mocks = vi.hoisted(()=>({ + get_config: vi.fn(), +})); +vi.mock('../../utils/config', ()=>({ + get: mocks.get_config, +})); describe('utils/output.serialize csv', ()=>{ + beforeEach(()=>{ + mocks.get_config.mockReturnValue(true); + }); + afterEach(()=>{ + mocks.get_config.mockReset(); + }); it('serializes array of flat objects as RFC 4180 CSV with header row', ()=>{ const rows = [ {url: 'https://a.test/1', title: 'A', price: 1.5}, @@ -46,6 +65,95 @@ describe('utils/output.serialize csv', ()=>{ const lines = out.trim().split('\n'); expect(lines[1]).toBe('1,"{""tag"":""x""}"'); }); + + it('sanitizes spreadsheet formula prefixes in CSV string cells', ()=>{ + const rows = [{ + equals: '=1+1', + space: ' =1+1', + multi_space: ' =1+1', + tab: '\t=1+1', + nbsp: '\u00a0=1+1', + plus: '+cmd', + minus: '-SUM(A1:A2)', + at: '@SUM(A1:A2)', + }]; + const out = serialize(rows, 'csv'); + expect(out).toContain("'=1+1"); + expect(out).toContain("' =1+1"); + expect(out).toContain("' =1+1"); + expect(out).toContain("'\t=1+1"); + expect(out).toContain("'\u00a0=1+1"); + expect(out).toContain("'+cmd"); + expect(out).toContain("'-SUM(A1:A2)"); + expect(out).toContain("'@SUM(A1:A2)"); + }); + + it('does not sanitize numeric values', ()=>{ + const out = serialize([{value: -100}], 'csv'); + const lines = out.trim().split('\n'); + expect(lines[1]).toBe('-100'); + }); + + it('preserves numeric-looking CSV strings', ()=>{ + const out = serialize([{negative: '-100', positive: '+15'}], 'csv'); + expect(out).toContain('-100,+15'); + }); + + it('does not sanitize CSV cells when sanitize_csv is false', ()=>{ + mocks.get_config.mockReturnValue(false); + const out = serialize([{value: '=1+1'}], 'csv'); + expect(out).toContain('=1+1'); + expect(out).not.toContain("'=1+1"); + }); + + it('does not sanitize serialized CSV when sanitize_csv is false', ()=>{ + mocks.get_config.mockReturnValue(false); + + const input = 'name,value\nfoo,=1+1\n'; + const out = sanitize_serialized_csv(input); + + expect(out).toBe(input); + }); + + it('reads sanitize_csv config once per CSV serialization', ()=>{ + serialize([ + {a: '=1+1', b: '+cmd'}, + {a: '-SUM(A1:A2)', b: '@SUM(A1:A2)'}, + ], 'csv'); + + expect(mocks.get_config).toHaveBeenCalledTimes(1); + expect(mocks.get_config).toHaveBeenCalledWith('sanitize_csv'); + }); + + it('sanitizes formula cells in serialized CSV', ()=>{ + const input = 'name,value\nfoo,"=SUM(1,2)"\n'; + const out = sanitize_serialized_csv(input); + expect(out).not.toContain('"=SUM(1,2)"'); + }); + + it('preserves quoted commas and newlines in serialized CSV', ()=>{ + const input = + 'name,note\n' + +'foo,"hello,world"\n' + +'bar,"line1\nline2"\n'; + const out = sanitize_serialized_csv(input); + expect(out).toContain('"hello,world"'); + expect(out).toContain('"line1\nline2"'); + }); + + it('preserves UTF-8 BOM when sanitizing serialized CSV', ()=>{ + const input = '\uFEFFname,value\nfoo,"=SUM(1,2)"\n'; + const out = sanitize_serialized_csv(input); + expect(out.charCodeAt(0)).toBe(0xFEFF); + expect(out).toContain("'=SUM(1,2)"); + }); + + it('throws actionable error when serialized CSV cannot be parsed', ()=>{ + const input = 'name,value\nfoo,"unterminated\n'; + expect(()=>sanitize_serialized_csv(input)).toThrow( + /Failed to sanitize server CSV: .*sanitize_csv=false/ + ); + }); }); describe('utils/output.serialize markdown', ()=>{ diff --git a/src/commands/config.ts b/src/commands/config.ts index 6a5a6d0..f4c3aa3 100644 --- a/src/commands/config.ts +++ b/src/commands/config.ts @@ -14,6 +14,7 @@ const CONFIG_KEYS: Config_key[] = [ 'default_zone_serp', 'default_format', 'api_url', + 'sanitize_csv', ]; const format_keys = ()=>CONFIG_KEYS.join(', '); @@ -50,12 +51,22 @@ const handle_get_config = (key: string)=>{ ); return; } - process.stdout.write(value+'\n'); + process.stdout.write(String(value)+'\n'); }; const handle_set_config = (key: string, value: string)=>{ const valid_key = ensure_valid_key(key); - set_config(valid_key, value); + if (valid_key == 'sanitize_csv') + { + if (value != 'true' && value != 'false') + { + fail('sanitize_csv must be true or false'); + return; + } + set_config(valid_key, value == 'true'); + } + else + set_config(valid_key, value); success(`Config updated: ${valid_key}=${value}`); }; diff --git a/src/commands/dataset.ts b/src/commands/dataset.ts index 667db33..824bc35 100644 --- a/src/commands/dataset.ts +++ b/src/commands/dataset.ts @@ -1,7 +1,7 @@ import {Command} from 'commander'; import {ensure_authenticated} from '../utils/auth'; import {get, post} from '../utils/client'; -import {print, dim, fail} from '../utils/output'; +import {print, dim, fail, sanitize_serialized_csv} from '../utils/output'; import {start as start_spinner} from '../utils/spinner'; import {parse_timeout, poll_until} from '../utils/polling'; import {add_examples} from '../utils/help'; @@ -287,7 +287,11 @@ const handle_pipelines = async( `Data received after ${poll_result.attempts} attempts` )); const result = poll_result.result; - const cleaned_result = format == 'json' ? strip_nulls(result) : result; + let cleaned_result = result; + if (format == 'json') + cleaned_result = strip_nulls(result); + else if (format == 'csv' && typeof result == 'string') + cleaned_result = sanitize_serialized_csv(result); print(cleaned_result, { json: opts.json, pretty: opts.pretty, diff --git a/src/utils/config.ts b/src/utils/config.ts index 0efa59b..f519da6 100644 --- a/src/utils/config.ts +++ b/src/utils/config.ts @@ -9,23 +9,40 @@ type Config = { default_zone_serp?: string; default_format?: string; api_url?: string; + sanitize_csv?: boolean; }; +type String_config_key = { + [K in keyof Config]-?: Config[K] extends string|undefined ? K : never +}[keyof Config]; + const DEFAULTS: Config = { default_format: 'markdown', api_url: 'https://api.brightdata.com', + sanitize_csv: true, }; const load = (): Config=>{ const config_path = get_config_path(); if (!fs.existsSync(config_path)) return {...DEFAULTS}; + let parsed: Record; try { const raw = fs.readFileSync(config_path, 'utf8'); - return {...DEFAULTS, ...JSON.parse(raw) as Config}; + parsed = JSON.parse(raw) as Record; } catch(e) { return {...DEFAULTS}; } + if (parsed.sanitize_csv == 'true') + parsed.sanitize_csv = true; + else if (parsed.sanitize_csv == 'false') + parsed.sanitize_csv = false; + else if (parsed.sanitize_csv !== undefined + && typeof parsed.sanitize_csv != 'boolean') + { + throw new Error('sanitize_csv must be true or false'); + } + return {...DEFAULTS, ...parsed} as Config; }; const save = (config: Config)=>{ @@ -35,12 +52,12 @@ const save = (config: Config)=>{ fs.writeFileSync(get_config_path(), JSON.stringify(config, null, 4)); }; -const get = (key: keyof Config): string|undefined=>{ +const get = (key: K): Config[K]=>{ const config = load(); return config[key]; }; -const set = (key: keyof Config, value: string)=>{ +const set = (key: K, value: Config[K])=>{ const config = load(); config[key] = value; save(config); @@ -50,7 +67,7 @@ const set = (key: keyof Config, value: string)=>{ const resolve = ( cli_val: string|undefined, env_key: string, - config_key: keyof Config + config_key: String_config_key ): string|undefined=>{ if (cli_val) return cli_val; diff --git a/src/utils/output.ts b/src/utils/output.ts index 75743c8..a14a53b 100644 --- a/src/utils/output.ts +++ b/src/utils/output.ts @@ -1,6 +1,10 @@ import fs from 'fs'; import path from 'path'; import { stripVTControlCharacters } from 'util'; +import {parse} from 'csv-parse/sync'; +import {stringify} from 'csv-stringify/sync'; + +import {get as get_config} from './config'; const terminal_safe = (val: unknown): string=> stripVTControlCharacters(String(val)) @@ -95,10 +99,46 @@ const cell_to_string = (val: unknown): string=>{ return JSON.stringify(val); }; -const csv_escape = (val: unknown): string=>{ - const s = cell_to_string(val); +const sanitize_csv_cell = (s: string): string=>{ + const trimmed = s.trim(); + if (/^[+-]?(?:\d+(?:\.\d+)?|\.\d+)$/.test(trimmed)) + return s; + if (/^[\s\u00a0]*[=+\-@]/.test(s)) + return "'" + s; + return s; +}; + +const sanitize_serialized_csv = (csv: string): string=>{ + const sanitize = get_config('sanitize_csv') !== false; + if (!sanitize || !csv) + return csv; + const has_bom = csv.charCodeAt(0) == 0xFEFF; + let rows: string[][]; + try { + rows = parse(csv, { + bom: true, + }) as string[][]; + } catch(e) { + throw new Error( + 'Failed to sanitize server CSV: ' + +(e as Error).message + +'. Set sanitize_csv=false to export it without sanitization.' + ); + } + const sanitized = rows.map(row=> + row.map(cell=>sanitize_csv_cell(cell)) + ); + return stringify(sanitized, { + bom: has_bom, + }); +}; + +const csv_escape = (val: unknown, sanitize: boolean): string=>{ + let s = cell_to_string(val); + if (typeof val == 'string' && sanitize) + s = sanitize_csv_cell(s); if (/[",\r\n]/.test(s)) - return '"'+s.replace(/"/g, '""')+'"'; + return '"' + s.replace(/"/g, '""') + '"'; return s; }; @@ -112,9 +152,10 @@ const serialize_csv = (data: unknown): string=>{ +'to JSON. Use --json to silence this warning.'); return JSON.stringify(data, null, 2); } + const sanitize = get_config('sanitize_csv') !== false; const keys = collect_keys(rows); - const header = keys.map(csv_escape).join(','); - const body = rows.map(r=>keys.map(k=>csv_escape(r[k])).join(',')).join('\n'); + const header = keys.map(k=>csv_escape(k, sanitize)).join(','); + const body = rows.map(r=>keys.map(k=>csv_escape(r[k], sanitize)).join(',')).join('\n'); return header+'\n'+body+'\n'; }; @@ -237,5 +278,6 @@ export { green, red, yellow, dim, success, warn, info, fail, format_from_ext, serialize, print, print_table, + sanitize_serialized_csv, }; export type {Output_format, Print_opts};