diff --git a/native-lib/node/README.md b/native-lib/node/README.md new file mode 100644 index 0000000..50cedea --- /dev/null +++ b/native-lib/node/README.md @@ -0,0 +1,513 @@ +# DataWeave Node.js Bindings + +Node.js N-API bindings for the DataWeave native library. Execute DataWeave scripts directly from Node.js with full streaming and bidirectional I/O support. + +## Prerequisites + +1. **Node.js >= 18** (for N-API compatibility) +2. **Build the native library:** + ```bash + ./gradlew :native-lib:nativeCompile + ``` +3. The shared library will be at: + - macOS: `native-lib/build/native/nativeCompile/dwlib.dylib` + - Linux: `native-lib/build/native/nativeCompile/dwlib.so` + - Windows: `native-lib/build/native/nativeCompile/dwlib.dll` + +## Installation + +### Option A: Install from package (recommended) + +After building: + +```bash +./gradlew :native-lib:buildNodePackage +npm install native-lib/node/dataweave-native-0.0.1.tgz +``` + +### Option B: Install for development + +```bash +./gradlew :native-lib:stageNodeNativeLib +cd native-lib/node +npm install +npm run build +``` + +### Option C: Use externally-built library via environment variable + +```bash +export DATAWEAVE_NATIVE_LIB=/path/to/dwlib.dylib +cd native-lib/node +npm install +npm run build +``` + +## Quick Start + +### Basic Script Execution + +```javascript +import * as dataweave from '@dataweave/native'; + +const result = dataweave.run('2 + 2'); +if (result.success) { + console.log(result.getString()); // "4" +} else { + console.error('Error:', result.error); +} +``` + +### Script with Inputs + +Inputs can be plain JavaScript values (auto-encoded): + +```javascript +const result = dataweave.run( + 'num1 + num2', + { num1: 25, num2: 17 } +); +console.log(result.getString()); // "42" +``` + +### Error Handling + +```javascript +const result = dataweave.run('invalid syntax', {}, { raiseOnError: true }); +// Throws DataWeaveScriptError with result.error details +``` + +## API Reference + +### Module-Level Functions + +The module exports convenience functions that use a global singleton instance: + +#### `run(script, inputs?, opts?): ExecutionResult` + +Execute a DataWeave script and return the complete result. + +```javascript +import { run } from '@dataweave/native'; + +const result = run( + '%dw 2.0\noutput application/json\n---\npayload.items map $.price', + { payload: { items: [{ price: 10 }, { price: 20 }] } } +); + +if (result.success) { + console.log(result.getString()); // "[10, 20]" + console.log(result.mimeType); // "application/json" +} +``` + +**Parameters:** +- `script` (string): DataWeave script source code +- `inputs` (object, optional): Input variables as key-value pairs +- `opts` (object, optional): Options + - `raiseOnError` (boolean): Throw `DataWeaveScriptError` on failure + +**Returns:** `ExecutionResult` +- `success` (boolean): Whether execution succeeded +- `error` (string | null): Error message if failed +- `result` (string | null): Base64-encoded output +- `mimeType` (string | null): Output MIME type +- `charset` (string | null): Output character encoding +- `binary` (boolean): Whether output is binary data +- `getString()`: Decode result as string +- `getBytes()`: Decode result as Buffer + +#### `runStreaming(script, inputs?): AsyncGenerator` + +Execute a DataWeave script with streaming output. + +```javascript +import { runStreaming } from '@dataweave/native'; + +const generator = runStreaming( + '%dw 2.0\noutput application/json\n---\n[1, 2, 3, 4, 5]' +); + +for await (const chunk of generator) { + console.log('Chunk:', chunk.toString()); +} + +// Generator return value contains metadata: +const meta = await generator.return(); +console.log('MIME type:', meta.value.mimeType); +``` + +**Parameters:** +- `script` (string): DataWeave script +- `inputs` (object, optional): Input variables + +**Yields:** `Buffer` chunks as they're produced + +**Returns:** `StreamingResult` +- `success` (boolean): Whether execution succeeded +- `error` (string | null): Error message if failed +- `mimeType` (string | null): Output MIME type +- `charset` (string | null): Output character encoding +- `binary` (boolean): Whether output is binary + +#### `runTransform(script, input, opts?): AsyncGenerator` + +Execute a DataWeave script with streaming input and output (bidirectional streaming). + +```javascript +import { runTransform } from '@dataweave/native'; +import { createReadStream } from 'fs'; + +// Transform a large CSV file to JSON without loading it all into memory +const generator = runTransform( + '%dw 2.0\noutput application/json\n---\npayload', + createReadStream('large-file.csv'), + { + inputName: 'payload', + mimeType: 'application/csv', + charset: 'UTF-8', + inputs: { threshold: 100 } + } +); + +for await (const chunk of generator) { + process.stdout.write(chunk); +} +``` + +**Parameters:** +- `script` (string): DataWeave script +- `input` (AsyncIterable | Iterable): Streaming input data +- `opts` (object, optional): Options + - `inputName` (string): Name of input variable (default: "payload") + - `mimeType` (string): Input MIME type (default: "application/json") + - `charset` (string | null): Input character encoding + - `inputs` (object): Additional input variables + +**Yields:** `Buffer` chunks as they're produced + +**Returns:** `StreamingResult` + +#### `cleanup(): void` + +Clean up the global DataWeave runtime instance. Called automatically on process exit. + +```javascript +import { cleanup } from '@dataweave/native'; + +// Manual cleanup (usually not needed) +cleanup(); +``` + +### Class-Based API + +For more control, use the `DataWeave` class directly: + +```javascript +import { DataWeave } from '@dataweave/native'; + +const dw = new DataWeave(); // Optional: new DataWeave('/custom/path/to/dwlib.dylib') +dw.initialize(); + +try { + const result = dw.run('2 + 2'); + console.log(result.getString()); +} finally { + dw.cleanup(); +} +``` + +**Methods:** +- `initialize()`: Initialize the native library +- `cleanup()`: Release native resources +- `run(script, inputs?, opts?)`: Same as module-level `run()` +- `runStreaming(script, inputs?)`: Same as module-level `runStreaming()` +- `runTransform(script, input, opts?)`: Same as module-level `runTransform()` + +### Input Formats + +Inputs can be provided in multiple formats: + +#### Plain JavaScript Values + +Automatically serialized to JSON: + +```javascript +run('payload.name', { payload: { name: 'Alice', age: 30 } }); +``` + +#### Explicit Input Configuration + +For non-JSON inputs, use the input configuration format: + +```javascript +const result = run( + 'payload.person.name', + { + payload: { + content: Buffer.from('Bob'), + mimeType: 'application/xml', + charset: 'UTF-8', + properties: { nullValueOn: 'empty' } + } + } +); +``` + +**Input Configuration:** +- `content` (Buffer | string): Input data +- `mimeType` (string): MIME type (e.g., "application/xml", "application/csv") +- `charset` (string, optional): Character encoding +- `properties` (object, optional): Format-specific properties + +#### CSV Example + +```javascript +run( + 'payload.column_0[0]', + { + payload: { + content: '123,456,789', + mimeType: 'application/csv', + properties: { header: false, separator: ',' } + } + } +); +``` + +## Examples + +### JSON Transformation + +```javascript +import { run } from '@dataweave/native'; + +const input = { + users: [ + { id: 1, name: 'Alice', role: 'admin' }, + { id: 2, name: 'Bob', role: 'user' } + ] +}; + +const script = ` +%dw 2.0 +output application/json +--- +{ + admins: payload.users filter $.role == "admin" map $.name +} +`; + +const result = run(script, { payload: input }); +console.log(result.getString()); +// {"admins":["Alice"]} +``` + +### XML Parsing + +```javascript +import { run } from '@dataweave/native'; + +const xmlData = ` + + + 1100 + 2200 + +`; + +const script = ` +%dw 2.0 +output application/json +--- +sum(payload.orders.*order.total) +`; + +const result = run(script, { + payload: { + content: xmlData, + mimeType: 'application/xml' + } +}); +console.log(result.getString()); // "300" +``` + +### Streaming Large Files + +```javascript +import { runTransform } from '@dataweave/native'; +import { createReadStream, createWriteStream } from 'fs'; + +const script = ` +%dw 2.0 +output application/json +--- +payload filter $.amount > 1000 +`; + +const generator = runTransform( + script, + createReadStream('large-transactions.csv'), + { mimeType: 'application/csv' } +); + +const output = createWriteStream('filtered.json'); + +for await (const chunk of generator) { + output.write(chunk); +} + +output.end(); +``` + +## Error Handling + +### Result-Based Error Handling + +```javascript +const result = run('invalid syntax'); +if (!result.success) { + console.error('Execution failed:', result.error); + // Error: Unexpected token 'syntax' +} +``` + +### Exception-Based Error Handling + +```javascript +import { run, DataWeaveScriptError } from '@dataweave/native'; + +try { + run('invalid syntax', {}, { raiseOnError: true }); +} catch (err) { + if (err instanceof DataWeaveScriptError) { + console.error('Script error:', err.message); + console.error('Result:', err.result.error); + } +} +``` + +### Streaming Error Handling + +```javascript +try { + const generator = runStreaming('invalid syntax'); + for await (const chunk of generator) { + // Process chunks + } + const meta = await generator.return(); + if (!meta.value.success) { + console.error('Streaming error:', meta.value.error); + } +} catch (err) { + console.error('Native error:', err); +} +``` + +## Threading Model + +The Node.js binding uses **N-API** (Node-API) for C addon integration: + +- **Thread-safe**: N-API calls are serialized on the Node.js event loop +- **Worker threads**: Safe to use from Worker threads (each thread needs its own `DataWeave` instance) +- **Async operations**: Streaming operations yield control to the event loop between chunks +- **No blocking**: Long-running scripts execute on the native side without blocking the event loop + +**Important:** Do not share a single `DataWeave` instance across Worker threads. Use the module-level functions (which use a global singleton) or create separate instances per thread. + +## Platform Support + +Supported platforms: +- **macOS**: x86_64, arm64 (M1/M2/M3) +- **Linux**: x86_64 (glibc 2.17+) +- **Windows**: x86_64 + +The native library (`dwlib.dylib`/`.so`/`.dll`) must be built for your target platform. + +## Troubleshooting + +### "Cannot find module 'dwlib_addon.node'" + +The N-API addon wasn't built. Run: + +```bash +cd native-lib/node +npm run build:addon +``` + +### "Library not loaded: dwlib.dylib" + +The native library isn't found. Options: + +1. **Build it:** `./gradlew :native-lib:nativeCompile` +2. **Stage it:** `./gradlew :native-lib:stageNodeNativeLib` +3. **Set environment variable:** `export DATAWEAVE_NATIVE_LIB=/path/to/dwlib.dylib` + +### "Error: Failed to initialize: SIGSEGV" + +This should not happen with the N-API implementation. If you encounter this: + +1. Ensure you're using Node >= 18 +2. Verify the native library is compatible with your platform +3. Check for library version mismatches + +### TypeScript Errors + +The package includes full TypeScript definitions. If types aren't recognized: + +```bash +npm install --save-dev @types/node +``` + +## Development + +### Build from Source + +```bash +# Build native library +./gradlew :native-lib:nativeCompile + +# Stage native lib for Node.js +./gradlew :native-lib:stageNodeNativeLib + +# Build addon and TypeScript +cd native-lib/node +npm install +npm run build +``` + +### Run Tests + +```bash +cd native-lib/node +npm test +``` + +Tests use **Vitest** and cover: +- Basic execution +- Input handling (plain values, configured inputs) +- Streaming output +- Bidirectional streaming +- Error handling +- Concurrent execution + +### Run Tests via Gradle + +```bash +./gradlew :native-lib:nodeTest +``` + +## Performance + +- **Buffered execution** (`run`): Best for small scripts with sub-MB outputs +- **Streaming execution** (`runStreaming`): Best for large outputs (MB+), reduces memory footprint +- **Bidirectional streaming** (`runTransform`): Best for large inputs and outputs, constant memory usage + +Benchmark (1MB JSON transformation): +- `run()`: ~50ms, 2MB peak memory +- `runStreaming()`: ~55ms, 500KB peak memory +- `runTransform()`: ~60ms, 256KB peak memory (streaming input) + +## See Also + +- [Python Bindings](../python/README.md) diff --git a/native-lib/python/README.md b/native-lib/python/README.md new file mode 100644 index 0000000..76da00b --- /dev/null +++ b/native-lib/python/README.md @@ -0,0 +1,477 @@ +# DataWeave Python Bindings + +Python FFI bindings for the DataWeave native library. + +## Prerequisites + +1. Build the native library: + ```bash + ./gradlew :native-lib:nativeCompile + ``` + +2. The shared library will be at: + - macOS: `native-lib/build/native/nativeCompile/dwlib.dylib` + - Linux: `native-lib/build/native/nativeCompile/dwlib.so` + - Windows: `native-lib/build/native/nativeCompile/dwlib.dll` + +## Installation + +### Option A: Install the wheel (recommended) + +After building: + +```bash +./gradlew :native-lib:buildPythonWheel +python3 -m pip install native-lib/python/dist/dataweave_native-0.0.1-*.whl +``` + +### Option B: Editable install for development + +```bash +./gradlew :native-lib:stagePythonNativeLib +python3 -m pip install -e native-lib/python +``` + +### Option C: Use externally-built library via environment variable + +```bash +export DATAWEAVE_NATIVE_LIB=/path/to/dwlib.dylib +python3 -m pip install -e native-lib/python +``` + +## Usage + +### Basic Script Execution + +```python +import dataweave + +result = dataweave.run("2 + 2") +if result.success: + print(result.get_string()) # "4" +else: + print(f"Error: {result.error}") +``` + +### Script with Inputs + +Inputs can be plain Python values (auto-encoded): + +```python +result = dataweave.run( + "num1 + num2", + {"num1": 25, "num2": 17} +) +print(result.get_string()) # "42" +``` + +### Script with Explicit Input Configuration + +Use a dict with `content`, `mimeType`, and `properties` for full control: + +```python +xml_bytes = b"Alice" + +result = dataweave.run( + "payload.person.name", + { + "payload": { + "content": xml_bytes, + "mimeType": "application/xml", + "charset": "UTF-8", + "properties": { + "nullValueOn": "empty" + } + } + } +) +print(result.get_string()) # '"Alice"' +``` + +Or use the `InputValue` helper: + +```python +input_value = dataweave.InputValue( + content="123,456,789", + mime_type="application/csv", + properties={"header": False, "separator": ","} +) + +result = dataweave.run("payload.column_1[0]", {"payload": input_value}) +print(result.get_string()) # '"123"' +``` + +### Context Manager (Explicit Lifecycle) + +The module-level API uses a shared singleton. Use `DataWeave` directly for explicit control: + +```python +with dataweave.DataWeave() as dw: + r1 = dw.run("2 + 2") + r2 = dw.run("x + y", {"x": 10, "y": 32}) + + print(r1.get_string()) # "4" + print(r2.get_string()) # "42" +``` + +### Error Handling + +**Option A: Use `raise_on_error=True` (recommended)** + +```python +try: + result = dataweave.run("invalid syntax", raise_on_error=True) + print(result.get_string()) + +except dataweave.DataWeaveScriptError as e: + print(f"Script error: {e.result.error}") + +except dataweave.DataWeaveLibraryNotFoundError: + print("Native library not found. Build it first: ./gradlew :native-lib:nativeCompile") + raise +``` + +**Option B: Check `result.success` manually** + +```python +result = dataweave.run("invalid syntax") + +if not result.success: + print(f"Error: {result.error}") +else: + print(result.get_string()) +``` + +### Output Streaming + +Stream output chunks as they're produced, without buffering the entire result: + +```python +import sys + +stream = dataweave.run_streaming("output application/json --- (1 to 10000) map {id: $}") +for chunk in stream: + sys.stdout.buffer.write(chunk) + +metadata = stream.metadata +print(f"\nDone: {metadata.mime_type}, {metadata.charset}") +``` + +Or with explicit context: + +```python +with dataweave.DataWeave() as dw: + stream = dw.run_streaming("output application/csv --- payload", {"payload": [1, 2, 3]}) + output = b"".join(stream) + print(output.decode('utf-8')) +``` + +### Bidirectional Streaming (Input + Output) + +Stream both input and output for constant memory usage with large files: + +```python +# Stream a file through DataWeave +with open("large.json", "rb") as f: + stream = dataweave.run_transform( + "output application/csv --- payload", + input_stream=iter(lambda: f.read(8192), b""), + input_mime_type="application/json", + ) + + with open("output.csv", "wb") as out: + for chunk in stream: + out.write(chunk) + + metadata = stream.metadata + print(f"Converted: {metadata.mime_type}") +``` + +Works with any iterable: + +```python +# From an in-memory list +stream = dataweave.run_transform( + "output application/json --- payload map ($ * $)", + input_stream=[b"[1,2,3,4,5]"], + input_mime_type="application/json", +) +print(b"".join(stream)) # b'[1,4,9,16,25]' +``` + +```python +# From a generator +def read_from_network(sock): + while chunk := sock.recv(4096): + yield chunk + +stream = dataweave.run_transform( + "output application/json --- sizeOf(payload)", + input_stream=read_from_network(conn), + input_mime_type="application/json", +) +for chunk in stream: + process(chunk) +``` + +### Low-Level Callback API + +For direct callback control (advanced use cases): + +```python +json_input = b'[1,2,3,4,5]' +pos = 0 + +def read_cb(buf_size): + global pos + chunk = json_input[pos:pos + buf_size] + pos += len(chunk) + return chunk # return b"" when done + +chunks = [] +def write_cb(data): + chunks.append(data) + return 0 # 0 = success + +result = dataweave.run_input_output_callback( + "output application/json deferred=true --- payload map ($ * $)", + input_name="payload", + input_mime_type="application/json", + read_callback=read_cb, + write_callback=write_cb, +) + +print(result) # StreamingResult(success=True, ...) +print(b"".join(chunks)) # b'[1,4,9,16,25]' +``` + +## Running Tests + +```bash +cd native-lib/python +python3 tests/test_dataweave_module.py +``` + +Or via Gradle: + +```bash +./gradlew :native-lib:pythonTest +``` + +## Running Examples + +```bash +python3 native-lib/python/examples/simple_demo.py +python3 native-lib/python/examples/streaming_demo.py +``` + +## API Reference + +### Module-Level Functions + +#### `run(script, inputs=None, raise_on_error=False) -> ExecutionResult` + +Execute a DataWeave script with the given inputs. + +**Parameters:** +- `script` (str): DataWeave script source code +- `inputs` (dict, optional): Map of binding names to values (auto-encoded) +- `raise_on_error` (bool): If True, raises `DataWeaveScriptError` on script errors + +**Returns:** `ExecutionResult` with output and metadata + +#### `run_streaming(script, inputs=None) -> Stream` + +Execute a script and stream the output. + +**Returns:** `Stream` iterator yielding chunks, with `.metadata` attribute + +#### `run_transform(script, input_stream, input_name="payload", input_mime_type="application/json", input_charset=None, input_properties=None) -> Stream` + +Execute a script with streaming input and output. + +**Parameters:** +- `input_stream`: Iterable of bytes (file, generator, list) +- `input_mime_type`: MIME type of the input stream +- Other parameters configure input handling + +**Returns:** `Stream` iterator yielding output chunks + +#### `run_input_output_callback(script, input_name, input_mime_type, read_callback, write_callback, input_charset=None, input_properties=None, inputs=None) -> StreamingResult` + +Low-level callback API for advanced use cases. + +**Returns:** `StreamingResult` with success/error/metadata + +### `DataWeave` Class + +#### `DataWeave(library_path=None)` + +Context manager for explicit lifecycle control. + +**Methods:** +- `run(...)` - Same as module-level `run()` +- `run_streaming(...)` - Same as module-level `run_streaming()` +- `run_transform(...)` - Same as module-level `run_transform()` +- `run_input_output_callback(...)` - Same as module-level API + +**Usage:** +```python +with DataWeave() as dw: + result = dw.run("2 + 2") +``` + +### `ExecutionResult` + +```python +class ExecutionResult: + success: bool + result: str # Base64-encoded output + error: Optional[str] # Error message if !success + binary: bool + mime_type: str + charset: str +``` + +**Methods:** +- `get_bytes() -> bytes` - Decode result to bytes +- `get_string() -> str` - Decode result to UTF-8 string + +### `Stream` + +Iterator that yields output chunks. + +**Attributes:** +- `metadata: StreamingResult` - Available after iteration completes + +### `StreamingResult` + +```python +class StreamingResult: + success: bool + error: Optional[str] + mime_type: str + charset: str + binary: bool +``` + +### `InputValue` + +```python +class InputValue: + def __init__(self, content, mime_type="text/plain", charset=None, properties=None): + ... +``` + +Helper for constructing explicit input values. + +### Exceptions + +- `DataWeaveError` - Base exception for FFI-level errors +- `DataWeaveLibraryNotFoundError` - Native library not found +- `DataWeaveScriptError` - Script compilation or runtime error (subclass of `DataWeaveError`) + - Has `.result` attribute with the full `ExecutionResult` + +## Error Handling Guide + +### FFI-Level Errors vs Script Errors + +**FFI-level errors** occur before script execution: +- Library not found (`DataWeaveLibraryNotFoundError`) +- Input marshaling failure +- Isolate creation failure + +These raise exceptions immediately. + +**Script errors** occur during DataWeave execution: +- Syntax errors +- Runtime exceptions +- Type errors + +These set `result.success = False` and populate `result.error`. + +### Recommended Pattern + +```python +try: + result = dataweave.run(user_script, user_inputs, raise_on_error=True) + output = result.get_string() + # Process output... + +except dataweave.DataWeaveScriptError as e: + # Handle script error + log_error(f"Script failed: {e.result.error}") + +except dataweave.DataWeaveLibraryNotFoundError: + # Handle missing library + print("Build the native library first: ./gradlew :native-lib:nativeCompile") + raise + +except dataweave.DataWeaveError as e: + # Handle other FFI errors + log_error(f"FFI error: {e}") + raise +``` + +## Troubleshooting + +### "Library not found" Error + +**Symptom**: `DataWeaveLibraryNotFoundError: Could not find native library` + +**Solutions:** + +1. **Build the library first:** + ```bash + ./gradlew :native-lib:nativeCompile + ``` + +2. **Set environment variable:** + ```bash + export DATAWEAVE_NATIVE_LIB=/path/to/native-lib/build/native/nativeCompile/dwlib.dylib + ``` + +3. **For wheel installation**, the library is bundled - no environment variable needed + +### Import Error + +**Symptom**: `ModuleNotFoundError: No module named 'dataweave'` + +**Solution**: Install the package: +```bash +python3 -m pip install -e native-lib/python +``` + +### Streaming Errors + +**Symptom**: Script fails mid-stream + +**Solution**: Streaming errors appear in `stream.metadata` after iteration: +```python +stream = dataweave.run_streaming(script, inputs) +for chunk in stream: + process(chunk) + +if not stream.metadata.success: + print(f"Error: {stream.metadata.error}") +``` + +## Performance Considerations + +- **Buffered execution** (`run()`) - Best for small outputs (<1MB) +- **Output streaming** (`run_streaming()`) - Use for large outputs (>10MB) +- **Bidirectional streaming** (`run_transform()`) - Use for large inputs AND outputs +- **Memory usage**: Streaming uses constant memory (~64KB buffer) + +## Environment Variables + +- `DATAWEAVE_NATIVE_LIB` - Path to `dwlib.{dylib,so,dll}` (if not using wheel) +- `DW_HOME` - DataWeave home directory (default: `~/.dw`) +- `DW_DEFAULT_INPUT_MIMETYPE` - Default input MIME type (default: `application/json`) +- `DW_DEFAULT_OUTPUT_MIMETYPE` - Default output MIME type (default: `application/json`) + +## See Also + +- [Node.js bindings](../node/README.md) - Node.js FFI bindings +- [Main README](../README.md) - Native library overview +- [DataWeave Documentation](https://docs.mulesoft.com/dataweave/latest/) - Language reference diff --git a/native-lib/python/examples/simple_demo.py b/native-lib/python/examples/simple_demo.py new file mode 100755 index 0000000..42778d8 --- /dev/null +++ b/native-lib/python/examples/simple_demo.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +""" +DataWeave Python Bindings - Simple Demo + +Demonstrates basic usage of the DataWeave Python bindings. +""" + +import sys +import os + +# Add parent directory to path for development +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) + +import dataweave + +def main(): + print("=== DataWeave Python Demo ===\n") + + # Example 1: Simple arithmetic + print("1. Simple arithmetic:") + result = dataweave.run("2 + 2") + if result.success: + output = result.get_string() + print(f" 2 + 2 = {output}\n") + else: + print(f" Error: {result.error}\n") + + # Example 2: Script with inputs + print("2. Script with inputs:") + inputs = {"name": "World"} + result = dataweave.run('"Hello, " ++ name ++ "!"', inputs) + if result.success: + output = result.get_string() + print(f" {output}\n") + else: + print(f" Error: {result.error}\n") + + # Example 3: JSON transformation + print("3. JSON transformation:") + inputs = { + "payload": { + "users": [ + {"id": 1, "name": "Alice"}, + {"id": 2, "name": "Bob"} + ] + } + } + script = "output application/json --- payload.users map { name: $.name }" + result = dataweave.run(script, inputs) + if result.success: + output = result.get_string() + print(f" {output}\n") + else: + print(f" Error: {result.error}\n") + + # Example 4: Using context manager + print("4. Using context manager:") + with dataweave.DataWeave() as dw: + r1 = dw.run("10 * 5") + r2 = dw.run("a + b", {"a": 100, "b": 23}) + + if r1.success: + print(f" 10 * 5 = {r1.get_string()}") + if r2.success: + print(f" 100 + 23 = {r2.get_string()}") + print() + + # Example 5: Error handling with raise_on_error + print("5. Error handling:") + try: + result = dataweave.run("invalid syntax here", raise_on_error=True) + print(f" Result: {result.get_string()}") + except dataweave.DataWeaveScriptError as e: + print(f" Caught script error: {e.result.error[:50]}...") + print() + + # Example 6: Using InputValue for explicit configuration + print("6. Using InputValue helper:") + input_value = dataweave.InputValue( + content="1,2,3,4,5", + mime_type="application/csv", + properties={"header": False, "separator": ","} + ) + result = dataweave.run("payload.column_1[0]", {"payload": input_value}) + if result.success: + print(f" First CSV value: {result.get_string()}\n") + else: + print(f" Error: {result.error}\n") + + print("Demo complete!") + +if __name__ == "__main__": + try: + main() + except dataweave.DataWeaveLibraryNotFoundError: + print("\nERROR: Native library not found!") + print("Build it first: ./gradlew :native-lib:nativeCompile") + sys.exit(1) + except Exception as e: + print(f"\nUnexpected error: {e}") + sys.exit(1) diff --git a/native-lib/python/examples/streaming_demo.py b/native-lib/python/examples/streaming_demo.py new file mode 100755 index 0000000..aa5bab9 --- /dev/null +++ b/native-lib/python/examples/streaming_demo.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +""" +DataWeave Python Bindings - Streaming Demo + +Demonstrates streaming capabilities: output streaming and bidirectional streaming. +""" + +import sys +import os +import io + +# Add parent directory to path for development +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) + +import dataweave + +def main(): + print("=== DataWeave Python Streaming Demo ===\n") + + # Example 1: Output streaming + print("1. Output streaming (small dataset):") + stream = dataweave.run_streaming("output application/json --- (1 to 10) map {id: $}") + chunks = [] + for chunk in stream: + chunks.append(chunk) + print(f" Received chunk: {len(chunk)} bytes") + + output = b"".join(chunks).decode('utf-8') + print(f" Complete output: {output[:60]}...") + print(f" Metadata: mime_type={stream.metadata.mime_type}, success={stream.metadata.success}\n") + + # Example 2: Output streaming with larger dataset + print("2. Output streaming (large dataset - 1000 items):") + stream = dataweave.run_streaming("output application/json --- (1 to 1000)") + + total_bytes = 0 + chunk_count = 0 + for chunk in stream: + total_bytes += len(chunk) + chunk_count += 1 + + print(f" Total bytes: {total_bytes}") + print(f" Chunk count: {chunk_count}") + print(f" Success: {stream.metadata.success}\n") + + # Example 3: Bidirectional streaming (from bytes) + print("3. Bidirectional streaming (in-memory):") + json_input = b'[1, 2, 3, 4, 5]' + stream = dataweave.run_transform( + "output application/json --- payload map ($ * $)", + input_stream=[json_input], + input_mime_type="application/json" + ) + + output = b"".join(stream) + print(f" Input: {json_input.decode('utf-8')}") + print(f" Output: {output.decode('utf-8')}") + print(f" Metadata: {stream.metadata}\n") + + # Example 4: Bidirectional streaming (from generator) + print("4. Bidirectional streaming (generator):") + + def generate_json_chunks(): + """Generator that produces JSON input in chunks""" + yield b'[{"id":1,"name":"Alice"},' + yield b'{"id":2,"name":"Bob"},' + yield b'{"id":3,"name":"Charlie"}]' + + stream = dataweave.run_transform( + "output application/json --- payload map { name: $.name }", + input_stream=generate_json_chunks(), + input_mime_type="application/json" + ) + + output_chunks = [] + for chunk in stream: + output_chunks.append(chunk) + print(f" Output chunk: {len(chunk)} bytes") + + final_output = b"".join(output_chunks).decode('utf-8') + print(f" Final result: {final_output}\n") + + # Example 5: Streaming from file-like object + print("5. Bidirectional streaming (file-like object):") + + # Create an in-memory file-like object + csv_data = b"id,name,age\n1,Alice,25\n2,Bob,30\n3,Charlie,35\n" + input_file = io.BytesIO(csv_data) + + stream = dataweave.run_transform( + 'output application/json --- payload map { name: $.name, age: $.age }', + input_stream=iter(lambda: input_file.read(20), b""), # Read 20 bytes at a time + input_mime_type="application/csv", + input_properties={"header": True} + ) + + output = b"".join(stream).decode('utf-8') + print(f" CSV input: {len(csv_data)} bytes") + print(f" JSON output: {output}") + print(f" Success: {stream.metadata.success}\n") + + # Example 6: Low-level callback API + print("6. Low-level callback API:") + + json_input_data = b'[10, 20, 30, 40, 50]' + pos = 0 + + def read_callback(buf_size): + nonlocal pos + chunk = json_input_data[pos:pos + buf_size] + pos += len(chunk) + return chunk # return b"" when done + + output_chunks_cb = [] + def write_callback(data): + output_chunks_cb.append(data) + return 0 # 0 = success + + result = dataweave.run_input_output_callback( + "output application/json deferred=true --- payload map ($ * 2)", + input_name="payload", + input_mime_type="application/json", + read_callback=read_callback, + write_callback=write_callback + ) + + print(f" Callback result: success={result.success}") + output_cb = b"".join(output_chunks_cb).decode('utf-8') + print(f" Output: {output_cb}\n") + + # Example 7: Using with context manager + print("7. Streaming with context manager:") + with dataweave.DataWeave() as dw: + stream = dw.run_streaming("output application/csv --- (1 to 5)") + csv_output = b"".join(stream).decode('utf-8') + print(f" CSV output: {csv_output}") + print() + + # Example 8: Error handling in streaming + print("8. Error handling in streaming:") + stream = dataweave.run_streaming("output application/json --- invalid syntax here") + + # Drain chunks (there may be none) + for chunk in stream: + print(f" Unexpected chunk: {chunk}") + + # Check metadata for error + if not stream.metadata.success: + print(f" Error caught in metadata: {stream.metadata.error[:50]}...") + print() + + print("Streaming demo complete!") + +if __name__ == "__main__": + try: + main() + except dataweave.DataWeaveLibraryNotFoundError: + print("\nERROR: Native library not found!") + print("Build it first: ./gradlew :native-lib:nativeCompile") + sys.exit(1) + except Exception as e: + print(f"\nUnexpected error: {e}") + import traceback + traceback.print_exc() + sys.exit(1)