diff --git a/doc/datatypes.md b/doc/datatypes.md index 578ce05..ad17b72 100644 --- a/doc/datatypes.md +++ b/doc/datatypes.md @@ -18,3 +18,6 @@ * * [bitfield](./datatypes/utils.md) * * [mapper](./datatypes/utils.md) * * [pstring](./datatypes/utils.md) +* [Extras](./datatypes/extras.md) +* * [loop](./datatypes/extras.md#loop) +* * [restBuffer](./datatypes/extras.md#restbuffer) diff --git a/doc/datatypes/extras.md b/doc/datatypes/extras.md new file mode 100644 index 0000000..a27aaaa --- /dev/null +++ b/doc/datatypes/extras.md @@ -0,0 +1,162 @@ +# Extras Datatypes + +This document describes the extra datatypes available in ProtoDef that provide advanced functionality for specific use cases. + +## loop + +The `loop` datatype reads a sequence of elements until a terminator or end of buffer is reached. + +### Parameters + +- `type` - The datatype of each element in the loop +- `nt` (null terminator) - Optional terminator value. Can be: + - `null` - Read until end of buffer + - A number - Read until this byte value is encountered (terminator is consumed) + +### Usage Examples + +#### Loop without terminator (read until EOF) +```javascript +const proto = new ProtoDef() + +// Define a protocol that reads i8 values until end of buffer +proto.addType('numbers', ['loop', { type: 'i8', nt: null }]) + +// Parse a buffer containing [1, 2, 3, 4, 5] +const buffer = Buffer.from([1, 2, 3, 4, 5]) +const result = proto.parsePacketBuffer('numbers', buffer) +console.log(result.data) // [1, 2, 3, 4, 5] +``` + +#### Loop with terminator +```javascript +const proto = new ProtoDef() + +// Define a protocol that reads i8 values until 0 is encountered +proto.addType('terminated_numbers', ['loop', { type: 'i8', nt: 0 }]) + +// Parse a buffer containing [1, 2, 3, 0, 99] - 99 won't be read +const buffer = Buffer.from([1, 2, 3, 0, 99]) +const result = proto.parsePacketBuffer('terminated_numbers', buffer) +console.log(result.data) // [1, 2, 3] +``` + +#### Loop with complex types +```javascript +const proto = new ProtoDef() + +// Define a container type +proto.addType('point', ['container', [ + { name: 'x', type: 'i16' }, + { name: 'y', type: 'i16' } +]]) + +// Define a loop of points until end of buffer +proto.addType('points', ['loop', { type: 'point', nt: null }]) + +// Parse multiple points +const buffer = Buffer.from([0, 10, 0, 20, 0, 30, 0, 40]) // 4 bytes per point +const result = proto.parsePacketBuffer('points', buffer) +console.log(result.data) // [{ x: 10, y: 20 }, { x: 30, y: 40 }] +``` + +### Writing with loop +```javascript +const proto = new ProtoDef() +proto.addType('numbers', ['loop', { type: 'i8', nt: 0 }]) + +// Write an array with terminator +const data = [1, 2, 3, 4, 5] +const buffer = proto.createPacketBuffer('numbers', data) +console.log(buffer) // Buffer containing [1, 2, 3, 4, 5, 0] +``` + +## restBuffer + +The `restBuffer` datatype reads all remaining bytes in the buffer and returns them as a Buffer object. + +### Parameters + +None - `restBuffer` takes no parameters. + +### Usage Examples + +#### Basic restBuffer usage +```javascript +const proto = new ProtoDef() + +// Define a protocol with a header and remaining data +proto.addType('packet', ['container', [ + { name: 'header', type: 'u8' }, + { name: 'payload', type: 'restBuffer' } +]]) + +// Parse a buffer where first byte is header, rest is payload +const buffer = Buffer.from([42, 0x48, 0x65, 0x6c, 0x6c, 0x6f]) // 42 + "Hello" +const result = proto.parsePacketBuffer('packet', buffer) +console.log(result.data.header) // 42 +console.log(result.data.payload) // Buffer containing [0x48, 0x65, 0x6c, 0x6c, 0x6f] +console.log(result.data.payload.toString()) // "Hello" +``` + +#### Protocol with multiple fields and restBuffer +```javascript +const proto = new ProtoDef() + +proto.addType('message', ['container', [ + { name: 'version', type: 'u8' }, + { name: 'type', type: 'u8' }, + { name: 'length', type: 'u16' }, + { name: 'data', type: 'restBuffer' } +]]) + +const buffer = Buffer.from([1, 2, 0, 10, ...Buffer.from('Binary data here')]) +const result = proto.parsePacketBuffer('message', buffer) +console.log(result.data) +/* +{ + version: 1, + type: 2, + length: 10, + data: Buffer containing "Binary data here" +} +*/ +``` + +#### Writing with restBuffer +```javascript +const proto = new ProtoDef() +proto.addType('simple', ['container', [ + { name: 'id', type: 'u8' }, + { name: 'payload', type: 'restBuffer' } +]]) + +const data = { + id: 123, + payload: Buffer.from('Some binary payload data') +} + +const buffer = proto.createPacketBuffer('simple', data) +// Buffer will contain: [123, ...payload bytes] +``` + +## Use Cases + +### loop +- **Variable-length arrays without length prefix**: When the array length isn't known ahead of time +- **Null-terminated lists**: Similar to C-style null-terminated strings but for any data type +- **Streaming data**: Reading elements until a sentinel value or end of stream +- **Protocol parsing**: Reading repeated elements until a stop condition + +### restBuffer +- **Flexible payload handling**: When you need to capture all remaining data as binary +- **Nested protocol parsing**: Pass the remaining buffer to another parser +- **Variable-length data**: When the remaining data length is implicit +- **Raw data preservation**: Keeping binary data intact for further processing + +## Performance Considerations + +- `loop` reads elements one by one, so it may be slower than fixed-size arrays for large datasets +- `restBuffer` is very efficient as it just slices the remaining buffer +- Both types are fully supported in compiled mode for optimal performance +- Use `loop` with terminators when possible to avoid reading the entire buffer unnecessarily diff --git a/schemas/datatype.json b/schemas/datatype.json index 66b37d8..36facf7 100644 --- a/schemas/datatype.json +++ b/schemas/datatype.json @@ -45,6 +45,9 @@ { "$ref": "buffer" }, { "$ref": "bitfield" }, { "$ref": "bitflags" }, - { "$ref": "mapper" } + { "$ref": "mapper" }, + + { "$ref": "loop" }, + { "$ref": "restBuffer" } ] } diff --git a/schemas/extras.json b/schemas/extras.json new file mode 100644 index 0000000..a5a96dc --- /dev/null +++ b/schemas/extras.json @@ -0,0 +1,42 @@ +{ + "loop": { + "title": "loop", + "type": "array", + "items": [ + { + "enum": ["loop"] + }, + { + "type": "object", + "properties": { + "type": { + "$ref": "dataType" + }, + "nt": { + "oneOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "required": ["type", "nt"] + } + ], + "additionalItems": false + }, + "restBuffer": { + "title": "restBuffer", + "type": "array", + "items": [ + { + "enum": ["restBuffer"] + } + ], + "additionalItems": false + } +} diff --git a/test/extras.json b/test/extras.json new file mode 100644 index 0000000..2c4af1a --- /dev/null +++ b/test/extras.json @@ -0,0 +1,103 @@ +[ + { + "type": "loop", + "subtypes": [ + { + "description": "loop without terminator", + "type": [ + "loop", + { + "type": "i8", + "nt": null + } + ], + "values": [ + { + "description": "empty loop without terminator", + "buffer": [], + "value": [] + }, + { + "description": "single element loop without terminator", + "buffer": ["0x05"], + "value": [5] + }, + { + "description": "multiple elements loop without terminator", + "buffer": ["0x01", "0x02", "0x03"], + "value": [1, 2, 3] + } + ] + }, + { + "description": "loop with terminator", + "type": [ + "loop", + { + "type": "i8", + "nt": 0 + } + ], + "values": [ + { + "description": "empty loop with terminator", + "buffer": ["0x00"], + "value": [] + }, + { + "description": "single element loop with terminator", + "buffer": ["0x05", "0x00"], + "value": [5] + }, + { + "description": "multiple elements loop with terminator", + "buffer": ["0x01", "0x02", "0x03", "0x00"], + "value": [1, 2, 3] + } + ] + }, + { + "description": "loop of 16-bit integers", + "type": [ + "loop", + { + "type": "i16", + "nt": null + } + ], + "values": [ + { + "description": "loop of 16-bit integers without terminator", + "buffer": ["0x00", "0x01", "0x00", "0x02"], + "value": [1, 2] + } + ] + } + ] + }, + { + "type": "restBuffer", + "values": [ + { + "description": "empty rest buffer", + "buffer": [], + "value": [] + }, + { + "description": "single byte rest buffer", + "buffer": ["0xFF"], + "value": ["0xFF"] + }, + { + "description": "multiple bytes rest buffer", + "buffer": ["0x48", "0x65", "0x6C", "0x6C", "0x6F"], + "value": ["0x48", "0x65", "0x6C", "0x6C", "0x6F"] + }, + { + "description": "binary data rest buffer", + "buffer": ["0x00", "0x01", "0x02", "0x03", "0x04", "0x05"], + "value": ["0x00", "0x01", "0x02", "0x03", "0x04", "0x05"] + } + ] + } +]