diff --git a/PHILOSOPHY.md b/PHILOSOPHY.md index 61f3fc2..8bb80df 100644 --- a/PHILOSOPHY.md +++ b/PHILOSOPHY.md @@ -54,12 +54,12 @@ These edge cases are handled in a way that is consistent with the principles abo | Missing final newline | `a,b,c` EOF | The parser should not require a final newline character at the end of the file, as per the RFC 4180 standard. | | Missing header row | - | The header row is optional, as per the RFC 4180 standard. The parser should be able to handle files without a header row. | -| Unsupported Cases | Example | Reasoning | -| --------------------------------------- | -------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | -| Empty lines | `a,b\n\nc` | Empty lines may be a result of a misconfiguration or an error in the data. The parser should fail-fast to avoid cascading errors. | -| Inconsistent row lengths | `a,b\nc,d,e` | Fail-fast on inconsistent row lengths to ensure data integrity. Each row should have the same number of fields. | -| Backslash escaping | `a\,b` → `["a,b"]` or `a,"\"",c` → `["a,\"", "c"]` | Backslash escaping is currently unsupported as it is not part of the RFC 4180 standard and could lead to ambiguity. | -| Fields with comment-style trailing text | `a,b # note` | Trailing comments are parsed verbatim and as part of the field. This should be avoided to prevent ambiguity. | +| Unsupported Cases | Example | Reasoning | +| --------------------------------------- | -------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Empty lines | `a,b\n\nc` | Empty lines may be a result of a misconfiguration or an error in the data. The parser should fail-fast to avoid cascading errors. | +| Inconsistent row lengths | `a,b\nc,d,e` | Fail-fast on inconsistent row lengths to ensure data integrity. Each row should have the same number of fields. | +| Backslash escaping | `a\,b` → `["a,b"]` or `a,"\"",c` → `["a,\"", "c"]` | Backslash escaping is currently unsupported as it is not part of the RFC 4180 standard and could lead to ambiguity. However, it may be enabled if required. | +| Fields with comment-style trailing text | `a,b # note` | Trailing comments are parsed verbatim and as part of the field. This should be avoided to prevent ambiguity. | # References diff --git a/README.md b/README.md index ebebf9c..02e0308 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ const Animal = struct { // Parse CSV data into a StructuredTable var table = csv.StructuredTable(Animal).init( allocator, - csv.Settings.default() + csv.LexerSettings.default() ); defer table.deinit(); try table.parse( @@ -116,7 +116,7 @@ const csv = @import("zig_csv"); const allocator = std.heap.page_allocator; // Parse CSV data -var table = csv.Table.init(allocator, csv.Settings.default()); +var table = csv.Table.init(allocator, csv.LexerSettings.default()); defer table.deinit(); try table.parse( \\id,animal,color diff --git a/build.zig.zon b/build.zig.zon index ed0f3b1..274de65 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -1,14 +1,15 @@ .{ .name = .zig_csv, - .version = "1.2.0", + .version = "2.2.0", .fingerprint = 0xb5163165a35b825b, .minimum_zig_version = "0.16.0", .dependencies = .{}, .paths = .{ + "LICENSE.txt", + "PHILOSOPHY.md", + "README.md", "build.zig", "build.zig.zon", "src", - "LICENSE.txt", - "README.md", }, } diff --git a/src/parser.zig b/src/parser.zig new file mode 100644 index 0000000..789195a --- /dev/null +++ b/src/parser.zig @@ -0,0 +1,231 @@ +const std = @import("std"); +const table = @import("table.zig"); +const ArrayList = std.ArrayList; +const Allocator = std.mem.Allocator; + +pub const Token = enum { + DELIMITER, + ESCAPE, + QUOTE, + TERMINATOR, + TEXT, +}; + +pub const TokenValuePair = struct { + token: Token, + value: ?[]const u8, +}; + +pub const LexerSettings = struct { + /// Byte sequence used to separate fields within a row. + delimiter: []const u8, + /// Byte sequence used to escape special syntax within a field. + /// A null value disables this functionality. + escape: ?[]const u8, + /// Byte sequence used to wrap fields as a whole to allow + /// use of delimiters and terminators inside them. + quote: []const u8, + /// Byte sequence used to end a row (i.e., terminates a record). + terminator: []const u8, + + pub fn default() LexerSettings { + return LexerSettings{ + .delimiter = ",", + .escape = null, + .quote = "\"", + .terminator = "\n", + }; + } +}; + +pub const Lexer = struct { + cursor: usize = 0, + data: []const u8, + settings: LexerSettings, + text_start: ?usize, + + pub const TOKEN_FIELDS = .{ + .{ .token = Token.DELIMITER, .field_name = "delimiter" }, + .{ .token = Token.ESCAPE, .field_name = "escape" }, + .{ .token = Token.QUOTE, .field_name = "quote" }, + .{ .token = Token.TERMINATOR, .field_name = "terminator" }, + }; + + pub fn init(settings: LexerSettings, csv_data: []const u8) Lexer { + return Lexer{ + .cursor = 0, + .data = csv_data, + .settings = settings, + .text_start = null, + }; + } + + pub fn next(self: *Lexer) ?TokenValuePair { + while (true) { + if (self.cursor == self.data.len and self.text_start != null) { + self.cursor += 1; + return .{ .token = Token.TEXT, .value = self.data[self.text_start orelse unreachable ..] }; + } + if (self.cursor >= self.data.len) return null; + const lookahead = self.data[self.cursor..]; + inline for (TOKEN_FIELDS) |token_field| { + const token_type = token_field.token; + const token_byteseq: ?[]const u8 = @field(self.settings, token_field.field_name); + if (token_byteseq != null and + std.mem.startsWith(u8, lookahead, token_byteseq.?)) + { + const text_start = self.text_start; + if (text_start != null) { + self.text_start = null; + return .{ .token = Token.TEXT, .value = self.data[text_start orelse unreachable .. self.cursor] }; + } + self.cursor += token_byteseq.?.len; + return .{ .token = token_type, .value = null }; + } + } + if (self.text_start == null) { + self.text_start = self.cursor; + } + self.cursor += 1; + } + } +}; + +pub const ParserError = error{ + /// Certain TokenValuePairs require a value other than null + ExpectedValueFoundNull, + /// Quote sequences are expeced at beginning and end of a field + IllegalQuotation, + /// Not enough memory + OutOfMemory, + /// Starting a field with the QUOTE sequence requires the field + /// to be closed with the same sequence as well + UnfinishedQuotation, +}; + +pub const ParsingResult = union(enum) { + field: []u8, + end_of_row: void, +}; + +pub const Parser = struct { + expect_field_end_after_quotation: bool, + in_quotes: bool, + is_escaped: bool, + is_row_finished: bool, + last_token_pair: ?TokenValuePair, + lexer: Lexer, + settings: LexerSettings, + + pub fn init(settings: LexerSettings, csv_data: []const u8) Parser { + const lexer = Lexer.init(settings, csv_data); + return Parser{ + .expect_field_end_after_quotation = false, + .in_quotes = false, + .is_escaped = false, + .is_row_finished = false, + .last_token_pair = null, + .lexer = lexer, + .settings = settings, + }; + } + + pub fn next(self: *Parser, allocator: Allocator) ParserError!?ParsingResult { + if (self.is_row_finished) { + self.is_row_finished = false; + return .end_of_row; + } + var field: ArrayList(u8) = .empty; + errdefer field.deinit(allocator); + while (true) { + const token_value_pair = self.lexer.next(); + if (self.expect_field_end_after_quotation and token_value_pair != null) { + switch (token_value_pair.?.token) { + Token.DELIMITER, Token.TERMINATOR => { + self.expect_field_end_after_quotation = false; + }, + // consecutive double quotes are ignored within an escaped field + Token.QUOTE => { + self.in_quotes = !self.in_quotes; + self.expect_field_end_after_quotation = false; + self.is_escaped = true; + }, + // confirm closing quote token only right before ending a field + else => { + return ParserError.IllegalQuotation; + }, + } + } + if (token_value_pair == null) break; + defer self.last_token_pair = token_value_pair; + if (self.is_escaped) { + self.is_escaped = false; + inline for (Lexer.TOKEN_FIELDS) |token_field| { + const token_type = token_field.token; + const token_byteseq: ?[]const u8 = @field(self.settings, token_field.field_name); + if (token_byteseq != null and token_value_pair.?.token == token_type) { + try field.appendSlice(allocator, token_byteseq.?); + break; + } + } + continue; + } + if (token_value_pair.?.token == Token.ESCAPE) { + self.is_escaped = true; + } else if (token_value_pair.?.token == Token.QUOTE) { + // confirm quote token right after starting new field + if ( + // quotation must not be toggled on for we assume + // a new quotation + !self.in_quotes and + // ensure the previous field was closed + self.last_token_pair != null and + (self.last_token_pair.?.token != Token.DELIMITER and + self.last_token_pair.?.token != Token.TERMINATOR)) + { + return ParserError.IllegalQuotation; + } + if (self.in_quotes) self.expect_field_end_after_quotation = true; + self.in_quotes = !self.in_quotes; + } else if (token_value_pair.?.token == Token.TEXT) { + if (token_value_pair.?.value == null) { + return ParserError.ExpectedValueFoundNull; + } + try field.appendSlice(allocator, token_value_pair.?.value.?); + } else if (self.is_escaped) { + continue; + } else if (token_value_pair.?.token == Token.DELIMITER or + token_value_pair.?.token == Token.TERMINATOR) + { + if (self.in_quotes) { + switch (token_value_pair.?.token) { + Token.DELIMITER => try field.appendSlice(allocator, self.settings.delimiter), + Token.TERMINATOR => try field.appendSlice(allocator, self.settings.terminator), + else => unreachable, + } + } else { + if (token_value_pair.?.token == Token.TERMINATOR) + self.is_row_finished = true; + return ParsingResult{ + .field = try field.toOwnedSlice(allocator), + }; + } + } + } + if (self.in_quotes) { + return ParserError.UnfinishedQuotation; + } + if (field.items.len > 0 or + (self.last_token_pair != null and (self.last_token_pair.?.token == Token.DELIMITER or + self.last_token_pair.?.token == Token.QUOTE))) + { + // ensure it does not match the condition on the + // next run, throwing the same field twice + self.last_token_pair = null; + return ParsingResult{ + .field = try field.toOwnedSlice(allocator), + }; + } + return null; + } +}; diff --git a/src/root.zig b/src/root.zig index adf1b14..8ce34ff 100644 --- a/src/root.zig +++ b/src/root.zig @@ -1,12 +1,14 @@ -const table = @import("table.zig"); -const schema = @import("schema.zig"); +pub const parser = @import("parser.zig"); +pub const schema = @import("schema.zig"); +pub const table = @import("table.zig"); /// Thin root module that re-exports the core Table implementation and the schema /// module. This avoids circular import issues by keeping the core implementation /// in `table.zig` while allowing consumers to import this single entrypoint. -pub const Table = table.Table; -pub const Settings = table.Settings; -pub const TableError = table.TableError; -pub const StructureError = schema.StructureError; +pub const LexerSettings = parser.LexerSettings; pub const ParseResult = schema.ParseResult; +pub const ParserError = parser.ParserError; +pub const StructureError = schema.StructureError; pub const StructuredTable = schema.StructuredTable; +pub const Table = table.Table; +pub const TableError = table.TableError; diff --git a/src/schema.zig b/src/schema.zig index 39611ab..325b0d7 100644 --- a/src/schema.zig +++ b/src/schema.zig @@ -1,10 +1,12 @@ const std = @import("std"); const table = @import("table.zig"); +const parser = @import("parser.zig"); const Allocator = std.mem.Allocator; const ArrayList = std.ArrayList; const Table = table.Table; const TableError = table.TableError; -const Settings = table.Settings; +const LexerSettings = parser.LexerSettings; +const ParserError = parser.ParserError; /// Errors that can occur when mapping CSV data to a structured type pub const StructureError = error{ @@ -51,7 +53,7 @@ pub fn StructuredTable(table_schema: type) type { /// The underlying CSV table table: Table, /// The settings that should be used when parsing the CSV data - settings: Settings, + settings: LexerSettings, /// The allocator used for memory management allocator: Allocator, /// An arena allocator for dangling allocations @@ -60,7 +62,7 @@ pub fn StructuredTable(table_schema: type) type { const Self = @This(); /// Initialize a new StructuredTable - pub fn init(allocator: Allocator, settings: Settings) Self { + pub fn init(allocator: Allocator, settings: LexerSettings) Self { return Self{ .table = Table.init(allocator, settings), .settings = settings, @@ -76,7 +78,7 @@ pub fn StructuredTable(table_schema: type) type { } /// Parse CSV data into the StructuredTable - pub fn parse(self: *Self, csv_data: []const u8) (TableError || StructureError)!void { + pub fn parse(self: *Self, csv_data: []const u8) (TableError || StructureError || ParserError)!void { try self.table.parse(csv_data); if (self.table.getColumnCount() != schema_info.@"struct".fields.len) return StructureError.InvalidColumnCount; } diff --git a/src/table.zig b/src/table.zig index cc7c0c9..0b5106b 100644 --- a/src/table.zig +++ b/src/table.zig @@ -1,21 +1,10 @@ const std = @import("std"); +const parser = @import("parser.zig"); const Allocator = std.mem.Allocator; const ArrayList = std.ArrayList; - -/// A structure for storing settings for use with struct Table -pub const Settings = struct { - /// The delimiter that separates the values (aka. separator) - delimiter: []const u8, - /// The terminator that defines when a row of delimiter-separated values is terminated - terminator: []const u8, - - pub fn default() Settings { - return Settings{ - .delimiter = ",", - .terminator = "\n", - }; - } -}; +const LexerSettings = parser.LexerSettings; +const Parser = parser.Parser; +const ParserError = parser.ParserError; /// Errors that may return from struct Table pub const TableError = error{ @@ -29,50 +18,59 @@ pub const TableError = error{ /// A structure for parsing and manipulating CSV data pub const Table = struct { - settings: Settings, allocator: Allocator, - expected_column_count: ?usize, data: ArrayList(ArrayList([]const u8)), + expected_column_count: ?usize, + settings: LexerSettings, - pub fn init(allocator: Allocator, settings: Settings) Table { + pub fn init(allocator: Allocator, settings: LexerSettings) Table { return Table{ - .settings = settings, .allocator = allocator, - .expected_column_count = null, .data = .empty, + .expected_column_count = null, + .settings = settings, }; } pub fn deinit(self: *Table) void { for (self.data.items) |*row| { + for (row.items) |field| self.allocator.free(field); row.deinit(self.allocator); } self.data.deinit(self.allocator); } - pub fn parse(self: *Table, csv_data: []const u8) TableError!void { - const csv_data_sanitized = std.mem.trimEnd(u8, csv_data, self.settings.terminator); - var rows = std.mem.splitSequence(u8, csv_data_sanitized, self.settings.terminator); - while (rows.next()) |row| { - const value_count = try self.parseRow(row); + pub fn parse(self: *Table, csv_data: []const u8) (TableError || ParserError)!void { + var csv_parser = Parser.init(self.settings, csv_data); + var row: ArrayList([]const u8) = .empty; + errdefer { + for (row.items) |field| self.allocator.free(field); + row.deinit(self.allocator); + } + while (try csv_parser.next(self.allocator)) |result| switch (result) { + .field => try row.append(self.allocator, result.field), + .end_of_row => { + if (self.expected_column_count == null) { + self.expected_column_count = row.items.len; + } + if (self.expected_column_count != row.items.len) { + return TableError.InconsistentRowLength; + } + try self.data.append(self.allocator, row); + row = .empty; + }, + }; + if (row.items.len > 0) { if (self.expected_column_count == null) { - self.expected_column_count = value_count; - } else if (value_count != self.expected_column_count) { + self.expected_column_count = row.items.len; + } + if (self.expected_column_count != row.items.len) { return TableError.InconsistentRowLength; } + try self.data.append(self.allocator, row); } } - fn parseRow(self: *Table, row: []const u8) TableError!usize { - var values: ArrayList([]const u8) = .empty; - var columns = std.mem.splitSequence(u8, row, self.settings.delimiter); - while (columns.next()) |value| { - try values.append(self.allocator, value); - } - try self.data.append(self.allocator, values); - return values.items.len; - } - pub fn getRowCount(self: Table) usize { return self.data.items.len; } @@ -160,13 +158,15 @@ pub const Table = struct { if (column_index >= (self.expected_column_count orelse 0)) return TableError.ColumnNotFound; if (std.mem.indexOf(u8, new_value, self.settings.delimiter) != null) return TableError.IllegalCharacter; if (std.mem.indexOf(u8, new_value, self.settings.terminator) != null) return TableError.IllegalCharacter; - self.data.items[row_index].items[column_index] = new_value; + self.allocator.free(self.data.items[row_index].items[column_index]); + self.data.items[row_index].items[column_index] = try self.allocator.dupe(u8, new_value); } pub fn deleteColumnByIndex(self: *Table, column_index: usize) TableError!void { if (self.expected_column_count == null) return TableError.ColumnNotFound; if (column_index >= (self.expected_column_count orelse 0)) return TableError.ColumnNotFound; for (self.data.items) |*row| { + self.allocator.free(row.items[column_index]); _ = row.orderedRemove(column_index); } self.expected_column_count = (self.expected_column_count orelse 0) - 1; @@ -174,6 +174,7 @@ pub const Table = struct { pub fn deleteRowByIndex(self: *Table, row_index: usize) TableError!void { if (row_index >= self.data.items.len) return TableError.RowNotFound; + for (self.data.items[row_index].items) |field| self.allocator.free(field); self.data.items[row_index].deinit(self.allocator); _ = self.data.orderedRemove(row_index); } @@ -188,7 +189,25 @@ pub const Table = struct { if (column_index > 0) { try csv.appendSlice(allocator, self.settings.delimiter); } - try csv.appendSlice(allocator, column); + const requires_quotation = (std.mem.containsAtLeast(u8, column, 1, self.settings.delimiter) or + std.mem.containsAtLeast(u8, column, 1, self.settings.terminator) or + std.mem.containsAtLeast(u8, column, 1, self.settings.quote)); + if (requires_quotation) try csv.appendSlice(allocator, self.settings.quote); + var cursor: usize = 0; + while (cursor < column.len) { + const lookahead = column[cursor..]; + if (self.settings.escape != null and + std.mem.startsWith(u8, lookahead, self.settings.escape.?)) + { + try csv.appendSlice(allocator, self.settings.escape.?); + } else if (std.mem.startsWith(u8, lookahead, self.settings.quote)) { + // double quote escapes the quote as per RFC + try csv.appendSlice(allocator, self.settings.quote); + } + try csv.append(allocator, column[cursor]); + cursor += 1; + } + if (requires_quotation) try csv.appendSlice(allocator, self.settings.quote); } } return csv.toOwnedSlice(allocator); diff --git a/src/tests/parser.zig b/src/tests/parser.zig new file mode 100644 index 0000000..d2a95d3 --- /dev/null +++ b/src/tests/parser.zig @@ -0,0 +1,204 @@ +const std = @import("std"); +const csv = @import("zig_csv"); +const allocator = std.testing.allocator; +const expect = std.testing.expect; +const Lexer = csv.parser.Lexer; +const LexerSettings = csv.parser.LexerSettings; +const Parser = csv.parser.Parser; +const ParserError = csv.parser.ParserError; +const Token = csv.parser.Token; +const TokenValuePair = csv.parser.TokenValuePair; + +fn expect_token_value_pair(token_value_pair: TokenValuePair, expected_token_value_pair: TokenValuePair) anyerror!void { + try expect(token_value_pair.token == expected_token_value_pair.token); + if (token_value_pair.value == null or expected_token_value_pair.value == null) { + try expect(token_value_pair.value == null); + try expect(expected_token_value_pair.value == null); + } else { + try expect(std.mem.eql(u8, token_value_pair.value.?, expected_token_value_pair.value.?)); + } +} + +fn expect_parser_field_next(parser: *Parser, expected: []const u8) anyerror!void { + const result = try parser.next(allocator); + try expect(result != null); + try expect(result.? == .field); + defer allocator.free(result.?.field); + try expect(std.mem.eql(u8, result.?.field, expected)); +} + +test "Lexer(Token.DELIMITER): basic test" { + const data = + \\ABC,DEF + ; + var lexer = Lexer.init(LexerSettings.default(), data); + try expect_token_value_pair(lexer.next().?, .{ .token = Token.TEXT, .value = "ABC" }); + try expect_token_value_pair(lexer.next().?, .{ .token = Token.DELIMITER, .value = null }); + try expect_token_value_pair(lexer.next().?, .{ .token = Token.TEXT, .value = "DEF" }); + try expect(lexer.next() == null); +} + +test "Lexer(Token.TERMINATOR): basic test" { + const data = + \\ABC + \\DEF + ; + var lexer = Lexer.init(LexerSettings.default(), data); + try expect_token_value_pair(lexer.next().?, .{ .token = Token.TEXT, .value = "ABC" }); + try expect_token_value_pair(lexer.next().?, .{ .token = Token.TERMINATOR, .value = null }); + try expect_token_value_pair(lexer.next().?, .{ .token = Token.TEXT, .value = "DEF" }); + try expect(lexer.next() == null); +} + +test "Lexer(Token.QUOTE): basic test" { + const data = + \\"ABC" + ; + var lexer = Lexer.init(LexerSettings.default(), data); + try expect_token_value_pair(lexer.next().?, .{ .token = Token.QUOTE, .value = null }); + try expect_token_value_pair(lexer.next().?, .{ .token = Token.TEXT, .value = "ABC" }); + try expect_token_value_pair(lexer.next().?, .{ .token = Token.QUOTE, .value = null }); + try expect(lexer.next() == null); +} + +test "Lexer(Token.ESCAPE): basic test" { + const data = + \\ABC\,DEF + ; + var settings = LexerSettings.default(); + settings.escape = "\\"; + var lexer = Lexer.init(settings, data); + try expect_token_value_pair(lexer.next().?, .{ .token = Token.TEXT, .value = "ABC" }); + try expect_token_value_pair(lexer.next().?, .{ .token = Token.ESCAPE, .value = null }); + try expect_token_value_pair(lexer.next().?, .{ .token = Token.DELIMITER, .value = null }); + try expect_token_value_pair(lexer.next().?, .{ .token = Token.TEXT, .value = "DEF" }); + try expect(lexer.next() == null); +} + +test "Parser(DELIMITER): basic test" { + const data = + \\ABC,DEF + ; + var parser = Parser.init(LexerSettings.default(), data); + try expect_parser_field_next(&parser, "ABC"); + try expect_parser_field_next(&parser, "DEF"); + try expect(try parser.next(allocator) == null); +} + +test "Parser(TERMINATOR): basic test" { + const data = + \\ABC + \\DEF + ; + var parser = Parser.init(LexerSettings.default(), data); + try expect_parser_field_next(&parser, "ABC"); + try expect((try parser.next(allocator)).? == .end_of_row); + try expect_parser_field_next(&parser, "DEF"); + try expect(try parser.next(allocator) == null); +} + +test "Parser(QUOTE): legal quotation cases" { + const data = + \\"" + \\"ABC" + \\"DEF",GHI + \\JKL,"MNO" + \\"PQR","STU" + ; + var parser = Parser.init(LexerSettings.default(), data); + try expect_parser_field_next(&parser, ""); + try expect((try parser.next(allocator)).? == .end_of_row); + try expect_parser_field_next(&parser, "ABC"); + try expect((try parser.next(allocator)).? == .end_of_row); + try expect_parser_field_next(&parser, "DEF"); + try expect_parser_field_next(&parser, "GHI"); + try expect((try parser.next(allocator)).? == .end_of_row); + try expect_parser_field_next(&parser, "JKL"); + try expect_parser_field_next(&parser, "MNO"); + try expect((try parser.next(allocator)).? == .end_of_row); + try expect_parser_field_next(&parser, "PQR"); + try expect_parser_field_next(&parser, "STU"); + try expect(try parser.next(allocator) == null); +} + +test "Parser(QUOTE): ignores delimiters" { + const data = + \\";" + ; + var parser = Parser.init(LexerSettings.default(), data); + try expect_parser_field_next(&parser, ";"); + try expect(try parser.next(allocator) == null); +} + +test "Parser(QUOTE): ignores terminators" { + const data = + \\" + \\" + ; + var parser = Parser.init(LexerSettings.default(), data); + try expect_parser_field_next(&parser, "\n"); + try expect(try parser.next(allocator) == null); +} + +test "Parser(QUOTE): illegal quotation" { + const data = + \\"ABC" DEF,GHI + ; + var parser = Parser.init(LexerSettings.default(), data); + try expect(parser.next(allocator) == ParserError.IllegalQuotation); +} + +test "Parser(QUOTE): unfinished quotation" { + const data = + \\"ABC,DEF + ; + var parser = Parser.init(LexerSettings.default(), data); + try expect(parser.next(allocator) == ParserError.UnfinishedQuotation); +} + +test "Parser(ESCAPE): ignores delimiters" { + const data = + \\\, + ; + var settings = LexerSettings.default(); + settings.escape = "\\"; + var parser = Parser.init(settings, data); + try expect_parser_field_next(&parser, ","); + try expect(try parser.next(allocator) == null); +} + +test "Parser(ESCAPE): ignores terminators" { + const data = + \\\ + \\ + ; + var settings = LexerSettings.default(); + settings.escape = "\\"; + var parser = Parser.init(settings, data); + try expect_parser_field_next(&parser, "\n"); + try expect(try parser.next(allocator) == null); +} + +test "Parser(ESCAPE): ignores quotes" { + const data = + \\\" + ; + var settings = LexerSettings.default(); + settings.escape = "\\"; + var parser = Parser.init(settings, data); + try expect_parser_field_next(&parser, "\""); + try expect(try parser.next(allocator) == null); +} + +test "Parser: empty fields" { + const data = + \\ + \\, + ; + var parser = Parser.init(LexerSettings.default(), data); + try expect_parser_field_next(&parser, ""); + try expect((try parser.next(allocator)).? == .end_of_row); + try expect_parser_field_next(&parser, ""); + try expect_parser_field_next(&parser, ""); + try expect(try parser.next(allocator) == null); +} diff --git a/src/tests/root.zig b/src/tests/root.zig index 6b3cb13..d504327 100644 --- a/src/tests/root.zig +++ b/src/tests/root.zig @@ -5,6 +5,7 @@ // Import test files as anonymous comptime blocks so they don't create duplicate // top-level symbols in this module. comptime { + _ = @import("parser.zig"); _ = @import("schema.zig"); _ = @import("table.zig"); } diff --git a/src/tests/schema.zig b/src/tests/schema.zig index c15198d..a9560e7 100644 --- a/src/tests/schema.zig +++ b/src/tests/schema.zig @@ -12,7 +12,7 @@ test "StructuredTable: Parse CSV into struct and access rows" { foo: f32, }; - var table = StructuredTable(DogTable).init(allocator, csv.Settings.default()); + var table = StructuredTable(DogTable).init(allocator, csv.LexerSettings.default()); defer table.deinit(); try table.parse( \\name,age,alive,foo @@ -46,7 +46,7 @@ test "StructuredTable: Edit struct row and export to CSV" { foo: f32, }; - var table = StructuredTable(DogTable).init(allocator, csv.Settings.default()); + var table = StructuredTable(DogTable).init(allocator, csv.LexerSettings.default()); defer table.deinit(); try table.parse( \\name,age,alive,foo @@ -81,7 +81,7 @@ test "StructuredTable: Delete struct row" { foo: f32, }; - var table = StructuredTable(DogTable).init(allocator, csv.Settings.default()); + var table = StructuredTable(DogTable).init(allocator, csv.LexerSettings.default()); defer table.deinit(); try table.parse( \\name,age,alive,foo @@ -111,7 +111,7 @@ test "StructuredTable: Create empty struct table and insert rows" { foo: f32, }; - var table = StructuredTable(DogTable).init(allocator, csv.Settings.default()); + var table = StructuredTable(DogTable).init(allocator, csv.LexerSettings.default()); defer table.deinit(); const new_row_1 = DogTable{ @@ -150,7 +150,7 @@ test "StructuredTable: Insert row at specific index" { foo: f32, }; - var table = StructuredTable(DogTable).init(allocator, csv.Settings.default()); + var table = StructuredTable(DogTable).init(allocator, csv.LexerSettings.default()); defer table.deinit(); try table.parse( \\name,age,alive,foo @@ -187,7 +187,7 @@ test "StructuredTable: Handle parsing error due to invalid csv type" { foo: f32, }; - var table = StructuredTable(DogTable).init(allocator, csv.Settings.default()); + var table = StructuredTable(DogTable).init(allocator, csv.LexerSettings.default()); defer table.deinit(); try table.parse( \\name,age,alive,foo @@ -209,7 +209,7 @@ test "StructuredTable: Optional fields parse and null behavior" { foo: ?f32, }; - var table = StructuredTable(DogTableOpt).init(allocator, csv.Settings.default()); + var table = StructuredTable(DogTableOpt).init(allocator, csv.LexerSettings.default()); defer table.deinit(); try table.parse( \\name,age,alive,foo @@ -242,7 +242,7 @@ test "StructuredTable: Optional fields edit writes empty when null" { foo: ?f32, }; - var table = StructuredTable(DogTableOpt).init(allocator, csv.Settings.default()); + var table = StructuredTable(DogTableOpt).init(allocator, csv.LexerSettings.default()); defer table.deinit(); try table.parse( \\name,age,alive,foo diff --git a/src/tests/table.zig b/src/tests/table.zig index 9243cfe..bf363ef 100644 --- a/src/tests/table.zig +++ b/src/tests/table.zig @@ -5,7 +5,7 @@ const allocator = std.testing.allocator; const StructuredTable = csv.StructuredTable; test "Initialize Table using Table.parse and export to CSV via Table.exportCSV" { - var table = csv.Table.init(allocator, csv.Settings.default()); + var table = csv.Table.init(allocator, csv.LexerSettings.default()); defer table.deinit(); const csv_data = \\id,shorthand,animal name,scientific name @@ -22,7 +22,7 @@ test "Initialize Table using Table.parse and export to CSV via Table.exportCSV" } test "Initialize Table using Table.parseRow and export to CSV via Table.exportCSV" { - var table = csv.Table.init(allocator, csv.Settings.default()); + var table = csv.Table.init(allocator, csv.LexerSettings.default()); defer table.deinit(); const csv_data = \\id,letter @@ -38,8 +38,10 @@ test "Initialize Table using Table.parseRow and export to CSV via Table.exportCS } test "Initialize Table using custom delimiter and terminator and export to CSV via Table.exportCSV" { - var table = csv.Table.init(allocator, csv.Settings{ + var table = csv.Table.init(allocator, csv.LexerSettings{ .delimiter = "-", + .escape = "\\", + .quote = "\"", .terminator = "|", }); defer table.deinit(); @@ -54,7 +56,7 @@ test "Initialize Table using custom delimiter and terminator and export to CSV v } test "Get number of rows using Table.getRowCount" { - var table = csv.Table.init(allocator, csv.Settings.default()); + var table = csv.Table.init(allocator, csv.LexerSettings.default()); defer table.deinit(); try table.parse( \\id,animal name,scientific name @@ -65,7 +67,7 @@ test "Get number of rows using Table.getRowCount" { } test "Get number of columns using Table.getColumnCount" { - var table = csv.Table.init(allocator, csv.Settings.default()); + var table = csv.Table.init(allocator, csv.LexerSettings.default()); defer table.deinit(); try table.parse( \\id,animal name,scientific name @@ -76,7 +78,7 @@ test "Get number of columns using Table.getColumnCount" { } test "Find indexes of columns using Table.findColumnIndexesByValue" { - var table = csv.Table.init(allocator, csv.Settings.default()); + var table = csv.Table.init(allocator, csv.LexerSettings.default()); defer table.deinit(); try table.parse( \\id,animal name,scientific name @@ -109,7 +111,7 @@ test "Find indexes of columns using Table.findColumnIndexesByValue" { } test "Find indexes of columns using Table.findRowIndexesByValue" { - var table = csv.Table.init(allocator, csv.Settings.default()); + var table = csv.Table.init(allocator, csv.LexerSettings.default()); defer table.deinit(); try table.parse( \\id,animal name,scientific name @@ -142,7 +144,7 @@ test "Find indexes of columns using Table.findRowIndexesByValue" { } test "Get column by index using Table.getColumnByIndex" { - var table = csv.Table.init(allocator, csv.Settings.default()); + var table = csv.Table.init(allocator, csv.LexerSettings.default()); defer table.deinit(); try table.parse( \\id,letter @@ -165,7 +167,7 @@ test "Get column by index using Table.getColumnByIndex" { } test "Get row by index using Table.getRowByIndex" { - var table = csv.Table.init(allocator, csv.Settings.default()); + var table = csv.Table.init(allocator, csv.LexerSettings.default()); defer table.deinit(); try table.parse( \\id,letter @@ -184,7 +186,7 @@ test "Get row by index using Table.getRowByIndex" { } test "Replace values using Table.replaceValue" { - var table = csv.Table.init(allocator, csv.Settings.default()); + var table = csv.Table.init(allocator, csv.LexerSettings.default()); defer table.deinit(); try table.parse( \\id,letter @@ -205,8 +207,10 @@ test "Replace values using Table.replaceValue" { } test "Replace values containing illegal characters using Table.replaceValues" { - var table = csv.Table.init(allocator, csv.Settings{ + var table = csv.Table.init(allocator, csv.LexerSettings{ .delimiter = ",", + .escape = "\\", + .quote = "\"", .terminator = "\n", }); defer table.deinit(); @@ -220,7 +224,7 @@ test "Replace values containing illegal characters using Table.replaceValues" { } test "Append row using Table.insertEmptyRow" { - var table = csv.Table.init(allocator, csv.Settings.default()); + var table = csv.Table.init(allocator, csv.LexerSettings.default()); defer table.deinit(); try table.parse( \\id,letter @@ -240,7 +244,7 @@ test "Append row using Table.insertEmptyRow" { } test "Insert row using Table.insertEmptyRow" { - var table = csv.Table.init(allocator, csv.Settings.default()); + var table = csv.Table.init(allocator, csv.LexerSettings.default()); defer table.deinit(); const csv_data = \\id,letter @@ -261,7 +265,7 @@ test "Insert row using Table.insertEmptyRow" { } test "Append column using Table.insertEmptyColumn" { - var table = csv.Table.init(allocator, csv.Settings.default()); + var table = csv.Table.init(allocator, csv.LexerSettings.default()); defer table.deinit(); try table.parse( \\id,letter @@ -280,7 +284,7 @@ test "Append column using Table.insertEmptyColumn" { } test "Insert column using Table.insertEmptyColumn" { - var table = csv.Table.init(allocator, csv.Settings.default()); + var table = csv.Table.init(allocator, csv.LexerSettings.default()); defer table.deinit(); const csv_data = \\id,letter @@ -300,7 +304,7 @@ test "Insert column using Table.insertEmptyColumn" { } test "Append row using Table.insertEmptyRow and Table.replaceValue" { - var table = csv.Table.init(allocator, csv.Settings.default()); + var table = csv.Table.init(allocator, csv.LexerSettings.default()); defer table.deinit(); try table.parse( \\id,letter @@ -323,7 +327,7 @@ test "Append row using Table.insertEmptyRow and Table.replaceValue" { } test "Append column using Table.insertEmptyColumn and Table.replaceValue" { - var table = csv.Table.init(allocator, csv.Settings.default()); + var table = csv.Table.init(allocator, csv.LexerSettings.default()); defer table.deinit(); try table.parse( \\id,letter @@ -346,7 +350,7 @@ test "Append column using Table.insertEmptyColumn and Table.replaceValue" { } test "Delete row using Table.deleteRowByIndex" { - var table = csv.Table.init(allocator, csv.Settings.default()); + var table = csv.Table.init(allocator, csv.LexerSettings.default()); defer table.deinit(); try table.parse( \\id,letter @@ -365,7 +369,7 @@ test "Delete row using Table.deleteRowByIndex" { } test "Delete column using Table.deleteColumnByIndex" { - var table = csv.Table.init(allocator, csv.Settings.default()); + var table = csv.Table.init(allocator, csv.LexerSettings.default()); defer table.deinit(); try table.parse( \\id,letter @@ -385,7 +389,7 @@ test "Delete column using Table.deleteColumnByIndex" { } test "Parse row with trailing delimiter" { - var table = csv.Table.init(allocator, csv.Settings.default()); + var table = csv.Table.init(allocator, csv.LexerSettings.default()); defer table.deinit(); try table.parse( \\a,b, @@ -399,7 +403,7 @@ test "Parse row with trailing delimiter" { } test "Parse multiple consecutive delimiters" { - var table = csv.Table.init(allocator, csv.Settings.default()); + var table = csv.Table.init(allocator, csv.LexerSettings.default()); defer table.deinit(); try table.parse( \\a,,,d @@ -414,7 +418,7 @@ test "Parse multiple consecutive delimiters" { } test "Parse unescaped empty field" { - var table = csv.Table.init(allocator, csv.Settings.default()); + var table = csv.Table.init(allocator, csv.LexerSettings.default()); defer table.deinit(); try table.parse( \\a,,c @@ -428,7 +432,7 @@ test "Parse unescaped empty field" { } test "Handle trailing empty row" { - var table = csv.Table.init(allocator, csv.Settings.default()); + var table = csv.Table.init(allocator, csv.LexerSettings.default()); defer table.deinit(); try table.parse( \\a,b @@ -446,7 +450,7 @@ test "Fail-fast on empty row" { \\ \\c,d ; - var table = csv.Table.init(allocator, csv.Settings.default()); + var table = csv.Table.init(allocator, csv.LexerSettings.default()); defer table.deinit(); const result = table.parse(data); try expect(result == csv.TableError.InconsistentRowLength); @@ -455,10 +459,149 @@ test "Fail-fast on empty row" { test "Fail-fast on inconsistent row lengths" { const data = \\a,b - \\c,d,e" + \\c,d,e ; - var table = csv.Table.init(allocator, csv.Settings.default()); + var table = csv.Table.init(allocator, csv.LexerSettings.default()); defer table.deinit(); const result = table.parse(data); try expect(result == csv.TableError.InconsistentRowLength); } + +test "Parse empty escaped field" { + var table = csv.Table.init(allocator, csv.LexerSettings.default()); + defer table.deinit(); + try table.parse( + \\a,"",c + ); + + const row = try table.getRowByIndex(0); + try expect(row.len == 3); + try expect(std.mem.eql(u8, row[0], "a")); + try expect(std.mem.eql(u8, row[1], "")); + try expect(std.mem.eql(u8, row[2], "c")); +} + +test "Parse with custom escape character" { + const data = + \\|a|,|b| + ; + var table = csv.Table.init(allocator, csv.LexerSettings{ + .delimiter = ",", + .escape = "\\", + .quote = "|", + .terminator = "\n", + }); + defer table.deinit(); + try table.parse(data); + + const exported = try table.exportCSV(allocator); + defer allocator.free(exported); + const expected_csv = + \\a,b + ; + try expect(std.mem.eql(u8, exported, expected_csv)); +} + +test "Parse escaped field with delimiter" { + const data = + \\a,"example string, with delimiter",c + ; + var table = csv.Table.init(allocator, csv.LexerSettings.default()); + defer table.deinit(); + try table.parse(data); + + const exported = try table.exportCSV(allocator); + defer allocator.free(exported); + try expect(std.mem.eql(u8, exported, data)); +} + +test "Parse escaped field with escaped quote characters" { + const data = + \\a,"example string, with ""escape character""",c + ; + var table = csv.Table.init(allocator, csv.LexerSettings.default()); + defer table.deinit(); + try table.parse(data); + + const exported = try table.exportCSV(allocator); + defer allocator.free(exported); + try expect(std.mem.eql(u8, exported, data)); +} + +test "Parse escaped field with newline" { + const data = + \\a,"example string, + \\with newline",c + ; + var table = csv.Table.init(allocator, csv.LexerSettings.default()); + defer table.deinit(); + try table.parse(data); + + const exported = try table.exportCSV(allocator); + defer allocator.free(exported); + try expect(std.mem.eql(u8, exported, data)); +} + +test "Fail-fast on unopened escaped field" { + const data = + \\a,example string",c + ; + var table = csv.Table.init(allocator, csv.LexerSettings.default()); + defer table.deinit(); + const result = table.parse(data); + try expect(result == csv.ParserError.IllegalQuotation); +} + +test "Fail-fast on unclosed escaped field" { + const data = + \\a,"example string,c + ; + var table = csv.Table.init(allocator, csv.LexerSettings.default()); + defer table.deinit(); + const result = table.parse(data); + try expect(result == csv.ParserError.UnfinishedQuotation); +} + +test "Fail-fast on whitespace between escape character and delimiter" { + const data = + \\"a, "example string",c + ; + var table = csv.Table.init(allocator, csv.LexerSettings.default()); + defer table.deinit(); + const result = table.parse(data); + try expect(result == csv.ParserError.IllegalQuotation); +} + +test "Fail-fast on unescaped escape character" { + const data = + \\a,example "str"ing,c + ; + var table = csv.Table.init(allocator, csv.LexerSettings.default()); + defer table.deinit(); + const result = table.parse(data); + try expect(result == csv.ParserError.IllegalQuotation); +} + +test "Handle custom escape character in exported CSV" { + var table = csv.Table.init(allocator, csv.LexerSettings{ + .delimiter = ",", + .escape = "|", + .quote = "\"", + .terminator = "\n", + }); + defer table.deinit(); + try table.parse( + \\id,letter + \\0,a + ); + + try table.replaceValue(1, 1, "example \"word\""); + + const exported: []const u8 = try table.exportCSV(allocator); + defer allocator.free(exported); + const expected_csv = + \\id,letter + \\0,"example ""word""" + ; + try expect(std.mem.eql(u8, exported, expected_csv)); +}