From 7688dc8fd4f32cc42241d74e591524900236d208 Mon Sep 17 00:00:00 2001 From: Hirogen Date: Sat, 26 Sep 2026 13:41:15 +0200 Subject: [PATCH 1/3] Fix CSV columnizer crash when first line has no field names (#707) CsvHelper's ReadHeader() throws when HasHeaderRecord is false, so Selected() failed and every line fell back to a single unsplit column. Read fields from the parser record instead, and name generated columns "Column 1".."Column N" (was string-concatenated to "Column 01", "Column 11"). --- src/CsvColumnizer/CsvColumnizer.cs | 31 +++++---- .../ColumnizerTests/CSVColumnizerTest.cs | 65 +++++++++++++++++++ 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/CsvColumnizer/CsvColumnizer.cs b/src/CsvColumnizer/CsvColumnizer.cs index e780fd2c..64d34ced 100644 --- a/src/CsvColumnizer/CsvColumnizer.cs +++ b/src/CsvColumnizer/CsvColumnizer.cs @@ -185,26 +185,21 @@ public void Selected (ILogLineMemoryColumnizerCallback callback) if (line != null) { - using CsvReader csv = new(new StringReader(line.FullLine.ToString()), _config.ReaderConfiguration); - _ = csv.Read(); - _ = csv.ReadHeader(); + var fields = ReadFields(line.FullLine.ToString()); - var fieldCount = csv.Parser.Count; - - var headerRecord = csv.HeaderRecord; - - if (_config.HasFieldNames && headerRecord != null) + if (_config.HasFieldNames && fields != null) { - foreach (var headerColumn in headerRecord) + foreach (var headerColumn in fields) { _columnList.Add(new CsvColumn(headerColumn)); } } else { + var fieldCount = fields?.Length ?? 0; for (var i = 0; i < fieldCount; ++i) { - _columnList.Add(new CsvColumn("Column " + i + 1)); + _columnList.Add(new CsvColumn(string.Format(CultureInfo.InvariantCulture, "Column {0}", i + 1))); } } } @@ -360,6 +355,15 @@ private void AutoDetectDelimiter (ReadOnlyMemory lineContent) } } + /// + /// Parses one line into its fields; avoids ReadHeader(), which throws when HasHeaderRecord is false. + /// + private string[]? ReadFields (string line) + { + using CsvReader csv = new(new StringReader(line), _config.ReaderConfiguration); + return csv.Read() ? csv.Parser.Record : null; + } + private ColumnizedLogLine SplitCsvLine (ILogLineMemory line) { if (line.FullLine.IsEmpty) @@ -374,12 +378,7 @@ private ColumnizedLogLine SplitCsvLine (ILogLineMemory line) try { - using CsvReader csv = new(new StringReader(line.FullLine.ToString()), _config.ReaderConfiguration); - _ = csv.Read(); - _ = csv.ReadHeader(); - - //we only read line by line and not the whole file so it is always the header - var records = csv.HeaderRecord; + var records = ReadFields(line.FullLine.ToString()); if (records != null) { diff --git a/src/LogExpert.Tests/ColumnizerTests/CSVColumnizerTest.cs b/src/LogExpert.Tests/ColumnizerTests/CSVColumnizerTest.cs index ed7a404d..ae367de1 100644 --- a/src/LogExpert.Tests/ColumnizerTests/CSVColumnizerTest.cs +++ b/src/LogExpert.Tests/ColumnizerTests/CSVColumnizerTest.cs @@ -62,6 +62,22 @@ private static void SetPrivateField (CsvColumnizer.CsvColumnizer columnizer, str field!.SetValue(columnizer, value); } + private const string HEADERLESS_LINE = "2026-09-01 07:09:58.1980; INFO; 2001:7c0:3006:3014::8602:1a1d; ; ; ; zrnsh01 logged in"; + + // Valid-CSV mode with "first line contains field names" unchecked + private static CsvColumnizer.CsvColumnizer CreateColumnizerWithoutFieldNames () + { + CsvColumnizerConfig config = new(); + config.InitDefaults(); + config.HasFieldNames = false; + config.ConfigureReaderConfiguration(); + + CsvColumnizer.CsvColumnizer columnizer = new(); + SetPrivateField(columnizer, "_config", config); + SetPrivateField(columnizer, "_isValidCsv", true); + return columnizer; + } + [Test] public void Selected_HasFieldNames_FirstLineNull_FallsBackToCallback () { @@ -81,6 +97,55 @@ public void Selected_HasFieldNames_FirstLineNull_FallsBackToCallback () Assert.That(columnizer.GetColumnNames(), Is.EqualTo(["name", "age", "city"])); } + [Test] + public void Selected_NoFieldNames_ColumnCountFromFirstLine () + { + var columnizer = CreateColumnizerWithoutFieldNames(); + var mockCallback = new Mock(); + _ = mockCallback.Setup(c => c.GetLogLineMemory(0)).Returns(new CsvLogLine(HEADERLESS_LINE, 0)); + + columnizer.Selected(mockCallback.Object); + + Assert.That(columnizer.GetColumnCount(), Is.EqualTo(7)); + } + + [Test] + public void Selected_NoFieldNames_GeneratesOneBasedColumnNames () + { + var columnizer = CreateColumnizerWithoutFieldNames(); + var mockCallback = new Mock(); + _ = mockCallback.Setup(c => c.GetLogLineMemory(0)).Returns(new CsvLogLine(HEADERLESS_LINE, 0)); + + columnizer.Selected(mockCallback.Object); + + Assert.That(columnizer.GetColumnNames(), Is.EqualTo(["Column 1", "Column 2", "Column 3", "Column 4", "Column 5", "Column 6", "Column 7"])); + } + + [Test] + public void SplitLine_NoFieldNames_SplitsIntoFields () + { + var columnizer = CreateColumnizerWithoutFieldNames(); + var line = new CsvLogLine(HEADERLESS_LINE, 0); + var mockCallback = new Mock(); + _ = mockCallback.Setup(c => c.GetLogLineMemory(0)).Returns(line); + columnizer.Selected(mockCallback.Object); + + var result = columnizer.SplitLine(mockCallback.Object, line); + + Assert.That(result.ColumnValues.Select(c => c.FullValue.ToString()), Is.EqualTo(["2026-09-01 07:09:58.1980", " INFO", " 2001:7c0:3006:3014::8602:1a1d", " ", " ", " ", " zrnsh01 logged in"])); + } + + [Test] + [System.Diagnostics.CodeAnalysis.SuppressMessage("Globalization", "CA1303:Do not pass literals as localized parameters", Justification = "Unit Tests")] + public void PreProcessLine_NoFieldNames_KeepsFirstLineVisible () + { + var columnizer = CreateColumnizerWithoutFieldNames(); + + var result = columnizer.PreProcessLine(HEADERLESS_LINE.AsMemory(), 0, 0); + + Assert.That(result.ToString(), Is.EqualTo(HEADERLESS_LINE)); + } + [Test] public void Selected_HasFieldNames_NoLineAvailable_FallsBackToTextColumn () { From b3449e3045287bed42b8a6be2c9f5f7297a41911 Mon Sep 17 00:00:00 2001 From: Hirogen Date: Sat, 26 Sep 2026 13:47:17 +0200 Subject: [PATCH 2/3] Fall back to a single Text column when the CSV first line is bad data (#707) Selected() let CsvHelper's BadDataException escape when the first line doesn't parse with the configured settings, e.g. a quoted ';' file read with a ',' delimiter loaded from csvcolumnizer.json. Catch it and use the same single-column fallback SplitCsvLine already uses. --- src/CsvColumnizer/CsvColumnizer.cs | 18 +++++++++++---- .../ColumnizerTests/CSVColumnizerTest.cs | 23 +++++++++++++++++++ 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/CsvColumnizer/CsvColumnizer.cs b/src/CsvColumnizer/CsvColumnizer.cs index 64d34ced..4f337283 100644 --- a/src/CsvColumnizer/CsvColumnizer.cs +++ b/src/CsvColumnizer/CsvColumnizer.cs @@ -183,11 +183,22 @@ public void Selected (ILogLineMemoryColumnizerCallback callback) ? _firstLine ?? callback.GetLogLineMemory(0) : callback.GetLogLineMemory(0); + string[]? fields = null; if (line != null) { - var fields = ReadFields(line.FullLine.ToString()); + try + { + fields = ReadFields(line.FullLine.ToString()); + } + catch (CsvHelperException) + { + // first line doesn't parse with the current settings (e.g. wrong delimiter); same fallback as SplitCsvLine + } + } - if (_config.HasFieldNames && fields != null) + if (fields != null) + { + if (_config.HasFieldNames) { foreach (var headerColumn in fields) { @@ -196,8 +207,7 @@ public void Selected (ILogLineMemoryColumnizerCallback callback) } else { - var fieldCount = fields?.Length ?? 0; - for (var i = 0; i < fieldCount; ++i) + for (var i = 0; i < fields.Length; ++i) { _columnList.Add(new CsvColumn(string.Format(CultureInfo.InvariantCulture, "Column {0}", i + 1))); } diff --git a/src/LogExpert.Tests/ColumnizerTests/CSVColumnizerTest.cs b/src/LogExpert.Tests/ColumnizerTests/CSVColumnizerTest.cs index ae367de1..26212d30 100644 --- a/src/LogExpert.Tests/ColumnizerTests/CSVColumnizerTest.cs +++ b/src/LogExpert.Tests/ColumnizerTests/CSVColumnizerTest.cs @@ -135,6 +135,29 @@ public void SplitLine_NoFieldNames_SplitsIntoFields () Assert.That(result.ColumnValues.Select(c => c.FullValue.ToString()), Is.EqualTo(["2026-09-01 07:09:58.1980", " INFO", " 2001:7c0:3006:3014::8602:1a1d", " ", " ", " ", " zrnsh01 logged in"])); } + [TestCase(true)] + [TestCase(false)] + public void Selected_FirstLineIsBadDataForDelimiter_FallsBackToTextColumn (bool hasFieldNames) + { + // Quoted ';'-separated line parsed with a ',' delimiter (e.g. delimiter loaded from a saved config) + CsvColumnizerConfig config = new(); + config.InitDefaults(); + config.DelimiterChar = ","; + config.HasFieldNames = hasFieldNames; + config.ConfigureReaderConfiguration(); + + CsvColumnizer.CsvColumnizer columnizer = new(); + SetPrivateField(columnizer, "_config", config); + SetPrivateField(columnizer, "_isValidCsv", true); + + var mockCallback = new Mock(); + _ = mockCallback.Setup(c => c.GetLogLineMemory(0)).Returns(new CsvLogLine("\"2021-12-12\";\"TRACE\";\"semicolon file \"", 0)); + + columnizer.Selected(mockCallback.Object); + + Assert.That(columnizer.GetColumnNames(), Is.EqualTo(["Text"])); + } + [Test] [System.Diagnostics.CodeAnalysis.SuppressMessage("Globalization", "CA1303:Do not pass literals as localized parameters", Justification = "Unit Tests")] public void PreProcessLine_NoFieldNames_KeepsFirstLineVisible () From 06954ec4f98c7fe1f12615b263c0fab4d3007cae Mon Sep 17 00:00:00 2001 From: Hirogen Date: Sat, 26 Sep 2026 13:56:25 +0200 Subject: [PATCH 3/3] Re-detect CSV delimiter on Selected and before the config dialog (#707) The columnizer instance the user selects or configures is often a clone that loaded csvcolumnizer.json and never pre-processed the file, so it kept the saved delimiter (e.g. ',' from an earlier comma file) instead of the file's own. Detect the delimiter from the file's first line in Selected() and before showing the config dialog; after OK, rebuild the columns without re-detecting so a delimiter typed by the user sticks. --- src/CsvColumnizer/CsvColumnizer.cs | 31 +++++++++- .../ColumnizerTests/CSVColumnizerTest.cs | 57 +++++++++++++++---- 2 files changed, 75 insertions(+), 13 deletions(-) diff --git a/src/CsvColumnizer/CsvColumnizer.cs b/src/CsvColumnizer/CsvColumnizer.cs index 4f337283..b5e0c8c8 100644 --- a/src/CsvColumnizer/CsvColumnizer.cs +++ b/src/CsvColumnizer/CsvColumnizer.cs @@ -176,6 +176,25 @@ public void Selected (ILogLineMemoryColumnizerCallback callback) { ArgumentNullException.ThrowIfNull(callback, nameof(callback)); + DetectDelimiterFromFile(callback); + BuildColumns(callback); + } + + /// + /// This instance may not have pre-processed the current file (e.g. a clone that loaded the saved config), + /// so re-detect the delimiter from the file's first line, like PreProcessLine() does on load. + /// + private void DetectDelimiterFromFile (ILogLineMemoryColumnizerCallback callback) + { + var line = _firstLine ?? callback.GetLogLineMemory(0); + if (line != null) + { + AutoDetectDelimiter(line.FullLine); + } + } + + private void BuildColumns (ILogLineMemoryColumnizerCallback callback) + { if (_isValidCsv) // see PreProcessLine() { _columnList.Clear(); @@ -259,6 +278,12 @@ public void Configure (ILogLineMemoryColumnizerCallback callback, string configD var configPath = configDir + "\\" + CONFIGFILENAME; FileInfo fileInfo = new(configPath); + // show the file's delimiter rather than the one from the saved config + if (callback != null) + { + DetectDelimiterFromFile(callback); + } + CsvColumnizerConfigDlg dlg = new(_config); if (dlg.ShowDialog() == DialogResult.OK) @@ -273,7 +298,11 @@ public void Configure (ILogLineMemoryColumnizerCallback callback, string configD _config.ConfigureReaderConfiguration(); - Selected(callback); + // no re-detection here: keep the delimiter the user chose in the dialog + if (callback != null) + { + BuildColumns(callback); + } } } diff --git a/src/LogExpert.Tests/ColumnizerTests/CSVColumnizerTest.cs b/src/LogExpert.Tests/ColumnizerTests/CSVColumnizerTest.cs index 26212d30..0e03e196 100644 --- a/src/LogExpert.Tests/ColumnizerTests/CSVColumnizerTest.cs +++ b/src/LogExpert.Tests/ColumnizerTests/CSVColumnizerTest.cs @@ -78,6 +78,21 @@ private static CsvColumnizer.CsvColumnizer CreateColumnizerWithoutFieldNames () return columnizer; } + // Valid-CSV mode with ',' as configured delimiter, as if loaded from a saved csvcolumnizer.json + private static CsvColumnizer.CsvColumnizer CreateColumnizerWithSavedCommaDelimiter (bool hasFieldNames) + { + CsvColumnizerConfig config = new(); + config.InitDefaults(); + config.DelimiterChar = ","; + config.HasFieldNames = hasFieldNames; + config.ConfigureReaderConfiguration(); + + CsvColumnizer.CsvColumnizer columnizer = new(); + SetPrivateField(columnizer, "_config", config); + SetPrivateField(columnizer, "_isValidCsv", true); + return columnizer; + } + [Test] public void Selected_HasFieldNames_FirstLineNull_FallsBackToCallback () { @@ -137,25 +152,43 @@ public void SplitLine_NoFieldNames_SplitsIntoFields () [TestCase(true)] [TestCase(false)] - public void Selected_FirstLineIsBadDataForDelimiter_FallsBackToTextColumn (bool hasFieldNames) + public void Selected_FirstLineIsBadData_FallsBackToTextColumn (bool hasFieldNames) { - // Quoted ';'-separated line parsed with a ',' delimiter (e.g. delimiter loaded from a saved config) - CsvColumnizerConfig config = new(); - config.InitDefaults(); - config.DelimiterChar = ","; - config.HasFieldNames = hasFieldNames; - config.ConfigureReaderConfiguration(); + var columnizer = CreateColumnizerWithSavedCommaDelimiter(hasFieldNames); + var mockCallback = new Mock(); + _ = mockCallback.Setup(c => c.GetLogLineMemory(0)).Returns(new CsvLogLine("\"2021-12-12\"x;\"TRACE\"y", 0)); - CsvColumnizer.CsvColumnizer columnizer = new(); - SetPrivateField(columnizer, "_config", config); - SetPrivateField(columnizer, "_isValidCsv", true); + columnizer.Selected(mockCallback.Object); + + Assert.That(columnizer.GetColumnNames(), Is.EqualTo(["Text"])); + } + [Test] + public void Selected_NoFieldNames_SavedDelimiterDiffersFromFile_DetectsFileDelimiter () + { + // Instance that never pre-processed this file, e.g. a clone that loaded ',' from csvcolumnizer.json + var columnizer = CreateColumnizerWithSavedCommaDelimiter(false); + var line = new CsvLogLine("\"2021-12-12\";\"TRACE\";\"semicolon file \"", 0); var mockCallback = new Mock(); - _ = mockCallback.Setup(c => c.GetLogLineMemory(0)).Returns(new CsvLogLine("\"2021-12-12\";\"TRACE\";\"semicolon file \"", 0)); + _ = mockCallback.Setup(c => c.GetLogLineMemory(0)).Returns(line); columnizer.Selected(mockCallback.Object); + var result = columnizer.SplitLine(mockCallback.Object, line); - Assert.That(columnizer.GetColumnNames(), Is.EqualTo(["Text"])); + Assert.That(columnizer.GetColumnNames(), Is.EqualTo(["Column 1", "Column 2", "Column 3"])); + Assert.That(result.ColumnValues.Select(c => c.FullValue.ToString()), Is.EqualTo(["2021-12-12", "TRACE", "semicolon file "])); + } + + [Test] + public void Selected_HasFieldNames_SavedDelimiterDiffersFromFile_DetectsFileDelimiter () + { + var columnizer = CreateColumnizerWithSavedCommaDelimiter(true); + SetPrivateField(columnizer, "_firstLine", new CsvLogLine("\"Date\";\"Level\";\"Message\"", 0)); + var mockCallback = new Mock(); + + columnizer.Selected(mockCallback.Object); + + Assert.That(columnizer.GetColumnNames(), Is.EqualTo(["Date", "Level", "Message"])); } [Test]