From 2bedeedc733f6a354eb3ddf90492c0121c2eff8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20O=2E=20S=C3=B8rensen?= Date: Wed, 5 Aug 2026 10:12:11 +0200 Subject: [PATCH] test: Run the upstream braille specs against the wrapper Adds a reader for liblouis's braille spec files and runs the three Danish ones through the managed wrapper. 10524 cases, in both directions, whose expectations were written by upstream rather than invented here. The files are not YAML mappings and cannot be deserialised: one document repeats table, flags and tests at the same level, and lou_checkyaml treats the file as an event stream where each key mutates parser state (tools/lou_checkyaml.c:1087-1139). The reader does the same over YamlDotNet's IParser. Consecutive table keys accumulate rather than replace, so a tests block runs once per accumulated table, which is why 5855 entries expand to 10524 cases. Two details of the format are easy to get wrong and both were, before the specs caught them: - the backward leg of bothDirections swaps input and expected, because back translation should turn the braille back into the text, while an explicit testmode: backward does not (lou_checkyaml.c:892 against 900). - a display table can be an inline table written as a block scalar rather than a file name, to include the standard one and override a character. That arrives as an ordinary scalar, so it was read as a path, and every test under it failed to translate. Those are now written out beside the tables, where their own includes resolve. Anything the reader does not model throws rather than being skipped, so a spec using an unsupported construct fails loudly instead of quietly testing less than it appears to. One test per spec file rather than per case: ten thousand xunit cases makes discovery slow and buries a real regression, where a single failure listing every mismatch does not. Co-Authored-By: Claude Opus 5 --- LibLouis.NET.Test/BrailleSpec.cs | 398 ++ LibLouis.NET.Test/BrailleSpecTests.cs | 194 + LibLouis.NET.Test/LibLouis.NET.Test.csproj | 10 + .../braille-specs/da-dk-6dot.yaml | 1124 ++++ .../braille-specs/da-dk-8dot.yaml | 1003 +++ .../braille-specs/da-dk_1993.yaml | 5970 +++++++++++++++++ 6 files changed, 8699 insertions(+) create mode 100644 LibLouis.NET.Test/BrailleSpec.cs create mode 100644 LibLouis.NET.Test/BrailleSpecTests.cs create mode 100644 LibLouis.NET.Test/braille-specs/da-dk-6dot.yaml create mode 100644 LibLouis.NET.Test/braille-specs/da-dk-8dot.yaml create mode 100644 LibLouis.NET.Test/braille-specs/da-dk_1993.yaml diff --git a/LibLouis.NET.Test/BrailleSpec.cs b/LibLouis.NET.Test/BrailleSpec.cs new file mode 100644 index 0000000..a3e944e --- /dev/null +++ b/LibLouis.NET.Test/BrailleSpec.cs @@ -0,0 +1,398 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; + +using YamlDotNet.Core; +using YamlDotNet.Core.Events; + +namespace LibLouis.NET.Test; + +/// +/// One translation case from an upstream braille spec. +/// +public sealed record BrailleSpecCase( + string SpecFile, + int Line, + string TableQuery, + string? AssertMatch, + string DisplayTable, + string Input, + string Expected, + TestDirection Direction, + bool ExpectedToFail) +{ + public override string ToString() => + $"{SpecFile}:{Line} {Direction} {Describe(Input)} -> {Describe(Expected)}"; + + // Braille output is mostly U+28xx, which is unreadable in a test runner's output, so show the + // code points for anything outside printable ASCII. + private static string Describe(string value) => + value.All(c => c is >= ' ' and <= '~') + ? $"\"{value}\"" + : string.Concat(value.Select(c => c is >= ' ' and <= '~' ? c.ToString() : $"\\u{(int)c:X4}")); +} + +public enum TestDirection +{ + Forward, + Backward, +} + +/// +/// How a spec entry's two values are used. The distinction matters: the backward leg of +/// bothDirections swaps input and expected (lou_checkyaml.c:900-902), while an explicit +/// backward testmode does not (lou_checkyaml.c:892-895). +/// +public enum TestMode +{ + Forward, + Backward, + BothDirections, +} + +/// +/// Reads liblouis braille spec files. +/// +/// +/// These files are not YAML mappings and cannot be deserialised. A single document repeats +/// table, flags and tests at the same level, which duplicate key handling +/// would collapse or reject. lou_checkyaml treats the file as an event stream where each key +/// mutates parser state, and tests executes against whatever is current +/// (tools/lou_checkyaml.c:1087-1139), so this reader does the same over YamlDotNet's IParser. +/// +/// Consecutive table keys accumulate rather than replace: the following tests block +/// runs once per accumulated table. flags persists until the next flags. +/// +/// Only the constructs the Danish specs actually use are supported. Anything else throws rather +/// than being skipped, so a spec using a feature this reader does not model fails loudly instead +/// of silently testing less than it appears to. +/// +public static class BrailleSpecReader +{ + public static IReadOnlyList Read(string path) + { + string specFile = Path.GetFileName(path); + var cases = new List(); + + using var reader = new StreamReader(path); + var parser = new Parser(reader); + + parser.Consume(); + parser.Consume(); + parser.Consume(); + + string displayTable = string.Empty; + var tables = new List<(string Query, string? AssertMatch)>(); + TestMode mode = TestMode.Forward; + + // Consecutive table keys accumulate, but the first one after a tests block starts a fresh + // set rather than adding to the one just used. + bool tablesUsed = false; + + while (parser.Current is not MappingEnd) + { + string key = parser.Consume().Value; + + switch (key) + { + case "display": + displayTable = ReadTableValue(parser).Query; + break; + + case "table": + if (tablesUsed) + { + tables.Clear(); + tablesUsed = false; + } + + tables.Add(ReadTableValue(parser)); + break; + + case "flags": + mode = ReadFlags(parser); + break; + + case "tests": + ReadTests(parser, specFile, tables, displayTable, mode, cases); + tablesUsed = true; + break; + + default: + throw new NotSupportedException( + $"{specFile}: unsupported top level key '{key}'. This reader models only the " + + "constructs the Danish specs use; see lou_checkyaml.c for the full format."); + } + + } + + return cases; + } + + /// + /// A table value is either a file name or a query mapping. Queries are passed to lou_findTable + /// as "key:value key:value"; __assert-match is a harness directive, not part of the query. + /// + private static (string Query, string? AssertMatch) ReadTableValue(IParser parser) + { + if (parser.Current is Scalar scalar) + { + parser.MoveNext(); + return (scalar.Value, null); + } + + parser.Consume(); + + var terms = new List(); + string? assertMatch = null; + + while (parser.Current is not MappingEnd) + { + string key = parser.Consume().Value; + string value = parser.Consume().Value; + + if (key == "__assert-match") + { + assertMatch = value; + } + else + { + terms.Add($"{key}:{value}"); + } + } + + parser.Consume(); + + return (string.Join(' ', terms), assertMatch); + } + + private static TestMode ReadFlags(IParser parser) + { + parser.Consume(); + + TestMode mode = TestMode.Forward; + + while (parser.Current is not MappingEnd) + { + string key = parser.Consume().Value; + string value = parser.Consume().Value; + + if (key != "testmode") + { + throw new NotSupportedException($"unsupported flag '{key}'"); + } + + mode = ParseTestMode(value); + } + + parser.Consume(); + + return mode; + } + + private static TestMode ParseTestMode(string value) => value switch + { + "forward" => TestMode.Forward, + "backward" => TestMode.Backward, + "bothDirections" => TestMode.BothDirections, + _ => throw new NotSupportedException($"unsupported testmode '{value}'"), + }; + + private static void ReadTests( + IParser parser, + string specFile, + List<(string Query, string? AssertMatch)> tables, + string displayTable, + TestMode mode, + List cases) + { + parser.Consume(); + + while (parser.Current is not SequenceEnd) + { + SequenceStart entryStart = parser.Consume(); + int line = (int)entryStart.Start.Line; + + string input = Unescape(parser.Consume().Value); + string expected = Unescape(parser.Consume().Value); + + var xfail = XFail.None; + TestMode entryMode = mode; + bool skip = false; + + if (parser.Current is MappingStart) + { + (xfail, entryMode, skip) = ReadTestOptions(parser, mode); + } + + parser.Consume(); + + if (skip) + { + continue; + } + + foreach ((string query, string? assertMatch) in tables) + { + // Forward compares translate(input) with expected. An explicit backward testmode + // means the entry is already written braille-first, so it is not swapped. The + // backward leg of bothDirections is: the expected braille is the input, and the + // original text is what back translation should produce. + if (entryMode is TestMode.Forward or TestMode.BothDirections) + { + cases.Add(new BrailleSpecCase( + specFile, line, query, assertMatch, displayTable, + input, expected, TestDirection.Forward, xfail.HasFlag(XFail.Forward))); + } + + if (entryMode == TestMode.Backward) + { + cases.Add(new BrailleSpecCase( + specFile, line, query, assertMatch, displayTable, + input, expected, TestDirection.Backward, xfail.HasFlag(XFail.Backward))); + } + else if (entryMode == TestMode.BothDirections) + { + cases.Add(new BrailleSpecCase( + specFile, line, query, assertMatch, displayTable, + expected, input, TestDirection.Backward, xfail.HasFlag(XFail.Backward))); + } + } + } + + parser.Consume(); + } + + [Flags] + private enum XFail + { + None = 0, + Forward = 1, + Backward = 2, + Both = Forward | Backward, + } + + private static (XFail XFail, TestMode Mode, bool Skip) ReadTestOptions( + IParser parser, TestMode mode) + { + parser.Consume(); + + var xfail = XFail.None; + bool skip = false; + + while (parser.Current is not MappingEnd) + { + string key = parser.Consume().Value; + + switch (key) + { + case "xfail": + xfail = ReadXFail(parser); + break; + + case "testmode": + mode = ParseTestMode(parser.Consume().Value); + break; + + // Emphasis is applied through typeform, which this prototype does not drive yet. + // Skip the value so the rest of the file still parses, and drop the case: silently + // running it without the typeform would compare against the wrong expectation. + case "typeform": + parser.SkipThisAndNestedEvents(); + skip = true; + break; + + default: + throw new NotSupportedException($"unsupported test option '{key}'"); + } + } + + parser.Consume(); + + return (xfail, mode, skip); + } + + /// + /// xfail is either a scalar, where only "false" and "off" are falsy + /// (tools/lou_checkyaml.c:379-389), or a mapping naming the failing directions. + /// + private static XFail ReadXFail(IParser parser) + { + if (parser.Current is Scalar scalar) + { + parser.MoveNext(); + return scalar.Value is "false" or "off" ? XFail.None : XFail.Both; + } + + parser.Consume(); + + var xfail = XFail.None; + + while (parser.Current is not MappingEnd) + { + string key = parser.Consume().Value; + string value = parser.Consume().Value; + bool set = value is not ("false" or "off"); + + if (set) + { + xfail |= key switch + { + "forward" => XFail.Forward, + "backward" => XFail.Backward, + _ => throw new NotSupportedException($"unsupported xfail direction '{key}'"), + }; + } + } + + parser.Consume(); + + return xfail; + } + + /// + /// The specs use single quoted scalars, where YAML performs no escape processing at all, and + /// rely on liblouis to interpret the escapes itself. Only the forms the Danish specs actually + /// use are handled: \xNNNN and \uNNNN code points, and \\ for a literal backslash. + /// Without the backslash case, 'at\\bliver' parses as two backslashes and translates to two + /// cells where upstream expects one. + /// + private static string Unescape(string value) + { + if (!value.Contains('\\', StringComparison.Ordinal)) + { + return value; + } + + var builder = new StringBuilder(value.Length); + + for (int i = 0; i < value.Length; i++) + { + if (value[i] == '\\' && i + 1 < value.Length) + { + if (value[i + 1] is 'x' or 'y' or 'u' && i + 5 < value.Length && + ushort.TryParse( + value.AsSpan(i + 2, 4), NumberStyles.HexNumber, CultureInfo.InvariantCulture, out ushort code)) + { + builder.Append((char)code); + i += 5; + continue; + } + + if (value[i + 1] is '\\' or '"') + { + builder.Append(value[i + 1]); + i++; + continue; + } + } + + builder.Append(value[i]); + } + + return builder.ToString(); + } +} diff --git a/LibLouis.NET.Test/BrailleSpecTests.cs b/LibLouis.NET.Test/BrailleSpecTests.cs new file mode 100644 index 0000000..3a8dc95 --- /dev/null +++ b/LibLouis.NET.Test/BrailleSpecTests.cs @@ -0,0 +1,194 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +using Xunit; + +namespace LibLouis.NET.Test; + +/// +/// Runs the upstream braille specs for Danish through the managed wrapper. +/// +/// +/// These are liblouis's own expectations, so they check the wrapper end to end against thousands of +/// cases nobody here had to invent: not just that a translation succeeds, but that it produces the +/// characters upstream says it should, in both directions. +/// +/// The specs live in braille-specs/ and are copied verbatim from +/// upstream/liblouis-<version>/tests/braille-specs/. Re-copy them when the upstream version is +/// bumped; the diff is the set of expectations that changed. +/// +/// One test per spec file rather than per case. Ten thousand xunit cases makes discovery slow and +/// buries a real regression in an unreadable log; a single failure listing every mismatch is more +/// use than ten thousand separate red entries. +/// +public class BrailleSpecTests +{ + private static readonly string SpecDirectory = + Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "braille-specs"); + + // tables/ is the upstream set. Nota's own tables live in nota-tables/ and no longer shadow it, + // so the specs can be checked against the tables they were actually written for. + private static readonly string TableDirectory = + Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "tables"); + + public static TheoryData SpecFiles() + { + var data = new TheoryData(); + + foreach (string path in Directory.EnumerateFiles(SpecDirectory, "*.yaml").OrderBy(f => f, StringComparer.Ordinal)) + { + data.Add(Path.GetFileName(path)); + } + + return data; + } + + [Theory] + [MemberData(nameof(SpecFiles))] + public void MatchesUpstreamExpectations(string specFile) + { + IReadOnlyList cases = BrailleSpecReader.Read(Path.Combine(SpecDirectory, specFile)); + + IndexUpstreamTables(); + + var mismatches = new List(); + var unexpectedPasses = new List(); + int checkedCount = 0; + + foreach (BrailleSpecCase testCase in cases) + { + + string table = ResolveTable(testCase, TableCache.Value); + string? actual = Run(testCase, table); + bool matched = actual == testCase.Expected; + checkedCount++; + + if (testCase.ExpectedToFail) + { + // Upstream reports an unexpected pass as a warning rather than an error: the case + // is known-broken, and it passing usually means the expectation moved. + if (matched) + { + unexpectedPasses.Add(testCase.ToString()); + } + } + else if (!matched) + { + mismatches.Add($"{testCase}\n actual: {Describe(actual)}"); + } + } + + + Assert.True( + mismatches.Count == 0, + $"{specFile}: {mismatches.Count} of {checkedCount} cases did not match upstream " + + $"({unexpectedPasses.Count} xfail cases passed unexpectedly).\n " + + string.Join("\n ", mismatches.Take(25)) + + (mismatches.Count > 25 ? $"\n ... and {mismatches.Count - 25} more" : string.Empty)); + } + + private static string? Run(BrailleSpecCase testCase, string table) + { + string[] tables = [ResolveDisplayTable(testCase.DisplayTable), table]; + int outputLength = Math.Max(testCase.Input.Length, testCase.Expected.Length) * 4; + + try + { + return testCase.Direction == TestDirection.Forward + ? LibLouis.Instance.Translate(tables, testCase.Input, outputLength, null, null, TranslationMode.Regular) + : LibLouis.Instance.BackTranslate(tables, testCase.Input, outputLength, null, null, TranslationMode.Regular); + } + catch (LibLouisException ex) + { + // A translation that fails outright is a mismatch, not an error: upstream marks some of + // these xfail, and letting it throw would stop the whole file at the first one. The + // message is carried into the comparison so a failure says why, rather than only that + // nothing came back. + return $""; + } + } + + // Every query from every spec is resolved once, up front, before any translation runs. + // lou_findTable's return value is freed by the marshaller with the wrong allocator (P/Invoke + // audit item 3), so interleaving these calls with translations corrupts the native heap. + private static readonly Lazy> TableCache = new(() => + { + LibLouis.Instance.IndexTables( + Directory.EnumerateFiles(TableDirectory) + .Where(f => Path.GetExtension(f) is ".ctb" or ".utb" or ".uti" or ".dis" or ".cti" or ".dic")); + + var resolved = new Dictionary(StringComparer.Ordinal); + + foreach (string query in Directory.EnumerateFiles(SpecDirectory, "*.yaml") + .SelectMany(BrailleSpecReader.Read) + .Select(c => c.TableQuery) + .Distinct(StringComparer.Ordinal)) + { + resolved[query] = LibLouis.Instance.FindTable(query) ?? string.Empty; + } + + return resolved; + }); + + private static void IndexUpstreamTables() => _ = TableCache.Value; + + /// + /// Resolving a table query is what lou_findTable does, and the specs assert which file a query + /// should select, so this covers table resolution as well as translation. + /// + private static string ResolveTable(BrailleSpecCase testCase, Dictionary cache) + { + if (!cache.TryGetValue(testCase.TableQuery, out string? resolved)) + { + resolved = LibLouis.Instance.FindTable(testCase.TableQuery) ?? string.Empty; + cache[testCase.TableQuery] = resolved; + } + + Assert.False( + string.IsNullOrEmpty(resolved), + $"No table matched the query '{testCase.TableQuery}' from {testCase.SpecFile}:{testCase.Line}"); + + if (testCase.AssertMatch is not null) + { + Assert.True( + string.Equals(Path.GetFileName(resolved), testCase.AssertMatch, StringComparison.Ordinal), + $"Query '{testCase.TableQuery}' resolved to {Path.GetFileName(resolved)}, " + + $"but {testCase.SpecFile}:{testCase.Line} asserts {testCase.AssertMatch}"); + } + + return resolved; + } + + /// + /// A spec's display table is usually a file name, but it can also be an inline table written as + /// a YAML block scalar, for instance to include the standard one and then override a character. + /// liblouis only takes paths, so the inline form is written out next to the tables, where the + /// includes inside it resolve. + /// + private static string ResolveDisplayTable(string display) + { + // A file name never contains a newline, so this distinguishes the two forms. + if (!display.Contains('\n', StringComparison.Ordinal)) + { + return Path.Combine(TableDirectory, display); + } + + string name = $"inline-{Convert.ToHexString(System.Security.Cryptography.MD5.HashData(System.Text.Encoding.UTF8.GetBytes(display)))[..8]}.dis"; + string path = Path.Combine(TableDirectory, name); + + if (!File.Exists(path)) + { + File.WriteAllText(path, display); + } + + return path; + } + + private static string Describe(string? value) => + value is null + ? "" + : string.Concat(value.Select(c => c is >= ' ' and <= '~' ? c.ToString() : $"\\u{(int)c:X4}")); +} diff --git a/LibLouis.NET.Test/LibLouis.NET.Test.csproj b/LibLouis.NET.Test/LibLouis.NET.Test.csproj index 40fc277..1e8ca27 100644 --- a/LibLouis.NET.Test/LibLouis.NET.Test.csproj +++ b/LibLouis.NET.Test/LibLouis.NET.Test.csproj @@ -17,6 +17,7 @@ + all runtime; build; native; contentfiles; analyzers; buildtransitive @@ -29,6 +30,15 @@ + + + PreserveNewest + +