Separate Nota's tables from the upstream set

.gitignore

@@ -15,3 +15,6 @@ runtime.*.liblouis/runtimes/
 
 # Rider / ReSharper per-user settings
 *.DotSettings.user
+
+# Worktrees created by spawned Claude Code sessions
+.claude/worktrees/

LibLouis.NET.Test/HyphenateTests.cs

@@ -18,7 +18,7 @@ public class HyphenateTests
     private static readonly string[] Tables = ["da-dk-braillo.dis", "da-dk-g26.ctb"];
 
     private static string[] TablePaths() =>
-        [.. Tables.Select(t => Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "tables", t))];
+        [.. Tables.Select(t => Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "nota-tables", t))];
 
     [Fact]
     public void Hyphenate_ReturnsOneHyphenationFlagPerCharacter()

LibLouis.NET.Test/IndexTablesTests.cs

@@ -19,7 +19,7 @@ public class IndexTablesTests
     private static readonly string[] Tables = ["da-dk-g26.ctb", "da-dk-g16-markers.ctb"];
 
     private static string[] TablePaths() =>
-        [.. Tables.Select(t => Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "tables", t))];
+        [.. Tables.Select(t => Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "nota-tables", t))];
 
     /// <summary>
     /// liblouis logs one "Analyzing table &lt;name&gt;" line per array entry it walks, so the number

LibLouis.NET.Test/InputLengthTests.cs

@@ -29,7 +29,7 @@ public class InputLengthTests
     private static readonly string[] Tables = ["da-dk-braillo.dis", "da-dk-g26.ctb"];
 
     private static string[] TablePaths() =>
-        [.. Tables.Select(t => Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "tables", t))];
+        [.. Tables.Select(t => Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "nota-tables", t))];
 
     /// <summary>
     /// Translate() only requires outputPosition to hold input.Length entries, so liblouis must

LibLouis.NET.Test/LibLouis.NET.Test.csproj

@@ -29,7 +29,31 @@
   </ItemGroup>
 
   <ItemGroup>
-    <Content Include="tables/**">
+    <!--
+      Nota's tables go to their own output directory. Five of them share a file name with an
+      upstream table and differ from it, and LibLouis.NET.Tables copies the upstream set into
+      tables/, so putting both in one directory makes which copy a test gets depend on MSBuild item
+      ordering. Keeping them apart makes every test say which set it means.
+    -->
+    <Content Include="nota-tables/**">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+
+    <!--
+      Nota's tables include a handful of general upstream tables. liblouis resolves an include
+      relative to the directory of the table doing the including, so those have to sit alongside
+      them for nota-tables/ to compile on its own. Copied from the staged upstream set rather than
+      committed, so they cannot drift from it. NotaTablesAreSelfContained asserts the list is
+      complete.
+    -->
+    <Content Include="../LibLouis.NET.Tables/tables/braille-patterns.cti;
+                      ../LibLouis.NET.Tables/tables/digits6Dots.uti;
+                      ../LibLouis.NET.Tables/tables/digits8Dots.uti;
+                      ../LibLouis.NET.Tables/tables/latinLetterDef6Dots.uti;
+                      ../LibLouis.NET.Tables/tables/latinLetterDef8Dots.uti;
+                      ../LibLouis.NET.Tables/tables/litdigits6Dots.uti;
+                      ../LibLouis.NET.Tables/tables/spaces.uti"
+             Link="nota-tables/%(Filename)%(Extension)">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
   </ItemGroup>

LibLouis.NET.Test/NativeLockTests.cs

@@ -27,7 +27,7 @@ public class NativeLockTests
     private static readonly string[] Tables = ["da-dk-braillo.dis", "da-dk-g26.ctb"];
 
     private static string[] TablePaths() =>
-        [.. Tables.Select(t => Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "tables", t))];
+        [.. Tables.Select(t => Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "nota-tables", t))];
 
     [Fact]
     public void VersionIsReported()

LibLouis.NET.Test/NativeMethodsTests.cs

@@ -23,7 +23,7 @@ public void SingleMode()
         Array.Fill(modes, TypeForm.ForeignLanguage);
 
         string resultString = LibLouis.Instance.Translate(
-            tables.Select(t => Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "tables", t)),
+            tables.Select(t => Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "nota-tables", t)),
             input,
             outputLength,
             modes,
@@ -48,7 +48,7 @@ public void NestedMode_EmphasisInForeignLanguage()
         modes[6] = TypeForm.ForeignLanguage | TypeForm.Emphasis;
 
         string resultString = LibLouis.Instance.Translate(
-            tables.Select(t => Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "tables", t)),
+            tables.Select(t => Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "nota-tables", t)),
             input,
             outputLength,
             modes,
@@ -89,7 +89,7 @@ public void NestedMode_ForeignLanguageInEmphasis()
         Assert.Equal(inputLength, modes.Length);
 
         string resultString = LibLouis.Instance.Translate(
-            tables.Select(t => Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "tables", t)),
+            tables.Select(t => Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "nota-tables", t)),
             input,
             outputLength,
             modes,
@@ -105,7 +105,7 @@ public void TestPositionResults()
         const string input = "Første linje. Anden linje, med kursiveret tekst. Tredje linje.";
         const string expected = "@fze linje. @anç linje, m kursi#rò ükz. @tàdje linje.";
 
-        string[] tables = ["tables/da-dk-braillo.dis", "tables/da-dk-g26.ctb"];
+        string[] tables = ["nota-tables/da-dk-braillo.dis", "nota-tables/da-dk-g26.ctb"];
 
         int outputLength = input.Length * 4;
         int cursorPosition = 0;

LibLouis.NET.Test/NonBmpTests.cs

@@ -23,7 +23,7 @@ public class NonBmpTests
     private static readonly string[] Tables = ["da-dk-braillo.dis", "da-dk-g26.ctb"];
 
     private static string[] TablePaths() =>
-        [.. Tables.Select(t => Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "tables", t))];
+        [.. Tables.Select(t => Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "nota-tables", t))];
 
     /// <summary>
     /// How many widechars liblouis sees for <paramref name="value"/>: whole characters on a UCS-4

LibLouis.NET.Test/NotaTablesTests.cs

@@ -0,0 +1,98 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text.RegularExpressions;
+
+using Xunit;
+
+namespace LibLouis.NET.Test;
+
+/// <summary>
+/// Guards the nota-tables/ directory, which the tests use in place of the upstream set.
+/// </summary>
+public class NotaTablesTests
+{
+    private static readonly string NotaTableDirectory =
+        Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "nota-tables");
+
+    private static readonly string UpstreamTableDirectory =
+        Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "tables");
+
+    /// <summary>
+    /// liblouis resolves an include relative to the directory of the table doing the including, so
+    /// every table nota-tables/ pulls in has to be there too. Most are Nota's own; the handful of
+    /// general upstream tables they need are copied in by an explicit list in the csproj, and an
+    /// explicit list is exactly the kind of thing that goes stale the moment a table gains an
+    /// include.
+    /// </summary>
+    [Fact]
+    public void NotaTablesAreSelfContained()
+    {
+        var present = TableFilesIn(NotaTableDirectory).ToHashSet(StringComparer.Ordinal);
+        var missing = new SortedSet<string>(StringComparer.Ordinal);
+
+        foreach (string file in present)
+        {
+            foreach (string included in IncludesOf(Path.Combine(NotaTableDirectory, file)))
+            {
+                if (!present.Contains(included))
+                {
+                    missing.Add($"{included} (included by {file})");
+                }
+            }
+        }
+
+        Assert.True(
+            missing.Count == 0,
+            "nota-tables/ is missing tables it includes, so those cannot be compiled from that " +
+            "directory alone. Add them to the upstream copy list in the csproj:\n  " +
+            string.Join("\n  ", missing));
+    }
+
+    /// <summary>
+    /// The separate directory exists so that nothing of Nota's shadows the upstream set. Sharing a
+    /// file name across the two directories is expected and fine - twenty two of Nota's tables are
+    /// forks of an upstream table of the same name. What must never happen is Nota's tables being
+    /// copied into tables/ as well, which is what the old single directory did.
+    ///
+    /// The tables that exist nowhere upstream are the reliable canary: if one of them turns up in
+    /// tables/, the copy is misconfigured, whatever the file names say.
+    /// </summary>
+    [Fact]
+    public void NotaTablesDoNotLeakIntoTheUpstreamDirectory()
+    {
+        Assert.True(Directory.Exists(UpstreamTableDirectory), "the upstream tables were not staged");
+
+        string[] notaOnly =
+        [
+            "da-dk-g16-markers.ctb",
+            "da-dk-g16_1993-markers.ctb",
+            "da-dk-braillo.dis",
+            "da-dk-g16-crossword.ctb",
+            "da-dk-g26l.ctb",
+            "da-dk-g26l-lit.ctb",
+            "da-dk-g28l.ctb",
+            "da-dk-g2_1993.dic",
+        ];
+
+        var upstream = TableFilesIn(UpstreamTableDirectory).ToHashSet(StringComparer.Ordinal);
+        var leaked = notaOnly.Where(upstream.Contains).OrderBy(f => f, StringComparer.Ordinal).ToList();
+
+        Assert.True(
+            leaked.Count == 0,
+            "tables/ should hold only the upstream set, but these tables of Nota's are in it, so " +
+            "the two are being copied to the same place again:\n  " + string.Join("\n  ", leaked));
+    }
+
+    private static IEnumerable<string> TableFilesIn(string directory) =>
+        Directory.EnumerateFiles(directory)
+            .Select(Path.GetFileName)
+            .Where(f => f is not null && !f.EndsWith(".md", StringComparison.OrdinalIgnoreCase))!;
+
+    private static IEnumerable<string> IncludesOf(string path) =>
+        File.ReadLines(path)
+            .Select(line => Regex.Match(line, @"^\s*include\s+(\S+)"))
+            .Where(m => m.Success)
+            .Select(m => m.Groups[1].Value);
+}

LibLouis.NET.Test/OutputDotsTests.cs

@@ -18,7 +18,7 @@ public class OutputDotsTests
     private static readonly string[] Tables = ["da-dk-braillo.dis", "da-dk-g08.ctb"];
 
     private static string[] EightDotTables() =>
-        [.. Tables.Select(t => Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "tables", t))];
+        [.. Tables.Select(t => Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "nota-tables", t))];
 
     private static TranslatedString TranslateWithTypeForm(string input)
     {

LibLouis.NET.Test/PositionMappingTests.cs

@@ -21,7 +21,7 @@ public class PositionMappingTests
     private static readonly string[] Tables = ["da-dk-braillo.dis", "da-dk-g26.ctb"];
 
     private static string[] TablePaths() =>
-        [.. Tables.Select(t => Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "tables", t))];
+        [.. Tables.Select(t => Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "nota-tables", t))];
 
     private static TranslatedString Translate(string input)
     {

LibLouis.NET.Test/ShutdownTests.cs

@@ -24,7 +24,7 @@ public class ShutdownTests
     private static readonly string[] Tables = ["da-dk-braillo.dis", "da-dk-g26.ctb"];
 
     private static string[] TablePaths() =>
-        [.. Tables.Select(t => Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "tables", t))];
+        [.. Tables.Select(t => Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "nota-tables", t))];
 
     private static FieldInfo ShutDownField =>
         typeof(LibLouis).GetField("_shutDown", BindingFlags.NonPublic | BindingFlags.Static)

LibLouis.NET.Test/SpacingTests.cs

@@ -15,7 +15,7 @@
 {
     private static readonly string[] Tables =
         new[] { "da-dk-braillo.dis", "da-dk-g26.ctb" }
-            .Select(t => Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "tables", t))
+            .Select(t => Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "nota-tables", t))
             .ToArray();
 
     private static TranslatedString Translate(string input, string? spacing, int outputLength)
@@ -62,7 +62,7 @@
         {
             if (translated.OutputSpacing[cell] == '3')
             {
                 Assert.Contains(translated.InputPosition[cell], new[] { 6, 7, 8 });
             }
         }
     }
@@ -78,8 +78,8 @@
         char[] request = new string('7', input.Length).ToCharArray();
 
         string[] markerTables =
             new[] { "da-dk-braillo.dis", "da-dk-g16-markers.ctb" }
-                .Select(t => Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "tables", t))
+                .Select(t => Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "nota-tables", t))
                 .ToArray();
 
         TypeForm[] formtype = new TypeForm[input.Length];

LibLouis.NET.Test/TypeFormBufferTests.cs

@@ -24,7 +24,7 @@ public class TypeFormBufferTests
     private static readonly string[] Tables = ["da-dk-braillo.dis", "da-dk-g16-markers.ctb"];
 
     private static string[] TablePaths() =>
-        [.. Tables.Select(t => Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "tables", t))];
+        [.. Tables.Select(t => Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "nota-tables", t))];
 
     /// <summary>
     /// Documents the native contract that makes the overrun possible, independent of the wrapper:

LibLouis.NET.Test/UTF8StringNoFreeMarshallerTests.cs

@@ -16,7 +16,7 @@ public unsafe class UTF8StringNoFreeMarshallerTests
     [Theory]
     [InlineData("")]
     [InlineData("a")]
-    [InlineData("tables/da-dk-g26.ctb")]
+    [InlineData("nota-tables/da-dk-g26.ctb")]
     [InlineData("Første linje")]      // multi-byte UTF-8
     [InlineData("\U0001D11E")]        // non-BMP, surrogate pair on the managed side
     public void ConvertToManaged_ReadsNulTerminatedUtf8(string value)

LibLouis.NET.Test/nota-tables/README.md

@@ -0,0 +1,31 @@
+# Nota's Danish tables, for the tests
+
+Kept in their own directory, and copied to `nota-tables/` in the output rather than `tables/`.
+
+`LibLouis.NET.Tables` copies the upstream table set into `tables/`. Five of the files here share a
+name with an upstream table and differ from it, so a single shared directory would make which copy
+a test gets depend on MSBuild item ordering. Separate directories mean every test states which set
+it means, and the upstream braille specs can be checked against upstream tables without this set
+shadowing them.
+
+## What these are
+
+A fork of an older upstream, not a patch on the current one. They add things upstream does not
+have, such as the `foreign` emphasis class behind `TypeForm.ForeignLanguage`, and they are missing
+things upstream has since fixed, such as the rules that remove the space between `§` and a following
+number. Do not assume a file here matches the upstream file of the same name in either direction.
+
+Two of them, `da-dk-g16-markers.ctb` and `da-dk-braillo.dis`, exist nowhere upstream, which is why
+the tests cannot simply use the upstream set.
+
+## These are not canonical
+
+The canonical Nota tables ship with the application. This is a copy, and a stale one: only seven of
+the thirty files are reachable from any test.
+
+Reachable: `da-dk-g16-markers.ctb`, `da-dk-braillo.dis`, `da-dk-g08.ctb`, `da-dk-g26.ctb`, and the
+three they include (`da-dk-6miscChars.cti`, `da-dk-octobraille.dis`, `da-dk-g2.dic`).
+
+The other twenty three are referenced by nothing. They are kept for now, but nothing keeps them in
+step with the application's copy, so treat any of them as evidence of nothing. Prefer adding a test
+that needs a file over adding a file that no test needs.