From 6368364fb0315feecd7807699099d92dc19e8601 Mon Sep 17 00:00:00 2001 From: mahach666 <79989386+mahach666@users.noreply.github.com> Date: Wed, 11 Nov 2026 21:39:57 +0300 Subject: [PATCH] Cut a partition stream by its exact framing when it holds The reader searched every partition stream for gzip headers. A stream whose framing holds is now cut into its segments, each one gzip member checked by its CRC and length; the search with its recovery of damaged members stays for a stream whose framing does not hold, and the diagnostics name the reason. On 52 project files the segments are the chunks of the search in all 434 partition streams, and the output of 14 files is identical. Refs #47 Refs #48 --- CHANGELOG.md | 1 + docs/test-fixtures.md | 2 +- src/RvtIO/Parsing/PartitionFraming.cs | 4 +- src/RvtIO/RvtFileReader.cs | 31 ++++++-- tests/RvtIO.Tests/PartitionCutTests.cs | 106 +++++++++++++++++++++++++ 5 files changed, 137 insertions(+), 7 deletions(-) create mode 100644 tests/RvtIO.Tests/PartitionCutTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 421cfd8..a1b67b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,7 @@ API between minor releases. ### Changed +- Reading: a partition stream whose framing holds (content blocks, segments, gzip members with their CRC and length, record lengths, the owners of each block) is cut into its segments by that framing; the search for gzip members and its recovery of damaged members are kept for a stream whose framing does not hold, and the diagnostics say which way a stream was read. On 52 project files of the releases 2020 to 2027 the segments are the chunks the search found, byte for byte, in all 434 partition streams; the IFC, the compact JSON and the export report of 14 of them, one or more of every release, are identical to the output of the main branch. - Default output, IFC and JSON: the readers behind the output read the values of an extensible storage entity by the schema of that entity, as `rvtio coverage` does, so the objects behind those values are read in their place instead of out of step, and the record of an element that keeps an entity is read to its end. An entity whose schema the file does not hold, or with a field type that is not read, is passed as before. The parameters that a store behind an entity would list as unset are still left out, as before; what that rule would give now is not measured. The bars of a rebar system that holds an entity are read now as well. A RebarInSystem that is the one RebarInSystem of its system takes the `GlobalId` of the system, as in the reference; the bar of a system with more keeps its own sub-element `GlobalId` (the number of its bars plus 3). Measured on 28 project files of the releases 2020 to 2026 against the output of the main branch, and for the IFC also against the reference: no IFC value became worse. On eight MEP files the in/out direction and the name of 887 distribution ports are now those of the reference (331, 331, 83, 83, 25, 25, 8 and 1), and on ten MEP files 3,221 assignments of a port or an element to a system are added that the reference has; 48 added assignments are not in the reference at that system (in four files; not checked in Revit). 6 systems the reference has are written that were missing; 26 pipe fitting types that the reference does not have are no longer written, and 9 others it does not have either are (types of a single fitting each, whose `GlobalId` changes with the parameters now read). In three files of one architectural project (releases 2021, 2023 and 2024) 42, 48 and 48 reinforcing bars that were left out are written, all with the `GlobalId` of the reference (in the file of release 2024, 26 of the 42 bars of its systems are the one bar of their system); the three files now write 159, 168 and 168 of the 169 bars of the reference (117, 120 and 120 before). The one bar of a system is not put into a group of its system, which it stands for. In the compact JSON 1,214 family instances of that project (three files) and 3,086 of another (one file) get the parameter list that the other instances of their categories already had, and 53 family types of one MEP file get theirs. Against Revit, on 172 of the 1,214 in the file of release 2021, every value the compact JSON writes as Revit shows it (type, mark, comments, phases, heights, levels, offsets and others, 1,561 values) is equal, and the fields the compact JSON writes in its own form (family, type and host as names, the design option as `None`, justifications as numbers, an empty level as `None`) are written as for 170 instances of the same categories that had their list before. The IFC of 16 of the 28 files and the JSON of 23 do not change. - The repository moved to the IOExtend organization. The project and repository URLs of the packages, the links in the documentation and the address of the documentation site () name the new place. The old repository address redirects to the new one; the old address of the documentation site does not. - `rvtio coverage` reads the values of an extensible storage entity by the schema of that entity. `ESEntity.m_blob` is an owned pointer whose class tag is not written: a handle of 0 is no entity and nothing follows; behind 0xFFFFFFFF stands the 16-byte GUID of the entity's schema where the class tag would be, and the values are queued like any owned object, so they lie behind every object queued earlier and in front of the objects queued later. The values are the fields of that schema in the order of `ESField.m_entryIndex` (the schema lists them by name), each one value, a count and the values of an array, or a count and the pairs of a map, of the type `ESField.m_fieldTypeName` names: a text, a scalar, or a class of the class schema read inline, among them an entity, which queues its own values. The entity schemas are read from `Global/Latest`, an `ADocument` that the same walk reads: `ESSchemaStorage.m_storedSchemas` in the class schemas up to release 2024, `ESSchemaStorage.m_schemaUsageMap` from release 2025. Before, the walk passed an entity as one without values, read the objects behind the values out of step and named the stop after whatever class came next. On 52 project files of the releases 2020 to 2027 the records that are not exact go from 27,847 to 117 of 29,794,710, and every `FamilyInstance` record of the project blocks is exact (24,044 of 131,261 were not); 154 class rows have more exact records, in files of every release but 2026, whose one file gains nothing, and no class row of any channel of any file has fewer. Left are 72 records in the documents of loaded families whose entity names a schema that `Global/Latest` of the project does not hold (the new reason `entity schema not stored`) and 45 records that stop at the 100,000 character limit of a text (4 of them inside the values of an entity). When the entity schemas of a file cannot be read, every entity with values stops its record with `entity schema not stored` instead of being passed; a handle other than 0 and 0xFFFFFFFF stops with `entity handle`, and a schema with a field the walk does not read with `entity field type` (neither occurs in the 52 files). Some kinds of values are read by the same grammar although no record of those files holds one (`short`, `int64_t`, a single `char`, arrays of `bool`, `float` and `GUIDvalue`, and maps that occur only empty). The readers of the export are not given the entity schemas and read an entity as before; their output was not compared. `rvtio coverage --trace ` is new: a developer aid that prints the walk of single records (the queued objects with their offsets, the fields of the class the walk stopped in, the bytes around the stop). See [the coverage report](docs/coverage-report.md). diff --git a/docs/test-fixtures.md b/docs/test-fixtures.md index 69b07d7..ec0e8a7 100644 --- a/docs/test-fixtures.md +++ b/docs/test-fixtures.md @@ -45,7 +45,7 @@ folder (`TestFiles.Scratch`); no test uses a fixed absolute path. ## What runs without the private files -With no private file at all, 9673 of the 10091 test cases run and pass and 418 are skipped: 412 need private files (listed +With no private file at all, 9676 of the 10094 test cases run and pass and 418 are skipped: 412 need private files (listed below; 17 of them only `bim_dump.txt` or `formats_latest_decompressed.bin`) and 6 need other inputs (two need symbolic links, three need `RVTIO_FREEFORM_FULL`, `RVTIO_CYL_FULL` or `RVTIO_MESH_FULL`, one is a known gap). The tests that run use synthetic data built in the tests and the small fixtures committed under `tests/RvtIO.Tests/Fixtures`. With the full private fixture folder the diff --git a/src/RvtIO/Parsing/PartitionFraming.cs b/src/RvtIO/Parsing/PartitionFraming.cs index 259abab..639300a 100644 --- a/src/RvtIO/Parsing/PartitionFraming.cs +++ b/src/RvtIO/Parsing/PartitionFraming.cs @@ -42,7 +42,7 @@ internal static class PartitionFraming /// Walks the stream; sees every record of every block in file order. Returns false /// with the reason in when the framing does not hold. is 4 or 8. internal static bool Walk(byte[] raw, Tags tags, int idWidth, Action onRecord, out string? failure, out int blocks, - List? segmentLengths = null) + List? segmentLengths = null, List? segmentBytes = null) { failure = null; blocks = 0; @@ -65,6 +65,7 @@ internal static bool Walk(byte[] raw, Tags tags, int idWidth, Action onR if (expectedOwners == uint.MaxValue) { Require(pointer == 0 && blocks > 0, "terminal-marker"); + // what follows the terminal is no part of the stream (measured: 432 of 434 partition streams of 52 files have bytes after it) return true; } Require((blocks != 0 || pointer == 0) && keys.Add(key), "map-key"); @@ -94,6 +95,7 @@ internal static bool Walk(byte[] raw, Tags tags, int idWidth, Action onR int before = (int)chain.Length; Inflate(payload, chain); segmentLengths?.Add((int)chain.Length - before); + segmentBytes?.Add(chain.GetBuffer().AsSpan(before, (int)chain.Length - before).ToArray()); countSum += count; sizeSum += unc; if (mode is not (0 or 1)) continue; diff --git a/src/RvtIO/RvtFileReader.cs b/src/RvtIO/RvtFileReader.cs index 9dcbc64..44e2f85 100644 --- a/src/RvtIO/RvtFileReader.cs +++ b/src/RvtIO/RvtFileReader.cs @@ -1120,6 +1120,10 @@ private List DecompressPartitions(RootStorage root, TextWriter? diag, partitionStreams = new List>(); ownPartitionWallParts = null; var routing = _options.ResolveCurrentCopies && partEntries.Count > 1 ? PlanRecordRouting(root, partStor, partEntries.Select(e => e.Name).ToList(), diag) : null; + // the exact framing of the partition streams (PartitionFraming), when the class table names its marker classes and the id width is known + var framing = FormatsSchema.TryRead(root) is { } formats && PartitionFraming.TagsOf(formats) is { } framingTags + && SchemaRelease.IdWidth(SchemaRelease.Effective(SchemaRelease.Saved(BasicFileInfoReader.Read(root).GetValueOrDefault("RevitVersion")), + SchemaRelease.FromSchema(formats))) is int framingIdWidth ? (Tags: framingTags, IdWidth: framingIdWidth) : ((PartitionFraming.Tags Tags, int IdWidth)?)null; foreach (var partEntry in partEntries) { @@ -1140,11 +1144,10 @@ private List DecompressPartitions(RootStorage root, TextWriter? diag, diag?.WriteLine($" Own partition wall parts: {ownPartitionWallParts?.Count.ToString() ?? "unavailable"} ({failure ?? "exact"})"); } - var chunks = decompressor.DecompressAll(raw); - var (members, decodedMembers, failedMembers, recoveredMembers, unverifiedMembers) = decompressor.LastStats; - RecordStream("Partitions/" + partEntry.Name, raw.Length, members, decodedMembers, - failedMembers, recoveredMembers, unverifiedMembers); - diag?.WriteLine($" Chunks: {chunks.Count} decompressed"); + var chunks = CutPartition(raw, framing, decompressor, out var stats, out string? framingFailure); + RecordStream("Partitions/" + partEntry.Name, raw.Length, stats.Members, stats.Decoded, stats.Failed, stats.Recovered, stats.Unverified); + diag?.WriteLine(framingFailure == null ? $" Chunks: {chunks.Count} segments of the exact framing" + : $" Chunks: {chunks.Count} decompressed by a search for gzip members (framing: {framingFailure})"); if (routing != null && int.TryParse(partEntry.Name, out int partitionNumber) && RecordRouting.Stale(routing.Census, routing.Current, partitionNumber) is { Count: > 0 } stale) { @@ -1166,6 +1169,24 @@ private List DecompressPartitions(RootStorage root, TextWriter? diag, private static Dictionary? Fail(string reason, out string? failure) { failure = reason; return null; } + /// The chunks of a partition stream: when its framing holds (), the output of each segment, every one a + /// gzip member checked by its CRC and length, and null; otherwise the members a search for gzip headers + /// finds, with the recovery of damaged ones (), and the reason the framing does not hold. + internal static List CutPartition(byte[] raw, (PartitionFraming.Tags Tags, int IdWidth)? framing, ChunkDecompressor decompressor, + out (int Members, int Decoded, int Failed, int Recovered, int Unverified) stats, out string? framingFailure) + { + var segments = new List(); + framingFailure = "the class table names no framing classes or the id width is not known"; + if (framing is var (tags, idWidth) && PartitionFraming.Walk(raw, tags, idWidth, _ => { }, out framingFailure, out _, null, segments)) + { + stats = (segments.Count, segments.Count, 0, 0, 0); + return segments; + } + var chunks = decompressor.DecompressAll(raw); + stats = decompressor.LastStats; + return chunks; + } + private sealed record RoutingPlan(RecordRouting.Census Census, Dictionary Current); /// The current copy of every element that several partition streams hold (), or null with the diff --git a/tests/RvtIO.Tests/PartitionCutTests.cs b/tests/RvtIO.Tests/PartitionCutTests.cs new file mode 100644 index 0000000..8709e9b --- /dev/null +++ b/tests/RvtIO.Tests/PartitionCutTests.cs @@ -0,0 +1,106 @@ +using RvtIO.Parsing; +using RvtIO.Writing; +using Xunit; +using static RvtIO.Tests.CoverageCensusTests; + +namespace RvtIO.Tests; + +/// How the reader cuts a partition stream into chunks (): by its segments when the framing +/// holds, else by a search for gzip members. +public class PartitionCutTests +{ + private static (PartitionFraming.Tags Tags, int IdWidth) Framing => (PartitionFraming.TagsOf(new FormatsSchema(Schema))!.Value, 4); + + private static byte[] TwoBlocks() => Partition(Block(false, R(1, Body("Plain", 5, 0, 0, 0))), Block(true, R(2, Body("Plain", 5, 0, 0, 0)))); + + [Fact] + public void AStreamWhoseFramingHolds_IsCutIntoItsSegments() + { + byte[] raw = TwoBlocks(); + + var chunks = RvtFileReader.CutPartition(raw, Framing, new ChunkDecompressor(), out var stats, out string? failure); + + Assert.Null(failure); + Assert.Equal((6, 6, 0, 0, 0), stats); + Assert.Equal(new ChunkDecompressor().DecompressAll(raw), chunks); + } + + [Fact] + public void AStreamWhoseFramingDoesNotHold_IsSearchedForGzipMembers() + { + byte[] raw = TwoBlocks(); + raw[^10] ^= 0xFF; // the class tag of the terminal marker: the framing does not end, the members are intact + + var search = new ChunkDecompressor(); + var chunks = RvtFileReader.CutPartition(raw, Framing, search, out var stats, out string? failure); + + Assert.NotNull(failure); + Assert.Equal(search.LastStats, stats); + Assert.Equal(new ChunkDecompressor().DecompressAll(raw), chunks); + // without the framing classes the stream is searched as well, and the reason says so + RvtFileReader.CutPartition(TwoBlocks(), null, new ChunkDecompressor(), out _, out failure); + Assert.Equal("the class table names no framing classes or the id width is not known", failure); + } + + [Fact] + public void TheSegmentsOfAChain_AreTheOutputOfEachOfItsMembers() + { + // one block whose channel 101 is a chain of two segments (the first continues into the second), channels 102 and 103 one segment each + byte[] first = Records(101, R(1, Body("Plain", 5, 0, 0, 0))), second = Records(101, R(2, Body("Plain", 5, 0, 0, 0))); + using var stream = new MemoryStream(); + using (var w = new BinaryWriter(stream, System.Text.Encoding.UTF8, leaveOpen: true)) + { + w.Write(9ul); + w.Write(Id("ContentMarker")); w.Write(0u); w.Write(2u); + Segment(w, 101, 2u, first, 1); + Segment(w, 101, 1u, second, 1); + Segment(w, 102, 0u, Records(102, R(1, Body("Plain", 5, 0, 0, 0)), R(2, Body("Plain", 5, 0, 0, 0))), 2); + Segment(w, 103, 0u, Records(103, R(1, Body("Plain", 5, 0, 0, 0)), R(2, Body("Plain", 5, 0, 0, 0))), 2); + w.Write(Id("SegmentMarker")); w.Write(new byte[16]); + w.Write(Id("SignatureMarker")); w.Write(0u); w.Write(0u); + w.Write(Id("ContentMarker")); w.Write(0u); w.Write(uint.MaxValue); + } + byte[] raw = stream.ToArray(); + + var chunks = RvtFileReader.CutPartition(raw, Framing, new ChunkDecompressor(), out _, out string? failure); + + Assert.Null(failure); + Assert.Equal(4, chunks.Count); + Assert.Equal(first, chunks[0]); + Assert.Equal(second, chunks[1]); + Assert.Equal(new ChunkDecompressor().DecompressAll(raw), chunks); + } + + // the items of records as a segment holds them inflated: owner, the checksum on channels 102 and 103, size, body, size + private static byte[] Records(uint channel, params ElementCensusTests.Rec[] records) + { + using var items = new MemoryStream(); + using var item = new BinaryWriter(items); + foreach (var record in records) + { + item.Write(record.Owner); + if (channel != 101) item.Write(record.Extra); + item.Write((uint)record.Body.Length); + item.Write(record.Body); + item.Write((uint)record.Body.Length); + } + item.Flush(); + return items.ToArray(); + } + + private static void Segment(BinaryWriter w, uint channel, uint flags, byte[] records, uint count) + { + byte[] payload = GzipMember.Write(records); + uint size = (uint)payload.Length + 8; + w.Write(Id("SegmentMarker")); + w.Write(flags); + w.Write(count); + w.Write(size); + w.Write((uint)(records.Length - count * (channel == 101 ? 12u : 16u))); + w.Write(channel); + w.Write(19u); + w.Write(payload); + w.Write(Id("SegmentCheckback")); + w.Write(size); + } +}