diff --git a/CHANGELOG.md b/CHANGELOG.md
index 65ec0a9..61148dc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -56,6 +56,7 @@ API between minor releases.
### Changed
+- Reading: the element index of the files with their own class schema (the header category, design option and outline of each element, the class of its root object and the field walk of that object) and the reinforcement reader read a partition stream whose framing holds by its records: the channel-101 records of the project's own block are the element headers and its channel-102 records the root objects, one of each per element. Before, a scan of the stream took any byte run that looked like an item, including the records of the documents of loaded families, whose element ids repeat project ids; a value on which such a copy differed was dropped. On 29 files the scan found 5,987,630 header-like and 6,640,008 block-like items, the framing gives 848,325 of each. A stream whose framing does not hold is scanned as before. On 52 project files of the releases 2020 to 2027 the IFC and the export report are identical to the output of the main branch; in the compact JSON of one Revit 2021 file one element (the pen width table) now gets its name and properties, as in the other files.
- Reading: a partition stream whose framing holds (content blocks, segments, gzip members with their CRC and length, record lengths, the owners of each block) is cut into its segments by that framing; the search for gzip members and its recovery of damaged members are kept for a stream whose framing does not hold, and the diagnostics say which way a stream was read. On 52 project files of the releases 2020 to 2027 the segments are the chunks the search found, byte for byte, in all 434 partition streams; the IFC, the compact JSON and the export report of 14 of them, one or more of every release, are identical to the output of the main branch.
- Field walk by the class schema of the file: a text is read as long as its count says, where the walk stopped at a text longer than 100,000 characters; a text longer than the rest of the record stops it as any read past the end does. `rvtio coverage` also reads an extensible storage entity in the block of a loaded family by the schemas of that family's own document: up to release 2024 a family keeps them in its `ADocument` in `Global/ContentDocuments` (one document per content block, under the key of the block), from release 2025 `Global/Latest` holds them. On 52 project files of the releases 2020 to 2027 every record of every partition stream is now walked to its last byte (29,794,710 records; 117 were not: 45 at a long text, 72 at an entity of a family document); no class row has fewer exact records. Output: of ten files that hold such a text, eight family types of lighting fixtures in two MEP files get their parameter list in the compact JSON, and in three files of one MEP project one view type and one sun study change the reason they are left out of the IFC for; the IFC of the ten files does not change.
- Default output, IFC and JSON: the readers behind the output read the values of an extensible storage entity by the schema of that entity, as `rvtio coverage` does, so the objects behind those values are read in their place instead of out of step, and the record of an element that keeps an entity is read to its end. An entity whose schema the file does not hold, or with a field type that is not read, is passed as before. The parameters that a store behind an entity would list as unset are still left out, as before; what that rule would give now is not measured. The bars of a rebar system that holds an entity are read now as well. A RebarInSystem that is the one RebarInSystem of its system takes the `GlobalId` of the system, as in the reference; the bar of a system with more keeps its own sub-element `GlobalId` (the number of its bars plus 3). Measured on 28 project files of the releases 2020 to 2026 against the output of the main branch, and for the IFC also against the reference: no IFC value became worse. On eight MEP files the in/out direction and the name of 887 distribution ports are now those of the reference (331, 331, 83, 83, 25, 25, 8 and 1), and on ten MEP files 3,221 assignments of a port or an element to a system are added that the reference has; 48 added assignments are not in the reference at that system (in four files; not checked in Revit). 6 systems the reference has are written that were missing; 26 pipe fitting types that the reference does not have are no longer written, and 9 others it does not have either are (types of a single fitting each, whose `GlobalId` changes with the parameters now read). In three files of one architectural project (releases 2021, 2023 and 2024) 42, 48 and 48 reinforcing bars that were left out are written, all with the `GlobalId` of the reference (in the file of release 2024, 26 of the 42 bars of its systems are the one bar of their system); the three files now write 159, 168 and 168 of the 169 bars of the reference (117, 120 and 120 before). The one bar of a system is not put into a group of its system, which it stands for. In the compact JSON 1,214 family instances of that project (three files) and 3,086 of another (one file) get the parameter list that the other instances of their categories already had, and 53 family types of one MEP file get theirs. Against Revit, on 172 of the 1,214 in the file of release 2021, every value the compact JSON writes as Revit shows it (type, mark, comments, phases, heights, levels, offsets and others, 1,561 values) is equal, and the fields the compact JSON writes in its own form (family, type and host as names, the design option as `None`, justifications as numbers, an empty level as `None`) are written as for 170 instances of the same categories that had their list before. The IFC of 16 of the 28 files and the JSON of 23 do not change.
diff --git a/docs/test-fixtures.md b/docs/test-fixtures.md
index 90f168e..aecd9d3 100644
--- a/docs/test-fixtures.md
+++ b/docs/test-fixtures.md
@@ -45,7 +45,7 @@ folder (`TestFiles.Scratch`); no test uses a fixed absolute path.
## What runs without the private files
-With no private file at all, 9683 of the 10101 test cases run and pass and 418 are skipped: 412 need private files (listed
+With no private file at all, 9685 of the 10103 test cases run and pass and 418 are skipped: 412 need private files (listed
below; 17 of them only `bim_dump.txt` or `formats_latest_decompressed.bin`) and 6 need other inputs (two need symbolic links, three
need `RVTIO_FREEFORM_FULL`, `RVTIO_CYL_FULL` or `RVTIO_MESH_FULL`, one is a known gap). The tests that run use synthetic data built
in the tests and the small fixtures committed under `tests/RvtIO.Tests/Fixtures`. With the full private fixture folder the
diff --git a/src/RvtIO/Parsing/PartitionFraming.cs b/src/RvtIO/Parsing/PartitionFraming.cs
index 639300a..1eb57d8 100644
--- a/src/RvtIO/Parsing/PartitionFraming.cs
+++ b/src/RvtIO/Parsing/PartitionFraming.cs
@@ -31,6 +31,15 @@ internal static class PartitionFraming
/// that is the concatenation of every segment's output in file order; the item is Item.Length bytes there.
internal readonly record struct Record(int Block, string Key, int Channel, ulong Owner, uint Extra, ReadOnlyMemory Body, ReadOnlyMemory Item, long Offset);
+ /// Where a record of the null-key map lies: its channel, owner, and the whole item (owner to closing size) at
+ /// for bytes in the stream's inflated output.
+ internal readonly record struct ProjectItem(int Channel, ulong Owner, long Offset, int Length);
+
+ /// A partition stream read by its framing: the output of every segment in file order, and the records of the project's own block
+ /// (the null-key map) in file order, without the chain terminators. The records of the keyed blocks (loaded family documents) are not listed.
+ /// is the byte width of the owner the walk read.
+ internal sealed record Framed(IReadOnlyList Segments, List Items, int IdWidth);
+
internal static Tags? TagsOf(FormatsSchema schema)
{
ushort? Id(string name) => schema.IdOf(name) is int id and <= ushort.MaxValue ? (ushort)id : null;
diff --git a/src/RvtIO/Parsing/RebarReader.cs b/src/RvtIO/Parsing/RebarReader.cs
index ea0a8ec..a452c80 100644
--- a/src/RvtIO/Parsing/RebarReader.cs
+++ b/src/RvtIO/Parsing/RebarReader.cs
@@ -123,7 +123,8 @@ internal static bool HeldLayout(FileSchema schema) =>
internal static ReinforcementData Read(
IReadOnlyList> streams, FileSchema schema, IReadOnlySet wanted,
- IReadOnlyDictionary? entitySchemas = null, bool passUnknownEntities = false)
+ IReadOnlyDictionary? entitySchemas = null, bool passUnknownEntities = false,
+ IReadOnlyList? framed = null)
{
var result = new Dictionary();
var joists = new Dictionary();
@@ -220,7 +221,7 @@ internal static ReinforcementData Read(
foreach (var ((layer, index), bar) in ReadHeld(walk.Captures)) held.TryAdd((id, layer, index), bar);
}
else if (ReadMember(walker, walk, byQueue) is { } member) members.TryAdd(id, member);
- });
+ }, framed);
RebarBars? Solve(PendingBar bar) =>
diameters.TryGetValue(bar.BarTypeId, out var barDiameters) && barKinds.TryGetValue(bar.BarTypeId, out var kind)
&& shapeDefinitions.TryGetValue(bar.ShapeId, out var definition) ? SolveShapeDriven(bar, definition, hookKinds, kind, barDiameters.Model) : null;
@@ -228,7 +229,7 @@ internal static ReinforcementData Read(
if (Solve(bar) is { } solved) result.TryAdd(id, solved);
// the items of a container that can be solved; their bar types come with the pass, like those of the other bars. They are in the
// coordinates of the host instance, so a container whose host has no frame is not written.
- var hostFrames = ReadHostFrames(walker, streams, schema, containers.Values.Select(c => (uint)c.HostId).ToHashSet());
+ var hostFrames = ReadHostFrames(walker, streams, framed, schema, containers.Values.Select(c => (uint)c.HostId).ToHashSet());
foreach (var (id, (typeId, hostId, inAssembly, items)) in containers)
{
if (!hostFrames.TryGetValue((uint)hostId, out var hostFrame)) continue;
@@ -299,7 +300,8 @@ private static (int TypeId, int HostId, bool InAssembly, List Items)
/// The frame (row-major 3x3 of the columns x direction (m_RefDir), its cross product with the axis, axis, then the origin: 12 numbers) of
/// each of the FamilyInstance elements (FamilyInstance.m_instOrigin, m_RefDir and m_zAxis); a host that is no
/// FamilyInstance or whose values are not a frame has none.
- private static Dictionary ReadHostFrames(WideBlockWalker walker, IReadOnlyList> streams, FileSchema schema, IReadOnlySet hosts)
+ private static Dictionary ReadHostFrames(WideBlockWalker walker, IReadOnlyList> streams,
+ IReadOnlyList? framed, FileSchema schema, IReadOnlySet hosts)
{
var frames = new Dictionary();
if (hosts.Count == 0) return frames;
@@ -320,7 +322,7 @@ private static Dictionary ReadHostFrames(WideBlockWalker walker,
if (Math.Abs(Dot(x, x) - 1) > 1e-9 || Math.Abs(Dot(z, z) - 1) > 1e-9 || Math.Abs(Dot(x, z)) > 1e-9) return;
double[] y = [z[1] * x[2] - z[2] * x[1], z[2] * x[0] - z[0] * x[2], z[0] * x[1] - z[1] * x[0]];
frames.TryAdd(id, [x[0], y[0], z[0], x[1], y[1], z[1], x[2], y[2], z[2], origin[0], origin[1], origin[2]]);
- });
+ }, framed);
return frames;
}
diff --git a/src/RvtIO/Parsing/RecordRouting.cs b/src/RvtIO/Parsing/RecordRouting.cs
index 86e6a47..c5e02d7 100644
--- a/src/RvtIO/Parsing/RecordRouting.cs
+++ b/src/RvtIO/Parsing/RecordRouting.cs
@@ -120,6 +120,14 @@ internal static Dictionary Current(Census census, IReadOnlyListThe records of the framing of one partition stream without those the given ranges () mark as not
+ /// current: both walks of the stream place an item at the same offset.
+ internal static void Drop(List items, IReadOnlyList<(long Start, int Length)> stale)
+ {
+ var staleAt = stale.Select(s => s.Start).ToHashSet();
+ items.RemoveAll(i => staleAt.Contains(i.Offset));
+ }
+
/// The chunks of one partition stream without the given byte ranges of its inflated stream. Null when the chunks are
/// not the segments of the framing (same count, same lengths), so a range could not be placed exactly.
internal static List? Prune(IReadOnlyList chunks, IReadOnlyList segmentLengths, IReadOnlyList<(long Start, int Length)> ranges)
@@ -133,6 +141,13 @@ internal static Dictionary Current(Census census, IReadOnlyList= ranges.Count || ranges[next].Start >= chunkEnd)
+ {
+ result.Add(chunk);
+ chunkStart = chunkEnd;
+ continue;
+ }
var kept = new MemoryStream(chunk.Length);
long position = chunkStart;
while (next < ranges.Count && ranges[next].Start < chunkEnd)
diff --git a/src/RvtIO/Parsing/WideElementRecords.cs b/src/RvtIO/Parsing/WideElementRecords.cs
index 7bd61cd..36a540f 100644
--- a/src/RvtIO/Parsing/WideElementRecords.cs
+++ b/src/RvtIO/Parsing/WideElementRecords.cs
@@ -11,8 +11,10 @@ namespace RvtIO.Parsing;
/// m_id, m_OwningElementId, m_partitionId); the rows are in strictly increasing m_id order. Revit's UniqueId of a row is
/// the episode GUID episodes[count - 1 - m_creationDate] and m_originalElementId (see ).
///
-/// Partition stream: back-to-back items found by a position-by-position walk. Every item opens with the 8-byte element id
-/// and closes with a 4-byte echo of its length, which is what validates a candidate:
+/// Partition stream: read by its framing () where it holds, the header records being the channel-101
+/// records and the blocks the channel-102 records of the project's own block; otherwise back-to-back items found by a
+/// position-by-position walk. Every item opens with the 8-byte element id and closes with a 4-byte echo of its length, which is
+/// what validates a candidate:
/// record [id:8][size:4][ElementHeader class id:2 ...][size:4] (item length 16 + size)
/// block [id:8][u32:4][size:4][class id:2 ...][size:4] (item length 20 + size)
/// The ElementHeader record opens with m_regenHistory (u32 count + count entries of the schema-declared fixed size), then
@@ -115,11 +117,14 @@ internal sealed class Index
/// Walks every partition stream (each given as its decoded chunks, which are concatenated) and collects,
/// for the ids in , the ElementHeader category and the Element-derived block class.
- /// Null when the schema does not declare the expected ElementHeader shape.
+ /// A stream whose entry in is set is read by its framing instead: the header records are the
+ /// channel-101 records and the blocks the channel-102 records of the project's own block, and the documents of loaded
+ /// families are not read. Null when the schema does not declare the expected ElementHeader shape.
internal static Index? BuildIndex(
IReadOnlyList> streams, FileSchema schema, IReadOnlySet wanted,
- Action? onRootBlock = null)
+ Action? onRootBlock = null, IReadOnlyList? framed = null)
{
+ if (framed != null && framed.Count != streams.Count) throw new ArgumentException("one framing entry per stream", nameof(framed));
var header = schema.ByName("ElementHeader");
var element = schema.ByName("Element");
if (header == null || element == null || header.Fields.Count < 2
@@ -143,8 +148,10 @@ internal sealed class Index
var conflictCategory = new HashSet();
var conflictDesignOption = new HashSet();
var conflictClass = new HashSet();
- foreach (var chunks in streams)
+ for (int s = 0; s < streams.Count; s++)
{
+ var frame = framed?[s] is { } f && f.IdWidth == width ? f : null;
+ var chunks = frame?.Segments ?? streams[s];
long total = 0;
foreach (var c in chunks) total += c.Length;
if (total > int.MaxValue - 64) return null;
@@ -154,7 +161,32 @@ internal sealed class Index
var recordIds = new HashSet();
var blockClasses = new Dictionary(); // -1: the stream holds blocks of two classes for the id
var rootBlocks = onRootBlock == null ? null : new List<(uint Id, int Offset, int Size, int Class)>();
- Walk(buf, header.Id, width, categoryWidth, known, isElement, entry, wanted, index, conflictCategory, conflictDesignOption, recordIds, blockClasses, rootBlocks, outlineTags);
+ void Header(int p, int size, uint id)
+ {
+ index.Records++;
+ if (!wanted.Contains(id)) return;
+ recordIds.Add(id);
+ RecordCategory(buf, p + width - 8, size, categoryWidth, entry, id, index, conflictCategory, conflictDesignOption, outlineTags);
+ }
+ void Block(int p, int size, uint id, int cls)
+ {
+ index.Blocks++;
+ if (!isElement[cls] || !wanted.Contains(id)) return;
+ blockClasses[id] = blockClasses.TryGetValue(id, out int prior) && prior != cls ? -1 : cls;
+ rootBlocks?.Add((id, p + width + 8, size, cls));
+ }
+ if (frame == null) Walk(buf, header.Id, width, known, Header, Block);
+ else
+ foreach (var item in frame.Items)
+ {
+ // an id the 32-bit element table cannot hold is skipped, as the scan skips it
+ if (item.Owner > uint.MaxValue) continue;
+ int p = (int)item.Offset;
+ if (item.Channel == 101 && item.Length >= width + 16 && BitConverter.ToUInt16(buf, p + width + 4) == header.Id)
+ Header(p, item.Length - width - 8, (uint)item.Owner);
+ else if (item.Channel == 102 && item.Length >= width + 20 && BitConverter.ToUInt16(buf, p + width + 8) is var cls && cls < known.Length && known[cls])
+ Block(p, item.Length - width - 12, (uint)item.Owner, cls);
+ }
// payload (class id first) of every Element-derived block whose stream holds the header record of its id
if (rootBlocks != null)
foreach (var (blockId, offset, size, cls) in rootBlocks)
@@ -173,10 +205,8 @@ internal sealed class Index
return index;
}
- private static void Walk(byte[] b, int headerId, int w, int categoryWidth, bool[] known, bool[] isElement, int entry,
- IReadOnlySet wanted, Index index, HashSet conflictCategory, HashSet conflictDesignOption,
- HashSet recordIds, Dictionary blockClasses,
- List<(uint Id, int Offset, int Size, int Class)>? rootBlocks, (int Outline, int Parents)? outlineTags)
+ /// The position-by-position scan: every candidate whose length echo holds is a header record or a block (item start, size, id).
+ private static void Walk(byte[] b, int headerId, int w, bool[] known, Action header, Action block)
{
int n = b.Length;
int p = 0;
@@ -189,12 +219,7 @@ private static void Walk(byte[] b, int headerId, int w, int categoryWidth, bool[
if (BitConverter.ToUInt16(b, p + w + 4) == headerId && size >= 8 && size < 4_000_000
&& p + w + 8L + size <= n && BitConverter.ToUInt32(b, p + w + 4 + (int)size) == size)
{
- index.Records++;
- if (wanted.Contains(id))
- {
- recordIds.Add(id);
- RecordCategory(b, p + w - 8, (int)size, categoryWidth, entry, id, index, conflictCategory, conflictDesignOption, outlineTags);
- }
+ header(p, (int)size, id);
p += w + 8 + (int)size;
continue;
}
@@ -204,12 +229,7 @@ private static void Walk(byte[] b, int headerId, int w, int categoryWidth, bool[
if (blockSize >= 8 && blockSize < 8_000_000 && cls < known.Length && known[cls]
&& p + w + 12L + blockSize <= n && BitConverter.ToUInt32(b, p + w + 8 + (int)blockSize) == blockSize)
{
- index.Blocks++;
- if (isElement[cls] && wanted.Contains(id))
- {
- blockClasses[id] = blockClasses.TryGetValue(id, out int prior) && prior != cls ? -1 : cls;
- rootBlocks?.Add((id, p + w + 8, (int)blockSize, cls));
- }
+ block(p, (int)blockSize, id, cls);
p += w + 12 + (int)blockSize;
continue;
}
diff --git a/src/RvtIO/RvtFileReader.cs b/src/RvtIO/RvtFileReader.cs
index 44e2f85..d8567d4 100644
--- a/src/RvtIO/RvtFileReader.cs
+++ b/src/RvtIO/RvtFileReader.cs
@@ -56,7 +56,7 @@ public RevitFileInfo Read(string rvtPath)
Dictionary? tfSchemas = null;
LogPhase("Decompressing partitions...");
- var chunks = DecompressPartitions(root, diag, out var ownPartitionWallParts, out var partitionStreams, out bool staleCopiesRemoved);
+ var chunks = DecompressPartitions(root, diag, out var ownPartitionWallParts, out var partitionStreams, out var framedStreams, out bool staleCopiesRemoved);
var fmtChunks = DecompressStorage(root, "Formats", diag);
if (fmtChunks.Count > 0)
{
@@ -104,8 +104,8 @@ public RevitFileInfo Read(string rvtPath)
R22View = r22View,
StaleCopiesRemoved = staleCopiesRemoved,
};
- ReadWideElementIndex(root, state, partitionStreams, wideSchema, diag);
- ReadRebarBars(root, state, partitionStreams, wideSchema);
+ ReadWideElementIndex(root, state, partitionStreams, framedStreams, wideSchema, diag);
+ ReadRebarBars(root, state, partitionStreams, framedStreams, wideSchema);
ClassifyAndParseRecords(chunks, state, diag);
LogPhase("Records parsed");
@@ -843,7 +843,7 @@ internal static void MergeElemScanStringsInChunkOrder(
var (previewBase64, previewMime) = PreviewImageReader.Read(root);
LogPhase("Decompressing partitions...");
- var chunks = DecompressPartitions(root, diag, out var ownPartitionWallParts, out var partitionStreams, out bool staleCopiesRemoved);
+ var chunks = DecompressPartitions(root, diag, out var ownPartitionWallParts, out var partitionStreams, out var framedStreams, out bool staleCopiesRemoved);
// Episode GUID table: nested stream Global/History (see Read() above).
var (elemRecHistories, elemRecOwners, wideSchema) = ReadElemRecs(root);
@@ -871,8 +871,8 @@ internal static void MergeElemScanStringsInChunkOrder(
R22View = r22View,
StaleCopiesRemoved = staleCopiesRemoved,
};
- ReadWideElementIndex(root, state, partitionStreams, wideSchema, diag);
- ReadRebarBars(root, state, partitionStreams, wideSchema);
+ ReadWideElementIndex(root, state, partitionStreams, framedStreams, wideSchema, diag);
+ ReadRebarBars(root, state, partitionStreams, framedStreams, wideSchema);
ClassifyAndParseRecords(chunks, state, diag);
ParseBlocks(state, diag);
RunTfStreamWalk(state);
@@ -939,17 +939,18 @@ private static (Dictionary? Histories, DictionaryThe bars of the reinforcement elements and the beam systems (), by the file's own class schema in every layout.
private static void ReadRebarBars(RootStorage root, ParsingState state,
- IReadOnlyList> partitionStreams, FileSchema? wideSchema)
+ IReadOnlyList> partitionStreams, IReadOnlyList framedStreams, FileSchema? wideSchema)
{
if ((wideSchema ?? FileSchema.Read(root)) is not { } schema) return;
- var reinforcement = RebarReader.Read(partitionStreams, schema, state.AllKnownIds, EntitySchemaReader.Read(root, schema.Document), passUnknownEntities: true);
+ var reinforcement = RebarReader.Read(partitionStreams, schema, state.AllKnownIds, EntitySchemaReader.Read(root, schema.Document), passUnknownEntities: true,
+ framed: framedStreams);
foreach (var (id, bars) in reinforcement.Bars) state.RebarBars[id] = bars;
foreach (var (id, joist) in reinforcement.Joists) state.JoistSystems[id] = joist;
}
/// Header category and root class of every ElemTable element of a 64-bit-id file (no-op for the other layouts).
private static void ReadWideElementIndex(RootStorage root, ParsingState state,
- IReadOnlyList> partitionStreams, FileSchema? schema, TextWriter? diag)
+ IReadOnlyList> partitionStreams, IReadOnlyList framedStreams, FileSchema? schema, TextWriter? diag)
{
if (schema == null) return;
state.WideSchema = schema;
@@ -981,7 +982,7 @@ private static void ReadWideElementIndex(RootStorage root, ParsingState state,
if (!drivers.TryGetValue(id, out var found)) drivers[id] = found = [];
BlockParser.ExtractCurveDriverFromBlock(data, offset, offset + size, id, drivers, wallShim);
}
- });
+ }, framedStreams);
if (index == null)
{
diag?.WriteLine("Wide element records: the file's schema does not declare the expected ElementHeader");
@@ -1105,7 +1106,7 @@ void Fail(string reason)
private List DecompressPartitions(RootStorage root, TextWriter? diag,
out Dictionary? ownPartitionWallParts,
- out List> partitionStreams, out bool staleCopiesRemoved)
+ out List> partitionStreams, out List framedStreams, out bool staleCopiesRemoved)
{
staleCopiesRemoved = false;
var partStor = root.OpenStorage("Partitions");
@@ -1118,6 +1119,7 @@ private List DecompressPartitions(RootStorage root, TextWriter? diag,
var decompressor = new ChunkDecompressor();
var allChunks = new List();
partitionStreams = new List>();
+ framedStreams = [];
ownPartitionWallParts = null;
var routing = _options.ResolveCurrentCopies && partEntries.Count > 1 ? PlanRecordRouting(root, partStor, partEntries.Select(e => e.Name).ToList(), diag) : null;
// the exact framing of the partition streams (PartitionFraming), when the class table names its marker classes and the id width is known
@@ -1144,7 +1146,7 @@ private List DecompressPartitions(RootStorage root, TextWriter? diag,
diag?.WriteLine($" Own partition wall parts: {ownPartitionWallParts?.Count.ToString() ?? "unavailable"} ({failure ?? "exact"})");
}
- var chunks = CutPartition(raw, framing, decompressor, out var stats, out string? framingFailure);
+ var chunks = CutPartition(raw, framing, decompressor, out var stats, out string? framingFailure, out var framed);
RecordStream("Partitions/" + partEntry.Name, raw.Length, stats.Members, stats.Decoded, stats.Failed, stats.Recovered, stats.Unverified);
diag?.WriteLine(framingFailure == null ? $" Chunks: {chunks.Count} segments of the exact framing"
: $" Chunks: {chunks.Count} decompressed by a search for gzip members (framing: {framingFailure})");
@@ -1158,9 +1160,12 @@ private List DecompressPartitions(RootStorage root, TextWriter? diag,
diag?.WriteLine($" Record routing: {stale.Count} records of {partEntry.Name} are not the current copy and were removed");
}
else diag?.WriteLine($" Record routing: the chunks of {partEntry.Name} are not the framed segments; {stale.Count} stale records kept");
+ // the records of the framing leave out the copies that are not current (both walks give the same offsets)
+ if (framed != null) RecordRouting.Drop(framed.Items, stale);
}
allChunks.AddRange(chunks);
partitionStreams.Add(chunks);
+ framedStreams.Add(framed);
}
diag?.WriteLine($"Total chunks: {allChunks.Count} decompressed");
@@ -1173,13 +1178,20 @@ private List DecompressPartitions(RootStorage root, TextWriter? diag,
/// gzip member checked by its CRC and length, and null; otherwise the members a search for gzip headers
/// finds, with the recovery of damaged ones (), and the reason the framing does not hold.
internal static List CutPartition(byte[] raw, (PartitionFraming.Tags Tags, int IdWidth)? framing, ChunkDecompressor decompressor,
- out (int Members, int Decoded, int Failed, int Recovered, int Unverified) stats, out string? framingFailure)
+ out (int Members, int Decoded, int Failed, int Recovered, int Unverified) stats, out string? framingFailure, out PartitionFraming.Framed? framed)
{
var segments = new List();
+ var items = new List();
+ framed = null;
framingFailure = "the class table names no framing classes or the id width is not known";
- if (framing is var (tags, idWidth) && PartitionFraming.Walk(raw, tags, idWidth, _ => { }, out framingFailure, out _, null, segments))
+ if (framing is var (tags, idWidth) && PartitionFraming.Walk(raw, tags, idWidth, r =>
+ {
+ if (r.Block == 0 && r.Owner != (idWidth == 8 ? ulong.MaxValue : uint.MaxValue))
+ items.Add(new(r.Channel, r.Owner, r.Offset, r.Item.Length));
+ }, out framingFailure, out _, null, segments))
{
stats = (segments.Count, segments.Count, 0, 0, 0);
+ framed = new PartitionFraming.Framed(segments, items, idWidth);
return segments;
}
var chunks = decompressor.DecompressAll(raw);
diff --git a/tests/RvtIO.Tests/PartitionCutTests.cs b/tests/RvtIO.Tests/PartitionCutTests.cs
index 8709e9b..55e7558 100644
--- a/tests/RvtIO.Tests/PartitionCutTests.cs
+++ b/tests/RvtIO.Tests/PartitionCutTests.cs
@@ -18,11 +18,20 @@ public void AStreamWhoseFramingHolds_IsCutIntoItsSegments()
{
byte[] raw = TwoBlocks();
- var chunks = RvtFileReader.CutPartition(raw, Framing, new ChunkDecompressor(), out var stats, out string? failure);
+ var chunks = RvtFileReader.CutPartition(raw, Framing, new ChunkDecompressor(), out var stats, out string? failure, out var framed);
Assert.Null(failure);
Assert.Equal((6, 6, 0, 0, 0), stats);
Assert.Equal(new ChunkDecompressor().DecompressAll(raw), chunks);
+ // the records of the project's block, one per channel, each item at its place in the inflated stream; the family's block is not listed
+ Assert.NotNull(framed);
+ Assert.Same(chunks, framed!.Segments);
+ Assert.Equal([101, 102, 103], framed.Items.Select(i => i.Channel));
+ Assert.All(framed.Items, i => Assert.Equal(1ul, i.Owner));
+ byte[] inflated = [.. chunks.SelectMany(c => c)];
+ Assert.All(framed.Items, i => Assert.Equal(1u, BitConverter.ToUInt32(inflated, (int)i.Offset)));
+ // the item ends with the size of its body: owner, the checksum on channels 102 and 103, size and closing size around it
+ Assert.All(framed.Items, i => Assert.Equal((uint)(i.Length - (i.Channel == 101 ? 12 : 16)), BitConverter.ToUInt32(inflated, (int)i.Offset + i.Length - 4)));
}
[Fact]
@@ -32,13 +41,14 @@ public void AStreamWhoseFramingDoesNotHold_IsSearchedForGzipMembers()
raw[^10] ^= 0xFF; // the class tag of the terminal marker: the framing does not end, the members are intact
var search = new ChunkDecompressor();
- var chunks = RvtFileReader.CutPartition(raw, Framing, search, out var stats, out string? failure);
+ var chunks = RvtFileReader.CutPartition(raw, Framing, search, out var stats, out string? failure, out var framed);
Assert.NotNull(failure);
+ Assert.Null(framed);
Assert.Equal(search.LastStats, stats);
Assert.Equal(new ChunkDecompressor().DecompressAll(raw), chunks);
// without the framing classes the stream is searched as well, and the reason says so
- RvtFileReader.CutPartition(TwoBlocks(), null, new ChunkDecompressor(), out _, out failure);
+ RvtFileReader.CutPartition(TwoBlocks(), null, new ChunkDecompressor(), out _, out failure, out _);
Assert.Equal("the class table names no framing classes or the id width is not known", failure);
}
@@ -46,25 +56,27 @@ public void AStreamWhoseFramingDoesNotHold_IsSearchedForGzipMembers()
public void TheSegmentsOfAChain_AreTheOutputOfEachOfItsMembers()
{
// one block whose channel 101 is a chain of two segments (the first continues into the second), channels 102 and 103 one segment each
- byte[] first = Records(101, R(1, Body("Plain", 5, 0, 0, 0))), second = Records(101, R(2, Body("Plain", 5, 0, 0, 0)));
+ // each channel closes with the chain terminator, owner all ones and no body, which the framing does not list as a record of the block
+ byte[] first = Records(101, R(1, Body("Plain", 5, 0, 0, 0))), second = Records(101, R(2, Body("Plain", 5, 0, 0, 0)), R(uint.MaxValue, []));
using var stream = new MemoryStream();
using (var w = new BinaryWriter(stream, System.Text.Encoding.UTF8, leaveOpen: true))
{
w.Write(9ul);
w.Write(Id("ContentMarker")); w.Write(0u); w.Write(2u);
Segment(w, 101, 2u, first, 1);
- Segment(w, 101, 1u, second, 1);
- Segment(w, 102, 0u, Records(102, R(1, Body("Plain", 5, 0, 0, 0)), R(2, Body("Plain", 5, 0, 0, 0))), 2);
- Segment(w, 103, 0u, Records(103, R(1, Body("Plain", 5, 0, 0, 0)), R(2, Body("Plain", 5, 0, 0, 0))), 2);
+ Segment(w, 101, 1u, second, 2);
+ Segment(w, 102, 0u, Records(102, R(1, Body("Plain", 5, 0, 0, 0)), R(2, Body("Plain", 5, 0, 0, 0)), R(uint.MaxValue, [])), 3);
+ Segment(w, 103, 0u, Records(103, R(1, Body("Plain", 5, 0, 0, 0)), R(2, Body("Plain", 5, 0, 0, 0)), R(uint.MaxValue, [])), 3);
w.Write(Id("SegmentMarker")); w.Write(new byte[16]);
w.Write(Id("SignatureMarker")); w.Write(0u); w.Write(0u);
w.Write(Id("ContentMarker")); w.Write(0u); w.Write(uint.MaxValue);
}
byte[] raw = stream.ToArray();
- var chunks = RvtFileReader.CutPartition(raw, Framing, new ChunkDecompressor(), out _, out string? failure);
+ var chunks = RvtFileReader.CutPartition(raw, Framing, new ChunkDecompressor(), out _, out string? failure, out var framed);
Assert.Null(failure);
+ Assert.Equal([(101, 1ul), (101, 2ul), (102, 1ul), (102, 2ul), (103, 1ul), (103, 2ul)], framed!.Items.Select(i => (i.Channel, i.Owner)));
Assert.Equal(4, chunks.Count);
Assert.Equal(first, chunks[0]);
Assert.Equal(second, chunks[1]);
diff --git a/tests/RvtIO.Tests/RecordRoutingTests.cs b/tests/RvtIO.Tests/RecordRoutingTests.cs
index ad573a9..558dea7 100644
--- a/tests/RvtIO.Tests/RecordRoutingTests.cs
+++ b/tests/RvtIO.Tests/RecordRoutingTests.cs
@@ -203,13 +203,25 @@ public void RemovesRecordRangesAcrossChunkBoundaries()
// ranges: [1,3) inside a chunk, [4,7) across the first boundary, [9,13) across the second and to the end
var pruned = RecordRouting.Prune(chunks, lengths, [(1, 2), (4, 3), (9, 4)])!;
Assert.Equal([[0, 3], [7, 8], []], pruned.Select(c => c.ToArray()).ToArray());
- // no ranges keep every byte
+ // no ranges keep every byte; a chunk no range reaches is the same array
Assert.Equal(chunks.Select(c => c.ToArray()), RecordRouting.Prune(chunks, lengths, [])!.Select(c => c.ToArray()));
+ var first = RecordRouting.Prune(chunks, lengths, [(1, 2)])!;
+ Assert.Same(chunks[1], first[1]);
+ Assert.Same(chunks[2], first[2]);
+ Assert.Equal([0, 3, 4], first[0]);
// chunks that are not the framed segments are not touched
Assert.Null(RecordRouting.Prune(chunks, [5, 5, 4], [(1, 2)]));
Assert.Null(RecordRouting.Prune(chunks, [5, 5], [(1, 2)]));
}
+ [Fact]
+ public void DropsTheFramedRecordsThatAreNotTheCurrentCopy()
+ {
+ var items = new List { new(101, 1, 0, 20), new(101, 2, 20, 20), new(102, 1, 40, 24), new(102, 2, 64, 24) };
+ RecordRouting.Drop(items, [(20, 20), (64, 24)]);
+ Assert.Equal([new(101, 1, 0, 20), new(102, 1, 40, 24)], items);
+ }
+
/// TESTFILE.rvt is a Revit 2022 project saved in five increments (partition streams 0, 1, 3, 4, 5). Measured
/// on the file: the null-key map holds 8050 + 88 + 1 + 90 + 0 element records, 68 element ids occur in more than one
/// partition, and the 50,851 keyed records of the loaded families never carry a project element id.
diff --git a/tests/RvtIO.Tests/WideElementRecordsTests.cs b/tests/RvtIO.Tests/WideElementRecordsTests.cs
index 673bf54..7af7559 100644
--- a/tests/RvtIO.Tests/WideElementRecordsTests.cs
+++ b/tests/RvtIO.Tests/WideElementRecordsTests.cs
@@ -305,6 +305,33 @@ public void BuildIndex_IgnoresAnElementBlockInAStreamWithoutTheHeaderRecordOfThe
Assert.Equal(ids.Wall, index.RootClasses[11]);
}
+ [Fact]
+ public void BuildIndex_ReadsAFramedStreamByTheRecordsOfTheProjectBlockOnly()
+ {
+ var (stream, ids) = BuildSchema();
+ var schema = FileSchema.Parse(stream)!;
+ byte[] header = Record(ids, 10, [], -2000011), root = Block(10, ids.Level);
+ // the same id in the document of a loaded family, a keyed block the framing does not list
+ var data = Join(header, root, Record(ids, 10, [], -5), Block(10, ids.Wall));
+ var framed = new PartitionFraming.Framed([data[..20], data[20..]],
+ [new(101, 10, 0, header.Length), new(102, 10, header.Length, root.Length)], 8);
+ var roots = new List<(uint Id, int Offset, int Size, int Class)>();
+
+ var index = WideElementRecords.BuildIndex([[data]], schema, new HashSet { 10 }, (id, _, offset, size, cls) => roots.Add((id, offset, size, cls)), [framed])!;
+
+ Assert.Equal(-2000011, index.HeaderCategories[10]);
+ Assert.Equal(ids.Level, index.RootClasses[10]);
+ Assert.Equal((1, 1), (index.Records, index.Blocks));
+ Assert.Equal([(10u, header.Length + 16, root.Length - 20, ids.Level)], roots);
+ // the scan takes the family's records for a second copy of the element, and the copies disagree
+ var scanned = WideElementRecords.BuildIndex([[data]], schema, new HashSet { 10 })!;
+ Assert.Null(scanned.HeaderCategories[10]);
+ Assert.False(scanned.RootClasses.ContainsKey(10));
+ // a framing read with another id width is not used: the stream is scanned
+ var otherWidth = WideElementRecords.BuildIndex([[data]], schema, new HashSet { 10 }, null, [framed with { IdWidth = 4 }])!;
+ Assert.Null(otherWidth.HeaderCategories[10]);
+ }
+
private sealed record OutlineIds(int Header, int Outline, int Parents, int Level);
/// A schema whose ElementHeader has the layout of the files: ids, m_viewRules, m_abFlags4Bytes, m_classDef and the pointers