diff --git a/Archive/ArchiveTool.cs b/Archive/ArchiveTool.cs index 60212e8..0766c1b 100644 --- a/Archive/ArchiveTool.cs +++ b/Archive/ArchiveTool.cs @@ -163,6 +163,16 @@ public static int PrintSummary(FileInfo filename, OutputFormat format) uncompressedDataSize += block.UncompressedSize; } + // The data section can be larger than the sum of the blocks, because the writer may leave + // padding between them. Only version 9 and later record where each block starts, so this is + // the only version where padding is detectable (and where it is currently written). + long paddingSize = 0; + if (blocks.Length > 0) + { + var lastBlock = blocks[blocks.Length - 1]; + paddingSize = lastBlock.Offset + lastBlock.CompressedSize - dataSize; + } + // Determine the compression algorithm by finding the first block that uses compression. // Individual blocks may be stored uncompressed even when compression is enabled, because // compression is skipped when it provides no size reduction. So the first compressed block @@ -193,6 +203,7 @@ public static int PrintSummary(FileInfo filename, OutputFormat format) unityVersion = header.UnityVersion, fileSize = header.Size, dataSize = dataSize, + blockPaddingSize = paddingSize, uncompressedDataSize = uncompressedDataSize, compressionRatio = Math.Round(compressionRatio, 2), compression = compression, @@ -208,6 +219,8 @@ public static int PrintSummary(FileInfo filename, OutputFormat format) Console.WriteLine($"{"Unity Version",-30} {header.UnityVersion}"); Console.WriteLine($"{"File Size",-30} {header.Size:N0} bytes"); Console.WriteLine($"{"Data Size",-30} {dataSize:N0} bytes"); + if (paddingSize > 0) + Console.WriteLine($"{"Block Padding Size",-30} {paddingSize:N0} bytes"); Console.WriteLine($"{"Uncompressed Data Size",-30} {uncompressedDataSize:N0} bytes"); Console.WriteLine($"{"Compression Ratio",-30} {compressionRatio:F2}x"); Console.WriteLine($"{"Compression",-30} {compression}"); @@ -263,10 +276,10 @@ static string FormatCompressionType(int compressionType) static readonly (uint bit, string name)[] KnownArchiveFlags = { - (0x40, "BlocksAndDirectoryInfoCombined"), - (0x80, "BlocksInfoAtTheEnd"), - (0x100, "OldWebPluginCompatibility"), - (0x200, "BlockInfoNeedPaddingAtStart"), + (ArchiveFlags.BlocksAndDirectoryInfoCombined, "BlocksAndDirectoryInfoCombined"), + (ArchiveFlags.BlocksInfoAtTheEnd, "BlocksInfoAtTheEnd"), + (ArchiveFlags.OldWebPluginCompatibility, "OldWebPluginCompatibility"), + (ArchiveFlags.BlockInfoNeedPaddingAtStart, "BlockInfoNeedPaddingAtStart"), }; static string[] GetArchiveFlagNames(uint flagBits) diff --git a/TestCommon/Data/LeadingEdgeBuilds/AGENTS.md b/TestCommon/Data/LeadingEdgeBuilds/AGENTS.md index 977dfac..89add21 100644 --- a/TestCommon/Data/LeadingEdgeBuilds/AGENTS.md +++ b/TestCommon/Data/LeadingEdgeBuilds/AGENTS.md @@ -6,7 +6,8 @@ The LeadingEdge build scripts regenerate this folder directly, so to update it, ## Layout -* `AssetBundles/` - the AssetBundle build: one bundle per asset (named after the asset) plus the `AssetBundles` manifest bundle. +* `AssetBundles/` - the AssetBundle build: one bundle per asset (named after the asset) plus the `AssetBundles` manifest bundle. LZMA compressed, so each archive has a single data block. +* `AssetBundlesLz4/` - the same bundle layout built with chunk-based (LZ4) compression, giving archives with multiple data blocks. The writer leaves padding between the blocks, so these exercise the `StorageBlock.Offset` field added in archive format version 9. * `ContentDirectory/` - the Content Directory build: content (`.cf`) files, `.resource` files and the build manifest. * `BuildReport-AssetBundles/LastBuild.buildreport` - the AssetBundle build report. * `BuildReport-ContentDirectory/` - the Content Directory build report folder, including `ContentLayout.json`. diff --git a/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/6 b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/6 new file mode 100644 index 0000000..de67805 Binary files /dev/null and b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/6 differ diff --git a/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/6.manifest b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/6.manifest new file mode 100644 index 0000000..e2fa23a --- /dev/null +++ b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/6.manifest @@ -0,0 +1,21 @@ +ManifestFileVersion: 0 +UnityVersion: 6000.7.0b1 +CRC: 2778854771 +Hashes: + AssetFileHash: + serializedVersion: 2 + Hash: a19e41b950fe653a3177d32160af00b9 + TypeTreeHash: + serializedVersion: 2 + Hash: 9a2ca7bdbd1871f7131daf57de908e0c + IncrementalBuildHash: + serializedVersion: 2 + Hash: 929d8f9d46e767e1fc66294cd024c1fa +HashAppended: 0 +ClassTypes: +- Class: 83 + Script: {instanceID: 0} +SerializeReferenceClassIdentifiers: [] +Assets: +- Assets/Audio/6.mp3 +Dependencies: [] diff --git a/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/AssetBundlesLz4 b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/AssetBundlesLz4 new file mode 100644 index 0000000..5c5edda Binary files /dev/null and b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/AssetBundlesLz4 differ diff --git a/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/AssetBundlesLz4.manifest b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/AssetBundlesLz4.manifest new file mode 100644 index 0000000..4196fa0 --- /dev/null +++ b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/AssetBundlesLz4.manifest @@ -0,0 +1,33 @@ +ManifestFileVersion: 0 +UnityVersion: 6000.7.0b1 +CRC: 318228433 +HashAppended: 0 +AssetBundleManifest: + AssetBundleInfos: + Info_0: + Name: assetbundleroot + Dependencies: + Dependency_0: directaudioclipreference + Dependency_1: singleaudioclipdirectreference + Dependency_2: serializationdemo + Info_1: + Name: directaudioclipreference + Dependencies: + Dependency_0: 6 + Dependency_1: a + Info_2: + Name: singleaudioclipdirectreference + Dependencies: + Dependency_0: a + Info_3: + Name: serializationdemo + Dependencies: {} + Info_4: + Name: 6 + Dependencies: {} + Info_5: + Name: a + Dependencies: {} + Info_6: + Name: scenes + Dependencies: {} diff --git a/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/a b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/a new file mode 100644 index 0000000..9a8d7ad Binary files /dev/null and b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/a differ diff --git a/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/a.manifest b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/a.manifest new file mode 100644 index 0000000..11d7c92 --- /dev/null +++ b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/a.manifest @@ -0,0 +1,21 @@ +ManifestFileVersion: 0 +UnityVersion: 6000.7.0b1 +CRC: 3846767907 +Hashes: + AssetFileHash: + serializedVersion: 2 + Hash: a408b749397654c01bfaa7832600a856 + TypeTreeHash: + serializedVersion: 2 + Hash: 9a2ca7bdbd1871f7131daf57de908e0c + IncrementalBuildHash: + serializedVersion: 2 + Hash: aeb2216eeb5ccfe1cb3f81e62f704746 +HashAppended: 0 +ClassTypes: +- Class: 83 + Script: {instanceID: 0} +SerializeReferenceClassIdentifiers: [] +Assets: +- Assets/Audio/a.mp3 +Dependencies: [] diff --git a/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/assetbundleroot b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/assetbundleroot new file mode 100644 index 0000000..c0fbfec Binary files /dev/null and b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/assetbundleroot differ diff --git a/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/assetbundleroot.manifest b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/assetbundleroot.manifest new file mode 100644 index 0000000..4f1cfef --- /dev/null +++ b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/assetbundleroot.manifest @@ -0,0 +1,34 @@ +ManifestFileVersion: 0 +UnityVersion: 6000.7.0b1 +CRC: 408809727 +Hashes: + AssetFileHash: + serializedVersion: 2 + Hash: f0b4981a1d1b8cf8a1bb7d53527a4e07 + TypeTreeHash: + serializedVersion: 2 + Hash: 819240184df2798191fd853444ad58c4 + IncrementalBuildHash: + serializedVersion: 2 + Hash: cf13ad57198a935f56c518df40c037d0 +HashAppended: 0 +ClassTypes: +- Class: 114 + Script: {fileID: 11500000, guid: d6330d3e9b8e5a0439e4dd147cec19dd, type: 3} +- Class: 114 + Script: {fileID: 11500000, guid: 8623c5efbb626994da80931050f0aba0, type: 3} +- Class: 114 + Script: {fileID: 11500000, guid: f44aeb02ae06dd84bb3ef76e1df8d525, type: 3} +- Class: 115 + Script: {instanceID: 0} +SerializeReferenceClassIdentifiers: +- AssemblyName: Assembly-CSharp + ClassName: SerializationDemo/SerializedData +- AssemblyName: UnityEngine.CoreModule + ClassName: UnityEngine.DictionarySerialization/SerializedKeyValue`2 +Assets: +- Assets/ScriptableObjects/AssetBundleRoot.asset +Dependencies: +- C:/UnitySrc/UnityDataTools/UnityProjects/LeadingEdge/../../TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/directaudioclipreference +- C:/UnitySrc/UnityDataTools/UnityProjects/LeadingEdge/../../TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/serializationdemo +- C:/UnitySrc/UnityDataTools/UnityProjects/LeadingEdge/../../TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/singleaudioclipdirectreference diff --git a/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/directaudioclipreference b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/directaudioclipreference new file mode 100644 index 0000000..5107907 Binary files /dev/null and b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/directaudioclipreference differ diff --git a/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/directaudioclipreference.manifest b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/directaudioclipreference.manifest new file mode 100644 index 0000000..b6df488 --- /dev/null +++ b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/directaudioclipreference.manifest @@ -0,0 +1,29 @@ +ManifestFileVersion: 0 +UnityVersion: 6000.7.0b1 +CRC: 3870990589 +Hashes: + AssetFileHash: + serializedVersion: 2 + Hash: ba2d63b18f0c278121a49435b585c703 + TypeTreeHash: + serializedVersion: 2 + Hash: 4c88c857f41d6968a1090f557f31c5ed + IncrementalBuildHash: + serializedVersion: 2 + Hash: 7b9a02ffb784fe748e77e41e9e934187 +HashAppended: 0 +ClassTypes: +- Class: 83 + Script: {instanceID: 0} +- Class: 114 + Script: {fileID: 11500000, guid: d6330d3e9b8e5a0439e4dd147cec19dd, type: 3} +- Class: 115 + Script: {instanceID: 0} +SerializeReferenceClassIdentifiers: +- AssemblyName: UnityEngine.CoreModule + ClassName: UnityEngine.DictionarySerialization/SerializedKeyValue`2 +Assets: +- Assets/ScriptableObjects/DirectAudioClipReference.asset +Dependencies: +- C:/UnitySrc/UnityDataTools/UnityProjects/LeadingEdge/../../TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/6 +- C:/UnitySrc/UnityDataTools/UnityProjects/LeadingEdge/../../TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/a diff --git a/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/scenes b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/scenes new file mode 100644 index 0000000..ddb0ede Binary files /dev/null and b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/scenes differ diff --git a/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/scenes.manifest b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/scenes.manifest new file mode 100644 index 0000000..7ef5241 --- /dev/null +++ b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/scenes.manifest @@ -0,0 +1,42 @@ +ManifestFileVersion: 0 +UnityVersion: 6000.7.0b1 +CRC: 2529189112 +Hashes: + AssetFileHash: + serializedVersion: 2 + Hash: 6065d3cb8b4bef9ba301ab70514653b1 + TypeTreeHash: + serializedVersion: 2 + Hash: 62ed4bb4a9e4f0abaa38cfea31ee7bca + IncrementalBuildHash: + serializedVersion: 2 + Hash: e0ff4bc9af5bb69181585c505b429585 +HashAppended: 0 +ClassTypes: +- Class: 1 + Script: {instanceID: 0} +- Class: 4 + Script: {instanceID: 0} +- Class: 21 + Script: {instanceID: 0} +- Class: 28 + Script: {instanceID: 0} +- Class: 48 + Script: {instanceID: 0} +- Class: 89 + Script: {instanceID: 0} +- Class: 104 + Script: {instanceID: 0} +- Class: 157 + Script: {instanceID: 0} +- Class: 196 + Script: {instanceID: 0} +- Class: 212 + Script: {instanceID: 0} +- Class: 213 + Script: {instanceID: 0} +SerializeReferenceClassIdentifiers: [] +Assets: +- Assets/Scenes/Scene1.unity +- Assets/Scenes/Scene2.unity +Dependencies: [] diff --git a/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/serializationdemo b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/serializationdemo new file mode 100644 index 0000000..23c393d Binary files /dev/null and b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/serializationdemo differ diff --git a/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/serializationdemo.manifest b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/serializationdemo.manifest new file mode 100644 index 0000000..fd18465 --- /dev/null +++ b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/serializationdemo.manifest @@ -0,0 +1,25 @@ +ManifestFileVersion: 0 +UnityVersion: 6000.7.0b1 +CRC: 2011437194 +Hashes: + AssetFileHash: + serializedVersion: 2 + Hash: 379f68b39b6bf066e0f11fd2e4be68c5 + TypeTreeHash: + serializedVersion: 2 + Hash: fc9cbcabf595bb240394111e433d9cd2 + IncrementalBuildHash: + serializedVersion: 2 + Hash: 8b1b2638a68e5b09acfce1a73c16f6e4 +HashAppended: 0 +ClassTypes: +- Class: 114 + Script: {fileID: 11500000, guid: f44aeb02ae06dd84bb3ef76e1df8d525, type: 3} +- Class: 115 + Script: {instanceID: 0} +SerializeReferenceClassIdentifiers: +- AssemblyName: Assembly-CSharp + ClassName: SerializationDemo/SerializedData +Assets: +- Assets/ScriptableObjects/SerializationDemo.asset +Dependencies: [] diff --git a/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/singleaudioclipdirectreference b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/singleaudioclipdirectreference new file mode 100644 index 0000000..7f3e9a3 Binary files /dev/null and b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/singleaudioclipdirectreference differ diff --git a/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/singleaudioclipdirectreference.manifest b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/singleaudioclipdirectreference.manifest new file mode 100644 index 0000000..727ae4a --- /dev/null +++ b/TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/singleaudioclipdirectreference.manifest @@ -0,0 +1,28 @@ +ManifestFileVersion: 0 +UnityVersion: 6000.7.0b1 +CRC: 2956055036 +Hashes: + AssetFileHash: + serializedVersion: 2 + Hash: 58b80138f3f51447bc34f00041a3c4b1 + TypeTreeHash: + serializedVersion: 2 + Hash: 4c88c857f41d6968a1090f557f31c5ed + IncrementalBuildHash: + serializedVersion: 2 + Hash: f286e63fa9c6774d5dffd02de8e71a2c +HashAppended: 0 +ClassTypes: +- Class: 83 + Script: {instanceID: 0} +- Class: 114 + Script: {fileID: 11500000, guid: d6330d3e9b8e5a0439e4dd147cec19dd, type: 3} +- Class: 115 + Script: {instanceID: 0} +SerializeReferenceClassIdentifiers: +- AssemblyName: UnityEngine.CoreModule + ClassName: UnityEngine.DictionarySerialization/SerializedKeyValue`2 +Assets: +- Assets/ScriptableObjects/SingleAudioClipDirectReference.asset +Dependencies: +- C:/UnitySrc/UnityDataTools/UnityProjects/LeadingEdge/../../TestCommon/Data/LeadingEdgeBuilds/AssetBundlesLz4/a diff --git a/UnityBinaryFormat/ArchiveDetector.cs b/UnityBinaryFormat/ArchiveDetector.cs index 9deb80f..e8a88c7 100644 --- a/UnityBinaryFormat/ArchiveDetector.cs +++ b/UnityBinaryFormat/ArchiveDetector.cs @@ -16,8 +16,9 @@ namespace UnityDataTools.BinaryFormat; /// - Data: One or more blocks of file content. Each block has its own compression type /// recorded in its per-block flags. The metadata section is required to interpret the data. /// A single file can span multiple blocks, and a single block can contain data for multiple files. -/// The blocks account for every byte of the data (there are no offsets stored - no overlapping or -/// gaps can be expressed). However the files could have padding between them. +/// The blocks account for every byte of the uncompressed data, which has no gaps. On disk the +/// writer may leave padding between the blocks; from format version 9 each block records its own +/// position so a reader never has to assume a padding rule. /// /// The metadata can appear directly after the header (default layout) or at the end of the /// file after the data (indicated by the BlocksInfoAtTheEnd flag). @@ -47,6 +48,24 @@ public class ArchiveHeaderInfo /// Archive flag bits (bits 6+ of Flags), with compression bits masked out. /// public uint ArchiveFlagBits => Flags & ~0x3Fu; + + /// + /// True when each StorageBlock records its own offset within the data section. Older archives + /// don't, and their blocks are stored contiguously. + /// + public bool HasStorageBlockOffsets => Signature == "UnityFS" && Version >= 9; +} + +/// +/// Archive header flag bits (bits 6 and up of the header Flags field). Bits 0-5 hold the +/// metadata CompressionType instead. +/// +public static class ArchiveFlags +{ + public const uint BlocksAndDirectoryInfoCombined = 0x40; + public const uint BlocksInfoAtTheEnd = 0x80; + public const uint OldWebPluginCompatibility = 0x100; + public const uint BlockInfoNeedPaddingAtStart = 0x200; } public class ArchiveStorageBlock @@ -57,6 +76,13 @@ public class ArchiveStorageBlock public int CompressionType => Flags & 0x3F; public bool IsStreamed => (Flags & 0x40) != 0; + /// + /// Offset of this block's stored bytes from the start of the data section. Serialized from + /// format version 9; for older archives the blocks are contiguous and this is accumulated + /// from the preceding compressed sizes. + /// + public long Offset { get; set; } + /// /// Offset of this block from the start of the archive file. /// Calculated after parsing — not stored in the serialized data. @@ -276,10 +302,7 @@ public static bool TryReadArchiveMetadata(string filePath, ArchiveHeaderInfo hea metadata = null; errorMessage = null; - const uint flagBlocksAndDirectoryInfoCombined = 0x40; - const uint flagBlocksInfoAtTheEnd = 0x80; - - if ((header.ArchiveFlagBits & flagBlocksAndDirectoryInfoCombined) == 0) + if ((header.ArchiveFlagBits & ArchiveFlags.BlocksAndDirectoryInfoCombined) == 0) { errorMessage = "This archive does not use the combined BlocksInfo+DirectoryInfo layout. Only the combined layout is supported."; return false; @@ -291,7 +314,7 @@ public static bool TryReadArchiveMetadata(string filePath, ArchiveHeaderInfo hea // Calculate where the metadata section starts. long metadataOffset; - if ((header.ArchiveFlagBits & flagBlocksInfoAtTheEnd) != 0) + if ((header.ArchiveFlagBits & ArchiveFlags.BlocksInfoAtTheEnd) != 0) metadataOffset = (long)(header.Size - header.CompressedMetadataSize); else metadataOffset = GetHeaderSize(header); @@ -334,17 +357,17 @@ public static bool TryReadArchiveMetadata(string filePath, ArchiveHeaderInfo hea using var memStream = new MemoryStream(uncompressedData); using var reader = new BinaryReader(memStream); - var blocksInfo = ParseBlocksInfo(reader); + var blocksInfo = ParseBlocksInfo(reader, header); var directoryInfo = ParseDirectoryInfo(reader); - // Populate calculated offsets on each block. - long fileOffset = GetDataOffset(header); + // Convert each block's data-section-relative Offset into a file position, and accumulate + // the offsets into the uncompressed data, which is contiguous. + long dataSectionStart = GetDataOffset(header); long dataOffset = 0; foreach (var block in blocksInfo.Blocks) { - block.FileOffset = fileOffset; + block.FileOffset = dataSectionStart + block.Offset; block.DataOffset = dataOffset; - fileOffset += block.CompressedSize; dataOffset += block.UncompressedSize; } @@ -371,14 +394,11 @@ public static bool TryReadArchiveMetadata(string filePath, ArchiveHeaderInfo hea /// public static long GetDataOffset(ArchiveHeaderInfo header) { - const uint flagBlocksInfoAtTheEnd = 0x80; - const uint flagBlockInfoNeedPaddingAtStart = 0x200; - long offset = GetHeaderSize(header); - if ((header.ArchiveFlagBits & flagBlocksInfoAtTheEnd) == 0) + if ((header.ArchiveFlagBits & ArchiveFlags.BlocksInfoAtTheEnd) == 0) { - if ((header.ArchiveFlagBits & flagBlockInfoNeedPaddingAtStart) != 0) + if ((header.ArchiveFlagBits & ArchiveFlags.BlockInfoNeedPaddingAtStart) != 0) offset += AlignTo16(header.CompressedMetadataSize); else offset += header.CompressedMetadataSize; @@ -430,10 +450,8 @@ static void ValidateMetadata(ArchiveBlocksInfo blocksInfo, ArchiveDirectoryInfo static int GetHeaderSize(ArchiveHeaderInfo header) { - const uint flagOldWebPluginCompatibility = 0x100; - int size; - if ((header.ArchiveFlagBits & flagOldWebPluginCompatibility) != 0) + if ((header.ArchiveFlagBits & ArchiveFlags.OldWebPluginCompatibility) != 0) size = 10; // Legacy web plugin signature portion else size = header.Signature.Length + 1; @@ -452,25 +470,45 @@ static int GetHeaderSize(ArchiveHeaderInfo header) return size; } - static long AlignTo16(uint value) + static long AlignTo16(long value) { return (value + 15) & ~15L; } - static ArchiveBlocksInfo ParseBlocksInfo(BinaryReader reader) + static ArchiveBlocksInfo ParseBlocksInfo(BinaryReader reader, ArchiveHeaderInfo header) { var hash = reader.ReadBytes(16); var blockCount = BinaryFileHelper.ReadUInt32(reader, true); + var hasOffsets = header.HasStorageBlockOffsets; var blocks = new ArchiveStorageBlock[blockCount]; + long contiguousOffset = 0; for (int i = 0; i < blockCount; i++) { - blocks[i] = new ArchiveStorageBlock + var block = new ArchiveStorageBlock { UncompressedSize = BinaryFileHelper.ReadUInt32(reader, true), CompressedSize = BinaryFileHelper.ReadUInt32(reader, true), Flags = BinaryFileHelper.ReadUInt16(reader, true), }; + + if (hasOffsets) + { + block.Offset = (long)BinaryFileHelper.ReadUInt64(reader, true); + + // Blocks are stored in order and must not overlap. Padding between them is allowed. + if (block.Offset < contiguousOffset) + throw new InvalidDataException( + $"Block {i} is stored at data offset {block.Offset}, which overlaps the preceding " + + $"block ending at {contiguousOffset}. The file may be corrupt."); + } + else + { + block.Offset = contiguousOffset; + } + + contiguousOffset = block.Offset + block.CompressedSize; + blocks[i] = block; } return new ArchiveBlocksInfo diff --git a/UnityDataTool.Tests/ArchiveTests.cs b/UnityDataTool.Tests/ArchiveTests.cs index b21d7de..f108899 100644 --- a/UnityDataTool.Tests/ArchiveTests.cs +++ b/UnityDataTool.Tests/ArchiveTests.cs @@ -17,12 +17,21 @@ public class ArchiveTests private string m_TestDataFolder; private string m_ArchivePath; + // An LZ4 (chunk-based) archive in format version 9, where each block records its own offset + // and the writer left padding between them. + private string m_Version9ArchivePath; + + // A version 8 archive with several blocks, which has no stored offsets. + private string m_LegacyMultiBlockArchivePath; + [OneTimeSetUp] public void OneTimeSetup() { m_TestOutputFolder = Path.Combine(TestContext.CurrentContext.TestDirectory, "test_folder"); m_TestDataFolder = Path.Combine(TestContext.CurrentContext.TestDirectory, "Data"); m_ArchivePath = Path.Combine(m_TestDataFolder, "AssetBundles", "2023.1.0a16", "scenes"); + m_Version9ArchivePath = Path.Combine(m_TestDataFolder, "LeadingEdgeBuilds", "AssetBundlesLz4", "scenes"); + m_LegacyMultiBlockArchivePath = Path.Combine(m_TestDataFolder, "PlayerDataCompressed", "data.unity3d"); Directory.CreateDirectory(m_TestOutputFolder); Directory.SetCurrentDirectory(m_TestOutputFolder); } @@ -389,4 +398,140 @@ public async Task ArchiveExtract_WithFilter_ExtractsOnlyMatchingFiles() Assert.IsFalse(File.Exists(Path.Combine(m_TestOutputFolder, "archive", file)), $"File should not have been extracted: {file}"); } } + + [Test] + public async Task ArchiveHeader_Version9_AllFlagsRecognized() + { + using var sw = new StringWriter(); + var currentOut = Console.Out; + try + { + Console.SetOut(sw); + + Assert.AreEqual(0, await Program.Main(new string[] { "archive", "header", m_Version9ArchivePath, "-f", "Json" })); + + var json = JsonDocument.Parse(sw.ToString()).RootElement; + + Assert.AreEqual(9u, json.GetProperty("version").GetUInt32()); + + // Unrecognized bits are reported as raw hex, so this catches a flag we don't know about. + var flags = json.GetProperty("flags").EnumerateArray().Select(f => f.GetString()).ToArray(); + Assert.That(flags, Has.None.StartsWith("0x"), $"Unrecognized archive flag bits: {string.Join(", ", flags)}"); + } + finally + { + Console.SetOut(currentOut); + } + } + + // From version 9 each block records where its stored bytes start, so the reader must take the + // position from the block list rather than accumulating the compressed sizes. This checks the + // parsed offsets against the actual file: the blocks are in order, the gaps between them are + // padding that belongs to no block (all zero bytes), and at least one block sits somewhere + // accumulation would not have put it - which is what proves the stored offset is being used. + [Test] + public async Task ArchiveBlocks_Version9_OffsetsComeFromTheBlockList() + { + using var sw = new StringWriter(); + var currentOut = Console.Out; + try + { + Console.SetOut(sw); + + Assert.AreEqual(0, await Program.Main(new string[] { "archive", "blocks", m_Version9ArchivePath, "-f", "Json" })); + + var blocks = JsonDocument.Parse(sw.ToString()).RootElement.GetProperty("blocks").EnumerateArray().ToArray(); + Assert.Greater(blocks.Length, 1, "The test archive is expected to have several chunks."); + + var archiveBytes = File.ReadAllBytes(m_Version9ArchivePath); + long totalPadding = 0; + + for (int i = 1; i < blocks.Length; i++) + { + var previousEnd = blocks[i - 1].GetProperty("fileOffset").GetInt64() + + blocks[i - 1].GetProperty("compressedSize").GetInt64(); + var offset = blocks[i].GetProperty("fileOffset").GetInt64(); + + Assert.GreaterOrEqual(offset, previousEnd, $"Block {i} overlaps the preceding block."); + + for (var p = previousEnd; p < offset; p++) + Assert.AreEqual(0, archiveBytes[p], $"Padding byte at offset {p} is not zero."); + + totalPadding += offset - previousEnd; + } + + Assert.Greater(totalPadding, 0, + "The archive's blocks are contiguous, so this test would pass even if the stored offsets were ignored. " + + "The fixture needs to be an archive the writer left padding in."); + } + finally + { + Console.SetOut(currentOut); + } + } + + // Version 8 and earlier don't store block offsets; their blocks are contiguous and the parser + // has to fall back to accumulating the compressed sizes. + [Test] + public async Task ArchiveBlocks_LegacyVersion8_BlocksAreContiguous() + { + using var sw = new StringWriter(); + var currentOut = Console.Out; + try + { + Console.SetOut(sw); + + Assert.AreEqual(0, await Program.Main(new string[] { "archive", "blocks", m_LegacyMultiBlockArchivePath, "-f", "Json" })); + + var blocks = JsonDocument.Parse(sw.ToString()).RootElement.GetProperty("blocks").EnumerateArray().ToArray(); + Assert.Greater(blocks.Length, 1, "The legacy test archive is expected to have several blocks."); + + for (int i = 1; i < blocks.Length; i++) + { + var previousEnd = blocks[i - 1].GetProperty("fileOffset").GetInt64() + + blocks[i - 1].GetProperty("compressedSize").GetInt64(); + Assert.AreEqual(previousEnd, blocks[i].GetProperty("fileOffset").GetInt64(), + $"Block {i} of a version 8 archive should directly follow the preceding block."); + } + } + finally + { + Console.SetOut(currentOut); + } + } + + [Test] + public async Task ArchiveInfo_Version9_ReportsPaddingSize() + { + using var sw = new StringWriter(); + var currentOut = Console.Out; + try + { + Console.SetOut(sw); + + Assert.AreEqual(0, await Program.Main(new string[] { "archive", "info", m_Version9ArchivePath, "-f", "Json" })); + + var json = JsonDocument.Parse(sw.ToString()).RootElement; + + Assert.AreEqual("Lz4HC", json.GetProperty("compression").GetString()); + Assert.Greater(json.GetProperty("blockPaddingSize").GetInt64(), 0); + } + finally + { + Console.SetOut(currentOut); + } + } + + // The native library reads the block offsets independently of the C# parser, so successful + // extraction of content spanning several non-contiguous blocks confirms the layout is understood. + [Test] + public async Task ArchiveExtract_Version9_FilesExtractedSuccessfully() + { + Assert.AreEqual(0, await Program.Main(new string[] { "archive", "extract", m_Version9ArchivePath })); + + // BuildPlayer-Scene1.sharedAssets spans the first eight blocks of the archive. + var extractedFile = new FileInfo(Path.Combine(m_TestOutputFolder, "archive", "BuildPlayer-Scene1.sharedAssets")); + Assert.IsTrue(extractedFile.Exists, "Expected file not found: BuildPlayer-Scene1.sharedAssets"); + Assert.AreEqual(761460, extractedFile.Length); + } } diff --git a/UnityProjects/LeadingEdge/AGENTS.md b/UnityProjects/LeadingEdge/AGENTS.md index def6bca..2e68086 100644 --- a/UnityProjects/LeadingEdge/AGENTS.md +++ b/UnityProjects/LeadingEdge/AGENTS.md @@ -15,7 +15,7 @@ Both builds start from a root ScriptableObject whose serialized dictionary maps ## Editor scripts (`Assets/Editor`, `ContentDirectory` menu) * `GenerateAssets.cs` - creates the ScriptableObject assets in `Assets/ScriptableObjects`, populating the serialized dictionaries before saving so the entries are serialized into the assets. -* `BuildAssetBundles.cs` - runs the AssetBundle build and copies its build report. +* `BuildAssetBundles.cs` - runs the AssetBundle build and copies its build report. It has a second entry point, `BuildLz4`, that repeats the build with chunk-based (LZ4) compression into a separate folder. * `BuildContentDirectory.cs` - runs the Content Directory build and copies its build report folder. Both build scripts write directly into `TestCommon/Data/LeadingEdgeBuilds` using paths relative to the project root. @@ -27,3 +27,5 @@ The root asset is `ContentDirectoryRoot.asset`. It directly references the `Load ## AssetBundle Build The root asset is `AssetBundleRoot.asset`. AssetBundles do not support `Loadable`, so this build uses the direct-reference variants of the assets instead. Each asset is placed in its own bundle (named after the asset) - a highly granular layout that guarantees no content is duplicated across bundles. The two scenes are placed together in a `scenes` bundle, since AssetBundles require scenes and assets in separate bundles. + +The default build uses LZMA, which produces a single streamed data block per archive. The `Build AssetBundles (LZ4)` menu item repeats the same bundle layout with chunk-based compression, which produces archives with many small blocks. The writer leaves padding between those blocks, so they exercise the per-block `StorageBlock.Offset` added in archive format version 9. diff --git a/UnityProjects/LeadingEdge/Assets/Audio/6.mp3.meta b/UnityProjects/LeadingEdge/Assets/Audio/6.mp3.meta index e3138db..7b8940e 100644 --- a/UnityProjects/LeadingEdge/Assets/Audio/6.mp3.meta +++ b/UnityProjects/LeadingEdge/Assets/Audio/6.mp3.meta @@ -1,2 +1,23 @@ fileFormatVersion: 2 guid: 278c261333bf8604eb5c83790d02004d +AudioImporter: + externalObjects: {} + serializedVersion: 8 + defaultSettings: + serializedVersion: 2 + loadType: 0 + sampleRateSetting: 0 + sampleRateOverride: 44100 + compressionFormat: 1 + quality: 1 + conversionMode: 0 + preloadAudioData: 0 + platformSettingOverrides: {} + forceToMono: 0 + normalize: 1 + loadInBackground: 0 + ambisonic: 0 + 3D: 1 + userData: + assetBundleName: + assetBundleVariant: diff --git a/UnityProjects/LeadingEdge/Assets/Audio/a.mp3.meta b/UnityProjects/LeadingEdge/Assets/Audio/a.mp3.meta index ef181d2..789f225 100644 --- a/UnityProjects/LeadingEdge/Assets/Audio/a.mp3.meta +++ b/UnityProjects/LeadingEdge/Assets/Audio/a.mp3.meta @@ -1,2 +1,23 @@ fileFormatVersion: 2 guid: b65a7916245593b4e89f4bd0aa920533 +AudioImporter: + externalObjects: {} + serializedVersion: 8 + defaultSettings: + serializedVersion: 2 + loadType: 0 + sampleRateSetting: 0 + sampleRateOverride: 44100 + compressionFormat: 1 + quality: 1 + conversionMode: 0 + preloadAudioData: 0 + platformSettingOverrides: {} + forceToMono: 0 + normalize: 1 + loadInBackground: 0 + ambisonic: 0 + 3D: 1 + userData: + assetBundleName: + assetBundleVariant: diff --git a/UnityProjects/LeadingEdge/Assets/Editor/BuildAssetBundles.cs b/UnityProjects/LeadingEdge/Assets/Editor/BuildAssetBundles.cs index 68a023b..23bac32 100644 --- a/UnityProjects/LeadingEdge/Assets/Editor/BuildAssetBundles.cs +++ b/UnityProjects/LeadingEdge/Assets/Editor/BuildAssetBundles.cs @@ -8,6 +8,10 @@ // Assets/Audio. This highly granular layout ensures no asset is duplicated across bundles - shared assets // (e.g. a.mp3) live in a single bundle that the others depend on. A scene bundle holds the two scenes (AssetBundles // require scenes and assets in separate bundles). The build report is copied alongside the output. +// +// Two variants are produced into separate folders: the default LZMA build (a single streamed data block) and an +// LZ4 chunk-based build (many small blocks). The writer leaves padding between those blocks, so the LZ4 archives +// exercise the per-block offset added to the block list in archive format version 9. public static class BuildAssetBundles { const string AudioFolder = "Assets/Audio"; @@ -16,6 +20,7 @@ public static class BuildAssetBundles const string TestDataFolder = "../../TestCommon/Data/LeadingEdgeBuilds"; const string OutputFolder = TestDataFolder + "/AssetBundles"; const string BuildReportFolder = TestDataFolder + "/BuildReport-AssetBundles"; + const string Lz4OutputFolder = TestDataFolder + "/AssetBundlesLz4"; static readonly string[] DirectAssets = { @@ -28,7 +33,19 @@ public static class BuildAssetBundles [MenuItem("ContentDirectory/Build AssetBundles")] public static void Build() { - Directory.CreateDirectory(OutputFolder); + Build(BuildAssetBundleOptions.None, OutputFolder, BuildReportFolder); + } + + [MenuItem("ContentDirectory/Build AssetBundles (LZ4)")] + public static void BuildLz4() + { + Build(BuildAssetBundleOptions.ChunkBasedCompression, Lz4OutputFolder, null); + } + + // buildReportFolder may be null to skip copying the report. + static void Build(BuildAssetBundleOptions options, string outputFolder, string buildReportFolder) + { + Directory.CreateDirectory(outputFolder); var bundles = new List(); @@ -59,9 +76,9 @@ public static void Build() var parameters = new BuildAssetBundlesParameters { - outputPath = OutputFolder, + outputPath = outputFolder, bundleDefinitions = bundles.ToArray(), - options = BuildAssetBundleOptions.None, + options = options, targetPlatform = EditorUserBuildSettings.activeBuildTarget }; @@ -72,9 +89,12 @@ public static void Build() return; } - Directory.CreateDirectory(BuildReportFolder); - File.Copy("Library/LastBuild.buildreport", $"{BuildReportFolder}/LastBuild.buildreport", true); + if (buildReportFolder != null) + { + Directory.CreateDirectory(buildReportFolder); + File.Copy("Library/LastBuild.buildreport", $"{buildReportFolder}/LastBuild.buildreport", true); + } - Debug.Log($"BuildAssetBundles: built {manifest.GetAllAssetBundles().Length} bundles into {OutputFolder}."); + Debug.Log($"BuildAssetBundles: built {manifest.GetAllAssetBundles().Length} bundles into {outputFolder}."); } }