From 9dff41d089061b09182558f7b42058bef7562e92 Mon Sep 17 00:00:00 2001 From: siddh34 Date: Sun, 2 Aug 2026 13:05:29 +0530 Subject: [PATCH] fix: memory over allocation due to last partial block group --- .../ContainerizationEXT4/EXT4+Formatter.swift | 237 ++++++++++++------ .../ContainerizationEXT4/EXT4+Journal.swift | 47 ++-- .../TestEXT4Format.swift | 193 ++++++++++++++ .../TestEXT4Journal.swift | 28 +++ 4 files changed, 415 insertions(+), 90 deletions(-) diff --git a/Sources/ContainerizationEXT4/EXT4+Formatter.swift b/Sources/ContainerizationEXT4/EXT4+Formatter.swift index f5870ac08..5f3530bd5 100644 --- a/Sources/ContainerizationEXT4/EXT4+Formatter.swift +++ b/Sources/ContainerizationEXT4/EXT4+Formatter.swift @@ -610,6 +610,28 @@ extension EXT4 { throw Error.unsupportedFiletype } + private func markAllocatedRange( + start: UInt64, + end: UInt64, + groupStart: UInt64, + groupEnd: UInt64, + bitmap: inout [UInt8] + ) -> UInt32 { + let clippedStart = max(start, groupStart) + let clippedEnd = min(end, groupEnd) + + guard clippedStart < clippedEnd else { + return 0 + } + + for block in clippedStart.. fsBlocks, fsBlocks > lastGroupStart { + let reducedMetadataEnd = packedMetadataEnd - metadataBlocksPerGroup + if reducedMetadataEnd <= lastGroupStart { + totalGroups -= 1 + extraGroupCount -= 1 + packedMetadataEnd = reducedMetadataEnd + fsBlocks = lastGroupStart + newSize = fsBlocks * UInt64(self.blockSize) + } } - let totalGroups = (((newSize / UInt64(self.blockSize)) - 1) / UInt64(self.blocksPerGroup)) + 1 - // If the provided disk size is not aligned to a blockgroup boundary, it needs to - // be expanded to the next blockgroup boundary. - // Example: - // Provided disk size: 2 GB + 100MB: 2148 MB - // BlockSize: 4096 - // Blockgroup size: 32768 blocks: 128MB - // Number of blocks: 549888 - // Number of blockgroups = 549888 / 32768 = 16.78125 - // Aligned disk size = 557056 blocks = 17 blockgroups: 2176 MB - if newSize < totalGroups * blocksPerGroup * blockSize { - newSize = UInt64(totalGroups * blocksPerGroup * blockSize) + if packedMetadataEnd > fsBlocks { + let blocksPerGroup = UInt64(self.blocksPerGroup) + let minimumGroupCount = + (packedBitmapStart - normalGroupCount * metadataBlocksPerGroup + blocksPerGroup - metadataBlocksPerGroup - 1) + / (blocksPerGroup - metadataBlocksPerGroup) + totalGroups = max(requestedGroupCount, minimumGroupCount) + extraGroupCount = totalGroups - normalGroupCount + packedMetadataEnd = packedBitmapStart + extraGroupCount * metadataBlocksPerGroup + fsBlocks = packedMetadataEnd + newSize = fsBlocks * UInt64(self.blockSize) } + + let groupDescriptorBlockCount: UInt32 = (UInt32(totalGroups) - 1) / self.groupsPerDescriptorBlock + 1 // round up to descriptor block boundary + guard groupDescriptorBlockCount <= self.groupDescriptorBlocks else { + throw Error.insufficientSpaceForGroupDescriptorBlocks + } + + let packedBitmapEnd = packedBitmapStart + extraGroupCount * 2 + let packedInodeTableStart = packedBitmapEnd + + // Only the normal metadata prefix is contiguous with group zero. + let reservedDataBlocks = dataBlocks // Snapshot groupDescriptorBlocks before self.size potentially changes: the bitmap // loop uses this to identify which GDT slots were physically reserved at init time, // so it can mark any unused slots as free without accidentally freeing content blocks // written starting at reservedDescriptorBlocks + 1. let reservedDescriptorBlocks = self.groupDescriptorBlocks - if self.size < newSize { + if self.size != newSize { guard newSize / UInt64(self.blockSize) <= UInt64(UInt32.max) else { throw Error.cannotResizeFS(newSize) } self.size = newSize - let pos = self.pos - guard lseek(self.handle.fileDescriptor, off_t(self.size - 1), 0) == self.size - 1 else { + // ftruncate preserves sparse holes and handles both growth and a removable + // trailing partial group without disturbing the file position. + guard ftruncate(self.handle.fileDescriptor, off_t(self.size)) == 0 else { throw Error.cannotResizeFS(self.size) } - let zero: [UInt8] = [0] - try self.handle.write(contentsOf: zero) - try self.handle.seek(toOffset: pos) } for group in 0...init(repeating: 0, count: 1024)) - let computedInodes = totalGroups * blockGroupSize.inodesPerGroup - var blocksCount = totalGroups * self.blocksPerGroup - while blocksCount < totalBlocks { - blocksCount = UInt64(totalBlocks) + if fsBlocks < totalBlocks { + fsBlocks = UInt64(totalBlocks) } let totalFreeBlocks: UInt64 - if totalBlocks > blocksCount { + if totalBlocks > fsBlocks { totalFreeBlocks = 0 } else { - totalFreeBlocks = blocksCount - totalBlocks + totalFreeBlocks = fsBlocks - totalBlocks } var superblock = SuperBlock() superblock.inodesCount = computedInodes.lo - superblock.blocksCountLow = blocksCount.lo - superblock.blocksCountHigh = blocksCount.hi + superblock.blocksCountLow = fsBlocks.lo + superblock.blocksCountHigh = fsBlocks.hi superblock.freeBlocksCountLow = totalFreeBlocks.lo superblock.freeBlocksCountHigh = totalFreeBlocks.hi let freeInodesCount = computedInodes.lo - totalInodes @@ -1023,11 +1116,9 @@ extension EXT4 { } let tableSize: UInt64 = UInt64(EXT4.InodeSize) * blockGroups * inodesPerGroup let rest = tableSize - UInt64(self.inodes.count) * EXT4.InodeSize - let zeroBlock = Array.init(repeating: 0, count: Int(self.blockSize)) - for _ in 0..<(rest / self.blockSize) { - try self.handle.write(contentsOf: zeroBlock) + if rest > 0 { + try self.handle.seek(toOffset: self.pos + rest) } - try self.handle.write(contentsOf: Array.init(repeating: 0, count: Int(rest % self.blockSize))) return inodeTableOffset } @@ -1337,6 +1428,8 @@ extension EXT4 { case cannotResizeFS(_ size: UInt64) case invalidBlockSize(_ size: UInt32) case journalTooSmall(_ size: UInt64) + case journalTooLarge(_ size: UInt64) + case internalInconsistency(_ message: String) public var description: String { switch self { case .notDirectory(let path): @@ -1373,6 +1466,10 @@ extension EXT4 { return "invalid block size \(size): must be 1024, 2048, or 4096" case .journalTooSmall(let size): return "requested journal size \(size) bytes is too small; minimum is \(EXT4.MinJournalBlocks) blocks (JBD2_MIN_JOURNAL_BLOCKS)" + case .internalInconsistency(let message): + return "internal inconsistency: \(message)" + case .journalTooLarge(let size): + return "requested journal size \(size) bytes is too large" } } } diff --git a/Sources/ContainerizationEXT4/EXT4+Journal.swift b/Sources/ContainerizationEXT4/EXT4+Journal.swift index 5e3511f6a..5b15d72d4 100644 --- a/Sources/ContainerizationEXT4/EXT4+Journal.swift +++ b/Sources/ContainerizationEXT4/EXT4+Journal.swift @@ -35,9 +35,17 @@ extension EXT4.Formatter { try self.seek(block: self.currentBlock + 1) } let journalStartBlock = self.currentBlock + let journalEndBlock = try calculateJournalEndBlock( + startBlock: journalStartBlock, + blockCount: journalBlocks + ) try writeJournalSuperblock(journalBlocks: journalBlocks, filesystemUUID: filesystemUUID) - try zeroJournalBlocks(count: journalBlocks - 1) - try setupJournalInode(startBlock: journalStartBlock, blockCount: journalBlocks) + try skipJournalBlocks(count: journalBlocks - 1) + try setupJournalInode( + startBlock: journalStartBlock, + blockCount: journalBlocks, + endBlock: journalEndBlock + ) return journalBlocks } @@ -51,8 +59,10 @@ extension EXT4.Formatter { guard blocks >= EXT4.MinJournalBlocks else { throw EXT4.Formatter.Error.journalTooSmall(size) } - // Safe: any journal large enough to overflow UInt32 (>16 TiB at 4 KiB block size) - // would fail at the I/O layer before this conversion is reached. + // Safe: blocks is guaranteed to be at least EXT4.MinJournalBlocks and at most UInt32.max. + guard blocks <= UInt64(UInt32.max) else { + throw EXT4.Formatter.Error.journalTooLarge(size) + } return UInt32(blocks) } // Default sizing: scale with the usable content area, with a floor determined by @@ -124,22 +134,22 @@ extension EXT4.Formatter { try self.handle.write(contentsOf: buf) } - private func zeroJournalBlocks(count: UInt32) throws { + private func skipJournalBlocks(count: UInt32) throws { guard count > 0 else { return } - let chunkSize = 1.mib() - // Safe: both operands are UInt32, so their product peaks at ~17 TiB, which fits - // in Int64 (the width of Int on all 64-bit Apple platforms). - let totalBytes = Int(count) * Int(self.blockSize) - let zeroBuf = [UInt8](repeating: 0, count: min(Int(chunkSize), totalBytes)) - var remaining = totalBytes - while remaining > 0 { - let toWrite = min(zeroBuf.count, remaining) - try self.handle.write(contentsOf: zeroBuf[0.. UInt32 { + let (endBlock, overflow) = startBlock.addingReportingOverflow(blockCount) + guard !overflow else { + throw EXT4.Formatter.Error.journalTooLarge(UInt64(blockCount) * UInt64(self.blockSize)) } + return endBlock } - private func setupJournalInode(startBlock: UInt32, blockCount: UInt32) throws { + private func setupJournalInode(startBlock: UInt32, blockCount: UInt32, endBlock: UInt32) throws { var journalInode = EXT4.Inode() journalInode.mode = EXT4.Inode.Mode(.S_IFREG, 0o600) journalInode.uid = 0 @@ -162,10 +172,7 @@ extension EXT4.Formatter { // Journal is one contiguous allocation → numExtents = 1 → extent tree fits inline // in the inode, so writeExtents needs no extra disk I/O for extent index blocks. - // Safe: blockCount is at most UInt32.max and startBlock ≥ 0, so the addition could - // theoretically overflow — but zeroJournalBlocks would have already failed with an - // I/O error if the journal extended past the end of the filesystem image. - journalInode = try self.writeExtents(journalInode, (startBlock, startBlock + blockCount)) + journalInode = try self.writeExtents(journalInode, (startBlock, endBlock)) self.inodes[Int(EXT4.JournalInode) - 1].pointee = journalInode } diff --git a/Tests/ContainerizationEXT4Tests/TestEXT4Format.swift b/Tests/ContainerizationEXT4Tests/TestEXT4Format.swift index f0505ce28..2e9b2a666 100644 --- a/Tests/ContainerizationEXT4Tests/TestEXT4Format.swift +++ b/Tests/ContainerizationEXT4Tests/TestEXT4Format.swift @@ -218,6 +218,199 @@ struct Ext4FormatTests: ~Copyable { #expect(regFile.mode.isReg()) #expect(regFile.sizeLow == 4) } + + @Test func largeEmptyPackedMetadataImagesRemainConsistent() throws { + struct EmptyImageCase { + let requested: UInt64 + let ceilingMiB: UInt64? + } + let testCases: [EmptyImageCase] = [ + // .init(requested: 32.kib(), ceilingMiB: nil), // out of scope + // .init(requested: 64.mib(), ceilingMiB: nil), // out of scope + .init(requested: 128.mib(), ceilingMiB: nil), + .init(requested: 128.mib() + 4.kib(), ceilingMiB: nil), + .init(requested: 130.mib() + 8.kib(), ceilingMiB: nil), + .init(requested: 160.mib(), ceilingMiB: nil), + .init(requested: 160.mib() + 4.kib(), ceilingMiB: nil), + .init(requested: 256.mib(), ceilingMiB: nil), + .init(requested: 1.gib(), ceilingMiB: nil), + .init(requested: 4.gib(), ceilingMiB: 32), + .init(requested: 63 * 128.mib(), ceilingMiB: 32), + .init(requested: 63 * 128.mib() + 4.kib(), ceilingMiB: 32), + .init(requested: 8.gib(), ceilingMiB: 32), + .init(requested: 16.gib(), ceilingMiB: 32), + ] + + for testCase in testCases { + let requested = testCase.requested + let fsPath = FilePath( + FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString, isDirectory: false) + ) + defer { try? FileManager.default.removeItem(at: fsPath.url) } + + let formatter = try EXT4.Formatter(fsPath, minDiskSize: requested) + try formatter.close() + + let file = try FileHandle(forReadingFrom: fsPath.url) + fsync(file.fileDescriptor) + + let fileSize = try file.seekToEnd() + + let ext4 = try EXT4.EXT4Reader(blockDevice: fsPath) + let sb = ext4.superBlock + let blocksCount = UInt64(sb.blocksCountLow) | (UInt64(sb.blocksCountHigh) << 32) + + #expect(fileSize == requested) + #expect(fileSize == blocksCount * UInt64(sb.blockSize)) + #expect(blocksCount == fileSize / UInt64(sb.blockSize)) + + if let ceilingMiB = testCase.ceilingMiB { + var fileStat = stat() + let result = stat(fsPath.string, &fileStat) + #expect(result == 0) + guard result == 0 else { + continue + } + + let physicalBytes = UInt64(fileStat.st_blocks) * 512 + let ceilingBytes = ceilingMiB * 1024 * 1024 + + #expect( + physicalBytes <= ceilingBytes, + "physicalBytes <= ceilingBytes = false; physicalBytes = \(physicalBytes); ceilingBytes = \(ceilingBytes)" + ) + } + + let groupCount = (blocksCount + UInt64(sb.blocksPerGroup) - 1) / UInt64(sb.blocksPerGroup) + if groupCount > 1 { + let gd1 = try ext4.getGroupDescriptor(1) + #expect(UInt64(gd1.inodeTableLow) < blocksCount) + #expect(UInt64(gd1.blockBitmapLow) < blocksCount) + #expect(UInt64(gd1.inodeBitmapLow) < blocksCount) + } + } + } + + @Test func largeContentPackedMetadataImagesRemainConsistent() throws { + let cases: [(requested: UInt64, contentBytes: UInt64)] = [ + (requested: 160.mib(), contentBytes: 10.mib()), + (requested: 128.mib(), contentBytes: 50.mib()), + (requested: 128.mib() + 160.kib(), contentBytes: 124.mib()), // Large content size + (requested: 132.mib(), contentBytes: 125.mib() + 218 * 4.kib()), // Large content size + (requested: 128.mib() + 60.kib(), contentBytes: 177 * 4.kib()), + (requested: 160.mib(), contentBytes: 126.mib()), + (requested: 256.mib(), contentBytes: 130.mib()), + (requested: 1.gib(), contentBytes: 200.mib()), + (requested: 4.gib(), contentBytes: 260.mib()), + (requested: 63 * 128.mib(), contentBytes: 500.mib()), + ] + + for (requested, contentBytes) in cases { + let fsPath = FilePath( + FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString, isDirectory: false) + ) + defer { try? FileManager.default.removeItem(at: fsPath.url) } + + let formatter = try EXT4.Formatter(fsPath, minDiskSize: requested) + let payload = Data(repeating: 0x41, count: Int(contentBytes)) + let inputStream = InputStream(data: payload) + inputStream.open() + try formatter.create(path: FilePath("/content"), mode: EXT4.Inode.Mode(.S_IFREG, 0o755), buf: inputStream) + inputStream.close() + try formatter.close() + + let file = try FileHandle(forReadingFrom: fsPath.url) + fsync(file.fileDescriptor) + + let fileSize = try file.seekToEnd() + + let ext4 = try EXT4.EXT4Reader(blockDevice: fsPath) + let sb = ext4.superBlock + let blocksCount = UInt64(sb.blocksCountLow) | (UInt64(sb.blocksCountHigh) << 32) + #expect(fileSize == requested) + #expect(fileSize == blocksCount * UInt64(sb.blockSize)) + let groupCount = (blocksCount + UInt64(sb.blocksPerGroup) - 1) / UInt64(sb.blocksPerGroup) + if groupCount > 1 { + let gd1 = try ext4.getGroupDescriptor(1) + #expect(UInt64(gd1.inodeTableLow) < blocksCount) + #expect(UInt64(gd1.blockBitmapLow) < blocksCount) + #expect(UInt64(gd1.inodeBitmapLow) < blocksCount) + } + } + } + + @Test func emptyFilesystemRoundsUpToMinimumGeometry() throws { + let fsPath = FilePath( + FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString, isDirectory: false) + ) + defer { try? FileManager.default.removeItem(at: fsPath.url) } + + let formatter = try EXT4.Formatter(fsPath, minDiskSize: 32.kib()) + try formatter.close() + + let file = try FileHandle(forReadingFrom: fsPath.url) + #expect(try file.seekToEnd() == 128.mib()) + } + + @Test func closePreservesByteTailOutsideFilesystem() throws { + let requested = 128.mib() + 100 + let fsPath = FilePath( + FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString, isDirectory: false) + ) + defer { try? FileManager.default.removeItem(at: fsPath.url) } + + let formatter = try EXT4.Formatter(fsPath, minDiskSize: requested) + try formatter.close() + + let fileSize = try FileHandle(forReadingFrom: fsPath.url).seekToEnd() + let superblock = try EXT4.EXT4Reader(blockDevice: fsPath).superBlock + let blocksCount = UInt64(superblock.blocksCountLow) | (UInt64(superblock.blocksCountHigh) << 32) + let filesystemSize = blocksCount * UInt64(superblock.blockSize) + + #expect(fileSize == requested) + #expect(blocksCount == fileSize / UInt64(superblock.blockSize)) + #expect(fileSize - filesystemSize == 100) + } + + @Test func closeDropsOrGrowsTrailingPartialGroupsAsNeeded() throws { + let cases: [(requested: UInt64, contentBytes: UInt64, expectedToGrow: Bool)] = [ + (requested: 128.mib() + 4.kib(), contentBytes: 124.mib(), expectedToGrow: false), + (requested: 128.mib() + 4.kib(), contentBytes: 125.mib() + 218 * 4.kib(), expectedToGrow: false), + (requested: 128.mib() + 4.kib(), contentBytes: 129.mib(), expectedToGrow: true), + ] + + for (requested, contentBytes, expectedToGrow) in cases { + let fsPath = FilePath( + FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString, isDirectory: false) + ) + defer { try? FileManager.default.removeItem(at: fsPath.url) } + + let formatter = try EXT4.Formatter(fsPath, minDiskSize: requested) + let payload = Data(repeating: 0x41, count: Int(contentBytes)) + let inputStream = InputStream(data: payload) + inputStream.open() + defer { inputStream.close() } + try formatter.create(path: FilePath("/content"), mode: EXT4.Inode.Mode(.S_IFREG, 0o755), buf: inputStream) + + try formatter.close() + + let file = try FileHandle(forReadingFrom: fsPath.url) + let fileSize = try file.seekToEnd() + let ext4 = try EXT4.EXT4Reader(blockDevice: fsPath) + let sb = ext4.superBlock + let blocksCount = UInt64(sb.blocksCountLow) | (UInt64(sb.blocksCountHigh) << 32) + + #expect(expectedToGrow ? fileSize > requested : fileSize < requested) + #expect(fileSize == blocksCount * UInt64(sb.blockSize)) + + let groupCount = (blocksCount + UInt64(sb.blocksPerGroup) - 1) / UInt64(sb.blocksPerGroup) + let lastGroup = try ext4.getGroupDescriptor(UInt32(groupCount - 1)) + #expect(UInt64(lastGroup.inodeTableLow) < blocksCount) + #expect(UInt64(lastGroup.blockBitmapLow) < blocksCount) + #expect(UInt64(lastGroup.inodeBitmapLow) < blocksCount) + } + } + } @Suite(.serialized) diff --git a/Tests/ContainerizationEXT4Tests/TestEXT4Journal.swift b/Tests/ContainerizationEXT4Tests/TestEXT4Journal.swift index 837032461..54694c518 100644 --- a/Tests/ContainerizationEXT4Tests/TestEXT4Journal.swift +++ b/Tests/ContainerizationEXT4Tests/TestEXT4Journal.swift @@ -163,4 +163,32 @@ struct JournalOverflowTests { try formatter.close() } } + + @Test func closeRejectsJournalBlockCountOverflowingUInt32() throws { + let path = FilePath( + FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: false).path) + defer { try? FileManager.default.removeItem(atPath: path.string) } + let formatter = try EXT4.Formatter( + path, + journal: .init(size: (UInt64(UInt32.max) + 1) * 4096) + ) + #expect(throws: EXT4.Formatter.Error.self) { + try formatter.close() + } + } + + @Test func closeRejectsJournalEndBlockOverflowingUInt32() throws { + let path = FilePath( + FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: false).path) + defer { try? FileManager.default.removeItem(atPath: path.string) } + let formatter = try EXT4.Formatter( + path, + journal: .init(size: UInt64(UInt32.max) * 4096) + ) + #expect(throws: EXT4.Formatter.Error.self) { + try formatter.close() + } + } }