From 805ecdcb56e4ef76c2452d07d8925efa2ace9583 Mon Sep 17 00:00:00 2001 From: alinpahontu2912 Date: Mon, 8 Jun 2026 14:22:01 +0300 Subject: [PATCH 1/2] Pool the dictionary buffer when training a Zstandard dictionary ZstandardDictionary.Train allocated 'new byte[maxDictionarySize]' on every call. Dictionary sizes are typically tens to hundreds of KB (zstd recommends up to ~100 KB, but the API allows more), so each training call paid for a fresh GC allocation that often landed on the LOH. Rent the buffer from ArrayPool.Shared instead. Create copies the trained slice into an exact-sized array before returning, so the rented buffer can be returned immediately. Use clearArray: true on Return because the trained dictionary is derived from caller-supplied samples and must not linger in the shared pool. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Zstandard/ZstandardDictionary.cs | 35 +++++++++++-------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/src/libraries/System.IO.Compression/src/System/IO/Compression/Zstandard/ZstandardDictionary.cs b/src/libraries/System.IO.Compression/src/System/IO/Compression/Zstandard/ZstandardDictionary.cs index 5672690ae458f0..110d164beffa77 100644 --- a/src/libraries/System.IO.Compression/src/System/IO/Compression/Zstandard/ZstandardDictionary.cs +++ b/src/libraries/System.IO.Compression/src/System/IO/Compression/Zstandard/ZstandardDictionary.cs @@ -127,23 +127,30 @@ public static ZstandardDictionary Train(ReadOnlySpan samples, ReadOnlySpan ArgumentOutOfRangeException.ThrowIfLessThan(maxDictionarySize, 256, nameof(maxDictionarySize)); - byte[] dictionaryBuffer = new byte[maxDictionarySize]; - - nuint dictSize; - - unsafe + byte[] dictionaryBuffer = ArrayPool.Shared.Rent(maxDictionarySize); + try { - fixed (byte* samplesPtr = &MemoryMarshal.GetReference(samples)) - fixed (byte* dictPtr = dictionaryBuffer) - fixed (nuint* lengthsAsNuintPtr = &MemoryMarshal.GetReference(lengthsAsNuint)) + nuint dictSize; + + unsafe { - dictSize = Interop.Zstd.ZDICT_trainFromBuffer( - dictPtr, (nuint)maxDictionarySize, - samplesPtr, lengthsAsNuintPtr, (uint)sampleLengths.Length); + fixed (byte* samplesPtr = &MemoryMarshal.GetReference(samples)) + fixed (byte* dictPtr = dictionaryBuffer) + fixed (nuint* lengthsAsNuintPtr = &MemoryMarshal.GetReference(lengthsAsNuint)) + { + dictSize = Interop.Zstd.ZDICT_trainFromBuffer( + dictPtr, (nuint)maxDictionarySize, + samplesPtr, lengthsAsNuintPtr, (uint)sampleLengths.Length); + } + + ZstandardUtils.ThrowIfError(dictSize); + return Create(dictionaryBuffer.AsSpan(0, (int)dictSize)); } - - ZstandardUtils.ThrowIfError(dictSize); - return Create(dictionaryBuffer.AsSpan(0, (int)dictSize)); + } + finally + { + // Clear before returning: the trained dictionary is derived from caller-supplied samples. + ArrayPool.Shared.Return(dictionaryBuffer, clearArray: true); } } finally From 6941ad183a356ad465ef174e06bf1ebcb8bf40e7 Mon Sep 17 00:00:00 2001 From: alinpahontu2912 Date: Tue, 9 Jun 2026 10:40:52 +0300 Subject: [PATCH 2/2] address comments --- .../Zstandard/ZstandardDictionary.cs | 39 +++++++++---------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/src/libraries/System.IO.Compression/src/System/IO/Compression/Zstandard/ZstandardDictionary.cs b/src/libraries/System.IO.Compression/src/System/IO/Compression/Zstandard/ZstandardDictionary.cs index 110d164beffa77..046e97a90bed3d 100644 --- a/src/libraries/System.IO.Compression/src/System/IO/Compression/Zstandard/ZstandardDictionary.cs +++ b/src/libraries/System.IO.Compression/src/System/IO/Compression/Zstandard/ZstandardDictionary.cs @@ -103,6 +103,7 @@ public static ZstandardDictionary Train(ReadOnlySpan samples, ReadOnlySpan // This incidentally also protects against concurrent modifications of the sampleLengths that could cause // access violations later in native code. byte[] lengthsArray = ArrayPool.Shared.Rent(sampleLengths.Length * Unsafe.SizeOf()); + byte[]? dictionaryBuffer = null; try { Span lengthsAsNuint = MemoryMarshal.Cast(lengthsArray.AsSpan(0, sampleLengths.Length * Unsafe.SizeOf())); @@ -127,34 +128,30 @@ public static ZstandardDictionary Train(ReadOnlySpan samples, ReadOnlySpan ArgumentOutOfRangeException.ThrowIfLessThan(maxDictionarySize, 256, nameof(maxDictionarySize)); - byte[] dictionaryBuffer = ArrayPool.Shared.Rent(maxDictionarySize); - try - { - nuint dictSize; + dictionaryBuffer = ArrayPool.Shared.Rent(maxDictionarySize); + nuint dictSize; - unsafe + unsafe + { + fixed (byte* samplesPtr = &MemoryMarshal.GetReference(samples)) + fixed (byte* dictPtr = dictionaryBuffer) + fixed (nuint* lengthsAsNuintPtr = &MemoryMarshal.GetReference(lengthsAsNuint)) { - fixed (byte* samplesPtr = &MemoryMarshal.GetReference(samples)) - fixed (byte* dictPtr = dictionaryBuffer) - fixed (nuint* lengthsAsNuintPtr = &MemoryMarshal.GetReference(lengthsAsNuint)) - { - dictSize = Interop.Zstd.ZDICT_trainFromBuffer( - dictPtr, (nuint)maxDictionarySize, - samplesPtr, lengthsAsNuintPtr, (uint)sampleLengths.Length); - } - - ZstandardUtils.ThrowIfError(dictSize); - return Create(dictionaryBuffer.AsSpan(0, (int)dictSize)); + dictSize = Interop.Zstd.ZDICT_trainFromBuffer( + dictPtr, (nuint)maxDictionarySize, + samplesPtr, lengthsAsNuintPtr, (uint)sampleLengths.Length); } - } - finally - { - // Clear before returning: the trained dictionary is derived from caller-supplied samples. - ArrayPool.Shared.Return(dictionaryBuffer, clearArray: true); + + ZstandardUtils.ThrowIfError(dictSize); + return Create(dictionaryBuffer.AsSpan(0, (int)dictSize)); } } finally { + if (dictionaryBuffer is not null) + { + ArrayPool.Shared.Return(dictionaryBuffer); + } ArrayPool.Shared.Return(lengthsArray); } }