From d88e266658f30bece7173ad23e8c8c137e6a0812 Mon Sep 17 00:00:00 2001 From: Asaf Ben Natan Date: Thu, 27 Aug 2026 14:14:10 +0300 Subject: [PATCH 1/4] feat: add optional zstd dictionary binary-diff backend Allow per-file zstd --patch-from patches (--binary-diff bsdiff|zstd|auto) with separate size caps. Deltas that can emit ZstdDict use tardf2 up front. Cap the zstd window at min(512MiB, zstd.MaxWindowSize) so older apply stays compatible if klauspost raises MaxWindowSize later. Signed-off-by: Asaf Ben Natan Made-with: Cursor --- .gitignore | 4 +- README.md | 8 + cmd/tar-diff/main.go | 26 ++- file-format.md | 25 ++- pkg/protocol/common.go | 16 +- pkg/tar-diff/bsdiff_test.go | 10 +- pkg/tar-diff/delta.go | 42 ++++- pkg/tar-diff/delta_test.go | 46 +++-- pkg/tar-diff/diff.go | 125 +++++++++++-- pkg/tar-diff/diff_test.go | 19 +- pkg/tar-diff/zstd_diff.go | 98 ++++++++++ pkg/tar-diff/zstd_diff_test.go | 316 +++++++++++++++++++++++++++++++++ pkg/tar-patch/apply.go | 33 +++- pkg/tar-patch/apply_test.go | 88 ++++++++- 14 files changed, 802 insertions(+), 54 deletions(-) create mode 100644 pkg/tar-diff/zstd_diff.go create mode 100644 pkg/tar-diff/zstd_diff_test.go diff --git a/.gitignore b/.gitignore index 61a0fb5..3f95fe8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ cmd/tar-diff/tar-diff cmd/tar-patch/tar-patch .secrets -tar-diff -tar-patch +/tar-diff +/tar-patch test/ diff --git a/README.md b/README.md index c7b4853..681e286 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,14 @@ This is particularly useful for `bootc` images, for example, where only the file will be available on the system. In that case you would run `tar-diff` with `--source-prefix=sysroot/ostree/repo/objects/`. +### Mixing zstd and bsdiff + +By default `tar-diff` uses bsdiff for similar files. `--binary-diff auto` uses zstd dictionary patches for files under `--max-zstd-diff-size` (apply holds the old file in RAM as the dictionary) and bsdiff for larger files. The default method remains `bsdiff`. `--binary-diff auto` and `zstd` write the `tardf2` magic; see [file-format.md](file-format.md). + +``` +$ tar-diff --binary-diff auto --max-zstd-diff-size 128 --max-bsdiff-size 0 old.tar new.tar delta.tardiff +``` + ## Build requirements - golang >= 1.26 (see [`go.mod`](go.mod)) diff --git a/cmd/tar-diff/main.go b/cmd/tar-diff/main.go index af8f71f..bf0fef1 100644 --- a/cmd/tar-diff/main.go +++ b/cmd/tar-diff/main.go @@ -25,8 +25,12 @@ func (p *prefixList) Set(value string) error { } var version = flag.Bool("version", false, "Show version") -var compressionLevel = flag.Int("compression-level", 3, "zstd compression level") -var maxBsdiffSize = flag.Int("max-bsdiff-size", 192, "Max file size in megabytes to consider using bsdiff, or 0 for no limit") +var compressionLevel = flag.Int("compression-level", 3, "zstd compression level for the outer delta stream") +var maxBsdiffSize = flag.Int("max-bsdiff-size", 192, "Max file size in megabytes to consider for bsdiff, or 0 for no limit") +var maxZstdDiffSize = flag.Int("max-zstd-diff-size", 128, "Max file size in megabytes to consider for zstd dictionary patches, or 0 for no extra cap") +var binaryDiff = flag.String("binary-diff", "bsdiff", "Per-file binary diff method: bsdiff, zstd, or auto") +var zstdDiffLevel = flag.Int("zstd-diff-level", -1, "zstd level for dictionary patches (-1 = use --compression-level)") +var zstdDiffWindow = flag.Int("zstd-diff-window", 0, "zstd window size in MiB for dictionary patches (0 = auto from source size, max 512)") var tmpDir = flag.String("tmp-dir", defaultTmpDir, "Directory for temporary files") var applyWhiteouts = flag.Bool("apply-whiteouts", false, "Apply docker/OCI whiteout files when analyzing old tar layers") var sourcePrefixes prefixList @@ -94,6 +98,24 @@ func realMain() int { options := tardiff.NewOptions() options.SetCompressionLevel(*compressionLevel) options.SetMaxBsdiffFileSize(int64(*maxBsdiffSize) * 1024 * 1024) + options.SetMaxZstdDiffFileSize(int64(*maxZstdDiffSize) * 1024 * 1024) + switch *binaryDiff { + case "bsdiff": + options.SetBinaryDiffMethod(tardiff.BinaryDiffBsdiff) + case "zstd": + options.SetBinaryDiffMethod(tardiff.BinaryDiffZstd) + case "auto": + options.SetBinaryDiffMethod(tardiff.BinaryDiffAuto) + default: + log.Printf("Error: invalid --binary-diff %q (want bsdiff, zstd, or auto)", *binaryDiff) + return 1 + } + options.SetZstdDiffLevel(*zstdDiffLevel) + if *zstdDiffWindow < 0 { + log.Printf("Error: invalid --zstd-diff-window %d", *zstdDiffWindow) + return 1 + } + options.SetZstdDiffWindow(*zstdDiffWindow * 1024 * 1024) if len(sourcePrefixes) > 0 { options.SetSourcePrefixes(sourcePrefixes) } diff --git a/file-format.md b/file-format.md index 57a2c48..6a7e7c4 100644 --- a/file-format.md +++ b/file-format.md @@ -2,12 +2,18 @@ File Format ----------- A tar-diff file (media type `application/vnd.tar-diff`) consists of a -header, with the fixed bytes: +header, with one of: ``` -{ 't', 'a', 'r', 'd', 'f', '1', '\n', 0} +{ 't', 'a', 'r', 'd', 'f', '1', '\n', 0} // v1 +{ 't', 'a', 'r', 'd', 'f', '2', '\n', 0} // v2 ``` +v2 is required if the file contains any `DeltaOpZstdDict` operation. +Generators emit v2 whenever zstd-dict ops are possible (`--binary-diff +auto` or `zstd`), even if a particular delta happens to contain none. +v1 files must not contain `DeltaOpZstdDict`. + Followed by a [zstd](https://facebook.github.io/zstd/) compressed stream, with a sequence of operations, each operation is encoded as follows: @@ -42,6 +48,7 @@ DeltaOpOpen = 1 DeltaOpCopy = 2 DeltaOpAddData = 3 DeltaOpSeek = 4 +DeltaOpZstdDict = 5 ``` ***DeltaOpData*** @@ -49,8 +56,9 @@ Emit the bytes from `` into the output stream. ***DeltaOpOpen*** `` is a the (relative) path to a file within the original -tarball. Set the source for subsequent `DeltaOpCopy` and `DeltaAddData` -operations to this file, and reset the source position to 0. +tarball. Set the source for subsequent `DeltaOpCopy`, `DeltaAddData`, +and `DeltaOpZstdDict` operations to this file, and reset the source +position to 0. Tar-diff generates normalized paths with no `.` or `..` elements, which this will never point outside the target directory. However, for @@ -68,3 +76,12 @@ stream. ***DeltaOpSeek*** Set the source position to `` + +***DeltaOpZstdDict*** +Only valid in v2 files. `` is a zstd frame compressed with the +currently open source file as a raw dictionary (dict id 0), matching +`zstd --patch-from` semantics. Seek the source file to offset 0, +decompress `` using the full source file content as the +dictionary, and emit the decompressed bytes to the output stream. +Generators cap the zstd window at 512 MiB so older apply +implementations can still decode the frame. diff --git a/pkg/protocol/common.go b/pkg/protocol/common.go index 80af8d3..8e6309e 100644 --- a/pkg/protocol/common.go +++ b/pkg/protocol/common.go @@ -5,16 +5,20 @@ import "path/filepath" // Delta operation constants define the types of operations in a delta file. const ( - DeltaOpData = iota // Raw data operation - DeltaOpOpen = iota // Open file operation - DeltaOpCopy = iota // Copy from source operation - DeltaOpAddData = iota // Add new data operation - DeltaOpSeek = iota // Seek operation + DeltaOpData = iota // Raw data operation + DeltaOpOpen = iota // Open file operation + DeltaOpCopy = iota // Copy from source operation + DeltaOpAddData = iota // Add new data operation + DeltaOpSeek = iota // Seek operation + DeltaOpZstdDict = iota // zstd dictionary patch against open source file ) -// DeltaHeader is the magic header bytes for tar-diff files. +// DeltaHeader is the magic for v1 tar-diff files (no DeltaOpZstdDict). var DeltaHeader = [...]byte{'t', 'a', 'r', 'd', 'f', '1', '\n', 0} +// DeltaHeaderv2 is the magic when zstd-dict ops are possible (auto or zstd mode). +var DeltaHeaderv2 = [...]byte{'t', 'a', 'r', 'd', 'f', '2', '\n', 0} + // CleanPath cleans up the path lexically and prevents path traversal attacks. // Any ".." that extends outside the first elements (or the root itself) is invalid and returns "". // Uses filepath.Clean for proper cross-platform path handling (Windows backslashes, drive letters). diff --git a/pkg/tar-diff/bsdiff_test.go b/pkg/tar-diff/bsdiff_test.go index 917b508..fbdaf61 100644 --- a/pkg/tar-diff/bsdiff_test.go +++ b/pkg/tar-diff/bsdiff_test.go @@ -93,7 +93,7 @@ func TestBsdiffBasic(t *testing.T) { // and produces output, but cannot verify correctness without a bspatch implementation. // The bsdiff algorithm is well-tested upstream; these tests ensure integration works. var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("Failed to create delta writer: %v", err) } @@ -126,7 +126,7 @@ func TestBsdiffBasic(t *testing.T) { func TestBsdiffIdentical(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("Failed to create delta writer: %v", err) } @@ -158,7 +158,7 @@ func TestBsdiffIdentical(t *testing.T) { func TestBsdiffEmpty(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("Failed to create delta writer: %v", err) } @@ -197,7 +197,7 @@ func TestBsdiffEmpty(t *testing.T) { func TestBsdiffLargeData(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("Failed to create delta writer: %v", err) } @@ -259,7 +259,7 @@ func TestSplitFunction(t *testing.T) { func TestBsdiffPartialMatch(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("Failed to create delta writer: %v", err) } diff --git a/pkg/tar-diff/delta.go b/pkg/tar-diff/delta.go index d3debba..278f93b 100644 --- a/pkg/tar-diff/delta.go +++ b/pkg/tar-diff/delta.go @@ -2,15 +2,35 @@ package tardiff import ( "encoding/binary" + "fmt" + "io" + "github.com/containers/tar-diff/pkg/protocol" "github.com/klauspost/compress/zstd" - "io" ) const ( deltaDataChunkSize = 4 * 1024 * 1024 ) +type deltaFormatVersion int + +const ( + deltaFormatV1 deltaFormatVersion = 1 + deltaFormatV2 deltaFormatVersion = 2 +) + +func (v deltaFormatVersion) header() ([]byte, error) { + switch v { + case deltaFormatV1: + return protocol.DeltaHeader[:], nil + case deltaFormatV2: + return protocol.DeltaHeaderv2[:], nil + default: + return nil, fmt.Errorf("unsupported delta format version %d", v) + } +} + type deltaWriter struct { writer *zstd.Encoder buffer []byte @@ -18,17 +38,23 @@ type deltaWriter struct { currentPos uint64 } -func newDeltaWriter(writer io.Writer, compressionLevel int) (*deltaWriter, error) { - _, err := writer.Write(protocol.DeltaHeader[:]) +func newDeltaWriter(writer io.Writer, compressionLevel int, version deltaFormatVersion) (*deltaWriter, error) { + header, err := version.header() if err != nil { return nil, err } + if _, err := writer.Write(header); err != nil { + return nil, err + } encoder, err := zstd.NewWriter(writer, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(compressionLevel))) if err != nil { return nil, err } - d := deltaWriter{writer: encoder, buffer: make([]byte, 0, deltaDataChunkSize)} + d := deltaWriter{ + writer: encoder, + buffer: make([]byte, 0, deltaDataChunkSize), + } return &d, nil } @@ -188,6 +214,14 @@ func (d *deltaWriter) WriteOldFile(filename string, size uint64) error { return nil } +func (d *deltaWriter) WriteZstdDict(data []byte) error { + if err := d.FlushBuffer(); err != nil { + return err + } + + return d.writeOp(protocol.DeltaOpZstdDict, uint64(len(data)), data) +} + func (d *deltaWriter) Write(data []byte) (int, error) { n := len(data) err := d.WriteContent(data) diff --git a/pkg/tar-diff/delta_test.go b/pkg/tar-diff/delta_test.go index 31607ba..1b067fb 100644 --- a/pkg/tar-diff/delta_test.go +++ b/pkg/tar-diff/delta_test.go @@ -9,7 +9,7 @@ import ( func TestNewDeltaWriter(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("newDeltaWriter failed: %v", err) } @@ -40,7 +40,7 @@ func TestNewDeltaWriter(t *testing.T) { func TestDeltaWriterClose(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("newDeltaWriter failed: %v", err) } @@ -75,7 +75,7 @@ func TestDeltaWriterClose(t *testing.T) { func TestDeltaWriterWriteContent(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("newDeltaWriter failed: %v", err) } @@ -104,7 +104,7 @@ func TestDeltaWriterWriteContent(t *testing.T) { func TestDeltaWriterFlushBuffer(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("newDeltaWriter failed: %v", err) } @@ -130,7 +130,7 @@ func TestDeltaWriterFlushBuffer(t *testing.T) { func TestDeltaWriterSetCurrentFile(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("newDeltaWriter failed: %v", err) } @@ -158,7 +158,7 @@ func TestDeltaWriterSetCurrentFile(t *testing.T) { func TestDeltaWriterSeek(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("newDeltaWriter failed: %v", err) } @@ -182,7 +182,7 @@ func TestDeltaWriterSeek(t *testing.T) { func TestDeltaWriterSeekForward(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("newDeltaWriter failed: %v", err) } @@ -209,7 +209,7 @@ func TestDeltaWriterSeekForward(t *testing.T) { func TestDeltaWriterCopyFile(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("newDeltaWriter failed: %v", err) } @@ -236,7 +236,7 @@ func TestDeltaWriterCopyFile(t *testing.T) { func TestDeltaWriterWriteAddContent(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("newDeltaWriter failed: %v", err) } @@ -263,7 +263,7 @@ func TestDeltaWriterWriteAddContent(t *testing.T) { func TestDeltaWriterWriteOldFile(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("newDeltaWriter failed: %v", err) } @@ -293,7 +293,7 @@ func TestDeltaWriterWriteOldFile(t *testing.T) { func TestDeltaWriterWrite(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("newDeltaWriter failed: %v", err) } @@ -322,7 +322,7 @@ func TestDeltaWriterWrite(t *testing.T) { func TestDeltaWriterCopyFileAt(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("newDeltaWriter failed: %v", err) } @@ -349,7 +349,7 @@ func TestDeltaWriterCopyFileAt(t *testing.T) { func TestDeltaWriterWriteOp(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("newDeltaWriter failed: %v", err) } @@ -382,10 +382,26 @@ func TestDeltaWriterWriteOp(t *testing.T) { t.Logf("writeOp wrote %d bytes (op + data + header)", output.Len()-initialLen) } +func TestDeltaWriterV2Header(t *testing.T) { + var output bytes.Buffer + + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV2) + if err != nil { + t.Fatalf("newDeltaWriter failed: %v", err) + } + if err := deltaWriter.Close(); err != nil { + t.Fatalf("Close failed: %v", err) + } + + if !bytes.Equal(output.Bytes()[:len(protocol.DeltaHeaderv2)], protocol.DeltaHeaderv2[:]) { + t.Fatalf("expected v2 header, got %q", output.Bytes()[:8]) + } +} + func TestDeltaWriterLargeContent(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("newDeltaWriter failed: %v", err) } @@ -417,7 +433,7 @@ func TestDeltaWriterLargeContent(t *testing.T) { func TestDeltaWriterSetCurrentFileTwice(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("newDeltaWriter failed: %v", err) } diff --git a/pkg/tar-diff/diff.go b/pkg/tar-diff/diff.go index bd6ff89..585d905 100644 --- a/pkg/tar-diff/diff.go +++ b/pkg/tar-diff/diff.go @@ -6,12 +6,14 @@ import ( "fmt" "io" "log" + "os" "github.com/containers/image/v5/pkg/compression" ) const ( - defaultMaxBsdiffSize = 192 * 1024 * 1024 + defaultMaxBsdiffSize = 192 * 1024 * 1024 + defaultMaxZstdDiffSize = 128 * 1024 * 1024 ) type deltaGenerator struct { @@ -100,6 +102,57 @@ func (g *deltaGenerator) generateForFileWithBsdiff(info *targetInfo) error { return nil } +func (g *deltaGenerator) generateForFileWithZstd(info *targetInfo) error { + file := info.file + source := info.source + + if err := g.deltaWriter.SetCurrentFile(info.source.sourcePath); err != nil { + return err + } + + oldData, err := g.readSourceData(source, 0, source.file.size) + if err != nil { + return err + } + + windowSize, err := zstdWindowSize(len(oldData), g.options.zstdDiffWindow) + if err != nil { + return err + } + + g.setSkip(true) + tmp, err := os.CreateTemp(g.options.tmpDir, "tar-diff-zstd-new-") + if err != nil { + return err + } + defer func() { + _ = tmp.Close() + _ = os.Remove(tmp.Name()) + }() + + if _, err := io.Copy(tmp, io.LimitReader(g.tarReader, file.size)); err != nil { + return err + } + if _, err := tmp.Seek(0, io.SeekStart); err != nil { + return err + } + + patch, err := zstdPatchFrom(oldData, tmp, g.options.effectiveZstdDiffLevel(), windowSize) + if err != nil { + return err + } + + if int64(len(patch)) >= file.size { + if _, err := tmp.Seek(0, io.SeekStart); err != nil { + return err + } + _, err := io.Copy(g.deltaWriter, tmp) + return err + } + + return g.deltaWriter.WriteZstdDict(patch) +} + func (g *deltaGenerator) generateForFileWithrollsums(info *targetInfo) error { file := info.file source := info.source @@ -156,8 +209,7 @@ func (g *deltaGenerator) generateForFileWithrollsums(info *targetInfo) error { func (g *deltaGenerator) generateForFile(info *targetInfo) error { file := info.file sourceFile := info.source.file - - maxBsdiffSize := g.options.maxBsdiffSize + method := g.options.binaryDiffMethod // For files that are smaller than the path to the delta source plus some small // space for the delta header, skip doing deltas, as delta data will be larger @@ -176,11 +228,10 @@ func (g *deltaGenerator) generateForFile(info *targetInfo) error { if err := g.skipRest(); err != nil { return err } - case maxBsdiffSize == 0 || (file.size < maxBsdiffSize && sourceFile.size < maxBsdiffSize): - // Use bsdiff to generate delta - if err := g.generateForFileWithBsdiff(info); err != nil { - return err - } + case (method == BinaryDiffZstd || method == BinaryDiffAuto) && zstdFitsLimits(file.size, sourceFile.size, g.options.maxZstdDiffSize): + return g.generateForFileWithZstd(info) + case (method == BinaryDiffBsdiff || method == BinaryDiffAuto) && sizeWithinLimit(file.size, g.options.maxBsdiffSize) && sizeWithinLimit(sourceFile.size, g.options.maxBsdiffSize): + return g.generateForFileWithBsdiff(info) case info.rollsumMatches != nil && info.rollsumMatches.matchRatio > 20: // Use rollsums to generate delta if err := g.generateForFileWithrollsums(info); err != nil { @@ -205,7 +256,11 @@ func generateDelta(newFile io.ReadSeeker, deltaFile io.Writer, analysis *deltaAn } }() - deltaWriter, err := newDeltaWriter(deltaFile, options.compressionLevel) + version := deltaFormatV1 + if options.binaryDiffMethod == BinaryDiffZstd || options.binaryDiffMethod == BinaryDiffAuto { + version = deltaFormatV2 + } + deltaWriter, err := newDeltaWriter(deltaFile, options.compressionLevel, version) if err != nil { return err } @@ -262,17 +317,33 @@ func generateDelta(newFile io.ReadSeeker, deltaFile io.Writer, analysis *deltaAn return nil } +// BinaryDiffMethod selects the per-file binary diff algorithm for similar files. +type BinaryDiffMethod int + +const ( + // BinaryDiffBsdiff uses the classic bsdiff algorithm (default). + BinaryDiffBsdiff BinaryDiffMethod = iota + // BinaryDiffZstd uses zstd dictionary compression (zstd --patch-from semantics). + BinaryDiffZstd + // BinaryDiffAuto uses zstd under the zstd size cap, then bsdiff under the bsdiff cap. + BinaryDiffAuto +) + // Options configures the behavior of the diff operation. type Options struct { compressionLevel int maxBsdiffSize int64 + maxZstdDiffSize int64 + binaryDiffMethod BinaryDiffMethod + zstdDiffLevel int // <0 means use compressionLevel + zstdDiffWindow int // bytes; 0 means auto from source size sourcePrefixes []string ignoreSourcePrefixes []string tmpDir string applyWhiteouts bool } -// SetCompressionLevel sets the compression level for the output diff file. +// SetCompressionLevel sets the zstd compression level for the outer delta stream. func (o *Options) SetCompressionLevel(compressionLevel int) { o.compressionLevel = compressionLevel } @@ -282,6 +353,36 @@ func (o *Options) SetMaxBsdiffFileSize(maxBsdiffSize int64) { o.maxBsdiffSize = maxBsdiffSize } +// SetMaxZstdDiffFileSize sets the maximum file size for zstd dictionary patches. +// Pass 0 for no extra cap (still limited by zstd.MaxWindowSize). +func (o *Options) SetMaxZstdDiffFileSize(maxZstdDiffSize int64) { + o.maxZstdDiffSize = maxZstdDiffSize +} + +// SetBinaryDiffMethod selects the per-file binary diff backend. +func (o *Options) SetBinaryDiffMethod(method BinaryDiffMethod) { + o.binaryDiffMethod = method +} + +// SetZstdDiffLevel sets the zstd level for per-file dictionary patches. +// Pass a negative value to reuse SetCompressionLevel. +func (o *Options) SetZstdDiffLevel(level int) { + o.zstdDiffLevel = level +} + +// SetZstdDiffWindow sets the zstd window size in bytes for dictionary patches. +// Pass 0 to size the window automatically from the source file (power of two). +func (o *Options) SetZstdDiffWindow(windowBytes int) { + o.zstdDiffWindow = windowBytes +} + +func (o *Options) effectiveZstdDiffLevel() int { + if o.zstdDiffLevel < 0 { + return o.compressionLevel + } + return o.zstdDiffLevel +} + // SetSourcePrefixes sets path prefixes to filter which source files can be used for delta. // Only files whose primary path starts with one of these prefixes will be used as delta sources. func (o *Options) SetSourcePrefixes(prefixes []string) { @@ -313,6 +414,10 @@ func NewOptions() *Options { return &Options{ compressionLevel: 3, maxBsdiffSize: defaultMaxBsdiffSize, + maxZstdDiffSize: defaultMaxZstdDiffSize, + binaryDiffMethod: BinaryDiffBsdiff, + zstdDiffLevel: -1, + zstdDiffWindow: 0, sourcePrefixes: nil, ignoreSourcePrefixes: nil, } diff --git a/pkg/tar-diff/diff_test.go b/pkg/tar-diff/diff_test.go index 9e96564..81e41fe 100644 --- a/pkg/tar-diff/diff_test.go +++ b/pkg/tar-diff/diff_test.go @@ -284,9 +284,16 @@ func TestNewOptions(t *testing.T) { if options == nil { t.Fatal("NewOptions() returned nil") - } else if options.compressionLevel != 3 { + } + if options.compressionLevel != 3 { t.Errorf("Expected default compression level 3, got %d", options.compressionLevel) } + if options.binaryDiffMethod != BinaryDiffBsdiff { + t.Errorf("Expected default BinaryDiffBsdiff, got %v", options.binaryDiffMethod) + } + if options.maxZstdDiffSize != defaultMaxZstdDiffSize { + t.Errorf("Expected default max zstd diff size %d, got %d", defaultMaxZstdDiffSize, options.maxZstdDiffSize) + } } func TestOptionsSetCompressionLevel(t *testing.T) { @@ -308,6 +315,16 @@ func TestOptionsSetMaxBsdiffFileSize(t *testing.T) { } } +func TestOptionsSetMaxZstdDiffFileSize(t *testing.T) { + options := NewOptions() + + newSize := int64(64 * 1024 * 1024) + options.SetMaxZstdDiffFileSize(newSize) + if options.maxZstdDiffSize != newSize { + t.Errorf("Expected max zstd diff file size %d, got %d", newSize, options.maxZstdDiffSize) + } +} + func TestOptionsSetSourcePrefixes(t *testing.T) { options := NewOptions() diff --git a/pkg/tar-diff/zstd_diff.go b/pkg/tar-diff/zstd_diff.go new file mode 100644 index 0000000..86c122e --- /dev/null +++ b/pkg/tar-diff/zstd_diff.go @@ -0,0 +1,98 @@ +package tardiff + +import ( + "bytes" + "fmt" + "io" + + "github.com/klauspost/compress/zstd" +) + +// zstdPatchDictID matches zstd --patch-from (raw dictionary id 0). +const zstdPatchDictID = 0 + +// zstdMaxCompatibleWindow is the largest window we emit so older apply +// implementations (and current klauspost) can still decode the frame. +const zstdMaxCompatibleWindow = 512 * 1024 * 1024 + +func zstdMaxWindow() int { + if zstd.MaxWindowSize < zstdMaxCompatibleWindow { + return zstd.MaxWindowSize + } + return zstdMaxCompatibleWindow +} + +func zstdPatchFrom(oldData []byte, newData io.Reader, compressionLevel int, windowSize int) ([]byte, error) { + level := zstd.EncoderLevelFromZstd(compressionLevel) + opts := []zstd.EOption{ + zstd.WithEncoderLevel(level), + zstd.WithEncoderDictRaw(zstdPatchDictID, oldData), + zstd.WithEncoderConcurrency(1), + zstd.WithSingleSegment(true), + } + if windowSize > 0 { + opts = append(opts, zstd.WithWindowSize(windowSize)) + } + + var buf bytes.Buffer + enc, err := zstd.NewWriter(&buf, opts...) + if err != nil { + return nil, fmt.Errorf("create zstd patch encoder: %w", err) + } + if _, err := io.Copy(enc, newData); err != nil { + _ = enc.Close() + return nil, fmt.Errorf("encode zstd patch: %w", err) + } + if err := enc.Close(); err != nil { + return nil, fmt.Errorf("close zstd patch encoder: %w", err) + } + return buf.Bytes(), nil +} + +func zstdFitsLimits(fileSize, sourceSize, maxZstd int64) bool { + maxWin := int64(zstdMaxWindow()) + if fileSize > maxWin || sourceSize > maxWin { + return false + } + if maxZstd == 0 { + return true + } + return fileSize < maxZstd && sourceSize < maxZstd +} + +func sizeWithinLimit(size, max int64) bool { + if max == 0 { + return true + } + return size < max +} + +// zstdWindowSize picks a power-of-two window large enough for the source +// dictionary (old file), capped at min(512MiB, zstd.MaxWindowSize). +// configuredBytes 0 means auto. +func zstdWindowSize(oldLen, configuredBytes int) (int, error) { + maxWin := zstdMaxWindow() + if configuredBytes > 0 { + if configuredBytes < zstd.MinWindowSize || configuredBytes > maxWin { + return 0, fmt.Errorf("zstd diff window %d out of range [%d, %d]", configuredBytes, zstd.MinWindowSize, maxWin) + } + if configuredBytes&(configuredBytes-1) != 0 { + return 0, fmt.Errorf("zstd diff window %d must be a power of two", configuredBytes) + } + return configuredBytes, nil + } + + need := oldLen + if need < zstd.MinWindowSize { + return zstd.MinWindowSize, nil + } + if need > maxWin { + return maxWin, nil + } + + w := zstd.MinWindowSize + for w < need { + w <<= 1 + } + return w, nil +} diff --git a/pkg/tar-diff/zstd_diff_test.go b/pkg/tar-diff/zstd_diff_test.go new file mode 100644 index 0000000..7f7d65f --- /dev/null +++ b/pkg/tar-diff/zstd_diff_test.go @@ -0,0 +1,316 @@ +package tardiff + +import ( + "archive/tar" + "bytes" + "io" + "os" + "path/filepath" + "testing" + + "github.com/containers/tar-diff/pkg/protocol" + tarpatch "github.com/containers/tar-diff/pkg/tar-patch" + "github.com/klauspost/compress/zstd" +) + +func TestZstdPatchFromRoundTrip(t *testing.T) { + oldData := []byte("The quick brown fox jumps over the lazy dog") + newData := []byte("The quick red fox runs over the lazy cat") + + window, err := zstdWindowSize(len(oldData), 0) + if err != nil { + t.Fatalf("zstdWindowSize failed: %v", err) + } + patch, err := zstdPatchFrom(oldData, bytes.NewReader(newData), 3, window) + if err != nil { + t.Fatalf("zstdPatchFrom failed: %v", err) + } + if len(patch) == 0 { + t.Fatal("expected non-empty patch") + } + + dec, err := zstd.NewReader(nil, zstd.WithDecoderDictRaw(zstdPatchDictID, oldData)) + if err != nil { + t.Fatalf("create decoder: %v", err) + } + defer dec.Close() + + got, err := dec.DecodeAll(patch, nil) + if err != nil { + t.Fatalf("DecodeAll failed: %v", err) + } + if !bytes.Equal(got, newData) { + t.Fatalf("round-trip mismatch:\n got %q\nwant %q", got, newData) + } +} + +func TestZstdPatchFromBestCompressionUsesDict(t *testing.T) { + oldData := bytes.Repeat([]byte("abcdefghijklmnop"), 12800) + newData := append([]byte{}, oldData...) + newData[100] ^= 0xff + newData[101] ^= 0xff + + window, err := zstdWindowSize(len(oldData), 0) + if err != nil { + t.Fatalf("zstdWindowSize failed: %v", err) + } + patch, err := zstdPatchFrom(oldData, bytes.NewReader(newData), 22, window) + if err != nil { + t.Fatalf("zstdPatchFrom failed: %v", err) + } + if len(patch) > 200 { + t.Fatalf("expected small dict patch at best compression, got %d bytes", len(patch)) + } + + dec, err := zstd.NewReader(nil, zstd.WithDecoderDictRaw(zstdPatchDictID, oldData)) + if err != nil { + t.Fatalf("create decoder: %v", err) + } + defer dec.Close() + + got, err := dec.DecodeAll(patch, nil) + if err != nil { + t.Fatalf("DecodeAll failed: %v", err) + } + if !bytes.Equal(got, newData) { + t.Fatal("round-trip mismatch at SpeedBestCompression") + } +} + +func TestZstdFitsLimits(t *testing.T) { + if !zstdFitsLimits(100, 100, defaultMaxZstdDiffSize) { + t.Fatal("expected small files to fit") + } + if zstdFitsLimits(int64(zstdMaxWindow())+1, 100, 0) { + t.Fatal("expected size over max window to be rejected") + } + if zstdFitsLimits(1000, 1000, 500) { + t.Fatal("expected size over maxZstd to be rejected") + } + if !zstdFitsLimits(1000, 1000, 0) { + t.Fatal("expected maxZstd 0 to mean no extra cap") + } +} + +func diffApplyZstd(t *testing.T, method BinaryDiffMethod, maxZstd, maxBsdiff int64, oldData, newData []byte) (delta bytes.Buffer, wantNew []byte) { + t.Helper() + oldTar, err := createTestTar([]tarEntry{{name: "file.txt", typeflag: tar.TypeReg, data: oldData}}) + if err != nil { + t.Fatalf("create old tar: %v", err) + } + newTar, err := createTestTar([]tarEntry{{name: "file.txt", typeflag: tar.TypeReg, data: newData}}) + if err != nil { + t.Fatalf("create new tar: %v", err) + } + wantNew, err = io.ReadAll(newTar) + if err != nil { + t.Fatalf("read new tar: %v", err) + } + if _, err := newTar.Seek(0, 0); err != nil { + t.Fatalf("seek new tar: %v", err) + } + + options := NewOptions() + options.SetBinaryDiffMethod(method) + options.SetMaxZstdDiffFileSize(maxZstd) + options.SetMaxBsdiffFileSize(maxBsdiff) + if err := Diff([]io.ReadSeeker{oldTar}, newTar, &delta, options); err != nil { + t.Fatalf("Diff failed: %v", err) + } + if delta.Len() == 0 { + t.Fatal("expected non-empty delta") + } + return delta, wantNew +} + +func applyDelta(t *testing.T, delta *bytes.Buffer, oldData, wantNew []byte) { + t.Helper() + tmpDir := t.TempDir() + if err := os.WriteFile(filepath.Join(tmpDir, "file.txt"), oldData, 0o644); err != nil { + t.Fatalf("write source file: %v", err) + } + var reconstructed bytes.Buffer + ds := tarpatch.NewFilesystemDataSource(tmpDir) + defer func() { _ = ds.Close() }() + if err := tarpatch.Apply(delta, ds, &reconstructed); err != nil { + t.Fatalf("Apply failed: %v", err) + } + if !bytes.Equal(reconstructed.Bytes(), wantNew) { + t.Fatalf("reconstructed tar mismatch (%d vs %d bytes)", reconstructed.Len(), len(wantNew)) + } +} + +func TestDiffApplyZstdBinaryDiff(t *testing.T) { + oldData := []byte("The quick brown fox jumps over the lazy dog") + newData := []byte("The quick red fox runs over the lazy cat") + delta, wantNew := diffApplyZstd(t, BinaryDiffZstd, defaultMaxZstdDiffSize, defaultMaxBsdiffSize, oldData, newData) + if !bytes.Equal(delta.Bytes()[:len(protocol.DeltaHeaderv2)], protocol.DeltaHeaderv2[:]) { + t.Fatalf("expected tardf2 header, got %q", delta.Bytes()[:8]) + } + applyDelta(t, &delta, oldData, wantNew) +} + +func TestDiffAutoUsesZstdUnderCap(t *testing.T) { + oldData := []byte("The quick brown fox jumps over the lazy dog") + newData := []byte("The quick red fox runs over the lazy cat") + delta, wantNew := diffApplyZstd(t, BinaryDiffAuto, defaultMaxZstdDiffSize, defaultMaxBsdiffSize, oldData, newData) + if !bytes.Equal(delta.Bytes()[:len(protocol.DeltaHeaderv2)], protocol.DeltaHeaderv2[:]) { + t.Fatalf("expected tardf2 header for auto zstd, got %q", delta.Bytes()[:8]) + } + applyDelta(t, &delta, oldData, wantNew) +} + +func TestDiffAutoUsesBsdiffOverZstdCap(t *testing.T) { + oldData := bytes.Repeat([]byte("abcdefghijklmnop"), 4096) + newData := append([]byte{}, oldData...) + newData[100] ^= 0xff + + maxZstd := int64(len(oldData) / 2) + delta, wantNew := diffApplyZstd(t, BinaryDiffAuto, maxZstd, defaultMaxBsdiffSize, oldData, newData) + if !bytes.Equal(delta.Bytes()[:len(protocol.DeltaHeaderv2)], protocol.DeltaHeaderv2[:]) { + t.Fatalf("expected tardf2 header for auto even when only bsdiff is used, got %q", delta.Bytes()[:8]) + } + applyDelta(t, &delta, oldData, wantNew) +} + +func TestDiffZstdDoesNotFallThroughToBsdiff(t *testing.T) { + oldData := bytes.Repeat([]byte("abcdefghijklmnop"), 4096) + newData := append([]byte{}, oldData...) + newData[100] ^= 0xff + + maxZstd := int64(len(oldData) / 2) + delta, wantNew := diffApplyZstd(t, BinaryDiffZstd, maxZstd, defaultMaxBsdiffSize, oldData, newData) + if !bytes.Equal(delta.Bytes()[:len(protocol.DeltaHeaderv2)], protocol.DeltaHeaderv2[:]) { + t.Fatalf("expected tardf2 header for zstd mode even when zstd is skipped, got %q", delta.Bytes()[:8]) + } + applyDelta(t, &delta, oldData, wantNew) +} + +func TestDiffAutoMixesZstdAndBsdiff(t *testing.T) { + smallOld := []byte("The quick brown fox jumps over the lazy dog") + smallNew := []byte("The quick red fox runs over the lazy cat") + largeOld := bytes.Repeat([]byte("abcdefghijklmnop"), 4096) + largeNew := append([]byte{}, largeOld...) + largeNew[100] ^= 0xff + + oldTar, err := createTestTar([]tarEntry{ + {name: "small.txt", typeflag: tar.TypeReg, data: smallOld}, + {name: "large.txt", typeflag: tar.TypeReg, data: largeOld}, + }) + if err != nil { + t.Fatalf("create old tar: %v", err) + } + newTar, err := createTestTar([]tarEntry{ + {name: "small.txt", typeflag: tar.TypeReg, data: smallNew}, + {name: "large.txt", typeflag: tar.TypeReg, data: largeNew}, + }) + if err != nil { + t.Fatalf("create new tar: %v", err) + } + wantNew, err := io.ReadAll(newTar) + if err != nil { + t.Fatalf("read new tar: %v", err) + } + if _, err := newTar.Seek(0, 0); err != nil { + t.Fatalf("seek new tar: %v", err) + } + + var delta bytes.Buffer + options := NewOptions() + options.SetBinaryDiffMethod(BinaryDiffAuto) + options.SetMaxZstdDiffFileSize(int64(len(largeOld) / 2)) + options.SetMaxBsdiffFileSize(defaultMaxBsdiffSize) + if err := Diff([]io.ReadSeeker{oldTar}, newTar, &delta, options); err != nil { + t.Fatalf("Diff failed: %v", err) + } + if !bytes.Equal(delta.Bytes()[:len(protocol.DeltaHeaderv2)], protocol.DeltaHeaderv2[:]) { + t.Fatalf("expected tardf2 header when mix includes zstd, got %q", delta.Bytes()[:8]) + } + + tmpDir := t.TempDir() + if err := os.WriteFile(filepath.Join(tmpDir, "small.txt"), smallOld, 0o644); err != nil { + t.Fatalf("write small source: %v", err) + } + if err := os.WriteFile(filepath.Join(tmpDir, "large.txt"), largeOld, 0o644); err != nil { + t.Fatalf("write large source: %v", err) + } + var reconstructed bytes.Buffer + ds := tarpatch.NewFilesystemDataSource(tmpDir) + defer func() { _ = ds.Close() }() + if err := tarpatch.Apply(&delta, ds, &reconstructed); err != nil { + t.Fatalf("Apply failed: %v", err) + } + if !bytes.Equal(reconstructed.Bytes(), wantNew) { + t.Fatalf("reconstructed tar mismatch (%d vs %d bytes)", reconstructed.Len(), len(wantNew)) + } +} + +func TestOptionsSetBinaryDiffMethod(t *testing.T) { + options := NewOptions() + if options.binaryDiffMethod != BinaryDiffBsdiff { + t.Fatalf("expected default BinaryDiffBsdiff, got %v", options.binaryDiffMethod) + } + + options.SetBinaryDiffMethod(BinaryDiffZstd) + if options.binaryDiffMethod != BinaryDiffZstd { + t.Fatalf("expected BinaryDiffZstd, got %v", options.binaryDiffMethod) + } + + options.SetBinaryDiffMethod(BinaryDiffAuto) + if options.binaryDiffMethod != BinaryDiffAuto { + t.Fatalf("expected BinaryDiffAuto, got %v", options.binaryDiffMethod) + } +} + +func TestZstdDiffOptionDefaults(t *testing.T) { + options := NewOptions() + if options.zstdDiffLevel != -1 { + t.Fatalf("expected zstdDiffLevel -1, got %d", options.zstdDiffLevel) + } + if options.effectiveZstdDiffLevel() != options.compressionLevel { + t.Fatalf("expected effective level %d, got %d", options.compressionLevel, options.effectiveZstdDiffLevel()) + } + if options.maxZstdDiffSize != defaultMaxZstdDiffSize { + t.Fatalf("expected maxZstdDiffSize %d, got %d", defaultMaxZstdDiffSize, options.maxZstdDiffSize) + } + + options.SetZstdDiffLevel(9) + if options.effectiveZstdDiffLevel() != 9 { + t.Fatalf("expected effective level 9, got %d", options.effectiveZstdDiffLevel()) + } +} + +func TestZstdWindowSize(t *testing.T) { + w, err := zstdWindowSize(100, 0) + if err != nil { + t.Fatal(err) + } + if w < 100 { + t.Fatalf("auto window %d smaller than source", w) + } + if w&(w-1) != 0 { + t.Fatalf("auto window %d not power of two", w) + } + + w, err = zstdWindowSize(100, 1<<20) + if err != nil { + t.Fatal(err) + } + if w != 1<<20 { + t.Fatalf("got window %d, want %d", w, 1<<20) + } + + if zstdMaxWindow() > zstdMaxCompatibleWindow { + t.Fatalf("max window %d exceeds compatible cap %d", zstdMaxWindow(), zstdMaxCompatibleWindow) + } + w, err = zstdWindowSize(zstdMaxCompatibleWindow*2, 0) + if err != nil { + t.Fatal(err) + } + if w != zstdMaxWindow() { + t.Fatalf("auto window %d, want cap %d", w, zstdMaxWindow()) + } + if _, err := zstdWindowSize(100, zstdMaxCompatibleWindow*2); err == nil { + t.Fatal("expected error for configured window over 512MiB") + } +} diff --git a/pkg/tar-patch/apply.go b/pkg/tar-patch/apply.go index 9418427..7ca946e 100644 --- a/pkg/tar-patch/apply.go +++ b/pkg/tar-patch/apply.go @@ -95,7 +95,8 @@ func Apply(delta io.Reader, dataSource DataSource, dst io.Writer) error { if err != nil { return err } - if !bytes.Equal(buf, protocol.DeltaHeader[:]) { + isV2 := bytes.Equal(buf, protocol.DeltaHeaderv2[:]) + if !isV2 && !bytes.Equal(buf, protocol.DeltaHeader[:]) { return fmt.Errorf("invalid delta format") } @@ -180,6 +181,13 @@ func Apply(delta io.Reader, dataSource DataSource, dst io.Writer) error { if err != nil { return err } + case protocol.DeltaOpZstdDict: + if !isV2 { + return fmt.Errorf("ZstdDict requires tardf2") + } + if err := applyZstdDict(r, dataSource, dst, size); err != nil { + return err + } default: return fmt.Errorf("unexpected delta op %d", op) } @@ -187,3 +195,26 @@ func Apply(delta io.Reader, dataSource DataSource, dst io.Writer) error { return nil } + +func applyZstdDict(r io.Reader, dataSource DataSource, dst io.Writer, size uint64) error { + // Dict is the whole source file; Copy/Seek ops may have left the cursor mid-file. + if _, err := dataSource.Seek(0, io.SeekStart); err != nil { + return err + } + + dict, err := io.ReadAll(dataSource) + if err != nil { + return err + } + + decoder, err := zstd.NewReader(io.LimitReader(r, int64(size)), zstd.WithDecoderDictRaw(0, dict), zstd.WithDecoderConcurrency(1)) + if err != nil { + return err + } + defer decoder.Close() + + if _, err := io.Copy(dst, decoder); err != nil { + return fmt.Errorf("ZstdDict decompress: %w", err) + } + return nil +} diff --git a/pkg/tar-patch/apply_test.go b/pkg/tar-patch/apply_test.go index 63e5828..437a986 100644 --- a/pkg/tar-patch/apply_test.go +++ b/pkg/tar-patch/apply_test.go @@ -17,22 +17,31 @@ import ( // Helper to create delta streams for testing func createDeltaStream(t *testing.T, ops []deltaOp) *bytes.Buffer { t.Helper() + header := protocol.DeltaHeader + for _, op := range ops { + if op.code == protocol.DeltaOpZstdDict { + header = protocol.DeltaHeaderv2 + break + } + } + return createDeltaStreamWithHeader(t, header, ops) +} + +func createDeltaStreamWithHeader(t *testing.T, header [8]byte, ops []deltaOp) *bytes.Buffer { + t.Helper() var buf bytes.Buffer - // Write header - _, err := buf.Write(protocol.DeltaHeader[:]) + _, err := buf.Write(header[:]) if err != nil { t.Fatalf("failed to write delta header: %v", err) } - // Create zstd encoder encoder, err := zstd.NewWriter(&buf) if err != nil { t.Fatalf("failed to create zstd encoder: %v", err) } - // Write operations for _, op := range ops { opBuf := make([]byte, 1+binary.MaxVarintLen64) opBuf[0] = op.code @@ -293,6 +302,77 @@ func TestApply_DeltaOpSeek(t *testing.T) { } } +func TestApply_DeltaOpZstdDict(t *testing.T) { + filename := "base.txt" + oldData := []byte("The quick brown fox jumps over the lazy dog") + newData := []byte("The quick red fox runs over the lazy cat") + + enc, err := zstd.NewWriter(nil, + zstd.WithEncoderLevel(zstd.SpeedDefault), + zstd.WithEncoderDictRaw(0, oldData), + zstd.WithEncoderConcurrency(1), + ) + if err != nil { + t.Fatalf("create encoder: %v", err) + } + frame := enc.EncodeAll(newData, nil) + if err := enc.Close(); err != nil { + t.Fatalf("close encoder: %v", err) + } + + delta := createDeltaStream(t, []deltaOp{ + {code: protocol.DeltaOpOpen, size: uint64(len(filename)), data: []byte(filename)}, + {code: protocol.DeltaOpZstdDict, size: uint64(len(frame)), data: frame}, + }) + + var output bytes.Buffer + ds := newMockDataSource() + ds.AddFile(filename, oldData) + + if err := Apply(delta, ds, &output); err != nil { + t.Fatalf("Apply failed: %v", err) + } + if !bytes.Equal(output.Bytes(), newData) { + t.Errorf("expected output %q, got %q", newData, output.Bytes()) + } +} + +func TestApply_ZstdDictRejectedOnV1(t *testing.T) { + filename := "base.txt" + oldData := []byte("The quick brown fox jumps over the lazy dog") + newData := []byte("The quick red fox runs over the lazy cat") + + enc, err := zstd.NewWriter(nil, + zstd.WithEncoderLevel(zstd.SpeedDefault), + zstd.WithEncoderDictRaw(0, oldData), + zstd.WithEncoderConcurrency(1), + ) + if err != nil { + t.Fatalf("create encoder: %v", err) + } + frame := enc.EncodeAll(newData, nil) + if err := enc.Close(); err != nil { + t.Fatalf("close encoder: %v", err) + } + + delta := createDeltaStreamWithHeader(t, protocol.DeltaHeader, []deltaOp{ + {code: protocol.DeltaOpOpen, size: uint64(len(filename)), data: []byte(filename)}, + {code: protocol.DeltaOpZstdDict, size: uint64(len(frame)), data: frame}, + }) + + var output bytes.Buffer + ds := newMockDataSource() + ds.AddFile(filename, oldData) + + err = Apply(delta, ds, &output) + if err == nil { + t.Fatal("expected error for ZstdDict on v1 header") + } + if !strings.Contains(err.Error(), "tardf2") { + t.Errorf("expected tardf2 error, got: %v", err) + } +} + func TestApply_InvalidHeader(t *testing.T) { delta := bytes.NewReader([]byte("invalid header data")) From 4e863cba9e94d976f0b915a512a358b3bfc8f1e1 Mon Sep 17 00:00:00 2001 From: Asaf Ben Natan Date: Thu, 27 Aug 2026 19:13:14 +0300 Subject: [PATCH 2/4] fix: track source cursor after ZstdDict apply Apply reads the whole source as the dictionary, so later reuse of that file must emit Seek(0). Bump golangci-lint to v2.13.1 for Go 1.27. Signed-off-by: Asaf Ben Natan Made-with: Cursor --- Makefile | 4 +-- pkg/tar-diff/delta.go | 10 ++++--- pkg/tar-diff/delta_test.go | 24 +++++++++++++++++ pkg/tar-diff/diff.go | 2 +- pkg/tar-diff/zstd_diff.go | 6 ++--- pkg/tar-diff/zstd_diff_test.go | 48 ++++++++++++++++++++++++++++++++++ 6 files changed, 85 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index eba0687..fa6df2b 100644 --- a/Makefile +++ b/Makefile @@ -55,8 +55,8 @@ install: tar-diff tar-patch tools: .install.golangci-lint .install.golangci-lint: - if [ ! -x "$(GOBIN)/golangci-lint" ]; then \ - curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/main/install.sh | sh -s -- -b $(GOBIN) v2.10.1; \ + if [ ! -x "$(GOBIN)/golangci-lint" ] || ! "$(GOBIN)/golangci-lint" version 2>/dev/null | grep -q 'v2.13.1'; then \ + curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/main/install.sh | sh -s -- -b $(GOBIN) v2.13.1; \ fi clean: diff --git a/pkg/tar-diff/delta.go b/pkg/tar-diff/delta.go index 278f93b..59bdf02 100644 --- a/pkg/tar-diff/delta.go +++ b/pkg/tar-diff/delta.go @@ -214,12 +214,16 @@ func (d *deltaWriter) WriteOldFile(filename string, size uint64) error { return nil } -func (d *deltaWriter) WriteZstdDict(data []byte) error { +func (d *deltaWriter) WriteZstdDict(data []byte, sourceSize uint64) error { if err := d.FlushBuffer(); err != nil { return err } - - return d.writeOp(protocol.DeltaOpZstdDict, uint64(len(data)), data) + if err := d.writeOp(protocol.DeltaOpZstdDict, uint64(len(data)), data); err != nil { + return err + } + // Apply reads the whole source as the dict, leaving the cursor at EOF. + d.currentPos = sourceSize + return nil } func (d *deltaWriter) Write(data []byte) (int, error) { diff --git a/pkg/tar-diff/delta_test.go b/pkg/tar-diff/delta_test.go index 1b067fb..800ca15 100644 --- a/pkg/tar-diff/delta_test.go +++ b/pkg/tar-diff/delta_test.go @@ -290,6 +290,30 @@ func TestDeltaWriterWriteOldFile(t *testing.T) { } } +func TestDeltaWriterWriteZstdDictSetsCurrentPos(t *testing.T) { + var output bytes.Buffer + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV2) + if err != nil { + t.Fatalf("newDeltaWriter failed: %v", err) + } + defer func() { + if err := deltaWriter.Close(); err != nil { + t.Logf("Failed to close deltaWriter: %v", err) + } + }() + + sourceSize := uint64(4096) + if err := deltaWriter.SetCurrentFile("shared.txt"); err != nil { + t.Fatalf("SetCurrentFile failed: %v", err) + } + if err := deltaWriter.WriteZstdDict([]byte("fake-zstd-frame"), sourceSize); err != nil { + t.Fatalf("WriteZstdDict failed: %v", err) + } + if deltaWriter.currentPos != sourceSize { + t.Fatalf("currentPos after WriteZstdDict = %d, want %d", deltaWriter.currentPos, sourceSize) + } +} + func TestDeltaWriterWrite(t *testing.T) { var output bytes.Buffer diff --git a/pkg/tar-diff/diff.go b/pkg/tar-diff/diff.go index 585d905..d956d60 100644 --- a/pkg/tar-diff/diff.go +++ b/pkg/tar-diff/diff.go @@ -150,7 +150,7 @@ func (g *deltaGenerator) generateForFileWithZstd(info *targetInfo) error { return err } - return g.deltaWriter.WriteZstdDict(patch) + return g.deltaWriter.WriteZstdDict(patch, uint64(source.file.size)) } func (g *deltaGenerator) generateForFileWithrollsums(info *targetInfo) error { diff --git a/pkg/tar-diff/zstd_diff.go b/pkg/tar-diff/zstd_diff.go index 86c122e..05bdf05 100644 --- a/pkg/tar-diff/zstd_diff.go +++ b/pkg/tar-diff/zstd_diff.go @@ -60,11 +60,11 @@ func zstdFitsLimits(fileSize, sourceSize, maxZstd int64) bool { return fileSize < maxZstd && sourceSize < maxZstd } -func sizeWithinLimit(size, max int64) bool { - if max == 0 { +func sizeWithinLimit(size, limit int64) bool { + if limit == 0 { return true } - return size < max + return size < limit } // zstdWindowSize picks a power-of-two window large enough for the source diff --git a/pkg/tar-diff/zstd_diff_test.go b/pkg/tar-diff/zstd_diff_test.go index 7f7d65f..03fb75c 100644 --- a/pkg/tar-diff/zstd_diff_test.go +++ b/pkg/tar-diff/zstd_diff_test.go @@ -245,6 +245,54 @@ func TestDiffAutoMixesZstdAndBsdiff(t *testing.T) { } } +func TestDiffApplyZstdThenCopySameSource(t *testing.T) { + shared := bytes.Repeat([]byte("The quick brown fox jumps over the lazy dog\n"), 40) + changed := append([]byte{}, shared...) + changed[10] ^= 0xff + + oldTar, err := createTestTar([]tarEntry{ + {name: "shared.txt", typeflag: tar.TypeReg, data: shared}, + }) + if err != nil { + t.Fatalf("create old tar: %v", err) + } + newTar, err := createTestTar([]tarEntry{ + {name: "shared.txt", typeflag: tar.TypeReg, data: changed}, + {name: "copy.txt", typeflag: tar.TypeReg, data: shared}, + }) + if err != nil { + t.Fatalf("create new tar: %v", err) + } + wantNew, err := io.ReadAll(newTar) + if err != nil { + t.Fatalf("read new tar: %v", err) + } + if _, err := newTar.Seek(0, 0); err != nil { + t.Fatalf("seek new tar: %v", err) + } + + var delta bytes.Buffer + options := NewOptions() + options.SetBinaryDiffMethod(BinaryDiffZstd) + if err := Diff([]io.ReadSeeker{oldTar}, newTar, &delta, options); err != nil { + t.Fatalf("Diff failed: %v", err) + } + + tmpDir := t.TempDir() + if err := os.WriteFile(filepath.Join(tmpDir, "shared.txt"), shared, 0o644); err != nil { + t.Fatalf("write source file: %v", err) + } + var reconstructed bytes.Buffer + ds := tarpatch.NewFilesystemDataSource(tmpDir) + defer func() { _ = ds.Close() }() + if err := tarpatch.Apply(&delta, ds, &reconstructed); err != nil { + t.Fatalf("Apply failed: %v", err) + } + if !bytes.Equal(reconstructed.Bytes(), wantNew) { + t.Fatalf("reconstructed tar mismatch (%d vs %d bytes)", reconstructed.Len(), len(wantNew)) + } +} + func TestOptionsSetBinaryDiffMethod(t *testing.T) { options := NewOptions() if options.binaryDiffMethod != BinaryDiffBsdiff { From 955f328d27e7f905b3cac860765a692f21db9ceb Mon Sep 17 00:00:00 2001 From: Asaf Ben Natan Date: Mon, 31 Aug 2026 09:46:11 +0300 Subject: [PATCH 3/4] fix: frame zstd fallback as chunked DeltaOpData WriteContent now splits at 4MiB so a large raw fallback (or steal copy) never emits one oversized data op. Signed-off-by: Asaf Ben Natan Made-with: Cursor --- pkg/tar-diff/delta.go | 28 ++++++++++++++--- pkg/tar-diff/delta_test.go | 57 +++++++++++++++++++++++++++++++++- pkg/tar-diff/diff.go | 10 +++--- pkg/tar-diff/zstd_diff_test.go | 18 +++++++++++ 4 files changed, 102 insertions(+), 11 deletions(-) diff --git a/pkg/tar-diff/delta.go b/pkg/tar-diff/delta.go index 59bdf02..7707c78 100644 --- a/pkg/tar-diff/delta.go +++ b/pkg/tar-diff/delta.go @@ -100,10 +100,25 @@ func (d *deltaWriter) Close() error { } func (d *deltaWriter) WriteContent(data []byte) error { - d.buffer = append(d.buffer, data...) - - if len(d.buffer) >= deltaDataChunkSize { - return d.FlushBuffer() + for len(data) > 0 { + space := deltaDataChunkSize - len(d.buffer) + if space <= 0 { + if err := d.FlushBuffer(); err != nil { + return err + } + continue + } + if space > len(data) { + space = len(data) + } + d.buffer = append(d.buffer, data[:space]...) + data = data[space:] + if len(d.buffer) < deltaDataChunkSize { + return nil + } + if err := d.FlushBuffer(); err != nil { + return err + } } return nil } @@ -231,3 +246,8 @@ func (d *deltaWriter) Write(data []byte) (int, error) { err := d.WriteContent(data) return n, err } + +func (d *deltaWriter) WriteContentFrom(r io.Reader) error { + _, err := io.Copy(d, r) + return err +} diff --git a/pkg/tar-diff/delta_test.go b/pkg/tar-diff/delta_test.go index 800ca15..9574781 100644 --- a/pkg/tar-diff/delta_test.go +++ b/pkg/tar-diff/delta_test.go @@ -1,9 +1,14 @@ package tardiff import ( + "bufio" "bytes" - "github.com/containers/tar-diff/pkg/protocol" + "encoding/binary" + "io" "testing" + + "github.com/containers/tar-diff/pkg/protocol" + "github.com/klauspost/compress/zstd" ) func TestNewDeltaWriter(t *testing.T) { @@ -452,6 +457,21 @@ func TestDeltaWriterLargeContent(t *testing.T) { if output.Len() <= initialOutputLen { t.Error("Expected large content to trigger buffer flush and write to output") } + + ops, sizes := decodeDeltaOps(t, output.Bytes()) + var total uint64 + for i, op := range ops { + if op != protocol.DeltaOpData { + t.Fatalf("op %d: got %d, want DeltaOpData", i, op) + } + if sizes[i] > uint64(deltaDataChunkSize) { + t.Fatalf("op %d size %d exceeds chunk %d", i, sizes[i], deltaDataChunkSize) + } + total += sizes[i] + } + if total != uint64(len(largeData)) { + t.Fatalf("DATA payload %d, want %d", total, len(largeData)) + } } func TestDeltaWriterSetCurrentFileTwice(t *testing.T) { @@ -483,3 +503,38 @@ func TestDeltaWriterSetCurrentFileTwice(t *testing.T) { t.Errorf("Expected currentFile 'file2.txt', got %s", deltaWriter.currentFile) } } + +func decodeDeltaOps(t *testing.T, delta []byte) (ops []byte, sizes []uint64) { + t.Helper() + if len(delta) < len(protocol.DeltaHeader) { + t.Fatalf("delta too short: %d", len(delta)) + } + dec, err := zstd.NewReader(bytes.NewReader(delta[len(protocol.DeltaHeader):])) + if err != nil { + t.Fatalf("zstd decoder: %v", err) + } + defer dec.Close() + + br := bufio.NewReader(dec) + for { + op, err := br.ReadByte() + if err == io.EOF { + return ops, sizes + } + if err != nil { + t.Fatalf("read op: %v", err) + } + size, err := binary.ReadUvarint(br) + if err != nil { + t.Fatalf("read size: %v", err) + } + ops = append(ops, op) + sizes = append(sizes, size) + if op == protocol.DeltaOpCopy || op == protocol.DeltaOpSeek { + continue + } + if _, err := io.CopyN(io.Discard, br, int64(size)); err != nil { + t.Fatalf("skip payload: %v", err) + } + } +} diff --git a/pkg/tar-diff/diff.go b/pkg/tar-diff/diff.go index d956d60..482f31d 100644 --- a/pkg/tar-diff/diff.go +++ b/pkg/tar-diff/diff.go @@ -106,10 +106,6 @@ func (g *deltaGenerator) generateForFileWithZstd(info *targetInfo) error { file := info.file source := info.source - if err := g.deltaWriter.SetCurrentFile(info.source.sourcePath); err != nil { - return err - } - oldData, err := g.readSourceData(source, 0, source.file.size) if err != nil { return err @@ -146,10 +142,12 @@ func (g *deltaGenerator) generateForFileWithZstd(info *targetInfo) error { if _, err := tmp.Seek(0, io.SeekStart); err != nil { return err } - _, err := io.Copy(g.deltaWriter, tmp) - return err + return g.deltaWriter.WriteContentFrom(tmp) } + if err := g.deltaWriter.SetCurrentFile(info.source.sourcePath); err != nil { + return err + } return g.deltaWriter.WriteZstdDict(patch, uint64(source.file.size)) } diff --git a/pkg/tar-diff/zstd_diff_test.go b/pkg/tar-diff/zstd_diff_test.go index 03fb75c..a9f540a 100644 --- a/pkg/tar-diff/zstd_diff_test.go +++ b/pkg/tar-diff/zstd_diff_test.go @@ -3,6 +3,7 @@ package tardiff import ( "archive/tar" "bytes" + "crypto/rand" "io" "os" "path/filepath" @@ -150,6 +151,23 @@ func TestDiffApplyZstdBinaryDiff(t *testing.T) { applyDelta(t, &delta, oldData, wantNew) } +func TestDiffZstdFallsBackToRawData(t *testing.T) { + oldData := bytes.Repeat([]byte("abcdefghijklmnop"), 256) + newData := make([]byte, 64*1024) + if _, err := rand.Read(newData); err != nil { + t.Fatalf("rand: %v", err) + } + + delta, wantNew := diffApplyZstd(t, BinaryDiffZstd, defaultMaxZstdDiffSize, defaultMaxBsdiffSize, oldData, newData) + ops, _ := decodeDeltaOps(t, delta.Bytes()) + for _, op := range ops { + if op == protocol.DeltaOpZstdDict { + t.Fatal("expected raw DATA fallback, got ZstdDict") + } + } + applyDelta(t, &delta, oldData, wantNew) +} + func TestDiffAutoUsesZstdUnderCap(t *testing.T) { oldData := []byte("The quick brown fox jumps over the lazy dog") newData := []byte("The quick red fox runs over the lazy cat") From ea40fe41708f08ae4eec138cfa33f6a972b096b0 Mon Sep 17 00:00:00 2001 From: Asaf Ben Natan Date: Mon, 31 Aug 2026 14:10:23 +0300 Subject: [PATCH 4/4] refactor: end WriteContent on remaining input, not buffer space Flush a full chunk after the loop so we do not return early while data is still being copied. Signed-off-by: Asaf Ben Natan Made-with: Cursor --- pkg/tar-diff/delta.go | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/pkg/tar-diff/delta.go b/pkg/tar-diff/delta.go index 7707c78..89de1a3 100644 --- a/pkg/tar-diff/delta.go +++ b/pkg/tar-diff/delta.go @@ -101,26 +101,22 @@ func (d *deltaWriter) Close() error { func (d *deltaWriter) WriteContent(data []byte) error { for len(data) > 0 { - space := deltaDataChunkSize - len(d.buffer) - if space <= 0 { + if len(d.buffer) >= deltaDataChunkSize { if err := d.FlushBuffer(); err != nil { return err } - continue } + space := deltaDataChunkSize - len(d.buffer) if space > len(data) { space = len(data) } d.buffer = append(d.buffer, data[:space]...) data = data[space:] - if len(d.buffer) < deltaDataChunkSize { - return nil - } - if err := d.FlushBuffer(); err != nil { - return err - } } - return nil + if len(d.buffer) < deltaDataChunkSize { + return nil + } + return d.FlushBuffer() } // Switches to new file if needed and ensures we're at the start of it