diff --git a/.gitignore b/.gitignore index 61a0fb5..3f95fe8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ cmd/tar-diff/tar-diff cmd/tar-patch/tar-patch .secrets -tar-diff -tar-patch +/tar-diff +/tar-patch test/ diff --git a/Makefile b/Makefile index eba0687..fa6df2b 100644 --- a/Makefile +++ b/Makefile @@ -55,8 +55,8 @@ install: tar-diff tar-patch tools: .install.golangci-lint .install.golangci-lint: - if [ ! -x "$(GOBIN)/golangci-lint" ]; then \ - curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/main/install.sh | sh -s -- -b $(GOBIN) v2.10.1; \ + if [ ! -x "$(GOBIN)/golangci-lint" ] || ! "$(GOBIN)/golangci-lint" version 2>/dev/null | grep -q 'v2.13.1'; then \ + curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/main/install.sh | sh -s -- -b $(GOBIN) v2.13.1; \ fi clean: diff --git a/README.md b/README.md index c7b4853..681e286 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,14 @@ This is particularly useful for `bootc` images, for example, where only the file will be available on the system. In that case you would run `tar-diff` with `--source-prefix=sysroot/ostree/repo/objects/`. +### Mixing zstd and bsdiff + +By default `tar-diff` uses bsdiff for similar files. `--binary-diff auto` uses zstd dictionary patches for files under `--max-zstd-diff-size` (apply holds the old file in RAM as the dictionary) and bsdiff for larger files. The default method remains `bsdiff`. `--binary-diff auto` and `zstd` write the `tardf2` magic; see [file-format.md](file-format.md). + +``` +$ tar-diff --binary-diff auto --max-zstd-diff-size 128 --max-bsdiff-size 0 old.tar new.tar delta.tardiff +``` + ## Build requirements - golang >= 1.26 (see [`go.mod`](go.mod)) diff --git a/cmd/tar-diff/main.go b/cmd/tar-diff/main.go index af8f71f..bf0fef1 100644 --- a/cmd/tar-diff/main.go +++ b/cmd/tar-diff/main.go @@ -25,8 +25,12 @@ func (p *prefixList) Set(value string) error { } var version = flag.Bool("version", false, "Show version") -var compressionLevel = flag.Int("compression-level", 3, "zstd compression level") -var maxBsdiffSize = flag.Int("max-bsdiff-size", 192, "Max file size in megabytes to consider using bsdiff, or 0 for no limit") +var compressionLevel = flag.Int("compression-level", 3, "zstd compression level for the outer delta stream") +var maxBsdiffSize = flag.Int("max-bsdiff-size", 192, "Max file size in megabytes to consider for bsdiff, or 0 for no limit") +var maxZstdDiffSize = flag.Int("max-zstd-diff-size", 128, "Max file size in megabytes to consider for zstd dictionary patches, or 0 for no extra cap") +var binaryDiff = flag.String("binary-diff", "bsdiff", "Per-file binary diff method: bsdiff, zstd, or auto") +var zstdDiffLevel = flag.Int("zstd-diff-level", -1, "zstd level for dictionary patches (-1 = use --compression-level)") +var zstdDiffWindow = flag.Int("zstd-diff-window", 0, "zstd window size in MiB for dictionary patches (0 = auto from source size, max 512)") var tmpDir = flag.String("tmp-dir", defaultTmpDir, "Directory for temporary files") var applyWhiteouts = flag.Bool("apply-whiteouts", false, "Apply docker/OCI whiteout files when analyzing old tar layers") var sourcePrefixes prefixList @@ -94,6 +98,24 @@ func realMain() int { options := tardiff.NewOptions() options.SetCompressionLevel(*compressionLevel) options.SetMaxBsdiffFileSize(int64(*maxBsdiffSize) * 1024 * 1024) + options.SetMaxZstdDiffFileSize(int64(*maxZstdDiffSize) * 1024 * 1024) + switch *binaryDiff { + case "bsdiff": + options.SetBinaryDiffMethod(tardiff.BinaryDiffBsdiff) + case "zstd": + options.SetBinaryDiffMethod(tardiff.BinaryDiffZstd) + case "auto": + options.SetBinaryDiffMethod(tardiff.BinaryDiffAuto) + default: + log.Printf("Error: invalid --binary-diff %q (want bsdiff, zstd, or auto)", *binaryDiff) + return 1 + } + options.SetZstdDiffLevel(*zstdDiffLevel) + if *zstdDiffWindow < 0 { + log.Printf("Error: invalid --zstd-diff-window %d", *zstdDiffWindow) + return 1 + } + options.SetZstdDiffWindow(*zstdDiffWindow * 1024 * 1024) if len(sourcePrefixes) > 0 { options.SetSourcePrefixes(sourcePrefixes) } diff --git a/file-format.md b/file-format.md index 57a2c48..6a7e7c4 100644 --- a/file-format.md +++ b/file-format.md @@ -2,12 +2,18 @@ File Format ----------- A tar-diff file (media type `application/vnd.tar-diff`) consists of a -header, with the fixed bytes: +header, with one of: ``` -{ 't', 'a', 'r', 'd', 'f', '1', '\n', 0} +{ 't', 'a', 'r', 'd', 'f', '1', '\n', 0} // v1 +{ 't', 'a', 'r', 'd', 'f', '2', '\n', 0} // v2 ``` +v2 is required if the file contains any `DeltaOpZstdDict` operation. +Generators emit v2 whenever zstd-dict ops are possible (`--binary-diff +auto` or `zstd`), even if a particular delta happens to contain none. +v1 files must not contain `DeltaOpZstdDict`. + Followed by a [zstd](https://facebook.github.io/zstd/) compressed stream, with a sequence of operations, each operation is encoded as follows: @@ -42,6 +48,7 @@ DeltaOpOpen = 1 DeltaOpCopy = 2 DeltaOpAddData = 3 DeltaOpSeek = 4 +DeltaOpZstdDict = 5 ``` ***DeltaOpData*** @@ -49,8 +56,9 @@ Emit the bytes from `` into the output stream. ***DeltaOpOpen*** `` is a the (relative) path to a file within the original -tarball. Set the source for subsequent `DeltaOpCopy` and `DeltaAddData` -operations to this file, and reset the source position to 0. +tarball. Set the source for subsequent `DeltaOpCopy`, `DeltaAddData`, +and `DeltaOpZstdDict` operations to this file, and reset the source +position to 0. Tar-diff generates normalized paths with no `.` or `..` elements, which this will never point outside the target directory. However, for @@ -68,3 +76,12 @@ stream. ***DeltaOpSeek*** Set the source position to `` + +***DeltaOpZstdDict*** +Only valid in v2 files. `` is a zstd frame compressed with the +currently open source file as a raw dictionary (dict id 0), matching +`zstd --patch-from` semantics. Seek the source file to offset 0, +decompress `` using the full source file content as the +dictionary, and emit the decompressed bytes to the output stream. +Generators cap the zstd window at 512 MiB so older apply +implementations can still decode the frame. diff --git a/pkg/protocol/common.go b/pkg/protocol/common.go index 80af8d3..8e6309e 100644 --- a/pkg/protocol/common.go +++ b/pkg/protocol/common.go @@ -5,16 +5,20 @@ import "path/filepath" // Delta operation constants define the types of operations in a delta file. const ( - DeltaOpData = iota // Raw data operation - DeltaOpOpen = iota // Open file operation - DeltaOpCopy = iota // Copy from source operation - DeltaOpAddData = iota // Add new data operation - DeltaOpSeek = iota // Seek operation + DeltaOpData = iota // Raw data operation + DeltaOpOpen = iota // Open file operation + DeltaOpCopy = iota // Copy from source operation + DeltaOpAddData = iota // Add new data operation + DeltaOpSeek = iota // Seek operation + DeltaOpZstdDict = iota // zstd dictionary patch against open source file ) -// DeltaHeader is the magic header bytes for tar-diff files. +// DeltaHeader is the magic for v1 tar-diff files (no DeltaOpZstdDict). var DeltaHeader = [...]byte{'t', 'a', 'r', 'd', 'f', '1', '\n', 0} +// DeltaHeaderv2 is the magic when zstd-dict ops are possible (auto or zstd mode). +var DeltaHeaderv2 = [...]byte{'t', 'a', 'r', 'd', 'f', '2', '\n', 0} + // CleanPath cleans up the path lexically and prevents path traversal attacks. // Any ".." that extends outside the first elements (or the root itself) is invalid and returns "". // Uses filepath.Clean for proper cross-platform path handling (Windows backslashes, drive letters). diff --git a/pkg/tar-diff/bsdiff_test.go b/pkg/tar-diff/bsdiff_test.go index 917b508..fbdaf61 100644 --- a/pkg/tar-diff/bsdiff_test.go +++ b/pkg/tar-diff/bsdiff_test.go @@ -93,7 +93,7 @@ func TestBsdiffBasic(t *testing.T) { // and produces output, but cannot verify correctness without a bspatch implementation. // The bsdiff algorithm is well-tested upstream; these tests ensure integration works. var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("Failed to create delta writer: %v", err) } @@ -126,7 +126,7 @@ func TestBsdiffBasic(t *testing.T) { func TestBsdiffIdentical(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("Failed to create delta writer: %v", err) } @@ -158,7 +158,7 @@ func TestBsdiffIdentical(t *testing.T) { func TestBsdiffEmpty(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("Failed to create delta writer: %v", err) } @@ -197,7 +197,7 @@ func TestBsdiffEmpty(t *testing.T) { func TestBsdiffLargeData(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("Failed to create delta writer: %v", err) } @@ -259,7 +259,7 @@ func TestSplitFunction(t *testing.T) { func TestBsdiffPartialMatch(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("Failed to create delta writer: %v", err) } diff --git a/pkg/tar-diff/delta.go b/pkg/tar-diff/delta.go index d3debba..89de1a3 100644 --- a/pkg/tar-diff/delta.go +++ b/pkg/tar-diff/delta.go @@ -2,15 +2,35 @@ package tardiff import ( "encoding/binary" + "fmt" + "io" + "github.com/containers/tar-diff/pkg/protocol" "github.com/klauspost/compress/zstd" - "io" ) const ( deltaDataChunkSize = 4 * 1024 * 1024 ) +type deltaFormatVersion int + +const ( + deltaFormatV1 deltaFormatVersion = 1 + deltaFormatV2 deltaFormatVersion = 2 +) + +func (v deltaFormatVersion) header() ([]byte, error) { + switch v { + case deltaFormatV1: + return protocol.DeltaHeader[:], nil + case deltaFormatV2: + return protocol.DeltaHeaderv2[:], nil + default: + return nil, fmt.Errorf("unsupported delta format version %d", v) + } +} + type deltaWriter struct { writer *zstd.Encoder buffer []byte @@ -18,17 +38,23 @@ type deltaWriter struct { currentPos uint64 } -func newDeltaWriter(writer io.Writer, compressionLevel int) (*deltaWriter, error) { - _, err := writer.Write(protocol.DeltaHeader[:]) +func newDeltaWriter(writer io.Writer, compressionLevel int, version deltaFormatVersion) (*deltaWriter, error) { + header, err := version.header() if err != nil { return nil, err } + if _, err := writer.Write(header); err != nil { + return nil, err + } encoder, err := zstd.NewWriter(writer, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(compressionLevel))) if err != nil { return nil, err } - d := deltaWriter{writer: encoder, buffer: make([]byte, 0, deltaDataChunkSize)} + d := deltaWriter{ + writer: encoder, + buffer: make([]byte, 0, deltaDataChunkSize), + } return &d, nil } @@ -74,12 +100,23 @@ func (d *deltaWriter) Close() error { } func (d *deltaWriter) WriteContent(data []byte) error { - d.buffer = append(d.buffer, data...) - - if len(d.buffer) >= deltaDataChunkSize { - return d.FlushBuffer() + for len(data) > 0 { + if len(d.buffer) >= deltaDataChunkSize { + if err := d.FlushBuffer(); err != nil { + return err + } + } + space := deltaDataChunkSize - len(d.buffer) + if space > len(data) { + space = len(data) + } + d.buffer = append(d.buffer, data[:space]...) + data = data[space:] } - return nil + if len(d.buffer) < deltaDataChunkSize { + return nil + } + return d.FlushBuffer() } // Switches to new file if needed and ensures we're at the start of it @@ -188,8 +225,25 @@ func (d *deltaWriter) WriteOldFile(filename string, size uint64) error { return nil } +func (d *deltaWriter) WriteZstdDict(data []byte, sourceSize uint64) error { + if err := d.FlushBuffer(); err != nil { + return err + } + if err := d.writeOp(protocol.DeltaOpZstdDict, uint64(len(data)), data); err != nil { + return err + } + // Apply reads the whole source as the dict, leaving the cursor at EOF. + d.currentPos = sourceSize + return nil +} + func (d *deltaWriter) Write(data []byte) (int, error) { n := len(data) err := d.WriteContent(data) return n, err } + +func (d *deltaWriter) WriteContentFrom(r io.Reader) error { + _, err := io.Copy(d, r) + return err +} diff --git a/pkg/tar-diff/delta_test.go b/pkg/tar-diff/delta_test.go index 31607ba..9574781 100644 --- a/pkg/tar-diff/delta_test.go +++ b/pkg/tar-diff/delta_test.go @@ -1,15 +1,20 @@ package tardiff import ( + "bufio" "bytes" - "github.com/containers/tar-diff/pkg/protocol" + "encoding/binary" + "io" "testing" + + "github.com/containers/tar-diff/pkg/protocol" + "github.com/klauspost/compress/zstd" ) func TestNewDeltaWriter(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("newDeltaWriter failed: %v", err) } @@ -40,7 +45,7 @@ func TestNewDeltaWriter(t *testing.T) { func TestDeltaWriterClose(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("newDeltaWriter failed: %v", err) } @@ -75,7 +80,7 @@ func TestDeltaWriterClose(t *testing.T) { func TestDeltaWriterWriteContent(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("newDeltaWriter failed: %v", err) } @@ -104,7 +109,7 @@ func TestDeltaWriterWriteContent(t *testing.T) { func TestDeltaWriterFlushBuffer(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("newDeltaWriter failed: %v", err) } @@ -130,7 +135,7 @@ func TestDeltaWriterFlushBuffer(t *testing.T) { func TestDeltaWriterSetCurrentFile(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("newDeltaWriter failed: %v", err) } @@ -158,7 +163,7 @@ func TestDeltaWriterSetCurrentFile(t *testing.T) { func TestDeltaWriterSeek(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("newDeltaWriter failed: %v", err) } @@ -182,7 +187,7 @@ func TestDeltaWriterSeek(t *testing.T) { func TestDeltaWriterSeekForward(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("newDeltaWriter failed: %v", err) } @@ -209,7 +214,7 @@ func TestDeltaWriterSeekForward(t *testing.T) { func TestDeltaWriterCopyFile(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("newDeltaWriter failed: %v", err) } @@ -236,7 +241,7 @@ func TestDeltaWriterCopyFile(t *testing.T) { func TestDeltaWriterWriteAddContent(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("newDeltaWriter failed: %v", err) } @@ -263,7 +268,7 @@ func TestDeltaWriterWriteAddContent(t *testing.T) { func TestDeltaWriterWriteOldFile(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("newDeltaWriter failed: %v", err) } @@ -290,10 +295,34 @@ func TestDeltaWriterWriteOldFile(t *testing.T) { } } +func TestDeltaWriterWriteZstdDictSetsCurrentPos(t *testing.T) { + var output bytes.Buffer + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV2) + if err != nil { + t.Fatalf("newDeltaWriter failed: %v", err) + } + defer func() { + if err := deltaWriter.Close(); err != nil { + t.Logf("Failed to close deltaWriter: %v", err) + } + }() + + sourceSize := uint64(4096) + if err := deltaWriter.SetCurrentFile("shared.txt"); err != nil { + t.Fatalf("SetCurrentFile failed: %v", err) + } + if err := deltaWriter.WriteZstdDict([]byte("fake-zstd-frame"), sourceSize); err != nil { + t.Fatalf("WriteZstdDict failed: %v", err) + } + if deltaWriter.currentPos != sourceSize { + t.Fatalf("currentPos after WriteZstdDict = %d, want %d", deltaWriter.currentPos, sourceSize) + } +} + func TestDeltaWriterWrite(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("newDeltaWriter failed: %v", err) } @@ -322,7 +351,7 @@ func TestDeltaWriterWrite(t *testing.T) { func TestDeltaWriterCopyFileAt(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("newDeltaWriter failed: %v", err) } @@ -349,7 +378,7 @@ func TestDeltaWriterCopyFileAt(t *testing.T) { func TestDeltaWriterWriteOp(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("newDeltaWriter failed: %v", err) } @@ -382,10 +411,26 @@ func TestDeltaWriterWriteOp(t *testing.T) { t.Logf("writeOp wrote %d bytes (op + data + header)", output.Len()-initialLen) } +func TestDeltaWriterV2Header(t *testing.T) { + var output bytes.Buffer + + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV2) + if err != nil { + t.Fatalf("newDeltaWriter failed: %v", err) + } + if err := deltaWriter.Close(); err != nil { + t.Fatalf("Close failed: %v", err) + } + + if !bytes.Equal(output.Bytes()[:len(protocol.DeltaHeaderv2)], protocol.DeltaHeaderv2[:]) { + t.Fatalf("expected v2 header, got %q", output.Bytes()[:8]) + } +} + func TestDeltaWriterLargeContent(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("newDeltaWriter failed: %v", err) } @@ -412,12 +457,27 @@ func TestDeltaWriterLargeContent(t *testing.T) { if output.Len() <= initialOutputLen { t.Error("Expected large content to trigger buffer flush and write to output") } + + ops, sizes := decodeDeltaOps(t, output.Bytes()) + var total uint64 + for i, op := range ops { + if op != protocol.DeltaOpData { + t.Fatalf("op %d: got %d, want DeltaOpData", i, op) + } + if sizes[i] > uint64(deltaDataChunkSize) { + t.Fatalf("op %d size %d exceeds chunk %d", i, sizes[i], deltaDataChunkSize) + } + total += sizes[i] + } + if total != uint64(len(largeData)) { + t.Fatalf("DATA payload %d, want %d", total, len(largeData)) + } } func TestDeltaWriterSetCurrentFileTwice(t *testing.T) { var output bytes.Buffer - deltaWriter, err := newDeltaWriter(&output, 1) + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) if err != nil { t.Fatalf("newDeltaWriter failed: %v", err) } @@ -443,3 +503,38 @@ func TestDeltaWriterSetCurrentFileTwice(t *testing.T) { t.Errorf("Expected currentFile 'file2.txt', got %s", deltaWriter.currentFile) } } + +func decodeDeltaOps(t *testing.T, delta []byte) (ops []byte, sizes []uint64) { + t.Helper() + if len(delta) < len(protocol.DeltaHeader) { + t.Fatalf("delta too short: %d", len(delta)) + } + dec, err := zstd.NewReader(bytes.NewReader(delta[len(protocol.DeltaHeader):])) + if err != nil { + t.Fatalf("zstd decoder: %v", err) + } + defer dec.Close() + + br := bufio.NewReader(dec) + for { + op, err := br.ReadByte() + if err == io.EOF { + return ops, sizes + } + if err != nil { + t.Fatalf("read op: %v", err) + } + size, err := binary.ReadUvarint(br) + if err != nil { + t.Fatalf("read size: %v", err) + } + ops = append(ops, op) + sizes = append(sizes, size) + if op == protocol.DeltaOpCopy || op == protocol.DeltaOpSeek { + continue + } + if _, err := io.CopyN(io.Discard, br, int64(size)); err != nil { + t.Fatalf("skip payload: %v", err) + } + } +} diff --git a/pkg/tar-diff/diff.go b/pkg/tar-diff/diff.go index bd6ff89..482f31d 100644 --- a/pkg/tar-diff/diff.go +++ b/pkg/tar-diff/diff.go @@ -6,12 +6,14 @@ import ( "fmt" "io" "log" + "os" "github.com/containers/image/v5/pkg/compression" ) const ( - defaultMaxBsdiffSize = 192 * 1024 * 1024 + defaultMaxBsdiffSize = 192 * 1024 * 1024 + defaultMaxZstdDiffSize = 128 * 1024 * 1024 ) type deltaGenerator struct { @@ -100,6 +102,55 @@ func (g *deltaGenerator) generateForFileWithBsdiff(info *targetInfo) error { return nil } +func (g *deltaGenerator) generateForFileWithZstd(info *targetInfo) error { + file := info.file + source := info.source + + oldData, err := g.readSourceData(source, 0, source.file.size) + if err != nil { + return err + } + + windowSize, err := zstdWindowSize(len(oldData), g.options.zstdDiffWindow) + if err != nil { + return err + } + + g.setSkip(true) + tmp, err := os.CreateTemp(g.options.tmpDir, "tar-diff-zstd-new-") + if err != nil { + return err + } + defer func() { + _ = tmp.Close() + _ = os.Remove(tmp.Name()) + }() + + if _, err := io.Copy(tmp, io.LimitReader(g.tarReader, file.size)); err != nil { + return err + } + if _, err := tmp.Seek(0, io.SeekStart); err != nil { + return err + } + + patch, err := zstdPatchFrom(oldData, tmp, g.options.effectiveZstdDiffLevel(), windowSize) + if err != nil { + return err + } + + if int64(len(patch)) >= file.size { + if _, err := tmp.Seek(0, io.SeekStart); err != nil { + return err + } + return g.deltaWriter.WriteContentFrom(tmp) + } + + if err := g.deltaWriter.SetCurrentFile(info.source.sourcePath); err != nil { + return err + } + return g.deltaWriter.WriteZstdDict(patch, uint64(source.file.size)) +} + func (g *deltaGenerator) generateForFileWithrollsums(info *targetInfo) error { file := info.file source := info.source @@ -156,8 +207,7 @@ func (g *deltaGenerator) generateForFileWithrollsums(info *targetInfo) error { func (g *deltaGenerator) generateForFile(info *targetInfo) error { file := info.file sourceFile := info.source.file - - maxBsdiffSize := g.options.maxBsdiffSize + method := g.options.binaryDiffMethod // For files that are smaller than the path to the delta source plus some small // space for the delta header, skip doing deltas, as delta data will be larger @@ -176,11 +226,10 @@ func (g *deltaGenerator) generateForFile(info *targetInfo) error { if err := g.skipRest(); err != nil { return err } - case maxBsdiffSize == 0 || (file.size < maxBsdiffSize && sourceFile.size < maxBsdiffSize): - // Use bsdiff to generate delta - if err := g.generateForFileWithBsdiff(info); err != nil { - return err - } + case (method == BinaryDiffZstd || method == BinaryDiffAuto) && zstdFitsLimits(file.size, sourceFile.size, g.options.maxZstdDiffSize): + return g.generateForFileWithZstd(info) + case (method == BinaryDiffBsdiff || method == BinaryDiffAuto) && sizeWithinLimit(file.size, g.options.maxBsdiffSize) && sizeWithinLimit(sourceFile.size, g.options.maxBsdiffSize): + return g.generateForFileWithBsdiff(info) case info.rollsumMatches != nil && info.rollsumMatches.matchRatio > 20: // Use rollsums to generate delta if err := g.generateForFileWithrollsums(info); err != nil { @@ -205,7 +254,11 @@ func generateDelta(newFile io.ReadSeeker, deltaFile io.Writer, analysis *deltaAn } }() - deltaWriter, err := newDeltaWriter(deltaFile, options.compressionLevel) + version := deltaFormatV1 + if options.binaryDiffMethod == BinaryDiffZstd || options.binaryDiffMethod == BinaryDiffAuto { + version = deltaFormatV2 + } + deltaWriter, err := newDeltaWriter(deltaFile, options.compressionLevel, version) if err != nil { return err } @@ -262,17 +315,33 @@ func generateDelta(newFile io.ReadSeeker, deltaFile io.Writer, analysis *deltaAn return nil } +// BinaryDiffMethod selects the per-file binary diff algorithm for similar files. +type BinaryDiffMethod int + +const ( + // BinaryDiffBsdiff uses the classic bsdiff algorithm (default). + BinaryDiffBsdiff BinaryDiffMethod = iota + // BinaryDiffZstd uses zstd dictionary compression (zstd --patch-from semantics). + BinaryDiffZstd + // BinaryDiffAuto uses zstd under the zstd size cap, then bsdiff under the bsdiff cap. + BinaryDiffAuto +) + // Options configures the behavior of the diff operation. type Options struct { compressionLevel int maxBsdiffSize int64 + maxZstdDiffSize int64 + binaryDiffMethod BinaryDiffMethod + zstdDiffLevel int // <0 means use compressionLevel + zstdDiffWindow int // bytes; 0 means auto from source size sourcePrefixes []string ignoreSourcePrefixes []string tmpDir string applyWhiteouts bool } -// SetCompressionLevel sets the compression level for the output diff file. +// SetCompressionLevel sets the zstd compression level for the outer delta stream. func (o *Options) SetCompressionLevel(compressionLevel int) { o.compressionLevel = compressionLevel } @@ -282,6 +351,36 @@ func (o *Options) SetMaxBsdiffFileSize(maxBsdiffSize int64) { o.maxBsdiffSize = maxBsdiffSize } +// SetMaxZstdDiffFileSize sets the maximum file size for zstd dictionary patches. +// Pass 0 for no extra cap (still limited by zstd.MaxWindowSize). +func (o *Options) SetMaxZstdDiffFileSize(maxZstdDiffSize int64) { + o.maxZstdDiffSize = maxZstdDiffSize +} + +// SetBinaryDiffMethod selects the per-file binary diff backend. +func (o *Options) SetBinaryDiffMethod(method BinaryDiffMethod) { + o.binaryDiffMethod = method +} + +// SetZstdDiffLevel sets the zstd level for per-file dictionary patches. +// Pass a negative value to reuse SetCompressionLevel. +func (o *Options) SetZstdDiffLevel(level int) { + o.zstdDiffLevel = level +} + +// SetZstdDiffWindow sets the zstd window size in bytes for dictionary patches. +// Pass 0 to size the window automatically from the source file (power of two). +func (o *Options) SetZstdDiffWindow(windowBytes int) { + o.zstdDiffWindow = windowBytes +} + +func (o *Options) effectiveZstdDiffLevel() int { + if o.zstdDiffLevel < 0 { + return o.compressionLevel + } + return o.zstdDiffLevel +} + // SetSourcePrefixes sets path prefixes to filter which source files can be used for delta. // Only files whose primary path starts with one of these prefixes will be used as delta sources. func (o *Options) SetSourcePrefixes(prefixes []string) { @@ -313,6 +412,10 @@ func NewOptions() *Options { return &Options{ compressionLevel: 3, maxBsdiffSize: defaultMaxBsdiffSize, + maxZstdDiffSize: defaultMaxZstdDiffSize, + binaryDiffMethod: BinaryDiffBsdiff, + zstdDiffLevel: -1, + zstdDiffWindow: 0, sourcePrefixes: nil, ignoreSourcePrefixes: nil, } diff --git a/pkg/tar-diff/diff_test.go b/pkg/tar-diff/diff_test.go index 9e96564..81e41fe 100644 --- a/pkg/tar-diff/diff_test.go +++ b/pkg/tar-diff/diff_test.go @@ -284,9 +284,16 @@ func TestNewOptions(t *testing.T) { if options == nil { t.Fatal("NewOptions() returned nil") - } else if options.compressionLevel != 3 { + } + if options.compressionLevel != 3 { t.Errorf("Expected default compression level 3, got %d", options.compressionLevel) } + if options.binaryDiffMethod != BinaryDiffBsdiff { + t.Errorf("Expected default BinaryDiffBsdiff, got %v", options.binaryDiffMethod) + } + if options.maxZstdDiffSize != defaultMaxZstdDiffSize { + t.Errorf("Expected default max zstd diff size %d, got %d", defaultMaxZstdDiffSize, options.maxZstdDiffSize) + } } func TestOptionsSetCompressionLevel(t *testing.T) { @@ -308,6 +315,16 @@ func TestOptionsSetMaxBsdiffFileSize(t *testing.T) { } } +func TestOptionsSetMaxZstdDiffFileSize(t *testing.T) { + options := NewOptions() + + newSize := int64(64 * 1024 * 1024) + options.SetMaxZstdDiffFileSize(newSize) + if options.maxZstdDiffSize != newSize { + t.Errorf("Expected max zstd diff file size %d, got %d", newSize, options.maxZstdDiffSize) + } +} + func TestOptionsSetSourcePrefixes(t *testing.T) { options := NewOptions() diff --git a/pkg/tar-diff/zstd_diff.go b/pkg/tar-diff/zstd_diff.go new file mode 100644 index 0000000..05bdf05 --- /dev/null +++ b/pkg/tar-diff/zstd_diff.go @@ -0,0 +1,98 @@ +package tardiff + +import ( + "bytes" + "fmt" + "io" + + "github.com/klauspost/compress/zstd" +) + +// zstdPatchDictID matches zstd --patch-from (raw dictionary id 0). +const zstdPatchDictID = 0 + +// zstdMaxCompatibleWindow is the largest window we emit so older apply +// implementations (and current klauspost) can still decode the frame. +const zstdMaxCompatibleWindow = 512 * 1024 * 1024 + +func zstdMaxWindow() int { + if zstd.MaxWindowSize < zstdMaxCompatibleWindow { + return zstd.MaxWindowSize + } + return zstdMaxCompatibleWindow +} + +func zstdPatchFrom(oldData []byte, newData io.Reader, compressionLevel int, windowSize int) ([]byte, error) { + level := zstd.EncoderLevelFromZstd(compressionLevel) + opts := []zstd.EOption{ + zstd.WithEncoderLevel(level), + zstd.WithEncoderDictRaw(zstdPatchDictID, oldData), + zstd.WithEncoderConcurrency(1), + zstd.WithSingleSegment(true), + } + if windowSize > 0 { + opts = append(opts, zstd.WithWindowSize(windowSize)) + } + + var buf bytes.Buffer + enc, err := zstd.NewWriter(&buf, opts...) + if err != nil { + return nil, fmt.Errorf("create zstd patch encoder: %w", err) + } + if _, err := io.Copy(enc, newData); err != nil { + _ = enc.Close() + return nil, fmt.Errorf("encode zstd patch: %w", err) + } + if err := enc.Close(); err != nil { + return nil, fmt.Errorf("close zstd patch encoder: %w", err) + } + return buf.Bytes(), nil +} + +func zstdFitsLimits(fileSize, sourceSize, maxZstd int64) bool { + maxWin := int64(zstdMaxWindow()) + if fileSize > maxWin || sourceSize > maxWin { + return false + } + if maxZstd == 0 { + return true + } + return fileSize < maxZstd && sourceSize < maxZstd +} + +func sizeWithinLimit(size, limit int64) bool { + if limit == 0 { + return true + } + return size < limit +} + +// zstdWindowSize picks a power-of-two window large enough for the source +// dictionary (old file), capped at min(512MiB, zstd.MaxWindowSize). +// configuredBytes 0 means auto. +func zstdWindowSize(oldLen, configuredBytes int) (int, error) { + maxWin := zstdMaxWindow() + if configuredBytes > 0 { + if configuredBytes < zstd.MinWindowSize || configuredBytes > maxWin { + return 0, fmt.Errorf("zstd diff window %d out of range [%d, %d]", configuredBytes, zstd.MinWindowSize, maxWin) + } + if configuredBytes&(configuredBytes-1) != 0 { + return 0, fmt.Errorf("zstd diff window %d must be a power of two", configuredBytes) + } + return configuredBytes, nil + } + + need := oldLen + if need < zstd.MinWindowSize { + return zstd.MinWindowSize, nil + } + if need > maxWin { + return maxWin, nil + } + + w := zstd.MinWindowSize + for w < need { + w <<= 1 + } + return w, nil +} diff --git a/pkg/tar-diff/zstd_diff_test.go b/pkg/tar-diff/zstd_diff_test.go new file mode 100644 index 0000000..a9f540a --- /dev/null +++ b/pkg/tar-diff/zstd_diff_test.go @@ -0,0 +1,382 @@ +package tardiff + +import ( + "archive/tar" + "bytes" + "crypto/rand" + "io" + "os" + "path/filepath" + "testing" + + "github.com/containers/tar-diff/pkg/protocol" + tarpatch "github.com/containers/tar-diff/pkg/tar-patch" + "github.com/klauspost/compress/zstd" +) + +func TestZstdPatchFromRoundTrip(t *testing.T) { + oldData := []byte("The quick brown fox jumps over the lazy dog") + newData := []byte("The quick red fox runs over the lazy cat") + + window, err := zstdWindowSize(len(oldData), 0) + if err != nil { + t.Fatalf("zstdWindowSize failed: %v", err) + } + patch, err := zstdPatchFrom(oldData, bytes.NewReader(newData), 3, window) + if err != nil { + t.Fatalf("zstdPatchFrom failed: %v", err) + } + if len(patch) == 0 { + t.Fatal("expected non-empty patch") + } + + dec, err := zstd.NewReader(nil, zstd.WithDecoderDictRaw(zstdPatchDictID, oldData)) + if err != nil { + t.Fatalf("create decoder: %v", err) + } + defer dec.Close() + + got, err := dec.DecodeAll(patch, nil) + if err != nil { + t.Fatalf("DecodeAll failed: %v", err) + } + if !bytes.Equal(got, newData) { + t.Fatalf("round-trip mismatch:\n got %q\nwant %q", got, newData) + } +} + +func TestZstdPatchFromBestCompressionUsesDict(t *testing.T) { + oldData := bytes.Repeat([]byte("abcdefghijklmnop"), 12800) + newData := append([]byte{}, oldData...) + newData[100] ^= 0xff + newData[101] ^= 0xff + + window, err := zstdWindowSize(len(oldData), 0) + if err != nil { + t.Fatalf("zstdWindowSize failed: %v", err) + } + patch, err := zstdPatchFrom(oldData, bytes.NewReader(newData), 22, window) + if err != nil { + t.Fatalf("zstdPatchFrom failed: %v", err) + } + if len(patch) > 200 { + t.Fatalf("expected small dict patch at best compression, got %d bytes", len(patch)) + } + + dec, err := zstd.NewReader(nil, zstd.WithDecoderDictRaw(zstdPatchDictID, oldData)) + if err != nil { + t.Fatalf("create decoder: %v", err) + } + defer dec.Close() + + got, err := dec.DecodeAll(patch, nil) + if err != nil { + t.Fatalf("DecodeAll failed: %v", err) + } + if !bytes.Equal(got, newData) { + t.Fatal("round-trip mismatch at SpeedBestCompression") + } +} + +func TestZstdFitsLimits(t *testing.T) { + if !zstdFitsLimits(100, 100, defaultMaxZstdDiffSize) { + t.Fatal("expected small files to fit") + } + if zstdFitsLimits(int64(zstdMaxWindow())+1, 100, 0) { + t.Fatal("expected size over max window to be rejected") + } + if zstdFitsLimits(1000, 1000, 500) { + t.Fatal("expected size over maxZstd to be rejected") + } + if !zstdFitsLimits(1000, 1000, 0) { + t.Fatal("expected maxZstd 0 to mean no extra cap") + } +} + +func diffApplyZstd(t *testing.T, method BinaryDiffMethod, maxZstd, maxBsdiff int64, oldData, newData []byte) (delta bytes.Buffer, wantNew []byte) { + t.Helper() + oldTar, err := createTestTar([]tarEntry{{name: "file.txt", typeflag: tar.TypeReg, data: oldData}}) + if err != nil { + t.Fatalf("create old tar: %v", err) + } + newTar, err := createTestTar([]tarEntry{{name: "file.txt", typeflag: tar.TypeReg, data: newData}}) + if err != nil { + t.Fatalf("create new tar: %v", err) + } + wantNew, err = io.ReadAll(newTar) + if err != nil { + t.Fatalf("read new tar: %v", err) + } + if _, err := newTar.Seek(0, 0); err != nil { + t.Fatalf("seek new tar: %v", err) + } + + options := NewOptions() + options.SetBinaryDiffMethod(method) + options.SetMaxZstdDiffFileSize(maxZstd) + options.SetMaxBsdiffFileSize(maxBsdiff) + if err := Diff([]io.ReadSeeker{oldTar}, newTar, &delta, options); err != nil { + t.Fatalf("Diff failed: %v", err) + } + if delta.Len() == 0 { + t.Fatal("expected non-empty delta") + } + return delta, wantNew +} + +func applyDelta(t *testing.T, delta *bytes.Buffer, oldData, wantNew []byte) { + t.Helper() + tmpDir := t.TempDir() + if err := os.WriteFile(filepath.Join(tmpDir, "file.txt"), oldData, 0o644); err != nil { + t.Fatalf("write source file: %v", err) + } + var reconstructed bytes.Buffer + ds := tarpatch.NewFilesystemDataSource(tmpDir) + defer func() { _ = ds.Close() }() + if err := tarpatch.Apply(delta, ds, &reconstructed); err != nil { + t.Fatalf("Apply failed: %v", err) + } + if !bytes.Equal(reconstructed.Bytes(), wantNew) { + t.Fatalf("reconstructed tar mismatch (%d vs %d bytes)", reconstructed.Len(), len(wantNew)) + } +} + +func TestDiffApplyZstdBinaryDiff(t *testing.T) { + oldData := []byte("The quick brown fox jumps over the lazy dog") + newData := []byte("The quick red fox runs over the lazy cat") + delta, wantNew := diffApplyZstd(t, BinaryDiffZstd, defaultMaxZstdDiffSize, defaultMaxBsdiffSize, oldData, newData) + if !bytes.Equal(delta.Bytes()[:len(protocol.DeltaHeaderv2)], protocol.DeltaHeaderv2[:]) { + t.Fatalf("expected tardf2 header, got %q", delta.Bytes()[:8]) + } + applyDelta(t, &delta, oldData, wantNew) +} + +func TestDiffZstdFallsBackToRawData(t *testing.T) { + oldData := bytes.Repeat([]byte("abcdefghijklmnop"), 256) + newData := make([]byte, 64*1024) + if _, err := rand.Read(newData); err != nil { + t.Fatalf("rand: %v", err) + } + + delta, wantNew := diffApplyZstd(t, BinaryDiffZstd, defaultMaxZstdDiffSize, defaultMaxBsdiffSize, oldData, newData) + ops, _ := decodeDeltaOps(t, delta.Bytes()) + for _, op := range ops { + if op == protocol.DeltaOpZstdDict { + t.Fatal("expected raw DATA fallback, got ZstdDict") + } + } + applyDelta(t, &delta, oldData, wantNew) +} + +func TestDiffAutoUsesZstdUnderCap(t *testing.T) { + oldData := []byte("The quick brown fox jumps over the lazy dog") + newData := []byte("The quick red fox runs over the lazy cat") + delta, wantNew := diffApplyZstd(t, BinaryDiffAuto, defaultMaxZstdDiffSize, defaultMaxBsdiffSize, oldData, newData) + if !bytes.Equal(delta.Bytes()[:len(protocol.DeltaHeaderv2)], protocol.DeltaHeaderv2[:]) { + t.Fatalf("expected tardf2 header for auto zstd, got %q", delta.Bytes()[:8]) + } + applyDelta(t, &delta, oldData, wantNew) +} + +func TestDiffAutoUsesBsdiffOverZstdCap(t *testing.T) { + oldData := bytes.Repeat([]byte("abcdefghijklmnop"), 4096) + newData := append([]byte{}, oldData...) + newData[100] ^= 0xff + + maxZstd := int64(len(oldData) / 2) + delta, wantNew := diffApplyZstd(t, BinaryDiffAuto, maxZstd, defaultMaxBsdiffSize, oldData, newData) + if !bytes.Equal(delta.Bytes()[:len(protocol.DeltaHeaderv2)], protocol.DeltaHeaderv2[:]) { + t.Fatalf("expected tardf2 header for auto even when only bsdiff is used, got %q", delta.Bytes()[:8]) + } + applyDelta(t, &delta, oldData, wantNew) +} + +func TestDiffZstdDoesNotFallThroughToBsdiff(t *testing.T) { + oldData := bytes.Repeat([]byte("abcdefghijklmnop"), 4096) + newData := append([]byte{}, oldData...) + newData[100] ^= 0xff + + maxZstd := int64(len(oldData) / 2) + delta, wantNew := diffApplyZstd(t, BinaryDiffZstd, maxZstd, defaultMaxBsdiffSize, oldData, newData) + if !bytes.Equal(delta.Bytes()[:len(protocol.DeltaHeaderv2)], protocol.DeltaHeaderv2[:]) { + t.Fatalf("expected tardf2 header for zstd mode even when zstd is skipped, got %q", delta.Bytes()[:8]) + } + applyDelta(t, &delta, oldData, wantNew) +} + +func TestDiffAutoMixesZstdAndBsdiff(t *testing.T) { + smallOld := []byte("The quick brown fox jumps over the lazy dog") + smallNew := []byte("The quick red fox runs over the lazy cat") + largeOld := bytes.Repeat([]byte("abcdefghijklmnop"), 4096) + largeNew := append([]byte{}, largeOld...) + largeNew[100] ^= 0xff + + oldTar, err := createTestTar([]tarEntry{ + {name: "small.txt", typeflag: tar.TypeReg, data: smallOld}, + {name: "large.txt", typeflag: tar.TypeReg, data: largeOld}, + }) + if err != nil { + t.Fatalf("create old tar: %v", err) + } + newTar, err := createTestTar([]tarEntry{ + {name: "small.txt", typeflag: tar.TypeReg, data: smallNew}, + {name: "large.txt", typeflag: tar.TypeReg, data: largeNew}, + }) + if err != nil { + t.Fatalf("create new tar: %v", err) + } + wantNew, err := io.ReadAll(newTar) + if err != nil { + t.Fatalf("read new tar: %v", err) + } + if _, err := newTar.Seek(0, 0); err != nil { + t.Fatalf("seek new tar: %v", err) + } + + var delta bytes.Buffer + options := NewOptions() + options.SetBinaryDiffMethod(BinaryDiffAuto) + options.SetMaxZstdDiffFileSize(int64(len(largeOld) / 2)) + options.SetMaxBsdiffFileSize(defaultMaxBsdiffSize) + if err := Diff([]io.ReadSeeker{oldTar}, newTar, &delta, options); err != nil { + t.Fatalf("Diff failed: %v", err) + } + if !bytes.Equal(delta.Bytes()[:len(protocol.DeltaHeaderv2)], protocol.DeltaHeaderv2[:]) { + t.Fatalf("expected tardf2 header when mix includes zstd, got %q", delta.Bytes()[:8]) + } + + tmpDir := t.TempDir() + if err := os.WriteFile(filepath.Join(tmpDir, "small.txt"), smallOld, 0o644); err != nil { + t.Fatalf("write small source: %v", err) + } + if err := os.WriteFile(filepath.Join(tmpDir, "large.txt"), largeOld, 0o644); err != nil { + t.Fatalf("write large source: %v", err) + } + var reconstructed bytes.Buffer + ds := tarpatch.NewFilesystemDataSource(tmpDir) + defer func() { _ = ds.Close() }() + if err := tarpatch.Apply(&delta, ds, &reconstructed); err != nil { + t.Fatalf("Apply failed: %v", err) + } + if !bytes.Equal(reconstructed.Bytes(), wantNew) { + t.Fatalf("reconstructed tar mismatch (%d vs %d bytes)", reconstructed.Len(), len(wantNew)) + } +} + +func TestDiffApplyZstdThenCopySameSource(t *testing.T) { + shared := bytes.Repeat([]byte("The quick brown fox jumps over the lazy dog\n"), 40) + changed := append([]byte{}, shared...) + changed[10] ^= 0xff + + oldTar, err := createTestTar([]tarEntry{ + {name: "shared.txt", typeflag: tar.TypeReg, data: shared}, + }) + if err != nil { + t.Fatalf("create old tar: %v", err) + } + newTar, err := createTestTar([]tarEntry{ + {name: "shared.txt", typeflag: tar.TypeReg, data: changed}, + {name: "copy.txt", typeflag: tar.TypeReg, data: shared}, + }) + if err != nil { + t.Fatalf("create new tar: %v", err) + } + wantNew, err := io.ReadAll(newTar) + if err != nil { + t.Fatalf("read new tar: %v", err) + } + if _, err := newTar.Seek(0, 0); err != nil { + t.Fatalf("seek new tar: %v", err) + } + + var delta bytes.Buffer + options := NewOptions() + options.SetBinaryDiffMethod(BinaryDiffZstd) + if err := Diff([]io.ReadSeeker{oldTar}, newTar, &delta, options); err != nil { + t.Fatalf("Diff failed: %v", err) + } + + tmpDir := t.TempDir() + if err := os.WriteFile(filepath.Join(tmpDir, "shared.txt"), shared, 0o644); err != nil { + t.Fatalf("write source file: %v", err) + } + var reconstructed bytes.Buffer + ds := tarpatch.NewFilesystemDataSource(tmpDir) + defer func() { _ = ds.Close() }() + if err := tarpatch.Apply(&delta, ds, &reconstructed); err != nil { + t.Fatalf("Apply failed: %v", err) + } + if !bytes.Equal(reconstructed.Bytes(), wantNew) { + t.Fatalf("reconstructed tar mismatch (%d vs %d bytes)", reconstructed.Len(), len(wantNew)) + } +} + +func TestOptionsSetBinaryDiffMethod(t *testing.T) { + options := NewOptions() + if options.binaryDiffMethod != BinaryDiffBsdiff { + t.Fatalf("expected default BinaryDiffBsdiff, got %v", options.binaryDiffMethod) + } + + options.SetBinaryDiffMethod(BinaryDiffZstd) + if options.binaryDiffMethod != BinaryDiffZstd { + t.Fatalf("expected BinaryDiffZstd, got %v", options.binaryDiffMethod) + } + + options.SetBinaryDiffMethod(BinaryDiffAuto) + if options.binaryDiffMethod != BinaryDiffAuto { + t.Fatalf("expected BinaryDiffAuto, got %v", options.binaryDiffMethod) + } +} + +func TestZstdDiffOptionDefaults(t *testing.T) { + options := NewOptions() + if options.zstdDiffLevel != -1 { + t.Fatalf("expected zstdDiffLevel -1, got %d", options.zstdDiffLevel) + } + if options.effectiveZstdDiffLevel() != options.compressionLevel { + t.Fatalf("expected effective level %d, got %d", options.compressionLevel, options.effectiveZstdDiffLevel()) + } + if options.maxZstdDiffSize != defaultMaxZstdDiffSize { + t.Fatalf("expected maxZstdDiffSize %d, got %d", defaultMaxZstdDiffSize, options.maxZstdDiffSize) + } + + options.SetZstdDiffLevel(9) + if options.effectiveZstdDiffLevel() != 9 { + t.Fatalf("expected effective level 9, got %d", options.effectiveZstdDiffLevel()) + } +} + +func TestZstdWindowSize(t *testing.T) { + w, err := zstdWindowSize(100, 0) + if err != nil { + t.Fatal(err) + } + if w < 100 { + t.Fatalf("auto window %d smaller than source", w) + } + if w&(w-1) != 0 { + t.Fatalf("auto window %d not power of two", w) + } + + w, err = zstdWindowSize(100, 1<<20) + if err != nil { + t.Fatal(err) + } + if w != 1<<20 { + t.Fatalf("got window %d, want %d", w, 1<<20) + } + + if zstdMaxWindow() > zstdMaxCompatibleWindow { + t.Fatalf("max window %d exceeds compatible cap %d", zstdMaxWindow(), zstdMaxCompatibleWindow) + } + w, err = zstdWindowSize(zstdMaxCompatibleWindow*2, 0) + if err != nil { + t.Fatal(err) + } + if w != zstdMaxWindow() { + t.Fatalf("auto window %d, want cap %d", w, zstdMaxWindow()) + } + if _, err := zstdWindowSize(100, zstdMaxCompatibleWindow*2); err == nil { + t.Fatal("expected error for configured window over 512MiB") + } +} diff --git a/pkg/tar-patch/apply.go b/pkg/tar-patch/apply.go index 9418427..7ca946e 100644 --- a/pkg/tar-patch/apply.go +++ b/pkg/tar-patch/apply.go @@ -95,7 +95,8 @@ func Apply(delta io.Reader, dataSource DataSource, dst io.Writer) error { if err != nil { return err } - if !bytes.Equal(buf, protocol.DeltaHeader[:]) { + isV2 := bytes.Equal(buf, protocol.DeltaHeaderv2[:]) + if !isV2 && !bytes.Equal(buf, protocol.DeltaHeader[:]) { return fmt.Errorf("invalid delta format") } @@ -180,6 +181,13 @@ func Apply(delta io.Reader, dataSource DataSource, dst io.Writer) error { if err != nil { return err } + case protocol.DeltaOpZstdDict: + if !isV2 { + return fmt.Errorf("ZstdDict requires tardf2") + } + if err := applyZstdDict(r, dataSource, dst, size); err != nil { + return err + } default: return fmt.Errorf("unexpected delta op %d", op) } @@ -187,3 +195,26 @@ func Apply(delta io.Reader, dataSource DataSource, dst io.Writer) error { return nil } + +func applyZstdDict(r io.Reader, dataSource DataSource, dst io.Writer, size uint64) error { + // Dict is the whole source file; Copy/Seek ops may have left the cursor mid-file. + if _, err := dataSource.Seek(0, io.SeekStart); err != nil { + return err + } + + dict, err := io.ReadAll(dataSource) + if err != nil { + return err + } + + decoder, err := zstd.NewReader(io.LimitReader(r, int64(size)), zstd.WithDecoderDictRaw(0, dict), zstd.WithDecoderConcurrency(1)) + if err != nil { + return err + } + defer decoder.Close() + + if _, err := io.Copy(dst, decoder); err != nil { + return fmt.Errorf("ZstdDict decompress: %w", err) + } + return nil +} diff --git a/pkg/tar-patch/apply_test.go b/pkg/tar-patch/apply_test.go index 63e5828..437a986 100644 --- a/pkg/tar-patch/apply_test.go +++ b/pkg/tar-patch/apply_test.go @@ -17,22 +17,31 @@ import ( // Helper to create delta streams for testing func createDeltaStream(t *testing.T, ops []deltaOp) *bytes.Buffer { t.Helper() + header := protocol.DeltaHeader + for _, op := range ops { + if op.code == protocol.DeltaOpZstdDict { + header = protocol.DeltaHeaderv2 + break + } + } + return createDeltaStreamWithHeader(t, header, ops) +} + +func createDeltaStreamWithHeader(t *testing.T, header [8]byte, ops []deltaOp) *bytes.Buffer { + t.Helper() var buf bytes.Buffer - // Write header - _, err := buf.Write(protocol.DeltaHeader[:]) + _, err := buf.Write(header[:]) if err != nil { t.Fatalf("failed to write delta header: %v", err) } - // Create zstd encoder encoder, err := zstd.NewWriter(&buf) if err != nil { t.Fatalf("failed to create zstd encoder: %v", err) } - // Write operations for _, op := range ops { opBuf := make([]byte, 1+binary.MaxVarintLen64) opBuf[0] = op.code @@ -293,6 +302,77 @@ func TestApply_DeltaOpSeek(t *testing.T) { } } +func TestApply_DeltaOpZstdDict(t *testing.T) { + filename := "base.txt" + oldData := []byte("The quick brown fox jumps over the lazy dog") + newData := []byte("The quick red fox runs over the lazy cat") + + enc, err := zstd.NewWriter(nil, + zstd.WithEncoderLevel(zstd.SpeedDefault), + zstd.WithEncoderDictRaw(0, oldData), + zstd.WithEncoderConcurrency(1), + ) + if err != nil { + t.Fatalf("create encoder: %v", err) + } + frame := enc.EncodeAll(newData, nil) + if err := enc.Close(); err != nil { + t.Fatalf("close encoder: %v", err) + } + + delta := createDeltaStream(t, []deltaOp{ + {code: protocol.DeltaOpOpen, size: uint64(len(filename)), data: []byte(filename)}, + {code: protocol.DeltaOpZstdDict, size: uint64(len(frame)), data: frame}, + }) + + var output bytes.Buffer + ds := newMockDataSource() + ds.AddFile(filename, oldData) + + if err := Apply(delta, ds, &output); err != nil { + t.Fatalf("Apply failed: %v", err) + } + if !bytes.Equal(output.Bytes(), newData) { + t.Errorf("expected output %q, got %q", newData, output.Bytes()) + } +} + +func TestApply_ZstdDictRejectedOnV1(t *testing.T) { + filename := "base.txt" + oldData := []byte("The quick brown fox jumps over the lazy dog") + newData := []byte("The quick red fox runs over the lazy cat") + + enc, err := zstd.NewWriter(nil, + zstd.WithEncoderLevel(zstd.SpeedDefault), + zstd.WithEncoderDictRaw(0, oldData), + zstd.WithEncoderConcurrency(1), + ) + if err != nil { + t.Fatalf("create encoder: %v", err) + } + frame := enc.EncodeAll(newData, nil) + if err := enc.Close(); err != nil { + t.Fatalf("close encoder: %v", err) + } + + delta := createDeltaStreamWithHeader(t, protocol.DeltaHeader, []deltaOp{ + {code: protocol.DeltaOpOpen, size: uint64(len(filename)), data: []byte(filename)}, + {code: protocol.DeltaOpZstdDict, size: uint64(len(frame)), data: frame}, + }) + + var output bytes.Buffer + ds := newMockDataSource() + ds.AddFile(filename, oldData) + + err = Apply(delta, ds, &output) + if err == nil { + t.Fatal("expected error for ZstdDict on v1 header") + } + if !strings.Contains(err.Error(), "tardf2") { + t.Errorf("expected tardf2 error, got: %v", err) + } +} + func TestApply_InvalidHeader(t *testing.T) { delta := bytes.NewReader([]byte("invalid header data"))