feat: optional zstd dictionary binary-diff backend

.gitignore

@@ -1,6 +1,6 @@
 cmd/tar-diff/tar-diff
 cmd/tar-patch/tar-patch
 .secrets
-tar-diff
-tar-patch
+/tar-diff
+/tar-patch
 test/

Makefile

@@ -55,8 +55,8 @@ install: tar-diff tar-patch
 tools: .install.golangci-lint
 
 .install.golangci-lint:
-	if [ ! -x "$(GOBIN)/golangci-lint" ]; then \
-		curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/main/install.sh | sh -s -- -b $(GOBIN) v2.10.1; \
+	if [ ! -x "$(GOBIN)/golangci-lint" ] || ! "$(GOBIN)/golangci-lint" version 2>/dev/null | grep -q 'v2.13.1'; then \
+		curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/main/install.sh | sh -s -- -b $(GOBIN) v2.13.1; \
 	fi
 
 clean:

README.md

@@ -53,6 +53,14 @@ This is particularly useful for `bootc` images, for example, where only the file
 will be available on the system. In that case you would run `tar-diff` with
 `--source-prefix=sysroot/ostree/repo/objects/`.
 
+### Mixing zstd and bsdiff
+
+By default `tar-diff` uses bsdiff for similar files. `--binary-diff auto` uses zstd dictionary patches for files under `--max-zstd-diff-size` (apply holds the old file in RAM as the dictionary) and bsdiff for larger files. The default method remains `bsdiff`. `--binary-diff auto` and `zstd` write the `tardf2` magic; see [file-format.md](file-format.md).
+
+```
+$ tar-diff --binary-diff auto --max-zstd-diff-size 128 --max-bsdiff-size 0 old.tar new.tar delta.tardiff
+```
+
 ## Build requirements
 
 - golang >= 1.26 (see [`go.mod`](go.mod))

cmd/tar-diff/main.go

@@ -25,8 +25,12 @@ func (p *prefixList) Set(value string) error {
 }
 
 var version = flag.Bool("version", false, "Show version")
-var compressionLevel = flag.Int("compression-level", 3, "zstd compression level")
-var maxBsdiffSize = flag.Int("max-bsdiff-size", 192, "Max file size in megabytes to consider using bsdiff, or 0 for no limit")
+var compressionLevel = flag.Int("compression-level", 3, "zstd compression level for the outer delta stream")
+var maxBsdiffSize = flag.Int("max-bsdiff-size", 192, "Max file size in megabytes to consider for bsdiff, or 0 for no limit")
+var maxZstdDiffSize = flag.Int("max-zstd-diff-size", 128, "Max file size in megabytes to consider for zstd dictionary patches, or 0 for no extra cap")
+var binaryDiff = flag.String("binary-diff", "bsdiff", "Per-file binary diff method: bsdiff, zstd, or auto")
+var zstdDiffLevel = flag.Int("zstd-diff-level", -1, "zstd level for dictionary patches (-1 = use --compression-level)")
+var zstdDiffWindow = flag.Int("zstd-diff-window", 0, "zstd window size in MiB for dictionary patches (0 = auto from source size, max 512)")
 var tmpDir = flag.String("tmp-dir", defaultTmpDir, "Directory for temporary files")
 var applyWhiteouts = flag.Bool("apply-whiteouts", false, "Apply docker/OCI whiteout files when analyzing old tar layers")
 var sourcePrefixes prefixList
@@ -94,6 +98,24 @@ func realMain() int {
 	options := tardiff.NewOptions()
 	options.SetCompressionLevel(*compressionLevel)
 	options.SetMaxBsdiffFileSize(int64(*maxBsdiffSize) * 1024 * 1024)
+	options.SetMaxZstdDiffFileSize(int64(*maxZstdDiffSize) * 1024 * 1024)
+	switch *binaryDiff {
+	case "bsdiff":
+		options.SetBinaryDiffMethod(tardiff.BinaryDiffBsdiff)
+	case "zstd":
+		options.SetBinaryDiffMethod(tardiff.BinaryDiffZstd)
+	case "auto":
+		options.SetBinaryDiffMethod(tardiff.BinaryDiffAuto)
+	default:
+		log.Printf("Error: invalid --binary-diff %q (want bsdiff, zstd, or auto)", *binaryDiff)
+		return 1
+	}
+	options.SetZstdDiffLevel(*zstdDiffLevel)
+	if *zstdDiffWindow < 0 {
+		log.Printf("Error: invalid --zstd-diff-window %d", *zstdDiffWindow)
+		return 1
+	}
+	options.SetZstdDiffWindow(*zstdDiffWindow * 1024 * 1024)
 	if len(sourcePrefixes) > 0 {
 		options.SetSourcePrefixes(sourcePrefixes)
 	}

file-format.md

@@ -2,12 +2,18 @@ File Format
 -----------
 
 A tar-diff file (media type `application/vnd.tar-diff`) consists of a
-header, with the fixed bytes:
+header, with one of:
 
 ```
-{ 't', 'a', 'r', 'd', 'f', '1', '\n', 0}
+{ 't', 'a', 'r', 'd', 'f', '1', '\n', 0}   // v1
+{ 't', 'a', 'r', 'd', 'f', '2', '\n', 0}   // v2
 ```
 
+v2 is required if the file contains any `DeltaOpZstdDict` operation.
+Generators emit v2 whenever zstd-dict ops are possible (`--binary-diff
+auto` or `zstd`), even if a particular delta happens to contain none.
+v1 files must not contain `DeltaOpZstdDict`.
+
 Followed by a [zstd](https://facebook.github.io/zstd/) compressed
 stream, with a sequence of operations, each operation is encoded as
 follows:
@@ -42,15 +48,17 @@ DeltaOpOpen = 1
 DeltaOpCopy = 2
 DeltaOpAddData = 3
 DeltaOpSeek = 4
+DeltaOpZstdDict = 5
 ```
 
 ***DeltaOpData***
 Emit the bytes from `<data>` into the output stream.
 
 ***DeltaOpOpen***
 `<data>` is a the (relative) path to a file within the original
-tarball. Set the source for subsequent `DeltaOpCopy` and `DeltaAddData`
-operations to this file, and reset the source position to 0.
+tarball. Set the source for subsequent `DeltaOpCopy`, `DeltaAddData`,
+and `DeltaOpZstdDict` operations to this file, and reset the source
+position to 0.
 
 Tar-diff generates normalized paths with no `.` or `..`  elements,
 which this will never point outside the target directory. However, for
@@ -68,3 +76,12 @@ stream.
 
 ***DeltaOpSeek***
 Set the source position to `<size>`
+
+***DeltaOpZstdDict***
+Only valid in v2 files. `<data>` is a zstd frame compressed with the
+currently open source file as a raw dictionary (dict id 0), matching
+`zstd --patch-from` semantics. Seek the source file to offset 0,
+decompress `<data>` using the full source file content as the
+dictionary, and emit the decompressed bytes to the output stream.
+Generators cap the zstd window at 512 MiB so older apply
+implementations can still decode the frame.

pkg/protocol/common.go

@@ -5,16 +5,20 @@ import "path/filepath"
 
 // Delta operation constants define the types of operations in a delta file.
 const (
-	DeltaOpData    = iota // Raw data operation
-	DeltaOpOpen    = iota // Open file operation
-	DeltaOpCopy    = iota // Copy from source operation
-	DeltaOpAddData = iota // Add new data operation
-	DeltaOpSeek    = iota // Seek operation
+	DeltaOpData     = iota // Raw data operation
+	DeltaOpOpen     = iota // Open file operation
+	DeltaOpCopy     = iota // Copy from source operation
+	DeltaOpAddData  = iota // Add new data operation
+	DeltaOpSeek     = iota // Seek operation
+	DeltaOpZstdDict = iota // zstd dictionary patch against open source file
 )
 
-// DeltaHeader is the magic header bytes for tar-diff files.
+// DeltaHeader is the magic for v1 tar-diff files (no DeltaOpZstdDict).
 var DeltaHeader = [...]byte{'t', 'a', 'r', 'd', 'f', '1', '\n', 0}
 
+// DeltaHeaderv2 is the magic when zstd-dict ops are possible (auto or zstd mode).
+var DeltaHeaderv2 = [...]byte{'t', 'a', 'r', 'd', 'f', '2', '\n', 0}
+
 // CleanPath cleans up the path lexically and prevents path traversal attacks.
 // Any ".." that extends outside the first elements (or the root itself) is invalid and returns "".
 // Uses filepath.Clean for proper cross-platform path handling (Windows backslashes, drive letters).

pkg/tar-diff/bsdiff_test.go

@@ -93,7 +93,7 @@ func TestBsdiffBasic(t *testing.T) {
 	// and produces output, but cannot verify correctness without a bspatch implementation.
 	// The bsdiff algorithm is well-tested upstream; these tests ensure integration works.
 	var output bytes.Buffer
-	deltaWriter, err := newDeltaWriter(&output, 1)
+	deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1)
 	if err != nil {
 		t.Fatalf("Failed to create delta writer: %v", err)
 	}
@@ -126,7 +126,7 @@ func TestBsdiffBasic(t *testing.T) {
 
 func TestBsdiffIdentical(t *testing.T) {
 	var output bytes.Buffer
-	deltaWriter, err := newDeltaWriter(&output, 1)
+	deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1)
 	if err != nil {
 		t.Fatalf("Failed to create delta writer: %v", err)
 	}
@@ -158,7 +158,7 @@ func TestBsdiffIdentical(t *testing.T) {
 
 func TestBsdiffEmpty(t *testing.T) {
 	var output bytes.Buffer
-	deltaWriter, err := newDeltaWriter(&output, 1)
+	deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1)
 	if err != nil {
 		t.Fatalf("Failed to create delta writer: %v", err)
 	}
@@ -197,7 +197,7 @@ func TestBsdiffEmpty(t *testing.T) {
 
 func TestBsdiffLargeData(t *testing.T) {
 	var output bytes.Buffer
-	deltaWriter, err := newDeltaWriter(&output, 1)
+	deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1)
 	if err != nil {
 		t.Fatalf("Failed to create delta writer: %v", err)
 	}
@@ -259,7 +259,7 @@ func TestSplitFunction(t *testing.T) {
 
 func TestBsdiffPartialMatch(t *testing.T) {
 	var output bytes.Buffer
-	deltaWriter, err := newDeltaWriter(&output, 1)
+	deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1)
 	if err != nil {
 		t.Fatalf("Failed to create delta writer: %v", err)
 	}

pkg/tar-diff/delta.go

@@ -2,33 +2,59 @@ package tardiff
 
 import (
 	"encoding/binary"
+	"fmt"
+	"io"
+
 	"github.com/containers/tar-diff/pkg/protocol"
 	"github.com/klauspost/compress/zstd"
-	"io"
 )
 
 const (
 	deltaDataChunkSize = 4 * 1024 * 1024
 )
 
+type deltaFormatVersion int
+
+const (
+	deltaFormatV1 deltaFormatVersion = 1
+	deltaFormatV2 deltaFormatVersion = 2
+)
+
+func (v deltaFormatVersion) header() ([]byte, error) {
+	switch v {
+	case deltaFormatV1:
+		return protocol.DeltaHeader[:], nil
+	case deltaFormatV2:
+		return protocol.DeltaHeaderv2[:], nil
+	default:
+		return nil, fmt.Errorf("unsupported delta format version %d", v)
+	}
+}
+
 type deltaWriter struct {
 	writer      *zstd.Encoder
 	buffer      []byte
 	currentFile string
 	currentPos  uint64
 }
 
-func newDeltaWriter(writer io.Writer, compressionLevel int) (*deltaWriter, error) {
-	_, err := writer.Write(protocol.DeltaHeader[:])
+func newDeltaWriter(writer io.Writer, compressionLevel int, version deltaFormatVersion) (*deltaWriter, error) {
+	header, err := version.header()
 	if err != nil {
 		return nil, err
 	}
+	if _, err := writer.Write(header); err != nil {
+		return nil, err
+	}
 
 	encoder, err := zstd.NewWriter(writer, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(compressionLevel)))
 	if err != nil {
 		return nil, err
 	}
-	d := deltaWriter{writer: encoder, buffer: make([]byte, 0, deltaDataChunkSize)}
+	d := deltaWriter{
+		writer: encoder,
+		buffer: make([]byte, 0, deltaDataChunkSize),
+	}
 	return &d, nil
 }
 
@@ -74,12 +100,23 @@ func (d *deltaWriter) Close() error {
 }
 
 func (d *deltaWriter) WriteContent(data []byte) error {
-	d.buffer = append(d.buffer, data...)
-
-	if len(d.buffer) >= deltaDataChunkSize {
-		return d.FlushBuffer()
+	for len(data) > 0 {
+		if len(d.buffer) >= deltaDataChunkSize {
+			if err := d.FlushBuffer(); err != nil {
+				return err
+			}
+		}
+		space := deltaDataChunkSize - len(d.buffer)
+		if space > len(data) {
+			space = len(data)
+		}
+		d.buffer = append(d.buffer, data[:space]...)
+		data = data[space:]
 	}
-	return nil
+	if len(d.buffer) < deltaDataChunkSize {
+		return nil
+	}
+	return d.FlushBuffer()
 }
 
 // Switches to new file if needed and ensures we're at the start of it
@@ -188,8 +225,25 @@ func (d *deltaWriter) WriteOldFile(filename string, size uint64) error {
 	return nil
 }
 
+func (d *deltaWriter) WriteZstdDict(data []byte, sourceSize uint64) error {
+	if err := d.FlushBuffer(); err != nil {
+		return err
+	}
+	if err := d.writeOp(protocol.DeltaOpZstdDict, uint64(len(data)), data); err != nil {
+		return err
+	}
+	// Apply reads the whole source as the dict, leaving the cursor at EOF.
+	d.currentPos = sourceSize
+	return nil
+}
+
 func (d *deltaWriter) Write(data []byte) (int, error) {
 	n := len(data)
 	err := d.WriteContent(data)
 	return n, err
 }
+
+func (d *deltaWriter) WriteContentFrom(r io.Reader) error {
+	_, err := io.Copy(d, r)
+	return err
+}