diff --git a/go.mod b/go.mod index 4c2e945f86..06d8bae7dd 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/google/addlicense v1.2.0 github.com/google/gnostic-models v0.7.1 github.com/google/go-cmp v0.7.0 - github.com/google/go-containerregistry v0.21.9 + github.com/google/go-containerregistry v0.22.0 github.com/google/go-licenses/v2 v2.0.1 github.com/google/uuid v1.6.0 github.com/jstemmer/go-junit-report/v2 v2.1.0 @@ -84,7 +84,7 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chai2010/gettext-go v1.0.2 // indirect github.com/distribution/reference v0.6.0 // indirect - github.com/docker/cli v29.6.2+incompatible // indirect + github.com/docker/cli v29.7.2+incompatible // indirect github.com/docker/docker-credential-helpers v0.9.3 // indirect github.com/emicklei/go-restful/v3 v3.12.2 // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect @@ -113,7 +113,7 @@ require ( github.com/jonboulle/clockwork v0.5.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.19.1 // indirect + github.com/klauspost/compress v1.19.2 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect diff --git a/go.sum b/go.sum index cc22f8f50c..9b3a9ddf1c 100644 --- a/go.sum +++ b/go.sum @@ -70,8 +70,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/cli v29.6.2+incompatible h1:/bjePvcbbFTnRrMfWJBY7AjfICdsiLVgHn6LwTVOcqw= -github.com/docker/cli v29.6.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v29.7.2+incompatible h1:dlkwallR8XqfeVnA2ELEhdwvb4lsSwuB4IgsG8Q9cLY= +github.com/docker/cli v29.7.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/docker-credential-helpers v0.9.3 h1:gAm/VtF9wgqJMoxzT3Gj5p4AqIjCBS4wrsOh9yRqcz8= github.com/docker/docker-credential-helpers v0.9.3/go.mod h1:x+4Gbw9aGmChi3qTLZj8Dfn0TD20M/fuWy0E5+WDeCo= github.com/elliotchance/orderedmap/v2 v2.7.0 h1:WHuf0DRo63uLnldCPp9ojm3gskYwEdIIfAUVG5KhoOc= @@ -160,8 +160,8 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/go-containerregistry v0.21.9 h1:F+D4uZ3iA3DLMJLfhaqMdHJbzeqm/216WGQq2dokuLs= -github.com/google/go-containerregistry v0.21.9/go.mod h1:dP5XNKcL7kMFF/TB3LfvWmVhAcv7iqkHb3oDK8aauTo= +github.com/google/go-containerregistry v0.22.0 h1:eGbCiPeYxAH/7WLLq6zTBALP0tUIFsoyRauhxXDJ53I= +github.com/google/go-containerregistry v0.22.0/go.mod h1:bJR35SK8XgisYmhg/FMQ/5RK0S/XrOAqLBV5/LR2XE0= github.com/google/go-licenses/v2 v2.0.1 h1:ti+9bi5o7DKbeeg5eBb/uZTgsaPNoJaLCh93cRcXsW8= github.com/google/go-licenses/v2 v2.0.1/go.mod h1:efibo0EDNGkau6AIMOViGW+rTNPudhxX9rCxtfw5zKE= github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQEEioJvFYxYcLRKzk= @@ -202,8 +202,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report/v2 v2.1.0 h1:X3+hPYlSczH9IMIpSC9CQSZA0L+BipYafciZUWHEmsc= github.com/jstemmer/go-junit-report/v2 v2.1.0/go.mod h1:mgHVr7VUo5Tn8OLVr1cKnLuEy0M92wdRntM99h7RkgQ= -github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= -github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= diff --git a/vendor/github.com/docker/cli/cli/config/credentials/file_store.go b/vendor/github.com/docker/cli/cli/config/credentials/file_store.go index e3ef8e25ed..d4037b7a48 100644 --- a/vendor/github.com/docker/cli/cli/config/credentials/file_store.go +++ b/vendor/github.com/docker/cli/cli/config/credentials/file_store.go @@ -104,7 +104,7 @@ func (c *fileStore) Store(authConfig types.AuthConfig) error { // stored as hostname or as hostname including scheme (in legacy configuration // files). // -// It's the equivalent to [registry.ConvertToHostname] in the daemon. +// It's based on [registry.ConvertToHostname] from Moby daemon. // // [registry.ConvertToHostname]: https://pkg.go.dev/github.com/moby/moby/v2@v2.0.0-beta.7/daemon/pkg/registry#ConvertToHostname func ConvertToHostname(maybeURL string) string { @@ -117,7 +117,48 @@ func ConvertToHostname(maybeURL string) string { } return net.JoinHostPort(u.Hostname(), u.Port()) } + + if hostName := hostFromURLFallback(stripped); hostName != "" { + return hostName + } } hostName, _, _ := strings.Cut(stripped, "/") return hostName } + +// hostFromURLFallback extracts a host from scheme URLs that net/url rejects. +// Go rejects unbracketed IPv6 literals in URL hosts since +// https://github.com/golang/go/commit/0c28789bd7dfc55099cac86a3212dda0d6c091f6 +func hostFromURLFallback(maybeURL string) string { + _, rest, ok := strings.Cut(maybeURL, "://") + if !ok { + return "" + } + + hostName, _, _ := strings.Cut(rest, "/") + if hostName == "" { + return "" + } + + if strings.Count(hostName, ":") > 1 && !strings.HasPrefix(hostName, "[") { + portStart := strings.LastIndex(hostName, ":") + addr, port := hostName[:portStart], hostName[portStart+1:] + if addr != "" && isPort(port) { + return net.JoinHostPort(addr, port) + } + } + + return hostName +} + +func isPort(port string) bool { + if port == "" { + return false + } + for _, r := range port { + if r < '0' || r > '9' { + return false + } + } + return true +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/repository.go b/vendor/github.com/google/go-containerregistry/pkg/name/repository.go index efde6e869d..ad7b329111 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/name/repository.go +++ b/vendor/github.com/google/go-containerregistry/pkg/name/repository.go @@ -72,7 +72,7 @@ func (r Repository) Scope(action string) string { } func checkRepository(repository string) error { - return checkElement("repository", repository, repositoryChars, 2, 255) + return checkElement("repository", repository, repositoryChars, 1, 255) } // NewRepository returns a new Repository representing the given name, according to the given strictness. diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go index 596c78cb26..0125bfc653 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go @@ -26,7 +26,6 @@ import ( "strings" "time" - "github.com/google/go-containerregistry/internal/gzip" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/match" @@ -469,7 +468,8 @@ func hasWindowsDrivePrefix(name string) bool { } // Time sets all timestamps in an image to the given timestamp. -func Time(img v1.Image, t time.Time) (v1.Image, error) { +// Layers are rewritten as dockerv2+gzip unless opts say otherwise. +func Time(img v1.Image, t time.Time, opts ...tarball.LayerOption) (v1.Image, error) { newImage := empty.Image layers, err := img.Layers() @@ -485,7 +485,7 @@ func Time(img v1.Image, t time.Time) (v1.Image, error) { addendums := make([]Addendum, max(len(ocf.History), len(layers))) var historyIdx, addendumIdx int for layerIdx := 0; layerIdx < len(layers); addendumIdx, layerIdx = addendumIdx+1, layerIdx+1 { - newLayer, err := layerTime(layers[layerIdx], t) + newLayer, err := layerTime(layers[layerIdx], t, opts...) if err != nil { return nil, fmt.Errorf("setting layer times: %w", err) } @@ -547,7 +547,7 @@ func Time(img v1.Image, t time.Time) (v1.Image, error) { return ConfigFile(newImage, cfg) } -func layerTime(layer v1.Layer, t time.Time) (v1.Layer, error) { +func layerTime(layer v1.Layer, t time.Time, opts ...tarball.LayerOption) (v1.Layer, error) { layerReader, err := layer.Uncompressed() if err != nil { return nil, fmt.Errorf("getting layer: %w", err) @@ -598,11 +598,10 @@ func layerTime(layer v1.Layer, t time.Time) (v1.Layer, error) { } b := w.Bytes() - // gzip the contents, then create the layer opener := func() (io.ReadCloser, error) { - return gzip.ReadCloser(io.NopCloser(bytes.NewReader(b))), nil + return io.NopCloser(bytes.NewReader(b)), nil } - layer, err = tarball.LayerFromOpener(opener) + layer, err = tarball.LayerFromOpener(opener, opts...) if err != nil { return nil, fmt.Errorf("creating layer: %w", err) } @@ -612,10 +611,10 @@ func layerTime(layer v1.Layer, t time.Time) (v1.Layer, error) { // Canonical is a helper function to combine Time and configFile // to remove any randomness during a docker build. -func Canonical(img v1.Image) (v1.Image, error) { +func Canonical(img v1.Image, opts ...tarball.LayerOption) (v1.Image, error) { // Set all timestamps to 0 created := time.Time{} - img, err := Time(img, created) + img, err := Time(img, created, opts...) if err != nil { return nil, err } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go index ac879a6737..8f5bd12633 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go @@ -32,9 +32,9 @@ import ( // // TODO(#412): Remove the need for this method. func CheckPushPermission(ref name.Reference, kc authn.Keychain, t http.RoundTripper) error { - auth, err := kc.Resolve(ref.Context().Registry) + auth, err := authn.Resolve(context.TODO(), kc, ref.Context()) if err != nil { - return fmt.Errorf("resolving authorization for %v failed: %w", ref.Context().Registry, err) + return fmt.Errorf("resolving authorization for %v failed: %w", ref.Context(), err) } scopes := []string{ref.Scope(transport.PushScope)} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/puller.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/puller.go index bcb0d7350f..094162ec44 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/puller.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/puller.go @@ -76,7 +76,11 @@ func (p *Puller) fetcher(ctx context.Context, target resource) (*fetcher, error) o: p.o, }) rr := v.(*reader) - return rr.f, rr.init(ctx) + if err := rr.init(ctx); err != nil { + p.readers.CompareAndDelete(target, rr) + return nil, err + } + return rr.f, nil } // Head is like remote.Head, but avoids re-authenticating when possible. diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/pusher.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/pusher.go index 3a1d16f544..3466c7a671 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/pusher.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/pusher.go @@ -122,7 +122,11 @@ func (p *Pusher) writer(ctx context.Context, repo name.Repository, o *options) ( o: o, }) rw := v.(*repoWriter) - return rw, rw.init(ctx) + if err := rw.init(ctx); err != nil { + p.writers.CompareAndDelete(repo, rw) + return nil, err + } + return rw, nil } func (p *Pusher) Put(ctx context.Context, ref name.Reference, t Taggable) error { diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go index 2d6ef64be1..7035d656d9 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go @@ -1,222 +1,49 @@ //go:build amd64 && !appengine && !noasm && gc -// This file contains the specialisation of Decoder.Decompress4X -// and Decoder.Decompress1X that use an asm implementation of thir main loops. +// amd64 stubs and dispatch for the asm loops used by decompress_asm.go. package huff0 import ( - "errors" - "fmt" - "github.com/klauspost/compress/internal/cpuinfo" ) -// decompress4x_main_loop_x86 is an x86 assembler implementation +// decompress4x_main_loop_amd64 is an x86 assembler implementation // of Decompress4X when tablelog > 8. // //go:noescape func decompress4x_main_loop_amd64(ctx *decompress4xContext) -// decompress4x_8b_loop_x86 is an x86 assembler implementation +// decompress4x_8b_main_loop_amd64 is an x86 assembler implementation // of Decompress4X when tablelog <= 8 which decodes 4 entries // per loop. // //go:noescape func decompress4x_8b_main_loop_amd64(ctx *decompress4xContext) -// fallback8BitSize is the size where using Go version is faster. -const fallback8BitSize = 800 - -type decompress4xContext struct { - pbr *[4]bitReaderShifted - peekBits uint8 - out *byte - dstEvery int - tbl *dEntrySingle - decoded int - limit *byte -} - -// Decompress4X will decompress a 4X encoded stream. -// The length of the supplied input must match the end of a block exactly. -// The *capacity* of the dst slice must match the destination size of -// the uncompressed data exactly. -func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { - if len(d.dt.single) == 0 { - return nil, errors.New("no table loaded") - } - if len(src) < 6+(4*1) { - return nil, errors.New("input too small") - } - - use8BitTables := d.actualTableLog <= 8 - if cap(dst) < fallback8BitSize && use8BitTables { - return d.decompress4X8bit(dst, src) - } - - var br [4]bitReaderShifted - // Decode "jump table" - start := 6 - for i := range 3 { - length := int(src[i*2]) | (int(src[i*2+1]) << 8) - if start+length >= len(src) { - return nil, errors.New("truncated input (or invalid offset)") - } - err := br[i].init(src[start : start+length]) - if err != nil { - return nil, err - } - start += length - } - err := br[3].init(src[start:]) - if err != nil { - return nil, err - } - - // destination, offset to match first output - dstSize := cap(dst) - dst = dst[:dstSize] - out := dst - dstEvery := (dstSize + 3) / 4 - - const tlSize = 1 << tableLogMax - const tlMask = tlSize - 1 - single := d.dt.single[:tlSize] - - var decoded int - - if len(out) > 4*4 && !(br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4) { - ctx := decompress4xContext{ - pbr: &br, - peekBits: uint8((64 - d.actualTableLog) & 63), // see: bitReaderShifted.peekBitsFast() - out: &out[0], - dstEvery: dstEvery, - tbl: &single[0], - limit: &out[dstEvery-4], // Always stop decoding when first buffer gets here to avoid writing OOB on last. - } - if use8BitTables { - decompress4x_8b_main_loop_amd64(&ctx) - } else { - decompress4x_main_loop_amd64(&ctx) - } - - decoded = ctx.decoded - out = out[decoded/4:] - } - - // Decode remaining. - remainBytes := dstEvery - (decoded / 4) - for i := range br { - offset := dstEvery * i - endsAt := min(offset+remainBytes, len(out)) - br := &br[i] - bitsLeft := br.remaining() - for bitsLeft > 0 { - br.fill() - if offset >= endsAt { - return nil, errors.New("corruption detected: stream overrun 4") - } - - // Read value and increment offset. - val := br.peekBitsFast(d.actualTableLog) - v := single[val&tlMask].entry - nBits := uint8(v) - br.advance(nBits) - bitsLeft -= uint(nBits) - out[offset] = uint8(v >> 8) - offset++ - } - if offset != endsAt { - return nil, fmt.Errorf("corruption detected: short output block %d, end %d != %d", i, offset, endsAt) - } - decoded += offset - dstEvery*i - err = br.close() - if err != nil { - return nil, err - } - } - if dstSize != decoded { - return nil, errors.New("corruption detected: short output block") - } - return dst, nil -} - -// decompress4x_main_loop_x86 is an x86 assembler implementation +// decompress1x_main_loop_amd64 is an x86 assembler implementation // of Decompress1X when tablelog > 8. // //go:noescape func decompress1x_main_loop_amd64(ctx *decompress1xContext) -// decompress4x_main_loop_x86 is an x86 with BMI2 assembler implementation +// decompress1x_main_loop_bmi2 is an x86 with BMI2 assembler implementation // of Decompress1X when tablelog > 8. // //go:noescape func decompress1x_main_loop_bmi2(ctx *decompress1xContext) -type decompress1xContext struct { - pbr *bitReaderShifted - peekBits uint8 - out *byte - outCap int - tbl *dEntrySingle - decoded int +func decompress4x_main_loop_asm(ctx *decompress4xContext) { + decompress4x_main_loop_amd64(ctx) } -// Error reported by asm implementations -const error_max_decoded_size_exeeded = -1 - -// Decompress1X will decompress a 1X encoded stream. -// The cap of the output buffer will be the maximum decompressed size. -// The length of the supplied input must match the end of a block exactly. -func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) { - if len(d.dt.single) == 0 { - return nil, errors.New("no table loaded") - } - var br bitReaderShifted - err := br.init(src) - if err != nil { - return dst, err - } - maxDecodedSize := cap(dst) - dst = dst[:maxDecodedSize] - - const tlSize = 1 << tableLogMax - const tlMask = tlSize - 1 - - if maxDecodedSize >= 4 { - ctx := decompress1xContext{ - pbr: &br, - out: &dst[0], - outCap: maxDecodedSize, - peekBits: uint8((64 - d.actualTableLog) & 63), // see: bitReaderShifted.peekBitsFast() - tbl: &d.dt.single[0], - } - - if cpuinfo.HasBMI2() { - decompress1x_main_loop_bmi2(&ctx) - } else { - decompress1x_main_loop_amd64(&ctx) - } - if ctx.decoded == error_max_decoded_size_exeeded { - return nil, ErrMaxDecodedSizeExceeded - } - - dst = dst[:ctx.decoded] - } +func decompress4x_8b_main_loop_asm(ctx *decompress4xContext) { + decompress4x_8b_main_loop_amd64(ctx) +} - // br < 8, so uint8 is fine - bitsLeft := uint8(br.off)*8 + 64 - br.bitsRead - for bitsLeft > 0 { - br.fill() - if len(dst) >= maxDecodedSize { - br.close() - return nil, ErrMaxDecodedSizeExceeded - } - v := d.dt.single[br.peekBitsFast(d.actualTableLog)&tlMask] - nBits := uint8(v.entry) - br.advance(nBits) - bitsLeft -= nBits - dst = append(dst, uint8(v.entry>>8)) +func decompress1x_main_loop_asm(ctx *decompress1xContext) { + if cpuinfo.HasBMI2() { + decompress1x_main_loop_bmi2(ctx) + } else { + decompress1x_main_loop_amd64(ctx) } - return dst, br.close() } diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s index c4c7ab2d1f..c5d4a710a0 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s +++ b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s @@ -1,6 +1,6 @@ -// Code generated by command: go run gen.go -out ../decompress_amd64.s -pkg=huff0. DO NOT EDIT. +// Code generated by command: go run gen.go -out ../decompress.s -arch amd64,arm64 -pkg=huff0. DO NOT EDIT. -//go:build amd64 && !appengine && !noasm && gc +//go:build !appengine && !noasm && gc // func decompress4x_main_loop_amd64(ctx *decompress4xContext) TEXT ·decompress4x_main_loop_amd64(SB), $0-8 diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_arm64.go b/vendor/github.com/klauspost/compress/huff0/decompress_arm64.go new file mode 100644 index 0000000000..8ba3c810e6 --- /dev/null +++ b/vendor/github.com/klauspost/compress/huff0/decompress_arm64.go @@ -0,0 +1,37 @@ +//go:build arm64 && !appengine && !noasm && gc + +// arm64 stubs and dispatch for the asm loops used by decompress_asm.go. +// The asm (decompress_arm64.s) is generated by the avo arm64 lowering +// printer from the same source as the amd64 asm; see _generate/gen.go. +package huff0 + +// decompress4x_main_loop_arm64 is an arm64 assembler implementation +// of Decompress4X when tablelog > 8. +// +//go:noescape +func decompress4x_main_loop_arm64(ctx *decompress4xContext) + +// decompress4x_8b_main_loop_arm64 is an arm64 assembler implementation +// of Decompress4X when tablelog <= 8 which decodes 4 entries +// per loop. +// +//go:noescape +func decompress4x_8b_main_loop_arm64(ctx *decompress4xContext) + +// decompress1x_main_loop_arm64 is an arm64 assembler implementation +// of Decompress1X when tablelog > 8. +// +//go:noescape +func decompress1x_main_loop_arm64(ctx *decompress1xContext) + +func decompress4x_main_loop_asm(ctx *decompress4xContext) { + decompress4x_main_loop_arm64(ctx) +} + +func decompress4x_8b_main_loop_asm(ctx *decompress4xContext) { + decompress4x_8b_main_loop_arm64(ctx) +} + +func decompress1x_main_loop_asm(ctx *decompress1xContext) { + decompress1x_main_loop_arm64(ctx) +} diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_arm64.s b/vendor/github.com/klauspost/compress/huff0/decompress_arm64.s new file mode 100644 index 0000000000..e16012721a --- /dev/null +++ b/vendor/github.com/klauspost/compress/huff0/decompress_arm64.s @@ -0,0 +1,851 @@ +// Code generated by command: go run gen.go -out ../decompress.s -arch amd64,arm64 -pkg=huff0. DO NOT EDIT. +// EXPERIMENTAL arm64 output lowered from an amd64 avo program. + +//go:build arm64 && !appengine && !noasm && gc + +// func decompress4x_main_loop_amd64(ctx *decompress4xContext) +TEXT ·decompress4x_main_loop_arm64(SB), $0-8 + // Preload values + MOVD ctx+0(FP), R0 + MOVBU 8(R0), R6 + MOVD 16(R0), R3 + MOVD 48(R0), R5 + MOVD 24(R0), R7 + MOVD 32(R0), R8 + MOVD (R0), R9 + + // Main loop +main_loop: + MOVD $0, R2 + CMP R5, R3 + CSET GE, R16 + BFI $0, R16, $8, R2 + + // br0.fillFast32() + MOVD 32(R9), R10 + MOVBU 40(R9), R11 + CMP $0x20, R11 + BLS skip_fill0 + MOVD 24(R9), R0 + SUB $0x20, R11, R11 + SUB $0x04, R0, R0 + MOVD (R9), R12 + + // b.value |= uint64(low) << (b.bitsRead & 63) + ADD R12, R0, R15 + MOVWU (R15), R12 + MOVD R11, R1 + LSL R1, R12, R12 + MOVD R0, 24(R9) + ORR R12, R10, R10 + + // exhausted += (br0.off < 4) + CMP $0x04, R0 + CSINC HS, R2, R2, R16 + BFI $0, R16, $8, R2 + +skip_fill0: + // val0 := br0.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v0 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br0.advance(uint8(v0.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val1 := br0.peekTopBits(peekBits) + MOVD R6, R1 + MOVD R10, R12 + LSR R1, R12, R12 + + // v1 := table[val1&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br0.advance(uint8(v1.entry)) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // these two writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + MOVH R0, (R3) + + // update the bitreader structure + MOVD R10, 32(R9) + MOVB R11, 40(R9) + + // br1.fillFast32() + MOVD 80(R9), R10 + MOVBU 88(R9), R11 + CMP $0x20, R11 + BLS skip_fill1 + MOVD 72(R9), R0 + SUB $0x20, R11, R11 + SUB $0x04, R0, R0 + MOVD 48(R9), R12 + + // b.value |= uint64(low) << (b.bitsRead & 63) + ADD R12, R0, R15 + MOVWU (R15), R12 + MOVD R11, R1 + LSL R1, R12, R12 + MOVD R0, 72(R9) + ORR R12, R10, R10 + + // exhausted += (br1.off < 4) + CMP $0x04, R0 + CSINC HS, R2, R2, R16 + BFI $0, R16, $8, R2 + +skip_fill1: + // val0 := br1.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v0 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br1.advance(uint8(v0.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val1 := br1.peekTopBits(peekBits) + MOVD R6, R1 + MOVD R10, R12 + LSR R1, R12, R12 + + // v1 := table[val1&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br1.advance(uint8(v1.entry)) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // these two writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + ADD R7, R3, R15 + MOVH R0, (R15) + + // update the bitreader structure + MOVD R10, 80(R9) + MOVB R11, 88(R9) + + // br2.fillFast32() + MOVD 128(R9), R10 + MOVBU 136(R9), R11 + CMP $0x20, R11 + BLS skip_fill2 + MOVD 120(R9), R0 + SUB $0x20, R11, R11 + SUB $0x04, R0, R0 + MOVD 96(R9), R12 + + // b.value |= uint64(low) << (b.bitsRead & 63) + ADD R12, R0, R15 + MOVWU (R15), R12 + MOVD R11, R1 + LSL R1, R12, R12 + MOVD R0, 120(R9) + ORR R12, R10, R10 + + // exhausted += (br2.off < 4) + CMP $0x04, R0 + CSINC HS, R2, R2, R16 + BFI $0, R16, $8, R2 + +skip_fill2: + // val0 := br2.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v0 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br2.advance(uint8(v0.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val1 := br2.peekTopBits(peekBits) + MOVD R6, R1 + MOVD R10, R12 + LSR R1, R12, R12 + + // v1 := table[val1&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br2.advance(uint8(v1.entry)) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // these two writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + ADD R7<<1, R3, R15 + MOVH R0, (R15) + + // update the bitreader structure + MOVD R10, 128(R9) + MOVB R11, 136(R9) + + // br3.fillFast32() + MOVD 176(R9), R10 + MOVBU 184(R9), R11 + CMP $0x20, R11 + BLS skip_fill3 + MOVD 168(R9), R0 + SUB $0x20, R11, R11 + SUB $0x04, R0, R0 + MOVD 144(R9), R12 + + // b.value |= uint64(low) << (b.bitsRead & 63) + ADD R12, R0, R15 + MOVWU (R15), R12 + MOVD R11, R1 + LSL R1, R12, R12 + MOVD R0, 168(R9) + ORR R12, R10, R10 + + // exhausted += (br3.off < 4) + CMP $0x04, R0 + CSINC HS, R2, R2, R16 + BFI $0, R16, $8, R2 + +skip_fill3: + // val0 := br3.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v0 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br3.advance(uint8(v0.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val1 := br3.peekTopBits(peekBits) + MOVD R6, R1 + MOVD R10, R12 + LSR R1, R12, R12 + + // v1 := table[val1&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br3.advance(uint8(v1.entry)) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // these two writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + ADD R7<<1, R7, R1 + ADD R1, R3, R15 + MOVH R0, (R15) + + // update the bitreader structure + MOVD R10, 176(R9) + MOVB R11, 184(R9) + ADD $0x02, R3, R3 + AND $0xff, R2, R15 + AND $0xff, R2, R16 + TST R16, R15 + BEQ main_loop + MOVD ctx+0(FP), R0 + MOVD 16(R0), R16 + SUB R16, R3, R3 + LSL $0x02, R3, R3 + MOVD R3, 40(R0) + RET + +// func decompress4x_8b_main_loop_amd64(ctx *decompress4xContext) +TEXT ·decompress4x_8b_main_loop_arm64(SB), $0-8 + // Preload values + MOVD ctx+0(FP), R1 + MOVBU 8(R1), R6 + MOVD 16(R1), R3 + MOVD 48(R1), R5 + MOVD 24(R1), R7 + MOVD 32(R1), R8 + MOVD (R1), R9 + + // Main loop +main_loop: + MOVD $0, R2 + CMP R5, R3 + CSET GE, R16 + BFI $0, R16, $8, R2 + + // br0.fillFast32() + MOVD 32(R9), R10 + MOVBU 40(R9), R11 + CMP $0x20, R11 + BLS skip_fill0 + MOVD 24(R9), R12 + SUB $0x20, R11, R11 + SUB $0x04, R12, R12 + MOVD (R9), R13 + + // b.value |= uint64(low) << (b.bitsRead & 63) + ADD R13, R12, R15 + MOVWU (R15), R13 + MOVD R11, R1 + LSL R1, R13, R13 + MOVD R12, 24(R9) + ORR R13, R10, R10 + + // exhausted += (br0.off < 4) + CMP $0x04, R12 + CSINC HS, R2, R2, R16 + BFI $0, R16, $8, R2 + +skip_fill0: + // val0 := br0.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v0 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br0.advance(uint8(v0.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val1 := br0.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v1 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br0.advance(uint8(v1.entry) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + REVW R0, R0 + + // val2 := br0.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v2 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br0.advance(uint8(v2.entry) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val3 := br0.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v3 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br0.advance(uint8(v3.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + REVW R0, R0 + + // these four writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + // out[id * dstEvery + 3] = uint8(v2.entry >> 8) + // out[id * dstEvery + 4] = uint8(v3.entry >> 8) + MOVW R0, (R3) + + // update the bitreader structure + MOVD R10, 32(R9) + MOVB R11, 40(R9) + + // br1.fillFast32() + MOVD 80(R9), R10 + MOVBU 88(R9), R11 + CMP $0x20, R11 + BLS skip_fill1 + MOVD 72(R9), R12 + SUB $0x20, R11, R11 + SUB $0x04, R12, R12 + MOVD 48(R9), R13 + + // b.value |= uint64(low) << (b.bitsRead & 63) + ADD R13, R12, R15 + MOVWU (R15), R13 + MOVD R11, R1 + LSL R1, R13, R13 + MOVD R12, 72(R9) + ORR R13, R10, R10 + + // exhausted += (br1.off < 4) + CMP $0x04, R12 + CSINC HS, R2, R2, R16 + BFI $0, R16, $8, R2 + +skip_fill1: + // val0 := br1.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v0 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br1.advance(uint8(v0.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val1 := br1.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v1 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br1.advance(uint8(v1.entry) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + REVW R0, R0 + + // val2 := br1.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v2 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br1.advance(uint8(v2.entry) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val3 := br1.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v3 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br1.advance(uint8(v3.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + REVW R0, R0 + + // these four writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + // out[id * dstEvery + 3] = uint8(v2.entry >> 8) + // out[id * dstEvery + 4] = uint8(v3.entry >> 8) + ADD R7, R3, R15 + MOVW R0, (R15) + + // update the bitreader structure + MOVD R10, 80(R9) + MOVB R11, 88(R9) + + // br2.fillFast32() + MOVD 128(R9), R10 + MOVBU 136(R9), R11 + CMP $0x20, R11 + BLS skip_fill2 + MOVD 120(R9), R12 + SUB $0x20, R11, R11 + SUB $0x04, R12, R12 + MOVD 96(R9), R13 + + // b.value |= uint64(low) << (b.bitsRead & 63) + ADD R13, R12, R15 + MOVWU (R15), R13 + MOVD R11, R1 + LSL R1, R13, R13 + MOVD R12, 120(R9) + ORR R13, R10, R10 + + // exhausted += (br2.off < 4) + CMP $0x04, R12 + CSINC HS, R2, R2, R16 + BFI $0, R16, $8, R2 + +skip_fill2: + // val0 := br2.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v0 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br2.advance(uint8(v0.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val1 := br2.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v1 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br2.advance(uint8(v1.entry) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + REVW R0, R0 + + // val2 := br2.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v2 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br2.advance(uint8(v2.entry) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val3 := br2.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v3 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br2.advance(uint8(v3.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + REVW R0, R0 + + // these four writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + // out[id * dstEvery + 3] = uint8(v2.entry >> 8) + // out[id * dstEvery + 4] = uint8(v3.entry >> 8) + ADD R7<<1, R3, R15 + MOVW R0, (R15) + + // update the bitreader structure + MOVD R10, 128(R9) + MOVB R11, 136(R9) + + // br3.fillFast32() + MOVD 176(R9), R10 + MOVBU 184(R9), R11 + CMP $0x20, R11 + BLS skip_fill3 + MOVD 168(R9), R12 + SUB $0x20, R11, R11 + SUB $0x04, R12, R12 + MOVD 144(R9), R13 + + // b.value |= uint64(low) << (b.bitsRead & 63) + ADD R13, R12, R15 + MOVWU (R15), R13 + MOVD R11, R1 + LSL R1, R13, R13 + MOVD R12, 168(R9) + ORR R13, R10, R10 + + // exhausted += (br3.off < 4) + CMP $0x04, R12 + CSINC HS, R2, R2, R16 + BFI $0, R16, $8, R2 + +skip_fill3: + // val0 := br3.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v0 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br3.advance(uint8(v0.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val1 := br3.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v1 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br3.advance(uint8(v1.entry) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + REVW R0, R0 + + // val2 := br3.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v2 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br3.advance(uint8(v2.entry) + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + + // val3 := br3.peekTopBits(peekBits) + MOVD R10, R12 + MOVD R6, R1 + LSR R1, R12, R12 + + // v3 := table[val0&mask] + ADD R12<<1, R8, R15 + MOVHU (R15), R1 + + // br3.advance(uint8(v3.entry) + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + LSL R1, R10, R10 + ADD R1, R11, R15 + BFI $0, R15, $8, R11 + REVW R0, R0 + + // these four writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + // out[id * dstEvery + 3] = uint8(v2.entry >> 8) + // out[id * dstEvery + 4] = uint8(v3.entry >> 8) + ADD R7<<1, R7, R1 + ADD R1, R3, R15 + MOVW R0, (R15) + + // update the bitreader structure + MOVD R10, 176(R9) + MOVB R11, 184(R9) + ADD $0x04, R3, R3 + AND $0xff, R2, R15 + AND $0xff, R2, R16 + TST R16, R15 + BEQ main_loop + MOVD ctx+0(FP), R0 + MOVD 16(R0), R16 + SUB R16, R3, R3 + LSL $0x02, R3, R3 + MOVD R3, 40(R0) + RET + +// func decompress1x_main_loop_amd64(ctx *decompress1xContext) +TEXT ·decompress1x_main_loop_arm64(SB), $0-8 + MOVD ctx+0(FP), R1 + MOVD 16(R1), R2 + MOVD 24(R1), R3 + CMP $0x04, R3 + BLO error_max_decoded_size_exceeded + ADD R3, R2, R3 + MOVD (R1), R5 + MOVD (R5), R7 + MOVD 24(R5), R8 + MOVD 32(R5), R9 + MOVBU 40(R5), R10 + MOVD 32(R1), R5 + MOVBU 8(R1), R6 + JMP loop_condition + +main_loop: + // Check if we have room for 4 bytes in the output buffer + ADD $4, R2, R1 + CMP R3, R1 + BGE error_max_decoded_size_exceeded + + // Decode 4 values + CMP $0x20, R10 + BLT bitReader_fillFast_1_end + SUB $0x20, R10, R10 + SUB $0x04, R8, R8 + ADD R8, R7, R15 + MOVWU (R15), R11 + MOVD R10, R1 + LSL R1, R11, R11 + ORR R11, R9, R9 + +bitReader_fillFast_1_end: + MOVD R6, R1 + MOVD R9, R11 + LSR R1, R11, R11 + ADD R11<<1, R5, R15 + MOVHU (R15), R1 + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + MOVBU R1, R1 + ADD R1, R10, R10 + LSL R1, R9, R9 + MOVD R6, R1 + MOVD R9, R11 + LSR R1, R11, R11 + ADD R11<<1, R5, R15 + MOVHU (R15), R1 + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + MOVBU R1, R1 + ADD R1, R10, R10 + LSL R1, R9, R9 + REVW R0, R0 + CMP $0x20, R10 + BLT bitReader_fillFast_2_end + SUB $0x20, R10, R10 + SUB $0x04, R8, R8 + ADD R8, R7, R15 + MOVWU (R15), R11 + MOVD R10, R1 + LSL R1, R11, R11 + ORR R11, R9, R9 + +bitReader_fillFast_2_end: + MOVD R6, R1 + MOVD R9, R11 + LSR R1, R11, R11 + ADD R11<<1, R5, R15 + MOVHU (R15), R1 + UBFX $8, R1, $8, R16 + BFI $8, R16, $8, R0 + MOVBU R1, R1 + ADD R1, R10, R10 + LSL R1, R9, R9 + MOVD R6, R1 + MOVD R9, R11 + LSR R1, R11, R11 + ADD R11<<1, R5, R15 + MOVHU (R15), R1 + UBFX $8, R1, $8, R16 + BFI $0, R16, $8, R0 + MOVBU R1, R1 + ADD R1, R10, R10 + LSL R1, R9, R9 + REVW R0, R0 + + // Store the decoded values + MOVW R0, (R2) + ADD $0x04, R2, R2 + +loop_condition: + CMP $0x08, R8 + BGE main_loop + + // Update ctx structure + MOVD ctx+0(FP), R0 + MOVD 16(R0), R16 + SUB R16, R2, R2 + MOVD R2, 40(R0) + MOVD (R0), R0 + MOVD R8, 24(R0) + MOVD R9, 32(R0) + MOVB R10, 40(R0) + RET + + // Report error +error_max_decoded_size_exceeded: + MOVD ctx+0(FP), R0 + MOVD $-1, R1 + MOVD R1, 40(R0) + RET + +// skipped decompress1x_main_loop_bmi2 (generic twin preferred on arm64) diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_asm.go b/vendor/github.com/klauspost/compress/huff0/decompress_asm.go new file mode 100644 index 0000000000..4854dd4365 --- /dev/null +++ b/vendor/github.com/klauspost/compress/huff0/decompress_asm.go @@ -0,0 +1,193 @@ +//go:build (amd64 || arm64) && !appengine && !noasm && gc + +// This file contains the specialisation of Decoder.Decompress4X +// and Decoder.Decompress1X that use an asm implementation of their main loops. +// The asm function stubs and any per-arch dispatch live in decompress_amd64.go +// and decompress_arm64.go. +package huff0 + +import ( + "errors" + "fmt" +) + +// fallback8BitSize is the size where using Go version is faster. +const fallback8BitSize = 800 + +type decompress4xContext struct { + pbr *[4]bitReaderShifted + peekBits uint8 + out *byte + dstEvery int + tbl *dEntrySingle + decoded int + limit *byte +} + +// Decompress4X will decompress a 4X encoded stream. +// The length of the supplied input must match the end of a block exactly. +// The *capacity* of the dst slice must match the destination size of +// the uncompressed data exactly. +func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { + if len(d.dt.single) == 0 { + return nil, errors.New("no table loaded") + } + if len(src) < 6+(4*1) { + return nil, errors.New("input too small") + } + + use8BitTables := d.actualTableLog <= 8 + if cap(dst) < fallback8BitSize && use8BitTables { + return d.decompress4X8bit(dst, src) + } + + var br [4]bitReaderShifted + // Decode "jump table" + start := 6 + for i := range 3 { + length := int(src[i*2]) | (int(src[i*2+1]) << 8) + if start+length >= len(src) { + return nil, errors.New("truncated input (or invalid offset)") + } + err := br[i].init(src[start : start+length]) + if err != nil { + return nil, err + } + start += length + } + err := br[3].init(src[start:]) + if err != nil { + return nil, err + } + + // destination, offset to match first output + dstSize := cap(dst) + dst = dst[:dstSize] + out := dst + dstEvery := (dstSize + 3) / 4 + + const tlSize = 1 << tableLogMax + const tlMask = tlSize - 1 + single := d.dt.single[:tlSize] + + var decoded int + + if len(out) > 4*4 && !(br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4) { + ctx := decompress4xContext{ + pbr: &br, + peekBits: uint8((64 - d.actualTableLog) & 63), // see: bitReaderShifted.peekBitsFast() + out: &out[0], + dstEvery: dstEvery, + tbl: &single[0], + limit: &out[dstEvery-4], // Always stop decoding when first buffer gets here to avoid writing OOB on last. + } + if use8BitTables { + decompress4x_8b_main_loop_asm(&ctx) + } else { + decompress4x_main_loop_asm(&ctx) + } + + decoded = ctx.decoded + out = out[decoded/4:] + } + + // Decode remaining. + remainBytes := dstEvery - (decoded / 4) + for i := range br { + offset := dstEvery * i + endsAt := min(offset+remainBytes, len(out)) + br := &br[i] + bitsLeft := br.remaining() + for bitsLeft > 0 { + br.fill() + if offset >= endsAt { + return nil, errors.New("corruption detected: stream overrun 4") + } + + // Read value and increment offset. + val := br.peekBitsFast(d.actualTableLog) + v := single[val&tlMask].entry + nBits := uint8(v) + br.advance(nBits) + bitsLeft -= uint(nBits) + out[offset] = uint8(v >> 8) + offset++ + } + if offset != endsAt { + return nil, fmt.Errorf("corruption detected: short output block %d, end %d != %d", i, offset, endsAt) + } + decoded += offset - dstEvery*i + err = br.close() + if err != nil { + return nil, err + } + } + if dstSize != decoded { + return nil, errors.New("corruption detected: short output block") + } + return dst, nil +} + +type decompress1xContext struct { + pbr *bitReaderShifted + peekBits uint8 + out *byte + outCap int + tbl *dEntrySingle + decoded int +} + +// Error reported by asm implementations +const error_max_decoded_size_exeeded = -1 + +// Decompress1X will decompress a 1X encoded stream. +// The cap of the output buffer will be the maximum decompressed size. +// The length of the supplied input must match the end of a block exactly. +func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) { + if len(d.dt.single) == 0 { + return nil, errors.New("no table loaded") + } + var br bitReaderShifted + err := br.init(src) + if err != nil { + return dst, err + } + maxDecodedSize := cap(dst) + dst = dst[:maxDecodedSize] + + const tlSize = 1 << tableLogMax + const tlMask = tlSize - 1 + + if maxDecodedSize >= 4 { + ctx := decompress1xContext{ + pbr: &br, + out: &dst[0], + outCap: maxDecodedSize, + peekBits: uint8((64 - d.actualTableLog) & 63), // see: bitReaderShifted.peekBitsFast() + tbl: &d.dt.single[0], + } + + decompress1x_main_loop_asm(&ctx) + if ctx.decoded == error_max_decoded_size_exeeded { + return nil, ErrMaxDecodedSizeExceeded + } + + dst = dst[:ctx.decoded] + } + + // br < 8, so uint8 is fine + bitsLeft := uint8(br.off)*8 + 64 - br.bitsRead + for bitsLeft > 0 { + br.fill() + if len(dst) >= maxDecodedSize { + br.close() + return nil, ErrMaxDecodedSizeExceeded + } + v := d.dt.single[br.peekBitsFast(d.actualTableLog)&tlMask] + nBits := uint8(v.entry) + br.advance(nBits) + bitsLeft -= nBits + dst = append(dst, uint8(v.entry>>8)) + } + return dst, br.close() +} diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_generic.go b/vendor/github.com/klauspost/compress/huff0/decompress_generic.go index 6103923222..e1bca5c686 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_generic.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress_generic.go @@ -1,4 +1,4 @@ -//go:build !amd64 || appengine || !gc || noasm +//go:build (!amd64 && !arm64) || appengine || !gc || noasm // This file contains a generic implementation of Decoder.Decompress4X. package huff0 diff --git a/vendor/github.com/klauspost/compress/zstd/blockdec.go b/vendor/github.com/klauspost/compress/zstd/blockdec.go index 2329e996f8..51f9da0387 100644 --- a/vendor/github.com/klauspost/compress/zstd/blockdec.go +++ b/vendor/github.com/klauspost/compress/zstd/blockdec.go @@ -400,8 +400,9 @@ func (b *blockDec) decodeLiterals(in []byte, hist *history) (remain []byte, err } } var err error - // Use our out buffer. - huff.MaxDecodedSize = litRegenSize + // Decoder.Decompress* uses cap(dst) for the size limit. Do not write + // MaxDecodedSize on hist.huffTree: with a trained dictionary that + // pointer aliases the shared dict.litEnc and concurrent DecodeAll races. if fourStreams { literals, err = huff.Decoder().Decompress4X(b.literalBuf[:0:litRegenSize], literals) } else { @@ -673,10 +674,6 @@ func (b *blockDec) executeSequences(hist *history) error { hbytes := hist.b if len(hbytes) > hist.windowSize { hbytes = hbytes[len(hbytes)-hist.windowSize:] - // We do not need history anymore. - if hist.dict != nil { - hist.dict.content = nil - } } hist.decoders.windowSize = hist.windowSize hist.decoders.out = b.dst[:0] diff --git a/vendor/github.com/klauspost/compress/zstd/dict.go b/vendor/github.com/klauspost/compress/zstd/dict.go index 4f1c4938cd..67f40e7914 100644 --- a/vendor/github.com/klauspost/compress/zstd/dict.go +++ b/vendor/github.com/klauspost/compress/zstd/dict.go @@ -296,40 +296,81 @@ func BuildDict(o BuildDictOptions) ([]byte, error) { if offset > 3 { newOffsets[offset-3]++ } else { - newOffsets[uint32(o.Offsets[offset-1])]++ + // Repeat codes reference the training Offsets. Skip unset + // (zero) entries so they are not ranked as real offsets. + prev := o.Offsets[offset-1] + if prev > 0 { + newOffsets[uint32(prev)]++ + } } } } // Find most used offsets. var sortedOffsets []uint32 for k := range newOffsets { + if k == 0 { + continue + } sortedOffsets = append(sortedOffsets, k) } sort.Slice(sortedOffsets, func(i, j int) bool { a, b := sortedOffsets[i], sortedOffsets[j] - if a == b { + ca, cb := newOffsets[a], newOffsets[b] + if ca == cb { // Prefer the longer offset - return sortedOffsets[i] > sortedOffsets[j] + return a > b } - return newOffsets[sortedOffsets[i]] > newOffsets[sortedOffsets[j]] + return ca > cb }) - if len(sortedOffsets) > 3 { - if debug { - print("Offsets:") - for i, v := range sortedOffsets { - if i > 20 { - break - } - printf("[%d: %d],", v, newOffsets[v]) + if debug { + print("Offsets:") + for i, v := range sortedOffsets { + if i > 20 { + break } - println("") + printf("[%d: %d],", v, newOffsets[v]) + } + println("") + } + // Dictionary recent-offsets must be three positive values within the + // history. Ranked matches may be fewer (or empty when only unset + // repeat codes were seen), so fill remaining slots with defaults. + used := make(map[int]bool, 3) + var finalOffsets [3]int + nOff := 0 + for _, v := range sortedOffsets { + iv := int(v) + if iv <= 0 || iv > len(hist) || used[iv] { + continue + } + finalOffsets[nOff] = iv + used[iv] = true + nOff++ + if nOff == 3 { + break } - - sortedOffsets = sortedOffsets[:3] } - for i, v := range sortedOffsets { - o.Offsets[i] = int(v) + for _, def := range []int{1, 4, 8} { + if nOff == 3 { + break + } + if def <= len(hist) && !used[def] { + finalOffsets[nOff] = def + used[def] = true + nOff++ + } + } + for def := 1; nOff < 3 && def <= len(hist); def++ { + if !used[def] { + finalOffsets[nOff] = def + used[def] = true + nOff++ + } + } + if nOff < 3 { + return nil, fmt.Errorf("could not determine 3 valid dictionary offsets (history size %d)", len(hist)) } + o.Offsets = finalOffsets if debug { println("New repeat offsets", o.Offsets) } @@ -337,6 +378,9 @@ func BuildDict(o BuildDictOptions) ([]byte, error) { if nUsed == 0 || seqs == 0 { return nil, fmt.Errorf("%d blocks, %d sequences found", nUsed, seqs) } + if litTotal == 0 { + return nil, errors.New("0 literals found") + } if debug { println("Sequences:", seqs, "Blocks:", nUsed, "Literals:", litTotal) } @@ -517,11 +561,10 @@ func BuildDict(o BuildDictOptions) ([]byte, error) { out.Write(binary.LittleEndian.AppendUint32(nil, uint32(o.Offsets[1]))) out.Write(binary.LittleEndian.AppendUint32(nil, uint32(o.Offsets[2]))) out.Write(hist) + if _, err := loadDict(out.Bytes()); err != nil { + return nil, fmt.Errorf("built dictionary failed validation: %w", err) + } if debug { - _, err := loadDict(out.Bytes()) - if err != nil { - panic(err) - } i, err := InspectDictionary(out.Bytes()) if err != nil { panic(err) diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s index 3fc381c7a7..4d3188ff49 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s @@ -2081,7 +2081,7 @@ sequenceDecs_decodeSync_amd64_match_len_ofs_ok: MOVQ 16(SP), R13 // Check if we have enough space in s.out - LEAQ (AX)(R13*1), R14 + LEAQ 16(AX)(R13*1), R14 ADDQ R10, R14 CMPQ R14, 32(SP) JA error_not_enough_space @@ -2601,7 +2601,7 @@ sequenceDecs_decodeSync_bmi2_match_len_ofs_ok: MOVQ 16(SP), R13 // Check if we have enough space in s.out - LEAQ (CX)(R13*1), R14 + LEAQ 16(CX)(R13*1), R14 ADDQ R9, R14 CMPQ R14, 32(SP) JA error_not_enough_space diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.s b/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.s index a468e5fc2c..6f54ef21ba 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.s +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_arm64.s @@ -12,7 +12,7 @@ TEXT ·sequenceDecs_decode_arm64(SB), $8-32 MOVD (R1), R0 MOVD 32(R1), R5 ADD R5, R0, R0 - MOVD R0, (RSP) + MOVD R0, 8(RSP) MOVD ctx+16(FP), R0 MOVD 72(R0), R6 MOVD 80(R0), R7 @@ -24,7 +24,7 @@ TEXT ·sequenceDecs_decode_arm64(SB), $8-32 MOVD 160(R0), R12 sequenceDecs_decode_amd64_main_loop: - MOVD (RSP), R13 + MOVD 8(RSP), R13 // Fill bitreader to have enough for the offset and match length. CMP $0x08, R5 @@ -60,7 +60,8 @@ sequenceDecs_decode_amd64_fill_end: MOVD R3, R1 MOVD R2, R14 LSL R1, R14, R14 - UBFX $8, R0, $8, R1 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decode_amd64_of_update_zero @@ -81,7 +82,8 @@ sequenceDecs_decode_amd64_of_update_zero: MOVD R3, R1 MOVD R2, R14 LSL R1, R14, R14 - UBFX $8, R0, $8, R1 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decode_amd64_ml_update_zero @@ -131,7 +133,8 @@ sequenceDecs_decode_amd64_fill_2_end: MOVD R3, R1 MOVD R2, R14 LSL R1, R14, R14 - UBFX $8, R0, $8, R1 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decode_amd64_ll_update_zero @@ -148,7 +151,7 @@ sequenceDecs_decode_amd64_ll_update_zero: MOVD R0, (R9) // Fill bitreader for state updates - MOVD R13, (RSP) + MOVD R13, 8(RSP) MOVD R8, R0 LSR $0x08, R0, R0 MOVBU R0, R0 @@ -166,7 +169,7 @@ sequenceDecs_decode_amd64_ll_update_zero: NEG R1, R16 ROR R16, R14, R14 MOVD $0x00000001, R4 - MOVB R13, R1 + BFI $0, R13, $8, R1 LSLW R1, R4, R4 SUBW $1, R4, R4 AND R4, R14, R14 @@ -187,7 +190,7 @@ sequenceDecs_decode_amd64_ll_update_zero: NEG R1, R16 ROR R16, R14, R14 MOVD $0x00000001, R4 - MOVB R13, R1 + BFI $0, R13, $8, R1 LSLW R1, R4, R4 SUBW $1, R4, R4 AND R4, R14, R14 @@ -208,7 +211,7 @@ sequenceDecs_decode_amd64_ll_update_zero: NEG R1, R16 ROR R16, R14, R14 MOVD $0x00000001, R4 - MOVB R13, R1 + BFI $0, R13, $8, R1 LSLW R1, R4, R4 SUBW $1, R4, R4 AND R4, R14, R14 @@ -360,7 +363,7 @@ TEXT ·sequenceDecs_decode_56_arm64(SB), $8-32 MOVD (R1), R0 MOVD 32(R1), R5 ADD R5, R0, R0 - MOVD R0, (RSP) + MOVD R0, 8(RSP) MOVD ctx+16(FP), R0 MOVD 72(R0), R6 MOVD 80(R0), R7 @@ -372,7 +375,7 @@ TEXT ·sequenceDecs_decode_56_arm64(SB), $8-32 MOVD 160(R0), R12 sequenceDecs_decode_56_amd64_main_loop: - MOVD (RSP), R13 + MOVD 8(RSP), R13 // Fill bitreader to have enough for the offset and match length. CMP $0x08, R5 @@ -408,7 +411,8 @@ sequenceDecs_decode_56_amd64_fill_end: MOVD R3, R1 MOVD R2, R14 LSL R1, R14, R14 - UBFX $8, R0, $8, R1 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decode_56_amd64_of_update_zero @@ -429,7 +433,8 @@ sequenceDecs_decode_56_amd64_of_update_zero: MOVD R3, R1 MOVD R2, R14 LSL R1, R14, R14 - UBFX $8, R0, $8, R1 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decode_56_amd64_ml_update_zero @@ -450,7 +455,8 @@ sequenceDecs_decode_56_amd64_ml_update_zero: MOVD R3, R1 MOVD R2, R14 LSL R1, R14, R14 - UBFX $8, R0, $8, R1 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decode_56_amd64_ll_update_zero @@ -467,7 +473,7 @@ sequenceDecs_decode_56_amd64_ll_update_zero: MOVD R0, (R9) // Fill bitreader for state updates - MOVD R13, (RSP) + MOVD R13, 8(RSP) MOVD R8, R0 LSR $0x08, R0, R0 MOVBU R0, R0 @@ -485,7 +491,7 @@ sequenceDecs_decode_56_amd64_ll_update_zero: NEG R1, R16 ROR R16, R14, R14 MOVD $0x00000001, R4 - MOVB R13, R1 + BFI $0, R13, $8, R1 LSLW R1, R4, R4 SUBW $1, R4, R4 AND R4, R14, R14 @@ -506,7 +512,7 @@ sequenceDecs_decode_56_amd64_ll_update_zero: NEG R1, R16 ROR R16, R14, R14 MOVD $0x00000001, R4 - MOVB R13, R1 + BFI $0, R13, $8, R1 LSLW R1, R4, R4 SUBW $1, R4, R4 AND R4, R14, R14 @@ -527,7 +533,7 @@ sequenceDecs_decode_56_amd64_ll_update_zero: NEG R1, R16 ROR R16, R14, R14 MOVD $0x00000001, R4 - MOVB R13, R1 + BFI $0, R13, $8, R1 LSLW R1, R4, R4 SUBW $1, R4, R4 AND R4, R14, R14 @@ -766,13 +772,14 @@ copy_4_small: JMP copy_4_move_8through16 copy_4_move_3: - MOVH (R13), R10 - MOVB 2(R13), R11 - MOVH R10, (R3) - MOVB R11, 2(R3) - ADD R12, R13, R13 - ADD R12, R3, R3 - JMP copy_4_end + MOVHU (R13), R10 + MOVBU 2(R13), R16 + BFI $0, R16, $8, R11 + MOVH R10, (R3) + MOVB R11, 2(R3) + ADD R12, R13, R13 + ADD R12, R3, R3 + JMP copy_4_end copy_4_move_4through7: MOVWU (R13), R10 @@ -834,24 +841,27 @@ copy_5_small: JMP copy_5_move_8through16 copy_5_move_1or2: - MOVB (R13), R14 - ADD R10, R13, R15 - MOVB -1(R15), R4 - MOVB R14, (R3) - ADD R10, R3, R15 - MOVB R4, -1(R15) - ADD R10, R13, R13 - ADD R10, R3, R3 - JMP copy_5_end + MOVBU (R13), R16 + BFI $0, R16, $8, R14 + ADD R10, R13, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R4 + MOVB R14, (R3) + ADD R10, R3, R15 + MOVB R4, -1(R15) + ADD R10, R13, R13 + ADD R10, R3, R3 + JMP copy_5_end copy_5_move_3: - MOVH (R13), R14 - MOVB 2(R13), R4 - MOVH R14, (R3) - MOVB R4, 2(R3) - ADD R10, R13, R13 - ADD R10, R3, R3 - JMP copy_5_end + MOVHU (R13), R14 + MOVBU 2(R13), R16 + BFI $0, R16, $8, R4 + MOVH R14, (R3) + MOVB R4, 2(R3) + ADD R10, R13, R13 + ADD R10, R3, R3 + JMP copy_5_end copy_5_move_4through7: MOVWU (R13), R14 @@ -906,12 +916,13 @@ copy_overlapping_match: ADD R12, R6, R6 copy_slow_3: - MOVB (R10), R11 - MOVB R11, (R3) - ADD $1, R10, R10 - ADD $1, R3, R3 - SUBS $1, R12, R12 - BNE copy_slow_3 + MOVBU (R10), R16 + BFI $0, R16, $8, R11 + MOVB R11, (R3) + ADD $1, R10, R10 + ADD $1, R3, R3 + SUBS $1, R12, R12 + BNE copy_slow_3 handle_loop: ADD $0x18, R0, R0 @@ -1016,24 +1027,27 @@ copy_1_small: JMP copy_1_move_8through16 copy_1_move_1or2: - MOVB (R5), R13 - ADD R10, R5, R15 - MOVB -1(R15), R14 - MOVB R13, (R3) - ADD R10, R3, R15 - MOVB R14, -1(R15) - ADD R10, R5, R5 - ADD R10, R3, R3 - JMP copy_1_end + MOVBU (R5), R16 + BFI $0, R16, $8, R13 + ADD R10, R5, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R14 + MOVB R13, (R3) + ADD R10, R3, R15 + MOVB R14, -1(R15) + ADD R10, R5, R5 + ADD R10, R3, R3 + JMP copy_1_end copy_1_move_3: - MOVH (R5), R13 - MOVB 2(R5), R14 - MOVH R13, (R3) - MOVB R14, 2(R3) - ADD R10, R5, R5 - ADD R10, R3, R3 - JMP copy_1_end + MOVHU (R5), R13 + MOVBU 2(R5), R16 + BFI $0, R16, $8, R14 + MOVH R13, (R3) + MOVB R14, 2(R3) + ADD R10, R5, R5 + ADD R10, R3, R3 + JMP copy_1_end copy_1_move_4through7: MOVWU (R5), R13 @@ -1104,13 +1118,14 @@ copy_4_small: JMP copy_4_move_8through16 copy_4_move_3: - MOVH (R13), R10 - MOVB 2(R13), R11 - MOVH R10, (R3) - MOVB R11, 2(R3) - ADD R12, R13, R13 - ADD R12, R3, R3 - JMP copy_4_end + MOVHU (R13), R10 + MOVBU 2(R13), R16 + BFI $0, R16, $8, R11 + MOVH R10, (R3) + MOVB R11, 2(R3) + ADD R12, R13, R13 + ADD R12, R3, R3 + JMP copy_4_end copy_4_move_4through7: MOVWU (R13), R10 @@ -1172,24 +1187,27 @@ copy_5_small: JMP copy_5_move_8through16 copy_5_move_1or2: - MOVB (R13), R14 - ADD R10, R13, R15 - MOVB -1(R15), R4 - MOVB R14, (R3) - ADD R10, R3, R15 - MOVB R4, -1(R15) - ADD R10, R13, R13 - ADD R10, R3, R3 - JMP copy_5_end + MOVBU (R13), R16 + BFI $0, R16, $8, R14 + ADD R10, R13, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R4 + MOVB R14, (R3) + ADD R10, R3, R15 + MOVB R4, -1(R15) + ADD R10, R13, R13 + ADD R10, R3, R3 + JMP copy_5_end copy_5_move_3: - MOVH (R13), R14 - MOVB 2(R13), R4 - MOVH R14, (R3) - MOVB R4, 2(R3) - ADD R10, R13, R13 - ADD R10, R3, R3 - JMP copy_5_end + MOVHU (R13), R14 + MOVBU 2(R13), R16 + BFI $0, R16, $8, R4 + MOVH R14, (R3) + MOVB R4, 2(R3) + ADD R10, R13, R13 + ADD R10, R3, R3 + JMP copy_5_end copy_5_move_4through7: MOVWU (R13), R14 @@ -1257,24 +1275,27 @@ copy_2_small: JMP copy_2_move_8through16 copy_2_move_1or2: - MOVB (R10), R11 - ADD R12, R10, R15 - MOVB -1(R15), R13 - MOVB R11, (R3) - ADD R12, R3, R15 - MOVB R13, -1(R15) - ADD R12, R10, R10 - ADD R12, R3, R3 - JMP copy_2_end + MOVBU (R10), R16 + BFI $0, R16, $8, R11 + ADD R12, R10, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R13 + MOVB R11, (R3) + ADD R12, R3, R15 + MOVB R13, -1(R15) + ADD R12, R10, R10 + ADD R12, R3, R3 + JMP copy_2_end copy_2_move_3: - MOVH (R10), R11 - MOVB 2(R10), R13 - MOVH R11, (R3) - MOVB R13, 2(R3) - ADD R12, R10, R10 - ADD R12, R3, R3 - JMP copy_2_end + MOVHU (R10), R11 + MOVBU 2(R10), R16 + BFI $0, R16, $8, R13 + MOVH R11, (R3) + MOVB R13, 2(R3) + ADD R12, R10, R10 + ADD R12, R3, R3 + JMP copy_2_end copy_2_move_4through7: MOVWU (R10), R11 @@ -1305,12 +1326,13 @@ copy_overlapping_match: ADD R12, R6, R6 copy_slow_3: - MOVB (R10), R11 - MOVB R11, (R3) - ADD $1, R10, R10 - ADD $1, R3, R3 - SUBS $1, R12, R12 - BNE copy_slow_3 + MOVBU (R10), R16 + BFI $0, R16, $8, R11 + MOVB R11, (R3) + ADD $1, R10, R10 + ADD $1, R3, R3 + SUBS $1, R12, R12 + BNE copy_slow_3 handle_loop: ADD $0x18, R0, R0 @@ -1361,41 +1383,41 @@ TEXT ·sequenceDecs_decodeSync_arm64(SB), $64-32 MOVD (R1), R0 MOVD 32(R1), R5 ADD R5, R0, R0 - MOVD R0, (RSP) + MOVD R0, 8(RSP) MOVD ctx+16(FP), R0 MOVD 72(R0), R6 MOVD 80(R0), R7 MOVD 88(R0), R8 MOVD $0, R1 - MOVD R1, 8(RSP) MOVD R1, 16(RSP) MOVD R1, 24(RSP) + MOVD R1, 32(RSP) MOVD 112(R0), R9 MOVD 128(R0), R1 - MOVD R1, 32(RSP) + MOVD R1, 40(RSP) MOVD 144(R0), R10 MOVD 136(R0), R11 MOVD 200(R0), R1 - MOVD R1, 56(RSP) + MOVD R1, 64(RSP) MOVD 176(R0), R1 - MOVD R1, 48(RSP) + MOVD R1, 56(RSP) MOVD 184(R0), R0 - MOVD R0, 40(RSP) - MOVD 40(RSP), R0 - MOVD 48(RSP), R16 + MOVD R0, 48(RSP) + MOVD 48(RSP), R0 + MOVD 56(RSP), R16 ADD R0, R16, R16 - MOVD R16, 48(RSP) + MOVD R16, 56(RSP) // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) - MOVD 32(RSP), R16 + MOVD 40(RSP), R16 ADD R9, R16, R16 - MOVD R16, 32(RSP) + MOVD R16, 40(RSP) // outBase += outPosition ADD R11, R9, R9 sequenceDecs_decodeSync_amd64_main_loop: - MOVD (RSP), R12 + MOVD 8(RSP), R12 // Fill bitreader to have enough for the offset and match length. CMP $0x08, R5 @@ -1431,7 +1453,8 @@ sequenceDecs_decodeSync_amd64_fill_end: MOVD R3, R1 MOVD R2, R13 LSL R1, R13, R13 - UBFX $8, R0, $8, R1 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decodeSync_amd64_of_update_zero @@ -1445,14 +1468,15 @@ sequenceDecs_decodeSync_amd64_fill_end: ADD R13, R0, R0 sequenceDecs_decodeSync_amd64_of_update_zero: - MOVD R0, 8(RSP) + MOVD R0, 16(RSP) // Update match length MOVD R7, R0 MOVD R3, R1 MOVD R2, R13 LSL R1, R13, R13 - UBFX $8, R0, $8, R1 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decodeSync_amd64_ml_update_zero @@ -1466,7 +1490,7 @@ sequenceDecs_decodeSync_amd64_of_update_zero: ADD R13, R0, R0 sequenceDecs_decodeSync_amd64_ml_update_zero: - MOVD R0, 16(RSP) + MOVD R0, 24(RSP) // Fill bitreader to have enough for the remaining CMP $0x08, R5 @@ -1502,7 +1526,8 @@ sequenceDecs_decodeSync_amd64_fill_2_end: MOVD R3, R1 MOVD R2, R13 LSL R1, R13, R13 - UBFX $8, R0, $8, R1 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decodeSync_amd64_ll_update_zero @@ -1516,10 +1541,10 @@ sequenceDecs_decodeSync_amd64_fill_2_end: ADD R13, R0, R0 sequenceDecs_decodeSync_amd64_ll_update_zero: - MOVD R0, 24(RSP) + MOVD R0, 32(RSP) // Fill bitreader for state updates - MOVD R12, (RSP) + MOVD R12, 8(RSP) MOVD R8, R0 LSR $0x08, R0, R0 MOVBU R0, R0 @@ -1537,7 +1562,7 @@ sequenceDecs_decodeSync_amd64_ll_update_zero: NEG R1, R16 ROR R16, R13, R13 MOVD $0x00000001, R14 - MOVB R12, R1 + BFI $0, R12, $8, R1 LSLW R1, R14, R14 SUBW $1, R14, R14 AND R14, R13, R13 @@ -1558,7 +1583,7 @@ sequenceDecs_decodeSync_amd64_ll_update_zero: NEG R1, R16 ROR R16, R13, R13 MOVD $0x00000001, R14 - MOVB R12, R1 + BFI $0, R12, $8, R1 LSLW R1, R14, R14 SUBW $1, R14, R14 AND R14, R13, R13 @@ -1579,7 +1604,7 @@ sequenceDecs_decodeSync_amd64_ll_update_zero: NEG R1, R16 ROR R16, R13, R13 MOVD $0x00000001, R14 - MOVB R12, R1 + BFI $0, R12, $8, R1 LSLW R1, R14, R14 SUBW $1, R14, R14 AND R14, R13, R13 @@ -1594,7 +1619,7 @@ sequenceDecs_decodeSync_amd64_ll_update_zero: sequenceDecs_decodeSync_amd64_skip_update: // Adjust offset MOVD s+0(FP), R1 - MOVD 8(RSP), R12 + MOVD 16(RSP), R12 CMP $0x01, R0 BLS sequenceDecs_decodeSync_amd64_adjust_offsetB_1_or_0 ADD $144, R1, R15 @@ -1605,7 +1630,7 @@ sequenceDecs_decodeSync_amd64_skip_update: JMP sequenceDecs_decodeSync_amd64_after_adjust sequenceDecs_decodeSync_amd64_adjust_offsetB_1_or_0: - MOVD 24(RSP), R16 + MOVD 32(RSP), R16 CMP $0x00000000, R16 BNE sequenceDecs_decodeSync_amd64_adjust_offset_maybezero ADD $1, R12, R12 @@ -1643,11 +1668,11 @@ sequenceDecs_decodeSync_amd64_adjust_skip: MOVD R13, R12 sequenceDecs_decodeSync_amd64_after_adjust: - MOVD R12, 8(RSP) + MOVD R12, 16(RSP) // Check values - MOVD 16(RSP), R0 - MOVD 24(RSP), R1 + MOVD 24(RSP), R0 + MOVD 32(RSP), R1 ADD R1, R0, R13 MOVD s+0(FP), R14 MOVD 256(R14), R16 @@ -1666,14 +1691,15 @@ sequenceDecs_decodeSync_amd64_after_adjust: BNE sequenceDecs_decodeSync_amd64_error_match_len_ofs_mismatch sequenceDecs_decodeSync_amd64_match_len_ofs_ok: - MOVD 24(RSP), R0 - MOVD 8(RSP), R1 - MOVD 16(RSP), R12 + MOVD 32(RSP), R0 + MOVD 16(RSP), R1 + MOVD 24(RSP), R12 // Check if we have enough space in s.out ADD R12, R0, R13 + ADD $16, R13, R13 ADD R9, R13, R13 - MOVD 32(RSP), R16 + MOVD 40(RSP), R16 CMP R16, R13 BHI error_not_enough_space @@ -1697,11 +1723,11 @@ copy_1: // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) check_offset: MOVD R11, R0 - MOVD 40(RSP), R16 + MOVD 48(RSP), R16 ADD R16, R0, R0 CMP R0, R1 BGT error_match_off_too_big - MOVD 56(RSP), R16 + MOVD 64(RSP), R16 CMP R16, R1 BGT error_match_off_too_big @@ -1709,7 +1735,7 @@ check_offset: MOVD R1, R0 SUBS R11, R0, R0 BLS copy_match - MOVD 48(RSP), R13 + MOVD 56(RSP), R13 SUB R0, R13, R13 CMP R0, R12 BGT copy_all_from_history @@ -1742,13 +1768,14 @@ copy_4_small: JMP copy_4_move_8through16 copy_4_move_3: - MOVH (R13), R0 - MOVB 2(R13), R1 - MOVH R0, (R9) - MOVB R1, 2(R9) - ADD R12, R13, R13 - ADD R12, R9, R9 - JMP copy_4_end + MOVHU (R13), R0 + MOVBU 2(R13), R16 + BFI $0, R16, $8, R1 + MOVH R0, (R9) + MOVB R1, 2(R9) + ADD R12, R13, R13 + ADD R12, R9, R9 + JMP copy_4_end copy_4_move_4through7: MOVWU (R13), R0 @@ -1807,24 +1834,27 @@ copy_5_small: JMP copy_5_move_8through16 copy_5_move_1or2: - MOVB (R13), R14 - ADD R0, R13, R15 - MOVB -1(R15), R4 - MOVB R14, (R9) - ADD R0, R9, R15 - MOVB R4, -1(R15) - ADD R0, R13, R13 - ADD R0, R9, R9 - JMP copy_5_end + MOVBU (R13), R16 + BFI $0, R16, $8, R14 + ADD R0, R13, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R4 + MOVB R14, (R9) + ADD R0, R9, R15 + MOVB R4, -1(R15) + ADD R0, R13, R13 + ADD R0, R9, R9 + JMP copy_5_end copy_5_move_3: - MOVH (R13), R14 - MOVB 2(R13), R4 - MOVH R14, (R9) - MOVB R4, 2(R9) - ADD R0, R13, R13 - ADD R0, R9, R9 - JMP copy_5_end + MOVHU (R13), R14 + MOVBU 2(R13), R16 + BFI $0, R16, $8, R4 + MOVH R14, (R9) + MOVB R4, 2(R9) + ADD R0, R13, R13 + ADD R0, R9, R9 + JMP copy_5_end copy_5_move_4through7: MOVWU (R13), R14 @@ -1879,12 +1909,13 @@ copy_overlapping_match: ADD R12, R11, R11 copy_slow_3: - MOVB (R0), R1 - MOVB R1, (R9) - ADD $1, R0, R0 - ADD $1, R9, R9 - SUBS $1, R12, R12 - BNE copy_slow_3 + MOVBU (R0), R16 + BFI $0, R16, $8, R1 + MOVB R1, (R9) + ADD $1, R0, R0 + ADD $1, R9, R9 + SUBS $1, R12, R12 + BNE copy_slow_3 handle_loop: MOVD ctx+16(FP), R0 @@ -1913,7 +1944,7 @@ loop_finished: // Return with match length error sequenceDecs_decodeSync_amd64_error_match_len_ofs_mismatch: - MOVD 16(RSP), R0 + MOVD 24(RSP), R0 MOVD ctx+16(FP), R1 MOVD R0, 216(R1) MOVD $0x00000001, R16 @@ -1923,7 +1954,7 @@ sequenceDecs_decodeSync_amd64_error_match_len_ofs_mismatch: // Return with match too long error sequenceDecs_decodeSync_amd64_error_match_len_too_big: MOVD ctx+16(FP), R0 - MOVD 16(RSP), R1 + MOVD 24(RSP), R1 MOVD R1, 216(R0) MOVD $0x00000002, R16 MOVD R16, ret+24(FP) @@ -1932,7 +1963,7 @@ sequenceDecs_decodeSync_amd64_error_match_len_too_big: // Return with match offset too long error error_match_off_too_big: MOVD ctx+16(FP), R0 - MOVD 8(RSP), R1 + MOVD 16(RSP), R1 MOVD R1, 224(R0) MOVD R11, 136(R0) MOVD $0x00000003, R16 @@ -1942,7 +1973,7 @@ error_match_off_too_big: // Return with not enough literals error error_not_enough_literals: MOVD ctx+16(FP), R0 - MOVD 24(RSP), R1 + MOVD 32(RSP), R1 MOVD R1, 208(R0) MOVD $0x00000004, R16 MOVD R16, ret+24(FP) @@ -1957,9 +1988,9 @@ error_overread: // Return with not enough output space error error_not_enough_space: MOVD ctx+16(FP), R0 - MOVD 24(RSP), R1 + MOVD 32(RSP), R1 MOVD R1, 208(R0) - MOVD 16(RSP), R1 + MOVD 24(RSP), R1 MOVD R1, 216(R0) MOVD R11, 136(R0) MOVD $0x00000005, R16 @@ -1977,41 +2008,41 @@ TEXT ·sequenceDecs_decodeSync_safe_arm64(SB), $64-32 MOVD (R1), R0 MOVD 32(R1), R5 ADD R5, R0, R0 - MOVD R0, (RSP) + MOVD R0, 8(RSP) MOVD ctx+16(FP), R0 MOVD 72(R0), R6 MOVD 80(R0), R7 MOVD 88(R0), R8 MOVD $0, R1 - MOVD R1, 8(RSP) MOVD R1, 16(RSP) MOVD R1, 24(RSP) + MOVD R1, 32(RSP) MOVD 112(R0), R9 MOVD 128(R0), R1 - MOVD R1, 32(RSP) + MOVD R1, 40(RSP) MOVD 144(R0), R10 MOVD 136(R0), R11 MOVD 200(R0), R1 - MOVD R1, 56(RSP) + MOVD R1, 64(RSP) MOVD 176(R0), R1 - MOVD R1, 48(RSP) + MOVD R1, 56(RSP) MOVD 184(R0), R0 - MOVD R0, 40(RSP) - MOVD 40(RSP), R0 - MOVD 48(RSP), R16 + MOVD R0, 48(RSP) + MOVD 48(RSP), R0 + MOVD 56(RSP), R16 ADD R0, R16, R16 - MOVD R16, 48(RSP) + MOVD R16, 56(RSP) // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) - MOVD 32(RSP), R16 + MOVD 40(RSP), R16 ADD R9, R16, R16 - MOVD R16, 32(RSP) + MOVD R16, 40(RSP) // outBase += outPosition ADD R11, R9, R9 sequenceDecs_decodeSync_safe_amd64_main_loop: - MOVD (RSP), R12 + MOVD 8(RSP), R12 // Fill bitreader to have enough for the offset and match length. CMP $0x08, R5 @@ -2047,7 +2078,8 @@ sequenceDecs_decodeSync_safe_amd64_fill_end: MOVD R3, R1 MOVD R2, R13 LSL R1, R13, R13 - UBFX $8, R0, $8, R1 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decodeSync_safe_amd64_of_update_zero @@ -2061,14 +2093,15 @@ sequenceDecs_decodeSync_safe_amd64_fill_end: ADD R13, R0, R0 sequenceDecs_decodeSync_safe_amd64_of_update_zero: - MOVD R0, 8(RSP) + MOVD R0, 16(RSP) // Update match length MOVD R7, R0 MOVD R3, R1 MOVD R2, R13 LSL R1, R13, R13 - UBFX $8, R0, $8, R1 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decodeSync_safe_amd64_ml_update_zero @@ -2082,7 +2115,7 @@ sequenceDecs_decodeSync_safe_amd64_of_update_zero: ADD R13, R0, R0 sequenceDecs_decodeSync_safe_amd64_ml_update_zero: - MOVD R0, 16(RSP) + MOVD R0, 24(RSP) // Fill bitreader to have enough for the remaining CMP $0x08, R5 @@ -2118,7 +2151,8 @@ sequenceDecs_decodeSync_safe_amd64_fill_2_end: MOVD R3, R1 MOVD R2, R13 LSL R1, R13, R13 - UBFX $8, R0, $8, R1 + UBFX $8, R0, $8, R16 + BFI $0, R16, $8, R1 LSR $0x20, R0, R0 TST R1, R1 BEQ sequenceDecs_decodeSync_safe_amd64_ll_update_zero @@ -2132,10 +2166,10 @@ sequenceDecs_decodeSync_safe_amd64_fill_2_end: ADD R13, R0, R0 sequenceDecs_decodeSync_safe_amd64_ll_update_zero: - MOVD R0, 24(RSP) + MOVD R0, 32(RSP) // Fill bitreader for state updates - MOVD R12, (RSP) + MOVD R12, 8(RSP) MOVD R8, R0 LSR $0x08, R0, R0 MOVBU R0, R0 @@ -2153,7 +2187,7 @@ sequenceDecs_decodeSync_safe_amd64_ll_update_zero: NEG R1, R16 ROR R16, R13, R13 MOVD $0x00000001, R14 - MOVB R12, R1 + BFI $0, R12, $8, R1 LSLW R1, R14, R14 SUBW $1, R14, R14 AND R14, R13, R13 @@ -2174,7 +2208,7 @@ sequenceDecs_decodeSync_safe_amd64_ll_update_zero: NEG R1, R16 ROR R16, R13, R13 MOVD $0x00000001, R14 - MOVB R12, R1 + BFI $0, R12, $8, R1 LSLW R1, R14, R14 SUBW $1, R14, R14 AND R14, R13, R13 @@ -2195,7 +2229,7 @@ sequenceDecs_decodeSync_safe_amd64_ll_update_zero: NEG R1, R16 ROR R16, R13, R13 MOVD $0x00000001, R14 - MOVB R12, R1 + BFI $0, R12, $8, R1 LSLW R1, R14, R14 SUBW $1, R14, R14 AND R14, R13, R13 @@ -2210,7 +2244,7 @@ sequenceDecs_decodeSync_safe_amd64_ll_update_zero: sequenceDecs_decodeSync_safe_amd64_skip_update: // Adjust offset MOVD s+0(FP), R1 - MOVD 8(RSP), R12 + MOVD 16(RSP), R12 CMP $0x01, R0 BLS sequenceDecs_decodeSync_safe_amd64_adjust_offsetB_1_or_0 ADD $144, R1, R15 @@ -2221,7 +2255,7 @@ sequenceDecs_decodeSync_safe_amd64_skip_update: JMP sequenceDecs_decodeSync_safe_amd64_after_adjust sequenceDecs_decodeSync_safe_amd64_adjust_offsetB_1_or_0: - MOVD 24(RSP), R16 + MOVD 32(RSP), R16 CMP $0x00000000, R16 BNE sequenceDecs_decodeSync_safe_amd64_adjust_offset_maybezero ADD $1, R12, R12 @@ -2259,11 +2293,11 @@ sequenceDecs_decodeSync_safe_amd64_adjust_skip: MOVD R13, R12 sequenceDecs_decodeSync_safe_amd64_after_adjust: - MOVD R12, 8(RSP) + MOVD R12, 16(RSP) // Check values - MOVD 16(RSP), R0 - MOVD 24(RSP), R1 + MOVD 24(RSP), R0 + MOVD 32(RSP), R1 ADD R1, R0, R13 MOVD s+0(FP), R14 MOVD 256(R14), R16 @@ -2282,14 +2316,14 @@ sequenceDecs_decodeSync_safe_amd64_after_adjust: BNE sequenceDecs_decodeSync_safe_amd64_error_match_len_ofs_mismatch sequenceDecs_decodeSync_safe_amd64_match_len_ofs_ok: - MOVD 24(RSP), R0 - MOVD 8(RSP), R1 - MOVD 16(RSP), R12 + MOVD 32(RSP), R0 + MOVD 16(RSP), R1 + MOVD 24(RSP), R12 // Check if we have enough space in s.out ADD R12, R0, R13 ADD R9, R13, R13 - MOVD 32(RSP), R16 + MOVD 40(RSP), R16 CMP R16, R13 BHI error_not_enough_space @@ -2326,24 +2360,27 @@ copy_1_small: JMP copy_1_move_8through16 copy_1_move_1or2: - MOVB (R10), R13 - ADD R0, R10, R15 - MOVB -1(R15), R14 - MOVB R13, (R9) - ADD R0, R9, R15 - MOVB R14, -1(R15) - ADD R0, R10, R10 - ADD R0, R9, R9 - JMP copy_1_end + MOVBU (R10), R16 + BFI $0, R16, $8, R13 + ADD R0, R10, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R14 + MOVB R13, (R9) + ADD R0, R9, R15 + MOVB R14, -1(R15) + ADD R0, R10, R10 + ADD R0, R9, R9 + JMP copy_1_end copy_1_move_3: - MOVH (R10), R13 - MOVB 2(R10), R14 - MOVH R13, (R9) - MOVB R14, 2(R9) - ADD R0, R10, R10 - ADD R0, R9, R9 - JMP copy_1_end + MOVHU (R10), R13 + MOVBU 2(R10), R16 + BFI $0, R16, $8, R14 + MOVH R13, (R9) + MOVB R14, 2(R9) + ADD R0, R10, R10 + ADD R0, R9, R9 + JMP copy_1_end copy_1_move_4through7: MOVWU (R10), R13 @@ -2372,11 +2409,11 @@ copy_1_end: // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) check_offset: MOVD R11, R0 - MOVD 40(RSP), R16 + MOVD 48(RSP), R16 ADD R16, R0, R0 CMP R0, R1 BGT error_match_off_too_big - MOVD 56(RSP), R16 + MOVD 64(RSP), R16 CMP R16, R1 BGT error_match_off_too_big @@ -2384,7 +2421,7 @@ check_offset: MOVD R1, R0 SUBS R11, R0, R0 BLS copy_match - MOVD 48(RSP), R13 + MOVD 56(RSP), R13 SUB R0, R13, R13 CMP R0, R12 BGT copy_all_from_history @@ -2417,13 +2454,14 @@ copy_4_small: JMP copy_4_move_8through16 copy_4_move_3: - MOVH (R13), R0 - MOVB 2(R13), R1 - MOVH R0, (R9) - MOVB R1, 2(R9) - ADD R12, R13, R13 - ADD R12, R9, R9 - JMP copy_4_end + MOVHU (R13), R0 + MOVBU 2(R13), R16 + BFI $0, R16, $8, R1 + MOVH R0, (R9) + MOVB R1, 2(R9) + ADD R12, R13, R13 + ADD R12, R9, R9 + JMP copy_4_end copy_4_move_4through7: MOVWU (R13), R0 @@ -2482,24 +2520,27 @@ copy_5_small: JMP copy_5_move_8through16 copy_5_move_1or2: - MOVB (R13), R14 - ADD R0, R13, R15 - MOVB -1(R15), R4 - MOVB R14, (R9) - ADD R0, R9, R15 - MOVB R4, -1(R15) - ADD R0, R13, R13 - ADD R0, R9, R9 - JMP copy_5_end + MOVBU (R13), R16 + BFI $0, R16, $8, R14 + ADD R0, R13, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R4 + MOVB R14, (R9) + ADD R0, R9, R15 + MOVB R4, -1(R15) + ADD R0, R13, R13 + ADD R0, R9, R9 + JMP copy_5_end copy_5_move_3: - MOVH (R13), R14 - MOVB 2(R13), R4 - MOVH R14, (R9) - MOVB R4, 2(R9) - ADD R0, R13, R13 - ADD R0, R9, R9 - JMP copy_5_end + MOVHU (R13), R14 + MOVBU 2(R13), R16 + BFI $0, R16, $8, R4 + MOVH R14, (R9) + MOVB R4, 2(R9) + ADD R0, R13, R13 + ADD R0, R9, R9 + JMP copy_5_end copy_5_move_4through7: MOVWU (R13), R14 @@ -2567,24 +2608,27 @@ copy_2_small: JMP copy_2_move_8through16 copy_2_move_1or2: - MOVB (R0), R1 - ADD R12, R0, R15 - MOVB -1(R15), R13 - MOVB R1, (R9) - ADD R12, R9, R15 - MOVB R13, -1(R15) - ADD R12, R0, R0 - ADD R12, R9, R9 - JMP copy_2_end + MOVBU (R0), R16 + BFI $0, R16, $8, R1 + ADD R12, R0, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R13 + MOVB R1, (R9) + ADD R12, R9, R15 + MOVB R13, -1(R15) + ADD R12, R0, R0 + ADD R12, R9, R9 + JMP copy_2_end copy_2_move_3: - MOVH (R0), R1 - MOVB 2(R0), R13 - MOVH R1, (R9) - MOVB R13, 2(R9) - ADD R12, R0, R0 - ADD R12, R9, R9 - JMP copy_2_end + MOVHU (R0), R1 + MOVBU 2(R0), R16 + BFI $0, R16, $8, R13 + MOVH R1, (R9) + MOVB R13, 2(R9) + ADD R12, R0, R0 + ADD R12, R9, R9 + JMP copy_2_end copy_2_move_4through7: MOVWU (R0), R1 @@ -2615,12 +2659,13 @@ copy_overlapping_match: ADD R12, R11, R11 copy_slow_3: - MOVB (R0), R1 - MOVB R1, (R9) - ADD $1, R0, R0 - ADD $1, R9, R9 - SUBS $1, R12, R12 - BNE copy_slow_3 + MOVBU (R0), R16 + BFI $0, R16, $8, R1 + MOVB R1, (R9) + ADD $1, R0, R0 + ADD $1, R9, R9 + SUBS $1, R12, R12 + BNE copy_slow_3 handle_loop: MOVD ctx+16(FP), R0 @@ -2649,7 +2694,7 @@ loop_finished: // Return with match length error sequenceDecs_decodeSync_safe_amd64_error_match_len_ofs_mismatch: - MOVD 16(RSP), R0 + MOVD 24(RSP), R0 MOVD ctx+16(FP), R1 MOVD R0, 216(R1) MOVD $0x00000001, R16 @@ -2659,7 +2704,7 @@ sequenceDecs_decodeSync_safe_amd64_error_match_len_ofs_mismatch: // Return with match too long error sequenceDecs_decodeSync_safe_amd64_error_match_len_too_big: MOVD ctx+16(FP), R0 - MOVD 16(RSP), R1 + MOVD 24(RSP), R1 MOVD R1, 216(R0) MOVD $0x00000002, R16 MOVD R16, ret+24(FP) @@ -2668,7 +2713,7 @@ sequenceDecs_decodeSync_safe_amd64_error_match_len_too_big: // Return with match offset too long error error_match_off_too_big: MOVD ctx+16(FP), R0 - MOVD 8(RSP), R1 + MOVD 16(RSP), R1 MOVD R1, 224(R0) MOVD R11, 136(R0) MOVD $0x00000003, R16 @@ -2678,7 +2723,7 @@ error_match_off_too_big: // Return with not enough literals error error_not_enough_literals: MOVD ctx+16(FP), R0 - MOVD 24(RSP), R1 + MOVD 32(RSP), R1 MOVD R1, 208(R0) MOVD $0x00000004, R16 MOVD R16, ret+24(FP) @@ -2693,9 +2738,9 @@ error_overread: // Return with not enough output space error error_not_enough_space: MOVD ctx+16(FP), R0 - MOVD 24(RSP), R1 + MOVD 32(RSP), R1 MOVD R1, 208(R0) - MOVD 16(RSP), R1 + MOVD 24(RSP), R1 MOVD R1, 216(R0) MOVD R11, 136(R0) MOVD $0x00000005, R16 diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_asm.go b/vendor/github.com/klauspost/compress/zstd/seqdec_asm.go index 55405f3914..42ce5de716 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_asm.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_asm.go @@ -77,6 +77,35 @@ const errorNotEnoughSpace = 5 // error reported when bits are overread. const errorOverread = 6 +// useSafeDecodeSync reports whether decodeSyncSimple must use the bounds-exact +// ("safe") copy variants for the current buffer geometry. +// +// When the output and literal buffers have compressedBlockOverAlloc (16) +// bytes of slack past their logical use, the assembly may use extended +// memory copies that read and write in 16-byte blocks, overrunning the end +// of a literal run or match by up to 15 bytes. Otherwise it must use the +// bounds-exact ("safe") copies. This mirrors the analogous, always-dynamic +// selection in executeSimple below. +// +// The unsafe copies were disabled in #644 (2022) as a mitigation for a +// crash, but that crash's root cause — an unguarded bitReader overread that +// produced out-of-range match offsets/lengths — was fixed three days later +// in #645, which also added the fuzz corpus that has guarded this path since. +// See #1168. An asan-instrumented fuzz job (see .github/workflows/go.yml) +// covers the extended-copy path, which -race and plain fuzzing cannot. +func (s *sequenceDecs) useSafeDecodeSync() bool { + if s.maxSyncLen == 0 && cap(s.out)-len(s.out) < maxCompressedBlockSizeAlloc { + return true + } + if s.maxSyncLen > 0 && cap(s.out)-len(s.out)-compressedBlockOverAlloc < int(s.maxSyncLen) { + return true + } + if cap(s.literals) < len(s.literals)+compressedBlockOverAlloc { + return true + } + return false +} + // decode sequences from the stream with the provided history but without a dictionary. func (s *sequenceDecs) decodeSyncSimple(hist []byte) (bool, error) { if len(s.dict) > 0 { @@ -86,9 +115,7 @@ func (s *sequenceDecs) decodeSyncSimple(hist []byte) (bool, error) { return false, nil } - // FIXME: Using unsafe memory copies leads to rare, random crashes - // with fuzz testing. It is therefore disabled for now. - const useSafe = true + useSafe := s.useSafeDecodeSync() br := s.br diff --git a/vendor/modules.txt b/vendor/modules.txt index 5a54911007..d8b5c12fb6 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -109,7 +109,7 @@ github.com/davecgh/go-spew/spew # github.com/distribution/reference v0.6.0 ## explicit; go 1.20 github.com/distribution/reference -# github.com/docker/cli v29.6.2+incompatible +# github.com/docker/cli v29.7.2+incompatible ## explicit github.com/docker/cli/cli/config github.com/docker/cli/cli/config/configfile @@ -212,7 +212,7 @@ github.com/google/go-cmp/cmp/internal/diff github.com/google/go-cmp/cmp/internal/flags github.com/google/go-cmp/cmp/internal/function github.com/google/go-cmp/cmp/internal/value -# github.com/google/go-containerregistry v0.21.9 +# github.com/google/go-containerregistry v0.22.0 ## explicit; go 1.25.0 github.com/google/go-containerregistry/cmd/crane github.com/google/go-containerregistry/cmd/crane/cmd @@ -334,7 +334,7 @@ github.com/jstemmer/go-junit-report/v2/junit github.com/jstemmer/go-junit-report/v2/parser/gotest github.com/jstemmer/go-junit-report/v2/parser/gotest/internal/collector github.com/jstemmer/go-junit-report/v2/parser/gotest/internal/reader -# github.com/klauspost/compress v1.19.1 +# github.com/klauspost/compress v1.19.2 ## explicit; go 1.24 github.com/klauspost/compress github.com/klauspost/compress/fse