From 0021cf876b558c2824495f0076c54b32a16b3c64 Mon Sep 17 00:00:00 2001 From: Juan Calderon-Perez <835733+gaby@users.noreply.github.com> Date: Thu, 11 Dec 2025 07:20:37 -0500 Subject: [PATCH 1/2] Harden latest-only filtering --- api/mirror.go | 4 +++- cmd/mirror_update.go | 4 +++- completion.d/_aptly | 1 + completion.d/aptly | 2 +- deb/list.go | 33 ++++++++++++++++++++++++++ deb/list_test.go | 49 +++++++++++++++++++++++++++++++++++++++ deb/remote.go | 14 ++++++++++- deb/remote_test.go | 55 +++++++++++++++++++++++++++++++++----------- man/aptly.1 | 4 ++++ 9 files changed, 149 insertions(+), 17 deletions(-) diff --git a/api/mirror.go b/api/mirror.go index 77de795c12..60c8f23c7f 100644 --- a/api/mirror.go +++ b/api/mirror.go @@ -343,6 +343,8 @@ type mirrorUpdateParams struct { ForceUpdate bool ` json:"ForceUpdate"` // Set "true" to skip downloading already downloaded packages SkipExistingPackages bool ` json:"SkipExistingPackages"` + // Set "true" to download only the latest version per package/architecture + LatestOnly bool ` json:"LatestOnly"` } // @Summary Update Mirror @@ -434,7 +436,7 @@ func apiMirrorsUpdate(c *gin.Context) { } queue, downloadSize, err := remote.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(), - collectionFactory.ChecksumCollection(nil), b.SkipExistingPackages) + collectionFactory.ChecksumCollection(nil), b.SkipExistingPackages, b.LatestOnly) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) } diff --git a/cmd/mirror_update.go b/cmd/mirror_update.go index 2f0fb524b3..df28efa71b 100644 --- a/cmd/mirror_update.go +++ b/cmd/mirror_update.go @@ -87,10 +87,11 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error { ) skipExistingPackages := context.Flags().Lookup("skip-existing-packages").Value.Get().(bool) + latestOnly := context.Flags().Lookup("latest").Value.Get().(bool) context.Progress().Printf("Building download queue...\n") queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(), - collectionFactory.ChecksumCollection(nil), skipExistingPackages) + collectionFactory.ChecksumCollection(nil), skipExistingPackages, latestOnly) if err != nil { return fmt.Errorf("unable to update: %s", err) @@ -292,6 +293,7 @@ Example: cmd.Flag.Bool("ignore-checksums", false, "ignore checksum mismatches while downloading package files and metadata") cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures") cmd.Flag.Bool("skip-existing-packages", false, "do not check file existence for packages listed in the internal database of the mirror") + cmd.Flag.Bool("latest", false, "download only latest version of each package (per architecture)") cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)") cmd.Flag.String("downloader", "default", "downloader to use (e.g. grab)") cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error") diff --git a/completion.d/_aptly b/completion.d/_aptly index 8e8d2bbd65..a1f4d85964 100644 --- a/completion.d/_aptly +++ b/completion.d/_aptly @@ -211,6 +211,7 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could $keyring \ "-max-tries=[max download tries till process fails with download error]:number: " \ "-skip-existing-packages=[do not check file existence for packages listed in the internal database of the mirror]:$bool" \ + "-latest=[download only latest version of each package (per architecture)]:$bool" \ "(-)2:mirror name:$mirrors" ;; rename) diff --git a/completion.d/aptly b/completion.d/aptly index ab8c4c15ac..aa5b506fa7 100644 --- a/completion.d/aptly +++ b/completion.d/aptly @@ -263,7 +263,7 @@ _aptly() "update") if [[ $numargs -eq 0 ]]; then if [[ "$cur" == -* ]]; then - COMPREPLY=($(compgen -W "-force -download-limit= -downloader= -ignore-checksums -ignore-signatures -keyring= -skip-existing-packages" -- ${cur})) + COMPREPLY=($(compgen -W "-force -download-limit= -downloader= -ignore-checksums -ignore-signatures -keyring= -skip-existing-packages -latest" -- ${cur})) else COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur})) fi diff --git a/deb/list.go b/deb/list.go index 25a2d2830b..19e37593cc 100644 --- a/deb/list.go +++ b/deb/list.go @@ -172,6 +172,39 @@ func (l *PackageList) ForEach(handler func(*Package) error) error { return err } +// FilterLatest creates a copy of the package list containing only the +// latest version for each package name/architecture pair. +func (l *PackageList) FilterLatest() (*PackageList, error) { + if l == nil { + return nil, fmt.Errorf("package list is nil") + } + + filtered := make(map[string]*Package, l.Len()) + + err := l.ForEach(func(p *Package) error { + key := p.Architecture + "|" + p.Name + + if existing, found := filtered[key]; !found || CompareVersions(p.Version, existing.Version) > 0 { + filtered[key] = p + } + + return nil + }) + if err != nil { + return nil, err + } + + result := NewPackageListWithDuplicates(l.duplicatesAllowed, len(filtered)) + + for _, pkg := range filtered { + if err = result.Add(pkg); err != nil { + return nil, err + } + } + + return result, nil +} + // ForEachIndexed calls handler for each package in list in indexed order func (l *PackageList) ForEachIndexed(handler func(*Package) error) error { if !l.indexed { diff --git a/deb/list_test.go b/deb/list_test.go index 464dd09e2e..4e4768e34e 100644 --- a/deb/list_test.go +++ b/deb/list_test.go @@ -503,3 +503,52 @@ func (s *PackageListSuite) TestArchitectures(c *C) { sort.Strings(archs) c.Check(archs, DeepEquals, []string{"amd64", "arm", "i386", "s390"}) } + +func (s *PackageListSuite) TestFilterLatest(c *C) { + list := NewPackageList() + + older := packageStanza.Copy() + older["Version"] = "1.0" + olderPkg := NewPackageFromControlFile(older) + _ = list.Add(olderPkg) + + newer := packageStanza.Copy() + newer["Version"] = "2.0" + newerPkg := NewPackageFromControlFile(newer) + _ = list.Add(newerPkg) + + shared := packageStanza.Copy() + shared["Architecture"] = ArchitectureAll + shared["Version"] = "3.0" + shared["Package"] = "shared" + sharedPkg := NewPackageFromControlFile(shared) + _ = list.Add(sharedPkg) + + filtered, err := list.FilterLatest() + c.Assert(err, IsNil) + c.Assert(filtered.Len(), Equals, 2) + c.Check(filtered.Has(newerPkg), Equals, true) + c.Check(filtered.Has(sharedPkg), Equals, true) +} + +func (s *PackageListSuite) TestFilterLatestPreservesDuplicatesFlag(c *C) { + list := NewPackageListWithDuplicates(true, 2) + + _ = list.Add(NewPackageFromControlFile(packageStanza.Copy())) + + another := packageStanza.Copy() + another["Version"] = "7.41-1" + _ = list.Add(NewPackageFromControlFile(another)) + + filtered, err := list.FilterLatest() + c.Assert(err, IsNil) + c.Assert(filtered.duplicatesAllowed, Equals, true) +} + +func (s *PackageListSuite) TestFilterLatestNil(c *C) { + var list *PackageList + + filtered, err := list.FilterLatest() + c.Assert(err, ErrorMatches, "package list is nil") + c.Assert(filtered, IsNil) +} diff --git a/deb/remote.go b/deb/remote.go index c6fe595c2f..8776b65a4b 100644 --- a/deb/remote.go +++ b/deb/remote.go @@ -612,7 +612,19 @@ func (repo *RemoteRepo) ApplyFilter(dependencyOptions int, filterQuery PackageQu } // BuildDownloadQueue builds queue, discards current PackageList -func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, packageCollection *PackageCollection, checksumStorage aptly.ChecksumStorage, skipExistingPackages bool) (queue []PackageDownloadTask, downloadSize int64, err error) { +func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, packageCollection *PackageCollection, checksumStorage aptly.ChecksumStorage, skipExistingPackages, latestOnly bool) (queue []PackageDownloadTask, downloadSize int64, err error) { + if repo.packageList == nil { + err = fmt.Errorf("package list is empty, please (re)download package indexes") + return + } + + if latestOnly { + repo.packageList, err = repo.packageList.FilterLatest() + if err != nil { + return + } + } + queue = make([]PackageDownloadTask, 0, repo.packageList.Len()) seen := make(map[string]int, repo.packageList.Len()) diff --git a/deb/remote_test.go b/deb/remote_test.go index c0eae7f5a9..881685850b 100644 --- a/deb/remote_test.go +++ b/deb/remote_test.go @@ -281,7 +281,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) { c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) - queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) + queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(3)) c.Check(queue, HasLen, 1) @@ -308,7 +308,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) { c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) - queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true) + queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(0)) c.Check(queue, HasLen, 0) @@ -329,7 +329,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) { c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) - queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) + queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(3)) c.Check(queue, HasLen, 1) @@ -356,7 +356,7 @@ func (s *RemoteRepoSuite) TestDownloadWithInstaller(c *C) { c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) - queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) + queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(3)+int64(len(exampleInstallerManifestFile))) c.Check(queue, HasLen, 2) @@ -382,6 +382,35 @@ func (s *RemoteRepoSuite) TestDownloadWithInstaller(c *C) { c.Check(pkg.Name, Equals, "installer") } +func (s *RemoteRepoSuite) TestBuildDownloadQueueLatestOnly(c *C) { + s.repo.Architectures = []string{"i386"} + + err := s.repo.Fetch(s.downloader, nil, true) + c.Assert(err, IsNil) + + s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.bz2", &http.Error{Code: 404}) + s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404}) + s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile) + + err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, true, false) + c.Assert(err, IsNil) + c.Assert(s.downloader.Empty(), Equals, true) + + stanza := packageStanza.Copy() + stanza["Package"] = "amanda-client" + stanza["Version"] = "1:3.4.0-1" + stanza["Filename"] = "pool/main/a/amanda/amanda-client_3.4.0-1_i386.deb" + + newest := NewPackageFromControlFile(stanza) + _ = s.repo.packageList.Add(newest) + + queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, true) + c.Assert(err, IsNil) + c.Check(queue, HasLen, 1) + c.Check(queue[0].File.DownloadURL(), Equals, "pool/main/a/amanda/amanda-client_3.4.0-1_i386.deb") + c.Check(size, Equals, int64(187518)) +} + func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) { s.repo.Architectures = []string{"i386"} s.repo.DownloadSources = true @@ -400,7 +429,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) { c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) - queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) + queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(15)) c.Check(queue, HasLen, 4) @@ -444,7 +473,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) { c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) - queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true) + queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(0)) c.Check(queue, HasLen, 0) @@ -469,7 +498,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) { c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) - queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) + queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(15)) c.Check(queue, HasLen, 4) @@ -493,7 +522,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) { c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) - queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) + queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(3)) c.Check(queue, HasLen, 1) @@ -521,7 +550,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) { c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) - queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true) + queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(0)) c.Check(queue, HasLen, 0) @@ -543,7 +572,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) { c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) - queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) + queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(3)) c.Check(queue, HasLen, 1) @@ -574,7 +603,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) { c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) - queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) + queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(15)) c.Check(queue, HasLen, 4) @@ -620,7 +649,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) { c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) - queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true) + queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(0)) c.Check(queue, HasLen, 0) @@ -646,7 +675,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) { c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) - queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) + queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(15)) c.Check(queue, HasLen, 4) diff --git a/man/aptly.1 b/man/aptly.1 index bd6ad22317..f1ab1eee43 100644 --- a/man/aptly.1 +++ b/man/aptly.1 @@ -738,6 +738,10 @@ max download tries till process fails with download error \-\fBskip\-existing\-packages\fR do not check file existence for packages listed in the internal database of the mirror . +.TP +\-\fBlatest\fR +download only latest version of each package (per architecture) +. .SH "RENAMES MIRROR" \fBaptly\fR \fBmirror\fR \fBrename\fR \fIold\-name\fR \fInew\-name\fR . From de699aebe55262fc69738846b5909e9a30b42ab3 Mon Sep 17 00:00:00 2001 From: Juan Calderon-Perez <835733+gaby@users.noreply.github.com> Date: Thu, 11 Dec 2025 09:42:15 -0500 Subject: [PATCH 2/2] Update AUTHORS list --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 2c067dd232..4f5efee0a2 100644 --- a/AUTHORS +++ b/AUTHORS @@ -74,3 +74,4 @@ List of contributors, in chronological order: * JupiterRider (https://github.com/JupiterRider) * Agustin Henze (https://github.com/agustinhenze) * Tobias Assarsson (https://github.com/daedaluz) +* Juan Calderon-Perez (https://github.com/gaby)