Add support for architecture variants

api/api.go

@@ -285,33 +285,11 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
 
 	// filter packages by version
 	if c.Request.URL.Query().Get("maximumVersion") == "1" {
-		list.PrepareIndex()
-		_ = list.ForEach(func(p *deb.Package) error {
-			versionQ, err := query.Parse(fmt.Sprintf("Name (%s), $Version (<= %s)", p.Name, p.Version))
-			if err != nil {
-				fmt.Println("filter packages by version, query string parse err: ", err)
-				_ = c.AbortWithError(500, fmt.Errorf("unable to parse %s maximum version query string: %s", p.Name, err))
-			} else {
-				tmpList, err := list.Filter(deb.FilterOptions{
-					Queries: []deb.PackageQuery{versionQ},
-				})
-
-				if err == nil {
-					if tmpList.Len() > 0 {
-						_ = tmpList.ForEach(func(tp *deb.Package) error {
-							list.Remove(tp)
-							return nil
-						})
-						_ = list.Add(p)
-					}
-				} else {
-					fmt.Println("filter packages by version, filter err: ", err)
-					_ = c.AbortWithError(500, fmt.Errorf("unable to get %s maximum version: %s", p.Name, err))
-				}
-			}
-
-			return nil
-		})
+		list, err = list.FilterLatest()
+		if err != nil {
+			AbortWithJSONError(c, 500, fmt.Errorf("unable to filter latest packages: %s", err))
+			return
+		}
 	}
 
 	if c.Request.URL.Query().Get("format") == "details" {

api/packages_maximum_version_test.go

@@ -0,0 +1,112 @@
+package api
+
+import (
+	"encoding/json"
+	"net/http/httptest"
+	"reflect"
+	"sort"
+	"testing"
+
+	"github.com/aptly-dev/aptly/database/goleveldb"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/gin-gonic/gin"
+)
+
+// Exercise the common API listing handler with persisted packages, checking
+// both reference and detailed JSON responses independently of response order.
+func TestPackagesMaximumVersionArchitectureVariant(t *testing.T) {
+	pkg := func(arch, variant, version string) *deb.Package {
+		stanza := deb.Stanza{"Package": "example", "Version": version, "Architecture": arch}
+		if variant != "" {
+			stanza["Architecture-Variant"] = variant
+		}
+		return deb.NewPackageFromControlFile(stanza)
+	}
+	n1, n3 := pkg("amd64", "", "1"), pkg("amd64", "", "3")
+	v1, v2 := pkg("amd64", "amd64v3", "1"), pkg("amd64", "amd64v3", "2")
+	a1, a4 := pkg("arm64", "", "1"), pkg("arm64", "", "4")
+	all := []*deb.Package{n1, n3, v1, v2, a1, a4}
+	cases := []struct {
+		name, option string
+		input, want  []*deb.Package
+	}{
+		{"same_version", "1", []*deb.Package{n1, v1}, []*deb.Package{n1, v1}},
+		{"independent_latest", "1", []*deb.Package{n1, n3, v1, v2}, []*deb.Package{n3, v2}},
+		{"ordinary_multi_architecture", "1", []*deb.Package{n1, n3, a1, a4}, []*deb.Package{n3, a4}},
+		{"ordinary_single_architecture", "1", []*deb.Package{n1, n3}, []*deb.Package{n3}},
+		{"disabled_absent", "", all, all},
+		{"disabled_zero", "0", all, all},
+		{"empty", "1", nil, nil},
+	}
+	for _, tc := range cases {
+		for _, format := range []string{"refs", "details"} {
+			t.Run(tc.name+"/"+format, func(t *testing.T) {
+				db, err := goleveldb.NewOpenDB(t.TempDir())
+				if err != nil {
+					t.Fatal(err)
+				}
+				t.Cleanup(func() {
+					if err := db.Close(); err != nil {
+						t.Error(err)
+					}
+				})
+				factory := deb.NewCollectionFactory(db)
+				list := deb.NewPackageList()
+				for _, p := range tc.input {
+					// Update offloads metadata; use a fresh package for each database.
+					p := pkg(p.Architecture, p.ArchitectureVariant, p.Version)
+					if err := factory.PackageCollection().Update(p); err != nil {
+						t.Fatal(err)
+					}
+					if err := list.Add(p); err != nil {
+						t.Fatal(err)
+					}
+				}
+				refs := deb.NewPackageRefListFromPackageList(list)
+				url := "/api/packages?format=" + format
+				if tc.option != "" {
+					url += "&maximumVersion=" + tc.option
+				}
+				response := httptest.NewRecorder()
+				c, _ := gin.CreateTestContext(response)
+				c.Request = httptest.NewRequest("GET", url, nil)
+				showPackages(c, refs, deb.NewCollectionFactory(db))
+				if response.Code != 200 {
+					t.Fatalf("status %d: %s", response.Code, response.Body.String())
+				}
+				got := []string{}
+				if format == "details" {
+					var details []map[string]string
+					if err := json.Unmarshal(response.Body.Bytes(), &details); err != nil {
+						t.Fatal(err)
+					}
+					for _, detail := range details {
+						got = append(got, detail["Key"])
+						// Metadata must remain separate from the identity encoded in Key.
+						for _, p := range tc.want {
+							if detail["Key"] == string(p.Key("")) {
+								if detail["Architecture"] != p.Architecture || detail["Architecture-Variant"] != p.ArchitectureVariant || detail["ShortKey"] != string(p.ShortKey("")) {
+									t.Errorf("incorrect package metadata: %v", detail)
+								}
+							}
+						}
+					}
+				} else if err := json.Unmarshal(response.Body.Bytes(), &got); err != nil {
+					t.Fatal(err)
+				}
+				want := []string{}
+				for _, p := range tc.want {
+					want = append(want, string(p.Key("")))
+				}
+				sort.Strings(got)
+				sort.Strings(want)
+				if !reflect.DeepEqual(got, want) {
+					t.Errorf("package references:\n got %v\nwant %v", got, want)
+				}
+				if !reflect.DeepEqual(refs.Strings(), deb.NewPackageRefListFromPackageList(list).Strings()) {
+					t.Error("listing mutated source references")
+				}
+			})
+		}
+	}
+}

api/snapshot.go

@@ -800,6 +800,10 @@ func apiSnapshotsPull(c *gin.Context) {
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
+		err = taskCollectionFactory.SnapshotCollection().LoadComplete(freshToSnapshot)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
 		freshSourceSnapshot, err := taskCollectionFactory.SnapshotCollection().ByName(body.Source)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
@@ -873,14 +877,17 @@ func apiSnapshotsPull(c *gin.Context) {
 		alreadySeen := map[string]bool{}
 
 		_ = destinationPackageList.ForEachIndexed(func(pkg *deb.Package) error {
-			key := pkg.Architecture + "_" + pkg.Name
+			key := pkg.IndexArchitecture() + "_" + pkg.Name
 			_, seen := alreadySeen[key]
 
 			// If we haven't seen such name-architecture pair and were instructed to remove, remove it
 			if !noRemove && !seen {
 				// Remove all packages with the same name and architecture
 				packageSearchResults := toPackageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}, true, false)
 				for _, p := range packageSearchResults {
+					if p.ArchitectureVariant != pkg.ArchitectureVariant {
+						continue
+					}
 					toPackageList.Remove(p)
 					removedPackages = append(removedPackages, p.String())
 				}

api/snapshot_pull_variant_test.go

@@ -0,0 +1,138 @@
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+
+	ctx "github.com/aptly-dev/aptly/context"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/smira/flag"
+)
+
+// Exercise the real pull entry point and persisted references, without package
+// files or mirrors. Architecture selection remains base-architecture selection.
+func TestSnapshotPullArchitectureVariant(t *testing.T) {
+	normal := &deb.Package{Name: "pull-test", Version: "1", Architecture: "amd64"}
+	variant := &deb.Package{Name: "pull-test", Version: "1", Architecture: "amd64", ArchitectureVariant: "amd64v3"}
+	normal2 := &deb.Package{Name: "pull-test", Version: "2", Architecture: "amd64"}
+	variant2 := &deb.Package{Name: "pull-test", Version: "2", Architecture: "amd64", ArchitectureVariant: "amd64v3"}
+	cases := []struct {
+		name                 string
+		dest, source, want   []*deb.Package
+		noRemove, allMatches bool
+	}{
+		{"normal", nil, []*deb.Package{normal}, []*deb.Package{normal}, false, false},
+		{"variant", nil, []*deb.Package{variant}, []*deb.Package{variant}, false, false},
+		{"both", nil, []*deb.Package{normal, variant}, []*deb.Package{normal, variant}, false, false},
+		{"variant_into_normal", []*deb.Package{normal}, []*deb.Package{variant}, []*deb.Package{normal, variant}, false, false},
+		{"normal_into_variant", []*deb.Package{variant}, []*deb.Package{normal}, []*deb.Package{normal, variant}, false, false},
+		{"both_all_matches", nil, []*deb.Package{normal, variant}, []*deb.Package{normal, variant}, false, true},
+		{"both_no_remove", nil, []*deb.Package{normal, variant}, []*deb.Package{normal, variant}, true, false},
+		{"variant_into_normal_no_remove", []*deb.Package{normal}, []*deb.Package{variant}, []*deb.Package{normal, variant}, true, false},
+		{"normal_into_variant_no_remove", []*deb.Package{variant}, []*deb.Package{normal}, []*deb.Package{normal, variant}, true, false},
+		{"variant_into_normal_all_matches", []*deb.Package{normal}, []*deb.Package{variant}, []*deb.Package{normal, variant}, false, true},
+		{"normal_into_variant_all_matches", []*deb.Package{variant}, []*deb.Package{normal}, []*deb.Package{normal, variant}, false, true},
+		{"normal_replace", []*deb.Package{normal}, []*deb.Package{normal2}, []*deb.Package{normal2}, false, false},
+		{"normal_no_remove", []*deb.Package{normal}, []*deb.Package{normal2}, []*deb.Package{normal, normal2}, true, false},
+		{"normal_latest", nil, []*deb.Package{normal, normal2}, []*deb.Package{normal2}, false, false},
+		{"normal_all_matches", nil, []*deb.Package{normal, normal2}, []*deb.Package{normal, normal2}, false, true},
+		{"variant_replace_preserves_normal", []*deb.Package{normal, variant}, []*deb.Package{variant2}, []*deb.Package{normal, variant2}, false, false},
+		{"latest_per_identity", nil, []*deb.Package{normal, normal2, variant, variant2}, []*deb.Package{normal2, variant2}, false, false},
+		{"all_versions_both_identities", nil, []*deb.Package{normal, normal2, variant, variant2}, []*deb.Package{normal, normal2, variant, variant2}, false, true},
+	}
+	for _, tc := range cases {
+		for _, mode := range []string{"inferred", "explicit"} {
+			t.Run(tc.name+"/"+mode, func(t *testing.T) {
+				savedConfig := utils.Config
+				t.Cleanup(func() { utils.Config = savedConfig })
+				dir := t.TempDir()
+				configPath := filepath.Join(dir, "aptly.conf")
+				configData, err := json.Marshal(map[string]interface{}{"rootDir": dir, "architectures": []string{}})
+				if err != nil {
+					t.Fatal(err)
+				}
+				if err = os.WriteFile(configPath, configData, 0600); err != nil {
+					t.Fatal(err)
+				}
+				flags := flag.NewFlagSet("pull-test", flag.ContinueOnError)
+				flags.String("config", configPath, "")
+				// Empty destinations need explicit architecture; populated destinations
+				// exercise automatic base-architecture discovery.
+				arch := ""
+				if len(tc.dest) == 0 || mode == "explicit" {
+					arch = "amd64"
+				}
+				flags.String("architectures", arch, "")
+				flags.Bool("no-lock", false, "")
+				flags.Int("db-open-attempts", 1, "")
+				flags.Bool("no-deps", true, "")
+				flags.Bool("no-remove", tc.noRemove, "")
+				flags.Bool("all-matches", tc.allMatches, "")
+				flags.Bool("dry-run", false, "")
+				testContext, err := ctx.NewContext(flags)
+				if err != nil {
+					t.Fatal(err)
+				}
+				t.Cleanup(testContext.Shutdown)
+				factory := testContext.NewCollectionFactory()
+				refs := func(packages []*deb.Package) *deb.PackageRefList {
+					list := deb.NewPackageList()
+					for _, p := range packages {
+						if err := list.Add(p); err != nil {
+							t.Fatal(err)
+						}
+					}
+					return deb.NewPackageRefListFromPackageList(list)
+				}
+				for _, p := range append(append([]*deb.Package{}, tc.dest...), tc.source...) {
+					if err := factory.PackageCollection().Update(p); err != nil {
+						t.Fatal(err)
+					}
+				}
+				for name, packages := range map[string][]*deb.Package{"base": tc.dest, "source": tc.source} {
+					if err := factory.SnapshotCollection().Add(deb.NewSnapshotFromRefList(name, nil, refs(packages), "")); err != nil {
+						t.Fatal(err)
+					}
+				}
+				body, err := json.Marshal(snapshotsPullParams{Source: "source", Destination: "result", Queries: []string{"pull-test"}})
+				if err != nil {
+					t.Fatal(err)
+				}
+				url := "/api/snapshots/base/pull?no-deps=1"
+				if tc.noRemove {
+					url += "&no-remove=1"
+				}
+				if tc.allMatches {
+					url += "&all-matches=1"
+				}
+				req := httptest.NewRequest("POST", url, bytes.NewReader(body))
+				req.Header.Set("Content-Type", "application/json")
+				response := httptest.NewRecorder()
+				savedContext := context
+				t.Cleanup(func() { context = savedContext })
+				Router(testContext).ServeHTTP(response, req)
+				if response.Code != 201 {
+					t.Fatalf("pull status %d: %s", response.Code, response.Body.String())
+				}
+				snapshots := testContext.NewCollectionFactory().SnapshotCollection()
+				result, err := snapshots.ByName("result")
+				if err != nil {
+					t.Fatal(err)
+				}
+				if err = snapshots.LoadComplete(result); err != nil {
+					t.Fatal(err)
+				}
+				got, want := result.RefList().Strings(), refs(tc.want).Strings()
+				if !reflect.DeepEqual(got, want) {
+					t.Errorf("persisted package identities:\n got %v\nwant %v", got, want)
+				}
+			})
+		}
+	}
+}

api/snapshot_test.go

@@ -1,7 +1,14 @@
 package api
 
 import (
+	"bytes"
 	"encoding/json"
+	"os"
+	"path/filepath"
+
+	ctx "github.com/aptly-dev/aptly/context"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/smira/flag"
 
 	"github.com/aptly-dev/aptly/deb"
 	. "gopkg.in/check.v1"
@@ -51,3 +58,58 @@
 	c.Assert(response.Code, Equals, 500)
 	c.Assert(response.Body.String(), Matches, ".*msgpack.*|.*decode.*")
 }
+
+// Pull must load the destination references before inferring architectures or
+// preserving existing packages. Both fixtures are ordinary amd64 packages.
+func (_ *SnapshotsSuite) TestPullPreservesDestinationPackages(c *C) {
+	savedConfig, savedContext := utils.Config, context
+	defer func() { utils.Config, context = savedConfig, savedContext }()
+	dir := c.MkDir()
+	configPath := filepath.Join(dir, "aptly.conf")
+	configData, err := json.Marshal(map[string]interface{}{"rootDir": dir, "architectures": []string{}})
+	c.Assert(err, IsNil)
+	c.Assert(os.WriteFile(configPath, configData, 0600), IsNil)
+	flags := flag.NewFlagSet("pull-test", flag.ContinueOnError)
+	flags.String("config", configPath, "")
+	flags.String("architectures", "", "")
+	flags.Bool("no-lock", false, "")
+	flags.Int("db-open-attempts", 1, "")
+	testContext, err := ctx.NewContext(flags)
+	c.Assert(err, IsNil)
+	defer testContext.Shutdown()
+	s := &APISuite{context: testContext, router: Router(testContext)}
+	factory := s.context.NewCollectionFactory()
+	for _, mode := range []string{"inferred", "explicit"} {
+		a := &deb.Package{Name: "package-a", Version: "1", Architecture: "amd64"}
+		b := &deb.Package{Name: "package-b", Version: "1", Architecture: "amd64"}
+		for name, pkg := range map[string]*deb.Package{"base": a, "source": b} {
+			c.Assert(factory.PackageCollection().Update(pkg), IsNil)
+			list := deb.NewPackageList()
+			c.Assert(list.Add(pkg), IsNil)
+			snapshot := deb.NewSnapshotFromPackageList("pull-"+name+"-"+mode, nil, list, "")
+			c.Assert(factory.SnapshotCollection().Add(snapshot), IsNil)
+		}
+		params := snapshotsPullParams{
+			Source:      "pull-source-" + mode,
+			Destination: "pull-result-" + mode,
+			Queries:     []string{"package-b"},
+		}
+		if mode == "explicit" {
+			params.Architectures = []string{"amd64"}
+		}
+		body, err := json.Marshal(params)
+		c.Assert(err, IsNil)
+		response, err := s.HTTPRequest("POST", "/api/snapshots/pull-base-"+mode+"/pull?no-remove=1&no-deps=1", bytes.NewReader(body))
+		c.Assert(err, IsNil)
+		if !c.Check(response.Code, Equals, 201, Commentf("%s: %s", mode, response.Body.String())) {
+			continue
+		}
+		snapshots := s.context.NewCollectionFactory().SnapshotCollection()
+		result, err := snapshots.ByName(params.Destination)
+		c.Assert(err, IsNil)
+		c.Assert(snapshots.LoadComplete(result), IsNil)
+		c.Check(result.RefList().Strings(), DeepEquals, []string{
+			string(a.Key("")), string(b.Key("")),
+		}, Commentf("%s: preserve A and add B", mode))
+	}
+}