diff --git a/api/api.go b/api/api.go index b13b40c1e..ee20d257d 100644 --- a/api/api.go +++ b/api/api.go @@ -285,33 +285,11 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory // filter packages by version if c.Request.URL.Query().Get("maximumVersion") == "1" { - list.PrepareIndex() - _ = list.ForEach(func(p *deb.Package) error { - versionQ, err := query.Parse(fmt.Sprintf("Name (%s), $Version (<= %s)", p.Name, p.Version)) - if err != nil { - fmt.Println("filter packages by version, query string parse err: ", err) - _ = c.AbortWithError(500, fmt.Errorf("unable to parse %s maximum version query string: %s", p.Name, err)) - } else { - tmpList, err := list.Filter(deb.FilterOptions{ - Queries: []deb.PackageQuery{versionQ}, - }) - - if err == nil { - if tmpList.Len() > 0 { - _ = tmpList.ForEach(func(tp *deb.Package) error { - list.Remove(tp) - return nil - }) - _ = list.Add(p) - } - } else { - fmt.Println("filter packages by version, filter err: ", err) - _ = c.AbortWithError(500, fmt.Errorf("unable to get %s maximum version: %s", p.Name, err)) - } - } - - return nil - }) + list, err = list.FilterLatest() + if err != nil { + AbortWithJSONError(c, 500, fmt.Errorf("unable to filter latest packages: %s", err)) + return + } } if c.Request.URL.Query().Get("format") == "details" { diff --git a/api/packages_maximum_version_test.go b/api/packages_maximum_version_test.go new file mode 100644 index 000000000..0a46a271d --- /dev/null +++ b/api/packages_maximum_version_test.go @@ -0,0 +1,112 @@ +package api + +import ( + "encoding/json" + "net/http/httptest" + "reflect" + "sort" + "testing" + + "github.com/aptly-dev/aptly/database/goleveldb" + "github.com/aptly-dev/aptly/deb" + "github.com/gin-gonic/gin" +) + +// Exercise the common API listing handler with persisted packages, checking +// both reference and detailed JSON responses independently of response order. +func TestPackagesMaximumVersionArchitectureVariant(t *testing.T) { + pkg := func(arch, variant, version string) *deb.Package { + stanza := deb.Stanza{"Package": "example", "Version": version, "Architecture": arch} + if variant != "" { + stanza["Architecture-Variant"] = variant + } + return deb.NewPackageFromControlFile(stanza) + } + n1, n3 := pkg("amd64", "", "1"), pkg("amd64", "", "3") + v1, v2 := pkg("amd64", "amd64v3", "1"), pkg("amd64", "amd64v3", "2") + a1, a4 := pkg("arm64", "", "1"), pkg("arm64", "", "4") + all := []*deb.Package{n1, n3, v1, v2, a1, a4} + cases := []struct { + name, option string + input, want []*deb.Package + }{ + {"same_version", "1", []*deb.Package{n1, v1}, []*deb.Package{n1, v1}}, + {"independent_latest", "1", []*deb.Package{n1, n3, v1, v2}, []*deb.Package{n3, v2}}, + {"ordinary_multi_architecture", "1", []*deb.Package{n1, n3, a1, a4}, []*deb.Package{n3, a4}}, + {"ordinary_single_architecture", "1", []*deb.Package{n1, n3}, []*deb.Package{n3}}, + {"disabled_absent", "", all, all}, + {"disabled_zero", "0", all, all}, + {"empty", "1", nil, nil}, + } + for _, tc := range cases { + for _, format := range []string{"refs", "details"} { + t.Run(tc.name+"/"+format, func(t *testing.T) { + db, err := goleveldb.NewOpenDB(t.TempDir()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := db.Close(); err != nil { + t.Error(err) + } + }) + factory := deb.NewCollectionFactory(db) + list := deb.NewPackageList() + for _, p := range tc.input { + // Update offloads metadata; use a fresh package for each database. + p := pkg(p.Architecture, p.ArchitectureVariant, p.Version) + if err := factory.PackageCollection().Update(p); err != nil { + t.Fatal(err) + } + if err := list.Add(p); err != nil { + t.Fatal(err) + } + } + refs := deb.NewPackageRefListFromPackageList(list) + url := "/api/packages?format=" + format + if tc.option != "" { + url += "&maximumVersion=" + tc.option + } + response := httptest.NewRecorder() + c, _ := gin.CreateTestContext(response) + c.Request = httptest.NewRequest("GET", url, nil) + showPackages(c, refs, deb.NewCollectionFactory(db)) + if response.Code != 200 { + t.Fatalf("status %d: %s", response.Code, response.Body.String()) + } + got := []string{} + if format == "details" { + var details []map[string]string + if err := json.Unmarshal(response.Body.Bytes(), &details); err != nil { + t.Fatal(err) + } + for _, detail := range details { + got = append(got, detail["Key"]) + // Metadata must remain separate from the identity encoded in Key. + for _, p := range tc.want { + if detail["Key"] == string(p.Key("")) { + if detail["Architecture"] != p.Architecture || detail["Architecture-Variant"] != p.ArchitectureVariant || detail["ShortKey"] != string(p.ShortKey("")) { + t.Errorf("incorrect package metadata: %v", detail) + } + } + } + } + } else if err := json.Unmarshal(response.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + want := []string{} + for _, p := range tc.want { + want = append(want, string(p.Key(""))) + } + sort.Strings(got) + sort.Strings(want) + if !reflect.DeepEqual(got, want) { + t.Errorf("package references:\n got %v\nwant %v", got, want) + } + if !reflect.DeepEqual(refs.Strings(), deb.NewPackageRefListFromPackageList(list).Strings()) { + t.Error("listing mutated source references") + } + }) + } + } +} diff --git a/api/snapshot.go b/api/snapshot.go index 7b6422f04..e193b25c2 100644 --- a/api/snapshot.go +++ b/api/snapshot.go @@ -800,6 +800,10 @@ func apiSnapshotsPull(c *gin.Context) { if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err } + err = taskCollectionFactory.SnapshotCollection().LoadComplete(freshToSnapshot) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err + } freshSourceSnapshot, err := taskCollectionFactory.SnapshotCollection().ByName(body.Source) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err @@ -873,7 +877,7 @@ func apiSnapshotsPull(c *gin.Context) { alreadySeen := map[string]bool{} _ = destinationPackageList.ForEachIndexed(func(pkg *deb.Package) error { - key := pkg.Architecture + "_" + pkg.Name + key := pkg.IndexArchitecture() + "_" + pkg.Name _, seen := alreadySeen[key] // If we haven't seen such name-architecture pair and were instructed to remove, remove it @@ -881,6 +885,9 @@ func apiSnapshotsPull(c *gin.Context) { // Remove all packages with the same name and architecture packageSearchResults := toPackageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}, true, false) for _, p := range packageSearchResults { + if p.ArchitectureVariant != pkg.ArchitectureVariant { + continue + } toPackageList.Remove(p) removedPackages = append(removedPackages, p.String()) } diff --git a/api/snapshot_pull_variant_test.go b/api/snapshot_pull_variant_test.go new file mode 100644 index 000000000..bad799e44 --- /dev/null +++ b/api/snapshot_pull_variant_test.go @@ -0,0 +1,138 @@ +package api + +import ( + "bytes" + "encoding/json" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "testing" + + ctx "github.com/aptly-dev/aptly/context" + "github.com/aptly-dev/aptly/deb" + "github.com/aptly-dev/aptly/utils" + "github.com/smira/flag" +) + +// Exercise the real pull entry point and persisted references, without package +// files or mirrors. Architecture selection remains base-architecture selection. +func TestSnapshotPullArchitectureVariant(t *testing.T) { + normal := &deb.Package{Name: "pull-test", Version: "1", Architecture: "amd64"} + variant := &deb.Package{Name: "pull-test", Version: "1", Architecture: "amd64", ArchitectureVariant: "amd64v3"} + normal2 := &deb.Package{Name: "pull-test", Version: "2", Architecture: "amd64"} + variant2 := &deb.Package{Name: "pull-test", Version: "2", Architecture: "amd64", ArchitectureVariant: "amd64v3"} + cases := []struct { + name string + dest, source, want []*deb.Package + noRemove, allMatches bool + }{ + {"normal", nil, []*deb.Package{normal}, []*deb.Package{normal}, false, false}, + {"variant", nil, []*deb.Package{variant}, []*deb.Package{variant}, false, false}, + {"both", nil, []*deb.Package{normal, variant}, []*deb.Package{normal, variant}, false, false}, + {"variant_into_normal", []*deb.Package{normal}, []*deb.Package{variant}, []*deb.Package{normal, variant}, false, false}, + {"normal_into_variant", []*deb.Package{variant}, []*deb.Package{normal}, []*deb.Package{normal, variant}, false, false}, + {"both_all_matches", nil, []*deb.Package{normal, variant}, []*deb.Package{normal, variant}, false, true}, + {"both_no_remove", nil, []*deb.Package{normal, variant}, []*deb.Package{normal, variant}, true, false}, + {"variant_into_normal_no_remove", []*deb.Package{normal}, []*deb.Package{variant}, []*deb.Package{normal, variant}, true, false}, + {"normal_into_variant_no_remove", []*deb.Package{variant}, []*deb.Package{normal}, []*deb.Package{normal, variant}, true, false}, + {"variant_into_normal_all_matches", []*deb.Package{normal}, []*deb.Package{variant}, []*deb.Package{normal, variant}, false, true}, + {"normal_into_variant_all_matches", []*deb.Package{variant}, []*deb.Package{normal}, []*deb.Package{normal, variant}, false, true}, + {"normal_replace", []*deb.Package{normal}, []*deb.Package{normal2}, []*deb.Package{normal2}, false, false}, + {"normal_no_remove", []*deb.Package{normal}, []*deb.Package{normal2}, []*deb.Package{normal, normal2}, true, false}, + {"normal_latest", nil, []*deb.Package{normal, normal2}, []*deb.Package{normal2}, false, false}, + {"normal_all_matches", nil, []*deb.Package{normal, normal2}, []*deb.Package{normal, normal2}, false, true}, + {"variant_replace_preserves_normal", []*deb.Package{normal, variant}, []*deb.Package{variant2}, []*deb.Package{normal, variant2}, false, false}, + {"latest_per_identity", nil, []*deb.Package{normal, normal2, variant, variant2}, []*deb.Package{normal2, variant2}, false, false}, + {"all_versions_both_identities", nil, []*deb.Package{normal, normal2, variant, variant2}, []*deb.Package{normal, normal2, variant, variant2}, false, true}, + } + for _, tc := range cases { + for _, mode := range []string{"inferred", "explicit"} { + t.Run(tc.name+"/"+mode, func(t *testing.T) { + savedConfig := utils.Config + t.Cleanup(func() { utils.Config = savedConfig }) + dir := t.TempDir() + configPath := filepath.Join(dir, "aptly.conf") + configData, err := json.Marshal(map[string]interface{}{"rootDir": dir, "architectures": []string{}}) + if err != nil { + t.Fatal(err) + } + if err = os.WriteFile(configPath, configData, 0600); err != nil { + t.Fatal(err) + } + flags := flag.NewFlagSet("pull-test", flag.ContinueOnError) + flags.String("config", configPath, "") + // Empty destinations need explicit architecture; populated destinations + // exercise automatic base-architecture discovery. + arch := "" + if len(tc.dest) == 0 || mode == "explicit" { + arch = "amd64" + } + flags.String("architectures", arch, "") + flags.Bool("no-lock", false, "") + flags.Int("db-open-attempts", 1, "") + flags.Bool("no-deps", true, "") + flags.Bool("no-remove", tc.noRemove, "") + flags.Bool("all-matches", tc.allMatches, "") + flags.Bool("dry-run", false, "") + testContext, err := ctx.NewContext(flags) + if err != nil { + t.Fatal(err) + } + t.Cleanup(testContext.Shutdown) + factory := testContext.NewCollectionFactory() + refs := func(packages []*deb.Package) *deb.PackageRefList { + list := deb.NewPackageList() + for _, p := range packages { + if err := list.Add(p); err != nil { + t.Fatal(err) + } + } + return deb.NewPackageRefListFromPackageList(list) + } + for _, p := range append(append([]*deb.Package{}, tc.dest...), tc.source...) { + if err := factory.PackageCollection().Update(p); err != nil { + t.Fatal(err) + } + } + for name, packages := range map[string][]*deb.Package{"base": tc.dest, "source": tc.source} { + if err := factory.SnapshotCollection().Add(deb.NewSnapshotFromRefList(name, nil, refs(packages), "")); err != nil { + t.Fatal(err) + } + } + body, err := json.Marshal(snapshotsPullParams{Source: "source", Destination: "result", Queries: []string{"pull-test"}}) + if err != nil { + t.Fatal(err) + } + url := "/api/snapshots/base/pull?no-deps=1" + if tc.noRemove { + url += "&no-remove=1" + } + if tc.allMatches { + url += "&all-matches=1" + } + req := httptest.NewRequest("POST", url, bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + response := httptest.NewRecorder() + savedContext := context + t.Cleanup(func() { context = savedContext }) + Router(testContext).ServeHTTP(response, req) + if response.Code != 201 { + t.Fatalf("pull status %d: %s", response.Code, response.Body.String()) + } + snapshots := testContext.NewCollectionFactory().SnapshotCollection() + result, err := snapshots.ByName("result") + if err != nil { + t.Fatal(err) + } + if err = snapshots.LoadComplete(result); err != nil { + t.Fatal(err) + } + got, want := result.RefList().Strings(), refs(tc.want).Strings() + if !reflect.DeepEqual(got, want) { + t.Errorf("persisted package identities:\n got %v\nwant %v", got, want) + } + }) + } + } +} diff --git a/api/snapshot_test.go b/api/snapshot_test.go index 36c508093..5cf484c1e 100644 --- a/api/snapshot_test.go +++ b/api/snapshot_test.go @@ -1,7 +1,14 @@ package api import ( + "bytes" "encoding/json" + "os" + "path/filepath" + + ctx "github.com/aptly-dev/aptly/context" + "github.com/aptly-dev/aptly/utils" + "github.com/smira/flag" "github.com/aptly-dev/aptly/deb" . "gopkg.in/check.v1" @@ -51,3 +58,58 @@ func (s *SnapshotsSuite) TestGetSnapshotsReturns500OnCorruptRefList(c *C) { c.Assert(response.Code, Equals, 500) c.Assert(response.Body.String(), Matches, ".*msgpack.*|.*decode.*") } + +// Pull must load the destination references before inferring architectures or +// preserving existing packages. Both fixtures are ordinary amd64 packages. +func (*SnapshotsSuite) TestPullPreservesDestinationPackages(c *C) { + savedConfig, savedContext := utils.Config, context + defer func() { utils.Config, context = savedConfig, savedContext }() + dir := c.MkDir() + configPath := filepath.Join(dir, "aptly.conf") + configData, err := json.Marshal(map[string]interface{}{"rootDir": dir, "architectures": []string{}}) + c.Assert(err, IsNil) + c.Assert(os.WriteFile(configPath, configData, 0600), IsNil) + flags := flag.NewFlagSet("pull-test", flag.ContinueOnError) + flags.String("config", configPath, "") + flags.String("architectures", "", "") + flags.Bool("no-lock", false, "") + flags.Int("db-open-attempts", 1, "") + testContext, err := ctx.NewContext(flags) + c.Assert(err, IsNil) + defer testContext.Shutdown() + s := &APISuite{context: testContext, router: Router(testContext)} + factory := s.context.NewCollectionFactory() + for _, mode := range []string{"inferred", "explicit"} { + a := &deb.Package{Name: "package-a", Version: "1", Architecture: "amd64"} + b := &deb.Package{Name: "package-b", Version: "1", Architecture: "amd64"} + for name, pkg := range map[string]*deb.Package{"base": a, "source": b} { + c.Assert(factory.PackageCollection().Update(pkg), IsNil) + list := deb.NewPackageList() + c.Assert(list.Add(pkg), IsNil) + snapshot := deb.NewSnapshotFromPackageList("pull-"+name+"-"+mode, nil, list, "") + c.Assert(factory.SnapshotCollection().Add(snapshot), IsNil) + } + params := snapshotsPullParams{ + Source: "pull-source-" + mode, + Destination: "pull-result-" + mode, + Queries: []string{"package-b"}, + } + if mode == "explicit" { + params.Architectures = []string{"amd64"} + } + body, err := json.Marshal(params) + c.Assert(err, IsNil) + response, err := s.HTTPRequest("POST", "/api/snapshots/pull-base-"+mode+"/pull?no-remove=1&no-deps=1", bytes.NewReader(body)) + c.Assert(err, IsNil) + if !c.Check(response.Code, Equals, 201, Commentf("%s: %s", mode, response.Body.String())) { + continue + } + snapshots := s.context.NewCollectionFactory().SnapshotCollection() + result, err := snapshots.ByName(params.Destination) + c.Assert(err, IsNil) + c.Assert(snapshots.LoadComplete(result), IsNil) + c.Check(result.RefList().Strings(), DeepEquals, []string{ + string(a.Key("")), string(b.Key("")), + }, Commentf("%s: preserve A and add B", mode)) + } +} diff --git a/cmd/cmd_test.go b/cmd/cmd_test.go new file mode 100644 index 000000000..4c86d9c6d --- /dev/null +++ b/cmd/cmd_test.go @@ -0,0 +1,12 @@ +package cmd + +import ( + "testing" + + "gopkg.in/check.v1" +) + +// Launch gocheck tests. +func Test(t *testing.T) { + check.TestingT(t) +} diff --git a/cmd/repo_add_variant_test.go b/cmd/repo_add_variant_test.go new file mode 100644 index 000000000..3481eb2a1 --- /dev/null +++ b/cmd/repo_add_variant_test.go @@ -0,0 +1,151 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + + ctx "github.com/aptly-dev/aptly/context" + "github.com/aptly-dev/aptly/deb" + "github.com/aptly-dev/aptly/utils" + "github.com/smira/flag" + + "gopkg.in/check.v1" +) + +type RepoAddVariantSuite struct{} + +var _ = check.Suite(&RepoAddVariantSuite{}) + +// Every fixture has the same uninformative basename. Architecture-Variant is +// supplied exclusively by control metadata, and rebuilt packages change payload. +func (*RepoAddVariantSuite) TestRepoAddArchitectureVariant(c *check.C) { + builder, err := exec.LookPath("dpkg-deb") + c.Assert(err, check.IsNil, check.Commentf("these real .deb import tests require dpkg-deb")) + type fixture struct { + path string + pkg *deb.Package + data []byte + } + fixtures := map[string]fixture{} + for _, spec := range []struct{ id, variant, version, payload string }{ + {"normal", "", "1", "original"}, {"variant", "amd64v3", "1", "original"}, + {"normal-rebuilt", "", "1", "rebuilt"}, {"variant-rebuilt", "amd64v3", "1", "rebuilt"}, + {"normal-v2", "", "2", "new version"}, {"variant-v2", "amd64v3", "2", "new version"}, + } { + dir := c.MkDir() + root := filepath.Join(dir, "root") + c.Assert(os.MkdirAll(filepath.Join(root, "DEBIAN"), 0755), check.IsNil) + control := fmt.Sprintf("Package: import-test\nVersion: %s\nArchitecture: amd64\nMaintainer: Test \nDescription: tiny import fixture\n", spec.version) + if spec.variant != "" { + control += "Architecture-Variant: " + spec.variant + "\n" + } + c.Assert(os.WriteFile(filepath.Join(root, "DEBIAN", "control"), []byte(control), 0644), check.IsNil) + c.Assert(os.WriteFile(filepath.Join(root, "payload"), []byte(spec.payload), 0644), check.IsNil) + path := filepath.Join(dir, "payload.deb") + output, err := exec.Command(builder, "--build", "--root-owner-group", root, path).CombinedOutput() + c.Assert(err, check.IsNil, check.Commentf("build fixture: %s", output)) + checksums, err := utils.ChecksumsForFile(path) + c.Assert(err, check.IsNil) + pkg := &deb.Package{Name: "import-test", Version: spec.version, Architecture: "amd64", ArchitectureVariant: spec.variant, V06Plus: true} + pkg.UpdateFiles(deb.PackageFiles{{Filename: "payload.deb", Checksums: checksums}}) + data, err := os.ReadFile(path) + c.Assert(err, check.IsNil) + fixtures[spec.id] = fixture{path, pkg, data} + } + for _, tc := range []struct { + name string + initial []string + incoming string + force, reject bool + want []string + }{ + {"normal_empty", nil, "normal", false, false, []string{"normal"}}, + {"variant_empty", nil, "variant", false, false, []string{"variant"}}, + {"normal_then_variant", []string{"normal"}, "variant", false, false, []string{"normal", "variant"}}, + {"variant_then_normal", []string{"variant"}, "normal", false, false, []string{"normal", "variant"}}, + {"identical_normal", []string{"normal", "variant"}, "normal", false, false, []string{"normal", "variant"}}, + {"identical_variant", []string{"normal", "variant"}, "variant", false, false, []string{"normal", "variant"}}, + {"conflicting_normal_without_force", []string{"normal", "variant"}, "normal-rebuilt", false, true, []string{"normal", "variant"}}, + {"conflicting_variant_without_force", []string{"normal", "variant"}, "variant-rebuilt", false, true, []string{"normal", "variant"}}, + {"force_normal_preserves_variant", []string{"normal", "variant"}, "normal-rebuilt", true, false, []string{"normal-rebuilt", "variant"}}, + {"force_variant_preserves_normal", []string{"normal", "variant"}, "variant-rebuilt", true, false, []string{"normal", "variant-rebuilt"}}, + {"force_identical_normal", []string{"normal", "variant"}, "normal", true, false, []string{"normal", "variant"}}, + {"force_identical_variant", []string{"normal", "variant"}, "variant", true, false, []string{"normal", "variant"}}, + {"force_new_variant_family", []string{"normal"}, "variant", true, false, []string{"normal", "variant"}}, + {"force_new_normal_family", []string{"variant"}, "normal", true, false, []string{"normal", "variant"}}, + {"force_normal_only", []string{"normal"}, "normal-rebuilt", true, false, []string{"normal-rebuilt"}}, + {"force_variant_only", []string{"variant"}, "variant-rebuilt", true, false, []string{"variant-rebuilt"}}, + {"force_normal_new_version", []string{"normal", "variant"}, "normal-v2", true, false, []string{"normal", "variant", "normal-v2"}}, + {"force_variant_new_version", []string{"normal", "variant"}, "variant-v2", true, false, []string{"normal", "variant", "variant-v2"}}, + } { + c.Logf("case: %s", tc.name) + func() { + savedConfig, savedContext := utils.Config, context + defer func() { utils.Config, context = savedConfig, savedContext }() + dir := c.MkDir() + configPath := filepath.Join(dir, "aptly.conf") + config, err := json.Marshal(map[string]interface{}{"rootDir": dir, "gpgProvider": "internal"}) + c.Assert(err, check.IsNil) + c.Assert(os.WriteFile(configPath, config, 0600), check.IsNil) + flags := flag.NewFlagSet("repo-add-test", flag.ContinueOnError) + flags.String("config", configPath, "") + flags.Bool("no-lock", false, "") + flags.Int("db-open-attempts", 1, "") + flags.Bool("force-replace", false, "") + flags.Bool("remove-files", false, "") + testContext, err := ctx.NewContext(flags) + c.Assert(err, check.IsNil) + context = testContext + defer testContext.Shutdown() + factory := testContext.NewCollectionFactory() + c.Assert(factory.LocalRepoCollection().Add(deb.NewLocalRepo("test", "")), check.IsNil) + add := func(id string) error { return aptlyRepoAdd(makeCmdRepoAdd(), []string{"test", fixtures[id].path}) } + for _, id := range tc.initial { + c.Assert(add(id), check.IsNil) + } + c.Assert(flags.Set("force-replace", fmt.Sprint(tc.force)), check.IsNil) + err = add(tc.incoming) + c.Assert(err != nil, check.Equals, tc.reject, check.Commentf("case %s: repo add error: %v", tc.name, err)) + // Reload persisted references and metadata rather than checking an in-memory list. + factory = testContext.NewCollectionFactory() + repo, err := factory.LocalRepoCollection().ByName("test") + c.Assert(err, check.IsNil) + c.Assert(factory.LocalRepoCollection().LoadComplete(repo), check.IsNil) + got := map[string]string{} + for _, key := range repo.RefList().Refs { + p, err := factory.PackageCollection().ByKey(key) + c.Assert(err, check.IsNil) + c.Check(p.Architecture, check.Equals, "amd64", check.Commentf("case %s", tc.name)) + c.Assert(p.Files(), check.HasLen, 1) + got[string(p.ShortKey(""))] = p.Files()[0].Checksums.SHA256 + } + want := map[string]string{} + for _, id := range tc.want { + p := fixtures[id].pkg + want[string(p.ShortKey(""))] = p.Files()[0].Checksums.SHA256 + } + c.Check(got, check.DeepEquals, want, check.Commentf("case %s: persisted package identities", tc.name)) + // Import stores files and full DB records before resolving repo conflicts. + // Neither rejection nor replacement should corrupt/remove these pool blobs. + for _, id := range append(append([]string{}, tc.initial...), tc.incoming) { + f := fixtures[id] + p, err := factory.PackageCollection().ByKey(f.pkg.Key("")) + c.Assert(err, check.IsNil) + c.Check(p.ArchitectureVariant, check.Equals, f.pkg.ArchitectureVariant, check.Commentf("case %s", tc.name)) + stream, err := testContext.PackagePool().Open(p.Files()[0].PoolPath) + c.Assert(err, check.IsNil) + data, err := io.ReadAll(stream) + closeErr := stream.Close() + c.Assert(err, check.IsNil) + c.Assert(closeErr, check.IsNil) + c.Check(data, check.DeepEquals, f.data, check.Commentf("case %s: pool content for %s", tc.name, id)) + _, err = os.Stat(f.path) + c.Check(err, check.IsNil, check.Commentf("case %s: input file %s", tc.name, f.path)) + } + }() + } +} diff --git a/cmd/snapshot_diff.go b/cmd/snapshot_diff.go index ccbea32ee..992cf80a8 100644 --- a/cmd/snapshot_diff.go +++ b/cmd/snapshot_diff.go @@ -60,10 +60,10 @@ func aptlySnapshotDiff(cmd *commander.Command, args []string) error { verA = "-" verB = pdiff.Right.Version pkg = pdiff.Right.Name - arch = pdiff.Right.Architecture + arch = pdiff.Right.IndexArchitecture() } else { pkg = pdiff.Left.Name - arch = pdiff.Left.Architecture + arch = pdiff.Left.IndexArchitecture() verA = pdiff.Left.Version if pdiff.Right == nil { verB = "-" diff --git a/cmd/snapshot_pull.go b/cmd/snapshot_pull.go index df0c93315..78806e596 100644 --- a/cmd/snapshot_pull.go +++ b/cmd/snapshot_pull.go @@ -117,7 +117,7 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error { alreadySeen := map[string]bool{} _ = result.ForEachIndexed(func(pkg *deb.Package) error { - key := pkg.Architecture + "_" + pkg.Name + key := pkg.IndexArchitecture() + "_" + pkg.Name _, seen := alreadySeen[key] // If we haven't seen such name-architecture pair and were instructed to remove, remove it @@ -125,6 +125,9 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error { // Remove all packages with the same name and architecture pS := packageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}, true, false) for _, p := range pS { + if p.ArchitectureVariant != pkg.ArchitectureVariant { + continue + } packageList.Remove(p) context.Progress().ColoredPrintf("@r[-]@| %s removed", p) } diff --git a/cmd/snapshot_pull_variant_test.go b/cmd/snapshot_pull_variant_test.go new file mode 100644 index 000000000..091179649 --- /dev/null +++ b/cmd/snapshot_pull_variant_test.go @@ -0,0 +1,105 @@ +package cmd + +import ( + "encoding/json" + "os" + "path/filepath" + + ctx "github.com/aptly-dev/aptly/context" + "github.com/aptly-dev/aptly/deb" + "github.com/aptly-dev/aptly/utils" + "github.com/smira/flag" + + "gopkg.in/check.v1" +) + +type SnapshotPullVariantSuite struct{} + +var _ = check.Suite(&SnapshotPullVariantSuite{}) + +// Exercise the real pull entry point and persisted references, without package +// files or mirrors. Architecture selection remains base-architecture selection. +func (*SnapshotPullVariantSuite) TestSnapshotPullArchitectureVariant(c *check.C) { + normal := &deb.Package{Name: "pull-test", Version: "1", Architecture: "amd64"} + variant := &deb.Package{Name: "pull-test", Version: "1", Architecture: "amd64", ArchitectureVariant: "amd64v3"} + normal2 := &deb.Package{Name: "pull-test", Version: "2", Architecture: "amd64"} + variant2 := &deb.Package{Name: "pull-test", Version: "2", Architecture: "amd64", ArchitectureVariant: "amd64v3"} + cases := []struct { + name string + dest, source, want []*deb.Package + noRemove, allMatches bool + }{ + {"normal", nil, []*deb.Package{normal}, []*deb.Package{normal}, false, false}, + {"variant", nil, []*deb.Package{variant}, []*deb.Package{variant}, false, false}, + {"both", nil, []*deb.Package{normal, variant}, []*deb.Package{normal, variant}, false, false}, + {"variant_into_normal", []*deb.Package{normal}, []*deb.Package{variant}, []*deb.Package{normal, variant}, false, false}, + {"normal_into_variant", []*deb.Package{variant}, []*deb.Package{normal}, []*deb.Package{normal, variant}, false, false}, + {"both_all_matches", nil, []*deb.Package{normal, variant}, []*deb.Package{normal, variant}, false, true}, + {"both_no_remove", nil, []*deb.Package{normal, variant}, []*deb.Package{normal, variant}, true, false}, + {"variant_into_normal_no_remove", []*deb.Package{normal}, []*deb.Package{variant}, []*deb.Package{normal, variant}, true, false}, + {"normal_into_variant_no_remove", []*deb.Package{variant}, []*deb.Package{normal}, []*deb.Package{normal, variant}, true, false}, + {"variant_into_normal_all_matches", []*deb.Package{normal}, []*deb.Package{variant}, []*deb.Package{normal, variant}, false, true}, + {"normal_into_variant_all_matches", []*deb.Package{variant}, []*deb.Package{normal}, []*deb.Package{normal, variant}, false, true}, + {"normal_replace", []*deb.Package{normal}, []*deb.Package{normal2}, []*deb.Package{normal2}, false, false}, + {"normal_no_remove", []*deb.Package{normal}, []*deb.Package{normal2}, []*deb.Package{normal, normal2}, true, false}, + {"normal_latest", nil, []*deb.Package{normal, normal2}, []*deb.Package{normal2}, false, false}, + {"normal_all_matches", nil, []*deb.Package{normal, normal2}, []*deb.Package{normal, normal2}, false, true}, + {"variant_replace_preserves_normal", []*deb.Package{normal, variant}, []*deb.Package{variant2}, []*deb.Package{normal, variant2}, false, false}, + {"latest_per_identity", nil, []*deb.Package{normal, normal2, variant, variant2}, []*deb.Package{normal2, variant2}, false, false}, + {"all_versions_both_identities", nil, []*deb.Package{normal, normal2, variant, variant2}, []*deb.Package{normal, normal2, variant, variant2}, false, true}, + } + for _, tc := range cases { + c.Logf("case: %s", tc.name) + func() { + savedConfig := utils.Config + defer func() { utils.Config = savedConfig }() + dir := c.MkDir() + configPath := filepath.Join(dir, "aptly.conf") + configData, err := json.Marshal(map[string]interface{}{"rootDir": dir, "architectures": []string{}}) + c.Assert(err, check.IsNil) + c.Assert(os.WriteFile(configPath, configData, 0600), check.IsNil) + flags := flag.NewFlagSet("pull-test", flag.ContinueOnError) + flags.String("config", configPath, "") + // Empty destinations need explicit architecture; populated destinations + // exercise automatic base-architecture discovery. + arch := "" + if len(tc.dest) == 0 { + arch = "amd64" + } + flags.String("architectures", arch, "") + flags.Bool("no-lock", false, "") + flags.Int("db-open-attempts", 1, "") + flags.Bool("no-deps", true, "") + flags.Bool("no-remove", tc.noRemove, "") + flags.Bool("all-matches", tc.allMatches, "") + flags.Bool("dry-run", false, "") + testContext, err := ctx.NewContext(flags) + c.Assert(err, check.IsNil) + defer testContext.Shutdown() + factory := testContext.NewCollectionFactory() + refs := func(packages []*deb.Package) *deb.PackageRefList { + list := deb.NewPackageList() + for _, p := range packages { + c.Assert(list.Add(p), check.IsNil) + } + return deb.NewPackageRefListFromPackageList(list) + } + for _, p := range append(append([]*deb.Package{}, tc.dest...), tc.source...) { + c.Assert(factory.PackageCollection().Update(p), check.IsNil) + } + for name, packages := range map[string][]*deb.Package{"base": tc.dest, "source": tc.source} { + c.Assert(factory.SnapshotCollection().Add(deb.NewSnapshotFromRefList(name, nil, refs(packages), "")), check.IsNil) + } + savedContext := context + context = testContext + defer func() { context = savedContext }() + c.Assert(aptlySnapshotPull(makeCmdSnapshotPull(), []string{"base", "source", "result", "pull-test"}), check.IsNil) + snapshots := testContext.NewCollectionFactory().SnapshotCollection() + result, err := snapshots.ByName("result") + c.Assert(err, check.IsNil) + c.Assert(snapshots.LoadComplete(result), check.IsNil) + got, want := result.RefList().Strings(), refs(tc.want).Strings() + c.Check(got, check.DeepEquals, want, check.Commentf("case %s: persisted package identities", tc.name)) + }() + } +} diff --git a/deb/import.go b/deb/import.go index 99d52be82..8acfe92d2 100644 --- a/deb/import.go +++ b/deb/import.go @@ -211,6 +211,9 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b if forceReplace { conflictingPackages := list.Search(Dependency{Pkg: p.Name, Version: p.Version, Relation: VersionEqual, Architecture: p.Architecture}, true, false) for _, cp := range conflictingPackages { + if cp.ArchitectureVariant != p.ArchitectureVariant { + continue + } reporter.Removed("%s removed due to conflict with package being added", cp) list.Remove(cp) } diff --git a/deb/list.go b/deb/list.go index 19e37593c..1a29f03a0 100644 --- a/deb/list.go +++ b/deb/list.go @@ -182,7 +182,7 @@ func (l *PackageList) FilterLatest() (*PackageList, error) { filtered := make(map[string]*Package, l.Len()) err := l.ForEach(func(p *Package) error { - key := p.Architecture + "|" + p.Name + key := p.IndexArchitecture() + "|" + p.Name if existing, found := filtered[key]; !found || CompareVersions(p.Version, existing.Version) > 0 { filtered[key] = p @@ -287,6 +287,19 @@ func (l *PackageList) Architectures(includeSource bool) (result []string) { return } +// IndexArchitectures returns repository index architectures present in the list. +// If includeSource is true, the source meta-architecture is included. +func (l *PackageList) IndexArchitectures(includeSource bool) (result []string) { + result = make([]string, 0, 10) + for _, pkg := range l.packages { + arch := pkg.IndexArchitecture() + if arch != ArchitectureAll && (arch != ArchitectureSource || includeSource) && !utils.StrSliceHasItem(result, arch) { + result = append(result, arch) + } + } + return +} + // Strings builds list of strings with package keys func (l *PackageList) Strings() []string { result := make([]string, l.Len()) diff --git a/deb/list_test.go b/deb/list_test.go index 4e4768e34..3f7d781da 100644 --- a/deb/list_test.go +++ b/deb/list_test.go @@ -134,6 +134,66 @@ func (s *PackageListSuite) TestAddLen(c *C) { c.Check(s.list.Add(s.p4), ErrorMatches, "package already exists and is different: .*") } +func (s *PackageListSuite) TestArchitectureVariantAdd(c *C) { + normal := &Package{Name: "example", Version: "1.0", Architecture: "amd64"} + variant := &Package{Name: "example", Version: "1.0", Architecture: "amd64", ArchitectureVariant: "amd64v3"} + + c.Assert(s.list.Add(normal), IsNil) + c.Assert(s.list.Add(variant), IsNil) + c.Check(s.list.Len(), Equals, 2) + var packages []*Package + c.Assert(s.list.ForEach(func(p *Package) error { + packages = append(packages, p) + return nil + }), IsNil) + c.Check(packages, HasLen, 2) + found := map[*Package]bool{} + for _, p := range packages { + found[p] = true + } + c.Check(found[normal], Equals, true) + c.Check(found[variant], Equals, true) +} + +func (s *PackageListSuite) TestArchitectureVariantHas(c *C) { + normal := &Package{Name: "example", Version: "1.0", Architecture: "amd64"} + variant := &Package{Name: "example", Version: "1.0", Architecture: "amd64", ArchitectureVariant: "amd64v3"} + + c.Check(s.list.Has(normal), Equals, false) + c.Check(s.list.Has(variant), Equals, false) + c.Assert(s.list.Add(normal), IsNil) + c.Check(s.list.Has(normal), Equals, true) + c.Check(s.list.Has(variant), Equals, false) + c.Assert(s.list.Add(variant), IsNil) + c.Check(s.list.Has(normal), Equals, true) + c.Check(s.list.Has(variant), Equals, true) +} + +func (s *PackageListSuite) TestArchitectureVariantRemove(c *C) { + normal := &Package{Name: "example", Version: "1.0", Architecture: "amd64"} + variant := &Package{Name: "example", Version: "1.0", Architecture: "amd64", ArchitectureVariant: "amd64v3"} + + c.Assert(s.list.Add(normal), IsNil) + c.Assert(s.list.Add(variant), IsNil) + s.list.Remove(variant) + c.Check(s.list.Len(), Equals, 1) + c.Check(s.list.Has(normal), Equals, true) + c.Check(s.list.Has(variant), Equals, false) +} + +func (s *PackageListSuite) TestArchitectureVariantFilterLatest(c *C) { + normal := &Package{Name: "example", Version: "1.0", Architecture: "amd64"} + variant := &Package{Name: "example", Version: "1.0", Architecture: "amd64", ArchitectureVariant: "amd64v3"} + + c.Assert(s.list.Add(normal), IsNil) + c.Assert(s.list.Add(variant), IsNil) + filtered, err := s.list.FilterLatest() + c.Assert(err, IsNil) + c.Check(filtered.Len(), Equals, 2) + c.Check(filtered.Has(normal), Equals, true) + c.Check(filtered.Has(variant), Equals, true) +} + func (s *PackageListSuite) TestRemove(c *C) { c.Check(s.list.Add(s.p1), IsNil) c.Check(s.list.Add(s.p3), IsNil) @@ -552,3 +612,34 @@ func (s *PackageListSuite) TestFilterLatestNil(c *C) { c.Assert(err, ErrorMatches, "package list is nil") c.Assert(filtered, IsNil) } + +func (s *PackageListSuite) TestArchitectureVariantIndexArchitectures(c *C) { + for _, p := range []*Package{ + {Name: "example", Version: "1.0", Architecture: "amd64"}, + {Name: "example", Version: "1.0", Architecture: "amd64", ArchitectureVariant: "amd64v3"}, + {Name: "example", Version: "2.0", Architecture: "amd64", ArchitectureVariant: "amd64v3"}, + {Name: "example", Version: "1.0", Architecture: "i386"}, + {Name: "data", Version: "1.0", Architecture: "all"}, + {Name: "example", Version: "1.0", Architecture: "source", IsSource: true}, + } { + c.Assert(s.list.Add(p), IsNil) + } + indexes := s.list.IndexArchitectures(false) + sort.Strings(indexes) + c.Check(indexes, DeepEquals, []string{"amd64", "amd64v3", "i386"}) + indexes = s.list.IndexArchitectures(true) + sort.Strings(indexes) + c.Check(indexes, DeepEquals, []string{"amd64", "amd64v3", "i386", "source"}) + base := s.list.Architectures(false) + sort.Strings(base) + c.Check(base, DeepEquals, []string{"amd64", "i386"}) +} + +func (s *PackageListSuite) TestArchitectureVariantFullNames(c *C) { + for _, variant := range []string{"", "amd64v3"} { + c.Assert(s.list.Add(&Package{Name: "test-package", Version: "1.0", Architecture: "amd64", ArchitectureVariant: variant}), IsNil) + } + names := s.list.FullNames() + sort.Strings(names) + c.Check(names, DeepEquals, []string{"test-package_1.0_amd64", "test-package_1.0_amd64v3"}) +} diff --git a/deb/package.go b/deb/package.go index 0af2a4b4a..ff3a2b799 100644 --- a/deb/package.go +++ b/deb/package.go @@ -16,9 +16,10 @@ import ( // Package is single instance of Debian package type Package struct { // Basic package properties - Name string - Version string - Architecture string + Name string + Version string + Architecture string + ArchitectureVariant string // If this source package, this field holds "real" architecture value, // while Architecture would be equal to "source" SourceArchitecture string @@ -68,16 +69,18 @@ var ( // NewPackageFromControlFile creates Package from parsed Debian control file func NewPackageFromControlFile(input Stanza) *Package { result := &Package{ - Name: input["Package"], - Version: input["Version"], - Architecture: input["Architecture"], - Source: input["Source"], - V06Plus: true, + Name: input["Package"], + Version: input["Version"], + Architecture: input["Architecture"], + ArchitectureVariant: input["Architecture-Variant"], + Source: input["Source"], + V06Plus: true, } delete(input, "Package") delete(input, "Version") delete(input, "Architecture") + delete(input, "Architecture-Variant") delete(input, "Source") filesize, _ := strconv.ParseInt(input["Size"], 10, 64) @@ -213,10 +216,18 @@ func NewInstallerPackageFromControlFile(input Stanza, repo *RemoteRepo, componen return p, nil } +// IndexArchitecture returns the variant when present, otherwise the base architecture. +func (p *Package) IndexArchitecture() string { + if p.ArchitectureVariant != "" { + return p.ArchitectureVariant + } + return p.Architecture +} + // Key returns unique key identifying package func (p *Package) Key(prefix string) []byte { if p.V06Plus { - return []byte(fmt.Sprintf("%sP%s %s %s %08x", prefix, p.Architecture, p.Name, p.Version, p.FilesHash)) + return []byte(fmt.Sprintf("%sP%s %s %s %08x", prefix, p.IndexArchitecture(), p.Name, p.Version, p.FilesHash)) } return p.ShortKey(prefix) @@ -224,12 +235,12 @@ func (p *Package) Key(prefix string) []byte { // ShortKey returns key for the package that should be unique in one list func (p *Package) ShortKey(prefix string) []byte { - return []byte(fmt.Sprintf("%sP%s %s %s", prefix, p.Architecture, p.Name, p.Version)) + return []byte(fmt.Sprintf("%sP%s %s %s", prefix, p.IndexArchitecture(), p.Name, p.Version)) } // String creates readable representation func (p *Package) String() string { - return fmt.Sprintf("%s_%s_%s", p.Name, p.Version, p.Architecture) + return fmt.Sprintf("%s_%s_%s", p.Name, p.Version, p.IndexArchitecture()) } // ExtendedStanza returns package stanza enhanced with aptly-specific fields @@ -292,6 +303,8 @@ func (p *Package) GetField(name string) string { return p.SourceArchitecture } return p.Architecture + case "Architecture-Variant": + return p.ArchitectureVariant case "Source": return p.Source case "Depends": @@ -339,6 +352,14 @@ func (p *Package) MatchesArchitecture(arch string) bool { return p.Architecture == arch } +// MatchesIndexArchitecture checks membership in a repository index architecture. +func (p *Package) MatchesIndexArchitecture(arch string) bool { + if p.Architecture == ArchitectureAll && arch != ArchitectureSource { + return true + } + return p.IndexArchitecture() == arch +} + func JoinErrors(errs ...error) error { var combinedErr error for _, err := range errs { @@ -437,7 +458,7 @@ func (p *Package) GetName() string { // GetFullName returns the package full name func (p *Package) GetFullName() string { - return strings.Join([]string{p.Name, p.Version, p.Architecture}, "_") + return strings.Join([]string{p.Name, p.Version, p.IndexArchitecture()}, "_") } // GetVersion returns package version @@ -594,6 +615,9 @@ func (p *Package) Stanza() (result Stanza) { result["Architecture"] = p.SourceArchitecture } else { result["Architecture"] = p.Architecture + if p.ArchitectureVariant != "" { + result["Architecture-Variant"] = p.ArchitectureVariant + } if p.Source != "" { result["Source"] = p.Source } @@ -682,7 +706,7 @@ func (p *Package) Stanza() (result Stanza) { func (p *Package) Equals(p2 *Package) bool { return p.Name == p2.Name && p.Version == p2.Version && p.SourceArchitecture == p2.SourceArchitecture && p.Architecture == p2.Architecture && p.Source == p2.Source && p.IsSource == p2.IsSource && - p.FilesHash == p2.FilesHash + p.ArchitectureVariant == p2.ArchitectureVariant && p.FilesHash == p2.FilesHash } // LinkFromPool links package file from pool to dist's pool location diff --git a/deb/package_collection.go b/deb/package_collection.go index 0f2e7b8f6..5be5583da 100644 --- a/deb/package_collection.go +++ b/deb/package_collection.go @@ -336,7 +336,7 @@ func (collection *PackageCollection) SearchByKey(arch, name, version string) (re panic(fmt.Sprintf("unable to load package: %s", err)) } - if pkg.Architecture == arch && pkg.Name == name && pkg.Version == version { + if pkg.IndexArchitecture() == arch && pkg.Name == name && pkg.Version == version { _ = result.Add(pkg) } } diff --git a/deb/package_collection_test.go b/deb/package_collection_test.go index 691a48c34..38d123cb8 100644 --- a/deb/package_collection_test.go +++ b/deb/package_collection_test.go @@ -65,6 +65,106 @@ func (s *PackageCollectionSuite) TestByKey(c *C) { c.Check(p2.Files()[0].Filename, Equals, "alien-arena-common_7.40-2_i386.deb") } +func architectureVariantCollectionPackage(variant string) *Package { + stanza := Stanza{ + "Package": "example", "Version": "1.0", "Architecture": "amd64", + "Filename": "pool/example_1.0_amd64.deb", "Size": "123", + "Depends": "test-dependency (>= 1.0)", "Priority": "optional", + } + if variant != "" { + stanza["Architecture-Variant"] = variant + stanza["Filename"] = "pool/example_1.0_" + variant + ".deb" + } + return NewPackageFromControlFile(stanza) +} + +func (s *PackageCollectionSuite) TestArchitectureVariantNormalRoundTrip(c *C) { + p := architectureVariantCollectionPackage("") + key := p.Key("") + c.Assert(s.collection.Update(p), IsNil) + loaded, err := s.collection.ByKey(key) + c.Assert(err, IsNil) + c.Check(loaded.Name, Equals, "example") + c.Check(loaded.Version, Equals, "1.0") + c.Check(loaded.Architecture, Equals, "amd64") + c.Check(loaded.ArchitectureVariant, Equals, "") + c.Check(loaded.IndexArchitecture(), Equals, "amd64") + c.Check(loaded.Key(""), DeepEquals, key) + c.Check(loaded.Equals(p), Equals, true) + c.Check(loaded.Files()[0].Filename, Equals, "example_1.0_amd64.deb") + c.Check(loaded.GetDependencies(0), DeepEquals, []string{"test-dependency (>= 1.0)"}) + c.Check(loaded.Extra()["Priority"], Equals, "optional") + _, hasVariant := loaded.Stanza()["Architecture-Variant"] + c.Check(hasVariant, Equals, false) +} + +func (s *PackageCollectionSuite) TestArchitectureVariantRoundTrip(c *C) { + p := architectureVariantCollectionPackage("amd64v3") + key := p.Key("") + c.Assert(s.collection.Update(p), IsNil) + loaded, err := s.collection.ByKey(key) + c.Assert(err, IsNil) + c.Check(loaded.Name, Equals, "example") + c.Check(loaded.Version, Equals, "1.0") + c.Check(loaded.Architecture, Equals, "amd64") + c.Check(loaded.ArchitectureVariant, Equals, "amd64v3") + c.Check(loaded.IndexArchitecture(), Equals, "amd64v3") + c.Check(loaded.Key(""), DeepEquals, key) + c.Check(loaded.Equals(p), Equals, true) + // Exercise offloaded records using the reloaded variant-aware key. + c.Check(loaded.Files()[0].Filename, Equals, "example_1.0_amd64v3.deb") + c.Check(loaded.GetDependencies(0), DeepEquals, []string{"test-dependency (>= 1.0)"}) + c.Check(loaded.Extra()["Priority"], Equals, "optional") + stanza := loaded.Stanza() + c.Check(stanza["Architecture"], Equals, "amd64") + c.Check(stanza["Architecture-Variant"], Equals, "amd64v3") +} + +func (s *PackageCollectionSuite) TestArchitectureVariantCoexistence(c *C) { + normal := architectureVariantCollectionPackage("") + variant := architectureVariantCollectionPackage("amd64v3") + c.Assert(s.collection.Update(normal), IsNil) + c.Assert(s.collection.Update(variant), IsNil) + refs := s.collection.AllPackageRefs() + c.Check(refs.Len(), Equals, 2) + c.Check(refs.Has(normal), Equals, true) + c.Check(refs.Has(variant), Equals, true) + for _, p := range []*Package{normal, variant} { + loaded, err := s.collection.ByKey(p.Key("")) + c.Assert(err, IsNil) + c.Check(loaded.Equals(p), Equals, true) + c.Check(loaded.Architecture, Equals, "amd64") + c.Check(loaded.ArchitectureVariant, Equals, p.ArchitectureVariant) + } +} + +func (s *PackageCollectionSuite) TestArchitectureVariantFullKeyLookup(c *C) { + normal := architectureVariantCollectionPackage("") + variant := architectureVariantCollectionPackage("amd64v3") + // Keep the file hash identical so the architecture token alone must + // distinguish the full keys and stored records. + variant.UpdateFiles(normal.Files()) + normalKey, variantKey := normal.Key(""), variant.Key("") + c.Assert(normalKey, Not(DeepEquals), variantKey) + c.Assert(s.collection.Update(normal), IsNil) + _, err := s.collection.ByKey(variantKey) + c.Assert(err, Equals, database.ErrNotFound) + c.Assert(s.collection.Update(variant), IsNil) + + loaded, err := s.collection.ByKey(variantKey) + c.Assert(err, IsNil) + c.Check(loaded.Key(""), DeepEquals, variantKey) + c.Check(loaded.Architecture, Equals, "amd64") + c.Check(loaded.ArchitectureVariant, Equals, "amd64v3") + c.Check(loaded.IndexArchitecture(), Equals, "amd64v3") + c.Check(loaded.Equals(variant), Equals, true) + c.Check(loaded.Equals(normal), Equals, false) + loadedNormal, err := s.collection.ByKey(normalKey) + c.Assert(err, IsNil) + c.Check(loadedNormal.Equals(normal), Equals, true) + c.Check(loadedNormal.ArchitectureVariant, Equals, "") +} + func (s *PackageCollectionSuite) TestByKeyOld0_3(c *C) { key := []byte("Pi386 vmware-view-open-client 4.5.0-297975+dfsg-4+b1") _ = s.db.Put(key, old0_3Package) diff --git a/deb/package_test.go b/deb/package_test.go index 165716b52..7d9d82ee1 100644 --- a/deb/package_test.go +++ b/deb/package_test.go @@ -43,6 +43,66 @@ func (s *PackageSuite) TestNewFromPara(c *C) { c.Check(p.deps.Depends, DeepEquals, []string{"libc6 (>= 2.7)", "alien-arena-data (>= 7.40)"}) } +func (s *PackageSuite) TestArchitectureVariantParsedSeparately(c *C) { + input, err := NewControlFileReader(bytes.NewBufferString("Package: example\nVersion: 1.0\nArchitecture: amd64\nArchitecture-Variant: amd64v3\nFilename: pool/example_1.0_amd64v3.deb\n"), false, false).ReadStanza() + c.Assert(err, IsNil) + p := NewPackageFromControlFile(input) + + c.Check(p.Architecture, Equals, "amd64") + c.Check(p.ArchitectureVariant, Equals, "amd64v3") + _, inExtra := p.Extra()["Architecture-Variant"] + c.Check(inExtra, Equals, false) + c.Check(p.GetField("Architecture"), Equals, "amd64") + c.Check(p.GetField("Architecture-Variant"), Equals, "amd64v3") +} + +func (s *PackageSuite) TestArchitectureVariantShortKey(c *C) { + input := Stanza{"Package": "example", "Version": "1.0", "Architecture": "amd64"} + normal := NewPackageFromControlFile(input.Copy()) + input["Architecture-Variant"] = "amd64v3" + variant := NewPackageFromControlFile(input) + + c.Check(normal.Architecture, Equals, "amd64") + c.Check(variant.Architecture, Equals, "amd64") + c.Check(string(normal.ShortKey("")), Equals, "Pamd64 example 1.0") + c.Check(string(variant.ShortKey("")), Equals, "Pamd64v3 example 1.0") + c.Check(string(variant.ShortKey("")), Not(Equals), string(normal.ShortKey(""))) + c.Check(string(variant.ShortKey("xD")), Equals, "xDPamd64v3 example 1.0") +} + +func (s *PackageSuite) TestArchitectureVariantKeyAndEquals(c *C) { + normal := &Package{Name: "example", Version: "1.0", Architecture: "amd64", V06Plus: true, FilesHash: 0x12345678} + variant := *normal + variant.ArchitectureVariant = "amd64v3" + + c.Check(normal.IndexArchitecture(), Equals, "amd64") + c.Check(variant.IndexArchitecture(), Equals, "amd64v3") + c.Check(string(normal.Key("")), Equals, "Pamd64 example 1.0 12345678") + c.Check(string(variant.Key("")), Equals, "Pamd64v3 example 1.0 12345678") + c.Check(string(variant.Key("xD")), Equals, "xDPamd64v3 example 1.0 12345678") + c.Check(normal.Equals(&variant), Equals, false) + c.Check(variant.Equals(normal), Equals, false) + otherVariant := variant + c.Check(variant.Equals(&otherVariant), Equals, true) + otherVariant.ArchitectureVariant = "amd64v4" + c.Check(variant.Equals(&otherVariant), Equals, false) + + variant.V06Plus = false + c.Check(string(variant.Key("")), Equals, "Pamd64v3 example 1.0") + c.Check(string(variant.Key("xD")), Equals, "xDPamd64v3 example 1.0") +} + +func (s *PackageSuite) TestArchitectureVariantStanza(c *C) { + s.stanza["Architecture"] = "amd64" + s.stanza["Architecture-Variant"] = "amd64v3" + p := NewPackageFromControlFile(s.stanza) + stanza := p.Stanza() + + c.Check(p.Architecture, Equals, "amd64") + c.Check(stanza["Architecture"], Equals, "amd64") + c.Check(stanza["Architecture-Variant"], Equals, "amd64v3") +} + func (s *PackageSuite) TestNewUdebFromPara(c *C) { stanza, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta), false, false).ReadStanza() p := NewUdebPackageFromControlFile(stanza) @@ -520,3 +580,39 @@ SHA256: bbb3a2cb07f741c3995b6d4bb08d772d83582b93a0236d4ea7736bc0370fc320` const installerPackageMeta = `9d8bb14044dee520f4706ab197dfff10e9e39ecb3c1a402331712154e8284b2e ./MANIFEST.udebs dab96042d8e25e0f6bbb8d7c5bd78543afb5eb31a4a8b122ece68ab197228028 ./udeb.list` + +func (s *PackageSuite) TestArchitectureVariantMatchesIndexArchitecture(c *C) { + for _, tc := range []struct { + base, variant, target string + matches bool + }{ + {"amd64", "", "amd64", true}, + {"amd64", "", "amd64v3", false}, + {"amd64", "amd64v3", "amd64", false}, + {"amd64", "amd64v3", "amd64v3", true}, + {"all", "", "amd64", true}, + {"all", "", "amd64v3", true}, + {"all", "", "source", false}, + {"source", "", "source", true}, + {"source", "", "amd64v3", false}, + } { + p := &Package{Architecture: tc.base, ArchitectureVariant: tc.variant, IsSource: tc.base == "source"} + c.Check(p.MatchesIndexArchitecture(tc.target), Equals, tc.matches, Commentf("base=%s variant=%s target=%s", tc.base, tc.variant, tc.target)) + } + p := &Package{Architecture: "amd64", ArchitectureVariant: "amd64v3"} + c.Check(p.MatchesArchitecture("amd64"), Equals, true) + c.Check(p.MatchesArchitecture("amd64v3"), Equals, false) +} + +func (s *PackageSuite) TestArchitectureVariantDisplayNames(c *C) { + for _, variant := range []string{"", "amd64v3"} { + p := &Package{Name: "test-package", Version: "1.0", Architecture: "amd64", ArchitectureVariant: variant} + expected := "test-package_1.0_amd64" + if variant != "" { + expected = "test-package_1.0_amd64v3" + } + c.Check(p.String(), Equals, expected) + c.Check(p.GetFullName(), Equals, expected) + c.Check(p.GetArchitecture(), Equals, "amd64") + } +} diff --git a/deb/publish.go b/deb/publish.go index 8ae71df94..ffdf83152 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -879,7 +879,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP if !p.rePublishing { if len(p.Architectures) == 0 { for _, list := range lists { - p.Architectures = append(p.Architectures, list.Architectures(true)...) + p.Architectures = append(p.Architectures, list.IndexArchitectures(true)...) } } @@ -937,7 +937,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP } for _, arch := range p.Architectures { - if pkg.MatchesArchitecture(arch) { + if pkg.MatchesIndexArchitecture(arch) { hadUdebs = hadUdebs || pkg.IsUdeb var relPath string @@ -975,7 +975,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP batch := tempDB.CreateBatch() for _, arch := range p.Architectures { - if pkg.MatchesArchitecture(arch) { + if pkg.MatchesIndexArchitecture(arch) { var bufWriter *bufio.Writer if !p.SkipContents && !pkg.IsInstaller { diff --git a/deb/publish_test.go b/deb/publish_test.go index 20f78c4f9..415653863 100644 --- a/deb/publish_test.go +++ b/deb/publish_test.go @@ -379,6 +379,83 @@ func (s *PublishedRepoSuite) TestUpdate(c *C) { c.Assert(result.RemovedComponents(), DeepEquals, []string{}) } +func (s *PublishedRepoSuite) checkArchitectureVariantPublication(c *C, architectures []string) { + list := NewPackageList() + for _, arch := range []string{"amd64", "amd64v3"} { + filename := "example_1.0_" + arch + ".deb" + stanza := Stanza{"Package": "example", "Version": "1.0", "Architecture": "amd64", "Filename": filename} + if arch == "amd64v3" { + stanza["Architecture-Variant"] = "amd64v3" + } + p := NewPackageFromControlFile(stanza) + inputPath := filepath.Join(c.MkDir(), filename) + c.Assert(os.WriteFile(inputPath, nil, 0644), IsNil) + var err error + p.Files()[0].PoolPath, err = s.packagePool.Import(inputPath, filename, &p.Files()[0].Checksums, false, s.cs) + c.Assert(err, IsNil) + p.UpdateFiles(p.Files()) + c.Assert(s.packageCollection.Update(p), IsNil) + c.Assert(list.Add(p), IsNil) + } + local := NewLocalRepo("architecture-variant", "") + local.packageRefs = NewPackageRefListFromPackageList(list) + c.Assert(s.factory.LocalRepoCollection().Add(local), IsNil) + repo, err := NewPublishedRepo("", "variant-test", "test", architectures, []string{"main"}, []interface{}{local}, s.factory, false) + c.Assert(err, IsNil) + repo.SkipContents = true + c.Assert(repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false, ""), IsNil) + + root := filepath.Join(s.publishedStorage.PublicPath(), "variant-test") + releaseData, err := os.ReadFile(filepath.Join(root, "dists/test/Release")) + c.Assert(err, IsNil) + release, err := NewControlFileReader(bytes.NewReader(releaseData), true, false).ReadStanza() + c.Assert(err, IsNil) + c.Check(release["Architectures"], Equals, "amd64 amd64v3") + + for _, arch := range []string{"amd64", "amd64v3"} { + filename := "example_1.0_" + arch + ".deb" + c.Check(filepath.Join(root, "pool/main/e/example", filename), PathExists) + indexPath := filepath.Join(root, "dists/test/main", "binary-"+arch, "Packages") + data, err := os.ReadFile(indexPath) + if !c.Check(err, IsNil, Commentf("index %s must exist", arch)) { + continue + } + reader := NewControlFileReader(bytes.NewReader(data), false, false) + var entries []Stanza + for { + entry, err := reader.ReadStanza() + c.Assert(err, IsNil) + if entry == nil { + break + } + entries = append(entries, entry) + } + c.Check(entries, HasLen, 1, Commentf("index %s must contain exactly one package", arch)) + var filenames []string + for _, entry := range entries { + filenames = append(filenames, entry["Filename"]) + c.Check(entry["Package"], Equals, "example") + c.Check(entry["Version"], Equals, "1.0") + c.Check(entry["Architecture"], Equals, "amd64") + if entry["Filename"] == "pool/main/e/example/example_1.0_amd64v3.deb" { + c.Check(entry["Architecture-Variant"], Equals, "amd64v3") + } else { + _, hasVariant := entry["Architecture-Variant"] + c.Check(hasVariant, Equals, false) + } + } + c.Check(filenames, DeepEquals, []string{"pool/main/e/example/" + filename}, Commentf("index %s package membership", arch)) + } +} + +func (s *PublishedRepoSuite) TestArchitectureVariantPublishAutoArchitectures(c *C) { + s.checkArchitectureVariantPublication(c, nil) +} + +func (s *PublishedRepoSuite) TestArchitectureVariantPublishExplicitArchitectures(c *C) { + s.checkArchitectureVariantPublication(c, []string{"amd64", "amd64v3"}) +} + func (s *PublishedRepoSuite) TestPublish(c *C) { err := s.repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false, "") c.Assert(err, IsNil) diff --git a/deb/query.go b/deb/query.go index ea5fd255c..e781cb764 100644 --- a/deb/query.go +++ b/deb/query.go @@ -261,7 +261,11 @@ func (q *DependencyQuery) String() string { // Matches on specific properties func (q *PkgQuery) Matches(pkg PackageLike) bool { - return pkg.GetName() == q.Pkg && pkg.GetVersion() == q.Version && pkg.GetArchitecture() == q.Arch + arch := pkg.GetArchitecture() + if indexed, ok := pkg.(interface{ IndexArchitecture() string }); ok { + arch = indexed.IndexArchitecture() + } + return pkg.GetName() == q.Pkg && pkg.GetVersion() == q.Version && arch == q.Arch } // Fast is always true for package query diff --git a/deb/reflist.go b/deb/reflist.go index 0a5c1831e..a88ba9f9c 100644 --- a/deb/reflist.go +++ b/deb/reflist.go @@ -240,7 +240,7 @@ func (l *PackageRefList) Diff(r *PackageRefList, packageCollection *PackageColle if rel < 0 { // compaction: +(,A) -(B,) --> !(A,B) if len(result) > 0 && result[len(result)-1].Left == nil && result[len(result)-1].Right.Name == pl.Name && - result[len(result)-1].Right.Architecture == pl.Architecture { + result[len(result)-1].Right.IndexArchitecture() == pl.IndexArchitecture() { result[len(result)-1] = PackageDiff{Left: pl, Right: result[len(result)-1].Right} } else { result = append(result, PackageDiff{Left: pl, Right: nil}) @@ -250,7 +250,7 @@ func (l *PackageRefList) Diff(r *PackageRefList, packageCollection *PackageColle } else { // compaction: -(A,) +(,B) --> !(A,B) if len(result) > 0 && result[len(result)-1].Right == nil && result[len(result)-1].Left.Name == pr.Name && - result[len(result)-1].Left.Architecture == pr.Architecture { + result[len(result)-1].Left.IndexArchitecture() == pr.IndexArchitecture() { result[len(result)-1] = PackageDiff{Left: result[len(result)-1].Left, Right: pr} } else { result = append(result, PackageDiff{Left: nil, Right: pr}) diff --git a/deb/reflist_test.go b/deb/reflist_test.go index 3a7297ea7..41c2f696f 100644 --- a/deb/reflist_test.go +++ b/deb/reflist_test.go @@ -398,3 +398,104 @@ func (s *PackageRefListSuite) TestFilterLatestRefs(c *C) { c.Check(toStrSlice(result), DeepEquals, []string{"Pi386 dpkg 1.6", "Pi386 lib 1.2"}) } + +func (s *PackageRefListSuite) TestArchitectureVariantCoexistence(c *C) { + normal := &Package{Name: "example", Version: "1.0", Architecture: "amd64", V06Plus: true} + variant := &Package{Name: "example", Version: "1.0", Architecture: "amd64", ArchitectureVariant: "amd64v3", V06Plus: true} + c.Assert(s.list.Add(normal), IsNil) + c.Assert(s.list.Add(variant), IsNil) + + refs := NewPackageRefListFromPackageList(s.list) + c.Check(toStrSlice(refs), DeepEquals, []string{ + "Pamd64 example 1.0 00000000", + "Pamd64v3 example 1.0 00000000", + }) + c.Check(refs.Has(normal), Equals, true) + c.Check(refs.Has(variant), Equals, true) +} + +func (s *PackageRefListSuite) TestArchitectureVariantMerge(c *C) { + normal := &Package{Name: "example", Version: "1.0", Architecture: "amd64", V06Plus: true} + variant := &Package{Name: "example", Version: "1.0", Architecture: "amd64", ArchitectureVariant: "amd64v3", V06Plus: true} + left, right := NewPackageList(), NewPackageList() + c.Assert(left.Add(normal), IsNil) + c.Assert(right.Add(variant), IsNil) + leftRefs := NewPackageRefListFromPackageList(left) + rightRefs := NewPackageRefListFromPackageList(right) + + for _, overrideMatching := range []bool{false, true} { + for _, ignoreConflicting := range []bool{false, true} { + for _, reverse := range []bool{false, true} { + l, r := leftRefs, rightRefs + if reverse { + l, r = r, l + } + merged := l.Merge(r, overrideMatching, ignoreConflicting) + c.Check(toStrSlice(merged), DeepEquals, []string{ + "Pamd64 example 1.0 00000000", + "Pamd64v3 example 1.0 00000000", + }, Commentf("overrideMatching=%v ignoreConflicting=%v reverse=%v", overrideMatching, ignoreConflicting, reverse)) + } + } + } +} + +func (s *PackageRefListSuite) TestArchitectureVariantFilterLatestRefs(c *C) { + for _, variant := range []string{"", "amd64v3"} { + for _, version := range []string{"1.0", "2.0"} { + c.Assert(s.list.Add(&Package{ + Name: "example", Version: version, Architecture: "amd64", ArchitectureVariant: variant, V06Plus: true, + }), IsNil) + } + } + refs := NewPackageRefListFromPackageList(s.list) + c.Assert(refs.Len(), Equals, 4) + refs.FilterLatestRefs() + c.Check(toStrSlice(refs), DeepEquals, []string{ + "Pamd64 example 2.0 00000000", + "Pamd64v3 example 2.0 00000000", + }) +} + +func (s *PackageRefListSuite) TestArchitectureVariantDiffCompaction(c *C) { + db, err := goleveldb.NewOpenDB(c.MkDir()) + c.Assert(err, IsNil) + defer func() { c.Check(db.Close(), IsNil) }() + coll := NewPackageCollection(db) + + normal := &Package{Name: "example", Version: "1.0", Architecture: "amd64", V06Plus: true} + // Equal and differing versions must both remain separate additions/removals. + for _, version := range []string{"1.0", "2.0"} { + variant := &Package{Name: "example", Version: version, Architecture: "amd64", ArchitectureVariant: "amd64v3", V06Plus: true} + c.Assert(coll.Update(normal), IsNil) + c.Assert(coll.Update(variant), IsNil) + left, right := NewPackageList(), NewPackageList() + c.Assert(left.Add(normal), IsNil) + c.Assert(right.Add(variant), IsNil) + leftRefs := NewPackageRefListFromPackageList(left) + rightRefs := NewPackageRefListFromPackageList(right) + + for _, reverse := range []bool{false, true} { + l, r := leftRefs, rightRefs + if reverse { + l, r = r, l + } + diff, err := l.Diff(r, coll) + c.Assert(err, IsNil) + context := Commentf("variant version=%s reverse=%v", version, reverse) + c.Check(diff, HasLen, 2, context) + var removed, added []string + for _, entry := range diff { + c.Check(entry.Left != nil && entry.Right != nil, Equals, false, context) + if entry.Left != nil { + removed = append(removed, string(entry.Left.Key(""))) + } + if entry.Right != nil { + added = append(added, string(entry.Right.Key(""))) + } + } + c.Check(removed, DeepEquals, toStrSlice(l), context) + c.Check(added, DeepEquals, toStrSlice(r), context) + } + } +} diff --git a/deb/remote.go b/deb/remote.go index a1a79d689..f302d12d4 100644 --- a/deb/remote.go +++ b/deb/remote.go @@ -682,6 +682,11 @@ func (repo *RemoteRepo) ApplyFilter(dependencyOptions int, filterQuery PackageQu emptyList := NewPackageList() emptyList.PrepareIndex() + architectures := repo.packageList.Architectures(false) + if len(architectures) == 0 { + architectures = repo.Architectures + } + oldLen = repo.packageList.Len() repo.packageList, err = repo.packageList.Filter(FilterOptions{ Queries: []PackageQuery{filterQuery}, @@ -689,7 +694,7 @@ func (repo *RemoteRepo) ApplyFilter(dependencyOptions int, filterQuery PackageQu Source: emptyList, WithSources: repo.DownloadSources, DependencyOptions: dependencyOptions, - Architectures: repo.Architectures, + Architectures: architectures, Progress: progress, }) if repo.packageList != nil { diff --git a/deb/remote_filter_variant_test.go b/deb/remote_filter_variant_test.go new file mode 100644 index 000000000..ddfec1ed2 --- /dev/null +++ b/deb/remote_filter_variant_test.go @@ -0,0 +1,107 @@ +package deb + +import ( + "reflect" + "sort" + "strings" + "testing" +) + +// ApplyFilter operates on the package list already parsed from downloaded +// indexes. All architecture information here comes from control metadata. +func TestRemoteRepoApplyFilterArchitectureVariant(t *testing.T) { + binary := func(name, variant, arch, depends, source string) *Package { + control := "Package: " + name + "\nVersion: 1\nArchitecture: " + arch + "\n" + if variant != "" { + control += "Architecture-Variant: " + variant + "\n" + } + if depends != "" { + control += "Depends: " + depends + "\n" + } + if source != "" { + control += "Source: " + source + "\n" + } + stanza, err := NewControlFileReader(strings.NewReader(control), false, false).ReadStanza() + if err != nil { + t.Fatal(err) + } + return NewPackageFromControlFile(stanza) + } + source, err := NewSourcePackageFromControlFile(Stanza{ + "Package": "source-app", "Version": "1", "Architecture": "any all", "Build-Depends": "build-tool", + }) + if err != nil { + t.Fatal(err) + } + appNormal := binary("app", "", "amd64", "helper", "") + appVariant := binary("app", "amd64v3", "amd64", "helper", "") + helperNormal := binary("helper", "", "amd64", "", "") + helperVariant := binary("helper", "amd64v3", "amd64", "", "") + helperAll := binary("helper", "", "all", "", "") + appAll := binary("app", "", "all", "helper", "") + sourceVariant := binary("app", "amd64v3", "amd64", "", "source-app") + sourceNormal := binary("app", "", "amd64", "", "source-app") + buildTool := binary("build-tool", "", "amd64", "", "") + for _, tc := range []struct { + name string + indexes []string + input, want []*Package + queryName string + withDeps, withSources bool + options int + }{ + {"variant_index_normal_dependency", []string{"amd64v3"}, []*Package{appVariant, helperNormal}, []*Package{appVariant, helperNormal}, "app", true, false, 0}, + {"variant_index_variant_dependency", []string{"amd64v3"}, []*Package{appVariant, helperVariant}, []*Package{appVariant, helperVariant}, "app", true, false, 0}, + {"variant_index_all_dependency", []string{"amd64v3"}, []*Package{appVariant, helperAll}, []*Package{appVariant, helperAll}, "app", true, false, 0}, + {"variant_index_all_selected", []string{"amd64v3"}, []*Package{appAll, helperVariant}, []*Package{appAll, helperVariant}, "app", true, false, 0}, + {"mixed_indexes_shared_dependency", []string{"amd64", "amd64v3"}, []*Package{appNormal, appVariant, helperNormal}, []*Package{appNormal, appVariant, helperNormal}, "app", true, false, 0}, + {"mixed_indexes_shared_all_dependency", []string{"amd64", "amd64v3"}, []*Package{appNormal, appVariant, helperAll}, []*Package{appNormal, appVariant, helperAll}, "app", true, false, 0}, + {"normal_index_binary_dependency", []string{"amd64"}, []*Package{appNormal, helperNormal}, []*Package{appNormal, helperNormal}, "app", true, false, 0}, + {"normal_index_all_dependency", []string{"amd64"}, []*Package{appNormal, helperAll}, []*Package{appNormal, helperAll}, "app", true, false, 0}, + {"dependencies_disabled", []string{"amd64v3"}, []*Package{appVariant, helperVariant}, []*Package{appVariant}, "app", false, false, 0}, + {"variant_follow_source", []string{"amd64v3"}, []*Package{sourceVariant, source}, []*Package{sourceVariant, source}, "app", true, false, DepFollowSource}, + {"normal_follow_source", []string{"amd64"}, []*Package{sourceNormal, source}, []*Package{sourceNormal, source}, "app", true, false, DepFollowSource}, + {"download_sources_explicit_name", []string{"amd64v3"}, []*Package{sourceVariant, source}, []*Package{sourceVariant, source}, "app", true, true, 0}, + // These protect existing behavior against blindly passing Architectures(false). + {"only_all_normal_index", []string{"amd64"}, []*Package{appAll, helperAll}, []*Package{appAll, helperAll}, "app", true, false, 0}, + {"only_all_variant_index", []string{"amd64v3"}, []*Package{appAll, helperAll}, []*Package{appAll, helperAll}, "app", true, false, 0}, + {"source_only", []string{"amd64"}, []*Package{source}, []*Package{source}, "source-app", true, true, DepFollowBuild}, + // Mirrors do not currently walk source-package build dependencies. Retaining + // that behavior avoids treating the source pseudo-architecture as a CPU arch. + {"source_build_dependencies_unchanged", []string{"amd64"}, []*Package{source, buildTool}, []*Package{source}, "source-app", true, true, DepFollowBuild}, + {"empty_selection", []string{"amd64v3"}, []*Package{appVariant, helperVariant}, nil, "absent", true, false, 0}, + } { + t.Run(tc.name, func(t *testing.T) { + list := NewPackageList() + for _, p := range tc.input { + if err := list.Add(p); err != nil { + t.Fatal(err) + } + } + bases := list.Architectures(false) + sort.Strings(bases) + t.Logf("index architectures=%v; metadata base architectures=%v", tc.indexes, bases) + repo := &RemoteRepo{Architectures: append([]string{}, tc.indexes...), FilterWithDeps: tc.withDeps, DownloadSources: tc.withSources, packageList: list} + oldLen, newLen, err := repo.ApplyFilter(tc.options, &FieldQuery{Field: "Name", Relation: VersionEqual, Value: tc.queryName}, nil) + if err != nil { + t.Fatal(err) + } + if oldLen != len(tc.input) { + t.Errorf("oldLen=%d, want %d", oldLen, len(tc.input)) + } + got := repo.packageList.FullNames() + want := make([]string, 0, len(tc.want)) + for _, p := range tc.want { + want = append(want, p.GetFullName()) + } + sort.Strings(got) + sort.Strings(want) + if newLen != len(tc.want) || !reflect.DeepEqual(got, want) { + t.Errorf("filtered packages (newLen=%d):\n got %v\nwant %v", newLen, got, want) + } + if !reflect.DeepEqual(repo.Architectures, tc.indexes) { + t.Errorf("index configuration changed: %v", repo.Architectures) + } + }) + } +} diff --git a/deb/remote_test.go b/deb/remote_test.go index 4154e0b8a..cf05f105c 100644 --- a/deb/remote_test.go +++ b/deb/remote_test.go @@ -1,6 +1,7 @@ package deb import ( + "crypto/sha256" "errors" "fmt" "io" @@ -308,6 +309,97 @@ func (s *RemoteRepoSuite) TestRefKey(c *C) { c.Assert(s.repo.RefKey()[1:], DeepEquals, s.repo.Key()[1:]) } +type architectureVariantRemoteProgress struct { + aptly.Progress + messages []string +} + +func (p *architectureVariantRemoteProgress) ColoredPrintf(format string, args ...interface{}) { + p.messages = append(p.messages, fmt.Sprintf(format, args...)) +} + +func (s *RemoteRepoSuite) loadArchitectureVariantIndexes(c *C) { + var err error + s.repo, err = NewRemoteRepo("variant-test", "http://mirror.example.invalid/repo", "test", []string{"main"}, nil, false, false, false, false) + c.Assert(err, IsNil) + const normalIndex = `Package: example +Version: 1.0 +Architecture: amd64 +Filename: pool/main/e/example/example_1.0_amd64.deb +Size: 3 +MD5sum: 900150983cd24fb0d6963f7d28e17f72 + +` + const variantIndex = `Package: example +Version: 1.0 +Architecture: amd64 +Architecture-Variant: amd64v3 +Filename: pool/main/e/example/example_1.0_amd64v3.deb +Size: 4 +MD5sum: e2fc714c4727ee9395f324cd2e7f331f + +` + const root = "http://mirror.example.invalid/repo/dists/test/" + release := fmt.Sprintf("Architectures: amd64 amd64v3\nComponents: main\nSHA256:\n %x %d main/binary-amd64/Packages\n %x %d main/binary-amd64v3/Packages\n", + sha256.Sum256([]byte(normalIndex)), len(normalIndex), sha256.Sum256([]byte(variantIndex)), len(variantIndex)) + s.downloader = http.NewFakeDownloader().ExpectResponse(root+"Release", release) + s.downloader.ExpectResponse(root+"main/binary-amd64/Packages", normalIndex) + s.downloader.ExpectResponse(root+"main/binary-amd64v3/Packages", variantIndex) + c.Assert(s.repo.Fetch(s.downloader, nil, true), IsNil) + c.Check(s.repo.Architectures, DeepEquals, []string{"amd64", "amd64v3"}) + + progress := &architectureVariantRemoteProgress{Progress: s.progress} + c.Assert(s.repo.DownloadPackageIndexes(progress, s.downloader, nil, s.collectionFactory, true, false), IsNil) + c.Check(s.downloader.Empty(), Equals, true) + // Duplicate rejection is reported through ColoredPrintf, not as an error. + c.Check(progress.messages, HasLen, 0) + c.Assert(s.repo.packageList.Len(), Equals, 2) +} + +func (s *RemoteRepoSuite) TestArchitectureVariantPackageIndexes(c *C) { + s.loadArchitectureVariantIndexes(c) + packages := map[string]*Package{} + c.Assert(s.repo.packageList.ForEach(func(p *Package) error { + packages[p.IndexArchitecture()] = p + return nil + }), IsNil) + c.Assert(packages, HasLen, 2) + for _, arch := range []string{"amd64", "amd64v3"} { + p := packages[arch] + c.Assert(p, NotNil) + c.Check(p.Name, Equals, "example") + c.Check(p.Version, Equals, "1.0") + c.Check(p.Architecture, Equals, "amd64") + variant := "" + if arch == "amd64v3" { + variant = "amd64v3" + } + c.Check(p.ArchitectureVariant, Equals, variant) + c.Check(p.IndexArchitecture(), Equals, arch) + c.Assert(p.Files(), HasLen, 1) + c.Check(p.Files()[0].Filename, Equals, "example_1.0_"+arch+".deb") + c.Check(p.Files()[0].DownloadURL(), Equals, "pool/main/e/example/example_1.0_"+arch+".deb") + } +} + +func (s *RemoteRepoSuite) TestArchitectureVariantDownloadQueue(c *C) { + s.loadArchitectureVariantIndexes(c) + queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false) + c.Assert(err, IsNil) + c.Check(queue, HasLen, 2) + c.Check(size, Equals, int64(7)) + var urls []string + for _, task := range queue { + urls = append(urls, task.File.DownloadURL()) + c.Check(task.Additional, HasLen, 0) + } + sort.Strings(urls) + c.Check(urls, DeepEquals, []string{ + "pool/main/e/example/example_1.0_amd64.deb", + "pool/main/e/example/example_1.0_amd64v3.deb", + }) +} + func (s *RemoteRepoSuite) TestDownload(c *C) { s.repo.Architectures = []string{"i386"} diff --git a/query/architecture_variant_test.go b/query/architecture_variant_test.go new file mode 100644 index 000000000..ea5055dc0 --- /dev/null +++ b/query/architecture_variant_test.go @@ -0,0 +1,113 @@ +package query + +import ( + "reflect" + "sort" + "testing" + + "github.com/aptly-dev/aptly/database/goleveldb" + "github.com/aptly-dev/aptly/deb" +) + +func TestArchitectureVariantQueries(t *testing.T) { + db, err := goleveldb.NewOpenDB(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := db.Close(); err != nil { + t.Error(err) + } + }() + collection := deb.NewPackageCollection(db) + list := deb.NewPackageList() + const normal = "test-package_1.0_amd64" + const variant = "test-package_1.0_amd64v3" + for _, value := range []string{"", "amd64v3"} { + p := deb.NewPackageFromControlFile(deb.Stanza{ + "Package": "test-package", "Version": "1.0", + "Architecture": "amd64", "Architecture-Variant": value, + }) + if err := collection.Update(p); err != nil { + t.Fatal(err) + } + if err := list.Add(p); err != nil { + t.Fatal(err) + } + } + list.PrepareIndex() + for _, catalog := range []struct { + name string + value deb.PackageCatalog + }{{"list", list}, {"collection", collection}} { + t.Run(catalog.name, func(t *testing.T) { + for _, tc := range []struct { + name, expression string + want []string + }{ + {"coexistence", "Name (= test-package)", []string{normal, variant}}, + {"exact_normal", normal, []string{normal}}, + {"exact_variant", variant, []string{variant}}, + {"base_architecture", "Architecture (= amd64)", []string{normal, variant}}, + {"variant_is_not_base_architecture", "Architecture (= amd64v3)", []string{}}, + {"variant_field", "Architecture-Variant (= amd64v3)", []string{variant}}, + {"special_base_architecture", "$Architecture (= amd64)", []string{normal, variant}}, + {"special_variant_architecture", "$Architecture (= amd64v3)", []string{}}, + {"normal_and_field", normal + ", Architecture (= amd64)", []string{normal}}, + {"variant_and_field", variant + ", Architecture (= amd64)", []string{variant}}, + // A field OR operand forces scanning, including PkgQuery.Matches. + {"scan_normal", normal + " | Name (= absent)", []string{normal}}, + {"scan_variant", variant + " | Name (= absent)", []string{variant}}, + {"not_normal", "!(" + normal + ")", []string{variant}}, + {"not_variant", "!(" + variant + ")", []string{normal}}, + } { + t.Run(tc.name, func(t *testing.T) { + q, err := Parse(tc.expression) + if err != nil { + t.Fatal(err) + } + got := q.Query(catalog.value).FullNames() + sort.Strings(got) + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("query %q: got %v; want %v", tc.expression, got, tc.want) + } + }) + } + }) + } +} + +func TestArchitectureVariantQueryParsing(t *testing.T) { + for _, arch := range []string{"amd64", "amd64v3"} { + q, err := Parse("test-package_1.0_" + arch) + if err != nil { + t.Fatal(err) + } + want := &deb.PkgQuery{Pkg: "test-package", Version: "1.0", Arch: arch} + if !reflect.DeepEqual(q, want) { + t.Errorf("got %#v; want %#v", q, want) + } + } +} + +func TestArchitectureVariantArchitectureAllQuerySemantics(t *testing.T) { + p := &deb.Package{Name: "data", Architecture: "all"} + for _, tc := range []struct { + expression string + want bool + }{ + {"Architecture (= amd64)", false}, + {"Architecture (= amd64v3)", false}, + {"$Architecture (= amd64)", true}, + {"$Architecture (= amd64v3)", true}, + {"$Architecture (= source)", false}, + } { + q, err := Parse(tc.expression) + if err != nil { + t.Fatal(err) + } + if got := q.Matches(p); got != tc.want { + t.Errorf("query %q: got %v; want %v", tc.expression, got, tc.want) + } + } +} diff --git a/system/t05_snapshot/diff.py b/system/t05_snapshot/diff.py index c19807496..8ca5befdb 100644 --- a/system/t05_snapshot/diff.py +++ b/system/t05_snapshot/diff.py @@ -1,4 +1,6 @@ import re +import tempfile +from pathlib import Path from lib import BaseTest @@ -80,3 +82,58 @@ class DiffSnapshot6Test(BaseTest): "aptly snapshot create snap2 from mirror wheezy-main", ] runCmd = "aptly snapshot diff snap1 snap2" + + +class DiffSnapshotArchitectureVariantTest(BaseTest): + """ + snapshot diff: display base and variant architectures independently + """ + runCmd = "aptly snapshot diff variant-empty variant-normal" + + def prepare_fixture(self): + super().prepare_fixture() + self.run_cmd("aptly repo create variant-display") + self.run_cmd("aptly snapshot create variant-empty empty") + with tempfile.TemporaryDirectory(prefix="aptly-diff-variant-") as tmp: + for arch in ("amd64", "amd64v3"): + package_dir = Path(tmp) / arch + control_dir = package_dir / "DEBIAN" + control_dir.mkdir(parents=True) + control = ( + "Package: test-package\n" + "Version: 1.0\n" + "Architecture: amd64\n" + "Maintainer: Aptly Test \n" + "Description: Snapshot diff display fixture\n" + ) + if arch == "amd64v3": + control += "Architecture-Variant: amd64v3\n" + (control_dir / "control").write_text(control) + package_file = str(Path(tmp) / ( + "test-package_1.0_" + arch + ".deb")) + self.run_cmd(["dpkg-deb", "--build", str(package_dir), package_file]) + self.run_cmd(["aptly", "repo", "add", "variant-display", package_file]) + snapshot = "variant-normal" if arch == "amd64" else "variant-both" + self.run_cmd([ + "aptly", "snapshot", "create", snapshot, + "from", "repo", "variant-display", + ]) + + def check(self): + def check_row(output, expected): + lines = output.strip().splitlines() + self.check_equal(len(lines), 2) + self.check_equal(" ".join(lines[0].split()), + "Arch | Package | Version in A | Version in B") + self.check_equal(" ".join(lines[1].split()), expected) + + check_row(self.output, "+ amd64 | test-package | - | 1.0") + for left, right, expected in ( + ("variant-normal", "variant-both", + "+ amd64v3 | test-package | - | 1.0"), + ("variant-both", "variant-normal", + "- amd64v3 | test-package | 1.0 | -"), + ("variant-normal", "variant-empty", + "- amd64 | test-package | 1.0 | -"), + ): + check_row(self.run_cmd(["aptly", "snapshot", "diff", left, right]), expected)