diff --git a/api/mirror.go b/api/mirror.go index afc7d0079..15642f923 100644 --- a/api/mirror.go +++ b/api/mirror.go @@ -520,8 +520,10 @@ type mirrorUpdateParams struct { ForceUpdate bool ` json:"ForceUpdate"` // Set "true" to skip downloading already downloaded packages SkipExistingPackages bool ` json:"SkipExistingPackages"` - // Set "true" to download only the latest version per package/architecture + // Set "true" to download only the latest version per package/architecture (same as `MaxVersions`: 1) LatestOnly bool ` json:"LatestOnly"` + // Download only the N most recent versions per package/architecture, 0 means all versions + MaxVersions int ` json:"MaxVersions" example:"5"` } // @Summary Update Mirror @@ -573,6 +575,14 @@ func apiMirrorsUpdate(c *gin.Context) { } } + if b.MaxVersions < 0 { + AbortWithJSONError(c, 400, fmt.Errorf("MaxVersions should be a positive number")) + return + } + if b.LatestOnly && b.MaxVersions == 0 { + b.MaxVersions = 1 + } + verifier, err := getVerifier(b.Keyrings) if err != nil { AbortWithJSONError(c, 400, fmt.Errorf("unable to initialize GPG verifier: %s", err)) @@ -639,8 +649,15 @@ func apiMirrorsUpdate(c *gin.Context) { } } + if b.MaxVersions > 0 { + _, _, err := remote.FilterLatest(b.MaxVersions) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) + } + } + queue, downloadSize, err := remote.BuildDownloadQueue(context.PackagePool(), taskCollectionFactory.PackageCollection(), - taskCollectionFactory.ChecksumCollection(nil), b.SkipExistingPackages, b.LatestOnly) + taskCollectionFactory.ChecksumCollection(nil), b.SkipExistingPackages) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) } diff --git a/cmd/mirror_update.go b/cmd/mirror_update.go index d67fce3ed..bede24959 100644 --- a/cmd/mirror_update.go +++ b/cmd/mirror_update.go @@ -23,6 +23,14 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error { name := args[0] + maxVersions := context.Flags().Lookup("max-versions").Value.Get().(int) + if maxVersions < 0 { + return fmt.Errorf("unable to update: -max-versions should be a positive number") + } + if maxVersions == 0 && context.Flags().Lookup("latest").Value.Get().(bool) { + maxVersions = 1 + } + collectionFactory := context.NewCollectionFactory() repo, err := collectionFactory.RemoteRepoCollection().ByName(name) if err != nil { @@ -90,17 +98,29 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error { context.Progress().Printf("Packages filtered: %d -> %d.\n", oldLen, newLen) } + if maxVersions > 0 { + var oldLen, newLen int + oldLen, newLen, err = repo.FilterLatest(maxVersions) + if err != nil { + return fmt.Errorf("unable to update: %s", err) + } + versionWord := "versions" + if maxVersions == 1 { + versionWord = "version" + } + context.Progress().Printf("Keeping at most %d %s of each package: %d -> %d.\n", maxVersions, versionWord, oldLen, newLen) + } + var ( downloadSize int64 queue []deb.PackageDownloadTask ) skipExistingPackages := context.Flags().Lookup("skip-existing-packages").Value.Get().(bool) - latestOnly := context.Flags().Lookup("latest").Value.Get().(bool) context.Progress().Printf("Building download queue...\n") queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(), - collectionFactory.ChecksumCollection(nil), skipExistingPackages, latestOnly) + collectionFactory.ChecksumCollection(nil), skipExistingPackages) if err != nil { return fmt.Errorf("unable to update: %s", err) @@ -302,7 +322,8 @@ Example: cmd.Flag.Bool("ignore-checksums", false, "ignore checksum mismatches while downloading package files and metadata") cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures") cmd.Flag.Bool("skip-existing-packages", false, "do not check file existence for packages listed in the internal database of the mirror") - cmd.Flag.Bool("latest", false, "download only latest version of each package (per architecture)") + cmd.Flag.Bool("latest", false, "download only latest version of each package (per architecture), same as -max-versions=1") + cmd.Flag.Int("max-versions", 0, "download only the N most recent versions of each package (per architecture), 0 means all versions") cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)") cmd.Flag.String("downloader", "default", "downloader to use (e.g. grab)") cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error") diff --git a/completion.d/_aptly b/completion.d/_aptly index 6a70dfe3c..0e9a96294 100644 --- a/completion.d/_aptly +++ b/completion.d/_aptly @@ -212,7 +212,8 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could $keyring \ "-max-tries=[max download tries till process fails with download error]:number: " \ "-skip-existing-packages=[do not check file existence for packages listed in the internal database of the mirror]:$bool" \ - "-latest=[download only latest version of each package (per architecture)]:$bool" \ + "-latest=[download only latest version of each package (per architecture), same as -max-versions=1]:$bool" \ + "-max-versions=[download only the N most recent versions of each package (per architecture), 0 means all versions]:number: " \ "(-)2:mirror name:$mirrors" ;; rename) diff --git a/completion.d/aptly b/completion.d/aptly index bdfbe722c..972f4449d 100644 --- a/completion.d/aptly +++ b/completion.d/aptly @@ -263,7 +263,7 @@ _aptly() "update") if [[ $numargs -eq 0 ]]; then if [[ "$cur" == -* ]]; then - COMPREPLY=($(compgen -W "-force -download-limit= -downloader= -ignore-checksums -ignore-signatures -keyring= -skip-existing-packages -latest" -- ${cur})) + COMPREPLY=($(compgen -W "-force -download-limit= -downloader= -ignore-checksums -ignore-signatures -keyring= -skip-existing-packages -latest -max-versions=" -- ${cur})) else COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur})) fi diff --git a/deb/list.go b/deb/list.go index 19e37593c..bf6507a47 100644 --- a/deb/list.go +++ b/deb/list.go @@ -173,20 +173,21 @@ func (l *PackageList) ForEach(handler func(*Package) error) error { } // FilterLatest creates a copy of the package list containing only the -// latest version for each package name/architecture pair. -func (l *PackageList) FilterLatest() (*PackageList, error) { +// count latest versions for each package name/architecture pair. +func (l *PackageList) FilterLatest(count int) (*PackageList, error) { if l == nil { return nil, fmt.Errorf("package list is nil") } - filtered := make(map[string]*Package, l.Len()) + if count < 1 { + return nil, fmt.Errorf("number of latest versions should be at least 1, got %d", count) + } + + groups := make(map[string][]*Package, l.Len()) err := l.ForEach(func(p *Package) error { key := p.Architecture + "|" + p.Name - - if existing, found := filtered[key]; !found || CompareVersions(p.Version, existing.Version) > 0 { - filtered[key] = p - } + groups[key] = append(groups[key], p) return nil }) @@ -194,11 +195,26 @@ func (l *PackageList) FilterLatest() (*PackageList, error) { return nil, err } - result := NewPackageListWithDuplicates(l.duplicatesAllowed, len(filtered)) + result := NewPackageListWithDuplicates(l.duplicatesAllowed, len(groups)*count) + + for _, pkgs := range groups { + // sort from latest to oldest + sort.Slice(pkgs, func(i, j int) bool { + return CompareVersions(pkgs[i].Version, pkgs[j].Version) > 0 + }) - for _, pkg := range filtered { - if err = result.Add(pkg); err != nil { - return nil, err + versions := 0 + for i, p := range pkgs { + if i == 0 || CompareVersions(p.Version, pkgs[i-1].Version) != 0 { + versions++ + if versions > count { + break + } + } + + if err = result.Add(p); err != nil { + return nil, err + } } } diff --git a/deb/list_test.go b/deb/list_test.go index 4e4768e34..f4fd59dda 100644 --- a/deb/list_test.go +++ b/deb/list_test.go @@ -524,13 +524,55 @@ func (s *PackageListSuite) TestFilterLatest(c *C) { sharedPkg := NewPackageFromControlFile(shared) _ = list.Add(sharedPkg) - filtered, err := list.FilterLatest() + filtered, err := list.FilterLatest(1) c.Assert(err, IsNil) c.Assert(filtered.Len(), Equals, 2) c.Check(filtered.Has(newerPkg), Equals, true) c.Check(filtered.Has(sharedPkg), Equals, true) } +func (s *PackageListSuite) TestFilterLatestMultiple(c *C) { + list := NewPackageList() + + pkgs := make([]*Package, 0, 4) + for _, version := range []string{"2.0", "1.0", "10.0", "1:0.1"} { + stanza := packageStanza.Copy() + stanza["Version"] = version + pkg := NewPackageFromControlFile(stanza) + _ = list.Add(pkg) + pkgs = append(pkgs, pkg) + } + + shared := packageStanza.Copy() + shared["Architecture"] = ArchitectureAll + shared["Version"] = "3.0" + shared["Package"] = "shared" + sharedPkg := NewPackageFromControlFile(shared) + _ = list.Add(sharedPkg) + + filtered, err := list.FilterLatest(3) + c.Assert(err, IsNil) + c.Assert(filtered.Len(), Equals, 4) + c.Check(filtered.Has(pkgs[0]), Equals, true) + c.Check(filtered.Has(pkgs[1]), Equals, false) + c.Check(filtered.Has(pkgs[2]), Equals, true) + c.Check(filtered.Has(pkgs[3]), Equals, true) + c.Check(filtered.Has(sharedPkg), Equals, true) + + // fewer versions available than requested: everything is kept + filtered, err = list.FilterLatest(5) + c.Assert(err, IsNil) + c.Check(filtered.Len(), Equals, 5) +} + +func (s *PackageListSuite) TestFilterLatestInvalidCount(c *C) { + list := NewPackageList() + + filtered, err := list.FilterLatest(0) + c.Assert(err, ErrorMatches, "number of latest versions should be at least 1, got 0") + c.Assert(filtered, IsNil) +} + func (s *PackageListSuite) TestFilterLatestPreservesDuplicatesFlag(c *C) { list := NewPackageListWithDuplicates(true, 2) @@ -540,7 +582,7 @@ func (s *PackageListSuite) TestFilterLatestPreservesDuplicatesFlag(c *C) { another["Version"] = "7.41-1" _ = list.Add(NewPackageFromControlFile(another)) - filtered, err := list.FilterLatest() + filtered, err := list.FilterLatest(1) c.Assert(err, IsNil) c.Assert(filtered.duplicatesAllowed, Equals, true) } @@ -548,7 +590,7 @@ func (s *PackageListSuite) TestFilterLatestPreservesDuplicatesFlag(c *C) { func (s *PackageListSuite) TestFilterLatestNil(c *C) { var list *PackageList - filtered, err := list.FilterLatest() + filtered, err := list.FilterLatest(1) c.Assert(err, ErrorMatches, "package list is nil") c.Assert(filtered, IsNil) } diff --git a/deb/remote.go b/deb/remote.go index a1a79d689..379661245 100644 --- a/deb/remote.go +++ b/deb/remote.go @@ -698,18 +698,26 @@ func (repo *RemoteRepo) ApplyFilter(dependencyOptions int, filterQuery PackageQu return } -// BuildDownloadQueue builds queue, discards current PackageList -func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, packageCollection *PackageCollection, checksumStorage aptly.ChecksumStorage, skipExistingPackages, latestOnly bool) (queue []PackageDownloadTask, downloadSize int64, err error) { +// FilterLatest keeps only count latest versions of each package (per architecture) +func (repo *RemoteRepo) FilterLatest(count int) (oldLen, newLen int, err error) { if repo.packageList == nil { err = fmt.Errorf("package list is empty, please (re)download package indexes") return } - if latestOnly { - repo.packageList, err = repo.packageList.FilterLatest() - if err != nil { - return - } + oldLen = repo.packageList.Len() + repo.packageList, err = repo.packageList.FilterLatest(count) + if repo.packageList != nil { + newLen = repo.packageList.Len() + } + return +} + +// BuildDownloadQueue builds queue, discards current PackageList +func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, packageCollection *PackageCollection, checksumStorage aptly.ChecksumStorage, skipExistingPackages bool) (queue []PackageDownloadTask, downloadSize int64, err error) { + if repo.packageList == nil { + err = fmt.Errorf("package list is empty, please (re)download package indexes") + return } queue = make([]PackageDownloadTask, 0, repo.packageList.Len()) diff --git a/deb/remote_test.go b/deb/remote_test.go index 4154e0b8a..875287d69 100644 --- a/deb/remote_test.go +++ b/deb/remote_test.go @@ -322,7 +322,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) { c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) - queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false) + queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(3)) c.Check(queue, HasLen, 1) @@ -349,7 +349,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) { c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) - queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true, false) + queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true) c.Assert(err, IsNil) c.Check(size, Equals, int64(0)) c.Check(queue, HasLen, 0) @@ -370,7 +370,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) { c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) - queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false) + queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(3)) c.Check(queue, HasLen, 1) @@ -397,7 +397,7 @@ func (s *RemoteRepoSuite) TestDownloadWithInstaller(c *C) { c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) - queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false) + queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(3)+int64(len(exampleInstallerManifestFile))) c.Check(queue, HasLen, 2) @@ -423,7 +423,7 @@ func (s *RemoteRepoSuite) TestDownloadWithInstaller(c *C) { c.Check(pkg.Name, Equals, "installer") } -func (s *RemoteRepoSuite) TestBuildDownloadQueueLatestOnly(c *C) { +func (s *RemoteRepoSuite) TestFilterLatest(c *C) { s.repo.Architectures = []string{"i386"} err := s.repo.Fetch(s.downloader, nil, true) @@ -445,13 +445,29 @@ func (s *RemoteRepoSuite) TestBuildDownloadQueueLatestOnly(c *C) { newest := NewPackageFromControlFile(stanza) _ = s.repo.packageList.Add(newest) - queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, true) + oldLen, newLen, err := s.repo.FilterLatest(1) + c.Assert(err, IsNil) + c.Check(oldLen, Equals, 2) + c.Check(newLen, Equals, 1) + + queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) c.Assert(err, IsNil) c.Check(queue, HasLen, 1) c.Check(queue[0].File.DownloadURL(), Equals, "pool/main/a/amanda/amanda-client_3.4.0-1_i386.deb") c.Check(size, Equals, int64(187518)) } +func (s *RemoteRepoSuite) TestFilterLatestErrors(c *C) { + _, _, err := s.repo.FilterLatest(1) + c.Assert(err, ErrorMatches, "package list is empty, please \\(re\\)download package indexes") + + s.repo.packageList = NewPackageList() + oldLen, newLen, err := s.repo.FilterLatest(0) + c.Assert(err, ErrorMatches, "number of latest versions should be at least 1, got 0") + c.Check(oldLen, Equals, 0) + c.Check(newLen, Equals, 0) +} + func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) { s.repo.Architectures = []string{"i386"} s.repo.DownloadSources = true @@ -470,7 +486,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) { c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) - queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false) + queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(15)) c.Check(queue, HasLen, 4) @@ -514,7 +530,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) { c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) - queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true, false) + queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true) c.Assert(err, IsNil) c.Check(size, Equals, int64(0)) c.Check(queue, HasLen, 0) @@ -539,7 +555,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) { c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) - queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false) + queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(15)) c.Check(queue, HasLen, 4) @@ -563,7 +579,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) { c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) - queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false) + queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(3)) c.Check(queue, HasLen, 1) @@ -591,7 +607,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) { c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) - queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true, false) + queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true) c.Assert(err, IsNil) c.Check(size, Equals, int64(0)) c.Check(queue, HasLen, 0) @@ -613,7 +629,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) { c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) - queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false) + queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(3)) c.Check(queue, HasLen, 1) @@ -644,7 +660,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) { c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) - queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false) + queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(15)) c.Check(queue, HasLen, 4) @@ -690,7 +706,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) { c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) - queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true, false) + queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true) c.Assert(err, IsNil) c.Check(size, Equals, int64(0)) c.Check(queue, HasLen, 0) @@ -716,7 +732,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) { c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) - queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false) + queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(15)) c.Check(queue, HasLen, 4) diff --git a/man/aptly.1 b/man/aptly.1 index 67351d7a7..b3fa89ba8 100644 --- a/man/aptly.1 +++ b/man/aptly.1 @@ -804,7 +804,11 @@ do not check file existence for packages listed in the internal database of the . .TP \-\fBlatest\fR -download only latest version of each package (per architecture) +download only latest version of each package (per architecture), same as \-max\-versions=1 +. +.TP +\-\fBmax\-versions\fR=0 +download only the N most recent versions of each package (per architecture), 0 means all versions . .SH "RENAMES MIRROR" \fBaptly\fR \fBmirror\fR \fBrename\fR \fIold\-name\fR \fInew\-name\fR diff --git a/system/t04_mirror/UpdateMirror29Test_gold b/system/t04_mirror/UpdateMirror29Test_gold new file mode 100644 index 000000000..020d37a6c --- /dev/null +++ b/system/t04_mirror/UpdateMirror29Test_gold @@ -0,0 +1,32 @@ +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/dists/wheezy/Release +Downloading & parsing package files... +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/dists/wheezy/main/binary-i386/Packages.bz2 +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/dists/wheezy/main/binary-amd64/Packages.bz2 +Keeping at most 2 versions of each package: 52 -> 23. +Building download queue... +Download queue: 23 items (8.41 MiB) +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.6-1%29/varnish_3.0.6-1~wheezy_i386.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish-agent%20%282.2.1%29/varnish-agent_2.2.1~wheezy_i386.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish-agent%20%283.0.0%29/varnish-agent_3.0.0~wheezy_amd64.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.7-1%29/varnish-dbg_3.0.7-1~wheezy_amd64.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.6-1%29/libvarnishapi1_3.0.6-1~wheezy_amd64.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.7-1%29/libvarnishapi-dev_3.0.7-1~wheezy_amd64.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.7-1%29/libvarnishapi1_3.0.7-1~wheezy_i386.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.6-1%29/libvarnishapi1_3.0.6-1~wheezy_i386.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.7-1%29/varnish_3.0.7-1~wheezy_i386.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish-agent%20%282.2.0%29/varnish-agent_2.2.0~wheezy_i386.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.7-1%29/libvarnishapi-dev_3.0.7-1~wheezy_i386.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.7-1%29/varnish_3.0.7-1~wheezy_amd64.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.6-1%29/varnish_3.0.6-1~wheezy_amd64.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.6-1%29/libvarnishapi-dev_3.0.6-1~wheezy_amd64.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish-agent%20%283.0.1%29/varnish-agent_3.0.1~wheezy_amd64.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.6-1%29/varnish-dbg_3.0.6-1~wheezy_amd64.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.6-1%29/libvarnishapi-dev_3.0.6-1~wheezy_i386.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.7-1%29/varnish-dbg_3.0.7-1~wheezy_i386.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.7-1%29/varnish-doc_3.0.7-1~wheezy_all.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.6-1%29/varnish-doc_3.0.6-1~wheezy_all.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.7-1%29/libvarnishapi1_3.0.7-1~wheezy_amd64.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish-agent%20%281.16.0%29/varnish-agent_1.16.0~wheezy_all.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.6-1%29/varnish-dbg_3.0.6-1~wheezy_i386.deb + +Mirror `varnish` has been updated successfully. diff --git a/system/t04_mirror/UpdateMirror29Test_mirror_show b/system/t04_mirror/UpdateMirror29Test_mirror_show new file mode 100644 index 000000000..7d29075e4 --- /dev/null +++ b/system/t04_mirror/UpdateMirror29Test_mirror_show @@ -0,0 +1,45 @@ +Name: varnish +Archive Root URL: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/ +Distribution: wheezy +Components: main +Architectures: i386, amd64 +Download Sources: no +Download .udebs: no +Download AppStream: no +Number of packages: 23 + +Information from release file: +Acquire-By-Hash: yes +Architectures: amd64 armel armhf i386 ia64 mips mipsel powerpc ppc64el s390 s390x sparc alpha avr32 hppa m32 m68k sh arm64 +Codename: wheezy +Components: main +Date: Tue, 22 Jan 2019 00:28:39 +0000 +Description: +Label: varnish30 +Origin: packagecloud.io/varnishcache/varnish30 +Suite: wheezy +Version: 1 +Packages: + libvarnishapi-dev_3.0.7-1~wheezy_amd64 + libvarnishapi-dev_3.0.7-1~wheezy_i386 + libvarnishapi-dev_3.0.6-1~wheezy_amd64 + libvarnishapi-dev_3.0.6-1~wheezy_i386 + libvarnishapi1_3.0.7-1~wheezy_amd64 + libvarnishapi1_3.0.7-1~wheezy_i386 + libvarnishapi1_3.0.6-1~wheezy_amd64 + libvarnishapi1_3.0.6-1~wheezy_i386 + varnish_3.0.7-1~wheezy_amd64 + varnish_3.0.7-1~wheezy_i386 + varnish_3.0.6-1~wheezy_amd64 + varnish_3.0.6-1~wheezy_i386 + varnish-agent_3.0.1~wheezy_amd64 + varnish-agent_3.0.0~wheezy_amd64 + varnish-agent_2.2.1~wheezy_i386 + varnish-agent_2.2.0~wheezy_i386 + varnish-agent_1.16.0~wheezy_all + varnish-dbg_3.0.7-1~wheezy_amd64 + varnish-dbg_3.0.7-1~wheezy_i386 + varnish-dbg_3.0.6-1~wheezy_amd64 + varnish-dbg_3.0.6-1~wheezy_i386 + varnish-doc_3.0.7-1~wheezy_all + varnish-doc_3.0.6-1~wheezy_all diff --git a/system/t04_mirror/UpdateMirror30Test_gold b/system/t04_mirror/UpdateMirror30Test_gold new file mode 100644 index 000000000..d3a2f11a7 --- /dev/null +++ b/system/t04_mirror/UpdateMirror30Test_gold @@ -0,0 +1,21 @@ +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/dists/wheezy/Release +Downloading & parsing package files... +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/dists/wheezy/main/binary-i386/Packages.bz2 +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/dists/wheezy/main/binary-amd64/Packages.bz2 +Keeping at most 1 version of each package: 52 -> 12. +Building download queue... +Download queue: 12 items (4.19 MiB) +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.7-1%29/varnish-dbg_3.0.7-1~wheezy_amd64.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.7-1%29/libvarnishapi1_3.0.7-1~wheezy_amd64.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish-agent%20%281.16.0%29/varnish-agent_1.16.0~wheezy_all.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish-agent%20%283.0.1%29/varnish-agent_3.0.1~wheezy_amd64.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.7-1%29/varnish_3.0.7-1~wheezy_amd64.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.7-1%29/varnish-dbg_3.0.7-1~wheezy_i386.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.7-1%29/varnish_3.0.7-1~wheezy_i386.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.7-1%29/libvarnishapi-dev_3.0.7-1~wheezy_i386.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.7-1%29/libvarnishapi1_3.0.7-1~wheezy_i386.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.7-1%29/libvarnishapi-dev_3.0.7-1~wheezy_amd64.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish-agent%20%282.2.1%29/varnish-agent_2.2.1~wheezy_i386.deb +Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.7-1%29/varnish-doc_3.0.7-1~wheezy_all.deb + +Mirror `varnish` has been updated successfully. diff --git a/system/t04_mirror/UpdateMirror30Test_mirror_show b/system/t04_mirror/UpdateMirror30Test_mirror_show new file mode 100644 index 000000000..0f65089ea --- /dev/null +++ b/system/t04_mirror/UpdateMirror30Test_mirror_show @@ -0,0 +1,34 @@ +Name: varnish +Archive Root URL: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/ +Distribution: wheezy +Components: main +Architectures: i386, amd64 +Download Sources: no +Download .udebs: no +Download AppStream: no +Number of packages: 12 + +Information from release file: +Acquire-By-Hash: yes +Architectures: amd64 armel armhf i386 ia64 mips mipsel powerpc ppc64el s390 s390x sparc alpha avr32 hppa m32 m68k sh arm64 +Codename: wheezy +Components: main +Date: Tue, 22 Jan 2019 00:28:39 +0000 +Description: +Label: varnish30 +Origin: packagecloud.io/varnishcache/varnish30 +Suite: wheezy +Version: 1 +Packages: + libvarnishapi-dev_3.0.7-1~wheezy_amd64 + libvarnishapi-dev_3.0.7-1~wheezy_i386 + libvarnishapi1_3.0.7-1~wheezy_amd64 + libvarnishapi1_3.0.7-1~wheezy_i386 + varnish_3.0.7-1~wheezy_amd64 + varnish_3.0.7-1~wheezy_i386 + varnish-agent_3.0.1~wheezy_amd64 + varnish-agent_2.2.1~wheezy_i386 + varnish-agent_1.16.0~wheezy_all + varnish-dbg_3.0.7-1~wheezy_amd64 + varnish-dbg_3.0.7-1~wheezy_i386 + varnish-doc_3.0.7-1~wheezy_all diff --git a/system/t04_mirror/UpdateMirror31Test_gold b/system/t04_mirror/UpdateMirror31Test_gold new file mode 100644 index 000000000..01721a044 --- /dev/null +++ b/system/t04_mirror/UpdateMirror31Test_gold @@ -0,0 +1 @@ +ERROR: unable to update: -max-versions should be a positive number diff --git a/system/t04_mirror/update.py b/system/t04_mirror/update.py index c04717679..e47d24d8c 100644 --- a/system/t04_mirror/update.py +++ b/system/t04_mirror/update.py @@ -502,3 +502,45 @@ def check(self): self.check_output() self.check_cmd_output("aptly mirror show appstream-test", "mirror_show", match_prepare=lambda s: re.sub(r"Last update: [0-9:+A-Za-z -]+\n", "", s)) + + +class UpdateMirror29Test(BaseTest): + """ + update mirrors: keep only the 2 most recent versions of each package + """ + sortOutput = True + fixtureCmds = [ + "aptly -architectures=i386,amd64 mirror create --ignore-signatures varnish http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/ wheezy main", + ] + runCmd = "aptly mirror update --ignore-signatures -max-versions=2 varnish" + outputMatchPrepare = filterOutRedirects + + def check(self): + self.check_output() + self.check_cmd_output("aptly mirror show -with-packages varnish", "mirror_show", + match_prepare=lambda s: re.sub(r"Last update: [0-9:+A-Za-z -]+\n", "", s)) + + +class UpdateMirror30Test(BaseTest): + """ + update mirrors: -latest keeps only the most recent version of each package + """ + sortOutput = True + fixtureCmds = [ + "aptly -architectures=i386,amd64 mirror create --ignore-signatures varnish http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/ wheezy main", + ] + runCmd = "aptly mirror update --ignore-signatures -latest varnish" + outputMatchPrepare = filterOutRedirects + + def check(self): + self.check_output() + self.check_cmd_output("aptly mirror show -with-packages varnish", "mirror_show", + match_prepare=lambda s: re.sub(r"Last update: [0-9:+A-Za-z -]+\n", "", s)) + + +class UpdateMirror31Test(BaseTest): + """ + update mirrors: negative -max-versions + """ + runCmd = "aptly mirror update -max-versions=-1 varnish" + expectedCode = 1 diff --git a/system/t12_api/mirrors.py b/system/t12_api/mirrors.py index a38efe3ef..b82e48eab 100644 --- a/system/t12_api/mirrors.py +++ b/system/t12_api/mirrors.py @@ -300,3 +300,47 @@ def check(self): resp = self.post("/api/mirrors/" + mirror_name, json=edit_params) self.check_equal(resp.status_code, 400) self.check_in("flat mirrors don't support udebs", resp.json()['error']) + + +class MirrorsAPITestUpdateMaxVersions(APITest): + """ + PUT /api/mirrors/:name with MaxVersions and LatestOnly + """ + def check(self): + mirror_name = self.random_name() + mirror_desc = {'Name': mirror_name, + 'ArchiveURL': 'http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/', + 'Distribution': 'wheezy', + 'Architectures': ["amd64"], + 'Components': ['main'], + 'IgnoreSignatures': True} + + resp = self.post("/api/mirrors", json=mirror_desc) + self.check_equal(resp.status_code, 201) + + def versions_per_package(): + resp = self.get("/api/mirrors/" + mirror_name + "/packages") + self.check_equal(resp.status_code, 200) + versions = {} + for key in resp.json(): + arch, name, version = key.split(" ")[:3] + versions.setdefault((arch, name), []).append(version) + return versions + + resp = self.put("/api/mirrors/" + mirror_name, json={'IgnoreSignatures': True, 'MaxVersions': -1}) + self.check_equal(resp.status_code, 400) + self.check_in("MaxVersions should be a positive number", resp.json()['error']) + + resp = self.put_task("/api/mirrors/" + mirror_name, json={'IgnoreSignatures': True, 'MaxVersions': 2}) + self.check_task(resp) + + versions = versions_per_package() + self.check_equal(sorted(versions[("Pamd64", "varnish")]), ["3.0.6-1~wheezy", "3.0.7-1~wheezy"]) + self.check_equal(max(len(v) for v in versions.values()), 2) + + resp = self.put_task("/api/mirrors/" + mirror_name, json={'IgnoreSignatures': True, 'LatestOnly': True}) + self.check_task(resp) + + versions = versions_per_package() + self.check_equal(versions[("Pamd64", "varnish")], ["3.0.7-1~wheezy"]) + self.check_equal(max(len(v) for v in versions.values()), 1)