From 4366eee39ade8de0acef831c300f88b770da1018 Mon Sep 17 00:00:00 2001 From: Wayne Warren Date: Mon, 9 Mar 2020 14:11:14 +0000 Subject: [PATCH 1/5] add Vacuum.RemoveLayerLink function --- registry/storage/vacuum.go | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/registry/storage/vacuum.go b/registry/storage/vacuum.go index a43db17a4..11a8e6ba3 100644 --- a/registry/storage/vacuum.go +++ b/registry/storage/vacuum.go @@ -84,6 +84,38 @@ func (v Vacuum) RemoveManifest(name string, dgst digest.Digest, tags []string) e return v.driver.Delete(v.ctx, manifestPath) } +// RemoveLayerLink removes a layer link from the filesystem +func (v Vacuum) RemoveLayerLink(manifestName, dgst string) error { + d, err := digest.Parse(dgst) + if err != nil { + return err + } + + layerLinkPath, err := pathFor(layerLinkPathSpec{name: manifestName, digest: d}) + if err != nil { + return err + } + + dcontext.GetLogger(v.ctx).Infof("Deleting layer link path : %s", layerLinkPath) + + _, err = v.driver.Stat(v.ctx, layerLinkPath) + if err != nil { + switch err := err.(type) { + case driver.PathNotFoundError: + return nil + default: + return err + } + } + + err = v.driver.Delete(v.ctx, layerLinkPath) + if err != nil { + return err + } + + return nil +} + // RemoveRepository removes a repository directory from the // filesystem func (v Vacuum) RemoveRepository(repoName string) error { From 3b7741805d84c3414f9c8b9aa891b8a565f89a09 Mon Sep 17 00:00:00 2001 From: Wayne Warren Date: Mon, 9 Mar 2020 16:36:20 +0000 Subject: [PATCH 2/5] Remove layer links along with manifest --- registry/storage/garbagecollect.go | 37 +++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/registry/storage/garbagecollect.go b/registry/storage/garbagecollect.go index 317c792da..a93a35520 100644 --- a/registry/storage/garbagecollect.go +++ b/registry/storage/garbagecollect.go @@ -25,6 +25,7 @@ type ManifestDel struct { Name string Digest digest.Digest Tags []string + Layers []digest.Digest } // MarkAndSweep performs a mark and sweep of registry data @@ -61,6 +62,11 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis } err = manifestEnumerator.Enumerate(ctx, func(dgst digest.Digest) error { + manifest, err := manifestService.Get(ctx, dgst) + if err != nil { + return fmt.Errorf("failed to retrieve manifest for digest %v: %v", dgst, err) + } + if opts.RemoveUntagged { // fetch all tags where this manifest is the latest one tags, err := repository.Tags(ctx).Lookup(ctx, distribution.Descriptor{Digest: dgst}) @@ -76,7 +82,23 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis if err != nil { return fmt.Errorf("failed to retrieve tags %v", err) } - manifestArr = append(manifestArr, ManifestDel{Name: repoName, Digest: dgst, Tags: allTags}) + + manifestDel := ManifestDel{ + Name: repoName, + Digest: dgst, + Tags: allTags, + Layers: []digest.Digest{}, + } + + for _, ref := range manifest.References() { + if ref.MediaType == schema2.MediaTypeLayer || + ref.MediaType == schema2.MediaTypeImageConfig { + manifestDel.Layers = append(manifestDel.Layers, ref.Digest) + } + } + + manifestArr = append(manifestArr, manifestDel) + return nil } } @@ -84,11 +106,6 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis emit("%s: marking manifest %s ", repoName, dgst) markSet[dgst] = struct{}{} - manifest, err := manifestService.Get(ctx, dgst) - if err != nil { - return fmt.Errorf("failed to retrieve manifest for digest %v: %v", dgst, err) - } - descriptors := manifest.References() for _, descriptor := range descriptors { markSet[descriptor.Digest] = struct{}{} @@ -122,6 +139,14 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis if err != nil { return fmt.Errorf("failed to delete manifest %s: %v", obj.Digest, err) } + for _, layerDgst := range obj.Layers { + if _, ok := markSet[layerDgst]; !ok { + err := vacuum.RemoveLayerLink(obj.Name, string(layerDgst)) + if err != nil { + return fmt.Errorf("failed to delete layer link %s for manifest %s: %v", layerDgst, obj.Name, err) + } + } + } } } blobService := registry.Blobs() From 5fcc8f55e21651902765030a27dafaa08097c475 Mon Sep 17 00:00:00 2001 From: Wayne Warren Date: Wed, 2 Sep 2020 21:53:31 +0000 Subject: [PATCH 3/5] need schema2 import --- registry/storage/garbagecollect.go | 1 + 1 file changed, 1 insertion(+) diff --git a/registry/storage/garbagecollect.go b/registry/storage/garbagecollect.go index a93a35520..4a8c26320 100644 --- a/registry/storage/garbagecollect.go +++ b/registry/storage/garbagecollect.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/docker/distribution" + "github.com/docker/distribution/manifest/schema2" "github.com/docker/distribution/reference" "github.com/docker/distribution/registry/storage/driver" "github.com/opencontainers/go-digest" From ec64b15a774d03b0a5c69852db6daf4207c2a86d Mon Sep 17 00:00:00 2001 From: Wayne Warren Date: Wed, 2 Sep 2020 21:53:43 +0000 Subject: [PATCH 4/5] fix signature on RemoveLayerLink --- registry/storage/garbagecollect.go | 2 +- registry/storage/vacuum.go | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/registry/storage/garbagecollect.go b/registry/storage/garbagecollect.go index 4a8c26320..887f35238 100644 --- a/registry/storage/garbagecollect.go +++ b/registry/storage/garbagecollect.go @@ -142,7 +142,7 @@ func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, regis } for _, layerDgst := range obj.Layers { if _, ok := markSet[layerDgst]; !ok { - err := vacuum.RemoveLayerLink(obj.Name, string(layerDgst)) + err := vacuum.RemoveLayerLink(obj.Name, layerDgst) if err != nil { return fmt.Errorf("failed to delete layer link %s for manifest %s: %v", layerDgst, obj.Name, err) } diff --git a/registry/storage/vacuum.go b/registry/storage/vacuum.go index 11a8e6ba3..92edbcf2f 100644 --- a/registry/storage/vacuum.go +++ b/registry/storage/vacuum.go @@ -85,13 +85,8 @@ func (v Vacuum) RemoveManifest(name string, dgst digest.Digest, tags []string) e } // RemoveLayerLink removes a layer link from the filesystem -func (v Vacuum) RemoveLayerLink(manifestName, dgst string) error { - d, err := digest.Parse(dgst) - if err != nil { - return err - } - - layerLinkPath, err := pathFor(layerLinkPathSpec{name: manifestName, digest: d}) +func (v Vacuum) RemoveLayerLink(manifestName string, dgst digest.Digest) error { + layerLinkPath, err := pathFor(layerLinkPathSpec{name: manifestName, digest: dgst}) if err != nil { return err } From 28166f73834fa17f51f973a655a2874dbba55336 Mon Sep 17 00:00:00 2001 From: Wayne Warren Date: Wed, 2 Sep 2020 21:54:36 +0000 Subject: [PATCH 5/5] simplify RemoveLayerLink function --- registry/storage/vacuum.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/registry/storage/vacuum.go b/registry/storage/vacuum.go index 92edbcf2f..74e581a2d 100644 --- a/registry/storage/vacuum.go +++ b/registry/storage/vacuum.go @@ -103,12 +103,7 @@ func (v Vacuum) RemoveLayerLink(manifestName string, dgst digest.Digest) error { } } - err = v.driver.Delete(v.ctx, layerLinkPath) - if err != nil { - return err - } - - return nil + return v.driver.Delete(v.ctx, layerLinkPath) } // RemoveRepository removes a repository directory from the