diff --git a/manifest/ocischema/index.go b/manifest/ocischema/index.go index add766f3e..f93925195 100644 --- a/manifest/ocischema/index.go +++ b/manifest/ocischema/index.go @@ -59,9 +59,16 @@ type ImageIndex struct { // MediaType is the media type of this schema. MediaType string `json:"mediaType,omitempty"` + // ArtifactType is the type of an artifact when the manifest is used for an + // artifact. + ArtifactType string `json:"artifactType,omitempty"` + // Manifests references a list of manifests Manifests []v1.Descriptor `json:"manifests"` + // Subject is the descriptor of a manifest referred to by this manifest. + Subject *v1.Descriptor `json:"subject,omitempty"` + // Annotations is an optional field that contains arbitrary metadata for the // image index Annotations map[string]string `json:"annotations,omitempty"` @@ -87,17 +94,22 @@ type DeserializedImageIndex struct { // and its JSON representation. If annotations is nil or empty then the // annotations property will be omitted from the JSON representation. func FromDescriptors(descriptors []v1.Descriptor, annotations map[string]string) (*DeserializedImageIndex, error) { - return fromDescriptorsWithMediaType(descriptors, annotations, v1.MediaTypeImageIndex) + return fromDescriptorsWithMediaType(descriptors, nil, annotations, v1.MediaTypeImageIndex) } // fromDescriptorsWithMediaType is for testing purposes, it's useful to be able to specify the media type explicitly -func fromDescriptorsWithMediaType(descriptors []v1.Descriptor, annotations map[string]string, mediaType string) (_ *DeserializedImageIndex, err error) { +func fromDescriptorsWithMediaType(descriptors []v1.Descriptor, subject *v1.Descriptor, annotations map[string]string, mediaType string) (_ *DeserializedImageIndex, err error) { m := ImageIndex{ Versioned: specs.Versioned{SchemaVersion: 2}, MediaType: mediaType, + Subject: subject, Annotations: annotations, } + if subject != nil { + m.ArtifactType = "application/text" + } + m.Manifests = make([]v1.Descriptor, len(descriptors)) copy(m.Manifests, descriptors) @@ -162,3 +174,17 @@ func validateIndex(b []byte) error { } return nil } + +// Subject returns a pointer to the subject of this manifest or nil if there is +// none +func (m *DeserializedImageIndex) Subject() *distribution.Descriptor { + return m.ImageIndex.Subject +} + +// Type returns the artifactType of the manifest +func (m *DeserializedImageIndex) Type() string { + if m.ArtifactType == "" { + return m.ImageIndex.MediaType + } + return m.ArtifactType +} diff --git a/manifest/ocischema/index_test.go b/manifest/ocischema/index_test.go index e5485c51e..43b38b840 100644 --- a/manifest/ocischema/index_test.go +++ b/manifest/ocischema/index_test.go @@ -85,7 +85,7 @@ func makeTestOCIImageIndex(t *testing.T, mediaType string) ([]v1.Descriptor, *De "com.example.locale": "en_GB", } - deserialized, err := fromDescriptorsWithMediaType(manifestDescriptors, annotations, mediaType) + deserialized, err := fromDescriptorsWithMediaType(manifestDescriptors, nil, annotations, mediaType) if err != nil { t.Fatalf("error creating DeserializedManifestList: %v", err) } diff --git a/manifest/ocischema/manifest.go b/manifest/ocischema/manifest.go index 2009e2f5a..0ce220d84 100644 --- a/manifest/ocischema/manifest.go +++ b/manifest/ocischema/manifest.go @@ -58,10 +58,17 @@ type Manifest struct { // Config references the image configuration as a blob. Config v1.Descriptor `json:"config"` + // ArtifactType is the type of an artifact when the manifest is used for an + // artifact. + ArtifactType string `json:"artifactType,omitempty"` + // Layers lists descriptors for the layers referenced by the // configuration. Layers []v1.Descriptor `json:"layers"` + // Subject is the descriptor of a manifest referred to by this manifest. + Subject *v1.Descriptor `json:"subject,omitempty"` + // Annotations contains arbitrary metadata for the image manifest. Annotations map[string]string `json:"annotations,omitempty"` } @@ -116,6 +123,21 @@ func (m *DeserializedManifest) UnmarshalJSON(b []byte) error { v1.MediaTypeImageManifest, mfst.MediaType) } + if mfst.Config.MediaType == v1.MediaTypeEmptyJSON && mfst.ArtifactType == "" { + return fmt.Errorf("if config.mediaType is '%s' then artifactType must be set", v1.MediaTypeEmptyJSON) + } + + // The subject if specified must be a manifest. This is validated here + // rather than in the storage manifest Put handler because the subject does + // not have to exist, so there is nothing to validate in the manifest store. + // If a non-compliant client provided the digest of a blob then this + // registry would still indicate that the referred manifest does not exist. + if mfst.Subject != nil { + if !distribution.ManifestMediaTypeSupported(mfst.Subject.MediaType) { + return fmt.Errorf("subject.mediaType must be a manifest, not '%s'", mfst.Subject.MediaType) + } + } + m.Manifest = mfst return nil @@ -137,6 +159,20 @@ func (m *DeserializedManifest) Payload() (string, []byte, error) { return v1.MediaTypeImageManifest, m.canonical, nil } +// Subject returns a pointer to the subject of this manifest or nil if there is +// none +func (m *DeserializedManifest) Subject() *distribution.Descriptor { + return m.Manifest.Subject +} + +// Type returns the artifactType of the manifest +func (m *DeserializedManifest) Type() string { + if m.ArtifactType == "" { + return m.Config.MediaType + } + return m.ArtifactType +} + // validateManifest returns an error if the byte slice is invalid JSON or if it // contains fields that belong to a index func validateManifest(b []byte) error { diff --git a/manifest/ocischema/manifest_test.go b/manifest/ocischema/manifest_test.go index 59d7ca389..776ea2891 100644 --- a/manifest/ocischema/manifest_test.go +++ b/manifest/ocischema/manifest_test.go @@ -6,6 +6,8 @@ import ( "reflect" "testing" + "github.com/distribution/distribution/v3/manifest/schema2" + "github.com/distribution/distribution/v3" "github.com/distribution/distribution/v3/manifest/manifestlist" "github.com/opencontainers/go-digest" @@ -39,6 +41,14 @@ const expectedManifestSerialization = `{ } }` +var ( + emptyJsonDescriptor = distribution.Descriptor{ + MediaType: v1.DescriptorEmptyJSON.MediaType, + Size: v1.DescriptorEmptyJSON.Size, + Digest: v1.DescriptorEmptyJSON.Digest, + } +) + func makeTestManifest(mediaType string) Manifest { return Manifest{ Versioned: specs.Versioned{SchemaVersion: 2}, @@ -243,3 +253,210 @@ func TestValidateManifest(t *testing.T) { } }) } + +func TestArtifactManifest(t *testing.T) { + for name, test := range map[string]struct { + manifest Manifest + expectValid bool + expectedArtifactType string + }{ + "not_artifact": { + manifest: Manifest{ + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + MediaType: v1.MediaTypeImageManifest, + Config: distribution.Descriptor{ + MediaType: v1.MediaTypeImageConfig, + Size: 200, + Digest: "sha256:4de6702c739d8c9ed907f4c031fd0abc54ee1bf372603a585e139730772cc0b8", + }, + Layers: []distribution.Descriptor{ + { + MediaType: v1.MediaTypeImageLayerGzip, + Size: 23423, + Digest: "sha256:ff1b4a27562d8ffc821b4d7368818ad7c759cfc2068b7adf0d2712315d67359a", + }, + }, + }, + expectValid: true, + expectedArtifactType: v1.MediaTypeImageConfig, + }, + "typical_artifact": { + manifest: Manifest{ + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + Config: distribution.Descriptor{ + MediaType: "application/vnd.example.thing", + Size: 200, + Digest: "sha256:4de6702c739d8c9ed907f4c031fd0abc54ee1bf372603a585e139730772cc0b8", + }, + Layers: []distribution.Descriptor{ + { + MediaType: v1.MediaTypeImageLayerGzip, + Size: 23423, + Digest: "sha256:ff1b4a27562d8ffc821b4d7368818ad7c759cfc2068b7adf0d2712315d67359a", + }, + }, + }, + expectValid: true, + expectedArtifactType: "application/vnd.example.thing", + }, + "also_typical_artifact": { + manifest: Manifest{ + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + ArtifactType: "application/vnd.example.sbom", + Config: distribution.Descriptor{ + MediaType: v1.MediaTypeImageConfig, + Size: 200, + Digest: "sha256:4de6702c739d8c9ed907f4c031fd0abc54ee1bf372603a585e139730772cc0b8", + }, + Layers: []distribution.Descriptor{ + { + MediaType: v1.MediaTypeImageLayerGzip, + Size: 23423, + Digest: "sha256:ff1b4a27562d8ffc821b4d7368818ad7c759cfc2068b7adf0d2712315d67359a", + }, + }, + }, + expectValid: true, + expectedArtifactType: "application/vnd.example.sbom", + }, + "configless_artifact": { + manifest: Manifest{ + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + ArtifactType: "application/vnd.example.catgif", + Config: emptyJsonDescriptor, + Layers: []distribution.Descriptor{ + { + MediaType: "image/gif", + Size: 23423, + Digest: "sha256:ff1b4a27562d8ffc821b4d7368818ad7c759cfc2068b7adf0d2712315d67359a", + }, + }, + }, + expectValid: true, + expectedArtifactType: "application/vnd.example.catgif", + }, + "invalid_artifact": { + manifest: Manifest{ + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + Config: emptyJsonDescriptor, // This MUST have an artifactType + Layers: []distribution.Descriptor{ + { + MediaType: "image/gif", + Size: 23423, + Digest: "sha256:ff1b4a27562d8ffc821b4d7368818ad7c759cfc2068b7adf0d2712315d67359a", + }, + }, + }, + expectValid: false, + }, + "annotation_artifact": { + manifest: Manifest{ + Versioned: specs.Versioned{ + SchemaVersion: 2., + }, + ArtifactType: "application/vnd.example.comment", + Config: emptyJsonDescriptor, + Layers: []distribution.Descriptor{ + emptyJsonDescriptor, + }, + Annotations: map[string]string{ + "com.example.data": "payload", + }, + }, + expectValid: true, + expectedArtifactType: "application/vnd.example.comment", + }, + "valid_subject": { + manifest: Manifest{ + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + ArtifactType: "application/vnd.example.comment", + Config: emptyJsonDescriptor, + Layers: []distribution.Descriptor{ + emptyJsonDescriptor, + }, + Subject: &distribution.Descriptor{ + MediaType: v1.MediaTypeImageManifest, + Size: 365, + Digest: "sha256:05b3abf2579a5eb66403cd78be557fd860633a1fe2103c7642030defe32c657f", + }, + Annotations: map[string]string{ + "com.example.data": "payload", + }, + }, + expectValid: true, + expectedArtifactType: "application/vnd.example.comment", + }, + "invalid_subject": { + manifest: Manifest{ + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + ArtifactType: "application/vnd.example.comment", + Config: emptyJsonDescriptor, + Layers: []distribution.Descriptor{ + emptyJsonDescriptor, + }, + Subject: &distribution.Descriptor{ + MediaType: v1.MediaTypeImageLayerGzip, // The subject is a manifest + Size: 365, + Digest: "sha256:05b3abf2579a5eb66403cd78be557fd860633a1fe2103c7642030defe32c657f", + }, + Annotations: map[string]string{ + "com.example.data": "payload", + }, + }, + expectValid: false, + }, + "docker_manifest_valid_as_subject": { + manifest: Manifest{ + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + ArtifactType: "application/vnd.example.comment", + Config: emptyJsonDescriptor, + Layers: []distribution.Descriptor{ + emptyJsonDescriptor, + }, + Subject: &distribution.Descriptor{ + MediaType: schema2.MediaTypeManifest, + Size: 365, + Digest: "sha256:05b3abf2579a5eb66403cd78be557fd860633a1fe2103c7642030defe32c657f", + }, + Annotations: map[string]string{ + "com.example.data": "payload", + }, + }, + expectValid: true, + expectedArtifactType: "application/vnd.example.comment", + }, + } { + t.Run(name, func(t *testing.T) { + dm, err := FromStruct(test.manifest) + if err != nil { + t.Fatalf("Error making DeserializedManifest from struct: %s", err) + } + m, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, dm.canonical) + if test.expectValid != (nil == err) { + t.Fatalf("expectValid=%t but got err=%v", test.expectValid, err) + } + if err != nil { + return + } + if artifactType := m.(distribution.Referrer).Type(); artifactType != test.expectedArtifactType { + t.Errorf("Expected artifactType to be %q but got %q", test.expectedArtifactType, artifactType) + } + }) + } +} diff --git a/manifests.go b/manifests.go index 290558dab..51ff93a5c 100644 --- a/manifests.go +++ b/manifests.go @@ -27,6 +27,18 @@ type Manifest interface { Payload() (mediaType string, payload []byte, err error) } +// Referrer represents a Manifest which can refer to a subject. +type Referrer interface { + // Subject returns a pointer to a Descriptor representing the manifest which + // this manifest refers to or nil if this manifest does not refer to a + // subject. + Subject() *Descriptor + + // Type returns the type of the referrer if there is one, otherwise it + // returns empty string + Type() string +} + // ManifestService describes operations on manifests. type ManifestService interface { // Exists returns true if the manifest exists. @@ -68,6 +80,12 @@ func ManifestMediaTypes() (mediaTypes []string) { return } +// ManifestMediaTypeSupported returns true if the given mediaType is supported. +func ManifestMediaTypeSupported(mediaType string) bool { + _, ok := mappings[mediaType] + return ok +} + // UnmarshalFunc implements manifest unmarshalling a given MediaType type UnmarshalFunc func([]byte) (Manifest, v1.Descriptor, error) diff --git a/registry/api/errcode/register.go b/registry/api/errcode/register.go index 6030c36a5..ef94f897e 100644 --- a/registry/api/errcode/register.go +++ b/registry/api/errcode/register.go @@ -178,7 +178,7 @@ var ( ErrorCodeManifestBlobUnknown = register(errGroup, ErrorDescriptor{ Value: "MANIFEST_BLOB_UNKNOWN", Message: "blob unknown to registry", - Description: `This error may be returned when a manifest blob is + Description: `This error may be returned when a manifest blob is unknown to the registry.`, HTTPStatusCode: http.StatusBadRequest, }) @@ -224,6 +224,16 @@ var ( the maximum allowed.`, HTTPStatusCode: http.StatusBadRequest, }) + + // ErrorCodeManifestNotAcceptable is returned when the manifest found is not + // acceptable according to the client's Accept header + ErrorCodeManifestNotAcceptable = register(errGroup, ErrorDescriptor{ + Value: "MANIFEST_NOT_ACCEPTABLE", + Message: "manifest does not match Accept header", + Description: `This is returned if the manifest known to the registry + has a different mediaType then the client's Accept header.`, + HTTPStatusCode: http.StatusNotAcceptable, + }) ) var ( diff --git a/registry/handlers/api_test.go b/registry/handlers/api_test.go index 754def6c4..64ded71e1 100644 --- a/registry/handlers/api_test.go +++ b/registry/handlers/api_test.go @@ -19,6 +19,9 @@ import ( "strings" "testing" + "github.com/distribution/distribution/v3/manifest/ocischema" + v1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/distribution/distribution/v3" "github.com/distribution/distribution/v3/configuration" "github.com/distribution/distribution/v3/manifest/manifestlist" @@ -33,12 +36,18 @@ import ( "github.com/gorilla/handlers" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/specs-go" - v1 "github.com/opencontainers/image-spec/specs-go/v1" ) -var headerConfig = http.Header{ - "X-Content-Type-Options": []string{"nosniff"}, -} +var ( + headerConfig = http.Header{ + "X-Content-Type-Options": []string{"nosniff"}, + } + emptyJsonDescriptor = distribution.Descriptor{ + MediaType: v1.DescriptorEmptyJSON.MediaType, + Size: v1.DescriptorEmptyJSON.Size, + Digest: v1.DescriptorEmptyJSON.Digest, + } +) const ( // digestSha256EmptyTar is the canonical sha256 digest of empty data @@ -2856,3 +2865,471 @@ func TestProxyManifestGetByTag(t *testing.T) { "Docker-Content-Digest": []string{newDigest.String()}, }) } + +func TestArtifactManifest(t *testing.T) { + for name, test := range map[string]struct { + manifest func(*testing.T, *testEnv, reference.Named) distribution.Manifest + deleteSubject bool + }{ + // The link is made when the subject already exists and is kept if the + // subject is deleted + "subject_exists": { + manifest: func(t *testing.T, testEnv *testEnv, repo reference.Named) distribution.Manifest { + args := testManifestAPISchema2(t, testEnv, repo, "schema2tag") + _, payload, err := args.manifest.Payload() + if err != nil { + t.Fatalf("Failed to get subject payload: %s", err) + } + + pushScratch(t, testEnv, repo) + + manifest, err := ocischema.FromStruct(ocischema.Manifest{ + Versioned: specs.Versioned{SchemaVersion: 2}, + ArtifactType: "application/vnd.example.sbom.v1", + Config: emptyJsonDescriptor, + Subject: &distribution.Descriptor{ + MediaType: args.mediaType, + Digest: args.dgst, + Size: int64(len(payload)), + }, + }) + if err != nil { + t.Fatalf("Failed to create manifest: %s", err) + } + return manifest + }, + deleteSubject: true, + }, + // When an OCI Image Manifest with a subject field is PUT before its + // subject, the subject's referrers link will be made in advance. + "image_manifest_with_subject": { + manifest: func(t *testing.T, testEnv *testEnv, repo reference.Named) distribution.Manifest { + config, configDigest, err := testutil.CreateRandomTarFile() + if err != nil { + t.Fatalf("Failed to create test blob: %s", err) + } + url, _ := startPushLayer(t, testEnv, repo) + pushLayer(t, testEnv.builder, repo, configDigest, url, config) + manifest, err := ocischema.FromStruct(ocischema.Manifest{ + Versioned: specs.Versioned{SchemaVersion: 2}, + Config: distribution.Descriptor{ + MediaType: v1.MediaTypeImageConfig, + Digest: configDigest, + }, + Subject: &distribution.Descriptor{ + MediaType: v1.MediaTypeImageManifest, + Digest: "sha256:ebe054f08821294feee7bc442014fdd38b4836d83781d8ba99d38eb50d0c9d85", + Size: 99, + }, + }) + if err != nil { + t.Fatalf("Failed to create manifest: %s", err) + } + return manifest + }, + }, + } { + t.Run(name, func(t *testing.T) { + testEnv := newTestEnv(t, true) + defer testEnv.Shutdown() + + repo, err := reference.WithName("myrepo/myimage") + if err != nil { + t.Fatalf("failed to make repo: %s", err) + } + + manifest := test.manifest(t, testEnv, repo) + + contentType, payload, err := manifest.Payload() + if err != nil { + t.Fatalf("Failed to get raw manifest: %s", err) + } + ref, err := reference.WithDigest(repo, digest.FromBytes(payload)) + if err != nil { + t.Fatalf("failed to make reference: %s", err) + } + + manifestURL, err := testEnv.builder.BuildManifestURL(ref) + if err != nil { + t.Fatalf("Failed to build manifest URL: %s", err) + } + + req, err := http.NewRequest(http.MethodPut, manifestURL, bytes.NewReader(payload)) + if err != nil { + t.Fatalf("Failed to create artifact PUT request: %s", err) + } + req.Header.Set("Content-Type", contentType) + + res, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("Failed to PUT manifest: %s", err) + } + defer res.Body.Close() + if res.StatusCode != http.StatusCreated { + t.Fatalf("Incorrect status code for manifest PUT: %d, expected: %d", res.StatusCode, http.StatusCreated) + } + if res.Header.Get("Docker-Content-Digest") != ref.Digest().String() { + t.Errorf("Incorrect Docker-Content-Digest header: %q, expected %q", res.Header.Get("Docker-Content-Digest"), ref.Digest().String()) + } + + // TODO(brackendawson): We should now try to get referrers for the subject + // (which should also eventually exist for that to work), but those APIs + // don't exist yet so for now just check the link was made. + referrer, ok := manifest.(distribution.Referrer) + if !ok { + t.Fatalf("Manifest should implement distribution.Referrer: %T", manifest) + } + link, err := testEnv.app.driver.GetContent(context.Background(), fmt.Sprintf("/docker/registry/v2/repositories/%s/_manifests/revisions/sha256/%s/_referrers/_%s/sha256/%s/link", + repo.Name(), referrer.Subject().Digest.Hex(), url.QueryEscape(referrer.Type()), ref.Digest().Hex())) + if err != nil { + t.Fatalf("Failed to get expected referrers link from subject with error: %s", err) + } + if string(link) != ref.Digest().String() { + t.Errorf("Subject's referrers link has incorrect content:\n%s\nexpected:\n%s", string(link), ref.Digest().String()) + } + + // When an artifact manifest has been PUT it can be retrieved with GET. + location := res.Header.Get("Location") + req, err = http.NewRequest(http.MethodGet, location, nil) + if err != nil { + t.Fatalf("Failed to create artifact GET request: %s", err) + } + + res, err = http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("Failed to GET manifest: %s", err) + } + defer res.Body.Close() + if res.StatusCode != http.StatusNotAcceptable { + t.Fatalf("Incorrect status code for manifest GET: %d, expected: %d", res.StatusCode, http.StatusNotAcceptable) + } + + req.Header.Set("Accept", contentType) + res, err = http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("Failed to GET manifest: %s", err) + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + t.Fatalf("Incorrect status code for manifest GET: %d, expected: %d", res.StatusCode, http.StatusOK) + } + if res.Header.Get("Content-Type") != contentType { + t.Errorf("Incorrect mediaType for manifest GET: %q, expected: %q", res.Header.Get("Content-Type"), contentType) + } + gotManifest, err := io.ReadAll(res.Body) + if err != nil { + t.Fatalf("Failed to read manifest GET body: %s", err) + } + if !reflect.DeepEqual(gotManifest, payload) { + t.Errorf("Pulled manifest does not match pushed manifest, got:\n%s\nexpected:\n%s", string(gotManifest), string(payload)) + } + + if test.deleteSubject { + // When a subject is deleted, it's referrers link remains + subjectRef, err := reference.WithDigest(repo, referrer.Subject().Digest) + if err != nil { + t.Fatalf("Failed to build subject reference: %s", err) + } + subjectURL, err := testEnv.builder.BuildManifestURL(subjectRef) + if err != nil { + t.Errorf("Failed to build subject URL: %s", err) + } + + req, err := http.NewRequest(http.MethodDelete, subjectURL, nil) + if err != nil { + t.Fatalf("Failed to create subject DELETE request: %s", err) + } + res, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("Failed to DELETE subject: %s", err) + } + defer res.Body.Close() + if res.StatusCode != http.StatusAccepted { + t.Fatalf("Incorrect status code for subject DELETE: %s", err) + } + + // TODO(brackendawson): We should now try to get referrers for the subject + // (which should also eventually exist for that to work), but those APIs + // don't exist yet so for now just check the link still exists. + link, err := testEnv.app.driver.GetContent(context.Background(), fmt.Sprintf("/docker/registry/v2/repositories/%s/_manifests/revisions/sha256/%s/_referrers/_%s/sha256/%s/link", + repo.Name(), referrer.Subject().Digest.Hex(), url.QueryEscape(referrer.Type()), ref.Digest().Hex())) + if err != nil { + t.Fatalf("Failed to get expected referrers link from subject with error: %s", err) + } + if string(link) != ref.Digest().String() { + t.Errorf("Subject's referrers link has incorrect content:\n%s\nexpected:\n%s", string(link), ref.Digest().String()) + } + } + + // When an artifact manifest is DELETEd then it will not be found if you GET + // it. Its subject's referrer link will be left dangling. + req, err = http.NewRequest(http.MethodDelete, location, nil) + if err != nil { + t.Fatalf("Failed to create artifact DELETE request: %s", err) + } + + res, err = http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("Failed to DELETE manifest: %s", err) + } + defer res.Body.Close() + if res.StatusCode != http.StatusAccepted { + t.Fatalf("Incorrect status code for manifest DELETE: %d, expected: %d", res.StatusCode, http.StatusAccepted) + } + + req, err = http.NewRequest(http.MethodGet, location, nil) + if err != nil { + t.Fatalf("Failed to create artifact GET request: %s", err) + } + req.Header.Set("Accept", v1.MediaTypeImageManifest) + res, err = http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("Failed to GET manifest: %s", err) + } + defer res.Body.Close() + if res.StatusCode != http.StatusNotFound { + t.Fatalf("Incorrect status code for manifest GET: %d, expected: %d", res.StatusCode, http.StatusNotFound) + } + }) + } +} + +func TestDockerManifestWithSubject(t *testing.T) { + // When a docker image manifest containing a "subject" field is uploaded + // then no referrer links are made for that invalid subject. + testEnv := newTestEnv(t, true) + defer testEnv.Shutdown() + + repo, err := reference.WithName("test/repo") + if err != nil { + t.Fatalf("Failed to build repo: %s", err) + } + + config, configDigest, err := testutil.CreateRandomTarFile() + if err != nil { + t.Fatalf("Failed to make config blob: %s", err) + } + url, _ := startPushLayer(t, testEnv, repo) + pushLayer(t, testEnv.builder, repo, configDigest, url, config) + layer, layerDigest, err := testutil.CreateRandomTarFile() + if err != nil { + t.Fatalf("Failed to make layer blob: %s", err) + } + url, _ = startPushLayer(t, testEnv, repo) + pushLayer(t, testEnv.builder, repo, layerDigest, url, layer) + + manifest, err := schema2.FromStruct(schema2.Manifest{ + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + MediaType: schema2.MediaTypeManifest, + Config: distribution.Descriptor{ + MediaType: schema2.MediaTypeImageConfig, + Digest: configDigest, + Size: testutil.MustSeekerLen(config), + }, + Layers: []distribution.Descriptor{ + { + MediaType: schema2.MediaTypeLayer, + Digest: layerDigest, + Size: testutil.MustSeekerLen(layer), + }, + }, + }) + if err != nil { + t.Fatalf("Failed to make base manifest: %s", err) + } + var manifestFields map[string]interface{} + _, payload, err := manifest.Payload() + if err != nil { + t.Fatalf("Failed to get base manifest payload: %s", err) + } + if err = json.Unmarshal(payload, &manifestFields); err != nil { + t.Fatalf("Failed to unmarshal base manifest: %s", err) + } + manifestFields["subject"] = distribution.Descriptor{ + MediaType: schema2.MediaTypeManifest, + Digest: "sha256:118011bef6c697f7107cc0d788664a0f8c7d0316ce8d17673634155f5ecdba39", + Size: 56, + } + rawManifest, _ := json.Marshal(manifestFields) + if err = manifest.UnmarshalJSON(rawManifest); err != nil { + t.Fatalf("Failed to re-build manifest: %s", err) + } + + ref, err := reference.WithTag(repo, "latest") + if err != nil { + t.Fatalf("Failed to build reference: %s", err) + } + if url, err = testEnv.builder.BuildManifestURL(ref); err != nil { + t.Fatalf("Failed to build manifest url: %s", err) + } + res := putManifest(t, "putting manifest", url, schema2.MediaTypeManifest, manifest) + if res.StatusCode != http.StatusCreated { + t.Fatalf("Incorrect status code from manifest PUT: %d, expected: %d", res.StatusCode, http.StatusCreated) + } + defer res.Body.Close() + + // TODO(brackendawson): We should now try to get referrers for the subject + // (which should also eventually exist for that to work), but those APIs + // don't exist yet so for now just check the link was not made. + _, err = testEnv.app.driver.Stat(context.Background(), fmt.Sprintf("/docker/registry/v2/repositories/%s/_manifests/revisions/sha256/%s/_referrers", + repo.Name(), res.Header.Get("Docker-Content-Digest"))) + var expectedErr storagedriver.InvalidPathError + if !errors.As(err, &expectedErr) { + t.Fatalf("Should have got invalid path error for referrer directory: %s", err) + } +} + +func TestArtifactManifestValidation(t *testing.T) { + for name, test := range map[string]struct { + config func(*testing.T, *testEnv, reference.Named) distribution.Descriptor + layers func(*testing.T, *testEnv, reference.Named) []distribution.Descriptor + wantCode int + }{ + "layers_must_exist": { + config: func(t *testing.T, testEnv *testEnv, repo reference.Named) distribution.Descriptor { + pushScratch(t, testEnv, repo) + return emptyJsonDescriptor + }, + layers: func(t *testing.T, te *testEnv, n reference.Named) []distribution.Descriptor { + // a layer which has not been uploaded + return []distribution.Descriptor{ + { + MediaType: v1.MediaTypeImageLayer, + Digest: "sha256:7688b6ef52555962d008fff894223582c484517cea7da49ee67800adc7fc8866", + Size: 56, + }, + } + }, + wantCode: 400, + }, + "config_must_be_set": { + config: func(t *testing.T, testEnv *testEnv, repo reference.Named) distribution.Descriptor { + return distribution.Descriptor{} // zero value + }, + layers: func(t *testing.T, testEnv *testEnv, repo reference.Named) []distribution.Descriptor { + layers := int64(10) + digests := make([]distribution.Descriptor, layers) + for i := int64(0); i < layers; i++ { + rs, digest, err := testutil.CreateRandomTarFile() + if err != nil { + t.Fatalf("Failed to create test blob: %s", err) + } + url, _ := startPushLayer(t, testEnv, repo) + pushLayer(t, testEnv.builder, repo, digest, url, rs) + digests[i] = distribution.Descriptor{ + MediaType: v1.MediaTypeImageLayer, + Digest: digest, + Size: testutil.MustSeekerLen(rs), + } + } + return digests + }, + wantCode: 400, + }, + "config_must_exist": { + config: func(t *testing.T, testEnv *testEnv, repo reference.Named) distribution.Descriptor { + return emptyJsonDescriptor // not uploaded + }, + layers: func(t *testing.T, testEnv *testEnv, repo reference.Named) []distribution.Descriptor { + layers := int64(10) + digests := make([]distribution.Descriptor, layers) + for i := int64(0); i < layers; i++ { + rs, digest, err := testutil.CreateRandomTarFile() + if err != nil { + t.Fatalf("Failed to create test blob: %s", err) + } + url, _ := startPushLayer(t, testEnv, repo) + pushLayer(t, testEnv.builder, repo, digest, url, rs) + digests[i] = distribution.Descriptor{ + MediaType: v1.MediaTypeImageLayer, + Digest: digest, + Size: testutil.MustSeekerLen(rs), + } + } + return digests + }, + wantCode: 400, + }, + "valid_blobs": { + config: func(t *testing.T, testEnv *testEnv, repo reference.Named) distribution.Descriptor { + pushScratch(t, testEnv, repo) + return emptyJsonDescriptor + }, + layers: func(t *testing.T, testEnv *testEnv, repo reference.Named) []distribution.Descriptor { + layers := int64(10) + digests := make([]distribution.Descriptor, layers) + for i := int64(0); i < layers; i++ { + rs, digest, err := testutil.CreateRandomTarFile() + if err != nil { + t.Fatalf("Failed to create test blob: %s", err) + } + url, _ := startPushLayer(t, testEnv, repo) + pushLayer(t, testEnv.builder, repo, digest, url, rs) + digests[i] = distribution.Descriptor{ + MediaType: v1.MediaTypeImageLayer, + Digest: digest, + Size: testutil.MustSeekerLen(rs), + } + } + return digests + }, + wantCode: 201, + }, + } { + t.Run(name, func(t *testing.T) { + testEnv := newTestEnv(t, true) + defer testEnv.Shutdown() + + repo, err := reference.WithName("myrepo/myimage") + if err != nil { + t.Fatalf("failed to make repo: %s", err) + } + + manifest, err := ocischema.FromStruct(ocischema.Manifest{ + Versioned: specs.Versioned{SchemaVersion: 2}, + Config: test.config(t, testEnv, repo), + ArtifactType: "application/vnd.example.sbom.v1", + Layers: test.layers(t, testEnv, repo), + }) + if err != nil { + t.Fatalf("Failed to make manifest: %s", err) + } + + contentType, payload, err := manifest.Payload() + if err != nil { + t.Fatalf("Failed to get raw manifest: %s", err) + } + ref, err := reference.WithDigest(repo, digest.FromBytes(payload)) + if err != nil { + t.Fatalf("failed to make reference: %s", err) + } + + manifestURL, err := testEnv.builder.BuildManifestURL(ref) + if err != nil { + t.Fatalf("Failed to build manifest URL: %s", err) + } + + req, err := http.NewRequest(http.MethodPut, manifestURL, bytes.NewReader(payload)) + if err != nil { + t.Fatalf("Failed to create artifact PUT request: %s", err) + } + req.Header.Set("Content-Type", contentType) + + res, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("Failed to PUT manifest: %s", err) + } + defer res.Body.Close() + if res.StatusCode != test.wantCode { + t.Fatalf("Incorrect status code for manifest PUT: %d, expected: %d", res.StatusCode, test.wantCode) + } + }) + } +} + +func pushScratch(t *testing.T, testEnv *testEnv, repo reference.Named) { + url, _ := startPushLayer(t, testEnv, repo) + pushLayer(t, testEnv.builder, repo, v1.DescriptorEmptyJSON.Digest, url, bytes.NewBuffer(v1.DescriptorEmptyJSON.Data)) +} diff --git a/registry/handlers/manifests.go b/registry/handlers/manifests.go index 6b7648c2b..75b3d29d4 100644 --- a/registry/handlers/manifests.go +++ b/registry/handlers/manifests.go @@ -161,11 +161,11 @@ func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request) } if manifestType == ociSchema && !supports[ociSchema] { - imh.Errors = append(imh.Errors, errcode.ErrorCodeManifestUnknown.WithMessage("OCI manifest found, but accept header does not support OCI manifests")) + imh.Errors = append(imh.Errors, errcode.ErrorCodeManifestNotAcceptable.WithMessage("OCI manifest found, but accept header does not support OCI manifests")) return } if manifestType == ociImageIndexSchema && !supports[ociImageIndexSchema] { - imh.Errors = append(imh.Errors, errcode.ErrorCodeManifestUnknown.WithMessage("OCI index found, but accept header does not support OCI indexes")) + imh.Errors = append(imh.Errors, errcode.ErrorCodeManifestNotAcceptable.WithMessage("OCI index found, but accept header does not support OCI indexes")) return } diff --git a/registry/storage/driver/storagedriver.go b/registry/storage/driver/storagedriver.go index c12f79de3..5e95359fa 100644 --- a/registry/storage/driver/storagedriver.go +++ b/registry/storage/driver/storagedriver.go @@ -134,9 +134,9 @@ type FileWriter interface { // PathRegexp is the regular expression which each file path must match. A // file path is absolute, beginning with a slash and containing a positive // number of path components separated by slashes, where each component is -// restricted to alphanumeric characters or a period, underscore, or -// hyphen. -var PathRegexp = regexp.MustCompile(`^(/[A-Za-z0-9._-]+)+$`) +// restricted to alphanumeric characters or a period, underscore, hyphen or +// percent. +var PathRegexp = regexp.MustCompile(`^(/[A-Za-z0-9._%-]+)+$`) // ErrUnsupportedMethod may be returned in the case where a StorageDriver implementation does not support an optional method. type ErrUnsupportedMethod struct { diff --git a/registry/storage/ocimanifesthandler.go b/registry/storage/ocimanifesthandler.go index 86e2368c3..985437323 100644 --- a/registry/storage/ocimanifesthandler.go +++ b/registry/storage/ocimanifesthandler.go @@ -18,6 +18,7 @@ type ocischemaManifestHandler struct { blobStore distribution.BlobStore ctx context.Context manifestURLs manifestURLs + references ReferenceService } var _ ManifestHandler = &ocischemaManifestHandler{} @@ -45,11 +46,19 @@ func (ms *ocischemaManifestHandler) Put(ctx context.Context, manifest distributi return "", err } - mt, payload, err := m.Payload() + mt, payload, err := manifest.Payload() if err != nil { return "", err } + if referrer, ok := manifest.(distribution.Referrer); ok { + if subject := referrer.Subject(); subject != nil { + if err := ms.references.Link(ctx, referrer.Type(), digest.FromBytes(payload), subject.Digest); err != nil { + return "", err + } + } + } + revision, err := ms.blobStore.Put(ctx, mt, payload) if err != nil { dcontext.GetLogger(ctx).Errorf("error putting payload into blobstore: %v", err) @@ -88,7 +97,7 @@ func (ms *ocischemaManifestHandler) verifyManifest(ctx context.Context, mnfst oc } switch descriptor.MediaType { - case v1.MediaTypeImageLayer, v1.MediaTypeImageLayerGzip, v1.MediaTypeImageLayerNonDistributable, v1.MediaTypeImageLayerNonDistributableGzip: //nolint:staticcheck // ignore A1019: v1.MediaTypeImageLayerNonDistributable is deprecated: Non-distributable layers are deprecated, and not recommended for future use. + case v1.MediaTypeImageLayer, v1.MediaTypeImageLayerGzip, v1.MediaTypeImageLayerNonDistributable, v1.MediaTypeImageLayerNonDistributableGzip: //nolint:staticcheck // Ignore SA1019 v1.MediaTypeImageLayerNonDistributable is deprecated, it is used for backwards compatibility allow := ms.manifestURLs.allow deny := ms.manifestURLs.deny for _, u := range descriptor.URLs { diff --git a/registry/storage/ocimanifesthandler_test.go b/registry/storage/ocimanifesthandler_test.go index c8e3b5efb..24a1320ef 100644 --- a/registry/storage/ocimanifesthandler_test.go +++ b/registry/storage/ocimanifesthandler_test.go @@ -36,7 +36,7 @@ func TestVerifyOCIManifestNonDistributableLayer(t *testing.T) { nonDistributableLayer := v1.Descriptor{ Digest: "sha256:463435349086340864309863409683460843608348608934092322395278926a", Size: 6323, - MediaType: v1.MediaTypeImageLayerNonDistributableGzip, //nolint:staticcheck // ignore A1019: v1.MediaTypeImageLayerNonDistributableGzip is deprecated: Non-distributable layers are deprecated, and not recommended for future use + MediaType: v1.MediaTypeImageLayerNonDistributableGzip, //nolint:staticcheck // Ignore SA1019 v1.MediaTypeImageLayerNonDistributable is deprecated, it is used for backwards compatibility } emptyLayer := v1.Descriptor{ diff --git a/registry/storage/paths.go b/registry/storage/paths.go index 7771b581c..4f606188e 100644 --- a/registry/storage/paths.go +++ b/registry/storage/paths.go @@ -2,6 +2,7 @@ package storage import ( "fmt" + "net/url" "path" "strings" @@ -114,6 +115,10 @@ const ( // blobPathSpec: /v2/blobs/// // blobDataPathSpec: /v2/blobs////data // +// Artifacts: +// +// subjectReferrerPathSpec /v2/repositories//_manifests/revisions///_referrers/_///link +// // For more information on the semantic meaning of each path and their // contents, please see the path spec documentation. func pathFor(spec pathSpec) (string, error) { @@ -250,6 +255,19 @@ func pathFor(spec pathSpec) (string, error) { return path.Join(append(repoPrefix, v.name, "_uploads", v.id, "hashstates", string(v.alg), offset)...), nil case repositoriesRootPathSpec: return path.Join(repoPrefix...), nil + case subjectReferrerLinkPathSpec: + manifestPath, err := pathFor(manifestRevisionPathSpec{name: v.name, revision: v.subject}) + if err != nil { + return "", err + } + + artifactType := "_" + url.QueryEscape(v.mediaType) + components, err := digestPathComponents(v.referrer, false) + if err != nil { + return "", err + } + + return path.Join(manifestPath, "_referrers", artifactType, path.Join(components...), "link"), nil default: // TODO(sday): This is an internal error. Ensure it doesn't escape (panic?). return "", fmt.Errorf("unknown path spec: %#v", v) @@ -450,6 +468,17 @@ type repositoriesRootPathSpec struct{} func (repositoriesRootPathSpec) pathSpec() {} +// subjectReferrerLinkPathSpec returns the describes the link of a subject to +// its referrers +type subjectReferrerLinkPathSpec struct { + name string + referrer digest.Digest + subject digest.Digest + mediaType string +} + +func (subjectReferrerLinkPathSpec) pathSpec() {} + // digestPathComponents provides a consistent path breakdown for a given // digest. For a generic digest, it will be as follows: // diff --git a/registry/storage/references.go b/registry/storage/references.go new file mode 100644 index 000000000..a230f3800 --- /dev/null +++ b/registry/storage/references.go @@ -0,0 +1,35 @@ +package storage + +import ( + "context" + + "github.com/distribution/distribution/v3" + "github.com/opencontainers/go-digest" +) + +// ReferenceService is a service to manage internal links from subjects back to +// their referrers. +type ReferenceService interface { + // Link creates a link from a subject back to a referrer + Link(ctx context.Context, mediaType string, referrer, subject digest.Digest) error +} + +type referenceHandler struct { + *blobStore + repository distribution.Repository + pathFn func(name, mediaType string, reference, artifact_subject_must_be_manifest digest.Digest) (string, error) +} + +func (r *referenceHandler) Link(ctx context.Context, artifactType string, referrer, subject digest.Digest) error { + path, err := r.pathFn(r.repository.Named().Name(), artifactType, referrer, subject) + if err != nil { + return err + } + + return r.blobStore.link(ctx, path, referrer) +} + +// subjectReferrerLinkPath provides the path to the subject's referrer link +func subjectReferrerLinkPath(name, mediaType string, referrer, subject digest.Digest) (string, error) { + return pathFor(subjectReferrerLinkPathSpec{name: name, mediaType: mediaType, referrer: referrer, subject: subject}) +} diff --git a/registry/storage/registry.go b/registry/storage/registry.go index 5b4d6c424..4f23e8f8c 100644 --- a/registry/storage/registry.go +++ b/registry/storage/registry.go @@ -305,6 +305,11 @@ func (repo *repository) Manifests(ctx context.Context, options ...distribution.M repository: repo, blobStore: blobStore, manifestURLs: repo.registry.manifestURLs, + references: &referenceHandler{ + blobStore: repo.blobStore, + repository: repo, + pathFn: subjectReferrerLinkPath, + }, }, ocischemaIndexHandler: &ocischemaIndexHandler{ manifestListHandler: manifestListHandler, diff --git a/testutil/tarfile.go b/testutil/tarfile.go index 84634cdbb..97c2c8e3e 100644 --- a/testutil/tarfile.go +++ b/testutil/tarfile.go @@ -114,3 +114,29 @@ func UploadBlobs(repository distribution.Repository, layers map[digest.Digest]io } return nil } + +// SeekerLen returns the apparent size of s +func SeekerLen(s io.Seeker) (int64, error) { + offset, err := s.Seek(0, io.SeekCurrent) + if err != nil { + return 0, fmt.Errorf("failed to read initial offset: %w", err) + } + size, err := s.Seek(0, io.SeekEnd) + if err != nil { + return 0, fmt.Errorf("failed to read size: %w", err) + } + _, err = s.Seek(offset, io.SeekStart) + if err != nil { + return 0, fmt.Errorf("failed to restore initial offset: %w", err) + } + return size, nil +} + +// MustSeekerLen returns the apparent size of s or panics +func MustSeekerLen(s io.Seeker) int64 { + size, err := SeekerLen(s) + if err != nil { + panic(err) + } + return size +}