diff --git a/pkg/registry/core/namespace/storage/storage.go b/pkg/registry/core/namespace/storage/storage.go index fecac7428c9..b76e48f01dd 100644 --- a/pkg/registry/core/namespace/storage/storage.go +++ b/pkg/registry/core/namespace/storage/storage.go @@ -125,7 +125,7 @@ func (r *REST) Export(ctx context.Context, name string, opts metav1.ExportOption } // Delete enforces life-cycle rules for namespace termination -func (r *REST) Delete(ctx context.Context, name string, options *metav1.DeleteOptions) (runtime.Object, bool, error) { +func (r *REST) Delete(ctx context.Context, name string, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions) (runtime.Object, bool, error) { nsObj, err := r.Get(ctx, name, &metav1.GetOptions{}) if err != nil { return nil, false, err @@ -162,6 +162,10 @@ func (r *REST) Delete(ctx context.Context, name string, options *metav1.DeleteOp // upon first request to delete, we switch the phase to start namespace termination // TODO: enhance graceful deletion's calls to DeleteStrategy to allow phase change and finalizer patterns if namespace.DeletionTimestamp.IsZero() { + if err := deleteValidation(nsObj); err != nil { + return nil, false, err + } + key, err := r.store.KeyFunc(ctx, name) if err != nil { return nil, false, err @@ -238,7 +242,7 @@ func (r *REST) Delete(ctx context.Context, name string, options *metav1.DeleteOp err = apierrors.NewConflict(api.Resource("namespaces"), namespace.Name, fmt.Errorf("The system is ensuring all content is removed from this namespace. Upon completion, this namespace will automatically be purged by the system.")) return nil, false, err } - return r.store.Delete(ctx, name, options) + return r.store.Delete(ctx, name, deleteValidation, options) } // ShouldDeleteNamespaceDuringUpdate adds namespace-specific spec.finalizer checks on top of the default generic ShouldDeleteDuringUpdate behavior diff --git a/pkg/registry/core/namespace/storage/storage_test.go b/pkg/registry/core/namespace/storage/storage_test.go index 68191b936b4..f36b645ce39 100644 --- a/pkg/registry/core/namespace/storage/storage_test.go +++ b/pkg/registry/core/namespace/storage/storage_test.go @@ -162,7 +162,7 @@ func TestDeleteNamespaceWithIncompleteFinalizers(t *testing.T) { if err := storage.store.Storage.Create(ctx, key, namespace, nil, 0, false); err != nil { t.Fatalf("unexpected error: %v", err) } - if _, _, err := storage.Delete(ctx, "foo", nil); err == nil { + if _, _, err := storage.Delete(ctx, "foo", rest.ValidateAllObjectFunc, nil); err == nil { t.Errorf("unexpected error: %v", err) } // should still exist @@ -375,7 +375,7 @@ func TestDeleteNamespaceWithCompleteFinalizers(t *testing.T) { if err := storage.store.Storage.Create(ctx, key, namespace, nil, 0, false); err != nil { t.Fatalf("unexpected error: %v", err) } - if _, _, err := storage.Delete(ctx, "foo", nil); err != nil { + if _, _, err := storage.Delete(ctx, "foo", rest.ValidateAllObjectFunc, nil); err != nil { t.Errorf("unexpected error: %v", err) } // should not exist diff --git a/pkg/registry/core/pod/storage/eviction.go b/pkg/registry/core/pod/storage/eviction.go index 61899427499..386244451b0 100644 --- a/pkg/registry/core/pod/storage/eviction.go +++ b/pkg/registry/core/pod/storage/eviction.go @@ -124,7 +124,7 @@ func (r *EvictionREST) Create(ctx context.Context, obj runtime.Object, createVal // Evicting a terminal pod should result in direct deletion of pod as it already caused disruption by the time we are evicting. // There is no need to check for pdb. if pod.Status.Phase == api.PodSucceeded || pod.Status.Phase == api.PodFailed { - _, _, err = r.store.Delete(ctx, eviction.Name, deletionOptions) + _, _, err = r.store.Delete(ctx, eviction.Name, rest.ValidateAllObjectFunc, deletionOptions) if err != nil { return nil, err } @@ -173,7 +173,7 @@ func (r *EvictionREST) Create(ctx context.Context, obj runtime.Object, createVal // At this point there was either no PDB or we succeeded in decrementing // Try the delete - _, _, err = r.store.Delete(ctx, eviction.Name, deletionOptions) + _, _, err = r.store.Delete(ctx, eviction.Name, rest.ValidateAllObjectFunc, deletionOptions) if err != nil { return nil, err } diff --git a/pkg/registry/core/pod/storage/eviction_test.go b/pkg/registry/core/pod/storage/eviction_test.go index 83590a22f95..e33a24f9d57 100644 --- a/pkg/registry/core/pod/storage/eviction_test.go +++ b/pkg/registry/core/pod/storage/eviction_test.go @@ -150,7 +150,7 @@ type FailDeleteUpdateStorage struct { storage.Interface } -func (f FailDeleteUpdateStorage) Delete(ctx context.Context, key string, out runtime.Object, precondition *storage.Preconditions) error { +func (f FailDeleteUpdateStorage) Delete(ctx context.Context, key string, out runtime.Object, precondition *storage.Preconditions, validateDeletion storage.ValidateObjectFunc) error { return storage.NewKeyNotFoundError(key, 0) } diff --git a/pkg/registry/core/pod/storage/storage_test.go b/pkg/registry/core/pod/storage/storage_test.go index 20b2b9443db..57d2804ffb5 100644 --- a/pkg/registry/core/pod/storage/storage_test.go +++ b/pkg/registry/core/pod/storage/storage_test.go @@ -183,7 +183,7 @@ func TestIgnoreDeleteNotFound(t *testing.T) { defer registry.Store.DestroyFunc() // should fail if pod A is not created yet. - _, _, err := registry.Delete(testContext, pod.Name, nil) + _, _, err := registry.Delete(testContext, pod.Name, rest.ValidateAllObjectFunc, nil) if !errors.IsNotFound(err) { t.Errorf("Unexpected error: %v", err) } @@ -198,7 +198,7 @@ func TestIgnoreDeleteNotFound(t *testing.T) { // registry shouldn't get any error since we ignore the NotFound error. zero := int64(0) opt := &metav1.DeleteOptions{GracePeriodSeconds: &zero} - obj, _, err := registry.Delete(testContext, pod.Name, opt) + obj, _, err := registry.Delete(testContext, pod.Name, rest.ValidateAllObjectFunc, opt) if err != nil { t.Fatalf("Unexpected error: %v", err) } diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go index 4ed99d17917..b8d4471a0da 100644 --- a/pkg/registry/core/service/storage/rest.go +++ b/pkg/registry/core/service/storage/rest.go @@ -220,9 +220,9 @@ func (rs *REST) Create(ctx context.Context, obj runtime.Object, createValidation return out, err } -func (rs *REST) Delete(ctx context.Context, id string, options *metav1.DeleteOptions) (runtime.Object, bool, error) { +func (rs *REST) Delete(ctx context.Context, id string, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions) (runtime.Object, bool, error) { // TODO: handle graceful - obj, _, err := rs.services.Delete(ctx, id, options) + obj, _, err := rs.services.Delete(ctx, id, deleteValidation, options) if err != nil { return nil, false, err } @@ -233,7 +233,7 @@ func (rs *REST) Delete(ctx context.Context, id string, options *metav1.DeleteOpt if !dryrun.IsDryRun(options.DryRun) { // TODO: can leave dangling endpoints, and potentially return incorrect // endpoints if a new service is created with the same name - _, _, err = rs.endpoints.Delete(ctx, id, &metav1.DeleteOptions{}) + _, _, err = rs.endpoints.Delete(ctx, id, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{}) if err != nil && !errors.IsNotFound(err) { return nil, false, err } diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go index b226a87eac3..db7f90e4aab 100644 --- a/pkg/registry/core/service/storage/rest_test.go +++ b/pkg/registry/core/service/storage/rest_test.go @@ -136,14 +136,14 @@ func (s *serviceStorage) Update(ctx context.Context, name string, objInfo rest.U return obj, s.Created, s.Err } -func (s *serviceStorage) Delete(ctx context.Context, name string, options *metav1.DeleteOptions) (runtime.Object, bool, error) { +func (s *serviceStorage) Delete(ctx context.Context, name string, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions) (runtime.Object, bool, error) { if !dryrun.IsDryRun(options.DryRun) { s.DeletedID = name } return s.Service, s.DeletedImmediately, s.Err } -func (s *serviceStorage) DeleteCollection(ctx context.Context, options *metav1.DeleteOptions, listOptions *metainternalversion.ListOptions) (runtime.Object, error) { +func (s *serviceStorage) DeleteCollection(ctx context.Context, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions, listOptions *metainternalversion.ListOptions) (runtime.Object, error) { panic("not implemented") } @@ -949,7 +949,7 @@ func TestServiceRegistryDelete(t *testing.T) { }, } registry.Create(ctx, svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{}) - storage.Delete(ctx, svc.Name, &metav1.DeleteOptions{}) + storage.Delete(ctx, svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{}) if e, a := "foo", registry.DeletedID; e != a { t.Errorf("Expected %v, but got %v", e, a) } @@ -1038,7 +1038,7 @@ func TestServiceRegistryDeleteExternal(t *testing.T) { }, } registry.Create(ctx, svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{}) - storage.Delete(ctx, svc.Name, &metav1.DeleteOptions{}) + storage.Delete(ctx, svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{}) if e, a := "foo", registry.DeletedID; e != a { t.Errorf("Expected %v, but got %v", e, a) } @@ -1440,7 +1440,7 @@ func TestServiceRegistryIPReallocation(t *testing.T) { t.Errorf("Unexpected ClusterIP: %s", created_service_1.Spec.ClusterIP) } - _, _, err := storage.Delete(ctx, created_service_1.Name, &metav1.DeleteOptions{}) + _, _, err := storage.Delete(ctx, created_service_1.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{}) if err != nil { t.Errorf("Unexpected error deleting service: %v", err) } diff --git a/pkg/registry/registrytest/endpoint.go b/pkg/registry/registrytest/endpoint.go index c36a494282b..2dc31543bed 100644 --- a/pkg/registry/registrytest/endpoint.go +++ b/pkg/registry/registrytest/endpoint.go @@ -111,7 +111,7 @@ func (e *EndpointRegistry) Update(ctx context.Context, name string, objInfo rest return endpoints, false, nil } -func (e *EndpointRegistry) Delete(ctx context.Context, name string, options *metav1.DeleteOptions) (runtime.Object, bool, error) { +func (e *EndpointRegistry) Delete(ctx context.Context, name string, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions) (runtime.Object, bool, error) { // TODO: support namespaces in this mock e.lock.Lock() defer e.lock.Unlock() @@ -130,6 +130,6 @@ func (e *EndpointRegistry) Delete(ctx context.Context, name string, options *met return nil, true, nil } -func (e *EndpointRegistry) DeleteCollection(ctx context.Context, _ *metav1.DeleteOptions, _ *metainternalversion.ListOptions) (runtime.Object, error) { +func (e *EndpointRegistry) DeleteCollection(ctx context.Context, _ rest.ValidateObjectFunc, _ *metav1.DeleteOptions, _ *metainternalversion.ListOptions) (runtime.Object, error) { return nil, fmt.Errorf("unimplemented!") } diff --git a/pkg/registry/scheduling/priorityclass/storage/BUILD b/pkg/registry/scheduling/priorityclass/storage/BUILD index 002dbc8441a..1898120bee1 100644 --- a/pkg/registry/scheduling/priorityclass/storage/BUILD +++ b/pkg/registry/scheduling/priorityclass/storage/BUILD @@ -20,6 +20,7 @@ go_test( "//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library", "//staging/src/k8s.io/apiserver/pkg/registry/generic:go_default_library", "//staging/src/k8s.io/apiserver/pkg/registry/generic/testing:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/registry/rest:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/etcd/testing:go_default_library", ], ) diff --git a/pkg/registry/scheduling/priorityclass/storage/storage.go b/pkg/registry/scheduling/priorityclass/storage/storage.go index dfdd67946bd..69e547116ce 100644 --- a/pkg/registry/scheduling/priorityclass/storage/storage.go +++ b/pkg/registry/scheduling/priorityclass/storage/storage.go @@ -68,12 +68,12 @@ func (r *REST) ShortNames() []string { } // Delete ensures that system priority classes are not deleted. -func (r *REST) Delete(ctx context.Context, name string, options *metav1.DeleteOptions) (runtime.Object, bool, error) { +func (r *REST) Delete(ctx context.Context, name string, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions) (runtime.Object, bool, error) { for _, spc := range scheduling.SystemPriorityClasses() { if name == spc.Name { return nil, false, apierrors.NewForbidden(scheduling.Resource("priorityclasses"), spc.Name, errors.New("this is a system priority class and cannot be deleted")) } } - return r.Store.Delete(ctx, name, options) + return r.Store.Delete(ctx, name, deleteValidation, options) } diff --git a/pkg/registry/scheduling/priorityclass/storage/storage_test.go b/pkg/registry/scheduling/priorityclass/storage/storage_test.go index a0963545864..d05e6da3f93 100644 --- a/pkg/registry/scheduling/priorityclass/storage/storage_test.go +++ b/pkg/registry/scheduling/priorityclass/storage/storage_test.go @@ -26,6 +26,7 @@ import ( genericapirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/generic" genericregistrytest "k8s.io/apiserver/pkg/registry/generic/testing" + "k8s.io/apiserver/pkg/registry/rest" etcdtesting "k8s.io/apiserver/pkg/storage/etcd/testing" "k8s.io/kubernetes/pkg/apis/scheduling" "k8s.io/kubernetes/pkg/registry/registrytest" @@ -117,7 +118,7 @@ func TestDeleteSystemPriorityClass(t *testing.T) { if err := storage.Store.Storage.Create(ctx, key, pc, nil, 0, false); err != nil { t.Fatalf("unexpected error: %v", err) } - if _, _, err := storage.Delete(ctx, pc.Name, nil); err == nil { + if _, _, err := storage.Delete(ctx, pc.Name, rest.ValidateAllObjectFunc, nil); err == nil { t.Error("expected to receive an error") } } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/finalizer/crd_finalizer.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/finalizer/crd_finalizer.go index 04caac82905..2730f006c59 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/finalizer/crd_finalizer.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/finalizer/crd_finalizer.go @@ -204,7 +204,7 @@ func (c *CRDFinalizer) deleteInstances(crd *apiextensions.CustomResourceDefiniti // don't retry deleting the same namespace deletedNamespaces.Insert(metadata.GetNamespace()) nsCtx := genericapirequest.WithNamespace(ctx, metadata.GetNamespace()) - if _, err := crClient.DeleteCollection(nsCtx, nil, nil); err != nil { + if _, err := crClient.DeleteCollection(nsCtx, rest.ValidateAllObjectFunc, nil, nil); err != nil { deleteErrors = append(deleteErrors, err) continue } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/etcd.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/etcd.go index 0bc9f64ddc8..9d020cab437 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/etcd.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/etcd.go @@ -67,7 +67,7 @@ func (r *REST) ShortNames() []string { } // Delete adds the CRD finalizer to the list -func (r *REST) Delete(ctx context.Context, name string, options *metav1.DeleteOptions) (runtime.Object, bool, error) { +func (r *REST) Delete(ctx context.Context, name string, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions) (runtime.Object, bool, error) { obj, err := r.Get(ctx, name, &metav1.GetOptions{}) if err != nil { return nil, false, err @@ -103,6 +103,10 @@ func (r *REST) Delete(ctx context.Context, name string, options *metav1.DeleteOp // upon first request to delete, add our finalizer and then delegate if crd.DeletionTimestamp.IsZero() { + if err := deleteValidation(obj); err != nil { + return nil, false, err + } + key, err := r.Store.KeyFunc(ctx, name) if err != nil { return nil, false, err @@ -153,7 +157,7 @@ func (r *REST) Delete(ctx context.Context, name string, options *metav1.DeleteOp return out, false, nil } - return r.Store.Delete(ctx, name, options) + return r.Store.Delete(ctx, name, deleteValidation, options) } // NewStatusREST makes a RESTStorage for status that has more limited options. diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/apiserver_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/apiserver_test.go index a40c41cc6e5..a0cd62fb435 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/apiserver_test.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/apiserver_test.go @@ -440,13 +440,16 @@ func (storage *SimpleRESTStorage) checkContext(ctx context.Context) { storage.actualNamespace, storage.namespacePresent = request.NamespaceFrom(ctx) } -func (storage *SimpleRESTStorage) Delete(ctx context.Context, id string, options *metav1.DeleteOptions) (runtime.Object, bool, error) { +func (storage *SimpleRESTStorage) Delete(ctx context.Context, id string, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions) (runtime.Object, bool, error) { storage.checkContext(ctx) storage.deleted = id storage.deleteOptions = options if err := storage.errors["delete"]; err != nil { return nil, false, err } + if err := deleteValidation(nil); err != nil { + return nil, false, err + } var obj runtime.Object = &metav1.Status{Status: metav1.StatusSuccess} var err error if storage.injectedFunction != nil { @@ -569,6 +572,18 @@ func (s *ConnecterRESTStorage) NewConnectOptions() (runtime.Object, bool, string return s.emptyConnectOptions, false, "" } +<<<<<<< HEAD +======= +type LegacyRESTStorage struct { + *SimpleRESTStorage +} + +func (storage LegacyRESTStorage) Delete(ctx context.Context, id string) (runtime.Object, error) { + obj, _, err := storage.SimpleRESTStorage.Delete(ctx, id, nil, nil) + return obj, err +} + +>>>>>>> ab07482ecb... validate deletion admission object type MetadataRESTStorage struct { *SimpleRESTStorage types []string @@ -4192,7 +4207,7 @@ type SimpleRESTStorageWithDeleteCollection struct { } // Delete collection doesn't do much, but let us test this path. -func (storage *SimpleRESTStorageWithDeleteCollection) DeleteCollection(ctx context.Context, options *metav1.DeleteOptions, listOptions *metainternalversion.ListOptions) (runtime.Object, error) { +func (storage *SimpleRESTStorageWithDeleteCollection) DeleteCollection(ctx context.Context, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions, listOptions *metainternalversion.ListOptions) (runtime.Object, error) { storage.checkContext(ctx) return nil, nil } @@ -4263,7 +4278,7 @@ func (storage *SimpleXGSubresourceRESTStorage) Get(ctx context.Context, id strin return storage.item.DeepCopyObject(), nil } -func (storage *SimpleXGSubresourceRESTStorage) Delete(ctx context.Context, name string, options *metav1.DeleteOptions) (runtime.Object, bool, error) { +func (storage *SimpleXGSubresourceRESTStorage) Delete(ctx context.Context, name string, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions) (runtime.Object, bool, error) { return nil, true, nil } diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/delete.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/delete.go index 7375c575eba..eef4b1f62b1 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/delete.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/delete.go @@ -115,28 +115,12 @@ func DeleteResource(r rest.GracefulDeleter, allowsOptions bool, scope *RequestSc } options.TypeMeta.SetGroupVersionKind(metav1.SchemeGroupVersion.WithKind("DeleteOptions")) - trace.Step("About to check admission control") - if admit != nil && admit.Handles(admission.Delete) { - userInfo, _ := request.UserFrom(ctx) - attrs := admission.NewAttributesRecord(nil, nil, scope.Kind, namespace, name, scope.Resource, scope.Subresource, admission.Delete, options, dryrun.IsDryRun(options.DryRun), userInfo) - if mutatingAdmission, ok := admit.(admission.MutationInterface); ok { - if err := mutatingAdmission.Admit(attrs, scope); err != nil { - scope.err(err, w, req) - return - } - } - if validatingAdmission, ok := admit.(admission.ValidationInterface); ok { - if err := validatingAdmission.Validate(attrs, scope); err != nil { - scope.err(err, w, req) - return - } - } - } - trace.Step("About to delete object from database") wasDeleted := true + userInfo, _ := request.UserFrom(ctx) + staticAdmissionAttrs := admission.NewAttributesRecord(nil, nil, scope.Kind, namespace, name, scope.Resource, scope.Subresource, admission.Delete, options, dryrun.IsDryRun(options.DryRun), userInfo) result, err := finishRequest(timeout, func() (runtime.Object, error) { - obj, deleted, err := r.Delete(ctx, name, options) + obj, deleted, err := r.Delete(ctx, name, rest.AdmissionToValidateObjectDeleteFunc(admit, staticAdmissionAttrs, scope), options) wasDeleted = deleted return obj, err }) @@ -196,6 +180,7 @@ func DeleteCollection(r rest.CollectionDeleter, checkBody bool, scope *RequestSc ctx := req.Context() ctx = request.WithNamespace(ctx, namespace) ae := request.AuditEventFrom(ctx) + admit = admission.WithAudit(admit, ae) outputMediaType, _, err := negotiation.NegotiateOutputMediaType(req, scope.Serializer, scope) if err != nil { @@ -267,29 +252,10 @@ func DeleteCollection(r rest.CollectionDeleter, checkBody bool, scope *RequestSc } options.TypeMeta.SetGroupVersionKind(metav1.SchemeGroupVersion.WithKind("DeleteOptions")) - admit = admission.WithAudit(admit, ae) - if admit != nil && admit.Handles(admission.Delete) { - userInfo, _ := request.UserFrom(ctx) - attrs := admission.NewAttributesRecord(nil, nil, scope.Kind, namespace, "", scope.Resource, scope.Subresource, admission.Delete, options, dryrun.IsDryRun(options.DryRun), userInfo) - if mutatingAdmission, ok := admit.(admission.MutationInterface); ok { - err = mutatingAdmission.Admit(attrs, scope) - if err != nil { - scope.err(err, w, req) - return - } - } - - if validatingAdmission, ok := admit.(admission.ValidationInterface); ok { - err = validatingAdmission.Validate(attrs, scope) - if err != nil { - scope.err(err, w, req) - return - } - } - } - + userInfo, _ := request.UserFrom(ctx) + staticAdmissionAttrs := admission.NewAttributesRecord(nil, nil, scope.Kind, namespace, "", scope.Resource, scope.Subresource, admission.Delete, options, dryrun.IsDryRun(options.DryRun), userInfo) result, err := finishRequest(timeout, func() (runtime.Object, error) { - return r.DeleteCollection(ctx, options, &listOptions) + return r.DeleteCollection(ctx, rest.AdmissionToValidateObjectDeleteFunc(admit, staticAdmissionAttrs, scope), options, &listOptions) }) if err != nil { scope.err(err, w, req) diff --git a/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go b/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go index 7b8aad1449d..c8bbfe6316d 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go +++ b/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go @@ -24,6 +24,7 @@ import ( "sync" "time" + "k8s.io/apimachinery/pkg/api/errors" kubeerr "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/validation/path" @@ -881,16 +882,28 @@ func (e *Store) updateForGracefulDeletionAndFinalizers(ctx context.Context, name } // Delete removes the item from storage. -func (e *Store) Delete(ctx context.Context, name string, options *metav1.DeleteOptions) (runtime.Object, bool, error) { +func (e *Store) Delete(ctx context.Context, name string, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions) (runtime.Object, bool, error) { key, err := e.KeyFunc(ctx, name) if err != nil { return nil, false, err } obj := e.NewFunc() qualifiedResource := e.qualifiedResourceFromContext(ctx) - if err := e.Storage.Get(ctx, key, "", obj, false); err != nil { - return nil, false, storeerr.InterpretDeleteError(err, qualifiedResource, name) + err = e.Storage.Get(ctx, key, "", obj, false) + if err != nil { + // Note that we continue the admission check on "Not Found" error is for the compatibility + // with the NodeRestriction admission controller. + if interpretedErr := storeerr.InterpretDeleteError(err, qualifiedResource, name); errors.IsNotFound(interpretedErr) { + if err := deleteValidation(obj); err != nil { + return nil, false, err + } + return nil, false, interpretedErr + } } + if err := deleteValidation(obj.DeepCopyObject()); err != nil { + return nil, false, err + } + // support older consumers of delete by treating "nil" as delete immediately if options == nil { options = metav1.NewDeleteOptions(0) @@ -972,13 +985,19 @@ func (e *Store) Delete(ctx context.Context, name string, options *metav1.DeleteO // are removing all objects of a given type) with the current API (it's technically // possibly with storage API, but watch is not delivered correctly then). // It will be possible to fix it with v3 etcd API. -func (e *Store) DeleteCollection(ctx context.Context, options *metav1.DeleteOptions, listOptions *metainternalversion.ListOptions) (runtime.Object, error) { +func (e *Store) DeleteCollection(ctx context.Context, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions, listOptions *metainternalversion.ListOptions) (runtime.Object, error) { if listOptions == nil { listOptions = &metainternalversion.ListOptions{} } else { listOptions = listOptions.DeepCopy() } + if deleteValidation != nil { + if err := deleteValidation(nil); err != nil { + return nil, err + } + } + listObj, err := e.List(ctx, listOptions) if err != nil { return nil, err @@ -1025,7 +1044,7 @@ func (e *Store) DeleteCollection(ctx context.Context, options *metav1.DeleteOpti errs <- err return } - if _, _, err := e.Delete(ctx, accessor.GetName(), options); err != nil && !kubeerr.IsNotFound(err) { + if _, _, err := e.Delete(ctx, accessor.GetName(), rest.ValidateAllObjectFunc, options); err != nil && !kubeerr.IsNotFound(err) { klog.V(4).Infof("Delete %s in DeleteCollection failed: %v", accessor.GetName(), err) errs <- err return diff --git a/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store_test.go b/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store_test.go index 7b16f614e5b..a919f193763 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store_test.go +++ b/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store_test.go @@ -357,7 +357,7 @@ func TestStoreCreate(t *testing.T) { // now delete pod with graceful period set delOpts := &metav1.DeleteOptions{GracePeriodSeconds: &gracefulPeriod} - _, _, err = registry.Delete(testContext, podA.Name, delOpts) + _, _, err = registry.Delete(testContext, podA.Name, rest.ValidateAllObjectFunc, delOpts) if err != nil { t.Fatalf("Failed to delete pod gracefully. Unexpected error: %v", err) } @@ -660,7 +660,7 @@ func TestStoreDelete(t *testing.T) { defer destroyFunc() // test failure condition - _, _, err := registry.Delete(testContext, podA.Name, nil) + _, _, err := registry.Delete(testContext, podA.Name, rest.ValidateAllObjectFunc, nil) if !errors.IsNotFound(err) { t.Errorf("Unexpected error: %v", err) } @@ -672,7 +672,7 @@ func TestStoreDelete(t *testing.T) { } // delete object - _, wasDeleted, err := registry.Delete(testContext, podA.Name, nil) + _, wasDeleted, err := registry.Delete(testContext, podA.Name, rest.ValidateAllObjectFunc, nil) if err != nil { t.Errorf("Unexpected error: %v", err) } @@ -749,7 +749,7 @@ func TestGracefulStoreHandleFinalizers(t *testing.T) { } // delete the pod with grace period=0, the pod should still exist because it has a finalizer - _, wasDeleted, err := registry.Delete(testContext, podWithFinalizer.Name, metav1.NewDeleteOptions(0)) + _, wasDeleted, err := registry.Delete(testContext, podWithFinalizer.Name, rest.ValidateAllObjectFunc, metav1.NewDeleteOptions(0)) if err != nil { t.Fatalf("Unexpected error: %v", err) } @@ -815,7 +815,7 @@ func TestNonGracefulStoreHandleFinalizers(t *testing.T) { } // delete object with nil delete options doesn't delete the object - _, wasDeleted, err := registry.Delete(testContext, podWithFinalizer.Name, nil) + _, wasDeleted, err := registry.Delete(testContext, podWithFinalizer.Name, rest.ValidateAllObjectFunc, nil) if err != nil { t.Errorf("Unexpected error: %v", err) } @@ -1115,7 +1115,7 @@ func TestStoreDeleteWithOrphanDependents(t *testing.T) { if err != nil { t.Fatalf("Unexpected error: %v", err) } - _, _, err = registry.Delete(testContext, tc.pod.Name, tc.options) + _, _, err = registry.Delete(testContext, tc.pod.Name, rest.ValidateAllObjectFunc, tc.options) if err != nil { t.Fatalf("Unexpected error: %v", err) } @@ -1334,7 +1334,7 @@ func TestStoreDeletionPropagation(t *testing.T) { if err != nil { t.Fatalf("Unexpected error: %v", err) } - _, _, err = registry.Delete(testContext, pod.Name, tc.options) + _, _, err = registry.Delete(testContext, pod.Name, rest.ValidateAllObjectFunc, tc.options) obj, err := registry.Get(testContext, pod.Name, &metav1.GetOptions{}) if tc.expectedNotFound { if err == nil || !errors.IsNotFound(err) { @@ -1382,7 +1382,7 @@ func TestStoreDeleteCollection(t *testing.T) { } // Delete all pods. - deleted, err := registry.DeleteCollection(testContext, nil, &metainternalversion.ListOptions{}) + deleted, err := registry.DeleteCollection(testContext, rest.ValidateAllObjectFunc, nil, &metainternalversion.ListOptions{}) if err != nil { t.Fatalf("Unexpected error: %v", err) } @@ -1423,7 +1423,7 @@ func TestStoreDeleteCollectionNotFound(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - _, err := registry.DeleteCollection(testContext, nil, &metainternalversion.ListOptions{}) + _, err := registry.DeleteCollection(testContext, rest.ValidateAllObjectFunc, nil, &metainternalversion.ListOptions{}) if err != nil { t.Fatalf("Unexpected error: %v", err) } @@ -1461,7 +1461,7 @@ func TestStoreDeleteCollectionWithWatch(t *testing.T) { } defer watcher.Stop() - if _, err := registry.DeleteCollection(testContext, nil, &metainternalversion.ListOptions{}); err != nil { + if _, err := registry.DeleteCollection(testContext, rest.ValidateAllObjectFunc, nil, &metainternalversion.ListOptions{}); err != nil { t.Fatalf("Unexpected error: %v", err) } @@ -1683,7 +1683,7 @@ func TestQualifiedResource(t *testing.T) { } // delete a non-exist object - _, _, err = registry.Delete(testContext, podA.Name, nil) + _, _, err = registry.Delete(testContext, podA.Name, rest.ValidateAllObjectFunc, nil) if !errors.IsNotFound(err) { t.Fatalf("Unexpected error: %v", err) diff --git a/staging/src/k8s.io/apiserver/pkg/registry/generic/testing/BUILD b/staging/src/k8s.io/apiserver/pkg/registry/generic/testing/BUILD index bc4ad5fdf03..f9494438db0 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/generic/testing/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/registry/generic/testing/BUILD @@ -14,6 +14,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apiserver/pkg/registry/generic/registry:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/registry/rest:go_default_library", "//staging/src/k8s.io/apiserver/pkg/registry/rest/resttest:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/testing:go_default_library", ], diff --git a/staging/src/k8s.io/apiserver/pkg/registry/generic/testing/tester.go b/staging/src/k8s.io/apiserver/pkg/registry/generic/testing/tester.go index 5db5c49d3f0..78cc15bf151 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/generic/testing/tester.go +++ b/staging/src/k8s.io/apiserver/pkg/registry/generic/testing/tester.go @@ -28,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" + "k8s.io/apiserver/pkg/registry/rest" "k8s.io/apiserver/pkg/registry/rest/resttest" storagetesting "k8s.io/apiserver/pkg/storage/testing" ) @@ -168,7 +169,7 @@ func (t *Tester) createObject(ctx context.Context, obj runtime.Object) error { func (t *Tester) setObjectsForList(objects []runtime.Object) []runtime.Object { key := t.storage.KeyRootFunc(t.tester.TestContext()) - if _, err := t.storage.DeleteCollection(t.tester.TestContext(), nil, nil); err != nil { + if _, err := t.storage.DeleteCollection(t.tester.TestContext(), rest.ValidateAllObjectFunc, nil, nil); err != nil { t.tester.Errorf("unable to clear collection: %v", err) return nil } @@ -192,7 +193,7 @@ func (t *Tester) emitObject(obj runtime.Object, action string) error { if err != nil { return err } - _, _, err = t.storage.Delete(ctx, accessor.GetName(), nil) + _, _, err = t.storage.Delete(ctx, accessor.GetName(), rest.ValidateAllObjectFunc, nil) default: err = fmt.Errorf("unexpected action: %v", action) } diff --git a/staging/src/k8s.io/apiserver/pkg/registry/rest/delete.go b/staging/src/k8s.io/apiserver/pkg/registry/rest/delete.go index 16c9e1b40db..661db668bf8 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/rest/delete.go +++ b/staging/src/k8s.io/apiserver/pkg/registry/rest/delete.go @@ -26,6 +26,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/validation" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apiserver/pkg/admission" ) // RESTDeleteStrategy defines deletion behavior on an object that follows Kubernetes @@ -140,3 +141,37 @@ func BeforeDelete(strategy RESTDeleteStrategy, ctx context.Context, obj runtime. } return true, false, nil } + +// AdmissionToValidateObjectDeleteFunc returns a admission validate func for object deletion +func AdmissionToValidateObjectDeleteFunc(admit admission.Interface, staticAttributes admission.Attributes, objInterfaces admission.ObjectInterfaces) ValidateObjectFunc { + mutatingAdmission, isMutatingAdmission := admit.(admission.MutationInterface) + validatingAdmission, isValidatingAdmission := admit.(admission.ValidationInterface) + + return func(old runtime.Object) error { + if !isMutatingAdmission && !isValidatingAdmission { + return nil + } + finalAttributes := admission.NewAttributesRecord( + nil, + old, + staticAttributes.GetKind(), + staticAttributes.GetNamespace(), + staticAttributes.GetName(), + staticAttributes.GetResource(), + staticAttributes.GetSubresource(), + staticAttributes.GetOperation(), + staticAttributes.GetUserInfo(), + ) + if isMutatingAdmission && mutatingAdmission.Handles(finalAttributes.GetOperation()) { + if err := mutatingAdmission.Admit(finalAttributes, objInterfaces); err != nil { + return err + } + } + if isValidatingAdmission && validatingAdmission.Handles(finalAttributes.GetOperation()) { + if err := validatingAdmission.Validate(finalAttributes, objInterfaces); err != nil { + return err + } + } + return nil + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/registry/rest/rest.go b/staging/src/k8s.io/apiserver/pkg/registry/rest/rest.go index b16f7f677bc..e4336fb52d6 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/rest/rest.go +++ b/staging/src/k8s.io/apiserver/pkg/registry/rest/rest.go @@ -156,7 +156,7 @@ type GracefulDeleter interface { // information about deletion. // It also returns a boolean which is set to true if the resource was instantly // deleted or false if it will be deleted asynchronously. - Delete(ctx context.Context, name string, options *metav1.DeleteOptions) (runtime.Object, bool, error) + Delete(ctx context.Context, name string, deleteValidation ValidateObjectFunc, options *metav1.DeleteOptions) (runtime.Object, bool, error) } // CollectionDeleter is an object that can delete a collection @@ -167,7 +167,7 @@ type CollectionDeleter interface { // them or return an invalid request error. // DeleteCollection may not be atomic - i.e. it may delete some objects and still // return an error after it. On success, returns a list of deleted objects. - DeleteCollection(ctx context.Context, options *metav1.DeleteOptions, listOptions *metainternalversion.ListOptions) (runtime.Object, error) + DeleteCollection(ctx context.Context, deleteValidation ValidateObjectFunc, options *metav1.DeleteOptions, listOptions *metainternalversion.ListOptions) (runtime.Object, error) } // Creater is an object that can create an instance of a RESTful object. diff --git a/staging/src/k8s.io/apiserver/pkg/registry/rest/resttest/resttest.go b/staging/src/k8s.io/apiserver/pkg/registry/rest/resttest/resttest.go index 6518968c418..7e8f9f2a20b 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/rest/resttest/resttest.go +++ b/staging/src/k8s.io/apiserver/pkg/registry/rest/resttest/resttest.go @@ -257,7 +257,7 @@ func (t *Tester) delete(ctx context.Context, obj runtime.Object) error { if !ok { return fmt.Errorf("Expected deleting storage, got %v", t.storage) } - _, _, err = deleter.Delete(ctx, objectMeta.GetName(), nil) + _, _, err = deleter.Delete(ctx, objectMeta.GetName(), rest.ValidateAllObjectFunc, nil) return err } @@ -840,7 +840,7 @@ func (t *Tester) testDeleteNoGraceful(obj runtime.Object, createFn CreateFunc, g if dryRun { opts.DryRun = []string{metav1.DryRunAll} } - obj, wasDeleted, err := t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.GetName(), opts) + obj, wasDeleted, err := t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.GetName(), rest.ValidateAllObjectFunc, opts) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -866,7 +866,7 @@ func (t *Tester) testDeleteNoGraceful(obj runtime.Object, createFn CreateFunc, g func (t *Tester) testDeleteNonExist(obj runtime.Object, opts metav1.DeleteOptions) { objectMeta := t.getObjectMetaOrFail(obj) - _, _, err := t.storage.(rest.GracefulDeleter).Delete(t.TestContext(), objectMeta.GetName(), &opts) + _, _, err := t.storage.(rest.GracefulDeleter).Delete(t.TestContext(), objectMeta.GetName(), rest.ValidateAllObjectFunc, &opts) if err == nil || !errors.IsNotFound(err) { t.Errorf("unexpected error: %v", err) } @@ -886,12 +886,12 @@ func (t *Tester) testDeleteWithUID(obj runtime.Object, createFn CreateFunc, getF t.Errorf("unexpected error: %v", err) } opts.Preconditions = metav1.NewPreconditionDeleteOptions("UID1111").Preconditions - obj, _, err := t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.GetName(), &opts) + obj, _, err := t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.GetName(), rest.ValidateAllObjectFunc, &opts) if err == nil || !errors.IsConflict(err) { t.Errorf("unexpected error: %v", err) } - obj, _, err = t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.GetName(), metav1.NewPreconditionDeleteOptions("UID0000")) + obj, _, err = t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.GetName(), rest.ValidateAllObjectFunc, metav1.NewPreconditionDeleteOptions("UID0000")) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -989,7 +989,7 @@ func (t *Tester) testDeleteGracefulHasDefault(obj runtime.Object, createFn Creat } objectMeta := t.getObjectMetaOrFail(foo) generation := objectMeta.GetGeneration() - _, wasDeleted, err := t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.GetName(), &metav1.DeleteOptions{}) + _, wasDeleted, err := t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.GetName(), rest.ValidateAllObjectFunc, &metav1.DeleteOptions{}) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -1023,7 +1023,7 @@ func (t *Tester) testDeleteGracefulWithValue(obj runtime.Object, createFn Create } objectMeta := t.getObjectMetaOrFail(foo) generation := objectMeta.GetGeneration() - _, wasDeleted, err := t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.GetName(), metav1.NewDeleteOptions(expectedGrace+2)) + _, wasDeleted, err := t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.GetName(), rest.ValidateAllObjectFunc, metav1.NewDeleteOptions(expectedGrace+2)) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -1057,7 +1057,7 @@ func (t *Tester) testDeleteGracefulExtend(obj runtime.Object, createFn CreateFun } objectMeta := t.getObjectMetaOrFail(foo) generation := objectMeta.GetGeneration() - _, wasDeleted, err := t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.GetName(), metav1.NewDeleteOptions(expectedGrace)) + _, wasDeleted, err := t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.GetName(), rest.ValidateAllObjectFunc, metav1.NewDeleteOptions(expectedGrace)) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -1069,7 +1069,7 @@ func (t *Tester) testDeleteGracefulExtend(obj runtime.Object, createFn CreateFun } // second delete duration is ignored - _, wasDeleted, err = t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.GetName(), metav1.NewDeleteOptions(expectedGrace+2)) + _, wasDeleted, err = t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.GetName(), rest.ValidateAllObjectFunc, metav1.NewDeleteOptions(expectedGrace+2)) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -1099,7 +1099,7 @@ func (t *Tester) testDeleteGracefulImmediate(obj runtime.Object, createFn Create } objectMeta := t.getObjectMetaOrFail(foo) generation := objectMeta.GetGeneration() - _, wasDeleted, err := t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.GetName(), metav1.NewDeleteOptions(expectedGrace)) + _, wasDeleted, err := t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.GetName(), rest.ValidateAllObjectFunc, metav1.NewDeleteOptions(expectedGrace)) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -1111,7 +1111,7 @@ func (t *Tester) testDeleteGracefulImmediate(obj runtime.Object, createFn Create } // second delete is immediate, resource is deleted - out, wasDeleted, err := t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.GetName(), metav1.NewDeleteOptions(0)) + out, wasDeleted, err := t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.GetName(), rest.ValidateAllObjectFunc, metav1.NewDeleteOptions(0)) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -1141,7 +1141,7 @@ func (t *Tester) testDeleteGracefulUsesZeroOnNil(obj runtime.Object, createFn Cr t.Errorf("unexpected error: %v", err) } objectMeta := t.getObjectMetaOrFail(foo) - _, wasDeleted, err := t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.GetName(), nil) + _, wasDeleted, err := t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.GetName(), rest.ValidateAllObjectFunc, nil) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -1167,7 +1167,7 @@ func (t *Tester) testDeleteGracefulShorten(obj runtime.Object, createFn CreateFu bigGrace = 2 * expectedGrace } objectMeta := t.getObjectMetaOrFail(foo) - _, wasDeleted, err := t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.GetName(), metav1.NewDeleteOptions(bigGrace)) + _, wasDeleted, err := t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.GetName(), rest.ValidateAllObjectFunc, metav1.NewDeleteOptions(bigGrace)) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -1182,7 +1182,7 @@ func (t *Tester) testDeleteGracefulShorten(obj runtime.Object, createFn CreateFu deletionTimestamp := *objectMeta.GetDeletionTimestamp() // second delete duration is ignored - _, wasDeleted, err = t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.GetName(), metav1.NewDeleteOptions(expectedGrace)) + _, wasDeleted, err = t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.GetName(), rest.ValidateAllObjectFunc, metav1.NewDeleteOptions(expectedGrace)) if err != nil { t.Errorf("unexpected error: %v", err) }