From f2abdcf43f5e0435824104fe6f1af9fb3871d455 Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Mon, 20 May 2019 14:36:19 -0400 Subject: [PATCH] Consider equivalent resources when calling webhook --- .../admission/plugin/webhook/generic/BUILD | 7 +- .../plugin/webhook/generic/webhook.go | 42 ++- .../plugin/webhook/generic/webhook_test.go | 316 ++++++++++++++++++ 3 files changed, 362 insertions(+), 3 deletions(-) create mode 100644 staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/generic/webhook_test.go diff --git a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/generic/BUILD b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/generic/BUILD index 390ea55183b..7cdf2e46b48 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/generic/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/generic/BUILD @@ -43,15 +43,20 @@ filegroup( go_test( name = "go_default_test", - srcs = ["conversion_test.go"], + srcs = [ + "conversion_test.go", + "webhook_test.go", + ], embed = [":go_default_library"], deps = [ + "//staging/src/k8s.io/api/admissionregistration/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/namespace:go_default_library", "//staging/src/k8s.io/apiserver/pkg/apis/example:go_default_library", "//staging/src/k8s.io/apiserver/pkg/apis/example/v1:go_default_library", "//staging/src/k8s.io/apiserver/pkg/apis/example2/v1:go_default_library", diff --git a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/generic/webhook.go b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/generic/webhook.go index 92c3de2caa1..30a9b5e7b5e 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/generic/webhook.go +++ b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/generic/webhook.go @@ -24,6 +24,7 @@ import ( admissionv1beta1 "k8s.io/api/admission/v1beta1" "k8s.io/api/admissionregistration/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/admission" genericadmissioninit "k8s.io/apiserver/pkg/admission/initializer" "k8s.io/apiserver/pkg/admission/plugin/webhook/config" @@ -127,7 +128,7 @@ func (a *Webhook) ValidateInitialization() error { // shouldCallHook returns invocation details if the webhook should be called, nil if the webhook should not be called, // or an error if an error was encountered during evaluation. -func (a *Webhook) shouldCallHook(h *v1beta1.Webhook, attr admission.Attributes) (*WebhookInvocation, *apierrors.StatusError) { +func (a *Webhook) shouldCallHook(h *v1beta1.Webhook, attr admission.Attributes, o admission.ObjectInterfaces) (*WebhookInvocation, *apierrors.StatusError) { var err *apierrors.StatusError var invocation *WebhookInvocation for _, r := range h.Rules { @@ -142,6 +143,36 @@ func (a *Webhook) shouldCallHook(h *v1beta1.Webhook, attr admission.Attributes) break } } + if invocation == nil && h.MatchPolicy != nil && *h.MatchPolicy == v1beta1.Equivalent { + attrWithOverride := &attrWithResourceOverride{Attributes: attr} + equivalents := o.GetEquivalentResourceMapper().EquivalentResourcesFor(attr.GetResource(), attr.GetSubresource()) + // honor earlier rules first + OuterLoop: + for _, r := range h.Rules { + // see if the rule matches any of the equivalent resources + for _, equivalent := range equivalents { + if equivalent == attr.GetResource() { + // exclude attr.GetResource(), which we already checked + continue + } + attrWithOverride.resource = equivalent + m := rules.Matcher{Rule: r, Attr: attrWithOverride} + if m.Matches() { + kind := o.GetEquivalentResourceMapper().KindFor(equivalent, attr.GetSubresource()) + if kind.Empty() { + return nil, apierrors.NewInternalError(fmt.Errorf("unable to convert to %v: unknown kind", equivalent)) + } + invocation = &WebhookInvocation{ + Webhook: h, + Resource: equivalent, + Subresource: attr.GetSubresource(), + Kind: kind, + } + break OuterLoop + } + } + } + } if invocation == nil { return nil, nil @@ -155,6 +186,13 @@ func (a *Webhook) shouldCallHook(h *v1beta1.Webhook, attr admission.Attributes) return invocation, nil } +type attrWithResourceOverride struct { + admission.Attributes + resource schema.GroupVersionResource +} + +func (a *attrWithResourceOverride) GetResource() schema.GroupVersionResource { return a.resource } + // Dispatch is called by the downstream Validate or Admit methods. func (a *Webhook) Dispatch(attr admission.Attributes, o admission.ObjectInterfaces) error { if rules.IsWebhookConfigurationResource(attr) { @@ -169,7 +207,7 @@ func (a *Webhook) Dispatch(attr admission.Attributes, o admission.ObjectInterfac var relevantHooks []*WebhookInvocation for i := range hooks { - invocation, err := a.shouldCallHook(&hooks[i], attr) + invocation, err := a.shouldCallHook(&hooks[i], attr, o) if err != nil { return err } diff --git a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/generic/webhook_test.go b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/generic/webhook_test.go new file mode 100644 index 00000000000..b214122ad38 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/generic/webhook_test.go @@ -0,0 +1,316 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package generic + +import ( + "strings" + "testing" + + "k8s.io/api/admissionregistration/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apiserver/pkg/admission" + "k8s.io/apiserver/pkg/admission/plugin/webhook/namespace" +) + +func TestShouldCallHook(t *testing.T) { + a := &Webhook{namespaceMatcher: &namespace.Matcher{}} + + allScopes := v1beta1.AllScopes + exactMatch := v1beta1.Exact + equivalentMatch := v1beta1.Equivalent + + mapper := runtime.NewEquivalentResourceRegistryWithIdentity(func(resource schema.GroupResource) string { + if resource.Resource == "deployments" { + // co-locate deployments in all API groups + return "/deployments" + } + return "" + }) + mapper.RegisterKindFor(schema.GroupVersionResource{"extensions", "v1beta1", "deployments"}, "", schema.GroupVersionKind{"extensions", "v1beta1", "Deployment"}) + mapper.RegisterKindFor(schema.GroupVersionResource{"apps", "v1", "deployments"}, "", schema.GroupVersionKind{"apps", "v1", "Deployment"}) + mapper.RegisterKindFor(schema.GroupVersionResource{"apps", "v1beta1", "deployments"}, "", schema.GroupVersionKind{"apps", "v1beta1", "Deployment"}) + mapper.RegisterKindFor(schema.GroupVersionResource{"apps", "v1alpha1", "deployments"}, "", schema.GroupVersionKind{"apps", "v1alpha1", "Deployment"}) + + mapper.RegisterKindFor(schema.GroupVersionResource{"extensions", "v1beta1", "deployments"}, "scale", schema.GroupVersionKind{"extensions", "v1beta1", "Scale"}) + mapper.RegisterKindFor(schema.GroupVersionResource{"apps", "v1", "deployments"}, "scale", schema.GroupVersionKind{"autoscaling", "v1", "Scale"}) + mapper.RegisterKindFor(schema.GroupVersionResource{"apps", "v1beta1", "deployments"}, "scale", schema.GroupVersionKind{"apps", "v1beta1", "Scale"}) + mapper.RegisterKindFor(schema.GroupVersionResource{"apps", "v1alpha1", "deployments"}, "scale", schema.GroupVersionKind{"apps", "v1alpha1", "Scale"}) + + // register invalid kinds to trigger an error + mapper.RegisterKindFor(schema.GroupVersionResource{"example.com", "v1", "widgets"}, "", schema.GroupVersionKind{"", "", ""}) + mapper.RegisterKindFor(schema.GroupVersionResource{"example.com", "v2", "widgets"}, "", schema.GroupVersionKind{"", "", ""}) + + interfaces := &admission.RuntimeObjectInterfaces{EquivalentResourceMapper: mapper} + + testcases := []struct { + name string + + webhook *v1beta1.Webhook + attrs admission.Attributes + + expectCall bool + expectErr string + expectCallResource schema.GroupVersionResource + expectCallSubresource string + expectCallKind schema.GroupVersionKind + }{ + { + name: "no rules (just write)", + webhook: &v1beta1.Webhook{Rules: []v1beta1.RuleWithOperations{}}, + attrs: admission.NewAttributesRecord(nil, nil, schema.GroupVersionKind{"apps", "v1", "Deployment"}, "ns", "name", schema.GroupVersionResource{"apps", "v1", "deployments"}, "", admission.Create, &metav1.CreateOptions{}, false, nil), + expectCall: false, + }, + { + name: "invalid kind lookup", + webhook: &v1beta1.Webhook{ + NamespaceSelector: &metav1.LabelSelector{}, + MatchPolicy: &equivalentMatch, + Rules: []v1beta1.RuleWithOperations{{ + Operations: []v1beta1.OperationType{"*"}, + Rule: v1beta1.Rule{APIGroups: []string{"example.com"}, APIVersions: []string{"v1"}, Resources: []string{"widgets"}, Scope: &allScopes}, + }}}, + attrs: admission.NewAttributesRecord(nil, nil, schema.GroupVersionKind{"example.com", "v2", "Widget"}, "ns", "name", schema.GroupVersionResource{"example.com", "v2", "widgets"}, "", admission.Create, &metav1.CreateOptions{}, false, nil), + expectCall: false, + expectErr: "unknown kind", + }, + { + name: "wildcard rule, match as requested", + webhook: &v1beta1.Webhook{ + NamespaceSelector: &metav1.LabelSelector{}, + Rules: []v1beta1.RuleWithOperations{{ + Operations: []v1beta1.OperationType{"*"}, + Rule: v1beta1.Rule{APIGroups: []string{"*"}, APIVersions: []string{"*"}, Resources: []string{"*"}, Scope: &allScopes}, + }}}, + attrs: admission.NewAttributesRecord(nil, nil, schema.GroupVersionKind{"apps", "v1", "Deployment"}, "ns", "name", schema.GroupVersionResource{"apps", "v1", "deployments"}, "", admission.Create, &metav1.CreateOptions{}, false, nil), + expectCall: true, + expectCallKind: schema.GroupVersionKind{"apps", "v1", "Deployment"}, + expectCallResource: schema.GroupVersionResource{"apps", "v1", "deployments"}, + expectCallSubresource: "", + }, + { + name: "specific rules, prefer exact match", + webhook: &v1beta1.Webhook{ + NamespaceSelector: &metav1.LabelSelector{}, + Rules: []v1beta1.RuleWithOperations{{ + Operations: []v1beta1.OperationType{"*"}, + Rule: v1beta1.Rule{APIGroups: []string{"extensions"}, APIVersions: []string{"v1beta1"}, Resources: []string{"deployments"}, Scope: &allScopes}, + }, { + Operations: []v1beta1.OperationType{"*"}, + Rule: v1beta1.Rule{APIGroups: []string{"apps"}, APIVersions: []string{"v1beta1"}, Resources: []string{"deployments"}, Scope: &allScopes}, + }, { + Operations: []v1beta1.OperationType{"*"}, + Rule: v1beta1.Rule{APIGroups: []string{"apps"}, APIVersions: []string{"v1"}, Resources: []string{"deployments"}, Scope: &allScopes}, + }}}, + attrs: admission.NewAttributesRecord(nil, nil, schema.GroupVersionKind{"apps", "v1", "Deployment"}, "ns", "name", schema.GroupVersionResource{"apps", "v1", "deployments"}, "", admission.Create, &metav1.CreateOptions{}, false, nil), + expectCall: true, + expectCallKind: schema.GroupVersionKind{"apps", "v1", "Deployment"}, + expectCallResource: schema.GroupVersionResource{"apps", "v1", "deployments"}, + expectCallSubresource: "", + }, + { + name: "specific rules, match miss", + webhook: &v1beta1.Webhook{ + NamespaceSelector: &metav1.LabelSelector{}, + Rules: []v1beta1.RuleWithOperations{{ + Operations: []v1beta1.OperationType{"*"}, + Rule: v1beta1.Rule{APIGroups: []string{"extensions"}, APIVersions: []string{"v1beta1"}, Resources: []string{"deployments"}, Scope: &allScopes}, + }, { + Operations: []v1beta1.OperationType{"*"}, + Rule: v1beta1.Rule{APIGroups: []string{"apps"}, APIVersions: []string{"v1beta1"}, Resources: []string{"deployments"}, Scope: &allScopes}, + }}}, + attrs: admission.NewAttributesRecord(nil, nil, schema.GroupVersionKind{"apps", "v1", "Deployment"}, "ns", "name", schema.GroupVersionResource{"apps", "v1", "deployments"}, "", admission.Create, &metav1.CreateOptions{}, false, nil), + expectCall: false, + }, + { + name: "specific rules, exact match miss", + webhook: &v1beta1.Webhook{ + MatchPolicy: &exactMatch, + NamespaceSelector: &metav1.LabelSelector{}, + Rules: []v1beta1.RuleWithOperations{{ + Operations: []v1beta1.OperationType{"*"}, + Rule: v1beta1.Rule{APIGroups: []string{"extensions"}, APIVersions: []string{"v1beta1"}, Resources: []string{"deployments"}, Scope: &allScopes}, + }, { + Operations: []v1beta1.OperationType{"*"}, + Rule: v1beta1.Rule{APIGroups: []string{"apps"}, APIVersions: []string{"v1beta1"}, Resources: []string{"deployments"}, Scope: &allScopes}, + }}}, + attrs: admission.NewAttributesRecord(nil, nil, schema.GroupVersionKind{"apps", "v1", "Deployment"}, "ns", "name", schema.GroupVersionResource{"apps", "v1", "deployments"}, "", admission.Create, &metav1.CreateOptions{}, false, nil), + expectCall: false, + }, + { + name: "specific rules, equivalent match, prefer extensions", + webhook: &v1beta1.Webhook{ + MatchPolicy: &equivalentMatch, + NamespaceSelector: &metav1.LabelSelector{}, + Rules: []v1beta1.RuleWithOperations{{ + Operations: []v1beta1.OperationType{"*"}, + Rule: v1beta1.Rule{APIGroups: []string{"extensions"}, APIVersions: []string{"v1beta1"}, Resources: []string{"deployments"}, Scope: &allScopes}, + }, { + Operations: []v1beta1.OperationType{"*"}, + Rule: v1beta1.Rule{APIGroups: []string{"apps"}, APIVersions: []string{"v1beta1"}, Resources: []string{"deployments"}, Scope: &allScopes}, + }}}, + attrs: admission.NewAttributesRecord(nil, nil, schema.GroupVersionKind{"apps", "v1", "Deployment"}, "ns", "name", schema.GroupVersionResource{"apps", "v1", "deployments"}, "", admission.Create, &metav1.CreateOptions{}, false, nil), + expectCall: true, + expectCallKind: schema.GroupVersionKind{"extensions", "v1beta1", "Deployment"}, + expectCallResource: schema.GroupVersionResource{"extensions", "v1beta1", "deployments"}, + expectCallSubresource: "", + }, + { + name: "specific rules, equivalent match, prefer apps", + webhook: &v1beta1.Webhook{ + MatchPolicy: &equivalentMatch, + NamespaceSelector: &metav1.LabelSelector{}, + Rules: []v1beta1.RuleWithOperations{{ + Operations: []v1beta1.OperationType{"*"}, + Rule: v1beta1.Rule{APIGroups: []string{"apps"}, APIVersions: []string{"v1beta1"}, Resources: []string{"deployments"}, Scope: &allScopes}, + }, { + Operations: []v1beta1.OperationType{"*"}, + Rule: v1beta1.Rule{APIGroups: []string{"extensions"}, APIVersions: []string{"v1beta1"}, Resources: []string{"deployments"}, Scope: &allScopes}, + }}}, + attrs: admission.NewAttributesRecord(nil, nil, schema.GroupVersionKind{"apps", "v1", "Deployment"}, "ns", "name", schema.GroupVersionResource{"apps", "v1", "deployments"}, "", admission.Create, &metav1.CreateOptions{}, false, nil), + expectCall: true, + expectCallKind: schema.GroupVersionKind{"apps", "v1beta1", "Deployment"}, + expectCallResource: schema.GroupVersionResource{"apps", "v1beta1", "deployments"}, + expectCallSubresource: "", + }, + + { + name: "specific rules, subresource prefer exact match", + webhook: &v1beta1.Webhook{ + NamespaceSelector: &metav1.LabelSelector{}, + Rules: []v1beta1.RuleWithOperations{{ + Operations: []v1beta1.OperationType{"*"}, + Rule: v1beta1.Rule{APIGroups: []string{"extensions"}, APIVersions: []string{"v1beta1"}, Resources: []string{"deployments", "deployments/scale"}, Scope: &allScopes}, + }, { + Operations: []v1beta1.OperationType{"*"}, + Rule: v1beta1.Rule{APIGroups: []string{"apps"}, APIVersions: []string{"v1beta1"}, Resources: []string{"deployments", "deployments/scale"}, Scope: &allScopes}, + }, { + Operations: []v1beta1.OperationType{"*"}, + Rule: v1beta1.Rule{APIGroups: []string{"apps"}, APIVersions: []string{"v1"}, Resources: []string{"deployments", "deployments/scale"}, Scope: &allScopes}, + }}}, + attrs: admission.NewAttributesRecord(nil, nil, schema.GroupVersionKind{"autoscaling", "v1", "Scale"}, "ns", "name", schema.GroupVersionResource{"apps", "v1", "deployments"}, "scale", admission.Create, &metav1.CreateOptions{}, false, nil), + expectCall: true, + expectCallKind: schema.GroupVersionKind{"autoscaling", "v1", "Scale"}, + expectCallResource: schema.GroupVersionResource{"apps", "v1", "deployments"}, + expectCallSubresource: "scale", + }, + { + name: "specific rules, subresource match miss", + webhook: &v1beta1.Webhook{ + NamespaceSelector: &metav1.LabelSelector{}, + Rules: []v1beta1.RuleWithOperations{{ + Operations: []v1beta1.OperationType{"*"}, + Rule: v1beta1.Rule{APIGroups: []string{"extensions"}, APIVersions: []string{"v1beta1"}, Resources: []string{"deployments", "deployments/scale"}, Scope: &allScopes}, + }, { + Operations: []v1beta1.OperationType{"*"}, + Rule: v1beta1.Rule{APIGroups: []string{"apps"}, APIVersions: []string{"v1beta1"}, Resources: []string{"deployments", "deployments/scale"}, Scope: &allScopes}, + }}}, + attrs: admission.NewAttributesRecord(nil, nil, schema.GroupVersionKind{"autoscaling", "v1", "Scale"}, "ns", "name", schema.GroupVersionResource{"apps", "v1", "deployments"}, "scale", admission.Create, &metav1.CreateOptions{}, false, nil), + expectCall: false, + }, + { + name: "specific rules, subresource exact match miss", + webhook: &v1beta1.Webhook{ + MatchPolicy: &exactMatch, + NamespaceSelector: &metav1.LabelSelector{}, + Rules: []v1beta1.RuleWithOperations{{ + Operations: []v1beta1.OperationType{"*"}, + Rule: v1beta1.Rule{APIGroups: []string{"extensions"}, APIVersions: []string{"v1beta1"}, Resources: []string{"deployments", "deployments/scale"}, Scope: &allScopes}, + }, { + Operations: []v1beta1.OperationType{"*"}, + Rule: v1beta1.Rule{APIGroups: []string{"apps"}, APIVersions: []string{"v1beta1"}, Resources: []string{"deployments", "deployments/scale"}, Scope: &allScopes}, + }}}, + attrs: admission.NewAttributesRecord(nil, nil, schema.GroupVersionKind{"autoscaling", "v1", "Scale"}, "ns", "name", schema.GroupVersionResource{"apps", "v1", "deployments"}, "scale", admission.Create, &metav1.CreateOptions{}, false, nil), + expectCall: false, + }, + { + name: "specific rules, subresource equivalent match, prefer extensions", + webhook: &v1beta1.Webhook{ + MatchPolicy: &equivalentMatch, + NamespaceSelector: &metav1.LabelSelector{}, + Rules: []v1beta1.RuleWithOperations{{ + Operations: []v1beta1.OperationType{"*"}, + Rule: v1beta1.Rule{APIGroups: []string{"extensions"}, APIVersions: []string{"v1beta1"}, Resources: []string{"deployments", "deployments/scale"}, Scope: &allScopes}, + }, { + Operations: []v1beta1.OperationType{"*"}, + Rule: v1beta1.Rule{APIGroups: []string{"apps"}, APIVersions: []string{"v1beta1"}, Resources: []string{"deployments", "deployments/scale"}, Scope: &allScopes}, + }}}, + attrs: admission.NewAttributesRecord(nil, nil, schema.GroupVersionKind{"autoscaling", "v1", "Scale"}, "ns", "name", schema.GroupVersionResource{"apps", "v1", "deployments"}, "scale", admission.Create, &metav1.CreateOptions{}, false, nil), + expectCall: true, + expectCallKind: schema.GroupVersionKind{"extensions", "v1beta1", "Scale"}, + expectCallResource: schema.GroupVersionResource{"extensions", "v1beta1", "deployments"}, + expectCallSubresource: "scale", + }, + { + name: "specific rules, subresource equivalent match, prefer apps", + webhook: &v1beta1.Webhook{ + MatchPolicy: &equivalentMatch, + NamespaceSelector: &metav1.LabelSelector{}, + Rules: []v1beta1.RuleWithOperations{{ + Operations: []v1beta1.OperationType{"*"}, + Rule: v1beta1.Rule{APIGroups: []string{"apps"}, APIVersions: []string{"v1beta1"}, Resources: []string{"deployments", "deployments/scale"}, Scope: &allScopes}, + }, { + Operations: []v1beta1.OperationType{"*"}, + Rule: v1beta1.Rule{APIGroups: []string{"extensions"}, APIVersions: []string{"v1beta1"}, Resources: []string{"deployments", "deployments/scale"}, Scope: &allScopes}, + }}}, + attrs: admission.NewAttributesRecord(nil, nil, schema.GroupVersionKind{"autoscaling", "v1", "Scale"}, "ns", "name", schema.GroupVersionResource{"apps", "v1", "deployments"}, "scale", admission.Create, &metav1.CreateOptions{}, false, nil), + expectCall: true, + expectCallKind: schema.GroupVersionKind{"apps", "v1beta1", "Scale"}, + expectCallResource: schema.GroupVersionResource{"apps", "v1beta1", "deployments"}, + expectCallSubresource: "scale", + }, + } + + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + invocation, err := a.shouldCallHook(testcase.webhook, testcase.attrs, interfaces) + if err != nil { + if len(testcase.expectErr) == 0 { + t.Fatal(err) + } + if !strings.Contains(err.Error(), testcase.expectErr) { + t.Fatalf("expected error containing %q, got %s", testcase.expectErr, err.Error()) + } + return + } else if len(testcase.expectErr) > 0 { + t.Fatalf("expected error %q, got no error and %#v", testcase.expectErr, invocation) + } + + if invocation == nil { + if testcase.expectCall { + t.Fatal("expected invocation, got nil") + } + return + } + + if !testcase.expectCall { + t.Fatal("unexpected invocation") + } + + if invocation.Kind != testcase.expectCallKind { + t.Fatalf("expected %#v, got %#v", testcase.expectCallKind, invocation.Kind) + } + if invocation.Resource != testcase.expectCallResource { + t.Fatalf("expected %#v, got %#v", testcase.expectCallResource, invocation.Resource) + } + if invocation.Subresource != testcase.expectCallSubresource { + t.Fatalf("expected %#v, got %#v", testcase.expectCallSubresource, invocation.Subresource) + } + }) + } +}