From be172d6900eb7d02adbd5931e91fe804bd86997d Mon Sep 17 00:00:00 2001 From: Haowei Cai Date: Fri, 8 Jan 2021 11:39:08 -0800 Subject: [PATCH 1/2] add rbac role for storage version GC --- .../rbac/bootstrappolicy/controller_policy.go | 14 ++++ .../authorizer/rbac/bootstrappolicy/policy.go | 35 ++++---- test/e2e/apimachinery/storage_version.go | 82 +++++++++++++++++++ 3 files changed, 114 insertions(+), 17 deletions(-) create mode 100644 test/e2e/apimachinery/storage_version.go diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go index 2d326c62767..4e4e81f4ad2 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go @@ -24,6 +24,7 @@ import ( capi "k8s.io/api/certificates/v1beta1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + genericfeatures "k8s.io/apiserver/pkg/features" utilfeature "k8s.io/apiserver/pkg/util/feature" rbacv1helpers "k8s.io/kubernetes/pkg/apis/rbac/v1" "k8s.io/kubernetes/pkg/features" @@ -411,6 +412,19 @@ func buildControllerRoles() ([]rbacv1.ClusterRole, []rbacv1.ClusterRoleBinding) }, }) } + if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.StorageVersionAPI) && + utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerIdentity) { + addControllerRole(&controllerRoles, &controllerRoleBindings, rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + "storage-version-garbage-collector"}, + Rules: []rbacv1.PolicyRule{ + rbacv1helpers.NewRule("get", "list", "watch").Groups(coordinationGroup).Resources("leases").RuleOrDie(), + rbacv1helpers.NewRule("get", "list", "watch", "patch", "update", "delete").Groups(internalAPIServerGroup). + Resources("storageversions").RuleOrDie(), + rbacv1helpers.NewRule("get", "patch", "update").Groups(internalAPIServerGroup). + Resources("storageversions/status").RuleOrDie(), + }, + }) + } return controllerRoles, controllerRoleBindings } diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go index 352543a2281..4818bbb1e0e 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go @@ -42,23 +42,24 @@ var ( ) const ( - legacyGroup = "" - appsGroup = "apps" - authenticationGroup = "authentication.k8s.io" - authorizationGroup = "authorization.k8s.io" - autoscalingGroup = "autoscaling" - batchGroup = "batch" - certificatesGroup = "certificates.k8s.io" - coordinationGroup = "coordination.k8s.io" - discoveryGroup = "discovery.k8s.io" - extensionsGroup = "extensions" - policyGroup = "policy" - rbacGroup = "rbac.authorization.k8s.io" - storageGroup = "storage.k8s.io" - resMetricsGroup = "metrics.k8s.io" - customMetricsGroup = "custom.metrics.k8s.io" - networkingGroup = "networking.k8s.io" - eventsGroup = "events.k8s.io" + legacyGroup = "" + appsGroup = "apps" + authenticationGroup = "authentication.k8s.io" + authorizationGroup = "authorization.k8s.io" + autoscalingGroup = "autoscaling" + batchGroup = "batch" + certificatesGroup = "certificates.k8s.io" + coordinationGroup = "coordination.k8s.io" + discoveryGroup = "discovery.k8s.io" + extensionsGroup = "extensions" + policyGroup = "policy" + rbacGroup = "rbac.authorization.k8s.io" + storageGroup = "storage.k8s.io" + resMetricsGroup = "metrics.k8s.io" + customMetricsGroup = "custom.metrics.k8s.io" + networkingGroup = "networking.k8s.io" + eventsGroup = "events.k8s.io" + internalAPIServerGroup = "internal.apiserver.k8s.io" ) func addDefaultMetadata(obj runtime.Object) { diff --git a/test/e2e/apimachinery/storage_version.go b/test/e2e/apimachinery/storage_version.go new file mode 100644 index 00000000000..f0fa4b95d6a --- /dev/null +++ b/test/e2e/apimachinery/storage_version.go @@ -0,0 +1,82 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apimachinery + +import ( + "context" + "time" + + apiserverinternalv1alpha1 "k8s.io/api/apiserverinternal/v1alpha1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/kubernetes/test/e2e/framework" + + "github.com/onsi/ginkgo" +) + +const ( + svName = "storageversion.e2e.test.foos" + idNonExist = "id-non-exist" +) + +var _ = SIGDescribe("StorageVersion resources [Feature:StorageVersionAPI]", func() { + f := framework.NewDefaultFramework("storage-version") + + ginkgo.It("storage version with non-existing id should be GC'ed", func() { + client := f.ClientSet + sv := &apiserverinternalv1alpha1.StorageVersion{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: svName, + }, + } + createdSV, err := client.InternalV1alpha1().StorageVersions().Create(context.TODO(), sv, metav1.CreateOptions{}) + framework.ExpectNoError(err, "creating storage version") + + // update the created sv with server storage version + version := "v1" + createdSV.Status = apiserverinternalv1alpha1.StorageVersionStatus{ + StorageVersions: []apiserverinternalv1alpha1.ServerStorageVersion{ + { + APIServerID: idNonExist, + EncodingVersion: version, + DecodableVersions: []string{version}, + }, + }, + CommonEncodingVersion: &version, + } + _, err = client.InternalV1alpha1().StorageVersions().UpdateStatus( + context.TODO(), createdSV, metav1.UpdateOptions{}) + framework.ExpectNoError(err, "updating storage version") + + // wait for sv to be GC'ed + framework.Logf("Waiting for storage version %v to be garbage collected", createdSV.Name) + err = wait.PollImmediate(100*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { + _, err := client.InternalV1alpha1().StorageVersions().Get( + context.TODO(), createdSV.Name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return true, nil + } + if err != nil { + return false, err + } + framework.Logf("The storage version %v hasn't been garbage collected yet. Retrying", createdSV.Name) + return false, nil + }) + framework.ExpectNoError(err, "garbage-collecting storage version") + }) +}) From 83b30bc92f582022ff2327c6b497a51397afe316 Mon Sep 17 00:00:00 2001 From: Haowei Cai Date: Fri, 8 Jan 2021 11:39:41 -0800 Subject: [PATCH 2/2] generated --- plugin/pkg/auth/authorizer/rbac/bootstrappolicy/BUILD | 1 + test/e2e/apimachinery/BUILD | 2 ++ 2 files changed, 3 insertions(+) diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/BUILD b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/BUILD index e8a672a2c51..5a17c9bb6e7 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/BUILD +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/BUILD @@ -24,6 +24,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/serviceaccount:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/features:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//vendor/k8s.io/klog/v2:go_default_library", ], diff --git a/test/e2e/apimachinery/BUILD b/test/e2e/apimachinery/BUILD index f72675fb0cc..d340a6b9e38 100644 --- a/test/e2e/apimachinery/BUILD +++ b/test/e2e/apimachinery/BUILD @@ -27,6 +27,7 @@ go_library( "protocol.go", "resource_quota.go", "server_version.go", + "storage_version.go", "table_conversion.go", "watch.go", "webhook.go", @@ -37,6 +38,7 @@ go_library( "//pkg/apis/rbac/v1:go_default_library", "//pkg/quota/v1/evaluator/core:go_default_library", "//staging/src/k8s.io/api/admissionregistration/v1:go_default_library", + "//staging/src/k8s.io/api/apiserverinternal/v1alpha1:go_default_library", "//staging/src/k8s.io/api/apps/v1:go_default_library", "//staging/src/k8s.io/api/authorization/v1:go_default_library", "//staging/src/k8s.io/api/batch/v1:go_default_library",