From efac533f9269ca1687d084a7360f87f7b4ace1fc Mon Sep 17 00:00:00 2001 From: WanLinghao Date: Thu, 18 Oct 2018 14:41:53 +0800 Subject: [PATCH] To inject ca.crt into container when projected volume was specified, configmap should be created in each namespace. This patch add a controller called "root-ca-cert-publisher" to complete above job as well as some bootstrap rbac policies. --- cmd/kube-controller-manager/app/BUILD | 1 + .../app/certificates.go | 33 +++ .../app/controllermanager.go | 24 +- pkg/controller/certificates/BUILD | 1 + .../certificates/rootcacertpublisher/BUILD | 51 ++++ .../root_ca_cert_publisher.go | 232 ++++++++++++++++++ .../root_ca_cert_publisher_test.go | 218 ++++++++++++++++ .../rbac/bootstrappolicy/controller_policy.go | 10 + .../testdata/controller-role-bindings.yaml | 17 ++ .../testdata/controller-roles.yaml | 25 ++ test/cmd/apps.sh | 3 +- test/cmd/core.sh | 15 +- test/cmd/get.sh | 7 +- test/e2e/scheduling/resource_quota.go | 24 +- 14 files changed, 646 insertions(+), 15 deletions(-) create mode 100644 pkg/controller/certificates/rootcacertpublisher/BUILD create mode 100644 pkg/controller/certificates/rootcacertpublisher/root_ca_cert_publisher.go create mode 100644 pkg/controller/certificates/rootcacertpublisher/root_ca_cert_publisher_test.go diff --git a/cmd/kube-controller-manager/app/BUILD b/cmd/kube-controller-manager/app/BUILD index 41024caab96..f5807e2dcb7 100644 --- a/cmd/kube-controller-manager/app/BUILD +++ b/cmd/kube-controller-manager/app/BUILD @@ -43,6 +43,7 @@ go_library( "//pkg/controller/bootstrap:go_default_library", "//pkg/controller/certificates/approver:go_default_library", "//pkg/controller/certificates/cleaner:go_default_library", + "//pkg/controller/certificates/rootcacertpublisher:go_default_library", "//pkg/controller/certificates/signer:go_default_library", "//pkg/controller/clusterroleaggregation:go_default_library", "//pkg/controller/cronjob:go_default_library", diff --git a/cmd/kube-controller-manager/app/certificates.go b/cmd/kube-controller-manager/app/certificates.go index 078a5f8452c..bb9ca7afe2c 100644 --- a/cmd/kube-controller-manager/app/certificates.go +++ b/cmd/kube-controller-manager/app/certificates.go @@ -29,10 +29,13 @@ import ( "net/http" "k8s.io/apimachinery/pkg/runtime/schema" + utilfeature "k8s.io/apiserver/pkg/util/feature" kubeoptions "k8s.io/kubernetes/cmd/kube-controller-manager/app/options" "k8s.io/kubernetes/pkg/controller/certificates/approver" "k8s.io/kubernetes/pkg/controller/certificates/cleaner" + "k8s.io/kubernetes/pkg/controller/certificates/rootcacertpublisher" "k8s.io/kubernetes/pkg/controller/certificates/signer" + "k8s.io/kubernetes/pkg/features" ) func startCSRSigningController(ctx ControllerContext) (http.Handler, bool, error) { @@ -120,3 +123,33 @@ func startCSRCleanerController(ctx ControllerContext) (http.Handler, bool, error go cleaner.Run(1, ctx.Stop) return nil, true, nil } + +func startRootCACertPublisher(ctx ControllerContext) (http.Handler, bool, error) { + if !utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) { + return nil, false, nil + } + + var ( + rootCA []byte + err error + ) + if ctx.ComponentConfig.SAController.RootCAFile != "" { + if rootCA, err = readCA(ctx.ComponentConfig.SAController.RootCAFile); err != nil { + return nil, true, fmt.Errorf("error parsing root-ca-file at %s: %v", ctx.ComponentConfig.SAController.RootCAFile, err) + } + } else { + rootCA = ctx.ClientBuilder.ConfigOrDie("root-ca-cert-publisher").CAData + } + + sac, err := rootcacertpublisher.NewPublisher( + ctx.InformerFactory.Core().V1().ConfigMaps(), + ctx.InformerFactory.Core().V1().Namespaces(), + ctx.ClientBuilder.ClientOrDie("root-ca-cert-publisher"), + rootCA, + ) + if err != nil { + return nil, true, fmt.Errorf("error creating root CA certificate publisher: %v", err) + } + go sac.Run(1, ctx.Stop) + return nil, true, nil +} diff --git a/cmd/kube-controller-manager/app/controllermanager.go b/cmd/kube-controller-manager/app/controllermanager.go index 1dab0461221..f60564224de 100644 --- a/cmd/kube-controller-manager/app/controllermanager.go +++ b/cmd/kube-controller-manager/app/controllermanager.go @@ -39,6 +39,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/mux" + utilfeature "k8s.io/apiserver/pkg/util/feature" apiserverflag "k8s.io/apiserver/pkg/util/flag" cacheddiscovery "k8s.io/client-go/discovery/cached" "k8s.io/client-go/informers" @@ -54,6 +55,7 @@ import ( "k8s.io/kubernetes/pkg/controller" kubectrlmgrconfig "k8s.io/kubernetes/pkg/controller/apis/config" serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/serviceaccount" "k8s.io/kubernetes/pkg/util/configz" utilflag "k8s.io/kubernetes/pkg/util/flag" @@ -333,6 +335,7 @@ func KnownControllers() []string { var ControllersDisabledByDefault = sets.NewString( "bootstrapsigner", "tokencleaner", + "root_ca_crt_publisher", ) const ( @@ -379,6 +382,9 @@ func NewControllerInitializers(loopMode ControllerLoopMode) map[string]InitFunc controllers["pvc-protection"] = startPVCProtectionController controllers["pv-protection"] = startPVProtectionController controllers["ttl-after-finished"] = startTTLAfterFinishedController + if utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) { + controllers["root_ca_crt_publisher"] = startRootCACertPublisher + } return controllers } @@ -524,11 +530,7 @@ func (c serviceAccountTokenControllerStarter) startServiceAccountTokenController var rootCA []byte if ctx.ComponentConfig.SAController.RootCAFile != "" { - rootCA, err = ioutil.ReadFile(ctx.ComponentConfig.SAController.RootCAFile) - if err != nil { - return nil, true, fmt.Errorf("error reading root-ca-file at %s: %v", ctx.ComponentConfig.SAController.RootCAFile, err) - } - if _, err := certutil.ParseCertsPEM(rootCA); err != nil { + if rootCA, err = readCA(ctx.ComponentConfig.SAController.RootCAFile); err != nil { return nil, true, fmt.Errorf("error parsing root-ca-file at %s: %v", ctx.ComponentConfig.SAController.RootCAFile, err) } } else { @@ -558,3 +560,15 @@ func (c serviceAccountTokenControllerStarter) startServiceAccountTokenController return nil, true, nil } + +func readCA(file string) ([]byte, error) { + rootCA, err := ioutil.ReadFile(file) + if err != nil { + return nil, err + } + if _, err := certutil.ParseCertsPEM(rootCA); err != nil { + return nil, err + } + + return rootCA, err +} diff --git a/pkg/controller/certificates/BUILD b/pkg/controller/certificates/BUILD index 8057088ca18..4502d2f4efd 100644 --- a/pkg/controller/certificates/BUILD +++ b/pkg/controller/certificates/BUILD @@ -43,6 +43,7 @@ filegroup( ":package-srcs", "//pkg/controller/certificates/approver:all-srcs", "//pkg/controller/certificates/cleaner:all-srcs", + "//pkg/controller/certificates/rootcacertpublisher:all-srcs", "//pkg/controller/certificates/signer:all-srcs", ], tags = ["automanaged"], diff --git a/pkg/controller/certificates/rootcacertpublisher/BUILD b/pkg/controller/certificates/rootcacertpublisher/BUILD new file mode 100644 index 00000000000..603de73b745 --- /dev/null +++ b/pkg/controller/certificates/rootcacertpublisher/BUILD @@ -0,0 +1,51 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["root_ca_cert_publisher.go"], + importpath = "k8s.io/kubernetes/pkg/controller/certificates/rootcacertpublisher", + visibility = ["//visibility:public"], + deps = [ + "//pkg/controller:go_default_library", + "//pkg/util/metrics:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/client-go/informers/core/v1:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", + "//staging/src/k8s.io/client-go/tools/cache:go_default_library", + "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", + "//vendor/github.com/golang/glog:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["root_ca_cert_publisher_test.go"], + embed = [":go_default_library"], + deps = [ + "//pkg/controller:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/client-go/informers:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", + "//staging/src/k8s.io/client-go/testing:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/pkg/controller/certificates/rootcacertpublisher/root_ca_cert_publisher.go b/pkg/controller/certificates/rootcacertpublisher/root_ca_cert_publisher.go new file mode 100644 index 00000000000..274518afe81 --- /dev/null +++ b/pkg/controller/certificates/rootcacertpublisher/root_ca_cert_publisher.go @@ -0,0 +1,232 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rootcacertpublisher + +import ( + "fmt" + "reflect" + "time" + + "github.com/golang/glog" + "k8s.io/api/core/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + coreinformers "k8s.io/client-go/informers/core/v1" + clientset "k8s.io/client-go/kubernetes" + corelisters "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + "k8s.io/kubernetes/pkg/controller" + "k8s.io/kubernetes/pkg/util/metrics" +) + +// RootCACertCofigMapName is name of the configmap which stores certificates to access api-server +const RootCACertCofigMapName = "kube-root-ca.crt" + +// NewPublisher construct a new controller which would manage the configmap which stores +// certificates in each namespace. It will make sure certificate configmap exists in each namespace. +func NewPublisher(cmInformer coreinformers.ConfigMapInformer, nsInformer coreinformers.NamespaceInformer, cl clientset.Interface, rootCA []byte) (*Publisher, error) { + e := &Publisher{ + client: cl, + configMap: v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: RootCACertCofigMapName, + }, + Data: map[string]string{ + "ca.crt": string(rootCA), + }, + }, + queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "root-ca-crt-publisher"), + } + if cl.CoreV1().RESTClient().GetRateLimiter() != nil { + if err := metrics.RegisterMetricAndTrackRateLimiterUsage("root_ca_crt_publisher", cl.CoreV1().RESTClient().GetRateLimiter()); err != nil { + return nil, err + } + } + + cmInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + DeleteFunc: e.configMapDeleted, + UpdateFunc: e.configMapUpdated, + }) + e.cmLister = cmInformer.Lister() + e.cmListerSynced = cmInformer.Informer().HasSynced + + nsInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: e.namespaceAdded, + UpdateFunc: e.namespaceUpdated, + }) + e.nsLister = nsInformer.Lister() + e.nsListerSynced = nsInformer.Informer().HasSynced + + e.syncHandler = e.syncNamespace + + return e, nil + +} + +// Publisher manages certificate ConfigMap objects inside Namespaces +type Publisher struct { + client clientset.Interface + configMap v1.ConfigMap + + // To allow injection for testing. + syncHandler func(key string) error + + cmLister corelisters.ConfigMapLister + cmListerSynced cache.InformerSynced + + nsLister corelisters.NamespaceLister + nsListerSynced cache.InformerSynced + + queue workqueue.RateLimitingInterface +} + +// Run starts process +func (c *Publisher) Run(workers int, stopCh <-chan struct{}) { + defer utilruntime.HandleCrash() + defer c.queue.ShutDown() + + glog.Infof("Starting root CA certificate configmap publisher") + defer glog.Infof("Shutting down root CA certificate configmap publisher") + + if !controller.WaitForCacheSync("crt configmap", stopCh, c.cmListerSynced, c.nsListerSynced) { + return + } + + for i := 0; i < workers; i++ { + go wait.Until(c.runWorker, time.Second, stopCh) + } + + <-stopCh +} + +func (c *Publisher) configMapDeleted(obj interface{}) { + cm, err := convertToCM(obj) + if err != nil { + utilruntime.HandleError(err) + return + } + if cm.Name != RootCACertCofigMapName { + return + } + c.queue.Add(cm.Namespace) +} + +func (c *Publisher) configMapUpdated(_, newObj interface{}) { + newConfigMap, err := convertToCM(newObj) + if err != nil { + utilruntime.HandleError(err) + return + } + if newConfigMap.Name != RootCACertCofigMapName { + return + } + + if reflect.DeepEqual(c.configMap.Data, newConfigMap.Data) { + return + } + + newConfigMap.Data = make(map[string]string) + newConfigMap.Data["ca.crt"] = c.configMap.Data["ca.crt"] + if _, err := c.client.CoreV1().ConfigMaps(newConfigMap.Namespace).Update(newConfigMap); err != nil && !apierrs.IsAlreadyExists(err) { + utilruntime.HandleError(fmt.Errorf("configmap creation failure:%v", err)) + } +} + +func (c *Publisher) namespaceAdded(obj interface{}) { + namespace := obj.(*v1.Namespace) + c.queue.Add(namespace.Name) +} + +func (c *Publisher) namespaceUpdated(oldObj interface{}, newObj interface{}) { + newNamespace := newObj.(*v1.Namespace) + if newNamespace.Status.Phase != v1.NamespaceActive { + return + } + c.queue.Add(newNamespace.Name) +} + +func (c *Publisher) runWorker() { + for c.processNextWorkItem() { + } +} + +// processNextWorkItem deals with one key off the queue. It returns false when it's time to quit. +func (c *Publisher) processNextWorkItem() bool { + key, quit := c.queue.Get() + if quit { + return false + } + defer c.queue.Done(key) + + err := c.syncHandler(key.(string)) + if err == nil { + c.queue.Forget(key) + return true + } + + utilruntime.HandleError(fmt.Errorf("%v failed with : %v", key, err)) + c.queue.AddRateLimited(key) + + return true +} + +func (c *Publisher) syncNamespace(key string) error { + startTime := time.Now() + defer func() { + glog.V(4).Infof("Finished syncing namespace %q (%v)", key, time.Since(startTime)) + }() + + ns, err := c.nsLister.Get(key) + if apierrs.IsNotFound(err) { + return nil + } + if err != nil { + return err + } + + switch _, err := c.cmLister.ConfigMaps(ns.Name).Get(c.configMap.Name); { + case err == nil: + return nil + case apierrs.IsNotFound(err): + case err != nil: + return err + } + + cm := c.configMap.DeepCopy() + if _, err := c.client.CoreV1().ConfigMaps(ns.Name).Create(cm); err != nil && !apierrs.IsAlreadyExists(err) { + return err + } + return nil +} + +func convertToCM(obj interface{}) (*v1.ConfigMap, error) { + cm, ok := obj.(*v1.ConfigMap) + if !ok { + tombstone, ok := obj.(cache.DeletedFinalStateUnknown) + if !ok { + return nil, fmt.Errorf("Couldn't get object from tombstone %#v", obj) + } + cm, ok = tombstone.Obj.(*v1.ConfigMap) + if !ok { + return nil, fmt.Errorf("Tombstone contained object that is not a ConfigMap %#v", obj) + } + } + return cm, nil +} diff --git a/pkg/controller/certificates/rootcacertpublisher/root_ca_cert_publisher_test.go b/pkg/controller/certificates/rootcacertpublisher/root_ca_cert_publisher_test.go new file mode 100644 index 00000000000..97a4043baf4 --- /dev/null +++ b/pkg/controller/certificates/rootcacertpublisher/root_ca_cert_publisher_test.go @@ -0,0 +1,218 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rootcacertpublisher + +import ( + "testing" + "time" + + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" + core "k8s.io/client-go/testing" + "k8s.io/kubernetes/pkg/controller" +) + +func TestConfigMapCreation(t *testing.T) { + ns := metav1.NamespaceDefault + fakeRootCA := []byte("fake-root-ca") + + caConfigMap := defaultCrtConfigMapPtr(fakeRootCA) + addFieldCM := defaultCrtConfigMapPtr(fakeRootCA) + addFieldCM.Data["test"] = "test" + modifyFieldCM := defaultCrtConfigMapPtr([]byte("abc")) + otherConfigMap := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "other", + Namespace: ns, + ResourceVersion: "1", + }, + } + updateOtherConfigMap := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "other", + Namespace: ns, + ResourceVersion: "1", + }, + Data: map[string]string{"aa": "bb"}, + } + + existNS := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: ns}, + Status: v1.NamespaceStatus{ + Phase: v1.NamespaceActive, + }, + } + newNs := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "new"}, + Status: v1.NamespaceStatus{ + Phase: v1.NamespaceActive, + }, + } + terminatingNS := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: ns}, + Status: v1.NamespaceStatus{ + Phase: v1.NamespaceTerminating, + }, + } + + type action struct { + verb string + name string + } + testcases := map[string]struct { + ExistingNamespace *v1.Namespace + ExistingConfigMaps []*v1.ConfigMap + AddedNamespace *v1.Namespace + UpdatedNamespace *v1.Namespace + DeletedConfigMap *v1.ConfigMap + UpdatedConfigMap []*v1.ConfigMap + ExpectActions []action + }{ + "create new namesapce": { + AddedNamespace: newNs, + ExpectActions: []action{{verb: "create", name: RootCACertCofigMapName}}, + }, + + "delete other configmap": { + ExistingNamespace: existNS, + ExistingConfigMaps: []*v1.ConfigMap{otherConfigMap, caConfigMap}, + DeletedConfigMap: otherConfigMap, + }, + "delete ca configmap": { + ExistingNamespace: existNS, + ExistingConfigMaps: []*v1.ConfigMap{otherConfigMap, caConfigMap}, + DeletedConfigMap: caConfigMap, + ExpectActions: []action{{verb: "create", name: RootCACertCofigMapName}}, + }, + "update ca configmap with adding field": { + ExistingNamespace: existNS, + ExistingConfigMaps: []*v1.ConfigMap{caConfigMap}, + UpdatedConfigMap: []*v1.ConfigMap{caConfigMap, addFieldCM}, + ExpectActions: []action{{verb: "update", name: RootCACertCofigMapName}}, + }, + "update ca configmap with modifying field": { + ExistingNamespace: existNS, + ExistingConfigMaps: []*v1.ConfigMap{caConfigMap}, + UpdatedConfigMap: []*v1.ConfigMap{caConfigMap, modifyFieldCM}, + ExpectActions: []action{{verb: "update", name: RootCACertCofigMapName}}, + }, + "update with other configmap": { + ExistingNamespace: existNS, + ExistingConfigMaps: []*v1.ConfigMap{caConfigMap, otherConfigMap}, + UpdatedConfigMap: []*v1.ConfigMap{otherConfigMap, updateOtherConfigMap}, + }, + "update namespace with terminating state": { + ExistingNamespace: existNS, + UpdatedNamespace: terminatingNS, + }, + } + + for k, tc := range testcases { + client := fake.NewSimpleClientset(caConfigMap, existNS) + informers := informers.NewSharedInformerFactory(fake.NewSimpleClientset(), controller.NoResyncPeriodFunc()) + cmInformer := informers.Core().V1().ConfigMaps() + nsInformer := informers.Core().V1().Namespaces() + controller, err := NewPublisher(cmInformer, nsInformer, client, fakeRootCA) + if err != nil { + t.Fatalf("error creating ServiceAccounts controller: %v", err) + } + controller.cmListerSynced = alwaysReady + controller.nsListerSynced = alwaysReady + + cmStore := cmInformer.Informer().GetStore() + nsStore := nsInformer.Informer().GetStore() + + syncCalls := make(chan struct{}) + controller.syncHandler = func(key string) error { + err := controller.syncNamespace(key) + if err != nil { + t.Logf("%s: %v", k, err) + } + syncCalls <- struct{}{} + return err + } + stopCh := make(chan struct{}) + defer close(stopCh) + go controller.Run(1, stopCh) + + if tc.ExistingNamespace != nil { + nsStore.Add(tc.ExistingNamespace) + } + for _, s := range tc.ExistingConfigMaps { + cmStore.Add(s) + } + + if tc.AddedNamespace != nil { + nsStore.Add(tc.AddedNamespace) + controller.namespaceAdded(tc.AddedNamespace) + } + if tc.UpdatedNamespace != nil { + controller.namespaceUpdated(tc.ExistingNamespace, tc.UpdatedNamespace) + } + + if tc.DeletedConfigMap != nil { + cmStore.Delete(tc.DeletedConfigMap) + controller.configMapDeleted(tc.DeletedConfigMap) + } + + if tc.UpdatedConfigMap != nil { + old := tc.UpdatedConfigMap[0] + new := tc.UpdatedConfigMap[1] + controller.configMapUpdated(old, new) + } + + // wait to be called + select { + case <-syncCalls: + case <-time.After(5 * time.Second): + } + + actions := client.Actions() + if len(tc.ExpectActions) != len(actions) { + t.Errorf("%s: Expected to create configmap %#v. Actual actions were: %#v", k, tc.ExpectActions, actions) + continue + } + for i, expectAction := range tc.ExpectActions { + action := actions[i] + if !action.Matches(expectAction.verb, "configmaps") { + t.Errorf("%s: Unexpected action %s", k, action) + break + } + cm := action.(core.CreateAction).GetObject().(*v1.ConfigMap) + if cm.Name != expectAction.name { + t.Errorf("%s: Expected %s to be %s, got %s be %s", k, expectAction.name, expectAction.verb, cm.Name, action.GetVerb()) + } + } + } +} + +var alwaysReady = func() bool { return true } + +func defaultCrtConfigMapPtr(rootCA []byte) *v1.ConfigMap { + tmp := v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: RootCACertCofigMapName, + }, + Data: map[string]string{ + "ca.crt": string(rootCA), + }, + } + tmp.Namespace = metav1.NamespaceDefault + return &tmp +} diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go index ef96322cad0..ba6fd6724e2 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go @@ -353,6 +353,16 @@ func buildControllerRoles() ([]rbacv1.ClusterRole, []rbacv1.ClusterRoleBinding) }) } + if utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) { + addControllerRole(&controllerRoles, &controllerRoleBindings, rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + "root-ca-cert-publisher"}, + Rules: []rbacv1.PolicyRule{ + rbacv1helpers.NewRule("create", "update").Groups(legacyGroup).Resources("configmaps").RuleOrDie(), + eventsRule(), + }, + }) + } + return controllerRoles, controllerRoleBindings } diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-role-bindings.yaml b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-role-bindings.yaml index 6d5cb73e50d..4f259b16819 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-role-bindings.yaml +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-role-bindings.yaml @@ -357,6 +357,23 @@ items: - kind: ServiceAccount name: resourcequota-controller namespace: kube-system +- apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + creationTimestamp: null + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:controller:root-ca-cert-publisher + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:controller:root-ca-cert-publisher + subjects: + - kind: ServiceAccount + name: root-ca-cert-publisher + namespace: kube-system - apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-roles.yaml b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-roles.yaml index 3344def7c28..a853a9d9e9b 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-roles.yaml +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-roles.yaml @@ -1031,6 +1031,31 @@ items: - create - patch - update +- apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + creationTimestamp: null + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:controller:root-ca-cert-publisher + rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - update + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update - apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/test/cmd/apps.sh b/test/cmd/apps.sh index e8a7ef30183..8932ba1a841 100755 --- a/test/cmd/apps.sh +++ b/test/cmd/apps.sh @@ -369,7 +369,8 @@ run_deployment_tests() { kubectl create -f hack/testdata/configmap.yaml "${kube_flags[@]}" kubectl create -f hack/testdata/secret.yaml "${kube_flags[@]}" kube::test::get_object_assert deployment "{{range.items}}{{$id_field}}:{{end}}" 'nginx-deployment:' - kube::test::get_object_assert configmap "{{range.items}}{{$id_field}}:{{end}}" 'test-set-env-config:' + #configmap is special here due to controller will create kube-root-ca.crt for each namespace automatically + kube::test::get_object_assert 'configmaps/test-set-env-config' "{{$id_field}}" 'test-set-env-config' kube::test::get_object_assert secret "{{range.items}}{{$id_field}}:{{end}}" 'test-set-env-secret:' # Set env of deployments by configmap from keys kubectl set env deployment nginx-deployment --keys=key-2 --from=configmap/test-set-env-config "${kube_flags[@]}" diff --git a/test/cmd/core.sh b/test/cmd/core.sh index 549703fa0ee..3516af33159 100755 --- a/test/cmd/core.sh +++ b/test/cmd/core.sh @@ -25,7 +25,7 @@ run_configmap_tests() { create_and_use_new_namespace kube::log::status "Testing configmaps" kubectl create -f test/fixtures/doc-yaml/user-guide/configmap/configmap.yaml - kube::test::get_object_assert configmap "{{range.items}}{{$id_field}}{{end}}" 'test-configmap' + kube::test::get_object_assert 'configmap/test-configmap' "{{$id_field}}" 'test-configmap' kubectl delete configmap test-configmap "${kube_flags[@]}" ### Create a new namespace @@ -37,8 +37,10 @@ run_configmap_tests() { kube::test::get_object_assert 'namespaces/test-configmaps' "{{$id_field}}" 'test-configmaps' ### Create a generic configmap in a specific namespace - # Pre-condition: no configmaps namespace exists - kube::test::get_object_assert 'configmaps --namespace=test-configmaps' "{{range.items}}{{$id_field}}:{{end}}" '' + # Pre-condition: configmap test-configmap and test-binary-configmap does not exist + kube::test::get_object_assert 'configmaps' '{{range.items}}{{ if eq $id_field \"test-configmap\" }}found{{end}}{{end}}:' ':' + kube::test::get_object_assert 'configmaps' '{{range.items}}{{ if eq $id_field \"test-binary-configmap\" }}found{{end}}{{end}}:' ':' + # Command kubectl create configmap test-configmap --from-literal=key1=value1 --namespace=test-configmaps kubectl create configmap test-binary-configmap --from-file <( head -c 256 /dev/urandom ) --namespace=test-configmaps @@ -222,8 +224,11 @@ run_pod_tests() { kube::test::get_object_assert 'secret/test-secret --namespace=test-kubectl-describe-pod' "{{$secret_type}}" 'test-type' ### Create a generic configmap - # Pre-condition: no CONFIGMAP exists - kube::test::get_object_assert 'configmaps --namespace=test-kubectl-describe-pod' "{{range.items}}{{$id_field}}:{{end}}" '' + # Pre-condition: CONFIGMAP test-configmap does not exist + #kube::test::get_object_assert 'configmap/test-configmap --namespace=test-kubectl-describe-pod' "{{$id_field}}" '' + kube::test::get_object_assert 'configmaps --namespace=test-kubectl-describe-pod' '{{range.items}}{{ if eq $id_field \"test-configmap\" }}found{{end}}{{end}}:' ':' + + #kube::test::get_object_assert 'configmaps --namespace=test-kubectl-describe-pod' "{{range.items}}{{$id_field}}:{{end}}" '' # Command kubectl create configmap test-configmap --from-literal=key-2=value2 --namespace=test-kubectl-describe-pod # Post-condition: configmap exists and has expected values diff --git a/test/cmd/get.sh b/test/cmd/get.sh index 089e5aa4e59..15549d3021e 100755 --- a/test/cmd/get.sh +++ b/test/cmd/get.sh @@ -130,8 +130,11 @@ run_kubectl_get_tests() { kube::test::if_has_string "${output_message}" "/clusterroles?limit=500 200 OK" ### Test kubectl get chunk size does not result in a --watch error when resource list is served in multiple chunks - # Pre-condition: no ConfigMaps exist - kube::test::get_object_assert configmap "{{range.items}}{{$id_field}}:{{end}}" '' + # Pre-condition: ConfigMap one two tree does not exist + kube::test::get_object_assert 'configmaps' '{{range.items}}{{ if eq $id_field \"one\" }}found{{end}}{{end}}:' ':' + kube::test::get_object_assert 'configmaps' '{{range.items}}{{ if eq $id_field \"two\" }}found{{end}}{{end}}:' ':' + kube::test::get_object_assert 'configmaps' '{{range.items}}{{ if eq $id_field \"three\" }}found{{end}}{{end}}:' ':' + # Post-condition: Create three configmaps and ensure that we can --watch them with a --chunk-size of 1 kubectl create cm one "${kube_flags[@]}" kubectl create cm two "${kube_flags[@]}" diff --git a/test/e2e/scheduling/resource_quota.go b/test/e2e/scheduling/resource_quota.go index 40f5b16f3cd..3c33caa3909 100644 --- a/test/e2e/scheduling/resource_quota.go +++ b/test/e2e/scheduling/resource_quota.go @@ -365,6 +365,22 @@ var _ = SIGDescribe("ResourceQuota", func() { }) It("should create a ResourceQuota and capture the life of a configMap.", func() { + found, unchanged := 0, 0 + wait.Poll(1*time.Second, 30*time.Second, func() (bool, error) { + configmaps, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).List(metav1.ListOptions{}) + Expect(err).NotTo(HaveOccurred()) + if len(configmaps.Items) == found { + // loop until the number of configmaps has stabilized for 5 seconds + unchanged++ + return unchanged > 4, nil + } + unchanged = 0 + found = len(configmaps.Items) + return false, nil + }) + defaultConfigMaps := fmt.Sprintf("%d", found) + hardConfigMaps := fmt.Sprintf("%d", found+1) + By("Creating a ResourceQuota") quotaName := "test-quota" resourceQuota := newTestResourceQuota(quotaName) @@ -374,6 +390,7 @@ var _ = SIGDescribe("ResourceQuota", func() { By("Ensuring resource quota status is calculated") usedResources := v1.ResourceList{} usedResources[v1.ResourceQuotas] = resource.MustParse("1") + usedResources[v1.ResourceConfigMaps] = resource.MustParse(defaultConfigMaps) err = waitForResourceQuota(f.ClientSet, f.Namespace.Name, quotaName, usedResources) Expect(err).NotTo(HaveOccurred()) @@ -385,7 +402,10 @@ var _ = SIGDescribe("ResourceQuota", func() { By("Ensuring resource quota status captures configMap creation") usedResources = v1.ResourceList{} usedResources[v1.ResourceQuotas] = resource.MustParse("1") - usedResources[v1.ResourceConfigMaps] = resource.MustParse("1") + // we expect there to be two configmaps because each namespace will receive + // a ca.crt configmap by default. + // ref:https://github.com/kubernetes/kubernetes/pull/68812 + usedResources[v1.ResourceConfigMaps] = resource.MustParse(hardConfigMaps) err = waitForResourceQuota(f.ClientSet, f.Namespace.Name, quotaName, usedResources) Expect(err).NotTo(HaveOccurred()) @@ -394,7 +414,7 @@ var _ = SIGDescribe("ResourceQuota", func() { Expect(err).NotTo(HaveOccurred()) By("Ensuring resource quota status released usage") - usedResources[v1.ResourceConfigMaps] = resource.MustParse("0") + usedResources[v1.ResourceConfigMaps] = resource.MustParse(defaultConfigMaps) err = waitForResourceQuota(f.ClientSet, f.Namespace.Name, quotaName, usedResources) Expect(err).NotTo(HaveOccurred()) })