diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index ca4872113d6..b1380edf72d 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -22,7 +22,6 @@ package app import ( "crypto/tls" "fmt" - "io/ioutil" "net" "net/http" "net/url" @@ -314,27 +313,9 @@ func CreateKubeAPIServerConfig( } } - clientCA, err := readCAorNil(s.Authentication.ClientCert.ClientCA) - if err != nil { - return nil, nil, nil, nil, err - } - requestHeaderProxyCA, err := readCAorNil(s.Authentication.RequestHeader.ClientCAFile) - if err != nil { - return nil, nil, nil, nil, err - } - config := &master.Config{ GenericConfig: genericConfig, ExtraConfig: master.ExtraConfig{ - ClientCARegistrationHook: master.ClientCARegistrationHook{ - ClientCA: clientCA, - RequestHeaderUsernameHeaders: s.Authentication.RequestHeader.UsernameHeaders, - RequestHeaderGroupHeaders: s.Authentication.RequestHeader.GroupHeaders, - RequestHeaderExtraHeaderPrefixes: s.Authentication.RequestHeader.ExtraHeaderPrefixes, - RequestHeaderCA: requestHeaderProxyCA, - RequestHeaderAllowedNames: s.Authentication.RequestHeader.AllowedNames, - }, - APIResourceConfigSource: storageFactory.APIResourceConfigSource, StorageFactory: storageFactory, EventTTL: s.EventTTL, @@ -362,6 +343,25 @@ func CreateKubeAPIServerConfig( VersionedInformers: versionedInformers, }, } + + clientCAProvider, err := s.Authentication.ClientCert.GetClientCAContentProvider() + if err != nil { + return nil, nil, nil, nil, err + } + config.ExtraConfig.ClusterAuthenticationInfo.ClientCA = clientCAProvider + + requestHeaderConfig, err := s.Authentication.RequestHeader.ToAuthenticationRequestHeaderConfig() + if err != nil { + return nil, nil, nil, nil, err + } + if requestHeaderConfig != nil { + config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderCA = requestHeaderConfig.CAContentProvider + config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderAllowedNames = requestHeaderConfig.AllowedClientNames + config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderExtraHeaderPrefixes = requestHeaderConfig.ExtraHeaderPrefixes + config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderGroupHeaders = requestHeaderConfig.GroupHeaders + config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderUsernameHeaders = requestHeaderConfig.UsernameHeaders + } + if err := config.GenericConfig.AddPostStartHook("start-kube-apiserver-admission-initializer", admissionPostStartHook); err != nil { return nil, nil, nil, nil, err } @@ -709,10 +709,3 @@ func buildServiceResolver(enabledAggregatorRouting bool, hostname string, inform } return serviceResolver } - -func readCAorNil(file string) ([]byte, error) { - if len(file) == 0 { - return nil, nil - } - return ioutil.ReadFile(file) -} diff --git a/pkg/master/BUILD b/pkg/master/BUILD index c1087d15249..cd46f4bf077 100644 --- a/pkg/master/BUILD +++ b/pkg/master/BUILD @@ -3,7 +3,6 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", srcs = [ - "client_ca_hook.go", "client_util.go", "controller.go", "doc.go", @@ -41,6 +40,7 @@ go_library( "//pkg/features:go_default_library", "//pkg/kubeapiserver/options:go_default_library", "//pkg/kubelet/client:go_default_library", + "//pkg/master/controller/clusterauthenticationtrust:go_default_library", "//pkg/master/reconcilers:go_default_library", "//pkg/master/tunneler:go_default_library", "//pkg/registry/admissionregistration/rest:go_default_library", @@ -108,7 +108,6 @@ go_library( "//staging/src/k8s.io/api/storage/v1:go_default_library", "//staging/src/k8s.io/api/storage/v1alpha1:go_default_library", "//staging/src/k8s.io/api/storage/v1beta1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", @@ -118,11 +117,13 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/endpoints/discovery:go_default_library", "//staging/src/k8s.io/apiserver/pkg/registry/generic:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/healthz:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/storage:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/discovery/v1alpha1:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", @@ -138,7 +139,6 @@ go_test( size = "medium", timeout = "long", srcs = [ - "client_ca_hook_test.go", "controller_test.go", "import_known_versions_test.go", "master_openapi_test.go", @@ -163,9 +163,7 @@ go_test( "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/apitesting/naming:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", @@ -203,6 +201,7 @@ filegroup( name = "all-srcs", srcs = [ ":package-srcs", + "//pkg/master/controller/clusterauthenticationtrust:all-srcs", "//pkg/master/controller/crdregistration:all-srcs", "//pkg/master/ports:all-srcs", "//pkg/master/reconcilers:all-srcs", diff --git a/pkg/master/client_ca_hook.go b/pkg/master/client_ca_hook.go deleted file mode 100644 index 4d13b0a51f7..00000000000 --- a/pkg/master/client_ca_hook.go +++ /dev/null @@ -1,142 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package master - -import ( - "encoding/json" - "fmt" - "time" - - corev1 "k8s.io/api/core/v1" - apiequality "k8s.io/apimachinery/pkg/api/equality" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/apimachinery/pkg/util/wait" - genericapiserver "k8s.io/apiserver/pkg/server" - corev1client "k8s.io/client-go/kubernetes/typed/core/v1" -) - -// ClientCARegistrationHook defines CA registration hook request structure -type ClientCARegistrationHook struct { - ClientCA []byte - - RequestHeaderUsernameHeaders []string - RequestHeaderGroupHeaders []string - RequestHeaderExtraHeaderPrefixes []string - RequestHeaderCA []byte - RequestHeaderAllowedNames []string -} - -// PostStartHook initializes client CA configmap for the API server -func (h ClientCARegistrationHook) PostStartHook(hookContext genericapiserver.PostStartHookContext) error { - // initializing CAs is important so that aggregated API servers can come up with "normal" config. - // We've seen lagging etcd before, so we want to retry this a few times before we decide to crashloop - // the API server on it. - err := wait.Poll(1*time.Second, 30*time.Second, func() (done bool, err error) { - // retry building the config since sometimes the server can be in an in-between state which caused - // some kind of auto detection failure as I recall from other post start hooks. - // TODO see if this is still true and fix the RBAC one too if it isn't. - client, err := corev1client.NewForConfig(hookContext.LoopbackClientConfig) - if err != nil { - utilruntime.HandleError(err) - return false, nil - } - - return h.tryToWriteClientCAs(client) - }) - - // if we're never able to make it through initialization, kill the API server - if err != nil { - return fmt.Errorf("unable to initialize client CA configmap: %v", err) - } - - return nil -} - -// tryToWriteClientCAs is here for unit testing with a fake client. This is a wait.ConditionFunc so the bool -// indicates if the condition was met. True when its finished, false when it should retry. -func (h ClientCARegistrationHook) tryToWriteClientCAs(client corev1client.CoreV1Interface) (bool, error) { - if err := createNamespaceIfNeeded(client, metav1.NamespaceSystem); err != nil { - utilruntime.HandleError(err) - return false, nil - } - - data := map[string]string{} - if len(h.ClientCA) > 0 { - data["client-ca-file"] = string(h.ClientCA) - } - - if len(h.RequestHeaderCA) > 0 { - var err error - - // encoding errors aren't going to get better, so just fail on them. - data["requestheader-username-headers"], err = jsonSerializeStringSlice(h.RequestHeaderUsernameHeaders) - if err != nil { - return false, err - } - data["requestheader-group-headers"], err = jsonSerializeStringSlice(h.RequestHeaderGroupHeaders) - if err != nil { - return false, err - } - data["requestheader-extra-headers-prefix"], err = jsonSerializeStringSlice(h.RequestHeaderExtraHeaderPrefixes) - if err != nil { - return false, err - } - data["requestheader-client-ca-file"] = string(h.RequestHeaderCA) - data["requestheader-allowed-names"], err = jsonSerializeStringSlice(h.RequestHeaderAllowedNames) - if err != nil { - return false, err - } - } - - // write errors may work next time if we retry, so queue for retry - if err := writeConfigMap(client, "extension-apiserver-authentication", data); err != nil { - utilruntime.HandleError(err) - return false, nil - } - - return true, nil -} - -func jsonSerializeStringSlice(in []string) (string, error) { - out, err := json.Marshal(in) - if err != nil { - return "", err - } - return string(out), err -} - -func writeConfigMap(client corev1client.ConfigMapsGetter, name string, data map[string]string) error { - existing, err := client.ConfigMaps(metav1.NamespaceSystem).Get(name, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - _, err := client.ConfigMaps(metav1.NamespaceSystem).Create(&corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: name}, - Data: data, - }) - return err - } - if err != nil { - return err - } - - if !apiequality.Semantic.DeepEqual(existing.Data, data) { - existing.Data = data - _, err = client.ConfigMaps(metav1.NamespaceSystem).Update(existing) - } - return err -} diff --git a/pkg/master/client_ca_hook_test.go b/pkg/master/client_ca_hook_test.go deleted file mode 100644 index 69baf28266e..00000000000 --- a/pkg/master/client_ca_hook_test.go +++ /dev/null @@ -1,247 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package master - -import ( - "reflect" - "testing" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/diff" - "k8s.io/client-go/kubernetes/fake" - clienttesting "k8s.io/client-go/testing" -) - -func TestWriteClientCAs(t *testing.T) { - tests := []struct { - name string - hook ClientCARegistrationHook - preexistingObjs []runtime.Object - expectedConfigMaps map[string]*corev1.ConfigMap - expectUpdate bool - }{ - { - name: "basic", - hook: ClientCARegistrationHook{ - ClientCA: []byte("foo"), - RequestHeaderUsernameHeaders: []string{"alfa", "bravo", "charlie"}, - RequestHeaderGroupHeaders: []string{"delta"}, - RequestHeaderExtraHeaderPrefixes: []string{"echo", "foxtrot"}, - RequestHeaderCA: []byte("bar"), - RequestHeaderAllowedNames: []string{"first", "second"}, - }, - expectedConfigMaps: map[string]*corev1.ConfigMap{ - "extension-apiserver-authentication": { - ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"}, - Data: map[string]string{ - "client-ca-file": "foo", - "requestheader-username-headers": `["alfa","bravo","charlie"]`, - "requestheader-group-headers": `["delta"]`, - "requestheader-extra-headers-prefix": `["echo","foxtrot"]`, - "requestheader-client-ca-file": "bar", - "requestheader-allowed-names": `["first","second"]`, - }, - }, - }, - }, - { - name: "skip extension-apiserver-authentication", - hook: ClientCARegistrationHook{ - RequestHeaderCA: []byte("bar"), - RequestHeaderAllowedNames: []string{"first", "second"}, - }, - expectedConfigMaps: map[string]*corev1.ConfigMap{ - "extension-apiserver-authentication": { - ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"}, - Data: map[string]string{ - "requestheader-username-headers": `null`, - "requestheader-group-headers": `null`, - "requestheader-extra-headers-prefix": `null`, - "requestheader-client-ca-file": "bar", - "requestheader-allowed-names": `["first","second"]`, - }, - }, - }, - }, - { - name: "skip extension-apiserver-authentication", - hook: ClientCARegistrationHook{ - ClientCA: []byte("foo"), - }, - expectedConfigMaps: map[string]*corev1.ConfigMap{ - "extension-apiserver-authentication": { - ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"}, - Data: map[string]string{ - "client-ca-file": "foo", - }, - }, - }, - }, - { - name: "empty allowed names", - hook: ClientCARegistrationHook{ - RequestHeaderCA: []byte("bar"), - }, - expectedConfigMaps: map[string]*corev1.ConfigMap{ - "extension-apiserver-authentication": { - ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"}, - Data: map[string]string{ - "requestheader-username-headers": `null`, - "requestheader-group-headers": `null`, - "requestheader-extra-headers-prefix": `null`, - "requestheader-client-ca-file": "bar", - "requestheader-allowed-names": `null`, - }, - }, - }, - }, - { - name: "overwrite extension-apiserver-authentication", - hook: ClientCARegistrationHook{ - ClientCA: []byte("foo"), - }, - preexistingObjs: []runtime.Object{ - &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"}, - Data: map[string]string{ - "client-ca-file": "other", - }, - }, - }, - expectedConfigMaps: map[string]*corev1.ConfigMap{ - "extension-apiserver-authentication": { - ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"}, - Data: map[string]string{ - "client-ca-file": "foo", - }, - }, - }, - expectUpdate: true, - }, - { - name: "overwrite extension-apiserver-authentication requestheader", - hook: ClientCARegistrationHook{ - RequestHeaderUsernameHeaders: []string{}, - RequestHeaderGroupHeaders: []string{}, - RequestHeaderExtraHeaderPrefixes: []string{}, - RequestHeaderCA: []byte("bar"), - RequestHeaderAllowedNames: []string{}, - }, - preexistingObjs: []runtime.Object{ - &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"}, - Data: map[string]string{ - "requestheader-username-headers": `null`, - "requestheader-group-headers": `null`, - "requestheader-extra-headers-prefix": `null`, - "requestheader-client-ca-file": "something", - "requestheader-allowed-names": `null`, - }, - }, - }, - expectedConfigMaps: map[string]*corev1.ConfigMap{ - "extension-apiserver-authentication": { - ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"}, - Data: map[string]string{ - "requestheader-username-headers": `[]`, - "requestheader-group-headers": `[]`, - "requestheader-extra-headers-prefix": `[]`, - "requestheader-client-ca-file": "bar", - "requestheader-allowed-names": `[]`, - }, - }, - }, - expectUpdate: true, - }, - { - name: "namespace exists", - hook: ClientCARegistrationHook{ - ClientCA: []byte("foo"), - }, - preexistingObjs: []runtime.Object{ - &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: metav1.NamespaceSystem}}, - }, - expectedConfigMaps: map[string]*corev1.ConfigMap{ - "extension-apiserver-authentication": { - ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"}, - Data: map[string]string{ - "client-ca-file": "foo", - }, - }, - }, - }, - { - name: "skip on no change", - hook: ClientCARegistrationHook{ - RequestHeaderUsernameHeaders: []string{}, - RequestHeaderGroupHeaders: []string{}, - RequestHeaderExtraHeaderPrefixes: []string{}, - RequestHeaderCA: []byte("bar"), - RequestHeaderAllowedNames: []string{}, - }, - preexistingObjs: []runtime.Object{ - &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"}, - Data: map[string]string{ - "requestheader-username-headers": `[]`, - "requestheader-group-headers": `[]`, - "requestheader-extra-headers-prefix": `[]`, - "requestheader-client-ca-file": "bar", - "requestheader-allowed-names": `[]`, - }, - }, - }, - expectedConfigMaps: map[string]*corev1.ConfigMap{}, - expectUpdate: false, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - client := fake.NewSimpleClientset(test.preexistingObjs...) - test.hook.tryToWriteClientCAs(client.CoreV1()) - - actualConfigMaps, updated := getFinalConfigMaps(client) - if !reflect.DeepEqual(test.expectedConfigMaps, actualConfigMaps) { - t.Fatalf("%s: %v", test.name, diff.ObjectReflectDiff(test.expectedConfigMaps, actualConfigMaps)) - } - if test.expectUpdate != updated { - t.Fatalf("%s: expected %v, got %v", test.name, test.expectUpdate, updated) - } - }) - } -} - -func getFinalConfigMaps(client *fake.Clientset) (map[string]*corev1.ConfigMap, bool) { - ret := map[string]*corev1.ConfigMap{} - updated := false - - for _, action := range client.Actions() { - if action.Matches("create", "configmaps") { - obj := action.(clienttesting.CreateAction).GetObject().(*corev1.ConfigMap) - ret[obj.Name] = obj - } - if action.Matches("update", "configmaps") { - updated = true - obj := action.(clienttesting.UpdateAction).GetObject().(*corev1.ConfigMap) - ret[obj.Name] = obj - } - } - return ret, updated -} diff --git a/pkg/master/controller/clusterauthenticationtrust/BUILD b/pkg/master/controller/clusterauthenticationtrust/BUILD new file mode 100644 index 00000000000..91324d2a235 --- /dev/null +++ b/pkg/master/controller/clusterauthenticationtrust/BUILD @@ -0,0 +1,64 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["cluster_authentication_trust_controller.go"], + importpath = "k8s.io/kubernetes/pkg/master/controller/clusterauthenticationtrust", + visibility = ["//visibility:public"], + deps = [ + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/authentication/request/headerrequest:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:go_default_library", + "//staging/src/k8s.io/client-go/informers/core/v1:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", + "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", + "//staging/src/k8s.io/client-go/tools/cache:go_default_library", + "//staging/src/k8s.io/client-go/util/cert:go_default_library", + "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["cluster_authentication_trust_controller_test.go"], + embed = [":go_default_library"], + deps = [ + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/authentication/request/headerrequest:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", + "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", + "//staging/src/k8s.io/client-go/testing:go_default_library", + "//staging/src/k8s.io/client-go/tools/cache:go_default_library", + "//vendor/github.com/davecgh/go-spew/spew:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/pkg/master/controller/clusterauthenticationtrust/cluster_authentication_trust_controller.go b/pkg/master/controller/clusterauthenticationtrust/cluster_authentication_trust_controller.go new file mode 100644 index 00000000000..3d0cf616270 --- /dev/null +++ b/pkg/master/controller/clusterauthenticationtrust/cluster_authentication_trust_controller.go @@ -0,0 +1,516 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package clusterauthenticationtrust + +import ( + "bytes" + "crypto/x509" + "encoding/json" + "encoding/pem" + "fmt" + "reflect" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" + "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/authentication/request/headerrequest" + "k8s.io/apiserver/pkg/server/dynamiccertificates" + corev1informers "k8s.io/client-go/informers/core/v1" + "k8s.io/client-go/kubernetes" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + corev1listers "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/cert" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog" +) + +const ( + configMapNamespace = "kube-system" + configMapName = "extension-apiserver-authentication" +) + +// Controller holds the running state for the controller +type Controller struct { + requiredAuthenticationData ClusterAuthenticationInfo + + configMapLister corev1listers.ConfigMapLister + configMapClient corev1client.ConfigMapsGetter + namespaceClient corev1client.NamespacesGetter + + // queue is where incoming work is placed to de-dup and to allow "easy" rate limited requeues on errors. + // we only ever place one entry in here, but it is keyed as usual: namespace/name + queue workqueue.RateLimitingInterface + + // kubeSystemConfigMapInformer is tracked so that we can start these on Run + kubeSystemConfigMapInformer cache.SharedIndexInformer + + // preRunCaches are the caches to sync before starting the work of this control loop + preRunCaches []cache.InformerSynced +} + +// ClusterAuthenticationInfo holds the information that will included in public configmap. +type ClusterAuthenticationInfo struct { + // ClientCA is the CA that can be used to verify the identity of normal clients + ClientCA dynamiccertificates.CAContentProvider + + // RequestHeaderUsernameHeaders are the headers used by this kube-apiserver to determine username + RequestHeaderUsernameHeaders headerrequest.StringSliceProvider + // RequestHeaderGroupHeaders are the headers used by this kube-apiserver to determine groups + RequestHeaderGroupHeaders headerrequest.StringSliceProvider + // RequestHeaderExtraHeaderPrefixes are the headers used by this kube-apiserver to determine user.extra + RequestHeaderExtraHeaderPrefixes headerrequest.StringSliceProvider + // RequestHeaderAllowedNames are the sujbects allowed to act as a front proxy + RequestHeaderAllowedNames headerrequest.StringSliceProvider + // RequestHeaderCA is the CA that can be used to verify the front proxy + RequestHeaderCA dynamiccertificates.CAContentProvider +} + +// NewClusterAuthenticationTrustController returns a controller that will maintain the kube-system configmap/extension-apiserver-authentication +// that holds information about how to aggregated apiservers are recommended (but not required) to configure themselves. +func NewClusterAuthenticationTrustController(requiredAuthenticationData ClusterAuthenticationInfo, kubeClient kubernetes.Interface) *Controller { + // we construct our own informer because we need such a small subset of the information available. Just one namespace. + kubeSystemConfigMapInformer := corev1informers.NewConfigMapInformer(kubeClient, configMapNamespace, 12*time.Hour, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + + c := &Controller{ + requiredAuthenticationData: requiredAuthenticationData, + configMapLister: corev1listers.NewConfigMapLister(kubeSystemConfigMapInformer.GetIndexer()), + configMapClient: kubeClient.CoreV1(), + namespaceClient: kubeClient.CoreV1(), + queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "cluster_authentication_trust_controller"), + preRunCaches: []cache.InformerSynced{kubeSystemConfigMapInformer.HasSynced}, + kubeSystemConfigMapInformer: kubeSystemConfigMapInformer, + } + + kubeSystemConfigMapInformer.AddEventHandler(cache.FilteringResourceEventHandler{ + FilterFunc: func(obj interface{}) bool { + if cast, ok := obj.(*corev1.ConfigMap); ok { + return cast.Name == configMapName + } + if tombstone, ok := obj.(cache.DeletedFinalStateUnknown); ok { + if cast, ok := tombstone.Obj.(*apiextensions.CustomResourceDefinition); ok { + return cast.Name == configMapName + } + } + return true // always return true just in case. The checks are fairly cheap + }, + Handler: cache.ResourceEventHandlerFuncs{ + // we have a filter, so any time we're called, we may as well queue. We only ever check one configmap + // so we don't have to be choosy about our key. + AddFunc: func(obj interface{}) { + c.queue.Add(keyFn()) + }, + UpdateFunc: func(oldObj, newObj interface{}) { + c.queue.Add(keyFn()) + }, + DeleteFunc: func(obj interface{}) { + c.queue.Add(keyFn()) + }, + }, + }) + + return c +} + +func (c *Controller) syncConfigMap() error { + originalAuthConfigMap, err := c.configMapLister.ConfigMaps(configMapNamespace).Get(configMapName) + if apierrors.IsNotFound(err) { + originalAuthConfigMap = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Namespace: configMapNamespace, Name: configMapName}, + } + } else if err != nil { + return err + } + // keep the original to diff against later before updating + authConfigMap := originalAuthConfigMap.DeepCopy() + + existingAuthenticationInfo, err := getClusterAuthenticationInfoFor(originalAuthConfigMap.Data) + if err != nil { + return err + } + combinedInfo, err := combinedClusterAuthenticationInfo(existingAuthenticationInfo, c.requiredAuthenticationData) + if err != nil { + return err + } + authConfigMap.Data, err = getConfigMapDataFor(combinedInfo) + if err != nil { + return err + } + + if equality.Semantic.DeepEqual(authConfigMap, originalAuthConfigMap) { + klog.V(5).Info("no changes to configmap") + return nil + } + klog.V(2).Infof("writing updated authentication info to %s configmaps/%s", configMapNamespace, configMapName) + + if err := createNamespaceIfNeeded(c.namespaceClient, authConfigMap.Namespace); err != nil { + return err + } + if err := writeConfigMap(c.configMapClient, authConfigMap); err != nil { + return err + } + + return nil +} + +func createNamespaceIfNeeded(nsClient corev1client.NamespacesGetter, ns string) error { + if _, err := nsClient.Namespaces().Get(ns, metav1.GetOptions{}); err == nil { + // the namespace already exists + return nil + } + newNs := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: ns, + Namespace: "", + }, + } + _, err := nsClient.Namespaces().Create(newNs) + if err != nil && apierrors.IsAlreadyExists(err) { + err = nil + } + return err +} + +func writeConfigMap(configMapClient corev1client.ConfigMapsGetter, required *corev1.ConfigMap) error { + _, err := configMapClient.ConfigMaps(required.Namespace).Update(required) + if apierrors.IsNotFound(err) { + _, err := configMapClient.ConfigMaps(required.Namespace).Create(required) + return err + } + + // If the configmap is too big, clear the entire thing and count on this controller (or another one) to add the correct data back. + // We return the original error which causes the controller to re-queue. + // Too big means + // 1. request is so big the generic request catcher finds it + // 2. the content is so large that that the server sends a validation error "Too long: must have at most 1048576 characters" + if apierrors.IsRequestEntityTooLargeError(err) || (apierrors.IsInvalid(err) && strings.Contains(err.Error(), "Too long")) { + if deleteErr := configMapClient.ConfigMaps(required.Namespace).Delete(required.Name, nil); deleteErr != nil { + return deleteErr + } + return err + } + + return err +} + +// combinedClusterAuthenticationInfo combines two sets of authentication information into a new one +func combinedClusterAuthenticationInfo(lhs, rhs ClusterAuthenticationInfo) (ClusterAuthenticationInfo, error) { + ret := ClusterAuthenticationInfo{ + RequestHeaderAllowedNames: combineUniqueStringSlices(lhs.RequestHeaderAllowedNames, rhs.RequestHeaderAllowedNames), + RequestHeaderExtraHeaderPrefixes: combineUniqueStringSlices(lhs.RequestHeaderExtraHeaderPrefixes, rhs.RequestHeaderExtraHeaderPrefixes), + RequestHeaderGroupHeaders: combineUniqueStringSlices(lhs.RequestHeaderGroupHeaders, rhs.RequestHeaderGroupHeaders), + RequestHeaderUsernameHeaders: combineUniqueStringSlices(lhs.RequestHeaderUsernameHeaders, rhs.RequestHeaderUsernameHeaders), + } + + var err error + ret.ClientCA, err = combineCertLists(lhs.ClientCA, rhs.ClientCA) + if err != nil { + return ClusterAuthenticationInfo{}, err + } + ret.RequestHeaderCA, err = combineCertLists(lhs.RequestHeaderCA, rhs.RequestHeaderCA) + if err != nil { + return ClusterAuthenticationInfo{}, err + } + + return ret, nil +} + +func getConfigMapDataFor(authenticationInfo ClusterAuthenticationInfo) (map[string]string, error) { + data := map[string]string{} + if authenticationInfo.ClientCA != nil { + if caBytes := authenticationInfo.ClientCA.CurrentCABundleContent(); len(caBytes) > 0 { + data["client-ca-file"] = string(caBytes) + } + } + + if authenticationInfo.RequestHeaderCA == nil { + return data, nil + } + + if caBytes := authenticationInfo.RequestHeaderCA.CurrentCABundleContent(); len(caBytes) > 0 { + var err error + + // encoding errors aren't going to get better, so just fail on them. + data["requestheader-username-headers"], err = jsonSerializeStringSlice(authenticationInfo.RequestHeaderUsernameHeaders.Value()) + if err != nil { + return nil, err + } + data["requestheader-group-headers"], err = jsonSerializeStringSlice(authenticationInfo.RequestHeaderGroupHeaders.Value()) + if err != nil { + return nil, err + } + data["requestheader-extra-headers-prefix"], err = jsonSerializeStringSlice(authenticationInfo.RequestHeaderExtraHeaderPrefixes.Value()) + if err != nil { + return nil, err + } + + data["requestheader-client-ca-file"] = string(caBytes) + data["requestheader-allowed-names"], err = jsonSerializeStringSlice(authenticationInfo.RequestHeaderAllowedNames.Value()) + if err != nil { + return nil, err + } + } + + return data, nil +} + +func getClusterAuthenticationInfoFor(data map[string]string) (ClusterAuthenticationInfo, error) { + ret := ClusterAuthenticationInfo{} + + var err error + ret.RequestHeaderGroupHeaders, err = jsonDeserializeStringSlice(data["requestheader-group-headers"]) + if err != nil { + return ClusterAuthenticationInfo{}, err + } + ret.RequestHeaderExtraHeaderPrefixes, err = jsonDeserializeStringSlice(data["requestheader-extra-headers-prefix"]) + if err != nil { + return ClusterAuthenticationInfo{}, err + } + ret.RequestHeaderAllowedNames, err = jsonDeserializeStringSlice(data["requestheader-allowed-names"]) + if err != nil { + return ClusterAuthenticationInfo{}, err + } + ret.RequestHeaderUsernameHeaders, err = jsonDeserializeStringSlice(data["requestheader-username-headers"]) + if err != nil { + return ClusterAuthenticationInfo{}, err + } + + if caBundle := data["requestheader-client-ca-file"]; len(caBundle) > 0 { + ret.RequestHeaderCA, err = dynamiccertificates.NewStaticCAContent("existing", []byte(caBundle)) + if err != nil { + return ClusterAuthenticationInfo{}, err + } + } + + if caBundle := data["client-ca-file"]; len(caBundle) > 0 { + ret.ClientCA, err = dynamiccertificates.NewStaticCAContent("existing", []byte(caBundle)) + if err != nil { + return ClusterAuthenticationInfo{}, err + } + } + + return ret, nil +} + +func jsonSerializeStringSlice(in []string) (string, error) { + out, err := json.Marshal(in) + if err != nil { + return "", err + } + return string(out), err +} + +func jsonDeserializeStringSlice(in string) (headerrequest.StringSliceProvider, error) { + if len(in) == 0 { + return nil, nil + } + + out := []string{} + if err := json.Unmarshal([]byte(in), &out); err != nil { + return nil, err + } + return headerrequest.StaticStringSlice(out), nil +} + +func combineUniqueStringSlices(lhs, rhs headerrequest.StringSliceProvider) headerrequest.StringSliceProvider { + ret := []string{} + present := sets.String{} + + if lhs != nil { + for _, curr := range lhs.Value() { + if present.Has(curr) { + continue + } + ret = append(ret, curr) + present.Insert(curr) + } + } + + if rhs != nil { + for _, curr := range rhs.Value() { + if present.Has(curr) { + continue + } + ret = append(ret, curr) + present.Insert(curr) + } + } + + return headerrequest.StaticStringSlice(ret) +} + +func combineCertLists(lhs, rhs dynamiccertificates.CAContentProvider) (dynamiccertificates.CAContentProvider, error) { + certificates := []*x509.Certificate{} + + if lhs != nil { + lhsCABytes := lhs.CurrentCABundleContent() + lhsCAs, err := cert.ParseCertsPEM(lhsCABytes) + if err != nil { + return nil, err + } + certificates = append(certificates, lhsCAs...) + } + if rhs != nil { + rhsCABytes := rhs.CurrentCABundleContent() + rhsCAs, err := cert.ParseCertsPEM(rhsCABytes) + if err != nil { + return nil, err + } + certificates = append(certificates, rhsCAs...) + } + + certificates = filterExpiredCerts(certificates...) + + finalCertificates := []*x509.Certificate{} + // now check for duplicates. n^2, but super simple + for i := range certificates { + found := false + for j := range finalCertificates { + if reflect.DeepEqual(certificates[i].Raw, finalCertificates[j].Raw) { + found = true + break + } + } + if !found { + finalCertificates = append(finalCertificates, certificates[i]) + } + } + + finalCABytes, err := encodeCertificates(finalCertificates...) + if err != nil { + return nil, err + } + + if len(finalCABytes) == 0 { + return nil, nil + } + // it makes sense for this list to be static because the combination of sources is only used just before writing and + // is recalculated + return dynamiccertificates.NewStaticCAContent("combined", finalCABytes) +} + +// filterExpiredCerts checks are all certificates in the bundle valid, i.e. they have not expired. +// The function returns new bundle with only valid certificates or error if no valid certificate is found. +// We allow five minutes of slack for NotAfter comparisons +func filterExpiredCerts(certs ...*x509.Certificate) []*x509.Certificate { + fiveMinutesAgo := time.Now().Add(-5 * time.Minute) + + var validCerts []*x509.Certificate + for _, c := range certs { + if c.NotAfter.After(fiveMinutesAgo) { + validCerts = append(validCerts, c) + } + } + + return validCerts +} + +// Enqueue a method to allow separate control loops to cause the controller to trigger and reconcile content. +func (c *Controller) Enqueue() { + c.queue.Add(keyFn()) +} + +// Run the controller until stopped. +func (c *Controller) Run(threadiness int, stopCh <-chan struct{}) { + defer utilruntime.HandleCrash() + // make sure the work queue is shutdown which will trigger workers to end + defer c.queue.ShutDown() + + klog.Infof("Starting cluster_authentication_trust_controller controller") + defer klog.Infof("Shutting down cluster_authentication_trust_controller controller") + + // we have a personal informer that is narrowly scoped, start it. + go c.kubeSystemConfigMapInformer.Run(stopCh) + + // wait for your secondary caches to fill before starting your work + if !cache.WaitForNamedCacheSync("cluster_authentication_trust_controller", stopCh, c.preRunCaches...) { + return + } + + // only run one worker + go wait.Until(c.runWorker, time.Second, stopCh) + + // checks are cheap. run once a minute just to be sure we stay in sync in case fsnotify fails again + // start timer that rechecks every minute, just in case. this also serves to prime the controller quickly. + _ = wait.PollImmediateUntil(1*time.Minute, func() (bool, error) { + c.queue.Add(keyFn()) + return false, nil + }, stopCh) + + // wait until we're told to stop + <-stopCh +} + +func (c *Controller) runWorker() { + // hot loop until we're told to stop. processNextWorkItem will automatically wait until there's work + // available, so we don't worry about secondary waits + for c.processNextWorkItem() { + } +} + +// processNextWorkItem deals with one key off the queue. It returns false when it's time to quit. +func (c *Controller) processNextWorkItem() bool { + // pull the next work item from queue. It should be a key we use to lookup something in a cache + key, quit := c.queue.Get() + if quit { + return false + } + // you always have to indicate to the queue that you've completed a piece of work + defer c.queue.Done(key) + + // do your work on the key. This method will contains your "do stuff" logic + err := c.syncConfigMap() + if err == nil { + // if you had no error, tell the queue to stop tracking history for your key. This will + // reset things like failure counts for per-item rate limiting + c.queue.Forget(key) + return true + } + + // there was a failure so be sure to report it. This method allows for pluggable error handling + // which can be used for things like cluster-monitoring + utilruntime.HandleError(fmt.Errorf("%v failed with : %v", key, err)) + // since we failed, we should requeue the item to work on later. This method will add a backoff + // to avoid hotlooping on particular items (they're probably still not going to work right away) + // and overall controller protection (everything I've done is broken, this controller needs to + // calm down or it can starve other useful work) cases. + c.queue.AddRateLimited(key) + + return true +} + +func keyFn() string { + // this format matches DeletionHandlingMetaNamespaceKeyFunc for our single key + return configMapNamespace + "/" + configMapName +} + +func encodeCertificates(certs ...*x509.Certificate) ([]byte, error) { + b := bytes.Buffer{} + for _, cert := range certs { + if err := pem.Encode(&b, &pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw}); err != nil { + return []byte{}, err + } + } + return b.Bytes(), nil +} diff --git a/pkg/master/controller/clusterauthenticationtrust/cluster_authentication_trust_controller_test.go b/pkg/master/controller/clusterauthenticationtrust/cluster_authentication_trust_controller_test.go new file mode 100644 index 00000000000..50b56816849 --- /dev/null +++ b/pkg/master/controller/clusterauthenticationtrust/cluster_authentication_trust_controller_test.go @@ -0,0 +1,386 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package clusterauthenticationtrust + +import ( + "reflect" + "testing" + + "github.com/davecgh/go-spew/spew" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/diff" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/apiserver/pkg/authentication/request/headerrequest" + "k8s.io/apiserver/pkg/server/dynamiccertificates" + "k8s.io/client-go/kubernetes/fake" + corev1listers "k8s.io/client-go/listers/core/v1" + clienttesting "k8s.io/client-go/testing" + "k8s.io/client-go/tools/cache" +) + +var ( + someRandomCA = []byte(`-----BEGIN CERTIFICATE----- +MIIBqDCCAU2gAwIBAgIUfbqeieihh/oERbfvRm38XvS/xHAwCgYIKoZIzj0EAwIw +GjEYMBYGA1UEAxMPSW50ZXJtZWRpYXRlLUNBMCAXDTE2MTAxMTA1MDYwMFoYDzIx +MTYwOTE3MDUwNjAwWjAUMRIwEAYDVQQDEwlNeSBDbGllbnQwWTATBgcqhkjOPQIB +BggqhkjOPQMBBwNCAARv6N4R/sjMR65iMFGNLN1GC/vd7WhDW6J4X/iAjkRLLnNb +KbRG/AtOUZ+7upJ3BWIRKYbOabbQGQe2BbKFiap4o3UwczAOBgNVHQ8BAf8EBAMC +BaAwEwYDVR0lBAwwCgYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU +K/pZOWpNcYai6eHFpmJEeFpeQlEwHwYDVR0jBBgwFoAUX6nQlxjfWnP6aM1meO/Q +a6b3a9kwCgYIKoZIzj0EAwIDSQAwRgIhAIWTKw/sjJITqeuNzJDAKU4xo1zL+xJ5 +MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps= +-----END CERTIFICATE----- +`) + anotherRandomCA = []byte(`-----BEGIN CERTIFICATE----- +MIIDQDCCAiigAwIBAgIJANWw74P5KJk2MA0GCSqGSIb3DQEBCwUAMDQxMjAwBgNV +BAMMKWdlbmVyaWNfd2ViaG9va19hZG1pc3Npb25fcGx1Z2luX3Rlc3RzX2NhMCAX +DTE3MTExNjAwMDUzOVoYDzIyOTEwOTAxMDAwNTM5WjAjMSEwHwYDVQQDExh3ZWJo +b29rLXRlc3QuZGVmYXVsdC5zdmMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQDXd/nQ89a5H8ifEsigmMd01Ib6NVR3bkJjtkvYnTbdfYEBj7UzqOQtHoLa +dIVmefny5uIHvj93WD8WDVPB3jX2JHrXkDTXd/6o6jIXHcsUfFTVLp6/bZ+Anqe0 +r/7hAPkzA2A7APyTWM3ZbEeo1afXogXhOJ1u/wz0DflgcB21gNho4kKTONXO3NHD +XLpspFqSkxfEfKVDJaYAoMnYZJtFNsa2OvsmLnhYF8bjeT3i07lfwrhUZvP+7Gsp +7UgUwc06WuNHjfx1s5e6ySzH0QioMD1rjYneqOvk0pKrMIhuAEWXqq7jlXcDtx1E +j+wnYbVqqVYheHZ8BCJoVAAQGs9/AgMBAAGjZDBiMAkGA1UdEwQCMAAwCwYDVR0P +BAQDAgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATApBgNVHREEIjAg +hwR/AAABghh3ZWJob29rLXRlc3QuZGVmYXVsdC5zdmMwDQYJKoZIhvcNAQELBQAD +ggEBAD/GKSPNyQuAOw/jsYZesb+RMedbkzs18sSwlxAJQMUrrXwlVdHrA8q5WhE6 +ABLqU1b8lQ8AWun07R8k5tqTmNvCARrAPRUqls/ryER+3Y9YEcxEaTc3jKNZFLbc +T6YtcnkdhxsiO136wtiuatpYL91RgCmuSpR8+7jEHhuFU01iaASu7ypFrUzrKHTF +bKwiLRQi1cMzVcLErq5CDEKiKhUkoDucyARFszrGt9vNIl/YCcBOkcNvM3c05Hn3 +M++C29JwS3Hwbubg6WO3wjFjoEhpCwU6qRYUz3MRp4tHO4kxKXx+oQnUiFnR7vW0 +YkNtGc1RUDHwecCTFpJtPb7Yu/E= +-----END CERTIFICATE----- +`) + + someRandomCAProvider dynamiccertificates.CAContentProvider + anotherRandomCAProvider dynamiccertificates.CAContentProvider +) + +func init() { + var err error + someRandomCAProvider, err = dynamiccertificates.NewStaticCAContent("foo", someRandomCA) + if err != nil { + panic(err) + } + anotherRandomCAProvider, err = dynamiccertificates.NewStaticCAContent("bar", anotherRandomCA) + if err != nil { + panic(err) + } +} + +func TestWriteClientCAs(t *testing.T) { + tests := []struct { + name string + clusterAuthInfo ClusterAuthenticationInfo + preexistingObjs []runtime.Object + expectedConfigMaps map[string]*corev1.ConfigMap + expectCreate bool + }{ + { + name: "basic", + clusterAuthInfo: ClusterAuthenticationInfo{ + ClientCA: someRandomCAProvider, + RequestHeaderUsernameHeaders: headerrequest.StaticStringSlice{"alfa", "bravo", "charlie"}, + RequestHeaderGroupHeaders: headerrequest.StaticStringSlice{"delta"}, + RequestHeaderExtraHeaderPrefixes: headerrequest.StaticStringSlice{"echo", "foxtrot"}, + RequestHeaderCA: anotherRandomCAProvider, + RequestHeaderAllowedNames: headerrequest.StaticStringSlice{"first", "second"}, + }, + expectedConfigMaps: map[string]*corev1.ConfigMap{ + "extension-apiserver-authentication": { + ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"}, + Data: map[string]string{ + "client-ca-file": string(someRandomCA), + "requestheader-username-headers": `["alfa","bravo","charlie"]`, + "requestheader-group-headers": `["delta"]`, + "requestheader-extra-headers-prefix": `["echo","foxtrot"]`, + "requestheader-client-ca-file": string(anotherRandomCA), + "requestheader-allowed-names": `["first","second"]`, + }, + }, + }, + expectCreate: true, + }, + { + name: "skip extension-apiserver-authentication", + clusterAuthInfo: ClusterAuthenticationInfo{ + RequestHeaderCA: anotherRandomCAProvider, + RequestHeaderAllowedNames: headerrequest.StaticStringSlice{"first", "second"}, + }, + expectedConfigMaps: map[string]*corev1.ConfigMap{ + "extension-apiserver-authentication": { + ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"}, + Data: map[string]string{ + "requestheader-username-headers": `[]`, + "requestheader-group-headers": `[]`, + "requestheader-extra-headers-prefix": `[]`, + "requestheader-client-ca-file": string(anotherRandomCA), + "requestheader-allowed-names": `["first","second"]`, + }, + }, + }, + expectCreate: true, + }, + { + name: "skip extension-apiserver-authentication", + clusterAuthInfo: ClusterAuthenticationInfo{ + ClientCA: someRandomCAProvider, + }, + expectedConfigMaps: map[string]*corev1.ConfigMap{ + "extension-apiserver-authentication": { + ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"}, + Data: map[string]string{ + "client-ca-file": string(someRandomCA), + }, + }, + }, + expectCreate: true, + }, + { + name: "empty allowed names", + clusterAuthInfo: ClusterAuthenticationInfo{ + RequestHeaderCA: anotherRandomCAProvider, + }, + expectedConfigMaps: map[string]*corev1.ConfigMap{ + "extension-apiserver-authentication": { + ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"}, + Data: map[string]string{ + "requestheader-username-headers": `[]`, + "requestheader-group-headers": `[]`, + "requestheader-extra-headers-prefix": `[]`, + "requestheader-client-ca-file": string(anotherRandomCA), + "requestheader-allowed-names": `[]`, + }, + }, + }, + expectCreate: true, + }, + { + name: "overwrite extension-apiserver-authentication", + clusterAuthInfo: ClusterAuthenticationInfo{ + ClientCA: someRandomCAProvider, + }, + preexistingObjs: []runtime.Object{ + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"}, + Data: map[string]string{ + "client-ca-file": string(anotherRandomCA), + }, + }, + }, + expectedConfigMaps: map[string]*corev1.ConfigMap{ + "extension-apiserver-authentication": { + ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"}, + Data: map[string]string{ + "client-ca-file": string(anotherRandomCA) + string(someRandomCA), + }, + }, + }, + }, + { + name: "overwrite extension-apiserver-authentication requestheader", + clusterAuthInfo: ClusterAuthenticationInfo{ + RequestHeaderUsernameHeaders: headerrequest.StaticStringSlice{}, + RequestHeaderGroupHeaders: headerrequest.StaticStringSlice{}, + RequestHeaderExtraHeaderPrefixes: headerrequest.StaticStringSlice{}, + RequestHeaderCA: anotherRandomCAProvider, + RequestHeaderAllowedNames: headerrequest.StaticStringSlice{}, + }, + preexistingObjs: []runtime.Object{ + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"}, + Data: map[string]string{ + "requestheader-username-headers": `[]`, + "requestheader-group-headers": `[]`, + "requestheader-extra-headers-prefix": `[]`, + "requestheader-client-ca-file": string(someRandomCA), + "requestheader-allowed-names": `[]`, + }, + }, + }, + expectedConfigMaps: map[string]*corev1.ConfigMap{ + "extension-apiserver-authentication": { + ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"}, + Data: map[string]string{ + "requestheader-username-headers": `[]`, + "requestheader-group-headers": `[]`, + "requestheader-extra-headers-prefix": `[]`, + "requestheader-client-ca-file": string(someRandomCA) + string(anotherRandomCA), + "requestheader-allowed-names": `[]`, + }, + }, + }, + }, + { + name: "namespace exists", + clusterAuthInfo: ClusterAuthenticationInfo{ + ClientCA: someRandomCAProvider, + }, + preexistingObjs: []runtime.Object{ + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: metav1.NamespaceSystem}}, + }, + expectedConfigMaps: map[string]*corev1.ConfigMap{ + "extension-apiserver-authentication": { + ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"}, + Data: map[string]string{ + "client-ca-file": string(someRandomCA), + }, + }, + }, + expectCreate: true, + }, + { + name: "skip on no change", + clusterAuthInfo: ClusterAuthenticationInfo{ + RequestHeaderUsernameHeaders: headerrequest.StaticStringSlice{}, + RequestHeaderGroupHeaders: headerrequest.StaticStringSlice{}, + RequestHeaderExtraHeaderPrefixes: headerrequest.StaticStringSlice{}, + RequestHeaderCA: anotherRandomCAProvider, + RequestHeaderAllowedNames: headerrequest.StaticStringSlice{}, + }, + preexistingObjs: []runtime.Object{ + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"}, + Data: map[string]string{ + "requestheader-username-headers": `[]`, + "requestheader-group-headers": `[]`, + "requestheader-extra-headers-prefix": `[]`, + "requestheader-client-ca-file": string(anotherRandomCA), + "requestheader-allowed-names": `[]`, + }, + }, + }, + expectedConfigMaps: map[string]*corev1.ConfigMap{}, + expectCreate: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + client := fake.NewSimpleClientset(test.preexistingObjs...) + configMapIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + for _, obj := range test.preexistingObjs { + configMapIndexer.Add(obj) + } + configmapLister := corev1listers.NewConfigMapLister(configMapIndexer) + + c := &Controller{ + configMapLister: configmapLister, + configMapClient: client.CoreV1(), + namespaceClient: client.CoreV1(), + requiredAuthenticationData: test.clusterAuthInfo, + } + + err := c.syncConfigMap() + if err != nil { + t.Fatal(err) + } + + actualConfigMaps, updated := getFinalConfigMaps(t, client) + if !reflect.DeepEqual(test.expectedConfigMaps, actualConfigMaps) { + t.Fatalf("%s: %v", test.name, diff.ObjectReflectDiff(test.expectedConfigMaps, actualConfigMaps)) + } + if test.expectCreate != updated { + t.Fatalf("%s: expected %v, got %v", test.name, test.expectCreate, updated) + } + }) + } +} + +func getFinalConfigMaps(t *testing.T, client *fake.Clientset) (map[string]*corev1.ConfigMap, bool) { + ret := map[string]*corev1.ConfigMap{} + created := false + + for _, action := range client.Actions() { + t.Log(spew.Sdump(action)) + if action.Matches("create", "configmaps") { + created = true + obj := action.(clienttesting.CreateAction).GetObject().(*corev1.ConfigMap) + ret[obj.Name] = obj + } + if action.Matches("update", "configmaps") { + obj := action.(clienttesting.UpdateAction).GetObject().(*corev1.ConfigMap) + ret[obj.Name] = obj + } + } + return ret, created +} + +func TestWriteConfigMapDeleted(t *testing.T) { + // the basics are tested above, this checks the deletion logic when the ca bundles are too large + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Namespace: metav1.NamespaceSystem, Name: "extension-apiserver-authentication"}, + Data: map[string]string{ + "requestheader-username-headers": `[]`, + "requestheader-group-headers": `[]`, + "requestheader-extra-headers-prefix": `[]`, + "requestheader-client-ca-file": string(anotherRandomCA), + "requestheader-allowed-names": `[]`, + }, + } + + t.Run("request entity too large", func(t *testing.T) { + client := fake.NewSimpleClientset() + client.PrependReactor("update", "configmaps", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, apierrors.NewRequestEntityTooLargeError("way too big") + }) + client.PrependReactor("delete", "configmaps", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, nil + }) + + err := writeConfigMap(client.CoreV1(), cm) + if err == nil || err.Error() != "Request entity too large: way too big" { + t.Fatal(err) + } + if len(client.Actions()) != 2 { + t.Fatal(client.Actions()) + } + _, ok := client.Actions()[1].(clienttesting.DeleteAction) + if !ok { + t.Fatal(client.Actions()) + } + }) + + t.Run("ca bundle too large", func(t *testing.T) { + client := fake.NewSimpleClientset() + client.PrependReactor("update", "configmaps", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, apierrors.NewInvalid(schema.GroupKind{Kind: "ConfigMap"}, cm.Name, field.ErrorList{field.TooLong(field.NewPath(""), cm, corev1.MaxSecretSize)}) + }) + client.PrependReactor("delete", "configmaps", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, nil + }) + + err := writeConfigMap(client.CoreV1(), cm) + if err == nil || err.Error() != `ConfigMap "extension-apiserver-authentication" is invalid: []: Too long: must have at most 1048576 bytes` { + t.Fatal(err) + } + if len(client.Actions()) != 2 { + t.Fatal(client.Actions()) + } + _, ok := client.Actions()[1].(clienttesting.DeleteAction) + if !ok { + t.Fatal(client.Actions()) + } + }) + +} diff --git a/pkg/master/master.go b/pkg/master/master.go index 0c4cc573c60..5d22feb8df4 100644 --- a/pkg/master/master.go +++ b/pkg/master/master.go @@ -64,20 +64,24 @@ import ( storageapiv1beta1 "k8s.io/api/storage/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilnet "k8s.io/apimachinery/pkg/util/net" + "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apiserver/pkg/endpoints/discovery" "k8s.io/apiserver/pkg/registry/generic" genericapiserver "k8s.io/apiserver/pkg/server" + "k8s.io/apiserver/pkg/server/dynamiccertificates" "k8s.io/apiserver/pkg/server/healthz" serverstorage "k8s.io/apiserver/pkg/server/storage" storagefactory "k8s.io/apiserver/pkg/storage/storagebackend/factory" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" discoveryclient "k8s.io/client-go/kubernetes/typed/discovery/v1alpha1" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/features" kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options" kubeletclient "k8s.io/kubernetes/pkg/kubelet/client" + "k8s.io/kubernetes/pkg/master/controller/clusterauthenticationtrust" "k8s.io/kubernetes/pkg/master/reconcilers" "k8s.io/kubernetes/pkg/master/tunneler" "k8s.io/kubernetes/pkg/routes" @@ -120,7 +124,7 @@ const ( // ExtraConfig defines extra configuration for the master type ExtraConfig struct { - ClientCARegistrationHook ClientCARegistrationHook + ClusterAuthenticationInfo clusterauthenticationtrust.ClusterAuthenticationInfo APIResourceConfigSource serverstorage.APIResourceConfigSource StorageFactory serverstorage.StorageFactory @@ -217,7 +221,7 @@ type EndpointReconcilerConfig struct { type Master struct { GenericAPIServer *genericapiserver.GenericAPIServer - ClientCARegistrationHook ClientCARegistrationHook + ClusterAuthenticationInfo clusterauthenticationtrust.ClusterAuthenticationInfo } func (c *Config) createMasterCountReconciler() reconcilers.EndpointReconciler { @@ -339,7 +343,8 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) } m := &Master{ - GenericAPIServer: s, + GenericAPIServer: s, + ClusterAuthenticationInfo: c.ExtraConfig.ClusterAuthenticationInfo, } // install legacy rest storage @@ -400,7 +405,42 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) m.installTunneler(c.ExtraConfig.Tunneler, corev1client.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig).Nodes()) } - m.GenericAPIServer.AddPostStartHookOrDie("ca-registration", c.ExtraConfig.ClientCARegistrationHook.PostStartHook) + m.GenericAPIServer.AddPostStartHookOrDie("start-cluster-authentication-info-controller", func(hookContext genericapiserver.PostStartHookContext) error { + kubeClient, err := kubernetes.NewForConfig(hookContext.LoopbackClientConfig) + if err != nil { + return err + } + controller := clusterauthenticationtrust.NewClusterAuthenticationTrustController(m.ClusterAuthenticationInfo, kubeClient) + + // prime values and start listeners + if m.ClusterAuthenticationInfo.ClientCA != nil { + if notifier, ok := m.ClusterAuthenticationInfo.ClientCA.(dynamiccertificates.Notifier); ok { + notifier.AddListener(controller) + } + if controller, ok := m.ClusterAuthenticationInfo.ClientCA.(dynamiccertificates.ControllerRunner); ok { + // runonce to be sure that we have a value. + if err := controller.RunOnce(); err != nil { + runtime.HandleError(err) + } + go controller.Run(1, hookContext.StopCh) + } + } + if m.ClusterAuthenticationInfo.RequestHeaderCA != nil { + if notifier, ok := m.ClusterAuthenticationInfo.RequestHeaderCA.(dynamiccertificates.Notifier); ok { + notifier.AddListener(controller) + } + if controller, ok := m.ClusterAuthenticationInfo.RequestHeaderCA.(dynamiccertificates.ControllerRunner); ok { + // runonce to be sure that we have a value. + if err := controller.RunOnce(); err != nil { + runtime.HandleError(err) + } + go controller.Run(1, hookContext.StopCh) + } + } + + go controller.Run(1, hookContext.StopCh) + return nil + }) return m, nil } diff --git a/staging/src/k8s.io/client-go/util/cert/pem.go b/staging/src/k8s.io/client-go/util/cert/pem.go index 9185e2e22d8..c77512315ab 100644 --- a/staging/src/k8s.io/client-go/util/cert/pem.go +++ b/staging/src/k8s.io/client-go/util/cert/pem.go @@ -17,6 +17,7 @@ limitations under the License. package cert import ( + "bytes" "crypto/x509" "encoding/pem" "errors" @@ -59,3 +60,14 @@ func ParseCertsPEM(pemCerts []byte) ([]*x509.Certificate, error) { } return certs, nil } + +// EncodeCertificates returns the PEM-encoded byte array that represents by the specified certs. +func EncodeCertificates(certs ...*x509.Certificate) ([]byte, error) { + b := bytes.Buffer{} + for _, cert := range certs { + if err := pem.Encode(&b, &pem.Block{Type: CertificateBlockType, Bytes: cert.Raw}); err != nil { + return []byte{}, err + } + } + return b.Bytes(), nil +} diff --git a/test/integration/apiserver/certreload/BUILD b/test/integration/apiserver/certreload/BUILD index 76ebb246a7b..5eb80156cbf 100644 --- a/test/integration/apiserver/certreload/BUILD +++ b/test/integration/apiserver/certreload/BUILD @@ -9,7 +9,11 @@ go_test( tags = ["integration"], deps = [ "//cmd/kube-apiserver/app/options:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/component-base/cli/flag:go_default_library", "//test/integration/framework:go_default_library", ], diff --git a/test/integration/apiserver/certreload/certreload_test.go b/test/integration/apiserver/certreload/certreload_test.go index f4d16681f9d..306c827993d 100644 --- a/test/integration/apiserver/certreload/certreload_test.go +++ b/test/integration/apiserver/certreload/certreload_test.go @@ -28,7 +28,11 @@ import ( "testing" "time" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/server/dynamiccertificates" + "k8s.io/client-go/kubernetes" "k8s.io/component-base/cli/flag" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" "k8s.io/kubernetes/test/integration/framework" @@ -76,7 +80,7 @@ MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps= clientCAFilename := "" frontProxyCAFilename := "" - _, kubeconfig := framework.StartTestServer(t, stopCh, framework.TestServerSetup{ + kubeClient, kubeconfig := framework.StartTestServer(t, stopCh, framework.TestServerSetup{ ModifyServerRunOptions: func(opts *options.ServerRunOptions) { opts.GenericServerRunOptions.MaxRequestBodyBytes = 1024 * 1024 clientCAFilename = opts.Authentication.ClientCert.ClientCA @@ -89,6 +93,17 @@ MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps= t.Fatal(err) } + // wait for request header info + err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "requestheader-client-ca-file", "-----BEGIN CERTIFICATE-----", 1)) + if err != nil { + t.Fatal(err) + } + // wait for client cert info + err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "client-ca-file", "-----BEGIN CERTIFICATE-----", 1)) + if err != nil { + t.Fatal(err) + } + // when we run this the second time, we know which one we are expecting acceptableCAs := []string{} tlsConfig := &tls.Config{ @@ -132,6 +147,44 @@ MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps= t.Errorf("expected %q, got %q", expectedCAs[i], acceptableCAs[i]) } } + + // wait for updated request header info that contains both + err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "requestheader-client-ca-file", "-----BEGIN CERTIFICATE-----", 2)) + if err != nil { + t.Error(err) + } + err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "requestheader-client-ca-file", "MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps=", 1)) + if err != nil { + t.Error(err) + } + // wait for updated client cert info that contains both + err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "client-ca-file", "-----BEGIN CERTIFICATE-----", 2)) + if err != nil { + t.Error(err) + } + err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "client-ca-file", "M++C29JwS3Hwbubg6WO3wjFjoEhpCwU6qRYUz3MRp4tHO4kxKXx+oQnUiFnR7vW0", 1)) + if err != nil { + t.Error(err) + } +} + +func waitForConfigMapCAContent(t *testing.T, kubeClient kubernetes.Interface, key, content string, count int) func() (bool, error) { + return func() (bool, error) { + clusterAuthInfo, err := kubeClient.CoreV1().ConfigMaps("kube-system").Get("extension-apiserver-authentication", metav1.GetOptions{}) + if errors.IsNotFound(err) { + return false, nil + } + if err != nil { + return false, err + } + + ca := clusterAuthInfo.Data[key] + if strings.Count(ca, content) == count { + return true, nil + } + t.Log(ca) + return false, nil + } } var serverKey = []byte(`-----BEGIN RSA PRIVATE KEY----- diff --git a/test/integration/framework/master_utils.go b/test/integration/framework/master_utils.go index 0b8b71c4617..6d6acb0907a 100644 --- a/test/integration/framework/master_utils.go +++ b/test/integration/framework/master_utils.go @@ -141,7 +141,9 @@ func startMasterOrDie(masterConfig *master.Config, incomingServer *httptest.Serv stopCh := make(chan struct{}) closeFn := func() { - m.GenericAPIServer.RunPreShutdownHooks() + if m != nil { + m.GenericAPIServer.RunPreShutdownHooks() + } close(stopCh) s.Close() } diff --git a/test/integration/serviceaccount/service_account_test.go b/test/integration/serviceaccount/service_account_test.go index d0efb0ee4fb..bf29cff7d1e 100644 --- a/test/integration/serviceaccount/service_account_test.go +++ b/test/integration/serviceaccount/service_account_test.go @@ -30,7 +30,7 @@ import ( "testing" "time" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -446,12 +446,13 @@ func startServiceAccountTestServer(t *testing.T) (*clientset.Clientset, restclie masterConfig.GenericConfig.Authentication.Authenticator = authenticator masterConfig.GenericConfig.Authorization.Authorizer = authorizer masterConfig.GenericConfig.AdmissionControl = serviceAccountAdmission - framework.RunAMasterUsingServer(masterConfig, apiServer, h) + _, _, kubeAPIServerCloseFn := framework.RunAMasterUsingServer(masterConfig, apiServer, h) // Start the service account and service account token controllers stopCh := make(chan struct{}) stop := func() { close(stopCh) + kubeAPIServerCloseFn() apiServer.Close() }