From 5f43150deb9c552bad63b000e4724a70aeaaf6f7 Mon Sep 17 00:00:00 2001 From: Haoran Wang Date: Mon, 26 Jun 2017 23:16:09 +0800 Subject: [PATCH] add e2e tests for bootstrap signer --- staging/src/k8s.io/api/core/v1/types.go | 5 + test/e2e/BUILD | 2 + test/e2e/bootstrap/BUILD | 36 ++++++ test/e2e/bootstrap/OWNERS | 14 +++ test/e2e/bootstrap/bootstrap_signer.go | 155 ++++++++++++++++++++++++ test/e2e/e2e_test.go | 1 + test/e2e/framework/bootstrap_util.go | 74 +++++++++++ 7 files changed, 287 insertions(+) create mode 100644 test/e2e/bootstrap/BUILD create mode 100644 test/e2e/bootstrap/OWNERS create mode 100644 test/e2e/bootstrap/bootstrap_signer.go create mode 100644 test/e2e/framework/bootstrap_util.go diff --git a/staging/src/k8s.io/api/core/v1/types.go b/staging/src/k8s.io/api/core/v1/types.go index e2d97f613f0..1c88dda8459 100644 --- a/staging/src/k8s.io/api/core/v1/types.go +++ b/staging/src/k8s.io/api/core/v1/types.go @@ -4395,6 +4395,11 @@ const ( // - Secret.Data["password"] - password or token needed for authentication SecretTypeBasicAuth SecretType = "kubernetes.io/basic-auth" + // SecretTypeBootstrapToken is used during the automated bootstrap process (first + // implemented by kubeadm). It stores tokens that are used to sign well known + // ConfigMaps. They may also eventually be used for authentication. + SecretTypeBootstrapToken SecretType = "bootstrap.kubernetes.io/token" + // BasicAuthUsernameKey is the key of the username for SecretTypeBasicAuth secrets BasicAuthUsernameKey = "username" // BasicAuthPasswordKey is the key of the password or token for SecretTypeBasicAuth secrets diff --git a/test/e2e/BUILD b/test/e2e/BUILD index f31fb09aee7..9e6b6167fa6 100644 --- a/test/e2e/BUILD +++ b/test/e2e/BUILD @@ -20,6 +20,7 @@ go_test( deps = [ "//test/e2e/apimachinery:go_default_library", "//test/e2e/autoscaling:go_default_library", + "//test/e2e/bootstrap:go_default_library", "//test/e2e/framework:go_default_library", "//test/e2e/instrumentation/logging:go_default_library", "//test/e2e/instrumentation/monitoring:go_default_library", @@ -173,6 +174,7 @@ filegroup( ":package-srcs", "//test/e2e/apimachinery:all-srcs", "//test/e2e/autoscaling:all-srcs", + "//test/e2e/bootstrap:all-srcs", "//test/e2e/chaosmonkey:all-srcs", "//test/e2e/common:all-srcs", "//test/e2e/framework:all-srcs", diff --git a/test/e2e/bootstrap/BUILD b/test/e2e/bootstrap/BUILD new file mode 100644 index 00000000000..7a4aaab087f --- /dev/null +++ b/test/e2e/bootstrap/BUILD @@ -0,0 +1,36 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", +) + +go_library( + name = "go_default_library", + srcs = ["bootstrap_signer.go"], + tags = ["automanaged"], + deps = [ + "//pkg/bootstrap/api:go_default_library", + "//pkg/client/clientset_generated/clientset:go_default_library", + "//test/e2e/framework:go_default_library", + "//vendor/github.com/onsi/ginkgo:go_default_library", + "//vendor/github.com/onsi/gomega:go_default_library", + "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/test/e2e/bootstrap/OWNERS b/test/e2e/bootstrap/OWNERS new file mode 100644 index 00000000000..7987f09f4d0 --- /dev/null +++ b/test/e2e/bootstrap/OWNERS @@ -0,0 +1,14 @@ +approvers: +- errordeveloper +- jbeda +- luxas +- mikedanese +- krousey +reviewers: +- mikedanese +- luxas +- justinsb +- errordeveloper +- lukemarsden +- dmmcquay +- krousey diff --git a/test/e2e/bootstrap/bootstrap_signer.go b/test/e2e/bootstrap/bootstrap_signer.go new file mode 100644 index 00000000000..be7514a9815 --- /dev/null +++ b/test/e2e/bootstrap/bootstrap_signer.go @@ -0,0 +1,155 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import ( + "crypto/rand" + "encoding/hex" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api" + "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" + "k8s.io/kubernetes/test/e2e/framework" +) + +const ( + TokenIDBytes = 3 +) + +var _ = framework.KubeDescribe("[Feature:BootstrapSigner]", func() { + + var c clientset.Interface + + f := framework.NewDefaultFramework("bootstrap-signer") + + BeforeEach(func() { + c = f.ClientSet + }) + + It("should sign the new added bootstrap tokens", func() { + By("create a new bootstrap token secret") + tokenId, err := generateTokenId() + Expect(err).NotTo(HaveOccurred()) + secret := newTokenSecret(tokenId, "tokenSecret") + _, err = c.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret) + defer func() { + By("delete the bootstrap token secret") + err := c.CoreV1().Secrets(metav1.NamespaceSystem).Delete(bootstrapapi.BootstrapTokenSecretPrefix+tokenId, &metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + }() + Expect(err).NotTo(HaveOccurred()) + + By("wait for the bootstrap token secret be signed") + err = framework.WaitforSignedBootStrapToken(c, tokenId) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should resign the bootstrap tokens when the clusterInfo ConfigMap updated [Serial][Disruptive]", func() { + By("create a new bootstrap token secret") + tokenId, err := generateTokenId() + Expect(err).NotTo(HaveOccurred()) + secret := newTokenSecret(tokenId, "tokenSecret") + secret, err = c.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret) + defer func() { + err := c.CoreV1().Secrets(metav1.NamespaceSystem).Delete(bootstrapapi.BootstrapTokenSecretPrefix+tokenId, &metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + }() + + By("wait for the bootstrap token secret be signed") + err = framework.WaitforSignedBootStrapToken(c, tokenId) + + cfgMap, err := f.ClientSet.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + signedToken, ok := cfgMap.Data[bootstrapapi.JWSSignatureKeyPrefix+tokenId] + Expect(ok).Should(Equal(true)) + + By("update the cluster-info ConfigMap") + originalData := cfgMap.Data[bootstrapapi.KubeConfigKey] + cfgMap.Data[bootstrapapi.KubeConfigKey] = "updated" + _, err = f.ClientSet.CoreV1().ConfigMaps(metav1.NamespacePublic).Update(cfgMap) + Expect(err).NotTo(HaveOccurred()) + defer func() { + By("update back the cluster-info ConfigMap") + cfgMap, err = f.ClientSet.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + cfgMap.Data[bootstrapapi.KubeConfigKey] = originalData + _, err = f.ClientSet.CoreV1().ConfigMaps(metav1.NamespacePublic).Update(cfgMap) + Expect(err).NotTo(HaveOccurred()) + }() + + By("wait for signed bootstrap token updated") + err = framework.WaitForSignedBootstrapTokenToGetUpdated(c, tokenId, signedToken) + Expect(err).NotTo(HaveOccurred()) + }) + + It("delete the signed bootstrap tokens from clusterInfo ConfigMap when bootstrap token is deleted", func() { + By("create a new bootstrap token secret") + tokenId, err := generateTokenId() + Expect(err).NotTo(HaveOccurred()) + secret := newTokenSecret(tokenId, "tokenSecret") + _, err = c.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret) + Expect(err).NotTo(HaveOccurred()) + + By("wait for the bootstrap secret be signed") + err = framework.WaitforSignedBootStrapToken(c, tokenId) + Expect(err).NotTo(HaveOccurred()) + + By("delete the bootstrap token secret") + err = c.CoreV1().Secrets(metav1.NamespaceSystem).Delete(bootstrapapi.BootstrapTokenSecretPrefix+tokenId, &metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("wait for the bootstrap token removed from cluster-info ConfigMap") + err = framework.WaitForSignedBootstrapTokenToDisappear(c, tokenId) + Expect(err).NotTo(HaveOccurred()) + }) +}) + +func newTokenSecret(tokenID, tokenSecret string) *v1.Secret { + return &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceSystem, + Name: bootstrapapi.BootstrapTokenSecretPrefix + tokenID, + }, + Type: v1.SecretTypeBootstrapToken, + Data: map[string][]byte{ + bootstrapapi.BootstrapTokenIDKey: []byte(tokenID), + bootstrapapi.BootstrapTokenSecretKey: []byte(tokenSecret), + bootstrapapi.BootstrapTokenUsageSigningKey: []byte("true"), + }, + } +} + +func generateTokenId() (string, error) { + tokenID, err := randBytes(TokenIDBytes) + if err != nil { + return "", err + } + return tokenID, nil +} + +func randBytes(length int) (string, error) { + b := make([]byte, length) + _, err := rand.Read(b) + if err != nil { + return "", err + } + return hex.EncodeToString(b), nil +} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 741191a774d..55f2f611c74 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -21,6 +21,7 @@ import ( _ "k8s.io/kubernetes/test/e2e/apimachinery" _ "k8s.io/kubernetes/test/e2e/autoscaling" + _ "k8s.io/kubernetes/test/e2e/bootstrap" "k8s.io/kubernetes/test/e2e/framework" _ "k8s.io/kubernetes/test/e2e/instrumentation/logging" _ "k8s.io/kubernetes/test/e2e/instrumentation/monitoring" diff --git a/test/e2e/framework/bootstrap_util.go b/test/e2e/framework/bootstrap_util.go new file mode 100644 index 00000000000..12aa797c45d --- /dev/null +++ b/test/e2e/framework/bootstrap_util.go @@ -0,0 +1,74 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api" + "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" +) + +func WaitforSignedBootStrapToken(c clientset.Interface, tokenID string) error { + + return wait.Poll(Poll, 2*time.Minute, func() (bool, error) { + cfgMap, err := c.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) + if err != nil { + Failf("Failed to get cluster-info configMap: %v", err) + return false, err + } + _, ok := cfgMap.Data[bootstrapapi.JWSSignatureKeyPrefix+tokenID] + if !ok { + return false, nil + } + return true, nil + }) +} + +func WaitForSignedBootstrapTokenToGetUpdated(c clientset.Interface, tokenID string, signedToken string) error { + + return wait.Poll(Poll, 2*time.Minute, func() (bool, error) { + cfgMap, err := c.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) + if err != nil { + Failf("Failed to get cluster-info configMap: %v", err) + return false, err + } + updated, ok := cfgMap.Data[bootstrapapi.JWSSignatureKeyPrefix+tokenID] + if !ok || updated == signedToken { + return false, nil + } + return true, nil + }) +} + +func WaitForSignedBootstrapTokenToDisappear(c clientset.Interface, tokenID string) error { + + return wait.Poll(Poll, 2*time.Minute, func() (bool, error) { + cfgMap, err := c.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) + if err != nil { + Failf("Failed to get cluster-info configMap: %v", err) + return false, err + } + _, ok := cfgMap.Data[bootstrapapi.JWSSignatureKeyPrefix+tokenID] + if ok { + return false, nil + } + return true, nil + }) +}