From 0f2b79978a577a1b79d659038f0bf84adff933e3 Mon Sep 17 00:00:00 2001 From: Haoran Wang Date: Tue, 27 Jun 2017 16:49:12 +0800 Subject: [PATCH] add e2e for bootstrap token cleaner --- staging/src/k8s.io/api/core/v1/types.go | 5 - test/e2e/bootstrap/BUILD | 8 +- test/e2e/bootstrap/OWNERS | 10 +- test/e2e/bootstrap/bootstrap_signer.go | 78 +++------ test/e2e/bootstrap/bootstrap_token_cleaner.go | 84 ++++++++++ test/e2e/bootstrap/util.go | 155 ++++++++++++++++++ test/e2e/framework/bootstrap_util.go | 74 --------- 7 files changed, 272 insertions(+), 142 deletions(-) create mode 100644 test/e2e/bootstrap/bootstrap_token_cleaner.go create mode 100644 test/e2e/bootstrap/util.go delete mode 100644 test/e2e/framework/bootstrap_util.go diff --git a/staging/src/k8s.io/api/core/v1/types.go b/staging/src/k8s.io/api/core/v1/types.go index 1c88dda8459..e2d97f613f0 100644 --- a/staging/src/k8s.io/api/core/v1/types.go +++ b/staging/src/k8s.io/api/core/v1/types.go @@ -4395,11 +4395,6 @@ const ( // - Secret.Data["password"] - password or token needed for authentication SecretTypeBasicAuth SecretType = "kubernetes.io/basic-auth" - // SecretTypeBootstrapToken is used during the automated bootstrap process (first - // implemented by kubeadm). It stores tokens that are used to sign well known - // ConfigMaps. They may also eventually be used for authentication. - SecretTypeBootstrapToken SecretType = "bootstrap.kubernetes.io/token" - // BasicAuthUsernameKey is the key of the username for SecretTypeBasicAuth secrets BasicAuthUsernameKey = "username" // BasicAuthPasswordKey is the key of the password or token for SecretTypeBasicAuth secrets diff --git a/test/e2e/bootstrap/BUILD b/test/e2e/bootstrap/BUILD index 7a4aaab087f..b41abbed7af 100644 --- a/test/e2e/bootstrap/BUILD +++ b/test/e2e/bootstrap/BUILD @@ -9,7 +9,11 @@ load( go_library( name = "go_default_library", - srcs = ["bootstrap_signer.go"], + srcs = [ + "bootstrap_signer.go", + "bootstrap_token_cleaner.go", + "util.go", + ], tags = ["automanaged"], deps = [ "//pkg/bootstrap/api:go_default_library", @@ -18,7 +22,9 @@ go_library( "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", ], ) diff --git a/test/e2e/bootstrap/OWNERS b/test/e2e/bootstrap/OWNERS index 7987f09f4d0..54f38546f7d 100644 --- a/test/e2e/bootstrap/OWNERS +++ b/test/e2e/bootstrap/OWNERS @@ -1,14 +1,10 @@ -approvers: -- errordeveloper +approvers: #sig-cluster-lifecycle is the owner of this feature - jbeda - luxas -- mikedanese -- krousey +- wanghaoran1988 reviewers: - mikedanese - luxas -- justinsb -- errordeveloper -- lukemarsden - dmmcquay - krousey +- wanghaoran1988 diff --git a/test/e2e/bootstrap/bootstrap_signer.go b/test/e2e/bootstrap/bootstrap_signer.go index be7514a9815..d6d7d51afad 100644 --- a/test/e2e/bootstrap/bootstrap_signer.go +++ b/test/e2e/bootstrap/bootstrap_signer.go @@ -17,13 +17,9 @@ limitations under the License. package bootstrap import ( - "crypto/rand" - "encoding/hex" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api" "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" @@ -31,15 +27,23 @@ import ( ) const ( - TokenIDBytes = 3 + TokenIDBytes = 3 + TokenSecretBytes = 8 ) -var _ = framework.KubeDescribe("[Feature:BootstrapSigner]", func() { +var _ = framework.KubeDescribe("[Feature:BootstrapTokens]", func() { var c clientset.Interface f := framework.NewDefaultFramework("bootstrap-signer") - + AfterEach(func() { + if len(secretNeedClean) > 0 { + By("delete the bootstrap token secret") + err := c.CoreV1().Secrets(metav1.NamespaceSystem).Delete(secretNeedClean, &metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + secretNeedClean = "" + } + }) BeforeEach(func() { c = f.ClientSet }) @@ -50,15 +54,12 @@ var _ = framework.KubeDescribe("[Feature:BootstrapSigner]", func() { Expect(err).NotTo(HaveOccurred()) secret := newTokenSecret(tokenId, "tokenSecret") _, err = c.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret) - defer func() { - By("delete the bootstrap token secret") - err := c.CoreV1().Secrets(metav1.NamespaceSystem).Delete(bootstrapapi.BootstrapTokenSecretPrefix+tokenId, &metav1.DeleteOptions{}) - Expect(err).NotTo(HaveOccurred()) - }() + secretNeedClean = bootstrapapi.BootstrapTokenSecretPrefix + tokenId + Expect(err).NotTo(HaveOccurred()) By("wait for the bootstrap token secret be signed") - err = framework.WaitforSignedBootStrapToken(c, tokenId) + err = WaitforSignedClusterInfoByBootStrapToken(c, tokenId) Expect(err).NotTo(HaveOccurred()) }) @@ -68,13 +69,10 @@ var _ = framework.KubeDescribe("[Feature:BootstrapSigner]", func() { Expect(err).NotTo(HaveOccurred()) secret := newTokenSecret(tokenId, "tokenSecret") secret, err = c.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret) - defer func() { - err := c.CoreV1().Secrets(metav1.NamespaceSystem).Delete(bootstrapapi.BootstrapTokenSecretPrefix+tokenId, &metav1.DeleteOptions{}) - Expect(err).NotTo(HaveOccurred()) - }() + secretNeedClean = bootstrapapi.BootstrapTokenSecretPrefix + tokenId By("wait for the bootstrap token secret be signed") - err = framework.WaitforSignedBootStrapToken(c, tokenId) + err = WaitforSignedClusterInfoByBootStrapToken(c, tokenId) cfgMap, err := f.ClientSet.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -83,7 +81,9 @@ var _ = framework.KubeDescribe("[Feature:BootstrapSigner]", func() { By("update the cluster-info ConfigMap") originalData := cfgMap.Data[bootstrapapi.KubeConfigKey] - cfgMap.Data[bootstrapapi.KubeConfigKey] = "updated" + updatedKubeConfig, err := randBytes(20) + Expect(err).NotTo(HaveOccurred()) + cfgMap.Data[bootstrapapi.KubeConfigKey] = updatedKubeConfig _, err = f.ClientSet.CoreV1().ConfigMaps(metav1.NamespacePublic).Update(cfgMap) Expect(err).NotTo(HaveOccurred()) defer func() { @@ -96,11 +96,11 @@ var _ = framework.KubeDescribe("[Feature:BootstrapSigner]", func() { }() By("wait for signed bootstrap token updated") - err = framework.WaitForSignedBootstrapTokenToGetUpdated(c, tokenId, signedToken) + err = WaitForSignedClusterInfoGetUpdatedByBootstrapToken(c, tokenId, signedToken) Expect(err).NotTo(HaveOccurred()) }) - It("delete the signed bootstrap tokens from clusterInfo ConfigMap when bootstrap token is deleted", func() { + It("should delete the signed bootstrap tokens from clusterInfo ConfigMap when bootstrap token is deleted", func() { By("create a new bootstrap token secret") tokenId, err := generateTokenId() Expect(err).NotTo(HaveOccurred()) @@ -109,7 +109,7 @@ var _ = framework.KubeDescribe("[Feature:BootstrapSigner]", func() { Expect(err).NotTo(HaveOccurred()) By("wait for the bootstrap secret be signed") - err = framework.WaitforSignedBootStrapToken(c, tokenId) + err = WaitforSignedClusterInfoByBootStrapToken(c, tokenId) Expect(err).NotTo(HaveOccurred()) By("delete the bootstrap token secret") @@ -117,39 +117,7 @@ var _ = framework.KubeDescribe("[Feature:BootstrapSigner]", func() { Expect(err).NotTo(HaveOccurred()) By("wait for the bootstrap token removed from cluster-info ConfigMap") - err = framework.WaitForSignedBootstrapTokenToDisappear(c, tokenId) + err = WaitForSignedClusterInfoByBootstrapTokenToDisappear(c, tokenId) Expect(err).NotTo(HaveOccurred()) }) }) - -func newTokenSecret(tokenID, tokenSecret string) *v1.Secret { - return &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: metav1.NamespaceSystem, - Name: bootstrapapi.BootstrapTokenSecretPrefix + tokenID, - }, - Type: v1.SecretTypeBootstrapToken, - Data: map[string][]byte{ - bootstrapapi.BootstrapTokenIDKey: []byte(tokenID), - bootstrapapi.BootstrapTokenSecretKey: []byte(tokenSecret), - bootstrapapi.BootstrapTokenUsageSigningKey: []byte("true"), - }, - } -} - -func generateTokenId() (string, error) { - tokenID, err := randBytes(TokenIDBytes) - if err != nil { - return "", err - } - return tokenID, nil -} - -func randBytes(length int) (string, error) { - b := make([]byte, length) - _, err := rand.Read(b) - if err != nil { - return "", err - } - return hex.EncodeToString(b), nil -} diff --git a/test/e2e/bootstrap/bootstrap_token_cleaner.go b/test/e2e/bootstrap/bootstrap_token_cleaner.go new file mode 100644 index 00000000000..d7c7f4ad360 --- /dev/null +++ b/test/e2e/bootstrap/bootstrap_token_cleaner.go @@ -0,0 +1,84 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import ( + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api" + "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" + "k8s.io/kubernetes/test/e2e/framework" +) + +var secretNeedClean string +var _ = framework.KubeDescribe("[Feature:BootstrapTokens]", func() { + + var c clientset.Interface + + f := framework.NewDefaultFramework("bootstrap-token-cleaner") + + BeforeEach(func() { + c = f.ClientSet + }) + + AfterEach(func() { + if len(secretNeedClean) > 0 { + By("delete the bootstrap token secret") + err := c.CoreV1().Secrets(metav1.NamespaceSystem).Delete(secretNeedClean, &metav1.DeleteOptions{}) + secretNeedClean = "" + Expect(err).NotTo(HaveOccurred()) + } + }) + It("should delete the token secret when the secret expired", func() { + By("create a new expired bootstrap token secret") + tokenId, err := generateTokenId() + Expect(err).NotTo(HaveOccurred()) + tokenSecret, err := generateTokenSecret() + Expect(err).NotTo(HaveOccurred()) + + secret := newTokenSecret(tokenId, tokenSecret) + addSecretExpiration(secret, timeStringFromNow(-time.Hour)) + _, err = c.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret) + + Expect(err).NotTo(HaveOccurred()) + + By("wait for the bootstrap token secret be deleted") + err = WaitForBootstrapTokenSecretToDisappear(c, tokenId) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should not delete the token secret when the secret is not expired", func() { + By("create a new expired bootstrap token secret") + tokenId, err := generateTokenId() + Expect(err).NotTo(HaveOccurred()) + tokenSecret, err := generateTokenSecret() + Expect(err).NotTo(HaveOccurred()) + secret := newTokenSecret(tokenId, tokenSecret) + addSecretExpiration(secret, timeStringFromNow(time.Hour)) + _, err = c.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret) + secretNeedClean = bootstrapapi.BootstrapTokenSecretPrefix + tokenId + Expect(err).NotTo(HaveOccurred()) + + By("wait for the bootstrap token secret not be deleted") + err = WaitForBootstrapTokenSecretNotDisappear(c, tokenId, 20*time.Second) + Expect(err).NotTo(HaveOccurred()) + }) +}) diff --git a/test/e2e/bootstrap/util.go b/test/e2e/bootstrap/util.go new file mode 100644 index 00000000000..019117cd139 --- /dev/null +++ b/test/e2e/bootstrap/util.go @@ -0,0 +1,155 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "time" + + v1 "k8s.io/api/core/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api" + "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" + "k8s.io/kubernetes/test/e2e/framework" +) + +func newTokenSecret(tokenID, tokenSecret string) *v1.Secret { + return &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceSystem, + Name: bootstrapapi.BootstrapTokenSecretPrefix + tokenID, + }, + Type: bootstrapapi.SecretTypeBootstrapToken, + Data: map[string][]byte{ + bootstrapapi.BootstrapTokenIDKey: []byte(tokenID), + bootstrapapi.BootstrapTokenSecretKey: []byte(tokenSecret), + bootstrapapi.BootstrapTokenUsageSigningKey: []byte("true"), + }, + } +} + +func generateTokenId() (string, error) { + tokenID, err := randBytes(TokenIDBytes) + if err != nil { + return "", err + } + return tokenID, nil +} +func generateTokenSecret() (string, error) { + tokenSecret, err := randBytes(TokenSecretBytes) + if err != nil { + return "", err + } + return tokenSecret, err +} + +func randBytes(length int) (string, error) { + b := make([]byte, length) + _, err := rand.Read(b) + if err != nil { + return "", err + } + return hex.EncodeToString(b), nil +} + +func addSecretExpiration(s *v1.Secret, expiration string) { + s.Data[bootstrapapi.BootstrapTokenExpirationKey] = []byte(expiration) +} + +func timeStringFromNow(delta time.Duration) string { + return time.Now().Add(delta).Format(time.RFC3339) +} + +func WaitforSignedClusterInfoByBootStrapToken(c clientset.Interface, tokenID string) error { + + return wait.Poll(framework.Poll, 2*time.Minute, func() (bool, error) { + cfgMap, err := c.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) + if err != nil { + framework.Failf("Failed to get cluster-info configMap: %v", err) + return false, err + } + _, ok := cfgMap.Data[bootstrapapi.JWSSignatureKeyPrefix+tokenID] + if !ok { + return false, nil + } + return true, nil + }) +} + +func WaitForSignedClusterInfoGetUpdatedByBootstrapToken(c clientset.Interface, tokenID string, signedToken string) error { + + return wait.Poll(framework.Poll, 2*time.Minute, func() (bool, error) { + cfgMap, err := c.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) + if err != nil { + framework.Failf("Failed to get cluster-info configMap: %v", err) + return false, err + } + updated, ok := cfgMap.Data[bootstrapapi.JWSSignatureKeyPrefix+tokenID] + if !ok || updated == signedToken { + return false, nil + } + return true, nil + }) +} + +func WaitForSignedClusterInfoByBootstrapTokenToDisappear(c clientset.Interface, tokenID string) error { + + return wait.Poll(framework.Poll, 2*time.Minute, func() (bool, error) { + cfgMap, err := c.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) + if err != nil { + framework.Failf("Failed to get cluster-info configMap: %v", err) + return false, err + } + _, ok := cfgMap.Data[bootstrapapi.JWSSignatureKeyPrefix+tokenID] + if ok { + return false, nil + } + return true, nil + }) +} + +func WaitForBootstrapTokenSecretToDisappear(c clientset.Interface, tokenID string) error { + + return wait.Poll(framework.Poll, 1*time.Minute, func() (bool, error) { + _, err := c.CoreV1().Secrets(metav1.NamespaceSystem).Get(bootstrapapi.BootstrapTokenSecretPrefix+tokenID, metav1.GetOptions{}) + if apierrs.IsNotFound(err) { + return true, nil + } + return false, nil + }) +} + +func WaitForBootstrapTokenSecretNotDisappear(c clientset.Interface, tokenID string, t time.Duration) error { + err := wait.Poll(framework.Poll, t, func() (bool, error) { + secret, err := c.CoreV1().Secrets(metav1.NamespaceSystem).Get(bootstrapapi.BootstrapTokenSecretPrefix+tokenID, metav1.GetOptions{}) + if apierrs.IsNotFound(err) { + return true, errors.New("secret not exits") + } + if secret != nil { + return false, nil + } + return true, err + }) + if err == wait.ErrWaitTimeout { + return nil + } + return err +} diff --git a/test/e2e/framework/bootstrap_util.go b/test/e2e/framework/bootstrap_util.go deleted file mode 100644 index 12aa797c45d..00000000000 --- a/test/e2e/framework/bootstrap_util.go +++ /dev/null @@ -1,74 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package framework - -import ( - "time" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" - bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api" - "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" -) - -func WaitforSignedBootStrapToken(c clientset.Interface, tokenID string) error { - - return wait.Poll(Poll, 2*time.Minute, func() (bool, error) { - cfgMap, err := c.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) - if err != nil { - Failf("Failed to get cluster-info configMap: %v", err) - return false, err - } - _, ok := cfgMap.Data[bootstrapapi.JWSSignatureKeyPrefix+tokenID] - if !ok { - return false, nil - } - return true, nil - }) -} - -func WaitForSignedBootstrapTokenToGetUpdated(c clientset.Interface, tokenID string, signedToken string) error { - - return wait.Poll(Poll, 2*time.Minute, func() (bool, error) { - cfgMap, err := c.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) - if err != nil { - Failf("Failed to get cluster-info configMap: %v", err) - return false, err - } - updated, ok := cfgMap.Data[bootstrapapi.JWSSignatureKeyPrefix+tokenID] - if !ok || updated == signedToken { - return false, nil - } - return true, nil - }) -} - -func WaitForSignedBootstrapTokenToDisappear(c clientset.Interface, tokenID string) error { - - return wait.Poll(Poll, 2*time.Minute, func() (bool, error) { - cfgMap, err := c.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) - if err != nil { - Failf("Failed to get cluster-info configMap: %v", err) - return false, err - } - _, ok := cfgMap.Data[bootstrapapi.JWSSignatureKeyPrefix+tokenID] - if ok { - return false, nil - } - return true, nil - }) -}