add e2e for bootstrap token cleaner

staging/src/k8s.io/api/core/v1/types.go

@@ -4395,11 +4395,6 @@ const (
 	// - Secret.Data["password"] - password or token needed for authentication
 	SecretTypeBasicAuth SecretType = "kubernetes.io/basic-auth"
 
-	// SecretTypeBootstrapToken is used during the automated bootstrap process (first
-	// implemented by kubeadm). It stores tokens that are used to sign well known
-	// ConfigMaps. They may also eventually be used for authentication.
-	SecretTypeBootstrapToken SecretType = "bootstrap.kubernetes.io/token"
-
 	// BasicAuthUsernameKey is the key of the username for SecretTypeBasicAuth secrets
 	BasicAuthUsernameKey = "username"
 	// BasicAuthPasswordKey is the key of the password or token for SecretTypeBasicAuth secrets

test/e2e/bootstrap/BUILD

@@ -9,7 +9,11 @@ load(
 
 go_library(
     name = "go_default_library",
-    srcs = ["bootstrap_signer.go"],
+    srcs = [
+        "bootstrap_signer.go",
+        "bootstrap_token_cleaner.go",
+        "util.go",
+    ],
     tags = ["automanaged"],
     deps = [
         "//pkg/bootstrap/api:go_default_library",
@@ -18,7 +22,9 @@ go_library(
         "//vendor/github.com/onsi/ginkgo:go_default_library",
         "//vendor/github.com/onsi/gomega:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
     ],
 )
 

test/e2e/bootstrap/OWNERS

@@ -1,14 +1,10 @@
-approvers:
+approvers:  #sig-cluster-lifecycle is the owner of this feature
-- errordeveloper
 - jbeda
 - luxas
-- mikedanese
+- wanghaoran1988
-- krousey
 reviewers:
 - mikedanese
 - luxas
-- justinsb
-- errordeveloper
-- lukemarsden
 - dmmcquay
 - krousey
+- wanghaoran1988

test/e2e/bootstrap/bootstrap_signer.go

@@ -17,13 +17,9 @@ limitations under the License.
 package bootstrap
 
 import (
-	"crypto/rand"
-	"encoding/hex"
-
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api"
 	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
@@ -31,15 +27,23 @@ import (
 )
 
 const (
-	TokenIDBytes = 3
+	TokenIDBytes     = 3
+	TokenSecretBytes = 8
 )
 
-var _ = framework.KubeDescribe("[Feature:BootstrapSigner]", func() {
+var _ = framework.KubeDescribe("[Feature:BootstrapTokens]", func() {
 
 	var c clientset.Interface
 
 	f := framework.NewDefaultFramework("bootstrap-signer")
-
+	AfterEach(func() {
+		if len(secretNeedClean) > 0 {
+			By("delete the bootstrap token secret")
+			err := c.CoreV1().Secrets(metav1.NamespaceSystem).Delete(secretNeedClean, &metav1.DeleteOptions{})
+			Expect(err).NotTo(HaveOccurred())
+			secretNeedClean = ""
+		}
+	})
 	BeforeEach(func() {
 		c = f.ClientSet
 	})
@@ -50,15 +54,12 @@ var _ = framework.KubeDescribe("[Feature:BootstrapSigner]", func() {
 		Expect(err).NotTo(HaveOccurred())
 		secret := newTokenSecret(tokenId, "tokenSecret")
 		_, err = c.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret)
-		defer func() {
+		secretNeedClean = bootstrapapi.BootstrapTokenSecretPrefix + tokenId
-			By("delete the bootstrap token secret")
+
-			err := c.CoreV1().Secrets(metav1.NamespaceSystem).Delete(bootstrapapi.BootstrapTokenSecretPrefix+tokenId, &metav1.DeleteOptions{})
-			Expect(err).NotTo(HaveOccurred())
-		}()
 		Expect(err).NotTo(HaveOccurred())
 
 		By("wait for the bootstrap token secret be signed")
-		err = framework.WaitforSignedBootStrapToken(c, tokenId)
+		err = WaitforSignedClusterInfoByBootStrapToken(c, tokenId)
 		Expect(err).NotTo(HaveOccurred())
 	})
 
@@ -68,13 +69,10 @@ var _ = framework.KubeDescribe("[Feature:BootstrapSigner]", func() {
 		Expect(err).NotTo(HaveOccurred())
 		secret := newTokenSecret(tokenId, "tokenSecret")
 		secret, err = c.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret)
-		defer func() {
+		secretNeedClean = bootstrapapi.BootstrapTokenSecretPrefix + tokenId
-			err := c.CoreV1().Secrets(metav1.NamespaceSystem).Delete(bootstrapapi.BootstrapTokenSecretPrefix+tokenId, &metav1.DeleteOptions{})
-			Expect(err).NotTo(HaveOccurred())
-		}()
 
 		By("wait for the bootstrap token secret be signed")
-		err = framework.WaitforSignedBootStrapToken(c, tokenId)
+		err = WaitforSignedClusterInfoByBootStrapToken(c, tokenId)
 
 		cfgMap, err := f.ClientSet.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{})
 		Expect(err).NotTo(HaveOccurred())
@@ -83,7 +81,9 @@ var _ = framework.KubeDescribe("[Feature:BootstrapSigner]", func() {
 
 		By("update the cluster-info ConfigMap")
 		originalData := cfgMap.Data[bootstrapapi.KubeConfigKey]
-		cfgMap.Data[bootstrapapi.KubeConfigKey] = "updated"
+		updatedKubeConfig, err := randBytes(20)
+		Expect(err).NotTo(HaveOccurred())
+		cfgMap.Data[bootstrapapi.KubeConfigKey] = updatedKubeConfig
 		_, err = f.ClientSet.CoreV1().ConfigMaps(metav1.NamespacePublic).Update(cfgMap)
 		Expect(err).NotTo(HaveOccurred())
 		defer func() {
@@ -96,11 +96,11 @@ var _ = framework.KubeDescribe("[Feature:BootstrapSigner]", func() {
 		}()
 
 		By("wait for signed bootstrap token updated")
-		err = framework.WaitForSignedBootstrapTokenToGetUpdated(c, tokenId, signedToken)
+		err = WaitForSignedClusterInfoGetUpdatedByBootstrapToken(c, tokenId, signedToken)
 		Expect(err).NotTo(HaveOccurred())
 	})
 
-	It("delete the signed bootstrap tokens from clusterInfo ConfigMap when bootstrap token is deleted", func() {
+	It("should delete the signed bootstrap tokens from clusterInfo ConfigMap when bootstrap token is deleted", func() {
 		By("create a new bootstrap token secret")
 		tokenId, err := generateTokenId()
 		Expect(err).NotTo(HaveOccurred())
@@ -109,7 +109,7 @@ var _ = framework.KubeDescribe("[Feature:BootstrapSigner]", func() {
 		Expect(err).NotTo(HaveOccurred())
 
 		By("wait for the bootstrap secret be signed")
-		err = framework.WaitforSignedBootStrapToken(c, tokenId)
+		err = WaitforSignedClusterInfoByBootStrapToken(c, tokenId)
 		Expect(err).NotTo(HaveOccurred())
 
 		By("delete the bootstrap token secret")
@@ -117,39 +117,7 @@ var _ = framework.KubeDescribe("[Feature:BootstrapSigner]", func() {
 		Expect(err).NotTo(HaveOccurred())
 
 		By("wait for the bootstrap token removed from cluster-info ConfigMap")
-		err = framework.WaitForSignedBootstrapTokenToDisappear(c, tokenId)
+		err = WaitForSignedClusterInfoByBootstrapTokenToDisappear(c, tokenId)
 		Expect(err).NotTo(HaveOccurred())
 	})
 })
-
-func newTokenSecret(tokenID, tokenSecret string) *v1.Secret {
-	return &v1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			Namespace: metav1.NamespaceSystem,
-			Name:      bootstrapapi.BootstrapTokenSecretPrefix + tokenID,
-		},
-		Type: v1.SecretTypeBootstrapToken,
-		Data: map[string][]byte{
-			bootstrapapi.BootstrapTokenIDKey:           []byte(tokenID),
-			bootstrapapi.BootstrapTokenSecretKey:       []byte(tokenSecret),
-			bootstrapapi.BootstrapTokenUsageSigningKey: []byte("true"),
-		},
-	}
-}
-
-func generateTokenId() (string, error) {
-	tokenID, err := randBytes(TokenIDBytes)
-	if err != nil {
-		return "", err
-	}
-	return tokenID, nil
-}
-
-func randBytes(length int) (string, error) {
-	b := make([]byte, length)
-	_, err := rand.Read(b)
-	if err != nil {
-		return "", err
-	}
-	return hex.EncodeToString(b), nil
-}

test/e2e/bootstrap/bootstrap_token_cleaner.go

@@ -0,0 +1,84 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package bootstrap
+
+import (
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api"
+	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+var secretNeedClean string
+var _ = framework.KubeDescribe("[Feature:BootstrapTokens]", func() {
+
+	var c clientset.Interface
+
+	f := framework.NewDefaultFramework("bootstrap-token-cleaner")
+
+	BeforeEach(func() {
+		c = f.ClientSet
+	})
+
+	AfterEach(func() {
+		if len(secretNeedClean) > 0 {
+			By("delete the bootstrap token secret")
+			err := c.CoreV1().Secrets(metav1.NamespaceSystem).Delete(secretNeedClean, &metav1.DeleteOptions{})
+			secretNeedClean = ""
+			Expect(err).NotTo(HaveOccurred())
+		}
+	})
+	It("should delete the token secret when the secret expired", func() {
+		By("create a new expired bootstrap token secret")
+		tokenId, err := generateTokenId()
+		Expect(err).NotTo(HaveOccurred())
+		tokenSecret, err := generateTokenSecret()
+		Expect(err).NotTo(HaveOccurred())
+
+		secret := newTokenSecret(tokenId, tokenSecret)
+		addSecretExpiration(secret, timeStringFromNow(-time.Hour))
+		_, err = c.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret)
+
+		Expect(err).NotTo(HaveOccurred())
+
+		By("wait for the bootstrap token secret be deleted")
+		err = WaitForBootstrapTokenSecretToDisappear(c, tokenId)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("should not delete the token secret when the secret is not expired", func() {
+		By("create a new expired bootstrap token secret")
+		tokenId, err := generateTokenId()
+		Expect(err).NotTo(HaveOccurred())
+		tokenSecret, err := generateTokenSecret()
+		Expect(err).NotTo(HaveOccurred())
+		secret := newTokenSecret(tokenId, tokenSecret)
+		addSecretExpiration(secret, timeStringFromNow(time.Hour))
+		_, err = c.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret)
+		secretNeedClean = bootstrapapi.BootstrapTokenSecretPrefix + tokenId
+		Expect(err).NotTo(HaveOccurred())
+
+		By("wait for the bootstrap token secret not be deleted")
+		err = WaitForBootstrapTokenSecretNotDisappear(c, tokenId, 20*time.Second)
+		Expect(err).NotTo(HaveOccurred())
+	})
+})

test/e2e/bootstrap/util.go

@@ -0,0 +1,155 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package bootstrap
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	"errors"
+	"time"
+
+	v1 "k8s.io/api/core/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api"
+	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+func newTokenSecret(tokenID, tokenSecret string) *v1.Secret {
+	return &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: metav1.NamespaceSystem,
+			Name:      bootstrapapi.BootstrapTokenSecretPrefix + tokenID,
+		},
+		Type: bootstrapapi.SecretTypeBootstrapToken,
+		Data: map[string][]byte{
+			bootstrapapi.BootstrapTokenIDKey:           []byte(tokenID),
+			bootstrapapi.BootstrapTokenSecretKey:       []byte(tokenSecret),
+			bootstrapapi.BootstrapTokenUsageSigningKey: []byte("true"),
+		},
+	}
+}
+
+func generateTokenId() (string, error) {
+	tokenID, err := randBytes(TokenIDBytes)
+	if err != nil {
+		return "", err
+	}
+	return tokenID, nil
+}
+func generateTokenSecret() (string, error) {
+	tokenSecret, err := randBytes(TokenSecretBytes)
+	if err != nil {
+		return "", err
+	}
+	return tokenSecret, err
+}
+
+func randBytes(length int) (string, error) {
+	b := make([]byte, length)
+	_, err := rand.Read(b)
+	if err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(b), nil
+}
+
+func addSecretExpiration(s *v1.Secret, expiration string) {
+	s.Data[bootstrapapi.BootstrapTokenExpirationKey] = []byte(expiration)
+}
+
+func timeStringFromNow(delta time.Duration) string {
+	return time.Now().Add(delta).Format(time.RFC3339)
+}
+
+func WaitforSignedClusterInfoByBootStrapToken(c clientset.Interface, tokenID string) error {
+
+	return wait.Poll(framework.Poll, 2*time.Minute, func() (bool, error) {
+		cfgMap, err := c.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{})
+		if err != nil {
+			framework.Failf("Failed to get cluster-info configMap: %v", err)
+			return false, err
+		}
+		_, ok := cfgMap.Data[bootstrapapi.JWSSignatureKeyPrefix+tokenID]
+		if !ok {
+			return false, nil
+		}
+		return true, nil
+	})
+}
+
+func WaitForSignedClusterInfoGetUpdatedByBootstrapToken(c clientset.Interface, tokenID string, signedToken string) error {
+
+	return wait.Poll(framework.Poll, 2*time.Minute, func() (bool, error) {
+		cfgMap, err := c.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{})
+		if err != nil {
+			framework.Failf("Failed to get cluster-info configMap: %v", err)
+			return false, err
+		}
+		updated, ok := cfgMap.Data[bootstrapapi.JWSSignatureKeyPrefix+tokenID]
+		if !ok || updated == signedToken {
+			return false, nil
+		}
+		return true, nil
+	})
+}
+
+func WaitForSignedClusterInfoByBootstrapTokenToDisappear(c clientset.Interface, tokenID string) error {
+
+	return wait.Poll(framework.Poll, 2*time.Minute, func() (bool, error) {
+		cfgMap, err := c.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{})
+		if err != nil {
+			framework.Failf("Failed to get cluster-info configMap: %v", err)
+			return false, err
+		}
+		_, ok := cfgMap.Data[bootstrapapi.JWSSignatureKeyPrefix+tokenID]
+		if ok {
+			return false, nil
+		}
+		return true, nil
+	})
+}
+
+func WaitForBootstrapTokenSecretToDisappear(c clientset.Interface, tokenID string) error {
+
+	return wait.Poll(framework.Poll, 1*time.Minute, func() (bool, error) {
+		_, err := c.CoreV1().Secrets(metav1.NamespaceSystem).Get(bootstrapapi.BootstrapTokenSecretPrefix+tokenID, metav1.GetOptions{})
+		if apierrs.IsNotFound(err) {
+			return true, nil
+		}
+		return false, nil
+	})
+}
+
+func WaitForBootstrapTokenSecretNotDisappear(c clientset.Interface, tokenID string, t time.Duration) error {
+	err := wait.Poll(framework.Poll, t, func() (bool, error) {
+		secret, err := c.CoreV1().Secrets(metav1.NamespaceSystem).Get(bootstrapapi.BootstrapTokenSecretPrefix+tokenID, metav1.GetOptions{})
+		if apierrs.IsNotFound(err) {
+			return true, errors.New("secret not exits")
+		}
+		if secret != nil {
+			return false, nil
+		}
+		return true, err
+	})
+	if err == wait.ErrWaitTimeout {
+		return nil
+	}
+	return err
+}

test/e2e/framework/bootstrap_util.go

@@ -1,74 +0,0 @@
-/*
-Copyright 2017 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package framework
-
-import (
-	"time"
-
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/wait"
-	bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api"
-	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
-)
-
-func WaitforSignedBootStrapToken(c clientset.Interface, tokenID string) error {
-
-	return wait.Poll(Poll, 2*time.Minute, func() (bool, error) {
-		cfgMap, err := c.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{})
-		if err != nil {
-			Failf("Failed to get cluster-info configMap: %v", err)
-			return false, err
-		}
-		_, ok := cfgMap.Data[bootstrapapi.JWSSignatureKeyPrefix+tokenID]
-		if !ok {
-			return false, nil
-		}
-		return true, nil
-	})
-}
-
-func WaitForSignedBootstrapTokenToGetUpdated(c clientset.Interface, tokenID string, signedToken string) error {
-
-	return wait.Poll(Poll, 2*time.Minute, func() (bool, error) {
-		cfgMap, err := c.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{})
-		if err != nil {
-			Failf("Failed to get cluster-info configMap: %v", err)
-			return false, err
-		}
-		updated, ok := cfgMap.Data[bootstrapapi.JWSSignatureKeyPrefix+tokenID]
-		if !ok || updated == signedToken {
-			return false, nil
-		}
-		return true, nil
-	})
-}
-
-func WaitForSignedBootstrapTokenToDisappear(c clientset.Interface, tokenID string) error {
-
-	return wait.Poll(Poll, 2*time.Minute, func() (bool, error) {
-		cfgMap, err := c.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{})
-		if err != nil {
-			Failf("Failed to get cluster-info configMap: %v", err)
-			return false, err
-		}
-		_, ok := cfgMap.Data[bootstrapapi.JWSSignatureKeyPrefix+tokenID]
-		if ok {
-			return false, nil
-		}
-		return true, nil
-	})
-}