From 5f43150deb9c552bad63b000e4724a70aeaaf6f7 Mon Sep 17 00:00:00 2001 From: Haoran Wang Date: Mon, 26 Jun 2017 23:16:09 +0800 Subject: [PATCH 1/3] add e2e tests for bootstrap signer --- staging/src/k8s.io/api/core/v1/types.go | 5 + test/e2e/BUILD | 2 + test/e2e/bootstrap/BUILD | 36 ++++++ test/e2e/bootstrap/OWNERS | 14 +++ test/e2e/bootstrap/bootstrap_signer.go | 155 ++++++++++++++++++++++++ test/e2e/e2e_test.go | 1 + test/e2e/framework/bootstrap_util.go | 74 +++++++++++ 7 files changed, 287 insertions(+) create mode 100644 test/e2e/bootstrap/BUILD create mode 100644 test/e2e/bootstrap/OWNERS create mode 100644 test/e2e/bootstrap/bootstrap_signer.go create mode 100644 test/e2e/framework/bootstrap_util.go diff --git a/staging/src/k8s.io/api/core/v1/types.go b/staging/src/k8s.io/api/core/v1/types.go index e2d97f613f0..1c88dda8459 100644 --- a/staging/src/k8s.io/api/core/v1/types.go +++ b/staging/src/k8s.io/api/core/v1/types.go @@ -4395,6 +4395,11 @@ const ( // - Secret.Data["password"] - password or token needed for authentication SecretTypeBasicAuth SecretType = "kubernetes.io/basic-auth" + // SecretTypeBootstrapToken is used during the automated bootstrap process (first + // implemented by kubeadm). It stores tokens that are used to sign well known + // ConfigMaps. They may also eventually be used for authentication. + SecretTypeBootstrapToken SecretType = "bootstrap.kubernetes.io/token" + // BasicAuthUsernameKey is the key of the username for SecretTypeBasicAuth secrets BasicAuthUsernameKey = "username" // BasicAuthPasswordKey is the key of the password or token for SecretTypeBasicAuth secrets diff --git a/test/e2e/BUILD b/test/e2e/BUILD index f31fb09aee7..9e6b6167fa6 100644 --- a/test/e2e/BUILD +++ b/test/e2e/BUILD @@ -20,6 +20,7 @@ go_test( deps = [ "//test/e2e/apimachinery:go_default_library", "//test/e2e/autoscaling:go_default_library", + "//test/e2e/bootstrap:go_default_library", "//test/e2e/framework:go_default_library", "//test/e2e/instrumentation/logging:go_default_library", "//test/e2e/instrumentation/monitoring:go_default_library", @@ -173,6 +174,7 @@ filegroup( ":package-srcs", "//test/e2e/apimachinery:all-srcs", "//test/e2e/autoscaling:all-srcs", + "//test/e2e/bootstrap:all-srcs", "//test/e2e/chaosmonkey:all-srcs", "//test/e2e/common:all-srcs", "//test/e2e/framework:all-srcs", diff --git a/test/e2e/bootstrap/BUILD b/test/e2e/bootstrap/BUILD new file mode 100644 index 00000000000..7a4aaab087f --- /dev/null +++ b/test/e2e/bootstrap/BUILD @@ -0,0 +1,36 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", +) + +go_library( + name = "go_default_library", + srcs = ["bootstrap_signer.go"], + tags = ["automanaged"], + deps = [ + "//pkg/bootstrap/api:go_default_library", + "//pkg/client/clientset_generated/clientset:go_default_library", + "//test/e2e/framework:go_default_library", + "//vendor/github.com/onsi/ginkgo:go_default_library", + "//vendor/github.com/onsi/gomega:go_default_library", + "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/test/e2e/bootstrap/OWNERS b/test/e2e/bootstrap/OWNERS new file mode 100644 index 00000000000..7987f09f4d0 --- /dev/null +++ b/test/e2e/bootstrap/OWNERS @@ -0,0 +1,14 @@ +approvers: +- errordeveloper +- jbeda +- luxas +- mikedanese +- krousey +reviewers: +- mikedanese +- luxas +- justinsb +- errordeveloper +- lukemarsden +- dmmcquay +- krousey diff --git a/test/e2e/bootstrap/bootstrap_signer.go b/test/e2e/bootstrap/bootstrap_signer.go new file mode 100644 index 00000000000..be7514a9815 --- /dev/null +++ b/test/e2e/bootstrap/bootstrap_signer.go @@ -0,0 +1,155 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import ( + "crypto/rand" + "encoding/hex" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api" + "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" + "k8s.io/kubernetes/test/e2e/framework" +) + +const ( + TokenIDBytes = 3 +) + +var _ = framework.KubeDescribe("[Feature:BootstrapSigner]", func() { + + var c clientset.Interface + + f := framework.NewDefaultFramework("bootstrap-signer") + + BeforeEach(func() { + c = f.ClientSet + }) + + It("should sign the new added bootstrap tokens", func() { + By("create a new bootstrap token secret") + tokenId, err := generateTokenId() + Expect(err).NotTo(HaveOccurred()) + secret := newTokenSecret(tokenId, "tokenSecret") + _, err = c.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret) + defer func() { + By("delete the bootstrap token secret") + err := c.CoreV1().Secrets(metav1.NamespaceSystem).Delete(bootstrapapi.BootstrapTokenSecretPrefix+tokenId, &metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + }() + Expect(err).NotTo(HaveOccurred()) + + By("wait for the bootstrap token secret be signed") + err = framework.WaitforSignedBootStrapToken(c, tokenId) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should resign the bootstrap tokens when the clusterInfo ConfigMap updated [Serial][Disruptive]", func() { + By("create a new bootstrap token secret") + tokenId, err := generateTokenId() + Expect(err).NotTo(HaveOccurred()) + secret := newTokenSecret(tokenId, "tokenSecret") + secret, err = c.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret) + defer func() { + err := c.CoreV1().Secrets(metav1.NamespaceSystem).Delete(bootstrapapi.BootstrapTokenSecretPrefix+tokenId, &metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + }() + + By("wait for the bootstrap token secret be signed") + err = framework.WaitforSignedBootStrapToken(c, tokenId) + + cfgMap, err := f.ClientSet.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + signedToken, ok := cfgMap.Data[bootstrapapi.JWSSignatureKeyPrefix+tokenId] + Expect(ok).Should(Equal(true)) + + By("update the cluster-info ConfigMap") + originalData := cfgMap.Data[bootstrapapi.KubeConfigKey] + cfgMap.Data[bootstrapapi.KubeConfigKey] = "updated" + _, err = f.ClientSet.CoreV1().ConfigMaps(metav1.NamespacePublic).Update(cfgMap) + Expect(err).NotTo(HaveOccurred()) + defer func() { + By("update back the cluster-info ConfigMap") + cfgMap, err = f.ClientSet.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + cfgMap.Data[bootstrapapi.KubeConfigKey] = originalData + _, err = f.ClientSet.CoreV1().ConfigMaps(metav1.NamespacePublic).Update(cfgMap) + Expect(err).NotTo(HaveOccurred()) + }() + + By("wait for signed bootstrap token updated") + err = framework.WaitForSignedBootstrapTokenToGetUpdated(c, tokenId, signedToken) + Expect(err).NotTo(HaveOccurred()) + }) + + It("delete the signed bootstrap tokens from clusterInfo ConfigMap when bootstrap token is deleted", func() { + By("create a new bootstrap token secret") + tokenId, err := generateTokenId() + Expect(err).NotTo(HaveOccurred()) + secret := newTokenSecret(tokenId, "tokenSecret") + _, err = c.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret) + Expect(err).NotTo(HaveOccurred()) + + By("wait for the bootstrap secret be signed") + err = framework.WaitforSignedBootStrapToken(c, tokenId) + Expect(err).NotTo(HaveOccurred()) + + By("delete the bootstrap token secret") + err = c.CoreV1().Secrets(metav1.NamespaceSystem).Delete(bootstrapapi.BootstrapTokenSecretPrefix+tokenId, &metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("wait for the bootstrap token removed from cluster-info ConfigMap") + err = framework.WaitForSignedBootstrapTokenToDisappear(c, tokenId) + Expect(err).NotTo(HaveOccurred()) + }) +}) + +func newTokenSecret(tokenID, tokenSecret string) *v1.Secret { + return &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceSystem, + Name: bootstrapapi.BootstrapTokenSecretPrefix + tokenID, + }, + Type: v1.SecretTypeBootstrapToken, + Data: map[string][]byte{ + bootstrapapi.BootstrapTokenIDKey: []byte(tokenID), + bootstrapapi.BootstrapTokenSecretKey: []byte(tokenSecret), + bootstrapapi.BootstrapTokenUsageSigningKey: []byte("true"), + }, + } +} + +func generateTokenId() (string, error) { + tokenID, err := randBytes(TokenIDBytes) + if err != nil { + return "", err + } + return tokenID, nil +} + +func randBytes(length int) (string, error) { + b := make([]byte, length) + _, err := rand.Read(b) + if err != nil { + return "", err + } + return hex.EncodeToString(b), nil +} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 741191a774d..55f2f611c74 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -21,6 +21,7 @@ import ( _ "k8s.io/kubernetes/test/e2e/apimachinery" _ "k8s.io/kubernetes/test/e2e/autoscaling" + _ "k8s.io/kubernetes/test/e2e/bootstrap" "k8s.io/kubernetes/test/e2e/framework" _ "k8s.io/kubernetes/test/e2e/instrumentation/logging" _ "k8s.io/kubernetes/test/e2e/instrumentation/monitoring" diff --git a/test/e2e/framework/bootstrap_util.go b/test/e2e/framework/bootstrap_util.go new file mode 100644 index 00000000000..12aa797c45d --- /dev/null +++ b/test/e2e/framework/bootstrap_util.go @@ -0,0 +1,74 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api" + "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" +) + +func WaitforSignedBootStrapToken(c clientset.Interface, tokenID string) error { + + return wait.Poll(Poll, 2*time.Minute, func() (bool, error) { + cfgMap, err := c.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) + if err != nil { + Failf("Failed to get cluster-info configMap: %v", err) + return false, err + } + _, ok := cfgMap.Data[bootstrapapi.JWSSignatureKeyPrefix+tokenID] + if !ok { + return false, nil + } + return true, nil + }) +} + +func WaitForSignedBootstrapTokenToGetUpdated(c clientset.Interface, tokenID string, signedToken string) error { + + return wait.Poll(Poll, 2*time.Minute, func() (bool, error) { + cfgMap, err := c.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) + if err != nil { + Failf("Failed to get cluster-info configMap: %v", err) + return false, err + } + updated, ok := cfgMap.Data[bootstrapapi.JWSSignatureKeyPrefix+tokenID] + if !ok || updated == signedToken { + return false, nil + } + return true, nil + }) +} + +func WaitForSignedBootstrapTokenToDisappear(c clientset.Interface, tokenID string) error { + + return wait.Poll(Poll, 2*time.Minute, func() (bool, error) { + cfgMap, err := c.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) + if err != nil { + Failf("Failed to get cluster-info configMap: %v", err) + return false, err + } + _, ok := cfgMap.Data[bootstrapapi.JWSSignatureKeyPrefix+tokenID] + if ok { + return false, nil + } + return true, nil + }) +} From 0f2b79978a577a1b79d659038f0bf84adff933e3 Mon Sep 17 00:00:00 2001 From: Haoran Wang Date: Tue, 27 Jun 2017 16:49:12 +0800 Subject: [PATCH 2/3] add e2e for bootstrap token cleaner --- staging/src/k8s.io/api/core/v1/types.go | 5 - test/e2e/bootstrap/BUILD | 8 +- test/e2e/bootstrap/OWNERS | 10 +- test/e2e/bootstrap/bootstrap_signer.go | 78 +++------ test/e2e/bootstrap/bootstrap_token_cleaner.go | 84 ++++++++++ test/e2e/bootstrap/util.go | 155 ++++++++++++++++++ test/e2e/framework/bootstrap_util.go | 74 --------- 7 files changed, 272 insertions(+), 142 deletions(-) create mode 100644 test/e2e/bootstrap/bootstrap_token_cleaner.go create mode 100644 test/e2e/bootstrap/util.go delete mode 100644 test/e2e/framework/bootstrap_util.go diff --git a/staging/src/k8s.io/api/core/v1/types.go b/staging/src/k8s.io/api/core/v1/types.go index 1c88dda8459..e2d97f613f0 100644 --- a/staging/src/k8s.io/api/core/v1/types.go +++ b/staging/src/k8s.io/api/core/v1/types.go @@ -4395,11 +4395,6 @@ const ( // - Secret.Data["password"] - password or token needed for authentication SecretTypeBasicAuth SecretType = "kubernetes.io/basic-auth" - // SecretTypeBootstrapToken is used during the automated bootstrap process (first - // implemented by kubeadm). It stores tokens that are used to sign well known - // ConfigMaps. They may also eventually be used for authentication. - SecretTypeBootstrapToken SecretType = "bootstrap.kubernetes.io/token" - // BasicAuthUsernameKey is the key of the username for SecretTypeBasicAuth secrets BasicAuthUsernameKey = "username" // BasicAuthPasswordKey is the key of the password or token for SecretTypeBasicAuth secrets diff --git a/test/e2e/bootstrap/BUILD b/test/e2e/bootstrap/BUILD index 7a4aaab087f..b41abbed7af 100644 --- a/test/e2e/bootstrap/BUILD +++ b/test/e2e/bootstrap/BUILD @@ -9,7 +9,11 @@ load( go_library( name = "go_default_library", - srcs = ["bootstrap_signer.go"], + srcs = [ + "bootstrap_signer.go", + "bootstrap_token_cleaner.go", + "util.go", + ], tags = ["automanaged"], deps = [ "//pkg/bootstrap/api:go_default_library", @@ -18,7 +22,9 @@ go_library( "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", ], ) diff --git a/test/e2e/bootstrap/OWNERS b/test/e2e/bootstrap/OWNERS index 7987f09f4d0..54f38546f7d 100644 --- a/test/e2e/bootstrap/OWNERS +++ b/test/e2e/bootstrap/OWNERS @@ -1,14 +1,10 @@ -approvers: -- errordeveloper +approvers: #sig-cluster-lifecycle is the owner of this feature - jbeda - luxas -- mikedanese -- krousey +- wanghaoran1988 reviewers: - mikedanese - luxas -- justinsb -- errordeveloper -- lukemarsden - dmmcquay - krousey +- wanghaoran1988 diff --git a/test/e2e/bootstrap/bootstrap_signer.go b/test/e2e/bootstrap/bootstrap_signer.go index be7514a9815..d6d7d51afad 100644 --- a/test/e2e/bootstrap/bootstrap_signer.go +++ b/test/e2e/bootstrap/bootstrap_signer.go @@ -17,13 +17,9 @@ limitations under the License. package bootstrap import ( - "crypto/rand" - "encoding/hex" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api" "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" @@ -31,15 +27,23 @@ import ( ) const ( - TokenIDBytes = 3 + TokenIDBytes = 3 + TokenSecretBytes = 8 ) -var _ = framework.KubeDescribe("[Feature:BootstrapSigner]", func() { +var _ = framework.KubeDescribe("[Feature:BootstrapTokens]", func() { var c clientset.Interface f := framework.NewDefaultFramework("bootstrap-signer") - + AfterEach(func() { + if len(secretNeedClean) > 0 { + By("delete the bootstrap token secret") + err := c.CoreV1().Secrets(metav1.NamespaceSystem).Delete(secretNeedClean, &metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + secretNeedClean = "" + } + }) BeforeEach(func() { c = f.ClientSet }) @@ -50,15 +54,12 @@ var _ = framework.KubeDescribe("[Feature:BootstrapSigner]", func() { Expect(err).NotTo(HaveOccurred()) secret := newTokenSecret(tokenId, "tokenSecret") _, err = c.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret) - defer func() { - By("delete the bootstrap token secret") - err := c.CoreV1().Secrets(metav1.NamespaceSystem).Delete(bootstrapapi.BootstrapTokenSecretPrefix+tokenId, &metav1.DeleteOptions{}) - Expect(err).NotTo(HaveOccurred()) - }() + secretNeedClean = bootstrapapi.BootstrapTokenSecretPrefix + tokenId + Expect(err).NotTo(HaveOccurred()) By("wait for the bootstrap token secret be signed") - err = framework.WaitforSignedBootStrapToken(c, tokenId) + err = WaitforSignedClusterInfoByBootStrapToken(c, tokenId) Expect(err).NotTo(HaveOccurred()) }) @@ -68,13 +69,10 @@ var _ = framework.KubeDescribe("[Feature:BootstrapSigner]", func() { Expect(err).NotTo(HaveOccurred()) secret := newTokenSecret(tokenId, "tokenSecret") secret, err = c.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret) - defer func() { - err := c.CoreV1().Secrets(metav1.NamespaceSystem).Delete(bootstrapapi.BootstrapTokenSecretPrefix+tokenId, &metav1.DeleteOptions{}) - Expect(err).NotTo(HaveOccurred()) - }() + secretNeedClean = bootstrapapi.BootstrapTokenSecretPrefix + tokenId By("wait for the bootstrap token secret be signed") - err = framework.WaitforSignedBootStrapToken(c, tokenId) + err = WaitforSignedClusterInfoByBootStrapToken(c, tokenId) cfgMap, err := f.ClientSet.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -83,7 +81,9 @@ var _ = framework.KubeDescribe("[Feature:BootstrapSigner]", func() { By("update the cluster-info ConfigMap") originalData := cfgMap.Data[bootstrapapi.KubeConfigKey] - cfgMap.Data[bootstrapapi.KubeConfigKey] = "updated" + updatedKubeConfig, err := randBytes(20) + Expect(err).NotTo(HaveOccurred()) + cfgMap.Data[bootstrapapi.KubeConfigKey] = updatedKubeConfig _, err = f.ClientSet.CoreV1().ConfigMaps(metav1.NamespacePublic).Update(cfgMap) Expect(err).NotTo(HaveOccurred()) defer func() { @@ -96,11 +96,11 @@ var _ = framework.KubeDescribe("[Feature:BootstrapSigner]", func() { }() By("wait for signed bootstrap token updated") - err = framework.WaitForSignedBootstrapTokenToGetUpdated(c, tokenId, signedToken) + err = WaitForSignedClusterInfoGetUpdatedByBootstrapToken(c, tokenId, signedToken) Expect(err).NotTo(HaveOccurred()) }) - It("delete the signed bootstrap tokens from clusterInfo ConfigMap when bootstrap token is deleted", func() { + It("should delete the signed bootstrap tokens from clusterInfo ConfigMap when bootstrap token is deleted", func() { By("create a new bootstrap token secret") tokenId, err := generateTokenId() Expect(err).NotTo(HaveOccurred()) @@ -109,7 +109,7 @@ var _ = framework.KubeDescribe("[Feature:BootstrapSigner]", func() { Expect(err).NotTo(HaveOccurred()) By("wait for the bootstrap secret be signed") - err = framework.WaitforSignedBootStrapToken(c, tokenId) + err = WaitforSignedClusterInfoByBootStrapToken(c, tokenId) Expect(err).NotTo(HaveOccurred()) By("delete the bootstrap token secret") @@ -117,39 +117,7 @@ var _ = framework.KubeDescribe("[Feature:BootstrapSigner]", func() { Expect(err).NotTo(HaveOccurred()) By("wait for the bootstrap token removed from cluster-info ConfigMap") - err = framework.WaitForSignedBootstrapTokenToDisappear(c, tokenId) + err = WaitForSignedClusterInfoByBootstrapTokenToDisappear(c, tokenId) Expect(err).NotTo(HaveOccurred()) }) }) - -func newTokenSecret(tokenID, tokenSecret string) *v1.Secret { - return &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: metav1.NamespaceSystem, - Name: bootstrapapi.BootstrapTokenSecretPrefix + tokenID, - }, - Type: v1.SecretTypeBootstrapToken, - Data: map[string][]byte{ - bootstrapapi.BootstrapTokenIDKey: []byte(tokenID), - bootstrapapi.BootstrapTokenSecretKey: []byte(tokenSecret), - bootstrapapi.BootstrapTokenUsageSigningKey: []byte("true"), - }, - } -} - -func generateTokenId() (string, error) { - tokenID, err := randBytes(TokenIDBytes) - if err != nil { - return "", err - } - return tokenID, nil -} - -func randBytes(length int) (string, error) { - b := make([]byte, length) - _, err := rand.Read(b) - if err != nil { - return "", err - } - return hex.EncodeToString(b), nil -} diff --git a/test/e2e/bootstrap/bootstrap_token_cleaner.go b/test/e2e/bootstrap/bootstrap_token_cleaner.go new file mode 100644 index 00000000000..d7c7f4ad360 --- /dev/null +++ b/test/e2e/bootstrap/bootstrap_token_cleaner.go @@ -0,0 +1,84 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import ( + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api" + "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" + "k8s.io/kubernetes/test/e2e/framework" +) + +var secretNeedClean string +var _ = framework.KubeDescribe("[Feature:BootstrapTokens]", func() { + + var c clientset.Interface + + f := framework.NewDefaultFramework("bootstrap-token-cleaner") + + BeforeEach(func() { + c = f.ClientSet + }) + + AfterEach(func() { + if len(secretNeedClean) > 0 { + By("delete the bootstrap token secret") + err := c.CoreV1().Secrets(metav1.NamespaceSystem).Delete(secretNeedClean, &metav1.DeleteOptions{}) + secretNeedClean = "" + Expect(err).NotTo(HaveOccurred()) + } + }) + It("should delete the token secret when the secret expired", func() { + By("create a new expired bootstrap token secret") + tokenId, err := generateTokenId() + Expect(err).NotTo(HaveOccurred()) + tokenSecret, err := generateTokenSecret() + Expect(err).NotTo(HaveOccurred()) + + secret := newTokenSecret(tokenId, tokenSecret) + addSecretExpiration(secret, timeStringFromNow(-time.Hour)) + _, err = c.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret) + + Expect(err).NotTo(HaveOccurred()) + + By("wait for the bootstrap token secret be deleted") + err = WaitForBootstrapTokenSecretToDisappear(c, tokenId) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should not delete the token secret when the secret is not expired", func() { + By("create a new expired bootstrap token secret") + tokenId, err := generateTokenId() + Expect(err).NotTo(HaveOccurred()) + tokenSecret, err := generateTokenSecret() + Expect(err).NotTo(HaveOccurred()) + secret := newTokenSecret(tokenId, tokenSecret) + addSecretExpiration(secret, timeStringFromNow(time.Hour)) + _, err = c.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret) + secretNeedClean = bootstrapapi.BootstrapTokenSecretPrefix + tokenId + Expect(err).NotTo(HaveOccurred()) + + By("wait for the bootstrap token secret not be deleted") + err = WaitForBootstrapTokenSecretNotDisappear(c, tokenId, 20*time.Second) + Expect(err).NotTo(HaveOccurred()) + }) +}) diff --git a/test/e2e/bootstrap/util.go b/test/e2e/bootstrap/util.go new file mode 100644 index 00000000000..019117cd139 --- /dev/null +++ b/test/e2e/bootstrap/util.go @@ -0,0 +1,155 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "time" + + v1 "k8s.io/api/core/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api" + "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" + "k8s.io/kubernetes/test/e2e/framework" +) + +func newTokenSecret(tokenID, tokenSecret string) *v1.Secret { + return &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceSystem, + Name: bootstrapapi.BootstrapTokenSecretPrefix + tokenID, + }, + Type: bootstrapapi.SecretTypeBootstrapToken, + Data: map[string][]byte{ + bootstrapapi.BootstrapTokenIDKey: []byte(tokenID), + bootstrapapi.BootstrapTokenSecretKey: []byte(tokenSecret), + bootstrapapi.BootstrapTokenUsageSigningKey: []byte("true"), + }, + } +} + +func generateTokenId() (string, error) { + tokenID, err := randBytes(TokenIDBytes) + if err != nil { + return "", err + } + return tokenID, nil +} +func generateTokenSecret() (string, error) { + tokenSecret, err := randBytes(TokenSecretBytes) + if err != nil { + return "", err + } + return tokenSecret, err +} + +func randBytes(length int) (string, error) { + b := make([]byte, length) + _, err := rand.Read(b) + if err != nil { + return "", err + } + return hex.EncodeToString(b), nil +} + +func addSecretExpiration(s *v1.Secret, expiration string) { + s.Data[bootstrapapi.BootstrapTokenExpirationKey] = []byte(expiration) +} + +func timeStringFromNow(delta time.Duration) string { + return time.Now().Add(delta).Format(time.RFC3339) +} + +func WaitforSignedClusterInfoByBootStrapToken(c clientset.Interface, tokenID string) error { + + return wait.Poll(framework.Poll, 2*time.Minute, func() (bool, error) { + cfgMap, err := c.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) + if err != nil { + framework.Failf("Failed to get cluster-info configMap: %v", err) + return false, err + } + _, ok := cfgMap.Data[bootstrapapi.JWSSignatureKeyPrefix+tokenID] + if !ok { + return false, nil + } + return true, nil + }) +} + +func WaitForSignedClusterInfoGetUpdatedByBootstrapToken(c clientset.Interface, tokenID string, signedToken string) error { + + return wait.Poll(framework.Poll, 2*time.Minute, func() (bool, error) { + cfgMap, err := c.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) + if err != nil { + framework.Failf("Failed to get cluster-info configMap: %v", err) + return false, err + } + updated, ok := cfgMap.Data[bootstrapapi.JWSSignatureKeyPrefix+tokenID] + if !ok || updated == signedToken { + return false, nil + } + return true, nil + }) +} + +func WaitForSignedClusterInfoByBootstrapTokenToDisappear(c clientset.Interface, tokenID string) error { + + return wait.Poll(framework.Poll, 2*time.Minute, func() (bool, error) { + cfgMap, err := c.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) + if err != nil { + framework.Failf("Failed to get cluster-info configMap: %v", err) + return false, err + } + _, ok := cfgMap.Data[bootstrapapi.JWSSignatureKeyPrefix+tokenID] + if ok { + return false, nil + } + return true, nil + }) +} + +func WaitForBootstrapTokenSecretToDisappear(c clientset.Interface, tokenID string) error { + + return wait.Poll(framework.Poll, 1*time.Minute, func() (bool, error) { + _, err := c.CoreV1().Secrets(metav1.NamespaceSystem).Get(bootstrapapi.BootstrapTokenSecretPrefix+tokenID, metav1.GetOptions{}) + if apierrs.IsNotFound(err) { + return true, nil + } + return false, nil + }) +} + +func WaitForBootstrapTokenSecretNotDisappear(c clientset.Interface, tokenID string, t time.Duration) error { + err := wait.Poll(framework.Poll, t, func() (bool, error) { + secret, err := c.CoreV1().Secrets(metav1.NamespaceSystem).Get(bootstrapapi.BootstrapTokenSecretPrefix+tokenID, metav1.GetOptions{}) + if apierrs.IsNotFound(err) { + return true, errors.New("secret not exits") + } + if secret != nil { + return false, nil + } + return true, err + }) + if err == wait.ErrWaitTimeout { + return nil + } + return err +} diff --git a/test/e2e/framework/bootstrap_util.go b/test/e2e/framework/bootstrap_util.go deleted file mode 100644 index 12aa797c45d..00000000000 --- a/test/e2e/framework/bootstrap_util.go +++ /dev/null @@ -1,74 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package framework - -import ( - "time" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" - bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api" - "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" -) - -func WaitforSignedBootStrapToken(c clientset.Interface, tokenID string) error { - - return wait.Poll(Poll, 2*time.Minute, func() (bool, error) { - cfgMap, err := c.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) - if err != nil { - Failf("Failed to get cluster-info configMap: %v", err) - return false, err - } - _, ok := cfgMap.Data[bootstrapapi.JWSSignatureKeyPrefix+tokenID] - if !ok { - return false, nil - } - return true, nil - }) -} - -func WaitForSignedBootstrapTokenToGetUpdated(c clientset.Interface, tokenID string, signedToken string) error { - - return wait.Poll(Poll, 2*time.Minute, func() (bool, error) { - cfgMap, err := c.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) - if err != nil { - Failf("Failed to get cluster-info configMap: %v", err) - return false, err - } - updated, ok := cfgMap.Data[bootstrapapi.JWSSignatureKeyPrefix+tokenID] - if !ok || updated == signedToken { - return false, nil - } - return true, nil - }) -} - -func WaitForSignedBootstrapTokenToDisappear(c clientset.Interface, tokenID string) error { - - return wait.Poll(Poll, 2*time.Minute, func() (bool, error) { - cfgMap, err := c.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) - if err != nil { - Failf("Failed to get cluster-info configMap: %v", err) - return false, err - } - _, ok := cfgMap.Data[bootstrapapi.JWSSignatureKeyPrefix+tokenID] - if ok { - return false, nil - } - return true, nil - }) -} From f02008338f261e689d625bda469b0617766cf64c Mon Sep 17 00:00:00 2001 From: Haoran Wang Date: Mon, 17 Jul 2017 19:35:27 +0800 Subject: [PATCH 3/3] add integration testing for bootstrap token auth --- hack/.golint_failures | 1 + pkg/api/types.go | 4 + test/e2e/BUILD | 3 +- test/e2e/e2e_test.go | 2 +- test/e2e/lifecycle/BUILD | 5 +- test/e2e/{ => lifecycle}/bootstrap/BUILD | 3 +- test/e2e/{ => lifecycle}/bootstrap/OWNERS | 0 .../bootstrap/bootstrap_signer.go | 11 +- .../bootstrap/bootstrap_token_cleaner.go | 17 +- test/e2e/{ => lifecycle}/bootstrap/util.go | 12 +- test/integration/auth/BUILD | 5 + test/integration/auth/bootstraptoken_test.go | 188 ++++++++++++++++++ test/integration/utils.go | 1 + 13 files changed, 228 insertions(+), 24 deletions(-) rename test/e2e/{ => lifecycle}/bootstrap/BUILD (90%) rename test/e2e/{ => lifecycle}/bootstrap/OWNERS (100%) rename test/e2e/{ => lifecycle}/bootstrap/bootstrap_signer.go (94%) rename test/e2e/{ => lifecycle}/bootstrap/bootstrap_token_cleaner.go (84%) rename test/e2e/{ => lifecycle}/bootstrap/util.go (94%) create mode 100644 test/integration/auth/bootstraptoken_test.go diff --git a/hack/.golint_failures b/hack/.golint_failures index 74f1571a0be..34d514c4847 100644 --- a/hack/.golint_failures +++ b/hack/.golint_failures @@ -811,6 +811,7 @@ test/e2e/instrumentation/logging test/e2e/instrumentation/monitoring test/e2e/kubectl test/e2e/lifecycle +test/e2e/lifecycle/bootstrap test/e2e/metrics test/e2e/scalability test/e2e/scheduling diff --git a/pkg/api/types.go b/pkg/api/types.go index 6a81d43a81c..8dcd21e1580 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -3868,6 +3868,10 @@ const ( TLSCertKey = "tls.crt" // TLSPrivateKeyKey is the key for the private key field in a TLS secret. TLSPrivateKeyKey = "tls.key" + // SecretTypeBootstrapToken is used during the automated bootstrap process (first + // implemented by kubeadm). It stores tokens that are used to sign well known + // ConfigMaps. They are used for authn. + SecretTypeBootstrapToken SecretType = "bootstrap.kubernetes.io/token" ) // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/test/e2e/BUILD b/test/e2e/BUILD index 9e6b6167fa6..7618d46bf26 100644 --- a/test/e2e/BUILD +++ b/test/e2e/BUILD @@ -20,12 +20,12 @@ go_test( deps = [ "//test/e2e/apimachinery:go_default_library", "//test/e2e/autoscaling:go_default_library", - "//test/e2e/bootstrap:go_default_library", "//test/e2e/framework:go_default_library", "//test/e2e/instrumentation/logging:go_default_library", "//test/e2e/instrumentation/monitoring:go_default_library", "//test/e2e/kubectl:go_default_library", "//test/e2e/lifecycle:go_default_library", + "//test/e2e/lifecycle/bootstrap:go_default_library", "//test/e2e/metrics:go_default_library", "//test/e2e/scalability:go_default_library", "//test/e2e/scheduling:go_default_library", @@ -174,7 +174,6 @@ filegroup( ":package-srcs", "//test/e2e/apimachinery:all-srcs", "//test/e2e/autoscaling:all-srcs", - "//test/e2e/bootstrap:all-srcs", "//test/e2e/chaosmonkey:all-srcs", "//test/e2e/common:all-srcs", "//test/e2e/framework:all-srcs", diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 55f2f611c74..ae07686919e 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -21,12 +21,12 @@ import ( _ "k8s.io/kubernetes/test/e2e/apimachinery" _ "k8s.io/kubernetes/test/e2e/autoscaling" - _ "k8s.io/kubernetes/test/e2e/bootstrap" "k8s.io/kubernetes/test/e2e/framework" _ "k8s.io/kubernetes/test/e2e/instrumentation/logging" _ "k8s.io/kubernetes/test/e2e/instrumentation/monitoring" _ "k8s.io/kubernetes/test/e2e/kubectl" _ "k8s.io/kubernetes/test/e2e/lifecycle" + _ "k8s.io/kubernetes/test/e2e/lifecycle/bootstrap" _ "k8s.io/kubernetes/test/e2e/scalability" _ "k8s.io/kubernetes/test/e2e/scheduling" _ "k8s.io/kubernetes/test/e2e/storage" diff --git a/test/e2e/lifecycle/BUILD b/test/e2e/lifecycle/BUILD index d4c74b960b1..717e3f47902 100644 --- a/test/e2e/lifecycle/BUILD +++ b/test/e2e/lifecycle/BUILD @@ -54,6 +54,9 @@ filegroup( filegroup( name = "all-srcs", - srcs = [":package-srcs"], + srcs = [ + ":package-srcs", + "//test/e2e/lifecycle/bootstrap:all-srcs", + ], tags = ["automanaged"], ) diff --git a/test/e2e/bootstrap/BUILD b/test/e2e/lifecycle/bootstrap/BUILD similarity index 90% rename from test/e2e/bootstrap/BUILD rename to test/e2e/lifecycle/bootstrap/BUILD index b41abbed7af..ca411e4a154 100644 --- a/test/e2e/bootstrap/BUILD +++ b/test/e2e/lifecycle/bootstrap/BUILD @@ -17,14 +17,15 @@ go_library( tags = ["automanaged"], deps = [ "//pkg/bootstrap/api:go_default_library", - "//pkg/client/clientset_generated/clientset:go_default_library", "//test/e2e/framework:go_default_library", + "//test/e2e/lifecycle:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//vendor/k8s.io/client-go/kubernetes:go_default_library", ], ) diff --git a/test/e2e/bootstrap/OWNERS b/test/e2e/lifecycle/bootstrap/OWNERS similarity index 100% rename from test/e2e/bootstrap/OWNERS rename to test/e2e/lifecycle/bootstrap/OWNERS diff --git a/test/e2e/bootstrap/bootstrap_signer.go b/test/e2e/lifecycle/bootstrap/bootstrap_signer.go similarity index 94% rename from test/e2e/bootstrap/bootstrap_signer.go rename to test/e2e/lifecycle/bootstrap/bootstrap_signer.go index d6d7d51afad..8b2c7fd7411 100644 --- a/test/e2e/bootstrap/bootstrap_signer.go +++ b/test/e2e/lifecycle/bootstrap/bootstrap_signer.go @@ -21,9 +21,10 @@ import ( . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api" - "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/lifecycle" ) const ( @@ -31,7 +32,7 @@ const ( TokenSecretBytes = 8 ) -var _ = framework.KubeDescribe("[Feature:BootstrapTokens]", func() { +var _ = lifecycle.SIGDescribe("[Feature:BootstrapTokens]", func() { var c clientset.Interface @@ -50,7 +51,7 @@ var _ = framework.KubeDescribe("[Feature:BootstrapTokens]", func() { It("should sign the new added bootstrap tokens", func() { By("create a new bootstrap token secret") - tokenId, err := generateTokenId() + tokenId, err := GenerateTokenId() Expect(err).NotTo(HaveOccurred()) secret := newTokenSecret(tokenId, "tokenSecret") _, err = c.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret) @@ -65,7 +66,7 @@ var _ = framework.KubeDescribe("[Feature:BootstrapTokens]", func() { It("should resign the bootstrap tokens when the clusterInfo ConfigMap updated [Serial][Disruptive]", func() { By("create a new bootstrap token secret") - tokenId, err := generateTokenId() + tokenId, err := GenerateTokenId() Expect(err).NotTo(HaveOccurred()) secret := newTokenSecret(tokenId, "tokenSecret") secret, err = c.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret) @@ -102,7 +103,7 @@ var _ = framework.KubeDescribe("[Feature:BootstrapTokens]", func() { It("should delete the signed bootstrap tokens from clusterInfo ConfigMap when bootstrap token is deleted", func() { By("create a new bootstrap token secret") - tokenId, err := generateTokenId() + tokenId, err := GenerateTokenId() Expect(err).NotTo(HaveOccurred()) secret := newTokenSecret(tokenId, "tokenSecret") _, err = c.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret) diff --git a/test/e2e/bootstrap/bootstrap_token_cleaner.go b/test/e2e/lifecycle/bootstrap/bootstrap_token_cleaner.go similarity index 84% rename from test/e2e/bootstrap/bootstrap_token_cleaner.go rename to test/e2e/lifecycle/bootstrap/bootstrap_token_cleaner.go index d7c7f4ad360..0acab639b69 100644 --- a/test/e2e/bootstrap/bootstrap_token_cleaner.go +++ b/test/e2e/lifecycle/bootstrap/bootstrap_token_cleaner.go @@ -23,13 +23,14 @@ import ( . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api" - "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/lifecycle" ) var secretNeedClean string -var _ = framework.KubeDescribe("[Feature:BootstrapTokens]", func() { +var _ = lifecycle.SIGDescribe("[Feature:BootstrapTokens]", func() { var c clientset.Interface @@ -49,13 +50,13 @@ var _ = framework.KubeDescribe("[Feature:BootstrapTokens]", func() { }) It("should delete the token secret when the secret expired", func() { By("create a new expired bootstrap token secret") - tokenId, err := generateTokenId() + tokenId, err := GenerateTokenId() Expect(err).NotTo(HaveOccurred()) - tokenSecret, err := generateTokenSecret() + tokenSecret, err := GenerateTokenSecret() Expect(err).NotTo(HaveOccurred()) secret := newTokenSecret(tokenId, tokenSecret) - addSecretExpiration(secret, timeStringFromNow(-time.Hour)) + addSecretExpiration(secret, TimeStringFromNow(-time.Hour)) _, err = c.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret) Expect(err).NotTo(HaveOccurred()) @@ -67,12 +68,12 @@ var _ = framework.KubeDescribe("[Feature:BootstrapTokens]", func() { It("should not delete the token secret when the secret is not expired", func() { By("create a new expired bootstrap token secret") - tokenId, err := generateTokenId() + tokenId, err := GenerateTokenId() Expect(err).NotTo(HaveOccurred()) - tokenSecret, err := generateTokenSecret() + tokenSecret, err := GenerateTokenSecret() Expect(err).NotTo(HaveOccurred()) secret := newTokenSecret(tokenId, tokenSecret) - addSecretExpiration(secret, timeStringFromNow(time.Hour)) + addSecretExpiration(secret, TimeStringFromNow(time.Hour)) _, err = c.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret) secretNeedClean = bootstrapapi.BootstrapTokenSecretPrefix + tokenId Expect(err).NotTo(HaveOccurred()) diff --git a/test/e2e/bootstrap/util.go b/test/e2e/lifecycle/bootstrap/util.go similarity index 94% rename from test/e2e/bootstrap/util.go rename to test/e2e/lifecycle/bootstrap/util.go index 019117cd139..5b3e663af2b 100644 --- a/test/e2e/bootstrap/util.go +++ b/test/e2e/lifecycle/bootstrap/util.go @@ -22,12 +22,12 @@ import ( "errors" "time" - v1 "k8s.io/api/core/v1" + "k8s.io/api/core/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api" - "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" "k8s.io/kubernetes/test/e2e/framework" ) @@ -46,14 +46,14 @@ func newTokenSecret(tokenID, tokenSecret string) *v1.Secret { } } -func generateTokenId() (string, error) { +func GenerateTokenId() (string, error) { tokenID, err := randBytes(TokenIDBytes) if err != nil { return "", err } return tokenID, nil } -func generateTokenSecret() (string, error) { +func GenerateTokenSecret() (string, error) { tokenSecret, err := randBytes(TokenSecretBytes) if err != nil { return "", err @@ -74,7 +74,7 @@ func addSecretExpiration(s *v1.Secret, expiration string) { s.Data[bootstrapapi.BootstrapTokenExpirationKey] = []byte(expiration) } -func timeStringFromNow(delta time.Duration) string { +func TimeStringFromNow(delta time.Duration) string { return time.Now().Add(delta).Format(time.RFC3339) } @@ -141,7 +141,7 @@ func WaitForBootstrapTokenSecretNotDisappear(c clientset.Interface, tokenID stri err := wait.Poll(framework.Poll, t, func() (bool, error) { secret, err := c.CoreV1().Secrets(metav1.NamespaceSystem).Get(bootstrapapi.BootstrapTokenSecretPrefix+tokenID, metav1.GetOptions{}) if apierrs.IsNotFound(err) { - return true, errors.New("secret not exits") + return true, errors.New("secret not exists") } if secret != nil { return false, nil diff --git a/test/integration/auth/BUILD b/test/integration/auth/BUILD index b825f610265..d58486d18d7 100644 --- a/test/integration/auth/BUILD +++ b/test/integration/auth/BUILD @@ -13,6 +13,7 @@ go_test( srcs = [ "accessreview_test.go", "auth_test.go", + "bootstraptoken_test.go", "main_test.go", "node_test.go", "rbac_test.go", @@ -30,6 +31,7 @@ go_test( "//pkg/apis/rbac:go_default_library", "//pkg/auth/authorizer/abac:go_default_library", "//pkg/auth/nodeidentifier:go_default_library", + "//pkg/bootstrap/api:go_default_library", "//pkg/client/clientset_generated/internalclientset:go_default_library", "//pkg/client/informers/informers_generated/internalversion:go_default_library", "//pkg/kubeapiserver/authorizer:go_default_library", @@ -44,8 +46,10 @@ go_test( "//pkg/registry/rbac/rolebinding/storage:go_default_library", "//plugin/pkg/admission/admit:go_default_library", "//plugin/pkg/admission/noderestriction:go_default_library", + "//plugin/pkg/auth/authenticator/token/bootstrap:go_default_library", "//plugin/pkg/auth/authorizer/rbac:go_default_library", "//plugin/pkg/auth/authorizer/rbac/bootstrappolicy:go_default_library", + "//test/e2e/lifecycle/bootstrap:go_default_library", "//test/integration:go_default_library", "//test/integration/framework:go_default_library", "//vendor/github.com/golang/glog:go_default_library", @@ -53,6 +57,7 @@ go_test( "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library", "//vendor/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library", diff --git a/test/integration/auth/bootstraptoken_test.go b/test/integration/auth/bootstraptoken_test.go new file mode 100644 index 00000000000..ab62aff8156 --- /dev/null +++ b/test/integration/auth/bootstraptoken_test.go @@ -0,0 +1,188 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "bytes" + "fmt" + "io/ioutil" + "net/http" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apiserver/pkg/authentication/request/bearertoken" + "k8s.io/kubernetes/pkg/api" + bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api" + "k8s.io/kubernetes/plugin/pkg/admission/admit" + "k8s.io/kubernetes/plugin/pkg/auth/authenticator/token/bootstrap" + bootstraputil "k8s.io/kubernetes/test/e2e/lifecycle/bootstrap" + "k8s.io/kubernetes/test/integration" + "k8s.io/kubernetes/test/integration/framework" +) + +type bootstrapSecrets []*api.Secret + +func (b bootstrapSecrets) List(selector labels.Selector) (ret []*api.Secret, err error) { + return b, nil +} + +func (b bootstrapSecrets) Get(name string) (*api.Secret, error) { + return b[0], nil +} + +// TestBootstrapTokenAuth tests the bootstrap token auth provider +func TestBootstrapTokenAuth(t *testing.T) { + tokenId, err := bootstraputil.GenerateTokenId() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + secret, err := bootstraputil.GenerateTokenSecret() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + var bootstrapSecretValid = &api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceSystem, + Name: bootstrapapi.BootstrapTokenSecretPrefix, + }, + Type: api.SecretTypeBootstrapToken, + Data: map[string][]byte{ + bootstrapapi.BootstrapTokenIDKey: []byte(tokenId), + bootstrapapi.BootstrapTokenSecretKey: []byte(secret), + bootstrapapi.BootstrapTokenUsageAuthentication: []byte("true"), + }, + } + var bootstrapSecretInvalid = &api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceSystem, + Name: bootstrapapi.BootstrapTokenSecretPrefix, + }, + Type: api.SecretTypeBootstrapToken, + Data: map[string][]byte{ + bootstrapapi.BootstrapTokenIDKey: []byte(tokenId), + bootstrapapi.BootstrapTokenSecretKey: []byte("invalid"), + bootstrapapi.BootstrapTokenUsageAuthentication: []byte("true"), + }, + } + var expiredBootstrapToken = &api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceSystem, + Name: bootstrapapi.BootstrapTokenSecretPrefix, + }, + Type: api.SecretTypeBootstrapToken, + Data: map[string][]byte{ + bootstrapapi.BootstrapTokenIDKey: []byte(tokenId), + bootstrapapi.BootstrapTokenSecretKey: []byte("invalid"), + bootstrapapi.BootstrapTokenUsageAuthentication: []byte("true"), + bootstrapapi.BootstrapTokenExpirationKey: []byte(bootstraputil.TimeStringFromNow(-time.Hour)), + }, + } + type request struct { + verb string + URL string + body string + statusCodes map[int]bool // Set of expected resp.StatusCode if all goes well. + } + tests := []struct { + name string + request request + secret *api.Secret + }{ + { + name: "valid token", + request: request{verb: "GET", URL: path("pods", "", ""), body: "", statusCodes: integration.Code200}, + secret: bootstrapSecretValid, + }, + { + name: "invalid token format", + request: request{verb: "GET", URL: path("pods", "", ""), body: "", statusCodes: integration.Code401}, + secret: bootstrapSecretInvalid, + }, + { + name: "invalid token expired", + request: request{verb: "GET", URL: path("pods", "", ""), body: "", statusCodes: integration.Code401}, + secret: expiredBootstrapToken, + }, + } + for _, test := range tests { + + authenticator := bearertoken.New(bootstrap.NewTokenAuthenticator(bootstrapSecrets{test.secret})) + // Set up a master + masterConfig := framework.NewIntegrationTestMasterConfig() + masterConfig.GenericConfig.Authenticator = authenticator + masterConfig.GenericConfig.AdmissionControl = admit.NewAlwaysAdmit() + _, s, closeFn := framework.RunAMaster(masterConfig) + defer closeFn() + + ns := framework.CreateTestingNamespace("auth-bootstrap-token", s, t) + defer framework.DeleteTestingNamespace(ns, s, t) + + previousResourceVersion := make(map[string]float64) + transport := http.DefaultTransport + + token := tokenId + "." + secret + var bodyStr string + if test.request.body != "" { + sub := "" + if test.request.verb == "PUT" { + // For update operations, insert previous resource version + if resVersion := previousResourceVersion[getPreviousResourceVersionKey(test.request.URL, "")]; resVersion != 0 { + sub += fmt.Sprintf(",\r\n\"resourceVersion\": \"%v\"", resVersion) + } + sub += fmt.Sprintf(",\r\n\"namespace\": %q", ns.Name) + } + bodyStr = fmt.Sprintf(test.request.body, sub) + } + test.request.body = bodyStr + bodyBytes := bytes.NewReader([]byte(bodyStr)) + req, err := http.NewRequest(test.request.verb, s.URL+test.request.URL, bodyBytes) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + if test.request.verb == "PATCH" { + req.Header.Set("Content-Type", "application/merge-patch+json") + } + + func() { + resp, err := transport.RoundTrip(req) + defer resp.Body.Close() + if err != nil { + t.Logf("case %v", test.name) + t.Fatalf("unexpected error: %v", err) + } + b, _ := ioutil.ReadAll(resp.Body) + if _, ok := test.request.statusCodes[resp.StatusCode]; !ok { + t.Logf("case %v", test.name) + t.Errorf("Expected status one of %v, but got %v", test.request.statusCodes, resp.StatusCode) + t.Errorf("Body: %v", string(b)) + } else { + if test.request.verb == "POST" { + // For successful create operations, extract resourceVersion + id, currentResourceVersion, err := parseResourceVersion(b) + if err == nil { + key := getPreviousResourceVersionKey(test.request.URL, id) + previousResourceVersion[key] = currentResourceVersion + } + } + } + + }() + } +} diff --git a/test/integration/utils.go b/test/integration/utils.go index f25e39a3403..629a58a2d87 100644 --- a/test/integration/utils.go +++ b/test/integration/utils.go @@ -38,6 +38,7 @@ func DeletePodOrErrorf(t *testing.T, c clientset.Interface, ns, name string) { var Code200 = map[int]bool{200: true} var Code201 = map[int]bool{201: true} var Code400 = map[int]bool{400: true} +var Code401 = map[int]bool{401: true} var Code403 = map[int]bool{403: true} var Code404 = map[int]bool{404: true} var Code405 = map[int]bool{405: true}