Merge pull request #62665 from liztio/kubeadm-e2e

Automatic merge from submit-queue (batch tested with PRs 62665, 62194, 63616, 63672, 63450). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Kubeadm e2e

**What this PR does / why we need it**:

Provides in-tree E2E tests for the Kubeadm subproject

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes [kubeadm#456](https://github.com/kubernetes/kubeadm/issues/456)

**Special notes for your reviewer**:

The weird way tests are executed mirrors `e2e_node`. A future pull request will add a frontend for these tests to kubetest, which will abstract away much of this detail.

**Release note**:
```release-note
NONE
```

hack/.golint_failures

@@ -1,3 +1,4 @@
+
 cluster/images/etcd-version-monitor
 cmd/hyperkube
 cmd/kube-apiserver/app
@@ -695,6 +696,7 @@ test/e2e/ui
 test/e2e/upgrades
 test/e2e/upgrades/apps
 test/e2e/upgrades/storage
+test/e2e_kubeadm
 test/e2e_node
 test/e2e_node/builder
 test/e2e_node/environment

hack/make-rules/test.sh

@@ -57,6 +57,7 @@ kube::test::find_dirs() {
           -o -path './target/*' \
           -o -path './test/e2e/*' \
           -o -path './test/e2e_node/*' \
+          -o -path './test/e2e_kubeadm/*' \
           -o -path './test/integration/*' \
           -o -path './third_party/*' \
           -o -path './staging/*' \

test/BUILD

@@ -13,6 +13,7 @@ filegroup(
         ":package-srcs",
         "//test/conformance:all-srcs",
         "//test/e2e:all-srcs",
+        "//test/e2e_kubeadm:all-srcs",
         "//test/e2e_node:all-srcs",
         "//test/fixtures:all-srcs",
         "//test/images:all-srcs",

test/e2e_kubeadm/BUILD

@@ -0,0 +1,65 @@
+package(default_visibility = ["//visibility:public"])
+
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "e2e_kubeadm_suite_test.go",
+        "kubeadm_test.go",
+    ],
+    embed = [":go_default_library"],
+    tags = ["e2e"],
+    deps = [
+        "//test/e2e/framework:go_default_library",
+        "//vendor/github.com/onsi/ginkgo:go_default_library",
+        "//vendor/github.com/onsi/gomega:go_default_library",
+        "//vendor/github.com/spf13/pflag:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/api/rbac/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/k8s.io/client-go/rest:go_default_library",
+        "//vendor/k8s.io/client-go/tools/bootstrap/token/api:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+genrule(
+    name = "gen_e2e_kubeadm.test",
+    testonly = 1,
+    srcs = [":go_default_test"],
+    outs = ["e2e_kubeadm.test"],
+    cmd = "srcs=($(SRCS)); cp $$(dirname $${srcs[0]})/go_default_test $@;",
+    output_to_bindir = 1,
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//test/e2e_kubeadm/runner/local:all-srcs",
+        "//test/e2e_kubeadm/tests:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["matchers.go"],
+    importpath = "k8s.io/kubernetes/test/e2e_kubeadm",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//vendor/github.com/onsi/gomega:go_default_library",
+        "//vendor/github.com/onsi/gomega/gstruct:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+    ],
+)

test/e2e_kubeadm/e2e_kubeadm_suite_test.go

@@ -0,0 +1,48 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_kubeadm
+
+import (
+	"flag"
+	"os"
+	"testing"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+	"github.com/spf13/pflag"
+
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+func init() {
+	framework.RegisterCommonFlags()
+	framework.RegisterClusterFlags()
+
+	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
+}
+
+func TestMain(m *testing.M) {
+	pflag.Parse()
+	framework.AfterReadingAllFlags(&framework.TestContext)
+	os.Exit(m.Run())
+}
+
+func TestE2E(t *testing.T) {
+	reporters := []ginkgo.Reporter{}
+	gomega.RegisterFailHandler(ginkgo.Fail)
+	ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "E2EKubadm suite", reporters)
+}

test/e2e_kubeadm/kubeadm_test.go

@@ -0,0 +1,136 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_kubeadm
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	bootstrapapi "k8s.io/client-go/tools/bootstrap/token/api"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+const (
+	masterTaint                  = "node-role.kubernetes.io/master"
+	kubeadmConfigNamespace       = "kube-system"
+	kubeadmConfigName            = "kubeadm-config"
+	clusterInfoNamespace         = "kube-public"
+	clusterInfoName              = "cluster-info"
+	bootstrapSignerRoleNamespace = "kube-system"
+	bootstrapSignerRoleName      = "system:controller:bootstrap-signer"
+)
+
+var _ = framework.KubeDescribe("Kubeadm [Feature:Kubeadm]", func() {
+	f := framework.NewDefaultFramework("kubeadm")
+
+	Describe("kubeadm master", func() {
+		It("should be labelled and tainted", func() {
+			selector := labels.Set{masterTaint: ""}.AsSelector()
+			master, err := f.ClientSet.CoreV1().Nodes().
+				List(metav1.ListOptions{LabelSelector: selector.String()})
+			framework.ExpectNoError(err, "couldn't find a master node")
+			Expect(master.Items).NotTo(BeEmpty())
+			for _, master := range master.Items {
+				Expect(master.Spec.Taints).To(
+					ContainElement(taint(masterTaint, corev1.TaintEffectNoSchedule)),
+				)
+			}
+		})
+	})
+
+	Describe("kubeadm-config config map", func() {
+		It("should exist", func() {
+			_, err := f.ClientSet.CoreV1().
+				ConfigMaps(kubeadmConfigNamespace).
+				Get(kubeadmConfigName, metav1.GetOptions{})
+			framework.ExpectNoError(err)
+		})
+	})
+
+	Describe("cluster-info", func() {
+		It("should have expected keys", func() {
+			clientInfo, err := f.ClientSet.CoreV1().
+				ConfigMaps(clusterInfoNamespace).
+				Get(clusterInfoName, metav1.GetOptions{})
+			framework.ExpectNoError(err, "couldn't find config map")
+
+			Expect(clientInfo.Data).To(HaveKey(HavePrefix(bootstrapapi.JWSSignatureKeyPrefix)))
+			Expect(clientInfo.Data).To(HaveKey(bootstrapapi.KubeConfigKey))
+		})
+
+		It("should be public", func() {
+			cfg, err := framework.LoadConfig()
+			framework.ExpectNoError(err, "couldn't get config")
+			cfg = rest.AnonymousClientConfig(cfg)
+			client, err := kubernetes.NewForConfig(cfg)
+			framework.ExpectNoError(err, "couldn't create client")
+
+			_, err = client.CoreV1().ConfigMaps(clusterInfoNamespace).
+				Get(clusterInfoName, metav1.GetOptions{})
+			framework.ExpectNoError(err, "couldn't anonymously access config")
+		})
+	})
+
+	Describe("bootstrap signer RBAC role", func() {
+		It("should exist", func() {
+			_, err := f.ClientSet.RbacV1().
+				Roles(bootstrapSignerRoleNamespace).
+				Get(bootstrapSignerRoleName, metav1.GetOptions{})
+			framework.ExpectNoError(err, "doesn't exist")
+		})
+	})
+
+	Describe("kubeadm:kubelet-bootstrap cluster role binding", func() {
+		It("should exist", func() {
+			binding, err := f.ClientSet.RbacV1().
+				ClusterRoleBindings().
+				Get("kubeadm:kubelet-bootstrap", metav1.GetOptions{})
+			framework.ExpectNoError(err, "couldn't get clusterrolebinding")
+			Expect(binding.Subjects).To(
+				ContainElement(subject(
+					"system:bootstrappers:kubeadm:default-node-token",
+					rbacv1.GroupKind,
+				)),
+			)
+			Expect(binding.RoleRef.Name).To(Equal("system:node-bootstrapper"))
+		})
+	})
+
+	Describe("autoapproval for new bootstrap token", func() {
+		It("should create a clusterrolebinding", func() {
+			binding, err := f.ClientSet.RbacV1().
+				ClusterRoleBindings().
+				Get("kubeadm:node-autoapprove-bootstrap", metav1.GetOptions{})
+			framework.ExpectNoError(err, "couldn't get clusterrolebinding")
+			Expect(binding.Subjects).To(
+				ContainElement(subject(
+					"system:bootstrappers:kubeadm:default-node-token",
+					rbacv1.GroupKind,
+				)),
+			)
+			Expect(binding.RoleRef.Name).To(
+				Equal("system:certificates.k8s.io:certificatesigningrequests:nodeclient"),
+			)
+		})
+	})
+})

test/e2e_kubeadm/matchers.go

@@ -0,0 +1,37 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_kubeadm
+
+import (
+	"github.com/onsi/gomega"
+	"github.com/onsi/gomega/gstruct"
+	corev1 "k8s.io/api/core/v1"
+)
+
+func subject(name, kind string) gomega.OmegaMatcher {
+	return gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
+		"Name": gomega.Equal(name),
+		"Kind": gomega.Equal(kind),
+	})
+}
+
+func taint(key string, effect corev1.TaintEffect) gomega.OmegaMatcher {
+	return gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
+		"Key":    gomega.Equal(key),
+		"Effect": gomega.Equal(effect),
+	})
+}

test/e2e_kubeadm/runner/local/BUILD

@@ -0,0 +1,32 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["run_local.go"],
+    importpath = "k8s.io/kubernetes/test/e2e_kubeadm/runner/local",
+    visibility = ["//visibility:private"],
+    deps = [
+        "//test/utils:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+    ],
+)
+
+go_binary(
+    name = "local",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

test/e2e_kubeadm/runner/local/run_local.go

@@ -0,0 +1,106 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	"github.com/golang/glog"
+	"k8s.io/kubernetes/test/utils"
+)
+
+func bazelBuild() error {
+	targets := []string{
+		"//vendor/github.com/onsi/ginkgo/ginkgo",
+		"//test/e2e_kubeadm:e2e_kubeadm.test",
+	}
+
+	args := append([]string{"build"}, targets...)
+
+	return execCommand("bazel", args...)
+}
+
+var ginkgoFlags = flag.String("ginkgo-flags", "", "Space-separated list of arguments to pass to Ginkgo test runner.")
+var testFlags = flag.String("test-flags", "", "Space-separated list of arguments to pass to node e2e test.")
+var build = flag.Bool("build", false, "use Bazel to build binaries before testing")
+
+func main() {
+	flag.Parse()
+
+	if *build {
+		if err := bazelBuild(); err != nil {
+			glog.Exitf("couldn't build with bazel: %v", err)
+		}
+	}
+
+	ginkgo, err := getBazelGinkgo()
+	if err != nil {
+		glog.Fatalf("Failed to get ginkgo binary: %v", err)
+	}
+
+	test, err := getBazelTestBin()
+	if err != nil {
+		glog.Fatalf("Failed to get test file: %v", err)
+	}
+
+	args := append(strings.Split(*ginkgoFlags, " "), test, "--")
+	args = append(args, strings.Split(*testFlags, " ")...)
+
+	if execCommand(ginkgo, args...); err != nil {
+		glog.Exitf("Test failed: %v", err)
+	}
+
+}
+
+func getBazelTestBin() (string, error) {
+	k8sRoot, err := utils.GetK8sRootDir()
+	if err != nil {
+		return "", err
+	}
+	buildFile := filepath.Join(k8sRoot, "bazel-bin/test/e2e_kubeadm/e2e_kubeadm.test")
+	if _, err := os.Stat(buildFile); err != nil {
+		return "", err
+	}
+	return buildFile, nil
+
+}
+
+func getBazelGinkgo() (string, error) {
+	k8sRoot, err := utils.GetK8sRootDir()
+	if err != nil {
+		return "", err
+	}
+	buildOutputDir := filepath.Join(k8sRoot, "bazel-bin", "vendor/github.com/onsi/ginkgo/ginkgo", fmt.Sprintf("%s_%s_stripped", runtime.GOOS, runtime.GOARCH), "ginkgo")
+	if _, err := os.Stat(buildOutputDir); err != nil {
+		return "", err
+	}
+	return buildOutputDir, nil
+}
+
+func execCommand(binary string, args ...string) error {
+	fmt.Printf("Running command: %v %v\n", binary, strings.Join(args, " "))
+	cmd := exec.Command(binary, args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}

test/e2e_kubeadm/tests/BUILD

@@ -0,0 +1,13 @@
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)