Merge pull request #43881 from mikedanese/kubeadm-validate

Automatic merge from submit-queue

move end to end validation to a dedicated phase

mostly reshuffling

cmd/kubeadm/app/BUILD

@@ -42,6 +42,7 @@ filegroup(
         "//cmd/kubeadm/app/phases/certs:all-srcs",
         "//cmd/kubeadm/app/phases/kubeconfig:all-srcs",
         "//cmd/kubeadm/app/phases/token:all-srcs",
+        "//cmd/kubeadm/app/phases/validate:all-srcs",
         "//cmd/kubeadm/app/preflight:all-srcs",
         "//cmd/kubeadm/app/util:all-srcs",
     ],

cmd/kubeadm/app/cmd/phases/BUILD

@@ -13,6 +13,7 @@ go_library(
         "certs.go",
         "kubeconfig.go",
         "phase.go",
+        "validate.go",
     ],
     tags = ["automanaged"],
     deps = [
@@ -21,6 +22,7 @@ go_library(
         "//cmd/kubeadm/app/apis/kubeadm/validation:go_default_library",
         "//cmd/kubeadm/app/phases/certs:go_default_library",
         "//cmd/kubeadm/app/phases/kubeconfig:go_default_library",
+        "//cmd/kubeadm/app/phases/validate:go_default_library",
         "//cmd/kubeadm/app/util:go_default_library",
         "//vendor:github.com/spf13/cobra",
         "//vendor:k8s.io/apimachinery/pkg/util/net",

cmd/kubeadm/app/cmd/phases/phase.go

@@ -29,8 +29,11 @@ func NewCmdPhase(out io.Writer) *cobra.Command {
 		Short: "Invoke subsets of kubeadm functions separately for a manual install.",
 		RunE:  subCmdRunE("phase"),
 	}
+
 	cmd.AddCommand(NewCmdKubeConfig(out))
 	cmd.AddCommand(NewCmdCerts())
+	cmd.AddCommand(NewCmdValidate())
+
 	return cmd
 }
 

cmd/kubeadm/app/cmd/phases/validate.go

@@ -14,30 +14,30 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package master
+package phases
 
-const (
+import (
-	DummyDeployment = `
+	"os"
-apiVersion: extensions/v1beta1
+
-kind: Deployment
+	"k8s.io/kubernetes/cmd/kubeadm/app/phases/validate"
-metadata:
+
-  labels:
+	"github.com/spf13/cobra"
-    app: dummy
-  name: dummy
-  namespace: kube-system
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: dummy
-  template:
-    metadata:
-      labels:
-        app: dummy
-    spec:
-      containers:
-      - image: {{ .ImageRepository }}/pause-{{ .Arch }}:3.0
-        name: dummy
-      hostNetwork: true
-`
 )
+
+func NewCmdValidate() *cobra.Command {
+	var kubeconfig string
+
+	cmd := &cobra.Command{
+		Use:   "validate",
+		Short: "Run end to end validation",
+		RunE: func(cmd *cobra.Command, args []string) error {
+
+			return validate.Validate(kubeconfig)
+		},
+	}
+
+	//TODO: what's the convention for defaulting a kubeconfig?
+	cmd.Flags().StringVar(&kubeconfig, "kubeconfig", os.Getenv("HOME")+"/.kube/config", "path to kubeconfig")
+
+	return cmd
+}

cmd/kubeadm/app/master/BUILD

@@ -14,14 +14,12 @@ go_library(
         "apiclient.go",
         "manifests.go",
         "selfhosted.go",
-        "templates.go",
     ],
     tags = ["automanaged"],
     deps = [
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
         "//cmd/kubeadm/app/constants:go_default_library",
         "//cmd/kubeadm/app/images:go_default_library",
-        "//cmd/kubeadm/app/util:go_default_library",
         "//cmd/kubeadm/app/util/kubeconfig:go_default_library",
         "//pkg/bootstrap/api:go_default_library",
         "//pkg/kubeapiserver/authorizer/modes:go_default_library",
@@ -30,11 +28,9 @@ go_library(
         "//vendor:k8s.io/apimachinery/pkg/api/errors",
         "//vendor:k8s.io/apimachinery/pkg/api/resource",
         "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1",
-        "//vendor:k8s.io/apimachinery/pkg/runtime",
         "//vendor:k8s.io/apimachinery/pkg/util/intstr",
         "//vendor:k8s.io/apimachinery/pkg/util/wait",
         "//vendor:k8s.io/client-go/kubernetes",
-        "//vendor:k8s.io/client-go/pkg/api",
         "//vendor:k8s.io/client-go/pkg/api/v1",
         "//vendor:k8s.io/client-go/pkg/apis/extensions/v1beta1",
     ],

cmd/kubeadm/app/master/apiclient.go

@@ -18,20 +18,14 @@ package master
 
 import (
 	"fmt"
-	"runtime"
 	"time"
 
 	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	kuberuntime "k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/pkg/api"
 	"k8s.io/client-go/pkg/api/v1"
-	extensions "k8s.io/client-go/pkg/apis/extensions/v1beta1"
-	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
-	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
 )
 
@@ -44,22 +38,6 @@ func CreateClientAndWaitForAPI(file string) (*clientset.Clientset, error) {
 	fmt.Println("[apiclient] Created API client, waiting for the control plane to become ready")
 	WaitForAPI(client)
 
-	fmt.Println("[apiclient] Waiting for at least one node to register")
-	start := time.Now()
-	wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
-		nodeList, err := client.Nodes().List(metav1.ListOptions{})
-		if err != nil {
-			fmt.Println("[apiclient] Temporarily unable to list nodes (will retry)")
-			return false, nil
-		}
-		if len(nodeList.Items) < 1 {
-			return false, nil
-		}
-
-		fmt.Printf("[apiclient] First node has registered after %f seconds\n", time.Since(start).Seconds())
-		return true, nil
-	})
-
 	return client, nil
 }
 
@@ -92,49 +70,3 @@ func WaitForAPI(client *clientset.Clientset) {
 		return true, nil
 	})
 }
-
-func createAndWaitForADummyDeployment(client *clientset.Clientset) error {
-	dummyDeploymentBytes, err := kubeadmutil.ParseTemplate(DummyDeployment, struct{ ImageRepository, Arch string }{
-		ImageRepository: kubeadmapi.GlobalEnvParams.RepositoryPrefix,
-		Arch:            runtime.GOARCH,
-	})
-	if err != nil {
-		return fmt.Errorf("error when parsing dummy deployment template: %v", err)
-	}
-
-	dummyDeployment := &extensions.Deployment{}
-	if err := kuberuntime.DecodeInto(api.Codecs.UniversalDecoder(), dummyDeploymentBytes, dummyDeployment); err != nil {
-		return fmt.Errorf("unable to decode dummy deployment %v", err)
-	}
-
-	wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
-		// TODO: we should check the error, as some cases may be fatal
-		if _, err := client.ExtensionsV1beta1().Deployments(metav1.NamespaceSystem).Create(dummyDeployment); err != nil {
-			fmt.Printf("[apiclient] Failed to create test deployment [%v] (will retry)\n", err)
-			return false, nil
-		}
-		return true, nil
-	})
-
-	wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
-		d, err := client.ExtensionsV1beta1().Deployments(metav1.NamespaceSystem).Get("dummy", metav1.GetOptions{})
-		if err != nil {
-			fmt.Printf("[apiclient] Failed to get test deployment [%v] (will retry)\n", err)
-			return false, nil
-		}
-		if d.Status.AvailableReplicas < 1 {
-			return false, nil
-		}
-		return true, nil
-	})
-
-	fmt.Println("[apiclient] Test deployment succeeded")
-
-	foreground := metav1.DeletePropagationForeground
-	if err := client.ExtensionsV1beta1().Deployments(metav1.NamespaceSystem).Delete("dummy", &metav1.DeleteOptions{
-		PropagationPolicy: &foreground,
-	}); err != nil {
-		fmt.Printf("[apiclient] Failed to delete test deployment [%v] (will ignore)\n", err)
-	}
-	return nil
-}

cmd/kubeadm/app/phases/validate/BUILD

@@ -0,0 +1,40 @@
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["validate.go"],
+    tags = ["automanaged"],
+    deps = [
+        "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
+        "//cmd/kubeadm/app/constants:go_default_library",
+        "//cmd/kubeadm/app/util:go_default_library",
+        "//cmd/kubeadm/app/util/kubeconfig:go_default_library",
+        "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1",
+        "//vendor:k8s.io/apimachinery/pkg/runtime",
+        "//vendor:k8s.io/apimachinery/pkg/util/wait",
+        "//vendor:k8s.io/client-go/kubernetes",
+        "//vendor:k8s.io/client-go/pkg/api",
+        "//vendor:k8s.io/client-go/pkg/api/v1",
+        "//vendor:k8s.io/client-go/pkg/apis/extensions/v1beta1",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

cmd/kubeadm/app/phases/validate/validate.go

@@ -0,0 +1,139 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package validate
+
+import (
+	"fmt"
+	"runtime"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	kuberuntime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/pkg/api"
+	"k8s.io/client-go/pkg/api/v1"
+	extensions "k8s.io/client-go/pkg/apis/extensions/v1beta1"
+	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
+	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
+)
+
+func Validate(kubeconfigPath string) error {
+	client, err := kubeconfigutil.ClientSetFromFile(kubeconfigPath)
+	if err != nil {
+		return err
+	}
+	fmt.Println("[validate] Waiting for at least one node to register and become ready")
+	start := time.Now()
+	wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
+		nodeList, err := client.Nodes().List(metav1.ListOptions{})
+		if err != nil {
+			fmt.Println("[validate] Temporarily unable to list nodes (will retry)")
+			return false, nil
+		}
+		if len(nodeList.Items) < 1 {
+			return false, nil
+		}
+		n := &nodeList.Items[0]
+		if !v1.IsNodeReady(n) {
+			fmt.Println("[validate] First node has registered, but is not ready yet")
+			return false, nil
+		}
+
+		fmt.Printf("[validate] First node is ready after %f seconds\n", time.Since(start).Seconds())
+		return true, nil
+	})
+
+	if err := createAndWaitForADummyDeployment(client); err != nil {
+		return err
+	}
+	return nil
+}
+
+func createAndWaitForADummyDeployment(client *clientset.Clientset) error {
+	dummyDeploymentBytes, err := kubeadmutil.ParseTemplate(DummyDeployment, struct{ ImageRepository, Arch string }{
+		ImageRepository: kubeadmapi.GlobalEnvParams.RepositoryPrefix,
+		Arch:            runtime.GOARCH,
+	})
+	if err != nil {
+		return fmt.Errorf("error when parsing dummy deployment template: %v", err)
+	}
+
+	dummyDeployment := &extensions.Deployment{}
+	if err := kuberuntime.DecodeInto(api.Codecs.UniversalDecoder(), dummyDeploymentBytes, dummyDeployment); err != nil {
+		return fmt.Errorf("unable to decode dummy deployment %v", err)
+	}
+
+	wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
+		// TODO: we should check the error, as some cases may be fatal
+		if _, err := client.ExtensionsV1beta1().Deployments(metav1.NamespaceSystem).Create(dummyDeployment); err != nil {
+			fmt.Printf("[validate] Failed to create test deployment [%v] (will retry)\n", err)
+			return false, nil
+		}
+		return true, nil
+	})
+
+	wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
+		d, err := client.ExtensionsV1beta1().Deployments(metav1.NamespaceSystem).Get("dummy", metav1.GetOptions{})
+		if err != nil {
+			fmt.Printf("[validate] Failed to get test deployment [%v] (will retry)\n", err)
+			return false, nil
+		}
+		if d.Status.AvailableReplicas < 1 {
+			return false, nil
+		}
+		return true, nil
+	})
+
+	fmt.Println("[validate] Test deployment succeeded")
+
+	foreground := metav1.DeletePropagationForeground
+	if err := client.ExtensionsV1beta1().Deployments(metav1.NamespaceSystem).Delete("dummy", &metav1.DeleteOptions{
+		PropagationPolicy: &foreground,
+	}); err != nil {
+		fmt.Printf("[validate] Failed to delete test deployment [%v] (will ignore)\n", err)
+	}
+	return nil
+}
+
+const (
+	DummyDeployment = `
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  labels:
+    app: dummy
+  name: dummy
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: dummy
+  template:
+    metadata:
+      labels:
+        app: dummy
+    spec:
+      containers:
+      - image: {{ .ImageRepository }}/pause-{{ .Arch }}:3.0
+        name: dummy
+      hostNetwork: true
+`
+)