Merge pull request #44002 from mikedanese/vers-apiserver

Automatic merge from submit-queue

kubeadm: don't add proxy flag to apiservers below v1.7

cmd/kubeadm/app/master/BUILD

@@ -24,6 +24,7 @@ go_library(
         "//pkg/bootstrap/api:go_default_library",
         "//pkg/kubeapiserver/authorizer/modes:go_default_library",
         "//pkg/kubectl/cmd/util:go_default_library",
+        "//pkg/util/version:go_default_library",
         "//vendor:github.com/ghodss/yaml",
         "//vendor:k8s.io/apimachinery/pkg/api/errors",
         "//vendor:k8s.io/apimachinery/pkg/api/resource",
@@ -43,6 +44,7 @@ go_test(
     tags = ["automanaged"],
     deps = [
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
+        "//pkg/util/version:go_default_library",
         "//vendor:k8s.io/apimachinery/pkg/util/intstr",
         "//vendor:k8s.io/apimachinery/pkg/util/yaml",
         "//vendor:k8s.io/client-go/pkg/api/v1",

cmd/kubeadm/app/master/manifests.go

@@ -35,6 +35,7 @@ import (
 	bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api"
 	authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
 	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
+	"k8s.io/kubernetes/pkg/util/version"
 )
 
 // Static pod definitions in golang form are included below so that `kubeadm init` can get going.
@@ -52,6 +53,10 @@ const (
 	kubeProxy             = "kube-proxy"
 )
 
+var (
+	v170 = version.MustParseSemantic("v1.7.0-alpha.0")
+)
+
 // WriteStaticPodManifests builds manifest objects based on user provided configuration and then dumps it to disk
 // where kubelet will pick and schedule them.
 func WriteStaticPodManifests(cfg *kubeadmapi.MasterConfiguration) error {
@@ -68,12 +73,17 @@ func WriteStaticPodManifests(cfg *kubeadmapi.MasterConfiguration) error {
 		volumeMounts = append(volumeMounts, pkiVolumeMount())
 	}
 
+	k8sVersion, err := version.ParseSemantic(cfg.KubernetesVersion)
+	if err != nil {
+		return err
+	}
+
 	// Prepare static pod specs
 	staticPodSpecs := map[string]api.Pod{
 		kubeAPIServer: componentPod(api.Container{
 			Name:          kubeAPIServer,
 			Image:         images.GetCoreImage(images.KubeAPIServerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage),
-			Command:       getAPIServerCommand(cfg, false),
+			Command:       getAPIServerCommand(cfg, false, k8sVersion),
 			VolumeMounts:  volumeMounts,
 			LivenessProbe: componentProbe(int(cfg.API.BindPort), "/healthz", api.URISchemeHTTPS),
 			Resources:     componentResources("250m"),
@@ -293,7 +303,7 @@ func getComponentBaseCommand(component string) []string {
 	return []string{"kube-" + component}
 }
 
-func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool) []string {
+func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool, k8sVersion *version.Version) []string {
 	var command []string
 
 	// self-hosted apiserver needs to wait on a lock
@@ -323,9 +333,11 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool) [
 		"requestheader-extra-headers-prefix": "X-Remote-Extra-",
 		"requestheader-client-ca-file":       path.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertName),
 		"requestheader-allowed-names":        "front-proxy-client",
+	}
+	if k8sVersion.AtLeast(v170) {
 		// add options which allow the kube-apiserver to act as a front-proxy to aggregated API servers
-		"proxy-client-cert-file": path.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientCertName),
+		defaultArguments["proxy-client-cert-file"] = path.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientCertName)
-		"proxy-client-key-file":  path.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientKeyName),
+		defaultArguments["proxy-client-key-file"] = path.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientKeyName)
 	}
 
 	command = getComponentBaseCommand(apiServer)

cmd/kubeadm/app/master/manifests_test.go

@@ -29,6 +29,7 @@ import (
 	api "k8s.io/client-go/pkg/api/v1"
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	"k8s.io/kubernetes/pkg/util/version"
 )
 
 const testCertsDir = "/var/lib/certs"
@@ -47,12 +48,12 @@ func TestWriteStaticPodManifests(t *testing.T) {
 
 	var tests = []struct {
 		cfg                  *kubeadmapi.MasterConfiguration
-		expected             bool
+		expectErr            bool
 		expectedAPIProbePort int32
 	}{
 		{
-			cfg:      &kubeadmapi.MasterConfiguration{},
+			cfg:       &kubeadmapi.MasterConfiguration{},
-			expected: true,
+			expectErr: true,
 		},
 		{
 			cfg: &kubeadmapi.MasterConfiguration{
@@ -60,25 +61,29 @@ func TestWriteStaticPodManifests(t *testing.T) {
 					BindPort: 443,
 				},
 			},
-			expected:             true,
+			expectErr:            true,
 			expectedAPIProbePort: 443,
 		},
 	}
 	for _, rt := range tests {
 		actual := WriteStaticPodManifests(rt.cfg)
-		if (actual == nil) != rt.expected {
+		if (actual == nil) && rt.expectErr {
-			t.Errorf(
+			t.Error("expected an error from WriteStaticPodManifests but got none")
-				"failed WriteStaticPodManifests with an error:\n\texpected: %t\n\t  actual: %t",
+			continue
-				rt.expected,
+		}
-				(actual == nil),
+		if (actual != nil) && !rt.expectErr {
-			)
+			t.Errorf("didn't expect an error from WriteStaticPodManifests but got: %v", err)
+			continue
+		}
+		if rt.expectErr {
 			continue
 		}
 
+		// Below is dead code.
 		if rt.expectedAPIProbePort != 0 {
-			manifest, err := os.Open(fmt.Sprintf("%s/manifests/kube-apiserver.yaml", kubeadmapi.GlobalEnvParams.KubernetesDir))
+			manifest, err := os.Open(kubeadmapi.GlobalEnvParams.KubernetesDir + "/manifests/kube-apiserver.yaml")
 			if err != nil {
-				t.Error("WriteStaticPodManifests: error opening manifests/kube-apiserver.yaml")
+				t.Errorf("WriteStaticPodManifests: %v", err)
 				continue
 			}
 
@@ -433,9 +438,10 @@ func TestGetAPIServerCommand(t *testing.T) {
 	}{
 		{
 			cfg: &kubeadmapi.MasterConfiguration{
-				API:             kubeadm.API{BindPort: 123, AdvertiseAddress: "1.2.3.4"},
+				API:               kubeadm.API{BindPort: 123, AdvertiseAddress: "1.2.3.4"},
-				Networking:      kubeadm.Networking{ServiceSubnet: "bar"},
+				Networking:        kubeadm.Networking{ServiceSubnet: "bar"},
-				CertificatesDir: testCertsDir,
+				CertificatesDir:   testCertsDir,
+				KubernetesVersion: "v1.6.0",
 			},
 			expected: []string{
 				"kube-apiserver",
@@ -453,8 +459,6 @@ func TestGetAPIServerCommand(t *testing.T) {
 				"--storage-backend=etcd3",
 				"--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
 				"--experimental-bootstrap-token-auth=true",
-				"--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt",
-				"--proxy-client-key-file=/var/lib/certs/front-proxy-client.key",
 				"--requestheader-username-headers=X-Remote-User",
 				"--requestheader-group-headers=X-Remote-Group",
 				"--requestheader-extra-headers-prefix=X-Remote-Extra-",
@@ -467,9 +471,10 @@ func TestGetAPIServerCommand(t *testing.T) {
 		},
 		{
 			cfg: &kubeadmapi.MasterConfiguration{
-				API:             kubeadm.API{BindPort: 123, AdvertiseAddress: "4.3.2.1"},
+				API:               kubeadm.API{BindPort: 123, AdvertiseAddress: "4.3.2.1"},
-				Networking:      kubeadm.Networking{ServiceSubnet: "bar"},
+				Networking:        kubeadm.Networking{ServiceSubnet: "bar"},
-				CertificatesDir: testCertsDir,
+				CertificatesDir:   testCertsDir,
+				KubernetesVersion: "v1.6.0",
 			},
 			expected: []string{
 				"kube-apiserver",
@@ -487,8 +492,6 @@ func TestGetAPIServerCommand(t *testing.T) {
 				"--storage-backend=etcd3",
 				"--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
 				"--experimental-bootstrap-token-auth=true",
-				"--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt",
-				"--proxy-client-key-file=/var/lib/certs/front-proxy-client.key",
 				"--requestheader-username-headers=X-Remote-User",
 				"--requestheader-group-headers=X-Remote-Group",
 				"--requestheader-extra-headers-prefix=X-Remote-Extra-",
@@ -501,10 +504,47 @@ func TestGetAPIServerCommand(t *testing.T) {
 		},
 		{
 			cfg: &kubeadmapi.MasterConfiguration{
-				API:             kubeadm.API{BindPort: 123, AdvertiseAddress: "4.3.2.1"},
+				API:               kubeadm.API{BindPort: 123, AdvertiseAddress: "4.3.2.1"},
-				Networking:      kubeadm.Networking{ServiceSubnet: "bar"},
+				Networking:        kubeadm.Networking{ServiceSubnet: "bar"},
-				Etcd:            kubeadm.Etcd{CertFile: "fiz", KeyFile: "faz"},
+				Etcd:              kubeadm.Etcd{CertFile: "fiz", KeyFile: "faz"},
-				CertificatesDir: testCertsDir,
+				CertificatesDir:   testCertsDir,
+				KubernetesVersion: "v1.6.0",
+			},
+			expected: []string{
+				"kube-apiserver",
+				"--insecure-port=0",
+				"--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds",
+				"--service-cluster-ip-range=bar",
+				"--service-account-key-file=" + testCertsDir + "/sa.pub",
+				"--client-ca-file=" + testCertsDir + "/ca.crt",
+				"--tls-cert-file=" + testCertsDir + "/apiserver.crt",
+				"--tls-private-key-file=" + testCertsDir + "/apiserver.key",
+				"--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt",
+				"--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key",
+				fmt.Sprintf("--secure-port=%d", 123),
+				"--allow-privileged=true",
+				"--storage-backend=etcd3",
+				"--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
+				"--experimental-bootstrap-token-auth=true",
+				"--requestheader-username-headers=X-Remote-User",
+				"--requestheader-group-headers=X-Remote-Group",
+				"--requestheader-extra-headers-prefix=X-Remote-Extra-",
+				"--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt",
+				"--requestheader-allowed-names=front-proxy-client",
+				"--authorization-mode=RBAC",
+				"--advertise-address=4.3.2.1",
+				"--etcd-servers=http://127.0.0.1:2379",
+				"--etcd-certfile=fiz",
+				"--etcd-keyfile=faz",
+			},
+		},
+		{
+			cfg: &kubeadmapi.MasterConfiguration{
+				API:               kubeadm.API{BindPort: 123, AdvertiseAddress: "4.3.2.1"},
+				Networking:        kubeadm.Networking{ServiceSubnet: "bar"},
+				Etcd:              kubeadm.Etcd{CertFile: "fiz", KeyFile: "faz"},
+				CertificatesDir:   testCertsDir,
+				KubernetesVersion: "v1.7.0",
 			},
 			expected: []string{
 				"kube-apiserver",
@@ -539,7 +579,7 @@ func TestGetAPIServerCommand(t *testing.T) {
 	}
 
 	for _, rt := range tests {
-		actual := getAPIServerCommand(rt.cfg, false)
+		actual := getAPIServerCommand(rt.cfg, false, version.MustParseSemantic(rt.cfg.KubernetesVersion))
 		sort.Strings(actual)
 		sort.Strings(rt.expected)
 		if !reflect.DeepEqual(actual, rt.expected) {
@@ -760,3 +800,18 @@ func TestGetExtraParameters(t *testing.T) {
 		}
 	}
 }
+
+func TestVersionCompare(t *testing.T) {
+	versions := []string{
+		"v1.7.0-alpha.1",
+		"v1.7.0-beta.0",
+		"v1.7.0-rc.0",
+		"v1.7.0",
+		"v1.7.1",
+	}
+	for _, v := range versions {
+		if !version.MustParseSemantic(v).AtLeast(v170) {
+			t.Errorf("err")
+		}
+	}
+}

cmd/kubeadm/app/master/selfhosted.go

@@ -31,6 +31,7 @@ import (
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/images"
+	"k8s.io/kubernetes/pkg/util/version"
 )
 
 var (
@@ -74,7 +75,11 @@ func CreateSelfHostedControlPlane(cfg *kubeadmapi.MasterConfiguration, client *c
 func launchSelfHostedAPIServer(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset, volumes []v1.Volume, volumeMounts []v1.VolumeMount) error {
 	start := time.Now()
 
-	apiServer := getAPIServerDS(cfg, volumes, volumeMounts)
+	kubeVersion, err := version.ParseSemantic(cfg.KubernetesVersion)
+	if err != nil {
+		return err
+	}
+	apiServer := getAPIServerDS(cfg, volumes, volumeMounts, kubeVersion)
 	if _, err := client.Extensions().DaemonSets(metav1.NamespaceSystem).Create(&apiServer); err != nil {
 		return fmt.Errorf("failed to create self-hosted %q daemon set [%v]", kubeAPIServer, err)
 	}
@@ -183,7 +188,7 @@ func waitForPodsWithLabel(client *clientset.Clientset, appLabel string, mustBeRu
 }
 
 // Sources from bootkube templates.go
-func getAPIServerDS(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, volumeMounts []v1.VolumeMount) ext.DaemonSet {
+func getAPIServerDS(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, volumeMounts []v1.VolumeMount, kubeVersion *version.Version) ext.DaemonSet {
 	ds := ext.DaemonSet{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: "extensions/v1beta1",
@@ -211,7 +216,7 @@ func getAPIServerDS(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, vo
 						{
 							Name:          "self-hosted-" + kubeAPIServer,
 							Image:         images.GetCoreImage(images.KubeAPIServerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage),
-							Command:       getAPIServerCommand(cfg, true),
+							Command:       getAPIServerCommand(cfg, true, kubeVersion),
 							Env:           getSelfHostedAPIServerEnv(),
 							VolumeMounts:  volumeMounts,
 							LivenessProbe: componentProbe(6443, "/healthz", v1.URISchemeHTTPS),