Merge pull request #50762 from jamiehannaford/kubeadm-gated-secrets

Automatic merge from submit-queue (batch tested with PRs 41901, 50762, 50756)

Feature-gate self-hosted secrets

**What this PR does / why we need it**:

Feature gates now select whether secrets are used for TLS cert storage in self-hosted clusters.

**Release note**:
```release-note
TLS cert storage for self-hosted clusters is now configurable. You can store them as secrets (alpha) or as usual host mounts.
```

/cc @luxas

cmd/kubeadm/app/cmd/features/features.go

@@ -33,8 +33,7 @@ type FeatureList map[utilfeature.Feature]utilfeature.FeatureSpec
 
 // Enabled indicates whether a feature name has been enabled
 func Enabled(featureList map[string]bool, featureName utilfeature.Feature) bool {
-	_, ok := featureList[string(featureName)]
+	return featureList[string(featureName)]
-	return ok
 }
 
 // Supports indicates whether a feature name is supported on the given

cmd/kubeadm/app/phases/selfhosting/BUILD

@@ -16,6 +16,7 @@ go_test(
     library = ":go_default_library",
     deps = [
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
+        "//cmd/kubeadm/app/cmd/features:go_default_library",
         "//cmd/kubeadm/app/constants:go_default_library",
         "//vendor/github.com/ghodss/yaml:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
@@ -31,6 +32,7 @@ go_library(
     ],
     deps = [
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
+        "//cmd/kubeadm/app/cmd/features:go_default_library",
         "//cmd/kubeadm/app/constants:go_default_library",
         "//cmd/kubeadm/app/util/apiclient:go_default_library",
         "//pkg/api:go_default_library",

cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go

@@ -79,7 +79,7 @@ func setRightDNSPolicyOnPodSpec(cfg *kubeadmapi.MasterConfiguration, podSpec *v1
 
 // setVolumesOnKubeAPIServerPodSpec makes sure the self-hosted api server has the required files
 func setVolumesOnKubeAPIServerPodSpec(cfg *kubeadmapi.MasterConfiguration, podSpec *v1.PodSpec) {
-	setK8sVolume(apiServerProjectedVolume, cfg, podSpec)
+	setK8sVolume(apiServerVolume, cfg, podSpec)
 	for _, c := range podSpec.Containers {
 		c.VolumeMounts = append(c.VolumeMounts, k8sSelfHostedVolumeMount())
 	}
@@ -87,7 +87,7 @@ func setVolumesOnKubeAPIServerPodSpec(cfg *kubeadmapi.MasterConfiguration, podSp
 
 // setVolumesOnKubeControllerManagerPodSpec makes sure the self-hosted controller manager has the required files
 func setVolumesOnKubeControllerManagerPodSpec(cfg *kubeadmapi.MasterConfiguration, podSpec *v1.PodSpec) {
-	setK8sVolume(controllerManagerProjectedVolume, cfg, podSpec)
+	setK8sVolume(controllerManagerVolume, cfg, podSpec)
 	for _, c := range podSpec.Containers {
 		c.VolumeMounts = append(c.VolumeMounts, k8sSelfHostedVolumeMount())
 	}
@@ -95,7 +95,7 @@ func setVolumesOnKubeControllerManagerPodSpec(cfg *kubeadmapi.MasterConfiguratio
 
 // setVolumesOnKubeSchedulerPodSpec makes sure the self-hosted scheduler has the required files
 func setVolumesOnKubeSchedulerPodSpec(cfg *kubeadmapi.MasterConfiguration, podSpec *v1.PodSpec) {
-	setK8sVolume(schedulerProjectedVolume, cfg, podSpec)
+	setK8sVolume(schedulerVolume, cfg, podSpec)
 	for _, c := range podSpec.Containers {
 		c.VolumeMounts = append(c.VolumeMounts, k8sSelfHostedVolumeMount())
 	}

cmd/kubeadm/app/phases/selfhosting/selfhosting.go

@@ -28,6 +28,7 @@ import (
 	kuberuntime "k8s.io/apimachinery/pkg/runtime"
 	clientset "k8s.io/client-go/kubernetes"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/features"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
 	"k8s.io/kubernetes/pkg/api"
@@ -46,12 +47,13 @@ import (
 // 9. Do that for the kube-apiserver, kube-controller-manager and kube-scheduler in a loop
 func CreateSelfHostedControlPlane(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error {
 
-	if err := createTLSSecrets(cfg, client); err != nil {
+	if features.Enabled(cfg.FeatureFlags, features.StoreCertsInSecrets) {
-		return err
+		if err := createTLSSecrets(cfg, client); err != nil {
-	}
+			return err
-
+		}
-	if err := createOpaqueSecrets(cfg, client); err != nil {
+		if err := createOpaqueSecrets(cfg, client); err != nil {
-		return err
+			return err
+		}
 	}
 
 	for _, componentName := range kubeadmconstants.MasterComponents {

cmd/kubeadm/app/phases/selfhosting/selfhosting_test.go

@@ -21,14 +21,88 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"strings"
 	"testing"
 
 	"github.com/ghodss/yaml"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/features"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 )
 
 const (
+	apiProjectedSecret = `- name: k8s
+    projected:
+      sources:
+      - secret:
+          items:
+          - key: tls.crt
+            path: ca.crt
+          - key: tls.key
+            path: ca.key
+          name: ca
+      - secret:
+          items:
+          - key: tls.crt
+            path: apiserver.crt
+          - key: tls.key
+            path: apiserver.key
+          name: apiserver
+      - secret:
+          items:
+          - key: tls.crt
+            path: apiserver-kubelet-client.crt
+          - key: tls.key
+            path: apiserver-kubelet-client.key
+          name: apiserver-kubelet-client
+      - secret:
+          items:
+          - key: tls.crt
+            path: sa.pub
+          - key: tls.key
+            path: sa.key
+          name: sa
+      - secret:
+          items:
+          - key: tls.crt
+            path: front-proxy-ca.crt
+          name: front-proxy-ca
+      - secret:
+          items:
+          - key: tls.crt
+            path: front-proxy-client.crt
+          - key: tls.key
+            path: front-proxy-client.key
+          name: front-proxy-client`
+
+	controllerManagerProjectedSecret = `- name: k8s
+    projected:
+      sources:
+      - secret:
+          name: controller-manager.conf
+      - secret:
+          items:
+          - key: tls.crt
+            path: ca.crt
+          - key: tls.key
+            path: ca.key
+          name: ca
+      - secret:
+          items:
+          - key: tls.key
+            path: sa.key
+          name: sa`
+
+	schedulerProjectedSecret = `- name: k8s
+    projected:
+      sources:
+      - secret:
+          name: scheduler.conf`
+
+	hostPathVol = `- hostPath:
+      path: /etc/kubernetes
+    name: k8s`
+
 	testAPIServerPod = `
 apiVersion: v1
 kind: Pod
@@ -92,49 +166,7 @@ spec:
       name: pki
   hostNetwork: true
   volumes:
-  - name: k8s
+  %s
-    projected:
-      sources:
-      - secret:
-          items:
-          - key: tls.crt
-            path: ca.crt
-          - key: tls.key
-            path: ca.key
-          name: ca
-      - secret:
-          items:
-          - key: tls.crt
-            path: apiserver.crt
-          - key: tls.key
-            path: apiserver.key
-          name: apiserver
-      - secret:
-          items:
-          - key: tls.crt
-            path: apiserver-kubelet-client.crt
-          - key: tls.key
-            path: apiserver-kubelet-client.key
-          name: apiserver-kubelet-client
-      - secret:
-          items:
-          - key: tls.crt
-            path: sa.pub
-          - key: tls.key
-            path: sa.key
-          name: sa
-      - secret:
-          items:
-          - key: tls.crt
-            path: front-proxy-ca.crt
-          name: front-proxy-ca
-      - secret:
-          items:
-          - key: tls.crt
-            path: front-proxy-client.crt
-          - key: tls.key
-            path: front-proxy-client.key
-          name: front-proxy-client
   - hostPath:
       path: /etc/ssl/certs
     name: certs
@@ -213,49 +245,7 @@ spec:
       - effect: NoSchedule
         key: node-role.kubernetes.io/master
       volumes:
-      - name: k8s
+      %s
-        projected:
-          sources:
-          - secret:
-              items:
-              - key: tls.crt
-                path: ca.crt
-              - key: tls.key
-                path: ca.key
-              name: ca
-          - secret:
-              items:
-              - key: tls.crt
-                path: apiserver.crt
-              - key: tls.key
-                path: apiserver.key
-              name: apiserver
-          - secret:
-              items:
-              - key: tls.crt
-                path: apiserver-kubelet-client.crt
-              - key: tls.key
-                path: apiserver-kubelet-client.key
-              name: apiserver-kubelet-client
-          - secret:
-              items:
-              - key: tls.crt
-                path: sa.pub
-              - key: tls.key
-                path: sa.key
-              name: sa
-          - secret:
-              items:
-              - key: tls.crt
-                path: front-proxy-ca.crt
-              name: front-proxy-ca
-          - secret:
-              items:
-              - key: tls.crt
-                path: front-proxy-client.crt
-              - key: tls.key
-                path: front-proxy-client.key
-              name: front-proxy-client
       - hostPath:
           path: /etc/ssl/certs
         name: certs
@@ -319,23 +309,7 @@ spec:
       name: pki
   hostNetwork: true
   volumes:
-  - name: k8s
+  %s
-    projected:
-      sources:
-      - secret:
-          name: controller-manager.conf
-      - secret:
-          items:
-          - key: tls.crt
-            path: ca.crt
-          - key: tls.key
-            path: ca.key
-          name: ca
-      - secret:
-          items:
-          - key: tls.key
-            path: sa.key
-          name: sa
   - hostPath:
       path: /etc/ssl/certs
     name: certs
@@ -400,23 +374,7 @@ spec:
       - effect: NoSchedule
         key: node-role.kubernetes.io/master
       volumes:
-      - name: k8s
+      %s
-        projected:
-          sources:
-          - secret:
-              name: controller-manager.conf
-          - secret:
-              items:
-              - key: tls.crt
-                path: ca.crt
-              - key: tls.key
-                path: ca.key
-              name: ca
-          - secret:
-              items:
-              - key: tls.key
-                path: sa.key
-              name: sa
       - hostPath:
           path: /etc/ssl/certs
         name: certs
@@ -470,11 +428,7 @@ spec:
       readOnly: true
   hostNetwork: true
   volumes:
-  - name: k8s
+  %s
-    projected:
-      sources:
-      - secret:
-          name: scheduler.conf
 status: {}
 `
 
@@ -523,11 +477,7 @@ spec:
       - effect: NoSchedule
         key: node-role.kubernetes.io/master
       volumes:
-      - name: k8s
+      %s
-        projected:
-          sources:
-          - secret:
-              name: scheduler.conf
   updateStrategy: {}
 status:
   currentNumberScheduled: 0
@@ -537,26 +487,67 @@ status:
 `
 )
 
+var (
+	testAPIServerSecretsPod  = fmt.Sprintf(testAPIServerPod, apiProjectedSecret)
+	testAPIServerSecretsDS   = fmt.Sprintf(testAPIServerDaemonSet, indentString(apiProjectedSecret, 4))
+	testAPIServerHostPathPod = fmt.Sprintf(testAPIServerPod, hostPathVol)
+	testAPIServerHostPathDS  = fmt.Sprintf(testAPIServerDaemonSet, indentString(hostPathVol, 4))
+
+	testSchedulerSecretsPod  = fmt.Sprintf(testSchedulerPod, schedulerProjectedSecret)
+	testSchedulerSecretsDS   = fmt.Sprintf(testSchedulerDaemonSet, indentString(schedulerProjectedSecret, 4))
+	testSchedulerHostPathPod = fmt.Sprintf(testSchedulerPod, hostPathVol)
+	testSchedulerHostPathDS  = fmt.Sprintf(testSchedulerDaemonSet, indentString(hostPathVol, 4))
+
+	testControllerManagerSecretsPod  = fmt.Sprintf(testControllerManagerPod, controllerManagerProjectedSecret)
+	testControllerManagerSecretsDS   = fmt.Sprintf(testControllerManagerDaemonSet, indentString(controllerManagerProjectedSecret, 4))
+	testControllerManagerHostPathPod = fmt.Sprintf(testControllerManagerPod, hostPathVol)
+	testControllerManagerHostPathDS  = fmt.Sprintf(testControllerManagerDaemonSet, indentString(hostPathVol, 4))
+)
+
 func TestBuildDaemonSet(t *testing.T) {
 	var tests = []struct {
-		component string
+		component         string
-		podBytes  []byte
+		podBytes          []byte
-		dsBytes   []byte
+		dsBytes           []byte
+		selfHostedSecrets bool
 	}{
+		// vols as secrets
 		{
-			component: kubeadmconstants.KubeAPIServer,
+			component:         kubeadmconstants.KubeAPIServer,
-			podBytes:  []byte(testAPIServerPod),
+			podBytes:          []byte(testAPIServerSecretsPod),
-			dsBytes:   []byte(testAPIServerDaemonSet),
+			dsBytes:           []byte(testAPIServerSecretsDS),
+			selfHostedSecrets: true,
 		},
 		{
-			component: kubeadmconstants.KubeControllerManager,
+			component:         kubeadmconstants.KubeControllerManager,
-			podBytes:  []byte(testControllerManagerPod),
+			podBytes:          []byte(testControllerManagerSecretsPod),
-			dsBytes:   []byte(testControllerManagerDaemonSet),
+			dsBytes:           []byte(testControllerManagerSecretsDS),
+			selfHostedSecrets: true,
 		},
 		{
-			component: kubeadmconstants.KubeScheduler,
+			component:         kubeadmconstants.KubeScheduler,
-			podBytes:  []byte(testSchedulerPod),
+			podBytes:          []byte(testSchedulerSecretsPod),
-			dsBytes:   []byte(testSchedulerDaemonSet),
+			dsBytes:           []byte(testSchedulerSecretsDS),
+			selfHostedSecrets: true,
+		},
+		// hostPath vols
+		{
+			component:         kubeadmconstants.KubeAPIServer,
+			podBytes:          []byte(testAPIServerHostPathPod),
+			dsBytes:           []byte(testAPIServerHostPathDS),
+			selfHostedSecrets: false,
+		},
+		{
+			component:         kubeadmconstants.KubeControllerManager,
+			podBytes:          []byte(testControllerManagerHostPathPod),
+			dsBytes:           []byte(testControllerManagerHostPathDS),
+			selfHostedSecrets: false,
+		},
+		{
+			component:         kubeadmconstants.KubeScheduler,
+			podBytes:          []byte(testSchedulerHostPathPod),
+			dsBytes:           []byte(testSchedulerHostPathDS),
+			selfHostedSecrets: false,
 		},
 	}
 
@@ -566,10 +557,13 @@ func TestBuildDaemonSet(t *testing.T) {
 
 		podSpec, err := loadPodSpecFromFile(tempFile)
 		if err != nil {
-			t.Fatalf("couldn't load the specified Pod")
+			t.Fatalf("couldn't load the specified Pod: %v", err)
+		}
+
+		cfg := &kubeadmapi.MasterConfiguration{
+			FeatureFlags: map[string]bool{string(features.StoreCertsInSecrets): rt.selfHostedSecrets},
 		}
 
-		cfg := &kubeadmapi.MasterConfiguration{}
 		ds := buildDaemonSet(cfg, rt.component, podSpec)
 		dsBytes, err := yaml.Marshal(ds)
 		if err != nil {
@@ -577,7 +571,7 @@ func TestBuildDaemonSet(t *testing.T) {
 		}
 
 		if !bytes.Equal(dsBytes, rt.dsBytes) {
-			t.Errorf("failed TestBuildDaemonSet:\nexpected:\n%s\nsaw:\n%s", rt.dsBytes, dsBytes)
+			t.Errorf("failed TestBuildDaemonSet for name=%s (secrets=%t):\nexpected:\n%s\nsaw:\n%s", rt.component, rt.selfHostedSecrets, rt.dsBytes, dsBytes)
 		}
 	}
 }
@@ -657,3 +651,18 @@ func createTempFileWithContent(content []byte) (string, error) {
 	}
 	return tempFile.Name(), nil
 }
+
+func indentString(input string, count int) string {
+	output := ""
+	lines := strings.Split(input, "\n")
+	for i, line := range lines {
+		if i > 0 {
+			output += strings.Repeat(" ", count)
+		}
+		output += line
+		if i < len(lines)-1 {
+			output += "\n"
+		}
+	}
+	return output
+}

cmd/kubeadm/app/phases/selfhosting/selfhosting_volumes.go

@@ -25,9 +25,15 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clientset "k8s.io/client-go/kubernetes"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/features"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 )
 
+const (
+	volumeName      = "k8s"
+	volumeMountName = "k8s"
+)
+
 type tlsKeyPair struct {
 	name string
 	cert string
@@ -36,16 +42,16 @@ type tlsKeyPair struct {
 
 func k8sSelfHostedVolumeMount() v1.VolumeMount {
 	return v1.VolumeMount{
-		Name:      "k8s",
+		Name:      volumeMountName,
 		MountPath: kubeadmconstants.KubernetesDir,
 		ReadOnly:  true,
 	}
 }
 
-func apiServerProjectedVolume(cfg *kubeadmapi.MasterConfiguration) v1.Volume {
+func apiServerVolume(cfg *kubeadmapi.MasterConfiguration) v1.Volume {
-	return v1.Volume{
+	var volumeSource v1.VolumeSource
-		Name: "k8s",
+	if features.Enabled(cfg.FeatureFlags, features.StoreCertsInSecrets) {
-		VolumeSource: v1.VolumeSource{
+		volumeSource = v1.VolumeSource{
 			Projected: &v1.ProjectedVolumeSource{
 				Sources: []v1.VolumeProjection{
 					{
@@ -148,14 +154,24 @@ func apiServerProjectedVolume(cfg *kubeadmapi.MasterConfiguration) v1.Volume {
 					},
 				},
 			},
-		},
+		}
+	} else {
+		volumeSource = v1.VolumeSource{
+			HostPath: &v1.HostPathVolumeSource{
+				Path: kubeadmconstants.KubernetesDir,
+			},
+		}
+	}
+	return v1.Volume{
+		Name:         volumeName,
+		VolumeSource: volumeSource,
 	}
 }
 
-func schedulerProjectedVolume(cfg *kubeadmapi.MasterConfiguration) v1.Volume {
+func schedulerVolume(cfg *kubeadmapi.MasterConfiguration) v1.Volume {
-	return v1.Volume{
+	var volumeSource v1.VolumeSource
-		Name: "k8s",
+	if features.Enabled(cfg.FeatureFlags, features.StoreCertsInSecrets) {
-		VolumeSource: v1.VolumeSource{
+		volumeSource = v1.VolumeSource{
 			Projected: &v1.ProjectedVolumeSource{
 				Sources: []v1.VolumeProjection{
 					{
@@ -167,14 +183,24 @@ func schedulerProjectedVolume(cfg *kubeadmapi.MasterConfiguration) v1.Volume {
 					},
 				},
 			},
-		},
+		}
+	} else {
+		volumeSource = v1.VolumeSource{
+			HostPath: &v1.HostPathVolumeSource{
+				Path: kubeadmconstants.KubernetesDir,
+			},
+		}
+	}
+	return v1.Volume{
+		Name:         volumeName,
+		VolumeSource: volumeSource,
 	}
 }
 
-func controllerManagerProjectedVolume(cfg *kubeadmapi.MasterConfiguration) v1.Volume {
+func controllerManagerVolume(cfg *kubeadmapi.MasterConfiguration) v1.Volume {
-	return v1.Volume{
+	var volumeSource v1.VolumeSource
-		Name: "k8s",
+	if features.Enabled(cfg.FeatureFlags, features.StoreCertsInSecrets) {
-		VolumeSource: v1.VolumeSource{
+		volumeSource = v1.VolumeSource{
 			Projected: &v1.ProjectedVolumeSource{
 				Sources: []v1.VolumeProjection{
 					{
@@ -216,7 +242,17 @@ func controllerManagerProjectedVolume(cfg *kubeadmapi.MasterConfiguration) v1.Vo
 					},
 				},
 			},
-		},
+		}
+	} else {
+		volumeSource = v1.VolumeSource{
+			HostPath: &v1.HostPathVolumeSource{
+				Path: kubeadmconstants.KubernetesDir,
+			},
+		}
+	}
+	return v1.Volume{
+		Name:         volumeName,
+		VolumeSource: volumeSource,
 	}
 }