Merge pull request #50762 from jamiehannaford/kubeadm-gated-secrets

Automatic merge from submit-queue (batch tested with PRs 41901, 50762, 50756) Feature-gate self-hosted secrets **What this PR does / why we need it**: Feature gates now select whether secrets are used for TLS cert storage in self-hosted clusters. **Release note**: ```release-note TLS cert storage for self-hosted clusters is now configurable. You can store them as secrets (alpha) or as usual host mounts. ``` /cc @luxas
2025-07-21 19:01:49 +00:00 · 2017-08-16 16:25:20 -07:00 · 2017-08-16 16:25:20 -07:00 · 4a15d32bec
commit 4a15d32bec
parent 4032896ef1 abedc49b71
6 changed files with 220 additions and 172 deletions
--- a/cmd/kubeadm/app/cmd/features/features.go
+++ b/cmd/kubeadm/app/cmd/features/features.go
@ -33,8 +33,7 @@ type FeatureList map[utilfeature.Feature]utilfeature.FeatureSpec

 // Enabled indicates whether a feature name has been enabled
 func Enabled(featureList map[string]bool, featureName utilfeature.Feature) bool {
-	_, ok := featureList[string(featureName)]
-	return ok
+	return featureList[string(featureName)]
 }

 // Supports indicates whether a feature name is supported on the given
--- a/cmd/kubeadm/app/phases/selfhosting/BUILD
+++ b/cmd/kubeadm/app/phases/selfhosting/BUILD
@ -16,6 +16,7 @@ go_test(
    library = ":go_default_library",
    deps = [
        "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
+        "//cmd/kubeadm/app/cmd/features:go_default_library",
        "//cmd/kubeadm/app/constants:go_default_library",
        "//vendor/github.com/ghodss/yaml:go_default_library",
        "//vendor/k8s.io/api/core/v1:go_default_library",
@ -31,6 +32,7 @@ go_library(
    ],
    deps = [
        "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
+        "//cmd/kubeadm/app/cmd/features:go_default_library",
        "//cmd/kubeadm/app/constants:go_default_library",
        "//cmd/kubeadm/app/util/apiclient:go_default_library",
        "//pkg/api:go_default_library",
--- a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go
+++ b/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go
@ -79,7 +79,7 @@ func setRightDNSPolicyOnPodSpec(cfg *kubeadmapi.MasterConfiguration, podSpec *v1

 // setVolumesOnKubeAPIServerPodSpec makes sure the self-hosted api server has the required files
 func setVolumesOnKubeAPIServerPodSpec(cfg *kubeadmapi.MasterConfiguration, podSpec *v1.PodSpec) {
-	setK8sVolume(apiServerProjectedVolume, cfg, podSpec)
+	setK8sVolume(apiServerVolume, cfg, podSpec)
 	for _, c := range podSpec.Containers {
 		c.VolumeMounts = append(c.VolumeMounts, k8sSelfHostedVolumeMount())
 	}
@ -87,7 +87,7 @@ func setVolumesOnKubeAPIServerPodSpec(cfg *kubeadmapi.MasterConfiguration, podSp

 // setVolumesOnKubeControllerManagerPodSpec makes sure the self-hosted controller manager has the required files
 func setVolumesOnKubeControllerManagerPodSpec(cfg *kubeadmapi.MasterConfiguration, podSpec *v1.PodSpec) {
-	setK8sVolume(controllerManagerProjectedVolume, cfg, podSpec)
+	setK8sVolume(controllerManagerVolume, cfg, podSpec)
 	for _, c := range podSpec.Containers {
 		c.VolumeMounts = append(c.VolumeMounts, k8sSelfHostedVolumeMount())
 	}
@ -95,7 +95,7 @@ func setVolumesOnKubeControllerManagerPodSpec(cfg *kubeadmapi.MasterConfiguratio

 // setVolumesOnKubeSchedulerPodSpec makes sure the self-hosted scheduler has the required files
 func setVolumesOnKubeSchedulerPodSpec(cfg *kubeadmapi.MasterConfiguration, podSpec *v1.PodSpec) {
-	setK8sVolume(schedulerProjectedVolume, cfg, podSpec)
+	setK8sVolume(schedulerVolume, cfg, podSpec)
 	for _, c := range podSpec.Containers {
 		c.VolumeMounts = append(c.VolumeMounts, k8sSelfHostedVolumeMount())
 	}
--- a/cmd/kubeadm/app/phases/selfhosting/selfhosting.go
+++ b/cmd/kubeadm/app/phases/selfhosting/selfhosting.go
@ -28,6 +28,7 @@ import (
 	kuberuntime "k8s.io/apimachinery/pkg/runtime"
 	clientset "k8s.io/client-go/kubernetes"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/features"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
 	"k8s.io/kubernetes/pkg/api"
@ -46,13 +47,14 @@ import (
 // 9. Do that for the kube-apiserver, kube-controller-manager and kube-scheduler in a loop
 func CreateSelfHostedControlPlane(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error {

+	if features.Enabled(cfg.FeatureFlags, features.StoreCertsInSecrets) {
 		if err := createTLSSecrets(cfg, client); err != nil {
 			return err
 		}
-
 		if err := createOpaqueSecrets(cfg, client); err != nil {
 			return err
 		}
+	}

 	for _, componentName := range kubeadmconstants.MasterComponents {
 		start := time.Now()
--- a/cmd/kubeadm/app/phases/selfhosting/selfhosting_test.go
+++ b/cmd/kubeadm/app/phases/selfhosting/selfhosting_test.go
@ -21,14 +21,88 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"strings"
 	"testing"

 	"github.com/ghodss/yaml"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/features"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 )

 const (
+	apiProjectedSecret = `- name: k8s
+    projected:
+      sources:
+      - secret:
+          items:
+          - key: tls.crt
+            path: ca.crt
+          - key: tls.key
+            path: ca.key
+          name: ca
+      - secret:
+          items:
+          - key: tls.crt
+            path: apiserver.crt
+          - key: tls.key
+            path: apiserver.key
+          name: apiserver
+      - secret:
+          items:
+          - key: tls.crt
+            path: apiserver-kubelet-client.crt
+          - key: tls.key
+            path: apiserver-kubelet-client.key
+          name: apiserver-kubelet-client
+      - secret:
+          items:
+          - key: tls.crt
+            path: sa.pub
+          - key: tls.key
+            path: sa.key
+          name: sa
+      - secret:
+          items:
+          - key: tls.crt
+            path: front-proxy-ca.crt
+          name: front-proxy-ca
+      - secret:
+          items:
+          - key: tls.crt
+            path: front-proxy-client.crt
+          - key: tls.key
+            path: front-proxy-client.key
+          name: front-proxy-client`
+
+	controllerManagerProjectedSecret = `- name: k8s
+    projected:
+      sources:
+      - secret:
+          name: controller-manager.conf
+      - secret:
+          items:
+          - key: tls.crt
+            path: ca.crt
+          - key: tls.key
+            path: ca.key
+          name: ca
+      - secret:
+          items:
+          - key: tls.key
+            path: sa.key
+          name: sa`
+
+	schedulerProjectedSecret = `- name: k8s
+    projected:
+      sources:
+      - secret:
+          name: scheduler.conf`
+
+	hostPathVol = `- hostPath:
+      path: /etc/kubernetes
+    name: k8s`
+
 	testAPIServerPod = `
 apiVersion: v1
 kind: Pod
@ -92,49 +166,7 @@ spec:
      name: pki
  hostNetwork: true
  volumes:
-  - name: k8s
-    projected:
-      sources:
-      - secret:
-          items:
-          - key: tls.crt
-            path: ca.crt
-          - key: tls.key
-            path: ca.key
-          name: ca
-      - secret:
-          items:
-          - key: tls.crt
-            path: apiserver.crt
-          - key: tls.key
-            path: apiserver.key
-          name: apiserver
-      - secret:
-          items:
-          - key: tls.crt
-            path: apiserver-kubelet-client.crt
-          - key: tls.key
-            path: apiserver-kubelet-client.key
-          name: apiserver-kubelet-client
-      - secret:
-          items:
-          - key: tls.crt
-            path: sa.pub
-          - key: tls.key
-            path: sa.key
-          name: sa
-      - secret:
-          items:
-          - key: tls.crt
-            path: front-proxy-ca.crt
-          name: front-proxy-ca
-      - secret:
-          items:
-          - key: tls.crt
-            path: front-proxy-client.crt
-          - key: tls.key
-            path: front-proxy-client.key
-          name: front-proxy-client
+  %s
  - hostPath:
      path: /etc/ssl/certs
    name: certs
@ -213,49 +245,7 @@ spec:
      - effect: NoSchedule
        key: node-role.kubernetes.io/master
      volumes:
-      - name: k8s
-        projected:
-          sources:
-          - secret:
-              items:
-              - key: tls.crt
-                path: ca.crt
-              - key: tls.key
-                path: ca.key
-              name: ca
-          - secret:
-              items:
-              - key: tls.crt
-                path: apiserver.crt
-              - key: tls.key
-                path: apiserver.key
-              name: apiserver
-          - secret:
-              items:
-              - key: tls.crt
-                path: apiserver-kubelet-client.crt
-              - key: tls.key
-                path: apiserver-kubelet-client.key
-              name: apiserver-kubelet-client
-          - secret:
-              items:
-              - key: tls.crt
-                path: sa.pub
-              - key: tls.key
-                path: sa.key
-              name: sa
-          - secret:
-              items:
-              - key: tls.crt
-                path: front-proxy-ca.crt
-              name: front-proxy-ca
-          - secret:
-              items:
-              - key: tls.crt
-                path: front-proxy-client.crt
-              - key: tls.key
-                path: front-proxy-client.key
-              name: front-proxy-client
+      %s
      - hostPath:
          path: /etc/ssl/certs
        name: certs
@ -319,23 +309,7 @@ spec:
      name: pki
  hostNetwork: true
  volumes:
-  - name: k8s
-    projected:
-      sources:
-      - secret:
-          name: controller-manager.conf
-      - secret:
-          items:
-          - key: tls.crt
-            path: ca.crt
-          - key: tls.key
-            path: ca.key
-          name: ca
-      - secret:
-          items:
-          - key: tls.key
-            path: sa.key
-          name: sa
+  %s
  - hostPath:
      path: /etc/ssl/certs
    name: certs
@ -400,23 +374,7 @@ spec:
      - effect: NoSchedule
        key: node-role.kubernetes.io/master
      volumes:
-      - name: k8s
-        projected:
-          sources:
-          - secret:
-              name: controller-manager.conf
-          - secret:
-              items:
-              - key: tls.crt
-                path: ca.crt
-              - key: tls.key
-                path: ca.key
-              name: ca
-          - secret:
-              items:
-              - key: tls.key
-                path: sa.key
-              name: sa
+      %s
      - hostPath:
          path: /etc/ssl/certs
        name: certs
@ -470,11 +428,7 @@ spec:
      readOnly: true
  hostNetwork: true
  volumes:
-  - name: k8s
-    projected:
-      sources:
-      - secret:
-          name: scheduler.conf
+  %s
 status: {}
 `

@ -523,11 +477,7 @@ spec:
      - effect: NoSchedule
        key: node-role.kubernetes.io/master
      volumes:
-      - name: k8s
-        projected:
-          sources:
-          - secret:
-              name: scheduler.conf
+      %s
  updateStrategy: {}
 status:
  currentNumberScheduled: 0
@ -537,26 +487,67 @@ status:
 `
 )

+var (
+	testAPIServerSecretsPod  = fmt.Sprintf(testAPIServerPod, apiProjectedSecret)
+	testAPIServerSecretsDS   = fmt.Sprintf(testAPIServerDaemonSet, indentString(apiProjectedSecret, 4))
+	testAPIServerHostPathPod = fmt.Sprintf(testAPIServerPod, hostPathVol)
+	testAPIServerHostPathDS  = fmt.Sprintf(testAPIServerDaemonSet, indentString(hostPathVol, 4))
+
+	testSchedulerSecretsPod  = fmt.Sprintf(testSchedulerPod, schedulerProjectedSecret)
+	testSchedulerSecretsDS   = fmt.Sprintf(testSchedulerDaemonSet, indentString(schedulerProjectedSecret, 4))
+	testSchedulerHostPathPod = fmt.Sprintf(testSchedulerPod, hostPathVol)
+	testSchedulerHostPathDS  = fmt.Sprintf(testSchedulerDaemonSet, indentString(hostPathVol, 4))
+
+	testControllerManagerSecretsPod  = fmt.Sprintf(testControllerManagerPod, controllerManagerProjectedSecret)
+	testControllerManagerSecretsDS   = fmt.Sprintf(testControllerManagerDaemonSet, indentString(controllerManagerProjectedSecret, 4))
+	testControllerManagerHostPathPod = fmt.Sprintf(testControllerManagerPod, hostPathVol)
+	testControllerManagerHostPathDS  = fmt.Sprintf(testControllerManagerDaemonSet, indentString(hostPathVol, 4))
+)
+
 func TestBuildDaemonSet(t *testing.T) {
 	var tests = []struct {
 		component         string
 		podBytes          []byte
 		dsBytes           []byte
+		selfHostedSecrets bool
 	}{
+		// vols as secrets
 		{
 			component:         kubeadmconstants.KubeAPIServer,
-			podBytes:  []byte(testAPIServerPod),
-			dsBytes:   []byte(testAPIServerDaemonSet),
+			podBytes:          []byte(testAPIServerSecretsPod),
+			dsBytes:           []byte(testAPIServerSecretsDS),
+			selfHostedSecrets: true,
 		},
 		{
 			component:         kubeadmconstants.KubeControllerManager,
-			podBytes:  []byte(testControllerManagerPod),
-			dsBytes:   []byte(testControllerManagerDaemonSet),
+			podBytes:          []byte(testControllerManagerSecretsPod),
+			dsBytes:           []byte(testControllerManagerSecretsDS),
+			selfHostedSecrets: true,
 		},
 		{
 			component:         kubeadmconstants.KubeScheduler,
-			podBytes:  []byte(testSchedulerPod),
-			dsBytes:   []byte(testSchedulerDaemonSet),
+			podBytes:          []byte(testSchedulerSecretsPod),
+			dsBytes:           []byte(testSchedulerSecretsDS),
+			selfHostedSecrets: true,
+		},
+		// hostPath vols
+		{
+			component:         kubeadmconstants.KubeAPIServer,
+			podBytes:          []byte(testAPIServerHostPathPod),
+			dsBytes:           []byte(testAPIServerHostPathDS),
+			selfHostedSecrets: false,
+		},
+		{
+			component:         kubeadmconstants.KubeControllerManager,
+			podBytes:          []byte(testControllerManagerHostPathPod),
+			dsBytes:           []byte(testControllerManagerHostPathDS),
+			selfHostedSecrets: false,
+		},
+		{
+			component:         kubeadmconstants.KubeScheduler,
+			podBytes:          []byte(testSchedulerHostPathPod),
+			dsBytes:           []byte(testSchedulerHostPathDS),
+			selfHostedSecrets: false,
 		},
 	}

@ -566,10 +557,13 @@ func TestBuildDaemonSet(t *testing.T) {

 		podSpec, err := loadPodSpecFromFile(tempFile)
 		if err != nil {
-			t.Fatalf("couldn't load the specified Pod")
+			t.Fatalf("couldn't load the specified Pod: %v", err)
+		}
+
+		cfg := &kubeadmapi.MasterConfiguration{
+			FeatureFlags: map[string]bool{string(features.StoreCertsInSecrets): rt.selfHostedSecrets},
 		}

-		cfg := &kubeadmapi.MasterConfiguration{}
 		ds := buildDaemonSet(cfg, rt.component, podSpec)
 		dsBytes, err := yaml.Marshal(ds)
 		if err != nil {
@ -577,7 +571,7 @@ func TestBuildDaemonSet(t *testing.T) {
 		}

 		if !bytes.Equal(dsBytes, rt.dsBytes) {
-			t.Errorf("failed TestBuildDaemonSet:\nexpected:\n%s\nsaw:\n%s", rt.dsBytes, dsBytes)
+			t.Errorf("failed TestBuildDaemonSet for name=%s (secrets=%t):\nexpected:\n%s\nsaw:\n%s", rt.component, rt.selfHostedSecrets, rt.dsBytes, dsBytes)
 		}
 	}
 }
@ -657,3 +651,18 @@ func createTempFileWithContent(content []byte) (string, error) {
 	}
 	return tempFile.Name(), nil
 }
+
+func indentString(input string, count int) string {
+	output := ""
+	lines := strings.Split(input, "\n")
+	for i, line := range lines {
+		if i > 0 {
+			output += strings.Repeat(" ", count)
+		}
+		output += line
+		if i < len(lines)-1 {
+			output += "\n"
+		}
+	}
+	return output
+}
--- a/cmd/kubeadm/app/phases/selfhosting/selfhosting_volumes.go
+++ b/cmd/kubeadm/app/phases/selfhosting/selfhosting_volumes.go
@ -25,9 +25,15 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clientset "k8s.io/client-go/kubernetes"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/features"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 )

+const (
+	volumeName      = "k8s"
+	volumeMountName = "k8s"
+)
+
 type tlsKeyPair struct {
 	name string
 	cert string
@ -36,16 +42,16 @@ type tlsKeyPair struct {

 func k8sSelfHostedVolumeMount() v1.VolumeMount {
 	return v1.VolumeMount{
-		Name:      "k8s",
+		Name:      volumeMountName,
 		MountPath: kubeadmconstants.KubernetesDir,
 		ReadOnly:  true,
 	}
 }

-func apiServerProjectedVolume(cfg *kubeadmapi.MasterConfiguration) v1.Volume {
-	return v1.Volume{
-		Name: "k8s",
-		VolumeSource: v1.VolumeSource{
+func apiServerVolume(cfg *kubeadmapi.MasterConfiguration) v1.Volume {
+	var volumeSource v1.VolumeSource
+	if features.Enabled(cfg.FeatureFlags, features.StoreCertsInSecrets) {
+		volumeSource = v1.VolumeSource{
 			Projected: &v1.ProjectedVolumeSource{
 				Sources: []v1.VolumeProjection{
 					{
@ -148,14 +154,24 @@ func apiServerProjectedVolume(cfg *kubeadmapi.MasterConfiguration) v1.Volume {
 					},
 				},
 			},
+		}
+	} else {
+		volumeSource = v1.VolumeSource{
+			HostPath: &v1.HostPathVolumeSource{
+				Path: kubeadmconstants.KubernetesDir,
 			},
 		}
+	}
+	return v1.Volume{
+		Name:         volumeName,
+		VolumeSource: volumeSource,
+	}
 }

-func schedulerProjectedVolume(cfg *kubeadmapi.MasterConfiguration) v1.Volume {
-	return v1.Volume{
-		Name: "k8s",
-		VolumeSource: v1.VolumeSource{
+func schedulerVolume(cfg *kubeadmapi.MasterConfiguration) v1.Volume {
+	var volumeSource v1.VolumeSource
+	if features.Enabled(cfg.FeatureFlags, features.StoreCertsInSecrets) {
+		volumeSource = v1.VolumeSource{
 			Projected: &v1.ProjectedVolumeSource{
 				Sources: []v1.VolumeProjection{
 					{
@ -167,14 +183,24 @@ func schedulerProjectedVolume(cfg *kubeadmapi.MasterConfiguration) v1.Volume {
 					},
 				},
 			},
+		}
+	} else {
+		volumeSource = v1.VolumeSource{
+			HostPath: &v1.HostPathVolumeSource{
+				Path: kubeadmconstants.KubernetesDir,
 			},
 		}
+	}
+	return v1.Volume{
+		Name:         volumeName,
+		VolumeSource: volumeSource,
+	}
 }

-func controllerManagerProjectedVolume(cfg *kubeadmapi.MasterConfiguration) v1.Volume {
-	return v1.Volume{
-		Name: "k8s",
-		VolumeSource: v1.VolumeSource{
+func controllerManagerVolume(cfg *kubeadmapi.MasterConfiguration) v1.Volume {
+	var volumeSource v1.VolumeSource
+	if features.Enabled(cfg.FeatureFlags, features.StoreCertsInSecrets) {
+		volumeSource = v1.VolumeSource{
 			Projected: &v1.ProjectedVolumeSource{
 				Sources: []v1.VolumeProjection{
 					{
@ -216,8 +242,18 @@ func controllerManagerProjectedVolume(cfg *kubeadmapi.MasterConfiguration) v1.Vo
 					},
 				},
 			},
+		}
+	} else {
+		volumeSource = v1.VolumeSource{
+			HostPath: &v1.HostPathVolumeSource{
+				Path: kubeadmconstants.KubernetesDir,
 			},
 		}
+	}
+	return v1.Volume{
+		Name:         volumeName,
+		VolumeSource: volumeSource,
+	}
 }

 func createTLSSecrets(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error {