diff --git a/cmd/kubeadm/app/cmd/features/features.go b/cmd/kubeadm/app/cmd/features/features.go index e77ef8412d2..62fa02574c6 100644 --- a/cmd/kubeadm/app/cmd/features/features.go +++ b/cmd/kubeadm/app/cmd/features/features.go @@ -33,8 +33,7 @@ type FeatureList map[utilfeature.Feature]utilfeature.FeatureSpec // Enabled indicates whether a feature name has been enabled func Enabled(featureList map[string]bool, featureName utilfeature.Feature) bool { - _, ok := featureList[string(featureName)] - return ok + return featureList[string(featureName)] } // Supports indicates whether a feature name is supported on the given diff --git a/cmd/kubeadm/app/phases/selfhosting/BUILD b/cmd/kubeadm/app/phases/selfhosting/BUILD index 564a1862295..4f541f6c0fe 100644 --- a/cmd/kubeadm/app/phases/selfhosting/BUILD +++ b/cmd/kubeadm/app/phases/selfhosting/BUILD @@ -16,6 +16,7 @@ go_test( library = ":go_default_library", deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", + "//cmd/kubeadm/app/cmd/features:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", "//vendor/github.com/ghodss/yaml:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", @@ -31,6 +32,7 @@ go_library( ], deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", + "//cmd/kubeadm/app/cmd/features:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/util/apiclient:go_default_library", "//pkg/api:go_default_library", diff --git a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go b/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go index bcddca3d74b..3bb36afca67 100644 --- a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go +++ b/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go @@ -79,7 +79,7 @@ func setRightDNSPolicyOnPodSpec(cfg *kubeadmapi.MasterConfiguration, podSpec *v1 // setVolumesOnKubeAPIServerPodSpec makes sure the self-hosted api server has the required files func setVolumesOnKubeAPIServerPodSpec(cfg *kubeadmapi.MasterConfiguration, podSpec *v1.PodSpec) { - setK8sVolume(apiServerProjectedVolume, cfg, podSpec) + setK8sVolume(apiServerVolume, cfg, podSpec) for _, c := range podSpec.Containers { c.VolumeMounts = append(c.VolumeMounts, k8sSelfHostedVolumeMount()) } @@ -87,7 +87,7 @@ func setVolumesOnKubeAPIServerPodSpec(cfg *kubeadmapi.MasterConfiguration, podSp // setVolumesOnKubeControllerManagerPodSpec makes sure the self-hosted controller manager has the required files func setVolumesOnKubeControllerManagerPodSpec(cfg *kubeadmapi.MasterConfiguration, podSpec *v1.PodSpec) { - setK8sVolume(controllerManagerProjectedVolume, cfg, podSpec) + setK8sVolume(controllerManagerVolume, cfg, podSpec) for _, c := range podSpec.Containers { c.VolumeMounts = append(c.VolumeMounts, k8sSelfHostedVolumeMount()) } @@ -95,7 +95,7 @@ func setVolumesOnKubeControllerManagerPodSpec(cfg *kubeadmapi.MasterConfiguratio // setVolumesOnKubeSchedulerPodSpec makes sure the self-hosted scheduler has the required files func setVolumesOnKubeSchedulerPodSpec(cfg *kubeadmapi.MasterConfiguration, podSpec *v1.PodSpec) { - setK8sVolume(schedulerProjectedVolume, cfg, podSpec) + setK8sVolume(schedulerVolume, cfg, podSpec) for _, c := range podSpec.Containers { c.VolumeMounts = append(c.VolumeMounts, k8sSelfHostedVolumeMount()) } diff --git a/cmd/kubeadm/app/phases/selfhosting/selfhosting.go b/cmd/kubeadm/app/phases/selfhosting/selfhosting.go index ec3338b9ffa..9016ce1881c 100644 --- a/cmd/kubeadm/app/phases/selfhosting/selfhosting.go +++ b/cmd/kubeadm/app/phases/selfhosting/selfhosting.go @@ -28,6 +28,7 @@ import ( kuberuntime "k8s.io/apimachinery/pkg/runtime" clientset "k8s.io/client-go/kubernetes" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/features" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" "k8s.io/kubernetes/pkg/api" @@ -46,12 +47,13 @@ import ( // 9. Do that for the kube-apiserver, kube-controller-manager and kube-scheduler in a loop func CreateSelfHostedControlPlane(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error { - if err := createTLSSecrets(cfg, client); err != nil { - return err - } - - if err := createOpaqueSecrets(cfg, client); err != nil { - return err + if features.Enabled(cfg.FeatureFlags, features.StoreCertsInSecrets) { + if err := createTLSSecrets(cfg, client); err != nil { + return err + } + if err := createOpaqueSecrets(cfg, client); err != nil { + return err + } } for _, componentName := range kubeadmconstants.MasterComponents { diff --git a/cmd/kubeadm/app/phases/selfhosting/selfhosting_test.go b/cmd/kubeadm/app/phases/selfhosting/selfhosting_test.go index f8d17589ee5..0a1afc69b7f 100644 --- a/cmd/kubeadm/app/phases/selfhosting/selfhosting_test.go +++ b/cmd/kubeadm/app/phases/selfhosting/selfhosting_test.go @@ -21,14 +21,88 @@ import ( "fmt" "io/ioutil" "os" + "strings" "testing" "github.com/ghodss/yaml" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/features" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" ) const ( + apiProjectedSecret = `- name: k8s + projected: + sources: + - secret: + items: + - key: tls.crt + path: ca.crt + - key: tls.key + path: ca.key + name: ca + - secret: + items: + - key: tls.crt + path: apiserver.crt + - key: tls.key + path: apiserver.key + name: apiserver + - secret: + items: + - key: tls.crt + path: apiserver-kubelet-client.crt + - key: tls.key + path: apiserver-kubelet-client.key + name: apiserver-kubelet-client + - secret: + items: + - key: tls.crt + path: sa.pub + - key: tls.key + path: sa.key + name: sa + - secret: + items: + - key: tls.crt + path: front-proxy-ca.crt + name: front-proxy-ca + - secret: + items: + - key: tls.crt + path: front-proxy-client.crt + - key: tls.key + path: front-proxy-client.key + name: front-proxy-client` + + controllerManagerProjectedSecret = `- name: k8s + projected: + sources: + - secret: + name: controller-manager.conf + - secret: + items: + - key: tls.crt + path: ca.crt + - key: tls.key + path: ca.key + name: ca + - secret: + items: + - key: tls.key + path: sa.key + name: sa` + + schedulerProjectedSecret = `- name: k8s + projected: + sources: + - secret: + name: scheduler.conf` + + hostPathVol = `- hostPath: + path: /etc/kubernetes + name: k8s` + testAPIServerPod = ` apiVersion: v1 kind: Pod @@ -92,49 +166,7 @@ spec: name: pki hostNetwork: true volumes: - - name: k8s - projected: - sources: - - secret: - items: - - key: tls.crt - path: ca.crt - - key: tls.key - path: ca.key - name: ca - - secret: - items: - - key: tls.crt - path: apiserver.crt - - key: tls.key - path: apiserver.key - name: apiserver - - secret: - items: - - key: tls.crt - path: apiserver-kubelet-client.crt - - key: tls.key - path: apiserver-kubelet-client.key - name: apiserver-kubelet-client - - secret: - items: - - key: tls.crt - path: sa.pub - - key: tls.key - path: sa.key - name: sa - - secret: - items: - - key: tls.crt - path: front-proxy-ca.crt - name: front-proxy-ca - - secret: - items: - - key: tls.crt - path: front-proxy-client.crt - - key: tls.key - path: front-proxy-client.key - name: front-proxy-client + %s - hostPath: path: /etc/ssl/certs name: certs @@ -213,49 +245,7 @@ spec: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - name: k8s - projected: - sources: - - secret: - items: - - key: tls.crt - path: ca.crt - - key: tls.key - path: ca.key - name: ca - - secret: - items: - - key: tls.crt - path: apiserver.crt - - key: tls.key - path: apiserver.key - name: apiserver - - secret: - items: - - key: tls.crt - path: apiserver-kubelet-client.crt - - key: tls.key - path: apiserver-kubelet-client.key - name: apiserver-kubelet-client - - secret: - items: - - key: tls.crt - path: sa.pub - - key: tls.key - path: sa.key - name: sa - - secret: - items: - - key: tls.crt - path: front-proxy-ca.crt - name: front-proxy-ca - - secret: - items: - - key: tls.crt - path: front-proxy-client.crt - - key: tls.key - path: front-proxy-client.key - name: front-proxy-client + %s - hostPath: path: /etc/ssl/certs name: certs @@ -319,23 +309,7 @@ spec: name: pki hostNetwork: true volumes: - - name: k8s - projected: - sources: - - secret: - name: controller-manager.conf - - secret: - items: - - key: tls.crt - path: ca.crt - - key: tls.key - path: ca.key - name: ca - - secret: - items: - - key: tls.key - path: sa.key - name: sa + %s - hostPath: path: /etc/ssl/certs name: certs @@ -400,23 +374,7 @@ spec: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - name: k8s - projected: - sources: - - secret: - name: controller-manager.conf - - secret: - items: - - key: tls.crt - path: ca.crt - - key: tls.key - path: ca.key - name: ca - - secret: - items: - - key: tls.key - path: sa.key - name: sa + %s - hostPath: path: /etc/ssl/certs name: certs @@ -470,11 +428,7 @@ spec: readOnly: true hostNetwork: true volumes: - - name: k8s - projected: - sources: - - secret: - name: scheduler.conf + %s status: {} ` @@ -523,11 +477,7 @@ spec: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - - name: k8s - projected: - sources: - - secret: - name: scheduler.conf + %s updateStrategy: {} status: currentNumberScheduled: 0 @@ -537,26 +487,67 @@ status: ` ) +var ( + testAPIServerSecretsPod = fmt.Sprintf(testAPIServerPod, apiProjectedSecret) + testAPIServerSecretsDS = fmt.Sprintf(testAPIServerDaemonSet, indentString(apiProjectedSecret, 4)) + testAPIServerHostPathPod = fmt.Sprintf(testAPIServerPod, hostPathVol) + testAPIServerHostPathDS = fmt.Sprintf(testAPIServerDaemonSet, indentString(hostPathVol, 4)) + + testSchedulerSecretsPod = fmt.Sprintf(testSchedulerPod, schedulerProjectedSecret) + testSchedulerSecretsDS = fmt.Sprintf(testSchedulerDaemonSet, indentString(schedulerProjectedSecret, 4)) + testSchedulerHostPathPod = fmt.Sprintf(testSchedulerPod, hostPathVol) + testSchedulerHostPathDS = fmt.Sprintf(testSchedulerDaemonSet, indentString(hostPathVol, 4)) + + testControllerManagerSecretsPod = fmt.Sprintf(testControllerManagerPod, controllerManagerProjectedSecret) + testControllerManagerSecretsDS = fmt.Sprintf(testControllerManagerDaemonSet, indentString(controllerManagerProjectedSecret, 4)) + testControllerManagerHostPathPod = fmt.Sprintf(testControllerManagerPod, hostPathVol) + testControllerManagerHostPathDS = fmt.Sprintf(testControllerManagerDaemonSet, indentString(hostPathVol, 4)) +) + func TestBuildDaemonSet(t *testing.T) { var tests = []struct { - component string - podBytes []byte - dsBytes []byte + component string + podBytes []byte + dsBytes []byte + selfHostedSecrets bool }{ + // vols as secrets { - component: kubeadmconstants.KubeAPIServer, - podBytes: []byte(testAPIServerPod), - dsBytes: []byte(testAPIServerDaemonSet), + component: kubeadmconstants.KubeAPIServer, + podBytes: []byte(testAPIServerSecretsPod), + dsBytes: []byte(testAPIServerSecretsDS), + selfHostedSecrets: true, }, { - component: kubeadmconstants.KubeControllerManager, - podBytes: []byte(testControllerManagerPod), - dsBytes: []byte(testControllerManagerDaemonSet), + component: kubeadmconstants.KubeControllerManager, + podBytes: []byte(testControllerManagerSecretsPod), + dsBytes: []byte(testControllerManagerSecretsDS), + selfHostedSecrets: true, }, { - component: kubeadmconstants.KubeScheduler, - podBytes: []byte(testSchedulerPod), - dsBytes: []byte(testSchedulerDaemonSet), + component: kubeadmconstants.KubeScheduler, + podBytes: []byte(testSchedulerSecretsPod), + dsBytes: []byte(testSchedulerSecretsDS), + selfHostedSecrets: true, + }, + // hostPath vols + { + component: kubeadmconstants.KubeAPIServer, + podBytes: []byte(testAPIServerHostPathPod), + dsBytes: []byte(testAPIServerHostPathDS), + selfHostedSecrets: false, + }, + { + component: kubeadmconstants.KubeControllerManager, + podBytes: []byte(testControllerManagerHostPathPod), + dsBytes: []byte(testControllerManagerHostPathDS), + selfHostedSecrets: false, + }, + { + component: kubeadmconstants.KubeScheduler, + podBytes: []byte(testSchedulerHostPathPod), + dsBytes: []byte(testSchedulerHostPathDS), + selfHostedSecrets: false, }, } @@ -566,10 +557,13 @@ func TestBuildDaemonSet(t *testing.T) { podSpec, err := loadPodSpecFromFile(tempFile) if err != nil { - t.Fatalf("couldn't load the specified Pod") + t.Fatalf("couldn't load the specified Pod: %v", err) + } + + cfg := &kubeadmapi.MasterConfiguration{ + FeatureFlags: map[string]bool{string(features.StoreCertsInSecrets): rt.selfHostedSecrets}, } - cfg := &kubeadmapi.MasterConfiguration{} ds := buildDaemonSet(cfg, rt.component, podSpec) dsBytes, err := yaml.Marshal(ds) if err != nil { @@ -577,7 +571,7 @@ func TestBuildDaemonSet(t *testing.T) { } if !bytes.Equal(dsBytes, rt.dsBytes) { - t.Errorf("failed TestBuildDaemonSet:\nexpected:\n%s\nsaw:\n%s", rt.dsBytes, dsBytes) + t.Errorf("failed TestBuildDaemonSet for name=%s (secrets=%t):\nexpected:\n%s\nsaw:\n%s", rt.component, rt.selfHostedSecrets, rt.dsBytes, dsBytes) } } } @@ -657,3 +651,18 @@ func createTempFileWithContent(content []byte) (string, error) { } return tempFile.Name(), nil } + +func indentString(input string, count int) string { + output := "" + lines := strings.Split(input, "\n") + for i, line := range lines { + if i > 0 { + output += strings.Repeat(" ", count) + } + output += line + if i < len(lines)-1 { + output += "\n" + } + } + return output +} diff --git a/cmd/kubeadm/app/phases/selfhosting/selfhosting_volumes.go b/cmd/kubeadm/app/phases/selfhosting/selfhosting_volumes.go index 63485bd52e0..4c5c7f37db9 100644 --- a/cmd/kubeadm/app/phases/selfhosting/selfhosting_volumes.go +++ b/cmd/kubeadm/app/phases/selfhosting/selfhosting_volumes.go @@ -25,9 +25,15 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/features" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" ) +const ( + volumeName = "k8s" + volumeMountName = "k8s" +) + type tlsKeyPair struct { name string cert string @@ -36,16 +42,16 @@ type tlsKeyPair struct { func k8sSelfHostedVolumeMount() v1.VolumeMount { return v1.VolumeMount{ - Name: "k8s", + Name: volumeMountName, MountPath: kubeadmconstants.KubernetesDir, ReadOnly: true, } } -func apiServerProjectedVolume(cfg *kubeadmapi.MasterConfiguration) v1.Volume { - return v1.Volume{ - Name: "k8s", - VolumeSource: v1.VolumeSource{ +func apiServerVolume(cfg *kubeadmapi.MasterConfiguration) v1.Volume { + var volumeSource v1.VolumeSource + if features.Enabled(cfg.FeatureFlags, features.StoreCertsInSecrets) { + volumeSource = v1.VolumeSource{ Projected: &v1.ProjectedVolumeSource{ Sources: []v1.VolumeProjection{ { @@ -148,14 +154,24 @@ func apiServerProjectedVolume(cfg *kubeadmapi.MasterConfiguration) v1.Volume { }, }, }, - }, + } + } else { + volumeSource = v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: kubeadmconstants.KubernetesDir, + }, + } + } + return v1.Volume{ + Name: volumeName, + VolumeSource: volumeSource, } } -func schedulerProjectedVolume(cfg *kubeadmapi.MasterConfiguration) v1.Volume { - return v1.Volume{ - Name: "k8s", - VolumeSource: v1.VolumeSource{ +func schedulerVolume(cfg *kubeadmapi.MasterConfiguration) v1.Volume { + var volumeSource v1.VolumeSource + if features.Enabled(cfg.FeatureFlags, features.StoreCertsInSecrets) { + volumeSource = v1.VolumeSource{ Projected: &v1.ProjectedVolumeSource{ Sources: []v1.VolumeProjection{ { @@ -167,14 +183,24 @@ func schedulerProjectedVolume(cfg *kubeadmapi.MasterConfiguration) v1.Volume { }, }, }, - }, + } + } else { + volumeSource = v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: kubeadmconstants.KubernetesDir, + }, + } + } + return v1.Volume{ + Name: volumeName, + VolumeSource: volumeSource, } } -func controllerManagerProjectedVolume(cfg *kubeadmapi.MasterConfiguration) v1.Volume { - return v1.Volume{ - Name: "k8s", - VolumeSource: v1.VolumeSource{ +func controllerManagerVolume(cfg *kubeadmapi.MasterConfiguration) v1.Volume { + var volumeSource v1.VolumeSource + if features.Enabled(cfg.FeatureFlags, features.StoreCertsInSecrets) { + volumeSource = v1.VolumeSource{ Projected: &v1.ProjectedVolumeSource{ Sources: []v1.VolumeProjection{ { @@ -216,7 +242,17 @@ func controllerManagerProjectedVolume(cfg *kubeadmapi.MasterConfiguration) v1.Vo }, }, }, - }, + } + } else { + volumeSource = v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: kubeadmconstants.KubernetesDir, + }, + } + } + return v1.Volume{ + Name: volumeName, + VolumeSource: volumeSource, } }