From e65d0bd514f5b27f828a65aae5317b39147f606b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20K=C3=A4ldstr=C3=B6m?= Date: Thu, 20 Jul 2017 20:17:28 +0300 Subject: [PATCH] kubeadm: Make the hostPath volume mount code more secure --- cmd/kubeadm/app/phases/controlplane/BUILD | 10 +- .../app/phases/controlplane/manifests.go | 179 ++---- .../app/phases/controlplane/manifests_test.go | 314 +--------- .../app/phases/controlplane/volumes.go | 183 ++++++ .../app/phases/controlplane/volumes_test.go | 534 ++++++++++++++++++ 5 files changed, 780 insertions(+), 440 deletions(-) create mode 100644 cmd/kubeadm/app/phases/controlplane/volumes.go create mode 100644 cmd/kubeadm/app/phases/controlplane/volumes_test.go diff --git a/cmd/kubeadm/app/phases/controlplane/BUILD b/cmd/kubeadm/app/phases/controlplane/BUILD index 83ef59764a3..4a3c9c8e1fb 100644 --- a/cmd/kubeadm/app/phases/controlplane/BUILD +++ b/cmd/kubeadm/app/phases/controlplane/BUILD @@ -10,7 +10,10 @@ load( go_test( name = "go_default_test", - srcs = ["manifests_test.go"], + srcs = [ + "manifests_test.go", + "volumes_test.go", + ], library = ":go_default_library", tags = ["automanaged"], deps = [ @@ -25,7 +28,10 @@ go_test( go_library( name = "go_default_library", - srcs = ["manifests.go"], + srcs = [ + "manifests.go", + "volumes.go", + ], tags = ["automanaged"], deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go index e807257386b..eaf61da5c51 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests.go @@ -55,69 +55,58 @@ const ( // WriteStaticPodManifests builds manifest objects based on user provided configuration and then dumps it to disk // where kubelet will pick and schedule them. func WriteStaticPodManifests(cfg *kubeadmapi.MasterConfiguration) error { - volumes := []v1.Volume{k8sVolume()} - volumeMounts := []v1.VolumeMount{k8sVolumeMount()} - - if isCertsVolumeMountNeeded() { - volumes = append(volumes, certsVolume(cfg)) - volumeMounts = append(volumeMounts, certsVolumeMount()) - } - - if isPkiVolumeMountNeeded() { - volumes = append(volumes, pkiVolume()) - volumeMounts = append(volumeMounts, pkiVolumeMount()) - } - - if !strings.HasPrefix(cfg.CertificatesDir, kubeadmapiext.DefaultCertificatesDir) { - volumes = append(volumes, newVolume("certdir", cfg.CertificatesDir)) - volumeMounts = append(volumeMounts, newVolumeMount("certdir", cfg.CertificatesDir)) - } + // TODO: Move the "pkg/util/version".Version object into the internal API instead of always parsing the string k8sVersion, err := version.ParseSemantic(cfg.KubernetesVersion) if err != nil { return err } + // Get the required hostpath mounts + mounts := getHostPathVolumesForTheControlPlane(cfg) + // Prepare static pod specs staticPodSpecs := map[string]v1.Pod{ kubeAPIServer: componentPod(v1.Container{ Name: kubeAPIServer, Image: images.GetCoreImage(images.KubeAPIServerImage, cfg, cfg.UnifiedControlPlaneImage), - Command: getAPIServerCommand(cfg, false, k8sVersion), - VolumeMounts: volumeMounts, + Command: getAPIServerCommand(cfg, k8sVersion), + VolumeMounts: mounts.GetVolumeMounts(kubeAPIServer), LivenessProbe: componentProbe(int(cfg.API.BindPort), "/healthz", v1.URISchemeHTTPS), Resources: componentResources("250m"), Env: getProxyEnvVars(), - }, volumes...), + }, mounts.GetVolumes(kubeAPIServer)), kubeControllerManager: componentPod(v1.Container{ Name: kubeControllerManager, Image: images.GetCoreImage(images.KubeControllerManagerImage, cfg, cfg.UnifiedControlPlaneImage), - Command: getControllerManagerCommand(cfg, false, k8sVersion), - VolumeMounts: volumeMounts, + Command: getControllerManagerCommand(cfg, k8sVersion), + VolumeMounts: mounts.GetVolumeMounts(kubeControllerManager), LivenessProbe: componentProbe(10252, "/healthz", v1.URISchemeHTTP), Resources: componentResources("200m"), Env: getProxyEnvVars(), - }, volumes...), + }, mounts.GetVolumes(kubeControllerManager)), kubeScheduler: componentPod(v1.Container{ Name: kubeScheduler, Image: images.GetCoreImage(images.KubeSchedulerImage, cfg, cfg.UnifiedControlPlaneImage), - Command: getSchedulerCommand(cfg, false), - VolumeMounts: []v1.VolumeMount{k8sVolumeMount()}, + Command: getSchedulerCommand(cfg), + VolumeMounts: mounts.GetVolumeMounts(kubeScheduler), LivenessProbe: componentProbe(10251, "/healthz", v1.URISchemeHTTP), Resources: componentResources("100m"), Env: getProxyEnvVars(), - }, k8sVolume()), + }, mounts.GetVolumes(kubeScheduler)), } // Add etcd static pod spec only if external etcd is not configured if len(cfg.Etcd.Endpoints) == 0 { + etcdPod := componentPod(v1.Container{ - Name: etcd, - Command: getEtcdCommand(cfg), - VolumeMounts: []v1.VolumeMount{certsVolumeMount(), etcdVolumeMount(cfg.Etcd.DataDir), k8sVolumeMount()}, - Image: images.GetCoreImage(images.KubeEtcdImage, cfg, cfg.Etcd.Image), + Name: etcd, + Command: getEtcdCommand(cfg), + Image: images.GetCoreImage(images.KubeEtcdImage, cfg, cfg.Etcd.Image), + // Mount the etcd datadir path read-write so etcd can store data in a more persistent manner + VolumeMounts: []v1.VolumeMount{newVolumeMount(etcdVolumeName, cfg.Etcd.DataDir, false)}, LivenessProbe: componentProbe(2379, "/health", v1.URISchemeHTTP), - }, certsVolume(cfg), etcdVolume(cfg), k8sVolume()) + }, []v1.Volume{newVolume(etcdVolumeName, cfg.Etcd.DataDir)}) etcdPod.Spec.SecurityContext = &v1.PodSecurityContext{ SELinuxOptions: &v1.SELinuxOptions{ @@ -146,106 +135,7 @@ func WriteStaticPodManifests(cfg *kubeadmapi.MasterConfiguration) error { return nil } -func newVolume(name, path string) v1.Volume { - return v1.Volume{ - Name: name, - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: path}, - }, - } -} - -func newVolumeMount(name, path string) v1.VolumeMount { - return v1.VolumeMount{ - Name: name, - MountPath: path, - } -} - -// etcdVolume exposes a path on the host in order to guarantee data survival during reboot. -func etcdVolume(cfg *kubeadmapi.MasterConfiguration) v1.Volume { - return v1.Volume{ - Name: "etcd", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: cfg.Etcd.DataDir}, - }, - } -} - -func etcdVolumeMount(dataDir string) v1.VolumeMount { - return v1.VolumeMount{ - Name: "etcd", - MountPath: dataDir, - } -} - -func isCertsVolumeMountNeeded() bool { - // Always return true for now. We may add conditional logic here for images which do not require host mounting /etc/ssl - // hyperkube for example already has valid ca-certificates installed - return true -} - -// certsVolume exposes host SSL certificates to pod containers. -func certsVolume(cfg *kubeadmapi.MasterConfiguration) v1.Volume { - return v1.Volume{ - Name: "certs", - VolumeSource: v1.VolumeSource{ - // TODO(phase1+) make path configurable - HostPath: &v1.HostPathVolumeSource{Path: "/etc/ssl/certs"}, - }, - } -} - -func certsVolumeMount() v1.VolumeMount { - return v1.VolumeMount{ - Name: "certs", - MountPath: "/etc/ssl/certs", - } -} - -func isPkiVolumeMountNeeded() bool { - // On some systems were we host-mount /etc/ssl/certs, it is also required to mount /etc/pki. This is needed - // due to symlinks pointing from files in /etc/ssl/certs into /etc/pki/ - if _, err := os.Stat("/etc/pki"); err == nil { - return true - } - return false -} - -func pkiVolume() v1.Volume { - return v1.Volume{ - Name: "pki", - VolumeSource: v1.VolumeSource{ - // TODO(phase1+) make path configurable - HostPath: &v1.HostPathVolumeSource{Path: "/etc/pki"}, - }, - } -} - -func pkiVolumeMount() v1.VolumeMount { - return v1.VolumeMount{ - Name: "pki", - MountPath: "/etc/pki", - } -} - -func k8sVolume() v1.Volume { - return v1.Volume{ - Name: "k8s", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: kubeadmconstants.KubernetesDir}, - }, - } -} - -func k8sVolumeMount() v1.VolumeMount { - return v1.VolumeMount{ - Name: "k8s", - MountPath: kubeadmconstants.KubernetesDir, - ReadOnly: true, - } -} - +// componentResources returns the v1.ResourceRequirements object needed for allocating a specified amount of the CPU func componentResources(cpu string) v1.ResourceRequirements { return v1.ResourceRequirements{ Requests: v1.ResourceList{ @@ -254,10 +144,12 @@ func componentResources(cpu string) v1.ResourceRequirements { } } +// componentProbe is a helper function building a ready v1.Probe object from some simple parameters func componentProbe(port int, path string, scheme v1.URIScheme) *v1.Probe { return &v1.Probe{ Handler: v1.Handler{ HTTPGet: &v1.HTTPGetAction{ + // Host has to be set to "127.0.0.1" here due to that our static Pods are on the host's network Host: "127.0.0.1", Path: path, Port: intstr.FromInt(port), @@ -270,7 +162,8 @@ func componentProbe(port int, path string, scheme v1.URIScheme) *v1.Probe { } } -func componentPod(container v1.Container, volumes ...v1.Volume) v1.Pod { +// componentPod returns a Pod object from the container and volume specifications +func componentPod(container v1.Container, volumes []v1.Volume) v1.Pod { return v1.Pod{ TypeMeta: metav1.TypeMeta{ APIVersion: "v1", @@ -278,9 +171,8 @@ func componentPod(container v1.Container, volumes ...v1.Volume) v1.Pod { }, ObjectMeta: metav1.ObjectMeta{ Name: container.Name, - Namespace: "kube-system", + Namespace: metav1.NamespaceSystem, Annotations: map[string]string{kubetypes.CriticalPodAnnotationKey: ""}, - Labels: map[string]string{"component": container.Name, "tier": "control-plane"}, }, Spec: v1.PodSpec{ Containers: []v1.Container{container}, @@ -290,8 +182,10 @@ func componentPod(container v1.Container, volumes ...v1.Volume) v1.Pod { } } -func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool, k8sVersion *version.Version) []string { +// getAPIServerCommand builds the right API server command from the given config object and version +func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.Version) []string { defaultArguments := map[string]string{ + "advertise-address": cfg.API.AdvertiseAddress, "insecure-port": "0", "admission-control": defaultv17AdmissionControl, "service-cluster-ip-range": cfg.Networking.ServiceSubnet, @@ -320,12 +214,6 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool, k command = append(command, getExtraParameters(cfg.APIServerExtraArgs, defaultArguments)...) command = append(command, getAuthzParameters(cfg.AuthorizationModes)...) - if selfHosted { - command = append(command, "--advertise-address=$(POD_IP)") - } else { - command = append(command, fmt.Sprintf("--advertise-address=%s", cfg.API.AdvertiseAddress)) - } - // Check if the user decided to use an external etcd cluster if len(cfg.Etcd.Endpoints) > 0 { command = append(command, fmt.Sprintf("--etcd-servers=%s", strings.Join(cfg.Etcd.Endpoints, ","))) @@ -355,6 +243,7 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool, k return command } +// getEtcdCommand builds the right etcd command from the given config object func getEtcdCommand(cfg *kubeadmapi.MasterConfiguration) []string { defaultArguments := map[string]string{ "listen-client-urls": "http://127.0.0.1:2379", @@ -367,7 +256,8 @@ func getEtcdCommand(cfg *kubeadmapi.MasterConfiguration) []string { return command } -func getControllerManagerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool, k8sVersion *version.Version) []string { +// getControllerManagerCommand builds the right controller manager command from the given config object and version +func getControllerManagerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.Version) []string { defaultArguments := map[string]string{ "address": "127.0.0.1", "leader-elect": "true", @@ -400,7 +290,8 @@ func getControllerManagerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted return command } -func getSchedulerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool) []string { +// getSchedulerCommand builds the right scheduler command from the given config object and version +func getSchedulerCommand(cfg *kubeadmapi.MasterConfiguration) []string { defaultArguments := map[string]string{ "address": "127.0.0.1", "leader-elect": "true", @@ -412,6 +303,7 @@ func getSchedulerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool) [ return command } +// getProxyEnvVars builds a list of environment variables to use in the control plane containers in order to use the right proxy func getProxyEnvVars() []v1.EnvVar { envs := []v1.EnvVar{} for _, env := range os.Environ() { @@ -452,6 +344,7 @@ func getAuthzParameters(modes []string) []string { return command } +// getExtraParameters builds a list of flag arguments two string-string maps, one with default, base commands and one with overrides func getExtraParameters(overrides map[string]string, defaults map[string]string) []string { var command []string for k, v := range overrides { diff --git a/cmd/kubeadm/app/phases/controlplane/manifests_test.go b/cmd/kubeadm/app/phases/controlplane/manifests_test.go index 8ba45c4f55e..25780b00c87 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests_test.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests_test.go @@ -28,7 +28,6 @@ import ( "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/yaml" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/pkg/util/version" @@ -48,6 +47,7 @@ func TestWriteStaticPodManifests(t *testing.T) { // set up tmp KubernetesDir for testing kubeadmconstants.KubernetesDir = fmt.Sprintf("%s/etc/kubernetes", tmpdir) + defer func() { kubeadmconstants.KubernetesDir = "/etc/kubernetes" }() var tests = []struct { cfg *kubeadmapi.MasterConfiguration @@ -125,282 +125,6 @@ func TestWriteStaticPodManifests(t *testing.T) { } } -func TestNewVolume(t *testing.T) { - var tests = []struct { - name string - path string - expected v1.Volume - }{ - { - name: "foo", - path: "/etc/foo", - expected: v1.Volume{ - Name: "foo", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: "/etc/foo"}, - }}, - }, - } - - for _, rt := range tests { - actual := newVolume(rt.name, rt.path) - if actual.Name != rt.expected.Name { - t.Errorf( - "failed newVolume:\n\texpected: %s\n\t actual: %s", - rt.expected.Name, - actual.Name, - ) - } - if actual.VolumeSource.HostPath.Path != rt.expected.VolumeSource.HostPath.Path { - t.Errorf( - "failed newVolume:\n\texpected: %s\n\t actual: %s", - rt.expected.VolumeSource.HostPath.Path, - actual.VolumeSource.HostPath.Path, - ) - } - } -} - -func TestNewVolumeMount(t *testing.T) { - var tests = []struct { - name string - path string - expected v1.VolumeMount - }{ - { - name: "foo", - path: "/etc/foo", - expected: v1.VolumeMount{ - Name: "foo", - MountPath: "/etc/foo", - }, - }, - } - - for _, rt := range tests { - actual := newVolumeMount(rt.name, rt.path) - if actual.Name != rt.expected.Name { - t.Errorf( - "failed newVolumeMount:\n\texpected: %s\n\t actual: %s", - rt.expected.Name, - actual.Name, - ) - } - if actual.MountPath != rt.expected.MountPath { - t.Errorf( - "failed newVolumeMount:\n\texpected: %s\n\t actual: %s", - rt.expected.MountPath, - actual.MountPath, - ) - } - } -} - -func TestEtcdVolume(t *testing.T) { - var tests = []struct { - cfg *kubeadmapi.MasterConfiguration - expected v1.Volume - }{ - { - cfg: &kubeadmapi.MasterConfiguration{ - Etcd: kubeadmapi.Etcd{DataDir: etcdDataDir}, - }, - expected: v1.Volume{ - Name: "etcd", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: etcdDataDir}, - }}, - }, - } - - for _, rt := range tests { - actual := etcdVolume(rt.cfg) - if actual.Name != rt.expected.Name { - t.Errorf( - "failed etcdVolume:\n\texpected: %s\n\t actual: %s", - rt.expected.Name, - actual.Name, - ) - } - if actual.VolumeSource.HostPath.Path != rt.expected.VolumeSource.HostPath.Path { - t.Errorf( - "failed etcdVolume:\n\texpected: %s\n\t actual: %s", - rt.expected.VolumeSource.HostPath.Path, - actual.VolumeSource.HostPath.Path, - ) - } - } -} - -func TestEtcdVolumeMount(t *testing.T) { - var tests = []struct { - expected v1.VolumeMount - }{ - { - expected: v1.VolumeMount{ - Name: "etcd", - MountPath: etcdDataDir, - }, - }, - } - - for _, rt := range tests { - actual := etcdVolumeMount(etcdDataDir) - if actual.Name != rt.expected.Name { - t.Errorf( - "failed etcdVolumeMount:\n\texpected: %s\n\t actual: %s", - rt.expected.Name, - actual.Name, - ) - } - if actual.MountPath != rt.expected.MountPath { - t.Errorf( - "failed etcdVolumeMount:\n\texpected: %s\n\t actual: %s", - rt.expected.MountPath, - actual.MountPath, - ) - } - } -} - -func TestCertsVolume(t *testing.T) { - var tests = []struct { - cfg *kubeadmapi.MasterConfiguration - expected v1.Volume - }{ - { - cfg: &kubeadmapi.MasterConfiguration{}, - expected: v1.Volume{ - Name: "certs", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/ssl/certs"}, - }}, - }, - } - - for _, rt := range tests { - actual := certsVolume(rt.cfg) - if actual.Name != rt.expected.Name { - t.Errorf( - "failed certsVolume:\n\texpected: %s\n\t actual: %s", - rt.expected.Name, - actual.Name, - ) - } - if actual.VolumeSource.HostPath.Path != rt.expected.VolumeSource.HostPath.Path { - t.Errorf( - "failed certsVolume:\n\texpected: %s\n\t actual: %s", - rt.expected.VolumeSource.HostPath.Path, - actual.VolumeSource.HostPath.Path, - ) - } - } -} - -func TestCertsVolumeMount(t *testing.T) { - var tests = []struct { - expected v1.VolumeMount - }{ - { - expected: v1.VolumeMount{ - Name: "certs", - MountPath: "/etc/ssl/certs", - }, - }, - } - - for _, rt := range tests { - actual := certsVolumeMount() - if actual.Name != rt.expected.Name { - t.Errorf( - "failed certsVolumeMount:\n\texpected: %s\n\t actual: %s", - rt.expected.Name, - actual.Name, - ) - } - if actual.MountPath != rt.expected.MountPath { - t.Errorf( - "failed certsVolumeMount:\n\texpected: %s\n\t actual: %s", - rt.expected.MountPath, - actual.MountPath, - ) - } - } -} - -func TestK8sVolume(t *testing.T) { - var tests = []struct { - expected v1.Volume - }{ - { - expected: v1.Volume{ - Name: "k8s", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: kubeadmconstants.KubernetesDir}, - }}, - }, - } - - for _, rt := range tests { - actual := k8sVolume() - if actual.Name != rt.expected.Name { - t.Errorf( - "failed k8sVolume:\n\texpected: %s\n\t actual: %s", - rt.expected.Name, - actual.Name, - ) - } - if actual.VolumeSource.HostPath.Path != rt.expected.VolumeSource.HostPath.Path { - t.Errorf( - "failed k8sVolume:\n\texpected: %s\n\t actual: %s", - rt.expected.VolumeSource.HostPath.Path, - actual.VolumeSource.HostPath.Path, - ) - } - } -} - -func TestK8sVolumeMount(t *testing.T) { - var tests = []struct { - expected v1.VolumeMount - }{ - { - expected: v1.VolumeMount{ - Name: "k8s", - MountPath: kubeadmconstants.KubernetesDir, - ReadOnly: true, - }, - }, - } - - for _, rt := range tests { - actual := k8sVolumeMount() - if actual.Name != rt.expected.Name { - t.Errorf( - "failed k8sVolumeMount:\n\texpected: %s\n\t actual: %s", - rt.expected.Name, - actual.Name, - ) - } - if actual.MountPath != rt.expected.MountPath { - t.Errorf( - "failed k8sVolumeMount:\n\texpected: %s\n\t actual: %s", - rt.expected.MountPath, - actual.MountPath, - ) - } - if actual.ReadOnly != rt.expected.ReadOnly { - t.Errorf( - "failed k8sVolumeMount:\n\texpected: %t\n\t actual: %t", - rt.expected.ReadOnly, - actual.ReadOnly, - ) - } - } -} - func TestComponentResources(t *testing.T) { a := componentResources("250m") if a.Requests == nil { @@ -464,7 +188,7 @@ func TestComponentPod(t *testing.T) { for _, rt := range tests { c := v1.Container{Name: rt.n} - v := v1.Volume{} + v := []v1.Volume{} actual := componentPod(c, v) if actual.ObjectMeta.Name != rt.n { t.Errorf( @@ -483,8 +207,8 @@ func TestGetAPIServerCommand(t *testing.T) { }{ { cfg: &kubeadmapi.MasterConfiguration{ - API: kubeadm.API{BindPort: 123, AdvertiseAddress: "1.2.3.4"}, - Networking: kubeadm.Networking{ServiceSubnet: "bar"}, + API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "1.2.3.4"}, + Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, CertificatesDir: testCertsDir, KubernetesVersion: "v1.7.0", }, @@ -517,8 +241,8 @@ func TestGetAPIServerCommand(t *testing.T) { }, { cfg: &kubeadmapi.MasterConfiguration{ - API: kubeadm.API{BindPort: 123, AdvertiseAddress: "4.3.2.1"}, - Networking: kubeadm.Networking{ServiceSubnet: "bar"}, + API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "4.3.2.1"}, + Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, CertificatesDir: testCertsDir, KubernetesVersion: "v1.7.1", }, @@ -551,9 +275,9 @@ func TestGetAPIServerCommand(t *testing.T) { }, { cfg: &kubeadmapi.MasterConfiguration{ - API: kubeadm.API{BindPort: 123, AdvertiseAddress: "4.3.2.1"}, - Networking: kubeadm.Networking{ServiceSubnet: "bar"}, - Etcd: kubeadm.Etcd{CertFile: "fiz", KeyFile: "faz"}, + API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "4.3.2.1"}, + Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, + Etcd: kubeadmapi.Etcd{CertFile: "fiz", KeyFile: "faz"}, CertificatesDir: testCertsDir, KubernetesVersion: "v1.7.2", }, @@ -588,9 +312,9 @@ func TestGetAPIServerCommand(t *testing.T) { }, { cfg: &kubeadmapi.MasterConfiguration{ - API: kubeadm.API{BindPort: 123, AdvertiseAddress: "4.3.2.1"}, - Networking: kubeadm.Networking{ServiceSubnet: "bar"}, - Etcd: kubeadm.Etcd{CertFile: "fiz", KeyFile: "faz"}, + API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "4.3.2.1"}, + Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, + Etcd: kubeadmapi.Etcd{CertFile: "fiz", KeyFile: "faz"}, CertificatesDir: testCertsDir, KubernetesVersion: "v1.7.3", }, @@ -625,9 +349,9 @@ func TestGetAPIServerCommand(t *testing.T) { }, { cfg: &kubeadmapi.MasterConfiguration{ - API: kubeadm.API{BindPort: 123, AdvertiseAddress: "2001:db8::1"}, - Networking: kubeadm.Networking{ServiceSubnet: "bar"}, - Etcd: kubeadm.Etcd{CertFile: "fiz", KeyFile: "faz"}, + API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "2001:db8::1"}, + Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, + Etcd: kubeadmapi.Etcd{CertFile: "fiz", KeyFile: "faz"}, CertificatesDir: testCertsDir, KubernetesVersion: "v1.7.0", }, @@ -663,7 +387,7 @@ func TestGetAPIServerCommand(t *testing.T) { } for _, rt := range tests { - actual := getAPIServerCommand(rt.cfg, false, version.MustParseSemantic(rt.cfg.KubernetesVersion)) + actual := getAPIServerCommand(rt.cfg, version.MustParseSemantic(rt.cfg.KubernetesVersion)) sort.Strings(actual) sort.Strings(rt.expected) if !reflect.DeepEqual(actual, rt.expected) { @@ -717,7 +441,7 @@ func TestGetControllerManagerCommand(t *testing.T) { }, { cfg: &kubeadmapi.MasterConfiguration{ - Networking: kubeadm.Networking{PodSubnet: "bar"}, + Networking: kubeadmapi.Networking{PodSubnet: "bar"}, CertificatesDir: testCertsDir, KubernetesVersion: "v1.7.0", }, @@ -739,7 +463,7 @@ func TestGetControllerManagerCommand(t *testing.T) { } for _, rt := range tests { - actual := getControllerManagerCommand(rt.cfg, false, version.MustParseSemantic(rt.cfg.KubernetesVersion)) + actual := getControllerManagerCommand(rt.cfg, version.MustParseSemantic(rt.cfg.KubernetesVersion)) sort.Strings(actual) sort.Strings(rt.expected) if !reflect.DeepEqual(actual, rt.expected) { @@ -821,7 +545,7 @@ func TestGetSchedulerCommand(t *testing.T) { } for _, rt := range tests { - actual := getSchedulerCommand(rt.cfg, false) + actual := getSchedulerCommand(rt.cfg) sort.Strings(actual) sort.Strings(rt.expected) if !reflect.DeepEqual(actual, rt.expected) { diff --git a/cmd/kubeadm/app/phases/controlplane/volumes.go b/cmd/kubeadm/app/phases/controlplane/volumes.go new file mode 100644 index 00000000000..4873595b365 --- /dev/null +++ b/cmd/kubeadm/app/phases/controlplane/volumes.go @@ -0,0 +1,183 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controlplane + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" +) + +const ( + k8sCertsVolumeName = "k8s-certs" + etcdVolumeName = "etcd" + caCertsVolumeName = "ca-certs" + caCertsVolumePath = "/etc/ssl/certs" + caCertsPkiVolumeName = "ca-certs-etc-pki" + kubeConfigVolumeName = "kubeconfig" +) + +// caCertsPkiVolumePath specifies the path that can be conditionally mounted into the apiserver and controller-manager containers +// as /etc/ssl/certs might be a symlink to it. It's a variable since it may be changed in unit testing. This var MUST NOT be changed +// in normal codepaths during runtime. +var caCertsPkiVolumePath = "/etc/pki" + +// getHostPathVolumesForTheControlPlane gets the required hostPath volumes and mounts for the control plane +func getHostPathVolumesForTheControlPlane(cfg *kubeadmapi.MasterConfiguration) controlPlaneHostPathMounts { + mounts := newControlPlaneHostPathMounts() + + // HostPath volumes for the API Server + // Read-only mount for the certificates directory + // TODO: Always mount the K8s Certificates directory to a static path inside of the container + mounts.NewHostPathMount(kubeAPIServer, k8sCertsVolumeName, cfg.CertificatesDir, cfg.CertificatesDir, true) + // Read-only mount for the ca certs (/etc/ssl/certs) directory + mounts.NewHostPathMount(kubeAPIServer, caCertsVolumeName, caCertsVolumePath, caCertsVolumePath, true) + + // If external etcd is specified, mount the directories needed for accessing the CA/serving certs and the private key + if len(cfg.Etcd.Endpoints) != 0 { + etcdVols, etcdVolMounts := getEtcdCertVolumes(cfg.Etcd) + mounts.AddHostPathMounts(kubeAPIServer, etcdVols, etcdVolMounts) + } + + // HostPath volumes for the controller manager + // Read-only mount for the certificates directory + // TODO: Always mount the K8s Certificates directory to a static path inside of the container + mounts.NewHostPathMount(kubeControllerManager, k8sCertsVolumeName, cfg.CertificatesDir, cfg.CertificatesDir, true) + // Read-only mount for the ca certs (/etc/ssl/certs) directory + mounts.NewHostPathMount(kubeControllerManager, caCertsVolumeName, caCertsVolumePath, caCertsVolumePath, true) + // Read-only mount for the controller manager kubeconfig file + controllerManagerKubeConfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ControllerManagerKubeConfigFileName) + mounts.NewHostPathMount(kubeControllerManager, kubeConfigVolumeName, controllerManagerKubeConfigFile, controllerManagerKubeConfigFile, true) + + // HostPath volumes for the scheduler + // Read-only mount for the scheduler kubeconfig file + schedulerKubeConfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.SchedulerKubeConfigFileName) + mounts.NewHostPathMount(kubeScheduler, kubeConfigVolumeName, schedulerKubeConfigFile, schedulerKubeConfigFile, true) + + // On some systems were we host-mount /etc/ssl/certs, it is also required to mount /etc/pki. This is needed + // due to symlinks pointing from files in /etc/ssl/certs into /etc/pki/ + if isPkiVolumeMountNeeded() { + mounts.NewHostPathMount(kubeAPIServer, caCertsPkiVolumeName, caCertsPkiVolumePath, caCertsPkiVolumePath, true) + mounts.NewHostPathMount(kubeControllerManager, caCertsPkiVolumeName, caCertsPkiVolumePath, caCertsPkiVolumePath, true) + } + + return mounts +} + +// controlPlaneHostPathMounts is a helper struct for handling all the control plane's hostPath mounts in an easy way +type controlPlaneHostPathMounts struct { + volumes map[string][]v1.Volume + volumeMounts map[string][]v1.VolumeMount +} + +func newControlPlaneHostPathMounts() controlPlaneHostPathMounts { + return controlPlaneHostPathMounts{ + volumes: map[string][]v1.Volume{}, + volumeMounts: map[string][]v1.VolumeMount{}, + } +} + +func (c *controlPlaneHostPathMounts) NewHostPathMount(component, mountName, hostPath, containerPath string, readOnly bool) { + c.volumes[component] = append(c.volumes[component], newVolume(mountName, hostPath)) + c.volumeMounts[component] = append(c.volumeMounts[component], newVolumeMount(mountName, containerPath, readOnly)) +} + +func (c *controlPlaneHostPathMounts) AddHostPathMounts(component string, vols []v1.Volume, volMounts []v1.VolumeMount) { + c.volumes[component] = append(c.volumes[component], vols...) + c.volumeMounts[component] = append(c.volumeMounts[component], volMounts...) +} + +func (c *controlPlaneHostPathMounts) GetVolumes(component string) []v1.Volume { + return c.volumes[component] +} + +func (c *controlPlaneHostPathMounts) GetVolumeMounts(component string) []v1.VolumeMount { + return c.volumeMounts[component] +} + +// newVolume creates a v1.Volume with a hostPath mount to the specified location +func newVolume(name, path string) v1.Volume { + return v1.Volume{ + Name: name, + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: path}, + }, + } +} + +// newVolumeMount creates a v1.VolumeMount to the specified location +func newVolumeMount(name, path string, readOnly bool) v1.VolumeMount { + return v1.VolumeMount{ + Name: name, + MountPath: path, + ReadOnly: readOnly, + } +} + +// getEtcdCertVolumes returns the volumes/volumemounts needed for talking to an external etcd cluster +func getEtcdCertVolumes(etcdCfg kubeadmapi.Etcd) ([]v1.Volume, []v1.VolumeMount) { + certPaths := []string{etcdCfg.CAFile, etcdCfg.CertFile, etcdCfg.KeyFile} + certDirs := sets.NewString() + for _, certPath := range certPaths { + certDir := filepath.Dir(certPath) + // Ignore ".", which is the result of passing an empty path. + // Also ignore the cert directories that already may be mounted; /etc/ssl/certs and /etc/pki. If the etcd certs are in there, it's okay, we don't have to do anything + if certDir == "." || strings.HasPrefix(certDir, caCertsVolumePath) || strings.HasPrefix(certDir, caCertsPkiVolumePath) { + continue + } + // Filter out any existing hostpath mounts in the list that contains a subset of the path + alreadyExists := false + for _, existingCertDir := range certDirs.List() { + // If the current directory is a parent of an existing one, remove the already existing one + if strings.HasPrefix(existingCertDir, certDir) { + certDirs.Delete(existingCertDir) + } else if strings.HasPrefix(certDir, existingCertDir) { + // If an existing directory is a parent of the current one, don't add the current one + alreadyExists = true + } + } + if alreadyExists { + continue + } + certDirs.Insert(certDir) + } + + volumes := []v1.Volume{} + volumeMounts := []v1.VolumeMount{} + for i, certDir := range certDirs.List() { + name := fmt.Sprintf("etcd-certs-%d", i) + volumes = append(volumes, newVolume(name, certDir)) + volumeMounts = append(volumeMounts, newVolumeMount(name, certDir, true)) + } + return volumes, volumeMounts +} + +// isPkiVolumeMountNeeded specifies whether /etc/pki should be host-mounted into the containers +// On some systems were we host-mount /etc/ssl/certs, it is also required to mount /etc/pki. This is needed +// due to symlinks pointing from files in /etc/ssl/certs into /etc/pki/ +func isPkiVolumeMountNeeded() bool { + if _, err := os.Stat(caCertsPkiVolumePath); err == nil { + return true + } + return false +} diff --git a/cmd/kubeadm/app/phases/controlplane/volumes_test.go b/cmd/kubeadm/app/phases/controlplane/volumes_test.go new file mode 100644 index 00000000000..9a9d521dfd5 --- /dev/null +++ b/cmd/kubeadm/app/phases/controlplane/volumes_test.go @@ -0,0 +1,534 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controlplane + +import ( + "fmt" + "io/ioutil" + "os" + "reflect" + "testing" + + "k8s.io/api/core/v1" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" +) + +func TestNewVolume(t *testing.T) { + var tests = []struct { + name string + path string + expected v1.Volume + }{ + { + name: "foo", + path: "/etc/foo", + expected: v1.Volume{ + Name: "foo", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: "/etc/foo"}, + }, + }, + }, + } + + for _, rt := range tests { + actual := newVolume(rt.name, rt.path) + if !reflect.DeepEqual(actual, rt.expected) { + t.Errorf( + "failed newVolume:\n\texpected: %v\n\t actual: %v", + rt.expected, + actual, + ) + } + } +} + +func TestNewVolumeMount(t *testing.T) { + var tests = []struct { + name string + path string + ro bool + expected v1.VolumeMount + }{ + { + name: "foo", + path: "/etc/foo", + ro: false, + expected: v1.VolumeMount{ + Name: "foo", + MountPath: "/etc/foo", + ReadOnly: false, + }, + }, + { + name: "bar", + path: "/etc/foo/bar", + ro: true, + expected: v1.VolumeMount{ + Name: "bar", + MountPath: "/etc/foo/bar", + ReadOnly: true, + }, + }, + } + + for _, rt := range tests { + actual := newVolumeMount(rt.name, rt.path, rt.ro) + if !reflect.DeepEqual(actual, rt.expected) { + t.Errorf( + "failed newVolumeMount:\n\texpected: %v\n\t actual: %v", + rt.expected, + actual, + ) + } + } +} + +func TestGetEtcdCertVolumes(t *testing.T) { + var tests = []struct { + ca, cert, key string + vol []v1.Volume + volMount []v1.VolumeMount + }{ + { + // Should ignore files in /etc/ssl/certs + ca: "/etc/ssl/certs/my-etcd-ca.crt", + cert: "/etc/ssl/certs/my-etcd.crt", + key: "/etc/ssl/certs/my-etcd.key", + vol: []v1.Volume{}, + volMount: []v1.VolumeMount{}, + }, + { + // Should ignore files in subdirs of /etc/ssl/certs + ca: "/etc/ssl/certs/etcd/my-etcd-ca.crt", + cert: "/etc/ssl/certs/etcd/my-etcd.crt", + key: "/etc/ssl/certs/etcd/my-etcd.key", + vol: []v1.Volume{}, + volMount: []v1.VolumeMount{}, + }, + { + // Should ignore files in /etc/pki + ca: "/etc/pki/my-etcd-ca.crt", + cert: "/etc/pki/my-etcd.crt", + key: "/etc/pki/my-etcd.key", + vol: []v1.Volume{}, + volMount: []v1.VolumeMount{}, + }, + { + // All in the same dir + ca: "/var/lib/certs/etcd/my-etcd-ca.crt", + cert: "/var/lib/certs/etcd/my-etcd.crt", + key: "/var/lib/certs/etcd/my-etcd.key", + vol: []v1.Volume{ + { + Name: "etcd-certs-0", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: "/var/lib/certs/etcd"}, + }, + }, + }, + volMount: []v1.VolumeMount{ + { + Name: "etcd-certs-0", + MountPath: "/var/lib/certs/etcd", + ReadOnly: true, + }, + }, + }, + { + // One file + two files in separate dirs + ca: "/etc/certs/etcd/my-etcd-ca.crt", + cert: "/var/lib/certs/etcd/my-etcd.crt", + key: "/var/lib/certs/etcd/my-etcd.key", + vol: []v1.Volume{ + { + Name: "etcd-certs-0", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: "/etc/certs/etcd"}, + }, + }, + { + Name: "etcd-certs-1", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: "/var/lib/certs/etcd"}, + }, + }, + }, + volMount: []v1.VolumeMount{ + { + Name: "etcd-certs-0", + MountPath: "/etc/certs/etcd", + ReadOnly: true, + }, + { + Name: "etcd-certs-1", + MountPath: "/var/lib/certs/etcd", + ReadOnly: true, + }, + }, + }, + { + // All three files in different directories + ca: "/etc/certs/etcd/my-etcd-ca.crt", + cert: "/var/lib/certs/etcd/my-etcd.crt", + key: "/var/lib/certs/private/my-etcd.key", + vol: []v1.Volume{ + { + Name: "etcd-certs-0", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: "/etc/certs/etcd"}, + }, + }, + { + Name: "etcd-certs-1", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: "/var/lib/certs/etcd"}, + }, + }, + { + Name: "etcd-certs-2", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: "/var/lib/certs/private"}, + }, + }, + }, + volMount: []v1.VolumeMount{ + { + Name: "etcd-certs-0", + MountPath: "/etc/certs/etcd", + ReadOnly: true, + }, + { + Name: "etcd-certs-1", + MountPath: "/var/lib/certs/etcd", + ReadOnly: true, + }, + { + Name: "etcd-certs-2", + MountPath: "/var/lib/certs/private", + ReadOnly: true, + }, + }, + }, + { + // The most top-level dir should be used + ca: "/etc/certs/etcd/my-etcd-ca.crt", + cert: "/etc/certs/etcd/serving/my-etcd.crt", + key: "/etc/certs/etcd/serving/my-etcd.key", + vol: []v1.Volume{ + { + Name: "etcd-certs-0", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: "/etc/certs/etcd"}, + }, + }, + }, + volMount: []v1.VolumeMount{ + { + Name: "etcd-certs-0", + MountPath: "/etc/certs/etcd", + ReadOnly: true, + }, + }, + }, + { + // The most top-level dir should be used, regardless of order + ca: "/etc/certs/etcd/ca/my-etcd-ca.crt", + cert: "/etc/certs/etcd/my-etcd.crt", + key: "/etc/certs/etcd/my-etcd.key", + vol: []v1.Volume{ + { + Name: "etcd-certs-0", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: "/etc/certs/etcd"}, + }, + }, + }, + volMount: []v1.VolumeMount{ + { + Name: "etcd-certs-0", + MountPath: "/etc/certs/etcd", + ReadOnly: true, + }, + }, + }, + } + + for _, rt := range tests { + actualVol, actualVolMount := getEtcdCertVolumes(kubeadmapi.Etcd{ + CAFile: rt.ca, + CertFile: rt.cert, + KeyFile: rt.key, + }) + if !reflect.DeepEqual(actualVol, rt.vol) { + t.Errorf( + "failed getEtcdCertVolumes:\n\texpected: %v\n\t actual: %v", + rt.vol, + actualVol, + ) + } + if !reflect.DeepEqual(actualVolMount, rt.volMount) { + t.Errorf( + "failed getEtcdCertVolumes:\n\texpected: %v\n\t actual: %v", + rt.volMount, + actualVolMount, + ) + } + } +} + +func TestGetHostPathVolumesForTheControlPlane(t *testing.T) { + var tests = []struct { + cfg *kubeadmapi.MasterConfiguration + vol map[string][]v1.Volume + volMount map[string][]v1.VolumeMount + }{ + { + // Should ignore files in /etc/ssl/certs + cfg: &kubeadmapi.MasterConfiguration{ + CertificatesDir: testCertsDir, + Etcd: kubeadmapi.Etcd{}, + }, + vol: map[string][]v1.Volume{ + kubeAPIServer: { + { + Name: "k8s-certs", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: testCertsDir}, + }, + }, + { + Name: "ca-certs", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: "/etc/ssl/certs"}, + }, + }, + }, + kubeControllerManager: { + { + Name: "k8s-certs", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: testCertsDir}, + }, + }, + { + Name: "ca-certs", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: "/etc/ssl/certs"}, + }, + }, + { + Name: "kubeconfig", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: "/etc/kubernetes/controller-manager.conf"}, + }, + }, + }, + kubeScheduler: { + { + Name: "kubeconfig", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: "/etc/kubernetes/scheduler.conf"}, + }, + }, + }, + }, + volMount: map[string][]v1.VolumeMount{ + kubeAPIServer: { + { + Name: "k8s-certs", + MountPath: testCertsDir, + ReadOnly: true, + }, + { + Name: "ca-certs", + MountPath: "/etc/ssl/certs", + ReadOnly: true, + }, + }, + kubeControllerManager: { + { + Name: "k8s-certs", + MountPath: testCertsDir, + ReadOnly: true, + }, + { + Name: "ca-certs", + MountPath: "/etc/ssl/certs", + ReadOnly: true, + }, + { + Name: "kubeconfig", + MountPath: "/etc/kubernetes/controller-manager.conf", + ReadOnly: true, + }, + }, + kubeScheduler: { + { + Name: "kubeconfig", + MountPath: "/etc/kubernetes/scheduler.conf", + ReadOnly: true, + }, + }, + }, + }, + { + // Should ignore files in /etc/ssl/certs + cfg: &kubeadmapi.MasterConfiguration{ + CertificatesDir: testCertsDir, + Etcd: kubeadmapi.Etcd{ + Endpoints: []string{"foo"}, + CAFile: "/etc/certs/etcd/my-etcd-ca.crt", + CertFile: "/var/lib/certs/etcd/my-etcd.crt", + KeyFile: "/var/lib/certs/etcd/my-etcd.key", + }, + }, + vol: map[string][]v1.Volume{ + kubeAPIServer: { + { + Name: "k8s-certs", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: testCertsDir}, + }, + }, + { + Name: "ca-certs", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: "/etc/ssl/certs"}, + }, + }, + { + Name: "etcd-certs-0", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: "/etc/certs/etcd"}, + }, + }, + { + Name: "etcd-certs-1", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: "/var/lib/certs/etcd"}, + }, + }, + }, + kubeControllerManager: { + { + Name: "k8s-certs", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: testCertsDir}, + }, + }, + { + Name: "ca-certs", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: "/etc/ssl/certs"}, + }, + }, + { + Name: "kubeconfig", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: "/etc/kubernetes/controller-manager.conf"}, + }, + }, + }, + kubeScheduler: { + { + Name: "kubeconfig", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: "/etc/kubernetes/scheduler.conf"}, + }, + }, + }, + }, + volMount: map[string][]v1.VolumeMount{ + kubeAPIServer: { + { + Name: "k8s-certs", + MountPath: testCertsDir, + ReadOnly: true, + }, + { + Name: "ca-certs", + MountPath: "/etc/ssl/certs", + ReadOnly: true, + }, + { + Name: "etcd-certs-0", + MountPath: "/etc/certs/etcd", + ReadOnly: true, + }, + { + Name: "etcd-certs-1", + MountPath: "/var/lib/certs/etcd", + ReadOnly: true, + }, + }, + kubeControllerManager: { + { + Name: "k8s-certs", + MountPath: testCertsDir, + ReadOnly: true, + }, + { + Name: "ca-certs", + MountPath: "/etc/ssl/certs", + ReadOnly: true, + }, + { + Name: "kubeconfig", + MountPath: "/etc/kubernetes/controller-manager.conf", + ReadOnly: true, + }, + }, + kubeScheduler: { + { + Name: "kubeconfig", + MountPath: "/etc/kubernetes/scheduler.conf", + ReadOnly: true, + }, + }, + }, + }, + } + + tmpdir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("Couldn't create tmpdir") + } + defer os.RemoveAll(tmpdir) + + // set up tmp caCertsPkiVolumePath for testing + caCertsPkiVolumePath = fmt.Sprintf("%s/etc/pki", tmpdir) + defer func() { caCertsPkiVolumePath = "/etc/pki" }() + + for _, rt := range tests { + mounts := getHostPathVolumesForTheControlPlane(rt.cfg) + if !reflect.DeepEqual(mounts.volumes, rt.vol) { + t.Errorf( + "failed getHostPathVolumesForTheControlPlane:\n\texpected: %v\n\t actual: %v", + rt.vol, + mounts.volumes, + ) + } + if !reflect.DeepEqual(mounts.volumeMounts, rt.volMount) { + t.Errorf( + "failed getHostPathVolumesForTheControlPlane:\n\texpected: %v\n\t actual: %v", + rt.volMount, + mounts.volumeMounts, + ) + } + } +}