From 8e49774f90a46988d67fa380b3667542068e8b20 Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Fri, 22 Dec 2017 11:32:03 -0500 Subject: [PATCH 1/2] Enable privileged containers for apiserver and controller In OpenStack environment, when there is no metadata service, we look at the config drive to figure out the metadata. Since we need to run commands like blkid, we need to ensure that api server and kube controller are running in the privileged mode. So add a new field in MasterConfiguration for specifying that the api server and controller manager (s) need extra privileges. Added a TODO to remove this code when we fully yank out cloud provider specific calls from these processes. --- cmd/kubeadm/app/apis/kubeadm/types.go | 5 ++ .../app/apis/kubeadm/v1alpha1/types.go | 5 ++ cmd/kubeadm/app/cmd/upgrade/common_test.go | 3 ++ cmd/kubeadm/app/phases/controlplane/BUILD | 1 + .../app/phases/controlplane/manifests.go | 13 +++++ .../app/phases/controlplane/manifests_test.go | 52 +++++++++++++++++++ 6 files changed, 79 insertions(+) diff --git a/cmd/kubeadm/app/apis/kubeadm/types.go b/cmd/kubeadm/app/apis/kubeadm/types.go index 659a8a5442b..176819374cf 100644 --- a/cmd/kubeadm/app/apis/kubeadm/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/types.go @@ -39,6 +39,11 @@ type MasterConfiguration struct { NodeName string AuthorizationModes []string + // Mark the controller and api server pods as privileged as some cloud + // controllers like openstack need escalated privileges under some conditions + // example - loading a config drive to fetch node information + PrivilegedPods bool + Token string TokenTTL *metav1.Duration diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go index ee7b84f1b1e..32d064bc80b 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go @@ -39,6 +39,11 @@ type MasterConfiguration struct { NodeName string `json:"nodeName"` AuthorizationModes []string `json:"authorizationModes,omitempty"` + // Mark the controller and api server pods as privileged as some cloud + // controllers like openstack need escalated privileges under some conditions + // example - loading a config drive to fetch node information + PrivilegedPods bool `json:"privilegedPods"` + Token string `json:"token"` TokenTTL *metav1.Duration `json:"tokenTTL,omitempty"` diff --git a/cmd/kubeadm/app/cmd/upgrade/common_test.go b/cmd/kubeadm/app/cmd/upgrade/common_test.go index 5ebff084e0a..61b2ee37e79 100644 --- a/cmd/kubeadm/app/cmd/upgrade/common_test.go +++ b/cmd/kubeadm/app/cmd/upgrade/common_test.go @@ -59,6 +59,7 @@ func TestPrintConfiguration(t *testing.T) { podSubnet: "" serviceSubnet: "" nodeName: "" + privilegedPods: false token: "" unifiedControlPlaneImage: "" `), @@ -92,6 +93,7 @@ func TestPrintConfiguration(t *testing.T) { podSubnet: "" serviceSubnet: 10.96.0.1/12 nodeName: "" + privilegedPods: false token: "" unifiedControlPlaneImage: "" `), @@ -135,6 +137,7 @@ func TestPrintConfiguration(t *testing.T) { podSubnet: "" serviceSubnet: "" nodeName: "" + privilegedPods: false token: "" unifiedControlPlaneImage: "" `), diff --git a/cmd/kubeadm/app/phases/controlplane/BUILD b/cmd/kubeadm/app/phases/controlplane/BUILD index 830d6ae90bd..c706fc98714 100644 --- a/cmd/kubeadm/app/phases/controlplane/BUILD +++ b/cmd/kubeadm/app/phases/controlplane/BUILD @@ -44,6 +44,7 @@ go_library( "//cmd/kubeadm/app/util/staticpod:go_default_library", "//pkg/kubeapiserver/authorizer/modes:go_default_library", "//pkg/master/reconcilers:go_default_library", + "//pkg/util/pointer:go_default_library", "//pkg/util/version:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go index f1ae775f864..c6fc386e551 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests.go @@ -35,6 +35,7 @@ import ( staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" "k8s.io/kubernetes/pkg/master/reconcilers" + utilpointer "k8s.io/kubernetes/pkg/util/pointer" "k8s.io/kubernetes/pkg/util/version" ) @@ -104,6 +105,18 @@ func GetStaticPodSpecs(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version. }, mounts.GetVolumes(kubeadmconstants.KubeScheduler)), } + // Some cloud providers need extra privileges for example to load node information from a config drive + // TODO: when we fully to external cloud providers and the api server and controller manager do not need + // to call out to cloud provider code, we can remove the support for the PrivilegedPods + if cfg.PrivilegedPods { + staticPodSpecs[kubeadmconstants.KubeAPIServer].Spec.Containers[0].SecurityContext = &v1.SecurityContext{ + Privileged: utilpointer.BoolPtr(true), + } + staticPodSpecs[kubeadmconstants.KubeControllerManager].Spec.Containers[0].SecurityContext = &v1.SecurityContext{ + Privileged: utilpointer.BoolPtr(true), + } + } + return staticPodSpecs } diff --git a/cmd/kubeadm/app/phases/controlplane/manifests_test.go b/cmd/kubeadm/app/phases/controlplane/manifests_test.go index 4764bf534f5..2b1e618967b 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests_test.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests_test.go @@ -135,6 +135,58 @@ func TestCreateStaticPodFilesAndWrappers(t *testing.T) { } } +func TestCreatePrivilegedContainerForOpenStack(t *testing.T) { + // Creates a Master Configuration with OpenStack cloud provider + var staticPodNames = []string{ + kubeadmconstants.KubeAPIServer, + kubeadmconstants.KubeControllerManager, + } + var assertions = []struct { + cloudProvider string + privilegedPods bool + expectedPrivilege bool + }{ + { + cloudProvider: "", + expectedPrivilege: false, + }, + { + cloudProvider: "aws", + expectedPrivilege: false, + }, + { + cloudProvider: "openstack", + privilegedPods: true, + expectedPrivilege: true, + }, + } + + for _, assertion := range assertions { + cfg := &kubeadmapi.MasterConfiguration{ + KubernetesVersion: "v1.9.0", + CloudProvider: assertion.cloudProvider, + PrivilegedPods: assertion.privilegedPods, + } + + k8sVersion, _ := version.ParseSemantic(cfg.KubernetesVersion) + specs := GetStaticPodSpecs(cfg, k8sVersion) + + for _, podname := range staticPodNames { + spec, _ := specs[podname] + sc := spec.Spec.Containers[0].SecurityContext + if assertion.expectedPrivilege == true { + if sc == nil || sc.Privileged == nil || *sc.Privileged == false { + t.Errorf("GetStaticPodSpecs did not enable privileged containers in %s pod for provider %s", podname, assertion.cloudProvider) + } + } else { + if sc != nil && sc.Privileged != nil && *sc.Privileged == true { + t.Errorf("GetStaticPodSpecs enabled privileged containers in %s pod for provider %s", podname, assertion.cloudProvider) + } + } + } + } +} + func TestGetAPIServerCommand(t *testing.T) { var tests = []struct { cfg *kubeadmapi.MasterConfiguration From 658a27cebfce8c9dd3f5a76af1a5e169e2bd9769 Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Wed, 17 Jan 2018 22:26:31 -0500 Subject: [PATCH 2/2] Update generated code --- .../app/apis/kubeadm/v1alpha1/zz_generated.conversion.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.conversion.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.conversion.go index 79e487a0802..17b5f458ecc 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.conversion.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.conversion.go @@ -202,6 +202,7 @@ func autoConvert_v1alpha1_MasterConfiguration_To_kubeadm_MasterConfiguration(in out.CloudProvider = in.CloudProvider out.NodeName = in.NodeName out.AuthorizationModes = *(*[]string)(unsafe.Pointer(&in.AuthorizationModes)) + out.PrivilegedPods = in.PrivilegedPods out.Token = in.Token out.TokenTTL = (*v1.Duration)(unsafe.Pointer(in.TokenTTL)) out.APIServerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.APIServerExtraArgs)) @@ -243,6 +244,7 @@ func autoConvert_kubeadm_MasterConfiguration_To_v1alpha1_MasterConfiguration(in out.CloudProvider = in.CloudProvider out.NodeName = in.NodeName out.AuthorizationModes = *(*[]string)(unsafe.Pointer(&in.AuthorizationModes)) + out.PrivilegedPods = in.PrivilegedPods out.Token = in.Token out.TokenTTL = (*v1.Duration)(unsafe.Pointer(in.TokenTTL)) out.APIServerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.APIServerExtraArgs))