Merge pull request #57561 from dims/enable-privileged-container-for-apiserver-and-controller

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Enable privileged containers for apiserver and controller

**What this PR does / why we need it**:

In OpenStack environment, when there is no metadata service, we
look at the config drive to figure out the metadata. Since we need
to run commands like blkid, we need to ensure that api server and
kube controller are running in the privileged mode.

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #47392
Fixes https://github.com/kubernetes/kubeadm/issues/588

**Special notes for your reviewer**:

**Release note**:

```release-note
Fix issue when using OpenStack config drive for node metadata
```

cmd/kubeadm/app/apis/kubeadm/types.go

@@ -39,6 +39,11 @@ type MasterConfiguration struct {
 	NodeName             string
 	AuthorizationModes   []string
 
+	// Mark the controller and api server pods as privileged as some cloud
+	// controllers like openstack need escalated privileges under some conditions
+	// example - loading a config drive to fetch node information
+	PrivilegedPods bool
+
 	Token    string
 	TokenTTL *metav1.Duration
 

cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go

@@ -39,6 +39,11 @@ type MasterConfiguration struct {
 	NodeName             string               `json:"nodeName"`
 	AuthorizationModes   []string             `json:"authorizationModes,omitempty"`
 
+	// Mark the controller and api server pods as privileged as some cloud
+	// controllers like openstack need escalated privileges under some conditions
+	// example - loading a config drive to fetch node information
+	PrivilegedPods bool `json:"privilegedPods"`
+
 	Token    string           `json:"token"`
 	TokenTTL *metav1.Duration `json:"tokenTTL,omitempty"`
 

cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.conversion.go

@@ -202,6 +202,7 @@ func autoConvert_v1alpha1_MasterConfiguration_To_kubeadm_MasterConfiguration(in
 	out.CloudProvider = in.CloudProvider
 	out.NodeName = in.NodeName
 	out.AuthorizationModes = *(*[]string)(unsafe.Pointer(&in.AuthorizationModes))
+	out.PrivilegedPods = in.PrivilegedPods
 	out.Token = in.Token
 	out.TokenTTL = (*v1.Duration)(unsafe.Pointer(in.TokenTTL))
 	out.APIServerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.APIServerExtraArgs))
@@ -243,6 +244,7 @@ func autoConvert_kubeadm_MasterConfiguration_To_v1alpha1_MasterConfiguration(in
 	out.CloudProvider = in.CloudProvider
 	out.NodeName = in.NodeName
 	out.AuthorizationModes = *(*[]string)(unsafe.Pointer(&in.AuthorizationModes))
+	out.PrivilegedPods = in.PrivilegedPods
 	out.Token = in.Token
 	out.TokenTTL = (*v1.Duration)(unsafe.Pointer(in.TokenTTL))
 	out.APIServerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.APIServerExtraArgs))

cmd/kubeadm/app/cmd/upgrade/common_test.go

@@ -59,6 +59,7 @@ func TestPrintConfiguration(t *testing.T) {
 	  podSubnet: ""
 	  serviceSubnet: ""
 	nodeName: ""
+	privilegedPods: false
 	token: ""
 	unifiedControlPlaneImage: ""
 `),
@@ -92,6 +93,7 @@ func TestPrintConfiguration(t *testing.T) {
 	  podSubnet: ""
 	  serviceSubnet: 10.96.0.1/12
 	nodeName: ""
+	privilegedPods: false
 	token: ""
 	unifiedControlPlaneImage: ""
 `),
@@ -135,6 +137,7 @@ func TestPrintConfiguration(t *testing.T) {
 	  podSubnet: ""
 	  serviceSubnet: ""
 	nodeName: ""
+	privilegedPods: false
 	token: ""
 	unifiedControlPlaneImage: ""
 `),

cmd/kubeadm/app/phases/controlplane/BUILD

@@ -44,6 +44,7 @@ go_library(
         "//cmd/kubeadm/app/util/staticpod:go_default_library",
         "//pkg/kubeapiserver/authorizer/modes:go_default_library",
         "//pkg/master/reconcilers:go_default_library",
+        "//pkg/util/pointer:go_default_library",
         "//pkg/util/version:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",

cmd/kubeadm/app/phases/controlplane/manifests.go

@@ -35,6 +35,7 @@ import (
 	staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
 	authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
 	"k8s.io/kubernetes/pkg/master/reconcilers"
+	utilpointer "k8s.io/kubernetes/pkg/util/pointer"
 	"k8s.io/kubernetes/pkg/util/version"
 )
 
@@ -104,6 +105,18 @@ func GetStaticPodSpecs(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.
 		}, mounts.GetVolumes(kubeadmconstants.KubeScheduler)),
 	}
 
+	// Some cloud providers need extra privileges for example to load node information from a config drive
+	// TODO: when we fully to external cloud providers and the api server and controller manager do not need
+	// to call out to cloud provider code, we can remove the support for the PrivilegedPods
+	if cfg.PrivilegedPods {
+		staticPodSpecs[kubeadmconstants.KubeAPIServer].Spec.Containers[0].SecurityContext = &v1.SecurityContext{
+			Privileged: utilpointer.BoolPtr(true),
+		}
+		staticPodSpecs[kubeadmconstants.KubeControllerManager].Spec.Containers[0].SecurityContext = &v1.SecurityContext{
+			Privileged: utilpointer.BoolPtr(true),
+		}
+	}
+
 	return staticPodSpecs
 }
 

cmd/kubeadm/app/phases/controlplane/manifests_test.go

@@ -135,6 +135,58 @@ func TestCreateStaticPodFilesAndWrappers(t *testing.T) {
 	}
 }
 
+func TestCreatePrivilegedContainerForOpenStack(t *testing.T) {
+	// Creates a Master Configuration with OpenStack cloud provider
+	var staticPodNames = []string{
+		kubeadmconstants.KubeAPIServer,
+		kubeadmconstants.KubeControllerManager,
+	}
+	var assertions = []struct {
+		cloudProvider     string
+		privilegedPods    bool
+		expectedPrivilege bool
+	}{
+		{
+			cloudProvider:     "",
+			expectedPrivilege: false,
+		},
+		{
+			cloudProvider:     "aws",
+			expectedPrivilege: false,
+		},
+		{
+			cloudProvider:     "openstack",
+			privilegedPods:    true,
+			expectedPrivilege: true,
+		},
+	}
+
+	for _, assertion := range assertions {
+		cfg := &kubeadmapi.MasterConfiguration{
+			KubernetesVersion: "v1.9.0",
+			CloudProvider:     assertion.cloudProvider,
+			PrivilegedPods:    assertion.privilegedPods,
+		}
+
+		k8sVersion, _ := version.ParseSemantic(cfg.KubernetesVersion)
+		specs := GetStaticPodSpecs(cfg, k8sVersion)
+
+		for _, podname := range staticPodNames {
+			spec, _ := specs[podname]
+			sc := spec.Spec.Containers[0].SecurityContext
+			if assertion.expectedPrivilege == true {
+				if sc == nil || sc.Privileged == nil || *sc.Privileged == false {
+					t.Errorf("GetStaticPodSpecs did not enable privileged containers in %s pod for provider %s", podname, assertion.cloudProvider)
+				}
+			} else {
+				if sc != nil && sc.Privileged != nil && *sc.Privileged == true {
+					t.Errorf("GetStaticPodSpecs enabled privileged containers in %s pod for provider %s", podname, assertion.cloudProvider)
+				}
+			}
+		}
+	}
+}
+
 func TestGetAPIServerCommand(t *testing.T) {
 	var tests = []struct {
 		cfg      *kubeadmapi.MasterConfiguration