From 59b4b124df03e496b5c05909d1ff15e916c2c4d8 Mon Sep 17 00:00:00 2001 From: Vinayak Goyal Date: Wed, 9 Jun 2021 14:59:03 -0700 Subject: [PATCH] Update kubeadm control-plane to run as non-root. --- .../app/phases/controlplane/manifests.go | 26 +++ cmd/kubeadm/app/util/staticpod/utils.go | 15 ++ cmd/kubeadm/app/util/staticpod/utils_linux.go | 158 ++++++++++++++++++ .../app/util/staticpod/utils_linux_test.go | 135 +++++++++++++++ .../app/util/staticpod/utils_others.go | 30 ++++ 5 files changed, 364 insertions(+) create mode 100644 cmd/kubeadm/app/util/staticpod/utils_linux.go create mode 100644 cmd/kubeadm/app/util/staticpod/utils_linux_test.go create mode 100644 cmd/kubeadm/app/util/staticpod/utils_others.go diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go index a7cf8e940a1..96c7d8c1bb9 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests.go @@ -34,6 +34,7 @@ import ( certphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" + "k8s.io/kubernetes/cmd/kubeadm/app/util/users" utilsnet "k8s.io/utils/net" ) @@ -96,6 +97,19 @@ func CreateStaticPodFiles(manifestDir, patchesDir string, cfg *kubeadmapi.Cluste klog.V(1).Infoln("[control-plane] getting StaticPodSpecs") specs := GetStaticPodSpecs(cfg, endpoint) + var usersAndGroups *users.UsersAndGroups + var err error + if features.Enabled(cfg.FeatureGates, features.RootlessControlPlane) { + if isDryRun { + fmt.Printf("[dryrun] Would create users and groups for %+v to run as non-root\n", componentNames) + } else { + usersAndGroups, err = staticpodutil.GetUsersAndGroups() + if err != nil { + return errors.Wrap(err, "failed to create users and groups") + } + } + } + // creates required static pod specs for _, componentName := range componentNames { // retrieves the StaticPodSpec for given component @@ -109,6 +123,18 @@ func CreateStaticPodFiles(manifestDir, patchesDir string, cfg *kubeadmapi.Cluste klog.V(2).Infof("[control-plane] adding volume %q for component %q", v.Name, componentName) } + if features.Enabled(cfg.FeatureGates, features.RootlessControlPlane) { + if isDryRun { + fmt.Printf("[dryrun] Would update static pod manifest for %q to run run as non-root\n", componentName) + } else { + if usersAndGroups != nil { + if err := staticpodutil.RunComponentAsNonRoot(componentName, &spec, usersAndGroups, cfg); err != nil { + return errors.Wrapf(err, "failed to run component %q as non-root", componentName) + } + } + } + } + // if patchesDir is defined, patch the static Pod manifest if patchesDir != "" { patchedSpec, err := staticpodutil.PatchStaticPod(&spec, patchesDir, os.Stdout) diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go index 59e09699785..68a9c16fcda 100644 --- a/cmd/kubeadm/app/util/staticpod/utils.go +++ b/cmd/kubeadm/app/util/staticpod/utils.go @@ -26,6 +26,7 @@ import ( "os" "sort" "strings" + "sync" "github.com/pkg/errors" @@ -37,6 +38,7 @@ import ( kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/cmd/kubeadm/app/util/patches" + "k8s.io/kubernetes/cmd/kubeadm/app/util/users" ) const ( @@ -47,6 +49,11 @@ const ( kubeSchedulerBindAddressArg = "bind-address" ) +var ( + usersAndGroups *users.UsersAndGroups + usersAndGroupsOnce sync.Once +) + // ComponentPod returns a Pod object from the container, volume and annotations specifications func ComponentPod(container v1.Container, volumes map[string]v1.Volume, annotations map[string]string) v1.Pod { return v1.Pod{ @@ -378,3 +385,11 @@ func getProbeAddress(addr string) string { } return addr } + +func GetUsersAndGroups() (*users.UsersAndGroups, error) { + var err error + usersAndGroupsOnce.Do(func() { + usersAndGroups, err = users.AddUsersAndGroups() + }) + return usersAndGroups, err +} diff --git a/cmd/kubeadm/app/util/staticpod/utils_linux.go b/cmd/kubeadm/app/util/staticpod/utils_linux.go new file mode 100644 index 00000000000..6b2425c3199 --- /dev/null +++ b/cmd/kubeadm/app/util/staticpod/utils_linux.go @@ -0,0 +1,158 @@ +// +build linux + +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package staticpod + +import ( + "fmt" + "path/filepath" + + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + certphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" + "k8s.io/kubernetes/cmd/kubeadm/app/util/users" + "k8s.io/utils/pointer" +) + +type pathOwerAndPermissionsUpdaterFunc func(path string, uid, gid int64, perms uint32) error + +// RunComponentAsNonRoot updates the pod manifest and the hostVolume permissions to run as non root. +func RunComponentAsNonRoot(componentName string, pod *v1.Pod, usersAndGroups *users.UsersAndGroups, cfg *kubeadmapi.ClusterConfiguration) error { + switch componentName { + case kubeadmconstants.KubeAPIServer: + return runKubeAPIServerAsNonRoot( + pod, + usersAndGroups.Users.ID(kubeadmconstants.KubeAPIServerUserName), + usersAndGroups.Groups.ID(kubeadmconstants.KubeAPIServerUserName), + usersAndGroups.Groups.ID(kubeadmconstants.ServiceAccountKeyReadersGroupName), + users.UpdatePathOwnerAndPermissions, + cfg, + ) + case kubeadmconstants.KubeControllerManager: + return runKubeControllerManagerAsNonRoot( + pod, + usersAndGroups.Users.ID(kubeadmconstants.KubeControllerManagerUserName), + usersAndGroups.Groups.ID(kubeadmconstants.KubeControllerManagerUserName), + usersAndGroups.Groups.ID(kubeadmconstants.ServiceAccountKeyReadersGroupName), + users.UpdatePathOwnerAndPermissions, + cfg, + ) + case kubeadmconstants.KubeScheduler: + return runKubeSchedulerAsNonRoot( + pod, + usersAndGroups.Users.ID(kubeadmconstants.KubeControllerManagerUserName), + usersAndGroups.Groups.ID(kubeadmconstants.KubeControllerManagerUserName), + users.UpdatePathOwnerAndPermissions, + ) + } + return errors.New(fmt.Sprintf("component name %q is not valid", componentName)) +} + +// runKubeAPIServerAsNonRoot updates the pod manifest and the hostVolume permissions to run kube-apiserver as non root. +func runKubeAPIServerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup, supplementalGroup *int64, updatePathOwnerAndPermissions pathOwerAndPermissionsUpdaterFunc, cfg *kubeadmapi.ClusterConfiguration) error { + saPublicKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPublicKeyName) + if err := updatePathOwnerAndPermissions(saPublicKeyFile, *runAsUser, *runAsGroup, 0600); err != nil { + return err + } + saPrivateKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPrivateKeyName) + if err := updatePathOwnerAndPermissions(saPrivateKeyFile, 0, *supplementalGroup, 0640); err != nil { + return err + } + apiServerKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKeyName) + if err := updatePathOwnerAndPermissions(apiServerKeyFile, *runAsUser, *runAsGroup, 0600); err != nil { + return err + } + apiServerKubeletClientKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKubeletClientKeyName) + if err := updatePathOwnerAndPermissions(apiServerKubeletClientKeyFile, *runAsUser, *runAsGroup, 0600); err != nil { + return err + } + frontProxyClientKeyName := filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientKeyName) + if err := updatePathOwnerAndPermissions(frontProxyClientKeyName, *runAsUser, *runAsGroup, 0600); err != nil { + return err + } + if cfg.Etcd.External == nil { + apiServerEtcdClientKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientKeyName) + if err := updatePathOwnerAndPermissions(apiServerEtcdClientKeyFile, *runAsUser, *runAsGroup, 0600); err != nil { + return err + } + } + pod.Spec.Containers[0].SecurityContext = &v1.SecurityContext{ + Capabilities: &v1.Capabilities{ + // We drop all capabilities that are added by default. + Drop: []v1.Capability{"ALL"}, + // kube-apiserver binary has the file capability cap_net_bind_service applied to it. + // This means that we must add this capability when running as non-root even if the + // capability is not required. + Add: []v1.Capability{"NET_BIND_SERVICE"}, + }, + } + pod.Spec.SecurityContext.RunAsGroup = runAsGroup + pod.Spec.SecurityContext.RunAsUser = runAsUser + pod.Spec.SecurityContext.SupplementalGroups = []int64{*supplementalGroup} + return nil +} + +// runKubeControllerManagerAsNonRoot updates the pod manifest and the hostVolume permissions to run kube-controller-manager as non root. +func runKubeControllerManagerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup, supplementalGroup *int64, updatePathOwnerAndPermissions pathOwerAndPermissionsUpdaterFunc, cfg *kubeadmapi.ClusterConfiguration) error { + kubeconfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ControllerManagerKubeConfigFileName) + if err := updatePathOwnerAndPermissions(kubeconfigFile, *runAsUser, *runAsGroup, 0600); err != nil { + return err + } + saPrivateKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPrivateKeyName) + if err := updatePathOwnerAndPermissions(saPrivateKeyFile, 0, *supplementalGroup, 0640); err != nil { + return err + } + if res, _ := certphase.UsingExternalCA(cfg); !res { + caKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.CAKeyName) + err := updatePathOwnerAndPermissions(caKeyFile, *runAsUser, *runAsGroup, 0600) + if err != nil { + return err + } + } + pod.Spec.Containers[0].SecurityContext = &v1.SecurityContext{ + AllowPrivilegeEscalation: pointer.Bool(false), + Capabilities: &v1.Capabilities{ + // We drop all capabilities that are added by default. + Drop: []v1.Capability{"ALL"}, + }, + } + pod.Spec.SecurityContext.RunAsUser = runAsUser + pod.Spec.SecurityContext.RunAsGroup = runAsGroup + pod.Spec.SecurityContext.SupplementalGroups = []int64{*supplementalGroup} + return nil +} + +// runKubeSchedulerAsNonRoot updates the pod manifest and the hostVolume permissions to run kube-scheduler as non root. +func runKubeSchedulerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup *int64, updatePathOwnerAndPermissions pathOwerAndPermissionsUpdaterFunc) error { + kubeconfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.SchedulerKubeConfigFileName) + if err := updatePathOwnerAndPermissions(kubeconfigFile, *runAsUser, *runAsGroup, 0600); err != nil { + return err + } + pod.Spec.Containers[0].SecurityContext = &v1.SecurityContext{ + AllowPrivilegeEscalation: pointer.Bool(false), + // We drop all capabilities that are added by default. + Capabilities: &v1.Capabilities{ + Drop: []v1.Capability{"ALL"}, + }, + } + pod.Spec.SecurityContext.RunAsUser = runAsUser + pod.Spec.SecurityContext.RunAsGroup = runAsGroup + return nil +} diff --git a/cmd/kubeadm/app/util/staticpod/utils_linux_test.go b/cmd/kubeadm/app/util/staticpod/utils_linux_test.go new file mode 100644 index 00000000000..f28112608b0 --- /dev/null +++ b/cmd/kubeadm/app/util/staticpod/utils_linux_test.go @@ -0,0 +1,135 @@ +// +build linux + +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package staticpod + +import ( + "path/filepath" + "reflect" + "testing" + + v1 "k8s.io/api/core/v1" + "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/utils/pointer" +) + +type ownerAndPermissions struct { + uid int64 + gid int64 + permissions uint32 +} + +func verifyPodSecurityContext(t *testing.T, pod *v1.Pod, wantRunAsUser, wantRunAsGroup int64, wantSupGroup []int64) { + t.Helper() + wantPodSecurityContext := &v1.PodSecurityContext{ + RunAsUser: pointer.Int64(wantRunAsUser), + RunAsGroup: pointer.Int64(wantRunAsGroup), + SupplementalGroups: wantSupGroup, + SeccompProfile: &v1.SeccompProfile{ + Type: v1.SeccompProfileTypeRuntimeDefault, + }, + } + if !reflect.DeepEqual(wantPodSecurityContext, pod.Spec.SecurityContext) { + t.Errorf("unexpected diff in PodSecurityContext, want: %+v, got: %+v", wantPodSecurityContext, pod.Spec.SecurityContext) + } +} + +func verifyContainerSecurityContext(t *testing.T, container v1.Container, addCaps, dropCaps []v1.Capability, allowPrivielege *bool) { + t.Helper() + wantContainerSecurityContext := &v1.SecurityContext{ + AllowPrivilegeEscalation: allowPrivielege, + Capabilities: &v1.Capabilities{ + Add: addCaps, + Drop: dropCaps, + }, + } + if !reflect.DeepEqual(wantContainerSecurityContext, container.SecurityContext) { + t.Errorf("unexpected diff in container SecurityContext, want: %+v, got: %+v", wantContainerSecurityContext, container.SecurityContext) + } +} + +func verifyFilePermissions(t *testing.T, updatedFiles, wantFiles map[string]ownerAndPermissions) { + t.Helper() + if !reflect.DeepEqual(updatedFiles, wantFiles) { + t.Errorf("unexpected diff in file owners and permissions want: %+v, got: %+v", wantFiles, updatedFiles) + } +} + +func TestRunKubeAPIServerAsNonRoot(t *testing.T) { + cfg := &kubeadm.ClusterConfiguration{} + pod := ComponentPod(v1.Container{Name: "kube-apiserver"}, nil, nil) + var runAsUser, runAsGroup, supGroup int64 = 1000, 1001, 1002 + updatedFiles := map[string]ownerAndPermissions{} + if err := runKubeAPIServerAsNonRoot(&pod, &runAsUser, &runAsGroup, &supGroup, func(path string, uid, gid int64, perms uint32) error { + updatedFiles[path] = ownerAndPermissions{uid: uid, gid: gid, permissions: perms} + return nil + }, cfg); err != nil { + t.Fatal(err) + } + verifyPodSecurityContext(t, &pod, runAsUser, runAsGroup, []int64{supGroup}) + verifyContainerSecurityContext(t, pod.Spec.Containers[0], []v1.Capability{"NET_BIND_SERVICE"}, []v1.Capability{"ALL"}, nil) + wantUpdateFiles := map[string]ownerAndPermissions{ + filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPublicKeyName): {uid: runAsUser, gid: runAsGroup, permissions: 0600}, + filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPrivateKeyName): {uid: 0, gid: supGroup, permissions: 0640}, + filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKeyName): {uid: runAsUser, gid: runAsGroup, permissions: 0600}, + filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKubeletClientKeyName): {uid: runAsUser, gid: runAsGroup, permissions: 0600}, + filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientKeyName): {uid: runAsUser, gid: runAsGroup, permissions: 0600}, + filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientKeyName): {uid: runAsUser, gid: runAsGroup, permissions: 0600}, + } + verifyFilePermissions(t, updatedFiles, wantUpdateFiles) +} + +func TestRunKubeControllerManagerAsNonRoot(t *testing.T) { + cfg := &kubeadm.ClusterConfiguration{} + pod := ComponentPod(v1.Container{Name: "kube-controller-manager"}, nil, nil) + var runAsUser, runAsGroup, supGroup int64 = 1000, 1001, 1002 + updatedFiles := map[string]ownerAndPermissions{} + if err := runKubeControllerManagerAsNonRoot(&pod, &runAsUser, &runAsGroup, &supGroup, func(path string, uid, gid int64, perms uint32) error { + updatedFiles[path] = ownerAndPermissions{uid: uid, gid: gid, permissions: perms} + return nil + }, cfg); err != nil { + t.Fatal(err) + } + verifyPodSecurityContext(t, &pod, runAsUser, runAsGroup, []int64{supGroup}) + verifyContainerSecurityContext(t, pod.Spec.Containers[0], nil, []v1.Capability{"ALL"}, pointer.Bool(false)) + wantUpdateFiles := map[string]ownerAndPermissions{ + filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ControllerManagerKubeConfigFileName): {uid: runAsUser, gid: runAsGroup, permissions: 0600}, + filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPrivateKeyName): {uid: 0, gid: supGroup, permissions: 0640}, + filepath.Join(cfg.CertificatesDir, kubeadmconstants.CAKeyName): {uid: runAsUser, gid: runAsGroup, permissions: 0600}, + } + verifyFilePermissions(t, updatedFiles, wantUpdateFiles) +} + +func TestRunKubeSchedulerAsNonRoot(t *testing.T) { + pod := ComponentPod(v1.Container{Name: "kube-scheduler"}, nil, nil) + var runAsUser, runAsGroup int64 = 1000, 1001 + updatedFiles := map[string]ownerAndPermissions{} + if err := runKubeSchedulerAsNonRoot(&pod, &runAsUser, &runAsGroup, func(path string, uid, gid int64, perms uint32) error { + updatedFiles[path] = ownerAndPermissions{uid: uid, gid: gid, permissions: perms} + return nil + }); err != nil { + t.Fatal(err) + } + verifyPodSecurityContext(t, &pod, runAsUser, runAsGroup, nil) + verifyContainerSecurityContext(t, pod.Spec.Containers[0], nil, []v1.Capability{"ALL"}, pointer.Bool(false)) + wantUpdateFiles := map[string]ownerAndPermissions{ + filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.SchedulerKubeConfigFileName): {uid: runAsUser, gid: runAsGroup, permissions: 0600}, + } + verifyFilePermissions(t, updatedFiles, wantUpdateFiles) +} diff --git a/cmd/kubeadm/app/util/staticpod/utils_others.go b/cmd/kubeadm/app/util/staticpod/utils_others.go new file mode 100644 index 00000000000..bbb37abae69 --- /dev/null +++ b/cmd/kubeadm/app/util/staticpod/utils_others.go @@ -0,0 +1,30 @@ +// +build !linux + +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package staticpod + +import ( + v1 "k8s.io/api/core/v1" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/util/users" +) + +// RunComponentAsNonRoot is a NO-OP on non linux. +func RunComponentAsNonRoot(componentName string, pod *v1.Pod, usersAndGroups *users.UsersAndGroups, cfg *kubeadmapi.ClusterConfiguration) error { + return nil +}