github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-24 20:24:09 +00:00
Code Issues Projects Releases Wiki Activity

Update etcd in kubeadm to run as non-root.

Browse Source
This commit is contained in:
Vinayak Goyal 2021-06-14 16:13:57 -07:00
parent 8a6a26714c
commit 5a0756c5f4
6 changed files with 138 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
cmd/kubeadm/app/cmd/phases/join/controlplanejoin.go
View File

@ -145,7 +145,8 @@ func runEtcdPhase(c workflow.RunData) error {
// because it needs two members as majority to agree on the consensus. You will only see this behavior between the time
// etcdctl member add informs the cluster about the new member and the new member successfully establishing a connection to the
// existing one."
if err := etcdphase.CreateStackedEtcdStaticPodManifestFile(client, kubeadmconstants.GetStaticPodDirectory(), data.PatchesDir(), cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint); err != nil {
// TODO: add support for join dry-run: https://github.com/kubernetes/kubeadm/issues/2505
if err := etcdphase.CreateStackedEtcdStaticPodManifestFile(client, kubeadmconstants.GetStaticPodDirectory(), data.PatchesDir(), cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, false /* isDryRun */); err != nil {
return errors.Wrap(err, "error creating local etcd static pod manifest file")
}

74
cmd/kubeadm/app/phases/etcd/local.go
View File

@ -34,10 +34,12 @@ import (
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/features"
"k8s.io/kubernetes/cmd/kubeadm/app/images"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd"
staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
"k8s.io/kubernetes/cmd/kubeadm/app/util/users"
)
const (
@ -54,20 +56,8 @@ func CreateLocalEtcdStaticPodManifestFile(manifestDir, patchesDir string, nodeNa
if cfg.Etcd.External != nil {
return errors.New("etcd static pod manifest cannot be generated for cluster using external etcd")
}
// gets etcd StaticPodSpec
spec := GetEtcdPodSpec(cfg, endpoint, nodeName, []etcdutil.Member{})
// if patchesDir is defined, patch the static Pod manifest
if patchesDir != "" {
patchedSpec, err := staticpodutil.PatchStaticPod(&spec, patchesDir, os.Stdout)
if err != nil {
return errors.Wrapf(err, "failed to patch static Pod manifest file for %q", kubeadmconstants.Etcd)
}
spec = *patchedSpec
}
// writes etcd StaticPod to disk
if err := staticpodutil.WriteStaticPodToDisk(kubeadmconstants.Etcd, manifestDir, spec); err != nil {
if err := prepareAndWriteEtcdStaticPod(manifestDir, patchesDir, cfg, endpoint, nodeName, []etcdutil.Member{}, isDryRun); err != nil {
return err
}
@ -143,7 +133,7 @@ func RemoveStackedEtcdMemberFromCluster(client clientset.Interface, cfg *kubeadm
// CreateStackedEtcdStaticPodManifestFile will write local etcd static pod manifest file
// for an additional etcd member that is joining an existing local/stacked etcd cluster.
// Other members of the etcd cluster will be notified of the joining node in beforehand as well.
func CreateStackedEtcdStaticPodManifestFile(client clientset.Interface, manifestDir, patchesDir string, nodeName string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint) error {
func CreateStackedEtcdStaticPodManifestFile(client clientset.Interface, manifestDir, patchesDir string, nodeName string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, isDryRun bool) error {
// creates an etcd client that connects to all the local/stacked etcd members
klog.V(1).Info("creating etcd client that connects to etcd pods")
etcdClient, err := etcdutil.NewFromCluster(client, cfg.CertificatesDir)
@ -188,20 +178,7 @@ func CreateStackedEtcdStaticPodManifestFile(client clientset.Interface, manifest
fmt.Printf("[etcd] Creating static Pod manifest for %q\n", kubeadmconstants.Etcd)
// gets etcd StaticPodSpec, actualized for the current InitConfiguration and the new list of etcd members
spec := GetEtcdPodSpec(cfg, endpoint, nodeName, initialCluster)
// if patchesDir is defined, patch the static Pod manifest
if patchesDir != "" {
patchedSpec, err := staticpodutil.PatchStaticPod(&spec, patchesDir, os.Stdout)
if err != nil {
return errors.Wrapf(err, "failed to patch static Pod manifest file for %q", kubeadmconstants.Etcd)
}
spec = *patchedSpec
}
// writes etcd StaticPod to disk
if err := staticpodutil.WriteStaticPodToDisk(kubeadmconstants.Etcd, manifestDir, spec); err != nil {
if err := prepareAndWriteEtcdStaticPod(manifestDir, patchesDir, cfg, endpoint, nodeName, initialCluster, isDryRun); err != nil {
return err
}
@ -292,3 +269,44 @@ func getEtcdCommand(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.A
command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.Etcd.Local.ExtraArgs)...)
return command
}
func prepareAndWriteEtcdStaticPod(manifestDir string, patchesDir string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, nodeName string, initialCluster []etcdutil.Member, isDryRun bool) error {
// gets etcd StaticPodSpec, actualized for the current ClusterConfiguration and the new list of etcd members
spec := GetEtcdPodSpec(cfg, endpoint, nodeName, initialCluster)
var usersAndGroups *users.UsersAndGroups
var err error
if features.Enabled(cfg.FeatureGates, features.RootlessControlPlane) {
if isDryRun {
fmt.Printf("[dryrun] Would create users and groups for %q to run as non-root\n", kubeadmconstants.Etcd)
fmt.Printf("[dryrun] Would update static pod manifest for %q to run run as non-root\n", kubeadmconstants.Etcd)
} else {
usersAndGroups, err = staticpodutil.GetUsersAndGroups()
if err != nil {
return errors.Wrap(err, "failed to create users and groups")
}
// usersAndGroups is nil on non-linux.
if usersAndGroups != nil {
if err := staticpodutil.RunComponentAsNonRoot(kubeadmconstants.Etcd, &spec, usersAndGroups, cfg); err != nil {
return errors.Wrapf(err, "failed to run component %q as non-root", kubeadmconstants.Etcd)
}
}
}
}
// if patchesDir is defined, patch the static Pod manifest
if patchesDir != "" {
patchedSpec, err := staticpodutil.PatchStaticPod(&spec, patchesDir, os.Stdout)
if err != nil {
return errors.Wrapf(err, "failed to patch static Pod manifest file for %q", kubeadmconstants.Etcd)
}
spec = *patchedSpec
}
// writes etcd StaticPod to disk
if err := staticpodutil.WriteStaticPodToDisk(kubeadmconstants.Etcd, manifestDir, spec); err != nil {
return err
}
return nil
}

43
cmd/kubeadm/app/util/staticpod/utils_linux.go
View File

@ -31,7 +31,8 @@ import (
"k8s.io/utils/pointer"
)
type pathOwerAndPermissionsUpdaterFunc func(path string, uid, gid int64, perms uint32) error
type pathOwnerAndPermissionsUpdaterFunc func(path string, uid, gid int64, perms uint32) error
type pathOwnerUpdaterFunc func(path string, uid, gid int64) error
// RunComponentAsNonRoot updates the pod manifest and the hostVolume permissions to run as non root.
func RunComponentAsNonRoot(componentName string, pod *v1.Pod, usersAndGroups *users.UsersAndGroups, cfg *kubeadmapi.ClusterConfiguration) error {
@ -61,12 +62,21 @@ func RunComponentAsNonRoot(componentName string, pod *v1.Pod, usersAndGroups *us
usersAndGroups.Groups.ID(kubeadmconstants.KubeControllerManagerUserName),
users.UpdatePathOwnerAndPermissions,
)
case kubeadmconstants.Etcd:
return runEtcdAsNonRoot(
pod,
usersAndGroups.Users.ID(kubeadmconstants.EtcdUserName),
usersAndGroups.Groups.ID(kubeadmconstants.EtcdUserName),
users.UpdatePathOwnerAndPermissions,
users.UpdatePathOwner,
cfg,
)
}
return errors.New(fmt.Sprintf("component name %q is not valid", componentName))
}
// runKubeAPIServerAsNonRoot updates the pod manifest and the hostVolume permissions to run kube-apiserver as non root.
func runKubeAPIServerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup, supplementalGroup *int64, updatePathOwnerAndPermissions pathOwerAndPermissionsUpdaterFunc, cfg *kubeadmapi.ClusterConfiguration) error {
func runKubeAPIServerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup, supplementalGroup *int64, updatePathOwnerAndPermissions pathOwnerAndPermissionsUpdaterFunc, cfg *kubeadmapi.ClusterConfiguration) error {
saPublicKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPublicKeyName)
if err := updatePathOwnerAndPermissions(saPublicKeyFile, *runAsUser, *runAsGroup, 0600); err != nil {
return err
@ -110,7 +120,7 @@ func runKubeAPIServerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup, supplementalG
}
// runKubeControllerManagerAsNonRoot updates the pod manifest and the hostVolume permissions to run kube-controller-manager as non root.
func runKubeControllerManagerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup, supplementalGroup *int64, updatePathOwnerAndPermissions pathOwerAndPermissionsUpdaterFunc, cfg *kubeadmapi.ClusterConfiguration) error {
func runKubeControllerManagerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup, supplementalGroup *int64, updatePathOwnerAndPermissions pathOwnerAndPermissionsUpdaterFunc, cfg *kubeadmapi.ClusterConfiguration) error {
kubeconfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ControllerManagerKubeConfigFileName)
if err := updatePathOwnerAndPermissions(kubeconfigFile, *runAsUser, *runAsGroup, 0600); err != nil {
return err
@ -140,7 +150,7 @@ func runKubeControllerManagerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup, suppl
}
// runKubeSchedulerAsNonRoot updates the pod manifest and the hostVolume permissions to run kube-scheduler as non root.
func runKubeSchedulerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup *int64, updatePathOwnerAndPermissions pathOwerAndPermissionsUpdaterFunc) error {
func runKubeSchedulerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup *int64, updatePathOwnerAndPermissions pathOwnerAndPermissionsUpdaterFunc) error {
kubeconfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.SchedulerKubeConfigFileName)
if err := updatePathOwnerAndPermissions(kubeconfigFile, *runAsUser, *runAsGroup, 0600); err != nil {
return err
@ -156,3 +166,28 @@ func runKubeSchedulerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup *int64, update
pod.Spec.SecurityContext.RunAsGroup = runAsGroup
return nil
}
// runEtcdAsNonRoot updates the pod manifest and the hostVolume permissions to run etcd as non root.
func runEtcdAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup *int64, updatePathOwnerAndPermissions pathOwnerAndPermissionsUpdaterFunc, updatePathOwner pathOwnerUpdaterFunc, cfg *kubeadmapi.ClusterConfiguration) error {
if err := updatePathOwner(cfg.Etcd.Local.DataDir, *runAsUser, *runAsGroup); err != nil {
return err
}
etcdServerKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerKeyName)
if err := updatePathOwnerAndPermissions(etcdServerKeyFile, *runAsUser, *runAsGroup, 0600); err != nil {
return err
}
etcdPeerKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerKeyName)
if err := updatePathOwnerAndPermissions(etcdPeerKeyFile, *runAsUser, *runAsGroup, 0600); err != nil {
return err
}
pod.Spec.Containers[0].SecurityContext = &v1.SecurityContext{
AllowPrivilegeEscalation: pointer.Bool(false),
// We drop all capabilities that are added by default.
Capabilities: &v1.Capabilities{
Drop: []v1.Capability{"ALL"},
},
}
pod.Spec.SecurityContext.RunAsUser = runAsUser
pod.Spec.SecurityContext.RunAsGroup = runAsGroup
return nil
}

31
cmd/kubeadm/app/util/staticpod/utils_linux_test.go
View File

@ -133,3 +133,34 @@ func TestRunKubeSchedulerAsNonRoot(t *testing.T) {
}
verifyFilePermissions(t, updatedFiles, wantUpdateFiles)
}
func TestRunEtcdAsNonRoot(t *testing.T) {
cfg := &kubeadm.ClusterConfiguration{
Etcd: kubeadm.Etcd{
Local: &kubeadm.LocalEtcd{
DataDir: "/var/lib/etcd/data",
},
},
}
pod := ComponentPod(v1.Container{Name: "etcd"}, nil, nil)
var runAsUser, runAsGroup int64 = 1000, 1001
updatedFiles := map[string]ownerAndPermissions{}
if err := runEtcdAsNonRoot(&pod, &runAsUser, &runAsGroup, func(path string, uid, gid int64, perms uint32) error {
updatedFiles[path] = ownerAndPermissions{uid: uid, gid: gid, permissions: perms}
return nil
},
func(path string, uid, gid int64) error {
updatedFiles[path] = ownerAndPermissions{uid: uid, gid: gid, permissions: 0700}
return nil
}, cfg); err != nil {
t.Fatal(err)
}
verifyPodSecurityContext(t, &pod, runAsUser, runAsGroup, nil)
verifyContainerSecurityContext(t, pod.Spec.Containers[0], nil, []v1.Capability{"ALL"}, pointer.Bool(false))
wantUpdateFiles := map[string]ownerAndPermissions{
cfg.Etcd.Local.DataDir: {uid: runAsUser, gid: runAsGroup, permissions: 0700},
filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerKeyName): {uid: runAsUser, gid: runAsGroup, permissions: 0600},
filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerKeyName): {uid: runAsUser, gid: runAsGroup, permissions: 0600},
}
verifyFilePermissions(t, updatedFiles, wantUpdateFiles)
}

23
cmd/kubeadm/app/util/users/users_linux.go
View File

@ -632,19 +632,26 @@ func writeFile(f *os.File, str string) error {
}
// UpdatePathOwnerAndPermissions updates the owner and permissions of the given path.
// If the path is a directory it updates its contents recursively.
func UpdatePathOwnerAndPermissions(dirPath string, uid, gid int64, perms uint32) error {
// If the path is a directory it is not recursively updated.
func UpdatePathOwnerAndPermissions(path string, uid, gid int64, perms uint32) error {
if err := os.Chown(path, int(uid), int(gid)); err != nil {
return errors.Wrapf(err, "failed to update owner of %q to uid: %d and gid: %d", path, uid, gid)
}
fm := os.FileMode(perms)
if err := os.Chmod(path, fm); err != nil {
return errors.Wrapf(err, "failed to update permissions of %q to %s", path, fm.String())
}
return nil
}
// UpdatePathOwner recursively updates the owners of a directory.
// It is equivalent to calling `chown -R uid:gid /path/to/dir`.
func UpdatePathOwner(dirPath string, uid, gid int64) error {
err := filepath.WalkDir(dirPath, func(path string, d os.DirEntry, err error) error {
if err := os.Chown(path, int(uid), int(gid)); err != nil {
return errors.Wrapf(err, "failed to update owner of %q to uid: %d and gid: %d", path, uid, gid)
}
fm := os.FileMode(perms)
if err := os.Chmod(path, fm); err != nil {
return errors.Wrapf(err, "failed to update permissions of %q to %s", path, fm.String())
}
return nil
})
return err
}

5
cmd/kubeadm/app/util/users/users_other.go
View File

@ -48,3 +48,8 @@ func RemoveUsersAndGroups() error {
func UpdatePathOwnerAndPermissions(path string, uid, gid int64, perms uint32) error {
return nil
}
// UpdatePathOwner is a NO-OP on non-Linux.
func UpdatePathOwner(dirPath string, uid, gid int64) error {
return nil
}