Update etcd in kubeadm to run as non-root.

cmd/kubeadm/app/cmd/phases/join/controlplanejoin.go

@@ -145,7 +145,8 @@ func runEtcdPhase(c workflow.RunData) error {
 	// because it needs two members as majority to agree on the consensus. You will only see this behavior between the time
 	// etcdctl member add informs the cluster about the new member and the new member successfully establishing a connection to the
 	// existing one."
-	if err := etcdphase.CreateStackedEtcdStaticPodManifestFile(client, kubeadmconstants.GetStaticPodDirectory(), data.PatchesDir(), cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint); err != nil {
+	// TODO: add support for join dry-run: https://github.com/kubernetes/kubeadm/issues/2505
+	if err := etcdphase.CreateStackedEtcdStaticPodManifestFile(client, kubeadmconstants.GetStaticPodDirectory(), data.PatchesDir(), cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, false /* isDryRun */); err != nil {
 		return errors.Wrap(err, "error creating local etcd static pod manifest file")
 	}
 

cmd/kubeadm/app/phases/etcd/local.go

@@ -34,10 +34,12 @@ import (
 
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/kubernetes/cmd/kubeadm/app/features"
 	"k8s.io/kubernetes/cmd/kubeadm/app/images"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd"
 	staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
+	"k8s.io/kubernetes/cmd/kubeadm/app/util/users"
 )
 
 const (
@@ -54,20 +56,8 @@ func CreateLocalEtcdStaticPodManifestFile(manifestDir, patchesDir string, nodeNa
 	if cfg.Etcd.External != nil {
 		return errors.New("etcd static pod manifest cannot be generated for cluster using external etcd")
 	}
-	// gets etcd StaticPodSpec
-	spec := GetEtcdPodSpec(cfg, endpoint, nodeName, []etcdutil.Member{})
 
-	// if patchesDir is defined, patch the static Pod manifest
+	if err := prepareAndWriteEtcdStaticPod(manifestDir, patchesDir, cfg, endpoint, nodeName, []etcdutil.Member{}, isDryRun); err != nil {
-	if patchesDir != "" {
-		patchedSpec, err := staticpodutil.PatchStaticPod(&spec, patchesDir, os.Stdout)
-		if err != nil {
-			return errors.Wrapf(err, "failed to patch static Pod manifest file for %q", kubeadmconstants.Etcd)
-		}
-		spec = *patchedSpec
-	}
-
-	// writes etcd StaticPod to disk
-	if err := staticpodutil.WriteStaticPodToDisk(kubeadmconstants.Etcd, manifestDir, spec); err != nil {
 		return err
 	}
 
@@ -143,7 +133,7 @@ func RemoveStackedEtcdMemberFromCluster(client clientset.Interface, cfg *kubeadm
 // CreateStackedEtcdStaticPodManifestFile will write local etcd static pod manifest file
 // for an additional etcd member that is joining an existing local/stacked etcd cluster.
 // Other members of the etcd cluster will be notified of the joining node in beforehand as well.
-func CreateStackedEtcdStaticPodManifestFile(client clientset.Interface, manifestDir, patchesDir string, nodeName string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint) error {
+func CreateStackedEtcdStaticPodManifestFile(client clientset.Interface, manifestDir, patchesDir string, nodeName string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, isDryRun bool) error {
 	// creates an etcd client that connects to all the local/stacked etcd members
 	klog.V(1).Info("creating etcd client that connects to etcd pods")
 	etcdClient, err := etcdutil.NewFromCluster(client, cfg.CertificatesDir)
@@ -188,20 +178,7 @@ func CreateStackedEtcdStaticPodManifestFile(client clientset.Interface, manifest
 
 	fmt.Printf("[etcd] Creating static Pod manifest for %q\n", kubeadmconstants.Etcd)
 
-	// gets etcd StaticPodSpec, actualized for the current InitConfiguration and the new list of etcd members
+	if err := prepareAndWriteEtcdStaticPod(manifestDir, patchesDir, cfg, endpoint, nodeName, initialCluster, isDryRun); err != nil {
-	spec := GetEtcdPodSpec(cfg, endpoint, nodeName, initialCluster)
-
-	// if patchesDir is defined, patch the static Pod manifest
-	if patchesDir != "" {
-		patchedSpec, err := staticpodutil.PatchStaticPod(&spec, patchesDir, os.Stdout)
-		if err != nil {
-			return errors.Wrapf(err, "failed to patch static Pod manifest file for %q", kubeadmconstants.Etcd)
-		}
-		spec = *patchedSpec
-	}
-
-	// writes etcd StaticPod to disk
-	if err := staticpodutil.WriteStaticPodToDisk(kubeadmconstants.Etcd, manifestDir, spec); err != nil {
 		return err
 	}
 
@@ -292,3 +269,44 @@ func getEtcdCommand(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.A
 	command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.Etcd.Local.ExtraArgs)...)
 	return command
 }
+
+func prepareAndWriteEtcdStaticPod(manifestDir string, patchesDir string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, nodeName string, initialCluster []etcdutil.Member, isDryRun bool) error {
+	// gets etcd StaticPodSpec, actualized for the current ClusterConfiguration and the new list of etcd members
+	spec := GetEtcdPodSpec(cfg, endpoint, nodeName, initialCluster)
+
+	var usersAndGroups *users.UsersAndGroups
+	var err error
+	if features.Enabled(cfg.FeatureGates, features.RootlessControlPlane) {
+		if isDryRun {
+			fmt.Printf("[dryrun] Would create users and groups for %q to run as non-root\n", kubeadmconstants.Etcd)
+			fmt.Printf("[dryrun] Would update static pod manifest for %q to run run as non-root\n", kubeadmconstants.Etcd)
+		} else {
+			usersAndGroups, err = staticpodutil.GetUsersAndGroups()
+			if err != nil {
+				return errors.Wrap(err, "failed to create users and groups")
+			}
+			// usersAndGroups is nil on non-linux.
+			if usersAndGroups != nil {
+				if err := staticpodutil.RunComponentAsNonRoot(kubeadmconstants.Etcd, &spec, usersAndGroups, cfg); err != nil {
+					return errors.Wrapf(err, "failed to run component %q as non-root", kubeadmconstants.Etcd)
+				}
+			}
+		}
+	}
+
+	// if patchesDir is defined, patch the static Pod manifest
+	if patchesDir != "" {
+		patchedSpec, err := staticpodutil.PatchStaticPod(&spec, patchesDir, os.Stdout)
+		if err != nil {
+			return errors.Wrapf(err, "failed to patch static Pod manifest file for %q", kubeadmconstants.Etcd)
+		}
+		spec = *patchedSpec
+	}
+
+	// writes etcd StaticPod to disk
+	if err := staticpodutil.WriteStaticPodToDisk(kubeadmconstants.Etcd, manifestDir, spec); err != nil {
+		return err
+	}
+
+	return nil
+}

cmd/kubeadm/app/util/staticpod/utils_linux.go

@@ -31,7 +31,8 @@ import (
 	"k8s.io/utils/pointer"
 )
 
-type pathOwerAndPermissionsUpdaterFunc func(path string, uid, gid int64, perms uint32) error
+type pathOwnerAndPermissionsUpdaterFunc func(path string, uid, gid int64, perms uint32) error
+type pathOwnerUpdaterFunc func(path string, uid, gid int64) error
 
 // RunComponentAsNonRoot updates the pod manifest and the hostVolume permissions to run as non root.
 func RunComponentAsNonRoot(componentName string, pod *v1.Pod, usersAndGroups *users.UsersAndGroups, cfg *kubeadmapi.ClusterConfiguration) error {
@@ -61,12 +62,21 @@ func RunComponentAsNonRoot(componentName string, pod *v1.Pod, usersAndGroups *us
 			usersAndGroups.Groups.ID(kubeadmconstants.KubeControllerManagerUserName),
 			users.UpdatePathOwnerAndPermissions,
 		)
+	case kubeadmconstants.Etcd:
+		return runEtcdAsNonRoot(
+			pod,
+			usersAndGroups.Users.ID(kubeadmconstants.EtcdUserName),
+			usersAndGroups.Groups.ID(kubeadmconstants.EtcdUserName),
+			users.UpdatePathOwnerAndPermissions,
+			users.UpdatePathOwner,
+			cfg,
+		)
 	}
 	return errors.New(fmt.Sprintf("component name %q is not valid", componentName))
 }
 
 // runKubeAPIServerAsNonRoot updates the pod manifest and the hostVolume permissions to run kube-apiserver as non root.
-func runKubeAPIServerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup, supplementalGroup *int64, updatePathOwnerAndPermissions pathOwerAndPermissionsUpdaterFunc, cfg *kubeadmapi.ClusterConfiguration) error {
+func runKubeAPIServerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup, supplementalGroup *int64, updatePathOwnerAndPermissions pathOwnerAndPermissionsUpdaterFunc, cfg *kubeadmapi.ClusterConfiguration) error {
 	saPublicKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPublicKeyName)
 	if err := updatePathOwnerAndPermissions(saPublicKeyFile, *runAsUser, *runAsGroup, 0600); err != nil {
 		return err
@@ -110,7 +120,7 @@ func runKubeAPIServerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup, supplementalG
 }
 
 // runKubeControllerManagerAsNonRoot updates the pod manifest and the hostVolume permissions to run kube-controller-manager as non root.
-func runKubeControllerManagerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup, supplementalGroup *int64, updatePathOwnerAndPermissions pathOwerAndPermissionsUpdaterFunc, cfg *kubeadmapi.ClusterConfiguration) error {
+func runKubeControllerManagerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup, supplementalGroup *int64, updatePathOwnerAndPermissions pathOwnerAndPermissionsUpdaterFunc, cfg *kubeadmapi.ClusterConfiguration) error {
 	kubeconfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ControllerManagerKubeConfigFileName)
 	if err := updatePathOwnerAndPermissions(kubeconfigFile, *runAsUser, *runAsGroup, 0600); err != nil {
 		return err
@@ -140,7 +150,7 @@ func runKubeControllerManagerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup, suppl
 }
 
 // runKubeSchedulerAsNonRoot updates the pod manifest and the hostVolume permissions to run kube-scheduler as non root.
-func runKubeSchedulerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup *int64, updatePathOwnerAndPermissions pathOwerAndPermissionsUpdaterFunc) error {
+func runKubeSchedulerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup *int64, updatePathOwnerAndPermissions pathOwnerAndPermissionsUpdaterFunc) error {
 	kubeconfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.SchedulerKubeConfigFileName)
 	if err := updatePathOwnerAndPermissions(kubeconfigFile, *runAsUser, *runAsGroup, 0600); err != nil {
 		return err
@@ -156,3 +166,28 @@ func runKubeSchedulerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup *int64, update
 	pod.Spec.SecurityContext.RunAsGroup = runAsGroup
 	return nil
 }
+
+// runEtcdAsNonRoot updates the pod manifest and the hostVolume permissions to run etcd as non root.
+func runEtcdAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup *int64, updatePathOwnerAndPermissions pathOwnerAndPermissionsUpdaterFunc, updatePathOwner pathOwnerUpdaterFunc, cfg *kubeadmapi.ClusterConfiguration) error {
+	if err := updatePathOwner(cfg.Etcd.Local.DataDir, *runAsUser, *runAsGroup); err != nil {
+		return err
+	}
+	etcdServerKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerKeyName)
+	if err := updatePathOwnerAndPermissions(etcdServerKeyFile, *runAsUser, *runAsGroup, 0600); err != nil {
+		return err
+	}
+	etcdPeerKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerKeyName)
+	if err := updatePathOwnerAndPermissions(etcdPeerKeyFile, *runAsUser, *runAsGroup, 0600); err != nil {
+		return err
+	}
+	pod.Spec.Containers[0].SecurityContext = &v1.SecurityContext{
+		AllowPrivilegeEscalation: pointer.Bool(false),
+		// We drop all capabilities that are added by default.
+		Capabilities: &v1.Capabilities{
+			Drop: []v1.Capability{"ALL"},
+		},
+	}
+	pod.Spec.SecurityContext.RunAsUser = runAsUser
+	pod.Spec.SecurityContext.RunAsGroup = runAsGroup
+	return nil
+}

cmd/kubeadm/app/util/staticpod/utils_linux_test.go

@@ -133,3 +133,34 @@ func TestRunKubeSchedulerAsNonRoot(t *testing.T) {
 	}
 	verifyFilePermissions(t, updatedFiles, wantUpdateFiles)
 }
+
+func TestRunEtcdAsNonRoot(t *testing.T) {
+	cfg := &kubeadm.ClusterConfiguration{
+		Etcd: kubeadm.Etcd{
+			Local: &kubeadm.LocalEtcd{
+				DataDir: "/var/lib/etcd/data",
+			},
+		},
+	}
+	pod := ComponentPod(v1.Container{Name: "etcd"}, nil, nil)
+	var runAsUser, runAsGroup int64 = 1000, 1001
+	updatedFiles := map[string]ownerAndPermissions{}
+	if err := runEtcdAsNonRoot(&pod, &runAsUser, &runAsGroup, func(path string, uid, gid int64, perms uint32) error {
+		updatedFiles[path] = ownerAndPermissions{uid: uid, gid: gid, permissions: perms}
+		return nil
+	},
+		func(path string, uid, gid int64) error {
+			updatedFiles[path] = ownerAndPermissions{uid: uid, gid: gid, permissions: 0700}
+			return nil
+		}, cfg); err != nil {
+		t.Fatal(err)
+	}
+	verifyPodSecurityContext(t, &pod, runAsUser, runAsGroup, nil)
+	verifyContainerSecurityContext(t, pod.Spec.Containers[0], nil, []v1.Capability{"ALL"}, pointer.Bool(false))
+	wantUpdateFiles := map[string]ownerAndPermissions{
+		cfg.Etcd.Local.DataDir: {uid: runAsUser, gid: runAsGroup, permissions: 0700},
+		filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerKeyName): {uid: runAsUser, gid: runAsGroup, permissions: 0600},
+		filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerKeyName):   {uid: runAsUser, gid: runAsGroup, permissions: 0600},
+	}
+	verifyFilePermissions(t, updatedFiles, wantUpdateFiles)
+}

cmd/kubeadm/app/util/users/users_linux.go

@@ -632,19 +632,26 @@ func writeFile(f *os.File, str string) error {
 }
 
 // UpdatePathOwnerAndPermissions updates the owner and permissions of the given path.
-// If the path is a directory it updates its contents recursively.
+// If the path is a directory it is not recursively updated.
-func UpdatePathOwnerAndPermissions(dirPath string, uid, gid int64, perms uint32) error {
+func UpdatePathOwnerAndPermissions(path string, uid, gid int64, perms uint32) error {
+	if err := os.Chown(path, int(uid), int(gid)); err != nil {
+		return errors.Wrapf(err, "failed to update owner of %q to uid: %d and gid: %d", path, uid, gid)
+	}
+	fm := os.FileMode(perms)
+	if err := os.Chmod(path, fm); err != nil {
+		return errors.Wrapf(err, "failed to update permissions of %q to %s", path, fm.String())
+	}
+	return nil
+}
+
+// UpdatePathOwner recursively updates the owners of a directory.
+// It is equivalent to calling `chown -R uid:gid /path/to/dir`.
+func UpdatePathOwner(dirPath string, uid, gid int64) error {
 	err := filepath.WalkDir(dirPath, func(path string, d os.DirEntry, err error) error {
 		if err := os.Chown(path, int(uid), int(gid)); err != nil {
 			return errors.Wrapf(err, "failed to update owner of %q to uid: %d and gid: %d", path, uid, gid)
 		}
-
-		fm := os.FileMode(perms)
-		if err := os.Chmod(path, fm); err != nil {
-			return errors.Wrapf(err, "failed to update permissions of %q to %s", path, fm.String())
-		}
 		return nil
 	})
-
 	return err
 }

cmd/kubeadm/app/util/users/users_other.go

@@ -48,3 +48,8 @@ func RemoveUsersAndGroups() error {
 func UpdatePathOwnerAndPermissions(path string, uid, gid int64, perms uint32) error {
 	return nil
 }
+
+// UpdatePathOwner is a NO-OP on non-Linux.
+func UpdatePathOwner(dirPath string, uid, gid int64) error {
+	return nil
+}