diff --git a/cmd/kubeadm/app/cmd/phases/join/controlplanejoin.go b/cmd/kubeadm/app/cmd/phases/join/controlplanejoin.go index 9780dca7021..4d3c476c0cd 100644 --- a/cmd/kubeadm/app/cmd/phases/join/controlplanejoin.go +++ b/cmd/kubeadm/app/cmd/phases/join/controlplanejoin.go @@ -145,7 +145,8 @@ func runEtcdPhase(c workflow.RunData) error { // because it needs two members as majority to agree on the consensus. You will only see this behavior between the time // etcdctl member add informs the cluster about the new member and the new member successfully establishing a connection to the // existing one." - if err := etcdphase.CreateStackedEtcdStaticPodManifestFile(client, kubeadmconstants.GetStaticPodDirectory(), data.PatchesDir(), cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint); err != nil { + // TODO: add support for join dry-run: https://github.com/kubernetes/kubeadm/issues/2505 + if err := etcdphase.CreateStackedEtcdStaticPodManifestFile(client, kubeadmconstants.GetStaticPodDirectory(), data.PatchesDir(), cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, false /* isDryRun */); err != nil { return errors.Wrap(err, "error creating local etcd static pod manifest file") } diff --git a/cmd/kubeadm/app/phases/etcd/local.go b/cmd/kubeadm/app/phases/etcd/local.go index dcbcfdb2ec5..70bc2f5466c 100644 --- a/cmd/kubeadm/app/phases/etcd/local.go +++ b/cmd/kubeadm/app/phases/etcd/local.go @@ -34,10 +34,12 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/features" "k8s.io/kubernetes/cmd/kubeadm/app/images" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd" staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" + "k8s.io/kubernetes/cmd/kubeadm/app/util/users" ) const ( @@ -54,20 +56,8 @@ func CreateLocalEtcdStaticPodManifestFile(manifestDir, patchesDir string, nodeNa if cfg.Etcd.External != nil { return errors.New("etcd static pod manifest cannot be generated for cluster using external etcd") } - // gets etcd StaticPodSpec - spec := GetEtcdPodSpec(cfg, endpoint, nodeName, []etcdutil.Member{}) - // if patchesDir is defined, patch the static Pod manifest - if patchesDir != "" { - patchedSpec, err := staticpodutil.PatchStaticPod(&spec, patchesDir, os.Stdout) - if err != nil { - return errors.Wrapf(err, "failed to patch static Pod manifest file for %q", kubeadmconstants.Etcd) - } - spec = *patchedSpec - } - - // writes etcd StaticPod to disk - if err := staticpodutil.WriteStaticPodToDisk(kubeadmconstants.Etcd, manifestDir, spec); err != nil { + if err := prepareAndWriteEtcdStaticPod(manifestDir, patchesDir, cfg, endpoint, nodeName, []etcdutil.Member{}, isDryRun); err != nil { return err } @@ -143,7 +133,7 @@ func RemoveStackedEtcdMemberFromCluster(client clientset.Interface, cfg *kubeadm // CreateStackedEtcdStaticPodManifestFile will write local etcd static pod manifest file // for an additional etcd member that is joining an existing local/stacked etcd cluster. // Other members of the etcd cluster will be notified of the joining node in beforehand as well. -func CreateStackedEtcdStaticPodManifestFile(client clientset.Interface, manifestDir, patchesDir string, nodeName string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint) error { +func CreateStackedEtcdStaticPodManifestFile(client clientset.Interface, manifestDir, patchesDir string, nodeName string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, isDryRun bool) error { // creates an etcd client that connects to all the local/stacked etcd members klog.V(1).Info("creating etcd client that connects to etcd pods") etcdClient, err := etcdutil.NewFromCluster(client, cfg.CertificatesDir) @@ -188,20 +178,7 @@ func CreateStackedEtcdStaticPodManifestFile(client clientset.Interface, manifest fmt.Printf("[etcd] Creating static Pod manifest for %q\n", kubeadmconstants.Etcd) - // gets etcd StaticPodSpec, actualized for the current InitConfiguration and the new list of etcd members - spec := GetEtcdPodSpec(cfg, endpoint, nodeName, initialCluster) - - // if patchesDir is defined, patch the static Pod manifest - if patchesDir != "" { - patchedSpec, err := staticpodutil.PatchStaticPod(&spec, patchesDir, os.Stdout) - if err != nil { - return errors.Wrapf(err, "failed to patch static Pod manifest file for %q", kubeadmconstants.Etcd) - } - spec = *patchedSpec - } - - // writes etcd StaticPod to disk - if err := staticpodutil.WriteStaticPodToDisk(kubeadmconstants.Etcd, manifestDir, spec); err != nil { + if err := prepareAndWriteEtcdStaticPod(manifestDir, patchesDir, cfg, endpoint, nodeName, initialCluster, isDryRun); err != nil { return err } @@ -292,3 +269,44 @@ func getEtcdCommand(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.A command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.Etcd.Local.ExtraArgs)...) return command } + +func prepareAndWriteEtcdStaticPod(manifestDir string, patchesDir string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, nodeName string, initialCluster []etcdutil.Member, isDryRun bool) error { + // gets etcd StaticPodSpec, actualized for the current ClusterConfiguration and the new list of etcd members + spec := GetEtcdPodSpec(cfg, endpoint, nodeName, initialCluster) + + var usersAndGroups *users.UsersAndGroups + var err error + if features.Enabled(cfg.FeatureGates, features.RootlessControlPlane) { + if isDryRun { + fmt.Printf("[dryrun] Would create users and groups for %q to run as non-root\n", kubeadmconstants.Etcd) + fmt.Printf("[dryrun] Would update static pod manifest for %q to run run as non-root\n", kubeadmconstants.Etcd) + } else { + usersAndGroups, err = staticpodutil.GetUsersAndGroups() + if err != nil { + return errors.Wrap(err, "failed to create users and groups") + } + // usersAndGroups is nil on non-linux. + if usersAndGroups != nil { + if err := staticpodutil.RunComponentAsNonRoot(kubeadmconstants.Etcd, &spec, usersAndGroups, cfg); err != nil { + return errors.Wrapf(err, "failed to run component %q as non-root", kubeadmconstants.Etcd) + } + } + } + } + + // if patchesDir is defined, patch the static Pod manifest + if patchesDir != "" { + patchedSpec, err := staticpodutil.PatchStaticPod(&spec, patchesDir, os.Stdout) + if err != nil { + return errors.Wrapf(err, "failed to patch static Pod manifest file for %q", kubeadmconstants.Etcd) + } + spec = *patchedSpec + } + + // writes etcd StaticPod to disk + if err := staticpodutil.WriteStaticPodToDisk(kubeadmconstants.Etcd, manifestDir, spec); err != nil { + return err + } + + return nil +} diff --git a/cmd/kubeadm/app/util/staticpod/utils_linux.go b/cmd/kubeadm/app/util/staticpod/utils_linux.go index 6b2425c3199..39e8039276c 100644 --- a/cmd/kubeadm/app/util/staticpod/utils_linux.go +++ b/cmd/kubeadm/app/util/staticpod/utils_linux.go @@ -31,7 +31,8 @@ import ( "k8s.io/utils/pointer" ) -type pathOwerAndPermissionsUpdaterFunc func(path string, uid, gid int64, perms uint32) error +type pathOwnerAndPermissionsUpdaterFunc func(path string, uid, gid int64, perms uint32) error +type pathOwnerUpdaterFunc func(path string, uid, gid int64) error // RunComponentAsNonRoot updates the pod manifest and the hostVolume permissions to run as non root. func RunComponentAsNonRoot(componentName string, pod *v1.Pod, usersAndGroups *users.UsersAndGroups, cfg *kubeadmapi.ClusterConfiguration) error { @@ -61,12 +62,21 @@ func RunComponentAsNonRoot(componentName string, pod *v1.Pod, usersAndGroups *us usersAndGroups.Groups.ID(kubeadmconstants.KubeControllerManagerUserName), users.UpdatePathOwnerAndPermissions, ) + case kubeadmconstants.Etcd: + return runEtcdAsNonRoot( + pod, + usersAndGroups.Users.ID(kubeadmconstants.EtcdUserName), + usersAndGroups.Groups.ID(kubeadmconstants.EtcdUserName), + users.UpdatePathOwnerAndPermissions, + users.UpdatePathOwner, + cfg, + ) } return errors.New(fmt.Sprintf("component name %q is not valid", componentName)) } // runKubeAPIServerAsNonRoot updates the pod manifest and the hostVolume permissions to run kube-apiserver as non root. -func runKubeAPIServerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup, supplementalGroup *int64, updatePathOwnerAndPermissions pathOwerAndPermissionsUpdaterFunc, cfg *kubeadmapi.ClusterConfiguration) error { +func runKubeAPIServerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup, supplementalGroup *int64, updatePathOwnerAndPermissions pathOwnerAndPermissionsUpdaterFunc, cfg *kubeadmapi.ClusterConfiguration) error { saPublicKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPublicKeyName) if err := updatePathOwnerAndPermissions(saPublicKeyFile, *runAsUser, *runAsGroup, 0600); err != nil { return err @@ -110,7 +120,7 @@ func runKubeAPIServerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup, supplementalG } // runKubeControllerManagerAsNonRoot updates the pod manifest and the hostVolume permissions to run kube-controller-manager as non root. -func runKubeControllerManagerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup, supplementalGroup *int64, updatePathOwnerAndPermissions pathOwerAndPermissionsUpdaterFunc, cfg *kubeadmapi.ClusterConfiguration) error { +func runKubeControllerManagerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup, supplementalGroup *int64, updatePathOwnerAndPermissions pathOwnerAndPermissionsUpdaterFunc, cfg *kubeadmapi.ClusterConfiguration) error { kubeconfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ControllerManagerKubeConfigFileName) if err := updatePathOwnerAndPermissions(kubeconfigFile, *runAsUser, *runAsGroup, 0600); err != nil { return err @@ -140,7 +150,7 @@ func runKubeControllerManagerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup, suppl } // runKubeSchedulerAsNonRoot updates the pod manifest and the hostVolume permissions to run kube-scheduler as non root. -func runKubeSchedulerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup *int64, updatePathOwnerAndPermissions pathOwerAndPermissionsUpdaterFunc) error { +func runKubeSchedulerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup *int64, updatePathOwnerAndPermissions pathOwnerAndPermissionsUpdaterFunc) error { kubeconfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.SchedulerKubeConfigFileName) if err := updatePathOwnerAndPermissions(kubeconfigFile, *runAsUser, *runAsGroup, 0600); err != nil { return err @@ -156,3 +166,28 @@ func runKubeSchedulerAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup *int64, update pod.Spec.SecurityContext.RunAsGroup = runAsGroup return nil } + +// runEtcdAsNonRoot updates the pod manifest and the hostVolume permissions to run etcd as non root. +func runEtcdAsNonRoot(pod *v1.Pod, runAsUser, runAsGroup *int64, updatePathOwnerAndPermissions pathOwnerAndPermissionsUpdaterFunc, updatePathOwner pathOwnerUpdaterFunc, cfg *kubeadmapi.ClusterConfiguration) error { + if err := updatePathOwner(cfg.Etcd.Local.DataDir, *runAsUser, *runAsGroup); err != nil { + return err + } + etcdServerKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerKeyName) + if err := updatePathOwnerAndPermissions(etcdServerKeyFile, *runAsUser, *runAsGroup, 0600); err != nil { + return err + } + etcdPeerKeyFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerKeyName) + if err := updatePathOwnerAndPermissions(etcdPeerKeyFile, *runAsUser, *runAsGroup, 0600); err != nil { + return err + } + pod.Spec.Containers[0].SecurityContext = &v1.SecurityContext{ + AllowPrivilegeEscalation: pointer.Bool(false), + // We drop all capabilities that are added by default. + Capabilities: &v1.Capabilities{ + Drop: []v1.Capability{"ALL"}, + }, + } + pod.Spec.SecurityContext.RunAsUser = runAsUser + pod.Spec.SecurityContext.RunAsGroup = runAsGroup + return nil +} diff --git a/cmd/kubeadm/app/util/staticpod/utils_linux_test.go b/cmd/kubeadm/app/util/staticpod/utils_linux_test.go index f28112608b0..6458af20473 100644 --- a/cmd/kubeadm/app/util/staticpod/utils_linux_test.go +++ b/cmd/kubeadm/app/util/staticpod/utils_linux_test.go @@ -133,3 +133,34 @@ func TestRunKubeSchedulerAsNonRoot(t *testing.T) { } verifyFilePermissions(t, updatedFiles, wantUpdateFiles) } + +func TestRunEtcdAsNonRoot(t *testing.T) { + cfg := &kubeadm.ClusterConfiguration{ + Etcd: kubeadm.Etcd{ + Local: &kubeadm.LocalEtcd{ + DataDir: "/var/lib/etcd/data", + }, + }, + } + pod := ComponentPod(v1.Container{Name: "etcd"}, nil, nil) + var runAsUser, runAsGroup int64 = 1000, 1001 + updatedFiles := map[string]ownerAndPermissions{} + if err := runEtcdAsNonRoot(&pod, &runAsUser, &runAsGroup, func(path string, uid, gid int64, perms uint32) error { + updatedFiles[path] = ownerAndPermissions{uid: uid, gid: gid, permissions: perms} + return nil + }, + func(path string, uid, gid int64) error { + updatedFiles[path] = ownerAndPermissions{uid: uid, gid: gid, permissions: 0700} + return nil + }, cfg); err != nil { + t.Fatal(err) + } + verifyPodSecurityContext(t, &pod, runAsUser, runAsGroup, nil) + verifyContainerSecurityContext(t, pod.Spec.Containers[0], nil, []v1.Capability{"ALL"}, pointer.Bool(false)) + wantUpdateFiles := map[string]ownerAndPermissions{ + cfg.Etcd.Local.DataDir: {uid: runAsUser, gid: runAsGroup, permissions: 0700}, + filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerKeyName): {uid: runAsUser, gid: runAsGroup, permissions: 0600}, + filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerKeyName): {uid: runAsUser, gid: runAsGroup, permissions: 0600}, + } + verifyFilePermissions(t, updatedFiles, wantUpdateFiles) +} diff --git a/cmd/kubeadm/app/util/users/users_linux.go b/cmd/kubeadm/app/util/users/users_linux.go index 9684849395b..aafb25613e1 100644 --- a/cmd/kubeadm/app/util/users/users_linux.go +++ b/cmd/kubeadm/app/util/users/users_linux.go @@ -632,19 +632,26 @@ func writeFile(f *os.File, str string) error { } // UpdatePathOwnerAndPermissions updates the owner and permissions of the given path. -// If the path is a directory it updates its contents recursively. -func UpdatePathOwnerAndPermissions(dirPath string, uid, gid int64, perms uint32) error { +// If the path is a directory it is not recursively updated. +func UpdatePathOwnerAndPermissions(path string, uid, gid int64, perms uint32) error { + if err := os.Chown(path, int(uid), int(gid)); err != nil { + return errors.Wrapf(err, "failed to update owner of %q to uid: %d and gid: %d", path, uid, gid) + } + fm := os.FileMode(perms) + if err := os.Chmod(path, fm); err != nil { + return errors.Wrapf(err, "failed to update permissions of %q to %s", path, fm.String()) + } + return nil +} + +// UpdatePathOwner recursively updates the owners of a directory. +// It is equivalent to calling `chown -R uid:gid /path/to/dir`. +func UpdatePathOwner(dirPath string, uid, gid int64) error { err := filepath.WalkDir(dirPath, func(path string, d os.DirEntry, err error) error { if err := os.Chown(path, int(uid), int(gid)); err != nil { return errors.Wrapf(err, "failed to update owner of %q to uid: %d and gid: %d", path, uid, gid) } - - fm := os.FileMode(perms) - if err := os.Chmod(path, fm); err != nil { - return errors.Wrapf(err, "failed to update permissions of %q to %s", path, fm.String()) - } return nil }) - return err } diff --git a/cmd/kubeadm/app/util/users/users_other.go b/cmd/kubeadm/app/util/users/users_other.go index 5614230feb4..388ef1efcd8 100644 --- a/cmd/kubeadm/app/util/users/users_other.go +++ b/cmd/kubeadm/app/util/users/users_other.go @@ -48,3 +48,8 @@ func RemoveUsersAndGroups() error { func UpdatePathOwnerAndPermissions(path string, uid, gid int64, perms uint32) error { return nil } + +// UpdatePathOwner is a NO-OP on non-Linux. +func UpdatePathOwner(dirPath string, uid, gid int64) error { + return nil +}