From b985e094b0f3f55163366bdb6b0143af7b63923d Mon Sep 17 00:00:00 2001 From: hwdef Date: Fri, 29 Oct 2021 09:31:36 +0800 Subject: [PATCH] kubeadm: fix absolute paths do not work properly in config files in windows --- cmd/kubeadm/app/apis/kubeadm/types.go | 3 + .../app/componentconfigs/fakeconfig_test.go | 4 ++ cmd/kubeadm/app/componentconfigs/kubelet.go | 58 +++++++++++++++++ .../app/componentconfigs/kubelet_test.go | 64 +++++++++++++++++++ cmd/kubeadm/app/componentconfigs/kubeproxy.go | 5 ++ cmd/kubeadm/app/phases/kubelet/config.go | 4 ++ 6 files changed, 138 insertions(+) diff --git a/cmd/kubeadm/app/apis/kubeadm/types.go b/cmd/kubeadm/app/apis/kubeadm/types.go index 954b2a38596..870775251af 100644 --- a/cmd/kubeadm/app/apis/kubeadm/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/types.go @@ -453,6 +453,9 @@ type ComponentConfig interface { // SetUserSupplied sets the state of the component config "user supplied" flag to, either true, or false. SetUserSupplied(userSupplied bool) + // Mutate allows applying pre-defined modifications to the config before it's marshaled. + Mutate() error + // Set can be used to set the internal configuration in the ComponentConfig Set(interface{}) diff --git a/cmd/kubeadm/app/componentconfigs/fakeconfig_test.go b/cmd/kubeadm/app/componentconfigs/fakeconfig_test.go index 95bd8eb3335..5dec200e8c8 100644 --- a/cmd/kubeadm/app/componentconfigs/fakeconfig_test.go +++ b/cmd/kubeadm/app/componentconfigs/fakeconfig_test.go @@ -110,6 +110,10 @@ func (cc *clusterConfig) Default(_ *kubeadmapi.ClusterConfiguration, _ *kubeadma cc.config.KubernetesVersion = "bar" } +func (cc *clusterConfig) Mutate() error { + return nil +} + // fakeKnown replaces temporarily during the execution of each test here known (in configset.go) var fakeKnown = []*handler{ &clusterConfigHandler, diff --git a/cmd/kubeadm/app/componentconfigs/kubelet.go b/cmd/kubeadm/app/componentconfigs/kubelet.go index 9a042a3f575..2acb5fd457f 100644 --- a/cmd/kubeadm/app/componentconfigs/kubelet.go +++ b/cmd/kubeadm/app/componentconfigs/kubelet.go @@ -17,8 +17,12 @@ limitations under the License. package componentconfigs import ( + "os" "path/filepath" + "runtime" + "strings" + "github.com/pkg/errors" "k8s.io/apimachinery/pkg/util/version" clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" @@ -212,6 +216,60 @@ func (kc *kubeletConfig) Default(cfg *kubeadmapi.ClusterConfiguration, _ *kubead } } +// Mutate modifies absolute path fields in the KubeletConfiguration to be Windows compatible absolute paths. +func (kc *kubeletConfig) Mutate() error { + // TODO: use build tags and move the Windows related logic to _windows.go files + // once the kubeadm code base is unit tested for Windows as part of CI - "GOOS=windows go test ...". + if runtime.GOOS != "windows" { + return nil + } + + // When "kubeadm join" downloads the KubeletConfiguration from the cluster on Windows + // nodes, it would contain absolute paths that may lack drive letters, since the config + // could have been generated on a Linux control-plane node. On Windows the + // Golang path.IsAbs() function returns false unless the path contains a drive letter. + // This trips client-go and the kubelet, creating problems on Windows nodes. + // Fixing it in client-go or the kubelet is a breaking change to existing Windows + // users that rely on relative paths: + // https://github.com/kubernetes/kubernetes/pull/77710#issuecomment-491989621 + // + // Thus, a workaround here is to adapt the KubeletConfiguration paths for Windows. + // Note this is currently bound to KubeletConfiguration v1beta1. + klog.V(2).Infoln("[componentconfig] Adapting the paths in the KubeletConfiguration for Windows...") + + // Get the drive from where the kubeadm binary was called. + exe, err := os.Executable() + if err != nil { + return errors.Wrap(err, "could not obtain information about the kubeadm executable") + } + drive := filepath.VolumeName(filepath.Dir(exe)) + klog.V(2).Infof("[componentconfig] Assuming Windows drive %q", drive) + + // Mutate the paths in the config. + mutatePathsOnWindows(&kc.config, drive) + return nil +} + +func mutatePathsOnWindows(cfg *kubeletconfig.KubeletConfiguration, drive string) { + mutateStringField := func(name string, field *string) { + // path.IsAbs() is not reliable here in the Windows runtime, so check if the + // path starts with "/" instead. This means the path originated from a Unix node and + // is an absolute path. + if !strings.HasPrefix(*field, "/") { + return + } + // Prepend the drive letter to the path and update the field. + *field = filepath.Join(drive, *field) + klog.V(2).Infof("[componentconfig] kubelet/Windows: adapted path for field %q to %q", name, *field) + } + + // Mutate the fields we care about. + klog.V(2).Infof("[componentconfig] kubelet/Windows: changing field \"resolverConfig\" to empty") + cfg.ResolverConfig = utilpointer.String("") + mutateStringField("staticPodPath", &cfg.StaticPodPath) + mutateStringField("authentication.x509.clientCAFile", &cfg.Authentication.X509.ClientCAFile) +} + // isServiceActive checks whether the given service exists and is running func isServiceActive(name string) (bool, error) { initSystem, err := initsystem.GetInitSystem() diff --git a/cmd/kubeadm/app/componentconfigs/kubelet_test.go b/cmd/kubeadm/app/componentconfigs/kubelet_test.go index c8b28523887..f2d28fbed95 100644 --- a/cmd/kubeadm/app/componentconfigs/kubelet_test.go +++ b/cmd/kubeadm/app/componentconfigs/kubelet_test.go @@ -288,3 +288,67 @@ func TestKubeletFromCluster(t *testing.T) { return kubeletHandler.FromCluster(client, testClusterCfg()) }) } + +func TestMutatePathsOnWindows(t *testing.T) { + const drive = "C:" + var fooResolverConfig string = "/foo/resolver" + + tests := []struct { + name string + cfg *kubeletconfig.KubeletConfiguration + expected *kubeletconfig.KubeletConfiguration + }{ + { + name: "valid: all fields are absolute paths", + cfg: &kubeletconfig.KubeletConfiguration{ + ResolverConfig: &fooResolverConfig, + StaticPodPath: "/foo/staticpods", + Authentication: kubeletconfig.KubeletAuthentication{ + X509: kubeletconfig.KubeletX509Authentication{ + ClientCAFile: "/foo/ca.crt", + }, + }, + }, + expected: &kubeletconfig.KubeletConfiguration{ + ResolverConfig: utilpointer.String(""), + StaticPodPath: filepath.Join(drive, "/foo/staticpods"), + Authentication: kubeletconfig.KubeletAuthentication{ + X509: kubeletconfig.KubeletX509Authentication{ + ClientCAFile: filepath.Join(drive, "/foo/ca.crt"), + }, + }, + }, + }, + { + name: "valid: some fields are not absolute paths", + cfg: &kubeletconfig.KubeletConfiguration{ + ResolverConfig: &fooResolverConfig, + StaticPodPath: "./foo/staticpods", // not an absolute Unix path + Authentication: kubeletconfig.KubeletAuthentication{ + X509: kubeletconfig.KubeletX509Authentication{ + ClientCAFile: "/foo/ca.crt", + }, + }, + }, + expected: &kubeletconfig.KubeletConfiguration{ + ResolverConfig: utilpointer.String(""), + StaticPodPath: "./foo/staticpods", + Authentication: kubeletconfig.KubeletAuthentication{ + X509: kubeletconfig.KubeletX509Authentication{ + ClientCAFile: filepath.Join(drive, "/foo/ca.crt"), + }, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + mutatePathsOnWindows(test.cfg, drive) + if !reflect.DeepEqual(test.cfg, test.expected) { + t.Errorf("Missmatch between expected and got:\nExpected:\n%+v\n---\nGot:\n%+v", + test.expected, test.cfg) + } + }) + } +} diff --git a/cmd/kubeadm/app/componentconfigs/kubeproxy.go b/cmd/kubeadm/app/componentconfigs/kubeproxy.go index aec7f35e769..bf11e34cccf 100644 --- a/cmd/kubeadm/app/componentconfigs/kubeproxy.go +++ b/cmd/kubeadm/app/componentconfigs/kubeproxy.go @@ -118,3 +118,8 @@ func (kp *kubeProxyConfig) Default(cfg *kubeadmapi.ClusterConfiguration, localAP warnDefaultComponentConfigValue(kind, "clientConnection.kubeconfig", kubeproxyKubeConfigFileName, kp.config.ClientConnection.Kubeconfig) } } + +// Mutate is NOP for the kube-proxy config +func (kp *kubeProxyConfig) Mutate() error { + return nil +} diff --git a/cmd/kubeadm/app/phases/kubelet/config.go b/cmd/kubeadm/app/phases/kubelet/config.go index 70b11f2742d..a8f852a6c9d 100644 --- a/cmd/kubeadm/app/phases/kubelet/config.go +++ b/cmd/kubeadm/app/phases/kubelet/config.go @@ -44,6 +44,10 @@ func WriteConfigToDisk(cfg *kubeadmapi.ClusterConfiguration, kubeletDir string) return errors.New("no kubelet component config found") } + if err := kubeletCfg.Mutate(); err != nil { + return err + } + kubeletBytes, err := kubeletCfg.Marshal() if err != nil { return err