From 7d6181a290ec926371e70f413a4e63cc08810f56 Mon Sep 17 00:00:00 2001 From: Luther Monson Date: Tue, 21 Jul 2020 13:35:36 -0700 Subject: [PATCH] add win_ params for prefix path, env, args and binds Problem: When building a hybrid cluster with windows nodes there is only a single set of overrides you can use per service. This limits configuring the node as service args and prefix_path sometimes need to be specific for the different OS. Solution: Add support for `win_` prefixed parameters for cluster level `path_prefix` and service level `extra_args`, `extra_env` and `extra_binds`. Params will work as before, passing in the non `win_` prefixed params, IF you set the `win_` prefixed params it willy only use those meaning you will need to duplicate the params in both config sections of your rke cluster yaml. --- cluster/cluster.go | 98 ++++++++-- cluster/hosts.go | 2 +- cluster/plan.go | 430 +++++++++++++++---------------------------- cluster/reconcile.go | 18 +- hosts/hosts.go | 85 ++++++++- hosts/tunnel.go | 8 +- 6 files changed, 328 insertions(+), 313 deletions(-) diff --git a/cluster/cluster.go b/cluster/cluster.go index ae6232dd..441d9339 100644 --- a/cluster/cluster.go +++ b/cluster/cluster.go @@ -124,7 +124,7 @@ func (c *Cluster) DeployControlPlane(ctx context.Context, svcOptionData map[stri if err != nil { return "", err } - etcdNodePlanMap[etcdHost.Address] = BuildRKEConfigNodePlan(ctx, c, etcdHost, etcdHost.DockerInfo, svcOptions) + etcdNodePlanMap[etcdHost.Address] = BuildRKEConfigNodePlan(ctx, c, etcdHost, svcOptions) } if len(c.Services.Etcd.ExternalURLs) > 0 { @@ -143,7 +143,7 @@ func (c *Cluster) DeployControlPlane(ctx context.Context, svcOptionData map[stri if err != nil { return "", err } - cpNodePlanMap[cpHost.Address] = BuildRKEConfigNodePlan(ctx, c, cpHost, cpHost.DockerInfo, svcOptions) + cpNodePlanMap[cpHost.Address] = BuildRKEConfigNodePlan(ctx, c, cpHost, svcOptions) } if !reconcileCluster { @@ -250,7 +250,7 @@ func (c *Cluster) DeployWorkerPlane(ctx context.Context, svcOptionData map[strin if err != nil { return "", err } - workerNodePlanMap[host.Address] = BuildRKEConfigNodePlan(ctx, c, host, host.DockerInfo, svcOptions) + workerNodePlanMap[host.Address] = BuildRKEConfigNodePlan(ctx, c, host, svcOptions) if host.IsControl { continue } @@ -1089,15 +1089,6 @@ func RestartClusterPods(ctx context.Context, kubeCluster *Cluster) error { return nil } -func (c *Cluster) GetHostInfoMap() map[string]types.Info { - hostsInfoMap := make(map[string]types.Info) - allHosts := hosts.GetUniqueHostList(c.EtcdHosts, c.ControlPlaneHosts, c.WorkerHosts) - for _, host := range allHosts { - hostsInfoMap[host.Address] = host.DockerInfo - } - return hostsInfoMap -} - func IsLegacyKubeAPI(ctx context.Context, kubeCluster *Cluster) (bool, error) { log.Infof(ctx, "[controlplane] Check if rotating a legacy cluster") for _, host := range kubeCluster.ControlPlaneHosts { @@ -1114,3 +1105,86 @@ func IsLegacyKubeAPI(ctx context.Context, kubeCluster *Cluster) (bool, error) { } return false, nil } + +func (c *Cluster) GetHostInfoMap() map[string]types.Info { + hostsInfoMap := make(map[string]types.Info) + allHosts := hosts.GetUniqueHostList(c.EtcdHosts, c.ControlPlaneHosts, c.WorkerHosts) + for _, host := range allHosts { + hostsInfoMap[host.Address] = host.DockerInfo + } + return hostsInfoMap +} + +func (c *Cluster) getPrefixPath(os string) string { + switch { + case os == "windows" && c.WindowsPrefixPath != "": + return c.WindowsPrefixPath + default: + return c.PrefixPath + } +} + +func (c *Cluster) getSidecarEntryPoint(os string) []string { + switch os { + case "windows": + return []string{"pwsh", "-NoLogo", "-NonInteractive", "-File", "c:/usr/bin/sidecar.ps1"} + default: + return []string{"/bin/bash"} + } +} + +func (c *Cluster) getNginxEntryPoint(os string) []string { + switch os { + case "windows": + return []string{"pwsh", "-NoLogo", "-NonInteractive", "-File", "c:/usr/bin/nginx-proxy.ps1"} + default: + return []string{"nginx-proxy"} + } +} + +func (c *Cluster) getRKEToolsEntryPoint(os, cmd string) []string { + var entrypoint []string + switch os { + case "windows": + entrypoint = c.getRKEToolsWindowsEntryPoint() + default: + entrypoint = c.getRKEToolsLinuxEntryPoint() + } + + return append(entrypoint, cmd) +} + +func (c *Cluster) getRKEToolsWindowsEntryPoint() []string { + return []string{"pwsh", "-NoLogo", "-NonInteractive", "-File", "c:/usr/bin/entrypoint.ps1"} +} + +func (c *Cluster) getRKEToolsLinuxEntryPoint() []string { + v := strings.Split(c.SystemImages.KubernetesServicesSidecar, ":") + last := v[len(v)-1] + + sv, err := util.StrToSemVer(last) + if err != nil { + return []string{DefaultToolsEntrypoint} + } + svdefault, err := util.StrToSemVer(DefaultToolsEntrypointVersion) + if err != nil { + return []string{DefaultToolsEntrypoint} + } + + if sv.LessThan(*svdefault) { + return []string{LegacyToolsEntrypoint} + } + return []string{DefaultToolsEntrypoint} +} + +func (c *Cluster) getWindowsEnv(host *hosts.Host) []string { + return []string{ + fmt.Sprintf("%s=%s", ClusterCIDREnv, c.ClusterCIDR), + fmt.Sprintf("%s=%s", ClusterDomainEnv, c.ClusterDomain), + fmt.Sprintf("%s=%s", ClusterDNSServerEnv, c.ClusterDNSServer), + fmt.Sprintf("%s=%s", ClusterServiceCIDREnv, c.Services.KubeController.ServiceClusterIPRange), + fmt.Sprintf("%s=%s", NodeAddressEnv, host.Address), + fmt.Sprintf("%s=%s", NodeInternalAddressEnv, host.InternalAddress), + fmt.Sprintf("%s=%s", CloudProviderNameEnv, c.CloudProvider.Name), + } +} diff --git a/cluster/hosts.go b/cluster/hosts.go index a111492e..5867b36b 100644 --- a/cluster/hosts.go +++ b/cluster/hosts.go @@ -39,7 +39,7 @@ func (c *Cluster) TunnelHosts(ctx context.Context, flags ExternalFlags) error { for _, uniqueHost := range uniqueHosts { runHost := uniqueHost errgrp.Go(func() error { - if err := runHost.TunnelUp(ctx, c.DockerDialerFactory, c.PrefixPath, c.Version); err != nil { + if err := runHost.TunnelUp(ctx, c.DockerDialerFactory, c.getPrefixPath(runHost.OS()), c.Version); err != nil { // Unsupported Docker version is NOT a connectivity problem that we can recover! So we bail out on it if strings.Contains(err.Error(), "Unsupported Docker version found") { return err diff --git a/cluster/plan.go b/cluster/plan.go index a5ce3979..114dd87e 100644 --- a/cluster/plan.go +++ b/cluster/plan.go @@ -85,39 +85,40 @@ func GeneratePlan(ctx context.Context, rkeConfig *v3.RancherKubernetesEngineConf if err != nil { return clusterPlan, err } - clusterPlan.Nodes = append(clusterPlan.Nodes, BuildRKEConfigNodePlan(ctx, myCluster, host, hostsInfoMap[host.Address], svcOptions)) + clusterPlan.Nodes = append(clusterPlan.Nodes, BuildRKEConfigNodePlan(ctx, myCluster, host, svcOptions)) } return clusterPlan, nil } -func BuildRKEConfigNodePlan(ctx context.Context, myCluster *Cluster, host *hosts.Host, hostDockerInfo types.Info, svcOptions v3.KubernetesServicesOptions) v3.RKEConfigNodePlan { - prefixPath := hosts.GetPrefixPath(hostDockerInfo.OperatingSystem, myCluster.PrefixPath) - processes := map[string]v3.Process{} - portChecks := []v3.PortCheck{} +func BuildRKEConfigNodePlan(ctx context.Context, myCluster *Cluster, host *hosts.Host, svcOptions v3.KubernetesServicesOptions) v3.RKEConfigNodePlan { + var portChecks []v3.PortCheck + processes := make(map[string]v3.Process) + host.SetPrefixPath(myCluster.getPrefixPath(host.OS())) + // Everybody gets a sidecar and a kubelet.. - processes[services.SidekickContainerName] = myCluster.BuildSidecarProcess(host, prefixPath) - processes[services.KubeletContainerName] = myCluster.BuildKubeletProcess(host, prefixPath, svcOptions) - processes[services.KubeproxyContainerName] = myCluster.BuildKubeProxyProcess(host, prefixPath, svcOptions) + processes[services.SidekickContainerName] = myCluster.BuildSidecarProcess(host) + processes[services.KubeletContainerName] = myCluster.BuildKubeletProcess(host, svcOptions) + processes[services.KubeproxyContainerName] = myCluster.BuildKubeProxyProcess(host, svcOptions) portChecks = append(portChecks, BuildPortChecksFromPortList(host, WorkerPortList, ProtocolTCP)...) // Do we need an nginxProxy for this one ? if !host.IsControl { - processes[services.NginxProxyContainerName] = myCluster.BuildProxyProcess(host, prefixPath) + processes[services.NginxProxyContainerName] = myCluster.BuildProxyProcess(host) } if host.IsControl { - processes[services.KubeAPIContainerName] = myCluster.BuildKubeAPIProcess(host, prefixPath, svcOptions) - processes[services.KubeControllerContainerName] = myCluster.BuildKubeControllerProcess(host, prefixPath, svcOptions) - processes[services.SchedulerContainerName] = myCluster.BuildSchedulerProcess(host, prefixPath, svcOptions) + processes[services.KubeAPIContainerName] = myCluster.BuildKubeAPIProcess(host, svcOptions) + processes[services.KubeControllerContainerName] = myCluster.BuildKubeControllerProcess(host, svcOptions) + processes[services.SchedulerContainerName] = myCluster.BuildSchedulerProcess(host, svcOptions) portChecks = append(portChecks, BuildPortChecksFromPortList(host, ControlPlanePortList, ProtocolTCP)...) } if host.IsEtcd { - processes[services.EtcdContainerName] = myCluster.BuildEtcdProcess(host, myCluster.EtcdReadyHosts, prefixPath, svcOptions) + processes[services.EtcdContainerName] = myCluster.BuildEtcdProcess(host, myCluster.EtcdReadyHosts, svcOptions) portChecks = append(portChecks, BuildPortChecksFromPortList(host, EtcdPortList, ProtocolTCP)...) } files := []v3.File{ - v3.File{ + { Name: cloudConfigFileName, Contents: b64.StdEncoding.EncodeToString([]byte(myCluster.CloudConfigFile)), }, @@ -130,7 +131,7 @@ func BuildRKEConfigNodePlan(ctx context.Context, myCluster *Cluster, host *hosts } return v3.RKEConfigNodePlan{ Address: host.Address, - Processes: osLimitationFilter(hostDockerInfo.OSType, processes), + Processes: host.ProcessFilter(processes), PortChecks: portChecks, Files: files, Annotations: map[string]string{ @@ -141,38 +142,7 @@ func BuildRKEConfigNodePlan(ctx context.Context, myCluster *Cluster, host *hosts } } -func osLimitationFilter(osType string, processes map[string]v3.Process) map[string]v3.Process { - if osType != "windows" { - return processes - } - - // windows limitations - for name, process := range processes { - // doesn't support host network on windows - if process.NetworkMode == "host" { - process.NetworkMode = "" - } - - // doesn't support PID on windows - if process.PidMode != "" { - process.PidMode = "" - } - - // doesn't support privileged mode on windows - if process.Privileged { - process.Privileged = false - } - - // doesn't execute health check - process.HealthCheck = v3.HealthCheck{} - - processes[name] = process - } - - return processes -} - -func (c *Cluster) BuildKubeAPIProcess(host *hosts.Host, prefixPath string, serviceOptions v3.KubernetesServicesOptions) v3.Process { +func (c *Cluster) BuildKubeAPIProcess(host *hosts.Host, serviceOptions v3.KubernetesServicesOptions) v3.Process { // check if external etcd is used etcdConnectionString := services.GetEtcdConnString(c.EtcdHosts, host.InternalAddress) etcdPathPrefix := EtcdPathPrefix @@ -188,11 +158,7 @@ func (c *Cluster) BuildKubeAPIProcess(host *hosts.Host, prefixPath string, servi etcdCAClientCert = pki.GetCertPath(pki.EtcdClientCACertName) } - Command := []string{ - c.getRKEToolsEntryPoint(), - "kube-apiserver", - } - + Command := c.getRKEToolsEntryPoint(host.OS(), "kube-apiserver") CommandArgs := map[string]string{ "client-ca-file": pki.GetCertPath(pki.CACertName), "cloud-provider": c.CloudProvider.Name, @@ -293,10 +259,10 @@ func (c *Cluster) BuildKubeAPIProcess(host *hosts.Host, prefixPath string, servi services.SidekickContainerName, } Binds := []string{ - fmt.Sprintf("%s:/etc/kubernetes:z", path.Join(prefixPath, "/etc/kubernetes")), + fmt.Sprintf("%s:/etc/kubernetes:z", path.Join(host.PrefixPath, "/etc/kubernetes")), } if c.Services.KubeAPI.AuditLog != nil && c.Services.KubeAPI.AuditLog.Enabled { - Binds = append(Binds, fmt.Sprintf("%s:/var/log/kube-audit:z", path.Join(prefixPath, "/var/log/kube-audit"))) + Binds = append(Binds, fmt.Sprintf("%s:/var/log/kube-audit:z", path.Join(host.PrefixPath, "/var/log/kube-audit"))) } // Override args if they exist, add additional args @@ -335,12 +301,8 @@ func (c *Cluster) BuildKubeAPIProcess(host *hosts.Host, prefixPath string, servi } } -func (c *Cluster) BuildKubeControllerProcess(host *hosts.Host, prefixPath string, serviceOptions v3.KubernetesServicesOptions) v3.Process { - Command := []string{ - c.getRKEToolsEntryPoint(), - "kube-controller-manager", - } - +func (c *Cluster) BuildKubeControllerProcess(host *hosts.Host, serviceOptions v3.KubernetesServicesOptions) v3.Process { + Command := c.getRKEToolsEntryPoint(host.OS(), "kube-controller-manager") CommandArgs := map[string]string{ "cloud-provider": c.CloudProvider.Name, "cluster-cidr": c.ClusterCIDR, @@ -381,7 +343,7 @@ func (c *Cluster) BuildKubeControllerProcess(host *hosts.Host, prefixPath string services.SidekickContainerName, } Binds := []string{ - fmt.Sprintf("%s:/etc/kubernetes:z", path.Join(prefixPath, "/etc/kubernetes")), + fmt.Sprintf("%s:/etc/kubernetes:z", path.Join(host.PrefixPath, "/etc/kubernetes")), } for arg, value := range c.Services.KubeController.ExtraArgs { @@ -420,32 +382,23 @@ func (c *Cluster) BuildKubeControllerProcess(host *hosts.Host, prefixPath string } } -func (c *Cluster) BuildKubeletProcess(host *hosts.Host, prefixPath string, serviceOptions v3.KubernetesServicesOptions) v3.Process { - Command := []string{ - c.getRKEToolsEntryPoint(), - "kubelet", - } - if host.DockerInfo.OSType == "windows" { // compatible with Windows - Command = []string{ - "pwsh", "-NoLogo", "-NonInteractive", "-File", "c:/usr/bin/entrypoint.ps1", - "kubelet", - } - } - +func (c *Cluster) BuildKubeletProcess(host *hosts.Host, serviceOptions v3.KubernetesServicesOptions) v3.Process { + kubelet := &c.Services.Kubelet + Command := c.getRKEToolsEntryPoint(host.OS(), "kubelet") CommandArgs := map[string]string{ "client-ca-file": pki.GetCertPath(pki.CACertName), "cloud-provider": c.CloudProvider.Name, "cluster-dns": c.ClusterDNSServer, "cluster-domain": c.ClusterDomain, - "fail-swap-on": strconv.FormatBool(c.Services.Kubelet.FailSwapOn), + "fail-swap-on": strconv.FormatBool(kubelet.FailSwapOn), "hostname-override": host.HostnameOverride, "kubeconfig": pki.GetConfigPath(pki.KubeNodeCertName), - "pod-infra-container-image": c.Services.Kubelet.InfraContainerImage, - "root-dir": path.Join(prefixPath, "/var/lib/kubelet"), + "pod-infra-container-image": kubelet.InfraContainerImage, + "root-dir": path.Join(host.PrefixPath, "/var/lib/kubelet"), } - if host.DockerInfo.OSType == "windows" { // compatible with Windows - CommandArgs["kubeconfig"] = path.Join(prefixPath, pki.GetConfigPath(pki.KubeNodeCertName)) - CommandArgs["client-ca-file"] = path.Join(prefixPath, pki.GetCertPath(pki.CACertName)) + if host.IsWindows() { // compatible with Windows + CommandArgs["kubeconfig"] = path.Join(host.PrefixPath, pki.GetConfigPath(pki.KubeNodeCertName)) + CommandArgs["client-ca-file"] = path.Join(host.PrefixPath, pki.GetCertPath(pki.CACertName)) // this's a stopgap, we could drop this after https://github.com/kubernetes/kubernetes/pull/75618 merged CommandArgs["pod-infra-container-image"] = c.SystemImages.WindowsPodInfraContainer } @@ -462,8 +415,8 @@ func (c *Cluster) BuildKubeletProcess(host *hosts.Host, prefixPath string, servi } if len(c.CloudProvider.Name) > 0 { CommandArgs["cloud-config"] = cloudConfigFileName - if host.DockerInfo.OSType == "windows" { // compatible with Windows - CommandArgs["cloud-config"] = path.Join(prefixPath, cloudConfigFileName) + if host.IsWindows() { // compatible with Windows + CommandArgs["cloud-config"] = path.Join(host.PrefixPath, cloudConfigFileName) } } if c.IsKubeletGenerateServingCertificateEnabled() { @@ -471,23 +424,6 @@ func (c *Cluster) BuildKubeletProcess(host *hosts.Host, prefixPath string, servi CommandArgs["tls-private-key-file"] = pki.GetCertPath(fmt.Sprintf("%s-key", pki.GetCrtNameForHost(host, pki.KubeletCertName))) } - if len(c.CloudProvider.Name) > 0 { - c.Services.Kubelet.ExtraEnv = append( - c.Services.Kubelet.ExtraEnv, - fmt.Sprintf("%s=%s", CloudConfigSumEnv, getCloudConfigChecksum(c.CloudConfigFile))) - } - if len(c.PrivateRegistriesMap) > 0 { - kubeletDockerConfig, _ := docker.GetKubeletDockerConfig(c.PrivateRegistriesMap) - c.Services.Kubelet.ExtraEnv = append( - c.Services.Kubelet.ExtraEnv, - fmt.Sprintf("%s=%s", KubeletDockerConfigEnv, - b64.StdEncoding.EncodeToString([]byte(kubeletDockerConfig)))) - - c.Services.Kubelet.ExtraEnv = append( - c.Services.Kubelet.ExtraEnv, - fmt.Sprintf("%s=%s", KubeletDockerConfigFileEnv, path.Join(prefixPath, KubeletDockerConfigPath))) - } - if serviceOptions.Kubelet != nil { for k, v := range serviceOptions.Kubelet { // if the value is empty, we remove that option @@ -503,10 +439,10 @@ func (c *Cluster) BuildKubeletProcess(host *hosts.Host, prefixPath string, servi continue } - // if the value has [PREFIX_PATH] prefix, we need to replace it with `prefixPath`, + // if the value has [PREFIX_PATH] prefix, we need to replace it with `host.PrefixPath`, // e.g.: windows allows to use other drivers than `c:` if strings.HasPrefix(v, "[PREFIX_PATH]") { - CommandArgs[k] = path.Join(prefixPath, strings.Replace(v, "[PREFIX_PATH]", "", -1)) + CommandArgs[k] = path.Join(host.PrefixPath, strings.Replace(v, "[PREFIX_PATH]", "", -1)) continue } @@ -517,58 +453,68 @@ func (c *Cluster) BuildKubeletProcess(host *hosts.Host, prefixPath string, servi VolumesFrom := []string{ services.SidekickContainerName, } - Binds := []string{ - fmt.Sprintf("%s:/etc/kubernetes:z", path.Join(prefixPath, "/etc/kubernetes")), - "/etc/cni:/etc/cni:rw,z", - "/opt/cni:/opt/cni:rw,z", - fmt.Sprintf("%s:/var/lib/cni:z", path.Join(prefixPath, "/var/lib/cni")), - "/var/lib/calico:/var/lib/calico:z", - "/etc/resolv.conf:/etc/resolv.conf", - "/sys:/sys:rprivate", - host.DockerInfo.DockerRootDir + ":" + host.DockerInfo.DockerRootDir + ":rw,rslave,z", - fmt.Sprintf("%s:%s:shared,z", path.Join(prefixPath, "/var/lib/kubelet"), path.Join(prefixPath, "/var/lib/kubelet")), - "/var/lib/rancher:/var/lib/rancher:shared,z", - "/var/run:/var/run:rw,rprivate", - "/run:/run:rprivate", - fmt.Sprintf("%s:/etc/ceph", path.Join(prefixPath, "/etc/ceph")), - "/dev:/host/dev:rprivate", - "/var/log/containers:/var/log/containers:z", - "/var/log/pods:/var/log/pods:z", - "/usr:/host/usr:ro", - "/etc:/host/etc:ro", - } - // Special case to simplify using flex volumes - if path.Join(prefixPath, "/var/lib/kubelet") != "/var/lib/kubelet" { - Binds = append(Binds, "/var/lib/kubelet/volumeplugins:/var/lib/kubelet/volumeplugins:shared,z") - } - if host.DockerInfo.OSType == "windows" { // compatible with Windows + + var Binds []string + if host.IsWindows() { // compatible with Windows Binds = []string{ // put the execution binaries and cloud provider configuration to the host - fmt.Sprintf("%s:c:/host/etc/kubernetes", path.Join(prefixPath, "/etc/kubernetes")), + fmt.Sprintf("%s:c:/host/etc/kubernetes", path.Join(host.PrefixPath, "/etc/kubernetes")), // put the flexvolume plugins or private registry docker configuration to the host - fmt.Sprintf("%s:c:/host/var/lib/kubelet", path.Join(prefixPath, "/var/lib/kubelet")), + fmt.Sprintf("%s:c:/host/var/lib/kubelet", path.Join(host.PrefixPath, "/var/lib/kubelet")), // exchange resources with other components - fmt.Sprintf("%s:c:/host/run", path.Join(prefixPath, "/run")), + fmt.Sprintf("%s:c:/host/run", path.Join(host.PrefixPath, "/run")), + } + } else { + Binds = []string{ + fmt.Sprintf("%s:/etc/kubernetes:z", path.Join(host.PrefixPath, "/etc/kubernetes")), + "/etc/cni:/etc/cni:rw,z", + "/opt/cni:/opt/cni:rw,z", + fmt.Sprintf("%s:/var/lib/cni:z", path.Join(host.PrefixPath, "/var/lib/cni")), + "/var/lib/calico:/var/lib/calico:z", + "/etc/resolv.conf:/etc/resolv.conf", + "/sys:/sys:rprivate", + host.DockerInfo.DockerRootDir + ":" + host.DockerInfo.DockerRootDir + ":rw,rslave,z", + fmt.Sprintf("%s:%s:shared,z", path.Join(host.PrefixPath, "/var/lib/kubelet"), path.Join(host.PrefixPath, "/var/lib/kubelet")), + "/var/lib/rancher:/var/lib/rancher:shared,z", + "/var/run:/var/run:rw,rprivate", + "/run:/run:rprivate", + fmt.Sprintf("%s:/etc/ceph", path.Join(host.PrefixPath, "/etc/ceph")), + "/dev:/host/dev:rprivate", + "/var/log/containers:/var/log/containers:z", + "/var/log/pods:/var/log/pods:z", + "/usr:/host/usr:ro", + "/etc:/host/etc:ro", + } + + // Special case to simplify using flex volumes + if path.Join(host.PrefixPath, "/var/lib/kubelet") != "/var/lib/kubelet" { + Binds = append(Binds, "/var/lib/kubelet/volumeplugins:/var/lib/kubelet/volumeplugins:shared,z") } } + Binds = append(Binds, host.GetExtraBinds(kubelet.BaseService)...) - Env := c.Services.Kubelet.ExtraEnv - if host.DockerInfo.OSType == "windows" { // compatible with Windows + Env := host.GetExtraEnv(kubelet.BaseService) + + if len(c.CloudProvider.Name) > 0 { Env = append(Env, - fmt.Sprintf("%s=%s", ClusterCIDREnv, c.ClusterCIDR), - fmt.Sprintf("%s=%s", ClusterDomainEnv, c.ClusterDomain), - fmt.Sprintf("%s=%s", ClusterDNSServerEnv, c.ClusterDNSServer), - fmt.Sprintf("%s=%s", ClusterServiceCIDREnv, c.Services.KubeController.ServiceClusterIPRange), - fmt.Sprintf("%s=%s", NodeAddressEnv, host.Address), - fmt.Sprintf("%s=%s", NodeInternalAddressEnv, host.InternalAddress), - fmt.Sprintf("%s=%s", CloudProviderNameEnv, c.CloudProvider.Name), - ) + fmt.Sprintf("%s=%s", CloudConfigSumEnv, getCloudConfigChecksum(c.CloudConfigFile))) + } + if len(c.PrivateRegistriesMap) > 0 { + kubeletDockerConfig, _ := docker.GetKubeletDockerConfig(c.PrivateRegistriesMap) + Env = append(Env, + fmt.Sprintf("%s=%s", KubeletDockerConfigEnv, + b64.StdEncoding.EncodeToString([]byte(kubeletDockerConfig)))) + + Env = append(Env, + fmt.Sprintf("%s=%s", KubeletDockerConfigFileEnv, path.Join(host.PrefixPath, KubeletDockerConfigPath))) } - for arg, value := range c.Services.Kubelet.ExtraArgs { - if _, ok := c.Services.Kubelet.ExtraArgs[arg]; ok { - CommandArgs[arg] = value - } + if host.IsWindows() { // compatible with Windows + Env = append(Env, c.getWindowsEnv(host)...) + } + + for arg, value := range host.GetExtraArgs(kubelet.BaseService) { + CommandArgs[arg] = value } // If nodelocal DNS is configured, set cluster-dns to local IP @@ -576,27 +522,20 @@ func (c *Cluster) BuildKubeletProcess(host *hosts.Host, prefixPath string, servi CommandArgs["cluster-dns"] = c.DNS.Nodelocal.IPAddress } - for arg, value := range CommandArgs { - cmd := fmt.Sprintf("--%s=%s", arg, value) - Command = append(Command, cmd) - } - - Binds = append(Binds, c.Services.Kubelet.ExtraBinds...) - healthCheck := v3.HealthCheck{ URL: services.GetHealthCheckURL(false, services.KubeletPort), } - registryAuthConfig, _, _ := docker.GetImageRegistryConfig(c.Services.Kubelet.Image, c.PrivateRegistriesMap) + registryAuthConfig, _, _ := docker.GetImageRegistryConfig(kubelet.Image, c.PrivateRegistriesMap) return v3.Process{ Name: services.KubeletContainerName, - Command: Command, + Command: appendArgs(Command, CommandArgs), VolumesFrom: VolumesFrom, Binds: getUniqStringList(Binds), Env: getUniqStringList(Env), NetworkMode: "host", RestartPolicy: "always", - Image: c.Services.Kubelet.Image, + Image: kubelet.Image, PidMode: "host", Privileged: true, HealthCheck: healthCheck, @@ -607,25 +546,16 @@ func (c *Cluster) BuildKubeletProcess(host *hosts.Host, prefixPath string, servi } } -func (c *Cluster) BuildKubeProxyProcess(host *hosts.Host, prefixPath string, serviceOptions v3.KubernetesServicesOptions) v3.Process { - Command := []string{ - c.getRKEToolsEntryPoint(), - "kube-proxy", - } - if host.DockerInfo.OSType == "windows" { // compatible with Windows - Command = []string{ - "pwsh", "-NoLogo", "-NonInteractive", "-File", "c:/usr/bin/entrypoint.ps1", - "kube-proxy", - } - } - +func (c *Cluster) BuildKubeProxyProcess(host *hosts.Host, serviceOptions v3.KubernetesServicesOptions) v3.Process { + kubeproxy := &c.Services.Kubeproxy + Command := c.getRKEToolsEntryPoint(host.OS(), "kube-proxy") CommandArgs := map[string]string{ "cluster-cidr": c.ClusterCIDR, "hostname-override": host.HostnameOverride, "kubeconfig": pki.GetConfigPath(pki.KubeProxyCertName), } - if host.DockerInfo.OSType == "windows" { // compatible with Windows - CommandArgs["kubeconfig"] = path.Join(prefixPath, pki.GetConfigPath(pki.KubeProxyCertName)) + if host.IsWindows() { // compatible with Windows + CommandArgs["kubeconfig"] = path.Join(host.PrefixPath, pki.GetConfigPath(pki.KubeProxyCertName)) } if serviceOptions.Kubeproxy != nil { @@ -655,61 +585,47 @@ func (c *Cluster) BuildKubeProxyProcess(host *hosts.Host, prefixPath string, ser VolumesFrom := []string{ services.SidekickContainerName, } + //TODO: we should reevaluate if any of the bind mounts here should be using read-only mode - Binds := []string{ - fmt.Sprintf("%s:/etc/kubernetes:z", path.Join(prefixPath, "/etc/kubernetes")), - "/run:/run", - } - - BindModules := "/lib/modules:/lib/modules:z,ro" - if hosts.IsEnterpriseLinuxHost(host) && hosts.IsDockerSELinuxEnabled(host) && !hosts.IsEnterpriseLinuxDocker(host) { - // Avoid relabing on Enterprise Linux with Docker SELinux and upstream Docker - BindModules = "/lib/modules:/lib/modules:ro" - } - Binds = append(Binds, BindModules) - - if host.DockerInfo.OSType == "windows" { // compatible with Windows + var Binds []string + if host.IsWindows() { // compatible with Windows Binds = []string{ // put the execution binaries to the host - fmt.Sprintf("%s:c:/host/etc/kubernetes", path.Join(prefixPath, "/etc/kubernetes")), + fmt.Sprintf("%s:c:/host/etc/kubernetes", path.Join(host.PrefixPath, "/etc/kubernetes")), // exchange resources with other components - fmt.Sprintf("%s:c:/host/run", path.Join(prefixPath, "/run")), + fmt.Sprintf("%s:c:/host/run", path.Join(host.PrefixPath, "/run")), } - } - - Env := c.Services.Kubeproxy.ExtraEnv - if host.DockerInfo.OSType == "windows" { // compatible with Windows - Env = append(Env, - fmt.Sprintf("%s=%s", ClusterCIDREnv, c.ClusterCIDR), - fmt.Sprintf("%s=%s", ClusterDomainEnv, c.ClusterDomain), - fmt.Sprintf("%s=%s", ClusterDNSServerEnv, c.ClusterDNSServer), - fmt.Sprintf("%s=%s", ClusterServiceCIDREnv, c.Services.KubeController.ServiceClusterIPRange), - fmt.Sprintf("%s=%s", NodeAddressEnv, host.Address), - fmt.Sprintf("%s=%s", NodeInternalAddressEnv, host.InternalAddress), - fmt.Sprintf("%s=%s", CloudProviderNameEnv, c.CloudProvider.Name), - ) - } - - for arg, value := range c.Services.Kubeproxy.ExtraArgs { - if _, ok := c.Services.Kubeproxy.ExtraArgs[arg]; ok { - CommandArgs[arg] = value + } else { + Binds = []string{ + fmt.Sprintf("%s:/etc/kubernetes:z", path.Join(host.PrefixPath, "/etc/kubernetes")), + "/run:/run", } + + BindModules := "/lib/modules:/lib/modules:z,ro" + if hosts.IsEnterpriseLinuxHost(host) && hosts.IsDockerSELinuxEnabled(host) && !hosts.IsEnterpriseLinuxDocker(host) { + // Avoid relabing on Enterprise Linux with Docker SELinux and upstream Docker + BindModules = "/lib/modules:/lib/modules:ro" + } + Binds = append(Binds, BindModules) + } + Binds = append(Binds, host.GetExtraBinds(kubeproxy.BaseService)...) + + Env := host.GetExtraEnv(kubeproxy.BaseService) + if host.IsWindows() { // compatible with Windows + Env = append(Env, c.getWindowsEnv(host)...) } - for arg, value := range CommandArgs { - cmd := fmt.Sprintf("--%s=%s", arg, value) - Command = append(Command, cmd) + for arg, value := range host.GetExtraArgs(kubeproxy.BaseService) { + CommandArgs[arg] = value } - Binds = append(Binds, c.Services.Kubeproxy.ExtraBinds...) - healthCheck := v3.HealthCheck{ URL: services.GetHealthCheckURL(false, services.KubeproxyPort), } - registryAuthConfig, _, _ := docker.GetImageRegistryConfig(c.Services.Kubeproxy.Image, c.PrivateRegistriesMap) + registryAuthConfig, _, _ := docker.GetImageRegistryConfig(kubeproxy.Image, c.PrivateRegistriesMap) return v3.Process{ Name: services.KubeproxyContainerName, - Command: Command, + Command: appendArgs(Command, CommandArgs), VolumesFrom: VolumesFrom, Binds: getUniqStringList(Binds), Env: getUniqStringList(Env), @@ -718,7 +634,7 @@ func (c *Cluster) BuildKubeProxyProcess(host *hosts.Host, prefixPath string, ser PidMode: "host", Privileged: true, HealthCheck: healthCheck, - Image: c.Services.Kubeproxy.Image, + Image: kubeproxy.Image, ImageRegistryAuthConfig: registryAuthConfig, Labels: map[string]string{ services.ContainerNameLabel: services.KubeproxyContainerName, @@ -726,16 +642,8 @@ func (c *Cluster) BuildKubeProxyProcess(host *hosts.Host, prefixPath string, ser } } -func (c *Cluster) BuildProxyProcess(host *hosts.Host, prefixPath string) v3.Process { - Command := []string{ - "nginx-proxy", - } - if host.DockerInfo.OSType == "windows" { // compatible with Windows - Command = []string{ - "pwsh", "-NoLogo", "-NonInteractive", "-File", "c:/usr/bin/nginx-proxy.ps1", - } - } - +func (c *Cluster) BuildProxyProcess(host *hosts.Host) v3.Process { + Command := c.getNginxEntryPoint(host.OS()) nginxProxyEnv := "" for i, host := range c.ControlPlaneHosts { nginxProxyEnv += fmt.Sprintf("%s", host.InternalAddress) @@ -746,19 +654,19 @@ func (c *Cluster) BuildProxyProcess(host *hosts.Host, prefixPath string) v3.Proc Env := []string{fmt.Sprintf("%s=%s", services.NginxProxyEnvName, nginxProxyEnv)} VolumesFrom := []string{} - if host.DockerInfo.OSType == "windows" { // compatible withe Windows + if host.IsWindows() { // compatible withe Windows VolumesFrom = []string{ services.SidekickContainerName, } } Binds := []string{} - if host.DockerInfo.OSType == "windows" { // compatible with Windows + if host.IsWindows() { // compatible with Windows Binds = []string{ // put the execution binaries and generate the configuration to the host - fmt.Sprintf("%s:c:/host/etc/nginx", path.Join(prefixPath, "/etc/nginx")), + fmt.Sprintf("%s:c:/host/etc/nginx", path.Join(host.PrefixPath, "/etc/nginx")), // exchange resources with other components - fmt.Sprintf("%s:c:/host/run", path.Join(prefixPath, "/run")), + fmt.Sprintf("%s:c:/host/run", path.Join(host.PrefixPath, "/run")), } } @@ -782,12 +690,8 @@ func (c *Cluster) BuildProxyProcess(host *hosts.Host, prefixPath string) v3.Proc } } -func (c *Cluster) BuildSchedulerProcess(host *hosts.Host, prefixPath string, serviceOptions v3.KubernetesServicesOptions) v3.Process { - Command := []string{ - c.getRKEToolsEntryPoint(), - "kube-scheduler", - } - +func (c *Cluster) BuildSchedulerProcess(host *hosts.Host, serviceOptions v3.KubernetesServicesOptions) v3.Process { + Command := c.getRKEToolsEntryPoint(host.OS(), "kube-scheduler") CommandArgs := map[string]string{ "kubeconfig": pki.GetConfigPath(pki.KubeSchedulerCertName), } @@ -812,7 +716,7 @@ func (c *Cluster) BuildSchedulerProcess(host *hosts.Host, prefixPath string, ser services.SidekickContainerName, } Binds := []string{ - fmt.Sprintf("%s:/etc/kubernetes:z", path.Join(prefixPath, "/etc/kubernetes")), + fmt.Sprintf("%s:/etc/kubernetes:z", path.Join(host.PrefixPath, "/etc/kubernetes")), } for arg, value := range c.Services.Scheduler.ExtraArgs { @@ -849,29 +753,13 @@ func (c *Cluster) BuildSchedulerProcess(host *hosts.Host, prefixPath string, ser } } -func (c *Cluster) BuildSidecarProcess(host *hosts.Host, prefixPath string) v3.Process { - Command := []string{ - "/bin/bash", - } - if host.DockerInfo.OSType == "windows" { // compatible with Windows - // windows docker doesn't support host network mode, - // so we can't use the network components installed by addon like Linux. - // we use sidecar container to maintain the network components - Command = []string{ - "pwsh", "-NoLogo", "-NonInteractive", "-File", "c:/usr/bin/sidecar.ps1", - } - } +func (c *Cluster) BuildSidecarProcess(host *hosts.Host) v3.Process { + Command := c.getSidecarEntryPoint(host.OS()) Env := []string{} - if host.DockerInfo.OSType == "windows" { // compatible with Windows + if host.IsWindows() { // compatible with Windows + Env = append(Env, c.getWindowsEnv(host)...) Env = append(Env, - fmt.Sprintf("%s=%s", ClusterCIDREnv, c.ClusterCIDR), - fmt.Sprintf("%s=%s", ClusterDomainEnv, c.ClusterDomain), - fmt.Sprintf("%s=%s", ClusterDNSServerEnv, c.ClusterDNSServer), - fmt.Sprintf("%s=%s", ClusterServiceCIDREnv, c.Services.KubeController.ServiceClusterIPRange), - fmt.Sprintf("%s=%s", NodeAddressEnv, host.Address), - fmt.Sprintf("%s=%s", NodeInternalAddressEnv, host.InternalAddress), - fmt.Sprintf("%s=%s", CloudProviderNameEnv, c.CloudProvider.Name), // sidekick needs the node name to drive the cni network management, e.g: flanneld fmt.Sprintf("%s=%s", NodeNameOverrideEnv, host.HostnameOverride), // sidekick use the network configuration to drive the cni network management, e.g: flanneld @@ -880,26 +768,26 @@ func (c *Cluster) BuildSidecarProcess(host *hosts.Host, prefixPath string) v3.Pr } Binds := []string{} - if host.DockerInfo.OSType == "windows" { // compatible with Windows + if host.IsWindows() { // compatible with Windows Binds = []string{ // put the execution binaries and the cni binaries to the host - fmt.Sprintf("%s:c:/host/opt", path.Join(prefixPath, "/opt")), + fmt.Sprintf("%s:c:/host/opt", path.Join(host.PrefixPath, "/opt")), // put the cni configuration to the host - fmt.Sprintf("%s:c:/host/etc/cni/net.d", path.Join(prefixPath, "/etc/cni/net.d")), + fmt.Sprintf("%s:c:/host/etc/cni/net.d", path.Join(host.PrefixPath, "/etc/cni/net.d")), // put the cni network component configuration to the host - fmt.Sprintf("%s:c:/host/etc/kube-flannel", path.Join(prefixPath, "/etc/kube-flannel")), + fmt.Sprintf("%s:c:/host/etc/kube-flannel", path.Join(host.PrefixPath, "/etc/kube-flannel")), // exchange resources with other components - fmt.Sprintf("%s:c:/host/run", path.Join(prefixPath, "/run")), + fmt.Sprintf("%s:c:/host/run", path.Join(host.PrefixPath, "/run")), } } RestartPolicy := "" - if host.DockerInfo.OSType == "windows" { // compatible with Windows + if host.IsWindows() { // compatible with Windows RestartPolicy = "always" } NetworkMode := "none" - if host.DockerInfo.OSType == "windows" { // compatible with Windows + if host.IsWindows() { // compatible with Windows NetworkMode = "" } @@ -920,7 +808,7 @@ func (c *Cluster) BuildSidecarProcess(host *hosts.Host, prefixPath string) v3.Pr } } -func (c *Cluster) BuildEtcdProcess(host *hosts.Host, etcdHosts []*hosts.Host, prefixPath string, serviceOptions v3.KubernetesServicesOptions) v3.Process { +func (c *Cluster) BuildEtcdProcess(host *hosts.Host, etcdHosts []*hosts.Host, serviceOptions v3.KubernetesServicesOptions) v3.Process { nodeName := pki.GetCrtNameForHost(host, pki.EtcdCertName) initCluster := "" architecture := "amd64" @@ -967,8 +855,8 @@ func (c *Cluster) BuildEtcdProcess(host *hosts.Host, etcdHosts []*hosts.Host, pr } Binds := []string{ - fmt.Sprintf("%s:%s:z", path.Join(prefixPath, "/var/lib/etcd"), services.EtcdDataDir), - fmt.Sprintf("%s:/etc/kubernetes:z", path.Join(prefixPath, "/etc/kubernetes")), + fmt.Sprintf("%s:%s:z", path.Join(host.PrefixPath, "/var/lib/etcd"), services.EtcdDataDir), + fmt.Sprintf("%s:/etc/kubernetes:z", path.Join(host.PrefixPath, "/etc/kubernetes")), } if serviceOptions.Etcd != nil { @@ -1159,25 +1047,6 @@ func getUniqStringList(l []string) []string { return ul } -func (c *Cluster) getRKEToolsEntryPoint() string { - v := strings.Split(c.SystemImages.KubernetesServicesSidecar, ":") - last := v[len(v)-1] - - sv, err := util.StrToSemVer(last) - if err != nil { - return DefaultToolsEntrypoint - } - svdefault, err := util.StrToSemVer(DefaultToolsEntrypointVersion) - if err != nil { - return DefaultToolsEntrypoint - } - - if sv.LessThan(*svdefault) { - return LegacyToolsEntrypoint - } - return DefaultToolsEntrypoint -} - func getNetworkJSON(netconfig v3.NetworkConfig) string { ret, err := json.Marshal(netconfig) if err != nil { @@ -1185,3 +1054,10 @@ func getNetworkJSON(netconfig v3.NetworkConfig) string { } return string(ret) } + +func appendArgs(command []string, args map[string]string) []string { + for arg, value := range args { + command = append(command, fmt.Sprintf("--%s=%s", arg, value)) + } + return command +} diff --git a/cluster/reconcile.go b/cluster/reconcile.go index b08b1526..787115d1 100644 --- a/cluster/reconcile.go +++ b/cluster/reconcile.go @@ -80,7 +80,7 @@ func reconcileWorker(ctx context.Context, currentCluster, kubeCluster *Cluster, return fmt.Errorf("Failed to delete worker node [%s] from cluster: %v", toDeleteHost.Address, err) } // attempting to clean services/files on the host - if err := reconcileHost(ctx, toDeleteHost, true, false, currentCluster.SystemImages.Alpine, currentCluster.DockerDialerFactory, currentCluster.PrivateRegistriesMap, currentCluster.PrefixPath, currentCluster.Version); err != nil { + if err := reconcileHost(ctx, toDeleteHost, true, false, currentCluster); err != nil { log.Warnf(ctx, "[reconcile] Couldn't clean up worker node [%s]: %v", toDeleteHost.Address, err) continue } @@ -133,12 +133,12 @@ func reconcileControl(ctx context.Context, currentCluster, kubeCluster *Cluster, return nil } -func reconcileHost(ctx context.Context, toDeleteHost *hosts.Host, worker, etcd bool, cleanerImage string, dialerFactory hosts.DialerFactory, prsMap map[string]v3.PrivateRegistry, clusterPrefixPath string, clusterVersion string) error { +func reconcileHost(ctx context.Context, toDeleteHost *hosts.Host, worker, etcd bool, cluster *Cluster) error { var retryErr error retries := 3 sleepSeconds := 3 for i := 0; i < retries; i++ { - if retryErr = toDeleteHost.TunnelUp(ctx, dialerFactory, clusterPrefixPath, clusterVersion); retryErr != nil { + if retryErr = toDeleteHost.TunnelUp(ctx, cluster.DockerDialerFactory, cluster.getPrefixPath(toDeleteHost.OS()), cluster.Version); retryErr != nil { logrus.Debugf("Failed to dial the host %s trying again in %d seconds", toDeleteHost.Address, sleepSeconds) time.Sleep(time.Second * time.Duration(sleepSeconds)) toDeleteHost.DClient = nil @@ -153,21 +153,21 @@ func reconcileHost(ctx context.Context, toDeleteHost *hosts.Host, worker, etcd b if err := services.RemoveWorkerPlane(ctx, []*hosts.Host{toDeleteHost}, false); err != nil { return fmt.Errorf("Couldn't remove worker plane: %v", err) } - if err := toDeleteHost.CleanUpWorkerHost(ctx, cleanerImage, prsMap); err != nil { + if err := toDeleteHost.CleanUpWorkerHost(ctx, cluster.SystemImages.Alpine, cluster.PrivateRegistriesMap); err != nil { return fmt.Errorf("Not able to clean the host: %v", err) } } else if etcd { if err := services.RemoveEtcdPlane(ctx, []*hosts.Host{toDeleteHost}, false); err != nil { return fmt.Errorf("Couldn't remove etcd plane: %v", err) } - if err := toDeleteHost.CleanUpEtcdHost(ctx, cleanerImage, prsMap); err != nil { + if err := toDeleteHost.CleanUpEtcdHost(ctx, cluster.SystemImages.Alpine, cluster.PrivateRegistriesMap); err != nil { return fmt.Errorf("Not able to clean the host: %v", err) } } else { if err := services.RemoveControlPlane(ctx, []*hosts.Host{toDeleteHost}, false); err != nil { return fmt.Errorf("Couldn't remove control plane: %v", err) } - if err := toDeleteHost.CleanUpControlHost(ctx, cleanerImage, prsMap); err != nil { + if err := toDeleteHost.CleanUpControlHost(ctx, cluster.SystemImages.Alpine, cluster.PrivateRegistriesMap); err != nil { return fmt.Errorf("Not able to clean the host: %v", err) } } @@ -227,7 +227,7 @@ func addEtcdMembers(ctx context.Context, currentCluster, kubeCluster *Cluster, k if err != nil { return err } - etcdNodePlanMap[etcdReadyHost.Address] = BuildRKEConfigNodePlan(ctx, kubeCluster, etcdReadyHost, etcdReadyHost.DockerInfo, svcOptions) + etcdNodePlanMap[etcdReadyHost.Address] = BuildRKEConfigNodePlan(ctx, kubeCluster, etcdReadyHost, svcOptions) } // this will start the newly added etcd node and make sure it started correctly before restarting other node // https://github.com/etcd-io/etcd/blob/master/Documentation/op-guide/runtime-configuration.md#add-a-new-member @@ -251,7 +251,7 @@ func deleteEtcdMembers(ctx context.Context, currentCluster, kubeCluster *Cluster continue } // attempting to clean services/files on the host - if err := reconcileHost(ctx, etcdHost, false, true, currentCluster.SystemImages.Alpine, currentCluster.DockerDialerFactory, currentCluster.PrivateRegistriesMap, currentCluster.PrefixPath, currentCluster.Version); err != nil { + if err := reconcileHost(ctx, etcdHost, false, true, currentCluster); err != nil { log.Warnf(ctx, "[reconcile] Couldn't clean up etcd node [%s]: %v", etcdHost.Address, err) continue } @@ -314,7 +314,7 @@ func cleanControlNode(ctx context.Context, kubeCluster, currentCluster *Cluster, } } // attempting to clean services/files on the host - if err := reconcileHost(ctx, toDeleteHost, false, false, currentCluster.SystemImages.Alpine, currentCluster.DockerDialerFactory, currentCluster.PrivateRegistriesMap, currentCluster.PrefixPath, currentCluster.Version); err != nil { + if err := reconcileHost(ctx, toDeleteHost, false, false, currentCluster); err != nil { log.Warnf(ctx, "[reconcile] Couldn't clean up controlplane node [%s]: %v", toDeleteHost.Address, err) } return nil diff --git a/hosts/hosts.go b/hosts/hosts.go index b753220b..e97ae7a1 100644 --- a/hosts/hosts.go +++ b/hosts/hosts.go @@ -147,6 +147,46 @@ func (h *Host) CleanUp(ctx context.Context, toCleanPaths []string, cleanerImage return nil } +func (h *Host) OS() string { + return h.DockerInfo.OSType +} + +func (h *Host) IsWindows() bool { + return h.DockerInfo.OSType == "windows" +} + +func (h *Host) IsLinux() bool { + return h.DockerInfo.OSType == "linux" +} + +func (h *Host) ProcessFilter(processes map[string]v3.Process) map[string]v3.Process { + if h.IsWindows() { + for name, process := range processes { + // doesn't support host network on windows + if process.NetworkMode == "host" { + process.NetworkMode = "" + } + + // doesn't support PID on windows + if process.PidMode != "" { + process.PidMode = "" + } + + // doesn't support privileged mode on windows + if process.Privileged { + process.Privileged = false + } + + // doesn't execute health check + process.HealthCheck = v3.HealthCheck{} + + processes[name] = process + } + } + + return processes +} + func DeleteNode(ctx context.Context, toDeleteHost *Host, kubeClient *kubernetes.Clientset, hasAnotherRole bool, cloudProvider string) error { if hasAnotherRole { log.Infof(ctx, "[hosts] host [%s] has another role, skipping delete from kubernetes cluster", toDeleteHost.Address) @@ -315,23 +355,50 @@ func GetUniqueHostList(etcdHosts, cpHosts, workerHosts []*Host) []*Host { return uniqHostList } -func GetPrefixPath(osType, ClusterPrefixPath string) string { +func (h *Host) SetPrefixPath(clusterPrefixPath string) { var prefixPath string switch { - case ClusterPrefixPath != "/": - prefixPath = ClusterPrefixPath - case strings.Contains(osType, B2DOS): + case clusterPrefixPath != "/": + prefixPath = clusterPrefixPath + case strings.Contains(h.DockerInfo.OperatingSystem, B2DOS): prefixPath = B2DPrefixPath - case strings.Contains(osType, ROS): + case strings.Contains(h.DockerInfo.OperatingSystem, ROS): prefixPath = ROSPrefixPath - case strings.Contains(osType, CoreOS): + case strings.Contains(h.DockerInfo.OperatingSystem, CoreOS): prefixPath = CoreOSPrefixPath - case strings.Contains(osType, WindowsOS): + case strings.Contains(h.DockerInfo.OperatingSystem, WindowsOS): prefixPath = WindowsPrefixPath default: - prefixPath = ClusterPrefixPath + prefixPath = clusterPrefixPath + } + + h.PrefixPath = prefixPath +} + +func (h *Host) GetExtraBinds(service v3.BaseService) []string { + switch { + case h.OS() == "windows" && len(service.WindowsExtraBinds) > 0: + return service.WindowsExtraBinds + default: + return service.ExtraBinds + } +} + +func (h *Host) GetExtraEnv(service v3.BaseService) []string { + switch { + case h.OS() == "windows" && len(service.WindowsExtraEnv) > 0: + return service.WindowsExtraEnv + default: + return service.ExtraEnv + } +} +func (h *Host) GetExtraArgs(service v3.BaseService) map[string]string { + switch { + case h.OS() == "windows" && len(service.WindowsExtraArgs) > 0: + return service.WindowsExtraArgs + default: + return service.ExtraArgs } - return prefixPath } func DoRunLogCleaner(ctx context.Context, host *Host, alpineImage string, prsMap map[string]v3.PrivateRegistry) error { diff --git a/hosts/tunnel.go b/hosts/tunnel.go index 209b6d3d..cdd393e4 100644 --- a/hosts/tunnel.go +++ b/hosts/tunnel.go @@ -4,16 +4,14 @@ import ( "context" "fmt" "io/ioutil" + "net" "os" "path/filepath" - "github.com/rancher/rke/metadata" - - "net" - "github.com/docker/docker/client" "github.com/rancher/rke/docker" "github.com/rancher/rke/log" + "github.com/rancher/rke/metadata" "github.com/rancher/rke/util" "github.com/sirupsen/logrus" "golang.org/x/crypto/ssh" @@ -44,7 +42,7 @@ func (h *Host) TunnelUp(ctx context.Context, dialerFactory DialerFactory, cluste if err := checkDockerVersion(ctx, h, clusterVersion); err != nil { return err } - h.PrefixPath = GetPrefixPath(h.DockerInfo.OperatingSystem, clusterPrefixPath) + h.SetPrefixPath(clusterPrefixPath) return nil }