add fields ExtraArgsArray and WindowsExtraArgsArray

cluster/plan.go

@@ -188,6 +188,8 @@ func (c *Cluster) BuildKubeAPIProcess(host *hosts.Host, serviceOptions v3.Kubern
 		"tls-cert-file":                pki.GetCertPath(pki.KubeAPICertName),
 		"tls-private-key-file":         pki.GetKeyPath(pki.KubeAPICertName),
 	}
+	CommandArrayArgs := make(map[string][]string, len(c.Services.KubeAPI.ExtraArgsArray))
+
 	if len(c.CloudProvider.Name) > 0 {
 		CommandArgs["cloud-config"] = cloudConfigFileName
 	}
@@ -300,11 +302,13 @@ func (c *Cluster) BuildKubeAPIProcess(host *hosts.Host, serviceOptions v3.Kubern
 		}
 	}
 
-	for arg, value := range CommandArgs {
+	for arg, value := range c.Services.KubeAPI.ExtraArgsArray {
-		cmd := fmt.Sprintf("--%s=%s", arg, value)
+		CommandArrayArgs[arg] = value
-		Command = append(Command, cmd)
 	}
 
+	Command = appendArgs(Command, CommandArgs)
+	Command = appendArrayArgs(Command, CommandArrayArgs)
+
 	Binds = append(Binds, c.Services.KubeAPI.ExtraBinds...)
 
 	healthCheck := v3.HealthCheck{
@@ -339,6 +343,7 @@ func (c *Cluster) BuildKubeControllerProcess(host *hosts.Host, serviceOptions v3
 		"service-account-private-key-file": pki.GetKeyPath(pki.ServiceAccountTokenKeyName),
 		"service-cluster-ip-range":         c.Services.KubeController.ServiceClusterIPRange,
 	}
+	CommandArrayArgs := make(map[string][]string, len(c.Services.KubeAPI.ExtraArgsArray))
 	// Best security practice is to listen on localhost, but DinD uses private container network instead of Host.
 	if c.DinD {
 		CommandArgs["address"] = "0.0.0.0"
@@ -392,10 +397,13 @@ func (c *Cluster) BuildKubeControllerProcess(host *hosts.Host, serviceOptions v3
 		}
 	}
 
-	for arg, value := range CommandArgs {
+	for arg, value := range c.Services.KubeController.ExtraArgsArray {
-		cmd := fmt.Sprintf("--%s=%s", arg, value)
+		CommandArrayArgs[arg] = value
-		Command = append(Command, cmd)
 	}
+
+	Command = appendArgs(Command, CommandArgs)
+	Command = appendArrayArgs(Command, CommandArrayArgs)
+
 	k8sTag, err := util.GetImageTagFromImage(c.SystemImages.Kubernetes)
 	if err != nil {
 		logrus.Warn(err)
@@ -595,6 +603,14 @@ func (c *Cluster) BuildKubeletProcess(host *hosts.Host, serviceOptions v3.Kubern
 	for arg, value := range host.GetExtraArgs(kubelet.BaseService) {
 		CommandArgs[arg] = value
 	}
+	extraArgsArray := host.GetExtraArgsArray(kubelet.BaseService)
+	CommandArrayArgs := make(map[string][]string, len(extraArgsArray))
+	for arg, value := range extraArgsArray {
+		CommandArrayArgs[arg] = value
+	}
+
+	Command = appendArgs(Command, CommandArgs)
+	Command = appendArrayArgs(Command, CommandArrayArgs)
 
 	// If nodelocal DNS is configured, set cluster-dns to local IP
 	if c.DNS.Nodelocal != nil && c.DNS.Nodelocal.IPAddress != "" {
@@ -608,7 +624,7 @@ func (c *Cluster) BuildKubeletProcess(host *hosts.Host, serviceOptions v3.Kubern
 
 	return v3.Process{
 		Name:                    services.KubeletContainerName,
-		Command:                 appendArgs(Command, CommandArgs),
+		Command:                 Command,
 		VolumesFrom:             VolumesFrom,
 		Binds:                   getUniqStringList(Binds),
 		Env:                     getUniqStringList(Env),
@@ -707,13 +723,22 @@ func (c *Cluster) BuildKubeProxyProcess(host *hosts.Host, serviceOptions v3.Kube
 		CommandArgs[arg] = value
 	}
 
+	extraArgsArray := host.GetExtraArgsArray(kubeproxy.BaseService)
+	CommandArrayArgs := make(map[string][]string, len(extraArgsArray))
+	for arg, value := range extraArgsArray {
+		CommandArrayArgs[arg] = value
+	}
+
+	Command = appendArgs(Command, CommandArgs)
+	Command = appendArrayArgs(Command, CommandArrayArgs)
+
 	healthCheck := v3.HealthCheck{
 		URL: services.GetHealthCheckURL(false, services.KubeproxyPort),
 	}
 	registryAuthConfig, _, _ := docker.GetImageRegistryConfig(kubeproxy.Image, c.PrivateRegistriesMap)
 	return v3.Process{
 		Name:                    services.KubeproxyContainerName,
-		Command:                 appendArgs(Command, CommandArgs),
+		Command:                 Command,
 		VolumesFrom:             VolumesFrom,
 		Binds:                   getUniqStringList(Binds),
 		Env:                     getUniqStringList(Env),
@@ -786,7 +811,7 @@ func (c *Cluster) BuildSchedulerProcess(host *hosts.Host, serviceOptions v3.Kube
 	CommandArgs := map[string]string{
 		"kubeconfig": pki.GetConfigPath(pki.KubeSchedulerCertName),
 	}
-
+	CommandArrayArgs := make(map[string][]string, len(c.Services.KubeAPI.ExtraArgsArray))
 	// Best security practice is to listen on localhost, but DinD uses private container network instead of Host.
 	if c.DinD {
 		CommandArgs["address"] = "0.0.0.0"
@@ -827,11 +852,13 @@ func (c *Cluster) BuildSchedulerProcess(host *hosts.Host, serviceOptions v3.Kube
 		}
 	}
 
-	for arg, value := range CommandArgs {
+	for arg, value := range c.Services.Scheduler.ExtraArgsArray {
-		cmd := fmt.Sprintf("--%s=%s", arg, value)
+		CommandArrayArgs[arg] = value
-		Command = append(Command, cmd)
 	}
 
+	Command = appendArgs(Command, CommandArgs)
+	Command = appendArrayArgs(Command, CommandArrayArgs)
+
 	Binds = append(Binds, c.Services.Scheduler.ExtraBinds...)
 
 	maxK8s122Version, err := util.StrToSemVer(MaxK8s122Version)
@@ -969,6 +996,7 @@ func (c *Cluster) BuildEtcdProcess(host *hosts.Host, etcdHosts []*hosts.Host, se
 		"peer-cert-file":              pki.GetCertPath(nodeName),
 		"peer-key-file":               pki.GetKeyPath(nodeName),
 	}
+	CommandArrayArgs := make(map[string][]string, len(c.Services.KubeAPI.ExtraArgsArray))
 
 	etcdTag, err := util.GetImageTagFromImage(c.Services.Etcd.Image)
 	if err != nil {
@@ -1035,6 +1063,10 @@ func (c *Cluster) BuildEtcdProcess(host *hosts.Host, etcdHosts []*hosts.Host, se
 		}
 	}
 
+	for arg, value := range c.Services.Etcd.ExtraArgsArray {
+		CommandArrayArgs[arg] = value
+	}
+
 	// adding the old default value from L922 if not present in metadata options or passed by user
 	if _, ok := CommandArgs["client-cert-auth"]; !ok {
 		args = append(args, "--client-cert-auth")
@@ -1043,10 +1075,8 @@ func (c *Cluster) BuildEtcdProcess(host *hosts.Host, etcdHosts []*hosts.Host, se
 		args = append(args, "--peer-client-cert-auth")
 	}
 
-	for arg, value := range CommandArgs {
+	args = appendArgs(args, CommandArgs)
-		cmd := fmt.Sprintf("--%s=%s", arg, value)
+	args = appendArrayArgs(args, CommandArrayArgs)
-		args = append(args, cmd)
-	}
 
 	Binds = append(Binds, c.Services.Etcd.ExtraBinds...)
 	healthCheck := v3.HealthCheck{
@@ -1213,6 +1243,15 @@ func appendArgs(command []string, args map[string]string) []string {
 	return command
 }
 
+func appendArrayArgs(command []string, args map[string][]string) []string {
+	for arg, value := range args {
+		for _, v := range value {
+			command = append(command, fmt.Sprintf("--%s=%s", arg, v))
+		}
+	}
+	return command
+}
+
 func (c *Cluster) IsCRIDockerdEnabled() bool {
 	if c == nil {
 		return false

hosts/hosts.go

@@ -420,6 +420,13 @@ func (h *Host) GetExtraArgs(service v3.BaseService) map[string]string {
 	}
 }
 
+func (h *Host) GetExtraArgsArray(service v3.BaseService) map[string][]string {
+	if h.OS() == "windows" && len(service.WindowsExtraArgsArray) > 0 {
+		return service.WindowsExtraArgsArray
+	}
+	return service.ExtraArgsArray
+}
+
 func DoRunLogCleaner(ctx context.Context, host *Host, alpineImage string, prsMap map[string]v3.PrivateRegistry) error {
 	logrus.Debugf("[cleanup] Starting log link cleanup on host [%s]", host.Address)
 	imageCfg := &container.Config{

types/rke_types.go

@@ -364,6 +364,8 @@ type BaseService struct {
 	Image string `yaml:"image" json:"image,omitempty"`
 	// Extra arguments that are added to the services
 	ExtraArgs map[string]string `yaml:"extra_args" json:"extraArgs,omitempty"`
+	// Extra arguments that can be specified multiple times which are added to the services
+	ExtraArgsArray map[string][]string `yaml:"extra_args_array" json:"extraArgsArray,omitempty"`
 	// Extra binds added to the nodes
 	ExtraBinds []string `yaml:"extra_binds" json:"extraBinds,omitempty"`
 	// this is to provide extra env variable to the docker container running kubernetes service
@@ -372,6 +374,8 @@ type BaseService struct {
 	// Windows nodes only of the same as the above
 	// Extra arguments that are added to the services
 	WindowsExtraArgs map[string]string `yaml:"win_extra_args" json:"winExtraArgs,omitempty"`
+	// Extra arguments that can be specified multiple times which are added to the services
+	WindowsExtraArgsArray map[string][]string `yaml:"win_extra_args_array" json:"winExtraArgsArray,omitempty"`
 	// Extra binds added to the nodes
 	WindowsExtraBinds []string `yaml:"win_extra_binds" json:"winExtraBinds,omitempty"`
 	// this is to provide extra env variable to the docker container running kubernetes service