Merge pull request #3547 from jiaqiluo/fix-kube-apiserver

cluster/plan.go

@@ -40,10 +40,11 @@ const (
 
 	NetworkConfigurationEnv = "RKE_NETWORK_CONFIGURATION"
 
-	EtcdPathPrefix       = "/registry"
+	EtcdPathPrefix        = "/registry"
-	CloudConfigSumEnv    = "RKE_CLOUD_CONFIG_CHECKSUM"
+	CloudConfigSumEnv     = "RKE_CLOUD_CONFIG_CHECKSUM"
-	CloudProviderNameEnv = "RKE_CLOUD_PROVIDER_NAME"
+	CloudProviderNameEnv  = "RKE_CLOUD_PROVIDER_NAME"
-	AuditLogConfigSumEnv = "RKE_AUDITLOG_CONFIG_CHECKSUM"
+	AuditLogConfigSumEnv  = "RKE_AUDITLOG_CONFIG_CHECKSUM"
+	AdmissionConfigSumEnv = "RKE_ADMISSION_CONFIG_CHECKSUM"
 
 	DefaultToolsEntrypoint        = "/opt/rke-tools/entrypoint.sh"
 	DefaultToolsEntrypointVersion = "0.1.13"
@@ -202,6 +203,7 @@ func (c *Cluster) BuildKubeAPIProcess(host *hosts.Host, serviceOptions v3.Kubern
 		"tls-private-key-file":          pki.GetKeyPath(pki.KubeAPICertName),
 	}
 	CommandArrayArgs := make(map[string][]string, len(c.Services.KubeAPI.ExtraArgsArray))
+	Env := make([]string, len(c.Services.KubeAPI.ExtraEnv))
 
 	if len(c.CloudProvider.Name) > 0 {
 		CommandArgs["cloud-config"] = cloudConfigFileName
@@ -211,9 +213,7 @@ func (c *Cluster) BuildKubeAPIProcess(host *hosts.Host, serviceOptions v3.Kubern
 		CommandArgs["authentication-token-webhook-cache-ttl"] = c.Authentication.Webhook.CacheTimeout
 	}
 	if len(c.CloudProvider.Name) > 0 {
-		c.Services.KubeAPI.ExtraEnv = append(
+		Env = append(Env, fmt.Sprintf("%s=%s", CloudConfigSumEnv, getStringChecksum(c.CloudConfigFile)))
-			c.Services.KubeAPI.ExtraEnv,
-			fmt.Sprintf("%s=%s", CloudConfigSumEnv, getStringChecksum(c.CloudConfigFile)))
 	}
 	if c.EncryptionConfig.EncryptionProviderFile != "" {
 		CommandArgs[EncryptionProviderConfigArgument] = EncryptionProviderFilePath
@@ -286,16 +286,24 @@ func (c *Cluster) BuildKubeAPIProcess(host *hosts.Host, serviceOptions v3.Kubern
 		fmt.Sprintf("%s:/etc/kubernetes:z", path.Join(host.PrefixPath, "/etc/kubernetes")),
 	}
 
+	if _, ok := c.Services.KubeAPI.ExtraArgs[KubeAPIArgAdmissionControlConfigFile]; !ok {
+		admissionConfig, err := c.getConsolidatedAdmissionConfiguration()
+		if err != nil {
+			logrus.Warnf("Error while getting consolidated admission configuration: %v", err)
+		}
+		bytes, err := yaml.Marshal(admissionConfig)
+		if err != nil {
+			logrus.Warnf("Error while marshalling admission configuration: %v", err)
+		}
+		Env = append(Env, fmt.Sprintf("%s=%s", AdmissionConfigSumEnv, getStringChecksum(string(bytes))))
+	}
 	if c.Services.KubeAPI.AuditLog != nil && c.Services.KubeAPI.AuditLog.Enabled {
 		Binds = append(Binds, fmt.Sprintf("%s:/var/log/kube-audit", path.Join(host.PrefixPath, "/var/log/kube-audit")))
 		bytes, err := yaml.Marshal(c.Services.KubeAPI.AuditLog.Configuration.Policy)
 		if err != nil {
 			logrus.Warnf("Error while marshalling auditlog policy: %v", err)
 		}
-
+		Env = append(Env, fmt.Sprintf("%s=%s", AuditLogConfigSumEnv, getStringChecksum(string(bytes))))
-		c.Services.KubeAPI.ExtraEnv = append(
-			c.Services.KubeAPI.ExtraEnv,
-			fmt.Sprintf("%s=%s", AuditLogConfigSumEnv, getStringChecksum(string(bytes))))
 	}
 
 	matchedRange, err := util.SemVerMatchRange(c.Version, util.SemVerK8sVersion122OrHigher)
@@ -328,12 +336,14 @@ func (c *Cluster) BuildKubeAPIProcess(host *hosts.Host, serviceOptions v3.Kubern
 	}
 	registryAuthConfig, _, _ := docker.GetImageRegistryConfig(c.Services.KubeAPI.Image, c.PrivateRegistriesMap)
 
+	Env = append(Env, c.Services.KubeAPI.ExtraEnv...)
+
 	return v3.Process{
 		Name:                    services.KubeAPIContainerName,
 		Command:                 Command,
 		VolumesFrom:             VolumesFrom,
 		Binds:                   getUniqStringList(Binds),
-		Env:                     getUniqStringList(c.Services.KubeAPI.ExtraEnv),
+		Env:                     getUniqStringList(Env),
 		NetworkMode:             "host",
 		RestartPolicy:           "always",
 		Image:                   c.Services.KubeAPI.Image,

cluster/reconcile.go

@@ -3,7 +3,6 @@ package cluster
 import (
 	"context"
 	"fmt"
-	"reflect"
 	"time"
 
 	"github.com/rancher/rke/docker"
@@ -489,18 +488,3 @@ func getTaintKey(taint v3.RKETaint) string {
 func getTaintValue(taint v3.RKETaint) string {
 	return fmt.Sprintf("%s=%s:%s", taint.Key, taint.Value, taint.Effect)
 }
-
-// RestartKubeAPIServerWhenConfigChanges restarts the kube-apiserver container on the control plane nodes
-// when changes are detected on the to-be-applied kube-api configuration. This is needed to handle the case
-// where changes happen on the generated admission-control-config-file but not on the kube-apiserver container
-func RestartKubeAPIServerWhenConfigChanges(ctx context.Context, kubeCluster, currentCluster *Cluster) error {
-	if currentCluster == nil {
-		return nil
-	}
-	if !reflect.DeepEqual(currentCluster.Services.KubeAPI, kubeCluster.Services.KubeAPI) {
-		for _, host := range kubeCluster.ControlPlaneHosts {
-			return services.RestartKubeAPI(ctx, host)
-		}
-	}
-	return nil
-}

cmd/up.go

@@ -200,10 +200,6 @@ func ClusterUp(ctx context.Context, dialersOptions hosts.DialersOptions, flags c
 		return APIURL, caCrt, clientCert, clientKey, nil, err
 	}
 
-	if err := cluster.RestartKubeAPIServerWhenConfigChanges(ctx, kubeCluster, currentCluster); err != nil {
-		return APIURL, caCrt, clientCert, clientKey, nil, err
-	}
-
 	if err := kubeCluster.PrePullK8sImages(ctx); err != nil {
 		return APIURL, caCrt, clientCert, clientKey, nil, err
 	}