Revert "Merge pull request #3531 from jiaqiluo/fix-kube-apiserver"

This reverts commit 5e050c58a9, reversing
changes made to 1951add95d.

cluster/plan.go

@@ -40,11 +40,10 @@ const (
 
 	NetworkConfigurationEnv = "RKE_NETWORK_CONFIGURATION"
 
-	EtcdPathPrefix        = "/registry"
+	EtcdPathPrefix       = "/registry"
-	CloudConfigSumEnv     = "RKE_CLOUD_CONFIG_CHECKSUM"
+	CloudConfigSumEnv    = "RKE_CLOUD_CONFIG_CHECKSUM"
-	CloudProviderNameEnv  = "RKE_CLOUD_PROVIDER_NAME"
+	CloudProviderNameEnv = "RKE_CLOUD_PROVIDER_NAME"
-	AuditLogConfigSumEnv  = "RKE_AUDITLOG_CONFIG_CHECKSUM"
+	AuditLogConfigSumEnv = "RKE_AUDITLOG_CONFIG_CHECKSUM"
-	AdmissionConfigSumEnv = "RKE_ADMISSION_CONFIG_CHECKSUM"
 
 	DefaultToolsEntrypoint        = "/opt/rke-tools/entrypoint.sh"
 	DefaultToolsEntrypointVersion = "0.1.13"
@@ -203,7 +202,6 @@ func (c *Cluster) BuildKubeAPIProcess(host *hosts.Host, serviceOptions v3.Kubern
 		"tls-private-key-file":          pki.GetKeyPath(pki.KubeAPICertName),
 	}
 	CommandArrayArgs := make(map[string][]string, len(c.Services.KubeAPI.ExtraArgsArray))
-	Env := make([]string, len(c.Services.KubeAPI.ExtraEnv))
 
 	if len(c.CloudProvider.Name) > 0 {
 		CommandArgs["cloud-config"] = cloudConfigFileName
@@ -213,7 +211,9 @@ func (c *Cluster) BuildKubeAPIProcess(host *hosts.Host, serviceOptions v3.Kubern
 		CommandArgs["authentication-token-webhook-cache-ttl"] = c.Authentication.Webhook.CacheTimeout
 	}
 	if len(c.CloudProvider.Name) > 0 {
-		Env = append(Env, fmt.Sprintf("%s=%s", CloudConfigSumEnv, getStringChecksum(c.CloudConfigFile)))
+		c.Services.KubeAPI.ExtraEnv = append(
+			c.Services.KubeAPI.ExtraEnv,
+			fmt.Sprintf("%s=%s", CloudConfigSumEnv, getStringChecksum(c.CloudConfigFile)))
 	}
 	if c.EncryptionConfig.EncryptionProviderFile != "" {
 		CommandArgs[EncryptionProviderConfigArgument] = EncryptionProviderFilePath
@@ -286,24 +286,16 @@ func (c *Cluster) BuildKubeAPIProcess(host *hosts.Host, serviceOptions v3.Kubern
 		fmt.Sprintf("%s:/etc/kubernetes:z", path.Join(host.PrefixPath, "/etc/kubernetes")),
 	}
 
-	if _, ok := c.Services.KubeAPI.ExtraArgs[KubeAPIArgAdmissionControlConfigFile]; !ok {
-		admissionConfig, err := c.getConsolidatedAdmissionConfiguration()
-		if err != nil {
-			logrus.Warnf("Error while getting consolidated admission configuration: %v", err)
-		}
-		bytes, err := yaml.Marshal(admissionConfig)
-		if err != nil {
-			logrus.Warnf("Error while marshalling admission configuration: %v", err)
-		}
-		Env = append(Env, fmt.Sprintf("%s=%s", AdmissionConfigSumEnv, getStringChecksum(string(bytes))))
-	}
 	if c.Services.KubeAPI.AuditLog != nil && c.Services.KubeAPI.AuditLog.Enabled {
 		Binds = append(Binds, fmt.Sprintf("%s:/var/log/kube-audit", path.Join(host.PrefixPath, "/var/log/kube-audit")))
 		bytes, err := yaml.Marshal(c.Services.KubeAPI.AuditLog.Configuration.Policy)
 		if err != nil {
 			logrus.Warnf("Error while marshalling auditlog policy: %v", err)
 		}
-		Env = append(Env, fmt.Sprintf("%s=%s", AuditLogConfigSumEnv, getStringChecksum(string(bytes))))
+
+		c.Services.KubeAPI.ExtraEnv = append(
+			c.Services.KubeAPI.ExtraEnv,
+			fmt.Sprintf("%s=%s", AuditLogConfigSumEnv, getStringChecksum(string(bytes))))
 	}
 
 	matchedRange, err := util.SemVerMatchRange(c.Version, util.SemVerK8sVersion122OrHigher)
@@ -336,14 +328,12 @@ func (c *Cluster) BuildKubeAPIProcess(host *hosts.Host, serviceOptions v3.Kubern
 	}
 	registryAuthConfig, _, _ := docker.GetImageRegistryConfig(c.Services.KubeAPI.Image, c.PrivateRegistriesMap)
 
-	Env = append(Env, c.Services.KubeAPI.ExtraEnv...)
-
 	return v3.Process{
 		Name:                    services.KubeAPIContainerName,
 		Command:                 Command,
 		VolumesFrom:             VolumesFrom,
 		Binds:                   getUniqStringList(Binds),
-		Env:                     getUniqStringList(Env),
+		Env:                     getUniqStringList(c.Services.KubeAPI.ExtraEnv),
 		NetworkMode:             "host",
 		RestartPolicy:           "always",
 		Image:                   c.Services.KubeAPI.Image,

cluster/reconcile.go

@@ -3,6 +3,7 @@ package cluster
 import (
 	"context"
 	"fmt"
+	"reflect"
 	"time"
 
 	"github.com/rancher/rke/docker"
@@ -488,3 +489,18 @@ func getTaintKey(taint v3.RKETaint) string {
 func getTaintValue(taint v3.RKETaint) string {
 	return fmt.Sprintf("%s=%s:%s", taint.Key, taint.Value, taint.Effect)
 }
+
+// RestartKubeAPIServerWhenConfigChanges restarts the kube-apiserver container on the control plane nodes
+// when changes are detected on the to-be-applied kube-api configuration. This is needed to handle the case
+// where changes happen on the generated admission-control-config-file but not on the kube-apiserver container
+func RestartKubeAPIServerWhenConfigChanges(ctx context.Context, kubeCluster, currentCluster *Cluster) error {
+	if currentCluster == nil {
+		return nil
+	}
+	if !reflect.DeepEqual(currentCluster.Services.KubeAPI, kubeCluster.Services.KubeAPI) {
+		for _, host := range kubeCluster.ControlPlaneHosts {
+			return services.RestartKubeAPI(ctx, host)
+		}
+	}
+	return nil
+}

cmd/up.go

@@ -200,6 +200,10 @@ func ClusterUp(ctx context.Context, dialersOptions hosts.DialersOptions, flags c
 		return APIURL, caCrt, clientCert, clientKey, nil, err
 	}
 
+	if err := cluster.RestartKubeAPIServerWhenConfigChanges(ctx, kubeCluster, currentCluster); err != nil {
+		return APIURL, caCrt, clientCert, clientKey, nil, err
+	}
+
 	if err := kubeCluster.PrePullK8sImages(ctx); err != nil {
 		return APIURL, caCrt, clientCert, clientKey, nil, err
 	}