Ability to disable custom encryption

cluster/cluster.go

@@ -73,26 +73,22 @@ type encryptionConfig struct {
 }
 
 const (
-	AuthnX509Provider       = "x509"
+	AuthnX509Provider      = "x509"
-	AuthnWebhookProvider    = "webhook"
+	AuthnWebhookProvider   = "webhook"
-	StateConfigMapName      = "cluster-state"
+	StateConfigMapName     = "cluster-state"
-	FullStateConfigMapName  = "full-cluster-state"
+	FullStateConfigMapName = "full-cluster-state"
-	UpdateStateTimeout      = 30
+	UpdateStateTimeout     = 30
-	GetStateTimeout         = 30
+	GetStateTimeout        = 30
-	KubernetesClientTimeOut = 30
+	SyncWorkers            = 10
-	SyncWorkers             = 10
+	NoneAuthorizationMode  = "none"
-	NoneAuthorizationMode   = "none"
+	LocalNodeAddress       = "127.0.0.1"
-	LocalNodeAddress        = "127.0.0.1"
+	LocalNodeHostname      = "localhost"
-	LocalNodeHostname       = "localhost"
+	LocalNodeUser          = "root"
-	LocalNodeUser           = "root"
+	CloudProvider          = "CloudProvider"
-	CloudProvider           = "CloudProvider"
+	ControlPlane           = "controlPlane"
-	ControlPlane            = "controlPlane"
+	KubeAppLabel           = "k8s-app"
-	WorkerPlane             = "workerPlan"
+	AppLabel               = "app"
-	EtcdPlane               = "etcd"
+	NameLabel              = "name"
-
-	KubeAppLabel = "k8s-app"
-	AppLabel     = "app"
-	NameLabel    = "name"
 
 	WorkerThreads = util.WorkerThreads
 
@@ -261,7 +257,7 @@ func ParseConfig(clusterFile string) (*v3.RancherKubernetesEngineConfig, error)
 	// the customConfig is mapped to a k8s type, which doesn't unmarshal well because it has a
 	// nested struct and no yaml tags. Therefor, we have to re-parse it again and assign it correctly.
 	// this only affects rke cli. Since rkeConfig is passed from rancher directly in the rancher use case.
-	clusterFile, secretConfig, err := resolvCustomEncryptionConfig(clusterFile)
+	clusterFile, secretConfig, err := resolveCustomEncryptionConfig(clusterFile)
 	if err != nil {
 		return nil, err
 	}

cluster/encryption.go

@@ -18,8 +18,8 @@ import (
 	"github.com/rancher/rke/services"
 	"github.com/rancher/rke/templates"
 	"github.com/rancher/rke/util"
-	"github.com/rancher/types/apis/management.cattle.io/v3"
+	v3 "github.com/rancher/types/apis/management.cattle.io/v3"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
@@ -57,25 +57,24 @@ func ReconcileEncryptionProviderConfig(ctx context.Context, kubeCluster, current
 	}
 	// disable encryption
 	if !kubeCluster.IsEncryptionEnabled() && currentCluster.IsEncryptionEnabled() {
-		if currentCluster.IsEncryptionCustomConfig() {
+		return kubeCluster.DisableSecretsEncryption(ctx, currentCluster, currentCluster.IsEncryptionCustomConfig())
-			// KubeAPI will be restarted for the last time during controlplane redeployment, since the
-			// Configuration file is now empty, the Process Plan will change.
-			kubeCluster.EncryptionConfig.EncryptionProviderFile = ""
-			return kubeCluster.DeployEncryptionProviderFile(ctx)
-		}
-		return kubeCluster.DisableSecretsEncryption(ctx, currentCluster)
 	}
 	return nil
 }
 
-func (c *Cluster) DisableSecretsEncryption(ctx context.Context, currentCluster *Cluster) error {
+func (c *Cluster) DisableSecretsEncryption(ctx context.Context, currentCluster *Cluster, custom bool) error {
 	log.Infof(ctx, "[%s] Disabling Secrets Encryption..", services.ControlRole)
 
 	if len(c.ControlPlaneHosts) == 0 {
 		return nil
 	}
 	var err error
-	c.EncryptionConfig.EncryptionProviderFile, err = currentCluster.generateDisabledEncryptionProviderFile()
+	if custom {
+		c.EncryptionConfig.EncryptionProviderFile, err = currentCluster.generateDisabledCustomEncryptionProviderFile()
+	} else {
+		c.EncryptionConfig.EncryptionProviderFile, err = currentCluster.generateDisabledEncryptionProviderFile()
+	}
+
 	if err != nil {
 		return err
 	}
@@ -83,7 +82,8 @@ func (c *Cluster) DisableSecretsEncryption(ctx context.Context, currentCluster *
 	if err := c.DeployEncryptionProviderFile(ctx); err != nil {
 		return err
 	}
-	if err := services.RestartKubeAPIWithHealthcheck(ctx, c.ControlPlaneHosts, c.LocalConnDialerFactory, c.Certificates); err != nil {
+	if err := services.RestartKubeAPIWithHealthcheck(ctx, c.ControlPlaneHosts, c.LocalConnDialerFactory,
+		c.Certificates); err != nil {
 		return err
 	}
 	if err := c.RewriteSecrets(ctx); err != nil {
@@ -104,7 +104,7 @@ func (c *Cluster) RewriteSecrets(ctx context.Context) error {
 	var errgrp errgroup.Group
 	k8sClient, err := k8s.NewClient(c.LocalKubeConfigPath, c.K8sWrapTransport)
 	if err != nil {
-		return fmt.Errorf("Failed to initialize new kubernetes client: %v", err)
+		return fmt.Errorf("failed to initialize new kubernetes client: %v", err)
 	}
 	secretsList, err := k8s.GetSecretsList(k8sClient, "")
 	if err != nil {
@@ -246,6 +246,39 @@ func (c *Cluster) extractActiveKey(s string) (*encryptionKey, error) {
 	}, nil
 }
 
+func (c *Cluster) generateDisabledCustomEncryptionProviderFile() (string, error) {
+	config := apiserverconfigv1.EncryptionConfiguration{}
+	if err := k8s.DecodeYamlResource(&config, c.EncryptionConfig.EncryptionProviderFile); err != nil {
+		return "", err
+	}
+
+	// 1. Prepend custom config providers with ignore provider
+	updatedProviders := []apiserverconfigv1.ProviderConfiguration{{
+		Identity: &apiserverconfigv1.IdentityConfiguration{},
+	}}
+
+	for _, provider := range config.Resources[0].Providers {
+		if provider.Identity != nil {
+			continue
+		}
+		updatedProviders = append(updatedProviders, provider)
+	}
+
+	config.Resources[0].Providers = updatedProviders
+
+	// 2. Generate custom config file
+	jsonConfig, err := json.Marshal(config)
+	if err != nil {
+		return "", err
+	}
+	yamlConfig, err := sigsyaml.JSONToYAML(jsonConfig)
+	if err != nil {
+		return "", nil
+	}
+
+	return string(yamlConfig), nil
+}
+
 func (c *Cluster) generateDisabledEncryptionProviderFile() (string, error) {
 	key, err := c.extractActiveKey(c.EncryptionConfig.EncryptionProviderFile)
 	if err != nil {
@@ -327,7 +360,7 @@ func (c *Cluster) readEncryptionCustomConfig() (string, error) {
 		struct{ CustomConfig string }{CustomConfig: string(yamlConfig)})
 }
 
-func resolvCustomEncryptionConfig(clusterFile string) (string, *apiserverconfig.EncryptionConfiguration, error) {
+func resolveCustomEncryptionConfig(clusterFile string) (string, *apiserverconfig.EncryptionConfiguration, error) {
 	var err error
 	var r map[string]interface{}
 	err = ghodssyaml.Unmarshal([]byte(clusterFile), &r)