diff --git a/cluster/cluster.go b/cluster/cluster.go index 82158e3c..4b88a1df 100644 --- a/cluster/cluster.go +++ b/cluster/cluster.go @@ -73,26 +73,22 @@ type encryptionConfig struct { } const ( - AuthnX509Provider = "x509" - AuthnWebhookProvider = "webhook" - StateConfigMapName = "cluster-state" - FullStateConfigMapName = "full-cluster-state" - UpdateStateTimeout = 30 - GetStateTimeout = 30 - KubernetesClientTimeOut = 30 - SyncWorkers = 10 - NoneAuthorizationMode = "none" - LocalNodeAddress = "127.0.0.1" - LocalNodeHostname = "localhost" - LocalNodeUser = "root" - CloudProvider = "CloudProvider" - ControlPlane = "controlPlane" - WorkerPlane = "workerPlan" - EtcdPlane = "etcd" - - KubeAppLabel = "k8s-app" - AppLabel = "app" - NameLabel = "name" + AuthnX509Provider = "x509" + AuthnWebhookProvider = "webhook" + StateConfigMapName = "cluster-state" + FullStateConfigMapName = "full-cluster-state" + UpdateStateTimeout = 30 + GetStateTimeout = 30 + SyncWorkers = 10 + NoneAuthorizationMode = "none" + LocalNodeAddress = "127.0.0.1" + LocalNodeHostname = "localhost" + LocalNodeUser = "root" + CloudProvider = "CloudProvider" + ControlPlane = "controlPlane" + KubeAppLabel = "k8s-app" + AppLabel = "app" + NameLabel = "name" WorkerThreads = util.WorkerThreads @@ -261,7 +257,7 @@ func ParseConfig(clusterFile string) (*v3.RancherKubernetesEngineConfig, error) // the customConfig is mapped to a k8s type, which doesn't unmarshal well because it has a // nested struct and no yaml tags. Therefor, we have to re-parse it again and assign it correctly. // this only affects rke cli. Since rkeConfig is passed from rancher directly in the rancher use case. - clusterFile, secretConfig, err := resolvCustomEncryptionConfig(clusterFile) + clusterFile, secretConfig, err := resolveCustomEncryptionConfig(clusterFile) if err != nil { return nil, err } diff --git a/cluster/encryption.go b/cluster/encryption.go index 2aa82e4e..041fd833 100644 --- a/cluster/encryption.go +++ b/cluster/encryption.go @@ -18,8 +18,8 @@ import ( "github.com/rancher/rke/services" "github.com/rancher/rke/templates" "github.com/rancher/rke/util" - "github.com/rancher/types/apis/management.cattle.io/v3" - "k8s.io/api/core/v1" + v3 "github.com/rancher/types/apis/management.cattle.io/v3" + v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" @@ -57,25 +57,24 @@ func ReconcileEncryptionProviderConfig(ctx context.Context, kubeCluster, current } // disable encryption if !kubeCluster.IsEncryptionEnabled() && currentCluster.IsEncryptionEnabled() { - if currentCluster.IsEncryptionCustomConfig() { - // KubeAPI will be restarted for the last time during controlplane redeployment, since the - // Configuration file is now empty, the Process Plan will change. - kubeCluster.EncryptionConfig.EncryptionProviderFile = "" - return kubeCluster.DeployEncryptionProviderFile(ctx) - } - return kubeCluster.DisableSecretsEncryption(ctx, currentCluster) + return kubeCluster.DisableSecretsEncryption(ctx, currentCluster, currentCluster.IsEncryptionCustomConfig()) } return nil } -func (c *Cluster) DisableSecretsEncryption(ctx context.Context, currentCluster *Cluster) error { +func (c *Cluster) DisableSecretsEncryption(ctx context.Context, currentCluster *Cluster, custom bool) error { log.Infof(ctx, "[%s] Disabling Secrets Encryption..", services.ControlRole) if len(c.ControlPlaneHosts) == 0 { return nil } var err error - c.EncryptionConfig.EncryptionProviderFile, err = currentCluster.generateDisabledEncryptionProviderFile() + if custom { + c.EncryptionConfig.EncryptionProviderFile, err = currentCluster.generateDisabledCustomEncryptionProviderFile() + } else { + c.EncryptionConfig.EncryptionProviderFile, err = currentCluster.generateDisabledEncryptionProviderFile() + } + if err != nil { return err } @@ -83,7 +82,8 @@ func (c *Cluster) DisableSecretsEncryption(ctx context.Context, currentCluster * if err := c.DeployEncryptionProviderFile(ctx); err != nil { return err } - if err := services.RestartKubeAPIWithHealthcheck(ctx, c.ControlPlaneHosts, c.LocalConnDialerFactory, c.Certificates); err != nil { + if err := services.RestartKubeAPIWithHealthcheck(ctx, c.ControlPlaneHosts, c.LocalConnDialerFactory, + c.Certificates); err != nil { return err } if err := c.RewriteSecrets(ctx); err != nil { @@ -104,7 +104,7 @@ func (c *Cluster) RewriteSecrets(ctx context.Context) error { var errgrp errgroup.Group k8sClient, err := k8s.NewClient(c.LocalKubeConfigPath, c.K8sWrapTransport) if err != nil { - return fmt.Errorf("Failed to initialize new kubernetes client: %v", err) + return fmt.Errorf("failed to initialize new kubernetes client: %v", err) } secretsList, err := k8s.GetSecretsList(k8sClient, "") if err != nil { @@ -246,6 +246,39 @@ func (c *Cluster) extractActiveKey(s string) (*encryptionKey, error) { }, nil } +func (c *Cluster) generateDisabledCustomEncryptionProviderFile() (string, error) { + config := apiserverconfigv1.EncryptionConfiguration{} + if err := k8s.DecodeYamlResource(&config, c.EncryptionConfig.EncryptionProviderFile); err != nil { + return "", err + } + + // 1. Prepend custom config providers with ignore provider + updatedProviders := []apiserverconfigv1.ProviderConfiguration{{ + Identity: &apiserverconfigv1.IdentityConfiguration{}, + }} + + for _, provider := range config.Resources[0].Providers { + if provider.Identity != nil { + continue + } + updatedProviders = append(updatedProviders, provider) + } + + config.Resources[0].Providers = updatedProviders + + // 2. Generate custom config file + jsonConfig, err := json.Marshal(config) + if err != nil { + return "", err + } + yamlConfig, err := sigsyaml.JSONToYAML(jsonConfig) + if err != nil { + return "", nil + } + + return string(yamlConfig), nil +} + func (c *Cluster) generateDisabledEncryptionProviderFile() (string, error) { key, err := c.extractActiveKey(c.EncryptionConfig.EncryptionProviderFile) if err != nil { @@ -327,7 +360,7 @@ func (c *Cluster) readEncryptionCustomConfig() (string, error) { struct{ CustomConfig string }{CustomConfig: string(yamlConfig)}) } -func resolvCustomEncryptionConfig(clusterFile string) (string, *apiserverconfig.EncryptionConfiguration, error) { +func resolveCustomEncryptionConfig(clusterFile string) (string, *apiserverconfig.EncryptionConfiguration, error) { var err error var r map[string]interface{} err = ghodssyaml.Unmarshal([]byte(clusterFile), &r)