Add support for Kubernetes API Authn Webhook

Allow multiple authn strategies to be defined, including new 'webhook'
strategy. Webhook strategy configuration contains the contents of the
authentication webhook file as well as the cache timeout period.

This change allows a Kubernetes API Auth service to authenticate
user requests without proxying through the Rancher server.

cluster/certificates.go

@@ -16,7 +16,7 @@ import (
 )
 
 func SetUpAuthentication(ctx context.Context, kubeCluster, currentCluster *Cluster, fullState *FullState) error {
-	if kubeCluster.Authentication.Strategy == X509AuthenticationProvider {
+	if kubeCluster.AuthnStrategies[AuthnX509Provider] {
 		kubeCluster.Certificates = fullState.DesiredState.CertificatesBundle
 		return nil
 	}

cluster/cloud-provider.go

@@ -1,62 +0,0 @@
-package cluster
-
-import (
-	"context"
-	"fmt"
-	"path"
-
-	"github.com/docker/docker/api/types/container"
-	"github.com/rancher/rke/docker"
-	"github.com/rancher/rke/hosts"
-	"github.com/rancher/rke/log"
-	"github.com/rancher/types/apis/management.cattle.io/v3"
-	"github.com/sirupsen/logrus"
-)
-
-const (
-	CloudConfigDeployer    = "cloud-config-deployer"
-	CloudConfigServiceName = "cloud"
-	CloudConfigPath        = "/etc/kubernetes/cloud-config"
-	CloudConfigEnv         = "RKE_CLOUD_CONFIG"
-)
-
-func deployCloudProviderConfig(ctx context.Context, uniqueHosts []*hosts.Host, alpineImage string, prsMap map[string]v3.PrivateRegistry, cloudConfig string) error {
-	for _, host := range uniqueHosts {
-		log.Infof(ctx, "[%s] Deploying cloud config file to node [%s]", CloudConfigServiceName, host.Address)
-		if err := doDeployConfigFile(ctx, host, cloudConfig, alpineImage, prsMap); err != nil {
-			return fmt.Errorf("Failed to deploy cloud config file on node [%s]: %v", host.Address, err)
-		}
-	}
-	return nil
-}
-
-func doDeployConfigFile(ctx context.Context, host *hosts.Host, cloudConfig, alpineImage string, prsMap map[string]v3.PrivateRegistry) error {
-	// remove existing container. Only way it's still here is if previous deployment failed
-	if err := docker.DoRemoveContainer(ctx, host.DClient, CloudConfigDeployer, host.Address); err != nil {
-		return err
-	}
-	containerEnv := []string{CloudConfigEnv + "=" + cloudConfig}
-	imageCfg := &container.Config{
-		Image: alpineImage,
-		Cmd: []string{
-			"sh",
-			"-c",
-			fmt.Sprintf("t=$(mktemp); echo -e \"$%s\" > $t && mv $t %s && chmod 644 %s", CloudConfigEnv, CloudConfigPath, CloudConfigPath),
-		},
-		Env: containerEnv,
-	}
-	hostCfg := &container.HostConfig{
-		Binds: []string{
-			fmt.Sprintf("%s:/etc/kubernetes:z", path.Join(host.PrefixPath, "/etc/kubernetes")),
-		},
-		Privileged: true,
-	}
-	if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, CloudConfigDeployer, host.Address, CloudConfigServiceName, prsMap); err != nil {
-		return err
-	}
-	if err := docker.DoRemoveContainer(ctx, host.DClient, CloudConfigDeployer, host.Address); err != nil {
-		return err
-	}
-	logrus.Debugf("[%s] Successfully started cloud config deployer container on node [%s]", CloudConfigServiceName, host.Address)
-	return nil
-}

cluster/cluster.go

@@ -28,6 +28,7 @@ import (
 )
 
 type Cluster struct {
+	AuthnStrategies                  map[string]bool
 	ConfigPath                       string
 	ConfigDir                        string
 	CloudConfigFile                  string
@@ -54,21 +55,22 @@ type Cluster struct {
 }
 
 const (
-	X509AuthenticationProvider = "x509"
-	StateConfigMapName         = "cluster-state"
-	FullStateConfigMapName     = "full-cluster-state"
-	UpdateStateTimeout         = 30
-	GetStateTimeout            = 30
-	KubernetesClientTimeOut    = 30
-	SyncWorkers                = 10
-	NoneAuthorizationMode      = "none"
-	LocalNodeAddress           = "127.0.0.1"
-	LocalNodeHostname          = "localhost"
-	LocalNodeUser              = "root"
-	CloudProvider              = "CloudProvider"
-	ControlPlane               = "controlPlane"
-	WorkerPlane                = "workerPlan"
-	EtcdPlane                  = "etcd"
+	AuthnX509Provider       = "x509"
+	AuthnWebhookProvider    = "webhook"
+	StateConfigMapName      = "cluster-state"
+	FullStateConfigMapName  = "full-cluster-state"
+	UpdateStateTimeout      = 30
+	GetStateTimeout         = 30
+	KubernetesClientTimeOut = 30
+	SyncWorkers             = 10
+	NoneAuthorizationMode   = "none"
+	LocalNodeAddress        = "127.0.0.1"
+	LocalNodeHostname       = "localhost"
+	LocalNodeUser           = "root"
+	CloudProvider           = "CloudProvider"
+	ControlPlane            = "controlPlane"
+	WorkerPlane             = "workerPlan"
+	EtcdPlane               = "etcd"
 
 	KubeAppLabel = "k8s-app"
 	AppLabel     = "app"
@@ -149,6 +151,7 @@ func ParseConfig(clusterFile string) (*v3.RancherKubernetesEngineConfig, error)
 func InitClusterObject(ctx context.Context, rkeConfig *v3.RancherKubernetesEngineConfig, flags ExternalFlags) (*Cluster, error) {
 	// basic cluster object from rkeConfig
 	c := &Cluster{
+		AuthnStrategies:               make(map[string]bool),
 		RancherKubernetesEngineConfig: *rkeConfig,
 		ConfigPath:                    flags.ClusterFilePath,
 		ConfigDir:                     flags.ConfigDir,
@@ -158,6 +161,7 @@ func InitClusterObject(ctx context.Context, rkeConfig *v3.RancherKubernetesEngin
 	if len(c.ConfigPath) == 0 {
 		c.ConfigPath = pki.ClusterConfig
 	}
+
 	// set kube_config and state file
 	c.LocalKubeConfigPath = pki.GetLocalKubeConfig(c.ConfigPath, c.ConfigDir)
 	c.StateFilePath = GetStateFilePath(c.ConfigPath, c.ConfigDir)
@@ -166,6 +170,7 @@ func InitClusterObject(ctx context.Context, rkeConfig *v3.RancherKubernetesEngin
 	c.setClusterDefaults(ctx)
 	// extract cluster network configuration
 	c.setNetworkOptions()
+
 	// Register cloud provider
 	if err := c.setCloudProvider(); err != nil {
 		return nil, fmt.Errorf("Failed to register cloud provider: %v", err)

cluster/defaults.go

@@ -10,6 +10,7 @@ import (
 	"github.com/rancher/rke/k8s"
 	"github.com/rancher/rke/log"
 	"github.com/rancher/rke/services"
+	"github.com/rancher/rke/templates"
 	"github.com/rancher/types/apis/management.cattle.io/v3"
 )
 
@@ -30,6 +31,9 @@ const (
 	DefaultAuthStrategy      = "x509"
 	DefaultAuthorizationMode = "rbac"
 
+	DefaultAuthnWebhookFile  = templates.AuthnWebhook
+	DefaultAuthnCacheTimeout = "5s"
+
 	DefaultNetworkPlugin        = "canal"
 	DefaultNetworkCloudProvider = "none"
 
@@ -137,6 +141,7 @@ func (c *Cluster) setClusterDefaults(ctx context.Context) {
 	c.setClusterImageDefaults()
 	c.setClusterServicesDefaults()
 	c.setClusterNetworkDefaults()
+	c.setClusterAuthnDefaults()
 }
 
 func (c *Cluster) setClusterServicesDefaults() {
@@ -162,7 +167,6 @@ func (c *Cluster) setClusterServicesDefaults() {
 		&c.Services.Kubelet.ClusterDNSServer:             DefaultClusterDNSService,
 		&c.Services.Kubelet.ClusterDomain:                DefaultClusterDomain,
 		&c.Services.Kubelet.InfraContainerImage:          c.SystemImages.PodInfraContainer,
-		&c.Authentication.Strategy:                       DefaultAuthStrategy,
 		&c.Services.Etcd.Creation:                        DefaultEtcdBackupCreationPeriod,
 		&c.Services.Etcd.Retention:                       DefaultEtcdBackupRetentionPeriod,
 	}
@@ -268,6 +272,28 @@ func (c *Cluster) setClusterNetworkDefaults() {
 	}
 }
 
+func (c *Cluster) setClusterAuthnDefaults() {
+	setDefaultIfEmpty(&c.Authentication.Strategy, DefaultAuthStrategy)
+
+	for _, strategy := range strings.Split(c.Authentication.Strategy, "|") {
+		strategy = strings.ToLower(strings.TrimSpace(strategy))
+		c.AuthnStrategies[strategy] = true
+	}
+
+	if c.AuthnStrategies[AuthnWebhookProvider] && c.Authentication.Webhook == nil {
+		c.Authentication.Webhook = &v3.AuthWebhookConfig{}
+	}
+	if c.Authentication.Webhook != nil {
+		webhookConfigDefaultsMap := map[*string]string{
+			&c.Authentication.Webhook.ConfigFile:   DefaultAuthnWebhookFile,
+			&c.Authentication.Webhook.CacheTimeout: DefaultAuthnCacheTimeout,
+		}
+		for k, v := range webhookConfigDefaultsMap {
+			setDefaultIfEmpty(k, v)
+		}
+	}
+}
+
 func d(image, defaultRegistryURL string) string {
 	if len(defaultRegistryURL) == 0 {
 		return image

cluster/file-deployer.go

@@ -0,0 +1,60 @@
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"path"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/rancher/rke/docker"
+	"github.com/rancher/rke/hosts"
+	"github.com/rancher/rke/log"
+	"github.com/rancher/types/apis/management.cattle.io/v3"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	ContainerName = "file-deployer"
+	ServiceName   = "file-deploy"
+	ConfigEnv     = "FILE_DEPLOY"
+)
+
+func deployFile(ctx context.Context, uniqueHosts []*hosts.Host, alpineImage string, prsMap map[string]v3.PrivateRegistry, fileName, fileContents string) error {
+	for _, host := range uniqueHosts {
+		log.Infof(ctx, "[%s] Deploying file '%s' to node [%s]", ServiceName, fileName, host.Address)
+		if err := doDeployFile(ctx, host, fileName, fileContents, alpineImage, prsMap); err != nil {
+			return fmt.Errorf("Failed to deploy file '%s' on node [%s]: %v", host.Address, fileName, err)
+		}
+	}
+	return nil
+}
+
+func doDeployFile(ctx context.Context, host *hosts.Host, fileName, fileContents, alpineImage string, prsMap map[string]v3.PrivateRegistry) error {
+	// remove existing container. Only way it's still here is if previous deployment failed
+	if err := docker.DoRemoveContainer(ctx, host.DClient, ContainerName, host.Address); err != nil {
+		return err
+	}
+	containerEnv := []string{ConfigEnv + "=" + fileContents}
+	imageCfg := &container.Config{
+		Image: alpineImage,
+		Cmd: []string{
+			"sh",
+			"-c",
+			fmt.Sprintf("t=$(mktemp); echo -e \"$%s\" > $t && mv $t %s && chmod 644 %s", ConfigEnv, fileName, fileName),
+		},
+		Env: containerEnv,
+	}
+	hostCfg := &container.HostConfig{
+		Binds: []string{
+			fmt.Sprintf("%s:/etc/kubernetes:z", path.Join(host.PrefixPath, "/etc/kubernetes")),
+		},
+	}
+	if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, ContainerName, host.Address, ServiceName, prsMap); err != nil {
+		return err
+	}
+	if err := docker.DoRemoveContainer(ctx, host.DClient, ContainerName, host.Address); err != nil {
+		return err
+	}
+	logrus.Debugf("[%s] Successfully deployed file '%s' on node [%s]", ServiceName, fileName, host.Address)
+	return nil
+}

cluster/hosts.go

@@ -21,6 +21,8 @@ const (
 	etcdRoleLabel         = "node-role.kubernetes.io/etcd"
 	controlplaneRoleLabel = "node-role.kubernetes.io/controlplane"
 	workerRoleLabel       = "node-role.kubernetes.io/worker"
+	cloudConfigFileName   = "/etc/kubernetes/cloud-config"
+	authnWebhookFileName  = "/etc/kubernetes/kube-api-authn-webhook.yaml"
 )
 
 func (c *Cluster) TunnelHosts(ctx context.Context, flags ExternalFlags) error {
@@ -117,7 +119,7 @@ func (c *Cluster) InvertIndexHosts() error {
 }
 
 func (c *Cluster) SetUpHosts(ctx context.Context, rotateCerts bool) error {
-	if c.Authentication.Strategy == X509AuthenticationProvider {
+	if c.AuthnStrategies[AuthnX509Provider] {
 		log.Infof(ctx, "[certificates] Deploying kubernetes certificates to Cluster nodes")
 		hostList := hosts.GetUniqueHostList(c.EtcdHosts, c.ControlPlaneHosts, c.WorkerHosts)
 		var errgrp errgroup.Group
@@ -144,10 +146,17 @@ func (c *Cluster) SetUpHosts(ctx context.Context, rotateCerts bool) error {
 		}
 		log.Infof(ctx, "[certificates] Successfully deployed kubernetes certificates to Cluster nodes")
 		if c.CloudProvider.Name != "" {
-			if err := deployCloudProviderConfig(ctx, hostList, c.SystemImages.Alpine, c.PrivateRegistriesMap, c.CloudConfigFile); err != nil {
+			if err := deployFile(ctx, hostList, c.SystemImages.Alpine, c.PrivateRegistriesMap, cloudConfigFileName, c.CloudConfigFile); err != nil {
 				return err
 			}
-			log.Infof(ctx, "[%s] Successfully deployed kubernetes cloud config to Cluster nodes", CloudConfigServiceName)
+			log.Infof(ctx, "[%s] Successfully deployed kubernetes cloud config to Cluster nodes", cloudConfigFileName)
+		}
+
+		if c.Authentication.Webhook != nil {
+			if err := deployFile(ctx, hostList, c.SystemImages.Alpine, c.PrivateRegistriesMap, authnWebhookFileName, c.Authentication.Webhook.ConfigFile); err != nil {
+				return err
+			}
+			log.Infof(ctx, "[%s] Successfully deployed authentication webhook config Cluster nodes", cloudConfigFileName)
 		}
 	}
 	return nil

cluster/plan.go

@@ -81,7 +81,7 @@ func BuildRKEConfigNodePlan(ctx context.Context, myCluster *Cluster, host *hosts
 		portChecks = append(portChecks, BuildPortChecksFromPortList(host, EtcdPortList, ProtocolTCP)...)
 	}
 	cloudConfig := v3.File{
-		Name:     CloudConfigPath,
+		Name:     cloudConfigFileName,
 		Contents: b64.StdEncoding.EncodeToString([]byte(myCluster.CloudConfigFile)),
 	}
 	return v3.RKEConfigNodePlan{
@@ -149,7 +149,11 @@ func (c *Cluster) BuildKubeAPIProcess(prefixPath string) v3.Process {
 		"requestheader-username-headers":     "X-Remote-User",
 	}
 	if len(c.CloudProvider.Name) > 0 && c.CloudProvider.Name != aws.AWSCloudProviderName {
-		CommandArgs["cloud-config"] = CloudConfigPath
+		CommandArgs["cloud-config"] = cloudConfigFileName
+	}
+	if c.Authentication.Webhook != nil {
+		CommandArgs["authentication-token-webhook-config-file"] = authnWebhookFileName
+		CommandArgs["authentication-token-webhook-cache-ttl"] = c.Authentication.Webhook.CacheTimeout
 	}
 	if len(c.CloudProvider.Name) > 0 {
 		c.Services.KubeAPI.ExtraEnv = append(
@@ -253,7 +257,7 @@ func (c *Cluster) BuildKubeControllerProcess(prefixPath string) v3.Process {
 		"root-ca-file":                     pki.GetCertPath(pki.CACertName),
 	}
 	if len(c.CloudProvider.Name) > 0 && c.CloudProvider.Name != aws.AWSCloudProviderName {
-		CommandArgs["cloud-config"] = CloudConfigPath
+		CommandArgs["cloud-config"] = cloudConfigFileName
 	}
 	if len(c.CloudProvider.Name) > 0 {
 		c.Services.KubeController.ExtraEnv = append(
@@ -359,7 +363,7 @@ func (c *Cluster) BuildKubeletProcess(host *hosts.Host, prefixPath string) v3.Pr
 		CommandArgs["node-ip"] = host.InternalAddress
 	}
 	if len(c.CloudProvider.Name) > 0 && c.CloudProvider.Name != aws.AWSCloudProviderName {
-		CommandArgs["cloud-config"] = CloudConfigPath
+		CommandArgs["cloud-config"] = cloudConfigFileName
 	}
 	if len(c.CloudProvider.Name) > 0 {
 		c.Services.Kubelet.ExtraEnv = append(

cluster/validation.go

@@ -39,8 +39,17 @@ func (c *Cluster) ValidateCluster() error {
 }
 
 func validateAuthOptions(c *Cluster) error {
-	if c.Authentication.Strategy != DefaultAuthStrategy {
-		return fmt.Errorf("Authentication strategy [%s] is not supported", c.Authentication.Strategy)
+	for strategy, enabled := range c.AuthnStrategies {
+		if !enabled {
+			continue
+		}
+		strategy = strings.ToLower(strategy)
+		if strategy != AuthnX509Provider && strategy != AuthnWebhookProvider {
+			return fmt.Errorf("Authentication strategy [%s] is not supported", strategy)
+		}
+	}
+	if !c.AuthnStrategies[AuthnX509Provider] {
+		return fmt.Errorf("Authentication strategy must contain [%s]", AuthnX509Provider)
 	}
 	return nil
 }

templates/authn-webhook.go

@@ -0,0 +1,23 @@
+package templates
+
+const (
+	AuthnWebhook = `
+apiVersion: v1
+kind: Config
+clusters:
+- name: Default
+  cluster:
+    insecure-skip-tls-verify: true
+    server: http://127.0.0.1:6440/v1/authenticate
+users:
+- name: Default
+  user:
+    insecure-skip-tls-verify: true
+current-context: webhook
+contexts:
+- name: webhook
+  context:
+    user: Default
+    cluster: Default
+`
+)