1
0
mirror of https://github.com/rancher/rke.git synced 2025-08-01 23:33:39 +00:00

Add support for Kubernetes API Authn Webhook

Allow multiple authn strategies to be defined, including new 'webhook'
strategy. Webhook strategy configuration contains the contents of the
authentication webhook file as well as the cache timeout period.

This change allows a Kubernetes API Auth service to authenticate
user requests without proxying through the Rancher server.
This commit is contained in:
Erik Wilson 2018-12-28 09:41:37 -07:00 committed by Craig Jellick
parent 30d8c8a30f
commit e04b7d4413
9 changed files with 162 additions and 88 deletions

View File

@ -16,7 +16,7 @@ import (
) )
func SetUpAuthentication(ctx context.Context, kubeCluster, currentCluster *Cluster, fullState *FullState) error { func SetUpAuthentication(ctx context.Context, kubeCluster, currentCluster *Cluster, fullState *FullState) error {
if kubeCluster.Authentication.Strategy == X509AuthenticationProvider { if kubeCluster.AuthnStrategies[AuthnX509Provider] {
kubeCluster.Certificates = fullState.DesiredState.CertificatesBundle kubeCluster.Certificates = fullState.DesiredState.CertificatesBundle
return nil return nil
} }

View File

@ -1,62 +0,0 @@
package cluster
import (
"context"
"fmt"
"path"
"github.com/docker/docker/api/types/container"
"github.com/rancher/rke/docker"
"github.com/rancher/rke/hosts"
"github.com/rancher/rke/log"
"github.com/rancher/types/apis/management.cattle.io/v3"
"github.com/sirupsen/logrus"
)
const (
CloudConfigDeployer = "cloud-config-deployer"
CloudConfigServiceName = "cloud"
CloudConfigPath = "/etc/kubernetes/cloud-config"
CloudConfigEnv = "RKE_CLOUD_CONFIG"
)
func deployCloudProviderConfig(ctx context.Context, uniqueHosts []*hosts.Host, alpineImage string, prsMap map[string]v3.PrivateRegistry, cloudConfig string) error {
for _, host := range uniqueHosts {
log.Infof(ctx, "[%s] Deploying cloud config file to node [%s]", CloudConfigServiceName, host.Address)
if err := doDeployConfigFile(ctx, host, cloudConfig, alpineImage, prsMap); err != nil {
return fmt.Errorf("Failed to deploy cloud config file on node [%s]: %v", host.Address, err)
}
}
return nil
}
func doDeployConfigFile(ctx context.Context, host *hosts.Host, cloudConfig, alpineImage string, prsMap map[string]v3.PrivateRegistry) error {
// remove existing container. Only way it's still here is if previous deployment failed
if err := docker.DoRemoveContainer(ctx, host.DClient, CloudConfigDeployer, host.Address); err != nil {
return err
}
containerEnv := []string{CloudConfigEnv + "=" + cloudConfig}
imageCfg := &container.Config{
Image: alpineImage,
Cmd: []string{
"sh",
"-c",
fmt.Sprintf("t=$(mktemp); echo -e \"$%s\" > $t && mv $t %s && chmod 644 %s", CloudConfigEnv, CloudConfigPath, CloudConfigPath),
},
Env: containerEnv,
}
hostCfg := &container.HostConfig{
Binds: []string{
fmt.Sprintf("%s:/etc/kubernetes:z", path.Join(host.PrefixPath, "/etc/kubernetes")),
},
Privileged: true,
}
if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, CloudConfigDeployer, host.Address, CloudConfigServiceName, prsMap); err != nil {
return err
}
if err := docker.DoRemoveContainer(ctx, host.DClient, CloudConfigDeployer, host.Address); err != nil {
return err
}
logrus.Debugf("[%s] Successfully started cloud config deployer container on node [%s]", CloudConfigServiceName, host.Address)
return nil
}

View File

@ -28,6 +28,7 @@ import (
) )
type Cluster struct { type Cluster struct {
AuthnStrategies map[string]bool
ConfigPath string ConfigPath string
ConfigDir string ConfigDir string
CloudConfigFile string CloudConfigFile string
@ -54,21 +55,22 @@ type Cluster struct {
} }
const ( const (
X509AuthenticationProvider = "x509" AuthnX509Provider = "x509"
StateConfigMapName = "cluster-state" AuthnWebhookProvider = "webhook"
FullStateConfigMapName = "full-cluster-state" StateConfigMapName = "cluster-state"
UpdateStateTimeout = 30 FullStateConfigMapName = "full-cluster-state"
GetStateTimeout = 30 UpdateStateTimeout = 30
KubernetesClientTimeOut = 30 GetStateTimeout = 30
SyncWorkers = 10 KubernetesClientTimeOut = 30
NoneAuthorizationMode = "none" SyncWorkers = 10
LocalNodeAddress = "127.0.0.1" NoneAuthorizationMode = "none"
LocalNodeHostname = "localhost" LocalNodeAddress = "127.0.0.1"
LocalNodeUser = "root" LocalNodeHostname = "localhost"
CloudProvider = "CloudProvider" LocalNodeUser = "root"
ControlPlane = "controlPlane" CloudProvider = "CloudProvider"
WorkerPlane = "workerPlan" ControlPlane = "controlPlane"
EtcdPlane = "etcd" WorkerPlane = "workerPlan"
EtcdPlane = "etcd"
KubeAppLabel = "k8s-app" KubeAppLabel = "k8s-app"
AppLabel = "app" AppLabel = "app"
@ -149,6 +151,7 @@ func ParseConfig(clusterFile string) (*v3.RancherKubernetesEngineConfig, error)
func InitClusterObject(ctx context.Context, rkeConfig *v3.RancherKubernetesEngineConfig, flags ExternalFlags) (*Cluster, error) { func InitClusterObject(ctx context.Context, rkeConfig *v3.RancherKubernetesEngineConfig, flags ExternalFlags) (*Cluster, error) {
// basic cluster object from rkeConfig // basic cluster object from rkeConfig
c := &Cluster{ c := &Cluster{
AuthnStrategies: make(map[string]bool),
RancherKubernetesEngineConfig: *rkeConfig, RancherKubernetesEngineConfig: *rkeConfig,
ConfigPath: flags.ClusterFilePath, ConfigPath: flags.ClusterFilePath,
ConfigDir: flags.ConfigDir, ConfigDir: flags.ConfigDir,
@ -158,6 +161,7 @@ func InitClusterObject(ctx context.Context, rkeConfig *v3.RancherKubernetesEngin
if len(c.ConfigPath) == 0 { if len(c.ConfigPath) == 0 {
c.ConfigPath = pki.ClusterConfig c.ConfigPath = pki.ClusterConfig
} }
// set kube_config and state file // set kube_config and state file
c.LocalKubeConfigPath = pki.GetLocalKubeConfig(c.ConfigPath, c.ConfigDir) c.LocalKubeConfigPath = pki.GetLocalKubeConfig(c.ConfigPath, c.ConfigDir)
c.StateFilePath = GetStateFilePath(c.ConfigPath, c.ConfigDir) c.StateFilePath = GetStateFilePath(c.ConfigPath, c.ConfigDir)
@ -166,6 +170,7 @@ func InitClusterObject(ctx context.Context, rkeConfig *v3.RancherKubernetesEngin
c.setClusterDefaults(ctx) c.setClusterDefaults(ctx)
// extract cluster network configuration // extract cluster network configuration
c.setNetworkOptions() c.setNetworkOptions()
// Register cloud provider // Register cloud provider
if err := c.setCloudProvider(); err != nil { if err := c.setCloudProvider(); err != nil {
return nil, fmt.Errorf("Failed to register cloud provider: %v", err) return nil, fmt.Errorf("Failed to register cloud provider: %v", err)

View File

@ -10,6 +10,7 @@ import (
"github.com/rancher/rke/k8s" "github.com/rancher/rke/k8s"
"github.com/rancher/rke/log" "github.com/rancher/rke/log"
"github.com/rancher/rke/services" "github.com/rancher/rke/services"
"github.com/rancher/rke/templates"
"github.com/rancher/types/apis/management.cattle.io/v3" "github.com/rancher/types/apis/management.cattle.io/v3"
) )
@ -30,6 +31,9 @@ const (
DefaultAuthStrategy = "x509" DefaultAuthStrategy = "x509"
DefaultAuthorizationMode = "rbac" DefaultAuthorizationMode = "rbac"
DefaultAuthnWebhookFile = templates.AuthnWebhook
DefaultAuthnCacheTimeout = "5s"
DefaultNetworkPlugin = "canal" DefaultNetworkPlugin = "canal"
DefaultNetworkCloudProvider = "none" DefaultNetworkCloudProvider = "none"
@ -137,6 +141,7 @@ func (c *Cluster) setClusterDefaults(ctx context.Context) {
c.setClusterImageDefaults() c.setClusterImageDefaults()
c.setClusterServicesDefaults() c.setClusterServicesDefaults()
c.setClusterNetworkDefaults() c.setClusterNetworkDefaults()
c.setClusterAuthnDefaults()
} }
func (c *Cluster) setClusterServicesDefaults() { func (c *Cluster) setClusterServicesDefaults() {
@ -162,7 +167,6 @@ func (c *Cluster) setClusterServicesDefaults() {
&c.Services.Kubelet.ClusterDNSServer: DefaultClusterDNSService, &c.Services.Kubelet.ClusterDNSServer: DefaultClusterDNSService,
&c.Services.Kubelet.ClusterDomain: DefaultClusterDomain, &c.Services.Kubelet.ClusterDomain: DefaultClusterDomain,
&c.Services.Kubelet.InfraContainerImage: c.SystemImages.PodInfraContainer, &c.Services.Kubelet.InfraContainerImage: c.SystemImages.PodInfraContainer,
&c.Authentication.Strategy: DefaultAuthStrategy,
&c.Services.Etcd.Creation: DefaultEtcdBackupCreationPeriod, &c.Services.Etcd.Creation: DefaultEtcdBackupCreationPeriod,
&c.Services.Etcd.Retention: DefaultEtcdBackupRetentionPeriod, &c.Services.Etcd.Retention: DefaultEtcdBackupRetentionPeriod,
} }
@ -268,6 +272,28 @@ func (c *Cluster) setClusterNetworkDefaults() {
} }
} }
func (c *Cluster) setClusterAuthnDefaults() {
setDefaultIfEmpty(&c.Authentication.Strategy, DefaultAuthStrategy)
for _, strategy := range strings.Split(c.Authentication.Strategy, "|") {
strategy = strings.ToLower(strings.TrimSpace(strategy))
c.AuthnStrategies[strategy] = true
}
if c.AuthnStrategies[AuthnWebhookProvider] && c.Authentication.Webhook == nil {
c.Authentication.Webhook = &v3.AuthWebhookConfig{}
}
if c.Authentication.Webhook != nil {
webhookConfigDefaultsMap := map[*string]string{
&c.Authentication.Webhook.ConfigFile: DefaultAuthnWebhookFile,
&c.Authentication.Webhook.CacheTimeout: DefaultAuthnCacheTimeout,
}
for k, v := range webhookConfigDefaultsMap {
setDefaultIfEmpty(k, v)
}
}
}
func d(image, defaultRegistryURL string) string { func d(image, defaultRegistryURL string) string {
if len(defaultRegistryURL) == 0 { if len(defaultRegistryURL) == 0 {
return image return image

60
cluster/file-deployer.go Normal file
View File

@ -0,0 +1,60 @@
package cluster
import (
"context"
"fmt"
"path"
"github.com/docker/docker/api/types/container"
"github.com/rancher/rke/docker"
"github.com/rancher/rke/hosts"
"github.com/rancher/rke/log"
"github.com/rancher/types/apis/management.cattle.io/v3"
"github.com/sirupsen/logrus"
)
const (
ContainerName = "file-deployer"
ServiceName = "file-deploy"
ConfigEnv = "FILE_DEPLOY"
)
func deployFile(ctx context.Context, uniqueHosts []*hosts.Host, alpineImage string, prsMap map[string]v3.PrivateRegistry, fileName, fileContents string) error {
for _, host := range uniqueHosts {
log.Infof(ctx, "[%s] Deploying file '%s' to node [%s]", ServiceName, fileName, host.Address)
if err := doDeployFile(ctx, host, fileName, fileContents, alpineImage, prsMap); err != nil {
return fmt.Errorf("Failed to deploy file '%s' on node [%s]: %v", host.Address, fileName, err)
}
}
return nil
}
func doDeployFile(ctx context.Context, host *hosts.Host, fileName, fileContents, alpineImage string, prsMap map[string]v3.PrivateRegistry) error {
// remove existing container. Only way it's still here is if previous deployment failed
if err := docker.DoRemoveContainer(ctx, host.DClient, ContainerName, host.Address); err != nil {
return err
}
containerEnv := []string{ConfigEnv + "=" + fileContents}
imageCfg := &container.Config{
Image: alpineImage,
Cmd: []string{
"sh",
"-c",
fmt.Sprintf("t=$(mktemp); echo -e \"$%s\" > $t && mv $t %s && chmod 644 %s", ConfigEnv, fileName, fileName),
},
Env: containerEnv,
}
hostCfg := &container.HostConfig{
Binds: []string{
fmt.Sprintf("%s:/etc/kubernetes:z", path.Join(host.PrefixPath, "/etc/kubernetes")),
},
}
if err := docker.DoRunContainer(ctx, host.DClient, imageCfg, hostCfg, ContainerName, host.Address, ServiceName, prsMap); err != nil {
return err
}
if err := docker.DoRemoveContainer(ctx, host.DClient, ContainerName, host.Address); err != nil {
return err
}
logrus.Debugf("[%s] Successfully deployed file '%s' on node [%s]", ServiceName, fileName, host.Address)
return nil
}

View File

@ -21,6 +21,8 @@ const (
etcdRoleLabel = "node-role.kubernetes.io/etcd" etcdRoleLabel = "node-role.kubernetes.io/etcd"
controlplaneRoleLabel = "node-role.kubernetes.io/controlplane" controlplaneRoleLabel = "node-role.kubernetes.io/controlplane"
workerRoleLabel = "node-role.kubernetes.io/worker" workerRoleLabel = "node-role.kubernetes.io/worker"
cloudConfigFileName = "/etc/kubernetes/cloud-config"
authnWebhookFileName = "/etc/kubernetes/kube-api-authn-webhook.yaml"
) )
func (c *Cluster) TunnelHosts(ctx context.Context, flags ExternalFlags) error { func (c *Cluster) TunnelHosts(ctx context.Context, flags ExternalFlags) error {
@ -117,7 +119,7 @@ func (c *Cluster) InvertIndexHosts() error {
} }
func (c *Cluster) SetUpHosts(ctx context.Context, rotateCerts bool) error { func (c *Cluster) SetUpHosts(ctx context.Context, rotateCerts bool) error {
if c.Authentication.Strategy == X509AuthenticationProvider { if c.AuthnStrategies[AuthnX509Provider] {
log.Infof(ctx, "[certificates] Deploying kubernetes certificates to Cluster nodes") log.Infof(ctx, "[certificates] Deploying kubernetes certificates to Cluster nodes")
hostList := hosts.GetUniqueHostList(c.EtcdHosts, c.ControlPlaneHosts, c.WorkerHosts) hostList := hosts.GetUniqueHostList(c.EtcdHosts, c.ControlPlaneHosts, c.WorkerHosts)
var errgrp errgroup.Group var errgrp errgroup.Group
@ -144,10 +146,17 @@ func (c *Cluster) SetUpHosts(ctx context.Context, rotateCerts bool) error {
} }
log.Infof(ctx, "[certificates] Successfully deployed kubernetes certificates to Cluster nodes") log.Infof(ctx, "[certificates] Successfully deployed kubernetes certificates to Cluster nodes")
if c.CloudProvider.Name != "" { if c.CloudProvider.Name != "" {
if err := deployCloudProviderConfig(ctx, hostList, c.SystemImages.Alpine, c.PrivateRegistriesMap, c.CloudConfigFile); err != nil { if err := deployFile(ctx, hostList, c.SystemImages.Alpine, c.PrivateRegistriesMap, cloudConfigFileName, c.CloudConfigFile); err != nil {
return err return err
} }
log.Infof(ctx, "[%s] Successfully deployed kubernetes cloud config to Cluster nodes", CloudConfigServiceName) log.Infof(ctx, "[%s] Successfully deployed kubernetes cloud config to Cluster nodes", cloudConfigFileName)
}
if c.Authentication.Webhook != nil {
if err := deployFile(ctx, hostList, c.SystemImages.Alpine, c.PrivateRegistriesMap, authnWebhookFileName, c.Authentication.Webhook.ConfigFile); err != nil {
return err
}
log.Infof(ctx, "[%s] Successfully deployed authentication webhook config Cluster nodes", cloudConfigFileName)
} }
} }
return nil return nil

View File

@ -81,7 +81,7 @@ func BuildRKEConfigNodePlan(ctx context.Context, myCluster *Cluster, host *hosts
portChecks = append(portChecks, BuildPortChecksFromPortList(host, EtcdPortList, ProtocolTCP)...) portChecks = append(portChecks, BuildPortChecksFromPortList(host, EtcdPortList, ProtocolTCP)...)
} }
cloudConfig := v3.File{ cloudConfig := v3.File{
Name: CloudConfigPath, Name: cloudConfigFileName,
Contents: b64.StdEncoding.EncodeToString([]byte(myCluster.CloudConfigFile)), Contents: b64.StdEncoding.EncodeToString([]byte(myCluster.CloudConfigFile)),
} }
return v3.RKEConfigNodePlan{ return v3.RKEConfigNodePlan{
@ -149,7 +149,11 @@ func (c *Cluster) BuildKubeAPIProcess(prefixPath string) v3.Process {
"requestheader-username-headers": "X-Remote-User", "requestheader-username-headers": "X-Remote-User",
} }
if len(c.CloudProvider.Name) > 0 && c.CloudProvider.Name != aws.AWSCloudProviderName { if len(c.CloudProvider.Name) > 0 && c.CloudProvider.Name != aws.AWSCloudProviderName {
CommandArgs["cloud-config"] = CloudConfigPath CommandArgs["cloud-config"] = cloudConfigFileName
}
if c.Authentication.Webhook != nil {
CommandArgs["authentication-token-webhook-config-file"] = authnWebhookFileName
CommandArgs["authentication-token-webhook-cache-ttl"] = c.Authentication.Webhook.CacheTimeout
} }
if len(c.CloudProvider.Name) > 0 { if len(c.CloudProvider.Name) > 0 {
c.Services.KubeAPI.ExtraEnv = append( c.Services.KubeAPI.ExtraEnv = append(
@ -253,7 +257,7 @@ func (c *Cluster) BuildKubeControllerProcess(prefixPath string) v3.Process {
"root-ca-file": pki.GetCertPath(pki.CACertName), "root-ca-file": pki.GetCertPath(pki.CACertName),
} }
if len(c.CloudProvider.Name) > 0 && c.CloudProvider.Name != aws.AWSCloudProviderName { if len(c.CloudProvider.Name) > 0 && c.CloudProvider.Name != aws.AWSCloudProviderName {
CommandArgs["cloud-config"] = CloudConfigPath CommandArgs["cloud-config"] = cloudConfigFileName
} }
if len(c.CloudProvider.Name) > 0 { if len(c.CloudProvider.Name) > 0 {
c.Services.KubeController.ExtraEnv = append( c.Services.KubeController.ExtraEnv = append(
@ -359,7 +363,7 @@ func (c *Cluster) BuildKubeletProcess(host *hosts.Host, prefixPath string) v3.Pr
CommandArgs["node-ip"] = host.InternalAddress CommandArgs["node-ip"] = host.InternalAddress
} }
if len(c.CloudProvider.Name) > 0 && c.CloudProvider.Name != aws.AWSCloudProviderName { if len(c.CloudProvider.Name) > 0 && c.CloudProvider.Name != aws.AWSCloudProviderName {
CommandArgs["cloud-config"] = CloudConfigPath CommandArgs["cloud-config"] = cloudConfigFileName
} }
if len(c.CloudProvider.Name) > 0 { if len(c.CloudProvider.Name) > 0 {
c.Services.Kubelet.ExtraEnv = append( c.Services.Kubelet.ExtraEnv = append(

View File

@ -39,8 +39,17 @@ func (c *Cluster) ValidateCluster() error {
} }
func validateAuthOptions(c *Cluster) error { func validateAuthOptions(c *Cluster) error {
if c.Authentication.Strategy != DefaultAuthStrategy { for strategy, enabled := range c.AuthnStrategies {
return fmt.Errorf("Authentication strategy [%s] is not supported", c.Authentication.Strategy) if !enabled {
continue
}
strategy = strings.ToLower(strategy)
if strategy != AuthnX509Provider && strategy != AuthnWebhookProvider {
return fmt.Errorf("Authentication strategy [%s] is not supported", strategy)
}
}
if !c.AuthnStrategies[AuthnX509Provider] {
return fmt.Errorf("Authentication strategy must contain [%s]", AuthnX509Provider)
} }
return nil return nil
} }

View File

@ -0,0 +1,23 @@
package templates
const (
AuthnWebhook = `
apiVersion: v1
kind: Config
clusters:
- name: Default
cluster:
insecure-skip-tls-verify: true
server: http://127.0.0.1:6440/v1/authenticate
users:
- name: Default
user:
insecure-skip-tls-verify: true
current-context: webhook
contexts:
- name: webhook
context:
user: Default
cluster: Default
`
)