mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-23 03:41:45 +00:00
Merge pull request #85468 from neolit123/1.17-discovery-token-fix
kubeadm: simplify discover/token and add detailed unit tests
This commit is contained in:
Kubernetes Prow Robot
GitHub
commit
85f8005cf0
@ -79,7 +79,7 @@ func DiscoverValidatedKubeConfig(cfg *kubeadmapi.JoinConfiguration) (*clientcmda
|
||||
}
|
||||
return file.RetrieveValidatedConfigInfo(kubeConfigPath, kubeadmapiv1beta2.DefaultClusterName, cfg.Discovery.Timeout.Duration)
|
||||
case cfg.Discovery.BootstrapToken != nil:
|
||||
return token.RetrieveValidatedConfigInfo(cfg)
|
||||
return token.RetrieveValidatedConfigInfo(&cfg.Discovery)
|
||||
default:
|
||||
return nil, errors.New("couldn't find a valid discovery configuration")
|
||||
}
|
||||
|
||||
@ -16,8 +16,10 @@ go_library(
|
||||
"//cmd/kubeadm/app/constants:go_default_library",
|
||||
"//cmd/kubeadm/app/util/kubeconfig:go_default_library",
|
||||
"//cmd/kubeadm/app/util/pubkeypin:go_default_library",
|
||||
"//staging/src/k8s.io/api/core/v1:go_default_library",
|
||||
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
|
||||
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
|
||||
"//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library",
|
||||
"//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library",
|
||||
"//staging/src/k8s.io/client-go/util/cert:go_default_library",
|
||||
@ -46,7 +48,15 @@ go_test(
|
||||
srcs = ["token_test.go"],
|
||||
embed = [":go_default_library"],
|
||||
deps = [
|
||||
"//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library",
|
||||
"//vendor/github.com/pkg/errors:go_default_library",
|
||||
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
|
||||
"//cmd/kubeadm/app/util/apiclient:go_default_library",
|
||||
"//staging/src/k8s.io/api/core/v1:go_default_library",
|
||||
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
|
||||
"//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
|
||||
"//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library",
|
||||
"//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library",
|
||||
"//staging/src/k8s.io/cluster-bootstrap/token/jws:go_default_library",
|
||||
"//vendor/github.com/pmezard/go-difflib/difflib:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
@ -18,14 +18,16 @@ package token
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
clientset "k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/tools/clientcmd"
|
||||
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
|
||||
certutil "k8s.io/client-go/util/cert"
|
||||
@ -43,120 +45,94 @@ import (
|
||||
const BootstrapUser = "token-bootstrap-client"
|
||||
|
||||
// RetrieveValidatedConfigInfo connects to the API Server and tries to fetch the cluster-info ConfigMap
|
||||
// It then makes sure it can trust the API Server by looking at the JWS-signed tokens and (if cfg.DiscoveryTokenCACertHashes is not empty)
|
||||
// It then makes sure it can trust the API Server by looking at the JWS-signed tokens and (if CACertHashes is not empty)
|
||||
// validating the cluster CA against a set of pinned public keys
|
||||
func RetrieveValidatedConfigInfo(cfg *kubeadmapi.JoinConfiguration) (*clientcmdapi.Config, error) {
|
||||
token, err := kubeadmapi.NewBootstrapTokenString(cfg.Discovery.BootstrapToken.Token)
|
||||
func RetrieveValidatedConfigInfo(cfg *kubeadmapi.Discovery) (*clientcmdapi.Config, error) {
|
||||
return retrieveValidatedConfigInfo(nil, cfg, constants.DiscoveryRetryInterval)
|
||||
}
|
||||
|
||||
// retrieveValidatedConfigInfo is a private implementation of RetrieveValidatedConfigInfo.
|
||||
// It accepts an optional clientset that can be used for testing purposes.
|
||||
func retrieveValidatedConfigInfo(client clientset.Interface, cfg *kubeadmapi.Discovery, interval time.Duration) (*clientcmdapi.Config, error) {
|
||||
token, err := kubeadmapi.NewBootstrapTokenString(cfg.BootstrapToken.Token)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Load the cfg.DiscoveryTokenCACertHashes into a pubkeypin.Set
|
||||
// Load the CACertHashes into a pubkeypin.Set
|
||||
pubKeyPins := pubkeypin.NewSet()
|
||||
err = pubKeyPins.Allow(cfg.Discovery.BootstrapToken.CACertHashes...)
|
||||
if err = pubKeyPins.Allow(cfg.BootstrapToken.CACertHashes...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
duration := cfg.Timeout.Duration
|
||||
// Make sure the interval is not bigger than the duration
|
||||
if interval > duration {
|
||||
interval = duration
|
||||
}
|
||||
|
||||
endpoint := cfg.BootstrapToken.APIServerEndpoint
|
||||
insecureBootstrapConfig := buildInsecureBootstrapKubeConfig(endpoint, kubeadmapiv1beta2.DefaultClusterName)
|
||||
clusterName := insecureBootstrapConfig.Contexts[insecureBootstrapConfig.CurrentContext].Cluster
|
||||
|
||||
klog.V(1).Infof("[discovery] Created cluster-info discovery client, requesting info from %q", endpoint)
|
||||
insecureClusterInfo, err := getClusterInfo(client, insecureBootstrapConfig, token, interval, duration)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// The function below runs for every endpoint, and all endpoints races with each other.
|
||||
// The endpoint that wins the race and completes the task first gets its kubeconfig returned below
|
||||
baseKubeConfig, err := fetchKubeConfigWithTimeout(cfg.Discovery.BootstrapToken.APIServerEndpoint, cfg.Discovery.Timeout.Duration, func(endpoint string) (*clientcmdapi.Config, error) {
|
||||
|
||||
insecureBootstrapConfig := buildInsecureBootstrapKubeConfig(endpoint, kubeadmapiv1beta2.DefaultClusterName)
|
||||
clusterName := insecureBootstrapConfig.Contexts[insecureBootstrapConfig.CurrentContext].Cluster
|
||||
|
||||
insecureClient, err := kubeconfigutil.ToClientSet(insecureBootstrapConfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
klog.V(1).Infof("[discovery] Created cluster-info discovery client, requesting info from %q\n", insecureBootstrapConfig.Clusters[clusterName].Server)
|
||||
|
||||
// Make an initial insecure connection to get the cluster-info ConfigMap
|
||||
insecureClusterInfo, err := insecureClient.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
klog.V(1).Infof("[discovery] Failed to request cluster info: [%s]\n", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Validate the MAC on the kubeconfig from the ConfigMap and load it
|
||||
insecureKubeconfigString, ok := insecureClusterInfo.Data[bootstrapapi.KubeConfigKey]
|
||||
if !ok || len(insecureKubeconfigString) == 0 {
|
||||
return nil, errors.Errorf("there is no %s key in the %s ConfigMap. This API Server isn't set up for token bootstrapping, can't connect",
|
||||
bootstrapapi.KubeConfigKey, bootstrapapi.ConfigMapClusterInfo)
|
||||
}
|
||||
detachedJWSToken, ok := insecureClusterInfo.Data[bootstrapapi.JWSSignatureKeyPrefix+token.ID]
|
||||
if !ok || len(detachedJWSToken) == 0 {
|
||||
return nil, errors.Errorf("token id %q is invalid for this cluster or it has expired. Use \"kubeadm token create\" on the control-plane node to create a new valid token", token.ID)
|
||||
}
|
||||
if !bootstrap.DetachedTokenIsValid(detachedJWSToken, insecureKubeconfigString, token.ID, token.Secret) {
|
||||
return nil, errors.New("failed to verify JWS signature of received cluster info object, can't trust this API Server")
|
||||
}
|
||||
insecureKubeconfigBytes := []byte(insecureKubeconfigString)
|
||||
insecureConfig, err := clientcmd.Load(insecureKubeconfigBytes)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "couldn't parse the kubeconfig file in the %s configmap", bootstrapapi.ConfigMapClusterInfo)
|
||||
}
|
||||
|
||||
// If no TLS root CA pinning was specified, we're done
|
||||
if pubKeyPins.Empty() {
|
||||
klog.V(1).Infof("[discovery] Cluster info signature and contents are valid and no TLS pinning was specified, will use API Server %q\n", endpoint)
|
||||
return insecureConfig, nil
|
||||
}
|
||||
|
||||
// Load the cluster CA from the Config
|
||||
if len(insecureConfig.Clusters) != 1 {
|
||||
return nil, errors.Errorf("expected the kubeconfig file in the %s configmap to have a single cluster, but it had %d", bootstrapapi.ConfigMapClusterInfo, len(insecureConfig.Clusters))
|
||||
}
|
||||
var clusterCABytes []byte
|
||||
for _, cluster := range insecureConfig.Clusters {
|
||||
clusterCABytes = cluster.CertificateAuthorityData
|
||||
}
|
||||
clusterCAs, err := certutil.ParseCertsPEM(clusterCABytes)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to parse cluster CA from the %s configmap", bootstrapapi.ConfigMapClusterInfo)
|
||||
|
||||
}
|
||||
|
||||
// Validate the cluster CA public key against the pinned set
|
||||
err = pubKeyPins.CheckAny(clusterCAs)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "cluster CA found in %s configmap is invalid", bootstrapapi.ConfigMapClusterInfo)
|
||||
}
|
||||
|
||||
// Now that we know the proported cluster CA, connect back a second time validating with that CA
|
||||
secureBootstrapConfig := buildSecureBootstrapKubeConfig(endpoint, clusterCABytes, clusterName)
|
||||
secureClient, err := kubeconfigutil.ToClientSet(secureBootstrapConfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
klog.V(1).Infof("[discovery] Requesting info from %q again to validate TLS against the pinned public key\n", insecureBootstrapConfig.Clusters[clusterName].Server)
|
||||
secureClusterInfo, err := secureClient.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
klog.V(1).Infof("[discovery] Failed to request cluster info: [%s]\n", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Pull the kubeconfig from the securely-obtained ConfigMap and validate that it's the same as what we found the first time
|
||||
secureKubeconfigBytes := []byte(secureClusterInfo.Data[bootstrapapi.KubeConfigKey])
|
||||
if !bytes.Equal(secureKubeconfigBytes, insecureKubeconfigBytes) {
|
||||
return nil, errors.Errorf("the second kubeconfig from the %s configmap (using validated TLS) was different from the first", bootstrapapi.ConfigMapClusterInfo)
|
||||
}
|
||||
|
||||
secureKubeconfig, err := clientcmd.Load(secureKubeconfigBytes)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "couldn't parse the kubeconfig file in the %s configmap", bootstrapapi.ConfigMapClusterInfo)
|
||||
}
|
||||
|
||||
klog.V(1).Infof("[discovery] Cluster info signature and contents are valid and TLS certificate validates against pinned roots, will use API Server %q\n", endpoint)
|
||||
return secureKubeconfig, nil
|
||||
})
|
||||
// Validate the token in the cluster info
|
||||
insecureKubeconfigBytes, err := validateClusterInfoToken(insecureClusterInfo, token)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return baseKubeConfig, nil
|
||||
// Load the insecure config
|
||||
insecureConfig, err := clientcmd.Load(insecureKubeconfigBytes)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "couldn't parse the kubeconfig file in the %s ConfigMap", bootstrapapi.ConfigMapClusterInfo)
|
||||
}
|
||||
|
||||
// The ConfigMap should contain a single cluster
|
||||
if len(insecureConfig.Clusters) != 1 {
|
||||
return nil, errors.Errorf("expected the kubeconfig file in the %s ConfigMap to have a single cluster, but it had %d", bootstrapapi.ConfigMapClusterInfo, len(insecureConfig.Clusters))
|
||||
}
|
||||
|
||||
// If no TLS root CA pinning was specified, we're done
|
||||
if pubKeyPins.Empty() {
|
||||
klog.V(1).Infof("[discovery] Cluster info signature and contents are valid and no TLS pinning was specified, will use API Server %q", endpoint)
|
||||
return insecureConfig, nil
|
||||
}
|
||||
|
||||
// Load and validate the cluster CA from the insecure kubeconfig
|
||||
clusterCABytes, err := validateClusterCA(insecureConfig, pubKeyPins)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Now that we know the cluster CA, connect back a second time validating with that CA
|
||||
secureBootstrapConfig := buildSecureBootstrapKubeConfig(endpoint, clusterCABytes, clusterName)
|
||||
|
||||
klog.V(1).Infof("[discovery] Requesting info from %q again to validate TLS against the pinned public key", endpoint)
|
||||
secureClusterInfo, err := getClusterInfo(client, secureBootstrapConfig, token, interval, duration)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Pull the kubeconfig from the securely-obtained ConfigMap and validate that it's the same as what we found the first time
|
||||
secureKubeconfigBytes := []byte(secureClusterInfo.Data[bootstrapapi.KubeConfigKey])
|
||||
if !bytes.Equal(secureKubeconfigBytes, insecureKubeconfigBytes) {
|
||||
return nil, errors.Errorf("the second kubeconfig from the %s ConfigMap (using validated TLS) was different from the first", bootstrapapi.ConfigMapClusterInfo)
|
||||
}
|
||||
|
||||
secureKubeconfig, err := clientcmd.Load(secureKubeconfigBytes)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "couldn't parse the kubeconfig file in the %s ConfigMap", bootstrapapi.ConfigMapClusterInfo)
|
||||
}
|
||||
|
||||
klog.V(1).Infof("[discovery] Cluster info signature and contents are valid and TLS certificate validates against pinned roots, will use API Server %q", endpoint)
|
||||
|
||||
return secureKubeconfig, nil
|
||||
}
|
||||
|
||||
// buildInsecureBootstrapKubeConfig makes a kubeconfig object that connects insecurely to the API Server for bootstrapping purposes
|
||||
@ -174,42 +150,85 @@ func buildSecureBootstrapKubeConfig(endpoint string, caCert []byte, clustername
|
||||
return bootstrapConfig
|
||||
}
|
||||
|
||||
// fetchKubeConfigWithTimeout tries to run fetchKubeConfigFunc on every DiscoveryRetryInterval, but until discoveryTimeout is reached
|
||||
func fetchKubeConfigWithTimeout(apiEndpoint string, discoveryTimeout time.Duration, fetchKubeConfigFunc func(string) (*clientcmdapi.Config, error)) (*clientcmdapi.Config, error) {
|
||||
stopChan := make(chan struct{})
|
||||
var resultingKubeConfig *clientcmdapi.Config
|
||||
var once sync.Once
|
||||
var wg sync.WaitGroup
|
||||
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
wait.Until(func() {
|
||||
klog.V(1).Infof("[discovery] Trying to connect to API Server %q\n", apiEndpoint)
|
||||
cfg, err := fetchKubeConfigFunc(apiEndpoint)
|
||||
if err != nil {
|
||||
klog.V(1).Infof("[discovery] Failed to connect to API Server %q: %v\n", apiEndpoint, err)
|
||||
return
|
||||
}
|
||||
klog.V(1).Infof("[discovery] Successfully established connection with API Server %q\n", apiEndpoint)
|
||||
once.Do(func() {
|
||||
resultingKubeConfig = cfg
|
||||
close(stopChan)
|
||||
})
|
||||
}, constants.DiscoveryRetryInterval, stopChan)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-time.After(discoveryTimeout):
|
||||
once.Do(func() {
|
||||
close(stopChan)
|
||||
})
|
||||
err := errors.Errorf("abort connecting to API servers after timeout of %v", discoveryTimeout)
|
||||
klog.V(1).Infof("[discovery] %v\n", err)
|
||||
wg.Wait()
|
||||
return nil, err
|
||||
case <-stopChan:
|
||||
wg.Wait()
|
||||
return resultingKubeConfig, nil
|
||||
// validateClusterInfoToken validates that the JWS token present in the cluster info ConfigMap is valid
|
||||
func validateClusterInfoToken(insecureClusterInfo *v1.ConfigMap, token *kubeadmapi.BootstrapTokenString) ([]byte, error) {
|
||||
insecureKubeconfigString, ok := insecureClusterInfo.Data[bootstrapapi.KubeConfigKey]
|
||||
if !ok || len(insecureKubeconfigString) == 0 {
|
||||
return nil, errors.Errorf("there is no %s key in the %s ConfigMap. This API Server isn't set up for token bootstrapping, can't connect",
|
||||
bootstrapapi.KubeConfigKey, bootstrapapi.ConfigMapClusterInfo)
|
||||
}
|
||||
|
||||
detachedJWSToken, ok := insecureClusterInfo.Data[bootstrapapi.JWSSignatureKeyPrefix+token.ID]
|
||||
if !ok || len(detachedJWSToken) == 0 {
|
||||
return nil, errors.Errorf("token id %q is invalid for this cluster or it has expired. Use \"kubeadm token create\" on the control-plane node to create a new valid token", token.ID)
|
||||
}
|
||||
|
||||
if !bootstrap.DetachedTokenIsValid(detachedJWSToken, insecureKubeconfigString, token.ID, token.Secret) {
|
||||
return nil, errors.New("failed to verify JWS signature of received cluster info object, can't trust this API Server")
|
||||
}
|
||||
|
||||
return []byte(insecureKubeconfigString), nil
|
||||
}
|
||||
|
||||
// validateClusterCA validates the cluster CA found in the insecure kubeconfig
|
||||
func validateClusterCA(insecureConfig *clientcmdapi.Config, pubKeyPins *pubkeypin.Set) ([]byte, error) {
|
||||
var clusterCABytes []byte
|
||||
for _, cluster := range insecureConfig.Clusters {
|
||||
clusterCABytes = cluster.CertificateAuthorityData
|
||||
}
|
||||
|
||||
clusterCAs, err := certutil.ParseCertsPEM(clusterCABytes)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to parse cluster CA from the %s ConfigMap", bootstrapapi.ConfigMapClusterInfo)
|
||||
}
|
||||
|
||||
// Validate the cluster CA public key against the pinned set
|
||||
err = pubKeyPins.CheckAny(clusterCAs)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "cluster CA found in %s ConfigMap is invalid", bootstrapapi.ConfigMapClusterInfo)
|
||||
}
|
||||
|
||||
return clusterCABytes, nil
|
||||
}
|
||||
|
||||
// getClusterInfo creates a client from the given kubeconfig if the given client is nil,
|
||||
// and requests the cluster info ConfigMap using PollImmediate.
|
||||
// If a client is provided it will be used instead.
|
||||
func getClusterInfo(client clientset.Interface, kubeconfig *clientcmdapi.Config, token *kubeadmapi.BootstrapTokenString, interval, duration time.Duration) (*v1.ConfigMap, error) {
|
||||
var cm *v1.ConfigMap
|
||||
var err error
|
||||
|
||||
// Create client from kubeconfig
|
||||
if client == nil {
|
||||
client, err = kubeconfigutil.ToClientSet(kubeconfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.TODO(), duration)
|
||||
defer cancel()
|
||||
|
||||
wait.JitterUntil(func() {
|
||||
cm, err = client.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
klog.V(1).Infof("[discovery] Failed to request cluster-info, will try again: %v", err)
|
||||
return
|
||||
}
|
||||
// Even if the ConfigMap is available the JWS signature is patched-in a bit later.
|
||||
// Make sure we retry util then.
|
||||
if _, ok := cm.Data[bootstrapapi.JWSSignatureKeyPrefix+token.ID]; !ok {
|
||||
klog.V(1).Infof("[discovery] The cluster-info ConfigMap does not yet contain a JWS signature for token ID %q, will try again", token.ID)
|
||||
err = errors.Errorf("could not find a JWS signature in the cluster-info ConfigMap for token ID %q", token.ID)
|
||||
return
|
||||
}
|
||||
// Cancel the context on success
|
||||
cancel()
|
||||
}, interval, 0.3, true, ctx.Done())
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return cm, nil
|
||||
}
|
||||
|
||||
@ -20,53 +20,293 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
clientset "k8s.io/client-go/kubernetes"
|
||||
fakeclient "k8s.io/client-go/kubernetes/fake"
|
||||
"k8s.io/client-go/tools/clientcmd"
|
||||
bootstrapapi "k8s.io/cluster-bootstrap/token/api"
|
||||
tokenjws "k8s.io/cluster-bootstrap/token/jws"
|
||||
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
|
||||
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
|
||||
|
||||
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
|
||||
"github.com/pmezard/go-difflib/difflib"
|
||||
)
|
||||
|
||||
func TestFetchKubeConfigWithTimeout(t *testing.T) {
|
||||
const testAPIEndpoint = "sample-endpoint:1234"
|
||||
func TestRetrieveValidatedConfigInfo(t *testing.T) {
|
||||
const (
|
||||
caCert = `-----BEGIN CERTIFICATE-----
|
||||
MIICyDCCAbCgAwIBAgIBADANBgkqhkiG9w0BAQsFADAVMRMwEQYDVQQDEwprdWJl
|
||||
cm5ldGVzMB4XDTE5MTEyMDAwNDk0MloXDTI5MTExNzAwNDk0MlowFTETMBEGA1UE
|
||||
AxMKa3ViZXJuZXRlczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMqQ
|
||||
ctECzA8yFSuVYupOUYgrTmfQeKe/9BaDWagaq7ow9+I2IvsfWFvlrD8QQr8sea6q
|
||||
xjq7TV67Vb4RxBaoYDA+yI5vIcujWUxULun64lu3Q6iC1sj2UnmUpIdgazRXXEkZ
|
||||
vxA6EbAnoxA0+lBOn1CZWl23IQ4s70o2hZ7wIp/vevB88RRRjqtvgc5elsjsbmDF
|
||||
LS7L1Zuye8c6gS93bR+VjVmSIfr1IEq0748tIIyXjAVCWPVCvuP41MlfPc/JVpZD
|
||||
uD2+pO6ZYREcdAnOf2eD4/eLOMKko4L1dSFy9JKM5PLnOC0Zk0AYOd1vS8DTAfxj
|
||||
XPEIY8OBYFhlsxf4TE8CAwEAAaMjMCEwDgYDVR0PAQH/BAQDAgKkMA8GA1UdEwEB
|
||||
/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAH/OYq8zyl1+zSTmuow3yI/15PL1
|
||||
dl8hB7IKnZNWmC/LTdm/+noh3Sb1IdRv6HkKg/GUn0UMuRUngLhju3EO4ozJPQcX
|
||||
quaxzgmTKNWJ6ErDvRvWhGX0ZcbdBfZv+dowyRqzd5nlJ49hC+NrtFFQq6P05BYn
|
||||
7SemguqeXmXwIj2Sa+1DeR6lRm9o8shAYjnyThUFqaMn18kI3SANJ5vk/3DFrPEO
|
||||
CKC9EzFku2kuxg2dM12PbRGZQ2o0K6HEZgrrIKTPOy3ocb8r9M0aSFhjOV/NqGA4
|
||||
SaupXSW6XfvIi/UHoIbU3pNcsnUJGnQfQvip95XKk/gqcUr+m50vxgumxtA=
|
||||
-----END CERTIFICATE-----`
|
||||
|
||||
caCertHash = "sha256:98be2e6d4d8a89aa308fb15de0c07e2531ce549c68dec1687cdd5c06f0826658"
|
||||
|
||||
expectedKubeconfig = `apiVersion: v1
|
||||
clusters:
|
||||
- cluster:
|
||||
certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN5RENDQWJDZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRFNU1URXlNREF3TkRrME1sb1hEVEk1TVRFeE56QXdORGswTWxvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTXFRCmN0RUN6QTh5RlN1Vll1cE9VWWdyVG1mUWVLZS85QmFEV2FnYXE3b3c5K0kySXZzZldGdmxyRDhRUXI4c2VhNnEKeGpxN1RWNjdWYjRSeEJhb1lEQSt5STV2SWN1aldVeFVMdW42NGx1M1E2aUMxc2oyVW5tVXBJZGdhelJYWEVrWgp2eEE2RWJBbm94QTArbEJPbjFDWldsMjNJUTRzNzBvMmhaN3dJcC92ZXZCODhSUlJqcXR2Z2M1ZWxzanNibURGCkxTN0wxWnV5ZThjNmdTOTNiUitWalZtU0lmcjFJRXEwNzQ4dElJeVhqQVZDV1BWQ3Z1UDQxTWxmUGMvSlZwWkQKdUQyK3BPNlpZUkVjZEFuT2YyZUQ0L2VMT01La280TDFkU0Z5OUpLTTVQTG5PQzBaazBBWU9kMXZTOERUQWZ4agpYUEVJWThPQllGaGxzeGY0VEU4Q0F3RUFBYU1qTUNFd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFIL09ZcTh6eWwxK3pTVG11b3czeUkvMTVQTDEKZGw4aEI3SUtuWk5XbUMvTFRkbS8rbm9oM1NiMUlkUnY2SGtLZy9HVW4wVU11UlVuZ0xoanUzRU80b3pKUFFjWApxdWF4emdtVEtOV0o2RXJEdlJ2V2hHWDBaY2JkQmZaditkb3d5UnF6ZDVubEo0OWhDK05ydEZGUXE2UDA1QlluCjdTZW1ndXFlWG1Yd0lqMlNhKzFEZVI2bFJtOW84c2hBWWpueVRoVUZxYU1uMThrSTNTQU5KNXZrLzNERnJQRU8KQ0tDOUV6Rmt1Mmt1eGcyZE0xMlBiUkdaUTJvMEs2SEVaZ3JySUtUUE95M29jYjhyOU0wYVNGaGpPVi9OcUdBNApTYXVwWFNXNlhmdklpL1VIb0liVTNwTmNzblVKR25RZlF2aXA5NVhLay9ncWNVcittNTB2eGd1bXh0QT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==
|
||||
server: https://127.0.0.1
|
||||
name: somecluster
|
||||
contexts:
|
||||
- context:
|
||||
cluster: somecluster
|
||||
user: token-bootstrap-client
|
||||
name: token-bootstrap-client@somecluster
|
||||
current-context: token-bootstrap-client@somecluster
|
||||
kind: Config
|
||||
preferences: {}
|
||||
users: null
|
||||
`
|
||||
)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
discoveryTimeout time.Duration
|
||||
shouldFail bool
|
||||
name string
|
||||
tokenID string
|
||||
tokenSecret string
|
||||
cfg *kubeadmapi.Discovery
|
||||
configMap *fakeConfigMap
|
||||
delayedJWSSignaturePatch bool
|
||||
expectedError bool
|
||||
}{
|
||||
{
|
||||
name: "Timeout if value is not returned on time",
|
||||
discoveryTimeout: 1 * time.Second,
|
||||
shouldFail: true,
|
||||
// This is the default behavior. The JWS signature is patched after the cluster-info ConfigMap is created
|
||||
name: "valid: retrieve a valid kubeconfig with CA verification and delayed JWS signature",
|
||||
tokenID: "123456",
|
||||
tokenSecret: "abcdef1234567890",
|
||||
cfg: &kubeadmapi.Discovery{
|
||||
BootstrapToken: &kubeadmapi.BootstrapTokenDiscovery{
|
||||
Token: "123456.abcdef1234567890",
|
||||
CACertHashes: []string{caCertHash},
|
||||
},
|
||||
},
|
||||
configMap: &fakeConfigMap{
|
||||
name: bootstrapapi.ConfigMapClusterInfo,
|
||||
data: map[string]string{},
|
||||
},
|
||||
delayedJWSSignaturePatch: true,
|
||||
},
|
||||
{
|
||||
name: "Don't timeout if value is returned on time",
|
||||
discoveryTimeout: 5 * time.Second,
|
||||
shouldFail: false,
|
||||
// Same as above expect this test creates the ConfigMap with the JWS signature
|
||||
name: "valid: retrieve a valid kubeconfig with CA verification",
|
||||
tokenID: "123456",
|
||||
tokenSecret: "abcdef1234567890",
|
||||
cfg: &kubeadmapi.Discovery{
|
||||
BootstrapToken: &kubeadmapi.BootstrapTokenDiscovery{
|
||||
Token: "123456.abcdef1234567890",
|
||||
CACertHashes: []string{caCertHash},
|
||||
},
|
||||
},
|
||||
configMap: &fakeConfigMap{
|
||||
name: bootstrapapi.ConfigMapClusterInfo,
|
||||
data: nil,
|
||||
},
|
||||
},
|
||||
{
|
||||
// Skipping CA verification is also supported
|
||||
name: "valid: retrieve a valid kubeconfig without CA verification",
|
||||
tokenID: "123456",
|
||||
tokenSecret: "abcdef1234567890",
|
||||
cfg: &kubeadmapi.Discovery{
|
||||
BootstrapToken: &kubeadmapi.BootstrapTokenDiscovery{
|
||||
Token: "123456.abcdef1234567890",
|
||||
},
|
||||
},
|
||||
configMap: &fakeConfigMap{
|
||||
name: bootstrapapi.ConfigMapClusterInfo,
|
||||
data: nil,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "invalid: token format is invalid",
|
||||
tokenID: "foo",
|
||||
tokenSecret: "bar",
|
||||
cfg: &kubeadmapi.Discovery{
|
||||
BootstrapToken: &kubeadmapi.BootstrapTokenDiscovery{
|
||||
Token: "foo.bar",
|
||||
},
|
||||
},
|
||||
configMap: &fakeConfigMap{
|
||||
name: bootstrapapi.ConfigMapClusterInfo,
|
||||
data: nil,
|
||||
},
|
||||
expectedError: true,
|
||||
},
|
||||
{
|
||||
name: "invalid: missing cluster-info ConfigMap",
|
||||
tokenID: "123456",
|
||||
tokenSecret: "abcdef1234567890",
|
||||
cfg: &kubeadmapi.Discovery{
|
||||
BootstrapToken: &kubeadmapi.BootstrapTokenDiscovery{
|
||||
Token: "123456.abcdef1234567890",
|
||||
},
|
||||
},
|
||||
configMap: &fakeConfigMap{
|
||||
name: "baz",
|
||||
data: nil,
|
||||
},
|
||||
expectedError: true,
|
||||
},
|
||||
{
|
||||
name: "invalid: wrong JWS signature",
|
||||
tokenID: "123456",
|
||||
tokenSecret: "abcdef1234567890",
|
||||
cfg: &kubeadmapi.Discovery{
|
||||
BootstrapToken: &kubeadmapi.BootstrapTokenDiscovery{
|
||||
Token: "123456.abcdef1234567890",
|
||||
},
|
||||
},
|
||||
configMap: &fakeConfigMap{
|
||||
name: bootstrapapi.ConfigMapClusterInfo,
|
||||
data: map[string]string{
|
||||
bootstrapapi.KubeConfigKey: "foo",
|
||||
bootstrapapi.JWSSignatureKeyPrefix + "123456": "bar",
|
||||
},
|
||||
},
|
||||
expectedError: true,
|
||||
},
|
||||
{
|
||||
name: "invalid: missing key for JWSSignatureKeyPrefix",
|
||||
tokenID: "123456",
|
||||
tokenSecret: "abcdef1234567890",
|
||||
cfg: &kubeadmapi.Discovery{
|
||||
BootstrapToken: &kubeadmapi.BootstrapTokenDiscovery{
|
||||
Token: "123456.abcdef1234567890",
|
||||
},
|
||||
},
|
||||
configMap: &fakeConfigMap{
|
||||
name: bootstrapapi.ConfigMapClusterInfo,
|
||||
data: map[string]string{
|
||||
bootstrapapi.KubeConfigKey: "foo",
|
||||
},
|
||||
},
|
||||
expectedError: true,
|
||||
},
|
||||
{
|
||||
name: "invalid: wrong CA cert hash",
|
||||
tokenID: "123456",
|
||||
tokenSecret: "abcdef1234567890",
|
||||
cfg: &kubeadmapi.Discovery{
|
||||
BootstrapToken: &kubeadmapi.BootstrapTokenDiscovery{
|
||||
Token: "123456.abcdef1234567890",
|
||||
CACertHashes: []string{"foo"},
|
||||
},
|
||||
},
|
||||
configMap: &fakeConfigMap{
|
||||
name: bootstrapapi.ConfigMapClusterInfo,
|
||||
data: nil,
|
||||
},
|
||||
expectedError: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
cfg, err := fetchKubeConfigWithTimeout(testAPIEndpoint, test.discoveryTimeout, func(apiEndpoint string) (*clientcmdapi.Config, error) {
|
||||
if apiEndpoint != testAPIEndpoint {
|
||||
return nil, errors.Errorf("unexpected API server endpoint:\n\texpected: %q\n\tgot: %q", testAPIEndpoint, apiEndpoint)
|
||||
}
|
||||
kubeconfig := buildSecureBootstrapKubeConfig("127.0.0.1", []byte(caCert), "somecluster")
|
||||
kubeconfigBytes, err := clientcmd.Write(*kubeconfig)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot marshal kubeconfig %v", err)
|
||||
}
|
||||
|
||||
time.Sleep(3 * time.Second)
|
||||
return &clientcmdapi.Config{}, nil
|
||||
})
|
||||
// Generate signature of the insecure kubeconfig
|
||||
sig, err := tokenjws.ComputeDetachedSignature(string(kubeconfigBytes), test.tokenID, test.tokenSecret)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot compute detached JWS signature: %v", err)
|
||||
}
|
||||
|
||||
if test.shouldFail {
|
||||
if err == nil {
|
||||
t.Fatal("unexpected success")
|
||||
// If the JWS signature is delayed, only add the kubeconfig
|
||||
if test.delayedJWSSignaturePatch {
|
||||
test.configMap.data = map[string]string{}
|
||||
test.configMap.data[bootstrapapi.KubeConfigKey] = string(kubeconfigBytes)
|
||||
}
|
||||
|
||||
// Populate the default cluster-info data
|
||||
if test.configMap.data == nil {
|
||||
test.configMap.data = map[string]string{}
|
||||
test.configMap.data[bootstrapapi.KubeConfigKey] = string(kubeconfigBytes)
|
||||
test.configMap.data[bootstrapapi.JWSSignatureKeyPrefix+test.tokenID] = sig
|
||||
}
|
||||
|
||||
// Create a fake client and create the cluster-info ConfigMap
|
||||
client := fakeclient.NewSimpleClientset()
|
||||
if err = test.configMap.createOrUpdate(client); err != nil {
|
||||
t.Fatalf("could not create ConfigMap: %v", err)
|
||||
}
|
||||
|
||||
// Set arbitrary discovery timeout and retry interval
|
||||
test.cfg.Timeout = &metav1.Duration{Duration: time.Millisecond * 200}
|
||||
interval := time.Millisecond * 20
|
||||
|
||||
// Patch the JWS signature after a short delay
|
||||
if test.delayedJWSSignaturePatch {
|
||||
test.configMap.data[bootstrapapi.JWSSignatureKeyPrefix+test.tokenID] = sig
|
||||
go func() {
|
||||
time.Sleep(time.Millisecond * 60)
|
||||
if err := test.configMap.createOrUpdate(client); err != nil {
|
||||
t.Errorf("could not update the cluster-info ConfigMap with a JWS signature: %v", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Retrieve validated configuration
|
||||
kubeconfig, err = retrieveValidatedConfigInfo(client, test.cfg, interval)
|
||||
if (err != nil) != test.expectedError {
|
||||
t.Errorf("expected error %v, got %v, error: %v", test.expectedError, err != nil, err)
|
||||
}
|
||||
|
||||
// Return if an error is expected
|
||||
if test.expectedError {
|
||||
return
|
||||
}
|
||||
|
||||
// Validate the resulted kubeconfig
|
||||
kubeconfigBytes, err = clientcmd.Write(*kubeconfig)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot marshal resulted kubeconfig %v", err)
|
||||
}
|
||||
if string(kubeconfigBytes) != expectedKubeconfig {
|
||||
t.Error("unexpected kubeconfig")
|
||||
diff := difflib.UnifiedDiff{
|
||||
A: difflib.SplitLines(expectedKubeconfig),
|
||||
B: difflib.SplitLines(string(kubeconfigBytes)),
|
||||
FromFile: "expected",
|
||||
ToFile: "got",
|
||||
Context: 10,
|
||||
}
|
||||
} else {
|
||||
diffstr, err := difflib.GetUnifiedDiffString(diff)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected failure: %v", err)
|
||||
}
|
||||
if cfg == nil {
|
||||
t.Fatal("cfg is nil")
|
||||
t.Fatalf("error generating unified diff string: %v", err)
|
||||
}
|
||||
t.Errorf("\n%s", diffstr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type fakeConfigMap struct {
|
||||
name string
|
||||
data map[string]string
|
||||
}
|
||||
|
||||
func (c *fakeConfigMap) createOrUpdate(client clientset.Interface) error {
|
||||
return apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: c.name,
|
||||
Namespace: metav1.NamespacePublic,
|
||||
},
|
||||
Data: c.data,
|
||||
})
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user