diff --git a/cmd/kubeadm/app/discovery/discovery.go b/cmd/kubeadm/app/discovery/discovery.go index dd11b1d5245..9a83190cb50 100644 --- a/cmd/kubeadm/app/discovery/discovery.go +++ b/cmd/kubeadm/app/discovery/discovery.go @@ -79,7 +79,7 @@ func DiscoverValidatedKubeConfig(cfg *kubeadmapi.JoinConfiguration) (*clientcmda } return file.RetrieveValidatedConfigInfo(kubeConfigPath, kubeadmapiv1beta2.DefaultClusterName, cfg.Discovery.Timeout.Duration) case cfg.Discovery.BootstrapToken != nil: - return token.RetrieveValidatedConfigInfo(cfg) + return token.RetrieveValidatedConfigInfo(&cfg.Discovery) default: return nil, errors.New("couldn't find a valid discovery configuration") } diff --git a/cmd/kubeadm/app/discovery/token/BUILD b/cmd/kubeadm/app/discovery/token/BUILD index 048fb51ea74..de7a4086c4a 100644 --- a/cmd/kubeadm/app/discovery/token/BUILD +++ b/cmd/kubeadm/app/discovery/token/BUILD @@ -16,8 +16,10 @@ go_library( "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/util/kubeconfig:go_default_library", "//cmd/kubeadm/app/util/pubkeypin:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", "//staging/src/k8s.io/client-go/util/cert:go_default_library", @@ -46,7 +48,15 @@ go_test( srcs = ["token_test.go"], embed = [":go_default_library"], deps = [ - "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", + "//cmd/kubeadm/app/apis/kubeadm:go_default_library", + "//cmd/kubeadm/app/util/apiclient:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", + "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", + "//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library", + "//staging/src/k8s.io/cluster-bootstrap/token/jws:go_default_library", + "//vendor/github.com/pmezard/go-difflib/difflib:go_default_library", ], ) diff --git a/cmd/kubeadm/app/discovery/token/token.go b/cmd/kubeadm/app/discovery/token/token.go index dff7d570781..3dfb35903d2 100644 --- a/cmd/kubeadm/app/discovery/token/token.go +++ b/cmd/kubeadm/app/discovery/token/token.go @@ -18,14 +18,16 @@ package token import ( "bytes" + "context" "fmt" - "sync" "time" "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" certutil "k8s.io/client-go/util/cert" @@ -43,120 +45,94 @@ import ( const BootstrapUser = "token-bootstrap-client" // RetrieveValidatedConfigInfo connects to the API Server and tries to fetch the cluster-info ConfigMap -// It then makes sure it can trust the API Server by looking at the JWS-signed tokens and (if cfg.DiscoveryTokenCACertHashes is not empty) +// It then makes sure it can trust the API Server by looking at the JWS-signed tokens and (if CACertHashes is not empty) // validating the cluster CA against a set of pinned public keys -func RetrieveValidatedConfigInfo(cfg *kubeadmapi.JoinConfiguration) (*clientcmdapi.Config, error) { - token, err := kubeadmapi.NewBootstrapTokenString(cfg.Discovery.BootstrapToken.Token) +func RetrieveValidatedConfigInfo(cfg *kubeadmapi.Discovery) (*clientcmdapi.Config, error) { + return retrieveValidatedConfigInfo(nil, cfg, constants.DiscoveryRetryInterval) +} + +// retrieveValidatedConfigInfo is a private implementation of RetrieveValidatedConfigInfo. +// It accepts an optional clientset that can be used for testing purposes. +func retrieveValidatedConfigInfo(client clientset.Interface, cfg *kubeadmapi.Discovery, interval time.Duration) (*clientcmdapi.Config, error) { + token, err := kubeadmapi.NewBootstrapTokenString(cfg.BootstrapToken.Token) if err != nil { return nil, err } - // Load the cfg.DiscoveryTokenCACertHashes into a pubkeypin.Set + // Load the CACertHashes into a pubkeypin.Set pubKeyPins := pubkeypin.NewSet() - err = pubKeyPins.Allow(cfg.Discovery.BootstrapToken.CACertHashes...) + if err = pubKeyPins.Allow(cfg.BootstrapToken.CACertHashes...); err != nil { + return nil, err + } + + duration := cfg.Timeout.Duration + // Make sure the interval is not bigger than the duration + if interval > duration { + interval = duration + } + + endpoint := cfg.BootstrapToken.APIServerEndpoint + insecureBootstrapConfig := buildInsecureBootstrapKubeConfig(endpoint, kubeadmapiv1beta2.DefaultClusterName) + clusterName := insecureBootstrapConfig.Contexts[insecureBootstrapConfig.CurrentContext].Cluster + + klog.V(1).Infof("[discovery] Created cluster-info discovery client, requesting info from %q", endpoint) + insecureClusterInfo, err := getClusterInfo(client, insecureBootstrapConfig, token, interval, duration) if err != nil { return nil, err } - // The function below runs for every endpoint, and all endpoints races with each other. - // The endpoint that wins the race and completes the task first gets its kubeconfig returned below - baseKubeConfig, err := fetchKubeConfigWithTimeout(cfg.Discovery.BootstrapToken.APIServerEndpoint, cfg.Discovery.Timeout.Duration, func(endpoint string) (*clientcmdapi.Config, error) { - - insecureBootstrapConfig := buildInsecureBootstrapKubeConfig(endpoint, kubeadmapiv1beta2.DefaultClusterName) - clusterName := insecureBootstrapConfig.Contexts[insecureBootstrapConfig.CurrentContext].Cluster - - insecureClient, err := kubeconfigutil.ToClientSet(insecureBootstrapConfig) - if err != nil { - return nil, err - } - - klog.V(1).Infof("[discovery] Created cluster-info discovery client, requesting info from %q\n", insecureBootstrapConfig.Clusters[clusterName].Server) - - // Make an initial insecure connection to get the cluster-info ConfigMap - insecureClusterInfo, err := insecureClient.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) - if err != nil { - klog.V(1).Infof("[discovery] Failed to request cluster info: [%s]\n", err) - return nil, err - } - - // Validate the MAC on the kubeconfig from the ConfigMap and load it - insecureKubeconfigString, ok := insecureClusterInfo.Data[bootstrapapi.KubeConfigKey] - if !ok || len(insecureKubeconfigString) == 0 { - return nil, errors.Errorf("there is no %s key in the %s ConfigMap. This API Server isn't set up for token bootstrapping, can't connect", - bootstrapapi.KubeConfigKey, bootstrapapi.ConfigMapClusterInfo) - } - detachedJWSToken, ok := insecureClusterInfo.Data[bootstrapapi.JWSSignatureKeyPrefix+token.ID] - if !ok || len(detachedJWSToken) == 0 { - return nil, errors.Errorf("token id %q is invalid for this cluster or it has expired. Use \"kubeadm token create\" on the control-plane node to create a new valid token", token.ID) - } - if !bootstrap.DetachedTokenIsValid(detachedJWSToken, insecureKubeconfigString, token.ID, token.Secret) { - return nil, errors.New("failed to verify JWS signature of received cluster info object, can't trust this API Server") - } - insecureKubeconfigBytes := []byte(insecureKubeconfigString) - insecureConfig, err := clientcmd.Load(insecureKubeconfigBytes) - if err != nil { - return nil, errors.Wrapf(err, "couldn't parse the kubeconfig file in the %s configmap", bootstrapapi.ConfigMapClusterInfo) - } - - // If no TLS root CA pinning was specified, we're done - if pubKeyPins.Empty() { - klog.V(1).Infof("[discovery] Cluster info signature and contents are valid and no TLS pinning was specified, will use API Server %q\n", endpoint) - return insecureConfig, nil - } - - // Load the cluster CA from the Config - if len(insecureConfig.Clusters) != 1 { - return nil, errors.Errorf("expected the kubeconfig file in the %s configmap to have a single cluster, but it had %d", bootstrapapi.ConfigMapClusterInfo, len(insecureConfig.Clusters)) - } - var clusterCABytes []byte - for _, cluster := range insecureConfig.Clusters { - clusterCABytes = cluster.CertificateAuthorityData - } - clusterCAs, err := certutil.ParseCertsPEM(clusterCABytes) - if err != nil { - return nil, errors.Wrapf(err, "failed to parse cluster CA from the %s configmap", bootstrapapi.ConfigMapClusterInfo) - - } - - // Validate the cluster CA public key against the pinned set - err = pubKeyPins.CheckAny(clusterCAs) - if err != nil { - return nil, errors.Wrapf(err, "cluster CA found in %s configmap is invalid", bootstrapapi.ConfigMapClusterInfo) - } - - // Now that we know the proported cluster CA, connect back a second time validating with that CA - secureBootstrapConfig := buildSecureBootstrapKubeConfig(endpoint, clusterCABytes, clusterName) - secureClient, err := kubeconfigutil.ToClientSet(secureBootstrapConfig) - if err != nil { - return nil, err - } - - klog.V(1).Infof("[discovery] Requesting info from %q again to validate TLS against the pinned public key\n", insecureBootstrapConfig.Clusters[clusterName].Server) - secureClusterInfo, err := secureClient.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) - if err != nil { - klog.V(1).Infof("[discovery] Failed to request cluster info: [%s]\n", err) - return nil, err - } - - // Pull the kubeconfig from the securely-obtained ConfigMap and validate that it's the same as what we found the first time - secureKubeconfigBytes := []byte(secureClusterInfo.Data[bootstrapapi.KubeConfigKey]) - if !bytes.Equal(secureKubeconfigBytes, insecureKubeconfigBytes) { - return nil, errors.Errorf("the second kubeconfig from the %s configmap (using validated TLS) was different from the first", bootstrapapi.ConfigMapClusterInfo) - } - - secureKubeconfig, err := clientcmd.Load(secureKubeconfigBytes) - if err != nil { - return nil, errors.Wrapf(err, "couldn't parse the kubeconfig file in the %s configmap", bootstrapapi.ConfigMapClusterInfo) - } - - klog.V(1).Infof("[discovery] Cluster info signature and contents are valid and TLS certificate validates against pinned roots, will use API Server %q\n", endpoint) - return secureKubeconfig, nil - }) + // Validate the token in the cluster info + insecureKubeconfigBytes, err := validateClusterInfoToken(insecureClusterInfo, token) if err != nil { return nil, err } - return baseKubeConfig, nil + // Load the insecure config + insecureConfig, err := clientcmd.Load(insecureKubeconfigBytes) + if err != nil { + return nil, errors.Wrapf(err, "couldn't parse the kubeconfig file in the %s ConfigMap", bootstrapapi.ConfigMapClusterInfo) + } + + // The ConfigMap should contain a single cluster + if len(insecureConfig.Clusters) != 1 { + return nil, errors.Errorf("expected the kubeconfig file in the %s ConfigMap to have a single cluster, but it had %d", bootstrapapi.ConfigMapClusterInfo, len(insecureConfig.Clusters)) + } + + // If no TLS root CA pinning was specified, we're done + if pubKeyPins.Empty() { + klog.V(1).Infof("[discovery] Cluster info signature and contents are valid and no TLS pinning was specified, will use API Server %q", endpoint) + return insecureConfig, nil + } + + // Load and validate the cluster CA from the insecure kubeconfig + clusterCABytes, err := validateClusterCA(insecureConfig, pubKeyPins) + if err != nil { + return nil, err + } + + // Now that we know the cluster CA, connect back a second time validating with that CA + secureBootstrapConfig := buildSecureBootstrapKubeConfig(endpoint, clusterCABytes, clusterName) + + klog.V(1).Infof("[discovery] Requesting info from %q again to validate TLS against the pinned public key", endpoint) + secureClusterInfo, err := getClusterInfo(client, secureBootstrapConfig, token, interval, duration) + if err != nil { + return nil, err + } + + // Pull the kubeconfig from the securely-obtained ConfigMap and validate that it's the same as what we found the first time + secureKubeconfigBytes := []byte(secureClusterInfo.Data[bootstrapapi.KubeConfigKey]) + if !bytes.Equal(secureKubeconfigBytes, insecureKubeconfigBytes) { + return nil, errors.Errorf("the second kubeconfig from the %s ConfigMap (using validated TLS) was different from the first", bootstrapapi.ConfigMapClusterInfo) + } + + secureKubeconfig, err := clientcmd.Load(secureKubeconfigBytes) + if err != nil { + return nil, errors.Wrapf(err, "couldn't parse the kubeconfig file in the %s ConfigMap", bootstrapapi.ConfigMapClusterInfo) + } + + klog.V(1).Infof("[discovery] Cluster info signature and contents are valid and TLS certificate validates against pinned roots, will use API Server %q", endpoint) + + return secureKubeconfig, nil } // buildInsecureBootstrapKubeConfig makes a kubeconfig object that connects insecurely to the API Server for bootstrapping purposes @@ -174,42 +150,85 @@ func buildSecureBootstrapKubeConfig(endpoint string, caCert []byte, clustername return bootstrapConfig } -// fetchKubeConfigWithTimeout tries to run fetchKubeConfigFunc on every DiscoveryRetryInterval, but until discoveryTimeout is reached -func fetchKubeConfigWithTimeout(apiEndpoint string, discoveryTimeout time.Duration, fetchKubeConfigFunc func(string) (*clientcmdapi.Config, error)) (*clientcmdapi.Config, error) { - stopChan := make(chan struct{}) - var resultingKubeConfig *clientcmdapi.Config - var once sync.Once - var wg sync.WaitGroup - - wg.Add(1) - go func() { - defer wg.Done() - wait.Until(func() { - klog.V(1).Infof("[discovery] Trying to connect to API Server %q\n", apiEndpoint) - cfg, err := fetchKubeConfigFunc(apiEndpoint) - if err != nil { - klog.V(1).Infof("[discovery] Failed to connect to API Server %q: %v\n", apiEndpoint, err) - return - } - klog.V(1).Infof("[discovery] Successfully established connection with API Server %q\n", apiEndpoint) - once.Do(func() { - resultingKubeConfig = cfg - close(stopChan) - }) - }, constants.DiscoveryRetryInterval, stopChan) - }() - - select { - case <-time.After(discoveryTimeout): - once.Do(func() { - close(stopChan) - }) - err := errors.Errorf("abort connecting to API servers after timeout of %v", discoveryTimeout) - klog.V(1).Infof("[discovery] %v\n", err) - wg.Wait() - return nil, err - case <-stopChan: - wg.Wait() - return resultingKubeConfig, nil +// validateClusterInfoToken validates that the JWS token present in the cluster info ConfigMap is valid +func validateClusterInfoToken(insecureClusterInfo *v1.ConfigMap, token *kubeadmapi.BootstrapTokenString) ([]byte, error) { + insecureKubeconfigString, ok := insecureClusterInfo.Data[bootstrapapi.KubeConfigKey] + if !ok || len(insecureKubeconfigString) == 0 { + return nil, errors.Errorf("there is no %s key in the %s ConfigMap. This API Server isn't set up for token bootstrapping, can't connect", + bootstrapapi.KubeConfigKey, bootstrapapi.ConfigMapClusterInfo) } + + detachedJWSToken, ok := insecureClusterInfo.Data[bootstrapapi.JWSSignatureKeyPrefix+token.ID] + if !ok || len(detachedJWSToken) == 0 { + return nil, errors.Errorf("token id %q is invalid for this cluster or it has expired. Use \"kubeadm token create\" on the control-plane node to create a new valid token", token.ID) + } + + if !bootstrap.DetachedTokenIsValid(detachedJWSToken, insecureKubeconfigString, token.ID, token.Secret) { + return nil, errors.New("failed to verify JWS signature of received cluster info object, can't trust this API Server") + } + + return []byte(insecureKubeconfigString), nil +} + +// validateClusterCA validates the cluster CA found in the insecure kubeconfig +func validateClusterCA(insecureConfig *clientcmdapi.Config, pubKeyPins *pubkeypin.Set) ([]byte, error) { + var clusterCABytes []byte + for _, cluster := range insecureConfig.Clusters { + clusterCABytes = cluster.CertificateAuthorityData + } + + clusterCAs, err := certutil.ParseCertsPEM(clusterCABytes) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse cluster CA from the %s ConfigMap", bootstrapapi.ConfigMapClusterInfo) + } + + // Validate the cluster CA public key against the pinned set + err = pubKeyPins.CheckAny(clusterCAs) + if err != nil { + return nil, errors.Wrapf(err, "cluster CA found in %s ConfigMap is invalid", bootstrapapi.ConfigMapClusterInfo) + } + + return clusterCABytes, nil +} + +// getClusterInfo creates a client from the given kubeconfig if the given client is nil, +// and requests the cluster info ConfigMap using PollImmediate. +// If a client is provided it will be used instead. +func getClusterInfo(client clientset.Interface, kubeconfig *clientcmdapi.Config, token *kubeadmapi.BootstrapTokenString, interval, duration time.Duration) (*v1.ConfigMap, error) { + var cm *v1.ConfigMap + var err error + + // Create client from kubeconfig + if client == nil { + client, err = kubeconfigutil.ToClientSet(kubeconfig) + if err != nil { + return nil, err + } + } + + ctx, cancel := context.WithTimeout(context.TODO(), duration) + defer cancel() + + wait.JitterUntil(func() { + cm, err = client.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) + if err != nil { + klog.V(1).Infof("[discovery] Failed to request cluster-info, will try again: %v", err) + return + } + // Even if the ConfigMap is available the JWS signature is patched-in a bit later. + // Make sure we retry util then. + if _, ok := cm.Data[bootstrapapi.JWSSignatureKeyPrefix+token.ID]; !ok { + klog.V(1).Infof("[discovery] The cluster-info ConfigMap does not yet contain a JWS signature for token ID %q, will try again", token.ID) + err = errors.Errorf("could not find a JWS signature in the cluster-info ConfigMap for token ID %q", token.ID) + return + } + // Cancel the context on success + cancel() + }, interval, 0.3, true, ctx.Done()) + + if err != nil { + return nil, err + } + + return cm, nil } diff --git a/cmd/kubeadm/app/discovery/token/token_test.go b/cmd/kubeadm/app/discovery/token/token_test.go index 61fdacfd35a..3f27812d2c1 100644 --- a/cmd/kubeadm/app/discovery/token/token_test.go +++ b/cmd/kubeadm/app/discovery/token/token_test.go @@ -20,53 +20,293 @@ import ( "testing" "time" - "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + fakeclient "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/clientcmd" + bootstrapapi "k8s.io/cluster-bootstrap/token/api" + tokenjws "k8s.io/cluster-bootstrap/token/jws" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + "github.com/pmezard/go-difflib/difflib" ) -func TestFetchKubeConfigWithTimeout(t *testing.T) { - const testAPIEndpoint = "sample-endpoint:1234" +func TestRetrieveValidatedConfigInfo(t *testing.T) { + const ( + caCert = `-----BEGIN CERTIFICATE----- +MIICyDCCAbCgAwIBAgIBADANBgkqhkiG9w0BAQsFADAVMRMwEQYDVQQDEwprdWJl +cm5ldGVzMB4XDTE5MTEyMDAwNDk0MloXDTI5MTExNzAwNDk0MlowFTETMBEGA1UE +AxMKa3ViZXJuZXRlczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMqQ +ctECzA8yFSuVYupOUYgrTmfQeKe/9BaDWagaq7ow9+I2IvsfWFvlrD8QQr8sea6q +xjq7TV67Vb4RxBaoYDA+yI5vIcujWUxULun64lu3Q6iC1sj2UnmUpIdgazRXXEkZ +vxA6EbAnoxA0+lBOn1CZWl23IQ4s70o2hZ7wIp/vevB88RRRjqtvgc5elsjsbmDF +LS7L1Zuye8c6gS93bR+VjVmSIfr1IEq0748tIIyXjAVCWPVCvuP41MlfPc/JVpZD +uD2+pO6ZYREcdAnOf2eD4/eLOMKko4L1dSFy9JKM5PLnOC0Zk0AYOd1vS8DTAfxj +XPEIY8OBYFhlsxf4TE8CAwEAAaMjMCEwDgYDVR0PAQH/BAQDAgKkMA8GA1UdEwEB +/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAH/OYq8zyl1+zSTmuow3yI/15PL1 +dl8hB7IKnZNWmC/LTdm/+noh3Sb1IdRv6HkKg/GUn0UMuRUngLhju3EO4ozJPQcX +quaxzgmTKNWJ6ErDvRvWhGX0ZcbdBfZv+dowyRqzd5nlJ49hC+NrtFFQq6P05BYn +7SemguqeXmXwIj2Sa+1DeR6lRm9o8shAYjnyThUFqaMn18kI3SANJ5vk/3DFrPEO +CKC9EzFku2kuxg2dM12PbRGZQ2o0K6HEZgrrIKTPOy3ocb8r9M0aSFhjOV/NqGA4 +SaupXSW6XfvIi/UHoIbU3pNcsnUJGnQfQvip95XKk/gqcUr+m50vxgumxtA= +-----END CERTIFICATE-----` + + caCertHash = "sha256:98be2e6d4d8a89aa308fb15de0c07e2531ce549c68dec1687cdd5c06f0826658" + + expectedKubeconfig = `apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN5RENDQWJDZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRFNU1URXlNREF3TkRrME1sb1hEVEk1TVRFeE56QXdORGswTWxvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTXFRCmN0RUN6QTh5RlN1Vll1cE9VWWdyVG1mUWVLZS85QmFEV2FnYXE3b3c5K0kySXZzZldGdmxyRDhRUXI4c2VhNnEKeGpxN1RWNjdWYjRSeEJhb1lEQSt5STV2SWN1aldVeFVMdW42NGx1M1E2aUMxc2oyVW5tVXBJZGdhelJYWEVrWgp2eEE2RWJBbm94QTArbEJPbjFDWldsMjNJUTRzNzBvMmhaN3dJcC92ZXZCODhSUlJqcXR2Z2M1ZWxzanNibURGCkxTN0wxWnV5ZThjNmdTOTNiUitWalZtU0lmcjFJRXEwNzQ4dElJeVhqQVZDV1BWQ3Z1UDQxTWxmUGMvSlZwWkQKdUQyK3BPNlpZUkVjZEFuT2YyZUQ0L2VMT01La280TDFkU0Z5OUpLTTVQTG5PQzBaazBBWU9kMXZTOERUQWZ4agpYUEVJWThPQllGaGxzeGY0VEU4Q0F3RUFBYU1qTUNFd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFIL09ZcTh6eWwxK3pTVG11b3czeUkvMTVQTDEKZGw4aEI3SUtuWk5XbUMvTFRkbS8rbm9oM1NiMUlkUnY2SGtLZy9HVW4wVU11UlVuZ0xoanUzRU80b3pKUFFjWApxdWF4emdtVEtOV0o2RXJEdlJ2V2hHWDBaY2JkQmZaditkb3d5UnF6ZDVubEo0OWhDK05ydEZGUXE2UDA1QlluCjdTZW1ndXFlWG1Yd0lqMlNhKzFEZVI2bFJtOW84c2hBWWpueVRoVUZxYU1uMThrSTNTQU5KNXZrLzNERnJQRU8KQ0tDOUV6Rmt1Mmt1eGcyZE0xMlBiUkdaUTJvMEs2SEVaZ3JySUtUUE95M29jYjhyOU0wYVNGaGpPVi9OcUdBNApTYXVwWFNXNlhmdklpL1VIb0liVTNwTmNzblVKR25RZlF2aXA5NVhLay9ncWNVcittNTB2eGd1bXh0QT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== + server: https://127.0.0.1 + name: somecluster +contexts: +- context: + cluster: somecluster + user: token-bootstrap-client + name: token-bootstrap-client@somecluster +current-context: token-bootstrap-client@somecluster +kind: Config +preferences: {} +users: null +` + ) + tests := []struct { - name string - discoveryTimeout time.Duration - shouldFail bool + name string + tokenID string + tokenSecret string + cfg *kubeadmapi.Discovery + configMap *fakeConfigMap + delayedJWSSignaturePatch bool + expectedError bool }{ { - name: "Timeout if value is not returned on time", - discoveryTimeout: 1 * time.Second, - shouldFail: true, + // This is the default behavior. The JWS signature is patched after the cluster-info ConfigMap is created + name: "valid: retrieve a valid kubeconfig with CA verification and delayed JWS signature", + tokenID: "123456", + tokenSecret: "abcdef1234567890", + cfg: &kubeadmapi.Discovery{ + BootstrapToken: &kubeadmapi.BootstrapTokenDiscovery{ + Token: "123456.abcdef1234567890", + CACertHashes: []string{caCertHash}, + }, + }, + configMap: &fakeConfigMap{ + name: bootstrapapi.ConfigMapClusterInfo, + data: map[string]string{}, + }, + delayedJWSSignaturePatch: true, }, { - name: "Don't timeout if value is returned on time", - discoveryTimeout: 5 * time.Second, - shouldFail: false, + // Same as above expect this test creates the ConfigMap with the JWS signature + name: "valid: retrieve a valid kubeconfig with CA verification", + tokenID: "123456", + tokenSecret: "abcdef1234567890", + cfg: &kubeadmapi.Discovery{ + BootstrapToken: &kubeadmapi.BootstrapTokenDiscovery{ + Token: "123456.abcdef1234567890", + CACertHashes: []string{caCertHash}, + }, + }, + configMap: &fakeConfigMap{ + name: bootstrapapi.ConfigMapClusterInfo, + data: nil, + }, + }, + { + // Skipping CA verification is also supported + name: "valid: retrieve a valid kubeconfig without CA verification", + tokenID: "123456", + tokenSecret: "abcdef1234567890", + cfg: &kubeadmapi.Discovery{ + BootstrapToken: &kubeadmapi.BootstrapTokenDiscovery{ + Token: "123456.abcdef1234567890", + }, + }, + configMap: &fakeConfigMap{ + name: bootstrapapi.ConfigMapClusterInfo, + data: nil, + }, + }, + { + name: "invalid: token format is invalid", + tokenID: "foo", + tokenSecret: "bar", + cfg: &kubeadmapi.Discovery{ + BootstrapToken: &kubeadmapi.BootstrapTokenDiscovery{ + Token: "foo.bar", + }, + }, + configMap: &fakeConfigMap{ + name: bootstrapapi.ConfigMapClusterInfo, + data: nil, + }, + expectedError: true, + }, + { + name: "invalid: missing cluster-info ConfigMap", + tokenID: "123456", + tokenSecret: "abcdef1234567890", + cfg: &kubeadmapi.Discovery{ + BootstrapToken: &kubeadmapi.BootstrapTokenDiscovery{ + Token: "123456.abcdef1234567890", + }, + }, + configMap: &fakeConfigMap{ + name: "baz", + data: nil, + }, + expectedError: true, + }, + { + name: "invalid: wrong JWS signature", + tokenID: "123456", + tokenSecret: "abcdef1234567890", + cfg: &kubeadmapi.Discovery{ + BootstrapToken: &kubeadmapi.BootstrapTokenDiscovery{ + Token: "123456.abcdef1234567890", + }, + }, + configMap: &fakeConfigMap{ + name: bootstrapapi.ConfigMapClusterInfo, + data: map[string]string{ + bootstrapapi.KubeConfigKey: "foo", + bootstrapapi.JWSSignatureKeyPrefix + "123456": "bar", + }, + }, + expectedError: true, + }, + { + name: "invalid: missing key for JWSSignatureKeyPrefix", + tokenID: "123456", + tokenSecret: "abcdef1234567890", + cfg: &kubeadmapi.Discovery{ + BootstrapToken: &kubeadmapi.BootstrapTokenDiscovery{ + Token: "123456.abcdef1234567890", + }, + }, + configMap: &fakeConfigMap{ + name: bootstrapapi.ConfigMapClusterInfo, + data: map[string]string{ + bootstrapapi.KubeConfigKey: "foo", + }, + }, + expectedError: true, + }, + { + name: "invalid: wrong CA cert hash", + tokenID: "123456", + tokenSecret: "abcdef1234567890", + cfg: &kubeadmapi.Discovery{ + BootstrapToken: &kubeadmapi.BootstrapTokenDiscovery{ + Token: "123456.abcdef1234567890", + CACertHashes: []string{"foo"}, + }, + }, + configMap: &fakeConfigMap{ + name: bootstrapapi.ConfigMapClusterInfo, + data: nil, + }, + expectedError: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - cfg, err := fetchKubeConfigWithTimeout(testAPIEndpoint, test.discoveryTimeout, func(apiEndpoint string) (*clientcmdapi.Config, error) { - if apiEndpoint != testAPIEndpoint { - return nil, errors.Errorf("unexpected API server endpoint:\n\texpected: %q\n\tgot: %q", testAPIEndpoint, apiEndpoint) - } + kubeconfig := buildSecureBootstrapKubeConfig("127.0.0.1", []byte(caCert), "somecluster") + kubeconfigBytes, err := clientcmd.Write(*kubeconfig) + if err != nil { + t.Fatalf("cannot marshal kubeconfig %v", err) + } - time.Sleep(3 * time.Second) - return &clientcmdapi.Config{}, nil - }) + // Generate signature of the insecure kubeconfig + sig, err := tokenjws.ComputeDetachedSignature(string(kubeconfigBytes), test.tokenID, test.tokenSecret) + if err != nil { + t.Fatalf("cannot compute detached JWS signature: %v", err) + } - if test.shouldFail { - if err == nil { - t.Fatal("unexpected success") + // If the JWS signature is delayed, only add the kubeconfig + if test.delayedJWSSignaturePatch { + test.configMap.data = map[string]string{} + test.configMap.data[bootstrapapi.KubeConfigKey] = string(kubeconfigBytes) + } + + // Populate the default cluster-info data + if test.configMap.data == nil { + test.configMap.data = map[string]string{} + test.configMap.data[bootstrapapi.KubeConfigKey] = string(kubeconfigBytes) + test.configMap.data[bootstrapapi.JWSSignatureKeyPrefix+test.tokenID] = sig + } + + // Create a fake client and create the cluster-info ConfigMap + client := fakeclient.NewSimpleClientset() + if err = test.configMap.createOrUpdate(client); err != nil { + t.Fatalf("could not create ConfigMap: %v", err) + } + + // Set arbitrary discovery timeout and retry interval + test.cfg.Timeout = &metav1.Duration{Duration: time.Millisecond * 200} + interval := time.Millisecond * 20 + + // Patch the JWS signature after a short delay + if test.delayedJWSSignaturePatch { + test.configMap.data[bootstrapapi.JWSSignatureKeyPrefix+test.tokenID] = sig + go func() { + time.Sleep(time.Millisecond * 60) + if err := test.configMap.createOrUpdate(client); err != nil { + t.Errorf("could not update the cluster-info ConfigMap with a JWS signature: %v", err) + } + }() + } + + // Retrieve validated configuration + kubeconfig, err = retrieveValidatedConfigInfo(client, test.cfg, interval) + if (err != nil) != test.expectedError { + t.Errorf("expected error %v, got %v, error: %v", test.expectedError, err != nil, err) + } + + // Return if an error is expected + if test.expectedError { + return + } + + // Validate the resulted kubeconfig + kubeconfigBytes, err = clientcmd.Write(*kubeconfig) + if err != nil { + t.Fatalf("cannot marshal resulted kubeconfig %v", err) + } + if string(kubeconfigBytes) != expectedKubeconfig { + t.Error("unexpected kubeconfig") + diff := difflib.UnifiedDiff{ + A: difflib.SplitLines(expectedKubeconfig), + B: difflib.SplitLines(string(kubeconfigBytes)), + FromFile: "expected", + ToFile: "got", + Context: 10, } - } else { + diffstr, err := difflib.GetUnifiedDiffString(diff) if err != nil { - t.Fatalf("unexpected failure: %v", err) - } - if cfg == nil { - t.Fatal("cfg is nil") + t.Fatalf("error generating unified diff string: %v", err) } + t.Errorf("\n%s", diffstr) } }) } } + +type fakeConfigMap struct { + name string + data map[string]string +} + +func (c *fakeConfigMap) createOrUpdate(client clientset.Interface) error { + return apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: c.name, + Namespace: metav1.NamespacePublic, + }, + Data: c.data, + }) +}