diff --git a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go index 42acb89bc62..c1fcca242b7 100644 --- a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go +++ b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go @@ -264,10 +264,36 @@ func validateKubeConfig(outDir, filename string, config *clientcmdapi.Config) er } caExpected := bytes.TrimSpace(config.Clusters[expectedCluster].CertificateAuthorityData) - // If the current CA cert on disk doesn't match the expected CA cert, error out because we have a file, but it's stale - if !bytes.Equal(caCurrent, caExpected) { - return errors.Errorf("a kubeconfig file %q exists already but has got the wrong CA cert", kubeConfigFilePath) + // Parse the current certificate authority data + currentCACerts, err := certutil.ParseCertsPEM(caCurrent) + if err != nil { + return errors.Errorf("the kubeconfig file %q contains an invalid CA cert", kubeConfigFilePath) } + + // Parse the expected certificate authority data + expectedCACerts, err := certutil.ParseCertsPEM(caExpected) + if err != nil { + return errors.Errorf("the expected base64 encoded CA cert could not be parsed as a PEM:\n%s\n", caExpected) + } + + // Only use the first certificate in the current CA cert list + currentCaCert := currentCACerts[0] + + // Find a common trust anchor + trustAnchorFound := false + for _, expectedCaCert := range expectedCACerts { + // Compare the current CA cert to the expected CA cert. + // If the certificates match then a common trust anchor was found. + if currentCaCert.Equal(expectedCaCert) { + trustAnchorFound = true + break + } + } + if !trustAnchorFound { + return errors.Errorf("a kubeconfig file %q exists but does not contain a trusted CA in its current context's "+ + "cluster. Total CA certificates found: %d", kubeConfigFilePath, len(currentCACerts)) + } + // If the current API Server location on disk doesn't match the expected API server, show a warning if currentConfig.Clusters[currentCluster].Server != config.Clusters[expectedCluster].Server { klog.Warningf("a kubeconfig file %q exists already but has an unexpected API Server URL: expected: %s, got: %s", @@ -386,12 +412,18 @@ func writeKubeConfigFromSpec(out io.Writer, spec *kubeConfigSpec, clustername st func ValidateKubeconfigsForExternalCA(outDir string, cfg *kubeadmapi.InitConfiguration) error { // Creates a kubeconfig file with the target CA and server URL // to be used as a input for validating user provided kubeconfig files - caCert, err := pkiutil.TryLoadCertFromDisk(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) + caCert, intermediaries, err := pkiutil.TryLoadCertChainFromDisk(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) if err != nil { return errors.Wrapf(err, "the CA file couldn't be loaded") } + + // Combine caCert and intermediaries into one array + caCertChain := append([]*x509.Certificate{caCert}, intermediaries...) + // Validate period - certsphase.CheckCertificatePeriodValidity(kubeadmconstants.CACertAndKeyBaseName, caCert) + for _, cert := range caCertChain { + certsphase.CheckCertificatePeriodValidity(kubeadmconstants.CACertAndKeyBaseName, cert) + } // validate user provided kubeconfig files for the scheduler and controller-manager localAPIEndpoint, err := kubeadmutil.GetLocalAPIEndpoint(&cfg.LocalAPIEndpoint) @@ -399,7 +431,12 @@ func ValidateKubeconfigsForExternalCA(outDir string, cfg *kubeadmapi.InitConfigu return err } - validationConfigLocal := kubeconfigutil.CreateBasic(localAPIEndpoint, "dummy", "dummy", pkiutil.EncodeCertPEM(caCert)) + caCertBytes, err := pkiutil.EncodeCertBundlePEM(caCertChain) + if err != nil { + return err + } + + validationConfigLocal := kubeconfigutil.CreateBasic(localAPIEndpoint, "dummy", "dummy", caCertBytes) kubeConfigFileNamesLocal := []string{ kubeadmconstants.ControllerManagerKubeConfigFileName, kubeadmconstants.SchedulerKubeConfigFileName, @@ -417,7 +454,7 @@ func ValidateKubeconfigsForExternalCA(outDir string, cfg *kubeadmapi.InitConfigu return err } - validationConfigCPE := kubeconfigutil.CreateBasic(controlPlaneEndpoint, "dummy", "dummy", pkiutil.EncodeCertPEM(caCert)) + validationConfigCPE := kubeconfigutil.CreateBasic(controlPlaneEndpoint, "dummy", "dummy", caCertBytes) kubeConfigFileNamesCPE := []string{ kubeadmconstants.AdminKubeConfigFileName, kubeadmconstants.SuperAdminKubeConfigFileName, diff --git a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go index f36dee1d3ba..0418beca5b5 100644 --- a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go +++ b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go @@ -608,7 +608,11 @@ func TestValidateKubeConfig(t *testing.T) { func TestValidateKubeconfigsForExternalCA(t *testing.T) { tmpDir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpDir) + defer func() { + if err := os.RemoveAll(tmpDir); err != nil { + t.Error(err) + } + }() pkiDir := filepath.Join(tmpDir, "pki") initConfig := &kubeadmapi.InitConfiguration{ @@ -623,11 +627,9 @@ func TestValidateKubeconfigsForExternalCA(t *testing.T) { // creates CA, write to pkiDir and remove ca.key to get into external CA condition caCert, caKey := certstestutil.SetupCertificateAuthority(t) - if err := pkiutil.WriteCertAndKey(pkiDir, kubeadmconstants.CACertAndKeyBaseName, caCert, caKey); err != nil { - t.Fatalf("failure while saving CA certificate and key: %v", err) - } - if err := os.Remove(filepath.Join(pkiDir, kubeadmconstants.CAKeyName)); err != nil { - t.Fatalf("failure while deleting ca.key: %v", err) + + if err := pkiutil.WriteCertBundle(pkiDir, kubeadmconstants.CACertAndKeyBaseName, []*x509.Certificate{caCert}); err != nil { + t.Fatalf("failure while saving CA certificate: %v", err) } notAfter, _ := time.Parse(time.RFC3339, "2026-01-02T15:04:05Z") @@ -697,7 +699,11 @@ func TestValidateKubeconfigsForExternalCA(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { tmpdir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpdir) + defer func() { + if err := os.RemoveAll(tmpdir); err != nil { + t.Error(err) + } + }() for name, config := range test.filesToWrite { if err := createKubeConfigFileIfNotExists(tmpdir, name, config); err != nil { @@ -719,6 +725,166 @@ func TestValidateKubeconfigsForExternalCA(t *testing.T) { } } +func TestValidateKubeconfigsForExternalCAMissingRoot(t *testing.T) { + tmpDir := testutil.SetupTempDir(t) + defer func() { + if err := os.RemoveAll(tmpDir); err != nil { + t.Error(err) + } + }() + pkiDir := filepath.Join(tmpDir, "pki") + + initConfig := &kubeadmapi.InitConfiguration{ + ClusterConfiguration: kubeadmapi.ClusterConfiguration{ + CertificatesDir: pkiDir, + }, + LocalAPIEndpoint: kubeadmapi.APIEndpoint{ + BindPort: 1234, + AdvertiseAddress: "1.2.3.4", + }, + } + + // Creates CA, write to pkiDir and remove ca.key to get into external CA mode + caCert, caKey := certstestutil.SetupCertificateAuthority(t) + + // Setup multiple intermediate certificate authorities (CAs) for testing purposes. + // This is "Root CA" signs "Intermediate Authority 1A" signs "Intermediate Authority 2A" + intermediateCACert1a, intermediateCAKey1a := certstestutil.SetupIntermediateCertificateAuthority(t, caCert, caKey, "Intermediate Authority 1A") + intermediateCACert2a, intermediateCAKey2a := certstestutil.SetupIntermediateCertificateAuthority(t, intermediateCACert1a, intermediateCAKey1a, "Intermediate Authority 1A") + + // These two CA certificates should both validate using the Intermediate CA 2B certificate + // This is "Root CA" signs "Intermediate Authority 1B" signs "Intermediate Authority 2B" + intermediateCACert1b, intermediateCAKey1b := certstestutil.SetupIntermediateCertificateAuthority(t, caCert, caKey, "Intermediate Authority 1B") + intermediateCACert2b, intermediateCAKey2b := certstestutil.SetupIntermediateCertificateAuthority(t, intermediateCACert1b, intermediateCAKey1b, "Intermediate Authority 2B") + + notAfter, _ := time.Parse(time.RFC3339, "2036-01-02T15:04:05Z") + clusterName := "myOrg1" + + var validCaCertBundle []*x509.Certificate + validCaCertBundle = append(validCaCertBundle, caCert, intermediateCACert1a, intermediateCACert2a) + multipleCAConfigRootCAIssuer := setupKubeConfigWithClientAuth(t, caCert, caKey, notAfter, "https://1.2.3.4:1234", "test-cluster", clusterName) + multipleCAConfigIntermediateCA1aIssuer := setupKubeConfigWithClientAuth(t, intermediateCACert1a, intermediateCAKey1a, notAfter, "https://1.2.3.4:1234", "test-cluster", clusterName) + multipleCAConfigIntermediateCA2aIssuer := setupKubeConfigWithClientAuth(t, intermediateCACert2a, intermediateCAKey2a, notAfter, "https://1.2.3.4:1234", "test-cluster", clusterName) + + var caBundleMissingRootCA []*x509.Certificate + caBundleMissingRootCA = append(caBundleMissingRootCA, intermediateCACert1b, intermediateCACert2b) + multipleCAConfigNoRootCA := setupKubeConfigWithClientAuth(t, intermediateCACert2b, intermediateCAKey2b, notAfter, "https://1.2.3.4:1234", "test-cluster", clusterName) + multipleCAConfigDifferentIssuer := setupKubeConfigWithClientAuth(t, intermediateCACert2a, intermediateCAKey2a, notAfter, "https://1.2.3.4:1234", "test-cluster", clusterName) + + var caBundlePartialChain []*x509.Certificate + caBundlePartialChain = append(caBundlePartialChain, intermediateCACert1a) + multipleCaPartialCA := setupKubeConfigWithClientAuth(t, intermediateCACert2b, intermediateCAKey2b, notAfter, "https://1.2.3.4:1234", "test-cluster", clusterName) + + tests := map[string]struct { + filesToWrite map[string]*clientcmdapi.Config + initConfig *kubeadmapi.InitConfiguration + expectedError bool + caCertificate []*x509.Certificate + }{ + // Positive test cases + "valid config issued from RootCA": { + filesToWrite: map[string]*clientcmdapi.Config{ + kubeadmconstants.AdminKubeConfigFileName: multipleCAConfigRootCAIssuer, + kubeadmconstants.SuperAdminKubeConfigFileName: multipleCAConfigRootCAIssuer, + kubeadmconstants.KubeletKubeConfigFileName: multipleCAConfigRootCAIssuer, + kubeadmconstants.ControllerManagerKubeConfigFileName: multipleCAConfigRootCAIssuer, + kubeadmconstants.SchedulerKubeConfigFileName: multipleCAConfigRootCAIssuer, + }, + caCertificate: validCaCertBundle, + initConfig: initConfig, + expectedError: false, + }, + "valid config issued from IntermediateCA 1A": { + filesToWrite: map[string]*clientcmdapi.Config{ + kubeadmconstants.AdminKubeConfigFileName: multipleCAConfigIntermediateCA1aIssuer, + kubeadmconstants.SuperAdminKubeConfigFileName: multipleCAConfigIntermediateCA1aIssuer, + kubeadmconstants.KubeletKubeConfigFileName: multipleCAConfigIntermediateCA1aIssuer, + kubeadmconstants.ControllerManagerKubeConfigFileName: multipleCAConfigIntermediateCA1aIssuer, + kubeadmconstants.SchedulerKubeConfigFileName: multipleCAConfigIntermediateCA1aIssuer, + }, + caCertificate: validCaCertBundle, + initConfig: initConfig, + expectedError: false, + }, + "valid config issued from IntermediateCA 2A": { + filesToWrite: map[string]*clientcmdapi.Config{ + kubeadmconstants.AdminKubeConfigFileName: multipleCAConfigIntermediateCA2aIssuer, + kubeadmconstants.SuperAdminKubeConfigFileName: multipleCAConfigIntermediateCA2aIssuer, + kubeadmconstants.KubeletKubeConfigFileName: multipleCAConfigIntermediateCA2aIssuer, + kubeadmconstants.ControllerManagerKubeConfigFileName: multipleCAConfigIntermediateCA2aIssuer, + kubeadmconstants.SchedulerKubeConfigFileName: multipleCAConfigIntermediateCA2aIssuer, + }, + caCertificate: validCaCertBundle, + initConfig: initConfig, + expectedError: false, + }, + "valid config issued from IntermediateCA 2B, CA missing root certificate": { + filesToWrite: map[string]*clientcmdapi.Config{ + kubeadmconstants.AdminKubeConfigFileName: multipleCAConfigNoRootCA, + kubeadmconstants.SuperAdminKubeConfigFileName: multipleCAConfigNoRootCA, + kubeadmconstants.KubeletKubeConfigFileName: multipleCAConfigNoRootCA, + kubeadmconstants.ControllerManagerKubeConfigFileName: multipleCAConfigNoRootCA, + kubeadmconstants.SchedulerKubeConfigFileName: multipleCAConfigNoRootCA, + }, + caCertificate: caBundleMissingRootCA, + initConfig: initConfig, + expectedError: false, + }, + // Negative test cases + "invalid config issued from IntermediateCA 2A, testing a chain with a different issuer": { + filesToWrite: map[string]*clientcmdapi.Config{ + kubeadmconstants.AdminKubeConfigFileName: multipleCAConfigDifferentIssuer, + kubeadmconstants.SuperAdminKubeConfigFileName: multipleCAConfigDifferentIssuer, + kubeadmconstants.KubeletKubeConfigFileName: multipleCAConfigDifferentIssuer, + kubeadmconstants.ControllerManagerKubeConfigFileName: multipleCAConfigDifferentIssuer, + kubeadmconstants.SchedulerKubeConfigFileName: multipleCAConfigDifferentIssuer, + }, + caCertificate: caBundleMissingRootCA, + initConfig: initConfig, + expectedError: true, + }, + "invalid config issued from IntermediateCA 2B chain, CA only contains Intermediate 1A": { + filesToWrite: map[string]*clientcmdapi.Config{ + kubeadmconstants.AdminKubeConfigFileName: multipleCaPartialCA, + kubeadmconstants.SuperAdminKubeConfigFileName: multipleCaPartialCA, + kubeadmconstants.KubeletKubeConfigFileName: multipleCaPartialCA, + kubeadmconstants.ControllerManagerKubeConfigFileName: multipleCaPartialCA, + kubeadmconstants.SchedulerKubeConfigFileName: multipleCaPartialCA, + }, + caCertificate: caBundlePartialChain, + initConfig: initConfig, + expectedError: true, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + tmpdir := testutil.SetupTempDir(t) + defer func() { + if err := os.RemoveAll(tmpdir); err != nil { + t.Error(err) + } + }() + + for name, config := range test.filesToWrite { + if err := createKubeConfigFileIfNotExists(tmpdir, name, config); err != nil { + t.Errorf("createKubeConfigFileIfNotExists failed: %v", err) + } + } + + if err := pkiutil.WriteCertBundle(pkiDir, kubeadmconstants.CACertAndKeyBaseName, test.caCertificate); err != nil { + t.Fatalf("Failure while saving CA certificate: %v", err) + } + + err := ValidateKubeconfigsForExternalCA(tmpdir, test.initConfig) + if (err != nil) != test.expectedError { + t.Fatalf("ValidateKubeconfigsForExternalCA failed\n%s\nexpected error: %t\n\tgot: %t\nerror: %v", + name, test.expectedError, (err != nil), err) + } + }) + } +} + // setupKubeConfigWithClientAuth is a test utility function that wraps buildKubeConfigFromSpec for building a KubeConfig object With ClientAuth func setupKubeConfigWithClientAuth(t *testing.T, caCert *x509.Certificate, caKey crypto.Signer, notAfter time.Time, apiServer, clientName, clustername string, organizations ...string) *clientcmdapi.Config { spec := &kubeConfigSpec{ @@ -740,7 +906,7 @@ func setupKubeConfigWithClientAuth(t *testing.T, caCert *x509.Certificate, caKey return config } -// setupKubeConfigWithClientAuth is a test utility function that wraps buildKubeConfigFromSpec for building a KubeConfig object With Token +// setupKubeConfigWithTokenAuth is a test utility function that wraps buildKubeConfigFromSpec for building a KubeConfig object With Token func setupKubeConfigWithTokenAuth(t *testing.T, caCert *x509.Certificate, apiServer, clientName, token, clustername string) *clientcmdapi.Config { spec := &kubeConfigSpec{ CACert: caCert, diff --git a/cmd/kubeadm/app/util/certs/util.go b/cmd/kubeadm/app/util/certs/util.go index 78198f02f18..4aefbbafc10 100644 --- a/cmd/kubeadm/app/util/certs/util.go +++ b/cmd/kubeadm/app/util/certs/util.go @@ -38,7 +38,20 @@ func SetupCertificateAuthority(t *testing.T) (*x509.Certificate, crypto.Signer) Config: certutil.Config{CommonName: "kubernetes"}, }) if err != nil { - t.Fatalf("failure while generating CA certificate and key: %v", err) + t.Fatalf("Failure while generating CA certificate and key: %v", err) + } + + return caCert, caKey +} + +// SetupIntermediateCertificateAuthority is a utility function for kubeadm testing that creates a +// Intermediate CertificateAuthority cert/key pair +func SetupIntermediateCertificateAuthority(t *testing.T, parentCert *x509.Certificate, parentKey crypto.Signer, cn string) (*x509.Certificate, crypto.Signer) { + caCert, caKey, err := pkiutil.NewIntermediateCertificateAuthority(parentCert, parentKey, &pkiutil.CertConfig{ + Config: certutil.Config{CommonName: cn}, + }) + if err != nil { + t.Fatalf("Failure while generating intermediate CA certificate and key: %v", err) } return caCert, caKey