diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index c099d35f8d0..60caf43bed6 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -277,14 +277,20 @@ func (i *Init) Run(out io.Writer) error { adminKubeConfigPath := filepath.Join(kubeConfigDir, kubeadmconstants.AdminKubeConfigFileName) - // PHASE 1: Generate certificates - if err := certsphase.CreatePKIAssets(i.cfg); err != nil { - return err - } + if res, _ := certsphase.UsingExternalCA(i.cfg); !res { - // PHASE 2: Generate kubeconfig files for the admin and the kubelet - if err := kubeconfigphase.CreateInitKubeConfigFiles(kubeConfigDir, i.cfg); err != nil { - return err + // PHASE 1: Generate certificates + if err := certsphase.CreatePKIAssets(i.cfg); err != nil { + return err + } + + // PHASE 2: Generate kubeconfig files for the admin and the kubelet + if err := kubeconfigphase.CreateInitKubeConfigFiles(kubeConfigDir, i.cfg); err != nil { + return err + } + + } else { + fmt.Println("[externalca] No ca.key detected, but all other certificates are available, so using external CA mode. Will not generate certs or kubeconfig.") } // Temporarily set cfg.CertificatesDir to the "real value" when writing controlplane manifests diff --git a/cmd/kubeadm/app/phases/certs/certs.go b/cmd/kubeadm/app/phases/certs/certs.go index a6b060a0da2..f96fe7ef7dd 100644 --- a/cmd/kubeadm/app/phases/certs/certs.go +++ b/cmd/kubeadm/app/phases/certs/certs.go @@ -21,6 +21,8 @@ import ( "crypto/x509" "fmt" "net" + "os" + "path/filepath" certutil "k8s.io/client-go/util/cert" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" @@ -398,6 +400,110 @@ func writeKeyFilesIfNotExist(pkiDir string, baseName string, key *rsa.PrivateKey return nil } +type certKeyLocation struct { + pkiDir string + caBaseName string + baseName string + uxName string +} + +// UsingExternalCA determines whether the user is relying on an external CA. We currently implicitly determine this is the case when the CA Cert +// is present but the CA Key is not. This allows us to, e.g., skip generating certs or not start the csr signing controller. +func UsingExternalCA(cfg *kubeadmapi.MasterConfiguration) (bool, error) { + + if err := validateCACert(certKeyLocation{cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName, "", "CA"}); err != nil { + return false, err + } + + caKeyPath := filepath.Join(cfg.CertificatesDir, kubeadmconstants.CAKeyName) + if _, err := os.Stat(caKeyPath); !os.IsNotExist(err) { + return false, fmt.Errorf("ca.key exists") + } + + if err := validateSignedCert(certKeyLocation{cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName, kubeadmconstants.APIServerCertAndKeyBaseName, "API server"}); err != nil { + return false, err + } + + if err := validateSignedCert(certKeyLocation{cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName, kubeadmconstants.APIServerKubeletClientCertAndKeyBaseName, "API server kubelet client"}); err != nil { + return false, err + } + + if err := validatePrivatePublicKey(certKeyLocation{cfg.CertificatesDir, "", kubeadmconstants.ServiceAccountKeyBaseName, "service account"}); err != nil { + return false, err + } + + if err := validateCACertAndKey(certKeyLocation{cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertAndKeyBaseName, "", "front-proxy CA"}); err != nil { + return false, err + } + + if err := validateSignedCert(certKeyLocation{cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertAndKeyBaseName, kubeadmconstants.FrontProxyClientCertAndKeyBaseName, "front-proxy client"}); err != nil { + return false, err + } + + return true, nil +} + +// validateCACert tries to load a x509 certificate from pkiDir and validates that it is a CA +func validateCACert(l certKeyLocation) error { + // Check CA Cert + caCert, err := pkiutil.TryLoadCertFromDisk(l.pkiDir, l.caBaseName) + if err != nil { + return fmt.Errorf("failure loading certificate for %s: %v", l.uxName, err) + } + + // Check if cert is a CA + if !caCert.IsCA { + return fmt.Errorf("certificate %s is not a CA", l.uxName) + } + return nil +} + +// validateCACertAndKey tries to load a x509 certificate and private key from pkiDir, +// and validates that the cert is a CA +func validateCACertAndKey(l certKeyLocation) error { + if err := validateCACert(l); err != nil { + return err + } + + _, err := pkiutil.TryLoadKeyFromDisk(l.pkiDir, l.caBaseName) + if err != nil { + return fmt.Errorf("failure loading key for %s: %v", l.uxName, err) + } + return nil +} + +// validateSignedCert tries to load a x509 certificate and private key from pkiDir and validates +// that the cert is signed by a given CA +func validateSignedCert(l certKeyLocation) error { + // Try to load CA + caCert, err := pkiutil.TryLoadCertFromDisk(l.pkiDir, l.caBaseName) + if err != nil { + return fmt.Errorf("failure loading certificate authorithy for %s: %v", l.uxName, err) + } + + // Try to load key and signed certificate + signedCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(l.pkiDir, l.baseName) + if err != nil { + return fmt.Errorf("failure loading certificate for %s: %v", l.uxName, err) + } + + // Check if the cert is signed by the CA + if err := signedCert.CheckSignatureFrom(caCert); err != nil { + return fmt.Errorf("certificate %s is not signed by corresponding CA", l.uxName) + } + return nil +} + +// validatePrivatePublicKey tries to load a private key from pkiDir +func validatePrivatePublicKey(l certKeyLocation) error { + // Try to load key + _, _, err := pkiutil.TryLoadPrivatePublicKeyFromDisk(l.pkiDir, l.baseName) + if err != nil { + return fmt.Errorf("failure loading key for %s: %v", l.uxName, err) + } + return nil +} + // getAltNames builds an AltNames object for to be used when generating apiserver certificate func getAltNames(cfg *kubeadmapi.MasterConfiguration) (*certutil.AltNames, error) { diff --git a/cmd/kubeadm/app/phases/certs/certs_test.go b/cmd/kubeadm/app/phases/certs/certs_test.go index 5840e648524..a22f2e4ae77 100644 --- a/cmd/kubeadm/app/phases/certs/certs_test.go +++ b/cmd/kubeadm/app/phases/certs/certs_test.go @@ -19,14 +19,15 @@ package certs import ( "crypto/rsa" "crypto/x509" + "fmt" "net" "os" + "path/filepath" "testing" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil" - testutil "k8s.io/kubernetes/cmd/kubeadm/test" certstestutil "k8s.io/kubernetes/cmd/kubeadm/test/certs" ) @@ -381,6 +382,151 @@ func TestNewFrontProxyClientCertAndKey(t *testing.T) { certstestutil.AssertCertificateHasClientAuthUsage(t, frontProxyClientCert) } +func TestUsingExternalCA(t *testing.T) { + + tests := []struct { + setupFuncs []func(cfg *kubeadmapi.MasterConfiguration) error + expected bool + }{ + { + setupFuncs: []func(cfg *kubeadmapi.MasterConfiguration) error{ + CreatePKIAssets, + }, + expected: false, + }, + { + setupFuncs: []func(cfg *kubeadmapi.MasterConfiguration) error{ + CreatePKIAssets, + deleteCAKey, + }, + expected: true, + }, + } + + for _, test := range tests { + dir := testutil.SetupTempDir(t) + defer os.RemoveAll(dir) + + cfg := &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4"}, + Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, + NodeName: "valid-hostname", + CertificatesDir: dir, + } + + for _, f := range test.setupFuncs { + if err := f(cfg); err != nil { + t.Errorf("error executing setup function: %v", err) + } + } + + if val, _ := UsingExternalCA(cfg); val != test.expected { + t.Errorf("UsingExternalCA did not match expected: %v", test.expected) + } + } +} + +func TestValidateMethods(t *testing.T) { + + tests := []struct { + name string + setupFuncs []func(cfg *kubeadmapi.MasterConfiguration) error + validateFunc func(l certKeyLocation) error + loc certKeyLocation + expectedSuccess bool + }{ + { + name: "validateCACert", + setupFuncs: []func(cfg *kubeadmapi.MasterConfiguration) error{ + CreateCACertAndKeyfiles, + }, + validateFunc: validateCACert, + loc: certKeyLocation{caBaseName: "ca", baseName: "", uxName: "CA"}, + expectedSuccess: true, + }, + { + name: "validateCACertAndKey (files present)", + setupFuncs: []func(cfg *kubeadmapi.MasterConfiguration) error{ + CreateCACertAndKeyfiles, + }, + validateFunc: validateCACertAndKey, + loc: certKeyLocation{caBaseName: "ca", baseName: "", uxName: "CA"}, + expectedSuccess: true, + }, + { + name: "validateCACertAndKey (key missing)", + setupFuncs: []func(cfg *kubeadmapi.MasterConfiguration) error{ + CreatePKIAssets, + deleteCAKey, + }, + validateFunc: validateCACertAndKey, + loc: certKeyLocation{caBaseName: "ca", baseName: "", uxName: "CA"}, + expectedSuccess: false, + }, + { + name: "validateSignedCert", + setupFuncs: []func(cfg *kubeadmapi.MasterConfiguration) error{ + CreateCACertAndKeyfiles, + CreateAPIServerCertAndKeyFiles, + }, + validateFunc: validateSignedCert, + loc: certKeyLocation{caBaseName: "ca", baseName: "apiserver", uxName: "apiserver"}, + expectedSuccess: true, + }, + { + name: "validatePrivatePublicKey", + setupFuncs: []func(cfg *kubeadmapi.MasterConfiguration) error{ + CreateServiceAccountKeyAndPublicKeyFiles, + }, + validateFunc: validatePrivatePublicKey, + loc: certKeyLocation{baseName: "sa", uxName: "service account"}, + expectedSuccess: true, + }, + } + + for _, test := range tests { + + dir := testutil.SetupTempDir(t) + defer os.RemoveAll(dir) + test.loc.pkiDir = dir + + cfg := &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4"}, + Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, + NodeName: "valid-hostname", + CertificatesDir: dir, + } + + fmt.Println("Testing", test.name) + + for _, f := range test.setupFuncs { + if err := f(cfg); err != nil { + t.Errorf("error executing setup function: %v", err) + } + } + + err := test.validateFunc(test.loc) + if test.expectedSuccess && err != nil { + t.Errorf("expected success, error executing validateFunc: %v, %v", test.name, err) + } else if !test.expectedSuccess && err == nil { + t.Errorf("expected failure, no error executing validateFunc: %v", test.name) + } + } +} + +func deleteCAKey(cfg *kubeadmapi.MasterConfiguration) error { + if err := os.Remove(filepath.Join(cfg.CertificatesDir, "ca.key")); err != nil { + return fmt.Errorf("failed removing ca.key: %v", err) + } + return nil +} + +func assertIsCa(t *testing.T, cert *x509.Certificate) { + if !cert.IsCA { + t.Error("cert is not a valida CA") + } +} + func TestCreateCertificateFilesMethods(t *testing.T) { var tests = []struct { diff --git a/cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers.go b/cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers.go index a3f512faf35..5ea8ed3f640 100644 --- a/cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers.go +++ b/cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers.go @@ -200,6 +200,35 @@ func TryLoadKeyFromDisk(pkiPath, name string) (*rsa.PrivateKey, error) { return key, nil } +// TryLoadPrivatePublicKeyFromDisk tries to load the key from the disk and validates that it is valid +func TryLoadPrivatePublicKeyFromDisk(pkiPath, name string) (*rsa.PrivateKey, *rsa.PublicKey, error) { + privateKeyPath := pathForKey(pkiPath, name) + + // Parse the private key from a file + privKey, err := certutil.PrivateKeyFromFile(privateKeyPath) + if err != nil { + return nil, nil, fmt.Errorf("couldn't load the private key file %s: %v", privateKeyPath, err) + } + + publicKeyPath := pathForPublicKey(pkiPath, name) + + // Parse the public key from a file + pubKeys, err := certutil.PublicKeysFromFile(publicKeyPath) + if err != nil { + return nil, nil, fmt.Errorf("couldn't load the public key file %s: %v", publicKeyPath, err) + } + + // Allow RSA format only + k, ok := privKey.(*rsa.PrivateKey) + if !ok { + return nil, nil, fmt.Errorf("the private key file %s isn't in RSA format", privateKeyPath) + } + + p := pubKeys[0].(*rsa.PublicKey) + + return k, p, nil +} + func pathsForCertAndKey(pkiPath, name string) (string, string) { return pathForCert(pkiPath, name), pathForKey(pkiPath, name) } diff --git a/cmd/kubeadm/app/phases/controlplane/BUILD b/cmd/kubeadm/app/phases/controlplane/BUILD index c88f404feff..99de76ec6ab 100644 --- a/cmd/kubeadm/app/phases/controlplane/BUILD +++ b/cmd/kubeadm/app/phases/controlplane/BUILD @@ -16,6 +16,7 @@ go_test( deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", + "//cmd/kubeadm/app/phases/certs:go_default_library", "//cmd/kubeadm/test:go_default_library", "//pkg/util/version:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", @@ -33,6 +34,7 @@ go_library( "//cmd/kubeadm/app/apis/kubeadm/v1alpha1:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/images:go_default_library", + "//cmd/kubeadm/app/phases/certs:go_default_library", "//cmd/kubeadm/app/util:go_default_library", "//cmd/kubeadm/app/util/staticpod:go_default_library", "//pkg/kubeapiserver/authorizer/modes:go_default_library", diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go index 19a4a5de55a..352c4e7c154 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests.go @@ -28,6 +28,7 @@ import ( kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/images" + certphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" @@ -203,6 +204,7 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion *versio // getControllerManagerCommand builds the right controller manager command from the given config object and version func getControllerManagerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.Version) []string { + defaultArguments := map[string]string{ "address": "127.0.0.1", "leader-elect": "true", @@ -215,6 +217,13 @@ func getControllerManagerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion "controllers": "*,bootstrapsigner,tokencleaner", } + // If using external CA, pass empty string to controller manager instead of ca.key/ca.crt path, + // so that the csrsigning controller fails to start + if res, _ := certphase.UsingExternalCA(cfg); res { + defaultArguments["cluster-signing-key-file"] = "" + defaultArguments["cluster-signing-cert-file"] = "" + } + command := []string{"kube-controller-manager"} command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.ControllerManagerExtraArgs)...) diff --git a/cmd/kubeadm/app/phases/controlplane/manifests_test.go b/cmd/kubeadm/app/phases/controlplane/manifests_test.go index 488e5a4df12..d8efb03841b 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests_test.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests_test.go @@ -26,6 +26,7 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" "k8s.io/kubernetes/pkg/util/version" testutil "k8s.io/kubernetes/cmd/kubeadm/test" @@ -438,6 +439,88 @@ func TestGetControllerManagerCommand(t *testing.T) { } } +func TestGetControllerManagerCommandExternalCA(t *testing.T) { + + tests := []struct { + cfg *kubeadmapi.MasterConfiguration + caKeyPresent bool + expectedArgFunc func(dir string) []string + }{ + { + cfg: &kubeadmapi.MasterConfiguration{ + KubernetesVersion: "v1.7.0", + API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4"}, + Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, + NodeName: "valid-hostname", + }, + caKeyPresent: false, + expectedArgFunc: func(tmpdir string) []string { + return []string{ + "kube-controller-manager", + "--address=127.0.0.1", + "--leader-elect=true", + "--kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf", + "--root-ca-file=" + tmpdir + "/ca.crt", + "--service-account-private-key-file=" + tmpdir + "/sa.key", + "--cluster-signing-cert-file=", + "--cluster-signing-key-file=", + "--use-service-account-credentials=true", + "--controllers=*,bootstrapsigner,tokencleaner", + } + }, + }, + { + cfg: &kubeadmapi.MasterConfiguration{ + KubernetesVersion: "v1.7.0", + API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4"}, + Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, + NodeName: "valid-hostname", + }, + caKeyPresent: true, + expectedArgFunc: func(tmpdir string) []string { + return []string{ + "kube-controller-manager", + "--address=127.0.0.1", + "--leader-elect=true", + "--kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf", + "--root-ca-file=" + tmpdir + "/ca.crt", + "--service-account-private-key-file=" + tmpdir + "/sa.key", + "--cluster-signing-cert-file=" + tmpdir + "/ca.crt", + "--cluster-signing-key-file=" + tmpdir + "/ca.key", + "--use-service-account-credentials=true", + "--controllers=*,bootstrapsigner,tokencleaner", + } + }, + }, + } + + for _, test := range tests { + // Create temp folder for the test case + tmpdir := testutil.SetupTempDir(t) + defer os.RemoveAll(tmpdir) + test.cfg.CertificatesDir = tmpdir + + if err := certs.CreatePKIAssets(test.cfg); err != nil { + t.Errorf("failed creating pki assets: %v", err) + } + + // delete ca.key if test.caKeyPresent is false + if !test.caKeyPresent { + if err := os.Remove(filepath.Join(test.cfg.CertificatesDir, "ca.key")); err != nil { + t.Errorf("failed removing ca.key: %v", err) + } + } + + actual := getControllerManagerCommand(test.cfg, version.MustParseSemantic(test.cfg.KubernetesVersion)) + expected := test.expectedArgFunc(tmpdir) + sort.Strings(actual) + sort.Strings(expected) + if !reflect.DeepEqual(actual, expected) { + t.Errorf("failed getControllerManagerCommand:\nexpected:\n%v\nsaw:\n%v", expected, actual) + } + } +} + func TestGetSchedulerCommand(t *testing.T) { var tests = []struct { cfg *kubeadmapi.MasterConfiguration