From 5801fa5f4de71fd9ee06f12345cb1c8d8e7eeca7 Mon Sep 17 00:00:00 2001 From: George Tankersley Date: Tue, 2 Aug 2016 17:33:22 -0700 Subject: [PATCH 1/4] util/certificates: Add CSR utility function. --- pkg/util/certificates/csr.go | 102 ++++++++++++++++++ pkg/util/certificates/csr_test.go | 21 ++++ .../certificates/testdata/dontUseThisKey.pem | 6 ++ 3 files changed, 129 insertions(+) create mode 100644 pkg/util/certificates/csr_test.go create mode 100644 pkg/util/certificates/testdata/dontUseThisKey.pem diff --git a/pkg/util/certificates/csr.go b/pkg/util/certificates/csr.go index 86b037df42c..72140720d33 100644 --- a/pkg/util/certificates/csr.go +++ b/pkg/util/certificates/csr.go @@ -17,9 +17,18 @@ limitations under the License. package certificates import ( + "crypto/ecdsa" + "crypto/elliptic" + cryptorand "crypto/rand" + "crypto/rsa" "crypto/x509" + "crypto/x509/pkix" "encoding/pem" "errors" + "fmt" + "io/ioutil" + "net" + "os" "k8s.io/kubernetes/pkg/apis/certificates" ) @@ -38,3 +47,96 @@ func ParseCertificateRequestObject(obj *certificates.CertificateSigningRequest) } return csr, nil } + +// NewCertificateRequest generates a PEM-encoded CSR using the supplied private +// key file, subject, and SANs. If the private key file does not exist, it generates a +// new ECDSA P256 key to use and writes it to the keyFile path. +func NewCertificateRequest(keyFile string, subject *pkix.Name, dnsSANs []string, ipSANs []net.IP) ([]byte, error) { + var privateKey interface{} + + if _, err := os.Stat(keyFile); os.IsNotExist(err) { + privateKey, err = ecdsa.GenerateKey(elliptic.P256(), cryptorand.Reader) + if err != nil { + return nil, err + } + + ecdsaKey := privateKey.(*ecdsa.PrivateKey) + derBytes, err := x509.MarshalECPrivateKey(ecdsaKey) + if err != nil { + return nil, err + } + + pemBlock := &pem.Block{ + Type: "EC PRIVATE KEY", + Bytes: derBytes, + } + + err = ioutil.WriteFile(keyFile, pem.EncodeToMemory(pemBlock), os.FileMode(0600)) + if err != nil { + return nil, err + } + } + + keyBytes, err := ioutil.ReadFile(keyFile) + if err != nil { + return nil, err + } + + var block *pem.Block + var sigType x509.SignatureAlgorithm + + block, _ = pem.Decode(keyBytes) + + switch block.Type { + case "EC PRIVATE KEY": + privateKey, err = x509.ParseECPrivateKey(block.Bytes) + if err != nil { + return nil, err + } + ecdsaKey := privateKey.(*ecdsa.PrivateKey) + switch ecdsaKey.Curve.Params().BitSize { + case 521: + sigType = x509.ECDSAWithSHA512 + case 384: + sigType = x509.ECDSAWithSHA384 + default: + sigType = x509.ECDSAWithSHA256 + } + case "RSA PRIVATE KEY": + privateKey, err = x509.ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + return nil, err + } + rsaKey := privateKey.(*rsa.PrivateKey) + keySize := rsaKey.N.BitLen() + switch { + case keySize >= 4096: + sigType = x509.SHA512WithRSA + case keySize >= 3072: + sigType = x509.SHA384WithRSA + default: + sigType = x509.SHA256WithRSA + } + default: + return nil, fmt.Errorf("unsupported key type: %s", block.Type) + } + + template := &x509.CertificateRequest{ + Subject: *subject, + SignatureAlgorithm: sigType, + DNSNames: dnsSANs, + IPAddresses: ipSANs, + } + + csr, err := x509.CreateCertificateRequest(cryptorand.Reader, template, privateKey) + if err != nil { + return nil, err + } + + pemBlock := &pem.Block{ + Type: "CERTIFICATE REQUEST", + Bytes: csr, + } + + return pem.EncodeToMemory(pemBlock), nil +} diff --git a/pkg/util/certificates/csr_test.go b/pkg/util/certificates/csr_test.go new file mode 100644 index 00000000000..12a99ba5564 --- /dev/null +++ b/pkg/util/certificates/csr_test.go @@ -0,0 +1,21 @@ +package certificates + +import ( + "crypto/x509/pkix" + "net" + "testing" +) + +func TestNewCertificateRequest(t *testing.T) { + keyFile := "testdata/dontUseThisKey.pem" + subject := &pkix.Name{ + CommonName: "kube-worker", + } + dnsSANs := []string{"localhost"} + ipSANs := []net.IP{net.ParseIP("127.0.0.1")} + + _, err := NewCertificateRequest(keyFile, subject, dnsSANs, ipSANs) + if err != nil { + t.Error(err) + } +} diff --git a/pkg/util/certificates/testdata/dontUseThisKey.pem b/pkg/util/certificates/testdata/dontUseThisKey.pem new file mode 100644 index 00000000000..d6432631dca --- /dev/null +++ b/pkg/util/certificates/testdata/dontUseThisKey.pem @@ -0,0 +1,6 @@ +-----BEGIN EC PRIVATE KEY----- +MIGkAgEBBDAPEbSXwyDfWf0+61Oofd7aHkmdX69mrzD2Xb1CHF5syfsoRIhnG0dJ +ozBulPZCDDWgBwYFK4EEACKhZANiAATjlMJAtKhEPqU/i7MsrgKcK/RmXHC6He7W +0p69+9qFXg2raJ9zvvbKxkiu2ELOYRDAz0utcFTBOIgoUJEzBVmsjZQ7dvFa1BKP +Ym7MFAKG3O2espBqXn+audgdHGh5B0I= +-----END EC PRIVATE KEY----- From 26babd4eba2edbaaf00e6396942d9bb1ef3bbef0 Mon Sep 17 00:00:00 2001 From: Yifan Gu Date: Fri, 12 Aug 2016 18:06:48 -0700 Subject: [PATCH 2/4] kubelet: Add --bootstrap-kubeconfig to get TLS client cert. Add --bootstrap-kubeconfig flag to kubelet. If the flag is non-empty and --kubeconfig doesn't exist, then the kubelet will use the bootstrap kubeconfig to create rest client and generate certificate signing request to request a client cert from API server. Once succeeds, the result cert will be written down to --cert-dir/kubelet-client.crt, and the kubeconfig will be populated with certfile, keyfile path pointing to the result certificate file, key file. (The key file is generated before creating the CSR). --- cmd/kubelet/app/options/options.go | 8 +- cmd/kubelet/app/server.go | 258 ++++++++++++++++++++++++++++- pkg/util/certificates/csr.go | 52 +++--- pkg/util/certificates/csr_test.go | 7 +- pkg/util/crypto/crypto.go | 38 ++++- 5 files changed, 319 insertions(+), 44 deletions(-) diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index 45aad5afa6e..3c81bf65cd8 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -41,7 +41,9 @@ const ( type KubeletServer struct { componentconfig.KubeletConfiguration - KubeConfig util.StringFlag + KubeConfig util.StringFlag + BootstrapKubeconfig string + // If true, an invalid KubeConfig will result in the Kubelet exiting with an error. RequireKubeConfig bool AuthPath util.StringFlag // Deprecated -- use KubeConfig instead @@ -97,6 +99,10 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&s.TLSPrivateKeyFile, "tls-private-key-file", s.TLSPrivateKeyFile, "File containing x509 private key matching --tls-cert-file.") fs.StringVar(&s.CertDirectory, "cert-dir", s.CertDirectory, "The directory where the TLS certs are located (by default /var/run/kubernetes). "+ "If --tls-cert-file and --tls-private-key-file are provided, this flag will be ignored.") + fs.StringVar(&s.BootstrapKubeconfig, "experimental-bootstrap-kubeconfig", s.BootstrapKubeconfig, " Path to a kubeconfig file that will be used to get client certificate for kubelet. "+ + "If the file specified by --kubeconfig does not exist, the bootstrap kubeconfig is used to request a client certificate from the API server. "+ + "On success, a kubeconfig file referencing the generated key and obtained certificate is written to the path specified by --kubeconfig. "+ + "The certificate and key file will be stored in /var/run/kubernetes/.") fs.StringVar(&s.HostnameOverride, "hostname-override", s.HostnameOverride, "If non-empty, will use this string as identification instead of the actual hostname.") fs.StringVar(&s.PodInfraContainerImage, "pod-infra-container-image", s.PodInfraContainerImage, "The image whose network/ipc namespaces containers in each pod will use.") fs.StringVar(&s.DockerEndpoint, "docker-endpoint", s.DockerEndpoint, "Use this for the docker endpoint to communicate with") diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index e7929d86646..e63965412c7 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -19,6 +19,8 @@ package app import ( "crypto/tls" + "crypto/x509/pkix" + "encoding/json" "errors" "fmt" "io/ioutil" @@ -39,11 +41,14 @@ import ( "k8s.io/kubernetes/cmd/kubelet/app/options" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/resource" + "k8s.io/kubernetes/pkg/api/unversioned" + "k8s.io/kubernetes/pkg/apis/certificates" "k8s.io/kubernetes/pkg/apis/componentconfig" kubeExternal "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1" "k8s.io/kubernetes/pkg/capabilities" "k8s.io/kubernetes/pkg/client/chaosclient" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" + unversionedcertificates "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/certificates/unversioned" unversionedcore "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/unversioned" "k8s.io/kubernetes/pkg/client/record" "k8s.io/kubernetes/pkg/client/restclient" @@ -64,6 +69,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/network" "k8s.io/kubernetes/pkg/kubelet/server" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" + utilcertificates "k8s.io/kubernetes/pkg/util/certificates" utilconfig "k8s.io/kubernetes/pkg/util/config" "k8s.io/kubernetes/pkg/util/configz" "k8s.io/kubernetes/pkg/util/crypto" @@ -77,6 +83,12 @@ import ( "k8s.io/kubernetes/pkg/util/wait" "k8s.io/kubernetes/pkg/version" "k8s.io/kubernetes/pkg/volume" + "k8s.io/kubernetes/pkg/watch" +) + +const ( + defaultKubeletClientCertificateFile = "/var/run/kubernetes/kubelet-client.crt" + defaultKubeletClientKeyFile = "/var/run/kubernetes/kubelet-client.key" ) // bootstrapping interface for kubelet, targets the initialization protocol @@ -335,6 +347,12 @@ func run(s *options.KubeletServer, kcfg *KubeletConfig) (err error) { if kcfg == nil { var kubeClient, eventClient *clientset.Clientset + if s.BootstrapKubeconfig != "" { + if err := bootstrapClientCert(s); err != nil { + return err + } + } + clientConfig, err := CreateAPIServerClientConfig(s) if err == nil { kubeClient, err = clientset.NewForConfig(clientConfig) @@ -439,6 +457,242 @@ func run(s *options.KubeletServer, kcfg *KubeletConfig) (err error) { return nil } +// bootstrapClientCert will request a client cert for kubelet. +// If the file specified by --kubeconfig does not exist, the bootstrap kubeconfig is used +// to request a client certificate from the API server. +// On success, a kubeconfig file referencing the generated key and obtained certificate is +// written to the path specified by --kubeconfig. +// The certificate and key file will be stored in /var/run/kubernetes/. +func bootstrapClientCert(s *options.KubeletServer) error { + // Check the if --kubeconfig already has sufficient TLS client info. + kcfg, err := (&clientcmd.ClientConfigLoadingRules{ExplicitPath: s.KubeConfig.Value()}).Load() + if err == nil { + authInfo, err := getCurrentContextAuthInfo(kcfg) + if err == nil { + if containsSufficientTLSInfo(authInfo) { + return nil + } + } + } + + // At this point, we need to use the bootstrap kubeconfig to generate TLS client cert, key, and a kubeconfig + // to stored in --kubeconfig. + glog.V(2).Info("Using bootstrap kubeconfig to generate TLS client cert, key and kubeconfig file") + + kcfg, err = (&clientcmd.ClientConfigLoadingRules{ExplicitPath: s.BootstrapKubeconfig}).Load() + if err != nil { + return fmt.Errorf("unable to load boostrap kubeconfig: %v", err) + } + authInfo, err := getCurrentContextAuthInfo(kcfg) + if err != nil { + return fmt.Errorf("unable to load auth info in bootstrap kubeconfig: %v", err) + } + + authInfo.ClientCertificate, authInfo.ClientKey, err = getClientCertAndKey(s, authInfo.ClientCertificate, authInfo.ClientKey) + if err != nil { + return fmt.Errorf("unable to get cert from API server: %v", err) + } + + // Marshal and write the kubeconfig to disk. + data, err := json.Marshal(kcfg) + if err != nil { + return fmt.Errorf("unable to marshal the kubeconfig: %v", err) + } + + if err := ioutil.WriteFile(s.KubeConfig.Value(), data, 0644); err != nil { + return fmt.Errorf("unable to write the kubeconfig file at %q: %v", s.KubeConfig.Value(), err) + } + + return nil +} + +// getCurrentContextAuthInfo returns the AuthInfo object that's referenced +// by the current context. +// If current context or auth info name is empty, then it will return an error. +// If AuthInfo is empty, a new auth info object will be created. +func getCurrentContextAuthInfo(config *clientcmdapi.Config) (*clientcmdapi.AuthInfo, error) { + ctx, ok := config.Contexts[config.CurrentContext] + if !ok { + return nil, fmt.Errorf("unable to find current context %q", config.CurrentContext) + } + + if ctx.AuthInfo == "" { + return nil, fmt.Errorf("unable to find the name of the authInfo in current context %q", config.CurrentContext) + } + + if _, ok := config.AuthInfos[ctx.AuthInfo]; !ok { + config.AuthInfos[ctx.AuthInfo] = clientcmdapi.NewAuthInfo() + } + return config.AuthInfos[ctx.AuthInfo], nil +} + +// TODO(yifan): More detailed check on the cert / key content. +// CheckTLSInfo returns true if the authInfo contains client certificate data for client key data. +// Or if the client certificate or key file exists. +func containsSufficientTLSInfo(authInfo *clientcmdapi.AuthInfo) bool { + // We use '||' so that we won't override existing data in case of wrong setup. + if len(authInfo.ClientCertificateData) > 0 || len(authInfo.ClientKeyData) > 0 { + return true + } + + if crypto.FoundCertOrKey(authInfo.ClientCertificate, authInfo.ClientKey) { + return true + } + + return false +} + +// getClientCertAndKey will: +// (1) Create a restful client for doing the certificate signing request. +// (2) Read existing key data from existingKeyPath if possible. +// (3) Pass 'requestClientCertificate()' the CSR client, existing key data, and node name to +// request for client certificate from the API server. +// (4) Once (3) succeeds, dump the certificate and key data to the given paths. +// On failure, the the certificate and key file will be cleaned up. +// If the existingCertPath or existingKeyPath is empty, then the function will use the default path, respectively: +// /var/run/kubernetes/kubelet-client.crt, /var/run/kubernetes/kubelet-client.key. +func getClientCertAndKey(s *options.KubeletServer, existingCertPath, existingKeyPath string) (certPath, keyPath string, err error) { + // (1). + clientConfig, err := kubeconfigClientConfig(s.BootstrapKubeconfig, s.APIServerList) + if err != nil { + return "", "", fmt.Errorf("unable to create client config: %v", err) + } + client, err := unversionedcertificates.NewForConfig(clientConfig) + if err != nil { + return "", "", fmt.Errorf("unable to create certificates signing request client: %v", err) + } + csrClient := client.CertificateSigningRequests() + + // (2). + certPath, keyPath = existingCertPath, existingKeyPath + if certPath == "" { + certPath = defaultKubeletClientCertificateFile + } + if keyPath == "" { + keyPath = defaultKubeletClientKeyFile + + } + existingKeyData, err := ioutil.ReadFile(keyPath) + if err != nil && !os.IsNotExist(err) { + return "", "", fmt.Errorf("unable to read key file %q: %v", keyPath, err) + } + + // (3). + nodeName, err := getNodeName(s) + if err != nil { + return "", "", fmt.Errorf("unable to get node name: %v", err) + } + certData, keyData, err := requestClientCertificate(csrClient, existingKeyData, nodeName) + if err != nil { + return "", "", fmt.Errorf("unable to request certificate from API server: %v", err) + } + + // (4). + if err = crypto.WriteCertToPath(certPath, certData); err != nil { + return "", "", fmt.Errorf("unable to write certificate file %q: %v", certPath, err) + } + if err = crypto.WriteKeyToPath(keyPath, keyData); err != nil { + if err := os.Remove(certPath); err != nil { + glog.Warningf("Cannot clean up the certificate file %q: %v", certPath, err) + } + return "", "", fmt.Errorf("unable to write key file %q: %v", keyPath, err) + } + + return certPath, keyPath, nil +} + +// getNodeName returns the node name according to the cloud provider +// if cloud provider is specified. Otherwise, returns the host name of the node. +func getNodeName(s *options.KubeletServer) (string, error) { + var err error + var cloud cloudprovider.Interface + + if s.CloudProvider != kubeExternal.AutoDetectCloudProvider { + cloud, err = cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile) + if err != nil { + return "", err + } + } + + hostName := nodeutil.GetHostname(s.HostnameOverride) + if cloud != nil { + instances, ok := cloud.Instances() + if !ok { + return "", fmt.Errorf("failed to get instances from cloud provider") + } + return instances.CurrentNodeName(hostName) + } + return hostName, nil +} + +// requestClientCertificate will create a certificate signing request and send it to API server, +// then it will watch the object's status, once approved by API server, it will return the API +// server's issued certificate (pem-encoded). If there is any errors, or the watch timeouts, +// it will return an error. +// If the existingKeyData is empty, a new private key will be generated to create the certificate +// signing request. +func requestClientCertificate(client unversionedcertificates.CertificateSigningRequestInterface, existingKeyData []byte, nodeName string) (certData []byte, keyData []byte, err error) { + subject := &pkix.Name{ + Organization: []string{"system:nodes"}, + CommonName: fmt.Sprintf("system:node:%s", nodeName), + } + + csr, keyData, err := utilcertificates.NewCertificateRequest(existingKeyData, subject, nil, nil) + if err != nil { + return nil, nil, fmt.Errorf("unable to generate certificate request: %v", err) + } + + req, err := client.Create(&certificates.CertificateSigningRequest{ + TypeMeta: unversioned.TypeMeta{Kind: "CertificateSigningRequest"}, + ObjectMeta: api.ObjectMeta{GenerateName: "csr-"}, + + // Username, UID, Groups will be injected by API server. + Spec: certificates.CertificateSigningRequestSpec{Request: csr}, + }) + if err != nil { + return nil, nil, fmt.Errorf("cannot create certificate signing request: %v", err) + + } + + // Make a default timeout = 3600s + var defaultTimeoutSeconds int64 = 3600 + resultCh, err := client.Watch(api.ListOptions{ + Watch: true, + TimeoutSeconds: &defaultTimeoutSeconds, + // Label and field selector are not used now. + }) + if err != nil { + return nil, nil, fmt.Errorf("cannot watch on the certificate signing request: %v", err) + } + + var status certificates.CertificateSigningRequestStatus + ch := resultCh.ResultChan() + + for { + event, ok := <-ch + if !ok { + break + } + + if event.Type == watch.Modified { + if event.Object.(*certificates.CertificateSigningRequest).UID != req.UID { + continue + } + status = event.Object.(*certificates.CertificateSigningRequest).Status + for _, c := range status.Conditions { + if c.Type == certificates.CertificateDenied { + return nil, nil, fmt.Errorf("certificate signing request is not approved: %v, %v", c.Reason, c.Message) + } + if c.Type == certificates.CertificateApproved && status.Certificate != nil { + return status.Certificate, keyData, nil + } + } + } + } + + return nil, nil, fmt.Errorf("watch channel closed") +} + // InitializeTLS checks for a configured TLSCertFile and TLSPrivateKeyFile: if unspecified a new self-signed // certificate and key file are generated. Returns a configured server.TLSOptions object. func InitializeTLS(s *options.KubeletServer) (*server.TLSOptions, error) { @@ -528,13 +782,13 @@ func createClientConfig(s *options.KubeletServer) (*restclient.Config, error) { return nil, fmt.Errorf("cannot specify both --kubeconfig and --auth-path") } if s.KubeConfig.Provided() { - return kubeconfigClientConfig(s) + return kubeconfigClientConfig(s.KubeConfig.Value(), s.APIServerList) } if s.AuthPath.Provided() { return authPathClientConfig(s, false) } // Try the kubeconfig default first, falling back to the auth path default. - clientConfig, err := kubeconfigClientConfig(s) + clientConfig, err := kubeconfigClientConfig(s.KubeConfig.Value(), s.APIServerList) if err != nil { glog.Warningf("Could not load kubeconfig file %s: %v. Trying auth path instead.", s.KubeConfig, err) return authPathClientConfig(s, true) diff --git a/pkg/util/certificates/csr.go b/pkg/util/certificates/csr.go index 72140720d33..29cae0c5fd5 100644 --- a/pkg/util/certificates/csr.go +++ b/pkg/util/certificates/csr.go @@ -26,9 +26,7 @@ import ( "encoding/pem" "errors" "fmt" - "io/ioutil" "net" - "os" "k8s.io/kubernetes/pkg/apis/certificates" ) @@ -49,49 +47,39 @@ func ParseCertificateRequestObject(obj *certificates.CertificateSigningRequest) } // NewCertificateRequest generates a PEM-encoded CSR using the supplied private -// key file, subject, and SANs. If the private key file does not exist, it generates a -// new ECDSA P256 key to use and writes it to the keyFile path. -func NewCertificateRequest(keyFile string, subject *pkix.Name, dnsSANs []string, ipSANs []net.IP) ([]byte, error) { +// key data, subject, and SANs. If the private key data is empty, it generates a +// new ECDSA P256 key to use and returns it together with the CSR data. +func NewCertificateRequest(keyData []byte, subject *pkix.Name, dnsSANs []string, ipSANs []net.IP) (csr []byte, key []byte, err error) { var privateKey interface{} + var privateKeyPemBlock *pem.Block - if _, err := os.Stat(keyFile); os.IsNotExist(err) { + if len(keyData) == 0 { privateKey, err = ecdsa.GenerateKey(elliptic.P256(), cryptorand.Reader) if err != nil { - return nil, err + return nil, nil, err } ecdsaKey := privateKey.(*ecdsa.PrivateKey) derBytes, err := x509.MarshalECPrivateKey(ecdsaKey) if err != nil { - return nil, err + return nil, nil, err } - pemBlock := &pem.Block{ + privateKeyPemBlock = &pem.Block{ Type: "EC PRIVATE KEY", Bytes: derBytes, } - - err = ioutil.WriteFile(keyFile, pem.EncodeToMemory(pemBlock), os.FileMode(0600)) - if err != nil { - return nil, err - } + } else { + privateKeyPemBlock, _ = pem.Decode(keyData) } - keyBytes, err := ioutil.ReadFile(keyFile) - if err != nil { - return nil, err - } - - var block *pem.Block var sigType x509.SignatureAlgorithm - block, _ = pem.Decode(keyBytes) - - switch block.Type { + switch privateKeyPemBlock.Type { case "EC PRIVATE KEY": - privateKey, err = x509.ParseECPrivateKey(block.Bytes) + privateKey, err = x509.ParseECPrivateKey(privateKeyPemBlock.Bytes) if err != nil { - return nil, err + return nil, nil, err } ecdsaKey := privateKey.(*ecdsa.PrivateKey) switch ecdsaKey.Curve.Params().BitSize { @@ -103,9 +91,9 @@ func NewCertificateRequest(keyFile string, subject *pkix.Name, dnsSANs []string, sigType = x509.ECDSAWithSHA256 } case "RSA PRIVATE KEY": - privateKey, err = x509.ParsePKCS1PrivateKey(block.Bytes) + privateKey, err = x509.ParsePKCS1PrivateKey(privateKeyPemBlock.Bytes) if err != nil { - return nil, err + return nil, nil, err } rsaKey := privateKey.(*rsa.PrivateKey) keySize := rsaKey.N.BitLen() @@ -118,7 +106,7 @@ func NewCertificateRequest(keyFile string, subject *pkix.Name, dnsSANs []string, sigType = x509.SHA256WithRSA } default: - return nil, fmt.Errorf("unsupported key type: %s", block.Type) + return nil, nil, fmt.Errorf("unsupported key type: %s", privateKeyPemBlock.Type) } template := &x509.CertificateRequest{ @@ -128,15 +116,15 @@ func NewCertificateRequest(keyFile string, subject *pkix.Name, dnsSANs []string, IPAddresses: ipSANs, } - csr, err := x509.CreateCertificateRequest(cryptorand.Reader, template, privateKey) + csr, err = x509.CreateCertificateRequest(cryptorand.Reader, template, privateKey) if err != nil { - return nil, err + return nil, nil, err } - pemBlock := &pem.Block{ + csrPemBlock := &pem.Block{ Type: "CERTIFICATE REQUEST", Bytes: csr, } - return pem.EncodeToMemory(pemBlock), nil + return pem.EncodeToMemory(csrPemBlock), pem.EncodeToMemory(privateKeyPemBlock), nil } diff --git a/pkg/util/certificates/csr_test.go b/pkg/util/certificates/csr_test.go index 12a99ba5564..89b886a5b5e 100644 --- a/pkg/util/certificates/csr_test.go +++ b/pkg/util/certificates/csr_test.go @@ -2,6 +2,7 @@ package certificates import ( "crypto/x509/pkix" + "io/ioutil" "net" "testing" ) @@ -14,7 +15,11 @@ func TestNewCertificateRequest(t *testing.T) { dnsSANs := []string{"localhost"} ipSANs := []net.IP{net.ParseIP("127.0.0.1")} - _, err := NewCertificateRequest(keyFile, subject, dnsSANs, ipSANs) + keyData, err := ioutil.ReadFile(keyFile) + if err != nil { + t.Fatal(err) + } + _, _, err = NewCertificateRequest(keyData, subject, dnsSANs, ipSANs) if err != nil { t.Error(err) } diff --git a/pkg/util/crypto/crypto.go b/pkg/util/crypto/crypto.go index b573c8a5945..c32357a1991 100644 --- a/pkg/util/crypto/crypto.go +++ b/pkg/util/crypto/crypto.go @@ -109,24 +109,46 @@ func GenerateSelfSignedCert(host, certPath, keyPath string, alternateIPs []net.I } // Write cert - if err := os.MkdirAll(filepath.Dir(certPath), os.FileMode(0755)); err != nil { - return err - } - if err := ioutil.WriteFile(certPath, certBuffer.Bytes(), os.FileMode(0644)); err != nil { + if err := WriteCertToPath(certPath, certBuffer.Bytes()); err != nil { return err } // Write key - if err := os.MkdirAll(filepath.Dir(keyPath), os.FileMode(0755)); err != nil { - return err - } - if err := ioutil.WriteFile(keyPath, keyBuffer.Bytes(), os.FileMode(0600)); err != nil { + if err := WriteKeyToPath(keyPath, keyBuffer.Bytes()); err != nil { return err } return nil } +// WriteCertToPath writes the pem-encoded certificate data to certPath. +// The certificate file will be created with file mode 0644. +// If the certificate file already exists, it will be overwritten. +// The parent directory of the certPath will be created as needed with file mode 0755. +func WriteCertToPath(certPath string, data []byte) error { + if err := os.MkdirAll(filepath.Dir(certPath), os.FileMode(0755)); err != nil { + return err + } + if err := ioutil.WriteFile(certPath, data, os.FileMode(0644)); err != nil { + return err + } + return nil +} + +// writeCertToPath writes the pem-encoded key data to keyPath. +// The key file will be created with file mode 0600. +// If the key file already exists, it will be overwritten. +// The parent directory of the keyPath will be created as needed with file mode 0755. +func WriteKeyToPath(keyPath string, data []byte) error { + if err := os.MkdirAll(filepath.Dir(keyPath), os.FileMode(0755)); err != nil { + return err + } + if err := ioutil.WriteFile(keyPath, data, os.FileMode(0600)); err != nil { + return err + } + return nil +} + // CertPoolFromFile returns an x509.CertPool containing the certificates in the given PEM-encoded file. // Returns an error if the file could not be read, a certificate could not be parsed, or if the file does not contain any certificates func CertPoolFromFile(filename string) (*x509.CertPool, error) { From 2e631d811ce75b29004601dfb4e73ca58421ee01 Mon Sep 17 00:00:00 2001 From: Yifan Gu Date: Fri, 19 Aug 2016 13:50:07 -0700 Subject: [PATCH 3/4] crypto.go: Rename ShouldGenSelfSignedCerts() to FoundCertOrKey(). Since the function only tests whether the files are on the disk, the original name is a little bit misleading. --- cmd/kubelet/app/server.go | 2 +- pkg/genericapiserver/genericapiserver.go | 2 +- pkg/util/crypto/crypto.go | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index e63965412c7..deb5f081b39 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -699,7 +699,7 @@ func InitializeTLS(s *options.KubeletServer) (*server.TLSOptions, error) { if s.TLSCertFile == "" && s.TLSPrivateKeyFile == "" { s.TLSCertFile = path.Join(s.CertDirectory, "kubelet.crt") s.TLSPrivateKeyFile = path.Join(s.CertDirectory, "kubelet.key") - if crypto.ShouldGenSelfSignedCerts(s.TLSCertFile, s.TLSPrivateKeyFile) { + if !crypto.FoundCertOrKey(s.TLSCertFile, s.TLSPrivateKeyFile) { if err := crypto.GenerateSelfSignedCert(nodeutil.GetHostname(s.HostnameOverride), s.TLSCertFile, s.TLSPrivateKeyFile, nil, nil); err != nil { return nil, fmt.Errorf("unable to generate self signed cert: %v", err) } diff --git a/pkg/genericapiserver/genericapiserver.go b/pkg/genericapiserver/genericapiserver.go index b6406b4d5a4..ed1c86b2317 100644 --- a/pkg/genericapiserver/genericapiserver.go +++ b/pkg/genericapiserver/genericapiserver.go @@ -689,7 +689,7 @@ func (s *GenericAPIServer) Run(options *options.ServerRunOptions) { alternateDNS := []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"} // It would be nice to set a fqdn subject alt name, but only the kubelets know, the apiserver is clueless // alternateDNS = append(alternateDNS, "kubernetes.default.svc.CLUSTER.DNS.NAME") - if crypto.ShouldGenSelfSignedCerts(options.TLSCertFile, options.TLSPrivateKeyFile) { + if !crypto.FoundCertOrKey(options.TLSCertFile, options.TLSPrivateKeyFile) { if err := crypto.GenerateSelfSignedCert(s.ClusterIP.String(), options.TLSCertFile, options.TLSPrivateKeyFile, alternateIPs, alternateDNS); err != nil { glog.Errorf("Unable to generate self signed cert: %v", err) } else { diff --git a/pkg/util/crypto/crypto.go b/pkg/util/crypto/crypto.go index c32357a1991..32650e4912a 100644 --- a/pkg/util/crypto/crypto.go +++ b/pkg/util/crypto/crypto.go @@ -33,14 +33,14 @@ import ( "time" ) -// ShouldGenSelfSignedCerts returns false if the certificate or key files already exists, -// otherwise returns true. -func ShouldGenSelfSignedCerts(certPath, keyPath string) bool { +// FoundCertOrKey returns true if the certificate or key files already exists, +// otherwise returns false. +func FoundCertOrKey(certPath, keyPath string) bool { if canReadFile(certPath) || canReadFile(keyPath) { - return false + return true } - return true + return false } // If the file represented by path exists and From 26a66232610e627d4912fbd3dfb37afa8036dcf8 Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Thu, 18 Aug 2016 00:56:52 -0400 Subject: [PATCH 4/4] kubelet: '--experimental-bootstrap-kubeconfig' refactor. Move bootstrap functions to separate files. Split some of the functions into small sub-functions for reusability. Other cleanups --- cmd/kubelet/app/bootstrap.go | 259 ++++++++++++++++++++++++ cmd/kubelet/app/bootstrap_test.go | 85 ++++++++ cmd/kubelet/app/options/options.go | 2 +- cmd/kubelet/app/server.go | 312 ++++------------------------- hack/verify-flags/known-flags.txt | 1 + pkg/util/certificates/csr.go | 104 +++++----- pkg/util/certificates/csr_test.go | 22 +- pkg/util/crypto/crypto.go | 2 +- 8 files changed, 465 insertions(+), 322 deletions(-) create mode 100644 cmd/kubelet/app/bootstrap.go create mode 100644 cmd/kubelet/app/bootstrap_test.go diff --git a/cmd/kubelet/app/bootstrap.go b/cmd/kubelet/app/bootstrap.go new file mode 100644 index 00000000000..16ae5d43363 --- /dev/null +++ b/cmd/kubelet/app/bootstrap.go @@ -0,0 +1,259 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "crypto/x509/pkix" + "fmt" + "io/ioutil" + _ "net/http/pprof" + "os" + "path/filepath" + + "github.com/golang/glog" + + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/unversioned" + "k8s.io/kubernetes/pkg/apis/certificates" + unversionedcertificates "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/certificates/unversioned" + "k8s.io/kubernetes/pkg/client/restclient" + "k8s.io/kubernetes/pkg/client/unversioned/clientcmd" + clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" + "k8s.io/kubernetes/pkg/fields" + utilcertificates "k8s.io/kubernetes/pkg/util/certificates" + "k8s.io/kubernetes/pkg/util/crypto" + "k8s.io/kubernetes/pkg/watch" +) + +const ( + defaultKubeletClientCertificateFile = "kubelet-client.crt" + defaultKubeletClientKeyFile = "kubelet-client.key" +) + +// bootstrapClientCert requests a client cert for kubelet if the kubeconfigPath file does not exist. +// The kubeconfig at bootstrapPath is used to request a client certificate from the API server. +// On success, a kubeconfig file referencing the generated key and obtained certificate is written to kubeconfigPath. +// The certificate and key file are stored in certDir. +func bootstrapClientCert(kubeconfigPath string, bootstrapPath string, certDir string, nodeName string) error { + // Short-circuit if the kubeconfig file already exists. + // TODO: inspect the kubeconfig, ensure a rest client can be built from it, verify client cert expiration, etc. + _, err := os.Stat(kubeconfigPath) + if err == nil { + glog.V(2).Infof("Kubeconfig %s exists, skipping bootstrap", kubeconfigPath) + return nil + } + if !os.IsNotExist(err) { + glog.Errorf("Error reading kubeconfig %s, skipping bootstrap: %v", kubeconfigPath, err) + return err + } + + glog.V(2).Info("Using bootstrap kubeconfig to generate TLS client cert, key and kubeconfig file") + + bootstrapClientConfig, err := loadRESTClientConfig(bootstrapPath) + if err != nil { + return fmt.Errorf("unable to load bootstrap kubeconfig: %v", err) + } + bootstrapClient, err := unversionedcertificates.NewForConfig(bootstrapClientConfig) + if err != nil { + return fmt.Errorf("unable to create certificates signing request client: %v", err) + } + + success := false + + // Get the private key. + keyPath, err := filepath.Abs(filepath.Join(certDir, defaultKubeletClientKeyFile)) + if err != nil { + return fmt.Errorf("unable to build bootstrap key path: %v", err) + } + keyData, generatedKeyFile, err := loadOrGenerateKeyFile(keyPath) + if err != nil { + return err + } + if generatedKeyFile { + defer func() { + if !success { + if err := os.Remove(keyPath); err != nil { + glog.Warningf("Cannot clean up the key file %q: %v", keyPath, err) + } + } + }() + } + + // Get the cert. + certPath, err := filepath.Abs(filepath.Join(certDir, defaultKubeletClientCertificateFile)) + if err != nil { + return fmt.Errorf("unable to build bootstrap client cert path: %v", err) + } + certData, err := RequestClientCertificate(bootstrapClient.CertificateSigningRequests(), keyData, nodeName) + if err != nil { + return err + } + if err := crypto.WriteCertToPath(certPath, certData); err != nil { + return err + } + defer func() { + if !success { + if err := os.Remove(certPath); err != nil { + glog.Warningf("Cannot clean up the cert file %q: %v", certPath, err) + } + } + }() + + // Get the CA data from the bootstrap client config. + caFile, caData := bootstrapClientConfig.CAFile, []byte{} + if len(caFile) == 0 { + caData = bootstrapClientConfig.CAData + } + + // Build resulting kubeconfig. + kubeconfigData := clientcmdapi.Config{ + // Define a cluster stanza based on the bootstrap kubeconfig. + Clusters: map[string]*clientcmdapi.Cluster{"default-cluster": { + Server: bootstrapClientConfig.Host, + InsecureSkipTLSVerify: bootstrapClientConfig.Insecure, + CertificateAuthority: caFile, + CertificateAuthorityData: caData, + }}, + // Define auth based on the obtained client cert. + AuthInfos: map[string]*clientcmdapi.AuthInfo{"default-auth": { + ClientCertificate: certPath, + ClientKey: keyPath, + }}, + // Define a context that connects the auth info and cluster, and set it as the default + Contexts: map[string]*clientcmdapi.Context{"default-context": { + Cluster: "default-cluster", + AuthInfo: "default-auth", + Namespace: "default", + }}, + CurrentContext: "default-context", + } + + // Marshal to disk + if err := clientcmd.WriteToFile(kubeconfigData, kubeconfigPath); err != nil { + return err + } + + success = true + return nil +} + +func loadRESTClientConfig(kubeconfig string) (*restclient.Config, error) { + // Load structured kubeconfig data from the given path. + loader := &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig} + loadedConfig, err := loader.Load() + if err != nil { + return nil, err + } + // Flatten the loaded data to a particular restclient.Config based on the current context. + return clientcmd.NewNonInteractiveClientConfig( + *loadedConfig, + loadedConfig.CurrentContext, + &clientcmd.ConfigOverrides{}, + loader, + ).ClientConfig() +} + +func loadOrGenerateKeyFile(keyPath string) (data []byte, wasGenerated bool, err error) { + loadedData, err := ioutil.ReadFile(keyPath) + if err == nil { + return loadedData, false, err + } + if !os.IsNotExist(err) { + return nil, false, fmt.Errorf("error loading key from %s: %v", keyPath, err) + } + + generatedData, err := utilcertificates.GeneratePrivateKey() + if err != nil { + return nil, false, fmt.Errorf("error generating key: %v", err) + } + if err := crypto.WriteKeyToPath(keyPath, generatedData); err != nil { + return nil, false, fmt.Errorf("error writing key to %s: %v", keyPath, err) + } + return generatedData, true, nil +} + +// RequestClientCertificate will create a certificate signing request and send it to API server, +// then it will watch the object's status, once approved by API server, it will return the API +// server's issued certificate (pem-encoded). If there is any errors, or the watch timeouts, +// it will return an error. +func RequestClientCertificate(client unversionedcertificates.CertificateSigningRequestInterface, privateKeyData []byte, nodeName string) (certData []byte, err error) { + subject := &pkix.Name{ + Organization: []string{"system:nodes"}, + CommonName: fmt.Sprintf("system:node:%s", nodeName), + } + + privateKey, err := utilcertificates.ParsePrivateKey(privateKeyData) + if err != nil { + return nil, fmt.Errorf("invalid private key for certificate request: %v", err) + } + csr, err := utilcertificates.NewCertificateRequest(privateKey, subject, nil, nil) + if err != nil { + return nil, fmt.Errorf("unable to generate certificate request: %v", err) + } + + req, err := client.Create(&certificates.CertificateSigningRequest{ + // Username, UID, Groups will be injected by API server. + TypeMeta: unversioned.TypeMeta{Kind: "CertificateSigningRequest"}, + ObjectMeta: api.ObjectMeta{GenerateName: "csr-"}, + + // TODO: For now, this is a request for a certificate with allowed usage of "TLS Web Client Authentication". + // Need to figure out whether/how to surface the allowed usage in the spec. + Spec: certificates.CertificateSigningRequestSpec{Request: csr}, + }) + if err != nil { + return nil, fmt.Errorf("cannot create certificate signing request: %v", err) + + } + + // Make a default timeout = 3600s. + var defaultTimeoutSeconds int64 = 3600 + resultCh, err := client.Watch(api.ListOptions{ + Watch: true, + TimeoutSeconds: &defaultTimeoutSeconds, + FieldSelector: fields.OneTermEqualSelector("metadata.name", req.Name), + }) + if err != nil { + return nil, fmt.Errorf("cannot watch on the certificate signing request: %v", err) + } + + var status certificates.CertificateSigningRequestStatus + ch := resultCh.ResultChan() + + for { + event, ok := <-ch + if !ok { + break + } + + if event.Type == watch.Modified || event.Type == watch.Added { + if event.Object.(*certificates.CertificateSigningRequest).UID != req.UID { + continue + } + status = event.Object.(*certificates.CertificateSigningRequest).Status + for _, c := range status.Conditions { + if c.Type == certificates.CertificateDenied { + return nil, fmt.Errorf("certificate signing request is not approved, reason: %v, message: %v", c.Reason, c.Message) + } + if c.Type == certificates.CertificateApproved && status.Certificate != nil { + return status.Certificate, nil + } + } + } + } + + return nil, fmt.Errorf("watch channel closed") +} diff --git a/cmd/kubelet/app/bootstrap_test.go b/cmd/kubelet/app/bootstrap_test.go new file mode 100644 index 00000000000..174f29ffa91 --- /dev/null +++ b/cmd/kubelet/app/bootstrap_test.go @@ -0,0 +1,85 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "io/ioutil" + "os" + "reflect" + "testing" + + "k8s.io/kubernetes/pkg/client/restclient" + "k8s.io/kubernetes/pkg/util/diff" +) + +func TestLoadRESTClientConfig(t *testing.T) { + testData := []byte(` +apiVersion: v1 +kind: Config +clusters: +- cluster: + certificate-authority: ca-a.crt + server: https://cluster-a.com + name: cluster-a +- cluster: + certificate-authority-data: VGVzdA== + server: https://cluster-b.com + name: cluster-b +contexts: +- context: + cluster: cluster-a + namespace: ns-a + user: user-a + name: context-a +- context: + cluster: cluster-b + namespace: ns-b + user: user-b + name: context-b +current-context: context-b +users: +- name: user-a + user: + token: mytoken-a +- name: user-b + user: + token: mytoken-b +`) + f, err := ioutil.TempFile("", "kubeconfig") + if err != nil { + t.Fatal(err) + } + defer os.Remove(f.Name()) + ioutil.WriteFile(f.Name(), testData, os.FileMode(0755)) + + config, err := loadRESTClientConfig(f.Name()) + if err != nil { + t.Fatal(err) + } + + expectedConfig := &restclient.Config{ + Host: "https://cluster-b.com", + TLSClientConfig: restclient.TLSClientConfig{ + CAData: []byte(`Test`), + }, + BearerToken: "mytoken-b", + } + + if !reflect.DeepEqual(config, expectedConfig) { + t.Errorf("Unexpected config: %s", diff.ObjectDiff(config, expectedConfig)) + } +} diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index 3c81bf65cd8..1e778045182 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -102,7 +102,7 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&s.BootstrapKubeconfig, "experimental-bootstrap-kubeconfig", s.BootstrapKubeconfig, " Path to a kubeconfig file that will be used to get client certificate for kubelet. "+ "If the file specified by --kubeconfig does not exist, the bootstrap kubeconfig is used to request a client certificate from the API server. "+ "On success, a kubeconfig file referencing the generated key and obtained certificate is written to the path specified by --kubeconfig. "+ - "The certificate and key file will be stored in /var/run/kubernetes/.") + "The certificate and key file will be stored in the directory pointed by --cert-dir.") fs.StringVar(&s.HostnameOverride, "hostname-override", s.HostnameOverride, "If non-empty, will use this string as identification instead of the actual hostname.") fs.StringVar(&s.PodInfraContainerImage, "pod-infra-container-image", s.PodInfraContainerImage, "The image whose network/ipc namespaces containers in each pod will use.") fs.StringVar(&s.DockerEndpoint, "docker-endpoint", s.DockerEndpoint, "Use this for the docker endpoint to communicate with") diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index deb5f081b39..9aefc2f281c 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -19,8 +19,6 @@ package app import ( "crypto/tls" - "crypto/x509/pkix" - "encoding/json" "errors" "fmt" "io/ioutil" @@ -41,14 +39,11 @@ import ( "k8s.io/kubernetes/cmd/kubelet/app/options" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/resource" - "k8s.io/kubernetes/pkg/api/unversioned" - "k8s.io/kubernetes/pkg/apis/certificates" "k8s.io/kubernetes/pkg/apis/componentconfig" kubeExternal "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1" "k8s.io/kubernetes/pkg/capabilities" "k8s.io/kubernetes/pkg/client/chaosclient" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" - unversionedcertificates "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/certificates/unversioned" unversionedcore "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/unversioned" "k8s.io/kubernetes/pkg/client/record" "k8s.io/kubernetes/pkg/client/restclient" @@ -69,7 +64,6 @@ import ( "k8s.io/kubernetes/pkg/kubelet/network" "k8s.io/kubernetes/pkg/kubelet/server" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" - utilcertificates "k8s.io/kubernetes/pkg/util/certificates" utilconfig "k8s.io/kubernetes/pkg/util/config" "k8s.io/kubernetes/pkg/util/configz" "k8s.io/kubernetes/pkg/util/crypto" @@ -83,12 +77,6 @@ import ( "k8s.io/kubernetes/pkg/util/wait" "k8s.io/kubernetes/pkg/version" "k8s.io/kubernetes/pkg/volume" - "k8s.io/kubernetes/pkg/watch" -) - -const ( - defaultKubeletClientCertificateFile = "/var/run/kubernetes/kubelet-client.crt" - defaultKubeletClientKeyFile = "/var/run/kubernetes/kubelet-client.key" ) // bootstrapping interface for kubelet, targets the initialization protocol @@ -347,8 +335,29 @@ func run(s *options.KubeletServer, kcfg *KubeletConfig) (err error) { if kcfg == nil { var kubeClient, eventClient *clientset.Clientset + var autoDetectCloudProvider bool + var cloud cloudprovider.Interface + + if s.CloudProvider == kubeExternal.AutoDetectCloudProvider { + autoDetectCloudProvider = true + } else { + cloud, err = cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile) + if err != nil { + return err + } + if cloud == nil { + glog.V(2).Infof("No cloud provider specified: %q from the config file: %q\n", s.CloudProvider, s.CloudConfigFile) + } else { + glog.V(2).Infof("Successfully initialized cloud provider: %q from the config file: %q\n", s.CloudProvider, s.CloudConfigFile) + } + } + if s.BootstrapKubeconfig != "" { - if err := bootstrapClientCert(s); err != nil { + nodeName, err := getNodeName(cloud, nodeutil.GetHostname(s.HostnameOverride)) + if err != nil { + return err + } + if err := bootstrapClientCert(s.KubeConfig.Value(), s.BootstrapKubeconfig, s.CertDirectory, nodeName); err != nil { return err } } @@ -377,24 +386,12 @@ func run(s *options.KubeletServer, kcfg *KubeletConfig) (err error) { if err != nil { return err } + kcfg = cfg + kcfg.AutoDetectCloudProvider = autoDetectCloudProvider + kcfg.Cloud = cloud kcfg.KubeClient = kubeClient kcfg.EventClient = eventClient - - if s.CloudProvider == kubeExternal.AutoDetectCloudProvider { - kcfg.AutoDetectCloudProvider = true - } else { - cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile) - if err != nil { - return err - } - if cloud == nil { - glog.V(2).Infof("No cloud provider specified: %q from the config file: %q\n", s.CloudProvider, s.CloudConfigFile) - } else { - glog.V(2).Infof("Successfully initialized cloud provider: %q from the config file: %q\n", s.CloudProvider, s.CloudConfigFile) - kcfg.Cloud = cloud - } - } } if kcfg.CAdvisorInterface == nil { @@ -457,240 +454,26 @@ func run(s *options.KubeletServer, kcfg *KubeletConfig) (err error) { return nil } -// bootstrapClientCert will request a client cert for kubelet. -// If the file specified by --kubeconfig does not exist, the bootstrap kubeconfig is used -// to request a client certificate from the API server. -// On success, a kubeconfig file referencing the generated key and obtained certificate is -// written to the path specified by --kubeconfig. -// The certificate and key file will be stored in /var/run/kubernetes/. -func bootstrapClientCert(s *options.KubeletServer) error { - // Check the if --kubeconfig already has sufficient TLS client info. - kcfg, err := (&clientcmd.ClientConfigLoadingRules{ExplicitPath: s.KubeConfig.Value()}).Load() - if err == nil { - authInfo, err := getCurrentContextAuthInfo(kcfg) - if err == nil { - if containsSufficientTLSInfo(authInfo) { - return nil - } - } - } - - // At this point, we need to use the bootstrap kubeconfig to generate TLS client cert, key, and a kubeconfig - // to stored in --kubeconfig. - glog.V(2).Info("Using bootstrap kubeconfig to generate TLS client cert, key and kubeconfig file") - - kcfg, err = (&clientcmd.ClientConfigLoadingRules{ExplicitPath: s.BootstrapKubeconfig}).Load() - if err != nil { - return fmt.Errorf("unable to load boostrap kubeconfig: %v", err) - } - authInfo, err := getCurrentContextAuthInfo(kcfg) - if err != nil { - return fmt.Errorf("unable to load auth info in bootstrap kubeconfig: %v", err) - } - - authInfo.ClientCertificate, authInfo.ClientKey, err = getClientCertAndKey(s, authInfo.ClientCertificate, authInfo.ClientKey) - if err != nil { - return fmt.Errorf("unable to get cert from API server: %v", err) - } - - // Marshal and write the kubeconfig to disk. - data, err := json.Marshal(kcfg) - if err != nil { - return fmt.Errorf("unable to marshal the kubeconfig: %v", err) - } - - if err := ioutil.WriteFile(s.KubeConfig.Value(), data, 0644); err != nil { - return fmt.Errorf("unable to write the kubeconfig file at %q: %v", s.KubeConfig.Value(), err) - } - - return nil -} - -// getCurrentContextAuthInfo returns the AuthInfo object that's referenced -// by the current context. -// If current context or auth info name is empty, then it will return an error. -// If AuthInfo is empty, a new auth info object will be created. -func getCurrentContextAuthInfo(config *clientcmdapi.Config) (*clientcmdapi.AuthInfo, error) { - ctx, ok := config.Contexts[config.CurrentContext] - if !ok { - return nil, fmt.Errorf("unable to find current context %q", config.CurrentContext) - } - - if ctx.AuthInfo == "" { - return nil, fmt.Errorf("unable to find the name of the authInfo in current context %q", config.CurrentContext) - } - - if _, ok := config.AuthInfos[ctx.AuthInfo]; !ok { - config.AuthInfos[ctx.AuthInfo] = clientcmdapi.NewAuthInfo() - } - return config.AuthInfos[ctx.AuthInfo], nil -} - -// TODO(yifan): More detailed check on the cert / key content. -// CheckTLSInfo returns true if the authInfo contains client certificate data for client key data. -// Or if the client certificate or key file exists. -func containsSufficientTLSInfo(authInfo *clientcmdapi.AuthInfo) bool { - // We use '||' so that we won't override existing data in case of wrong setup. - if len(authInfo.ClientCertificateData) > 0 || len(authInfo.ClientKeyData) > 0 { - return true - } - - if crypto.FoundCertOrKey(authInfo.ClientCertificate, authInfo.ClientKey) { - return true - } - - return false -} - -// getClientCertAndKey will: -// (1) Create a restful client for doing the certificate signing request. -// (2) Read existing key data from existingKeyPath if possible. -// (3) Pass 'requestClientCertificate()' the CSR client, existing key data, and node name to -// request for client certificate from the API server. -// (4) Once (3) succeeds, dump the certificate and key data to the given paths. -// On failure, the the certificate and key file will be cleaned up. -// If the existingCertPath or existingKeyPath is empty, then the function will use the default path, respectively: -// /var/run/kubernetes/kubelet-client.crt, /var/run/kubernetes/kubelet-client.key. -func getClientCertAndKey(s *options.KubeletServer, existingCertPath, existingKeyPath string) (certPath, keyPath string, err error) { - // (1). - clientConfig, err := kubeconfigClientConfig(s.BootstrapKubeconfig, s.APIServerList) - if err != nil { - return "", "", fmt.Errorf("unable to create client config: %v", err) - } - client, err := unversionedcertificates.NewForConfig(clientConfig) - if err != nil { - return "", "", fmt.Errorf("unable to create certificates signing request client: %v", err) - } - csrClient := client.CertificateSigningRequests() - - // (2). - certPath, keyPath = existingCertPath, existingKeyPath - if certPath == "" { - certPath = defaultKubeletClientCertificateFile - } - if keyPath == "" { - keyPath = defaultKubeletClientKeyFile - - } - existingKeyData, err := ioutil.ReadFile(keyPath) - if err != nil && !os.IsNotExist(err) { - return "", "", fmt.Errorf("unable to read key file %q: %v", keyPath, err) - } - - // (3). - nodeName, err := getNodeName(s) - if err != nil { - return "", "", fmt.Errorf("unable to get node name: %v", err) - } - certData, keyData, err := requestClientCertificate(csrClient, existingKeyData, nodeName) - if err != nil { - return "", "", fmt.Errorf("unable to request certificate from API server: %v", err) - } - - // (4). - if err = crypto.WriteCertToPath(certPath, certData); err != nil { - return "", "", fmt.Errorf("unable to write certificate file %q: %v", certPath, err) - } - if err = crypto.WriteKeyToPath(keyPath, keyData); err != nil { - if err := os.Remove(certPath); err != nil { - glog.Warningf("Cannot clean up the certificate file %q: %v", certPath, err) - } - return "", "", fmt.Errorf("unable to write key file %q: %v", keyPath, err) - } - - return certPath, keyPath, nil -} - // getNodeName returns the node name according to the cloud provider -// if cloud provider is specified. Otherwise, returns the host name of the node. -func getNodeName(s *options.KubeletServer) (string, error) { - var err error - var cloud cloudprovider.Interface - - if s.CloudProvider != kubeExternal.AutoDetectCloudProvider { - cloud, err = cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile) - if err != nil { - return "", err - } +// if cloud provider is specified. Otherwise, returns the hostname of the node. +func getNodeName(cloud cloudprovider.Interface, hostname string) (string, error) { + if cloud == nil { + return hostname, nil } - hostName := nodeutil.GetHostname(s.HostnameOverride) - if cloud != nil { - instances, ok := cloud.Instances() - if !ok { - return "", fmt.Errorf("failed to get instances from cloud provider") - } - return instances.CurrentNodeName(hostName) - } - return hostName, nil -} - -// requestClientCertificate will create a certificate signing request and send it to API server, -// then it will watch the object's status, once approved by API server, it will return the API -// server's issued certificate (pem-encoded). If there is any errors, or the watch timeouts, -// it will return an error. -// If the existingKeyData is empty, a new private key will be generated to create the certificate -// signing request. -func requestClientCertificate(client unversionedcertificates.CertificateSigningRequestInterface, existingKeyData []byte, nodeName string) (certData []byte, keyData []byte, err error) { - subject := &pkix.Name{ - Organization: []string{"system:nodes"}, - CommonName: fmt.Sprintf("system:node:%s", nodeName), + instances, ok := cloud.Instances() + if !ok { + return "", fmt.Errorf("failed to get instances from cloud provider") } - csr, keyData, err := utilcertificates.NewCertificateRequest(existingKeyData, subject, nil, nil) + nodeName, err := instances.CurrentNodeName(hostname) if err != nil { - return nil, nil, fmt.Errorf("unable to generate certificate request: %v", err) + return "", fmt.Errorf("error fetching current instance name from cloud provider: %v", err) } - req, err := client.Create(&certificates.CertificateSigningRequest{ - TypeMeta: unversioned.TypeMeta{Kind: "CertificateSigningRequest"}, - ObjectMeta: api.ObjectMeta{GenerateName: "csr-"}, + glog.V(2).Infof("cloud provider determined current node name to be %s", nodeName) - // Username, UID, Groups will be injected by API server. - Spec: certificates.CertificateSigningRequestSpec{Request: csr}, - }) - if err != nil { - return nil, nil, fmt.Errorf("cannot create certificate signing request: %v", err) - - } - - // Make a default timeout = 3600s - var defaultTimeoutSeconds int64 = 3600 - resultCh, err := client.Watch(api.ListOptions{ - Watch: true, - TimeoutSeconds: &defaultTimeoutSeconds, - // Label and field selector are not used now. - }) - if err != nil { - return nil, nil, fmt.Errorf("cannot watch on the certificate signing request: %v", err) - } - - var status certificates.CertificateSigningRequestStatus - ch := resultCh.ResultChan() - - for { - event, ok := <-ch - if !ok { - break - } - - if event.Type == watch.Modified { - if event.Object.(*certificates.CertificateSigningRequest).UID != req.UID { - continue - } - status = event.Object.(*certificates.CertificateSigningRequest).Status - for _, c := range status.Conditions { - if c.Type == certificates.CertificateDenied { - return nil, nil, fmt.Errorf("certificate signing request is not approved: %v, %v", c.Reason, c.Message) - } - if c.Type == certificates.CertificateApproved && status.Certificate != nil { - return status.Certificate, keyData, nil - } - } - } - } - - return nil, nil, fmt.Errorf("watch channel closed") + return nodeName, nil } // InitializeTLS checks for a configured TLSCertFile and TLSPrivateKeyFile: if unspecified a new self-signed @@ -782,13 +565,13 @@ func createClientConfig(s *options.KubeletServer) (*restclient.Config, error) { return nil, fmt.Errorf("cannot specify both --kubeconfig and --auth-path") } if s.KubeConfig.Provided() { - return kubeconfigClientConfig(s.KubeConfig.Value(), s.APIServerList) + return kubeconfigClientConfig(s) } if s.AuthPath.Provided() { return authPathClientConfig(s, false) } // Try the kubeconfig default first, falling back to the auth path default. - clientConfig, err := kubeconfigClientConfig(s.KubeConfig.Value(), s.APIServerList) + clientConfig, err := kubeconfigClientConfig(s) if err != nil { glog.Warningf("Could not load kubeconfig file %s: %v. Trying auth path instead.", s.KubeConfig, err) return authPathClientConfig(s, true) @@ -932,23 +715,10 @@ func RunKubelet(kcfg *KubeletConfig) error { kcfg.Hostname = nodeutil.GetHostname(kcfg.HostnameOverride) if len(kcfg.NodeName) == 0 { - // Query the cloud provider for our node name, default to Hostname - nodeName := kcfg.Hostname - if kcfg.Cloud != nil { - var err error - instances, ok := kcfg.Cloud.Instances() - if !ok { - return fmt.Errorf("failed to get instances from cloud provider") - } - - nodeName, err = instances.CurrentNodeName(kcfg.Hostname) - if err != nil { - return fmt.Errorf("error fetching current instance name from cloud provider: %v", err) - } - - glog.V(2).Infof("cloud provider determined current node name to be %s", nodeName) + nodeName, err := getNodeName(kcfg.Cloud, kcfg.Hostname) + if err != nil { + return err } - kcfg.NodeName = nodeName } diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index c1b81a0dcb5..7be58994885 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -163,6 +163,7 @@ executor-logv executor-path executor-suicide-timeout exit-on-lock-contention +experimental-bootstrap-kubeconfig experimental-flannel-overlay experimental-keystone-url experimental-nvidia-gpus diff --git a/pkg/util/certificates/csr.go b/pkg/util/certificates/csr.go index 29cae0c5fd5..6f5e78348f2 100644 --- a/pkg/util/certificates/csr.go +++ b/pkg/util/certificates/csr.go @@ -46,57 +46,64 @@ func ParseCertificateRequestObject(obj *certificates.CertificateSigningRequest) return csr, nil } -// NewCertificateRequest generates a PEM-encoded CSR using the supplied private -// key data, subject, and SANs. If the private key data is empty, it generates a -// new ECDSA P256 key to use and returns it together with the CSR data. -func NewCertificateRequest(keyData []byte, subject *pkix.Name, dnsSANs []string, ipSANs []net.IP) (csr []byte, key []byte, err error) { - var privateKey interface{} - var privateKeyPemBlock *pem.Block - - if len(keyData) == 0 { - privateKey, err = ecdsa.GenerateKey(elliptic.P256(), cryptorand.Reader) - if err != nil { - return nil, nil, err - } - - ecdsaKey := privateKey.(*ecdsa.PrivateKey) - derBytes, err := x509.MarshalECPrivateKey(ecdsaKey) - if err != nil { - return nil, nil, err - } - - privateKeyPemBlock = &pem.Block{ - Type: "EC PRIVATE KEY", - Bytes: derBytes, - } - } else { - privateKeyPemBlock, _ = pem.Decode(keyData) +// GeneratePrivateKey returns PEM data containing a generated ECDSA private key +func GeneratePrivateKey() ([]byte, error) { + privateKey, err := ecdsa.GenerateKey(elliptic.P256(), cryptorand.Reader) + if err != nil { + return nil, err } + derBytes, err := x509.MarshalECPrivateKey(privateKey) + if err != nil { + return nil, err + } + + privateKeyPemBlock := &pem.Block{ + Type: "EC PRIVATE KEY", + Bytes: derBytes, + } + return pem.EncodeToMemory(privateKeyPemBlock), nil +} + +// ParsePrivateKey returns a private key parsed from a PEM block in the supplied data. +// Recognizes PEM blocks for "EC PRIVATE KEY" and "RSA PRIVATE KEY" +func ParsePrivateKey(keyData []byte) (interface{}, error) { + for { + var privateKeyPemBlock *pem.Block + privateKeyPemBlock, keyData = pem.Decode(keyData) + if privateKeyPemBlock == nil { + // we read all the PEM blocks and didn't recognize one + return nil, fmt.Errorf("no private key PEM block found") + } + + switch privateKeyPemBlock.Type { + case "EC PRIVATE KEY": + return x509.ParseECPrivateKey(privateKeyPemBlock.Bytes) + case "RSA PRIVATE KEY": + return x509.ParsePKCS1PrivateKey(privateKeyPemBlock.Bytes) + } + } +} + +// NewCertificateRequest generates a PEM-encoded CSR using the supplied private key, subject, and SANs. +// privateKey must be a *ecdsa.PrivateKey or *rsa.PrivateKey. +func NewCertificateRequest(privateKey interface{}, subject *pkix.Name, dnsSANs []string, ipSANs []net.IP) (csr []byte, err error) { var sigType x509.SignatureAlgorithm - switch privateKeyPemBlock.Type { - case "EC PRIVATE KEY": - privateKey, err = x509.ParseECPrivateKey(privateKeyPemBlock.Bytes) - if err != nil { - return nil, nil, err - } - ecdsaKey := privateKey.(*ecdsa.PrivateKey) - switch ecdsaKey.Curve.Params().BitSize { - case 521: - sigType = x509.ECDSAWithSHA512 - case 384: - sigType = x509.ECDSAWithSHA384 - default: + switch privateKey := privateKey.(type) { + case *ecdsa.PrivateKey: + switch privateKey.Curve { + case elliptic.P224(), elliptic.P256(): sigType = x509.ECDSAWithSHA256 + case elliptic.P384(): + sigType = x509.ECDSAWithSHA384 + case elliptic.P521(): + sigType = x509.ECDSAWithSHA512 + default: + return nil, fmt.Errorf("unknown elliptic curve: %v", privateKey.Curve) } - case "RSA PRIVATE KEY": - privateKey, err = x509.ParsePKCS1PrivateKey(privateKeyPemBlock.Bytes) - if err != nil { - return nil, nil, err - } - rsaKey := privateKey.(*rsa.PrivateKey) - keySize := rsaKey.N.BitLen() + case *rsa.PrivateKey: + keySize := privateKey.N.BitLen() switch { case keySize >= 4096: sigType = x509.SHA512WithRSA @@ -105,8 +112,9 @@ func NewCertificateRequest(keyData []byte, subject *pkix.Name, dnsSANs []string, default: sigType = x509.SHA256WithRSA } + default: - return nil, nil, fmt.Errorf("unsupported key type: %s", privateKeyPemBlock.Type) + return nil, fmt.Errorf("unsupported key type: %T", privateKey) } template := &x509.CertificateRequest{ @@ -118,7 +126,7 @@ func NewCertificateRequest(keyData []byte, subject *pkix.Name, dnsSANs []string, csr, err = x509.CreateCertificateRequest(cryptorand.Reader, template, privateKey) if err != nil { - return nil, nil, err + return nil, err } csrPemBlock := &pem.Block{ @@ -126,5 +134,5 @@ func NewCertificateRequest(keyData []byte, subject *pkix.Name, dnsSANs []string, Bytes: csr, } - return pem.EncodeToMemory(csrPemBlock), pem.EncodeToMemory(privateKeyPemBlock), nil + return pem.EncodeToMemory(csrPemBlock), nil } diff --git a/pkg/util/certificates/csr_test.go b/pkg/util/certificates/csr_test.go index 89b886a5b5e..3ca229fb649 100644 --- a/pkg/util/certificates/csr_test.go +++ b/pkg/util/certificates/csr_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package certificates import ( @@ -19,7 +35,11 @@ func TestNewCertificateRequest(t *testing.T) { if err != nil { t.Fatal(err) } - _, _, err = NewCertificateRequest(keyData, subject, dnsSANs, ipSANs) + key, err := ParsePrivateKey(keyData) + if err != nil { + t.Fatal(err) + } + _, err = NewCertificateRequest(key, subject, dnsSANs, ipSANs) if err != nil { t.Error(err) } diff --git a/pkg/util/crypto/crypto.go b/pkg/util/crypto/crypto.go index 32650e4912a..7e926450844 100644 --- a/pkg/util/crypto/crypto.go +++ b/pkg/util/crypto/crypto.go @@ -135,7 +135,7 @@ func WriteCertToPath(certPath string, data []byte) error { return nil } -// writeCertToPath writes the pem-encoded key data to keyPath. +// WriteKeyToPath writes the pem-encoded key data to keyPath. // The key file will be created with file mode 0600. // If the key file already exists, it will be overwritten. // The parent directory of the keyPath will be created as needed with file mode 0755.