diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index 45aad5afa6e..3c81bf65cd8 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -41,7 +41,9 @@ const ( type KubeletServer struct { componentconfig.KubeletConfiguration - KubeConfig util.StringFlag + KubeConfig util.StringFlag + BootstrapKubeconfig string + // If true, an invalid KubeConfig will result in the Kubelet exiting with an error. RequireKubeConfig bool AuthPath util.StringFlag // Deprecated -- use KubeConfig instead @@ -97,6 +99,10 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&s.TLSPrivateKeyFile, "tls-private-key-file", s.TLSPrivateKeyFile, "File containing x509 private key matching --tls-cert-file.") fs.StringVar(&s.CertDirectory, "cert-dir", s.CertDirectory, "The directory where the TLS certs are located (by default /var/run/kubernetes). "+ "If --tls-cert-file and --tls-private-key-file are provided, this flag will be ignored.") + fs.StringVar(&s.BootstrapKubeconfig, "experimental-bootstrap-kubeconfig", s.BootstrapKubeconfig, " Path to a kubeconfig file that will be used to get client certificate for kubelet. "+ + "If the file specified by --kubeconfig does not exist, the bootstrap kubeconfig is used to request a client certificate from the API server. "+ + "On success, a kubeconfig file referencing the generated key and obtained certificate is written to the path specified by --kubeconfig. "+ + "The certificate and key file will be stored in /var/run/kubernetes/.") fs.StringVar(&s.HostnameOverride, "hostname-override", s.HostnameOverride, "If non-empty, will use this string as identification instead of the actual hostname.") fs.StringVar(&s.PodInfraContainerImage, "pod-infra-container-image", s.PodInfraContainerImage, "The image whose network/ipc namespaces containers in each pod will use.") fs.StringVar(&s.DockerEndpoint, "docker-endpoint", s.DockerEndpoint, "Use this for the docker endpoint to communicate with") diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index e7929d86646..e63965412c7 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -19,6 +19,8 @@ package app import ( "crypto/tls" + "crypto/x509/pkix" + "encoding/json" "errors" "fmt" "io/ioutil" @@ -39,11 +41,14 @@ import ( "k8s.io/kubernetes/cmd/kubelet/app/options" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/resource" + "k8s.io/kubernetes/pkg/api/unversioned" + "k8s.io/kubernetes/pkg/apis/certificates" "k8s.io/kubernetes/pkg/apis/componentconfig" kubeExternal "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1" "k8s.io/kubernetes/pkg/capabilities" "k8s.io/kubernetes/pkg/client/chaosclient" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" + unversionedcertificates "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/certificates/unversioned" unversionedcore "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/unversioned" "k8s.io/kubernetes/pkg/client/record" "k8s.io/kubernetes/pkg/client/restclient" @@ -64,6 +69,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/network" "k8s.io/kubernetes/pkg/kubelet/server" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" + utilcertificates "k8s.io/kubernetes/pkg/util/certificates" utilconfig "k8s.io/kubernetes/pkg/util/config" "k8s.io/kubernetes/pkg/util/configz" "k8s.io/kubernetes/pkg/util/crypto" @@ -77,6 +83,12 @@ import ( "k8s.io/kubernetes/pkg/util/wait" "k8s.io/kubernetes/pkg/version" "k8s.io/kubernetes/pkg/volume" + "k8s.io/kubernetes/pkg/watch" +) + +const ( + defaultKubeletClientCertificateFile = "/var/run/kubernetes/kubelet-client.crt" + defaultKubeletClientKeyFile = "/var/run/kubernetes/kubelet-client.key" ) // bootstrapping interface for kubelet, targets the initialization protocol @@ -335,6 +347,12 @@ func run(s *options.KubeletServer, kcfg *KubeletConfig) (err error) { if kcfg == nil { var kubeClient, eventClient *clientset.Clientset + if s.BootstrapKubeconfig != "" { + if err := bootstrapClientCert(s); err != nil { + return err + } + } + clientConfig, err := CreateAPIServerClientConfig(s) if err == nil { kubeClient, err = clientset.NewForConfig(clientConfig) @@ -439,6 +457,242 @@ func run(s *options.KubeletServer, kcfg *KubeletConfig) (err error) { return nil } +// bootstrapClientCert will request a client cert for kubelet. +// If the file specified by --kubeconfig does not exist, the bootstrap kubeconfig is used +// to request a client certificate from the API server. +// On success, a kubeconfig file referencing the generated key and obtained certificate is +// written to the path specified by --kubeconfig. +// The certificate and key file will be stored in /var/run/kubernetes/. +func bootstrapClientCert(s *options.KubeletServer) error { + // Check the if --kubeconfig already has sufficient TLS client info. + kcfg, err := (&clientcmd.ClientConfigLoadingRules{ExplicitPath: s.KubeConfig.Value()}).Load() + if err == nil { + authInfo, err := getCurrentContextAuthInfo(kcfg) + if err == nil { + if containsSufficientTLSInfo(authInfo) { + return nil + } + } + } + + // At this point, we need to use the bootstrap kubeconfig to generate TLS client cert, key, and a kubeconfig + // to stored in --kubeconfig. + glog.V(2).Info("Using bootstrap kubeconfig to generate TLS client cert, key and kubeconfig file") + + kcfg, err = (&clientcmd.ClientConfigLoadingRules{ExplicitPath: s.BootstrapKubeconfig}).Load() + if err != nil { + return fmt.Errorf("unable to load boostrap kubeconfig: %v", err) + } + authInfo, err := getCurrentContextAuthInfo(kcfg) + if err != nil { + return fmt.Errorf("unable to load auth info in bootstrap kubeconfig: %v", err) + } + + authInfo.ClientCertificate, authInfo.ClientKey, err = getClientCertAndKey(s, authInfo.ClientCertificate, authInfo.ClientKey) + if err != nil { + return fmt.Errorf("unable to get cert from API server: %v", err) + } + + // Marshal and write the kubeconfig to disk. + data, err := json.Marshal(kcfg) + if err != nil { + return fmt.Errorf("unable to marshal the kubeconfig: %v", err) + } + + if err := ioutil.WriteFile(s.KubeConfig.Value(), data, 0644); err != nil { + return fmt.Errorf("unable to write the kubeconfig file at %q: %v", s.KubeConfig.Value(), err) + } + + return nil +} + +// getCurrentContextAuthInfo returns the AuthInfo object that's referenced +// by the current context. +// If current context or auth info name is empty, then it will return an error. +// If AuthInfo is empty, a new auth info object will be created. +func getCurrentContextAuthInfo(config *clientcmdapi.Config) (*clientcmdapi.AuthInfo, error) { + ctx, ok := config.Contexts[config.CurrentContext] + if !ok { + return nil, fmt.Errorf("unable to find current context %q", config.CurrentContext) + } + + if ctx.AuthInfo == "" { + return nil, fmt.Errorf("unable to find the name of the authInfo in current context %q", config.CurrentContext) + } + + if _, ok := config.AuthInfos[ctx.AuthInfo]; !ok { + config.AuthInfos[ctx.AuthInfo] = clientcmdapi.NewAuthInfo() + } + return config.AuthInfos[ctx.AuthInfo], nil +} + +// TODO(yifan): More detailed check on the cert / key content. +// CheckTLSInfo returns true if the authInfo contains client certificate data for client key data. +// Or if the client certificate or key file exists. +func containsSufficientTLSInfo(authInfo *clientcmdapi.AuthInfo) bool { + // We use '||' so that we won't override existing data in case of wrong setup. + if len(authInfo.ClientCertificateData) > 0 || len(authInfo.ClientKeyData) > 0 { + return true + } + + if crypto.FoundCertOrKey(authInfo.ClientCertificate, authInfo.ClientKey) { + return true + } + + return false +} + +// getClientCertAndKey will: +// (1) Create a restful client for doing the certificate signing request. +// (2) Read existing key data from existingKeyPath if possible. +// (3) Pass 'requestClientCertificate()' the CSR client, existing key data, and node name to +// request for client certificate from the API server. +// (4) Once (3) succeeds, dump the certificate and key data to the given paths. +// On failure, the the certificate and key file will be cleaned up. +// If the existingCertPath or existingKeyPath is empty, then the function will use the default path, respectively: +// /var/run/kubernetes/kubelet-client.crt, /var/run/kubernetes/kubelet-client.key. +func getClientCertAndKey(s *options.KubeletServer, existingCertPath, existingKeyPath string) (certPath, keyPath string, err error) { + // (1). + clientConfig, err := kubeconfigClientConfig(s.BootstrapKubeconfig, s.APIServerList) + if err != nil { + return "", "", fmt.Errorf("unable to create client config: %v", err) + } + client, err := unversionedcertificates.NewForConfig(clientConfig) + if err != nil { + return "", "", fmt.Errorf("unable to create certificates signing request client: %v", err) + } + csrClient := client.CertificateSigningRequests() + + // (2). + certPath, keyPath = existingCertPath, existingKeyPath + if certPath == "" { + certPath = defaultKubeletClientCertificateFile + } + if keyPath == "" { + keyPath = defaultKubeletClientKeyFile + + } + existingKeyData, err := ioutil.ReadFile(keyPath) + if err != nil && !os.IsNotExist(err) { + return "", "", fmt.Errorf("unable to read key file %q: %v", keyPath, err) + } + + // (3). + nodeName, err := getNodeName(s) + if err != nil { + return "", "", fmt.Errorf("unable to get node name: %v", err) + } + certData, keyData, err := requestClientCertificate(csrClient, existingKeyData, nodeName) + if err != nil { + return "", "", fmt.Errorf("unable to request certificate from API server: %v", err) + } + + // (4). + if err = crypto.WriteCertToPath(certPath, certData); err != nil { + return "", "", fmt.Errorf("unable to write certificate file %q: %v", certPath, err) + } + if err = crypto.WriteKeyToPath(keyPath, keyData); err != nil { + if err := os.Remove(certPath); err != nil { + glog.Warningf("Cannot clean up the certificate file %q: %v", certPath, err) + } + return "", "", fmt.Errorf("unable to write key file %q: %v", keyPath, err) + } + + return certPath, keyPath, nil +} + +// getNodeName returns the node name according to the cloud provider +// if cloud provider is specified. Otherwise, returns the host name of the node. +func getNodeName(s *options.KubeletServer) (string, error) { + var err error + var cloud cloudprovider.Interface + + if s.CloudProvider != kubeExternal.AutoDetectCloudProvider { + cloud, err = cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile) + if err != nil { + return "", err + } + } + + hostName := nodeutil.GetHostname(s.HostnameOverride) + if cloud != nil { + instances, ok := cloud.Instances() + if !ok { + return "", fmt.Errorf("failed to get instances from cloud provider") + } + return instances.CurrentNodeName(hostName) + } + return hostName, nil +} + +// requestClientCertificate will create a certificate signing request and send it to API server, +// then it will watch the object's status, once approved by API server, it will return the API +// server's issued certificate (pem-encoded). If there is any errors, or the watch timeouts, +// it will return an error. +// If the existingKeyData is empty, a new private key will be generated to create the certificate +// signing request. +func requestClientCertificate(client unversionedcertificates.CertificateSigningRequestInterface, existingKeyData []byte, nodeName string) (certData []byte, keyData []byte, err error) { + subject := &pkix.Name{ + Organization: []string{"system:nodes"}, + CommonName: fmt.Sprintf("system:node:%s", nodeName), + } + + csr, keyData, err := utilcertificates.NewCertificateRequest(existingKeyData, subject, nil, nil) + if err != nil { + return nil, nil, fmt.Errorf("unable to generate certificate request: %v", err) + } + + req, err := client.Create(&certificates.CertificateSigningRequest{ + TypeMeta: unversioned.TypeMeta{Kind: "CertificateSigningRequest"}, + ObjectMeta: api.ObjectMeta{GenerateName: "csr-"}, + + // Username, UID, Groups will be injected by API server. + Spec: certificates.CertificateSigningRequestSpec{Request: csr}, + }) + if err != nil { + return nil, nil, fmt.Errorf("cannot create certificate signing request: %v", err) + + } + + // Make a default timeout = 3600s + var defaultTimeoutSeconds int64 = 3600 + resultCh, err := client.Watch(api.ListOptions{ + Watch: true, + TimeoutSeconds: &defaultTimeoutSeconds, + // Label and field selector are not used now. + }) + if err != nil { + return nil, nil, fmt.Errorf("cannot watch on the certificate signing request: %v", err) + } + + var status certificates.CertificateSigningRequestStatus + ch := resultCh.ResultChan() + + for { + event, ok := <-ch + if !ok { + break + } + + if event.Type == watch.Modified { + if event.Object.(*certificates.CertificateSigningRequest).UID != req.UID { + continue + } + status = event.Object.(*certificates.CertificateSigningRequest).Status + for _, c := range status.Conditions { + if c.Type == certificates.CertificateDenied { + return nil, nil, fmt.Errorf("certificate signing request is not approved: %v, %v", c.Reason, c.Message) + } + if c.Type == certificates.CertificateApproved && status.Certificate != nil { + return status.Certificate, keyData, nil + } + } + } + } + + return nil, nil, fmt.Errorf("watch channel closed") +} + // InitializeTLS checks for a configured TLSCertFile and TLSPrivateKeyFile: if unspecified a new self-signed // certificate and key file are generated. Returns a configured server.TLSOptions object. func InitializeTLS(s *options.KubeletServer) (*server.TLSOptions, error) { @@ -528,13 +782,13 @@ func createClientConfig(s *options.KubeletServer) (*restclient.Config, error) { return nil, fmt.Errorf("cannot specify both --kubeconfig and --auth-path") } if s.KubeConfig.Provided() { - return kubeconfigClientConfig(s) + return kubeconfigClientConfig(s.KubeConfig.Value(), s.APIServerList) } if s.AuthPath.Provided() { return authPathClientConfig(s, false) } // Try the kubeconfig default first, falling back to the auth path default. - clientConfig, err := kubeconfigClientConfig(s) + clientConfig, err := kubeconfigClientConfig(s.KubeConfig.Value(), s.APIServerList) if err != nil { glog.Warningf("Could not load kubeconfig file %s: %v. Trying auth path instead.", s.KubeConfig, err) return authPathClientConfig(s, true) diff --git a/pkg/util/certificates/csr.go b/pkg/util/certificates/csr.go index 72140720d33..29cae0c5fd5 100644 --- a/pkg/util/certificates/csr.go +++ b/pkg/util/certificates/csr.go @@ -26,9 +26,7 @@ import ( "encoding/pem" "errors" "fmt" - "io/ioutil" "net" - "os" "k8s.io/kubernetes/pkg/apis/certificates" ) @@ -49,49 +47,39 @@ func ParseCertificateRequestObject(obj *certificates.CertificateSigningRequest) } // NewCertificateRequest generates a PEM-encoded CSR using the supplied private -// key file, subject, and SANs. If the private key file does not exist, it generates a -// new ECDSA P256 key to use and writes it to the keyFile path. -func NewCertificateRequest(keyFile string, subject *pkix.Name, dnsSANs []string, ipSANs []net.IP) ([]byte, error) { +// key data, subject, and SANs. If the private key data is empty, it generates a +// new ECDSA P256 key to use and returns it together with the CSR data. +func NewCertificateRequest(keyData []byte, subject *pkix.Name, dnsSANs []string, ipSANs []net.IP) (csr []byte, key []byte, err error) { var privateKey interface{} + var privateKeyPemBlock *pem.Block - if _, err := os.Stat(keyFile); os.IsNotExist(err) { + if len(keyData) == 0 { privateKey, err = ecdsa.GenerateKey(elliptic.P256(), cryptorand.Reader) if err != nil { - return nil, err + return nil, nil, err } ecdsaKey := privateKey.(*ecdsa.PrivateKey) derBytes, err := x509.MarshalECPrivateKey(ecdsaKey) if err != nil { - return nil, err + return nil, nil, err } - pemBlock := &pem.Block{ + privateKeyPemBlock = &pem.Block{ Type: "EC PRIVATE KEY", Bytes: derBytes, } - - err = ioutil.WriteFile(keyFile, pem.EncodeToMemory(pemBlock), os.FileMode(0600)) - if err != nil { - return nil, err - } + } else { + privateKeyPemBlock, _ = pem.Decode(keyData) } - keyBytes, err := ioutil.ReadFile(keyFile) - if err != nil { - return nil, err - } - - var block *pem.Block var sigType x509.SignatureAlgorithm - block, _ = pem.Decode(keyBytes) - - switch block.Type { + switch privateKeyPemBlock.Type { case "EC PRIVATE KEY": - privateKey, err = x509.ParseECPrivateKey(block.Bytes) + privateKey, err = x509.ParseECPrivateKey(privateKeyPemBlock.Bytes) if err != nil { - return nil, err + return nil, nil, err } ecdsaKey := privateKey.(*ecdsa.PrivateKey) switch ecdsaKey.Curve.Params().BitSize { @@ -103,9 +91,9 @@ func NewCertificateRequest(keyFile string, subject *pkix.Name, dnsSANs []string, sigType = x509.ECDSAWithSHA256 } case "RSA PRIVATE KEY": - privateKey, err = x509.ParsePKCS1PrivateKey(block.Bytes) + privateKey, err = x509.ParsePKCS1PrivateKey(privateKeyPemBlock.Bytes) if err != nil { - return nil, err + return nil, nil, err } rsaKey := privateKey.(*rsa.PrivateKey) keySize := rsaKey.N.BitLen() @@ -118,7 +106,7 @@ func NewCertificateRequest(keyFile string, subject *pkix.Name, dnsSANs []string, sigType = x509.SHA256WithRSA } default: - return nil, fmt.Errorf("unsupported key type: %s", block.Type) + return nil, nil, fmt.Errorf("unsupported key type: %s", privateKeyPemBlock.Type) } template := &x509.CertificateRequest{ @@ -128,15 +116,15 @@ func NewCertificateRequest(keyFile string, subject *pkix.Name, dnsSANs []string, IPAddresses: ipSANs, } - csr, err := x509.CreateCertificateRequest(cryptorand.Reader, template, privateKey) + csr, err = x509.CreateCertificateRequest(cryptorand.Reader, template, privateKey) if err != nil { - return nil, err + return nil, nil, err } - pemBlock := &pem.Block{ + csrPemBlock := &pem.Block{ Type: "CERTIFICATE REQUEST", Bytes: csr, } - return pem.EncodeToMemory(pemBlock), nil + return pem.EncodeToMemory(csrPemBlock), pem.EncodeToMemory(privateKeyPemBlock), nil } diff --git a/pkg/util/certificates/csr_test.go b/pkg/util/certificates/csr_test.go index 12a99ba5564..89b886a5b5e 100644 --- a/pkg/util/certificates/csr_test.go +++ b/pkg/util/certificates/csr_test.go @@ -2,6 +2,7 @@ package certificates import ( "crypto/x509/pkix" + "io/ioutil" "net" "testing" ) @@ -14,7 +15,11 @@ func TestNewCertificateRequest(t *testing.T) { dnsSANs := []string{"localhost"} ipSANs := []net.IP{net.ParseIP("127.0.0.1")} - _, err := NewCertificateRequest(keyFile, subject, dnsSANs, ipSANs) + keyData, err := ioutil.ReadFile(keyFile) + if err != nil { + t.Fatal(err) + } + _, _, err = NewCertificateRequest(keyData, subject, dnsSANs, ipSANs) if err != nil { t.Error(err) } diff --git a/pkg/util/crypto/crypto.go b/pkg/util/crypto/crypto.go index b573c8a5945..c32357a1991 100644 --- a/pkg/util/crypto/crypto.go +++ b/pkg/util/crypto/crypto.go @@ -109,24 +109,46 @@ func GenerateSelfSignedCert(host, certPath, keyPath string, alternateIPs []net.I } // Write cert - if err := os.MkdirAll(filepath.Dir(certPath), os.FileMode(0755)); err != nil { - return err - } - if err := ioutil.WriteFile(certPath, certBuffer.Bytes(), os.FileMode(0644)); err != nil { + if err := WriteCertToPath(certPath, certBuffer.Bytes()); err != nil { return err } // Write key - if err := os.MkdirAll(filepath.Dir(keyPath), os.FileMode(0755)); err != nil { - return err - } - if err := ioutil.WriteFile(keyPath, keyBuffer.Bytes(), os.FileMode(0600)); err != nil { + if err := WriteKeyToPath(keyPath, keyBuffer.Bytes()); err != nil { return err } return nil } +// WriteCertToPath writes the pem-encoded certificate data to certPath. +// The certificate file will be created with file mode 0644. +// If the certificate file already exists, it will be overwritten. +// The parent directory of the certPath will be created as needed with file mode 0755. +func WriteCertToPath(certPath string, data []byte) error { + if err := os.MkdirAll(filepath.Dir(certPath), os.FileMode(0755)); err != nil { + return err + } + if err := ioutil.WriteFile(certPath, data, os.FileMode(0644)); err != nil { + return err + } + return nil +} + +// writeCertToPath writes the pem-encoded key data to keyPath. +// The key file will be created with file mode 0600. +// If the key file already exists, it will be overwritten. +// The parent directory of the keyPath will be created as needed with file mode 0755. +func WriteKeyToPath(keyPath string, data []byte) error { + if err := os.MkdirAll(filepath.Dir(keyPath), os.FileMode(0755)); err != nil { + return err + } + if err := ioutil.WriteFile(keyPath, data, os.FileMode(0600)); err != nil { + return err + } + return nil +} + // CertPoolFromFile returns an x509.CertPool containing the certificates in the given PEM-encoded file. // Returns an error if the file could not be read, a certificate could not be parsed, or if the file does not contain any certificates func CertPoolFromFile(filename string) (*x509.CertPool, error) {