diff --git a/cmd/BUILD b/cmd/BUILD index bfa8e5ccd07..37e9ef85e60 100644 --- a/cmd/BUILD +++ b/cmd/BUILD @@ -22,6 +22,7 @@ filegroup( "//cmd/genswaggertypedocs:all-srcs", "//cmd/genutils:all-srcs", "//cmd/genyaml:all-srcs", + "//cmd/gke-certificates-controller:all-srcs", "//cmd/hyperkube:all-srcs", "//cmd/kube-apiserver:all-srcs", "//cmd/kube-controller-manager:all-srcs", diff --git a/cmd/gke-certificates-controller/BUILD b/cmd/gke-certificates-controller/BUILD new file mode 100644 index 00000000000..0c10d81a0c9 --- /dev/null +++ b/cmd/gke-certificates-controller/BUILD @@ -0,0 +1,44 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_binary", + "go_library", +) + +go_library( + name = "go_default_library", + srcs = ["main.go"], + tags = ["automanaged"], + deps = [ + "//cmd/gke-certificates-controller/app:go_default_library", + "//pkg/util/logs:go_default_library", + "//pkg/version/verflag:go_default_library", + "//vendor:github.com/spf13/pflag", + "//vendor:k8s.io/apiserver/pkg/util/flag", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [ + ":package-srcs", + "//cmd/gke-certificates-controller/app:all-srcs", + ], + tags = ["automanaged"], +) + +go_binary( + name = "gke-certificates-controller", + library = ":go_default_library", + tags = ["automanaged"], +) diff --git a/cmd/gke-certificates-controller/OWNERS b/cmd/gke-certificates-controller/OWNERS new file mode 100644 index 00000000000..8ab9ac5f08c --- /dev/null +++ b/cmd/gke-certificates-controller/OWNERS @@ -0,0 +1,8 @@ +approvers: +- pipejakob +- mikedanese +- roberthbailey +reviewers: +- pipejakob +- mikedanese +- roberthbailey diff --git a/cmd/gke-certificates-controller/app/BUILD b/cmd/gke-certificates-controller/app/BUILD new file mode 100644 index 00000000000..72207eeac94 --- /dev/null +++ b/cmd/gke-certificates-controller/app/BUILD @@ -0,0 +1,60 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", + "go_test", +) + +go_library( + name = "go_default_library", + srcs = [ + "gke_certificates_controller.go", + "gke_signer.go", + "options.go", + ], + tags = ["automanaged"], + deps = [ + "//pkg/api:go_default_library", + "//pkg/apis/certificates/install:go_default_library", + "//pkg/apis/certificates/v1beta1:go_default_library", + "//pkg/client/clientset_generated/clientset:go_default_library", + "//pkg/client/informers/informers_generated/externalversions:go_default_library", + "//pkg/controller:go_default_library", + "//pkg/controller/certificates:go_default_library", + "//vendor:github.com/golang/glog", + "//vendor:github.com/spf13/cobra", + "//vendor:github.com/spf13/pflag", + "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1", + "//vendor:k8s.io/apimachinery/pkg/runtime/schema", + "//vendor:k8s.io/apiserver/pkg/util/webhook", + "//vendor:k8s.io/client-go/kubernetes/typed/core/v1", + "//vendor:k8s.io/client-go/plugin/pkg/client/auth", + "//vendor:k8s.io/client-go/rest", + "//vendor:k8s.io/client-go/tools/clientcmd", + "//vendor:k8s.io/client-go/tools/record", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) + +go_test( + name = "go_default_test", + srcs = ["gke_signer_test.go"], + library = ":go_default_library", + tags = ["automanaged"], + deps = ["//pkg/apis/certificates/v1beta1:go_default_library"], +) diff --git a/cmd/gke-certificates-controller/app/gke_certificates_controller.go b/cmd/gke-certificates-controller/app/gke_certificates_controller.go new file mode 100644 index 00000000000..71de2888b00 --- /dev/null +++ b/cmd/gke-certificates-controller/app/gke_certificates_controller.go @@ -0,0 +1,90 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package app implements a server that runs a stand-alone version of the +// certificates controller for GKE clusters. +package app + +import ( + "time" + + v1core "k8s.io/client-go/kubernetes/typed/core/v1" + restclient "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/record" + "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" + informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/externalversions" + "k8s.io/kubernetes/pkg/controller" + "k8s.io/kubernetes/pkg/controller/certificates" + + // Install all auth plugins + _ "k8s.io/client-go/plugin/pkg/client/auth" + + "github.com/golang/glog" + "github.com/spf13/cobra" +) + +// NewGKECertificatesControllerCommand creates a new *cobra.Command with default parameters. +func NewGKECertificatesControllerCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "gke-certificates-controller", + Long: `The Kubernetes GKE certificates controller is a daemon that +handles auto-approving and signing certificates for GKE clusters.`, + } + + return cmd +} + +// Run runs the GKECertificatesController. This should never exit. +func Run(s *GKECertificatesController) error { + kubeconfig, err := clientcmd.BuildConfigFromFlags("", s.Kubeconfig) + if err != nil { + return err + } + + kubeClient, err := clientset.NewForConfig(restclient.AddUserAgent(kubeconfig, "gke-certificates-controller")) + if err != nil { + return err + } + + eventBroadcaster := record.NewBroadcaster() + eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: v1core.New(kubeClient.Core().RESTClient()).Events("")}) + + clientBuilder := controller.SimpleControllerClientBuilder{ClientConfig: kubeconfig} + client := clientBuilder.ClientOrDie("certificate-controller") + + sharedInformers := informers.NewSharedInformerFactory(client, time.Duration(12)*time.Hour) + + signer, err := NewGKESigner(s.ClusterSigningGKEKubeconfig, s.ClusterSigningGKERetryBackoff.Duration) + if err != nil { + return err + } + + controller, err := certificates.NewCertificateController( + client, + sharedInformers.Certificates().V1beta1().CertificateSigningRequests(), + signer, + certificates.NewGroupApprover(s.ApproveAllKubeletCSRsForGroup), + ) + if err != nil { + return err + } + + sharedInformers.Start(nil) + controller.Run(1, nil) // runs forever + return nil +} diff --git a/cmd/gke-certificates-controller/app/gke_signer.go b/cmd/gke-certificates-controller/app/gke_signer.go new file mode 100644 index 00000000000..4496eb477f3 --- /dev/null +++ b/cmd/gke-certificates-controller/app/gke_signer.go @@ -0,0 +1,90 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "fmt" + "time" + + "github.com/golang/glog" + + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apiserver/pkg/util/webhook" + "k8s.io/client-go/rest" + "k8s.io/kubernetes/pkg/api" + _ "k8s.io/kubernetes/pkg/apis/certificates/install" + certificates "k8s.io/kubernetes/pkg/apis/certificates/v1beta1" +) + +var ( + groupVersions = []schema.GroupVersion{certificates.SchemeGroupVersion} +) + +// GKESigner uses external calls to GKE in order to sign certificate signing +// requests. +type GKESigner struct { + webhook *webhook.GenericWebhook + kubeConfigFile string + retryBackoff time.Duration +} + +// NewGKESigner will create a new instance of a GKESigner. +func NewGKESigner(kubeConfigFile string, retryBackoff time.Duration) (*GKESigner, error) { + webhook, err := webhook.NewGenericWebhook(api.Registry, api.Codecs, kubeConfigFile, groupVersions, retryBackoff) + if err != nil { + return nil, err + } + + return &GKESigner{ + webhook: webhook, + kubeConfigFile: kubeConfigFile, + retryBackoff: retryBackoff, + }, nil +} + +// Sign will make an external call to GKE order to sign the given +// *certificates.CertificateSigningRequest, using the GKESigner's +// kubeConfigFile. +func (s *GKESigner) Sign(csr *certificates.CertificateSigningRequest) (*certificates.CertificateSigningRequest, error) { + result := s.webhook.WithExponentialBackoff(func() rest.Result { + return s.webhook.RestClient.Post().Body(csr).Do() + }) + + if err := result.Error(); err != nil { + return nil, s.webhookError(csr, err) + } + + var statusCode int + if result.StatusCode(&statusCode); statusCode < 200 || statusCode >= 300 { + return nil, s.webhookError(csr, fmt.Errorf("received unsuccessful response code from webhook: %d", statusCode)) + } + + result_csr := &certificates.CertificateSigningRequest{} + + if err := result.Into(result_csr); err != nil { + return nil, s.webhookError(result_csr, err) + } + + // Keep the original CSR intact, and only update fields we expect to change. + csr.Status.Certificate = result_csr.Status.Certificate + return csr, nil +} + +func (s *GKESigner) webhookError(csr *certificates.CertificateSigningRequest, err error) error { + glog.V(2).Infof("error contacting webhook backend: %s", err) + return err +} diff --git a/cmd/gke-certificates-controller/app/gke_signer_test.go b/cmd/gke-certificates-controller/app/gke_signer_test.go new file mode 100644 index 00000000000..a56653cf9f2 --- /dev/null +++ b/cmd/gke-certificates-controller/app/gke_signer_test.go @@ -0,0 +1,153 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "bytes" + "encoding/json" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + "text/template" + "time" + + certificates "k8s.io/kubernetes/pkg/apis/certificates/v1beta1" +) + +const kubeConfigTmpl = ` +clusters: +- cluster: + server: {{ .Server }} + name: testcluster +users: +- user: + username: admin + password: mypass +` + +func TestGKESigner(t *testing.T) { + goodResponse := &certificates.CertificateSigningRequest{ + Status: certificates.CertificateSigningRequestStatus{ + Certificate: []byte("fake certificate"), + }, + } + + invalidResponse := "{ \"status\": \"Not a properly formatted CSR response\" }" + + cases := []struct { + mockResponse interface{} + expected []byte + failCalls int + wantErr bool + }{ + { + mockResponse: goodResponse, + expected: goodResponse.Status.Certificate, + wantErr: false, + }, + { + mockResponse: goodResponse, + expected: goodResponse.Status.Certificate, + failCalls: 3, + wantErr: false, + }, + { + mockResponse: goodResponse, + failCalls: 20, + wantErr: true, + }, + { + mockResponse: invalidResponse, + wantErr: true, + }, + } + + for _, c := range cases { + server, err := newTestServer(c.mockResponse, c.failCalls) + if err != nil { + t.Fatalf("error creating test server") + } + + kubeConfig, err := ioutil.TempFile("", "kubeconfig") + if err != nil { + t.Fatalf("error creating kubeconfig tempfile: %v", err) + } + + tmpl, err := template.New("kubeconfig").Parse(kubeConfigTmpl) + if err != nil { + t.Fatalf("error creating kubeconfig template: %v", err) + } + + data := struct{ Server string }{server.httpserver.URL} + + if err := tmpl.Execute(kubeConfig, data); err != nil { + t.Fatalf("error executing kubeconfig template: %v", err) + } + + if err := kubeConfig.Close(); err != nil { + t.Fatalf("error closing kubeconfig template: %v", err) + } + + signer, err := NewGKESigner(kubeConfig.Name(), time.Duration(500)*time.Millisecond) + if err != nil { + t.Fatalf("error creating GKESigner: %v", err) + } + + cert, err := signer.Sign(&certificates.CertificateSigningRequest{}) + + if c.wantErr { + if err == nil { + t.Errorf("wanted error during GKE.Sign() call, got not none") + } + } else { + if err != nil { + t.Errorf("error while signing: %v", err) + } + + if !bytes.Equal(cert.Status.Certificate, c.expected) { + t.Errorf("response certificate didn't match expected %v: %v", c.expected, cert) + } + } + } +} + +type testServer struct { + httpserver *httptest.Server + failCalls int + response interface{} +} + +func newTestServer(response interface{}, failCalls int) (*testServer, error) { + server := &testServer{ + response: response, + failCalls: failCalls, + } + + server.httpserver = httptest.NewServer(server) + return server, nil +} + +func (s *testServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if s.failCalls > 0 { + http.Error(w, "Service unavailable", 500) + s.failCalls-- + } else { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(s.response) + } +} diff --git a/cmd/gke-certificates-controller/app/options.go b/cmd/gke-certificates-controller/app/options.go new file mode 100644 index 00000000000..56ce237fc70 --- /dev/null +++ b/cmd/gke-certificates-controller/app/options.go @@ -0,0 +1,54 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package app implements a server that runs a stand-alone version of the +// certificates controller. +package app + +import ( + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/spf13/pflag" +) + +// GKECertificatesController is the main context object for the package. +type GKECertificatesController struct { + Kubeconfig string + ClusterSigningGKEKubeconfig string + ClusterSigningGKERetryBackoff metav1.Duration + ApproveAllKubeletCSRsForGroup string +} + +// Create a new instance of a GKECertificatesController with default parameters. +func NewGKECertificatesController() *GKECertificatesController { + s := &GKECertificatesController{ + ClusterSigningGKERetryBackoff: metav1.Duration{Duration: 500 * time.Millisecond}, + } + return s +} + +// AddFlags adds flags for a specific GKECertificatesController to the +// specified FlagSet. +func (s *GKECertificatesController) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization and master location information.") + + fs.StringVar(&s.ClusterSigningGKEKubeconfig, "cluster-signing-gke-kubeconfig", s.ClusterSigningGKEKubeconfig, "If set, use the kubeconfig file to call GKE to sign cluster-scoped certificates instead of using a local private key.") + fs.DurationVar(&s.ClusterSigningGKERetryBackoff.Duration, "cluster-signing-gke-retry-backoff", s.ClusterSigningGKERetryBackoff.Duration, "The initial backoff to use when retrying requests to GKE. Additional attempts will use exponential backoff.") + + fs.StringVar(&s.ApproveAllKubeletCSRsForGroup, "insecure-experimental-approve-all-kubelet-csrs-for-group", s.ApproveAllKubeletCSRsForGroup, "The group for which the controller-manager will auto approve all CSRs for kubelet client certificates.") +} diff --git a/cmd/gke-certificates-controller/main.go b/cmd/gke-certificates-controller/main.go new file mode 100644 index 00000000000..21e07157cd4 --- /dev/null +++ b/cmd/gke-certificates-controller/main.go @@ -0,0 +1,49 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// The GKE certificates controller is responsible for monitoring certificate +// signing requests and (potentially) auto-approving and signing them within +// GKE. +package main + +import ( + "fmt" + "os" + + "k8s.io/apiserver/pkg/util/flag" + "k8s.io/kubernetes/cmd/gke-certificates-controller/app" + "k8s.io/kubernetes/pkg/util/logs" + "k8s.io/kubernetes/pkg/version/verflag" + + "github.com/spf13/pflag" +) + +// TODO(pipejakob): Move this entire cmd directory into its own repo +func main() { + s := app.NewGKECertificatesController() + s.AddFlags(pflag.CommandLine) + + flag.InitFlags() + logs.InitLogs() + defer logs.FlushLogs() + + verflag.PrintAndExitIfRequested() + + if err := app.Run(s); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } +} diff --git a/hack/.linted_packages b/hack/.linted_packages index 3d83b3c9ed3..c1c63ef9861 100644 --- a/hack/.linted_packages +++ b/hack/.linted_packages @@ -8,6 +8,7 @@ cmd/genkubedocs cmd/genman cmd/genswaggertypedocs cmd/genyaml +cmd/gke-certificates-controller cmd/kube-apiserver cmd/kube-apiserver/app cmd/kube-apiserver/app/options diff --git a/hack/lib/golang.sh b/hack/lib/golang.sh index a8022826c96..26082647771 100755 --- a/hack/lib/golang.sh +++ b/hack/lib/golang.sh @@ -19,8 +19,6 @@ readonly KUBE_GO_PACKAGE=k8s.io/kubernetes readonly KUBE_GOPATH="${KUBE_OUTPUT}/go" # The set of server targets that we are only building for Linux -# Note: if you are adding something here, you might need to add it to -# kube::build::source_targets in build/common.sh as well. # If you update this list, please also update build/release-tars/BUILD. kube::golang::server_targets() { local targets=( @@ -174,11 +172,15 @@ readonly KUBE_TEST_SERVER_PLATFORMS=("${KUBE_SERVER_PLATFORMS[@]}") # laptops-versus-not. readonly KUBE_PARALLEL_BUILD_MEMORY=11 +# TODO(pipejakob) gke-certificates-controller is included here to exercise its +# compilation, but it doesn't need to be distributed in any of our tars. Its +# code is only living in this repo temporarily until it finds a new home. readonly KUBE_ALL_TARGETS=( "${KUBE_SERVER_TARGETS[@]}" "${KUBE_CLIENT_TARGETS[@]}" "${KUBE_TEST_TARGETS[@]}" "${KUBE_TEST_SERVER_TARGETS[@]}" + cmd/gke-certificates-controller ) readonly KUBE_ALL_BINARIES=("${KUBE_ALL_TARGETS[@]##*/}") diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index b0449a7bd2f..85a6e0b48b6 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -87,6 +87,8 @@ cluster-ip cluster-monitor-period cluster-name cluster-signing-cert-file +cluster-signing-gke-kubeconfig +cluster-signing-gke-retry-backoff cluster-signing-key-file cluster-tag cni-bin-dir