diff --git a/test/integration/client/cert_rotation_test.go b/test/integration/client/cert_rotation_test.go index ebeae2689d2..fd5c76d6768 100644 --- a/test/integration/client/cert_rotation_test.go +++ b/test/integration/client/cert_rotation_test.go @@ -45,34 +45,21 @@ func TestCertRotation(t *testing.T) { stopCh := make(chan struct{}) defer close(stopCh) - clientSigningKey, err := utils.NewPrivateKey() - if err != nil { - t.Fatal(err) - } - clientSigningCert, err := cert.NewSelfSignedCACert(cert.Config{CommonName: "client-ca"}, clientSigningKey) - if err != nil { - t.Fatal(err) - } - transport.CertCallbackRefreshDuration = 1 * time.Second certDir := os.TempDir() - clientCAFilename := path.Join(certDir, "ca.crt") - - if err := ioutil.WriteFile(clientCAFilename, utils.EncodeCertPEM(clientSigningCert), 0644); err != nil { - t.Fatal(err) - } + clientCAFilename, clientSigningCert, clientSigningKey := writeCACertFiles(t, certDir) server := apiservertesting.StartTestServerOrDie(t, apiservertesting.NewDefaultTestServerOptions(), []string{ "--client-ca-file=" + clientCAFilename, }, framework.SharedEtcd()) defer server.TearDownFn() - writeCerts(t, clientSigningCert, clientSigningKey, certDir, 30*time.Second) + clientCertFilename, clientKeyFilename := writeCerts(t, clientSigningCert, clientSigningKey, certDir, 30*time.Second) kubeconfig := server.ClientConfig - kubeconfig.CertFile = path.Join(certDir, "client.crt") - kubeconfig.KeyFile = path.Join(certDir, "client.key") + kubeconfig.CertFile = clientCertFilename + kubeconfig.KeyFile = clientKeyFilename kubeconfig.BearerToken = "" client := clientset.NewForConfigOrDie(kubeconfig) @@ -115,34 +102,21 @@ func TestCertRotationContinuousRequests(t *testing.T) { stopCh := make(chan struct{}) defer close(stopCh) - clientSigningKey, err := utils.NewPrivateKey() - if err != nil { - t.Fatal(err) - } - clientSigningCert, err := cert.NewSelfSignedCACert(cert.Config{CommonName: "client-ca"}, clientSigningKey) - if err != nil { - t.Fatal(err) - } - transport.CertCallbackRefreshDuration = 1 * time.Second certDir := os.TempDir() - clientCAFilename := path.Join(certDir, "ca.crt") - - if err := ioutil.WriteFile(clientCAFilename, utils.EncodeCertPEM(clientSigningCert), 0644); err != nil { - t.Fatal(err) - } + clientCAFilename, clientSigningCert, clientSigningKey := writeCACertFiles(t, certDir) server := apiservertesting.StartTestServerOrDie(t, apiservertesting.NewDefaultTestServerOptions(), []string{ "--client-ca-file=" + clientCAFilename, }, framework.SharedEtcd()) defer server.TearDownFn() - writeCerts(t, clientSigningCert, clientSigningKey, certDir, 30*time.Second) + clientCertFilename, clientKeyFilename := writeCerts(t, clientSigningCert, clientSigningKey, certDir, 30*time.Second) kubeconfig := server.ClientConfig - kubeconfig.CertFile = path.Join(certDir, "client.crt") - kubeconfig.KeyFile = path.Join(certDir, "client.key") + kubeconfig.CertFile = clientCertFilename + kubeconfig.KeyFile = clientKeyFilename kubeconfig.BearerToken = "" client := clientset.NewForConfigOrDie(kubeconfig) @@ -171,7 +145,26 @@ func TestCertRotationContinuousRequests(t *testing.T) { } } -func writeCerts(t *testing.T, clientSigningCert *x509.Certificate, clientSigningKey *rsa.PrivateKey, certDir string, duration time.Duration) { +func writeCACertFiles(t *testing.T, certDir string) (string, *x509.Certificate, *rsa.PrivateKey) { + clientSigningKey, err := utils.NewPrivateKey() + if err != nil { + t.Fatal(err) + } + clientSigningCert, err := cert.NewSelfSignedCACert(cert.Config{CommonName: "client-ca"}, clientSigningKey) + if err != nil { + t.Fatal(err) + } + + clientCAFilename := path.Join(certDir, "ca.crt") + + if err := ioutil.WriteFile(clientCAFilename, utils.EncodeCertPEM(clientSigningCert), 0644); err != nil { + t.Fatal(err) + } + + return clientCAFilename, clientSigningCert, clientSigningKey +} + +func writeCerts(t *testing.T, clientSigningCert *x509.Certificate, clientSigningKey *rsa.PrivateKey, certDir string, duration time.Duration) (string, string) { clientKey, err := utils.NewPrivateKey() if err != nil { t.Fatal(err) @@ -211,4 +204,6 @@ func writeCerts(t *testing.T, clientSigningCert *x509.Certificate, clientSigning if err := ioutil.WriteFile(path.Join(certDir, "client.crt"), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDERBytes}), 0666); err != nil { t.Fatal(err) } + + return path.Join(certDir, "client.crt"), path.Join(certDir, "client.key") } diff --git a/test/integration/client/exec_test.go b/test/integration/client/exec_test.go new file mode 100644 index 00000000000..11529bd1744 --- /dev/null +++ b/test/integration/client/exec_test.go @@ -0,0 +1,466 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "fmt" + "io/ioutil" + "net/http" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + "k8s.io/client-go/transport" + "k8s.io/client-go/util/cert" + + kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" + "k8s.io/kubernetes/test/integration/framework" +) + +// This file tests the client-go credential plugin feature. + +type roundTripperFunc func(*http.Request) (*http.Response, error) + +func (f roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req) +} + +type syncedHeaderValues struct { + mu sync.Mutex + data [][]string +} + +func (s *syncedHeaderValues) append(values []string) { + s.mu.Lock() + defer s.mu.Unlock() + s.data = append(s.data, values) +} + +func (s *syncedHeaderValues) get() [][]string { + s.mu.Lock() + defer s.mu.Unlock() + return s.data +} + +func TestExecPlugin(t *testing.T) { + // These constants are used to communicate behavior to the testdata/exec-plugin.sh test fixture. + const ( + outputEnvVar = "EXEC_PLUGIN_OUTPUT" + ) + + const ( + clientAuthorizedToken = "authorized-token" + clientUnauthorizedToken = "unauthorized-token" + ) + + certDir, err := ioutil.TempDir("", "kubernetes-client-exec-test-cert-dir-*") + if err != nil { + t.Fatal(err) + } + + tokenFileName := writeTokenFile(t, clientAuthorizedToken) + clientCAFileName, clientSigningCert, clientSigningKey := writeCACertFiles(t, certDir) + clientCertFileName, clientKeyFileName := writeCerts(t, clientSigningCert, clientSigningKey, certDir, 30*time.Second) + result := kubeapiservertesting.StartTestServerOrDie( + t, + nil, + []string{ + "--token-auth-file", tokenFileName, + "--client-ca-file=" + clientCAFileName, + }, + framework.SharedEtcd(), + ) + t.Cleanup(result.TearDownFn) + + unauthorizedCert, unauthorizedKey, err := cert.GenerateSelfSignedCertKey("some-host", nil, nil) + if err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + clientConfigFunc func(*rest.Config) + wantAuthorizationHeaderValues [][]string + wantCertificate *tls.Certificate + wantClientErrorPrefix string + }{ + { + name: "unauthorized token", + clientConfigFunc: func(c *rest.Config) { + c.ExecProvider.Env = []clientcmdapi.ExecEnvVar{ + { + Name: outputEnvVar, + Value: `{ + "kind": "ExecCredential", + "apiVersion": "client.authentication.k8s.io/v1beta1", + "status": { + "token": "unauthorized" + } + }`, + }, + } + }, + wantAuthorizationHeaderValues: [][]string{{"Bearer unauthorized"}}, + wantCertificate: &tls.Certificate{}, + wantClientErrorPrefix: "Unauthorized", + }, + { + name: "unauthorized certificate", + clientConfigFunc: func(c *rest.Config) { + c.ExecProvider.Env = []clientcmdapi.ExecEnvVar{ + { + Name: outputEnvVar, + Value: fmt.Sprintf(`{ + "kind": "ExecCredential", + "apiVersion": "client.authentication.k8s.io/v1beta1", + "status": { + "clientCertificateData": %q, + "clientKeyData": %q + } + }`, unauthorizedCert, unauthorizedKey), + }, + } + }, + wantAuthorizationHeaderValues: [][]string{nil}, + wantCertificate: x509KeyPair(unauthorizedCert, unauthorizedKey, true), + wantClientErrorPrefix: "Unauthorized", + }, + { + name: "authorized token", + clientConfigFunc: func(c *rest.Config) { + c.ExecProvider.Env = []clientcmdapi.ExecEnvVar{ + { + Name: outputEnvVar, + Value: fmt.Sprintf(`{ + "kind": "ExecCredential", + "apiVersion": "client.authentication.k8s.io/v1beta1", + "status": { + "token": "%s" + } + }`, clientAuthorizedToken), + }, + } + }, + wantAuthorizationHeaderValues: [][]string{{"Bearer " + clientAuthorizedToken}}, + wantCertificate: &tls.Certificate{}, + }, + { + name: "authorized certificate", + clientConfigFunc: func(c *rest.Config) { + c.ExecProvider.Env = []clientcmdapi.ExecEnvVar{ + { + Name: outputEnvVar, + Value: fmt.Sprintf(`{ + "kind": "ExecCredential", + "apiVersion": "client.authentication.k8s.io/v1beta1", + "status": { + "clientCertificateData": %s, + "clientKeyData": %s + } + }`, read(t, clientCertFileName), read(t, clientKeyFileName)), + }, + } + }, + wantAuthorizationHeaderValues: [][]string{nil}, + wantCertificate: loadX509KeyPair(clientCertFileName, clientKeyFileName), + }, + { + name: "authorized token and certificate", + clientConfigFunc: func(c *rest.Config) { + c.ExecProvider.Env = []clientcmdapi.ExecEnvVar{ + { + Name: outputEnvVar, + Value: fmt.Sprintf(`{ + "kind": "ExecCredential", + "apiVersion": "client.authentication.k8s.io/v1beta1", + "status": { + "token": "%s", + "clientCertificateData": %s, + "clientKeyData": %s + } + }`, clientAuthorizedToken, read(t, clientCertFileName), read(t, clientKeyFileName)), + }, + } + }, + wantAuthorizationHeaderValues: [][]string{{"Bearer " + clientAuthorizedToken}}, + wantCertificate: loadX509KeyPair(clientCertFileName, clientKeyFileName), + }, + { + name: "unauthorized token and authorized certificate favors authorized certificate", + clientConfigFunc: func(c *rest.Config) { + c.ExecProvider.Env = []clientcmdapi.ExecEnvVar{ + { + Name: outputEnvVar, + Value: fmt.Sprintf(`{ + "kind": "ExecCredential", + "apiVersion": "client.authentication.k8s.io/v1beta1", + "status": { + "token": "%s", + "clientCertificateData": %s, + "clientKeyData": %s + } + }`, clientUnauthorizedToken, read(t, clientCertFileName), read(t, clientKeyFileName)), + }, + } + }, + wantAuthorizationHeaderValues: [][]string{{"Bearer " + clientUnauthorizedToken}}, + wantCertificate: loadX509KeyPair(clientCertFileName, clientKeyFileName), + }, + { + name: "authorized token and unauthorized certificate favors authorized token", + clientConfigFunc: func(c *rest.Config) { + c.ExecProvider.Env = []clientcmdapi.ExecEnvVar{ + { + Name: outputEnvVar, + Value: fmt.Sprintf(`{ + "kind": "ExecCredential", + "apiVersion": "client.authentication.k8s.io/v1beta1", + "status": { + "token": "%s", + "clientCertificateData": %q, + "clientKeyData": %q + } + }`, clientAuthorizedToken, string(unauthorizedCert), string(unauthorizedKey)), + }, + } + }, + wantAuthorizationHeaderValues: [][]string{{"Bearer " + clientAuthorizedToken}}, + wantCertificate: x509KeyPair([]byte(unauthorizedCert), []byte(unauthorizedKey), true), + }, + { + name: "unauthorized token and unauthorized certificate", + clientConfigFunc: func(c *rest.Config) { + c.ExecProvider.Env = []clientcmdapi.ExecEnvVar{ + { + Name: outputEnvVar, + Value: fmt.Sprintf(`{ + "kind": "ExecCredential", + "apiVersion": "client.authentication.k8s.io/v1beta1", + "status": { + "token": "%s", + "clientCertificateData": %q, + "clientKeyData": %q + } + }`, clientUnauthorizedToken, string(unauthorizedCert), string(unauthorizedKey)), + }, + } + }, + wantAuthorizationHeaderValues: [][]string{{"Bearer " + clientUnauthorizedToken}}, + wantCertificate: x509KeyPair([]byte(unauthorizedCert), []byte(unauthorizedKey), true), + wantClientErrorPrefix: "Unauthorized", + }, + { + name: "good token with static auth basic creds favors static auth basic creds", + clientConfigFunc: func(c *rest.Config) { + c.ExecProvider.Env = []clientcmdapi.ExecEnvVar{ + { + Name: outputEnvVar, + Value: fmt.Sprintf(`{ + "kind": "ExecCredential", + "apiVersion": "client.authentication.k8s.io/v1beta1", + "status": { + "token": "%s" + } + }`, clientAuthorizedToken), + }, + } + c.Username = "unauthorized" + c.Password = "unauthorized" + }, + wantAuthorizationHeaderValues: [][]string{{"Basic " + basicAuthHeaderValue("unauthorized", "unauthorized")}}, + wantCertificate: &tls.Certificate{}, + wantClientErrorPrefix: "Unauthorized", + }, + { + name: "good token with static auth bearer token favors static auth bearer token", + clientConfigFunc: func(c *rest.Config) { + c.ExecProvider.Env = []clientcmdapi.ExecEnvVar{ + { + Name: outputEnvVar, + Value: fmt.Sprintf(`{ + "kind": "ExecCredential", + "apiVersion": "client.authentication.k8s.io/v1beta1", + "status": { + "token": "%s" + } + }`, clientAuthorizedToken), + }, + } + c.BearerToken = "some-unauthorized-token" + }, + wantAuthorizationHeaderValues: [][]string{{"Bearer some-unauthorized-token"}}, + wantClientErrorPrefix: "Unauthorized", + }, + { + // This is not the behavior we would expect, see + // https://github.com/kubernetes/kubernetes/issues/99603 + name: "good token with static auth cert and key favors exec plugin", + clientConfigFunc: func(c *rest.Config) { + c.ExecProvider.Env = []clientcmdapi.ExecEnvVar{ + { + Name: outputEnvVar, + Value: fmt.Sprintf(`{ + "kind": "ExecCredential", + "apiVersion": "client.authentication.k8s.io/v1beta1", + "status": { + "token": "%s" + } + }`, clientAuthorizedToken), + }, + } + c.CertData = unauthorizedCert + c.KeyData = unauthorizedKey + }, + wantAuthorizationHeaderValues: [][]string{{"Bearer " + clientAuthorizedToken}}, + wantCertificate: x509KeyPair([]byte(unauthorizedCert), []byte(unauthorizedKey), false), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "kubernetes-client-exec-test-plugin-dir-*") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpDir) + + var authorizationHeaderValues syncedHeaderValues + clientConfig := rest.AnonymousClientConfig(result.ClientConfig) + clientConfig.ExecProvider = &clientcmdapi.ExecConfig{ + Command: "testdata/exec-plugin.sh", + // TODO(ankeesler): move to v1 once exec plugins go GA. + APIVersion: "client.authentication.k8s.io/v1beta1", + } + clientConfig.Wrap(transport.WrapperFunc(func(rt http.RoundTripper) http.RoundTripper { + return roundTripperFunc(func(req *http.Request) (*http.Response, error) { + authorizationHeaderValues.append(req.Header.Values("Authorization")) + return rt.RoundTrip(req) + }) + })) + + if test.clientConfigFunc != nil { + test.clientConfigFunc(clientConfig) + } + client := clientset.NewForConfigOrDie(clientConfig) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + // Validate that the client works as expected on its own. + _, err = client.CoreV1().ConfigMaps("default").List(ctx, metav1.ListOptions{}) + if test.wantClientErrorPrefix != "" { + if err == nil || !strings.HasPrefix(err.Error(), test.wantClientErrorPrefix) { + t.Fatalf(`got %q, wanted "%s..."`, err, test.wantClientErrorPrefix) + } + } else if err != nil { + t.Fatal(err) + } + + // Validate that the right token is used. + if diff := cmp.Diff(test.wantAuthorizationHeaderValues, authorizationHeaderValues.get()); diff != "" { + t.Error("unexpected authorization header values; -want, +got:\n" + diff) + } + + // Validate that the right certs are used. + tlsConfig, err := rest.TLSConfigFor(clientConfig) + if err != nil { + t.Fatal(err) + } + if tlsConfig.GetClientCertificate == nil { + if test.wantCertificate != nil { + t.Error("GetClientCertificate is nil, but we expected a certificate") + } + } else { + cert, err := tlsConfig.GetClientCertificate(&tls.CertificateRequestInfo{}) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(test.wantCertificate, cert); diff != "" { + t.Error("unexpected certificate; -want, +got:\n" + diff) + } + } + }) + } +} + +func writeTokenFile(t *testing.T, goodToken string) string { + t.Helper() + + tokenFile, err := ioutil.TempFile("", "kubernetes-client-exec-test-token-file-*") + if err != nil { + t.Fatal(err) + } + + if _, err := tokenFile.WriteString(fmt.Sprintf(`%s,admin,uid1,"system:masters"`, goodToken)); err != nil { + t.Fatal(err) + } + + if err := tokenFile.Close(); err != nil { + t.Fatal(err) + } + + return tokenFile.Name() +} + +func read(t *testing.T, fileName string) string { + t.Helper() + data, err := ioutil.ReadFile(fileName) + if err != nil { + t.Fatal(err) + } + return fmt.Sprintf("%q", string(data)) +} + +func basicAuthHeaderValue(username, password string) string { + return base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", username, password))) +} + +func x509KeyPair(certPEMBlock, keyPEMBlock []byte, leaf bool) *tls.Certificate { + cert, err := tls.X509KeyPair(certPEMBlock, keyPEMBlock) + if err != nil { + panic(err) + } + if leaf { + cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + panic(err) + } + } + return &cert +} + +func loadX509KeyPair(certFile, keyFile string) *tls.Certificate { + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + panic(err) + } + cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + panic(err) + } + return &cert +} diff --git a/test/integration/client/testdata/exec-plugin.sh b/test/integration/client/testdata/exec-plugin.sh new file mode 100755 index 00000000000..6ad3e6ef6ab --- /dev/null +++ b/test/integration/client/testdata/exec-plugin.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +echo "$EXEC_PLUGIN_OUTPUT"