Fix client-ca dynamic reload in apiserver

This commit is contained in:
Tomas Nozicka 2020-04-29 16:03:09 +02:00
parent 47daccb272
commit b22a170d46
8 changed files with 280 additions and 122 deletions

View File

@ -13,7 +13,6 @@ go_library(
"//cmd/kube-apiserver/app/options:go_default_library",
"//pkg/api/legacyscheme:go_default_library",
"//pkg/capabilities:go_default_library",
"//pkg/controller/serviceaccount:go_default_library",
"//pkg/features:go_default_library",
"//pkg/generated/openapi:go_default_library",
"//pkg/kubeapiserver:go_default_library",
@ -29,7 +28,6 @@ go_library(
"//pkg/registry/cachesize:go_default_library",
"//pkg/registry/rbac/rest:go_default_library",
"//pkg/serviceaccount:go_default_library",
"//plugin/pkg/auth/authenticator/token/bootstrap:go_default_library",
"//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1:go_default_library",
"//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
"//staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver:go_default_library",
@ -43,7 +41,6 @@ go_library(
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/admission:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/endpoints/openapi:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/features:go_default_library",
@ -76,12 +73,17 @@ go_library(
"//staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1:go_default_library",
"//staging/src/k8s.io/kube-aggregator/pkg/client/informers/externalversions/apiregistration/v1:go_default_library",
"//staging/src/k8s.io/kube-aggregator/pkg/controllers/autoregister:go_default_library",
"//vendor/github.com/go-openapi/spec:go_default_library",
"//vendor/github.com/spf13/cobra:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
],
)
go_test(
name = "go_default_test",
srcs = ["server_test.go"],
embed = [":go_default_library"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
@ -99,9 +101,3 @@ filegroup(
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
go_test(
name = "go_default_test",
srcs = ["server_test.go"],
embed = [":go_default_library"],
)

View File

@ -30,17 +30,14 @@ import (
"strings"
"time"
"github.com/go-openapi/spec"
"github.com/spf13/cobra"
extensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
utilnet "k8s.io/apimachinery/pkg/util/net"
"k8s.io/apimachinery/pkg/util/sets"
utilwait "k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/authentication/authenticator"
"k8s.io/apiserver/pkg/authorization/authorizer"
openapinamer "k8s.io/apiserver/pkg/endpoints/openapi"
genericfeatures "k8s.io/apiserver/pkg/features"
@ -70,7 +67,6 @@ import (
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
"k8s.io/kubernetes/pkg/api/legacyscheme"
"k8s.io/kubernetes/pkg/capabilities"
serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount"
"k8s.io/kubernetes/pkg/features"
generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi"
"k8s.io/kubernetes/pkg/kubeapiserver"
@ -85,7 +81,6 @@ import (
"k8s.io/kubernetes/pkg/registry/cachesize"
rbacrest "k8s.io/kubernetes/pkg/registry/rbac/rest"
"k8s.io/kubernetes/pkg/serviceaccount"
"k8s.io/kubernetes/plugin/pkg/auth/authenticator/token/bootstrap"
)
const (
@ -440,9 +435,6 @@ func buildGenericConfig(
if lastErr = s.SecureServing.ApplyTo(&genericConfig.SecureServing, &genericConfig.LoopbackClientConfig); lastErr != nil {
return
}
if lastErr = s.Authentication.ApplyTo(genericConfig); lastErr != nil {
return
}
if lastErr = s.Features.ApplyTo(genericConfig); lastErr != nil {
return
}
@ -498,9 +490,8 @@ func buildGenericConfig(
}
versionedInformers = clientgoinformers.NewSharedInformerFactory(clientgoExternalClient, 10*time.Minute)
genericConfig.Authentication.Authenticator, genericConfig.OpenAPIConfig.SecurityDefinitions, err = BuildAuthenticator(s, genericConfig.EgressSelector, clientgoExternalClient, versionedInformers)
if err != nil {
lastErr = fmt.Errorf("invalid authentication config: %v", err)
// Authentication.ApplyTo requires already applied OpenAPIConfig and EgressSelector if present
if lastErr = s.Authentication.ApplyTo(&genericConfig.Authentication, genericConfig.SecureServing, genericConfig.EgressSelector, genericConfig.OpenAPIConfig, clientgoExternalClient, versionedInformers); lastErr != nil {
return
}
@ -559,35 +550,6 @@ func buildGenericConfig(
return
}
// BuildAuthenticator constructs the authenticator
func BuildAuthenticator(s *options.ServerRunOptions, EgressSelector *egressselector.EgressSelector, extclient clientgoclientset.Interface, versionedInformer clientgoinformers.SharedInformerFactory) (authenticator.Request, *spec.SecurityDefinitions, error) {
authenticatorConfig, err := s.Authentication.ToAuthenticationConfig()
if err != nil {
return nil, nil, err
}
if s.Authentication.ServiceAccounts.Lookup || utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) {
authenticatorConfig.ServiceAccountTokenGetter = serviceaccountcontroller.NewGetterFromClient(
extclient,
versionedInformer.Core().V1().Secrets().Lister(),
versionedInformer.Core().V1().ServiceAccounts().Lister(),
versionedInformer.Core().V1().Pods().Lister(),
)
}
authenticatorConfig.BootstrapTokenAuthenticator = bootstrap.NewTokenAuthenticator(
versionedInformer.Core().V1().Secrets().Lister().Secrets(v1.NamespaceSystem),
)
if EgressSelector != nil {
egressDialer, err := EgressSelector.Lookup(egressselector.Master.AsNetworkContext())
if err != nil {
return nil, nil, err
}
authenticatorConfig.CustomDial = egressDialer
}
return authenticatorConfig.New()
}
// BuildAuthorizer constructs the authorizer
func BuildAuthorizer(s *options.ServerRunOptions, EgressSelector *egressselector.EgressSelector, versionedInformers clientgoinformers.SharedInformerFactory) (authorizer.Authorizer, authorizer.RuleResolver, error) {
authorizationConfig := s.Authorization.ToAuthorizationConfig(versionedInformers)

View File

@ -36,7 +36,8 @@ import (
)
// BuildAuth creates an authenticator, an authorizer, and a matching authorizer attributes getter compatible with the kubelet's needs
func BuildAuth(nodeName types.NodeName, client clientset.Interface, config kubeletconfig.KubeletConfiguration) (server.AuthInterface, error) {
// It returns AuthInterface, a run method to start internal controllers (like cert reloading) and error.
func BuildAuth(nodeName types.NodeName, client clientset.Interface, config kubeletconfig.KubeletConfiguration) (server.AuthInterface, func(<-chan struct{}), error) {
// Get clients, if provided
var (
tokenClient authenticationclient.TokenReviewInterface
@ -47,47 +48,55 @@ func BuildAuth(nodeName types.NodeName, client clientset.Interface, config kubel
sarClient = client.AuthorizationV1().SubjectAccessReviews()
}
authenticator, err := BuildAuthn(tokenClient, config.Authentication)
authenticator, runAuthenticatorCAReload, err := BuildAuthn(tokenClient, config.Authentication)
if err != nil {
return nil, err
return nil, nil, err
}
attributes := server.NewNodeAuthorizerAttributesGetter(nodeName)
authorizer, err := BuildAuthz(sarClient, config.Authorization)
if err != nil {
return nil, err
return nil, nil, err
}
return server.NewKubeletAuth(authenticator, attributes, authorizer), nil
return server.NewKubeletAuth(authenticator, attributes, authorizer), runAuthenticatorCAReload, nil
}
// BuildAuthn creates an authenticator compatible with the kubelet's needs
func BuildAuthn(client authenticationclient.TokenReviewInterface, authn kubeletconfig.KubeletAuthentication) (authenticator.Request, error) {
var clientCertificateCAContentProvider authenticatorfactory.CAContentProvider
func BuildAuthn(client authenticationclient.TokenReviewInterface, authn kubeletconfig.KubeletAuthentication) (authenticator.Request, func(<-chan struct{}), error) {
var dynamicCAContentFromFile *dynamiccertificates.DynamicFileCAContent
var err error
if len(authn.X509.ClientCAFile) > 0 {
clientCertificateCAContentProvider, err = dynamiccertificates.NewDynamicCAContentFromFile("client-ca-bundle", authn.X509.ClientCAFile)
dynamicCAContentFromFile, err = dynamiccertificates.NewDynamicCAContentFromFile("client-ca-bundle", authn.X509.ClientCAFile)
if err != nil {
return nil, err
return nil, nil, err
}
}
authenticatorConfig := authenticatorfactory.DelegatingAuthenticatorConfig{
Anonymous: authn.Anonymous.Enabled,
CacheTTL: authn.Webhook.CacheTTL.Duration,
ClientCertificateCAContentProvider: clientCertificateCAContentProvider,
ClientCertificateCAContentProvider: dynamicCAContentFromFile,
}
if authn.Webhook.Enabled {
if client == nil {
return nil, errors.New("no client provided, cannot use webhook authentication")
return nil, nil, errors.New("no client provided, cannot use webhook authentication")
}
authenticatorConfig.TokenAccessReviewClient = client
}
authenticator, _, err := authenticatorConfig.New()
return authenticator, err
if err != nil {
return nil, nil, err
}
return authenticator, func(stopCh <-chan struct{}) {
if dynamicCAContentFromFile != nil {
go dynamicCAContentFromFile.Run(1, stopCh)
}
}, err
}
// BuildAuthz creates an authorizer compatible with the kubelet's needs

View File

@ -599,11 +599,12 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, featureGate f
}
if kubeDeps.Auth == nil {
auth, err := BuildAuth(nodeName, kubeDeps.KubeClient, s.KubeletConfiguration)
auth, runAuthenticatorCAReload, err := BuildAuth(nodeName, kubeDeps.KubeClient, s.KubeletConfiguration)
if err != nil {
return err
}
kubeDeps.Auth = auth
runAuthenticatorCAReload(stopCh)
}
var cgroupRoots []string

View File

@ -14,6 +14,7 @@ go_library(
importpath = "k8s.io/kubernetes/pkg/kubeapiserver/options",
visibility = ["//visibility:public"],
deps = [
"//pkg/controller/serviceaccount:go_default_library",
"//pkg/features:go_default_library",
"//pkg/kubeapiserver/authenticator:go_default_library",
"//pkg/kubeapiserver/authorizer:go_default_library",
@ -50,6 +51,8 @@ go_library(
"//plugin/pkg/admission/storage/persistentvolume/resize:go_default_library",
"//plugin/pkg/admission/storage/storageclass/setdefault:go_default_library",
"//plugin/pkg/admission/storage/storageobjectinuseprotection:go_default_library",
"//plugin/pkg/auth/authenticator/token/bootstrap:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/admission:go_default_library",
@ -58,14 +61,17 @@ go_library(
"//staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/validating:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/server:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/server/egressselector:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//staging/src/k8s.io/client-go/informers:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/rest:go_default_library",
"//staging/src/k8s.io/component-base/cli/flag:go_default_library",
"//staging/src/k8s.io/component-base/featuregate:go_default_library",
"//vendor/github.com/spf13/pflag:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
"//vendor/k8s.io/kube-openapi/pkg/common:go_default_library",
],
)

View File

@ -24,17 +24,24 @@ import (
"time"
"github.com/spf13/pflag"
"k8s.io/klog"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apiserver/pkg/authentication/authenticator"
genericapiserver "k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/server/egressselector"
genericoptions "k8s.io/apiserver/pkg/server/options"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/klog"
openapicommon "k8s.io/kube-openapi/pkg/common"
serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount"
"k8s.io/kubernetes/pkg/features"
kubeauthenticator "k8s.io/kubernetes/pkg/kubeapiserver/authenticator"
authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
"k8s.io/kubernetes/plugin/pkg/auth/authenticator/token/bootstrap"
)
type BuiltInAuthenticationOptions struct {
@ -406,35 +413,60 @@ func (s *BuiltInAuthenticationOptions) ToAuthenticationConfig() (kubeauthenticat
return ret, nil
}
func (o *BuiltInAuthenticationOptions) ApplyTo(c *genericapiserver.Config) error {
// ApplyTo requires already applied OpenAPIConfig and EgressSelector if present.
func (o *BuiltInAuthenticationOptions) ApplyTo(authInfo *genericapiserver.AuthenticationInfo, secureServing *genericapiserver.SecureServingInfo, egressSelector *egressselector.EgressSelector, openAPIConfig *openapicommon.Config, extclient kubernetes.Interface, versionedInformer informers.SharedInformerFactory) error {
if o == nil {
return nil
}
if o.ClientCert != nil {
clientCertificateCAContentProvider, err := o.ClientCert.GetClientCAContentProvider()
if err != nil {
return fmt.Errorf("unable to load client CA file: %v", err)
}
if err = c.Authentication.ApplyClientCert(clientCertificateCAContentProvider, c.SecureServing); err != nil {
if openAPIConfig == nil {
return errors.New("uninitialized OpenAPIConfig")
}
authenticatorConfig, err := o.ToAuthenticationConfig()
if err != nil {
return err
}
if authenticatorConfig.ClientCAContentProvider != nil {
if err = authInfo.ApplyClientCert(authenticatorConfig.ClientCAContentProvider, secureServing); err != nil {
return fmt.Errorf("unable to load client CA file: %v", err)
}
}
if o.RequestHeader != nil {
requestHeaderConfig, err := o.RequestHeader.ToAuthenticationRequestHeaderConfig()
if err != nil {
return fmt.Errorf("unable to create request header authentication config: %v", err)
}
if requestHeaderConfig != nil {
if err = c.Authentication.ApplyClientCert(requestHeaderConfig.CAContentProvider, c.SecureServing); err != nil {
return fmt.Errorf("unable to load client CA file: %v", err)
}
if authenticatorConfig.RequestHeaderConfig != nil && authenticatorConfig.RequestHeaderConfig.CAContentProvider != nil {
if err = authInfo.ApplyClientCert(authenticatorConfig.RequestHeaderConfig.CAContentProvider, secureServing); err != nil {
return fmt.Errorf("unable to load client CA file: %v", err)
}
}
c.Authentication.APIAudiences = o.APIAudiences
authInfo.APIAudiences = o.APIAudiences
if o.ServiceAccounts != nil && o.ServiceAccounts.Issuer != "" && len(o.APIAudiences) == 0 {
c.Authentication.APIAudiences = authenticator.Audiences{o.ServiceAccounts.Issuer}
authInfo.APIAudiences = authenticator.Audiences{o.ServiceAccounts.Issuer}
}
if o.ServiceAccounts.Lookup || utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) {
authenticatorConfig.ServiceAccountTokenGetter = serviceaccountcontroller.NewGetterFromClient(
extclient,
versionedInformer.Core().V1().Secrets().Lister(),
versionedInformer.Core().V1().ServiceAccounts().Lister(),
versionedInformer.Core().V1().Pods().Lister(),
)
}
authenticatorConfig.BootstrapTokenAuthenticator = bootstrap.NewTokenAuthenticator(
versionedInformer.Core().V1().Secrets().Lister().Secrets(metav1.NamespaceSystem),
)
if egressSelector != nil {
egressDialer, err := egressSelector.Lookup(egressselector.Master.AsNetworkContext())
if err != nil {
return err
}
authenticatorConfig.CustomDial = egressDialer
}
authInfo.Authenticator, openAPIConfig.SecurityDefinitions, err = authenticatorConfig.New()
if err != nil {
return err
}
return nil

View File

@ -9,11 +9,14 @@ go_test(
tags = ["integration"],
deps = [
"//cmd/kube-apiserver/app/options:go_default_library",
"//staging/src/k8s.io/api/authorization/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library",
"//staging/src/k8s.io/client-go/rest:go_default_library",
"//staging/src/k8s.io/client-go/util/cert:go_default_library",
"//staging/src/k8s.io/component-base/cli/flag:go_default_library",
"//test/integration/framework:go_default_library",

View File

@ -19,63 +19,150 @@ package podlogs
import (
"bytes"
"context"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"fmt"
"io/ioutil"
"math/big"
"path"
"strings"
"testing"
"time"
authorizationv1 "k8s.io/api/authorization/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/server/dynamiccertificates"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"k8s.io/client-go/util/cert"
"k8s.io/component-base/cli/flag"
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
"k8s.io/kubernetes/test/integration/framework"
)
type caWithClient struct {
CACert []byte
ClientCert []byte
ClientKey []byte
}
func newTestCAWithClient(caSubject pkix.Name, caSerial *big.Int, clientSubject pkix.Name, subjectSerial *big.Int) (*caWithClient, error) {
ca := &x509.Certificate{
SerialNumber: caSerial,
Subject: caSubject,
NotBefore: time.Now(),
NotAfter: time.Now().Add(24 * time.Hour),
IsCA: true,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
BasicConstraintsValid: true,
}
caPrivateKey, err := rsa.GenerateKey(rand.Reader, 4096)
if err != nil {
return nil, err
}
caBytes, err := x509.CreateCertificate(rand.Reader, ca, ca, &caPrivateKey.PublicKey, caPrivateKey)
if err != nil {
return nil, err
}
caPEM := new(bytes.Buffer)
err = pem.Encode(caPEM, &pem.Block{
Type: "CERTIFICATE",
Bytes: caBytes,
})
if err != nil {
return nil, err
}
clientCert := &x509.Certificate{
SerialNumber: subjectSerial,
Subject: clientSubject,
NotBefore: time.Now(),
NotAfter: time.Now().Add(24 * time.Hour),
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
KeyUsage: x509.KeyUsageDigitalSignature,
}
clientCertPrivateKey, err := rsa.GenerateKey(rand.Reader, 4096)
if err != nil {
return nil, err
}
clientCertPrivateKeyPEM := new(bytes.Buffer)
err = pem.Encode(clientCertPrivateKeyPEM, &pem.Block{
Type: "RSA PRIVATE KEY",
Bytes: x509.MarshalPKCS1PrivateKey(clientCertPrivateKey),
})
if err != nil {
return nil, err
}
clientCertBytes, err := x509.CreateCertificate(rand.Reader, clientCert, ca, &clientCertPrivateKey.PublicKey, caPrivateKey)
if err != nil {
return nil, err
}
clientCertPEM := new(bytes.Buffer)
err = pem.Encode(clientCertPEM, &pem.Block{
Type: "CERTIFICATE",
Bytes: clientCertBytes,
})
if err != nil {
return nil, err
}
return &caWithClient{
CACert: caPEM.Bytes(),
ClientCert: clientCertPEM.Bytes(),
ClientKey: clientCertPrivateKeyPEM.Bytes(),
}, nil
}
func TestClientCA(t *testing.T) {
stopCh := make(chan struct{})
defer close(stopCh)
// I have no idea what this cert is, but it doesn't matter, we just want something that always fails validation
differentClientCA := []byte(`-----BEGIN CERTIFICATE-----
MIIDQDCCAiigAwIBAgIJANWw74P5KJk2MA0GCSqGSIb3DQEBCwUAMDQxMjAwBgNV
BAMMKWdlbmVyaWNfd2ViaG9va19hZG1pc3Npb25fcGx1Z2luX3Rlc3RzX2NhMCAX
DTE3MTExNjAwMDUzOVoYDzIyOTEwOTAxMDAwNTM5WjAjMSEwHwYDVQQDExh3ZWJo
b29rLXRlc3QuZGVmYXVsdC5zdmMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
AoIBAQDXd/nQ89a5H8ifEsigmMd01Ib6NVR3bkJjtkvYnTbdfYEBj7UzqOQtHoLa
dIVmefny5uIHvj93WD8WDVPB3jX2JHrXkDTXd/6o6jIXHcsUfFTVLp6/bZ+Anqe0
r/7hAPkzA2A7APyTWM3ZbEeo1afXogXhOJ1u/wz0DflgcB21gNho4kKTONXO3NHD
XLpspFqSkxfEfKVDJaYAoMnYZJtFNsa2OvsmLnhYF8bjeT3i07lfwrhUZvP+7Gsp
7UgUwc06WuNHjfx1s5e6ySzH0QioMD1rjYneqOvk0pKrMIhuAEWXqq7jlXcDtx1E
j+wnYbVqqVYheHZ8BCJoVAAQGs9/AgMBAAGjZDBiMAkGA1UdEwQCMAAwCwYDVR0P
BAQDAgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATApBgNVHREEIjAg
hwR/AAABghh3ZWJob29rLXRlc3QuZGVmYXVsdC5zdmMwDQYJKoZIhvcNAQELBQAD
ggEBAD/GKSPNyQuAOw/jsYZesb+RMedbkzs18sSwlxAJQMUrrXwlVdHrA8q5WhE6
ABLqU1b8lQ8AWun07R8k5tqTmNvCARrAPRUqls/ryER+3Y9YEcxEaTc3jKNZFLbc
T6YtcnkdhxsiO136wtiuatpYL91RgCmuSpR8+7jEHhuFU01iaASu7ypFrUzrKHTF
bKwiLRQi1cMzVcLErq5CDEKiKhUkoDucyARFszrGt9vNIl/YCcBOkcNvM3c05Hn3
M++C29JwS3Hwbubg6WO3wjFjoEhpCwU6qRYUz3MRp4tHO4kxKXx+oQnUiFnR7vW0
YkNtGc1RUDHwecCTFpJtPb7Yu/E=
-----END CERTIFICATE-----
`)
differentFrontProxyCA := []byte(`-----BEGIN CERTIFICATE-----
MIIBqDCCAU2gAwIBAgIUfbqeieihh/oERbfvRm38XvS/xHAwCgYIKoZIzj0EAwIw
GjEYMBYGA1UEAxMPSW50ZXJtZWRpYXRlLUNBMCAXDTE2MTAxMTA1MDYwMFoYDzIx
MTYwOTE3MDUwNjAwWjAUMRIwEAYDVQQDEwlNeSBDbGllbnQwWTATBgcqhkjOPQIB
BggqhkjOPQMBBwNCAARv6N4R/sjMR65iMFGNLN1GC/vd7WhDW6J4X/iAjkRLLnNb
KbRG/AtOUZ+7upJ3BWIRKYbOabbQGQe2BbKFiap4o3UwczAOBgNVHQ8BAf8EBAMC
BaAwEwYDVR0lBAwwCgYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU
K/pZOWpNcYai6eHFpmJEeFpeQlEwHwYDVR0jBBgwFoAUX6nQlxjfWnP6aM1meO/Q
a6b3a9kwCgYIKoZIzj0EAwIDSQAwRgIhAIWTKw/sjJITqeuNzJDAKU4xo1zL+xJ5
MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps=
-----END CERTIFICATE-----
frontProxyCA, err := newTestCAWithClient(
pkix.Name{
CommonName: "test-front-proxy-ca",
},
big.NewInt(43),
pkix.Name{
CommonName: "test-aggregated-apiserver",
Organization: []string{"system:masters"},
},
big.NewInt(86),
)
if err != nil {
t.Error(err)
return
}
clientCA, err := newTestCAWithClient(
pkix.Name{
CommonName: "test-client-ca",
},
big.NewInt(42),
pkix.Name{
CommonName: "system:admin",
Organization: []string{"system:masters"},
},
big.NewInt(84),
)
if err != nil {
t.Error(err)
return
}
`)
clientCAFilename := ""
frontProxyCAFilename := ""
@ -84,12 +171,13 @@ MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps=
opts.GenericServerRunOptions.MaxRequestBodyBytes = 1024 * 1024
clientCAFilename = opts.Authentication.ClientCert.ClientCA
frontProxyCAFilename = opts.Authentication.RequestHeader.ClientCAFile
opts.Authentication.RequestHeader.AllowedNames = append(opts.Authentication.RequestHeader.AllowedNames, "test-aggregated-apiserver")
dynamiccertificates.FileRefreshDuration = 1 * time.Second
},
})
// wait for request header info
err := wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "requestheader-client-ca-file", "-----BEGIN CERTIFICATE-----", 1))
err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "requestheader-client-ca-file", "-----BEGIN CERTIFICATE-----", 1))
if err != nil {
t.Fatal(err)
}
@ -100,10 +188,10 @@ MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps=
}
// when we run this the second time, we know which one we are expecting
if err := ioutil.WriteFile(clientCAFilename, differentClientCA, 0644); err != nil {
if err := ioutil.WriteFile(clientCAFilename, clientCA.CACert, 0644); err != nil {
t.Fatal(err)
}
if err := ioutil.WriteFile(frontProxyCAFilename, differentFrontProxyCA, 0644); err != nil {
if err := ioutil.WriteFile(frontProxyCAFilename, frontProxyCA.CACert, 0644); err != nil {
t.Fatal(err)
}
@ -114,7 +202,7 @@ MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps=
t.Fatal(err)
}
expectedCAs := []string{"webhook-test.default.svc", "My Client"}
expectedCAs := []string{"test-client-ca", "test-front-proxy-ca"}
if len(expectedCAs) != len(acceptableCAs) {
t.Fatal(strings.Join(acceptableCAs, ":"))
}
@ -129,7 +217,7 @@ MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps=
if err != nil {
t.Error(err)
}
err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "requestheader-client-ca-file", "MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps=", 1))
err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "requestheader-client-ca-file", string(frontProxyCA.CACert), 1))
if err != nil {
t.Error(err)
}
@ -138,7 +226,68 @@ MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps=
if err != nil {
t.Error(err)
}
err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "client-ca-file", "M++C29JwS3Hwbubg6WO3wjFjoEhpCwU6qRYUz3MRp4tHO4kxKXx+oQnUiFnR7vW0", 1))
err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "client-ca-file", string(clientCA.CACert), 1))
if err != nil {
t.Error(err)
}
// Test an aggregated apiserver client (signed by the new front proxy CA) is authorized
extensionApiserverClient, err := kubernetes.NewForConfig(&rest.Config{
Host: kubeconfig.Host,
TLSClientConfig: rest.TLSClientConfig{
CAData: kubeconfig.TLSClientConfig.CAData,
CAFile: kubeconfig.TLSClientConfig.CAFile,
ServerName: kubeconfig.TLSClientConfig.ServerName,
KeyData: frontProxyCA.ClientKey,
CertData: frontProxyCA.ClientCert,
},
})
if err != nil {
t.Error(err)
return
}
// Call an endpoint to make sure we are authenticated
err = extensionApiserverClient.AuthorizationV1().RESTClient().
Post().
Resource("subjectaccessreviews").
VersionedParams(&metav1.CreateOptions{}, scheme.ParameterCodec).
Body(&authorizationv1.SubjectAccessReview{
Spec: authorizationv1.SubjectAccessReviewSpec{
ResourceAttributes: &authorizationv1.ResourceAttributes{
Verb: "create",
Resource: "pods",
Namespace: "default",
},
User: "deads2k",
},
}).
SetHeader("X-Remote-User", "test-aggregated-apiserver").
SetHeader("X-Remote-Group", "system:masters").
Do(context.Background()).
Into(&authorizationv1.SubjectAccessReview{})
if err != nil {
t.Error(err)
}
// Test a client signed by the new ClientCA is authorized
testClient, err := kubernetes.NewForConfig(&rest.Config{
Host: kubeconfig.Host,
TLSClientConfig: rest.TLSClientConfig{
CAData: kubeconfig.TLSClientConfig.CAData,
CAFile: kubeconfig.TLSClientConfig.CAFile,
ServerName: kubeconfig.TLSClientConfig.ServerName,
KeyData: clientCA.ClientKey,
CertData: clientCA.ClientCert,
},
})
if err != nil {
t.Error(err)
return
}
// Call an endpoint to make sure we are authenticated
_, err = testClient.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{})
if err != nil {
t.Error(err)
}