mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-24 12:15:52 +00:00
Fix client-ca dynamic reload in apiserver
This commit is contained in:
Tomas Nozicka
parent
47daccb272
commit
b22a170d46
@ -13,7 +13,6 @@ go_library(
|
||||
"//cmd/kube-apiserver/app/options:go_default_library",
|
||||
"//pkg/api/legacyscheme:go_default_library",
|
||||
"//pkg/capabilities:go_default_library",
|
||||
"//pkg/controller/serviceaccount:go_default_library",
|
||||
"//pkg/features:go_default_library",
|
||||
"//pkg/generated/openapi:go_default_library",
|
||||
"//pkg/kubeapiserver:go_default_library",
|
||||
@ -29,7 +28,6 @@ go_library(
|
||||
"//pkg/registry/cachesize:go_default_library",
|
||||
"//pkg/registry/rbac/rest:go_default_library",
|
||||
"//pkg/serviceaccount:go_default_library",
|
||||
"//plugin/pkg/auth/authenticator/token/bootstrap:go_default_library",
|
||||
"//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1:go_default_library",
|
||||
"//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
|
||||
"//staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver:go_default_library",
|
||||
@ -43,7 +41,6 @@ go_library(
|
||||
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
|
||||
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
|
||||
"//staging/src/k8s.io/apiserver/pkg/admission:go_default_library",
|
||||
"//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library",
|
||||
"//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library",
|
||||
"//staging/src/k8s.io/apiserver/pkg/endpoints/openapi:go_default_library",
|
||||
"//staging/src/k8s.io/apiserver/pkg/features:go_default_library",
|
||||
@ -76,12 +73,17 @@ go_library(
|
||||
"//staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1:go_default_library",
|
||||
"//staging/src/k8s.io/kube-aggregator/pkg/client/informers/externalversions/apiregistration/v1:go_default_library",
|
||||
"//staging/src/k8s.io/kube-aggregator/pkg/controllers/autoregister:go_default_library",
|
||||
"//vendor/github.com/go-openapi/spec:go_default_library",
|
||||
"//vendor/github.com/spf13/cobra:go_default_library",
|
||||
"//vendor/k8s.io/klog:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
go_test(
|
||||
name = "go_default_test",
|
||||
srcs = ["server_test.go"],
|
||||
embed = [":go_default_library"],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "package-srcs",
|
||||
srcs = glob(["**"]),
|
||||
@ -99,9 +101,3 @@ filegroup(
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
go_test(
|
||||
name = "go_default_test",
|
||||
srcs = ["server_test.go"],
|
||||
embed = [":go_default_library"],
|
||||
)
|
||||
|
||||
@ -30,17 +30,14 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-openapi/spec"
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
extensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
|
||||
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
utilerrors "k8s.io/apimachinery/pkg/util/errors"
|
||||
utilnet "k8s.io/apimachinery/pkg/util/net"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
utilwait "k8s.io/apimachinery/pkg/util/wait"
|
||||
"k8s.io/apiserver/pkg/admission"
|
||||
"k8s.io/apiserver/pkg/authentication/authenticator"
|
||||
"k8s.io/apiserver/pkg/authorization/authorizer"
|
||||
openapinamer "k8s.io/apiserver/pkg/endpoints/openapi"
|
||||
genericfeatures "k8s.io/apiserver/pkg/features"
|
||||
@ -70,7 +67,6 @@ import (
|
||||
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
|
||||
"k8s.io/kubernetes/pkg/api/legacyscheme"
|
||||
"k8s.io/kubernetes/pkg/capabilities"
|
||||
serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount"
|
||||
"k8s.io/kubernetes/pkg/features"
|
||||
generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi"
|
||||
"k8s.io/kubernetes/pkg/kubeapiserver"
|
||||
@ -85,7 +81,6 @@ import (
|
||||
"k8s.io/kubernetes/pkg/registry/cachesize"
|
||||
rbacrest "k8s.io/kubernetes/pkg/registry/rbac/rest"
|
||||
"k8s.io/kubernetes/pkg/serviceaccount"
|
||||
"k8s.io/kubernetes/plugin/pkg/auth/authenticator/token/bootstrap"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -440,9 +435,6 @@ func buildGenericConfig(
|
||||
if lastErr = s.SecureServing.ApplyTo(&genericConfig.SecureServing, &genericConfig.LoopbackClientConfig); lastErr != nil {
|
||||
return
|
||||
}
|
||||
if lastErr = s.Authentication.ApplyTo(genericConfig); lastErr != nil {
|
||||
return
|
||||
}
|
||||
if lastErr = s.Features.ApplyTo(genericConfig); lastErr != nil {
|
||||
return
|
||||
}
|
||||
@ -498,9 +490,8 @@ func buildGenericConfig(
|
||||
}
|
||||
versionedInformers = clientgoinformers.NewSharedInformerFactory(clientgoExternalClient, 10*time.Minute)
|
||||
|
||||
genericConfig.Authentication.Authenticator, genericConfig.OpenAPIConfig.SecurityDefinitions, err = BuildAuthenticator(s, genericConfig.EgressSelector, clientgoExternalClient, versionedInformers)
|
||||
if err != nil {
|
||||
lastErr = fmt.Errorf("invalid authentication config: %v", err)
|
||||
// Authentication.ApplyTo requires already applied OpenAPIConfig and EgressSelector if present
|
||||
if lastErr = s.Authentication.ApplyTo(&genericConfig.Authentication, genericConfig.SecureServing, genericConfig.EgressSelector, genericConfig.OpenAPIConfig, clientgoExternalClient, versionedInformers); lastErr != nil {
|
||||
return
|
||||
}
|
||||
|
||||
@ -559,35 +550,6 @@ func buildGenericConfig(
|
||||
return
|
||||
}
|
||||
|
||||
// BuildAuthenticator constructs the authenticator
|
||||
func BuildAuthenticator(s *options.ServerRunOptions, EgressSelector *egressselector.EgressSelector, extclient clientgoclientset.Interface, versionedInformer clientgoinformers.SharedInformerFactory) (authenticator.Request, *spec.SecurityDefinitions, error) {
|
||||
authenticatorConfig, err := s.Authentication.ToAuthenticationConfig()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if s.Authentication.ServiceAccounts.Lookup || utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) {
|
||||
authenticatorConfig.ServiceAccountTokenGetter = serviceaccountcontroller.NewGetterFromClient(
|
||||
extclient,
|
||||
versionedInformer.Core().V1().Secrets().Lister(),
|
||||
versionedInformer.Core().V1().ServiceAccounts().Lister(),
|
||||
versionedInformer.Core().V1().Pods().Lister(),
|
||||
)
|
||||
}
|
||||
authenticatorConfig.BootstrapTokenAuthenticator = bootstrap.NewTokenAuthenticator(
|
||||
versionedInformer.Core().V1().Secrets().Lister().Secrets(v1.NamespaceSystem),
|
||||
)
|
||||
|
||||
if EgressSelector != nil {
|
||||
egressDialer, err := EgressSelector.Lookup(egressselector.Master.AsNetworkContext())
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
authenticatorConfig.CustomDial = egressDialer
|
||||
}
|
||||
|
||||
return authenticatorConfig.New()
|
||||
}
|
||||
|
||||
// BuildAuthorizer constructs the authorizer
|
||||
func BuildAuthorizer(s *options.ServerRunOptions, EgressSelector *egressselector.EgressSelector, versionedInformers clientgoinformers.SharedInformerFactory) (authorizer.Authorizer, authorizer.RuleResolver, error) {
|
||||
authorizationConfig := s.Authorization.ToAuthorizationConfig(versionedInformers)
|
||||
|
||||
@ -36,7 +36,8 @@ import (
|
||||
)
|
||||
|
||||
// BuildAuth creates an authenticator, an authorizer, and a matching authorizer attributes getter compatible with the kubelet's needs
|
||||
func BuildAuth(nodeName types.NodeName, client clientset.Interface, config kubeletconfig.KubeletConfiguration) (server.AuthInterface, error) {
|
||||
// It returns AuthInterface, a run method to start internal controllers (like cert reloading) and error.
|
||||
func BuildAuth(nodeName types.NodeName, client clientset.Interface, config kubeletconfig.KubeletConfiguration) (server.AuthInterface, func(<-chan struct{}), error) {
|
||||
// Get clients, if provided
|
||||
var (
|
||||
tokenClient authenticationclient.TokenReviewInterface
|
||||
@ -47,47 +48,55 @@ func BuildAuth(nodeName types.NodeName, client clientset.Interface, config kubel
|
||||
sarClient = client.AuthorizationV1().SubjectAccessReviews()
|
||||
}
|
||||
|
||||
authenticator, err := BuildAuthn(tokenClient, config.Authentication)
|
||||
authenticator, runAuthenticatorCAReload, err := BuildAuthn(tokenClient, config.Authentication)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
attributes := server.NewNodeAuthorizerAttributesGetter(nodeName)
|
||||
|
||||
authorizer, err := BuildAuthz(sarClient, config.Authorization)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return server.NewKubeletAuth(authenticator, attributes, authorizer), nil
|
||||
return server.NewKubeletAuth(authenticator, attributes, authorizer), runAuthenticatorCAReload, nil
|
||||
}
|
||||
|
||||
// BuildAuthn creates an authenticator compatible with the kubelet's needs
|
||||
func BuildAuthn(client authenticationclient.TokenReviewInterface, authn kubeletconfig.KubeletAuthentication) (authenticator.Request, error) {
|
||||
var clientCertificateCAContentProvider authenticatorfactory.CAContentProvider
|
||||
func BuildAuthn(client authenticationclient.TokenReviewInterface, authn kubeletconfig.KubeletAuthentication) (authenticator.Request, func(<-chan struct{}), error) {
|
||||
var dynamicCAContentFromFile *dynamiccertificates.DynamicFileCAContent
|
||||
var err error
|
||||
if len(authn.X509.ClientCAFile) > 0 {
|
||||
clientCertificateCAContentProvider, err = dynamiccertificates.NewDynamicCAContentFromFile("client-ca-bundle", authn.X509.ClientCAFile)
|
||||
dynamicCAContentFromFile, err = dynamiccertificates.NewDynamicCAContentFromFile("client-ca-bundle", authn.X509.ClientCAFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
authenticatorConfig := authenticatorfactory.DelegatingAuthenticatorConfig{
|
||||
Anonymous: authn.Anonymous.Enabled,
|
||||
CacheTTL: authn.Webhook.CacheTTL.Duration,
|
||||
ClientCertificateCAContentProvider: clientCertificateCAContentProvider,
|
||||
ClientCertificateCAContentProvider: dynamicCAContentFromFile,
|
||||
}
|
||||
|
||||
if authn.Webhook.Enabled {
|
||||
if client == nil {
|
||||
return nil, errors.New("no client provided, cannot use webhook authentication")
|
||||
return nil, nil, errors.New("no client provided, cannot use webhook authentication")
|
||||
}
|
||||
authenticatorConfig.TokenAccessReviewClient = client
|
||||
}
|
||||
|
||||
authenticator, _, err := authenticatorConfig.New()
|
||||
return authenticator, err
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return authenticator, func(stopCh <-chan struct{}) {
|
||||
if dynamicCAContentFromFile != nil {
|
||||
go dynamicCAContentFromFile.Run(1, stopCh)
|
||||
}
|
||||
}, err
|
||||
}
|
||||
|
||||
// BuildAuthz creates an authorizer compatible with the kubelet's needs
|
||||
|
||||
@ -599,11 +599,12 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, featureGate f
|
||||
}
|
||||
|
||||
if kubeDeps.Auth == nil {
|
||||
auth, err := BuildAuth(nodeName, kubeDeps.KubeClient, s.KubeletConfiguration)
|
||||
auth, runAuthenticatorCAReload, err := BuildAuth(nodeName, kubeDeps.KubeClient, s.KubeletConfiguration)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
kubeDeps.Auth = auth
|
||||
runAuthenticatorCAReload(stopCh)
|
||||
}
|
||||
|
||||
var cgroupRoots []string
|
||||
|
||||
@ -14,6 +14,7 @@ go_library(
|
||||
importpath = "k8s.io/kubernetes/pkg/kubeapiserver/options",
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//pkg/controller/serviceaccount:go_default_library",
|
||||
"//pkg/features:go_default_library",
|
||||
"//pkg/kubeapiserver/authenticator:go_default_library",
|
||||
"//pkg/kubeapiserver/authorizer:go_default_library",
|
||||
@ -50,6 +51,8 @@ go_library(
|
||||
"//plugin/pkg/admission/storage/persistentvolume/resize:go_default_library",
|
||||
"//plugin/pkg/admission/storage/storageclass/setdefault:go_default_library",
|
||||
"//plugin/pkg/admission/storage/storageobjectinuseprotection:go_default_library",
|
||||
"//plugin/pkg/auth/authenticator/token/bootstrap:go_default_library",
|
||||
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||
"//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
|
||||
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
|
||||
"//staging/src/k8s.io/apiserver/pkg/admission:go_default_library",
|
||||
@ -58,14 +61,17 @@ go_library(
|
||||
"//staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/validating:go_default_library",
|
||||
"//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library",
|
||||
"//staging/src/k8s.io/apiserver/pkg/server:go_default_library",
|
||||
"//staging/src/k8s.io/apiserver/pkg/server/egressselector:go_default_library",
|
||||
"//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library",
|
||||
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
|
||||
"//staging/src/k8s.io/client-go/informers:go_default_library",
|
||||
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
|
||||
"//staging/src/k8s.io/client-go/rest:go_default_library",
|
||||
"//staging/src/k8s.io/component-base/cli/flag:go_default_library",
|
||||
"//staging/src/k8s.io/component-base/featuregate:go_default_library",
|
||||
"//vendor/github.com/spf13/pflag:go_default_library",
|
||||
"//vendor/k8s.io/klog:go_default_library",
|
||||
"//vendor/k8s.io/kube-openapi/pkg/common:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@ -24,17 +24,24 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/spf13/pflag"
|
||||
"k8s.io/klog"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
"k8s.io/apiserver/pkg/authentication/authenticator"
|
||||
genericapiserver "k8s.io/apiserver/pkg/server"
|
||||
"k8s.io/apiserver/pkg/server/egressselector"
|
||||
genericoptions "k8s.io/apiserver/pkg/server/options"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
"k8s.io/client-go/informers"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
cliflag "k8s.io/component-base/cli/flag"
|
||||
"k8s.io/klog"
|
||||
openapicommon "k8s.io/kube-openapi/pkg/common"
|
||||
serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount"
|
||||
"k8s.io/kubernetes/pkg/features"
|
||||
kubeauthenticator "k8s.io/kubernetes/pkg/kubeapiserver/authenticator"
|
||||
authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
|
||||
"k8s.io/kubernetes/plugin/pkg/auth/authenticator/token/bootstrap"
|
||||
)
|
||||
|
||||
type BuiltInAuthenticationOptions struct {
|
||||
@ -406,35 +413,60 @@ func (s *BuiltInAuthenticationOptions) ToAuthenticationConfig() (kubeauthenticat
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
func (o *BuiltInAuthenticationOptions) ApplyTo(c *genericapiserver.Config) error {
|
||||
// ApplyTo requires already applied OpenAPIConfig and EgressSelector if present.
|
||||
func (o *BuiltInAuthenticationOptions) ApplyTo(authInfo *genericapiserver.AuthenticationInfo, secureServing *genericapiserver.SecureServingInfo, egressSelector *egressselector.EgressSelector, openAPIConfig *openapicommon.Config, extclient kubernetes.Interface, versionedInformer informers.SharedInformerFactory) error {
|
||||
if o == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if o.ClientCert != nil {
|
||||
clientCertificateCAContentProvider, err := o.ClientCert.GetClientCAContentProvider()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to load client CA file: %v", err)
|
||||
}
|
||||
if err = c.Authentication.ApplyClientCert(clientCertificateCAContentProvider, c.SecureServing); err != nil {
|
||||
if openAPIConfig == nil {
|
||||
return errors.New("uninitialized OpenAPIConfig")
|
||||
}
|
||||
|
||||
authenticatorConfig, err := o.ToAuthenticationConfig()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if authenticatorConfig.ClientCAContentProvider != nil {
|
||||
if err = authInfo.ApplyClientCert(authenticatorConfig.ClientCAContentProvider, secureServing); err != nil {
|
||||
return fmt.Errorf("unable to load client CA file: %v", err)
|
||||
}
|
||||
}
|
||||
if o.RequestHeader != nil {
|
||||
requestHeaderConfig, err := o.RequestHeader.ToAuthenticationRequestHeaderConfig()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to create request header authentication config: %v", err)
|
||||
}
|
||||
if requestHeaderConfig != nil {
|
||||
if err = c.Authentication.ApplyClientCert(requestHeaderConfig.CAContentProvider, c.SecureServing); err != nil {
|
||||
return fmt.Errorf("unable to load client CA file: %v", err)
|
||||
}
|
||||
if authenticatorConfig.RequestHeaderConfig != nil && authenticatorConfig.RequestHeaderConfig.CAContentProvider != nil {
|
||||
if err = authInfo.ApplyClientCert(authenticatorConfig.RequestHeaderConfig.CAContentProvider, secureServing); err != nil {
|
||||
return fmt.Errorf("unable to load client CA file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
c.Authentication.APIAudiences = o.APIAudiences
|
||||
authInfo.APIAudiences = o.APIAudiences
|
||||
if o.ServiceAccounts != nil && o.ServiceAccounts.Issuer != "" && len(o.APIAudiences) == 0 {
|
||||
c.Authentication.APIAudiences = authenticator.Audiences{o.ServiceAccounts.Issuer}
|
||||
authInfo.APIAudiences = authenticator.Audiences{o.ServiceAccounts.Issuer}
|
||||
}
|
||||
|
||||
if o.ServiceAccounts.Lookup || utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) {
|
||||
authenticatorConfig.ServiceAccountTokenGetter = serviceaccountcontroller.NewGetterFromClient(
|
||||
extclient,
|
||||
versionedInformer.Core().V1().Secrets().Lister(),
|
||||
versionedInformer.Core().V1().ServiceAccounts().Lister(),
|
||||
versionedInformer.Core().V1().Pods().Lister(),
|
||||
)
|
||||
}
|
||||
authenticatorConfig.BootstrapTokenAuthenticator = bootstrap.NewTokenAuthenticator(
|
||||
versionedInformer.Core().V1().Secrets().Lister().Secrets(metav1.NamespaceSystem),
|
||||
)
|
||||
|
||||
if egressSelector != nil {
|
||||
egressDialer, err := egressSelector.Lookup(egressselector.Master.AsNetworkContext())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
authenticatorConfig.CustomDial = egressDialer
|
||||
}
|
||||
|
||||
authInfo.Authenticator, openAPIConfig.SecurityDefinitions, err = authenticatorConfig.New()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@ -9,11 +9,14 @@ go_test(
|
||||
tags = ["integration"],
|
||||
deps = [
|
||||
"//cmd/kube-apiserver/app/options:go_default_library",
|
||||
"//staging/src/k8s.io/api/authorization/v1:go_default_library",
|
||||
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
|
||||
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
|
||||
"//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:go_default_library",
|
||||
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
|
||||
"//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library",
|
||||
"//staging/src/k8s.io/client-go/rest:go_default_library",
|
||||
"//staging/src/k8s.io/client-go/util/cert:go_default_library",
|
||||
"//staging/src/k8s.io/component-base/cli/flag:go_default_library",
|
||||
"//test/integration/framework:go_default_library",
|
||||
|
||||
@ -19,63 +19,150 @@ package podlogs
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math/big"
|
||||
"path"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
authorizationv1 "k8s.io/api/authorization/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
"k8s.io/apiserver/pkg/server/dynamiccertificates"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/kubernetes/scheme"
|
||||
"k8s.io/client-go/rest"
|
||||
"k8s.io/client-go/util/cert"
|
||||
"k8s.io/component-base/cli/flag"
|
||||
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
|
||||
"k8s.io/kubernetes/test/integration/framework"
|
||||
)
|
||||
|
||||
type caWithClient struct {
|
||||
CACert []byte
|
||||
ClientCert []byte
|
||||
ClientKey []byte
|
||||
}
|
||||
|
||||
func newTestCAWithClient(caSubject pkix.Name, caSerial *big.Int, clientSubject pkix.Name, subjectSerial *big.Int) (*caWithClient, error) {
|
||||
ca := &x509.Certificate{
|
||||
SerialNumber: caSerial,
|
||||
Subject: caSubject,
|
||||
NotBefore: time.Now(),
|
||||
NotAfter: time.Now().Add(24 * time.Hour),
|
||||
IsCA: true,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
|
||||
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
|
||||
BasicConstraintsValid: true,
|
||||
}
|
||||
|
||||
caPrivateKey, err := rsa.GenerateKey(rand.Reader, 4096)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
caBytes, err := x509.CreateCertificate(rand.Reader, ca, ca, &caPrivateKey.PublicKey, caPrivateKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
caPEM := new(bytes.Buffer)
|
||||
err = pem.Encode(caPEM, &pem.Block{
|
||||
Type: "CERTIFICATE",
|
||||
Bytes: caBytes,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
clientCert := &x509.Certificate{
|
||||
SerialNumber: subjectSerial,
|
||||
Subject: clientSubject,
|
||||
NotBefore: time.Now(),
|
||||
NotAfter: time.Now().Add(24 * time.Hour),
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
|
||||
KeyUsage: x509.KeyUsageDigitalSignature,
|
||||
}
|
||||
|
||||
clientCertPrivateKey, err := rsa.GenerateKey(rand.Reader, 4096)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
clientCertPrivateKeyPEM := new(bytes.Buffer)
|
||||
err = pem.Encode(clientCertPrivateKeyPEM, &pem.Block{
|
||||
Type: "RSA PRIVATE KEY",
|
||||
Bytes: x509.MarshalPKCS1PrivateKey(clientCertPrivateKey),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
clientCertBytes, err := x509.CreateCertificate(rand.Reader, clientCert, ca, &clientCertPrivateKey.PublicKey, caPrivateKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
clientCertPEM := new(bytes.Buffer)
|
||||
err = pem.Encode(clientCertPEM, &pem.Block{
|
||||
Type: "CERTIFICATE",
|
||||
Bytes: clientCertBytes,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &caWithClient{
|
||||
CACert: caPEM.Bytes(),
|
||||
ClientCert: clientCertPEM.Bytes(),
|
||||
ClientKey: clientCertPrivateKeyPEM.Bytes(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func TestClientCA(t *testing.T) {
|
||||
stopCh := make(chan struct{})
|
||||
defer close(stopCh)
|
||||
|
||||
// I have no idea what this cert is, but it doesn't matter, we just want something that always fails validation
|
||||
differentClientCA := []byte(`-----BEGIN CERTIFICATE-----
|
||||
MIIDQDCCAiigAwIBAgIJANWw74P5KJk2MA0GCSqGSIb3DQEBCwUAMDQxMjAwBgNV
|
||||
BAMMKWdlbmVyaWNfd2ViaG9va19hZG1pc3Npb25fcGx1Z2luX3Rlc3RzX2NhMCAX
|
||||
DTE3MTExNjAwMDUzOVoYDzIyOTEwOTAxMDAwNTM5WjAjMSEwHwYDVQQDExh3ZWJo
|
||||
b29rLXRlc3QuZGVmYXVsdC5zdmMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
|
||||
AoIBAQDXd/nQ89a5H8ifEsigmMd01Ib6NVR3bkJjtkvYnTbdfYEBj7UzqOQtHoLa
|
||||
dIVmefny5uIHvj93WD8WDVPB3jX2JHrXkDTXd/6o6jIXHcsUfFTVLp6/bZ+Anqe0
|
||||
r/7hAPkzA2A7APyTWM3ZbEeo1afXogXhOJ1u/wz0DflgcB21gNho4kKTONXO3NHD
|
||||
XLpspFqSkxfEfKVDJaYAoMnYZJtFNsa2OvsmLnhYF8bjeT3i07lfwrhUZvP+7Gsp
|
||||
7UgUwc06WuNHjfx1s5e6ySzH0QioMD1rjYneqOvk0pKrMIhuAEWXqq7jlXcDtx1E
|
||||
j+wnYbVqqVYheHZ8BCJoVAAQGs9/AgMBAAGjZDBiMAkGA1UdEwQCMAAwCwYDVR0P
|
||||
BAQDAgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATApBgNVHREEIjAg
|
||||
hwR/AAABghh3ZWJob29rLXRlc3QuZGVmYXVsdC5zdmMwDQYJKoZIhvcNAQELBQAD
|
||||
ggEBAD/GKSPNyQuAOw/jsYZesb+RMedbkzs18sSwlxAJQMUrrXwlVdHrA8q5WhE6
|
||||
ABLqU1b8lQ8AWun07R8k5tqTmNvCARrAPRUqls/ryER+3Y9YEcxEaTc3jKNZFLbc
|
||||
T6YtcnkdhxsiO136wtiuatpYL91RgCmuSpR8+7jEHhuFU01iaASu7ypFrUzrKHTF
|
||||
bKwiLRQi1cMzVcLErq5CDEKiKhUkoDucyARFszrGt9vNIl/YCcBOkcNvM3c05Hn3
|
||||
M++C29JwS3Hwbubg6WO3wjFjoEhpCwU6qRYUz3MRp4tHO4kxKXx+oQnUiFnR7vW0
|
||||
YkNtGc1RUDHwecCTFpJtPb7Yu/E=
|
||||
-----END CERTIFICATE-----
|
||||
`)
|
||||
differentFrontProxyCA := []byte(`-----BEGIN CERTIFICATE-----
|
||||
MIIBqDCCAU2gAwIBAgIUfbqeieihh/oERbfvRm38XvS/xHAwCgYIKoZIzj0EAwIw
|
||||
GjEYMBYGA1UEAxMPSW50ZXJtZWRpYXRlLUNBMCAXDTE2MTAxMTA1MDYwMFoYDzIx
|
||||
MTYwOTE3MDUwNjAwWjAUMRIwEAYDVQQDEwlNeSBDbGllbnQwWTATBgcqhkjOPQIB
|
||||
BggqhkjOPQMBBwNCAARv6N4R/sjMR65iMFGNLN1GC/vd7WhDW6J4X/iAjkRLLnNb
|
||||
KbRG/AtOUZ+7upJ3BWIRKYbOabbQGQe2BbKFiap4o3UwczAOBgNVHQ8BAf8EBAMC
|
||||
BaAwEwYDVR0lBAwwCgYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU
|
||||
K/pZOWpNcYai6eHFpmJEeFpeQlEwHwYDVR0jBBgwFoAUX6nQlxjfWnP6aM1meO/Q
|
||||
a6b3a9kwCgYIKoZIzj0EAwIDSQAwRgIhAIWTKw/sjJITqeuNzJDAKU4xo1zL+xJ5
|
||||
MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps=
|
||||
-----END CERTIFICATE-----
|
||||
frontProxyCA, err := newTestCAWithClient(
|
||||
pkix.Name{
|
||||
CommonName: "test-front-proxy-ca",
|
||||
},
|
||||
big.NewInt(43),
|
||||
pkix.Name{
|
||||
CommonName: "test-aggregated-apiserver",
|
||||
Organization: []string{"system:masters"},
|
||||
},
|
||||
big.NewInt(86),
|
||||
)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
clientCA, err := newTestCAWithClient(
|
||||
pkix.Name{
|
||||
CommonName: "test-client-ca",
|
||||
},
|
||||
big.NewInt(42),
|
||||
pkix.Name{
|
||||
CommonName: "system:admin",
|
||||
Organization: []string{"system:masters"},
|
||||
},
|
||||
big.NewInt(84),
|
||||
)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
`)
|
||||
clientCAFilename := ""
|
||||
frontProxyCAFilename := ""
|
||||
|
||||
@ -84,12 +171,13 @@ MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps=
|
||||
opts.GenericServerRunOptions.MaxRequestBodyBytes = 1024 * 1024
|
||||
clientCAFilename = opts.Authentication.ClientCert.ClientCA
|
||||
frontProxyCAFilename = opts.Authentication.RequestHeader.ClientCAFile
|
||||
opts.Authentication.RequestHeader.AllowedNames = append(opts.Authentication.RequestHeader.AllowedNames, "test-aggregated-apiserver")
|
||||
dynamiccertificates.FileRefreshDuration = 1 * time.Second
|
||||
},
|
||||
})
|
||||
|
||||
// wait for request header info
|
||||
err := wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "requestheader-client-ca-file", "-----BEGIN CERTIFICATE-----", 1))
|
||||
err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "requestheader-client-ca-file", "-----BEGIN CERTIFICATE-----", 1))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -100,10 +188,10 @@ MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps=
|
||||
}
|
||||
|
||||
// when we run this the second time, we know which one we are expecting
|
||||
if err := ioutil.WriteFile(clientCAFilename, differentClientCA, 0644); err != nil {
|
||||
if err := ioutil.WriteFile(clientCAFilename, clientCA.CACert, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ioutil.WriteFile(frontProxyCAFilename, differentFrontProxyCA, 0644); err != nil {
|
||||
if err := ioutil.WriteFile(frontProxyCAFilename, frontProxyCA.CACert, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@ -114,7 +202,7 @@ MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps=
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
expectedCAs := []string{"webhook-test.default.svc", "My Client"}
|
||||
expectedCAs := []string{"test-client-ca", "test-front-proxy-ca"}
|
||||
if len(expectedCAs) != len(acceptableCAs) {
|
||||
t.Fatal(strings.Join(acceptableCAs, ":"))
|
||||
}
|
||||
@ -129,7 +217,7 @@ MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps=
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "requestheader-client-ca-file", "MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps=", 1))
|
||||
err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "requestheader-client-ca-file", string(frontProxyCA.CACert), 1))
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
@ -138,7 +226,68 @@ MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps=
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "client-ca-file", "M++C29JwS3Hwbubg6WO3wjFjoEhpCwU6qRYUz3MRp4tHO4kxKXx+oQnUiFnR7vW0", 1))
|
||||
err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "client-ca-file", string(clientCA.CACert), 1))
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
// Test an aggregated apiserver client (signed by the new front proxy CA) is authorized
|
||||
extensionApiserverClient, err := kubernetes.NewForConfig(&rest.Config{
|
||||
Host: kubeconfig.Host,
|
||||
TLSClientConfig: rest.TLSClientConfig{
|
||||
CAData: kubeconfig.TLSClientConfig.CAData,
|
||||
CAFile: kubeconfig.TLSClientConfig.CAFile,
|
||||
ServerName: kubeconfig.TLSClientConfig.ServerName,
|
||||
KeyData: frontProxyCA.ClientKey,
|
||||
CertData: frontProxyCA.ClientCert,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
// Call an endpoint to make sure we are authenticated
|
||||
err = extensionApiserverClient.AuthorizationV1().RESTClient().
|
||||
Post().
|
||||
Resource("subjectaccessreviews").
|
||||
VersionedParams(&metav1.CreateOptions{}, scheme.ParameterCodec).
|
||||
Body(&authorizationv1.SubjectAccessReview{
|
||||
Spec: authorizationv1.SubjectAccessReviewSpec{
|
||||
ResourceAttributes: &authorizationv1.ResourceAttributes{
|
||||
Verb: "create",
|
||||
Resource: "pods",
|
||||
Namespace: "default",
|
||||
},
|
||||
User: "deads2k",
|
||||
},
|
||||
}).
|
||||
SetHeader("X-Remote-User", "test-aggregated-apiserver").
|
||||
SetHeader("X-Remote-Group", "system:masters").
|
||||
Do(context.Background()).
|
||||
Into(&authorizationv1.SubjectAccessReview{})
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
// Test a client signed by the new ClientCA is authorized
|
||||
testClient, err := kubernetes.NewForConfig(&rest.Config{
|
||||
Host: kubeconfig.Host,
|
||||
TLSClientConfig: rest.TLSClientConfig{
|
||||
CAData: kubeconfig.TLSClientConfig.CAData,
|
||||
CAFile: kubeconfig.TLSClientConfig.CAFile,
|
||||
ServerName: kubeconfig.TLSClientConfig.ServerName,
|
||||
KeyData: clientCA.ClientKey,
|
||||
CertData: clientCA.ClientCert,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
// Call an endpoint to make sure we are authenticated
|
||||
_, err = testClient.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{})
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user