add ability to authenticators for dynamic update of certs

This commit is contained in:
David Eads 2019-09-05 09:59:59 -04:00
parent c2c821534b
commit 51195dd860
40 changed files with 505 additions and 84 deletions

View File

@ -525,7 +525,10 @@ func buildGenericConfig(
// BuildAuthenticator constructs the authenticator
func BuildAuthenticator(s *options.ServerRunOptions, extclient clientgoclientset.Interface, versionedInformer clientgoinformers.SharedInformerFactory) (authenticator.Request, *spec.SecurityDefinitions, error) {
authenticatorConfig := s.Authentication.ToAuthenticationConfig()
authenticatorConfig, err := s.Authentication.ToAuthenticationConfig()
if err != nil {
return nil, nil, err
}
if s.Authentication.ServiceAccounts.Lookup || utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) {
authenticatorConfig.ServiceAccountTokenGetter = serviceaccountcontroller.NewGetterFromClient(
extclient,

View File

@ -118,6 +118,7 @@ go_library(
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authentication/request/x509:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/server:go_default_library",

View File

@ -24,6 +24,7 @@ import (
"k8s.io/apimachinery/pkg/types"
"k8s.io/apiserver/pkg/authentication/authenticator"
"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
"k8s.io/apiserver/pkg/authentication/request/x509"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/authorization/authorizerfactory"
clientset "k8s.io/client-go/kubernetes"
@ -63,10 +64,15 @@ func BuildAuth(nodeName types.NodeName, client clientset.Interface, config kubel
// BuildAuthn creates an authenticator compatible with the kubelet's needs
func BuildAuthn(client authenticationclient.TokenReviewInterface, authn kubeletconfig.KubeletAuthentication) (authenticator.Request, error) {
clientCertVerifier, err := x509.NewStaticVerifierFromFile(authn.X509.ClientCAFile)
if err != nil {
return nil, err
}
authenticatorConfig := authenticatorfactory.DelegatingAuthenticatorConfig{
Anonymous: authn.Anonymous.Enabled,
CacheTTL: authn.Webhook.CacheTTL.Duration,
ClientCAFile: authn.X509.ClientCAFile,
Anonymous: authn.Anonymous.Enabled,
CacheTTL: authn.Webhook.CacheTTL.Duration,
ClientVerifyOptionFn: clientCertVerifier,
}
if authn.Webhook.Enabled {

View File

@ -30,7 +30,6 @@ go_library(
"//staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/oidc:go_default_library",
"//staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook:go_default_library",
"//staging/src/k8s.io/client-go/plugin/pkg/client/auth:go_default_library",
"//staging/src/k8s.io/client-go/util/cert:go_default_library",
"//staging/src/k8s.io/client-go/util/keyutil:go_default_library",
"//vendor/github.com/go-openapi/spec:go_default_library",
],

View File

@ -41,7 +41,6 @@ import (
// Initialize all known client auth plugins.
_ "k8s.io/client-go/plugin/pkg/client/auth"
certutil "k8s.io/client-go/util/cert"
"k8s.io/client-go/util/keyutil"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/serviceaccount"
@ -49,10 +48,10 @@ import (
// Config contains the data on how to authenticate a request to the Kube API Server
type Config struct {
Anonymous bool
BasicAuthFile string
BootstrapToken bool
ClientCAFile string
Anonymous bool
BasicAuthFile string
BootstrapToken bool
TokenAuthFile string
OIDCIssuerURL string
OIDCClientID string
@ -78,6 +77,10 @@ type Config struct {
// TODO, this is the only non-serializable part of the entire config. Factor it out into a clientconfig
ServiceAccountTokenGetter serviceaccount.ServiceAccountTokenGetter
BootstrapTokenAuthenticator authenticator.Token
// ClientVerifyOptionFn are the options for verifying incoming connections using mTLS and directly assigning to users.
// Generally this is the CA bundle file used to authenticate client certificates
// If this value is nil, then mutual TLS is disabled.
ClientVerifyOptionFn x509.VerifyOptionFunc
}
// New returns an authenticator.Request or an error that supports the standard
@ -90,8 +93,8 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, er
// front-proxy, BasicAuth methods, local first, then remote
// Add the front proxy authenticator if requested
if config.RequestHeaderConfig != nil {
requestHeaderAuthenticator, err := headerrequest.NewSecure(
config.RequestHeaderConfig.ClientCA,
requestHeaderAuthenticator, err := headerrequest.NewDynamicVerifyOptionsSecure(
config.RequestHeaderConfig.VerifyOptionFn,
config.RequestHeaderConfig.AllowedClientNames,
config.RequestHeaderConfig.UsernameHeaders,
config.RequestHeaderConfig.GroupHeaders,
@ -120,11 +123,8 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, er
}
// X509 methods
if len(config.ClientCAFile) > 0 {
certAuth, err := newAuthenticatorFromClientCAFile(config.ClientCAFile)
if err != nil {
return nil, nil, err
}
if config.ClientVerifyOptionFn != nil {
certAuth := x509.NewDynamic(config.ClientVerifyOptionFn, x509.CommonNameUserConversion)
authenticators = append(authenticators, certAuth)
}
@ -307,19 +307,6 @@ func newServiceAccountAuthenticator(iss string, keyfiles []string, apiAudiences
return tokenAuthenticator, nil
}
// newAuthenticatorFromClientCAFile returns an authenticator.Request or an error
func newAuthenticatorFromClientCAFile(clientCAFile string) (authenticator.Request, error) {
roots, err := certutil.NewPool(clientCAFile)
if err != nil {
return nil, err
}
opts := x509.DefaultVerifyOptions()
opts.Roots = roots
return x509.New(opts, x509.CommonNameUserConversion), nil
}
func newWebhookTokenAuthenticator(webhookConfigFile string, ttl time.Duration, implicitAuds authenticator.Audiences) (authenticator.Token, error) {
webhookTokenAuthenticator, err := webhook.New(webhookConfigFile, implicitAuds)
if err != nil {

View File

@ -89,6 +89,12 @@ go_test(
"authentication_test.go",
"authorization_test.go",
],
data = [
"testdata/client-expired.pem",
"testdata/client-valid.pem",
"testdata/intermediate.pem",
"testdata/root.pem",
],
embed = [":go_default_library"],
deps = [
"//pkg/kubeapiserver/authenticator:go_default_library",
@ -97,5 +103,6 @@ go_test(
"//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library",
"//vendor/github.com/google/go-cmp/cmp:go_default_library",
],
)

View File

@ -308,7 +308,7 @@ func (s *BuiltInAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
}
}
func (s *BuiltInAuthenticationOptions) ToAuthenticationConfig() kubeauthenticator.Config {
func (s *BuiltInAuthenticationOptions) ToAuthenticationConfig() (kubeauthenticator.Config, error) {
ret := kubeauthenticator.Config{
TokenSuccessCacheTTL: s.TokenSuccessCacheTTL,
TokenFailureCacheTTL: s.TokenFailureCacheTTL,
@ -323,7 +323,11 @@ func (s *BuiltInAuthenticationOptions) ToAuthenticationConfig() kubeauthenticato
}
if s.ClientCert != nil {
ret.ClientCAFile = s.ClientCert.ClientCA
var err error
ret.ClientVerifyOptionFn, err = s.ClientCert.GetClientVerifyOptionFn()
if err != nil {
return kubeauthenticator.Config{}, err
}
}
if s.OIDC != nil {
@ -343,7 +347,11 @@ func (s *BuiltInAuthenticationOptions) ToAuthenticationConfig() kubeauthenticato
}
if s.RequestHeader != nil {
ret.RequestHeaderConfig = s.RequestHeader.ToAuthenticationRequestHeaderConfig()
var err error
ret.RequestHeaderConfig, err = s.RequestHeader.ToAuthenticationRequestHeaderConfig()
if err != nil {
return kubeauthenticator.Config{}, err
}
}
ret.APIAudiences = s.APIAudiences
@ -374,7 +382,7 @@ func (s *BuiltInAuthenticationOptions) ToAuthenticationConfig() kubeauthenticato
}
}
return ret
return ret, nil
}
func (o *BuiltInAuthenticationOptions) ApplyTo(c *genericapiserver.Config) error {

View File

@ -22,6 +22,8 @@ import (
"testing"
"time"
"github.com/google/go-cmp/cmp"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apiserver/pkg/authentication/authenticator"
"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
@ -101,7 +103,7 @@ func TestToAuthenticationConfig(t *testing.T) {
Allow: false,
},
ClientCert: &apiserveroptions.ClientCertAuthenticationOptions{
ClientCA: "/client-ca",
ClientCA: "testdata/root.pem",
},
WebHook: &WebHookAuthenticationOptions{
CacheTTL: 180000000000,
@ -124,7 +126,7 @@ func TestToAuthenticationConfig(t *testing.T) {
UsernameHeaders: []string{"x-remote-user"},
GroupHeaders: []string{"x-remote-group"},
ExtraHeaderPrefixes: []string{"x-remote-extra-"},
ClientCAFile: "/testClientCAFile",
ClientCAFile: "testdata/root.pem",
AllowedNames: []string{"kube-aggregator"},
},
ServiceAccounts: &ServiceAccountAuthenticationOptions{
@ -143,7 +145,7 @@ func TestToAuthenticationConfig(t *testing.T) {
Anonymous: false,
BasicAuthFile: "/testBasicAuthFile",
BootstrapToken: false,
ClientCAFile: "/client-ca",
ClientVerifyOptionFn: nil, // this is nil because you can't compare functions
TokenAuthFile: "/testTokenFile",
OIDCIssuerURL: "testIssuerURL",
OIDCClientID: "testClientID",
@ -162,13 +164,27 @@ func TestToAuthenticationConfig(t *testing.T) {
UsernameHeaders: []string{"x-remote-user"},
GroupHeaders: []string{"x-remote-group"},
ExtraHeaderPrefixes: []string{"x-remote-extra-"},
ClientCA: "/testClientCAFile",
VerifyOptionFn: nil, // this is nil because you can't compare functions
AllowedClientNames: []string{"kube-aggregator"},
},
}
resultConfig := testOptions.ToAuthenticationConfig()
resultConfig, err := testOptions.ToAuthenticationConfig()
if err != nil {
t.Fatal(err)
}
// nil these out because you cannot compare pointers. Ensure they are non-nil first
if resultConfig.ClientVerifyOptionFn == nil {
t.Error("missing client verify")
}
if resultConfig.RequestHeaderConfig.VerifyOptionFn == nil {
t.Error("missing requestheader verify")
}
resultConfig.ClientVerifyOptionFn = nil
resultConfig.RequestHeaderConfig.VerifyOptionFn = nil
if !reflect.DeepEqual(resultConfig, expectConfig) {
t.Errorf("Got AuthenticationConfig:\n\t%v\nExpected AuthenticationConfig:\n\t%v", resultConfig, expectConfig)
t.Error(cmp.Diff(resultConfig, expectConfig))
}
}

View File

@ -0,0 +1,11 @@
-----BEGIN CERTIFICATE-----
MIIBpTCCAUugAwIBAgIUPV4LAC5KK8YWY1FegyTuhkGUr3EwCgYIKoZIzj0EAwIw
GjEYMBYGA1UEAxMPSW50ZXJtZWRpYXRlLUNBMB4XDTkwMTIzMTIzNTkwMFoXDTkw
MTIzMTIzNTkwMFowFDESMBAGA1UEAxMJTXkgQ2xpZW50MFkwEwYHKoZIzj0CAQYI
KoZIzj0DAQcDQgAEyYUnseNUN87rfHgekrfZu5sj4wlt5LYr3JYZZkfSbsb+BW3/
RzX02ifjp+8w7mI4qUGg6y6J7oXHGFT3uj9kj6N1MHMwDgYDVR0PAQH/BAQDAgWg
MBMGA1UdJQQMMAoGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFKsX
EnXwDg8j2LIEM1QzmFrE6537MB8GA1UdIwQYMBaAFF+p0JcY31pz+mjNZnjv0Gum
92vZMAoGCCqGSM49BAMCA0gAMEUCIG4FBcb57oqOCoaFiJ+Yx6S0zkaash7bTv3V
CIy9JvFdAiEAy8bf2S9EkvZyURZ6ycgEMnekll57Ebze6rjlPx8+B1Y=
-----END CERTIFICATE-----

View File

@ -0,0 +1,11 @@
-----BEGIN CERTIFICATE-----
MIIBqDCCAU2gAwIBAgIUfbqeieihh/oERbfvRm38XvS/xHAwCgYIKoZIzj0EAwIw
GjEYMBYGA1UEAxMPSW50ZXJtZWRpYXRlLUNBMCAXDTE2MTAxMTA1MDYwMFoYDzIx
MTYwOTE3MDUwNjAwWjAUMRIwEAYDVQQDEwlNeSBDbGllbnQwWTATBgcqhkjOPQIB
BggqhkjOPQMBBwNCAARv6N4R/sjMR65iMFGNLN1GC/vd7WhDW6J4X/iAjkRLLnNb
KbRG/AtOUZ+7upJ3BWIRKYbOabbQGQe2BbKFiap4o3UwczAOBgNVHQ8BAf8EBAMC
BaAwEwYDVR0lBAwwCgYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU
K/pZOWpNcYai6eHFpmJEeFpeQlEwHwYDVR0jBBgwFoAUX6nQlxjfWnP6aM1meO/Q
a6b3a9kwCgYIKoZIzj0EAwIDSQAwRgIhAIWTKw/sjJITqeuNzJDAKU4xo1zL+xJ5
MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps=
-----END CERTIFICATE-----

View File

@ -0,0 +1,24 @@
{
"signing": {
"profiles": {
"valid": {
"expiry": "876000h",
"usages": [
"signing",
"key encipherment",
"client auth"
]
},
"expired": {
"expiry": "1h",
"not_before": "1990-12-31T23:59:00Z",
"not_after": "1990-12-31T23:59:00Z",
"usages": [
"signing",
"key encipherment",
"client auth"
]
}
}
}
}

View File

@ -0,0 +1,3 @@
{
"CN": "My Client"
}

View File

@ -0,0 +1,24 @@
#!/usr/bin/env bash
# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
cfssl gencert -initca root.csr.json | cfssljson -bare root
cfssl gencert -initca intermediate.csr.json | cfssljson -bare intermediate
cfssl sign -ca root.pem -ca-key root-key.pem -config intermediate.config.json intermediate.csr | cfssljson -bare intermediate
cfssl gencert -ca intermediate.pem -ca-key intermediate-key.pem -config client.config.json --profile=valid client.csr.json | cfssljson -bare client-valid
cfssl gencert -ca intermediate.pem -ca-key intermediate-key.pem -config client.config.json --profile=expired client.csr.json | cfssljson -bare client-expired

View File

@ -0,0 +1,18 @@
{
"signing": {
"default": {
"usages": [
"digital signature",
"cert sign",
"crl sign",
"signing",
"key encipherment",
"client auth"
],
"expiry": "876000h",
"ca_constraint": {
"is_ca": true
}
}
}
}

View File

@ -0,0 +1,6 @@
{
"CN": "Intermediate-CA",
"ca": {
"expiry": "876000h"
}
}

View File

@ -0,0 +1,11 @@
-----BEGIN CERTIFICATE-----
MIIBqDCCAU6gAwIBAgIUfqZtjoFgczZ+oQZbEC/BDSS2J6wwCgYIKoZIzj0EAwIw
EjEQMA4GA1UEAxMHUm9vdC1DQTAgFw0xNjEwMTEwNTA2MDBaGA8yMTE2MDkxNzA1
MDYwMFowGjEYMBYGA1UEAxMPSW50ZXJtZWRpYXRlLUNBMFkwEwYHKoZIzj0CAQYI
KoZIzj0DAQcDQgAEyWHEMMCctJg8Xa5YWLqaCPbk3MjB+uvXac42JM9pj4k9jedD
kpUJRkWIPzgJI8Zk/3cSzluUTixP6JBSDKtwwaN4MHYwDgYDVR0PAQH/BAQDAgGm
MBMGA1UdJQQMMAoGCCsGAQUFBwMCMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYE
FF+p0JcY31pz+mjNZnjv0Gum92vZMB8GA1UdIwQYMBaAFB7P6+i4/pfNjqZgJv/b
dgA7Fe4tMAoGCCqGSM49BAMCA0gAMEUCIQCTT1YWQZaAqfQ2oBxzOkJE2BqLFxhz
3smQlrZ5gCHddwIgcvT7puhYOzAgcvMn9+SZ1JOyZ7edODjshCVCRnuHK2c=
-----END CERTIFICATE-----

View File

@ -0,0 +1,6 @@
{
"CN": "Root-CA",
"ca": {
"expiry": "876000h"
}
}

View File

@ -0,0 +1,11 @@
-----BEGIN CERTIFICATE-----
MIIBizCCATGgAwIBAgIUH4plk9qwD61FVXgiOTngFU5FeSkwCgYIKoZIzj0EAwIw
EjEQMA4GA1UEAxMHUm9vdC1DQTAgFw0xNjEwMTEwNTA2MDBaGA8yMTE2MDkxNzA1
MDYwMFowEjEQMA4GA1UEAxMHUm9vdC1DQTBZMBMGByqGSM49AgEGCCqGSM49AwEH
A0IABI2CsrAnMGT8P2VGU2MLo5pv86Z74kcV9hgkLJUkSaeNyc1s89w7X5V2wvwu
iWEJRGm5RoZJausmyZLZEoKEVXejYzBhMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMB
Af8EBTADAQH/MB0GA1UdDgQWBBQez+vouP6XzY6mYCb/23YAOxXuLTAfBgNVHSME
GDAWgBQez+vouP6XzY6mYCb/23YAOxXuLTAKBggqhkjOPQQDAgNIADBFAiBGclts
vJRM+QMVoV/1L9b+hvhgLIp/OupUFsSOReefIwIhALY06hBklyh8eFwuBtyX2VcE
8xlVn4/5idUvc3Xv2h9s
-----END CERTIFICATE-----

View File

@ -28,7 +28,6 @@ go_library(
"//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library",
"//staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/typed/authentication/v1beta1:go_default_library",
"//staging/src/k8s.io/client-go/util/cert:go_default_library",
"//vendor/github.com/go-openapi/spec:go_default_library",
],
)

View File

@ -18,7 +18,6 @@ package authenticatorfactory
import (
"errors"
"fmt"
"time"
"github.com/go-openapi/spec"
@ -31,10 +30,10 @@ import (
unionauth "k8s.io/apiserver/pkg/authentication/request/union"
"k8s.io/apiserver/pkg/authentication/request/websocket"
"k8s.io/apiserver/pkg/authentication/request/x509"
x509request "k8s.io/apiserver/pkg/authentication/request/x509"
"k8s.io/apiserver/pkg/authentication/token/cache"
webhooktoken "k8s.io/apiserver/plugin/pkg/authenticator/token/webhook"
authenticationclient "k8s.io/client-go/kubernetes/typed/authentication/v1beta1"
"k8s.io/client-go/util/cert"
)
// DelegatingAuthenticatorConfig is the minimal configuration needed to create an authenticator
@ -48,8 +47,9 @@ type DelegatingAuthenticatorConfig struct {
// CacheTTL is the length of time that a token authentication answer will be cached.
CacheTTL time.Duration
// ClientCAFile is the CA bundle file used to authenticate client certificates
ClientCAFile string
// ClientVerifyOptionFn are the options for verifying incoming connections using mTLS and directly assigning to users.
// Generally this is the CA bundle file used to authenticate client certificates
ClientVerifyOptionFn x509request.VerifyOptionFunc
APIAudiences authenticator.Audiences
@ -63,8 +63,8 @@ func (c DelegatingAuthenticatorConfig) New() (authenticator.Request, *spec.Secur
// front-proxy first, then remote
// Add the front proxy authenticator if requested
if c.RequestHeaderConfig != nil {
requestHeaderAuthenticator, err := headerrequest.NewSecure(
c.RequestHeaderConfig.ClientCA,
requestHeaderAuthenticator, err := headerrequest.NewDynamicVerifyOptionsSecure(
c.RequestHeaderConfig.VerifyOptionFn,
c.RequestHeaderConfig.AllowedClientNames,
c.RequestHeaderConfig.UsernameHeaders,
c.RequestHeaderConfig.GroupHeaders,
@ -77,14 +77,8 @@ func (c DelegatingAuthenticatorConfig) New() (authenticator.Request, *spec.Secur
}
// x509 client cert auth
if len(c.ClientCAFile) > 0 {
clientCAs, err := cert.NewPool(c.ClientCAFile)
if err != nil {
return nil, nil, fmt.Errorf("unable to load client CA file %s: %v", c.ClientCAFile, err)
}
verifyOpts := x509.DefaultVerifyOptions()
verifyOpts.Roots = clientCAs
authenticators = append(authenticators, x509.New(verifyOpts, x509.CommonNameUserConversion))
if c.ClientVerifyOptionFn != nil {
authenticators = append(authenticators, x509.NewDynamic(c.ClientVerifyOptionFn, x509.CommonNameUserConversion))
}
if c.TokenAccessReviewClient != nil {

View File

@ -16,6 +16,10 @@ limitations under the License.
package authenticatorfactory
import (
x509request "k8s.io/apiserver/pkg/authentication/request/x509"
)
type RequestHeaderConfig struct {
// UsernameHeaders are the headers to check (in order, case-insensitively) for an identity. The first header with a value wins.
UsernameHeaders []string
@ -24,8 +28,9 @@ type RequestHeaderConfig struct {
// ExtraHeaderPrefixes are the head prefixes to check (case-insentively) for filling in
// the user.Info.Extra. All values of all matching headers will be added.
ExtraHeaderPrefixes []string
// ClientCA points to CA bundle file which is used verify the identity of the front proxy
ClientCA string
// VerifyOptionFn are the options for verifying incoming connections using mTLS. Generally this points to CA bundle file which is used verify the identity of the front proxy.
// It may produce different options at will.
VerifyOptionFn x509request.VerifyOptionFunc
// AllowedClientNames is a list of common names that may be presented by the authenticating front proxy. Empty means: accept any.
AllowedClientNames []string
}

View File

@ -78,11 +78,6 @@ func trimHeaders(headerNames ...string) ([]string, error) {
}
func NewSecure(clientCA string, proxyClientNames []string, nameHeaders []string, groupHeaders []string, extraHeaderPrefixes []string) (authenticator.Request, error) {
headerAuthenticator, err := New(nameHeaders, groupHeaders, extraHeaderPrefixes)
if err != nil {
return nil, err
}
if len(clientCA) == 0 {
return nil, fmt.Errorf("missing clientCA file")
}
@ -102,7 +97,17 @@ func NewSecure(clientCA string, proxyClientNames []string, nameHeaders []string,
opts.Roots.AddCert(cert)
}
return x509request.NewVerifier(opts, headerAuthenticator, sets.NewString(proxyClientNames...)), nil
return NewDynamicVerifyOptionsSecure(x509request.StaticVerifierFn(opts), proxyClientNames, nameHeaders, groupHeaders, extraHeaderPrefixes)
}
// TODO make the string slices dynamic too.
func NewDynamicVerifyOptionsSecure(verifyOptionFn x509request.VerifyOptionFunc, proxyClientNames []string, nameHeaders []string, groupHeaders []string, extraHeaderPrefixes []string) (authenticator.Request, error) {
headerAuthenticator, err := New(nameHeaders, groupHeaders, extraHeaderPrefixes)
if err != nil {
return nil, err
}
return x509request.NewDynamicCAVerifier(verifyOptionFn, headerAuthenticator, sets.NewString(proxyClientNames...)), nil
}
func (a *requestHeaderAuthRequestHandler) AuthenticateRequest(req *http.Request) (*authenticator.Response, bool, error) {

View File

@ -27,6 +27,7 @@ go_library(
name = "go_default_library",
srcs = [
"doc.go",
"verify_options.go",
"x509.go",
],
importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/authentication/request/x509",
@ -36,6 +37,7 @@ go_library(
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library",
"//staging/src/k8s.io/client-go/util/cert:go_default_library",
"//staging/src/k8s.io/component-base/metrics:go_default_library",
"//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library",
],

View File

@ -0,0 +1,49 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package x509
import (
"crypto/x509"
"fmt"
"k8s.io/client-go/util/cert"
)
// StaticVerifierFn is a VerifyOptionFunc that always returns the same value. This allows verify options that cannot change.
func StaticVerifierFn(opts x509.VerifyOptions) VerifyOptionFunc {
return func() x509.VerifyOptions {
return opts
}
}
// NewStaticVerifierFromFile creates a new verification func from a file. It reads the content and then fails.
// It will return a nil function if you pass an empty CA file.
func NewStaticVerifierFromFile(clientCA string) (VerifyOptionFunc, error) {
if len(clientCA) == 0 {
return nil, nil
}
// Wrap with an x509 verifier
var err error
opts := DefaultVerifyOptions()
opts.Roots, err = cert.NewPool(clientCA)
if err != nil {
return nil, fmt.Errorf("error loading certs from %s: %v", clientCA, err)
}
return StaticVerifierFn(opts), nil
}

View File

@ -82,16 +82,26 @@ func (f UserConversionFunc) User(chain []*x509.Certificate) (*authenticator.Resp
return f(chain)
}
// VerifyOptionFunc is function which provides a shallow copy of the VerifyOptions to the authenticator. This allows
// for cases where the options (particularly the CAs) can change.
type VerifyOptionFunc func() x509.VerifyOptions
// Authenticator implements request.Authenticator by extracting user info from verified client certificates
type Authenticator struct {
opts x509.VerifyOptions
user UserConversion
verifyOptionsFn VerifyOptionFunc
user UserConversion
}
// New returns a request.Authenticator that verifies client certificates using the provided
// VerifyOptions, and converts valid certificate chains into user.Info using the provided UserConversion
func New(opts x509.VerifyOptions, user UserConversion) *Authenticator {
return &Authenticator{opts, user}
return NewDynamic(StaticVerifierFn(opts), user)
}
// NewDynamic returns a request.Authenticator that verifies client certificates using the provided
// VerifyOptionFunc (which may be dynamic), and converts valid certificate chains into user.Info using the provided UserConversion
func NewDynamic(verifyOptionsFn VerifyOptionFunc, user UserConversion) *Authenticator {
return &Authenticator{verifyOptionsFn, user}
}
// AuthenticateRequest authenticates the request using presented client certificates
@ -101,7 +111,7 @@ func (a *Authenticator) AuthenticateRequest(req *http.Request) (*authenticator.R
}
// Use intermediates, if provided
optsCopy := a.opts
optsCopy := a.verifyOptionsFn()
if optsCopy.Intermediates == nil && len(req.TLS.PeerCertificates) > 1 {
optsCopy.Intermediates = x509.NewCertPool()
for _, intermediate := range req.TLS.PeerCertificates[1:] {
@ -133,8 +143,8 @@ func (a *Authenticator) AuthenticateRequest(req *http.Request) (*authenticator.R
// Verifier implements request.Authenticator by verifying a client cert on the request, then delegating to the wrapped auth
type Verifier struct {
opts x509.VerifyOptions
auth authenticator.Request
verifyOptionsFn VerifyOptionFunc
auth authenticator.Request
// allowedCommonNames contains the common names which a verified certificate is allowed to have.
// If empty, all verified certificates are allowed.
@ -143,7 +153,13 @@ type Verifier struct {
// NewVerifier create a request.Authenticator by verifying a client cert on the request, then delegating to the wrapped auth
func NewVerifier(opts x509.VerifyOptions, auth authenticator.Request, allowedCommonNames sets.String) authenticator.Request {
return &Verifier{opts, auth, allowedCommonNames}
return NewDynamicCAVerifier(StaticVerifierFn(opts), auth, allowedCommonNames)
}
// NewDynamicCAVerifier create a request.Authenticator by verifying a client cert on the request, then delegating to the wrapped auth
// TODO make the allowedCommonNames dynamic
func NewDynamicCAVerifier(verifyOptionsFn VerifyOptionFunc, auth authenticator.Request, allowedCommonNames sets.String) authenticator.Request {
return &Verifier{verifyOptionsFn, auth, allowedCommonNames}
}
// AuthenticateRequest verifies the presented client certificate, then delegates to the wrapped auth
@ -153,7 +169,7 @@ func (a *Verifier) AuthenticateRequest(req *http.Request) (*authenticator.Respon
}
// Use intermediates, if provided
optsCopy := a.opts
optsCopy := a.verifyOptionsFn()
if optsCopy.Intermediates == nil && len(req.TLS.PeerCertificates) > 1 {
optsCopy.Intermediates = x509.NewCertPool()
for _, intermediate := range req.TLS.PeerCertificates[1:] {

View File

@ -657,6 +657,7 @@ func TestX509(t *testing.T) {
req.TLS = &tls.ConnectionState{PeerCertificates: testCase.Certs}
}
// this effectively tests the simple dynamic verify function.
a := New(testCase.Opts, testCase.User)
resp, ok, err := a.AuthenticateRequest(req)

View File

@ -49,6 +49,7 @@ go_library(
"//staging/src/k8s.io/apiserver/pkg/audit:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/audit/policy:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authentication/request/x509:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authorization/path:go_default_library",

View File

@ -23,16 +23,18 @@ import (
"time"
"github.com/spf13/pflag"
"k8s.io/klog"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
"k8s.io/apiserver/pkg/authentication/request/x509"
"k8s.io/apiserver/pkg/server"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/util/cert"
"k8s.io/klog"
openapicommon "k8s.io/kube-openapi/pkg/common"
)
@ -74,23 +76,48 @@ func (s *RequestHeaderAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
// ToAuthenticationRequestHeaderConfig returns a RequestHeaderConfig config object for these options
// if necessary, nil otherwise.
func (s *RequestHeaderAuthenticationOptions) ToAuthenticationRequestHeaderConfig() *authenticatorfactory.RequestHeaderConfig {
func (s *RequestHeaderAuthenticationOptions) ToAuthenticationRequestHeaderConfig() (*authenticatorfactory.RequestHeaderConfig, error) {
if len(s.ClientCAFile) == 0 {
return nil
return nil, nil
}
verifyFn, err := x509.NewStaticVerifierFromFile(s.ClientCAFile)
if err != nil {
return nil, err
}
return &authenticatorfactory.RequestHeaderConfig{
UsernameHeaders: s.UsernameHeaders,
GroupHeaders: s.GroupHeaders,
ExtraHeaderPrefixes: s.ExtraHeaderPrefixes,
ClientCA: s.ClientCAFile,
VerifyOptionFn: verifyFn,
AllowedClientNames: s.AllowedNames,
}
}, nil
}
// ClientCertAuthenticationOptions provides different options for client cert auth. You should use `GetClientVerifyOptionFn` to
// get the verify options for your authenticator.
type ClientCertAuthenticationOptions struct {
// ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates
ClientCA string
// ClientVerifyOptionFn are the options for verifying incoming connections using mTLS and directly assigning to users.
// Generally this is the CA bundle file used to authenticate client certificates
// If non-nil, this takes priority over the ClientCA file.
ClientVerifyOptionFn x509.VerifyOptionFunc
}
// GetClientVerifyOptionFn provides verify options for your authenticator while respecting the preferred order of verifiers.
func (s *ClientCertAuthenticationOptions) GetClientVerifyOptionFn() (x509.VerifyOptionFunc, error) {
if s.ClientVerifyOptionFn != nil {
return s.ClientVerifyOptionFn, nil
}
if len(s.ClientCA) == 0 {
return nil, nil
}
return x509.NewStaticVerifierFromFile(s.ClientCA)
}
func (s *ClientCertAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
@ -206,12 +233,18 @@ func (s *DelegatingAuthenticationOptions) ApplyTo(c *server.AuthenticationInfo,
}
// configure AuthenticationInfo config
cfg.ClientCAFile = s.ClientCert.ClientCA
cfg.ClientVerifyOptionFn, err = s.ClientCert.GetClientVerifyOptionFn()
if err != nil {
return fmt.Errorf("unable to load client CA file: %v", err)
}
if err = c.ApplyClientCert(s.ClientCert.ClientCA, servingInfo); err != nil {
return fmt.Errorf("unable to load client CA file: %v", err)
}
cfg.RequestHeaderConfig = s.RequestHeader.ToAuthenticationRequestHeaderConfig()
cfg.RequestHeaderConfig, err = s.RequestHeader.ToAuthenticationRequestHeaderConfig()
if err != nil {
return fmt.Errorf("unable to create request header authentication config: %v", err)
}
if err = c.ApplyClientCert(s.RequestHeader.ClientCAFile, servingInfo); err != nil {
return fmt.Errorf("unable to load client CA file: %v", err)
}
@ -307,6 +340,16 @@ func inClusterClientCA(authConfigMap *v1.ConfigMap) (*ClientCertAuthenticationOp
return nil, nil
}
clientCAs, err := cert.NewPoolFromBytes([]byte(clientCA))
if err != nil {
return nil, fmt.Errorf("unable to load client CA from configmap: %v", err)
}
verifyOpts := x509.DefaultVerifyOptions()
verifyOpts.Roots = clientCAs
// we still need to write out the client-ca-file for now because it is used to plumb the options through the apiserver's
// configuration to hint clients.
// TODO deads2k this should eventually be made dynamic along with the authenticator. I'm just wiring them one at at time.
f, err := ioutil.TempFile("", "client-ca-file")
if err != nil {
return nil, err
@ -314,7 +357,11 @@ func inClusterClientCA(authConfigMap *v1.ConfigMap) (*ClientCertAuthenticationOp
if err := ioutil.WriteFile(f.Name(), []byte(clientCA), 0600); err != nil {
return nil, err
}
return &ClientCertAuthenticationOptions{ClientCA: f.Name()}, nil
return &ClientCertAuthenticationOptions{
ClientCA: f.Name(),
ClientVerifyOptionFn: x509.StaticVerifierFn(verifyOpts),
}, nil
}
func inClusterRequestHeader(authConfigMap *v1.ConfigMap) (*RequestHeaderAuthenticationOptions, error) {

View File

@ -46,7 +46,7 @@ func TestToAuthenticationRequestHeaderConfig(t *testing.T) {
{
name: "test when ClientCAFile is not nil",
testOptions: &RequestHeaderAuthenticationOptions{
ClientCAFile: "/testClientCAFile",
ClientCAFile: "testdata/root.pem",
UsernameHeaders: []string{"x-remote-user"},
GroupHeaders: []string{"x-remote-group"},
ExtraHeaderPrefixes: []string{"x-remote-extra-"},
@ -56,7 +56,7 @@ func TestToAuthenticationRequestHeaderConfig(t *testing.T) {
UsernameHeaders: []string{"x-remote-user"},
GroupHeaders: []string{"x-remote-group"},
ExtraHeaderPrefixes: []string{"x-remote-extra-"},
ClientCA: "/testClientCAFile",
VerifyOptionFn: nil, // this is nil because you can't compare functions
AllowedClientNames: []string{"kube-aggregator"},
},
},
@ -64,7 +64,17 @@ func TestToAuthenticationRequestHeaderConfig(t *testing.T) {
for _, testcase := range testCases {
t.Run(testcase.name, func(t *testing.T) {
resultConfig := testcase.testOptions.ToAuthenticationRequestHeaderConfig()
resultConfig, err := testcase.testOptions.ToAuthenticationRequestHeaderConfig()
if err != nil {
t.Fatal(err)
}
if resultConfig != nil {
if resultConfig.VerifyOptionFn == nil {
t.Error("missing requestheader verify")
}
resultConfig.VerifyOptionFn = nil
}
if !reflect.DeepEqual(resultConfig, testcase.expectConfig) {
t.Errorf("got RequestHeaderConfig: %#v, expected RequestHeaderConfig: %#v", resultConfig, testcase.expectConfig)
}

View File

@ -0,0 +1,11 @@
-----BEGIN CERTIFICATE-----
MIIBpTCCAUugAwIBAgIUPV4LAC5KK8YWY1FegyTuhkGUr3EwCgYIKoZIzj0EAwIw
GjEYMBYGA1UEAxMPSW50ZXJtZWRpYXRlLUNBMB4XDTkwMTIzMTIzNTkwMFoXDTkw
MTIzMTIzNTkwMFowFDESMBAGA1UEAxMJTXkgQ2xpZW50MFkwEwYHKoZIzj0CAQYI
KoZIzj0DAQcDQgAEyYUnseNUN87rfHgekrfZu5sj4wlt5LYr3JYZZkfSbsb+BW3/
RzX02ifjp+8w7mI4qUGg6y6J7oXHGFT3uj9kj6N1MHMwDgYDVR0PAQH/BAQDAgWg
MBMGA1UdJQQMMAoGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFKsX
EnXwDg8j2LIEM1QzmFrE6537MB8GA1UdIwQYMBaAFF+p0JcY31pz+mjNZnjv0Gum
92vZMAoGCCqGSM49BAMCA0gAMEUCIG4FBcb57oqOCoaFiJ+Yx6S0zkaash7bTv3V
CIy9JvFdAiEAy8bf2S9EkvZyURZ6ycgEMnekll57Ebze6rjlPx8+B1Y=
-----END CERTIFICATE-----

View File

@ -0,0 +1,11 @@
-----BEGIN CERTIFICATE-----
MIIBqDCCAU2gAwIBAgIUfbqeieihh/oERbfvRm38XvS/xHAwCgYIKoZIzj0EAwIw
GjEYMBYGA1UEAxMPSW50ZXJtZWRpYXRlLUNBMCAXDTE2MTAxMTA1MDYwMFoYDzIx
MTYwOTE3MDUwNjAwWjAUMRIwEAYDVQQDEwlNeSBDbGllbnQwWTATBgcqhkjOPQIB
BggqhkjOPQMBBwNCAARv6N4R/sjMR65iMFGNLN1GC/vd7WhDW6J4X/iAjkRLLnNb
KbRG/AtOUZ+7upJ3BWIRKYbOabbQGQe2BbKFiap4o3UwczAOBgNVHQ8BAf8EBAMC
BaAwEwYDVR0lBAwwCgYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU
K/pZOWpNcYai6eHFpmJEeFpeQlEwHwYDVR0jBBgwFoAUX6nQlxjfWnP6aM1meO/Q
a6b3a9kwCgYIKoZIzj0EAwIDSQAwRgIhAIWTKw/sjJITqeuNzJDAKU4xo1zL+xJ5
MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps=
-----END CERTIFICATE-----

View File

@ -0,0 +1,24 @@
{
"signing": {
"profiles": {
"valid": {
"expiry": "876000h",
"usages": [
"signing",
"key encipherment",
"client auth"
]
},
"expired": {
"expiry": "1h",
"not_before": "1990-12-31T23:59:00Z",
"not_after": "1990-12-31T23:59:00Z",
"usages": [
"signing",
"key encipherment",
"client auth"
]
}
}
}
}

View File

@ -0,0 +1,3 @@
{
"CN": "My Client"
}

View File

@ -0,0 +1,24 @@
#!/usr/bin/env bash
# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
cfssl gencert -initca root.csr.json | cfssljson -bare root
cfssl gencert -initca intermediate.csr.json | cfssljson -bare intermediate
cfssl sign -ca root.pem -ca-key root-key.pem -config intermediate.config.json intermediate.csr | cfssljson -bare intermediate
cfssl gencert -ca intermediate.pem -ca-key intermediate-key.pem -config client.config.json --profile=valid client.csr.json | cfssljson -bare client-valid
cfssl gencert -ca intermediate.pem -ca-key intermediate-key.pem -config client.config.json --profile=expired client.csr.json | cfssljson -bare client-expired

View File

@ -0,0 +1,18 @@
{
"signing": {
"default": {
"usages": [
"digital signature",
"cert sign",
"crl sign",
"signing",
"key encipherment",
"client auth"
],
"expiry": "876000h",
"ca_constraint": {
"is_ca": true
}
}
}
}

View File

@ -0,0 +1,6 @@
{
"CN": "Intermediate-CA",
"ca": {
"expiry": "876000h"
}
}

View File

@ -0,0 +1,11 @@
-----BEGIN CERTIFICATE-----
MIIBqDCCAU6gAwIBAgIUfqZtjoFgczZ+oQZbEC/BDSS2J6wwCgYIKoZIzj0EAwIw
EjEQMA4GA1UEAxMHUm9vdC1DQTAgFw0xNjEwMTEwNTA2MDBaGA8yMTE2MDkxNzA1
MDYwMFowGjEYMBYGA1UEAxMPSW50ZXJtZWRpYXRlLUNBMFkwEwYHKoZIzj0CAQYI
KoZIzj0DAQcDQgAEyWHEMMCctJg8Xa5YWLqaCPbk3MjB+uvXac42JM9pj4k9jedD
kpUJRkWIPzgJI8Zk/3cSzluUTixP6JBSDKtwwaN4MHYwDgYDVR0PAQH/BAQDAgGm
MBMGA1UdJQQMMAoGCCsGAQUFBwMCMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYE
FF+p0JcY31pz+mjNZnjv0Gum92vZMB8GA1UdIwQYMBaAFB7P6+i4/pfNjqZgJv/b
dgA7Fe4tMAoGCCqGSM49BAMCA0gAMEUCIQCTT1YWQZaAqfQ2oBxzOkJE2BqLFxhz
3smQlrZ5gCHddwIgcvT7puhYOzAgcvMn9+SZ1JOyZ7edODjshCVCRnuHK2c=
-----END CERTIFICATE-----

View File

@ -0,0 +1,6 @@
{
"CN": "Root-CA",
"ca": {
"expiry": "876000h"
}
}

View File

@ -0,0 +1,11 @@
-----BEGIN CERTIFICATE-----
MIIBizCCATGgAwIBAgIUH4plk9qwD61FVXgiOTngFU5FeSkwCgYIKoZIzj0EAwIw
EjEQMA4GA1UEAxMHUm9vdC1DQTAgFw0xNjEwMTEwNTA2MDBaGA8yMTE2MDkxNzA1
MDYwMFowEjEQMA4GA1UEAxMHUm9vdC1DQTBZMBMGByqGSM49AgEGCCqGSM49AwEH
A0IABI2CsrAnMGT8P2VGU2MLo5pv86Z74kcV9hgkLJUkSaeNyc1s89w7X5V2wvwu
iWEJRGm5RoZJausmyZLZEoKEVXejYzBhMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMB
Af8EBTADAQH/MB0GA1UdDgQWBBQez+vouP6XzY6mYCb/23YAOxXuLTAfBgNVHSME
GDAWgBQez+vouP6XzY6mYCb/23YAOxXuLTAKBggqhkjOPQQDAgNIADBFAiBGclts
vJRM+QMVoV/1L9b+hvhgLIp/OupUFsSOReefIwIhALY06hBklyh8eFwuBtyX2VcE
8xlVn4/5idUvc3Xv2h9s
-----END CERTIFICATE-----

View File

@ -72,7 +72,22 @@ func WriteCert(certPath string, data []byte) error {
// NewPool returns an x509.CertPool containing the certificates in the given PEM-encoded file.
// Returns an error if the file could not be read, a certificate could not be parsed, or if the file does not contain any certificates
func NewPool(filename string) (*x509.CertPool, error) {
certs, err := CertsFromFile(filename)
pemBlock, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
pool, err := NewPoolFromBytes(pemBlock)
if err != nil {
return nil, fmt.Errorf("error creating pool from %s: %s", filename, err)
}
return pool, nil
}
// NewPoolFromBytes returns an x509.CertPool containing the certificates in the given PEM-encoded bytes.
// Returns an error if the file could not be read, a certificate could not be parsed, or if the file does not contain any certificates
func NewPoolFromBytes(pemBlock []byte) (*x509.CertPool, error) {
certs, err := ParseCertsPEM(pemBlock)
if err != nil {
return nil, err
}