Merge pull request #55132 from caesarxuchao/webhook-move-shared-code

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Reorganize admission webhook code

ref: https://github.com/kubernetes/features/issues/492

* Moved client and kubeconfig related code to webhook/config;
* Moved the rule matcher to webhook/rules;
* Left TODOs saying we are going to move some other common utilities;
* Other code is moved to webhook/validation.


This is to prepare adding the mutating webhook. See https://github.com/kubernetes/kubernetes/pull/54892.
This commit is contained in:
Kubernetes Submit Queue 2017-11-14 17:50:54 -08:00 committed by GitHub
commit ff7934fdee
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
33 changed files with 503 additions and 236 deletions

View File

@ -59,7 +59,7 @@ go_library(
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/apiserver/pkg/admission:go_default_library", "//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
"//vendor/k8s.io/apiserver/pkg/admission/plugin/webhook:go_default_library", "//vendor/k8s.io/apiserver/pkg/admission/plugin/webhook/config:go_default_library",
"//vendor/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library", "//vendor/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library",
"//vendor/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", "//vendor/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library",
"//vendor/k8s.io/apiserver/pkg/server:go_default_library", "//vendor/k8s.io/apiserver/pkg/server:go_default_library",

View File

@ -44,7 +44,7 @@ import (
"k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/sets"
utilwait "k8s.io/apimachinery/pkg/util/wait" utilwait "k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/admission/plugin/webhook" webhookconfig "k8s.io/apiserver/pkg/admission/plugin/webhook/config"
"k8s.io/apiserver/pkg/authentication/authenticator" "k8s.io/apiserver/pkg/authentication/authenticator"
"k8s.io/apiserver/pkg/authorization/authorizer" "k8s.io/apiserver/pkg/authorization/authorizer"
genericapiserver "k8s.io/apiserver/pkg/server" genericapiserver "k8s.io/apiserver/pkg/server"
@ -446,8 +446,8 @@ func BuildGenericConfig(s *options.ServerRunOptions, proxyTransport *http.Transp
genericConfig.DisabledPostStartHooks.Insert(rbacrest.PostStartHookName) genericConfig.DisabledPostStartHooks.Insert(rbacrest.PostStartHookName)
} }
webhookAuthResolver := func(delegate webhook.AuthenticationInfoResolver) webhook.AuthenticationInfoResolver { webhookAuthResolver := func(delegate webhookconfig.AuthenticationInfoResolver) webhookconfig.AuthenticationInfoResolver {
return webhook.AuthenticationInfoResolverFunc(func(server string) (*rest.Config, error) { return webhookconfig.AuthenticationInfoResolverFunc(func(server string) (*rest.Config, error) {
if server == "kubernetes.default.svc" { if server == "kubernetes.default.svc" {
return genericConfig.LoopbackClientConfig, nil return genericConfig.LoopbackClientConfig, nil
} }
@ -486,7 +486,7 @@ func BuildGenericConfig(s *options.ServerRunOptions, proxyTransport *http.Transp
} }
// BuildAdmissionPluginInitializer constructs the admission plugin initializer // BuildAdmissionPluginInitializer constructs the admission plugin initializer
func BuildAdmissionPluginInitializer(s *options.ServerRunOptions, client internalclientset.Interface, sharedInformers informers.SharedInformerFactory, serviceResolver aggregatorapiserver.ServiceResolver, webhookAuthWrapper webhook.AuthenticationInfoResolverWrapper) (admission.PluginInitializer, error) { func BuildAdmissionPluginInitializer(s *options.ServerRunOptions, client internalclientset.Interface, sharedInformers informers.SharedInformerFactory, serviceResolver aggregatorapiserver.ServiceResolver, webhookAuthWrapper webhookconfig.AuthenticationInfoResolverWrapper) (admission.PluginInitializer, error) {
var cloudConfig []byte var cloudConfig []byte
if s.CloudProvider.CloudConfigFile != "" { if s.CloudProvider.CloudConfigFile != "" {

View File

@ -542,7 +542,8 @@ staging/src/k8s.io/apiserver/pkg/admission
staging/src/k8s.io/apiserver/pkg/admission/configuration staging/src/k8s.io/apiserver/pkg/admission/configuration
staging/src/k8s.io/apiserver/pkg/admission/plugin/initialization staging/src/k8s.io/apiserver/pkg/admission/plugin/initialization
staging/src/k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle staging/src/k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle
staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/config
staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/validating
staging/src/k8s.io/apiserver/pkg/apis/apiserver staging/src/k8s.io/apiserver/pkg/apis/apiserver
staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1 staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1
staging/src/k8s.io/apiserver/pkg/apis/audit staging/src/k8s.io/apiserver/pkg/apis/audit

View File

@ -13,7 +13,7 @@ go_test(
library = ":go_default_library", library = ":go_default_library",
deps = [ deps = [
"//vendor/k8s.io/apiserver/pkg/admission:go_default_library", "//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
"//vendor/k8s.io/apiserver/pkg/admission/plugin/webhook:go_default_library", "//vendor/k8s.io/apiserver/pkg/admission/plugin/webhook/config:go_default_library",
], ],
) )
@ -27,7 +27,7 @@ go_library(
"//pkg/quota:go_default_library", "//pkg/quota:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library",
"//vendor/k8s.io/apiserver/pkg/admission:go_default_library", "//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
"//vendor/k8s.io/apiserver/pkg/admission/plugin/webhook:go_default_library", "//vendor/k8s.io/apiserver/pkg/admission/plugin/webhook/config:go_default_library",
"//vendor/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", "//vendor/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library",
], ],

View File

@ -21,7 +21,7 @@ import (
"testing" "testing"
"k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/admission/plugin/webhook" "k8s.io/apiserver/pkg/admission/plugin/webhook/config"
) )
type doNothingAdmission struct{} type doNothingAdmission struct{}
@ -61,7 +61,7 @@ type serviceWanter struct {
got ServiceResolver got ServiceResolver
} }
func (s *serviceWanter) SetServiceResolver(sr webhook.ServiceResolver) { s.got = sr } func (s *serviceWanter) SetServiceResolver(sr config.ServiceResolver) { s.got = sr }
func TestWantsServiceResolver(t *testing.T) { func TestWantsServiceResolver(t *testing.T) {
sw := &serviceWanter{} sw := &serviceWanter{}

View File

@ -21,7 +21,7 @@ import (
"k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/admission/plugin/webhook" webhookconfig "k8s.io/apiserver/pkg/admission/plugin/webhook/config"
"k8s.io/apiserver/pkg/authorization/authorizer" "k8s.io/apiserver/pkg/authorization/authorizer"
clientset "k8s.io/client-go/kubernetes" clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
@ -62,7 +62,7 @@ type WantsQuotaConfiguration interface {
// WantsServiceResolver defines a fuction that accepts a ServiceResolver for // WantsServiceResolver defines a fuction that accepts a ServiceResolver for
// admission plugins that need to make calls to services. // admission plugins that need to make calls to services.
type WantsServiceResolver interface { type WantsServiceResolver interface {
SetServiceResolver(webhook.ServiceResolver) SetServiceResolver(webhookconfig.ServiceResolver)
} }
// ServiceResolver knows how to convert a service reference into an actual // ServiceResolver knows how to convert a service reference into an actual
@ -74,7 +74,7 @@ type ServiceResolver interface {
// WantsAuthenticationInfoResolverWrapper defines a function that wraps the standard AuthenticationInfoResolver // WantsAuthenticationInfoResolverWrapper defines a function that wraps the standard AuthenticationInfoResolver
// to allow the apiserver to control what is returned as auth info // to allow the apiserver to control what is returned as auth info
type WantsAuthenticationInfoResolverWrapper interface { type WantsAuthenticationInfoResolverWrapper interface {
SetAuthenticationInfoResolverWrapper(webhook.AuthenticationInfoResolverWrapper) SetAuthenticationInfoResolverWrapper(webhookconfig.AuthenticationInfoResolverWrapper)
admission.InitializationValidator admission.InitializationValidator
} }
@ -86,8 +86,8 @@ type PluginInitializer struct {
cloudConfig []byte cloudConfig []byte
restMapper meta.RESTMapper restMapper meta.RESTMapper
quotaConfiguration quota.Configuration quotaConfiguration quota.Configuration
serviceResolver webhook.ServiceResolver serviceResolver webhookconfig.ServiceResolver
authenticationInfoResolverWrapper webhook.AuthenticationInfoResolverWrapper authenticationInfoResolverWrapper webhookconfig.AuthenticationInfoResolverWrapper
} }
var _ admission.PluginInitializer = &PluginInitializer{} var _ admission.PluginInitializer = &PluginInitializer{}
@ -101,8 +101,8 @@ func NewPluginInitializer(
cloudConfig []byte, cloudConfig []byte,
restMapper meta.RESTMapper, restMapper meta.RESTMapper,
quotaConfiguration quota.Configuration, quotaConfiguration quota.Configuration,
authenticationInfoResolverWrapper webhook.AuthenticationInfoResolverWrapper, authenticationInfoResolverWrapper webhookconfig.AuthenticationInfoResolverWrapper,
serviceResolver webhook.ServiceResolver, serviceResolver webhookconfig.ServiceResolver,
) *PluginInitializer { ) *PluginInitializer {
return &PluginInitializer{ return &PluginInitializer{
internalClient: internalClient, internalClient: internalClient,

View File

@ -875,7 +875,15 @@
"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
}, },
{ {
"ImportPath": "k8s.io/apiserver/pkg/admission/plugin/webhook", "ImportPath": "k8s.io/apiserver/pkg/admission/plugin/webhook/config",
"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
},
{
"ImportPath": "k8s.io/apiserver/pkg/admission/plugin/webhook/rules",
"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
},
{
"ImportPath": "k8s.io/apiserver/pkg/admission/plugin/webhook/validating",
"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
}, },
{ {

View File

@ -78,7 +78,9 @@ filegroup(
"//staging/src/k8s.io/apiserver/pkg/admission/initializer:all-srcs", "//staging/src/k8s.io/apiserver/pkg/admission/initializer:all-srcs",
"//staging/src/k8s.io/apiserver/pkg/admission/plugin/initialization:all-srcs", "//staging/src/k8s.io/apiserver/pkg/admission/plugin/initialization:all-srcs",
"//staging/src/k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle:all-srcs", "//staging/src/k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle:all-srcs",
"//staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook:all-srcs", "//staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/config:all-srcs",
"//staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/rules:all-srcs",
"//staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/validating:all-srcs",
], ],
tags = ["automanaged"], tags = ["automanaged"],
) )

View File

@ -0,0 +1,54 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
srcs = [
"authentication.go",
"client.go",
"errors.go",
"kubeconfig.go",
"serviceresolver.go",
],
importpath = "k8s.io/apiserver/pkg/admission/plugin/webhook/config",
visibility = ["//visibility:public"],
deps = [
"//vendor/github.com/hashicorp/golang-lru:go_default_library",
"//vendor/k8s.io/api/admissionregistration/v1alpha1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/yaml:go_default_library",
"//vendor/k8s.io/client-go/rest:go_default_library",
"//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
"//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library",
],
)
go_test(
name = "go_default_test",
srcs = [
"authentication_test.go",
"serviceresolver_test.go",
],
importpath = "k8s.io/apiserver/pkg/admission/plugin/webhook/config",
library = ":go_default_library",
deps = [
"//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/diff:go_default_library",
"//vendor/k8s.io/client-go/rest:go_default_library",
"//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
package webhook package config
import ( import (
"fmt" "fmt"
@ -27,14 +27,19 @@ import (
clientcmdapi "k8s.io/client-go/tools/clientcmd/api" clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
) )
// AuthenticationInfoResolverWrapper can be used to inject Dial function to the
// rest.Config generated by the resolver.
type AuthenticationInfoResolverWrapper func(AuthenticationInfoResolver) AuthenticationInfoResolver type AuthenticationInfoResolverWrapper func(AuthenticationInfoResolver) AuthenticationInfoResolver
// AuthenticationInfoResolver builds rest.Config base on the server name.
type AuthenticationInfoResolver interface { type AuthenticationInfoResolver interface {
ClientConfigFor(server string) (*rest.Config, error) ClientConfigFor(server string) (*rest.Config, error)
} }
// AuthenticationInfoResolverFunc implements AuthenticationInfoResolver.
type AuthenticationInfoResolverFunc func(server string) (*rest.Config, error) type AuthenticationInfoResolverFunc func(server string) (*rest.Config, error)
//ClientConfigFor implements AuthenticationInfoResolver.
func (a AuthenticationInfoResolverFunc) ClientConfigFor(server string) (*rest.Config, error) { func (a AuthenticationInfoResolverFunc) ClientConfigFor(server string) (*rest.Config, error) {
return a(server) return a(server)
} }
@ -43,7 +48,10 @@ type defaultAuthenticationInfoResolver struct {
kubeconfig clientcmdapi.Config kubeconfig clientcmdapi.Config
} }
func newDefaultAuthenticationInfoResolver(kubeconfigFile string) (AuthenticationInfoResolver, error) { // NewDefaultAuthenticationInfoResolver generates an AuthenticationInfoResolver
// that builds rest.Config based on the kubeconfig file. kubeconfigFile is the
// path to the kubeconfig.
func NewDefaultAuthenticationInfoResolver(kubeconfigFile string) (AuthenticationInfoResolver, error) {
if len(kubeconfigFile) == 0 { if len(kubeconfigFile) == 0 {
return &defaultAuthenticationInfoResolver{}, nil return &defaultAuthenticationInfoResolver{}, nil
} }

View File

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
package webhook package config
import ( import (
"testing" "testing"

View File

@ -0,0 +1,174 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package config
import (
"encoding/json"
"errors"
"fmt"
"net"
"net/url"
lru "github.com/hashicorp/golang-lru"
"k8s.io/api/admissionregistration/v1alpha1"
"k8s.io/apimachinery/pkg/runtime"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/client-go/rest"
)
const (
defaultCacheSize = 200
)
var (
ErrNeedServiceOrURL = errors.New("webhook configuration must have either service or URL")
)
// ClientManager builds REST clients to talk to webhooks. It caches the clients
// to avoid duplicate creation.
type ClientManager struct {
authInfoResolver AuthenticationInfoResolver
serviceResolver ServiceResolver
negotiatedSerializer runtime.NegotiatedSerializer
cache *lru.Cache
}
// NewClientManager creates a ClientManager.
func NewClientManager() (ClientManager, error) {
cache, err := lru.New(defaultCacheSize)
if err != nil {
return ClientManager{}, err
}
return ClientManager{
cache: cache,
}, nil
}
// SetAuthenticationInfoResolverWrapper sets the
// AuthenticationInfoResolverWrapper.
func (cm *ClientManager) SetAuthenticationInfoResolverWrapper(wrapper AuthenticationInfoResolverWrapper) {
if wrapper != nil {
cm.authInfoResolver = wrapper(cm.authInfoResolver)
}
}
// SetAuthenticationInfoResolver sets the AuthenticationInfoResolver.
func (cm *ClientManager) SetAuthenticationInfoResolver(resolver AuthenticationInfoResolver) {
cm.authInfoResolver = resolver
}
// SetServiceResolver sets the ServiceResolver.
func (cm *ClientManager) SetServiceResolver(sr ServiceResolver) {
if sr != nil {
cm.serviceResolver = sr
}
}
// SetNegotiatedSerializer sets the NegotiatedSerializer.
func (cm *ClientManager) SetNegotiatedSerializer(n runtime.NegotiatedSerializer) {
cm.negotiatedSerializer = n
}
// Validate checks if ClientManager is properly set up.
func (cm *ClientManager) Validate() error {
var errs []error
if cm.negotiatedSerializer == nil {
errs = append(errs, fmt.Errorf("the ClientManager requires a negotiatedSerializer"))
}
if cm.serviceResolver == nil {
errs = append(errs, fmt.Errorf("the ClientManager requires a serviceResolver"))
}
if cm.authInfoResolver == nil {
errs = append(errs, fmt.Errorf("the ClientManager requires an authInfoResolver"))
}
return utilerrors.NewAggregate(errs)
}
// HookClient get a RESTClient from the cache, or constructs one based on the
// webhook configuration.
func (cm *ClientManager) HookClient(h *v1alpha1.Webhook) (*rest.RESTClient, error) {
cacheKey, err := json.Marshal(h.ClientConfig)
if err != nil {
return nil, err
}
if client, ok := cm.cache.Get(string(cacheKey)); ok {
return client.(*rest.RESTClient), nil
}
complete := func(cfg *rest.Config) (*rest.RESTClient, error) {
cfg.TLSClientConfig.CAData = h.ClientConfig.CABundle
cfg.ContentConfig.NegotiatedSerializer = cm.negotiatedSerializer
cfg.ContentConfig.ContentType = runtime.ContentTypeJSON
client, err := rest.UnversionedRESTClientFor(cfg)
if err == nil {
cm.cache.Add(string(cacheKey), client)
}
return client, err
}
if svc := h.ClientConfig.Service; svc != nil {
serverName := svc.Name + "." + svc.Namespace + ".svc"
restConfig, err := cm.authInfoResolver.ClientConfigFor(serverName)
if err != nil {
return nil, err
}
cfg := rest.CopyConfig(restConfig)
host := serverName + ":443"
cfg.Host = "https://" + host
if svc.Path != nil {
cfg.APIPath = *svc.Path
}
cfg.TLSClientConfig.ServerName = serverName
delegateDialer := cfg.Dial
if delegateDialer == nil {
delegateDialer = net.Dial
}
cfg.Dial = func(network, addr string) (net.Conn, error) {
if addr == host {
u, err := cm.serviceResolver.ResolveEndpoint(svc.Namespace, svc.Name)
if err != nil {
return nil, err
}
addr = u.Host
}
return delegateDialer(network, addr)
}
return complete(cfg)
}
if h.ClientConfig.URL == nil {
return nil, &ErrCallingWebhook{WebhookName: h.Name, Reason: ErrNeedServiceOrURL}
}
u, err := url.Parse(*h.ClientConfig.URL)
if err != nil {
return nil, &ErrCallingWebhook{WebhookName: h.Name, Reason: fmt.Errorf("Unparsable URL: %v", err)}
}
restConfig, err := cm.authInfoResolver.ClientConfigFor(u.Host)
if err != nil {
return nil, err
}
cfg := rest.CopyConfig(restConfig)
cfg.Host = u.Host
cfg.APIPath = u.Path
return complete(cfg)
}

View File

@ -0,0 +1,34 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package config
import "fmt"
// ErrCallingWebhook is returned for transport-layer errors calling webhooks. It
// represents a failure to talk to the webhook, not the webhook rejecting a
// request.
type ErrCallingWebhook struct {
WebhookName string
Reason error
}
func (e *ErrCallingWebhook) Error() string {
if e.Reason != nil {
return fmt.Sprintf("failed calling admission webhook %q: %v", e.WebhookName, e.Reason)
}
return fmt.Sprintf("failed calling admission webhook %q; no further details available", e.WebhookName)
}

View File

@ -14,9 +14,32 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
package webhook package config
import (
"io"
"k8s.io/apimachinery/pkg/util/yaml"
)
// AdmissionConfig holds config data that is unique to each API server. // AdmissionConfig holds config data that is unique to each API server.
type AdmissionConfig struct { type AdmissionConfig struct {
// KubeConfigFile is the path to the kubeconfig file.
KubeConfigFile string `json:"kubeConfigFile"` KubeConfigFile string `json:"kubeConfigFile"`
} }
// LoadConfig extract the KubeConfigFile from configFile
func LoadConfig(configFile io.Reader) (string, error) {
var kubeconfigFile string
if configFile != nil {
// TODO: move this to a versioned configuration file format
var config AdmissionConfig
d := yaml.NewYAMLOrJSONDecoder(configFile, 4096)
err := d.Decode(&config)
if err != nil {
return "", err
}
kubeconfigFile = config.KubeConfigFile
}
return kubeconfigFile, nil
}

View File

@ -14,8 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
// Package webhook checks a webhook for configured operation admission package config
package webhook
import ( import (
"errors" "errors"
@ -23,8 +22,17 @@ import (
"net/url" "net/url"
) )
// ServiceResolver knows how to convert a service reference into an actual location.
type ServiceResolver interface {
ResolveEndpoint(namespace, name string) (*url.URL, error)
}
type defaultServiceResolver struct{} type defaultServiceResolver struct{}
func NewDefaultServiceResolver() ServiceResolver {
return &defaultServiceResolver{}
}
// ResolveEndpoint constructs a service URL from a given namespace and name // ResolveEndpoint constructs a service URL from a given namespace and name
// note that the name and namespace are required and by default all created addresses use HTTPS scheme. // note that the name and namespace are required and by default all created addresses use HTTPS scheme.
// for example: // for example:

View File

@ -14,8 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
// Package webhook checks a webhook for configured operation admission package config
package webhook
import ( import (
"fmt" "fmt"

View File

@ -0,0 +1,38 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
srcs = ["rules.go"],
importpath = "k8s.io/apiserver/pkg/admission/plugin/webhook/rules",
visibility = ["//visibility:public"],
deps = [
"//vendor/k8s.io/api/admissionregistration/v1alpha1:go_default_library",
"//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
],
)
go_test(
name = "go_default_test",
srcs = ["rules_test.go"],
importpath = "k8s.io/apiserver/pkg/admission/plugin/webhook/rules",
library = ":go_default_library",
deps = [
"//vendor/k8s.io/api/admissionregistration/v1alpha1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
"//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@ -14,8 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
// Package webhook checks a webhook for configured operation admission package rules
package webhook
import ( import (
"strings" "strings"
@ -24,12 +23,14 @@ import (
"k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/admission"
) )
type RuleMatcher struct { // Matcher determines if the Attr matches the Rule.
type Matcher struct {
Rule v1alpha1.RuleWithOperations Rule v1alpha1.RuleWithOperations
Attr admission.Attributes Attr admission.Attributes
} }
func (r *RuleMatcher) Matches() bool { // Matches returns if the Attr matches the Rule.
func (r *Matcher) Matches() bool {
return r.operation() && return r.operation() &&
r.group() && r.group() &&
r.version() && r.version() &&
@ -49,15 +50,15 @@ func exactOrWildcard(items []string, requested string) bool {
return false return false
} }
func (r *RuleMatcher) group() bool { func (r *Matcher) group() bool {
return exactOrWildcard(r.Rule.APIGroups, r.Attr.GetResource().Group) return exactOrWildcard(r.Rule.APIGroups, r.Attr.GetResource().Group)
} }
func (r *RuleMatcher) version() bool { func (r *Matcher) version() bool {
return exactOrWildcard(r.Rule.APIVersions, r.Attr.GetResource().Version) return exactOrWildcard(r.Rule.APIVersions, r.Attr.GetResource().Version)
} }
func (r *RuleMatcher) operation() bool { func (r *Matcher) operation() bool {
attrOp := r.Attr.GetOperation() attrOp := r.Attr.GetOperation()
for _, op := range r.Rule.Operations { for _, op := range r.Rule.Operations {
if op == v1alpha1.OperationAll { if op == v1alpha1.OperationAll {
@ -80,7 +81,7 @@ func splitResource(resSub string) (res, sub string) {
return parts[0], "" return parts[0], ""
} }
func (r *RuleMatcher) resource() bool { func (r *Matcher) resource() bool {
opRes, opSub := r.Attr.GetResource().Resource, r.Attr.GetSubresource() opRes, opSub := r.Attr.GetResource().Resource, r.Attr.GetSubresource()
for _, res := range r.Rule.Resources { for _, res := range r.Rule.Resources {
res, sub := splitResource(res) res, sub := splitResource(res)

View File

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
package webhook package rules
import ( import (
"testing" "testing"
@ -77,13 +77,13 @@ func TestGroup(t *testing.T) {
for name, tt := range table { for name, tt := range table {
for _, m := range tt.match { for _, m := range tt.match {
r := RuleMatcher{tt.rule, m} r := Matcher{tt.rule, m}
if !r.group() { if !r.group() {
t.Errorf("%v: expected match %#v", name, m) t.Errorf("%v: expected match %#v", name, m)
} }
} }
for _, m := range tt.noMatch { for _, m := range tt.noMatch {
r := RuleMatcher{tt.rule, m} r := Matcher{tt.rule, m}
if r.group() { if r.group() {
t.Errorf("%v: expected no match %#v", name, m) t.Errorf("%v: expected no match %#v", name, m)
} }
@ -121,13 +121,13 @@ func TestVersion(t *testing.T) {
} }
for name, tt := range table { for name, tt := range table {
for _, m := range tt.match { for _, m := range tt.match {
r := RuleMatcher{tt.rule, m} r := Matcher{tt.rule, m}
if !r.version() { if !r.version() {
t.Errorf("%v: expected match %#v", name, m) t.Errorf("%v: expected match %#v", name, m)
} }
} }
for _, m := range tt.noMatch { for _, m := range tt.noMatch {
r := RuleMatcher{tt.rule, m} r := Matcher{tt.rule, m}
if r.version() { if r.version() {
t.Errorf("%v: expected no match %#v", name, m) t.Errorf("%v: expected no match %#v", name, m)
} }
@ -204,13 +204,13 @@ func TestOperation(t *testing.T) {
} }
for name, tt := range table { for name, tt := range table {
for _, m := range tt.match { for _, m := range tt.match {
r := RuleMatcher{tt.rule, m} r := Matcher{tt.rule, m}
if !r.operation() { if !r.operation() {
t.Errorf("%v: expected match %#v", name, m) t.Errorf("%v: expected match %#v", name, m)
} }
} }
for _, m := range tt.noMatch { for _, m := range tt.noMatch {
r := RuleMatcher{tt.rule, m} r := Matcher{tt.rule, m}
if r.operation() { if r.operation() {
t.Errorf("%v: expected no match %#v", name, m) t.Errorf("%v: expected no match %#v", name, m)
} }
@ -285,13 +285,13 @@ func TestResource(t *testing.T) {
} }
for name, tt := range table { for name, tt := range table {
for _, m := range tt.match { for _, m := range tt.match {
r := RuleMatcher{tt.rule, m} r := Matcher{tt.rule, m}
if !r.resource() { if !r.resource() {
t.Errorf("%v: expected match %#v", name, m) t.Errorf("%v: expected match %#v", name, m)
} }
} }
for _, m := range tt.noMatch { for _, m := range tt.noMatch {
r := RuleMatcher{tt.rule, m} r := Matcher{tt.rule, m}
if r.resource() { if r.resource() {
t.Errorf("%v: expected no match %#v", name, m) t.Errorf("%v: expected no match %#v", name, m)
} }

View File

@ -5,17 +5,12 @@ go_library(
srcs = [ srcs = [
"admission.go", "admission.go",
"admissionreview.go", "admissionreview.go",
"authentication.go",
"config.go",
"doc.go", "doc.go",
"rules.go",
"serviceresolver.go",
], ],
importpath = "k8s.io/apiserver/pkg/admission/plugin/webhook", importpath = "k8s.io/apiserver/pkg/admission/plugin/webhook/validating",
visibility = ["//visibility:public"], visibility = ["//visibility:public"],
deps = [ deps = [
"//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/golang/glog:go_default_library",
"//vendor/github.com/hashicorp/golang-lru:go_default_library",
"//vendor/k8s.io/api/admission/v1alpha1:go_default_library", "//vendor/k8s.io/api/admission/v1alpha1:go_default_library",
"//vendor/k8s.io/api/admissionregistration/v1alpha1:go_default_library", "//vendor/k8s.io/api/admissionregistration/v1alpha1:go_default_library",
"//vendor/k8s.io/api/authentication/v1:go_default_library", "//vendor/k8s.io/api/authentication/v1:go_default_library",
@ -28,16 +23,14 @@ go_library(
"//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/yaml:go_default_library",
"//vendor/k8s.io/apiserver/pkg/admission:go_default_library", "//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
"//vendor/k8s.io/apiserver/pkg/admission/configuration:go_default_library", "//vendor/k8s.io/apiserver/pkg/admission/configuration:go_default_library",
"//vendor/k8s.io/apiserver/pkg/admission/initializer:go_default_library", "//vendor/k8s.io/apiserver/pkg/admission/initializer:go_default_library",
"//vendor/k8s.io/apiserver/pkg/admission/plugin/webhook/config:go_default_library",
"//vendor/k8s.io/apiserver/pkg/admission/plugin/webhook/rules:go_default_library",
"//vendor/k8s.io/client-go/informers:go_default_library", "//vendor/k8s.io/client-go/informers:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/listers/core/v1:go_default_library", "//vendor/k8s.io/client-go/listers/core/v1:go_default_library",
"//vendor/k8s.io/client-go/rest:go_default_library",
"//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
"//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library",
], ],
) )
@ -45,33 +38,28 @@ go_test(
name = "go_default_test", name = "go_default_test",
srcs = [ srcs = [
"admission_test.go", "admission_test.go",
"authentication_test.go",
"certs_test.go", "certs_test.go",
"conversion_test.go", "conversion_test.go",
"rules_test.go",
"serviceresolver_test.go",
], ],
importpath = "k8s.io/apiserver/pkg/admission/plugin/webhook", importpath = "k8s.io/apiserver/pkg/admission/plugin/webhook/validating",
library = ":go_default_library", library = ":go_default_library",
deps = [ deps = [
"//vendor/k8s.io/api/admission/v1alpha1:go_default_library", "//vendor/k8s.io/api/admission/v1alpha1:go_default_library",
"//vendor/k8s.io/api/admissionregistration/v1alpha1:go_default_library", "//vendor/k8s.io/api/admissionregistration/v1alpha1:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/labels:go_default_library", "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/diff:go_default_library",
"//vendor/k8s.io/apiserver/pkg/admission:go_default_library", "//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
"//vendor/k8s.io/apiserver/pkg/admission/plugin/webhook/config:go_default_library",
"//vendor/k8s.io/apiserver/pkg/apis/example:go_default_library", "//vendor/k8s.io/apiserver/pkg/apis/example:go_default_library",
"//vendor/k8s.io/apiserver/pkg/apis/example/v1:go_default_library", "//vendor/k8s.io/apiserver/pkg/apis/example/v1:go_default_library",
"//vendor/k8s.io/apiserver/pkg/apis/example2/v1:go_default_library", "//vendor/k8s.io/apiserver/pkg/apis/example2/v1:go_default_library",
"//vendor/k8s.io/apiserver/pkg/authentication/user:go_default_library", "//vendor/k8s.io/apiserver/pkg/authentication/user:go_default_library",
"//vendor/k8s.io/client-go/rest:go_default_library", "//vendor/k8s.io/client-go/rest:go_default_library",
"//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library",
], ],
) )

View File

@ -14,22 +14,18 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
// Package webhook delegates admission checks to dynamically configured webhooks. // Package validating delegates admission checks to dynamically configured
package webhook // validating webhooks.
package validating
import ( import (
"context" "context"
"encoding/json"
"errors"
"fmt" "fmt"
"io" "io"
"net"
"net/url"
"sync" "sync"
"time" "time"
"github.com/golang/glog" "github.com/golang/glog"
lru "github.com/hashicorp/golang-lru"
admissionv1alpha1 "k8s.io/api/admission/v1alpha1" admissionv1alpha1 "k8s.io/api/admission/v1alpha1"
"k8s.io/api/admissionregistration/v1alpha1" "k8s.io/api/admissionregistration/v1alpha1"
@ -42,38 +38,21 @@ import (
"k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/runtime/serializer"
utilruntime "k8s.io/apimachinery/pkg/util/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apimachinery/pkg/util/yaml"
"k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/admission/configuration" "k8s.io/apiserver/pkg/admission/configuration"
genericadmissioninit "k8s.io/apiserver/pkg/admission/initializer" genericadmissioninit "k8s.io/apiserver/pkg/admission/initializer"
"k8s.io/apiserver/pkg/admission/plugin/webhook/config"
"k8s.io/apiserver/pkg/admission/plugin/webhook/rules"
"k8s.io/client-go/informers" "k8s.io/client-go/informers"
clientset "k8s.io/client-go/kubernetes" clientset "k8s.io/client-go/kubernetes"
corelisters "k8s.io/client-go/listers/core/v1" corelisters "k8s.io/client-go/listers/core/v1"
"k8s.io/client-go/rest"
) )
const ( const (
// Name of admission plug-in // Name of admission plug-in
PluginName = "GenericAdmissionWebhook" PluginName = "GenericAdmissionWebhook"
defaultCacheSize = 200
) )
var (
ErrNeedServiceOrURL = errors.New("webhook configuration must have either service or URL")
)
type ErrCallingWebhook struct {
WebhookName string
Reason error
}
func (e *ErrCallingWebhook) Error() string {
if e.Reason != nil {
return fmt.Sprintf("failed calling admission webhook %q: %v", e.WebhookName, e.Reason)
}
return fmt.Sprintf("failed calling admission webhook %q; no further details available", e.WebhookName)
}
// Register registers a plugin // Register registers a plugin
func Register(plugins *admission.Plugins) { func Register(plugins *admission.Plugins) {
plugins.Register(PluginName, func(configFile io.Reader) (admission.Interface, error) { plugins.Register(PluginName, func(configFile io.Reader) (admission.Interface, error) {
@ -94,26 +73,22 @@ type WebhookSource interface {
// NewGenericAdmissionWebhook returns a generic admission webhook plugin. // NewGenericAdmissionWebhook returns a generic admission webhook plugin.
func NewGenericAdmissionWebhook(configFile io.Reader) (*GenericAdmissionWebhook, error) { func NewGenericAdmissionWebhook(configFile io.Reader) (*GenericAdmissionWebhook, error) {
kubeconfigFile := "" kubeconfigFile, err := config.LoadConfig(configFile)
if configFile != nil {
// TODO: move this to a versioned configuration file format
var config AdmissionConfig
d := yaml.NewYAMLOrJSONDecoder(configFile, 4096)
err := d.Decode(&config)
if err != nil {
return nil, err
}
kubeconfigFile = config.KubeConfigFile
}
authInfoResolver, err := newDefaultAuthenticationInfoResolver(kubeconfigFile)
if err != nil { if err != nil {
return nil, err return nil, err
} }
cache, err := lru.New(defaultCacheSize) cm, err := config.NewClientManager()
if err != nil { if err != nil {
return nil, err return nil, err
} }
authInfoResolver, err := config.NewDefaultAuthenticationInfoResolver(kubeconfigFile)
if err != nil {
return nil, err
}
// Set defaults which may be overridden later.
cm.SetAuthenticationInfoResolver(authInfoResolver)
cm.SetServiceResolver(config.NewDefaultServiceResolver())
return &GenericAdmissionWebhook{ return &GenericAdmissionWebhook{
Handler: admission.NewHandler( Handler: admission.NewHandler(
@ -122,30 +97,19 @@ func NewGenericAdmissionWebhook(configFile io.Reader) (*GenericAdmissionWebhook,
admission.Delete, admission.Delete,
admission.Update, admission.Update,
), ),
authInfoResolver: authInfoResolver, clientManager: cm,
serviceResolver: defaultServiceResolver{},
cache: cache,
}, nil }, nil
} }
// GenericAdmissionWebhook is an implementation of admission.Interface. // GenericAdmissionWebhook is an implementation of admission.Interface.
type GenericAdmissionWebhook struct { type GenericAdmissionWebhook struct {
*admission.Handler *admission.Handler
hookSource WebhookSource hookSource WebhookSource
serviceResolver ServiceResolver namespaceLister corelisters.NamespaceLister
negotiatedSerializer runtime.NegotiatedSerializer client clientset.Interface
namespaceLister corelisters.NamespaceLister convertor runtime.ObjectConvertor
client clientset.Interface creator runtime.ObjectCreater
convertor runtime.ObjectConvertor clientManager config.ClientManager
creator runtime.ObjectCreater
authInfoResolver AuthenticationInfoResolver
cache *lru.Cache
}
// serviceResolver knows how to convert a service reference into an actual location.
type ServiceResolver interface {
ResolveEndpoint(namespace, name string) (*url.URL, error)
} }
var ( var (
@ -153,26 +117,22 @@ var (
) )
// TODO find a better way wire this, but keep this pull small for now. // TODO find a better way wire this, but keep this pull small for now.
func (a *GenericAdmissionWebhook) SetAuthenticationInfoResolverWrapper(wrapper AuthenticationInfoResolverWrapper) { func (a *GenericAdmissionWebhook) SetAuthenticationInfoResolverWrapper(wrapper config.AuthenticationInfoResolverWrapper) {
if wrapper != nil { a.clientManager.SetAuthenticationInfoResolverWrapper(wrapper)
a.authInfoResolver = wrapper(a.authInfoResolver)
}
} }
// SetServiceResolver sets a service resolver for the webhook admission plugin. // SetServiceResolver sets a service resolver for the webhook admission plugin.
// Passing a nil resolver does not have an effect, instead a default one will be used. // Passing a nil resolver does not have an effect, instead a default one will be used.
func (a *GenericAdmissionWebhook) SetServiceResolver(sr ServiceResolver) { func (a *GenericAdmissionWebhook) SetServiceResolver(sr config.ServiceResolver) {
if sr != nil { a.clientManager.SetServiceResolver(sr)
a.serviceResolver = sr
}
} }
// SetScheme sets a serializer(NegotiatedSerializer) which is derived from the scheme // SetScheme sets a serializer(NegotiatedSerializer) which is derived from the scheme
func (a *GenericAdmissionWebhook) SetScheme(scheme *runtime.Scheme) { func (a *GenericAdmissionWebhook) SetScheme(scheme *runtime.Scheme) {
if scheme != nil { if scheme != nil {
a.negotiatedSerializer = serializer.NegotiatedSerializerWrapper(runtime.SerializerInfo{ a.clientManager.SetNegotiatedSerializer(serializer.NegotiatedSerializerWrapper(runtime.SerializerInfo{
Serializer: serializer.NewCodecFactory(scheme).LegacyCodec(admissionv1alpha1.SchemeGroupVersion), Serializer: serializer.NewCodecFactory(scheme).LegacyCodec(admissionv1alpha1.SchemeGroupVersion),
}) }))
a.convertor = scheme a.convertor = scheme
a.creator = scheme a.creator = scheme
} }
@ -196,12 +156,12 @@ func (a *GenericAdmissionWebhook) ValidateInitialization() error {
if a.hookSource == nil { if a.hookSource == nil {
return fmt.Errorf("the GenericAdmissionWebhook admission plugin requires a Kubernetes client to be provided") return fmt.Errorf("the GenericAdmissionWebhook admission plugin requires a Kubernetes client to be provided")
} }
if a.negotiatedSerializer == nil {
return fmt.Errorf("the GenericAdmissionWebhook admission plugin requires a runtime.Scheme to be provided to derive a serializer")
}
if a.namespaceLister == nil { if a.namespaceLister == nil {
return fmt.Errorf("the GenericAdmissionWebhook admission plugin requires a namespaceLister") return fmt.Errorf("the GenericAdmissionWebhook admission plugin requires a namespaceLister")
} }
if err := a.clientManager.Validate(); err != nil {
return fmt.Errorf("the GenericAdmissionWebhook.clientManager is not properly setup: %v", err)
}
go a.hookSource.Run(wait.NeverStop) go a.hookSource.Run(wait.NeverStop)
return nil return nil
} }
@ -225,6 +185,7 @@ func (a *GenericAdmissionWebhook) loadConfiguration(attr admission.Attributes) (
return hookConfig, nil return hookConfig, nil
} }
// TODO: move this object to a common package
type versionedAttributes struct { type versionedAttributes struct {
admission.Attributes admission.Attributes
oldObject runtime.Object oldObject runtime.Object
@ -239,6 +200,7 @@ func (v versionedAttributes) GetOldObject() runtime.Object {
return v.oldObject return v.oldObject
} }
// TODO: move this method to a common package
func (a *GenericAdmissionWebhook) convertToGVK(obj runtime.Object, gvk schema.GroupVersionKind) (runtime.Object, error) { func (a *GenericAdmissionWebhook) convertToGVK(obj runtime.Object, gvk schema.GroupVersionKind) (runtime.Object, error) {
// Unlike other resources, custom resources do not have internal version, so // Unlike other resources, custom resources do not have internal version, so
// if obj is a custom resource, it should not need conversion. // if obj is a custom resource, it should not need conversion.
@ -315,7 +277,7 @@ func (a *GenericAdmissionWebhook) Admit(attr admission.Attributes) error {
} }
ignoreClientCallFailures := hook.FailurePolicy != nil && *hook.FailurePolicy == v1alpha1.Ignore ignoreClientCallFailures := hook.FailurePolicy != nil && *hook.FailurePolicy == v1alpha1.Ignore
if callErr, ok := err.(*ErrCallingWebhook); ok { if callErr, ok := err.(*config.ErrCallingWebhook); ok {
if ignoreClientCallFailures { if ignoreClientCallFailures {
glog.Warningf("Failed calling webhook, failing open %v: %v", hook.Name, callErr) glog.Warningf("Failed calling webhook, failing open %v: %v", hook.Name, callErr)
utilruntime.HandleError(callErr) utilruntime.HandleError(callErr)
@ -351,6 +313,7 @@ func (a *GenericAdmissionWebhook) Admit(attr admission.Attributes) error {
return errs[0] return errs[0]
} }
// TODO: move this method to a common package
func (a *GenericAdmissionWebhook) getNamespaceLabels(attr admission.Attributes) (map[string]string, error) { func (a *GenericAdmissionWebhook) getNamespaceLabels(attr admission.Attributes) (map[string]string, error) {
// If the request itself is creating or updating a namespace, then get the // If the request itself is creating or updating a namespace, then get the
// labels from attr.Object, because namespaceLister doesn't have the latest // labels from attr.Object, because namespaceLister doesn't have the latest
@ -385,6 +348,7 @@ func (a *GenericAdmissionWebhook) getNamespaceLabels(attr admission.Attributes)
return namespace.Labels, nil return namespace.Labels, nil
} }
// TODO: move this method to a common package
// whether the request is exempted by the webhook because of the // whether the request is exempted by the webhook because of the
// namespaceSelector of the webhook. // namespaceSelector of the webhook.
func (a *GenericAdmissionWebhook) exemptedByNamespaceSelector(h *v1alpha1.Webhook, attr admission.Attributes) (bool, *apierrors.StatusError) { func (a *GenericAdmissionWebhook) exemptedByNamespaceSelector(h *v1alpha1.Webhook, attr admission.Attributes) (bool, *apierrors.StatusError) {
@ -417,10 +381,11 @@ func (a *GenericAdmissionWebhook) exemptedByNamespaceSelector(h *v1alpha1.Webhoo
return !selector.Matches(labels.Set(namespaceLabels)), nil return !selector.Matches(labels.Set(namespaceLabels)), nil
} }
// TODO: move this method to a common package
func (a *GenericAdmissionWebhook) shouldCallHook(h *v1alpha1.Webhook, attr admission.Attributes) (bool, *apierrors.StatusError) { func (a *GenericAdmissionWebhook) shouldCallHook(h *v1alpha1.Webhook, attr admission.Attributes) (bool, *apierrors.StatusError) {
var matches bool var matches bool
for _, r := range h.Rules { for _, r := range h.Rules {
m := RuleMatcher{Rule: r, Attr: attr} m := rules.Matcher{Rule: r, Attr: attr}
if m.Matches() { if m.Matches() {
matches = true matches = true
break break
@ -440,13 +405,13 @@ func (a *GenericAdmissionWebhook) shouldCallHook(h *v1alpha1.Webhook, attr admis
func (a *GenericAdmissionWebhook) callHook(ctx context.Context, h *v1alpha1.Webhook, attr admission.Attributes) error { func (a *GenericAdmissionWebhook) callHook(ctx context.Context, h *v1alpha1.Webhook, attr admission.Attributes) error {
// Make the webhook request // Make the webhook request
request := createAdmissionReview(attr) request := createAdmissionReview(attr)
client, err := a.hookClient(h) client, err := a.clientManager.HookClient(h)
if err != nil { if err != nil {
return &ErrCallingWebhook{WebhookName: h.Name, Reason: err} return &config.ErrCallingWebhook{WebhookName: h.Name, Reason: err}
} }
response := &admissionv1alpha1.AdmissionReview{} response := &admissionv1alpha1.AdmissionReview{}
if err := client.Post().Context(ctx).Body(&request).Do().Into(response); err != nil { if err := client.Post().Context(ctx).Body(&request).Do().Into(response); err != nil {
return &ErrCallingWebhook{WebhookName: h.Name, Reason: err} return &config.ErrCallingWebhook{WebhookName: h.Name, Reason: err}
} }
if response.Status.Allowed { if response.Status.Allowed {
@ -456,6 +421,7 @@ func (a *GenericAdmissionWebhook) callHook(ctx context.Context, h *v1alpha1.Webh
return toStatusErr(h.Name, response.Status.Result) return toStatusErr(h.Name, response.Status.Result)
} }
// TODO: move this function to a common package
// toStatusErr returns a StatusError with information about the webhook controller // toStatusErr returns a StatusError with information about the webhook controller
func toStatusErr(name string, result *metav1.Status) *apierrors.StatusError { func toStatusErr(name string, result *metav1.Status) *apierrors.StatusError {
deniedBy := fmt.Sprintf("admission webhook %q denied the request", name) deniedBy := fmt.Sprintf("admission webhook %q denied the request", name)
@ -478,76 +444,3 @@ func toStatusErr(name string, result *metav1.Status) *apierrors.StatusError {
ErrStatus: *result, ErrStatus: *result,
} }
} }
func (a *GenericAdmissionWebhook) hookClient(h *v1alpha1.Webhook) (*rest.RESTClient, error) {
cacheKey, err := json.Marshal(h.ClientConfig)
if err != nil {
return nil, err
}
if client, ok := a.cache.Get(string(cacheKey)); ok {
return client.(*rest.RESTClient), nil
}
complete := func(cfg *rest.Config) (*rest.RESTClient, error) {
cfg.TLSClientConfig.CAData = h.ClientConfig.CABundle
cfg.ContentConfig.NegotiatedSerializer = a.negotiatedSerializer
cfg.ContentConfig.ContentType = runtime.ContentTypeJSON
client, err := rest.UnversionedRESTClientFor(cfg)
if err == nil {
a.cache.Add(string(cacheKey), client)
}
return client, err
}
if svc := h.ClientConfig.Service; svc != nil {
serverName := svc.Name + "." + svc.Namespace + ".svc"
restConfig, err := a.authInfoResolver.ClientConfigFor(serverName)
if err != nil {
return nil, err
}
cfg := rest.CopyConfig(restConfig)
host := serverName + ":443"
cfg.Host = "https://" + host
if svc.Path != nil {
cfg.APIPath = *svc.Path
}
cfg.TLSClientConfig.ServerName = serverName
delegateDialer := cfg.Dial
if delegateDialer == nil {
delegateDialer = net.Dial
}
cfg.Dial = func(network, addr string) (net.Conn, error) {
if addr == host {
u, err := a.serviceResolver.ResolveEndpoint(svc.Namespace, svc.Name)
if err != nil {
return nil, err
}
addr = u.Host
}
return delegateDialer(network, addr)
}
return complete(cfg)
}
if h.ClientConfig.URL == nil {
return nil, &ErrCallingWebhook{WebhookName: h.Name, Reason: ErrNeedServiceOrURL}
}
u, err := url.Parse(*h.ClientConfig.URL)
if err != nil {
return nil, &ErrCallingWebhook{WebhookName: h.Name, Reason: fmt.Errorf("Unparsable URL: %v", err)}
}
restConfig, err := a.authInfoResolver.ClientConfigFor(u.Host)
if err != nil {
return nil, err
}
cfg := rest.CopyConfig(restConfig)
cfg.Host = u.Host
cfg.APIPath = u.Path
return complete(cfg)
}

View File

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
package webhook package validating
import ( import (
"crypto/tls" "crypto/tls"
@ -39,6 +39,7 @@ import (
"k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/admission/plugin/webhook/config"
"k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/authentication/user"
"k8s.io/client-go/rest" "k8s.io/client-go/rest"
) )
@ -133,9 +134,17 @@ func TestAdmit(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
wh.authInfoResolver = newFakeAuthenticationInfoResolver() cm, err := config.NewClientManager()
wh.serviceResolver = fakeServiceResolver{base: *serverURL} if err != nil {
t.Fatalf("cannot create client manager: %v", err)
}
cm.SetAuthenticationInfoResolver(newFakeAuthenticationInfoResolver(new(int32)))
cm.SetServiceResolver(fakeServiceResolver{base: *serverURL})
wh.clientManager = cm
wh.SetScheme(scheme) wh.SetScheme(scheme)
if err = wh.clientManager.Validate(); err != nil {
t.Fatal(err)
}
namespace := "webhook-test" namespace := "webhook-test"
wh.namespaceLister = fakeNamespaceLister{map[string]*corev1.Namespace{ wh.namespaceLister = fakeNamespaceLister{map[string]*corev1.Namespace{
namespace: { namespace: {
@ -397,8 +406,12 @@ func TestAdmitCachedClient(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
wh.authInfoResolver = newFakeAuthenticationInfoResolver() cm, err := config.NewClientManager()
wh.serviceResolver = fakeServiceResolver{base: *serverURL} if err != nil {
t.Fatalf("cannot create client manager: %v", err)
}
cm.SetServiceResolver(fakeServiceResolver{base: *serverURL})
wh.clientManager = cm
wh.SetScheme(scheme) wh.SetScheme(scheme)
namespace := "webhook-test" namespace := "webhook-test"
wh.namespaceLister = fakeNamespaceLister{map[string]*corev1.Namespace{ wh.namespaceLister = fakeNamespaceLister{map[string]*corev1.Namespace{
@ -519,18 +532,23 @@ func TestAdmitCachedClient(t *testing.T) {
for _, testcase := range cases { for _, testcase := range cases {
t.Run(testcase.name, func(t *testing.T) { t.Run(testcase.name, func(t *testing.T) {
wh.hookSource = &testcase.hookSource wh.hookSource = &testcase.hookSource
wh.authInfoResolver.(*fakeAuthenticationInfoResolver).cachedCount = 0 authInfoResolverCount := new(int32)
r := newFakeAuthenticationInfoResolver(authInfoResolverCount)
wh.clientManager.SetAuthenticationInfoResolver(r)
if err = wh.clientManager.Validate(); err != nil {
t.Fatal(err)
}
err = wh.Admit(admission.NewAttributesRecord(&object, &oldObject, kind, namespace, testcase.name, resource, subResource, operation, &userInfo)) err = wh.Admit(admission.NewAttributesRecord(&object, &oldObject, kind, namespace, testcase.name, resource, subResource, operation, &userInfo))
if testcase.expectAllow != (err == nil) { if testcase.expectAllow != (err == nil) {
t.Errorf("expected allowed=%v, but got err=%v", testcase.expectAllow, err) t.Errorf("expected allowed=%v, but got err=%v", testcase.expectAllow, err)
} }
if testcase.expectCache && wh.authInfoResolver.(*fakeAuthenticationInfoResolver).cachedCount != 1 { if testcase.expectCache && *authInfoResolverCount != 1 {
t.Errorf("expected cacheclient, but got none") t.Errorf("expected cacheclient, but got none")
} }
if !testcase.expectCache && wh.authInfoResolver.(*fakeAuthenticationInfoResolver).cachedCount != 0 { if !testcase.expectCache && *authInfoResolverCount != 0 {
t.Errorf("expected not cacheclient, but got cache") t.Errorf("expected not cacheclient, but got cache")
} }
}) })
@ -597,7 +615,7 @@ func webhookHandler(w http.ResponseWriter, r *http.Request) {
} }
} }
func newFakeAuthenticationInfoResolver() *fakeAuthenticationInfoResolver { func newFakeAuthenticationInfoResolver(count *int32) *fakeAuthenticationInfoResolver {
return &fakeAuthenticationInfoResolver{ return &fakeAuthenticationInfoResolver{
restConfig: &rest.Config{ restConfig: &rest.Config{
TLSClientConfig: rest.TLSClientConfig{ TLSClientConfig: rest.TLSClientConfig{
@ -606,16 +624,17 @@ func newFakeAuthenticationInfoResolver() *fakeAuthenticationInfoResolver {
KeyData: clientKey, KeyData: clientKey,
}, },
}, },
cachedCount: count,
} }
} }
type fakeAuthenticationInfoResolver struct { type fakeAuthenticationInfoResolver struct {
restConfig *rest.Config restConfig *rest.Config
cachedCount int32 cachedCount *int32
} }
func (c *fakeAuthenticationInfoResolver) ClientConfigFor(server string) (*rest.Config, error) { func (c *fakeAuthenticationInfoResolver) ClientConfigFor(server string) (*rest.Config, error) {
atomic.AddInt32(&c.cachedCount, 1) atomic.AddInt32(c.cachedCount, 1)
return c.restConfig, nil return c.restConfig, nil
} }

View File

@ -15,7 +15,7 @@ limitations under the License.
*/ */
// Package webhook delegates admission checks to dynamically configured webhooks. // Package webhook delegates admission checks to dynamically configured webhooks.
package webhook package validating
import ( import (
admissionv1alpha1 "k8s.io/api/admission/v1alpha1" admissionv1alpha1 "k8s.io/api/admission/v1alpha1"
@ -25,6 +25,7 @@ import (
"k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/admission"
) )
// TODO: move this function to a common package
// createAdmissionReview creates an AdmissionReview for the provided admission.Attributes // createAdmissionReview creates an AdmissionReview for the provided admission.Attributes
func createAdmissionReview(attr admission.Attributes) admissionv1alpha1.AdmissionReview { func createAdmissionReview(attr admission.Attributes) admissionv1alpha1.AdmissionReview {
gvk := attr.GetKind() gvk := attr.GetKind()

View File

@ -17,7 +17,7 @@ limitations under the License.
// This file was generated using openssl by the gencerts.sh script // This file was generated using openssl by the gencerts.sh script
// and holds raw certificates for the webhook tests. // and holds raw certificates for the webhook tests.
package webhook package validating
var caKey = []byte(`-----BEGIN RSA PRIVATE KEY----- var caKey = []byte(`-----BEGIN RSA PRIVATE KEY-----
MIIEpAIBAAKCAQEAt8E1XykA4860Tj7mypnsSU+hW0taUEvz26a5rgFSrwgKe1g+ MIIEpAIBAAKCAQEAt8E1XykA4860Tj7mypnsSU+hW0taUEvz26a5rgFSrwgKe1g+

View File

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
package webhook package validating
import ( import (
"reflect" "reflect"

View File

@ -14,5 +14,5 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
// Package webhook checks a webhook for configured operation admission // Package validating checks a non-mutating webhook for configured operation admission
package webhook // import "k8s.io/apiserver/pkg/admission/plugin/webhook" package validating // import "k8s.io/apiserver/pkg/admission/plugin/webhook/validating"

View File

@ -95,7 +95,7 @@ EOF
echo "// This file was generated using openssl by the gencerts.sh script" >> $outfile echo "// This file was generated using openssl by the gencerts.sh script" >> $outfile
echo "// and holds raw certificates for the webhook tests." >> $outfile echo "// and holds raw certificates for the webhook tests." >> $outfile
echo "" >> $outfile echo "" >> $outfile
echo "package webhook" >> $outfile echo "package validating" >> $outfile
for file in caKey caCert badCAKey badCACert serverKey serverCert clientKey clientCert; do for file in caKey caCert badCAKey badCACert serverKey serverCert clientKey clientCert; do
data=$(cat ${file}.pem) data=$(cat ${file}.pem)
echo "" >> $outfile echo "" >> $outfile

View File

@ -88,7 +88,7 @@ go_library(
"//vendor/k8s.io/apiserver/pkg/admission:go_default_library", "//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
"//vendor/k8s.io/apiserver/pkg/admission/plugin/initialization:go_default_library", "//vendor/k8s.io/apiserver/pkg/admission/plugin/initialization:go_default_library",
"//vendor/k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle:go_default_library", "//vendor/k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle:go_default_library",
"//vendor/k8s.io/apiserver/pkg/admission/plugin/webhook:go_default_library", "//vendor/k8s.io/apiserver/pkg/admission/plugin/webhook/validating:go_default_library",
"//vendor/k8s.io/apiserver/pkg/apis/apiserver/install:go_default_library", "//vendor/k8s.io/apiserver/pkg/apis/apiserver/install:go_default_library",
"//vendor/k8s.io/apiserver/pkg/audit:go_default_library", "//vendor/k8s.io/apiserver/pkg/audit:go_default_library",
"//vendor/k8s.io/apiserver/pkg/audit/policy:go_default_library", "//vendor/k8s.io/apiserver/pkg/audit/policy:go_default_library",

View File

@ -32,7 +32,7 @@ go_library(
"//vendor/k8s.io/apiserver/pkg/admission/initializer:go_default_library", "//vendor/k8s.io/apiserver/pkg/admission/initializer:go_default_library",
"//vendor/k8s.io/apiserver/pkg/admission/plugin/initialization:go_default_library", "//vendor/k8s.io/apiserver/pkg/admission/plugin/initialization:go_default_library",
"//vendor/k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle:go_default_library", "//vendor/k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle:go_default_library",
"//vendor/k8s.io/apiserver/pkg/admission/plugin/webhook:go_default_library", "//vendor/k8s.io/apiserver/pkg/admission/plugin/webhook/validating:go_default_library",
"//vendor/k8s.io/apiserver/pkg/apis/audit/v1beta1:go_default_library", "//vendor/k8s.io/apiserver/pkg/apis/audit/v1beta1:go_default_library",
"//vendor/k8s.io/apiserver/pkg/audit:go_default_library", "//vendor/k8s.io/apiserver/pkg/audit:go_default_library",
"//vendor/k8s.io/apiserver/pkg/audit/policy:go_default_library", "//vendor/k8s.io/apiserver/pkg/audit/policy:go_default_library",

View File

@ -26,7 +26,7 @@ import (
"k8s.io/apiserver/pkg/admission/initializer" "k8s.io/apiserver/pkg/admission/initializer"
"k8s.io/apiserver/pkg/admission/plugin/initialization" "k8s.io/apiserver/pkg/admission/plugin/initialization"
"k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle" "k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle"
"k8s.io/apiserver/pkg/admission/plugin/webhook" validatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/validating"
"k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server"
"k8s.io/client-go/informers" "k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes"
@ -56,8 +56,8 @@ func NewAdmissionOptions() *AdmissionOptions {
options := &AdmissionOptions{ options := &AdmissionOptions{
Plugins: &admission.Plugins{}, Plugins: &admission.Plugins{},
PluginNames: []string{}, PluginNames: []string{},
RecommendedPluginOrder: []string{lifecycle.PluginName, initialization.PluginName, webhook.PluginName}, RecommendedPluginOrder: []string{lifecycle.PluginName, initialization.PluginName, validatingwebhook.PluginName},
DefaultOffPlugins: []string{initialization.PluginName, webhook.PluginName}, DefaultOffPlugins: []string{initialization.PluginName, validatingwebhook.PluginName},
} }
server.RegisterAllAdmissionPlugins(options.Plugins) server.RegisterAllAdmissionPlugins(options.Plugins)
return options return options

View File

@ -21,12 +21,12 @@ import (
"k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/admission/plugin/initialization" "k8s.io/apiserver/pkg/admission/plugin/initialization"
"k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle" "k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle"
"k8s.io/apiserver/pkg/admission/plugin/webhook" validatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/validating"
) )
// RegisterAllAdmissionPlugins registers all admission plugins // RegisterAllAdmissionPlugins registers all admission plugins
func RegisterAllAdmissionPlugins(plugins *admission.Plugins) { func RegisterAllAdmissionPlugins(plugins *admission.Plugins) {
lifecycle.Register(plugins) lifecycle.Register(plugins)
initialization.Register(plugins) initialization.Register(plugins)
webhook.Register(plugins) validatingwebhook.Register(plugins)
} }

View File

@ -843,7 +843,15 @@
"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
}, },
{ {
"ImportPath": "k8s.io/apiserver/pkg/admission/plugin/webhook", "ImportPath": "k8s.io/apiserver/pkg/admission/plugin/webhook/config",
"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
},
{
"ImportPath": "k8s.io/apiserver/pkg/admission/plugin/webhook/rules",
"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
},
{
"ImportPath": "k8s.io/apiserver/pkg/admission/plugin/webhook/validating",
"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
}, },
{ {

View File

@ -839,7 +839,15 @@
"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
}, },
{ {
"ImportPath": "k8s.io/apiserver/pkg/admission/plugin/webhook", "ImportPath": "k8s.io/apiserver/pkg/admission/plugin/webhook/config",
"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
},
{
"ImportPath": "k8s.io/apiserver/pkg/admission/plugin/webhook/rules",
"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
},
{
"ImportPath": "k8s.io/apiserver/pkg/admission/plugin/webhook/validating",
"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
}, },
{ {