Merge pull request #82252 from liggitt/webhook-client-auth-test

Match webhook client auth with ports consistently
This commit is contained in:
Kubernetes Prow Robot 2019-09-03 16:01:51 -07:00 committed by GitHub
commit b47349ad20
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 443 additions and 30 deletions

View File

@ -52,12 +52,12 @@ type authenticationInfoResolver struct {
cacheMisses *int32
}
func (a *authenticationInfoResolver) ClientConfigFor(server string) (*rest.Config, error) {
func (a *authenticationInfoResolver) ClientConfigFor(hostPort string) (*rest.Config, error) {
atomic.AddInt32(a.cacheMisses, 1)
return a.restConfig, nil
}
func (a *authenticationInfoResolver) ClientConfigForService(serviceName, serviceNamespace string) (*rest.Config, error) {
func (a *authenticationInfoResolver) ClientConfigForService(serviceName, serviceNamespace string, servicePort int) (*rest.Config, error) {
atomic.AddInt32(a.cacheMisses, 1)
return a.restConfig, nil
}

View File

@ -19,7 +19,9 @@ package webhook
import (
"fmt"
"io/ioutil"
"net"
"net/http"
"strconv"
"strings"
"time"
@ -40,17 +42,17 @@ func NewDefaultAuthenticationInfoResolverWrapper(
webhookAuthResolverWrapper := func(delegate AuthenticationInfoResolver) AuthenticationInfoResolver {
return &AuthenticationInfoResolverDelegator{
ClientConfigForFunc: func(server string) (*rest.Config, error) {
if server == "kubernetes.default.svc" {
ClientConfigForFunc: func(hostPort string) (*rest.Config, error) {
if hostPort == "kubernetes.default.svc:443" {
return kubeapiserverClientConfig, nil
}
return delegate.ClientConfigFor(server)
return delegate.ClientConfigFor(hostPort)
},
ClientConfigForServiceFunc: func(serviceName, serviceNamespace string) (*rest.Config, error) {
if serviceName == "kubernetes" && serviceNamespace == corev1.NamespaceDefault {
ClientConfigForServiceFunc: func(serviceName, serviceNamespace string, servicePort int) (*rest.Config, error) {
if serviceName == "kubernetes" && serviceNamespace == corev1.NamespaceDefault && servicePort == 443 {
return kubeapiserverClientConfig, nil
}
ret, err := delegate.ClientConfigForService(serviceName, serviceNamespace)
ret, err := delegate.ClientConfigForService(serviceName, serviceNamespace, servicePort)
if err != nil {
return nil, err
}
@ -67,27 +69,27 @@ func NewDefaultAuthenticationInfoResolverWrapper(
// AuthenticationInfoResolver builds rest.Config base on the server or service
// name and service namespace.
type AuthenticationInfoResolver interface {
// ClientConfigFor builds rest.Config based on the server.
ClientConfigFor(server string) (*rest.Config, error)
// ClientConfigFor builds rest.Config based on the hostPort.
ClientConfigFor(hostPort string) (*rest.Config, error)
// ClientConfigForService builds rest.Config based on the serviceName and
// serviceNamespace.
ClientConfigForService(serviceName, serviceNamespace string) (*rest.Config, error)
ClientConfigForService(serviceName, serviceNamespace string, servicePort int) (*rest.Config, error)
}
// AuthenticationInfoResolverDelegator implements AuthenticationInfoResolver.
type AuthenticationInfoResolverDelegator struct {
ClientConfigForFunc func(server string) (*rest.Config, error)
ClientConfigForServiceFunc func(serviceName, serviceNamespace string) (*rest.Config, error)
ClientConfigForFunc func(hostPort string) (*rest.Config, error)
ClientConfigForServiceFunc func(serviceName, serviceNamespace string, servicePort int) (*rest.Config, error)
}
// ClientConfigFor returns client config for given server.
func (a *AuthenticationInfoResolverDelegator) ClientConfigFor(server string) (*rest.Config, error) {
return a.ClientConfigForFunc(server)
// ClientConfigFor returns client config for given hostPort.
func (a *AuthenticationInfoResolverDelegator) ClientConfigFor(hostPort string) (*rest.Config, error) {
return a.ClientConfigForFunc(hostPort)
}
// ClientConfigForService returns client config for given service.
func (a *AuthenticationInfoResolverDelegator) ClientConfigForService(serviceName, serviceNamespace string) (*rest.Config, error) {
return a.ClientConfigForServiceFunc(serviceName, serviceNamespace)
func (a *AuthenticationInfoResolverDelegator) ClientConfigForService(serviceName, serviceNamespace string, servicePort int) (*rest.Config, error) {
return a.ClientConfigForServiceFunc(serviceName, serviceNamespace, servicePort)
}
type defaultAuthenticationInfoResolver struct {
@ -113,12 +115,12 @@ func NewDefaultAuthenticationInfoResolver(kubeconfigFile string) (Authentication
return &defaultAuthenticationInfoResolver{kubeconfig: clientConfig}, nil
}
func (c *defaultAuthenticationInfoResolver) ClientConfigFor(server string) (*rest.Config, error) {
return c.clientConfig(server)
func (c *defaultAuthenticationInfoResolver) ClientConfigFor(hostPort string) (*rest.Config, error) {
return c.clientConfig(hostPort)
}
func (c *defaultAuthenticationInfoResolver) ClientConfigForService(serviceName, serviceNamespace string) (*rest.Config, error) {
return c.clientConfig(serviceName + "." + serviceNamespace + ".svc")
func (c *defaultAuthenticationInfoResolver) ClientConfigForService(serviceName, serviceNamespace string, servicePort int) (*rest.Config, error) {
return c.clientConfig(net.JoinHostPort(serviceName+"."+serviceNamespace+".svc", strconv.Itoa(servicePort)))
}
func (c *defaultAuthenticationInfoResolver) clientConfig(target string) (*rest.Config, error) {
@ -136,8 +138,25 @@ func (c *defaultAuthenticationInfoResolver) clientConfig(target string) (*rest.C
}
}
// If target included the default https port (443), search again without the port
if target, port, err := net.SplitHostPort(target); err == nil && port == "443" {
// exact match without port
if authConfig, ok := c.kubeconfig.AuthInfos[target]; ok {
return restConfigFromKubeconfig(authConfig)
}
// star prefixed match without port
serverSteps := strings.Split(target, ".")
for i := 1; i < len(serverSteps); i++ {
nickName := "*." + strings.Join(serverSteps[i:], ".")
if authConfig, ok := c.kubeconfig.AuthInfos[nickName]; ok {
return restConfigFromKubeconfig(authConfig)
}
}
}
// if we're trying to hit the kube-apiserver and there wasn't an explicit config, use the in-cluster config
if target == "kubernetes.default.svc" {
if target == "kubernetes.default.svc:443" {
// if we can find an in-cluster-config use that. If we can't, fall through.
inClusterConfig, err := rest.InClusterConfig()
if err == nil {

View File

@ -109,6 +109,90 @@ func TestAuthenticationDetection(t *testing.T) {
},
expected: rest.Config{BearerToken: "first"},
},
{
name: "exact match with default https port",
serverName: "one.two.three.com:443",
kubeconfig: clientcmdapi.Config{
AuthInfos: map[string]*clientcmdapi.AuthInfo{
"one.two.three.com:443": {Token: "exact"},
"*.two.three.com": {Token: "first"},
"*.three.com": {Token: "second"},
"*.com": {Token: "third"},
"*": {Token: "fallback"},
},
},
expected: rest.Config{BearerToken: "exact"},
},
{
name: "wildcard match with default https port",
serverName: "one.two.three.com:443",
kubeconfig: clientcmdapi.Config{
AuthInfos: map[string]*clientcmdapi.AuthInfo{
"*.two.three.com:443": {Token: "first-with-port"},
"*.two.three.com": {Token: "first"},
"*.three.com": {Token: "second"},
"*.com": {Token: "third"},
"*": {Token: "fallback"},
},
},
expected: rest.Config{BearerToken: "first-with-port"},
},
{
name: "wildcard match without default https port",
serverName: "one.two.three.com:443",
kubeconfig: clientcmdapi.Config{
AuthInfos: map[string]*clientcmdapi.AuthInfo{
"*.two.three.com": {Token: "first"},
"*.three.com": {Token: "second"},
"*.com": {Token: "third"},
"*": {Token: "fallback"},
},
},
expected: rest.Config{BearerToken: "first"},
},
{
name: "exact match with non-default https port",
serverName: "one.two.three.com:8443",
kubeconfig: clientcmdapi.Config{
AuthInfos: map[string]*clientcmdapi.AuthInfo{
"one.two.three.com:8443": {Token: "exact"},
"*.two.three.com": {Token: "first"},
"*.three.com": {Token: "second"},
"*.com": {Token: "third"},
"*": {Token: "fallback"},
},
},
expected: rest.Config{BearerToken: "exact"},
},
{
name: "wildcard match with non-default https port",
serverName: "one.two.three.com:8443",
kubeconfig: clientcmdapi.Config{
AuthInfos: map[string]*clientcmdapi.AuthInfo{
"*.two.three.com:8443": {Token: "first-with-port"},
"one.two.three.com": {Token: "first-without-port"},
"*.two.three.com": {Token: "first"},
"*.three.com": {Token: "second"},
"*.com": {Token: "third"},
"*": {Token: "fallback"},
},
},
expected: rest.Config{BearerToken: "first-with-port"},
},
{
name: "wildcard match without non-default https port",
serverName: "one.two.three.com:8443",
kubeconfig: clientcmdapi.Config{
AuthInfos: map[string]*clientcmdapi.AuthInfo{
"one.two.three.com": {Token: "first-without-port"},
"*.two.three.com": {Token: "first"},
"*.three.com": {Token: "second"},
"*.com": {Token: "third"},
"*": {Token: "fallback"},
},
},
expected: rest.Config{BearerToken: "fallback"},
},
}
for _, tc := range tests {

View File

@ -23,6 +23,7 @@ import (
"fmt"
"net"
"net/url"
"strconv"
"github.com/hashicorp/golang-lru"
"k8s.io/apimachinery/pkg/runtime"
@ -151,13 +152,20 @@ func (cm *ClientManager) HookClient(cc ClientConfig) (*rest.RESTClient, error) {
}
if cc.Service != nil {
restConfig, err := cm.authInfoResolver.ClientConfigForService(cc.Service.Name, cc.Service.Namespace)
port := cc.Service.Port
if port == 0 {
// Default to port 443 if no service port is specified
port = 443
}
restConfig, err := cm.authInfoResolver.ClientConfigForService(cc.Service.Name, cc.Service.Namespace, int(port))
if err != nil {
return nil, err
}
cfg := rest.CopyConfig(restConfig)
serverName := cc.Service.Name + "." + cc.Service.Namespace + ".svc"
host := serverName + ":443"
host := net.JoinHostPort(serverName, strconv.Itoa(int(port)))
cfg.Host = "https://" + host
cfg.APIPath = cc.Service.Path
// Set the server name if not already set
@ -172,10 +180,6 @@ func (cm *ClientManager) HookClient(cc ClientConfig) (*rest.RESTClient, error) {
}
cfg.Dial = func(ctx context.Context, network, addr string) (net.Conn, error) {
if addr == host {
port := cc.Service.Port
if port == 0 {
port = 443
}
u, err := cm.serviceResolver.ResolveEndpoint(cc.Service.Namespace, cc.Service.Name, port)
if err != nil {
return nil, err
@ -197,7 +201,13 @@ func (cm *ClientManager) HookClient(cc ClientConfig) (*rest.RESTClient, error) {
return nil, &ErrCallingWebhook{WebhookName: cc.Name, Reason: fmt.Errorf("Unparsable URL: %v", err)}
}
restConfig, err := cm.authInfoResolver.ClientConfigFor(u.Host)
hostPort := u.Host
if len(u.Port()) == 0 {
// Default to port 443 if no port is specified
hostPort = net.JoinHostPort(hostPort, "443")
}
restConfig, err := cm.authInfoResolver.ClientConfigFor(hostPort)
if err != nil {
return nil, err
}

View File

@ -5,6 +5,7 @@ go_test(
srcs = [
"admission_test.go",
"broken_webhook_test.go",
"client_auth_test.go",
"load_balance_test.go",
"main_test.go",
"reinvocation_test.go",

View File

@ -0,0 +1,299 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package admissionwebhook
import (
"crypto/tls"
"crypto/x509"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"net/url"
"os"
"sync"
"testing"
"time"
"k8s.io/api/admission/v1beta1"
admissionv1beta1 "k8s.io/api/admissionregistration/v1beta1"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
"k8s.io/kubernetes/test/integration/framework"
)
const (
testClientAuthClientUsername = "webhook-client-auth-integration-client"
)
// TestWebhookClientAuthWithAggregatorRouting ensures client auth is used for requests to URL backends
func TestWebhookClientAuthWithAggregatorRouting(t *testing.T) {
testWebhookClientAuth(t, true)
}
// TestWebhookClientAuthWithoutAggregatorRouting ensures client auth is used for requests to URL backends
func TestWebhookClientAuthWithoutAggregatorRouting(t *testing.T) {
testWebhookClientAuth(t, false)
}
func testWebhookClientAuth(t *testing.T, enableAggregatorRouting bool) {
roots := x509.NewCertPool()
if !roots.AppendCertsFromPEM(localhostCert) {
t.Fatal("Failed to append Cert from PEM")
}
cert, err := tls.X509KeyPair(localhostCert, localhostKey)
if err != nil {
t.Fatalf("Failed to build cert with error: %+v", err)
}
recorder := &clientAuthRecorder{}
webhookServer := httptest.NewUnstartedServer(newClientAuthWebhookHandler(t, recorder))
webhookServer.TLS = &tls.Config{
RootCAs: roots,
Certificates: []tls.Certificate{cert},
}
webhookServer.StartTLS()
defer webhookServer.Close()
webhookServerURL, err := url.Parse(webhookServer.URL)
if err != nil {
t.Fatal(err)
}
kubeConfigFile, err := ioutil.TempFile("", "admission-config.yaml")
if err != nil {
t.Fatal(err)
}
defer os.Remove(kubeConfigFile.Name())
if err := ioutil.WriteFile(kubeConfigFile.Name(), []byte(`
apiVersion: v1
kind: Config
users:
- name: "`+webhookServerURL.Host+`"
user:
token: "localhost-match-with-port"
- name: "`+webhookServerURL.Hostname()+`"
user:
token: "localhost-match-without-port"
- name: "*.localhost"
user:
token: "localhost-prefix"
- name: "*"
user:
token: "fallback"
`), os.FileMode(0755)); err != nil {
t.Fatal(err)
}
admissionConfigFile, err := ioutil.TempFile("", "admission-config.yaml")
if err != nil {
t.Fatal(err)
}
defer os.Remove(admissionConfigFile.Name())
if err := ioutil.WriteFile(admissionConfigFile.Name(), []byte(`
apiVersion: apiserver.k8s.io/v1alpha1
kind: AdmissionConfiguration
plugins:
- name: ValidatingAdmissionWebhook
configuration:
apiVersion: apiserver.config.k8s.io/v1alpha1
kind: WebhookAdmission
kubeConfigFile: "`+kubeConfigFile.Name()+`"
- name: MutatingAdmissionWebhook
configuration:
apiVersion: apiserver.config.k8s.io/v1alpha1
kind: WebhookAdmission
kubeConfigFile: "`+kubeConfigFile.Name()+`"
`), os.FileMode(0755)); err != nil {
t.Fatal(err)
}
s := kubeapiservertesting.StartTestServerOrDie(t, kubeapiservertesting.NewDefaultTestServerOptions(), []string{
"--disable-admission-plugins=ServiceAccount",
fmt.Sprintf("--enable-aggregator-routing=%v", enableAggregatorRouting),
"--admission-control-config-file=" + admissionConfigFile.Name(),
}, framework.SharedEtcd())
defer s.TearDownFn()
// Configure a client with a distinct user name so that it is easy to distinguish requests
// made by the client from requests made by controllers. We use this to filter out requests
// before recording them to ensure we don't accidentally mistake requests from controllers
// as requests made by the client.
clientConfig := rest.CopyConfig(s.ClientConfig)
clientConfig.Impersonate.UserName = testClientAuthClientUsername
clientConfig.Impersonate.Groups = []string{"system:masters", "system:authenticated"}
client, err := clientset.NewForConfig(clientConfig)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
_, err = client.CoreV1().Pods("default").Create(clientAuthMarkerFixture)
if err != nil {
t.Fatal(err)
}
upCh := recorder.Reset()
ns := "load-balance"
_, err = client.CoreV1().Namespaces().Create(&v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}})
if err != nil {
t.Fatal(err)
}
fail := admissionv1beta1.Fail
mutatingCfg, err := client.AdmissionregistrationV1beta1().MutatingWebhookConfigurations().Create(&admissionv1beta1.MutatingWebhookConfiguration{
ObjectMeta: metav1.ObjectMeta{Name: "admission.integration.test"},
Webhooks: []admissionv1beta1.MutatingWebhook{{
Name: "admission.integration.test",
ClientConfig: admissionv1beta1.WebhookClientConfig{
URL: &webhookServer.URL,
CABundle: localhostCert,
},
Rules: []admissionv1beta1.RuleWithOperations{{
Operations: []admissionv1beta1.OperationType{admissionv1beta1.OperationAll},
Rule: admissionv1beta1.Rule{APIGroups: []string{""}, APIVersions: []string{"v1"}, Resources: []string{"pods"}},
}},
FailurePolicy: &fail,
AdmissionReviewVersions: []string{"v1beta1"},
}},
})
if err != nil {
t.Fatal(err)
}
defer func() {
err := client.AdmissionregistrationV1beta1().MutatingWebhookConfigurations().Delete(mutatingCfg.GetName(), &metav1.DeleteOptions{})
if err != nil {
t.Fatal(err)
}
}()
// wait until new webhook is called
if err := wait.PollImmediate(time.Millisecond*5, wait.ForeverTestTimeout, func() (bool, error) {
_, err = client.CoreV1().Pods("default").Patch(clientAuthMarkerFixture.Name, types.JSONPatchType, []byte("[]"))
if t.Failed() {
return true, nil
}
select {
case <-upCh:
return true, nil
default:
t.Logf("Waiting for webhook to become effective, getting marker object: %v", err)
return false, nil
}
}); err != nil {
t.Fatal(err)
}
}
type clientAuthRecorder struct {
mu sync.Mutex
upCh chan struct{}
upOnce sync.Once
}
// Reset zeros out all counts and returns a channel that is closed when the first admission of the
// marker object is received.
func (i *clientAuthRecorder) Reset() chan struct{} {
i.mu.Lock()
defer i.mu.Unlock()
i.upCh = make(chan struct{})
i.upOnce = sync.Once{}
return i.upCh
}
func (i *clientAuthRecorder) MarkerReceived() {
i.mu.Lock()
defer i.mu.Unlock()
i.upOnce.Do(func() {
close(i.upCh)
})
}
func newClientAuthWebhookHandler(t *testing.T, recorder *clientAuthRecorder) http.Handler {
allow := func(w http.ResponseWriter) {
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(&v1beta1.AdmissionReview{
Response: &v1beta1.AdmissionResponse{
Allowed: true,
},
})
}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
data, err := ioutil.ReadAll(r.Body)
if err != nil {
http.Error(w, err.Error(), 400)
}
review := v1beta1.AdmissionReview{}
if err := json.Unmarshal(data, &review); err != nil {
http.Error(w, err.Error(), 400)
}
if review.Request.UserInfo.Username != testClientAuthClientUsername {
// skip requests not originating from this integration test's client
allow(w)
return
}
if authz := r.Header.Get("Authorization"); authz != "Bearer localhost-match-with-port" {
t.Errorf("unexpected authz header: %q", authz)
http.Error(w, "Invalid auth", 401)
return
}
if len(review.Request.Object.Raw) == 0 {
http.Error(w, err.Error(), 400)
return
}
pod := &corev1.Pod{}
if err := json.Unmarshal(review.Request.Object.Raw, pod); err != nil {
http.Error(w, err.Error(), 400)
return
}
// When resetting between tests, a marker object is patched until this webhook
// observes it, at which point it is considered ready.
if pod.Namespace == clientAuthMarkerFixture.Namespace && pod.Name == clientAuthMarkerFixture.Name {
recorder.MarkerReceived()
allow(w)
return
}
})
}
var clientAuthMarkerFixture = &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Namespace: "default",
Name: "marker",
},
Spec: corev1.PodSpec{
Containers: []v1.Container{{
Name: "fake-name",
Image: "fakeimage",
}},
},
}