From a0aebf96ec2eef6517e2611335f0e6c9375dd807 Mon Sep 17 00:00:00 2001 From: Andrew Sy Kim Date: Mon, 26 Oct 2020 10:24:16 -0400 Subject: [PATCH 1/4] apiserver: support egress selection name 'controlplane' and deprecate 'master' Signed-off-by: Andrew Sy Kim --- cmd/kube-apiserver/app/server.go | 2 +- pkg/kubeapiserver/options/authentication.go | 2 +- .../k8s.io/apiserver/pkg/apis/apiserver/types.go | 3 ++- .../pkg/apis/apiserver/v1alpha1/types.go | 3 ++- .../apiserver/pkg/apis/apiserver/v1beta1/types.go | 3 ++- .../pkg/server/egressselector/egress_selector.go | 15 ++++++++++----- .../server/egressselector/egress_selector_test.go | 2 +- .../k8s.io/apiserver/pkg/server/options/audit.go | 2 +- .../apiserver/pkg/util/webhook/authentication.go | 2 +- 9 files changed, 21 insertions(+), 13 deletions(-) diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index eccf058b917..ef1b9fa2d45 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -562,7 +562,7 @@ func BuildAuthorizer(s *options.ServerRunOptions, EgressSelector *egressselector authorizationConfig := s.Authorization.ToAuthorizationConfig(versionedInformers) if EgressSelector != nil { - egressDialer, err := EgressSelector.Lookup(egressselector.Master.AsNetworkContext()) + egressDialer, err := EgressSelector.Lookup(egressselector.ControlPlane.AsNetworkContext()) if err != nil { return nil, nil, err } diff --git a/pkg/kubeapiserver/options/authentication.go b/pkg/kubeapiserver/options/authentication.go index f01118ff8d2..44e14d314b4 100644 --- a/pkg/kubeapiserver/options/authentication.go +++ b/pkg/kubeapiserver/options/authentication.go @@ -477,7 +477,7 @@ func (o *BuiltInAuthenticationOptions) ApplyTo(authInfo *genericapiserver.Authen ) if egressSelector != nil { - egressDialer, err := egressSelector.Lookup(egressselector.Master.AsNetworkContext()) + egressDialer, err := egressSelector.Lookup(egressselector.ControlPlane.AsNetworkContext()) if err != nil { return err } diff --git a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/types.go b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/types.go index 4e414944129..f0614be3a00 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/types.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/types.go @@ -62,7 +62,8 @@ type EgressSelectorConfiguration struct { // EgressSelection provides the configuration for a single egress selection client. type EgressSelection struct { // Name is the name of the egress selection. - // Currently supported values are "Master", "Etcd" and "Cluster" + // Currently supported values are "ControlPlane", "Master", "Etcd" and "Cluster" + // The "Master" egress selector is deprecated in favor of "ControlPlane" Name string // Connection is the exact information used to configure the egress selection diff --git a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/types.go b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/types.go index 7b9aacae86f..53e5595967f 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/types.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/types.go @@ -62,7 +62,8 @@ type EgressSelectorConfiguration struct { // EgressSelection provides the configuration for a single egress selection client. type EgressSelection struct { // name is the name of the egress selection. - // Currently supported values are "Master", "Etcd" and "Cluster" + // Currently supported values are "ControlPlane", "Master", "Etcd" and "Cluster" + // The "Master" egress selector is deprecated in favor of "ControlPlane" Name string `json:"name"` // connection is the exact information used to configure the egress selection diff --git a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1beta1/types.go b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1beta1/types.go index 0a6fd07321e..03bac01bd7d 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1beta1/types.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1beta1/types.go @@ -33,7 +33,8 @@ type EgressSelectorConfiguration struct { // EgressSelection provides the configuration for a single egress selection client. type EgressSelection struct { // name is the name of the egress selection. - // Currently supported values are "Master", "Etcd" and "Cluster" + // Currently supported values are "ControlPlane", "Master", "Etcd" and "Cluster" + // The "Master" egress selector is deprecated in favor of "ControlPlane" Name string `json:"name"` // connection is the exact information used to configure the egress selection diff --git a/staging/src/k8s.io/apiserver/pkg/server/egressselector/egress_selector.go b/staging/src/k8s.io/apiserver/pkg/server/egressselector/egress_selector.go index a41a85403c1..a849575b8ef 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/egressselector/egress_selector.go +++ b/staging/src/k8s.io/apiserver/pkg/server/egressselector/egress_selector.go @@ -51,8 +51,8 @@ type EgressSelector struct { type EgressType int const ( - // Master is the EgressType for traffic intended to go to the control plane. - Master EgressType = iota + // ControlPlane is the EgressType for traffic intended to go to the control plane. + ControlPlane EgressType = iota // Etcd is the EgressType for traffic intended to go to Kubernetes persistence store. Etcd // Cluster is the EgressType for traffic intended to go to the system being managed by Kubernetes. @@ -73,8 +73,8 @@ type Lookup func(networkContext NetworkContext) (utilnet.DialFunc, error) // String returns the canonical string representation of the egress type func (s EgressType) String() string { switch s { - case Master: - return "master" + case ControlPlane: + return "controlplane" case Etcd: return "etcd" case Cluster: @@ -91,8 +91,12 @@ func (s EgressType) AsNetworkContext() NetworkContext { func lookupServiceName(name string) (EgressType, error) { switch strings.ToLower(name) { + // 'master' is deprecated, interpret "master" as controlplane internally until removed in v1.22. case "master": - return Master, nil + klog.Warning("EgressSelection name 'master' is deprecated, use 'controlplane' instead") + return ControlPlane, nil + case "controlplane": + return ControlPlane, nil case "etcd": return Etcd, nil case "cluster": @@ -364,5 +368,6 @@ func (cs *EgressSelector) Lookup(networkContext NetworkContext) (utilnet.DialFun // The round trip wrapper will over-ride the dialContext method appropriately return nil, nil } + return cs.egressToDialer[networkContext.EgressSelectionName], nil } diff --git a/staging/src/k8s.io/apiserver/pkg/server/egressselector/egress_selector_test.go b/staging/src/k8s.io/apiserver/pkg/server/egressselector/egress_selector_test.go index 8ff72fe8a9e..2b31ccc41b6 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/egressselector/egress_selector_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/egressselector/egress_selector_test.go @@ -90,7 +90,7 @@ func TestEgressSelector(t *testing.T) { nil, }, { - Master, + ControlPlane, validateDirectDialer, nil, nil, diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/audit.go b/staging/src/k8s.io/apiserver/pkg/server/options/audit.go index 06ff8a3efcc..1798eb71aa8 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/audit.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/audit.go @@ -306,7 +306,7 @@ func (o *AuditOptions) ApplyTo( klog.V(2).Info("No audit policy file provided, no events will be recorded for webhook backend") } else { if c.EgressSelector != nil { - egressDialer, err := c.EgressSelector.Lookup(egressselector.Master.AsNetworkContext()) + egressDialer, err := c.EgressSelector.Lookup(egressselector.ControlPlane.AsNetworkContext()) if err != nil { return err } diff --git a/staging/src/k8s.io/apiserver/pkg/util/webhook/authentication.go b/staging/src/k8s.io/apiserver/pkg/util/webhook/authentication.go index 042879dade9..965bc8b58c8 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/webhook/authentication.go +++ b/staging/src/k8s.io/apiserver/pkg/util/webhook/authentication.go @@ -55,7 +55,7 @@ func NewDefaultAuthenticationInfoResolverWrapper( } if egressSelector != nil { - networkContext := egressselector.Master.AsNetworkContext() + networkContext := egressselector.ControlPlane.AsNetworkContext() var egressDialer utilnet.DialFunc egressDialer, err = egressSelector.Lookup(networkContext) From 30d43806c3fe283c8a0b02f050a7ad93dcb1a759 Mon Sep 17 00:00:00 2001 From: Andrew Sy Kim Date: Mon, 26 Oct 2020 10:24:16 -0400 Subject: [PATCH 2/4] apiserver: add validation for EgressSelection names in EgressSelectorConfiguration API Signed-off-by: Andrew Sy Kim --- .../apiserver/pkg/server/egressselector/BUILD | 1 + .../pkg/server/egressselector/config.go | 29 +++++++++++++ .../pkg/server/egressselector/config_test.go | 42 +++++++++++++++++++ 3 files changed, 72 insertions(+) diff --git a/staging/src/k8s.io/apiserver/pkg/server/egressselector/BUILD b/staging/src/k8s.io/apiserver/pkg/server/egressselector/BUILD index bc730f43ff6..e9f44cd55c6 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/egressselector/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/egressselector/BUILD @@ -12,6 +12,7 @@ go_library( deps = [ "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", "//staging/src/k8s.io/apiserver/pkg/apis/apiserver:go_default_library", "//staging/src/k8s.io/apiserver/pkg/apis/apiserver/install:go_default_library", diff --git a/staging/src/k8s.io/apiserver/pkg/server/egressselector/config.go b/staging/src/k8s.io/apiserver/pkg/server/egressselector/config.go index e608565c076..b7e378e430f 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/egressselector/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/egressselector/config.go @@ -22,6 +22,7 @@ import ( "strings" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/apiserver/pkg/apis/apiserver" "k8s.io/apiserver/pkg/apis/apiserver/install" @@ -32,6 +33,10 @@ import ( var cfgScheme = runtime.NewScheme() +// validEgressSelectorNames contains the set of valid egress selctor names. +// 'master' is deprecated in favor of 'controlplane' and will be removed in v1.22. +var validEgressSelectorNames = sets.NewString("master", "controlplane", "cluster", "etcd") + func init() { install.Install(cfgScheme) } @@ -97,6 +102,30 @@ func ValidateEgressSelectorConfiguration(config *apiserver.EgressSelectorConfigu })) } } + + var foundControlPlane, foundMaster bool + for _, service := range config.EgressSelections { + canonicalName := strings.ToLower(service.Name) + + if !validEgressSelectorNames.Has(canonicalName) { + allErrs = append(allErrs, field.NotSupported(field.NewPath("egressSelection", "name"), canonicalName, validEgressSelectorNames.List())) + continue + } + + if canonicalName == "master" { + foundMaster = true + } + + if canonicalName == "controlplane" { + foundControlPlane = true + } + } + + // error if both master and controlplane egress selectors are set + if foundMaster && foundControlPlane { + allErrs = append(allErrs, field.Forbidden(field.NewPath("egressSelection", "name"), "both egressSelection names 'master' and 'controlplane' are specified, only one is allowed")) + } + return allErrs } diff --git a/staging/src/k8s.io/apiserver/pkg/server/egressselector/config_test.go b/staging/src/k8s.io/apiserver/pkg/server/egressselector/config_test.go index 9067938e7d5..2ddc3e69a81 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/egressselector/config_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/egressselector/config_test.go @@ -413,6 +413,48 @@ func TestValidateEgressSelectorConfiguration(t *testing.T) { }, }, }, + { + name: "invalid egress selection name", + expectError: true, + contents: &apiserver.EgressSelectorConfiguration{ + TypeMeta: metav1.TypeMeta{ + Kind: "", + APIVersion: "", + }, + EgressSelections: []apiserver.EgressSelection{ + { + Name: "invalid", + Connection: apiserver.Connection{ + ProxyProtocol: apiserver.ProtocolDirect, + }, + }, + }, + }, + }, + { + name: "both master and controlplane egress selection configured", + expectError: true, + contents: &apiserver.EgressSelectorConfiguration{ + TypeMeta: metav1.TypeMeta{ + Kind: "", + APIVersion: "", + }, + EgressSelections: []apiserver.EgressSelection{ + { + Name: "controlplane", + Connection: apiserver.Connection{ + ProxyProtocol: apiserver.ProtocolDirect, + }, + }, + { + Name: "master", + Connection: apiserver.Connection{ + ProxyProtocol: apiserver.ProtocolDirect, + }, + }, + }, + }, + }, } for _, tc := range testcases { From 5a0932586484b8d5e261cfcbd587fb3ce0c72b27 Mon Sep 17 00:00:00 2001 From: Andrew Sy Kim Date: Mon, 26 Oct 2020 10:24:16 -0400 Subject: [PATCH 3/4] apiserver: update TestReadEgressSelectorConfiguration to use 'controlplane' egress selector name Signed-off-by: Andrew Sy Kim --- .../pkg/server/egressselector/config_test.go | 82 ++++++++++++++++++- 1 file changed, 81 insertions(+), 1 deletion(-) diff --git a/staging/src/k8s.io/apiserver/pkg/server/egressselector/config_test.go b/staging/src/k8s.io/apiserver/pkg/server/egressselector/config_test.go index 2ddc3e69a81..2cb0e1231cf 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/egressselector/config_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/egressselector/config_test.go @@ -60,6 +60,86 @@ func TestReadEgressSelectorConfiguration(t *testing.T) { apiVersion: apiserver.k8s.io/v1beta1 kind: EgressSelectorConfiguration egressSelections: +- name: "cluster" + connection: + proxyProtocol: "HTTPConnect" + transport: + tcp: + url: "https://127.0.0.1:8131" + tlsConfig: + caBundle: "/etc/srv/kubernetes/pki/konnectivity-server/ca.crt" + clientKey: "/etc/srv/kubernetes/pki/konnectivity-server/client.key" + clientCert: "/etc/srv/kubernetes/pki/konnectivity-server/client.crt" +- name: "controlplane" + connection: + proxyProtocol: "HTTPConnect" + transport: + tcp: + url: "https://127.0.0.1:8132" + tlsConfig: + caBundle: "/etc/srv/kubernetes/pki/konnectivity-server-master/ca.crt" + clientKey: "/etc/srv/kubernetes/pki/konnectivity-server-master/client.key" + clientCert: "/etc/srv/kubernetes/pki/konnectivity-server-master/client.crt" +- name: "etcd" + connection: + proxyProtocol: "Direct" +`, + expectedResult: &apiserver.EgressSelectorConfiguration{ + TypeMeta: metav1.TypeMeta{ + Kind: "", + APIVersion: "", + }, + EgressSelections: []apiserver.EgressSelection{ + { + Name: "cluster", + Connection: apiserver.Connection{ + ProxyProtocol: "HTTPConnect", + Transport: &apiserver.Transport{ + TCP: &apiserver.TCPTransport{ + URL: "https://127.0.0.1:8131", + + TLSConfig: &apiserver.TLSConfig{ + CABundle: "/etc/srv/kubernetes/pki/konnectivity-server/ca.crt", + ClientKey: "/etc/srv/kubernetes/pki/konnectivity-server/client.key", + ClientCert: "/etc/srv/kubernetes/pki/konnectivity-server/client.crt", + }, + }, + }, + }, + }, + { + Name: "controlplane", + Connection: apiserver.Connection{ + ProxyProtocol: "HTTPConnect", + Transport: &apiserver.Transport{ + TCP: &apiserver.TCPTransport{ + URL: "https://127.0.0.1:8132", + TLSConfig: &apiserver.TLSConfig{ + CABundle: "/etc/srv/kubernetes/pki/konnectivity-server-master/ca.crt", + ClientKey: "/etc/srv/kubernetes/pki/konnectivity-server-master/client.key", + ClientCert: "/etc/srv/kubernetes/pki/konnectivity-server-master/client.crt", + }, + }, + }, + }, + }, + { + Name: "etcd", + Connection: apiserver.Connection{ + ProxyProtocol: "Direct", + }, + }, + }, + }, + expectedError: nil, + }, + { + name: "v1beta1 using deprecated 'master' type", + createFile: true, + contents: ` +apiVersion: apiserver.k8s.io/v1beta1 +kind: EgressSelectorConfiguration +egressSelections: - name: "cluster" connection: proxyProtocol: "HTTPConnect" @@ -240,7 +320,7 @@ func TestValidateEgressSelectorConfiguration(t *testing.T) { }, EgressSelections: []apiserver.EgressSelection{ { - Name: "master", + Name: "controlplane", Connection: apiserver.Connection{ ProxyProtocol: apiserver.ProtocolDirect, }, From e4b45d944d54c239e1ad40af17221420f349c4f8 Mon Sep 17 00:00:00 2001 From: Andrew Sy Kim Date: Mon, 26 Oct 2020 10:24:16 -0400 Subject: [PATCH 4/4] apiserver: use canonical egress selection names in EgressSelectorConfiguration API docs Signed-off-by: Andrew Sy Kim --- staging/src/k8s.io/apiserver/pkg/apis/apiserver/types.go | 4 ++-- .../src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/types.go | 4 ++-- .../src/k8s.io/apiserver/pkg/apis/apiserver/v1beta1/types.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/types.go b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/types.go index f0614be3a00..197ce653731 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/types.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/types.go @@ -62,8 +62,8 @@ type EgressSelectorConfiguration struct { // EgressSelection provides the configuration for a single egress selection client. type EgressSelection struct { // Name is the name of the egress selection. - // Currently supported values are "ControlPlane", "Master", "Etcd" and "Cluster" - // The "Master" egress selector is deprecated in favor of "ControlPlane" + // Currently supported values are "controlplane", "master", "etcd" and "cluster" + // The "master" egress selector is deprecated in favor of "controlplane" Name string // Connection is the exact information used to configure the egress selection diff --git a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/types.go b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/types.go index 53e5595967f..49509bf22c7 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/types.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/types.go @@ -62,8 +62,8 @@ type EgressSelectorConfiguration struct { // EgressSelection provides the configuration for a single egress selection client. type EgressSelection struct { // name is the name of the egress selection. - // Currently supported values are "ControlPlane", "Master", "Etcd" and "Cluster" - // The "Master" egress selector is deprecated in favor of "ControlPlane" + // Currently supported values are "controlplane", "master", "etcd" and "cluster" + // The "master" egress selector is deprecated in favor of "controlplane" Name string `json:"name"` // connection is the exact information used to configure the egress selection diff --git a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1beta1/types.go b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1beta1/types.go index 03bac01bd7d..ea22b403a33 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1beta1/types.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1beta1/types.go @@ -33,8 +33,8 @@ type EgressSelectorConfiguration struct { // EgressSelection provides the configuration for a single egress selection client. type EgressSelection struct { // name is the name of the egress selection. - // Currently supported values are "ControlPlane", "Master", "Etcd" and "Cluster" - // The "Master" egress selector is deprecated in favor of "ControlPlane" + // Currently supported values are "controlplane", "master", "etcd" and "cluster" + // The "master" egress selector is deprecated in favor of "controlplane" Name string `json:"name"` // connection is the exact information used to configure the egress selection