fix review

Signed-off-by: Patrik Cyvoct <patrik@ptrk.io>
This commit is contained in:
Patrik Cyvoct 2020-10-22 10:44:04 +02:00
parent d562b6924a
commit 0153b96ab8
No known key found for this signature in database
GPG Key ID: 4334D82B950FB63A
11 changed files with 242 additions and 115 deletions

View File

@ -3509,10 +3509,10 @@ type LoadBalancerIngress struct {
// +optional
Hostname string
// RouteType specifies the type of route to use for this ingress
// Defaults to `VIP`
// IPMode specifies the IP mode to use for this ingress
// Defaults to `VIP` if IP is set, null otherwise
// +optional
RouteType LoadBalancerRouteType
IPMode *LoadBalancerIPMode
}
const (
@ -3520,14 +3520,16 @@ const (
MaxServiceTopologyKeys = 16
)
// LoadBalancerRouteType represents the type of route available for a LoadBalancer ingress
type LoadBalancerRouteType string
// LoadBalancerIPMode represents the mode of the LoadBalancer ingress IP
type LoadBalancerIPMode string
const (
// LoadBalancerRouteTypeVIP is the type of route used by a LoadBalancer where dstIP = lbIP
LoadBalancerRouteTypeVIP LoadBalancerRouteType = "VIP"
// LoadBalancerRouteTypeProxy is the type of route used by a proxy like LoadBalancer
LoadBalancerRouteTypeProxy LoadBalancerRouteType = "Proxy"
// LoadBalancerIPModeVIP indicates that the traffic passing through this LoadBalancer
// is delivered with the destination IP set to the specified LoadBalancer IP
LoadBalancerIPModeVIP LoadBalancerIPMode = "VIP"
// LoadBalancerIPModeProxy indicates that the specified LoadBalancer acts like a proxy,
// changing the destination IP to the node IP and the source IP to the LoadBalaner (mostly private) IP
LoadBalancerIPModeProxy LoadBalancerIPMode = "Proxy"
)
// IPFamily represents the IP Family (IPv4 or IPv6). This type is used

View File

@ -19,7 +19,7 @@ package v1
import (
"time"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/kubernetes/pkg/util/parsers"
@ -161,10 +161,17 @@ func SetDefaults_Service(obj *v1.Service) {
// NOTE: strategy handles cases where ClusterIPs is used (but not ClusterIP).
}
}
}
// any other defaulting depends on cluster configuration.
// further IPFamilies, IPFamilyPolicy defaulting is in ClusterIP alloc/reserve logic
// note: conversion logic handles cases where ClusterIPs is used (but not ClusterIP).
if utilfeature.DefaultFeatureGate.Enabled(features.LoadBalancerIPMode) &&
obj.Spec.Type == v1.ServiceTypeLoadBalancer {
for _, ing := range obj.Status.LoadBalancer.Ingress {
if ing.IPMode == nil {
ipMode := v1.LoadBalancerIPModeVIP
ing.IPMode = &ipMode
}
}
}
}
func SetDefaults_Pod(obj *v1.Pod) {

View File

@ -5973,6 +5973,10 @@ func ValidatePodLogOptions(opts *core.PodLogOptions) field.ErrorList {
return allErrs
}
var (
supportedLoadBalancerIPMode = sets.NewString(string(core.LoadBalancerIPModeVIP), string(core.LoadBalancerIPModeProxy))
)
// ValidateLoadBalancerStatus validates required fields on a LoadBalancerStatus
func ValidateLoadBalancerStatus(status *core.LoadBalancerStatus, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
@ -5983,6 +5987,22 @@ func ValidateLoadBalancerStatus(status *core.LoadBalancerStatus, fldPath *field.
allErrs = append(allErrs, field.Invalid(idxPath.Child("ip"), ingress.IP, "must be a valid IP address"))
}
}
if utilfeature.DefaultFeatureGate.Enabled(features.LoadBalancerIPMode) {
if len(ingress.IP) > 0 && ingress.IPMode != nil {
switch *ingress.IPMode {
case core.LoadBalancerIPModeVIP, core.LoadBalancerIPModeProxy:
break
default:
allErrs = append(allErrs, field.NotSupported(idxPath.Child("ipMode"), ingress.IPMode, supportedLoadBalancerIPMode.List()))
}
} else if len(ingress.IP) > 0 && ingress.IPMode == nil {
allErrs = append(allErrs, field.Required(idxPath.Child("ipMode"), "must be specified when `ip` is set"))
} else if len(ingress.IP) == 0 && ingress.IPMode != nil {
allErrs = append(allErrs, field.Forbidden(idxPath.Child("ipMode"), "may not be used when `ip` is not set"))
}
}
if len(ingress.Hostname) > 0 {
for _, msg := range validation.IsDNS1123Subdomain(ingress.Hostname) {
allErrs = append(allErrs, field.Invalid(idxPath.Child("hostname"), ingress.Hostname, msg))
@ -5991,6 +6011,7 @@ func ValidateLoadBalancerStatus(status *core.LoadBalancerStatus, fldPath *field.
allErrs = append(allErrs, field.Invalid(idxPath.Child("hostname"), ingress.Hostname, "must be a DNS name, not an IP address"))
}
}
}
return allErrs
}

View File

@ -663,6 +663,11 @@ const (
//
// Enables kubelet support to size memory backed volumes
SizeMemoryBackedVolumes featuregate.Feature = "SizeMemoryBackedVolumes"
// owner: TBD ?
// alpha: v1.21
// LoadBalancerIPMode enables the IPMode field in the LoadBalancerIngress status of a Service
LoadBalancerIPMode featuregate.Feature = "LoadBalancerIPMode"
)
func init() {
@ -763,6 +768,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
HPAContainerMetrics: {Default: false, PreRelease: featuregate.Alpha},
RootCAConfigMap: {Default: true, PreRelease: featuregate.Beta},
SizeMemoryBackedVolumes: {Default: false, PreRelease: featuregate.Alpha},
LoadBalancerIPMode: {Default: false, PreRelease: featuregate.Alpha},
// inherited features from generic apiserver, relisted here to get a conflict if it is changed
// unintentionally on either side:

View File

@ -1175,7 +1175,7 @@ func (proxier *Proxier) syncProxyRules() {
// This currently works for loadbalancers that preserves source ips.
// For loadbalancers which direct traffic to service NodePort, the firewall rules will not apply.
if ingress.RouteType != v1.LoadBalancerRouteTypeProxy {
if !utilfeature.DefaultFeatureGate.Enabled(features.LoadBalancerIPMode) || ingress.IPMode == nil || *ingress.IPMode == v1.LoadBalancerIPModeVIP {
args = append(args[:0],
"-A", string(kubeServicesChain),
"-m", "comment", "--comment", fmt.Sprintf(`"%s loadbalancer IP"`, svcNameString),

View File

@ -26,6 +26,7 @@ import (
"testing"
"time"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/klog/v2"
"github.com/stretchr/testify/assert"
@ -34,6 +35,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/proxy"
"k8s.io/kubernetes/pkg/proxy/healthcheck"
utilproxy "k8s.io/kubernetes/pkg/proxy/util"
@ -46,6 +48,8 @@ import (
"k8s.io/utils/exec"
fakeexec "k8s.io/utils/exec/testing"
utilpointer "k8s.io/utils/pointer"
featuregatetesting "k8s.io/component-base/featuregate/testing"
)
func checkAllLines(t *testing.T, table utiliptables.Table, save []byte, expectedLines map[utiliptables.Chain]string) {
@ -2683,59 +2687,89 @@ COMMIT
}
func TestLoadBalancerIngressRouteTypeProxy(t *testing.T) {
ipt := iptablestest.NewFake()
fp := NewFakeProxier(ipt, false)
svcIP := "10.20.30.41"
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LoadBalancerIPMode, true)()
ipModeProxy := v1.LoadBalancerIPModeProxy
ipModeVIP := v1.LoadBalancerIPModeVIP
testCases := []struct {
svcIP string
svcLBIP string
ipMode *v1.LoadBalancerIPMode
expectedRule bool
}{
{
svcIP: "10.20.30.41",
svcLBIP: "1.2.3.4",
ipMode: &ipModeProxy,
expectedRule: false,
},
{
svcIP: "10.20.30.42",
svcLBIP: "1.2.3.5",
ipMode: &ipModeVIP,
expectedRule: true,
},
{
svcIP: "10.20.30.43",
svcLBIP: "1.2.3.6",
ipMode: nil,
expectedRule: true,
},
}
svcPort := 80
svcNodePort := 3001
svcLBIP := "1.2.3.4"
svcPortName := proxy.ServicePortName{
NamespacedName: makeNSN("ns1", "svc1"),
Port: "p80",
Protocol: v1.ProtocolTCP,
}
makeServiceMap(fp,
makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) {
svc.Spec.Type = "LoadBalancer"
svc.Spec.ClusterIP = svcIP
svc.Spec.Ports = []v1.ServicePort{{
Name: svcPortName.Port,
Port: int32(svcPort),
Protocol: v1.ProtocolTCP,
NodePort: int32(svcNodePort),
}}
svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
IP: svcLBIP,
RouteType: v1.LoadBalancerRouteTypeProxy,
}}
}),
)
epIP := "10.180.0.1"
makeEndpointsMap(fp,
makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *v1.Endpoints) {
ept.Subsets = []v1.EndpointSubset{{
Addresses: []v1.EndpointAddress{{
IP: epIP,
}},
Ports: []v1.EndpointPort{{
for _, testCase := range testCases {
ipt := iptablestest.NewFake()
fp := NewFakeProxier(ipt, false)
makeServiceMap(fp,
makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) {
svc.Spec.Type = "LoadBalancer"
svc.Spec.ClusterIP = testCase.svcIP
svc.Spec.Ports = []v1.ServicePort{{
Name: svcPortName.Port,
Port: int32(svcPort),
Protocol: v1.ProtocolTCP,
}},
}}
}),
)
NodePort: int32(svcNodePort),
}}
svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
IP: testCase.svcLBIP,
IPMode: testCase.ipMode,
}}
}),
)
fp.syncProxyRules()
epIP := "10.180.0.1"
makeEndpointsMap(fp,
makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *v1.Endpoints) {
ept.Subsets = []v1.EndpointSubset{{
Addresses: []v1.EndpointAddress{{
IP: epIP,
}},
Ports: []v1.EndpointPort{{
Name: svcPortName.Port,
Port: int32(svcPort),
Protocol: v1.ProtocolTCP,
}},
}}
}),
)
proto := strings.ToLower(string(v1.ProtocolTCP))
fwChain := string(serviceFirewallChainName(svcPortName.String(), proto))
fp.syncProxyRules()
kubeSvcRules := ipt.GetRules(string(kubeServicesChain))
if hasJump(kubeSvcRules, fwChain, svcLBIP, svcPort) {
errorf(fmt.Sprintf("Found jump to firewall chain %v", fwChain), kubeSvcRules, t)
proto := strings.ToLower(string(v1.ProtocolTCP))
fwChain := string(serviceFirewallChainName(svcPortName.String(), proto))
kubeSvcRules := ipt.GetRules(string(kubeServicesChain))
if hasJump(kubeSvcRules, fwChain, testCase.svcLBIP, svcPort) != testCase.expectedRule {
errorf(fmt.Sprintf("Found jump to firewall chain %v", fwChain), kubeSvcRules, t)
}
}
}

View File

@ -1333,7 +1333,7 @@ func (proxier *Proxier) syncProxyRules() {
// Capture load-balancer ingress.
for _, ingress := range svcInfo.LoadBalancerIngress() {
if ingress.IP != "" && ingress.RouteType != v1.LoadBalancerRouteTypeProxy {
if ingress.IP != "" && (!utilfeature.DefaultFeatureGate.Enabled(features.LoadBalancerIPMode) || ingress.IPMode == nil || *ingress.IPMode == v1.LoadBalancerIPModeVIP) {
// ipset call
entry = &utilipset.Entry{
IP: ingress.IP,

View File

@ -33,6 +33,8 @@ import (
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apimachinery/pkg/util/sets"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/proxy"
"k8s.io/kubernetes/pkg/proxy/healthcheck"
netlinktest "k8s.io/kubernetes/pkg/proxy/ipvs/testing"
@ -51,6 +53,8 @@ import (
utilpointer "k8s.io/utils/pointer"
utilnet "k8s.io/utils/net"
featuregatetesting "k8s.io/component-base/featuregate/testing"
)
const testHostname = "test-hostname"
@ -4322,55 +4326,85 @@ func TestFilterCIDRs(t *testing.T) {
}
func TestLoadBalancerIngressRouteTypeProxy(t *testing.T) {
_, fp := buildFakeProxier()
svcIP := "10.20.30.41"
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LoadBalancerIPMode, true)()
ipModeProxy := v1.LoadBalancerIPModeProxy
ipModeVIP := v1.LoadBalancerIPModeVIP
testCases := []struct {
svcIP string
svcLBIP string
ipMode *v1.LoadBalancerIPMode
expectedServices int
}{
{
svcIP: "10.20.30.41",
svcLBIP: "1.2.3.4",
ipMode: &ipModeProxy,
expectedServices: 1,
},
{
svcIP: "10.20.30.42",
svcLBIP: "1.2.3.5",
ipMode: &ipModeVIP,
expectedServices: 2,
},
{
svcIP: "10.20.30.43",
svcLBIP: "1.2.3.6",
ipMode: nil,
expectedServices: 2,
},
}
svcPort := 80
svcNodePort := 3001
svcLBIP := "1.2.3.4"
svcPortName := proxy.ServicePortName{
NamespacedName: makeNSN("ns1", "svc1"),
Port: "p80",
}
makeServiceMap(fp,
makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) {
svc.Spec.Type = "LoadBalancer"
svc.Spec.ClusterIP = svcIP
svc.Spec.Ports = []v1.ServicePort{{
Name: svcPortName.Port,
Port: int32(svcPort),
Protocol: v1.ProtocolTCP,
NodePort: int32(svcNodePort),
}}
svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
IP: svcLBIP,
RouteType: v1.LoadBalancerRouteTypeProxy,
}}
}),
)
for _, testCase := range testCases {
_, fp := buildFakeProxier()
makeServiceMap(fp,
makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) {
svc.Spec.Type = "LoadBalancer"
svc.Spec.ClusterIP = testCase.svcIP
svc.Spec.Ports = []v1.ServicePort{{
Name: svcPortName.Port,
Port: int32(svcPort),
Protocol: v1.ProtocolTCP,
NodePort: int32(svcNodePort),
}}
svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
IP: testCase.svcLBIP,
IPMode: testCase.ipMode,
}}
}),
)
epIP := "10.180.0.1"
makeEndpointsMap(fp,
makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *v1.Endpoints) {
ept.Subsets = []v1.EndpointSubset{{
Addresses: []v1.EndpointAddress{{
IP: epIP,
}},
Ports: []v1.EndpointPort{{
Name: svcPortName.Port,
Port: int32(svcPort),
}},
}}
}),
)
epIP := "10.180.0.1"
makeEndpointsMap(fp,
makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *v1.Endpoints) {
ept.Subsets = []v1.EndpointSubset{{
Addresses: []v1.EndpointAddress{{
IP: epIP,
}},
Ports: []v1.EndpointPort{{
Name: svcPortName.Port,
Port: int32(svcPort),
}},
}}
}),
)
fp.syncProxyRules()
fp.syncProxyRules()
services, err := fp.ipvs.GetVirtualServers()
if err != nil {
t.Errorf("Failed to get ipvs services, err: %v", err)
}
if len(services) != 1 {
t.Errorf("Expect 1 ipvs services, got %d", len(services))
services, err := fp.ipvs.GetVirtualServers()
if err != nil {
t.Errorf("Failed to get ipvs services, err: %v", err)
}
if len(services) != testCase.expectedServices {
t.Errorf("Expect %d ipvs services, got %d", testCase.expectedServices, len(services))
}
}
}

View File

@ -164,22 +164,24 @@ func (sct *ServiceChangeTracker) newBaseServiceInfo(port *v1.ServicePort, servic
}
// Obtain Load Balancer Ingress IPs
var ips []string
var allIncorrectIPs []string
for _, ing := range service.Status.LoadBalancer.Ingress {
ips = append(ips, ing.IP)
correctIPs, incorrectIPs := utilproxy.FilterIncorrectIPVersion([]string{ing.IP}, sct.ipFamily)
// len is either 1 or 0
if len(correctIPs) == 1 {
// Update the LoadBalancerStatus with the filtered IPs
info.loadBalancerStatus.Ingress = append(info.loadBalancerStatus.Ingress, ing)
continue
}
// here len(incorrectIPs) == 1
allIncorrectIPs = append(allIncorrectIPs, incorrectIPs[0])
}
if len(ips) > 0 {
correctIPs, incorrectIPs := utilproxy.FilterIncorrectIPVersion(ips, sct.ipFamily)
if len(incorrectIPs) > 0 {
klog.V(4).Infof("service change tracker(%v) ignored the following load balancer(%s) ingress ips for service %v/%v as they don't match IPFamily", sct.ipFamily, strings.Join(incorrectIPs, ","), service.Namespace, service.Name)
if len(incorrectIPs) > 0 {
klog.V(4).Infof("service change tracker(%v) ignored the following load balancer(%s) ingress ips for service %v/%v as they don't match IPFamily", sct.ipFamily, strings.Join(incorrectIPs, ","), service.Namespace, service.Name)
}
// Create the LoadBalancerStatus with the filtered IPs
for _, ip := range correctIPs {
info.loadBalancerStatus.Ingress = append(info.loadBalancerStatus.Ingress, v1.LoadBalancerIngress{IP: ip})
}
}
if apiservice.NeedsHealthCheck(service) {

View File

@ -179,6 +179,12 @@ func dropServiceDisabledFields(newSvc *api.Service, oldSvc *api.Service) {
if !utilfeature.DefaultFeatureGate.Enabled(features.ServiceTopology) && !topologyKeysInUse(oldSvc) {
newSvc.Spec.TopologyKeys = nil
}
if !utilfeature.DefaultFeatureGate.Enabled(features.LoadBalancerIPMode) && !loadbalancerIPModeInUse(oldSvc) {
for _, ing := range newSvc.Status.LoadBalancer.Ingress {
ing.IPMode = nil
}
}
}
// returns true if svc.Spec.ServiceIPFamily field is in use
@ -202,6 +208,19 @@ func topologyKeysInUse(svc *api.Service) bool {
return len(svc.Spec.TopologyKeys) > 0
}
// returns true is svc.Status.LoadBalancer.Ingress[].IPMode fields are in use
func loadbalancerIPModeInUse(svc *api.Service) bool {
if svc == nil {
return false
}
for _, ing := range svc.Status.LoadBalancer.Ingress {
if ing.IPMode != nil {
return true
}
}
return false
}
type serviceStatusStrategy struct {
Strategy
}

View File

@ -3972,10 +3972,10 @@ type LoadBalancerIngress struct {
// +optional
Hostname string `json:"hostname,omitempty" protobuf:"bytes,2,opt,name=hostname"`
// RouteType specifies the type of route to use for this ingress
// Defaults to `VIP`
// IPMode specifies the IP mode to use for this ingress
// Defaults to `VIP` if IP is set, null otherwise
// +optional
RouteType LoadBalancerRouteType `json:"routeType,omitempty" protobuf:"bytes,3,opt,name=routeType"`
IPMode *LoadBalancerIPMode `json:"ipMode,omitempty" protobuf:"bytes,3,opt,name=ipMode"`
}
const (
@ -3983,14 +3983,16 @@ const (
MaxServiceTopologyKeys = 16
)
// LoadBalancerRouteType represents the type of route available for a LoadBalancer ingress
type LoadBalancerRouteType string
// LoadBalancerIPMode represents the mode of the LoadBalancer ingress IP
type LoadBalancerIPMode string
const (
// LoadBalancerRouteTypeVIP is the type of route used by a LoadBalancer where dstIP = lbIP
LoadBalancerRouteTypeVIP LoadBalancerRouteType = "VIP"
// LoadBalancerRouteTypeProxy is the type of route used by a proxy like LoadBalancer
LoadBalancerRouteTypeProxy LoadBalancerRouteType = "Proxy"
// LoadBalancerIPModeVIP indicates that the traffic passing through this LoadBalancer
// is delivered with the destination IP set to the specified LoadBalancer IP
LoadBalancerIPModeVIP LoadBalancerIPMode = "VIP"
// LoadBalancerIPModeProxy indicates that the specified LoadBalancer acts like a proxy,
// changing the destination IP to the node IP and the source IP to the LoadBalaner (mostly private) IP
LoadBalancerIPModeProxy LoadBalancerIPMode = "Proxy"
)
// IPFamily represents the IP Family (IPv4 or IPv6). This type is used