github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-30 15:05:27 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #96338 from nilo19/feature/support-additional-probe-protocol

Browse Source 
Support customize load balancer health probe protocol and request path
This commit is contained in:
Kubernetes Prow Robot 2020-11-10 03:12:39 -08:00 committed by GitHub
commit 59405cc2b4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 122 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go
View File

@ -104,6 +104,15 @@ const (
// TODO(feiskyer): disable-tcp-reset annotations has been depracated since v1.18, it would removed on v1.20.
ServiceAnnotationLoadBalancerDisableTCPReset = "service.beta.kubernetes.io/azure-load-balancer-disable-tcp-reset"
// ServiceAnnotationLoadBalancerHealthProbeProtocol determines the network protocol that the load balancer health probe use.
// If not set, the local service would use the HTTP and the cluster service would use the TCP by default.
ServiceAnnotationLoadBalancerHealthProbeProtocol = "service.beta.kubernetes.io/azure-load-balancer-health-probe-protocol"
// ServiceAnnotationLoadBalancerHealthProbeRequestPath determines the request path of the load balancer health probe.
// This is only useful for the HTTP and HTTPS, and would be ignored when using TCP. If not set,
// `/healthz` would be configured by default.
ServiceAnnotationLoadBalancerHealthProbeRequestPath = "service.beta.kubernetes.io/azure-load-balancer-health-probe-request-path"
// serviceTagKey is the service key applied for public IP tags.
serviceTagKey = "service"
// clusterNameKey is the cluster name key applied for public IP tags.
@ -1561,6 +1570,23 @@ func (az *Cloud) checkLoadBalancerResourcesConflicted(
return nil
}
func parseHealthProbeProtocolAndPath(service *v1.Service) (string, string) {
var protocol, path string
if v, ok := service.Annotations[ServiceAnnotationLoadBalancerHealthProbeProtocol]; ok {
protocol = v
} else {
return protocol, path
}
// ignore the request path if using TCP
if strings.EqualFold(protocol, string(network.ProbeProtocolHTTP)) ||
strings.EqualFold(protocol, string(network.ProbeProtocolHTTPS)) {
if v, ok := service.Annotations[ServiceAnnotationLoadBalancerHealthProbeRequestPath]; ok {
path = v
}
}
return protocol, path
}
func (az *Cloud) reconcileLoadBalancerRule(
service *v1.Service,
wantLb bool,
@ -1606,14 +1632,21 @@ func (az *Cloud) reconcileLoadBalancerRule(
return expectedProbes, expectedRules, err
}
probeProtocol, requestPath := parseHealthProbeProtocolAndPath(service)
if servicehelpers.NeedsHealthCheck(service) {
podPresencePath, podPresencePort := servicehelpers.GetServiceHealthCheckPathPort(service)
if probeProtocol == "" {
probeProtocol = string(network.ProbeProtocolHTTP)
}
if requestPath == "" {
requestPath = podPresencePath
}
expectedProbes = append(expectedProbes, network.Probe{
Name: &lbRuleName,
ProbePropertiesFormat: &network.ProbePropertiesFormat{
RequestPath: to.StringPtr(podPresencePath),
Protocol: network.ProbeProtocolHTTP,
RequestPath: to.StringPtr(requestPath),
Protocol: network.ProbeProtocol(probeProtocol),
Port: to.Int32Ptr(podPresencePort),
IntervalInSeconds: to.Int32Ptr(5),
NumberOfProbes: to.Int32Ptr(2),
@ -1621,10 +1654,22 @@ func (az *Cloud) reconcileLoadBalancerRule(
})
} else if protocol != v1.ProtocolUDP && protocol != v1.ProtocolSCTP {
// we only add the expected probe if we're doing TCP
if probeProtocol == "" {
probeProtocol = string(*probeProto)
}
var actualPath *string
if !strings.EqualFold(probeProtocol, string(network.ProbeProtocolTCP)) {
if requestPath != "" {
actualPath = to.StringPtr(requestPath)
} else {
actualPath = to.StringPtr("/healthz")
}
}
expectedProbes = append(expectedProbes, network.Probe{
Name: &lbRuleName,
ProbePropertiesFormat: &network.ProbePropertiesFormat{
Protocol: *probeProto,
Protocol: network.ProbeProtocol(probeProtocol),
RequestPath: actualPath,
Port: to.Int32Ptr(port.NodePort),
IntervalInSeconds: to.Int32Ptr(5),
NumberOfProbes: to.Int32Ptr(2),

182
staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer_test.go
View File

@ -1547,6 +1547,8 @@ func TestReconcileLoadBalancerRule(t *testing.T) {
service v1.Service
loadBalancerSku string
wantLb bool
probeProtocol string
probePath string
expectedProbes []network.Probe
expectedRules []network.LoadBalancingRule
expectedErr error
@ -1561,131 +1563,46 @@ func TestReconcileLoadBalancerRule(t *testing.T) {
service: getTestService("test1", v1.ProtocolTCP, map[string]string{"service.beta.kubernetes.io/azure-load-balancer-disable-tcp-reset": "true"}, false, 80),
loadBalancerSku: "basic",
wantLb: true,
expectedProbes: []network.Probe{
{
Name: to.StringPtr("atest1-TCP-80"),
ProbePropertiesFormat: &network.ProbePropertiesFormat{
Protocol: network.ProbeProtocol("Tcp"),
Port: to.Int32Ptr(10080),
IntervalInSeconds: to.Int32Ptr(5),
NumberOfProbes: to.Int32Ptr(2),
},
},
},
expectedRules: []network.LoadBalancingRule{
{
Name: to.StringPtr("atest1-TCP-80"),
LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
Protocol: network.TransportProtocol("Tcp"),
FrontendIPConfiguration: &network.SubResource{
ID: to.StringPtr("frontendIPConfigID"),
},
BackendAddressPool: &network.SubResource{
ID: to.StringPtr("backendPoolID"),
},
LoadDistribution: "Default",
FrontendPort: to.Int32Ptr(80),
BackendPort: to.Int32Ptr(80),
EnableFloatingIP: to.BoolPtr(true),
DisableOutboundSnat: to.BoolPtr(false),
IdleTimeoutInMinutes: to.Int32Ptr(0),
Probe: &network.SubResource{
ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/" +
"Microsoft.Network/loadBalancers/lbname/probes/atest1-TCP-80"),
},
EnableTCPReset: nil,
},
},
},
expectedProbes: getDefaultTestProbes("Tcp", ""),
expectedRules: getDefaultTestRules(false),
},
{
desc: "reconcileLoadBalancerRule shall return corresponding probe and lbRule (slb without tcp reset)",
service: getTestService("test1", v1.ProtocolTCP, map[string]string{"service.beta.kubernetes.io/azure-load-balancer-disable-tcp-reset": "True"}, false, 80),
loadBalancerSku: "standard",
wantLb: true,
expectedProbes: []network.Probe{
{
Name: to.StringPtr("atest1-TCP-80"),
ProbePropertiesFormat: &network.ProbePropertiesFormat{
Protocol: network.ProbeProtocol("Tcp"),
Port: to.Int32Ptr(10080),
IntervalInSeconds: to.Int32Ptr(5),
NumberOfProbes: to.Int32Ptr(2),
},
},
},
expectedRules: []network.LoadBalancingRule{
{
Name: to.StringPtr("atest1-TCP-80"),
LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
Protocol: network.TransportProtocol("Tcp"),
FrontendIPConfiguration: &network.SubResource{
ID: to.StringPtr("frontendIPConfigID"),
},
BackendAddressPool: &network.SubResource{
ID: to.StringPtr("backendPoolID"),
},
LoadDistribution: "Default",
FrontendPort: to.Int32Ptr(80),
BackendPort: to.Int32Ptr(80),
EnableFloatingIP: to.BoolPtr(true),
DisableOutboundSnat: to.BoolPtr(false),
IdleTimeoutInMinutes: to.Int32Ptr(0),
Probe: &network.SubResource{
ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/" +
"Microsoft.Network/loadBalancers/lbname/probes/atest1-TCP-80"),
},
EnableTCPReset: to.BoolPtr(true),
},
},
},
expectedProbes: getDefaultTestProbes("Tcp", ""),
expectedRules: getDefaultTestRules(true),
},
{
desc: "reconcileLoadBalancerRule shall return corresponding probe and lbRule(slb with tcp reset)",
service: getTestService("test1", v1.ProtocolTCP, nil, false, 80),
loadBalancerSku: "standard",
wantLb: true,
expectedProbes: []network.Probe{
{
Name: to.StringPtr("atest1-TCP-80"),
ProbePropertiesFormat: &network.ProbePropertiesFormat{
Protocol: network.ProbeProtocol("Tcp"),
Port: to.Int32Ptr(10080),
IntervalInSeconds: to.Int32Ptr(5),
NumberOfProbes: to.Int32Ptr(2),
},
},
},
expectedRules: []network.LoadBalancingRule{
{
Name: to.StringPtr("atest1-TCP-80"),
LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
Protocol: network.TransportProtocol("Tcp"),
FrontendIPConfiguration: &network.SubResource{
ID: to.StringPtr("frontendIPConfigID"),
},
BackendAddressPool: &network.SubResource{
ID: to.StringPtr("backendPoolID"),
},
LoadDistribution: "Default",
FrontendPort: to.Int32Ptr(80),
BackendPort: to.Int32Ptr(80),
EnableFloatingIP: to.BoolPtr(true),
DisableOutboundSnat: to.BoolPtr(false),
IdleTimeoutInMinutes: to.Int32Ptr(0),
Probe: &network.SubResource{
ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/" +
"Microsoft.Network/loadBalancers/lbname/probes/atest1-TCP-80"),
},
EnableTCPReset: to.BoolPtr(true),
},
},
},
expectedProbes: getDefaultTestProbes("Tcp", ""),
expectedRules: getDefaultTestRules(true),
},
{
desc: "reconcileLoadBalancerRule shall respect the probe protocol and path configuration in the config file",
service: getTestService("test1", v1.ProtocolTCP, nil, false, 80),
loadBalancerSku: "standard",
wantLb: true,
probeProtocol: "http",
probePath: "/healthy",
expectedProbes: getDefaultTestProbes("http", "/healthy"),
expectedRules: getDefaultTestRules(true),
},
}
for i, test := range testCases {
az := GetTestCloud(ctrl)
az.Config.LoadBalancerSku = test.loadBalancerSku
service := test.service
if test.probeProtocol != "" {
service.Annotations[ServiceAnnotationLoadBalancerHealthProbeProtocol] = test.probeProtocol
}
if test.probePath != "" {
service.Annotations[ServiceAnnotationLoadBalancerHealthProbeRequestPath] = test.probePath
}
probe, lbrule, err := az.reconcileLoadBalancerRule(&test.service, test.wantLb,
"frontendIPConfigID", "backendPoolID", "lbname", to.Int32Ptr(0))
@ -1699,6 +1616,55 @@ func TestReconcileLoadBalancerRule(t *testing.T) {
}
}
func getDefaultTestProbes(protocol, path string) []network.Probe {
expectedProbes := []network.Probe{
{
Name: to.StringPtr("atest1-TCP-80"),
ProbePropertiesFormat: &network.ProbePropertiesFormat{
Protocol: network.ProbeProtocol(protocol),
Port: to.Int32Ptr(10080),
IntervalInSeconds: to.Int32Ptr(5),
NumberOfProbes: to.Int32Ptr(2),
},
},
}
if path != "" {
expectedProbes[0].RequestPath = to.StringPtr(path)
}
return expectedProbes
}
func getDefaultTestRules(enableTCPReset bool) []network.LoadBalancingRule {
expectedRules := []network.LoadBalancingRule{
{
Name: to.StringPtr("atest1-TCP-80"),
LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
Protocol: network.TransportProtocol("Tcp"),
FrontendIPConfiguration: &network.SubResource{
ID: to.StringPtr("frontendIPConfigID"),
},
BackendAddressPool: &network.SubResource{
ID: to.StringPtr("backendPoolID"),
},
LoadDistribution: "Default",
FrontendPort: to.Int32Ptr(80),
BackendPort: to.Int32Ptr(80),
EnableFloatingIP: to.BoolPtr(true),
DisableOutboundSnat: to.BoolPtr(false),
IdleTimeoutInMinutes: to.Int32Ptr(0),
Probe: &network.SubResource{
ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/" +
"Microsoft.Network/loadBalancers/lbname/probes/atest1-TCP-80"),
},
},
},
}
if enableTCPReset {
expectedRules[0].EnableTCPReset = to.BoolPtr(true)
}
return expectedRules
}
func getTestLoadBalancer(name, rgName, clusterName, identifier *string, service v1.Service, lbSku string) network.LoadBalancer {
lb := network.LoadBalancer{
Name: name,