mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-24 04:06:03 +00:00
Merge pull request #96338 from nilo19/feature/support-additional-probe-protocol
Support customize load balancer health probe protocol and request path
This commit is contained in:
commit
59405cc2b4
@ -104,6 +104,15 @@ const (
|
||||
// TODO(feiskyer): disable-tcp-reset annotations has been depracated since v1.18, it would removed on v1.20.
|
||||
ServiceAnnotationLoadBalancerDisableTCPReset = "service.beta.kubernetes.io/azure-load-balancer-disable-tcp-reset"
|
||||
|
||||
// ServiceAnnotationLoadBalancerHealthProbeProtocol determines the network protocol that the load balancer health probe use.
|
||||
// If not set, the local service would use the HTTP and the cluster service would use the TCP by default.
|
||||
ServiceAnnotationLoadBalancerHealthProbeProtocol = "service.beta.kubernetes.io/azure-load-balancer-health-probe-protocol"
|
||||
|
||||
// ServiceAnnotationLoadBalancerHealthProbeRequestPath determines the request path of the load balancer health probe.
|
||||
// This is only useful for the HTTP and HTTPS, and would be ignored when using TCP. If not set,
|
||||
// `/healthz` would be configured by default.
|
||||
ServiceAnnotationLoadBalancerHealthProbeRequestPath = "service.beta.kubernetes.io/azure-load-balancer-health-probe-request-path"
|
||||
|
||||
// serviceTagKey is the service key applied for public IP tags.
|
||||
serviceTagKey = "service"
|
||||
// clusterNameKey is the cluster name key applied for public IP tags.
|
||||
@ -1561,6 +1570,23 @@ func (az *Cloud) checkLoadBalancerResourcesConflicted(
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseHealthProbeProtocolAndPath(service *v1.Service) (string, string) {
|
||||
var protocol, path string
|
||||
if v, ok := service.Annotations[ServiceAnnotationLoadBalancerHealthProbeProtocol]; ok {
|
||||
protocol = v
|
||||
} else {
|
||||
return protocol, path
|
||||
}
|
||||
// ignore the request path if using TCP
|
||||
if strings.EqualFold(protocol, string(network.ProbeProtocolHTTP)) ||
|
||||
strings.EqualFold(protocol, string(network.ProbeProtocolHTTPS)) {
|
||||
if v, ok := service.Annotations[ServiceAnnotationLoadBalancerHealthProbeRequestPath]; ok {
|
||||
path = v
|
||||
}
|
||||
}
|
||||
return protocol, path
|
||||
}
|
||||
|
||||
func (az *Cloud) reconcileLoadBalancerRule(
|
||||
service *v1.Service,
|
||||
wantLb bool,
|
||||
@ -1606,14 +1632,21 @@ func (az *Cloud) reconcileLoadBalancerRule(
|
||||
return expectedProbes, expectedRules, err
|
||||
}
|
||||
|
||||
probeProtocol, requestPath := parseHealthProbeProtocolAndPath(service)
|
||||
if servicehelpers.NeedsHealthCheck(service) {
|
||||
podPresencePath, podPresencePort := servicehelpers.GetServiceHealthCheckPathPort(service)
|
||||
if probeProtocol == "" {
|
||||
probeProtocol = string(network.ProbeProtocolHTTP)
|
||||
}
|
||||
if requestPath == "" {
|
||||
requestPath = podPresencePath
|
||||
}
|
||||
|
||||
expectedProbes = append(expectedProbes, network.Probe{
|
||||
Name: &lbRuleName,
|
||||
ProbePropertiesFormat: &network.ProbePropertiesFormat{
|
||||
RequestPath: to.StringPtr(podPresencePath),
|
||||
Protocol: network.ProbeProtocolHTTP,
|
||||
RequestPath: to.StringPtr(requestPath),
|
||||
Protocol: network.ProbeProtocol(probeProtocol),
|
||||
Port: to.Int32Ptr(podPresencePort),
|
||||
IntervalInSeconds: to.Int32Ptr(5),
|
||||
NumberOfProbes: to.Int32Ptr(2),
|
||||
@ -1621,10 +1654,22 @@ func (az *Cloud) reconcileLoadBalancerRule(
|
||||
})
|
||||
} else if protocol != v1.ProtocolUDP && protocol != v1.ProtocolSCTP {
|
||||
// we only add the expected probe if we're doing TCP
|
||||
if probeProtocol == "" {
|
||||
probeProtocol = string(*probeProto)
|
||||
}
|
||||
var actualPath *string
|
||||
if !strings.EqualFold(probeProtocol, string(network.ProbeProtocolTCP)) {
|
||||
if requestPath != "" {
|
||||
actualPath = to.StringPtr(requestPath)
|
||||
} else {
|
||||
actualPath = to.StringPtr("/healthz")
|
||||
}
|
||||
}
|
||||
expectedProbes = append(expectedProbes, network.Probe{
|
||||
Name: &lbRuleName,
|
||||
ProbePropertiesFormat: &network.ProbePropertiesFormat{
|
||||
Protocol: *probeProto,
|
||||
Protocol: network.ProbeProtocol(probeProtocol),
|
||||
RequestPath: actualPath,
|
||||
Port: to.Int32Ptr(port.NodePort),
|
||||
IntervalInSeconds: to.Int32Ptr(5),
|
||||
NumberOfProbes: to.Int32Ptr(2),
|
||||
|
@ -1547,6 +1547,8 @@ func TestReconcileLoadBalancerRule(t *testing.T) {
|
||||
service v1.Service
|
||||
loadBalancerSku string
|
||||
wantLb bool
|
||||
probeProtocol string
|
||||
probePath string
|
||||
expectedProbes []network.Probe
|
||||
expectedRules []network.LoadBalancingRule
|
||||
expectedErr error
|
||||
@ -1561,131 +1563,46 @@ func TestReconcileLoadBalancerRule(t *testing.T) {
|
||||
service: getTestService("test1", v1.ProtocolTCP, map[string]string{"service.beta.kubernetes.io/azure-load-balancer-disable-tcp-reset": "true"}, false, 80),
|
||||
loadBalancerSku: "basic",
|
||||
wantLb: true,
|
||||
expectedProbes: []network.Probe{
|
||||
{
|
||||
Name: to.StringPtr("atest1-TCP-80"),
|
||||
ProbePropertiesFormat: &network.ProbePropertiesFormat{
|
||||
Protocol: network.ProbeProtocol("Tcp"),
|
||||
Port: to.Int32Ptr(10080),
|
||||
IntervalInSeconds: to.Int32Ptr(5),
|
||||
NumberOfProbes: to.Int32Ptr(2),
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedRules: []network.LoadBalancingRule{
|
||||
{
|
||||
Name: to.StringPtr("atest1-TCP-80"),
|
||||
LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
|
||||
Protocol: network.TransportProtocol("Tcp"),
|
||||
FrontendIPConfiguration: &network.SubResource{
|
||||
ID: to.StringPtr("frontendIPConfigID"),
|
||||
},
|
||||
BackendAddressPool: &network.SubResource{
|
||||
ID: to.StringPtr("backendPoolID"),
|
||||
},
|
||||
LoadDistribution: "Default",
|
||||
FrontendPort: to.Int32Ptr(80),
|
||||
BackendPort: to.Int32Ptr(80),
|
||||
EnableFloatingIP: to.BoolPtr(true),
|
||||
DisableOutboundSnat: to.BoolPtr(false),
|
||||
IdleTimeoutInMinutes: to.Int32Ptr(0),
|
||||
Probe: &network.SubResource{
|
||||
ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/" +
|
||||
"Microsoft.Network/loadBalancers/lbname/probes/atest1-TCP-80"),
|
||||
},
|
||||
EnableTCPReset: nil,
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedProbes: getDefaultTestProbes("Tcp", ""),
|
||||
expectedRules: getDefaultTestRules(false),
|
||||
},
|
||||
{
|
||||
desc: "reconcileLoadBalancerRule shall return corresponding probe and lbRule (slb without tcp reset)",
|
||||
service: getTestService("test1", v1.ProtocolTCP, map[string]string{"service.beta.kubernetes.io/azure-load-balancer-disable-tcp-reset": "True"}, false, 80),
|
||||
loadBalancerSku: "standard",
|
||||
wantLb: true,
|
||||
expectedProbes: []network.Probe{
|
||||
{
|
||||
Name: to.StringPtr("atest1-TCP-80"),
|
||||
ProbePropertiesFormat: &network.ProbePropertiesFormat{
|
||||
Protocol: network.ProbeProtocol("Tcp"),
|
||||
Port: to.Int32Ptr(10080),
|
||||
IntervalInSeconds: to.Int32Ptr(5),
|
||||
NumberOfProbes: to.Int32Ptr(2),
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedRules: []network.LoadBalancingRule{
|
||||
{
|
||||
Name: to.StringPtr("atest1-TCP-80"),
|
||||
LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
|
||||
Protocol: network.TransportProtocol("Tcp"),
|
||||
FrontendIPConfiguration: &network.SubResource{
|
||||
ID: to.StringPtr("frontendIPConfigID"),
|
||||
},
|
||||
BackendAddressPool: &network.SubResource{
|
||||
ID: to.StringPtr("backendPoolID"),
|
||||
},
|
||||
LoadDistribution: "Default",
|
||||
FrontendPort: to.Int32Ptr(80),
|
||||
BackendPort: to.Int32Ptr(80),
|
||||
EnableFloatingIP: to.BoolPtr(true),
|
||||
DisableOutboundSnat: to.BoolPtr(false),
|
||||
IdleTimeoutInMinutes: to.Int32Ptr(0),
|
||||
Probe: &network.SubResource{
|
||||
ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/" +
|
||||
"Microsoft.Network/loadBalancers/lbname/probes/atest1-TCP-80"),
|
||||
},
|
||||
EnableTCPReset: to.BoolPtr(true),
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedProbes: getDefaultTestProbes("Tcp", ""),
|
||||
expectedRules: getDefaultTestRules(true),
|
||||
},
|
||||
{
|
||||
desc: "reconcileLoadBalancerRule shall return corresponding probe and lbRule(slb with tcp reset)",
|
||||
service: getTestService("test1", v1.ProtocolTCP, nil, false, 80),
|
||||
loadBalancerSku: "standard",
|
||||
wantLb: true,
|
||||
expectedProbes: []network.Probe{
|
||||
{
|
||||
Name: to.StringPtr("atest1-TCP-80"),
|
||||
ProbePropertiesFormat: &network.ProbePropertiesFormat{
|
||||
Protocol: network.ProbeProtocol("Tcp"),
|
||||
Port: to.Int32Ptr(10080),
|
||||
IntervalInSeconds: to.Int32Ptr(5),
|
||||
NumberOfProbes: to.Int32Ptr(2),
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedRules: []network.LoadBalancingRule{
|
||||
{
|
||||
Name: to.StringPtr("atest1-TCP-80"),
|
||||
LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
|
||||
Protocol: network.TransportProtocol("Tcp"),
|
||||
FrontendIPConfiguration: &network.SubResource{
|
||||
ID: to.StringPtr("frontendIPConfigID"),
|
||||
},
|
||||
BackendAddressPool: &network.SubResource{
|
||||
ID: to.StringPtr("backendPoolID"),
|
||||
},
|
||||
LoadDistribution: "Default",
|
||||
FrontendPort: to.Int32Ptr(80),
|
||||
BackendPort: to.Int32Ptr(80),
|
||||
EnableFloatingIP: to.BoolPtr(true),
|
||||
DisableOutboundSnat: to.BoolPtr(false),
|
||||
IdleTimeoutInMinutes: to.Int32Ptr(0),
|
||||
Probe: &network.SubResource{
|
||||
ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/" +
|
||||
"Microsoft.Network/loadBalancers/lbname/probes/atest1-TCP-80"),
|
||||
},
|
||||
EnableTCPReset: to.BoolPtr(true),
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedProbes: getDefaultTestProbes("Tcp", ""),
|
||||
expectedRules: getDefaultTestRules(true),
|
||||
},
|
||||
{
|
||||
desc: "reconcileLoadBalancerRule shall respect the probe protocol and path configuration in the config file",
|
||||
service: getTestService("test1", v1.ProtocolTCP, nil, false, 80),
|
||||
loadBalancerSku: "standard",
|
||||
wantLb: true,
|
||||
probeProtocol: "http",
|
||||
probePath: "/healthy",
|
||||
expectedProbes: getDefaultTestProbes("http", "/healthy"),
|
||||
expectedRules: getDefaultTestRules(true),
|
||||
},
|
||||
}
|
||||
for i, test := range testCases {
|
||||
az := GetTestCloud(ctrl)
|
||||
az.Config.LoadBalancerSku = test.loadBalancerSku
|
||||
service := test.service
|
||||
if test.probeProtocol != "" {
|
||||
service.Annotations[ServiceAnnotationLoadBalancerHealthProbeProtocol] = test.probeProtocol
|
||||
}
|
||||
if test.probePath != "" {
|
||||
service.Annotations[ServiceAnnotationLoadBalancerHealthProbeRequestPath] = test.probePath
|
||||
}
|
||||
probe, lbrule, err := az.reconcileLoadBalancerRule(&test.service, test.wantLb,
|
||||
"frontendIPConfigID", "backendPoolID", "lbname", to.Int32Ptr(0))
|
||||
|
||||
@ -1699,6 +1616,55 @@ func TestReconcileLoadBalancerRule(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func getDefaultTestProbes(protocol, path string) []network.Probe {
|
||||
expectedProbes := []network.Probe{
|
||||
{
|
||||
Name: to.StringPtr("atest1-TCP-80"),
|
||||
ProbePropertiesFormat: &network.ProbePropertiesFormat{
|
||||
Protocol: network.ProbeProtocol(protocol),
|
||||
Port: to.Int32Ptr(10080),
|
||||
IntervalInSeconds: to.Int32Ptr(5),
|
||||
NumberOfProbes: to.Int32Ptr(2),
|
||||
},
|
||||
},
|
||||
}
|
||||
if path != "" {
|
||||
expectedProbes[0].RequestPath = to.StringPtr(path)
|
||||
}
|
||||
return expectedProbes
|
||||
}
|
||||
|
||||
func getDefaultTestRules(enableTCPReset bool) []network.LoadBalancingRule {
|
||||
expectedRules := []network.LoadBalancingRule{
|
||||
{
|
||||
Name: to.StringPtr("atest1-TCP-80"),
|
||||
LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
|
||||
Protocol: network.TransportProtocol("Tcp"),
|
||||
FrontendIPConfiguration: &network.SubResource{
|
||||
ID: to.StringPtr("frontendIPConfigID"),
|
||||
},
|
||||
BackendAddressPool: &network.SubResource{
|
||||
ID: to.StringPtr("backendPoolID"),
|
||||
},
|
||||
LoadDistribution: "Default",
|
||||
FrontendPort: to.Int32Ptr(80),
|
||||
BackendPort: to.Int32Ptr(80),
|
||||
EnableFloatingIP: to.BoolPtr(true),
|
||||
DisableOutboundSnat: to.BoolPtr(false),
|
||||
IdleTimeoutInMinutes: to.Int32Ptr(0),
|
||||
Probe: &network.SubResource{
|
||||
ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/" +
|
||||
"Microsoft.Network/loadBalancers/lbname/probes/atest1-TCP-80"),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
if enableTCPReset {
|
||||
expectedRules[0].EnableTCPReset = to.BoolPtr(true)
|
||||
}
|
||||
return expectedRules
|
||||
}
|
||||
|
||||
func getTestLoadBalancer(name, rgName, clusterName, identifier *string, service v1.Service, lbSku string) network.LoadBalancer {
|
||||
lb := network.LoadBalancer{
|
||||
Name: name,
|
||||
|
Loading…
Reference in New Issue
Block a user