Merge pull request #96338 from nilo19/feature/support-additional-probe-protocol

Support customize load balancer health probe protocol and request path

staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go

@@ -104,6 +104,15 @@ const (
 	// TODO(feiskyer): disable-tcp-reset annotations has been depracated since v1.18, it would removed on v1.20.
 	ServiceAnnotationLoadBalancerDisableTCPReset = "service.beta.kubernetes.io/azure-load-balancer-disable-tcp-reset"
 
+	// ServiceAnnotationLoadBalancerHealthProbeProtocol determines the network protocol that the load balancer health probe use.
+	// If not set, the local service would use the HTTP and the cluster service would use the TCP by default.
+	ServiceAnnotationLoadBalancerHealthProbeProtocol = "service.beta.kubernetes.io/azure-load-balancer-health-probe-protocol"
+
+	// ServiceAnnotationLoadBalancerHealthProbeRequestPath determines the request path of the load balancer health probe.
+	// This is only useful for the HTTP and HTTPS, and would be ignored when using TCP. If not set,
+	// `/healthz` would be configured by default.
+	ServiceAnnotationLoadBalancerHealthProbeRequestPath = "service.beta.kubernetes.io/azure-load-balancer-health-probe-request-path"
+
 	// serviceTagKey is the service key applied for public IP tags.
 	serviceTagKey = "service"
 	// clusterNameKey is the cluster name key applied for public IP tags.
@@ -1561,6 +1570,23 @@ func (az *Cloud) checkLoadBalancerResourcesConflicted(
 	return nil
 }
 
+func parseHealthProbeProtocolAndPath(service *v1.Service) (string, string) {
+	var protocol, path string
+	if v, ok := service.Annotations[ServiceAnnotationLoadBalancerHealthProbeProtocol]; ok {
+		protocol = v
+	} else {
+		return protocol, path
+	}
+	// ignore the request path if using TCP
+	if strings.EqualFold(protocol, string(network.ProbeProtocolHTTP)) ||
+		strings.EqualFold(protocol, string(network.ProbeProtocolHTTPS)) {
+		if v, ok := service.Annotations[ServiceAnnotationLoadBalancerHealthProbeRequestPath]; ok {
+			path = v
+		}
+	}
+	return protocol, path
+}
+
 func (az *Cloud) reconcileLoadBalancerRule(
 	service *v1.Service,
 	wantLb bool,
@@ -1606,14 +1632,21 @@ func (az *Cloud) reconcileLoadBalancerRule(
 				return expectedProbes, expectedRules, err
 			}
 
+			probeProtocol, requestPath := parseHealthProbeProtocolAndPath(service)
 			if servicehelpers.NeedsHealthCheck(service) {
 				podPresencePath, podPresencePort := servicehelpers.GetServiceHealthCheckPathPort(service)
+				if probeProtocol == "" {
+					probeProtocol = string(network.ProbeProtocolHTTP)
+				}
+				if requestPath == "" {
+					requestPath = podPresencePath
+				}
 
 				expectedProbes = append(expectedProbes, network.Probe{
 					Name: &lbRuleName,
 					ProbePropertiesFormat: &network.ProbePropertiesFormat{
-						RequestPath:       to.StringPtr(podPresencePath),
+						RequestPath:       to.StringPtr(requestPath),
-						Protocol:          network.ProbeProtocolHTTP,
+						Protocol:          network.ProbeProtocol(probeProtocol),
 						Port:              to.Int32Ptr(podPresencePort),
 						IntervalInSeconds: to.Int32Ptr(5),
 						NumberOfProbes:    to.Int32Ptr(2),
@@ -1621,10 +1654,22 @@ func (az *Cloud) reconcileLoadBalancerRule(
 				})
 			} else if protocol != v1.ProtocolUDP && protocol != v1.ProtocolSCTP {
 				// we only add the expected probe if we're doing TCP
+				if probeProtocol == "" {
+					probeProtocol = string(*probeProto)
+				}
+				var actualPath *string
+				if !strings.EqualFold(probeProtocol, string(network.ProbeProtocolTCP)) {
+					if requestPath != "" {
+						actualPath = to.StringPtr(requestPath)
+					} else {
+						actualPath = to.StringPtr("/healthz")
+					}
+				}
 				expectedProbes = append(expectedProbes, network.Probe{
 					Name: &lbRuleName,
 					ProbePropertiesFormat: &network.ProbePropertiesFormat{
-						Protocol:          *probeProto,
+						Protocol:          network.ProbeProtocol(probeProtocol),
+						RequestPath:       actualPath,
 						Port:              to.Int32Ptr(port.NodePort),
 						IntervalInSeconds: to.Int32Ptr(5),
 						NumberOfProbes:    to.Int32Ptr(2),

staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer_test.go

@@ -1547,6 +1547,8 @@ func TestReconcileLoadBalancerRule(t *testing.T) {
 		service         v1.Service
 		loadBalancerSku string
 		wantLb          bool
+		probeProtocol   string
+		probePath       string
 		expectedProbes  []network.Probe
 		expectedRules   []network.LoadBalancingRule
 		expectedErr     error
@@ -1561,131 +1563,46 @@ func TestReconcileLoadBalancerRule(t *testing.T) {
 			service:         getTestService("test1", v1.ProtocolTCP, map[string]string{"service.beta.kubernetes.io/azure-load-balancer-disable-tcp-reset": "true"}, false, 80),
 			loadBalancerSku: "basic",
 			wantLb:          true,
-			expectedProbes: []network.Probe{
+			expectedProbes:  getDefaultTestProbes("Tcp", ""),
-				{
+			expectedRules:   getDefaultTestRules(false),
-					Name: to.StringPtr("atest1-TCP-80"),
-					ProbePropertiesFormat: &network.ProbePropertiesFormat{
-						Protocol:          network.ProbeProtocol("Tcp"),
-						Port:              to.Int32Ptr(10080),
-						IntervalInSeconds: to.Int32Ptr(5),
-						NumberOfProbes:    to.Int32Ptr(2),
-					},
-				},
-			},
-			expectedRules: []network.LoadBalancingRule{
-				{
-					Name: to.StringPtr("atest1-TCP-80"),
-					LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
-						Protocol: network.TransportProtocol("Tcp"),
-						FrontendIPConfiguration: &network.SubResource{
-							ID: to.StringPtr("frontendIPConfigID"),
-						},
-						BackendAddressPool: &network.SubResource{
-							ID: to.StringPtr("backendPoolID"),
-						},
-						LoadDistribution:     "Default",
-						FrontendPort:         to.Int32Ptr(80),
-						BackendPort:          to.Int32Ptr(80),
-						EnableFloatingIP:     to.BoolPtr(true),
-						DisableOutboundSnat:  to.BoolPtr(false),
-						IdleTimeoutInMinutes: to.Int32Ptr(0),
-						Probe: &network.SubResource{
-							ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/" +
-								"Microsoft.Network/loadBalancers/lbname/probes/atest1-TCP-80"),
-						},
-						EnableTCPReset: nil,
-					},
-				},
-			},
 		},
 		{
 			desc:            "reconcileLoadBalancerRule shall return corresponding probe and lbRule (slb without tcp reset)",
 			service:         getTestService("test1", v1.ProtocolTCP, map[string]string{"service.beta.kubernetes.io/azure-load-balancer-disable-tcp-reset": "True"}, false, 80),
 			loadBalancerSku: "standard",
 			wantLb:          true,
-			expectedProbes: []network.Probe{
+			expectedProbes:  getDefaultTestProbes("Tcp", ""),
-				{
+			expectedRules:   getDefaultTestRules(true),
-					Name: to.StringPtr("atest1-TCP-80"),
-					ProbePropertiesFormat: &network.ProbePropertiesFormat{
-						Protocol:          network.ProbeProtocol("Tcp"),
-						Port:              to.Int32Ptr(10080),
-						IntervalInSeconds: to.Int32Ptr(5),
-						NumberOfProbes:    to.Int32Ptr(2),
-					},
-				},
-			},
-			expectedRules: []network.LoadBalancingRule{
-				{
-					Name: to.StringPtr("atest1-TCP-80"),
-					LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
-						Protocol: network.TransportProtocol("Tcp"),
-						FrontendIPConfiguration: &network.SubResource{
-							ID: to.StringPtr("frontendIPConfigID"),
-						},
-						BackendAddressPool: &network.SubResource{
-							ID: to.StringPtr("backendPoolID"),
-						},
-						LoadDistribution:     "Default",
-						FrontendPort:         to.Int32Ptr(80),
-						BackendPort:          to.Int32Ptr(80),
-						EnableFloatingIP:     to.BoolPtr(true),
-						DisableOutboundSnat:  to.BoolPtr(false),
-						IdleTimeoutInMinutes: to.Int32Ptr(0),
-						Probe: &network.SubResource{
-							ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/" +
-								"Microsoft.Network/loadBalancers/lbname/probes/atest1-TCP-80"),
-						},
-						EnableTCPReset: to.BoolPtr(true),
-					},
-				},
-			},
 		},
 		{
 			desc:            "reconcileLoadBalancerRule shall return corresponding probe and lbRule(slb with tcp reset)",
 			service:         getTestService("test1", v1.ProtocolTCP, nil, false, 80),
 			loadBalancerSku: "standard",
 			wantLb:          true,
-			expectedProbes: []network.Probe{
+			expectedProbes:  getDefaultTestProbes("Tcp", ""),
-				{
+			expectedRules:   getDefaultTestRules(true),
-					Name: to.StringPtr("atest1-TCP-80"),
+		},
-					ProbePropertiesFormat: &network.ProbePropertiesFormat{
+		{
-						Protocol:          network.ProbeProtocol("Tcp"),
+			desc:            "reconcileLoadBalancerRule shall respect the probe protocol and path configuration in the config file",
-						Port:              to.Int32Ptr(10080),
+			service:         getTestService("test1", v1.ProtocolTCP, nil, false, 80),
-						IntervalInSeconds: to.Int32Ptr(5),
+			loadBalancerSku: "standard",
-						NumberOfProbes:    to.Int32Ptr(2),
+			wantLb:          true,
-					},
+			probeProtocol:   "http",
-				},
+			probePath:       "/healthy",
-			},
+			expectedProbes:  getDefaultTestProbes("http", "/healthy"),
-			expectedRules: []network.LoadBalancingRule{
+			expectedRules:   getDefaultTestRules(true),
-				{
-					Name: to.StringPtr("atest1-TCP-80"),
-					LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
-						Protocol: network.TransportProtocol("Tcp"),
-						FrontendIPConfiguration: &network.SubResource{
-							ID: to.StringPtr("frontendIPConfigID"),
-						},
-						BackendAddressPool: &network.SubResource{
-							ID: to.StringPtr("backendPoolID"),
-						},
-						LoadDistribution:     "Default",
-						FrontendPort:         to.Int32Ptr(80),
-						BackendPort:          to.Int32Ptr(80),
-						EnableFloatingIP:     to.BoolPtr(true),
-						DisableOutboundSnat:  to.BoolPtr(false),
-						IdleTimeoutInMinutes: to.Int32Ptr(0),
-						Probe: &network.SubResource{
-							ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/" +
-								"Microsoft.Network/loadBalancers/lbname/probes/atest1-TCP-80"),
-						},
-						EnableTCPReset: to.BoolPtr(true),
-					},
-				},
-			},
 		},
 	}
 	for i, test := range testCases {
 		az := GetTestCloud(ctrl)
 		az.Config.LoadBalancerSku = test.loadBalancerSku
+		service := test.service
+		if test.probeProtocol != "" {
+			service.Annotations[ServiceAnnotationLoadBalancerHealthProbeProtocol] = test.probeProtocol
+		}
+		if test.probePath != "" {
+			service.Annotations[ServiceAnnotationLoadBalancerHealthProbeRequestPath] = test.probePath
+		}
 		probe, lbrule, err := az.reconcileLoadBalancerRule(&test.service, test.wantLb,
 			"frontendIPConfigID", "backendPoolID", "lbname", to.Int32Ptr(0))
 
@@ -1699,6 +1616,55 @@ func TestReconcileLoadBalancerRule(t *testing.T) {
 	}
 }
 
+func getDefaultTestProbes(protocol, path string) []network.Probe {
+	expectedProbes := []network.Probe{
+		{
+			Name: to.StringPtr("atest1-TCP-80"),
+			ProbePropertiesFormat: &network.ProbePropertiesFormat{
+				Protocol:          network.ProbeProtocol(protocol),
+				Port:              to.Int32Ptr(10080),
+				IntervalInSeconds: to.Int32Ptr(5),
+				NumberOfProbes:    to.Int32Ptr(2),
+			},
+		},
+	}
+	if path != "" {
+		expectedProbes[0].RequestPath = to.StringPtr(path)
+	}
+	return expectedProbes
+}
+
+func getDefaultTestRules(enableTCPReset bool) []network.LoadBalancingRule {
+	expectedRules := []network.LoadBalancingRule{
+		{
+			Name: to.StringPtr("atest1-TCP-80"),
+			LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
+				Protocol: network.TransportProtocol("Tcp"),
+				FrontendIPConfiguration: &network.SubResource{
+					ID: to.StringPtr("frontendIPConfigID"),
+				},
+				BackendAddressPool: &network.SubResource{
+					ID: to.StringPtr("backendPoolID"),
+				},
+				LoadDistribution:     "Default",
+				FrontendPort:         to.Int32Ptr(80),
+				BackendPort:          to.Int32Ptr(80),
+				EnableFloatingIP:     to.BoolPtr(true),
+				DisableOutboundSnat:  to.BoolPtr(false),
+				IdleTimeoutInMinutes: to.Int32Ptr(0),
+				Probe: &network.SubResource{
+					ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/" +
+						"Microsoft.Network/loadBalancers/lbname/probes/atest1-TCP-80"),
+				},
+			},
+		},
+	}
+	if enableTCPReset {
+		expectedRules[0].EnableTCPReset = to.BoolPtr(true)
+	}
+	return expectedRules
+}
+
 func getTestLoadBalancer(name, rgName, clusterName, identifier *string, service v1.Service, lbSku string) network.LoadBalancer {
 	lb := network.LoadBalancer{
 		Name: name,