github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-23 19:56:01 +00:00
Code Issues Projects Releases Wiki Activity

Implements Service Internal Traffic Policy

Browse Source 
1. Add API definitions;
2. Add feature gate and drops the field when feature gate is not on;
3. Set default values for the field;
4. Add API Validation
5. add kube-proxy iptables and ipvs implementations
6. add tests
This commit is contained in:
Fangyuan Li 2020-11-15 23:59:58 -08:00
parent 9a9a9b014c
commit 7ed2f1d94d
28 changed files with 3463 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2561
api/openapi-spec/swagger.json generated
View File

File diff suppressed because it is too large Load Diff

11
pkg/api/v1/service/util.go
View File

@ -20,7 +20,7 @@ import (
"fmt"
"strings"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
utilnet "k8s.io/utils/net"
)
@ -76,6 +76,15 @@ func RequestsOnlyLocalTraffic(service *v1.Service) bool {
return service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal
}
// RequestsOnlyLocalTrafficForInternal checks if service prefers Node Local
// endpoints for internal traffic
func RequestsOnlyLocalTrafficForInternal(service *v1.Service) bool {
if service.Spec.InternalTrafficPolicy == nil {
return false
}
return *service.Spec.InternalTrafficPolicy == v1.ServiceInternalTrafficPolicyLocal
}
// NeedsHealthCheck checks if service needs health check.
func NeedsHealthCheck(service *v1.Service) bool {
if service.Spec.Type != v1.ServiceTypeLoadBalancer {

29
pkg/api/v1/service/util_test.go
View File

@ -20,7 +20,7 @@ import (
"strings"
"testing"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
utilnet "k8s.io/utils/net"
)
@ -214,3 +214,30 @@ func TestNeedsHealthCheck(t *testing.T) {
},
})
}
func TestRequestsOnlyLocalTrafficForInternal(t *testing.T) {
checkRequestsOnlyLocalTrafficForInternal := func(expected bool, service *v1.Service) {
res := RequestsOnlyLocalTrafficForInternal(service)
if res != expected {
t.Errorf("Expected internal local traffic = %v, got %v",
expected, res)
}
}
// default InternalTrafficPolicy is nil
checkRequestsOnlyLocalTrafficForInternal(false, &v1.Service{})
local := v1.ServiceInternalTrafficPolicyLocal
checkRequestsOnlyLocalTrafficForInternal(true, &v1.Service{
Spec: v1.ServiceSpec{
InternalTrafficPolicy: &local,
},
})
cluster := v1.ServiceInternalTrafficPolicyCluster
checkRequestsOnlyLocalTrafficForInternal(false, &v1.Service{
Spec: v1.ServiceSpec{
InternalTrafficPolicy: &cluster,
},
})
}

25
pkg/apis/core/types.go
View File

@ -3475,6 +3475,19 @@ const (
ServiceTypeExternalName ServiceType = "ExternalName"
)
// ServiceInternalTrafficPolicyType describes the type of traffic routing for
// internal traffic
type ServiceInternalTrafficPolicyType string
const (
// ServiceInternalTrafficPolicyCluster routes traffic to all endpoints
ServiceInternalTrafficPolicyCluster ServiceInternalTrafficPolicyType = "Cluster"
// ServiceInternalTrafficPolicyLocal only routes to node-local
// endpoints, otherwise drops the traffic
ServiceInternalTrafficPolicyLocal ServiceInternalTrafficPolicyType = "Local"
)
// ServiceExternalTrafficPolicyType string
type ServiceExternalTrafficPolicyType string
@ -3739,9 +3752,19 @@ type ServiceSpec struct {
// implementation (e.g. cloud providers) should ignore Services that set this field.
// This field can only be set when creating or updating a Service to type 'LoadBalancer'.
// Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type.
// featureGate=LoadBalancerClass
// +featureGate=LoadBalancerClass
// +optional
LoadBalancerClass *string
// InternalTrafficPolicy specifies if the cluster internal traffic
// should be routed to all endpoints or node-local endpoints only.
// "Cluster" routes internal traffic to a Service to all endpoints.
// "Local" routes traffic to node-local endpoints only, traffic is
// dropped if no node-local endpoints are ready.
// The default value is "Cluster".
// +featureGate=ServiceInternalTrafficPolicy
// +optional
InternalTrafficPolicy *ServiceInternalTrafficPolicyType
}
// ServicePort represents the port on which the service is exposed

5
pkg/apis/core/v1/defaults.go
View File

@ -131,6 +131,11 @@ func SetDefaults_Service(obj *v1.Service) {
obj.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeCluster
}
if utilfeature.DefaultFeatureGate.Enabled(features.ServiceInternalTrafficPolicy) && obj.Spec.InternalTrafficPolicy == nil {
serviceInternalTrafficPolicyCluster := v1.ServiceInternalTrafficPolicyCluster
obj.Spec.InternalTrafficPolicy = &serviceInternalTrafficPolicyCluster
}
if utilfeature.DefaultFeatureGate.Enabled(features.ServiceLBNodePortControl) {
if obj.Spec.Type == v1.ServiceTypeLoadBalancer {
if obj.Spec.AllocateLoadBalancerNodePorts == nil {

64
pkg/apis/core/v1/defaults_test.go
View File

@ -29,8 +29,11 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/diff"
"k8s.io/apimachinery/pkg/util/intstr"
utilfeature "k8s.io/apiserver/pkg/util/feature"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/kubernetes/pkg/api/legacyscheme"
corev1 "k8s.io/kubernetes/pkg/apis/core/v1"
"k8s.io/kubernetes/pkg/features"
utilpointer "k8s.io/utils/pointer"
// ensure types are installed
@ -1798,3 +1801,64 @@ func TestSetDefaultEnableServiceLinks(t *testing.T) {
t.Errorf("Expected enableServiceLinks value: %+v\ngot: %+v\n", v1.DefaultEnableServiceLinks, *output.Spec.EnableServiceLinks)
}
}
func TestSetDefaultServiceInternalTrafficPolicy(t *testing.T) {
cluster := v1.ServiceInternalTrafficPolicyCluster
local := v1.ServiceInternalTrafficPolicyLocal
testCases := []struct {
name string
expectedInternalTrafficPolicy v1.ServiceInternalTrafficPolicyType
svc v1.Service
featureGateOn bool
}{
{
name: "must set default internalTrafficPolicy",
expectedInternalTrafficPolicy: v1.ServiceInternalTrafficPolicyCluster,
svc: v1.Service{},
featureGateOn: true,
},
{
name: "must not set default internalTrafficPolicy when it's cluster",
expectedInternalTrafficPolicy: v1.ServiceInternalTrafficPolicyCluster,
svc: v1.Service{
Spec: v1.ServiceSpec{
InternalTrafficPolicy: &cluster,
},
},
featureGateOn: true,
},
{
name: "must not set default internalTrafficPolicy when it's local",
expectedInternalTrafficPolicy: v1.ServiceInternalTrafficPolicyLocal,
svc: v1.Service{
Spec: v1.ServiceSpec{
InternalTrafficPolicy: &local,
},
},
featureGateOn: true,
},
{
name: "must not set default internalTrafficPolicy when gate is disabled",
expectedInternalTrafficPolicy: "",
svc: v1.Service{},
featureGateOn: false,
},
}
for _, test := range testCases {
t.Run(test.name, func(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ServiceInternalTrafficPolicy, test.featureGateOn)()
obj := roundTrip(t, runtime.Object(&test.svc))
svc := obj.(*v1.Service)
if test.expectedInternalTrafficPolicy == "" {
if svc.Spec.InternalTrafficPolicy != nil {
t.Fatalf("expected .spec.internalTrafficPolicy: null, got %v", *svc.Spec.InternalTrafficPolicy)
}
} else {
if *svc.Spec.InternalTrafficPolicy != test.expectedInternalTrafficPolicy {
t.Fatalf("expected .spec.internalTrafficPolicy: %v got %v", test.expectedInternalTrafficPolicy, *svc.Spec.InternalTrafficPolicy)
}
}
})
}
}

2
pkg/apis/core/v1/zz_generated.conversion.go generated
View File

@ -7636,6 +7636,7 @@ func autoConvert_v1_ServiceSpec_To_core_ServiceSpec(in *v1.ServiceSpec, out *cor
out.IPFamilyPolicy = (*core.IPFamilyPolicyType)(unsafe.Pointer(in.IPFamilyPolicy))
out.AllocateLoadBalancerNodePorts = (*bool)(unsafe.Pointer(in.AllocateLoadBalancerNodePorts))
out.LoadBalancerClass = (*string)(unsafe.Pointer(in.LoadBalancerClass))
out.InternalTrafficPolicy = (*core.ServiceInternalTrafficPolicyType)(unsafe.Pointer(in.InternalTrafficPolicy))
return nil
}
@ -7664,6 +7665,7 @@ func autoConvert_core_ServiceSpec_To_v1_ServiceSpec(in *core.ServiceSpec, out *v
out.TopologyKeys = *(*[]string)(unsafe.Pointer(&in.TopologyKeys))
out.AllocateLoadBalancerNodePorts = (*bool)(unsafe.Pointer(in.AllocateLoadBalancerNodePorts))
out.LoadBalancerClass = (*string)(unsafe.Pointer(in.LoadBalancerClass))
out.InternalTrafficPolicy = (*v1.ServiceInternalTrafficPolicyType)(unsafe.Pointer(in.InternalTrafficPolicy))
return nil
}

24
pkg/apis/core/validation/validation.go
View File

@ -4171,6 +4171,8 @@ var supportedSessionAffinityType = sets.NewString(string(core.ServiceAffinityCli
var supportedServiceType = sets.NewString(string(core.ServiceTypeClusterIP), string(core.ServiceTypeNodePort),
string(core.ServiceTypeLoadBalancer), string(core.ServiceTypeExternalName))
var supportedServiceInternalTrafficPolicy = sets.NewString(string(core.ServiceInternalTrafficPolicyCluster), string(core.ServiceExternalTrafficPolicyTypeLocal))
var supportedServiceIPFamily = sets.NewString(string(core.IPv4Protocol), string(core.IPv6Protocol))
var supportedServiceIPFamilyPolicy = sets.NewString(string(core.IPFamilyPolicySingleStack), string(core.IPFamilyPolicyPreferDualStack), string(core.IPFamilyPolicyRequireDualStack))
@ -4378,6 +4380,10 @@ func ValidateService(service *core.Service) field.ErrorList {
// external traffic fields
allErrs = append(allErrs, validateServiceExternalTrafficFieldsValue(service)...)
// internal traffic policy field
allErrs = append(allErrs, validateServiceInternalTrafficFieldsValue(service)...)
return allErrs
}
@ -4446,6 +4452,24 @@ func validateServiceExternalTrafficFieldsValue(service *core.Service) field.Erro
return allErrs
}
// validateServiceInternalTrafficFieldsValue validates InternalTraffic related
// spec have legal value.
func validateServiceInternalTrafficFieldsValue(service *core.Service) field.ErrorList {
allErrs := field.ErrorList{}
if utilfeature.DefaultFeatureGate.Enabled(features.ServiceInternalTrafficPolicy) {
if service.Spec.InternalTrafficPolicy == nil {
allErrs = append(allErrs, field.Required(field.NewPath("spec").Child("internalTrafficPolicy"), ""))
}
}
if service.Spec.InternalTrafficPolicy != nil && !supportedServiceInternalTrafficPolicy.Has(string(*service.Spec.InternalTrafficPolicy)) {
allErrs = append(allErrs, field.NotSupported(field.NewPath("spec").Child("internalTrafficPolicy"), *service.Spec.InternalTrafficPolicy, supportedServiceInternalTrafficPolicy.List()))
}
return allErrs
}
// ValidateServiceExternalTrafficFieldsCombination validates if ExternalTrafficPolicy,
// HealthCheckNodePort and Type combination are legal. For update, it should be called
// after clearing externalTraffic related fields for the ease of transitioning between

27
pkg/apis/core/validation/validation_test.go
View File

@ -33,6 +33,7 @@ import (
"k8s.io/apimachinery/pkg/util/validation"
"k8s.io/apimachinery/pkg/util/validation/field"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/component-base/featuregate"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/kubernetes/pkg/apis/core"
"k8s.io/kubernetes/pkg/capabilities"
@ -10135,9 +10136,10 @@ func TestValidateServiceCreate(t *testing.T) {
preferDualStack := core.IPFamilyPolicyPreferDualStack
testCases := []struct {
name string
tweakSvc func(svc *core.Service) // given a basic valid service, each test case can customize it
numErrs int
name string
tweakSvc func(svc *core.Service) // given a basic valid service, each test case can customize it
numErrs int
featureGates []featuregate.Feature
}{
{
name: "missing namespace",
@ -10750,6 +10752,22 @@ func TestValidateServiceCreate(t *testing.T) {
},
numErrs: 1,
},
{
name: "nil internalTraffic field when feature gate is on",
tweakSvc: func(s *core.Service) {
s.Spec.InternalTrafficPolicy = nil
},
featureGates: []featuregate.Feature{features.ServiceInternalTrafficPolicy},
numErrs: 1,
},
{
name: "invalid internalTraffic field",
tweakSvc: func(s *core.Service) {
invalid := core.ServiceInternalTrafficPolicyType("invalid")
s.Spec.InternalTrafficPolicy = &invalid
},
numErrs: 1,
},
{
name: "nagative healthCheckNodePort field",
tweakSvc: func(s *core.Service) {
@ -11323,6 +11341,9 @@ func TestValidateServiceCreate(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
for i := range tc.featureGates {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, tc.featureGates[i], true)()
}
svc := makeValidService()
tc.tweakSvc(&svc)
errs := ValidateServiceCreate(&svc)

5
pkg/apis/core/zz_generated.deepcopy.go generated
View File

@ -5335,6 +5335,11 @@ func (in *ServiceSpec) DeepCopyInto(out *ServiceSpec) {
*out = new(string)
**out = **in
}
if in.InternalTrafficPolicy != nil {
in, out := &in.InternalTrafficPolicy, &out.InternalTrafficPolicy
*out = new(ServiceInternalTrafficPolicyType)
**out = **in
}
return
}

7
pkg/features/kube_features.go
View File

@ -708,6 +708,12 @@ const (
//
// Enable Scope and Namespace fields on IngressClassParametersReference.
IngressClassNamespacedParams featuregate.Feature = "IngressClassNamespacedParams"
// owner: @maplain @andrewsykim
// alpha: v1.21
//
// Enables node-local routing for Service internal traffic
ServiceInternalTrafficPolicy featuregate.Feature = "ServiceInternalTrafficPolicy"
)
func init() {
@ -815,6 +821,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
ServiceLoadBalancerClass: {Default: false, PreRelease: featuregate.Alpha},
LogarithmicScaleDown: {Default: false, PreRelease: featuregate.Alpha},
IngressClassNamespacedParams: {Default: false, PreRelease: featuregate.Alpha},
ServiceInternalTrafficPolicy: {Default: false, PreRelease: featuregate.Alpha},
// inherited features from generic apiserver, relisted here to get a conflict if it is changed
// unintentionally on either side:

7
pkg/proxy/iptables/proxier.go
View File

@ -1029,6 +1029,13 @@ func (proxier *Proxier) syncProxyRules() {
allEndpoints = proxy.FilterTopologyEndpoint(proxier.nodeLabels, svcInfo.TopologyKeys(), allEndpoints)
}
// Service InternalTrafficPolicy is only enabled when all of the
// following are true:
// 1. InternalTrafficPolicy is Local
// 2. ServiceInternalTrafficPolicy feature gate is on
if utilfeature.DefaultFeatureGate.Enabled(features.ServiceInternalTrafficPolicy) && svcInfo.OnlyNodeLocalEndpointsForInternal() {
allEndpoints = proxy.FilterLocalEndpoint(svcInfo.InternalTrafficPolicy(), allEndpoints)
}
readyEndpoints := make([]proxy.Endpoint, 0, len(allEndpoints))
for _, endpoint := range allEndpoints {
if !endpoint.IsReady() {

219
pkg/proxy/iptables/proxier_test.go
View File

@ -32,8 +32,11 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
utilfeature "k8s.io/apiserver/pkg/util/feature"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/component-base/metrics/testutil"
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/proxy"
"k8s.io/kubernetes/pkg/proxy/metrics"
@ -3056,3 +3059,219 @@ func TestProxierMetricsIptablesTotalRules(t *testing.T) {
}
// TODO(thockin): add *more* tests for syncProxyRules() or break it down further and test the pieces.
// This test ensures that the iptables proxier supports translating Endpoints to
// iptables output when internalTrafficPolicy is specified
func TestInternalTrafficPolicyE2E(t *testing.T) {
type endpoint struct {
ip string
hostname string
}
cluster := v1.ServiceInternalTrafficPolicyCluster
local := v1.ServiceInternalTrafficPolicyLocal
clusterExpectedIPTables := `*filter
:KUBE-SERVICES - [0:0]
:KUBE-EXTERNAL-SERVICES - [0:0]
:KUBE-FORWARD - [0:0]
:KUBE-NODEPORTS - [0:0]
-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP
-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT
-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
COMMIT
*nat
:KUBE-SERVICES - [0:0]
:KUBE-NODEPORTS - [0:0]
:KUBE-POSTROUTING - [0:0]
:KUBE-MARK-MASQ - [0:0]
:KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0]
:KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0]
:KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0]
:KUBE-SEP-XGJFVO3L2O5SRFNT - [0:0]
-A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN
-A KUBE-POSTROUTING -j MARK --xor-mark 0x4000
-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE
-A KUBE-MARK-MASQ -j MARK --or-mark 0x4000
-A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.20.1.1/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ
-A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.20.1.1/32 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP
-A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m statistic --mode random --probability 0.3333333333 -j KUBE-SEP-3JOIVZTXZZRGORX4
-A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -s 10.0.1.1/32 -j KUBE-MARK-MASQ
-A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80
-A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m statistic --mode random --probability 0.5000000000 -j KUBE-SEP-IO5XOSKPAXIFQXAJ
-A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -s 10.0.1.2/32 -j KUBE-MARK-MASQ
-A KUBE-SEP-IO5XOSKPAXIFQXAJ -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.2:80
-A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -j KUBE-SEP-XGJFVO3L2O5SRFNT
-A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -s 10.0.1.3/32 -j KUBE-MARK-MASQ
-A KUBE-SEP-XGJFVO3L2O5SRFNT -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.3:80
-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS
COMMIT
`
testCases := []struct {
name string
internalTrafficPolicy *v1.ServiceInternalTrafficPolicyType
featureGateOn bool
endpoints []endpoint
expectEndpointRule bool
expectedIPTablesWithSlice string
}{
{
name: "internalTrafficPolicy is cluster",
internalTrafficPolicy: &cluster,
featureGateOn: true,
endpoints: []endpoint{
{"10.0.1.1", testHostname},
{"10.0.1.2", "host1"},
{"10.0.1.3", "host2"},
},
expectEndpointRule: true,
expectedIPTablesWithSlice: clusterExpectedIPTables,
},
{
name: "internalTrafficPolicy is local and there is non-zero local endpoints",
internalTrafficPolicy: &local,
featureGateOn: true,
endpoints: []endpoint{
{"10.0.1.1", testHostname},
{"10.0.1.2", "host1"},
{"10.0.1.3", "host2"},
},
expectEndpointRule: true,
expectedIPTablesWithSlice: `*filter
:KUBE-SERVICES - [0:0]
:KUBE-EXTERNAL-SERVICES - [0:0]
:KUBE-FORWARD - [0:0]
:KUBE-NODEPORTS - [0:0]
-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP
-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT
-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
COMMIT
*nat
:KUBE-SERVICES - [0:0]
:KUBE-NODEPORTS - [0:0]
:KUBE-POSTROUTING - [0:0]
:KUBE-MARK-MASQ - [0:0]
:KUBE-SVC-AQI2S6QIMU7PVVRP - [0:0]
:KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0]
-A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN
-A KUBE-POSTROUTING -j MARK --xor-mark 0x4000
-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE
-A KUBE-MARK-MASQ -j MARK --or-mark 0x4000
-A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.20.1.1/32 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ
-A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.20.1.1/32 --dport 80 -j KUBE-SVC-AQI2S6QIMU7PVVRP
-A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -j KUBE-SEP-3JOIVZTXZZRGORX4
-A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -s 10.0.1.1/32 -j KUBE-MARK-MASQ
-A KUBE-SEP-3JOIVZTXZZRGORX4 -m comment --comment ns1/svc1 -m tcp -p tcp -j DNAT --to-destination 10.0.1.1:80
-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS
COMMIT
`,
},
{
name: "internalTrafficPolicy is local and there is zero local endpoint",
internalTrafficPolicy: &local,
featureGateOn: true,
endpoints: []endpoint{
{"10.0.1.1", "host0"},
{"10.0.1.2", "host1"},
{"10.0.1.3", "host2"},
},
expectEndpointRule: false,
expectedIPTablesWithSlice: `*filter
:KUBE-SERVICES - [0:0]
:KUBE-EXTERNAL-SERVICES - [0:0]
:KUBE-FORWARD - [0:0]
:KUBE-NODEPORTS - [0:0]
-A KUBE-SERVICES -m comment --comment "ns1/svc1 has no endpoints" -m tcp -p tcp -d 172.20.1.1/32 --dport 80 -j REJECT
-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP
-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT
-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
COMMIT
*nat
:KUBE-SERVICES - [0:0]
:KUBE-NODEPORTS - [0:0]
:KUBE-POSTROUTING - [0:0]
:KUBE-MARK-MASQ - [0:0]
-A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN
-A KUBE-POSTROUTING -j MARK --xor-mark 0x4000
-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE
-A KUBE-MARK-MASQ -j MARK --or-mark 0x4000
-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS
COMMIT
`,
},
{
name: "internalTrafficPolicy is local and there is non-zero local endpoint with feature gate off",
internalTrafficPolicy: &local,
featureGateOn: false,
endpoints: []endpoint{
{"10.0.1.1", testHostname},
{"10.0.1.2", "host1"},
{"10.0.1.3", "host2"},
},
expectEndpointRule: false,
expectedIPTablesWithSlice: clusterExpectedIPTables,
},
}
for _, tc := range testCases {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ServiceInternalTrafficPolicy, tc.featureGateOn)()
ipt := iptablestest.NewFake()
fp := NewFakeProxier(ipt, true)
fp.OnServiceSynced()
fp.OnEndpointsSynced()
fp.OnEndpointSlicesSynced()
serviceName := "svc1"
namespaceName := "ns1"
svc := &v1.Service{
ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: namespaceName},
Spec: v1.ServiceSpec{
ClusterIP: "172.20.1.1",
Selector: map[string]string{"foo": "bar"},
Ports: []v1.ServicePort{{Name: "", Port: 80, Protocol: v1.ProtocolTCP}},
},
}
if tc.internalTrafficPolicy != nil {
svc.Spec.InternalTrafficPolicy = tc.internalTrafficPolicy
}
fp.OnServiceAdd(svc)
tcpProtocol := v1.ProtocolTCP
endpointSlice := &discovery.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-1", serviceName),
Namespace: namespaceName,
Labels: map[string]string{discovery.LabelServiceName: serviceName},
},
Ports: []discovery.EndpointPort{{
Name: utilpointer.StringPtr(""),
Port: utilpointer.Int32Ptr(80),
Protocol: &tcpProtocol,
}},
AddressType: discovery.AddressTypeIPv4,
}
for _, ep := range tc.endpoints {
endpointSlice.Endpoints = append(endpointSlice.Endpoints, discovery.Endpoint{
Addresses: []string{ep.ip},
Conditions: discovery.EndpointConditions{Ready: utilpointer.BoolPtr(true)},
Topology: map[string]string{"kubernetes.io/hostname": ep.hostname},
})
}
fp.OnEndpointSliceAdd(endpointSlice)
fp.syncProxyRules()
assert.Equal(t, tc.expectedIPTablesWithSlice, fp.iptablesData.String())
if tc.expectEndpointRule {
fp.OnEndpointSliceDelete(endpointSlice)
fp.syncProxyRules()
assert.NotEqual(t, tc.expectedIPTablesWithSlice, fp.iptablesData.String())
}
}
}

19
pkg/proxy/ipvs/proxier.go
View File

@ -1236,7 +1236,7 @@ func (proxier *Proxier) syncProxyRules() {
activeBindAddrs[serv.Address.String()] = true
// ExternalTrafficPolicy only works for NodePort and external LB traffic, does not affect ClusterIP
// So we still need clusterIP rules in onlyNodeLocalEndpoints mode.
if err := proxier.syncEndpoint(svcName, false, serv); err != nil {
if err := proxier.syncEndpoint(svcName, false, svcInfo.OnlyNodeLocalEndpointsForInternal(), serv); err != nil {
klog.Errorf("Failed to sync endpoint for service: %v, err: %v", serv, err)
}
} else {
@ -1319,7 +1319,8 @@ func (proxier *Proxier) syncProxyRules() {
activeBindAddrs[serv.Address.String()] = true
onlyNodeLocalEndpoints := svcInfo.OnlyNodeLocalEndpoints()
if err := proxier.syncEndpoint(svcName, onlyNodeLocalEndpoints, serv); err != nil {
onlyNodeLocalEndpointsForInternal := svcInfo.OnlyNodeLocalEndpointsForInternal()
if err := proxier.syncEndpoint(svcName, onlyNodeLocalEndpoints, onlyNodeLocalEndpointsForInternal, serv); err != nil {
klog.Errorf("Failed to sync endpoint for service: %v, err: %v", serv, err)
}
} else {
@ -1420,7 +1421,7 @@ func (proxier *Proxier) syncProxyRules() {
if err := proxier.syncService(svcNameString, serv, true, bindedAddresses); err == nil {
activeIPVSServices[serv.String()] = true
activeBindAddrs[serv.Address.String()] = true
if err := proxier.syncEndpoint(svcName, svcInfo.OnlyNodeLocalEndpoints(), serv); err != nil {
if err := proxier.syncEndpoint(svcName, svcInfo.OnlyNodeLocalEndpoints(), svcInfo.OnlyNodeLocalEndpointsForInternal(), serv); err != nil {
klog.Errorf("Failed to sync endpoint for service: %v, err: %v", serv, err)
}
} else {
@ -1579,7 +1580,7 @@ func (proxier *Proxier) syncProxyRules() {
// There is no need to bind Node IP to dummy interface, so set parameter `bindAddr` to `false`.
if err := proxier.syncService(svcNameString, serv, false, bindedAddresses); err == nil {
activeIPVSServices[serv.String()] = true
if err := proxier.syncEndpoint(svcName, svcInfo.OnlyNodeLocalEndpoints(), serv); err != nil {
if err := proxier.syncEndpoint(svcName, svcInfo.OnlyNodeLocalEndpoints(), svcInfo.OnlyNodeLocalEndpointsForInternal(), serv); err != nil {
klog.Errorf("Failed to sync endpoint for service: %v, err: %v", serv, err)
}
} else {
@ -2030,7 +2031,7 @@ func (proxier *Proxier) syncService(svcName string, vs *utilipvs.VirtualServer,
return nil
}
func (proxier *Proxier) syncEndpoint(svcPortName proxy.ServicePortName, onlyNodeLocalEndpoints bool, vs *utilipvs.VirtualServer) error {
func (proxier *Proxier) syncEndpoint(svcPortName proxy.ServicePortName, onlyNodeLocalEndpoints bool, onlyNodeLocalEndpointsForInternal bool, vs *utilipvs.VirtualServer) error {
appliedVirtualServer, err := proxier.ipvs.GetVirtualServer(vs)
if err != nil {
klog.Errorf("Failed to get IPVS service, error: %v", err)
@ -2065,6 +2066,14 @@ func (proxier *Proxier) syncEndpoint(svcPortName proxy.ServicePortName, onlyNode
endpoints = proxy.FilterTopologyEndpoint(proxier.nodeLabels, proxier.serviceMap[svcPortName].TopologyKeys(), endpoints)
}
// Service InternalTrafficPolicy is only enabled when all of the
// following are true:
// 1. InternalTrafficPolicy is PreferLocal or Local
// 2. ServiceInternalTrafficPolicy feature gate is on
if utilfeature.DefaultFeatureGate.Enabled(features.ServiceInternalTrafficPolicy) && onlyNodeLocalEndpointsForInternal {
endpoints = proxy.FilterLocalEndpoint(proxier.serviceMap[svcPortName].InternalTrafficPolicy(), endpoints)
}
for _, epInfo := range endpoints {
if !epInfo.IsReady() {
continue

183
pkg/proxy/ipvs/proxier_test.go
View File

@ -33,6 +33,9 @@ import (
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apimachinery/pkg/util/sets"
utilfeature "k8s.io/apiserver/pkg/util/feature"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/proxy"
"k8s.io/kubernetes/pkg/proxy/healthcheck"
netlinktest "k8s.io/kubernetes/pkg/proxy/ipvs/testing"
@ -4467,3 +4470,183 @@ func TestCreateAndLinkKubeChain(t *testing.T) {
assert.Equal(t, expectedNATChains, fp.natChains.String())
assert.Equal(t, expectedFilterChains, fp.filterChains.String())
}
// This test ensures that the iptables proxier supports translating Endpoints to
// iptables output when internalTrafficPolicy is specified
func TestTestInternalTrafficPolicyE2E(t *testing.T) {
type endpoint struct {
ip string
hostname string
}
cluster := v1.ServiceInternalTrafficPolicyCluster
local := v1.ServiceInternalTrafficPolicyLocal
testCases := []struct {
name string
internalTrafficPolicy *v1.ServiceInternalTrafficPolicyType
endpoints []endpoint
expectVirtualServer bool
expectLocalEntries bool
expectLocalRealServerNum int
expectLocalRealServers []string
}{
{
name: "internalTrafficPolicy is cluster with non-zero local endpoints",
internalTrafficPolicy: &cluster,
endpoints: []endpoint{
{"10.0.1.1", testHostname},
{"10.0.1.2", "host1"},
{"10.0.1.3", "host2"},
},
expectVirtualServer: true,
expectLocalEntries: true,
expectLocalRealServerNum: 3,
expectLocalRealServers: []string{
"10.0.1.1:80",
"10.0.1.2:80",
"10.0.1.3:80",
},
},
{
name: "internalTrafficPolicy is cluster with zero local endpoints",
internalTrafficPolicy: &cluster,
endpoints: []endpoint{
{"10.0.1.1", "host0"},
{"10.0.1.2", "host1"},
{"10.0.1.3", "host2"},
},
expectVirtualServer: false,
expectLocalEntries: false,
expectLocalRealServerNum: 3,
expectLocalRealServers: []string{
"10.0.1.1:80",
"10.0.1.2:80",
"10.0.1.3:80",
},
},
{
name: "internalTrafficPolicy is local with non-zero local endpoints",
internalTrafficPolicy: &local,
endpoints: []endpoint{
{"10.0.1.1", testHostname},
{"10.0.1.2", "host1"},
{"10.0.1.3", "host2"},
},
expectVirtualServer: true,
expectLocalEntries: true,
expectLocalRealServerNum: 1,
expectLocalRealServers: []string{
"10.0.1.1:80",
},
},
{
name: "internalTrafficPolicy is local with zero local endpoints",
internalTrafficPolicy: &local,
endpoints: []endpoint{
{"10.0.1.1", "host0"},
{"10.0.1.2", "host1"},
{"10.0.1.3", "host2"},
},
expectVirtualServer: false,
expectLocalEntries: false,
expectLocalRealServerNum: 0,
expectLocalRealServers: []string{},
},
}
for _, tc := range testCases {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ServiceInternalTrafficPolicy, true)()
ipt := iptablestest.NewFake()
ipvs := ipvstest.NewFake()
ipset := ipsettest.NewFake(testIPSetVersion)
fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, true, v1.IPv4Protocol)
fp.servicesSynced = true
fp.endpointsSynced = true
fp.endpointSlicesSynced = true
// Add initial service
serviceName := "svc1"
namespaceName := "ns1"
svc := &v1.Service{
ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: namespaceName},
Spec: v1.ServiceSpec{
ClusterIP: "172.20.1.1",
Selector: map[string]string{"foo": "bar"},
Ports: []v1.ServicePort{{Name: "", TargetPort: intstr.FromInt(80), Protocol: v1.ProtocolTCP}},
},
}
if tc.internalTrafficPolicy != nil {
svc.Spec.InternalTrafficPolicy = tc.internalTrafficPolicy
}
fp.OnServiceAdd(svc)
// Add initial endpoint slice
tcpProtocol := v1.ProtocolTCP
endpointSlice := &discovery.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-1", serviceName),
Namespace: namespaceName,
Labels: map[string]string{discovery.LabelServiceName: serviceName},
},
Ports: []discovery.EndpointPort{{
Name: utilpointer.StringPtr(""),
Port: utilpointer.Int32Ptr(80),
Protocol: &tcpProtocol,
}},
AddressType: discovery.AddressTypeIPv4,
}
for _, ep := range tc.endpoints {
endpointSlice.Endpoints = append(endpointSlice.Endpoints, discovery.Endpoint{
Addresses: []string{ep.ip},
Conditions: discovery.EndpointConditions{Ready: utilpointer.BoolPtr(true)},
Topology: map[string]string{"kubernetes.io/hostname": ep.hostname},
})
}
fp.OnEndpointSliceAdd(endpointSlice)
fp.syncProxyRules()
// Ensure that Proxier updates ipvs appropriately after EndpointSlice update
assert.NotNil(t, fp.ipsetList["KUBE-LOOP-BACK"])
activeEntries1 := fp.ipsetList["KUBE-LOOP-BACK"].activeEntries
if tc.expectLocalEntries {
assert.Equal(t, 1, activeEntries1.Len(), "Expected 1 active entry in KUBE-LOOP-BACK")
} else {
assert.Equal(t, 0, activeEntries1.Len(), "Expected no active entry in KUBE-LOOP-BACK")
}
if tc.expectVirtualServer {
virtualServers1, vsErr1 := ipvs.GetVirtualServers()
assert.Nil(t, vsErr1, "Expected no error getting virtual servers")
assert.Len(t, virtualServers1, 1, "Expected 1 virtual server")
realServers1, rsErr1 := ipvs.GetRealServers(virtualServers1[0])
assert.Nil(t, rsErr1, "Expected no error getting real servers")
assert.Len(t, realServers1, tc.expectLocalRealServerNum, fmt.Sprintf("Expected %d real servers", tc.expectLocalRealServerNum))
for i := 0; i < tc.expectLocalRealServerNum; i++ {
assert.Equal(t, realServers1[i].String(), tc.expectLocalRealServers[i])
}
}
fp.OnEndpointSliceDelete(endpointSlice)
fp.syncProxyRules()
// Ensure that Proxier updates ipvs appropriately after EndpointSlice delete
assert.NotNil(t, fp.ipsetList["KUBE-LOOP-BACK"])
activeEntries3 := fp.ipsetList["KUBE-LOOP-BACK"].activeEntries
assert.Equal(t, 0, activeEntries3.Len(), "Expected 0 active entries in KUBE-LOOP-BACK")
virtualServers2, vsErr2 := ipvs.GetVirtualServers()
assert.Nil(t, vsErr2, "Expected no error getting virtual servers")
assert.Len(t, virtualServers2, 1, "Expected 1 virtual server")
realServers2, rsErr2 := ipvs.GetRealServers(virtualServers2[0])
assert.Nil(t, rsErr2, "Expected no error getting real servers")
assert.Len(t, realServers2, 0, "Expected 0 real servers")
}
}

60
pkg/proxy/service.go
View File

@ -28,8 +28,10 @@ import (
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/tools/record"
apiservice "k8s.io/kubernetes/pkg/api/v1/service"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/proxy/metrics"
utilproxy "k8s.io/kubernetes/pkg/proxy/util"
)
@ -39,18 +41,20 @@ import (
// or can be used for constructing a more specific ServiceInfo struct
// defined by the proxier if needed.
type BaseServiceInfo struct {
clusterIP net.IP
port int
protocol v1.Protocol
nodePort int
loadBalancerStatus v1.LoadBalancerStatus
sessionAffinityType v1.ServiceAffinity
stickyMaxAgeSeconds int
externalIPs []string
loadBalancerSourceRanges []string
healthCheckNodePort int
onlyNodeLocalEndpoints bool
topologyKeys []string
clusterIP net.IP
port int
protocol v1.Protocol
nodePort int
loadBalancerStatus v1.LoadBalancerStatus
sessionAffinityType v1.ServiceAffinity
stickyMaxAgeSeconds int
externalIPs []string
loadBalancerSourceRanges []string
healthCheckNodePort int
onlyNodeLocalEndpoints bool
onlyNodeLocalEndpointsForInternal bool
internalTrafficPolicy *v1.ServiceInternalTrafficPolicyType
topologyKeys []string
}
var _ ServicePort = &BaseServiceInfo{}
@ -119,6 +123,16 @@ func (info *BaseServiceInfo) OnlyNodeLocalEndpoints() bool {
return info.onlyNodeLocalEndpoints
}
// OnlyNodeLocalEndpointsForInternal is part of ServicePort interface
func (info *BaseServiceInfo) OnlyNodeLocalEndpointsForInternal() bool {
return info.onlyNodeLocalEndpointsForInternal
}
// InternalTrafficPolicy is part of ServicePort interface
func (info *BaseServiceInfo) InternalTrafficPolicy() *v1.ServiceInternalTrafficPolicyType {
return info.internalTrafficPolicy
}
// TopologyKeys is part of ServicePort interface.
func (info *BaseServiceInfo) TopologyKeys() []string {
return info.topologyKeys
@ -129,6 +143,10 @@ func (sct *ServiceChangeTracker) newBaseServiceInfo(port *v1.ServicePort, servic
if apiservice.RequestsOnlyLocalTraffic(service) {
onlyNodeLocalEndpoints = true
}
onlyNodeLocalEndpointsForInternal := false
if utilfeature.DefaultFeatureGate.Enabled(features.ServiceInternalTrafficPolicy) {
onlyNodeLocalEndpointsForInternal = apiservice.RequestsOnlyLocalTrafficForInternal(service)
}
var stickyMaxAgeSeconds int
if service.Spec.SessionAffinity == v1.ServiceAffinityClientIP {
// Kube-apiserver side guarantees SessionAffinityConfig won't be nil when session affinity type is ClientIP
@ -137,14 +155,16 @@ func (sct *ServiceChangeTracker) newBaseServiceInfo(port *v1.ServicePort, servic
clusterIP := utilproxy.GetClusterIPByFamily(sct.ipFamily, service)
info := &BaseServiceInfo{
clusterIP: net.ParseIP(clusterIP),
port: int(port.Port),
protocol: port.Protocol,
nodePort: int(port.NodePort),
sessionAffinityType: service.Spec.SessionAffinity,
stickyMaxAgeSeconds: stickyMaxAgeSeconds,
onlyNodeLocalEndpoints: onlyNodeLocalEndpoints,
topologyKeys: service.Spec.TopologyKeys,
clusterIP: net.ParseIP(clusterIP),
port: int(port.Port),
protocol: port.Protocol,
nodePort: int(port.NodePort),
sessionAffinityType: service.Spec.SessionAffinity,
stickyMaxAgeSeconds: stickyMaxAgeSeconds,
onlyNodeLocalEndpoints: onlyNodeLocalEndpoints,
onlyNodeLocalEndpointsForInternal: onlyNodeLocalEndpointsForInternal,
internalTrafficPolicy: service.Spec.InternalTrafficPolicy,
topologyKeys: service.Spec.TopologyKeys,
}
loadBalancerSourceRanges := make([]string, len(service.Spec.LoadBalancerSourceRanges))

42
pkg/proxy/topology.go
View File

@ -18,6 +18,8 @@ package proxy
import (
v1 "k8s.io/api/core/v1"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/kubernetes/pkg/features"
)
// FilterTopologyEndpoint returns the appropriate endpoints based on the cluster
@ -44,7 +46,7 @@ func FilterTopologyEndpoint(nodeLabels map[string]string, topologyKeys []string,
return endpoints
}
filteredEndpoint := []Endpoint{}
filteredEndpoints := []Endpoint{}
if len(nodeLabels) == 0 {
if topologyKeys[len(topologyKeys)-1] == v1.TopologyKeyAny {
@ -54,7 +56,7 @@ func FilterTopologyEndpoint(nodeLabels map[string]string, topologyKeys []string,
}
// edge case: do not include any endpoints if topology key "Any" is
// not specified when we cannot determine current node's topology.
return filteredEndpoint
return filteredEndpoints
}
for _, key := range topologyKeys {
@ -69,12 +71,40 @@ func FilterTopologyEndpoint(nodeLabels map[string]string, topologyKeys []string,
for _, ep := range endpoints {
topology := ep.GetTopology()
if value, found := topology[key]; found && value == topologyValue {
filteredEndpoint = append(filteredEndpoint, ep)
filteredEndpoints = append(filteredEndpoints, ep)
}
}
if len(filteredEndpoint) > 0 {
return filteredEndpoint
if len(filteredEndpoints) > 0 {
return filteredEndpoints
}
}
return filteredEndpoint
return filteredEndpoints
}
// FilterLocalEndpoint returns the node local endpoints based on configured
// InternalTrafficPolicy.
//
// If ServiceInternalTrafficPolicy feature gate is off, returns the original
// endpoints slice.
// Otherwise, if InternalTrafficPolicy is Local, only return the node local endpoints.
func FilterLocalEndpoint(internalTrafficPolicy *v1.ServiceInternalTrafficPolicyType, endpoints []Endpoint) []Endpoint {
if !utilfeature.DefaultFeatureGate.Enabled(features.ServiceInternalTrafficPolicy) {
return endpoints
}
if internalTrafficPolicy == nil || *internalTrafficPolicy == v1.ServiceInternalTrafficPolicyCluster {
return endpoints
}
var filteredEndpoints []Endpoint
// Get all the local endpoints
for _, ep := range endpoints {
if ep.GetIsLocal() {
filteredEndpoints = append(filteredEndpoints, ep)
}
}
// When internalTrafficPolicy is Local, only return the node local
// endpoints
return filteredEndpoints
}

97
pkg/proxy/topology_test.go
View File

@ -22,6 +22,9 @@ import (
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
utilfeature "k8s.io/apiserver/pkg/util/feature"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/kubernetes/pkg/features"
)
func TestFilterTopologyEndpoint(t *testing.T) {
@ -476,3 +479,97 @@ func TestFilterTopologyEndpoint(t *testing.T) {
})
}
}
func TestFilterLocalEndpoint(t *testing.T) {
cluster := v1.ServiceInternalTrafficPolicyCluster
local := v1.ServiceInternalTrafficPolicyLocal
testCases := []struct {
name string
internalTrafficPolicy *v1.ServiceInternalTrafficPolicyType
endpoints []Endpoint
expected []Endpoint
featureGateOn bool
}{
{
name: "no internalTrafficPolicy with empty endpoints",
internalTrafficPolicy: nil,
endpoints: []Endpoint{},
expected: []Endpoint{},
featureGateOn: true,
},
{
name: "no internalTrafficPolicy with non-empty endpoints",
internalTrafficPolicy: nil,
endpoints: []Endpoint{
&BaseEndpointInfo{Endpoint: "10.0.0.0:80", IsLocal: true},
&BaseEndpointInfo{Endpoint: "10.0.0.1:80", IsLocal: false},
},
expected: []Endpoint{
&BaseEndpointInfo{Endpoint: "10.0.0.0:80", IsLocal: true},
&BaseEndpointInfo{Endpoint: "10.0.0.1:80", IsLocal: false},
},
featureGateOn: true,
},
{
name: "internalTrafficPolicy is cluster",
internalTrafficPolicy: &cluster,
endpoints: []Endpoint{
&BaseEndpointInfo{Endpoint: "10.0.0.0:80", IsLocal: true},
&BaseEndpointInfo{Endpoint: "10.0.0.1:80", IsLocal: false},
},
expected: []Endpoint{
&BaseEndpointInfo{Endpoint: "10.0.0.0:80", IsLocal: true},
&BaseEndpointInfo{Endpoint: "10.0.0.1:80", IsLocal: false},
},
featureGateOn: true,
},
{
name: "internalTrafficPolicy is local with non-zero local endpoints",
internalTrafficPolicy: &local,
endpoints: []Endpoint{
&BaseEndpointInfo{Endpoint: "10.0.0.0:80", IsLocal: true},
&BaseEndpointInfo{Endpoint: "10.0.0.1:80", IsLocal: false},
},
expected: []Endpoint{
&BaseEndpointInfo{Endpoint: "10.0.0.0:80", IsLocal: true},
},
featureGateOn: true,
},
{
name: "internalTrafficPolicy is local with zero local endpoints",
internalTrafficPolicy: &local,
endpoints: []Endpoint{
&BaseEndpointInfo{Endpoint: "10.0.0.0:80", IsLocal: false},
&BaseEndpointInfo{Endpoint: "10.0.0.1:80", IsLocal: false},
&BaseEndpointInfo{Endpoint: "10.0.0.2:80", IsLocal: false},
},
expected: nil,
featureGateOn: true,
},
{
name: "feature gate is off, internalTrafficPolicy is local with non-empty endpoints",
internalTrafficPolicy: &local,
endpoints: []Endpoint{
&BaseEndpointInfo{Endpoint: "10.0.0.0:80", IsLocal: true},
&BaseEndpointInfo{Endpoint: "10.0.0.1:80", IsLocal: false},
&BaseEndpointInfo{Endpoint: "10.0.0.2:80", IsLocal: false},
},
expected: []Endpoint{
&BaseEndpointInfo{Endpoint: "10.0.0.0:80", IsLocal: true},
&BaseEndpointInfo{Endpoint: "10.0.0.1:80", IsLocal: false},
&BaseEndpointInfo{Endpoint: "10.0.0.2:80", IsLocal: false},
},
featureGateOn: false,
},
}
for _, tc := range testCases {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ServiceInternalTrafficPolicy, tc.featureGateOn)()
t.Run(tc.name, func(t *testing.T) {
filteredEndpoint := FilterLocalEndpoint(tc.internalTrafficPolicy, tc.endpoints)
if !reflect.DeepEqual(filteredEndpoint, tc.expected) {
t.Errorf("expected %v, got %v", tc.expected, filteredEndpoint)
}
})
}
}

4
pkg/proxy/types.go
View File

@ -85,6 +85,10 @@ type ServicePort interface {
NodePort() int
// GetOnlyNodeLocalEndpoints returns if a service has only node local endpoints
OnlyNodeLocalEndpoints() bool
// OnlyNodeLocalEndpointsForInternal returns if a service prefers routing to node local endpoints for internal traffic
OnlyNodeLocalEndpointsForInternal() bool
// InternalTrafficPolicy returns service InternalTrafficPolicy
InternalTrafficPolicy() *v1.ServiceInternalTrafficPolicyType
// TopologyKeys returns service TopologyKeys as a string array.
TopologyKeys() []string
}

16
pkg/registry/core/service/strategy.go
View File

@ -156,7 +156,7 @@ func dropServiceDisabledFields(newSvc *api.Service, oldSvc *api.Service) {
newSvc.Spec.TopologyKeys = nil
}
// Clear AllocateLoadBalancerNodePorts if ServiceLBNodePortControl if not enabled
// Clear AllocateLoadBalancerNodePorts if ServiceLBNodePortControl is not enabled
if !utilfeature.DefaultFeatureGate.Enabled(features.ServiceLBNodePortControl) {
if !allocateLoadBalancerNodePortsInUse(oldSvc) {
newSvc.Spec.AllocateLoadBalancerNodePorts = nil
@ -180,6 +180,13 @@ func dropServiceDisabledFields(newSvc *api.Service, oldSvc *api.Service) {
newSvc.Spec.LoadBalancerClass = nil
}
}
// Clear InternalTrafficPolicy if not enabled
if !utilfeature.DefaultFeatureGate.Enabled(features.ServiceInternalTrafficPolicy) {
if !serviceInternalTrafficPolicyInUse(oldSvc) {
newSvc.Spec.InternalTrafficPolicy = nil
}
}
}
// returns true if svc.Spec.AllocateLoadBalancerNodePorts field is in use
@ -240,6 +247,13 @@ func loadBalancerClassInUse(svc *api.Service) bool {
return svc.Spec.LoadBalancerClass != nil
}
func serviceInternalTrafficPolicyInUse(svc *api.Service) bool {
if svc == nil {
return false
}
return svc.Spec.InternalTrafficPolicy != nil
}
type serviceStatusStrategy struct {
Strategy
}

50
pkg/registry/core/service/strategy_test.go
View File

@ -257,19 +257,30 @@ func makeServiceWithLoadBalancerClass(loadBalancerClass *string) *api.Service {
}
}
func makeServiceWithInternalTrafficPolicy(policy *api.ServiceInternalTrafficPolicyType) *api.Service {
return &api.Service{
Spec: api.ServiceSpec{
InternalTrafficPolicy: policy,
},
}
}
func TestDropDisabledField(t *testing.T) {
requireDualStack := api.IPFamilyPolicyRequireDualStack
preferDualStack := api.IPFamilyPolicyPreferDualStack
singleStack := api.IPFamilyPolicySingleStack
localInternalTrafficPolicy := api.ServiceInternalTrafficPolicyLocal
testCases := []struct {
name string
enableDualStack bool
enableMixedProtocol bool
enableLoadBalancerClass bool
svc *api.Service
oldSvc *api.Service
compareSvc *api.Service
name string
enableDualStack bool
enableMixedProtocol bool
enableLoadBalancerClass bool
enableInternalTrafficPolicy bool
svc *api.Service
oldSvc *api.Service
compareSvc *api.Service
}{
{
name: "not dual stack, field not used",
@ -500,6 +511,28 @@ func TestDropDisabledField(t *testing.T) {
oldSvc: makeServiceWithLoadBalancerClass(utilpointer.StringPtr("test.com/test")),
compareSvc: makeServiceWithLoadBalancerClass(nil),
},
/* svc.spec.internalTrafficPolicy */
{
name: "internal traffic policy not enabled, field used in old, not used in new",
enableInternalTrafficPolicy: false,
svc: makeServiceWithInternalTrafficPolicy(nil),
oldSvc: makeServiceWithInternalTrafficPolicy(&localInternalTrafficPolicy),
compareSvc: makeServiceWithInternalTrafficPolicy(nil),
},
{
name: "internal traffic policy not enabled, field not used in old, used in new",
enableInternalTrafficPolicy: false,
svc: makeServiceWithInternalTrafficPolicy(&localInternalTrafficPolicy),
oldSvc: makeServiceWithInternalTrafficPolicy(nil),
compareSvc: makeServiceWithInternalTrafficPolicy(nil),
},
{
name: "internal traffic policy enabled, field not used in old, used in new",
enableInternalTrafficPolicy: true,
svc: makeServiceWithInternalTrafficPolicy(&localInternalTrafficPolicy),
oldSvc: makeServiceWithInternalTrafficPolicy(nil),
compareSvc: makeServiceWithInternalTrafficPolicy(&localInternalTrafficPolicy),
},
/* add more tests for other dropped fields as needed */
}
for _, tc := range testCases {
@ -507,12 +540,13 @@ func TestDropDisabledField(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, tc.enableDualStack)()
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.MixedProtocolLBService, tc.enableMixedProtocol)()
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ServiceLoadBalancerClass, tc.enableLoadBalancerClass)()
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ServiceInternalTrafficPolicy, tc.enableInternalTrafficPolicy)()
old := tc.oldSvc.DeepCopy()
// to test against user using IPFamily not set on cluster
dropServiceDisabledFields(tc.svc, tc.oldSvc)
// old node should never be changed
// old node should never be changed
if !reflect.DeepEqual(tc.oldSvc, old) {
t.Errorf("%v: old svc changed: %v", tc.name, diff.ObjectReflectDiff(tc.oldSvc, old))
}

25
staging/src/k8s.io/api/core/v1/types.go
View File

@ -3940,6 +3940,19 @@ const (
ServiceTypeExternalName ServiceType = "ExternalName"
)
// ServiceInternalTrafficPolicyType describes the type of traffic routing for
// internal traffic
type ServiceInternalTrafficPolicyType string
const (
// ServiceInternalTrafficPolicyCluster routes traffic to all endpoints
ServiceInternalTrafficPolicyCluster ServiceInternalTrafficPolicyType = "Cluster"
// ServiceInternalTrafficPolicyLocal only routes to node-local
// endpoints, otherwise drops the traffic
ServiceInternalTrafficPolicyLocal ServiceInternalTrafficPolicyType = "Local"
)
// Service External Traffic Policy Type string
type ServiceExternalTrafficPolicyType string
@ -4272,9 +4285,19 @@ type ServiceSpec struct {
// implementation (e.g. cloud providers) should ignore Services that set this field.
// This field can only be set when creating or updating a Service to type 'LoadBalancer'.
// Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type.
// featureGate=LoadBalancerClass
// +featureGate=LoadBalancerClass
// +optional
LoadBalancerClass *string `json:"loadBalancerClass,omitempty" protobuf:"bytes,21,opt,name=loadBalancerClass"`
// InternalTrafficPolicy specifies if the cluster internal traffic
// should be routed to all endpoints or node-local endpoints only.
// "Cluster" routes internal traffic to a Service to all endpoints.
// "Local" routes traffic to node-local endpoints only, traffic is
// dropped if no node-local endpoints are ready.
// The default value is "Cluster".
// +featureGate=ServiceInternalTrafficPolicy
// +optional
InternalTrafficPolicy *ServiceInternalTrafficPolicyType `json:"internalTrafficPolicy,omitempty" protobuf:"bytes,22,opt,name=internalTrafficPolicy"`
}
// ServicePort contains information on service's port.

1
staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go generated
View File

@ -2258,6 +2258,7 @@ var map_ServiceSpec = map[string]string{
"ipFamilyPolicy": "IPFamilyPolicy represents the dual-stack-ness requested or required by this Service, and is gated by the \"IPv6DualStack\" feature gate. If there is no value provided, then this field will be set to SingleStack. Services can be \"SingleStack\" (a single IP family), \"PreferDualStack\" (two IP families on dual-stack configured clusters or a single IP family on single-stack clusters), or \"RequireDualStack\" (two IP families on dual-stack configured clusters, otherwise fail). The ipFamilies and clusterIPs fields depend on the value of this field. This field will be wiped when updating a service to type ExternalName.",
"allocateLoadBalancerNodePorts": "allocateLoadBalancerNodePorts defines if NodePorts will be automatically allocated for services with type LoadBalancer. Default is \"true\". It may be set to \"false\" if the cluster load-balancer does not rely on NodePorts. allocateLoadBalancerNodePorts may only be set for services with type LoadBalancer and will be cleared if the type is changed to any other type. This field is alpha-level and is only honored by servers that enable the ServiceLBNodePortControl feature.",
"loadBalancerClass": "loadBalancerClass is the class of the load balancer implementation this Service belongs to. If specified, the value of this field must be a label-style identifier, with an optional prefix, e.g. \"internal-vip\" or \"example.com/internal-vip\". Unprefixed names are reserved for end-users. This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load balancer implementation is used, today this is typically done through the cloud provider integration, but should apply for any default implementation. If set, it is assumed that a load balancer implementation is watching for Services with a matching class. Any default load balancer implementation (e.g. cloud providers) should ignore Services that set this field. This field can only be set when creating or updating a Service to type 'LoadBalancer'. Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. featureGate=LoadBalancerClass",
"internalTrafficPolicy": "InternalTrafficPolicy specifies if the cluster internal traffic should be routed to all endpoints or node-local endpoints only. \"Cluster\" routes internal traffic to a Service to all endpoints. \"Local\" routes traffic to node-local endpoints only, traffic is dropped if no node-local endpoints are ready. featureGate=ServiceInternalTrafficPolicy",
}
func (ServiceSpec) SwaggerDoc() map[string]string {

5
staging/src/k8s.io/api/core/v1/zz_generated.deepcopy.go generated
View File

@ -5350,6 +5350,11 @@ func (in *ServiceSpec) DeepCopyInto(out *ServiceSpec) {
*out = new(string)
**out = **in
}
if in.InternalTrafficPolicy != nil {
in, out := &in.InternalTrafficPolicy, &out.InternalTrafficPolicy
*out = new(ServiceInternalTrafficPolicyType)
**out = **in
}
return
}

13
staging/src/k8s.io/api/testdata/HEAD/core.v1.Service.json vendored
View File

@ -84,7 +84,8 @@
],
"ipFamilyPolicy": "9ȫŚ",
"allocateLoadBalancerNodePorts": true,
"loadBalancerClass": "31"
"loadBalancerClass": "31",
"internalTrafficPolicy": ""
},
"status": {
"loadBalancer": {
@ -94,8 +95,8 @@
"hostname": "33",
"ports": [
{
"port": -907310967,
"protocol": "喂ƈ斎AO6",
"port": 684408190,
"protocol": "ƈ斎AO6ĴC浔Ű",
"error": "34"
}
]
@ -105,9 +106,9 @@
"conditions": [
{
"type": "35",
"status": "C",
"observedGeneration": -2492120148461555858,
"lastTransitionTime": "2392-12-09T15:37:55Z",
"status": "ž(-譵",
"observedGeneration": -8651056334266075769,
"lastTransitionTime": "2404-01-10T11:35:42Z",
"reason": "36",
"message": "37"
}

BIN
staging/src/k8s.io/api/testdata/HEAD/core.v1.Service.pb vendored
View File

Binary file not shown.

11
staging/src/k8s.io/api/testdata/HEAD/core.v1.Service.yaml vendored
View File

@ -39,6 +39,7 @@ spec:
externalName: "29"
externalTrafficPolicy: ƏS$+½H牗洝尿
healthCheckNodePort: -1965738697
internalTrafficPolicy: ""
ipFamilies:
- 斬³;Ơ歿:狞夌碕ʂɭîcP$Iņɖ
ipFamilyPolicy: 9ȫŚ
@ -65,11 +66,11 @@ spec:
type: 鮽ort昍řČ扷5ƗǸƢ6/ʕVŚ(Ŀ
status:
conditions:
- lastTransitionTime: "2392-12-09T15:37:55Z"
- lastTransitionTime: "2404-01-10T11:35:42Z"
message: "37"
observedGeneration: -2492120148461555858
observedGeneration: -8651056334266075769
reason: "36"
status: C
status: ž(-譵
type: "35"
loadBalancer:
ingress:
@ -77,5 +78,5 @@ status:
ip: "32"
ports:
- error: "34"
port: -907310967
protocol: 喂ƈ斎AO6
port: 684408190
protocol: ƈ斎AO6ĴC浔Ű

9
staging/src/k8s.io/client-go/applyconfigurations/core/v1/servicespec.go
View File

@ -44,6 +44,7 @@ type ServiceSpecApplyConfiguration struct {
IPFamilyPolicy *corev1.IPFamilyPolicyType `json:"ipFamilyPolicy,omitempty"`
AllocateLoadBalancerNodePorts *bool `json:"allocateLoadBalancerNodePorts,omitempty"`
LoadBalancerClass *string `json:"loadBalancerClass,omitempty"`
InternalTrafficPolicy *corev1.ServiceInternalTrafficPolicyType `json:"internalTrafficPolicy,omitempty"`
}
// ServiceSpecApplyConfiguration constructs an declarative configuration of the ServiceSpec type for use with
@ -224,3 +225,11 @@ func (b *ServiceSpecApplyConfiguration) WithLoadBalancerClass(value string) *Ser
b.LoadBalancerClass = &value
return b
}
// WithInternalTrafficPolicy sets the InternalTrafficPolicy field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the InternalTrafficPolicy field is set to the value of the last call.
func (b *ServiceSpecApplyConfiguration) WithInternalTrafficPolicy(value corev1.ServiceInternalTrafficPolicyType) *ServiceSpecApplyConfiguration {
b.InternalTrafficPolicy = &value
return b
}