From 826a5219dacb9922aef74ecbc2b53e5059017339 Mon Sep 17 00:00:00 2001 From: Andrew Sy Kim Date: Thu, 8 Jul 2021 17:34:10 -0400 Subject: [PATCH 1/2] promote EndpointSliceTerminatingCondition to Beta Signed-off-by: Andrew Sy Kim --- pkg/features/kube_features.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 841686d615a..91522445226 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -557,6 +557,7 @@ const ( // owner: @andrewsykim // kep: http://kep.k8s.io/1672 // alpha: v1.20 + // beta: v1.22 // // Enable Terminating condition in Endpoint Slices. EndpointSliceTerminatingCondition featuregate.Feature = "EndpointSliceTerminatingCondition" @@ -854,7 +855,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS IPv6DualStack: {Default: true, PreRelease: featuregate.Beta}, EndpointSlice: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.25 EndpointSliceProxying: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.25 - EndpointSliceTerminatingCondition: {Default: false, PreRelease: featuregate.Alpha}, + EndpointSliceTerminatingCondition: {Default: true, PreRelease: featuregate.Beta}, ProxyTerminatingEndpoints: {Default: false, PreRelease: featuregate.Alpha}, EndpointSliceNodeName: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, //remove in 1.25 WindowsEndpointSliceProxying: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.25 From fd0db61d6c36cc442fd670dc02017ab50e85a7ea Mon Sep 17 00:00:00 2001 From: Andrew Sy Kim Date: Thu, 8 Jul 2021 17:54:56 -0400 Subject: [PATCH 2/2] test/intergration/endpointslice: add tests for endpointslice terminating condition Signed-off-by: Andrew Sy Kim --- .../endpointsliceterminating_test.go | 333 ++++++++++++++++++ 1 file changed, 333 insertions(+) create mode 100644 test/integration/endpointslice/endpointsliceterminating_test.go diff --git a/test/integration/endpointslice/endpointsliceterminating_test.go b/test/integration/endpointslice/endpointsliceterminating_test.go new file mode 100644 index 00000000000..2cd5be85e81 --- /dev/null +++ b/test/integration/endpointslice/endpointsliceterminating_test.go @@ -0,0 +1,333 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package endpointslice + +import ( + "context" + "reflect" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + discovery "k8s.io/api/discovery/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/wait" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/client-go/informers" + clientset "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/kubernetes/pkg/controller/endpointslice" + "k8s.io/kubernetes/pkg/features" + "k8s.io/kubernetes/test/integration/framework" + utilpointer "k8s.io/utils/pointer" +) + +// TestEndpointSliceTerminating tests that terminating pods are NOT included in EndpointSlice when +// the feature gate EndpointSliceTerminatingCondition is off. If the gate is on, it tests that +// terminating endpoints are included but with the correct conditions set for ready, serving and terminating. +func TestEndpointSliceTerminating(t *testing.T) { + testcases := []struct { + name string + podStatus corev1.PodStatus + expectedEndpoints []discovery.Endpoint + terminatingGate bool + }{ + { + name: "ready terminating pods not included, terminating gate off", + podStatus: corev1.PodStatus{ + Phase: corev1.PodRunning, + Conditions: []corev1.PodCondition{ + { + Type: corev1.PodReady, + Status: corev1.ConditionTrue, + }, + }, + PodIP: "10.0.0.1", + PodIPs: []corev1.PodIP{ + { + IP: "10.0.0.1", + }, + }, + }, + expectedEndpoints: []discovery.Endpoint{}, + terminatingGate: false, + }, + { + name: "not ready terminating pods not included, terminating gate off", + podStatus: corev1.PodStatus{ + Phase: corev1.PodRunning, + Conditions: []corev1.PodCondition{ + { + Type: corev1.PodReady, + Status: corev1.ConditionFalse, + }, + }, + PodIP: "10.0.0.1", + PodIPs: []corev1.PodIP{ + { + IP: "10.0.0.1", + }, + }, + }, + expectedEndpoints: []discovery.Endpoint{}, + terminatingGate: false, + }, + { + name: "ready terminating pods included, terminating gate on", + podStatus: corev1.PodStatus{ + Phase: corev1.PodRunning, + Conditions: []corev1.PodCondition{ + { + Type: corev1.PodReady, + Status: corev1.ConditionTrue, + }, + }, + PodIP: "10.0.0.1", + PodIPs: []corev1.PodIP{ + { + IP: "10.0.0.1", + }, + }, + }, + expectedEndpoints: []discovery.Endpoint{ + { + Addresses: []string{"10.0.0.1"}, + Conditions: discovery.EndpointConditions{ + Ready: utilpointer.BoolPtr(false), + Serving: utilpointer.BoolPtr(true), + Terminating: utilpointer.BoolPtr(true), + }, + }, + }, + terminatingGate: true, + }, + { + name: "not ready terminating pods included, terminating gate on", + podStatus: corev1.PodStatus{ + Phase: corev1.PodRunning, + Conditions: []corev1.PodCondition{ + { + Type: corev1.PodReady, + Status: corev1.ConditionFalse, + }, + }, + PodIP: "10.0.0.1", + PodIPs: []corev1.PodIP{ + { + IP: "10.0.0.1", + }, + }, + }, + expectedEndpoints: []discovery.Endpoint{ + { + Addresses: []string{"10.0.0.1"}, + Conditions: discovery.EndpointConditions{ + Ready: utilpointer.BoolPtr(false), + Serving: utilpointer.BoolPtr(false), + Terminating: utilpointer.BoolPtr(true), + }, + }, + }, + terminatingGate: true, + }, + } + + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.EndpointSliceTerminatingCondition, testcase.terminatingGate)() + + controlPlaneConfig := framework.NewIntegrationTestControlPlaneConfig() + _, server, closeFn := framework.RunAnAPIServer(controlPlaneConfig) + defer closeFn() + + config := restclient.Config{Host: server.URL} + client, err := clientset.NewForConfig(&config) + if err != nil { + t.Fatalf("Error creating clientset: %v", err) + } + + resyncPeriod := 12 * time.Hour + informers := informers.NewSharedInformerFactory(client, resyncPeriod) + + epsController := endpointslice.NewController( + informers.Core().V1().Pods(), + informers.Core().V1().Services(), + informers.Core().V1().Nodes(), + informers.Discovery().V1().EndpointSlices(), + int32(100), + client, + 1*time.Second) + + // Start informer and controllers + stopCh := make(chan struct{}) + defer close(stopCh) + informers.Start(stopCh) + go epsController.Run(1, stopCh) + + // Create namespace + ns := framework.CreateTestingNamespace("test-endpoints-terminating", server, t) + defer framework.DeleteTestingNamespace(ns, server, t) + + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-node", + }, + } + + _, err = client.CoreV1().Nodes().Create(context.TODO(), node, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Failed to create test node: %v", err) + } + + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-service", + Namespace: ns.Name, + Labels: map[string]string{ + "foo": "bar", + }, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{ + "foo": "bar", + }, + Ports: []corev1.ServicePort{ + {Name: "port-443", Port: 443, Protocol: "TCP", TargetPort: intstr.FromInt(443)}, + }, + }, + } + + _, err = client.CoreV1().Services(ns.Name).Create(context.TODO(), svc, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Failed to create test Service: %v", err) + } + + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Labels: map[string]string{ + "foo": "bar", + }, + }, + Spec: corev1.PodSpec{ + NodeName: "fake-node", + Containers: []corev1.Container{ + { + Name: "fakename", + Image: "fakeimage", + Ports: []corev1.ContainerPort{ + { + Name: "port-443", + ContainerPort: 443, + }, + }, + }, + }, + }, + } + + pod, err = client.CoreV1().Pods(ns.Name).Create(context.TODO(), pod, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Failed to create test ready pod: %v", err) + } + + pod.Status = testcase.podStatus + _, err = client.CoreV1().Pods(ns.Name).UpdateStatus(context.TODO(), pod, metav1.UpdateOptions{}) + if err != nil { + t.Fatalf("Failed to update status for test ready pod: %v", err) + } + + // first check that endpoints are included, test should always have 1 initial endpoint + err = wait.PollImmediate(1*time.Second, 10*time.Second, func() (bool, error) { + esList, err := client.DiscoveryV1().EndpointSlices(ns.Name).List(context.TODO(), metav1.ListOptions{ + LabelSelector: discovery.LabelServiceName + "=" + svc.Name, + }) + + if err != nil { + return false, err + } + + if len(esList.Items) == 0 { + return false, nil + } + + numEndpoints := 0 + for _, slice := range esList.Items { + numEndpoints += len(slice.Endpoints) + } + + if numEndpoints > 0 { + return true, nil + } + + return false, nil + }) + if err != nil { + t.Errorf("Error waiting for endpoint slices: %v", err) + } + + // Delete pod and check endpoints slice conditions + err = client.CoreV1().Pods(ns.Name).Delete(context.TODO(), pod.Name, metav1.DeleteOptions{}) + if err != nil { + t.Fatalf("Failed to delete pod in terminating state: %v", err) + } + + // Validate that terminating the endpoint will result in the expected endpoints in EndpointSlice. + // Use a stricter timeout value here since we should try to catch regressions in the time it takes to remove terminated endpoints. + var endpoints []discovery.Endpoint + err = wait.PollImmediate(1*time.Second, 10*time.Second, func() (bool, error) { + esList, err := client.DiscoveryV1().EndpointSlices(ns.Name).List(context.TODO(), metav1.ListOptions{ + LabelSelector: discovery.LabelServiceName + "=" + svc.Name, + }) + + if err != nil { + return false, err + } + + if len(esList.Items) == 0 { + return false, nil + } + + endpoints = esList.Items[0].Endpoints + if len(endpoints) == 0 && len(testcase.expectedEndpoints) == 0 { + return true, nil + } + + if len(endpoints) != len(testcase.expectedEndpoints) { + return false, nil + } + + if !reflect.DeepEqual(endpoints[0].Addresses, testcase.expectedEndpoints[0].Addresses) { + return false, nil + } + + if !reflect.DeepEqual(endpoints[0].Conditions, testcase.expectedEndpoints[0].Conditions) { + return false, nil + } + + return true, nil + }) + if err != nil { + t.Logf("actual endpoints: %v", endpoints) + t.Logf("expected endpoints: %v", testcase.expectedEndpoints) + t.Errorf("unexpected endpoints: %v", err) + } + }) + } +}