From 2a5ad65a9ab2b143ec4f86c4706598a4d7fdea88 Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Mon, 30 Aug 2021 17:30:00 +0200 Subject: [PATCH] e2e test apiserver endpoint and endpointslices The e2e test "should have Endpoints and EndpointSlices pointing to the API Server Service" was veryfing the current endpoints reconciler implementation on the apiservers, however, users may disable the endpoint reconciler and create their own. This e2e test is also a conformance test, so we should test the behaviour and not the implementation details. The test verifies that a kubernetes.default service exist, an endpoint and endpoint slices object referencing that service exist and are equivalent. --- test/conformance/testdata/conformance.yaml | 5 +- test/e2e/network/endpointslice.go | 87 ++++++++++++++++++---- 2 files changed, 77 insertions(+), 15 deletions(-) diff --git a/test/conformance/testdata/conformance.yaml b/test/conformance/testdata/conformance.yaml index 8bdf2e478db..4acc4164e05 100755 --- a/test/conformance/testdata/conformance.yaml +++ b/test/conformance/testdata/conformance.yaml @@ -1271,8 +1271,9 @@ description: The discovery.k8s.io API group MUST exist in the /apis discovery document. The discovery.k8s.io/v1 API group/version MUST exist in the /apis/discovery.k8s.io discovery document. The endpointslices resource MUST exist in the /apis/discovery.k8s.io/v1 - discovery document. API Server should create self referential Endpoints and EndpointSlices - named "kubernetes" in the default namespace. + discovery document. The cluster MUST have a service named "kubernetes" on the + default namespace referencing the API servers. The "kubernetes.default" service + MUST have Endpoints and EndpointSlices pointing to each API server instance. release: v1.21 file: test/e2e/network/endpointslice.go - testname: EndpointSlice API diff --git a/test/e2e/network/endpointslice.go b/test/e2e/network/endpointslice.go index 8a7373c8b1f..1caf3a95d95 100644 --- a/test/e2e/network/endpointslice.go +++ b/test/e2e/network/endpointslice.go @@ -57,26 +57,33 @@ var _ = common.SIGDescribe("EndpointSlice", func() { Description: The discovery.k8s.io API group MUST exist in the /apis discovery document. The discovery.k8s.io/v1 API group/version MUST exist in the /apis/discovery.k8s.io discovery document. The endpointslices resource MUST exist in the /apis/discovery.k8s.io/v1 discovery document. - API Server should create self referential Endpoints and EndpointSlices named "kubernetes" in the default namespace. + The cluster MUST have a service named "kubernetes" on the default namespace referencing the API servers. + The "kubernetes.default" service MUST have Endpoints and EndpointSlices pointing to each API server instance. */ framework.ConformanceIt("should have Endpoints and EndpointSlices pointing to API Server", func() { namespace := "default" name := "kubernetes" + // verify "kubernetes.default" service exist + _, err := cs.CoreV1().Services(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + framework.ExpectNoError(err, "error obtaining API server \"kubernetes\" Service resource on \"default\" namespace") + + // verify Endpoints for the API servers exist endpoints, err := cs.CoreV1().Endpoints(namespace).Get(context.TODO(), name, metav1.GetOptions{}) - framework.ExpectNoError(err, "error creating Endpoints resource") - if len(endpoints.Subsets) != 1 { - framework.Failf("Expected 1 subset in endpoints, got %d: %#v", len(endpoints.Subsets), endpoints.Subsets) + framework.ExpectNoError(err, "error obtaining API server \"kubernetes\" Endpoint resource on \"default\" namespace") + if len(endpoints.Subsets) == 0 { + framework.Failf("Expected at least 1 subset in endpoints, got %d: %#v", len(endpoints.Subsets), endpoints.Subsets) + } + // verify EndpointSlices for the API servers exist + endpointSliceList, err := cs.DiscoveryV1().EndpointSlices(namespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: "kubernetes.io/service-name=" + name, + }) + framework.ExpectNoError(err, "error obtaining API server \"kubernetes\" EndpointSlice resource on \"default\" namespace") + if len(endpointSliceList.Items) == 0 { + framework.Failf("Expected at least 1 EndpointSlice, got %d: %#v", len(endpoints.Subsets), endpoints.Subsets) } - endpointSubset := endpoints.Subsets[0] - endpointSlice, err := cs.DiscoveryV1().EndpointSlices(namespace).Get(context.TODO(), name, metav1.GetOptions{}) - framework.ExpectNoError(err, "error creating EndpointSlice resource") - if len(endpointSlice.Ports) != len(endpointSubset.Ports) { - framework.Failf("Expected EndpointSlice to have %d ports, got %d: %#v", len(endpointSubset.Ports), len(endpointSlice.Ports), endpointSlice.Ports) - } - numExpectedEndpoints := len(endpointSubset.Addresses) + len(endpointSubset.NotReadyAddresses) - if len(endpointSlice.Endpoints) != numExpectedEndpoints { - framework.Failf("Expected EndpointSlice to have %d endpoints, got %d: %#v", numExpectedEndpoints, len(endpointSlice.Endpoints), endpointSlice.Endpoints) + if !endpointSlicesEqual(endpoints, endpointSliceList) { + framework.Failf("Expected EndpointSlice to have same addresses and port as Endpoints, got %#v: %#v", endpoints, endpointSliceList) } }) @@ -785,3 +792,57 @@ func createServiceReportErr(cs clientset.Interface, ns string, service *v1.Servi framework.ExpectNoError(err, "error deleting Service") return svc } + +// endpointSlicesEqual compare if the Endpoint and the EndpointSliceList contains the same endpoints values +// as in addresses and ports, considering Ready and Unready addresses +func endpointSlicesEqual(endpoints *v1.Endpoints, endpointSliceList *discoveryv1.EndpointSliceList) bool { + // get the apiserver endpoint addresses + epAddresses := sets.NewString() + epPorts := sets.NewInt32() + for _, subset := range endpoints.Subsets { + for _, addr := range subset.Addresses { + epAddresses.Insert(addr.IP) + } + for _, addr := range subset.NotReadyAddresses { + epAddresses.Insert(addr.IP) + } + for _, port := range subset.Ports { + epPorts.Insert(port.Port) + } + } + framework.Logf("Endpoints addresses: %v , ports: %v", epAddresses.List(), epPorts.List()) + + // Endpoints are single stack, and must match the primary IP family of the Service kubernetes.default + // However, EndpointSlices can be IPv4 or IPv6, we can only compare the Slices that match the same IP family + // framework.TestContext.ClusterIsIPv6() reports the IP family of the kubernetes.default service + var addrType discoveryv1.AddressType + if framework.TestContext.ClusterIsIPv6() { + addrType = discoveryv1.AddressTypeIPv6 + } else { + addrType = discoveryv1.AddressTypeIPv4 + } + + // get the apiserver addresses from the endpoint slice list + sliceAddresses := sets.NewString() + slicePorts := sets.NewInt32() + for _, slice := range endpointSliceList.Items { + if slice.AddressType != addrType { + framework.Logf("Skipping slice %s: wanted %s family, got %s", slice.Name, addrType, slice.AddressType) + continue + } + for _, s := range slice.Endpoints { + sliceAddresses.Insert(s.Addresses...) + } + for _, ports := range slice.Ports { + if ports.Port != nil { + slicePorts.Insert(*ports.Port) + } + } + } + + framework.Logf("EndpointSlices addresses: %v , ports: %v", sliceAddresses.List(), slicePorts.List()) + if sliceAddresses.Equal(epAddresses) && slicePorts.Equal(epPorts) { + return true + } + return false +}