mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-08-04 09:49:50 +00:00
add integration test
Since ServiceCIDR and IPAddresses are mostly API driven integration test will give us a good coverage, exercising real use cases like the migration from one ServiceCIDR range to a new range.
This commit is contained in:
Antonio Ojea
parent
4ff80864e1
commit
ccbcadb032
281
test/integration/servicecidr/migration_test.go
Normal file
281
test/integration/servicecidr/migration_test.go
Normal file
@ -0,0 +1,281 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2023 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package servicecidr
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
v1 "k8s.io/api/core/v1"
|
||||||
|
networkingv1alpha1 "k8s.io/api/networking/v1alpha1"
|
||||||
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||||
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
"k8s.io/apimachinery/pkg/util/wait"
|
||||||
|
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||||
|
"k8s.io/client-go/informers"
|
||||||
|
clientset "k8s.io/client-go/kubernetes"
|
||||||
|
featuregatetesting "k8s.io/component-base/featuregate/testing"
|
||||||
|
kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
|
||||||
|
"k8s.io/kubernetes/pkg/controller/servicecidrs"
|
||||||
|
"k8s.io/kubernetes/pkg/controlplane/controller/defaultservicecidr"
|
||||||
|
"k8s.io/kubernetes/pkg/features"
|
||||||
|
"k8s.io/kubernetes/test/integration/framework"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestMigrateServiceCIDR validates the steps necessary to migrate a cluster default ServiceCIDR
|
||||||
|
// including the existing kubernetes.default service.
|
||||||
|
// 1. start apiserver with --service-cluster-ip-range 192.168.0.0/29"
|
||||||
|
// 2. create services to use some addresses on the cidr
|
||||||
|
// 3. create a temporary new ServiceCIDR 10.168.0.0/24 to migrate the cluster to it
|
||||||
|
// 4. delete the default service CIDR so the allocators ignore it (it will be pending because of the finalizer and having still IPs)
|
||||||
|
// 5. recreate the services, the allocator should pick the temporary ServiceCIDR
|
||||||
|
// 6. start the new apiserver with the new ServiceCIDRs on the flags and shutdown the old one
|
||||||
|
// 7. delete the kubernetes.default service, the new apiserver will recreate it within the new ServiceCIDR
|
||||||
|
func TestMigrateServiceCIDR(t *testing.T) {
|
||||||
|
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.MultiCIDRServiceAllocator, true)()
|
||||||
|
ctx, cancelFn := context.WithCancel(context.Background())
|
||||||
|
defer cancelFn()
|
||||||
|
|
||||||
|
cidr1 := "192.168.0.0/29"
|
||||||
|
cidr2 := "10.168.0.0/24"
|
||||||
|
|
||||||
|
etcdOptions := framework.SharedEtcd()
|
||||||
|
apiServerOptions := kubeapiservertesting.NewDefaultTestServerOptions()
|
||||||
|
s1 := kubeapiservertesting.StartTestServerOrDie(t,
|
||||||
|
apiServerOptions,
|
||||||
|
[]string{
|
||||||
|
"--runtime-config=networking.k8s.io/v1alpha1=true",
|
||||||
|
"--service-cluster-ip-range=" + cidr1,
|
||||||
|
"--advertise-address=10.1.1.1",
|
||||||
|
"--disable-admission-plugins=ServiceAccount",
|
||||||
|
},
|
||||||
|
etcdOptions)
|
||||||
|
|
||||||
|
client1, err := clientset.NewForConfig(s1.ClientConfig)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ns := framework.CreateNamespaceOrDie(client1, "test-migrate-service-cidr", t)
|
||||||
|
|
||||||
|
resyncPeriod := 12 * time.Hour
|
||||||
|
informers1 := informers.NewSharedInformerFactory(client1, resyncPeriod)
|
||||||
|
// ServiceCIDR controller
|
||||||
|
go servicecidrs.NewController(
|
||||||
|
informers1.Networking().V1alpha1().ServiceCIDRs(),
|
||||||
|
informers1.Networking().V1alpha1().IPAddresses(),
|
||||||
|
client1,
|
||||||
|
).Run(ctx, 5)
|
||||||
|
informers1.Start(ctx.Done())
|
||||||
|
|
||||||
|
// the default serviceCIDR should have a finalizer and ready condition set to true
|
||||||
|
if err := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, time.Minute, false, func(ctx context.Context) (bool, error) {
|
||||||
|
cidr, err := client1.NetworkingV1alpha1().ServiceCIDRs().Get(context.TODO(), defaultservicecidr.DefaultServiceCIDRName, metav1.GetOptions{})
|
||||||
|
if err != nil && !apierrors.IsNotFound(err) {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
if len(cidr.Finalizers) == 0 {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return isServiceCIDRReady(cidr), nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("waiting for default service cidr ready condition set to false: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
svc := func(i int) *v1.Service {
|
||||||
|
return &v1.Service{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Name: fmt.Sprintf("svc-%v", i),
|
||||||
|
},
|
||||||
|
Spec: v1.ServiceSpec{
|
||||||
|
Type: v1.ServiceTypeClusterIP,
|
||||||
|
Ports: []v1.ServicePort{
|
||||||
|
{Port: 80},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// make 2 services , there will be still 3 free addresses
|
||||||
|
for i := 0; i < 2; i++ {
|
||||||
|
if _, err := client1.CoreV1().Services(ns.Name).Create(context.TODO(), svc(i), metav1.CreateOptions{}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Add a new service CIDR to be able to migrate the apiserver
|
||||||
|
if _, err := client1.NetworkingV1alpha1().ServiceCIDRs().Create(context.Background(), makeServiceCIDR("migration-cidr", cidr2, ""), metav1.CreateOptions{}); err != nil {
|
||||||
|
t.Fatalf("got unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// wait ServiceCIDR is ready
|
||||||
|
if err := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, time.Minute, false, func(ctx context.Context) (bool, error) {
|
||||||
|
cidr, err := client1.NetworkingV1alpha1().ServiceCIDRs().Get(context.TODO(), "migration-cidr", metav1.GetOptions{})
|
||||||
|
if err != nil && !apierrors.IsNotFound(err) {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return isServiceCIDRReady(cidr), nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("waiting for default service cidr ready condition set to false: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// delete the default ServiceCIDR so is no longer used for allocating IPs
|
||||||
|
if err := client1.NetworkingV1alpha1().ServiceCIDRs().Delete(context.Background(), defaultservicecidr.DefaultServiceCIDRName, metav1.DeleteOptions{}); err != nil {
|
||||||
|
t.Fatalf("got unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// the default serviceCIDR should be pending deletion with Ready condition set to false
|
||||||
|
if err := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, time.Minute, false, func(ctx context.Context) (bool, error) {
|
||||||
|
cidr, err := client1.NetworkingV1alpha1().ServiceCIDRs().Get(context.TODO(), defaultservicecidr.DefaultServiceCIDRName, metav1.GetOptions{})
|
||||||
|
if err != nil && !apierrors.IsNotFound(err) {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
for _, condition := range cidr.Status.Conditions {
|
||||||
|
if condition.Type == networkingv1alpha1.ServiceCIDRConditionReady {
|
||||||
|
return condition.Status == metav1.ConditionFalse, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false, nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("waiting for default service cidr ready condition set to false: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Migrate the services, delete the existing ones and recreate without specifying the ClusterIP
|
||||||
|
services, err := client1.CoreV1().Services("").List(context.TODO(), metav1.ListOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, svc := range services.Items {
|
||||||
|
// skip the default service since is managed by the apiserver
|
||||||
|
// and we want the new apiserver with the new service cidr to take over
|
||||||
|
if svc.Name == "kubernetes" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// wipe the necessary fields so we can recreate the Service
|
||||||
|
svc.ResourceVersion = ""
|
||||||
|
svc.Spec.ClusterIP = ""
|
||||||
|
svc.Spec.ClusterIPs = nil
|
||||||
|
svc.Status = v1.ServiceStatus{}
|
||||||
|
if err := client1.CoreV1().Services(svc.Namespace).Delete(context.Background(), svc.Name, metav1.DeleteOptions{}); err != nil {
|
||||||
|
t.Fatalf("got unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
svc, err := client1.CoreV1().Services(svc.Namespace).Create(context.Background(), &svc, metav1.CreateOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("got unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if !cidrContainsIP(cidr2, svc.Spec.ClusterIP) {
|
||||||
|
t.Fatalf("Service expected to have an ip in range 10.168.0.0/24, got %s", svc.Spec.ClusterIP)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// start second apiserver with the new range and new service cidr controller
|
||||||
|
s2 := kubeapiservertesting.StartTestServerOrDie(t,
|
||||||
|
apiServerOptions,
|
||||||
|
[]string{
|
||||||
|
"--runtime-config=networking.k8s.io/v1alpha1=true",
|
||||||
|
"--service-cluster-ip-range=" + cidr2,
|
||||||
|
"--advertise-address=10.1.1.1",
|
||||||
|
"--disable-admission-plugins=ServiceAccount",
|
||||||
|
},
|
||||||
|
etcdOptions)
|
||||||
|
defer s2.TearDownFn()
|
||||||
|
|
||||||
|
client2, err := clientset.NewForConfig(s2.ClientConfig)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
defer framework.DeleteNamespaceOrDie(client2, ns, t)
|
||||||
|
|
||||||
|
// switch the controller to the new apiserver
|
||||||
|
cancelFn()
|
||||||
|
s1.TearDownFn()
|
||||||
|
|
||||||
|
// ServiceCIDR controller
|
||||||
|
ctx2, cancelFn2 := context.WithCancel(context.Background())
|
||||||
|
defer cancelFn2()
|
||||||
|
informers2 := informers.NewSharedInformerFactory(client2, resyncPeriod)
|
||||||
|
go servicecidrs.NewController(
|
||||||
|
informers2.Networking().V1alpha1().ServiceCIDRs(),
|
||||||
|
informers2.Networking().V1alpha1().IPAddresses(),
|
||||||
|
client2,
|
||||||
|
).Run(ctx2, 5)
|
||||||
|
informers2.Start(ctx2.Done())
|
||||||
|
|
||||||
|
// delete the kubernetes.default service so the old DefaultServiceCIDR can be deleted
|
||||||
|
// and the new apiserver can take over
|
||||||
|
if err := client2.CoreV1().Services(metav1.NamespaceDefault).Delete(context.Background(), "kubernetes", metav1.DeleteOptions{}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// the default serviceCIDR should be the new one
|
||||||
|
if err := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, time.Minute, false, func(ctx context.Context) (bool, error) {
|
||||||
|
cidr, err := client2.NetworkingV1alpha1().ServiceCIDRs().Get(context.TODO(), defaultservicecidr.DefaultServiceCIDRName, metav1.GetOptions{})
|
||||||
|
if err != nil && !apierrors.IsNotFound(err) {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if cidr.Spec.IPv4 != cidr2 {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(cidr.Finalizers) == 0 {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, condition := range cidr.Status.Conditions {
|
||||||
|
if condition.Type == networkingv1alpha1.ServiceCIDRConditionReady {
|
||||||
|
return condition.Status == metav1.ConditionTrue, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false, nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("waiting for default service cidr ready condition set to true: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, time.Minute, false, func(ctx context.Context) (bool, error) {
|
||||||
|
svc, err := client2.CoreV1().Services(metav1.NamespaceDefault).Get(context.TODO(), "kubernetes", metav1.GetOptions{})
|
||||||
|
if err != nil && !apierrors.IsNotFound(err) {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if svc.Spec.ClusterIP != "10.168.0.1" {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
return true, nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("waiting for default service kubernetes.default to be migrated: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The temporary ServiceCIDR can be deleted now since the Default ServiceCIDR will cover it
|
||||||
|
if err := client2.NetworkingV1alpha1().ServiceCIDRs().Delete(context.Background(), "migration-cidr", metav1.DeleteOptions{}); err != nil {
|
||||||
|
t.Fatalf("got unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// wait ServiceCIDR no longer exist
|
||||||
|
if err := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, time.Minute, false, func(ctx context.Context) (bool, error) {
|
||||||
|
_, err := client2.NetworkingV1alpha1().ServiceCIDRs().Get(context.TODO(), "migration-cidr", metav1.GetOptions{})
|
||||||
|
if err != nil && !apierrors.IsNotFound(err) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
return true, nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("waiting for the migration service cidr to be deleted: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
353
test/integration/servicecidr/servicecidr_test.go
Normal file
353
test/integration/servicecidr/servicecidr_test.go
Normal file
@ -0,0 +1,353 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2023 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package servicecidr
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"net/netip"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
v1 "k8s.io/api/core/v1"
|
||||||
|
networkingv1alpha1 "k8s.io/api/networking/v1alpha1"
|
||||||
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||||
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
"k8s.io/apimachinery/pkg/util/wait"
|
||||||
|
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||||
|
"k8s.io/client-go/informers"
|
||||||
|
"k8s.io/client-go/kubernetes"
|
||||||
|
featuregatetesting "k8s.io/component-base/featuregate/testing"
|
||||||
|
kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
|
||||||
|
"k8s.io/kubernetes/pkg/controller/servicecidrs"
|
||||||
|
"k8s.io/kubernetes/pkg/features"
|
||||||
|
"k8s.io/kubernetes/test/integration/framework"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestServiceAllocNewServiceCIDR(t *testing.T) {
|
||||||
|
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.MultiCIDRServiceAllocator, true)()
|
||||||
|
|
||||||
|
etcdOptions := framework.SharedEtcd()
|
||||||
|
apiServerOptions := kubeapiservertesting.NewDefaultTestServerOptions()
|
||||||
|
s := kubeapiservertesting.StartTestServerOrDie(t,
|
||||||
|
apiServerOptions,
|
||||||
|
[]string{
|
||||||
|
"--runtime-config=networking.k8s.io/v1alpha1=true",
|
||||||
|
"--service-cluster-ip-range=192.168.0.0/29",
|
||||||
|
"--advertise-address=10.1.1.1",
|
||||||
|
"--disable-admission-plugins=ServiceAccount",
|
||||||
|
},
|
||||||
|
etcdOptions)
|
||||||
|
defer s.TearDownFn()
|
||||||
|
|
||||||
|
client, err := kubernetes.NewForConfig(s.ClientConfig)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
// ServiceCIDR controller
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
resyncPeriod := 12 * time.Hour
|
||||||
|
informerFactory := informers.NewSharedInformerFactory(client, resyncPeriod)
|
||||||
|
go servicecidrs.NewController(
|
||||||
|
informerFactory.Networking().V1alpha1().ServiceCIDRs(),
|
||||||
|
informerFactory.Networking().V1alpha1().IPAddresses(),
|
||||||
|
client,
|
||||||
|
).Run(ctx, 5)
|
||||||
|
informerFactory.Start(ctx.Done())
|
||||||
|
|
||||||
|
// /29 = 6 services, kubernetes.default takes the first address
|
||||||
|
// make 5 more services to take up all IPs
|
||||||
|
for i := 0; i < 5; i++ {
|
||||||
|
if _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(context.Background(), makeService(fmt.Sprintf("service-%d", i)), metav1.CreateOptions{}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Make another service. It will fail because we're out of cluster IPs
|
||||||
|
if _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(context.Background(), makeService("fail"), metav1.CreateOptions{}); err != nil {
|
||||||
|
if !strings.Contains(err.Error(), "range is full") {
|
||||||
|
t.Fatalf("unexpected error text: %v", err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
svcs, err := client.CoreV1().Services(metav1.NamespaceAll).List(context.Background(), metav1.ListOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error getting the services: %v", err)
|
||||||
|
}
|
||||||
|
allIPs := []string{}
|
||||||
|
for _, s := range svcs.Items {
|
||||||
|
allIPs = append(allIPs, s.Spec.ClusterIP)
|
||||||
|
}
|
||||||
|
t.Fatalf("unexpected creation success. The following IPs exist: %#v. It should only be possible to allocate 6 IP addresses in this cluster.\n\n%#v", allIPs, svcs)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add a new service CIDR to be able to create new IPs.
|
||||||
|
cidr := makeServiceCIDR("test2", "10.168.0.0/24", "")
|
||||||
|
if _, err := client.NetworkingV1alpha1().ServiceCIDRs().Create(context.Background(), cidr, metav1.CreateOptions{}); err != nil {
|
||||||
|
t.Fatalf("got unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
// wait ServiceCIDR is ready
|
||||||
|
if err := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, time.Minute, false, func(ctx context.Context) (bool, error) {
|
||||||
|
cidr, err := client.NetworkingV1alpha1().ServiceCIDRs().Get(context.TODO(), cidr.Name, metav1.GetOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return isServiceCIDRReady(cidr), nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("waiting for default service cidr ready condition set to false: %v", err)
|
||||||
|
}
|
||||||
|
// This time creating more Services should work
|
||||||
|
for i := 10; i < 150; i++ {
|
||||||
|
if _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(context.Background(), makeService(fmt.Sprintf("service-%d", i)), metav1.CreateOptions{}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A ServiceCIDR can be deleted if there are no orphan IPs or if the existing IPs are contained in other ServiceCIDR
|
||||||
|
// that is not being deleted.
|
||||||
|
// The test starts the apiserver with the range "192.168.0.0/29"
|
||||||
|
// Create Services to fill the range
|
||||||
|
// Creates a new ServiceCIDR cidr1 with the same range as the one defined in the apiserver
|
||||||
|
// Deletes cidr1 object will work since its range is covered by the default ServiceCIDR created by the apiserver flags
|
||||||
|
// Creates a new cidr2 with a different range than cidr1
|
||||||
|
// Creates a new service so it picks an IPAddress on this range because "192.168.0.0/29" is full at this point
|
||||||
|
// Creates a new cidr3 that contains cidr2
|
||||||
|
// Deletes cidr2 since it is covered by cidr3
|
||||||
|
// Tries to delete cidr3 but is blocked since there is an IPAddress
|
||||||
|
// Deletes the Service with the IPAddress blocking the deletion
|
||||||
|
// cidr3 must not exist at this point
|
||||||
|
func TestServiceCIDRDeletion(t *testing.T) {
|
||||||
|
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.MultiCIDRServiceAllocator, true)()
|
||||||
|
cidr1 := "192.168.0.0/29" // same as the default
|
||||||
|
cidr2 := "10.0.0.0/24" // new range
|
||||||
|
cidr3 := "10.0.0.0/16" // contains cidr2
|
||||||
|
|
||||||
|
etcdOptions := framework.SharedEtcd()
|
||||||
|
apiServerOptions := kubeapiservertesting.NewDefaultTestServerOptions()
|
||||||
|
s := kubeapiservertesting.StartTestServerOrDie(t,
|
||||||
|
apiServerOptions,
|
||||||
|
[]string{
|
||||||
|
"--runtime-config=networking.k8s.io/v1alpha1=true",
|
||||||
|
"--service-cluster-ip-range=" + cidr1,
|
||||||
|
"--advertise-address=172.16.1.1",
|
||||||
|
"--disable-admission-plugins=ServiceAccount",
|
||||||
|
},
|
||||||
|
etcdOptions)
|
||||||
|
defer s.TearDownFn()
|
||||||
|
|
||||||
|
client, err := kubernetes.NewForConfig(s.ClientConfig)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ns := framework.CreateNamespaceOrDie(client, "test-service-cidr-deletion", t)
|
||||||
|
defer framework.DeleteNamespaceOrDie(client, ns, t)
|
||||||
|
|
||||||
|
// ServiceCIDR controller
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
resyncPeriod := 12 * time.Hour
|
||||||
|
informerFactory := informers.NewSharedInformerFactory(client, resyncPeriod)
|
||||||
|
go servicecidrs.NewController(
|
||||||
|
informerFactory.Networking().V1alpha1().ServiceCIDRs(),
|
||||||
|
informerFactory.Networking().V1alpha1().IPAddresses(),
|
||||||
|
client,
|
||||||
|
).Run(ctx, 5)
|
||||||
|
informerFactory.Start(ctx.Done())
|
||||||
|
|
||||||
|
// /29 = 6 services, kubernetes.default takes the first address
|
||||||
|
// make 5 more services to take up all IPs
|
||||||
|
for i := 0; i < 5; i++ {
|
||||||
|
if _, err := client.CoreV1().Services(ns.Name).Create(context.Background(), makeService(fmt.Sprintf("service-%d", i)), metav1.CreateOptions{}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// create a new ServiceCIDRs that overlaps the default one
|
||||||
|
_, err = client.NetworkingV1alpha1().ServiceCIDRs().Create(ctx, makeServiceCIDR("cidr1", cidr1, ""), metav1.CreateOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal((err))
|
||||||
|
}
|
||||||
|
// Wait until is ready.
|
||||||
|
if err := wait.PollUntilContextTimeout(context.Background(), 250*time.Millisecond, 30*time.Second, false, func(ctx context.Context) (bool, error) {
|
||||||
|
cidr, err := client.NetworkingV1alpha1().ServiceCIDRs().Get(ctx, "cidr1", metav1.GetOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
return isServiceCIDRReady(cidr), nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("cidr1 is not ready")
|
||||||
|
}
|
||||||
|
// we should be able to delete the ServiceCIDR despite it contains IP addresses as it overlaps with the default ServiceCIDR
|
||||||
|
err = client.NetworkingV1alpha1().ServiceCIDRs().Delete(ctx, "cidr1", metav1.DeleteOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal((err))
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := wait.PollUntilContextTimeout(context.Background(), 250*time.Millisecond, 30*time.Second, false, func(ctx context.Context) (bool, error) {
|
||||||
|
_, err := client.NetworkingV1alpha1().ServiceCIDRs().Get(ctx, "cidr1", metav1.GetOptions{})
|
||||||
|
if err != nil && apierrors.IsNotFound(err) {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
return false, nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("cidr1 has not been deleted")
|
||||||
|
}
|
||||||
|
|
||||||
|
// add a new ServiceCIDR with a new range
|
||||||
|
_, err = client.NetworkingV1alpha1().ServiceCIDRs().Create(ctx, makeServiceCIDR("cidr2", cidr2, ""), metav1.CreateOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal((err))
|
||||||
|
}
|
||||||
|
// wait the allocator process the new ServiceCIDR
|
||||||
|
// Wait until is ready.
|
||||||
|
if err := wait.PollUntilContextTimeout(context.Background(), 250*time.Millisecond, 30*time.Second, false, func(ctx context.Context) (bool, error) {
|
||||||
|
cidr, err := client.NetworkingV1alpha1().ServiceCIDRs().Get(ctx, "cidr2", metav1.GetOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
return isServiceCIDRReady(cidr), nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("cidr2 is not ready")
|
||||||
|
}
|
||||||
|
// create a new Service so it will take a new IP address from the new range
|
||||||
|
svc, err := client.CoreV1().Services(ns.Name).Create(context.Background(), makeService("new-cidr-service"), metav1.CreateOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !cidrContainsIP(cidr2, svc.Spec.ClusterIP) {
|
||||||
|
t.Fatalf("Service %s expected to have an IP on range %s, got %s", svc.Name, cidr2, svc.Spec.ClusterIP)
|
||||||
|
}
|
||||||
|
|
||||||
|
// add a new ServiceCIDR that overlaps the existing one
|
||||||
|
_, err = client.NetworkingV1alpha1().ServiceCIDRs().Create(ctx, makeServiceCIDR("cidr3", cidr3, ""), metav1.CreateOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal((err))
|
||||||
|
}
|
||||||
|
// Wait until is ready.
|
||||||
|
if err := wait.PollUntilContextTimeout(context.Background(), 250*time.Millisecond, 30*time.Second, false, func(ctx context.Context) (bool, error) {
|
||||||
|
cidr, err := client.NetworkingV1alpha1().ServiceCIDRs().Get(ctx, "cidr3", metav1.GetOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
return isServiceCIDRReady(cidr), nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("cidr3 is not ready")
|
||||||
|
}
|
||||||
|
// we should be able to delete the ServiceCIDR2 despite it contains IP addresses as it is contained on ServiceCIDR3
|
||||||
|
err = client.NetworkingV1alpha1().ServiceCIDRs().Delete(ctx, "cidr2", metav1.DeleteOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal((err))
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := wait.PollUntilContextTimeout(context.Background(), 250*time.Millisecond, 30*time.Second, false, func(ctx context.Context) (bool, error) {
|
||||||
|
_, err := client.NetworkingV1alpha1().ServiceCIDRs().Get(ctx, "cidr2", metav1.GetOptions{})
|
||||||
|
if err != nil && apierrors.IsNotFound(err) {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
return false, nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("cidr2 has not been deleted")
|
||||||
|
}
|
||||||
|
|
||||||
|
// serviceCIDR3 will not be able to be deleted until the IPAddress is removed
|
||||||
|
err = client.NetworkingV1alpha1().ServiceCIDRs().Delete(ctx, "cidr3", metav1.DeleteOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal((err))
|
||||||
|
}
|
||||||
|
// Wait until is not ready.
|
||||||
|
if err := wait.PollUntilContextTimeout(context.Background(), 250*time.Millisecond, 30*time.Second, false, func(ctx context.Context) (bool, error) {
|
||||||
|
cidr, err := client.NetworkingV1alpha1().ServiceCIDRs().Get(ctx, "cidr3", metav1.GetOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
for _, condition := range cidr.Status.Conditions {
|
||||||
|
if condition.Type == networkingv1alpha1.ServiceCIDRConditionReady {
|
||||||
|
return condition.Status == metav1.ConditionStatus(metav1.ConditionFalse), nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false, nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("cidr3 is ready")
|
||||||
|
}
|
||||||
|
|
||||||
|
// delete the service blocking the deletion
|
||||||
|
if err := client.CoreV1().Services(ns.Name).Delete(context.Background(), "new-cidr-service", metav1.DeleteOptions{}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// cidr3 must not exist
|
||||||
|
if err := wait.PollUntilContextTimeout(context.Background(), 250*time.Millisecond, 30*time.Second, false, func(ctx context.Context) (bool, error) {
|
||||||
|
_, err := client.NetworkingV1alpha1().ServiceCIDRs().Get(ctx, "cidr3", metav1.GetOptions{})
|
||||||
|
if err != nil && apierrors.IsNotFound(err) {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
return false, nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("cidr3 has not been deleted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func makeServiceCIDR(name, ipv4, ipv6 string) *networkingv1alpha1.ServiceCIDR {
|
||||||
|
serviceCIDR := &networkingv1alpha1.ServiceCIDR{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Name: name,
|
||||||
|
},
|
||||||
|
Spec: networkingv1alpha1.ServiceCIDRSpec{
|
||||||
|
IPv4: ipv4,
|
||||||
|
IPv6: ipv6,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
return serviceCIDR
|
||||||
|
}
|
||||||
|
|
||||||
|
func makeService(name string) *v1.Service {
|
||||||
|
return &v1.Service{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Name: name,
|
||||||
|
},
|
||||||
|
Spec: v1.ServiceSpec{
|
||||||
|
Type: v1.ServiceTypeClusterIP,
|
||||||
|
Ports: []v1.ServicePort{
|
||||||
|
{Port: 80},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// returns true of the ServiceCIDRConditionReady is true
|
||||||
|
func isServiceCIDRReady(serviceCIDR *networkingv1alpha1.ServiceCIDR) bool {
|
||||||
|
if serviceCIDR == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, condition := range serviceCIDR.Status.Conditions {
|
||||||
|
if condition.Type == networkingv1alpha1.ServiceCIDRConditionReady {
|
||||||
|
return condition.Status == metav1.ConditionStatus(metav1.ConditionTrue)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func cidrContainsIP(cidr, ip string) bool {
|
||||||
|
prefix := netip.MustParsePrefix(cidr)
|
||||||
|
address := netip.MustParseAddr(ip)
|
||||||
|
return prefix.Contains(address)
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user