diff --git a/test/integration/clustercidr/ipam_test.go b/test/integration/clustercidr/ipam_test.go new file mode 100644 index 00000000000..167370e157c --- /dev/null +++ b/test/integration/clustercidr/ipam_test.go @@ -0,0 +1,249 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package clustercidr + +import ( + "context" + "net" + "reflect" + "testing" + "time" + + v1 "k8s.io/api/core/v1" + networkingv1alpha1 "k8s.io/api/networking/v1alpha1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/client-go/informers" + clientset "k8s.io/client-go/kubernetes" + featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/kubernetes/cmd/kube-apiserver/app/options" + "k8s.io/kubernetes/pkg/controller/nodeipam" + "k8s.io/kubernetes/pkg/controller/nodeipam/ipam" + "k8s.io/kubernetes/pkg/features" + "k8s.io/kubernetes/test/integration/framework" + netutils "k8s.io/utils/net" +) + +func TestIPAMMultiCIDRRangeAllocatorType(t *testing.T) { + + // set the feature gate to enable MultiCIDRRangeAllocator + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.MultiCIDRRangeAllocator, true)() + + _, kubeConfig, tearDownFn := framework.StartTestServer(t, framework.TestServerSetup{ + ModifyServerRunOptions: func(opts *options.ServerRunOptions) { + // Disable ServiceAccount admission plugin as we don't have serviceaccount controller running. + opts.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount", "TaintNodesByCondition"} + opts.APIEnablement.RuntimeConfig.Set("networking.k8s.io/v1alpha1=true") + }, + }) + defer tearDownFn() + + clientSet := clientset.NewForConfigOrDie(kubeConfig) + sharedInformer := informers.NewSharedInformerFactory(clientSet, 1*time.Hour) + + ipamController := booststrapMultiCIDRRangeAllocator(t, clientSet, sharedInformer) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go ipamController.Run(ctx.Done()) + sharedInformer.Start(ctx.Done()) + + tests := []struct { + name string + clusterCIDR *networkingv1alpha1.ClusterCIDR + node *v1.Node + expectedPodCIDRs []string + }{ + { + name: "Default dualstack Pod CIDRs assigned to a node, node labels matching no ClusterCIDR nodeSelectors", + clusterCIDR: nil, + node: makeNode("default-node", map[string]string{"label": "unmatched"}), + expectedPodCIDRs: []string{"10.96.0.0/24", "fd00:10:96::/120"}, + }, + { + name: "Dualstack Pod CIDRs assigned to a node from a CC created during bootstrap", + clusterCIDR: nil, + node: makeNode("bootstrap-node", map[string]string{"bootstrap": "true"}), + expectedPodCIDRs: []string{"10.2.1.0/24", "fd00:20:96::100/120"}, + }, + { + name: "Single stack IPv4 Pod CIDR assigned to a node", + clusterCIDR: makeClusterCIDR("ipv4-cc", "10.0.0.0/16", "", nodeSelector(map[string][]string{"ipv4": {"true"}, "singlestack": {"true"}})), + node: makeNode("ipv4-node", map[string]string{"ipv4": "true", "singlestack": "true"}), + expectedPodCIDRs: []string{"10.0.0.0/24"}, + }, + { + name: "Single stack IPv6 Pod CIDR assigned to a node", + clusterCIDR: makeClusterCIDR("ipv6-cc", "", "fd00:20:100::/112", nodeSelector(map[string][]string{"ipv6": {"true"}})), + node: makeNode("ipv6-node", map[string]string{"ipv6": "true"}), + expectedPodCIDRs: []string{"fd00:20:100::/120"}, + }, + { + name: "DualStack Pod CIDRs assigned to a node", + clusterCIDR: makeClusterCIDR("dualstack-cc", "192.168.0.0/16", "fd00:30:100::/112", nodeSelector(map[string][]string{"ipv4": {"true"}, "ipv6": {"true"}})), + node: makeNode("dualstack-node", map[string]string{"ipv4": "true", "ipv6": "true"}), + expectedPodCIDRs: []string{"192.168.0.0/24", "fd00:30:100::/120"}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if test.clusterCIDR != nil { + // Create the test ClusterCIDR + if _, err := clientSet.NetworkingV1alpha1().ClusterCIDRs().Create(context.TODO(), test.clusterCIDR, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + + // Wait for the ClusterCIDR to be created + if err := wait.PollImmediate(time.Second, 5*time.Second, func() (bool, error) { + cc, err := clientSet.NetworkingV1alpha1().ClusterCIDRs().Get(context.TODO(), test.clusterCIDR.Name, metav1.GetOptions{}) + if err != nil { + return false, err + } + return cc != nil, nil + }); err != nil { + t.Fatalf("failed while waiting for ClusterCIDR %q to be created: %v", test.clusterCIDR.Name, err) + } + } + + // Sleep for one second to make sure the controller process the new created ClusterCIDR. + time.Sleep(1 * time.Second) + + if _, err := clientSet.CoreV1().Nodes().Create(context.TODO(), test.node, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + if gotPodCIDRs, err := nodePodCIDRs(clientSet, test.node.Name); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(test.expectedPodCIDRs, gotPodCIDRs) { + t.Errorf("unexpected result, expected Pod CIDRs %v but got %v", test.expectedPodCIDRs, gotPodCIDRs) + } + }) + } +} + +func booststrapMultiCIDRRangeAllocator(t *testing.T, + clientSet clientset.Interface, + sharedInformer informers.SharedInformerFactory, +) *nodeipam.Controller { + _, clusterCIDRv4, _ := netutils.ParseCIDRSloppy("10.96.0.0/12") // allows up to 8K nodes + _, clusterCIDRv6, _ := netutils.ParseCIDRSloppy("fd00:10:96::/112") // allows up to 8K nodes + _, serviceCIDR, _ := netutils.ParseCIDRSloppy("10.94.0.0/24") // does not matter for test - pick upto 250 services + _, secServiceCIDR, _ := netutils.ParseCIDRSloppy("2001:db2::/120") // does not matter for test - pick upto 250 services + + // order is ipv4 - ipv6 by convention for dual stack + clusterCIDRs := []*net.IPNet{clusterCIDRv4, clusterCIDRv6} + nodeMaskCIDRs := []int{24, 120} + + // set the current state of the informer, we can preseed nodes and ClusterCIDRs so we + // can simulate the bootstrap + initialCC := makeClusterCIDR("initial-cc", "10.2.0.0/16", "fd00:20:96::/112", nodeSelector(map[string][]string{"bootstrap": {"true"}})) + if _, err := clientSet.NetworkingV1alpha1().ClusterCIDRs().Create(context.TODO(), initialCC, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + + initialNode := makeNode("initial-node", map[string]string{"bootstrap": "true"}) + if _, err := clientSet.CoreV1().Nodes().Create(context.TODO(), initialNode, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + + ipamController, err := nodeipam.NewNodeIpamController( + sharedInformer.Core().V1().Nodes(), + sharedInformer.Networking().V1alpha1().ClusterCIDRs(), + nil, + clientSet, + clusterCIDRs, + serviceCIDR, + secServiceCIDR, + nodeMaskCIDRs, + ipam.MultiCIDRRangeAllocatorType, + ) + if err != nil { + t.Fatal(err) + } + + return ipamController +} + +func makeNode(name string, labels map[string]string) *v1.Node { + return &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: labels, + }, + Status: v1.NodeStatus{ + Capacity: v1.ResourceList{ + v1.ResourcePods: *resource.NewQuantity(110, resource.DecimalSI), + v1.ResourceCPU: resource.MustParse("4"), + v1.ResourceMemory: resource.MustParse("32Gi"), + }, + Phase: v1.NodeRunning, + Conditions: []v1.NodeCondition{ + {Type: v1.NodeReady, Status: v1.ConditionTrue}, + }, + }, + } +} + +func makeClusterCIDR(name, ipv4CIDR, ipv6CIDR string, nodeSelector *v1.NodeSelector) *networkingv1alpha1.ClusterCIDR { + return &networkingv1alpha1.ClusterCIDR{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: networkingv1alpha1.ClusterCIDRSpec{ + PerNodeHostBits: 8, + IPv4: ipv4CIDR, + IPv6: ipv6CIDR, + NodeSelector: nodeSelector, + }, + } +} + +func nodeSelector(labels map[string][]string) *v1.NodeSelector { + testNodeSelector := &v1.NodeSelector{} + + for key, values := range labels { + nst := v1.NodeSelectorTerm{ + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: key, + Operator: v1.NodeSelectorOpIn, + Values: values, + }, + }, + } + testNodeSelector.NodeSelectorTerms = append(testNodeSelector.NodeSelectorTerms, nst) + } + + return testNodeSelector +} + +func nodePodCIDRs(c clientset.Interface, name string) ([]string, error) { + var node *v1.Node + nodePollErr := wait.PollImmediate(time.Second, 5*time.Second, func() (bool, error) { + var err error + node, err = c.CoreV1().Nodes().Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + return false, err + } + return len(node.Spec.PodCIDRs) > 0, nil + }) + + return node.Spec.PodCIDRs, nodePollErr +} diff --git a/test/integration/clustercidr/main_test.go b/test/integration/clustercidr/main_test.go new file mode 100644 index 00000000000..cf920a27032 --- /dev/null +++ b/test/integration/clustercidr/main_test.go @@ -0,0 +1,27 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package clustercidr + +import ( + "testing" + + "k8s.io/kubernetes/test/integration/framework" +) + +func TestMain(m *testing.M) { + framework.EtcdMain(m.Run) +}