diff --git a/test/integration/volumescheduling/BUILD b/test/integration/volumescheduling/BUILD index 16ed1b528d0..66a834f42ae 100644 --- a/test/integration/volumescheduling/BUILD +++ b/test/integration/volumescheduling/BUILD @@ -17,19 +17,23 @@ go_test( tags = ["integration"], deps = [ "//pkg/controller/volume/persistentvolume:go_default_library", + "//pkg/features:go_default_library", "//pkg/scheduler/framework/plugins/nodevolumelimits:go_default_library", "//pkg/volume:go_default_library", "//pkg/volume/testing:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/storage/v1:go_default_library", + "//staging/src/k8s.io/api/storage/v1alpha1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/rand:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", + "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library", "//test/integration/framework:go_default_library", "//test/utils/image:go_default_library", "//vendor/k8s.io/klog/v2:go_default_library", @@ -55,6 +59,8 @@ go_library( importpath = "k8s.io/kubernetes/test/integration/volumescheduling", deps = [ "//pkg/api/v1/pod:go_default_library", + "//pkg/features:go_default_library", + "//pkg/master:go_default_library", "//pkg/scheduler:go_default_library", "//pkg/scheduler/profile:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", @@ -63,6 +69,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", diff --git a/test/integration/volumescheduling/util.go b/test/integration/volumescheduling/util.go index a4efc0fde6f..045ab0dbb6f 100644 --- a/test/integration/volumescheduling/util.go +++ b/test/integration/volumescheduling/util.go @@ -29,11 +29,14 @@ import ( "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/admission" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/events" podutil "k8s.io/kubernetes/pkg/api/v1/pod" + "k8s.io/kubernetes/pkg/features" + "k8s.io/kubernetes/pkg/master" "k8s.io/kubernetes/pkg/scheduler" "k8s.io/kubernetes/pkg/scheduler/profile" "k8s.io/kubernetes/test/integration/framework" @@ -52,7 +55,8 @@ type testContext struct { } // initTestMaster initializes a test environment and creates a master with default -// configuration. +// configuration. Alpha resources are enabled automatically if the corresponding feature +// is enabled. func initTestMaster(t *testing.T, nsPrefix string, admission admission.Interface) *testContext { ctx, cancelFunc := context.WithCancel(context.Background()) testCtx := testContext{ @@ -68,6 +72,14 @@ func initTestMaster(t *testing.T, nsPrefix string, admission admission.Interface })) masterConfig := framework.NewIntegrationTestMasterConfig() + resourceConfig := master.DefaultAPIResourceConfigSource() + if utilfeature.DefaultFeatureGate.Enabled(features.CSIStorageCapacity) { + resourceConfig.EnableVersions(schema.GroupVersion{ + Group: "storage.k8s.io", + Version: "v1alpha1", + }) + } + masterConfig.ExtraConfig.APIResourceConfigSource = resourceConfig if admission != nil { masterConfig.GenericConfig.AdmissionControl = admission diff --git a/test/integration/volumescheduling/volume_binding_test.go b/test/integration/volumescheduling/volume_binding_test.go index ff6ae43d359..bda15d111c0 100644 --- a/test/integration/volumescheduling/volume_binding_test.go +++ b/test/integration/volumescheduling/volume_binding_test.go @@ -31,15 +31,19 @@ import ( v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" + storagev1alpha1 "k8s.io/api/storage/v1alpha1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/rand" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/util/workqueue" + featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/controller/volume/persistentvolume" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodevolumelimits" "k8s.io/kubernetes/pkg/volume" volumetest "k8s.io/kubernetes/pkg/volume/testing" @@ -78,7 +82,7 @@ const ( podLimit = 50 volsPerPod = 3 nodeAffinityLabelKey = "kubernetes.io/hostname" - provisionerPluginName = "kubernetes.io/mock-provisioner" + provisionerPluginName = "mock-provisioner.kubernetes.io" ) type testPV struct { @@ -700,10 +704,17 @@ func TestPVAffinityConflict(t *testing.T) { } func TestVolumeProvision(t *testing.T) { + t.Run("with CSIStorageCapacity", func(t *testing.T) { testVolumeProvision(t, true) }) + t.Run("without CSIStorageCapacity", func(t *testing.T) { testVolumeProvision(t, false) }) +} + +func testVolumeProvision(t *testing.T, storageCapacity bool) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIStorageCapacity, storageCapacity)() + config := setupCluster(t, "volume-scheduling", 1, 0, 0) defer config.teardown() - cases := map[string]struct { + type testcaseType struct { pod *v1.Pod pvs []*testPV boundPvcs []*testPVC @@ -711,7 +722,9 @@ func TestVolumeProvision(t *testing.T) { // Create these, but they should not be bound in the end unboundPvcs []*testPVC shouldFail bool - }{ + } + + cases := map[string]testcaseType{ "wait provisioned": { pod: makePod("pod-pvc-canprovision", config.ns, []string{"pvc-canprovision"}), provisionedPvcs: []*testPVC{{"pvc-canprovision", classWait, ""}}, @@ -748,9 +761,7 @@ func TestVolumeProvision(t *testing.T) { }, } - for name, test := range cases { - klog.Infof("Running test %v", name) - + run := func(t *testing.T, test testcaseType) { // Create StorageClasses suffix := rand.String(4) classes := map[string]*storagev1.StorageClass{} @@ -831,6 +842,152 @@ func TestVolumeProvision(t *testing.T) { // Force delete objects, but they still may not be immediately removed deleteTestObjects(config.client, config.ns, deleteOption) } + + for name, test := range cases { + t.Run(name, func(t *testing.T) { run(t, test) }) + } +} + +// TestCapacity covers different scenarios involving CSIStorageCapacity objects. +func TestCapacity(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIStorageCapacity, true)() + + config := setupCluster(t, "volume-scheduling", 1, 0, 0) + defer config.teardown() + + type testcaseType struct { + pod *v1.Pod + pvcs []*testPVC + haveCapacity bool + capacitySupported bool + } + + cases := map[string]testcaseType{ + "baseline": { + pod: makePod("pod-pvc-canprovision", config.ns, []string{"pvc-canprovision"}), + pvcs: []*testPVC{{"pvc-canprovision", classWait, ""}}, + }, + "out of space": { + pod: makePod("pod-pvc-canprovision", config.ns, []string{"pvc-canprovision"}), + pvcs: []*testPVC{{"pvc-canprovision", classWait, ""}}, + capacitySupported: true, + }, + "with space": { + pod: makePod("pod-pvc-canprovision", config.ns, []string{"pvc-canprovision"}), + pvcs: []*testPVC{{"pvc-canprovision", classWait, ""}}, + capacitySupported: true, + haveCapacity: true, + }, + "ignored": { + pod: makePod("pod-pvc-canprovision", config.ns, []string{"pvc-canprovision"}), + pvcs: []*testPVC{{"pvc-canprovision", classWait, ""}}, + haveCapacity: true, + }, + } + + run := func(t *testing.T, test testcaseType) { + // Create StorageClasses + suffix := rand.String(4) + classes := map[string]*storagev1.StorageClass{} + classes[classImmediate] = makeDynamicProvisionerStorageClass(fmt.Sprintf("immediate-%v", suffix), &modeImmediate, nil) + classes[classWait] = makeDynamicProvisionerStorageClass(fmt.Sprintf("wait-%v", suffix), &modeWait, nil) + topo := []v1.TopologySelectorTerm{ + { + MatchLabelExpressions: []v1.TopologySelectorLabelRequirement{ + { + Key: nodeAffinityLabelKey, + Values: []string{node2}, + }, + }, + }, + } + classes[classTopoMismatch] = makeDynamicProvisionerStorageClass(fmt.Sprintf("topomismatch-%v", suffix), &modeWait, topo) + for _, sc := range classes { + if _, err := config.client.StorageV1().StorageClasses().Create(context.TODO(), sc, metav1.CreateOptions{}); err != nil { + t.Fatalf("Failed to create StorageClass %q: %v", sc.Name, err) + } + } + + // The provisioner isn't actually a CSI driver, but + // that doesn't matter here. + if test.capacitySupported { + if _, err := config.client.StorageV1().CSIDrivers().Create(context.TODO(), + &storagev1.CSIDriver{ + ObjectMeta: metav1.ObjectMeta{ + Name: provisionerPluginName, + }, + Spec: storagev1.CSIDriverSpec{ + StorageCapacity: &test.capacitySupported, + }, + }, + metav1.CreateOptions{}); err != nil { + t.Fatalf("Failed to create CSIDriver: %v", err) + } + + // kube-scheduler may need some time before it gets the CSIDriver object. + // Without it, scheduling will happen without considering capacity, which + // is not what we want to test. + time.Sleep(5 * time.Second) + } + + // Create CSIStorageCapacity + if test.haveCapacity { + if _, err := config.client.StorageV1alpha1().CSIStorageCapacities("default").Create(context.TODO(), + &storagev1alpha1.CSIStorageCapacity{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "foo-", + }, + StorageClassName: classes[classWait].Name, + NodeTopology: &metav1.LabelSelector{}, + // More than the 5Gi used in makePVC. + Capacity: resource.NewQuantity(6*1024*1024*1024, resource.BinarySI), + }, + metav1.CreateOptions{}); err != nil { + t.Fatalf("Failed to create CSIStorageCapacity: %v", err) + } + } + + // Create PVCs + for _, pvcConfig := range test.pvcs { + pvc := makePVC(pvcConfig.name, config.ns, &classes[pvcConfig.scName].Name, pvcConfig.preboundPV) + if _, err := config.client.CoreV1().PersistentVolumeClaims(config.ns).Create(context.TODO(), pvc, metav1.CreateOptions{}); err != nil { + t.Fatalf("Failed to create PersistentVolumeClaim %q: %v", pvc.Name, err) + } + } + + // Create Pod + if _, err := config.client.CoreV1().Pods(config.ns).Create(context.TODO(), test.pod, metav1.CreateOptions{}); err != nil { + t.Fatalf("Failed to create Pod %q: %v", test.pod.Name, err) + } + + // Lack of capacity prevents pod scheduling and binding. + shouldFail := test.capacitySupported && !test.haveCapacity + if shouldFail { + if err := waitForPodUnschedulable(config.client, test.pod); err != nil { + t.Errorf("Pod %q was not unschedulable: %v", test.pod.Name, err) + } + } else { + if err := waitForPodToSchedule(config.client, test.pod); err != nil { + t.Errorf("Failed to schedule Pod %q: %v", test.pod.Name, err) + } + } + + // Validate + for _, pvc := range test.pvcs { + if shouldFail { + validatePVCPhase(t, config.client, pvc.name, config.ns, v1.ClaimPending, false) + } else { + validatePVCPhase(t, config.client, pvc.name, config.ns, v1.ClaimBound, true) + } + } + + // Force delete objects, but they still may not be immediately removed + deleteTestObjects(config.client, config.ns, deleteOption) + } + + for name, test := range cases { + t.Run(name, func(t *testing.T) { run(t, test) }) + } } // TestRescheduleProvisioning validate that PV controller will remove @@ -988,6 +1145,8 @@ func deleteTestObjects(client clientset.Interface, ns string, option metav1.Dele client.CoreV1().PersistentVolumeClaims(ns).DeleteCollection(context.TODO(), option, metav1.ListOptions{}) client.CoreV1().PersistentVolumes().DeleteCollection(context.TODO(), option, metav1.ListOptions{}) client.StorageV1().StorageClasses().DeleteCollection(context.TODO(), option, metav1.ListOptions{}) + client.StorageV1().CSIDrivers().DeleteCollection(context.TODO(), option, metav1.ListOptions{}) + client.StorageV1alpha1().CSIStorageCapacities("default").DeleteCollection(context.TODO(), option, metav1.ListOptions{}) } func makeStorageClass(name string, mode *storagev1.VolumeBindingMode) *storagev1.StorageClass { @@ -1148,7 +1307,7 @@ func validatePVCPhase(t *testing.T, client clientset.Interface, pvcName string, // Check whether the bound claim is provisioned/bound as expect. if phase == v1.ClaimBound { if err := validateProvisionAnn(claim, isProvisioned); err != nil { - t.Errorf("Provisoning annotaion on PVC %v/%v not bahaviors as expected: %v", ns, pvcName, err) + t.Errorf("Provisoning annotation on PVC %v/%v not as expected: %v", ns, pvcName, err) } } }