CSIStorageCapacity: integration test

This is similar to the E2E test, it just doesn't need a real cluster.

test/integration/volumescheduling/BUILD

@@ -17,19 +17,23 @@ go_test(
     tags = ["integration"],
     deps = [
         "//pkg/controller/volume/persistentvolume:go_default_library",
+        "//pkg/features:go_default_library",
         "//pkg/scheduler/framework/plugins/nodevolumelimits:go_default_library",
         "//pkg/volume:go_default_library",
         "//pkg/volume/testing:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/api/storage/v1:go_default_library",
+        "//staging/src/k8s.io/api/storage/v1alpha1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/rand:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//staging/src/k8s.io/client-go/informers:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//staging/src/k8s.io/client-go/util/workqueue:go_default_library",
+        "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library",
         "//test/integration/framework:go_default_library",
         "//test/utils/image:go_default_library",
         "//vendor/k8s.io/klog/v2:go_default_library",
@@ -55,6 +59,8 @@ go_library(
     importpath = "k8s.io/kubernetes/test/integration/volumescheduling",
     deps = [
         "//pkg/api/v1/pod:go_default_library",
+        "//pkg/features:go_default_library",
+        "//pkg/master:go_default_library",
         "//pkg/scheduler:go_default_library",
         "//pkg/scheduler/profile:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
@@ -63,6 +69,7 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//staging/src/k8s.io/client-go/informers:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//staging/src/k8s.io/client-go/rest:go_default_library",

test/integration/volumescheduling/util.go

@@ -29,11 +29,14 @@ import (
 	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/admission"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/informers"
 	clientset "k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/events"
 	podutil "k8s.io/kubernetes/pkg/api/v1/pod"
+	"k8s.io/kubernetes/pkg/features"
+	"k8s.io/kubernetes/pkg/master"
 	"k8s.io/kubernetes/pkg/scheduler"
 	"k8s.io/kubernetes/pkg/scheduler/profile"
 	"k8s.io/kubernetes/test/integration/framework"
@@ -52,7 +55,8 @@ type testContext struct {
 }
 
 // initTestMaster initializes a test environment and creates a master with default
-// configuration.
+// configuration. Alpha resources are enabled automatically if the corresponding feature
+// is enabled.
 func initTestMaster(t *testing.T, nsPrefix string, admission admission.Interface) *testContext {
 	ctx, cancelFunc := context.WithCancel(context.Background())
 	testCtx := testContext{
@@ -68,6 +72,14 @@ func initTestMaster(t *testing.T, nsPrefix string, admission admission.Interface
 	}))
 
 	masterConfig := framework.NewIntegrationTestMasterConfig()
+	resourceConfig := master.DefaultAPIResourceConfigSource()
+	if utilfeature.DefaultFeatureGate.Enabled(features.CSIStorageCapacity) {
+		resourceConfig.EnableVersions(schema.GroupVersion{
+			Group:   "storage.k8s.io",
+			Version: "v1alpha1",
+		})
+	}
+	masterConfig.ExtraConfig.APIResourceConfigSource = resourceConfig
 
 	if admission != nil {
 		masterConfig.GenericConfig.AdmissionControl = admission

test/integration/volumescheduling/volume_binding_test.go

@@ -31,15 +31,19 @@ import (
 
 	v1 "k8s.io/api/core/v1"
 	storagev1 "k8s.io/api/storage/v1"
+	storagev1alpha1 "k8s.io/api/storage/v1alpha1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/rand"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/informers"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/util/workqueue"
+	featuregatetesting "k8s.io/component-base/featuregate/testing"
 	"k8s.io/kubernetes/pkg/controller/volume/persistentvolume"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodevolumelimits"
 	"k8s.io/kubernetes/pkg/volume"
 	volumetest "k8s.io/kubernetes/pkg/volume/testing"
@@ -78,7 +82,7 @@ const (
 	podLimit              = 50
 	volsPerPod            = 3
 	nodeAffinityLabelKey  = "kubernetes.io/hostname"
-	provisionerPluginName = "kubernetes.io/mock-provisioner"
+	provisionerPluginName = "mock-provisioner.kubernetes.io"
 )
 
 type testPV struct {
@@ -700,10 +704,17 @@ func TestPVAffinityConflict(t *testing.T) {
 }
 
 func TestVolumeProvision(t *testing.T) {
+	t.Run("with CSIStorageCapacity", func(t *testing.T) { testVolumeProvision(t, true) })
+	t.Run("without CSIStorageCapacity", func(t *testing.T) { testVolumeProvision(t, false) })
+}
+
+func testVolumeProvision(t *testing.T, storageCapacity bool) {
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIStorageCapacity, storageCapacity)()
+
 	config := setupCluster(t, "volume-scheduling", 1, 0, 0)
 	defer config.teardown()
 
-	cases := map[string]struct {
+	type testcaseType struct {
 		pod             *v1.Pod
 		pvs             []*testPV
 		boundPvcs       []*testPVC
@@ -711,7 +722,9 @@ func TestVolumeProvision(t *testing.T) {
 		// Create these, but they should not be bound in the end
 		unboundPvcs []*testPVC
 		shouldFail  bool
-	}{
+	}
+
+	cases := map[string]testcaseType{
 		"wait provisioned": {
 			pod:             makePod("pod-pvc-canprovision", config.ns, []string{"pvc-canprovision"}),
 			provisionedPvcs: []*testPVC{{"pvc-canprovision", classWait, ""}},
@@ -748,9 +761,7 @@ func TestVolumeProvision(t *testing.T) {
 		},
 	}
 
-	for name, test := range cases {
+	run := func(t *testing.T, test testcaseType) {
-		klog.Infof("Running test %v", name)
-
 		// Create StorageClasses
 		suffix := rand.String(4)
 		classes := map[string]*storagev1.StorageClass{}
@@ -831,6 +842,152 @@ func TestVolumeProvision(t *testing.T) {
 		// Force delete objects, but they still may not be immediately removed
 		deleteTestObjects(config.client, config.ns, deleteOption)
 	}
+
+	for name, test := range cases {
+		t.Run(name, func(t *testing.T) { run(t, test) })
+	}
+}
+
+// TestCapacity covers different scenarios involving CSIStorageCapacity objects.
+func TestCapacity(t *testing.T) {
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIStorageCapacity, true)()
+
+	config := setupCluster(t, "volume-scheduling", 1, 0, 0)
+	defer config.teardown()
+
+	type testcaseType struct {
+		pod               *v1.Pod
+		pvcs              []*testPVC
+		haveCapacity      bool
+		capacitySupported bool
+	}
+
+	cases := map[string]testcaseType{
+		"baseline": {
+			pod:  makePod("pod-pvc-canprovision", config.ns, []string{"pvc-canprovision"}),
+			pvcs: []*testPVC{{"pvc-canprovision", classWait, ""}},
+		},
+		"out of space": {
+			pod:               makePod("pod-pvc-canprovision", config.ns, []string{"pvc-canprovision"}),
+			pvcs:              []*testPVC{{"pvc-canprovision", classWait, ""}},
+			capacitySupported: true,
+		},
+		"with space": {
+			pod:               makePod("pod-pvc-canprovision", config.ns, []string{"pvc-canprovision"}),
+			pvcs:              []*testPVC{{"pvc-canprovision", classWait, ""}},
+			capacitySupported: true,
+			haveCapacity:      true,
+		},
+		"ignored": {
+			pod:          makePod("pod-pvc-canprovision", config.ns, []string{"pvc-canprovision"}),
+			pvcs:         []*testPVC{{"pvc-canprovision", classWait, ""}},
+			haveCapacity: true,
+		},
+	}
+
+	run := func(t *testing.T, test testcaseType) {
+		// Create StorageClasses
+		suffix := rand.String(4)
+		classes := map[string]*storagev1.StorageClass{}
+		classes[classImmediate] = makeDynamicProvisionerStorageClass(fmt.Sprintf("immediate-%v", suffix), &modeImmediate, nil)
+		classes[classWait] = makeDynamicProvisionerStorageClass(fmt.Sprintf("wait-%v", suffix), &modeWait, nil)
+		topo := []v1.TopologySelectorTerm{
+			{
+				MatchLabelExpressions: []v1.TopologySelectorLabelRequirement{
+					{
+						Key:    nodeAffinityLabelKey,
+						Values: []string{node2},
+					},
+				},
+			},
+		}
+		classes[classTopoMismatch] = makeDynamicProvisionerStorageClass(fmt.Sprintf("topomismatch-%v", suffix), &modeWait, topo)
+		for _, sc := range classes {
+			if _, err := config.client.StorageV1().StorageClasses().Create(context.TODO(), sc, metav1.CreateOptions{}); err != nil {
+				t.Fatalf("Failed to create StorageClass %q: %v", sc.Name, err)
+			}
+		}
+
+		// The provisioner isn't actually a CSI driver, but
+		// that doesn't matter here.
+		if test.capacitySupported {
+			if _, err := config.client.StorageV1().CSIDrivers().Create(context.TODO(),
+				&storagev1.CSIDriver{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: provisionerPluginName,
+					},
+					Spec: storagev1.CSIDriverSpec{
+						StorageCapacity: &test.capacitySupported,
+					},
+				},
+				metav1.CreateOptions{}); err != nil {
+				t.Fatalf("Failed to create CSIDriver: %v", err)
+			}
+
+			// kube-scheduler may need some time before it gets the CSIDriver object.
+			// Without it, scheduling will happen without considering capacity, which
+			// is not what we want to test.
+			time.Sleep(5 * time.Second)
+		}
+
+		// Create CSIStorageCapacity
+		if test.haveCapacity {
+			if _, err := config.client.StorageV1alpha1().CSIStorageCapacities("default").Create(context.TODO(),
+				&storagev1alpha1.CSIStorageCapacity{
+					ObjectMeta: metav1.ObjectMeta{
+						GenerateName: "foo-",
+					},
+					StorageClassName: classes[classWait].Name,
+					NodeTopology:     &metav1.LabelSelector{},
+					// More than the 5Gi used in makePVC.
+					Capacity: resource.NewQuantity(6*1024*1024*1024, resource.BinarySI),
+				},
+				metav1.CreateOptions{}); err != nil {
+				t.Fatalf("Failed to create CSIStorageCapacity: %v", err)
+			}
+		}
+
+		// Create PVCs
+		for _, pvcConfig := range test.pvcs {
+			pvc := makePVC(pvcConfig.name, config.ns, &classes[pvcConfig.scName].Name, pvcConfig.preboundPV)
+			if _, err := config.client.CoreV1().PersistentVolumeClaims(config.ns).Create(context.TODO(), pvc, metav1.CreateOptions{}); err != nil {
+				t.Fatalf("Failed to create PersistentVolumeClaim %q: %v", pvc.Name, err)
+			}
+		}
+
+		// Create Pod
+		if _, err := config.client.CoreV1().Pods(config.ns).Create(context.TODO(), test.pod, metav1.CreateOptions{}); err != nil {
+			t.Fatalf("Failed to create Pod %q: %v", test.pod.Name, err)
+		}
+
+		// Lack of capacity prevents pod scheduling and binding.
+		shouldFail := test.capacitySupported && !test.haveCapacity
+		if shouldFail {
+			if err := waitForPodUnschedulable(config.client, test.pod); err != nil {
+				t.Errorf("Pod %q was not unschedulable: %v", test.pod.Name, err)
+			}
+		} else {
+			if err := waitForPodToSchedule(config.client, test.pod); err != nil {
+				t.Errorf("Failed to schedule Pod %q: %v", test.pod.Name, err)
+			}
+		}
+
+		// Validate
+		for _, pvc := range test.pvcs {
+			if shouldFail {
+				validatePVCPhase(t, config.client, pvc.name, config.ns, v1.ClaimPending, false)
+			} else {
+				validatePVCPhase(t, config.client, pvc.name, config.ns, v1.ClaimBound, true)
+			}
+		}
+
+		// Force delete objects, but they still may not be immediately removed
+		deleteTestObjects(config.client, config.ns, deleteOption)
+	}
+
+	for name, test := range cases {
+		t.Run(name, func(t *testing.T) { run(t, test) })
+	}
 }
 
 // TestRescheduleProvisioning validate that PV controller will remove
@@ -988,6 +1145,8 @@ func deleteTestObjects(client clientset.Interface, ns string, option metav1.Dele
 	client.CoreV1().PersistentVolumeClaims(ns).DeleteCollection(context.TODO(), option, metav1.ListOptions{})
 	client.CoreV1().PersistentVolumes().DeleteCollection(context.TODO(), option, metav1.ListOptions{})
 	client.StorageV1().StorageClasses().DeleteCollection(context.TODO(), option, metav1.ListOptions{})
+	client.StorageV1().CSIDrivers().DeleteCollection(context.TODO(), option, metav1.ListOptions{})
+	client.StorageV1alpha1().CSIStorageCapacities("default").DeleteCollection(context.TODO(), option, metav1.ListOptions{})
 }
 
 func makeStorageClass(name string, mode *storagev1.VolumeBindingMode) *storagev1.StorageClass {
@@ -1148,7 +1307,7 @@ func validatePVCPhase(t *testing.T, client clientset.Interface, pvcName string,
 	// Check whether the bound claim is provisioned/bound as expect.
 	if phase == v1.ClaimBound {
 		if err := validateProvisionAnn(claim, isProvisioned); err != nil {
-			t.Errorf("Provisoning annotaion on PVC %v/%v not bahaviors as expected: %v", ns, pvcName, err)
+			t.Errorf("Provisoning annotation on PVC %v/%v not as expected: %v", ns, pvcName, err)
 		}
 	}
 }