github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-21 10:51:29 +00:00
Code Issues Projects Releases Wiki Activity

CSIStorageCapacity: integration test

Browse Source 
This is similar to the E2E test, it just doesn't need a real cluster.
This commit is contained in:
Patrick Ohly 2020-06-25 11:05:40 +02:00
parent 567ce87aee
commit cf735a3bf7
3 changed files with 186 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
test/integration/volumescheduling/BUILD
View File

@ -17,19 +17,23 @@ go_test(
tags = ["integration"],
deps = [
"//pkg/controller/volume/persistentvolume:go_default_library",
"//pkg/features:go_default_library",
"//pkg/scheduler/framework/plugins/nodevolumelimits:go_default_library",
"//pkg/volume:go_default_library",
"//pkg/volume/testing:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/api/storage/v1:go_default_library",
"//staging/src/k8s.io/api/storage/v1alpha1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/rand:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//staging/src/k8s.io/client-go/informers:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/util/workqueue:go_default_library",
"//staging/src/k8s.io/component-base/featuregate/testing:go_default_library",
"//test/integration/framework:go_default_library",
"//test/utils/image:go_default_library",
"//vendor/k8s.io/klog/v2:go_default_library",
@ -55,6 +59,8 @@ go_library(
importpath = "k8s.io/kubernetes/test/integration/volumescheduling",
deps = [
"//pkg/api/v1/pod:go_default_library",
"//pkg/features:go_default_library",
"//pkg/master:go_default_library",
"//pkg/scheduler:go_default_library",
"//pkg/scheduler/profile:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
@ -63,6 +69,7 @@ go_library(
"//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/admission:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//staging/src/k8s.io/client-go/informers:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/rest:go_default_library",

14
test/integration/volumescheduling/util.go
View File

@ -29,11 +29,14 @@ import (
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/admission"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/informers"
clientset "k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/events"
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/master"
"k8s.io/kubernetes/pkg/scheduler"
"k8s.io/kubernetes/pkg/scheduler/profile"
"k8s.io/kubernetes/test/integration/framework"
@ -52,7 +55,8 @@ type testContext struct {
}
// initTestMaster initializes a test environment and creates a master with default
// configuration.
// configuration. Alpha resources are enabled automatically if the corresponding feature
// is enabled.
func initTestMaster(t *testing.T, nsPrefix string, admission admission.Interface) *testContext {
ctx, cancelFunc := context.WithCancel(context.Background())
testCtx := testContext{
@ -68,6 +72,14 @@ func initTestMaster(t *testing.T, nsPrefix string, admission admission.Interface
}))
masterConfig := framework.NewIntegrationTestMasterConfig()
resourceConfig := master.DefaultAPIResourceConfigSource()
if utilfeature.DefaultFeatureGate.Enabled(features.CSIStorageCapacity) {
resourceConfig.EnableVersions(schema.GroupVersion{
Group: "storage.k8s.io",
Version: "v1alpha1",
})
}
masterConfig.ExtraConfig.APIResourceConfigSource = resourceConfig
if admission != nil {
masterConfig.GenericConfig.AdmissionControl = admission

173
test/integration/volumescheduling/volume_binding_test.go
View File

@ -31,15 +31,19 @@ import (
v1 "k8s.io/api/core/v1"
storagev1 "k8s.io/api/storage/v1"
storagev1alpha1 "k8s.io/api/storage/v1alpha1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/rand"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/informers"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/util/workqueue"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/kubernetes/pkg/controller/volume/persistentvolume"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodevolumelimits"
"k8s.io/kubernetes/pkg/volume"
volumetest "k8s.io/kubernetes/pkg/volume/testing"
@ -78,7 +82,7 @@ const (
podLimit = 50
volsPerPod = 3
nodeAffinityLabelKey = "kubernetes.io/hostname"
provisionerPluginName = "kubernetes.io/mock-provisioner"
provisionerPluginName = "mock-provisioner.kubernetes.io"
)
type testPV struct {
@ -700,10 +704,17 @@ func TestPVAffinityConflict(t *testing.T) {
}
func TestVolumeProvision(t *testing.T) {
t.Run("with CSIStorageCapacity", func(t *testing.T) { testVolumeProvision(t, true) })
t.Run("without CSIStorageCapacity", func(t *testing.T) { testVolumeProvision(t, false) })
}
func testVolumeProvision(t *testing.T, storageCapacity bool) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIStorageCapacity, storageCapacity)()
config := setupCluster(t, "volume-scheduling", 1, 0, 0)
defer config.teardown()
cases := map[string]struct {
type testcaseType struct {
pod *v1.Pod
pvs []*testPV
boundPvcs []*testPVC
@ -711,7 +722,9 @@ func TestVolumeProvision(t *testing.T) {
// Create these, but they should not be bound in the end
unboundPvcs []*testPVC
shouldFail bool
}{
}
cases := map[string]testcaseType{
"wait provisioned": {
pod: makePod("pod-pvc-canprovision", config.ns, []string{"pvc-canprovision"}),
provisionedPvcs: []*testPVC{{"pvc-canprovision", classWait, ""}},
@ -748,9 +761,7 @@ func TestVolumeProvision(t *testing.T) {
},
}
for name, test := range cases {
klog.Infof("Running test %v", name)
run := func(t *testing.T, test testcaseType) {
// Create StorageClasses
suffix := rand.String(4)
classes := map[string]*storagev1.StorageClass{}
@ -831,6 +842,152 @@ func TestVolumeProvision(t *testing.T) {
// Force delete objects, but they still may not be immediately removed
deleteTestObjects(config.client, config.ns, deleteOption)
}
for name, test := range cases {
t.Run(name, func(t *testing.T) { run(t, test) })
}
}
// TestCapacity covers different scenarios involving CSIStorageCapacity objects.
func TestCapacity(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIStorageCapacity, true)()
config := setupCluster(t, "volume-scheduling", 1, 0, 0)
defer config.teardown()
type testcaseType struct {
pod *v1.Pod
pvcs []*testPVC
haveCapacity bool
capacitySupported bool
}
cases := map[string]testcaseType{
"baseline": {
pod: makePod("pod-pvc-canprovision", config.ns, []string{"pvc-canprovision"}),
pvcs: []*testPVC{{"pvc-canprovision", classWait, ""}},
},
"out of space": {
pod: makePod("pod-pvc-canprovision", config.ns, []string{"pvc-canprovision"}),
pvcs: []*testPVC{{"pvc-canprovision", classWait, ""}},
capacitySupported: true,
},
"with space": {
pod: makePod("pod-pvc-canprovision", config.ns, []string{"pvc-canprovision"}),
pvcs: []*testPVC{{"pvc-canprovision", classWait, ""}},
capacitySupported: true,
haveCapacity: true,
},
"ignored": {
pod: makePod("pod-pvc-canprovision", config.ns, []string{"pvc-canprovision"}),
pvcs: []*testPVC{{"pvc-canprovision", classWait, ""}},
haveCapacity: true,
},
}
run := func(t *testing.T, test testcaseType) {
// Create StorageClasses
suffix := rand.String(4)
classes := map[string]*storagev1.StorageClass{}
classes[classImmediate] = makeDynamicProvisionerStorageClass(fmt.Sprintf("immediate-%v", suffix), &modeImmediate, nil)
classes[classWait] = makeDynamicProvisionerStorageClass(fmt.Sprintf("wait-%v", suffix), &modeWait, nil)
topo := []v1.TopologySelectorTerm{
{
MatchLabelExpressions: []v1.TopologySelectorLabelRequirement{
{
Key: nodeAffinityLabelKey,
Values: []string{node2},
},
},
},
}
classes[classTopoMismatch] = makeDynamicProvisionerStorageClass(fmt.Sprintf("topomismatch-%v", suffix), &modeWait, topo)
for _, sc := range classes {
if _, err := config.client.StorageV1().StorageClasses().Create(context.TODO(), sc, metav1.CreateOptions{}); err != nil {
t.Fatalf("Failed to create StorageClass %q: %v", sc.Name, err)
}
}
// The provisioner isn't actually a CSI driver, but
// that doesn't matter here.
if test.capacitySupported {
if _, err := config.client.StorageV1().CSIDrivers().Create(context.TODO(),
&storagev1.CSIDriver{
ObjectMeta: metav1.ObjectMeta{
Name: provisionerPluginName,
},
Spec: storagev1.CSIDriverSpec{
StorageCapacity: &test.capacitySupported,
},
},
metav1.CreateOptions{}); err != nil {
t.Fatalf("Failed to create CSIDriver: %v", err)
}
// kube-scheduler may need some time before it gets the CSIDriver object.
// Without it, scheduling will happen without considering capacity, which
// is not what we want to test.
time.Sleep(5 * time.Second)
}
// Create CSIStorageCapacity
if test.haveCapacity {
if _, err := config.client.StorageV1alpha1().CSIStorageCapacities("default").Create(context.TODO(),
&storagev1alpha1.CSIStorageCapacity{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "foo-",
},
StorageClassName: classes[classWait].Name,
NodeTopology: &metav1.LabelSelector{},
// More than the 5Gi used in makePVC.
Capacity: resource.NewQuantity(6*1024*1024*1024, resource.BinarySI),
},
metav1.CreateOptions{}); err != nil {
t.Fatalf("Failed to create CSIStorageCapacity: %v", err)
}
}
// Create PVCs
for _, pvcConfig := range test.pvcs {
pvc := makePVC(pvcConfig.name, config.ns, &classes[pvcConfig.scName].Name, pvcConfig.preboundPV)
if _, err := config.client.CoreV1().PersistentVolumeClaims(config.ns).Create(context.TODO(), pvc, metav1.CreateOptions{}); err != nil {
t.Fatalf("Failed to create PersistentVolumeClaim %q: %v", pvc.Name, err)
}
}
// Create Pod
if _, err := config.client.CoreV1().Pods(config.ns).Create(context.TODO(), test.pod, metav1.CreateOptions{}); err != nil {
t.Fatalf("Failed to create Pod %q: %v", test.pod.Name, err)
}
// Lack of capacity prevents pod scheduling and binding.
shouldFail := test.capacitySupported && !test.haveCapacity
if shouldFail {
if err := waitForPodUnschedulable(config.client, test.pod); err != nil {
t.Errorf("Pod %q was not unschedulable: %v", test.pod.Name, err)
}
} else {
if err := waitForPodToSchedule(config.client, test.pod); err != nil {
t.Errorf("Failed to schedule Pod %q: %v", test.pod.Name, err)
}
}
// Validate
for _, pvc := range test.pvcs {
if shouldFail {
validatePVCPhase(t, config.client, pvc.name, config.ns, v1.ClaimPending, false)
} else {
validatePVCPhase(t, config.client, pvc.name, config.ns, v1.ClaimBound, true)
}
}
// Force delete objects, but they still may not be immediately removed
deleteTestObjects(config.client, config.ns, deleteOption)
}
for name, test := range cases {
t.Run(name, func(t *testing.T) { run(t, test) })
}
}
// TestRescheduleProvisioning validate that PV controller will remove
@ -988,6 +1145,8 @@ func deleteTestObjects(client clientset.Interface, ns string, option metav1.Dele
client.CoreV1().PersistentVolumeClaims(ns).DeleteCollection(context.TODO(), option, metav1.ListOptions{})
client.CoreV1().PersistentVolumes().DeleteCollection(context.TODO(), option, metav1.ListOptions{})
client.StorageV1().StorageClasses().DeleteCollection(context.TODO(), option, metav1.ListOptions{})
client.StorageV1().CSIDrivers().DeleteCollection(context.TODO(), option, metav1.ListOptions{})
client.StorageV1alpha1().CSIStorageCapacities("default").DeleteCollection(context.TODO(), option, metav1.ListOptions{})
}
func makeStorageClass(name string, mode *storagev1.VolumeBindingMode) *storagev1.StorageClass {
@ -1148,7 +1307,7 @@ func validatePVCPhase(t *testing.T, client clientset.Interface, pvcName string,
// Check whether the bound claim is provisioned/bound as expect.
if phase == v1.ClaimBound {
if err := validateProvisionAnn(claim, isProvisioned); err != nil {
t.Errorf("Provisoning annotaion on PVC %v/%v not bahaviors as expected: %v", ns, pvcName, err)
t.Errorf("Provisoning annotation on PVC %v/%v not as expected: %v", ns, pvcName, err)
}
}
}