diff --git a/test/integration/BUILD b/test/integration/BUILD index 138752de4bd..1ef467a0b48 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -75,6 +75,7 @@ filegroup( "//test/integration/ttlcontroller:all-srcs", "//test/integration/util:all-srcs", "//test/integration/volume:all-srcs", + "//test/integration/volumescheduling:all-srcs", ], tags = ["automanaged"], ) diff --git a/test/integration/scheduler/BUILD b/test/integration/scheduler/BUILD index 427ef698d5d..72432413c0a 100644 --- a/test/integration/scheduler/BUILD +++ b/test/integration/scheduler/BUILD @@ -18,7 +18,6 @@ go_test( "priorities_test.go", "scheduler_test.go", "taint_test.go", - "volume_binding_test.go", ], embed = [":go_default_library"], tags = ["integration"], @@ -26,8 +25,6 @@ go_test( "//pkg/api/legacyscheme:go_default_library", "//pkg/api/v1/pod:go_default_library", "//pkg/controller/nodelifecycle:go_default_library", - "//pkg/controller/volume/persistentvolume:go_default_library", - "//pkg/controller/volume/persistentvolume/options:go_default_library", "//pkg/features:go_default_library", "//pkg/scheduler:go_default_library", "//pkg/scheduler/algorithm/predicates:go_default_library", @@ -38,13 +35,10 @@ go_test( "//pkg/scheduler/framework/v1alpha1:go_default_library", "//pkg/scheduler/nodeinfo:go_default_library", "//pkg/scheduler/testing:go_default_library", - "//pkg/volume:go_default_library", - "//pkg/volume/testing:go_default_library", "//plugin/pkg/admission/podtolerationrestriction:go_default_library", "//plugin/pkg/admission/podtolerationrestriction/apis/podtolerationrestriction:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/policy/v1beta1:go_default_library", - "//staging/src/k8s.io/api/storage/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", @@ -52,7 +46,6 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/rand:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", diff --git a/test/integration/volumescheduling/BUILD b/test/integration/volumescheduling/BUILD new file mode 100644 index 00000000000..36687ea9b58 --- /dev/null +++ b/test/integration/volumescheduling/BUILD @@ -0,0 +1,81 @@ +package(default_visibility = ["//visibility:public"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", + "go_test", +) + +go_test( + name = "go_default_test", + size = "large", + srcs = [ + "main_test.go", + "volume_binding_test.go", + ], + embed = [":go_default_library"], + tags = ["integration"], + deps = [ + "//pkg/controller/volume/persistentvolume:go_default_library", + "//pkg/controller/volume/persistentvolume/options:go_default_library", + "//pkg/features:go_default_library", + "//pkg/scheduler/algorithm/predicates:go_default_library", + "//pkg/volume:go_default_library", + "//pkg/volume/testing:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/api/storage/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/rand:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/client-go/informers:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library", + "//test/integration/framework:go_default_library", + "//test/utils/image:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) + +go_library( + name = "go_default_library", + srcs = ["util.go"], + importpath = "k8s.io/kubernetes/test/integration/volumescheduling", + deps = [ + "//pkg/api/legacyscheme:go_default_library", + "//pkg/api/v1/pod:go_default_library", + "//pkg/scheduler:go_default_library", + "//pkg/scheduler/algorithmprovider/defaults:go_default_library", + "//pkg/scheduler/api:go_default_library", + "//pkg/scheduler/apis/config:go_default_library", + "//pkg/scheduler/factory:go_default_library", + "//pkg/scheduler/framework/v1alpha1:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", + "//staging/src/k8s.io/client-go/informers:go_default_library", + "//staging/src/k8s.io/client-go/informers/core/v1:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//staging/src/k8s.io/client-go/rest:go_default_library", + "//staging/src/k8s.io/client-go/tools/events:go_default_library", + "//test/integration/framework:go_default_library", + ], +) diff --git a/test/integration/volumescheduling/main_test.go b/test/integration/volumescheduling/main_test.go new file mode 100644 index 00000000000..d7a2d57a9ec --- /dev/null +++ b/test/integration/volumescheduling/main_test.go @@ -0,0 +1,27 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volumescheduling + +import ( + "testing" + + "k8s.io/kubernetes/test/integration/framework" +) + +func TestMain(m *testing.M) { + framework.EtcdMain(m.Run) +} diff --git a/test/integration/volumescheduling/util.go b/test/integration/volumescheduling/util.go new file mode 100644 index 00000000000..f7d59b7a2b6 --- /dev/null +++ b/test/integration/volumescheduling/util.go @@ -0,0 +1,254 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volumescheduling + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/admission" + "k8s.io/client-go/informers" + coreinformers "k8s.io/client-go/informers/core/v1" + clientset "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + "k8s.io/client-go/tools/events" + "k8s.io/kubernetes/pkg/api/legacyscheme" + podutil "k8s.io/kubernetes/pkg/api/v1/pod" + "k8s.io/kubernetes/pkg/scheduler" + schedulerapi "k8s.io/kubernetes/pkg/scheduler/api" + schedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" + "k8s.io/kubernetes/pkg/scheduler/factory" + schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" + "k8s.io/kubernetes/test/integration/framework" + + // Install "DefaultProvider" algorithprovider + _ "k8s.io/kubernetes/pkg/scheduler/algorithmprovider/defaults" +) + +type testContext struct { + closeFn framework.CloseFunc + httpServer *httptest.Server + ns *v1.Namespace + clientSet *clientset.Clientset + informerFactory informers.SharedInformerFactory + schedulerConfigArgs *factory.ConfigFactoryArgs + schedulerConfig *factory.Config + scheduler *scheduler.Scheduler + stopCh chan struct{} +} + +// initTestMaster initializes a test environment and creates a master with default +// configuration. +func initTestMaster(t *testing.T, nsPrefix string, admission admission.Interface) *testContext { + context := testContext{ + stopCh: make(chan struct{}), + } + + // 1. Create master + h := &framework.MasterHolder{Initialized: make(chan struct{})} + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + <-h.Initialized + h.M.GenericAPIServer.Handler.ServeHTTP(w, req) + })) + + masterConfig := framework.NewIntegrationTestMasterConfig() + + if admission != nil { + masterConfig.GenericConfig.AdmissionControl = admission + } + + _, context.httpServer, context.closeFn = framework.RunAMasterUsingServer(masterConfig, s, h) + + if nsPrefix != "default" { + context.ns = framework.CreateTestingNamespace(nsPrefix+string(uuid.NewUUID()), s, t) + } else { + context.ns = framework.CreateTestingNamespace("default", s, t) + } + + // 2. Create kubeclient + context.clientSet = clientset.NewForConfigOrDie( + &restclient.Config{ + QPS: -1, Host: s.URL, + ContentConfig: restclient.ContentConfig{ + GroupVersion: &schema.GroupVersion{Group: "", Version: "v1"}, + }, + }, + ) + return &context +} + +// initTestSchedulerWithOptions initializes a test environment and creates a scheduler with default +// configuration and other options. +func initTestSchedulerWithOptions( + t *testing.T, + context *testContext, + resyncPeriod time.Duration, +) *testContext { + // 1. Create scheduler + context.informerFactory = informers.NewSharedInformerFactory(context.clientSet, resyncPeriod) + + podInformer := context.informerFactory.Core().V1().Pods() + + context.schedulerConfigArgs = createConfiguratorArgsWithPodInformer( + context.clientSet, podInformer, context.informerFactory, schedulerframework.NewRegistry(), nil, + []schedulerconfig.PluginConfig{}, context.stopCh) + configFactory := factory.NewConfigFactory(context.schedulerConfigArgs) + + var err error + context.schedulerConfig, err = configFactory.Create() + if err != nil { + t.Fatalf("Couldn't create scheduler config: %v", err) + } + + // set DisablePreemption option + context.schedulerConfig.DisablePreemption = false + + context.scheduler = scheduler.NewFromConfig(context.schedulerConfig) + + scheduler.AddAllEventHandlers(context.scheduler, + v1.DefaultSchedulerName, + context.informerFactory.Core().V1().Nodes(), + podInformer, + context.informerFactory.Core().V1().PersistentVolumes(), + context.informerFactory.Core().V1().PersistentVolumeClaims(), + context.informerFactory.Core().V1().Services(), + context.informerFactory.Storage().V1().StorageClasses(), + context.informerFactory.Storage().V1beta1().CSINodes(), + ) + + eventBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{ + Interface: context.clientSet.EventsV1beta1().Events(""), + }) + context.schedulerConfig.Recorder = eventBroadcaster.NewRecorder( + legacyscheme.Scheme, + v1.DefaultSchedulerName, + ) + stopCh := make(chan struct{}) + eventBroadcaster.StartRecordingToSink(stopCh) + + context.informerFactory.Start(context.schedulerConfig.StopEverything) + context.informerFactory.WaitForCacheSync(context.schedulerConfig.StopEverything) + + context.scheduler.Run() + return context +} + +// createConfiguratorWithPodInformer creates a configurator for scheduler. +func createConfiguratorArgsWithPodInformer( + clientSet clientset.Interface, + podInformer coreinformers.PodInformer, + informerFactory informers.SharedInformerFactory, + pluginRegistry schedulerframework.Registry, + plugins *schedulerconfig.Plugins, + pluginConfig []schedulerconfig.PluginConfig, + stopCh <-chan struct{}, +) *factory.ConfigFactoryArgs { + return &factory.ConfigFactoryArgs{ + Client: clientSet, + NodeInformer: informerFactory.Core().V1().Nodes(), + PodInformer: podInformer, + PvInformer: informerFactory.Core().V1().PersistentVolumes(), + PvcInformer: informerFactory.Core().V1().PersistentVolumeClaims(), + ReplicationControllerInformer: informerFactory.Core().V1().ReplicationControllers(), + ReplicaSetInformer: informerFactory.Apps().V1().ReplicaSets(), + StatefulSetInformer: informerFactory.Apps().V1().StatefulSets(), + ServiceInformer: informerFactory.Core().V1().Services(), + PdbInformer: informerFactory.Policy().V1beta1().PodDisruptionBudgets(), + StorageClassInformer: informerFactory.Storage().V1().StorageClasses(), + CSINodeInformer: informerFactory.Storage().V1beta1().CSINodes(), + Registry: pluginRegistry, + Plugins: plugins, + PluginConfig: pluginConfig, + HardPodAffinitySymmetricWeight: v1.DefaultHardPodAffinitySymmetricWeight, + DisablePreemption: false, + PercentageOfNodesToScore: schedulerapi.DefaultPercentageOfNodesToScore, + BindTimeoutSeconds: 600, + StopCh: stopCh, + } +} + +// cleanupTest deletes the scheduler and the test namespace. It should be called +// at the end of a test. +func cleanupTest(t *testing.T, context *testContext) { + // Kill the scheduler. + close(context.stopCh) + // Cleanup nodes. + context.clientSet.CoreV1().Nodes().DeleteCollection(nil, metav1.ListOptions{}) + framework.DeleteTestingNamespace(context.ns, context.httpServer, t) + context.closeFn() +} + +// waitForPodToScheduleWithTimeout waits for a pod to get scheduled and returns +// an error if it does not scheduled within the given timeout. +func waitForPodToScheduleWithTimeout(cs clientset.Interface, pod *v1.Pod, timeout time.Duration) error { + return wait.Poll(100*time.Millisecond, timeout, podScheduled(cs, pod.Namespace, pod.Name)) +} + +// waitForPodToSchedule waits for a pod to get scheduled and returns an error if +// it does not get scheduled within the timeout duration (30 seconds). +func waitForPodToSchedule(cs clientset.Interface, pod *v1.Pod) error { + return waitForPodToScheduleWithTimeout(cs, pod, 30*time.Second) +} + +// waitForPodUnscheduleWithTimeout waits for a pod to fail scheduling and returns +// an error if it does not become unschedulable within the given timeout. +func waitForPodUnschedulableWithTimeout(cs clientset.Interface, pod *v1.Pod, timeout time.Duration) error { + return wait.Poll(100*time.Millisecond, timeout, podUnschedulable(cs, pod.Namespace, pod.Name)) +} + +// waitForPodUnschedule waits for a pod to fail scheduling and returns +// an error if it does not become unschedulable within the timeout duration (30 seconds). +func waitForPodUnschedulable(cs clientset.Interface, pod *v1.Pod) error { + return waitForPodUnschedulableWithTimeout(cs, pod, 30*time.Second) +} + +// podScheduled returns true if a node is assigned to the given pod. +func podScheduled(c clientset.Interface, podNamespace, podName string) wait.ConditionFunc { + return func() (bool, error) { + pod, err := c.CoreV1().Pods(podNamespace).Get(podName, metav1.GetOptions{}) + if err != nil { + // This could be a connection error so we want to retry. + return false, nil + } + if pod.Spec.NodeName == "" { + return false, nil + } + return true, nil + } +} + +// podUnschedulable returns a condition function that returns true if the given pod +// gets unschedulable status. +func podUnschedulable(c clientset.Interface, podNamespace, podName string) wait.ConditionFunc { + return func() (bool, error) { + pod, err := c.CoreV1().Pods(podNamespace).Get(podName, metav1.GetOptions{}) + if err != nil { + // This could be a connection error so we want to retry. + return false, nil + } + _, cond := podutil.GetPodCondition(&pod.Status, v1.PodScheduled) + return cond != nil && cond.Status == v1.ConditionFalse && + cond.Reason == v1.PodReasonUnschedulable, nil + } +} diff --git a/test/integration/volume_scheduling/volume_binding_test.go b/test/integration/volumescheduling/volume_binding_test.go similarity index 99% rename from test/integration/volume_scheduling/volume_binding_test.go rename to test/integration/volumescheduling/volume_binding_test.go index f1ecdc90b3c..203a8a51b2f 100644 --- a/test/integration/volume_scheduling/volume_binding_test.go +++ b/test/integration/volumescheduling/volume_binding_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package scheduler +package volumescheduling // This file tests the VolumeScheduling feature. @@ -28,7 +28,7 @@ import ( "k8s.io/klog" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -43,7 +43,6 @@ import ( persistentvolumeoptions "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/options" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" - schedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/volume" volumetest "k8s.io/kubernetes/pkg/volume/testing" imageutils "k8s.io/kubernetes/test/utils/image" @@ -883,8 +882,7 @@ func TestRescheduleProvisioning(t *testing.T) { } func setupCluster(t *testing.T, nsName string, numberOfNodes int, resyncPeriod time.Duration, provisionDelaySeconds int) *testConfig { - context := initTestSchedulerWithOptions(t, initTestMaster(t, nsName, nil), false, nil, nil, - nil, []schedulerconfig.PluginConfig{}, false, resyncPeriod) + context := initTestSchedulerWithOptions(t, initTestMaster(t, nsName, nil), resyncPeriod) clientset := context.clientSet ns := context.ns.Name