From ce34d0a49dc41e5dd0119b2ed86a0b100d5dcac7 Mon Sep 17 00:00:00 2001 From: danielqsj Date: Mon, 28 Oct 2019 12:04:43 +0800 Subject: [PATCH] refactor events utils in e2e --- test/e2e/common/BUILD | 2 +- test/e2e/common/container_probe.go | 3 +- test/e2e/common/runtimeclass.go | 3 +- test/e2e/framework/BUILD | 1 + test/e2e/framework/events/BUILD | 34 +++++++++++++++++++ .../{common => framework/events}/events.go | 23 +++++++------ test/e2e/scheduling/BUILD | 2 +- test/e2e/scheduling/predicates.go | 10 +++--- test/e2e/scheduling/priorities.go | 4 +-- test/e2e/storage/testsuites/BUILD | 1 + test/e2e/storage/testsuites/volumemode.go | 18 +++------- 11 files changed, 65 insertions(+), 36 deletions(-) create mode 100644 test/e2e/framework/events/BUILD rename test/e2e/{common => framework/events}/events.go (85%) diff --git a/test/e2e/common/BUILD b/test/e2e/common/BUILD index 30033f0702a..9c76ee276fa 100644 --- a/test/e2e/common/BUILD +++ b/test/e2e/common/BUILD @@ -17,7 +17,6 @@ go_library( "downward_api.go", "downwardapi_volume.go", "empty_dir.go", - "events.go", "expansion.go", "host_path.go", "init_container.go", @@ -73,6 +72,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/watch:go_default_library", "//test/e2e/framework:go_default_library", + "//test/e2e/framework/events:go_default_library", "//test/e2e/framework/kubelet:go_default_library", "//test/e2e/framework/network:go_default_library", "//test/e2e/framework/node:go_default_library", diff --git a/test/e2e/common/container_probe.go b/test/e2e/common/container_probe.go index 8b7e6050ce3..eb60c250089 100644 --- a/test/e2e/common/container_probe.go +++ b/test/e2e/common/container_probe.go @@ -29,6 +29,7 @@ import ( podutil "k8s.io/kubernetes/pkg/api/v1/pod" "k8s.io/kubernetes/pkg/kubelet/events" "k8s.io/kubernetes/test/e2e/framework" + e2eevents "k8s.io/kubernetes/test/e2e/framework/events" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" testutils "k8s.io/kubernetes/test/utils" @@ -258,7 +259,7 @@ var _ = framework.KubeDescribe("Probing container", func() { "involvedObject.namespace": f.Namespace.Name, "reason": events.ContainerProbeWarning, }.AsSelector().String() - framework.ExpectNoError(WaitTimeoutForEvent( + framework.ExpectNoError(e2eevents.WaitTimeoutForEvent( f.ClientSet, f.Namespace.Name, expectedEvent, "0.0.0.0", framework.PodEventTimeout)) }) }) diff --git a/test/e2e/common/runtimeclass.go b/test/e2e/common/runtimeclass.go index 577233315c7..890ffe5245a 100644 --- a/test/e2e/common/runtimeclass.go +++ b/test/e2e/common/runtimeclass.go @@ -29,6 +29,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/events" runtimeclasstest "k8s.io/kubernetes/pkg/kubelet/runtimeclass/testing" "k8s.io/kubernetes/test/e2e/framework" + e2eevents "k8s.io/kubernetes/test/e2e/framework/events" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" imageutils "k8s.io/kubernetes/test/utils/image" utilpointer "k8s.io/utils/pointer" @@ -144,6 +145,6 @@ func expectSandboxFailureEvent(f *framework.Framework, pod *v1.Pod, msg string) "involvedObject.namespace": f.Namespace.Name, "reason": events.FailedCreatePodSandBox, }.AsSelector().String() - framework.ExpectNoError(WaitTimeoutForEvent( + framework.ExpectNoError(e2eevents.WaitTimeoutForEvent( f.ClientSet, f.Namespace.Name, eventSelector, msg, framework.PodEventTimeout)) } diff --git a/test/e2e/framework/BUILD b/test/e2e/framework/BUILD index 767dfabd131..a7c0fd48fa4 100644 --- a/test/e2e/framework/BUILD +++ b/test/e2e/framework/BUILD @@ -112,6 +112,7 @@ filegroup( "//test/e2e/framework/deployment:all-srcs", "//test/e2e/framework/deviceplugin:all-srcs", "//test/e2e/framework/endpoints:all-srcs", + "//test/e2e/framework/events:all-srcs", "//test/e2e/framework/ginkgowrapper:all-srcs", "//test/e2e/framework/gpu:all-srcs", "//test/e2e/framework/ingress:all-srcs", diff --git a/test/e2e/framework/events/BUILD b/test/e2e/framework/events/BUILD new file mode 100644 index 00000000000..998dd597ec1 --- /dev/null +++ b/test/e2e/framework/events/BUILD @@ -0,0 +1,34 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["events.go"], + importpath = "k8s.io/kubernetes/test/e2e/framework/events", + visibility = ["//visibility:public"], + deps = [ + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//staging/src/k8s.io/client-go/tools/cache:go_default_library", + "//test/e2e/framework:go_default_library", + "//vendor/github.com/onsi/ginkgo:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/test/e2e/common/events.go b/test/e2e/framework/events/events.go similarity index 85% rename from test/e2e/common/events.go rename to test/e2e/framework/events/events.go index 1d3b7f45a0e..098c0688774 100644 --- a/test/e2e/common/events.go +++ b/test/e2e/framework/events/events.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package common +package events import ( "fmt" @@ -35,11 +35,12 @@ import ( "github.com/onsi/ginkgo" ) +// Action is a function to be performed by the system. type Action func() error -// Returns true if a node update matching the predicate was emitted from the -// system after performing the supplied action. -func ObserveNodeUpdateAfterAction(f *framework.Framework, nodeName string, nodePredicate func(*v1.Node) bool, action Action) (bool, error) { +// ObserveNodeUpdateAfterAction returns true if a node update matching the predicate was emitted +// from the system after performing the supplied action. +func ObserveNodeUpdateAfterAction(c clientset.Interface, nodeName string, nodePredicate func(*v1.Node) bool, action Action) (bool, error) { observedMatchingNode := false nodeSelector := fields.OneTermEqualSelector("metadata.name", nodeName) informerStartedChan := make(chan struct{}) @@ -49,14 +50,14 @@ func ObserveNodeUpdateAfterAction(f *framework.Framework, nodeName string, nodeP &cache.ListWatch{ ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { options.FieldSelector = nodeSelector.String() - ls, err := f.ClientSet.CoreV1().Nodes().List(options) + ls, err := c.CoreV1().Nodes().List(options) return ls, err }, WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { // Signal parent goroutine that watching has begun. defer informerStartedGuard.Do(func() { close(informerStartedChan) }) options.FieldSelector = nodeSelector.String() - w, err := f.ClientSet.CoreV1().Nodes().Watch(options) + w, err := c.CoreV1().Nodes().Watch(options) return w, err }, }, @@ -95,9 +96,9 @@ func ObserveNodeUpdateAfterAction(f *framework.Framework, nodeName string, nodeP return err == nil, err } -// Returns true if an event matching the predicate was emitted from the system -// after performing the supplied action. -func ObserveEventAfterAction(f *framework.Framework, eventPredicate func(*v1.Event) bool, action Action) (bool, error) { +// ObserveEventAfterAction returns true if an event matching the predicate was emitted +// from the system after performing the supplied action. +func ObserveEventAfterAction(c clientset.Interface, ns string, eventPredicate func(*v1.Event) bool, action Action) (bool, error) { observedMatchingEvent := false informerStartedChan := make(chan struct{}) var informerStartedGuard sync.Once @@ -106,13 +107,13 @@ func ObserveEventAfterAction(f *framework.Framework, eventPredicate func(*v1.Eve _, controller := cache.NewInformer( &cache.ListWatch{ ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - ls, err := f.ClientSet.CoreV1().Events(f.Namespace.Name).List(options) + ls, err := c.CoreV1().Events(ns).List(options) return ls, err }, WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { // Signal parent goroutine that watching has begun. defer informerStartedGuard.Do(func() { close(informerStartedChan) }) - w, err := f.ClientSet.CoreV1().Events(f.Namespace.Name).Watch(options) + w, err := c.CoreV1().Events(ns).Watch(options) return w, err }, }, diff --git a/test/e2e/scheduling/BUILD b/test/e2e/scheduling/BUILD index ef83680617b..b002950093d 100644 --- a/test/e2e/scheduling/BUILD +++ b/test/e2e/scheduling/BUILD @@ -38,8 +38,8 @@ go_library( "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/watch:go_default_library", - "//test/e2e/common:go_default_library", "//test/e2e/framework:go_default_library", + "//test/e2e/framework/events:go_default_library", "//test/e2e/framework/gpu:go_default_library", "//test/e2e/framework/job:go_default_library", "//test/e2e/framework/kubelet:go_default_library", diff --git a/test/e2e/scheduling/predicates.go b/test/e2e/scheduling/predicates.go index c6913c85431..006c58da68c 100644 --- a/test/e2e/scheduling/predicates.go +++ b/test/e2e/scheduling/predicates.go @@ -27,8 +27,8 @@ import ( "k8s.io/apimachinery/pkg/util/uuid" utilversion "k8s.io/apimachinery/pkg/util/version" clientset "k8s.io/client-go/kubernetes" - "k8s.io/kubernetes/test/e2e/common" "k8s.io/kubernetes/test/e2e/framework" + e2eevents "k8s.io/kubernetes/test/e2e/framework/events" e2ekubelet "k8s.io/kubernetes/test/e2e/framework/kubelet" e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" @@ -722,7 +722,7 @@ func getRequestedStorageEphemeralStorage(pod v1.Pod) int64 { // removeTaintFromNodeAction returns a closure that removes the given taint // from the given node upon invocation. -func removeTaintFromNodeAction(cs clientset.Interface, nodeName string, testTaint v1.Taint) common.Action { +func removeTaintFromNodeAction(cs clientset.Interface, nodeName string, testTaint v1.Taint) e2eevents.Action { return func() error { framework.RemoveTaintOffNode(cs, nodeName, testTaint) return nil @@ -730,7 +730,7 @@ func removeTaintFromNodeAction(cs clientset.Interface, nodeName string, testTain } // createPausePodAction returns a closure that creates a pause pod upon invocation. -func createPausePodAction(f *framework.Framework, conf pausePodConfig) common.Action { +func createPausePodAction(f *framework.Framework, conf pausePodConfig) e2eevents.Action { return func() error { _, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(initPausePod(f, conf)) return err @@ -739,12 +739,12 @@ func createPausePodAction(f *framework.Framework, conf pausePodConfig) common.Ac // WaitForSchedulerAfterAction performs the provided action and then waits for // scheduler to act on the given pod. -func WaitForSchedulerAfterAction(f *framework.Framework, action common.Action, ns, podName string, expectSuccess bool) { +func WaitForSchedulerAfterAction(f *framework.Framework, action e2eevents.Action, ns, podName string, expectSuccess bool) { predicate := scheduleFailureEvent(podName) if expectSuccess { predicate = scheduleSuccessEvent(ns, podName, "" /* any node */) } - success, err := common.ObserveEventAfterAction(f, predicate, action) + success, err := e2eevents.ObserveEventAfterAction(f.ClientSet, f.Namespace.Name, predicate, action) framework.ExpectNoError(err) framework.ExpectEqual(success, true) } diff --git a/test/e2e/scheduling/priorities.go b/test/e2e/scheduling/priorities.go index 64ac2ead093..386cb12e5d0 100644 --- a/test/e2e/scheduling/priorities.go +++ b/test/e2e/scheduling/priorities.go @@ -36,8 +36,8 @@ import ( clientset "k8s.io/client-go/kubernetes" v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos" priorityutil "k8s.io/kubernetes/pkg/scheduler/algorithm/priorities/util" - "k8s.io/kubernetes/test/e2e/common" "k8s.io/kubernetes/test/e2e/framework" + e2eevents "k8s.io/kubernetes/test/e2e/framework/events" e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" testutils "k8s.io/kubernetes/test/utils" @@ -282,7 +282,7 @@ var _ = SIGDescribe("SchedulerPriorities [Serial]", func() { } return node.Annotations[v1.PreferAvoidPodsAnnotationKey] == string(val) } - success, err := common.ObserveNodeUpdateAfterAction(f, nodeName, predicate, action) + success, err := e2eevents.ObserveNodeUpdateAfterAction(f.ClientSet, nodeName, predicate, action) framework.ExpectNoError(err) framework.ExpectEqual(success, true) diff --git a/test/e2e/storage/testsuites/BUILD b/test/e2e/storage/testsuites/BUILD index a6b6c7e418c..d7eb7fc9e20 100644 --- a/test/e2e/storage/testsuites/BUILD +++ b/test/e2e/storage/testsuites/BUILD @@ -45,6 +45,7 @@ go_library( "//staging/src/k8s.io/csi-translation-lib:go_default_library", "//staging/src/k8s.io/csi-translation-lib/plugins:go_default_library", "//test/e2e/framework:go_default_library", + "//test/e2e/framework/events:go_default_library", "//test/e2e/framework/metrics:go_default_library", "//test/e2e/framework/node:go_default_library", "//test/e2e/framework/pod:go_default_library", diff --git a/test/e2e/storage/testsuites/volumemode.go b/test/e2e/storage/testsuites/volumemode.go index 4cdab57e715..3ccedf7a3ec 100644 --- a/test/e2e/storage/testsuites/volumemode.go +++ b/test/e2e/storage/testsuites/volumemode.go @@ -20,7 +20,6 @@ import ( "fmt" "path/filepath" "strings" - "time" "github.com/onsi/ginkgo" "github.com/onsi/gomega" @@ -35,6 +34,7 @@ import ( volevents "k8s.io/kubernetes/pkg/controller/volume/events" "k8s.io/kubernetes/pkg/kubelet/events" "k8s.io/kubernetes/test/e2e/framework" + e2eevents "k8s.io/kubernetes/test/e2e/framework/events" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2epv "k8s.io/kubernetes/test/e2e/framework/pv" "k8s.io/kubernetes/test/e2e/framework/volume" @@ -230,7 +230,7 @@ func (t *volumeModeTestSuite) defineTests(driver TestDriver, pattern testpattern }.AsSelector().String() msg := "Unable to attach or mount volumes" - err = waitTimeoutForEvent(l.cs, l.ns.Name, eventSelector, msg, framework.PodStartTimeout) + err = e2eevents.WaitTimeoutForEvent(l.cs, l.ns.Name, eventSelector, msg, framework.PodStartTimeout) // Events are unreliable, don't depend on the event. It's used only to speed up the test. if err != nil { framework.Logf("Warning: did not get event about FailedMountVolume") @@ -267,7 +267,7 @@ func (t *volumeModeTestSuite) defineTests(driver TestDriver, pattern testpattern }.AsSelector().String() msg := "does not support block volume provisioning" - err = waitTimeoutForEvent(l.cs, l.ns.Name, eventSelector, msg, framework.ClaimProvisionTimeout) + err = e2eevents.WaitTimeoutForEvent(l.cs, l.ns.Name, eventSelector, msg, framework.ClaimProvisionTimeout) // Events are unreliable, don't depend on the event. It's used only to speed up the test. if err != nil { framework.Logf("Warning: did not get event about provisioing failed") @@ -318,7 +318,7 @@ func (t *volumeModeTestSuite) defineTests(driver TestDriver, pattern testpattern } else { msg = "has volumeMode Filesystem, but is specified in volumeDevices" } - err = waitTimeoutForEvent(l.cs, l.ns.Name, eventSelector, msg, framework.PodStartTimeout) + err = e2eevents.WaitTimeoutForEvent(l.cs, l.ns.Name, eventSelector, msg, framework.PodStartTimeout) // Events are unreliable, don't depend on them. They're used only to speed up the test. if err != nil { framework.Logf("Warning: did not get event about mismatched volume use") @@ -481,16 +481,6 @@ func listPodDirectory(h utils.HostExec, path string, node *v1.Node) ([]string, e return strings.Split(out, "\n"), nil } -// NOTE(avalluri): The below code is intentionally copied from e2e/common package. -// Testsuites depending on common package is not desirable as that pulls quite -// many tests which are not interested by storage suites. -// -// waitTimeoutForEvent waits the given timeout duration for an event to occur. -func waitTimeoutForEvent(c clientset.Interface, namespace, eventSelector, msg string, timeout time.Duration) error { - interval := 2 * time.Second - return wait.PollImmediate(interval, timeout, eventOccurred(c, namespace, eventSelector, msg)) -} - // NOTE(avalluri): The below code is intentionally copied from e2e/common package. // Testsuites depending on common package is not desirable as that pulls quite // many tests which are not interested by storage suites.