Merge pull request #33159 from nikhiljindal/fedTestEvents

Automatic merge from submit-queue

Dumping federation events if federation e2e test failed

Updating the e2e framework to dump events in federation control plane if a federation e2e test failed.

This should help in debugging https://github.com/kubernetes/kubernetes/issues/32733

cc @kubernetes/sig-cluster-federation

test/e2e/e2e.go

@@ -98,6 +98,10 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
 	if err != nil {
 		glog.Fatal("Error loading client: ", err)
 	}
+	clientset, err := framework.LoadClientset()
+	if err != nil {
+		glog.Fatal("Error loading clientset: ", err)
+	}
 
 	// Delete any namespaces except default and kube-system. This ensures no
 	// lingering resources are left over from a previous test run.
@@ -118,7 +122,7 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
 	// ready will fail).
 	podStartupTimeout := framework.TestContext.SystemPodsStartupTimeout
 	if err := framework.WaitForPodsRunningReady(c, api.NamespaceSystem, int32(framework.TestContext.MinStartupPods), podStartupTimeout, framework.ImagePullerLabels); err != nil {
-		framework.DumpAllNamespaceInfo(c, api.NamespaceSystem)
+		framework.DumpAllNamespaceInfo(c, clientset, api.NamespaceSystem)
 		framework.LogFailedContainers(c, api.NamespaceSystem)
 		framework.RunKubernetesServiceTestContainer(c, api.NamespaceDefault)
 		framework.Failf("Error waiting for all pods to be running and ready: %v", err)

test/e2e/framework/framework.go

@@ -356,10 +356,15 @@ func (f *Framework) AfterEach() {
 
 	// Print events if the test failed.
 	if CurrentGinkgoTestDescription().Failed && TestContext.DumpLogsOnFailure {
-		DumpAllNamespaceInfo(f.Client, f.Namespace.Name)
+		// Pass both unversioned client and and versioned clientset, till we have removed all uses of the unversioned client.
+		DumpAllNamespaceInfo(f.Client, f.Clientset_1_5, f.Namespace.Name)
 		By(fmt.Sprintf("Dumping a list of prepulled images on each node"))
 		LogContainersInPodsWithLabels(f.Client, api.NamespaceSystem, ImagePullerLabels, "image-puller")
 		if f.federated {
+			// Dump federation events in federation namespace.
+			DumpEventsInNamespace(func(opts api.ListOptions, ns string) (*v1.EventList, error) {
+				return f.FederationClientset_1_4.Core().Events(ns).List(opts)
+			}, f.FederationNamespace.Name)
 			// Print logs of federation control plane pods (federation-apiserver and federation-controller-manager)
 			LogPodsWithLabels(f.Client, "federation", map[string]string{"app": "federated-cluster"})
 			// Print logs of kube-dns pod

test/e2e/framework/util.go

@@ -52,6 +52,7 @@ import (
 	"k8s.io/kubernetes/pkg/apis/extensions"
 	"k8s.io/kubernetes/pkg/client/cache"
 	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
+	"k8s.io/kubernetes/pkg/client/clientset_generated/release_1_5"
 	"k8s.io/kubernetes/pkg/client/restclient"
 	"k8s.io/kubernetes/pkg/client/typed/discovery"
 	"k8s.io/kubernetes/pkg/client/typed/dynamic"
@@ -1978,6 +1979,14 @@ func LoadClient() (*client.Client, error) {
 	return loadClientFromConfig(config)
 }
 
+func LoadClientset() (*release_1_5.Clientset, error) {
+	config, err := LoadConfig()
+	if err != nil {
+		return nil, fmt.Errorf("error creating client: %v", err.Error())
+	}
+	return release_1_5.NewForConfig(config)
+}
+
 // randomSuffix provides a random string to append to pods,services,rcs.
 // TODO: Allow service names to have the same form as names
 //       for pods and replication controllers so we don't
@@ -2815,9 +2824,11 @@ func dumpPodDebugInfo(c *client.Client, pods []*api.Pod) {
 	DumpNodeDebugInfo(c, badNodes.List())
 }
 
-func DumpAllNamespaceInfo(c *client.Client, namespace string) {
+type EventsLister func(opts api.ListOptions, ns string) (*v1.EventList, error)
+
+func DumpEventsInNamespace(eventsLister EventsLister, namespace string) {
 	By(fmt.Sprintf("Collecting events from namespace %q.", namespace))
-	events, err := c.Events(namespace).List(api.ListOptions{})
+	events, err := eventsLister(api.ListOptions{}, namespace)
 	Expect(err).NotTo(HaveOccurred())
 
 	// Sort events by their first timestamp
@@ -2831,6 +2842,12 @@ func DumpAllNamespaceInfo(c *client.Client, namespace string) {
 	// Note that we don't wait for any Cleanup to propagate, which means
 	// that if you delete a bunch of pods right before ending your test,
 	// you may or may not see the killing/deletion/Cleanup events.
+}
+
+func DumpAllNamespaceInfo(c *client.Client, cs *release_1_5.Clientset, namespace string) {
+	DumpEventsInNamespace(func(opts api.ListOptions, ns string) (*v1.EventList, error) {
+		return cs.Core().Events(ns).List(opts)
+	}, namespace)
 
 	// If cluster is large, then the following logs are basically useless, because:
 	// 1. it takes tens of minutes or hours to grab all of them
@@ -2850,7 +2867,7 @@ func DumpAllNamespaceInfo(c *client.Client, namespace string) {
 }
 
 // byFirstTimestamp sorts a slice of events by first timestamp, using their involvedObject's name as a tie breaker.
-type byFirstTimestamp []api.Event
+type byFirstTimestamp []v1.Event
 
 func (o byFirstTimestamp) Len() int      { return len(o) }
 func (o byFirstTimestamp) Swap(i, j int) { o[i], o[j] = o[j], o[i] }

test/e2e/kubectl.go

@@ -199,7 +199,7 @@ var _ = framework.KubeDescribe("Kubectl client", func() {
 		pods, err := clusterState().WaitFor(atLeast, framework.PodStartTimeout)
 		if err != nil || len(pods) < atLeast {
 			// TODO: Generalize integrating debug info into these tests so we always get debug info when we need it
-			framework.DumpAllNamespaceInfo(c, ns)
+			framework.DumpAllNamespaceInfo(c, f.Clientset_1_5, ns)
 			framework.Failf("Verified %v of %v pods , error : %v", len(pods), atLeast, err)
 		}
 	}