From 5dac1122b48ec4b5f4efa965d016a8de100a8223 Mon Sep 17 00:00:00 2001 From: bprashanth Date: Wed, 31 Aug 2016 16:44:30 -0700 Subject: [PATCH 1/3] Logging for service restart e2e clarity --- pkg/controller/endpoint/endpoints_controller.go | 5 +++++ test/e2e/framework/util.go | 6 +++++- test/e2e/service.go | 4 +++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/pkg/controller/endpoint/endpoints_controller.go b/pkg/controller/endpoint/endpoints_controller.go index 06eb1768a6a..9c9f06e7d02 100644 --- a/pkg/controller/endpoint/endpoints_controller.go +++ b/pkg/controller/endpoint/endpoints_controller.go @@ -380,6 +380,8 @@ func (e *EndpointController) syncService(key string) { } } + readyEps := 0 + notReadyEps := 0 for i := range pods { // TODO: Do we need to copy here? pod := &(*pods[i]) @@ -432,12 +434,14 @@ func (e *EndpointController) syncService(key string) { Addresses: []api.EndpointAddress{epa}, Ports: []api.EndpointPort{epp}, }) + readyEps++ } else { glog.V(5).Infof("Pod is out of service: %v/%v", pod.Namespace, pod.Name) subsets = append(subsets, api.EndpointSubset{ NotReadyAddresses: []api.EndpointAddress{epa}, Ports: []api.EndpointPort{epp}, }) + notReadyEps++ } } } @@ -490,6 +494,7 @@ func (e *EndpointController) syncService(key string) { newEndpoints.Annotations[endpoints.PodHostnamesAnnotation] = serializedPodHostNames } + glog.V(4).Infof("Update endpoints for %v/%v, ready: %d not ready: %d", service.Namespace, service.Name, readyEps, notReadyEps) createEndpoints := len(currentEndpoints.ResourceVersion) == 0 if createEndpoints { // No previous endpoints, create them diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index 6ce06bb9f1a..323b7aad7d8 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -4071,14 +4071,17 @@ func RestartKubeProxy(host string) error { return fmt.Errorf("unsupported provider: %s", TestContext.Provider) } // kubelet will restart the kube-proxy since it's running in a static pod + Logf("Killing kube-proxy on node %v", host) result, err := SSH("sudo pkill kube-proxy", host, TestContext.Provider) if err != nil || result.Code != 0 { LogSSHResult(result) return fmt.Errorf("couldn't restart kube-proxy: %v", err) } // wait for kube-proxy to come back up + sshCmd := "sudo /bin/sh -c 'pgrep kube-proxy | wc -l'" err = wait.Poll(5*time.Second, 60*time.Second, func() (bool, error) { - result, err := SSH("sudo /bin/sh -c 'pgrep kube-proxy | wc -l'", host, TestContext.Provider) + Logf("Waiting for kubeproxy to come back up with %v on %v", sshCmd, host) + result, err := SSH(sshCmd, host, TestContext.Provider) if err != nil { return false, err } @@ -4125,6 +4128,7 @@ func sshRestartMaster() error { } else { command = "sudo /etc/init.d/kube-apiserver restart" } + Logf("Restarting master via ssh, running: %v", command) result, err := SSH(command, GetMasterHost()+":22", TestContext.Provider) if err != nil || result.Code != 0 { LogSSHResult(result) diff --git a/test/e2e/service.go b/test/e2e/service.go index 9c1d61037c6..1cf5fcc0c85 100644 --- a/test/e2e/service.go +++ b/test/e2e/service.go @@ -312,7 +312,7 @@ var _ = framework.KubeDescribe("Services", func() { framework.ExpectNoError(verifyServeHostnameServiceUp(c, ns, host, podNames1, svc1IP, servicePort)) framework.ExpectNoError(verifyServeHostnameServiceUp(c, ns, host, podNames2, svc2IP, servicePort)) - By("Restarting kube-proxy") + By(fmt.Sprintf("Restarting kube-proxy on %v", host)) if err := framework.RestartKubeProxy(host); err != nil { framework.Failf("error restarting kube-proxy: %v", err) } @@ -353,9 +353,11 @@ var _ = framework.KubeDescribe("Services", func() { framework.ExpectNoError(verifyServeHostnameServiceUp(c, ns, host, podNames1, svc1IP, servicePort)) // Restart apiserver + By("Restarting apiserver") if err := framework.RestartApiserver(c); err != nil { framework.Failf("error restarting apiserver: %v", err) } + By("Waiting for apiserver to come up by polling /healthz") if err := framework.WaitForApiserverUp(c); err != nil { framework.Failf("error while waiting for apiserver up: %v", err) } From 9481354d77cd491cb4b0e98db3a51a1758cdf0f8 Mon Sep 17 00:00:00 2001 From: bprashanth Date: Thu, 1 Sep 2016 14:26:44 -0700 Subject: [PATCH 2/3] Fix race in scheduledjob e2e --- test/e2e/scheduledjob.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/e2e/scheduledjob.go b/test/e2e/scheduledjob.go index a88175d299e..bc36cf587dc 100644 --- a/test/e2e/scheduledjob.go +++ b/test/e2e/scheduledjob.go @@ -275,8 +275,11 @@ func waitForJobReplaced(c *client.Client, ns, previousJobName string) error { if err != nil { return false, err } - if len(jobs.Items) != 1 { - return false, fmt.Errorf("More than one job is running") + if len(jobs.Items) > 1 { + return false, fmt.Errorf("More than one job is running %+v", jobs.Items) + } else if len(jobs.Items) == 0 { + framework.Logf("Warning: Found 0 jobs in namespace %v", ns) + return false, nil } return jobs.Items[0].Name != previousJobName, nil }) From a0c0dcd8bd8247b8ada3c546f55e3ca8031c882d Mon Sep 17 00:00:00 2001 From: bprashanth Date: Thu, 1 Sep 2016 17:32:50 -0700 Subject: [PATCH 3/3] Log pod ip at V(2) --- pkg/kubelet/dockertools/docker_manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/kubelet/dockertools/docker_manager.go b/pkg/kubelet/dockertools/docker_manager.go index 8b6429f9437..053249ee717 100644 --- a/pkg/kubelet/dockertools/docker_manager.go +++ b/pkg/kubelet/dockertools/docker_manager.go @@ -2159,7 +2159,7 @@ func (dm *DockerManager) SyncPod(pod *api.Pod, _ api.PodStatus, podStatus *kubec result.Fail(err) return } - glog.V(4).Infof("Determined pod ip after infra change: %q: %q", format.Pod(pod), podIP) + glog.Infof("Determined pod ip after infra change: %q: %q", format.Pod(pod), podIP) } }