diff --git a/test/e2e/autoscaling_utils.go b/test/e2e/autoscaling_utils.go index 7b7ff7eafa6..b2c594fbb1d 100644 --- a/test/e2e/autoscaling_utils.go +++ b/test/e2e/autoscaling_utils.go @@ -17,6 +17,7 @@ limitations under the License. package e2e import ( + "context" "fmt" "strconv" "time" @@ -202,7 +203,12 @@ func (rc *ResourceConsumer) makeConsumeCustomMetric() { func (rc *ResourceConsumer) sendConsumeCPURequest(millicores int) { proxyRequest, err := framework.GetServicesProxyRequest(rc.framework.ClientSet, rc.framework.ClientSet.Core().RESTClient().Post()) framework.ExpectNoError(err) + + ctx, cancel := context.WithTimeout(context.Background(), framework.SingleCallTimeout) + defer cancel() + req := proxyRequest.Namespace(rc.framework.Namespace.Name). + Context(ctx). Name(rc.controllerName). Suffix("ConsumeCPU"). Param("millicores", strconv.Itoa(millicores)). @@ -217,7 +223,12 @@ func (rc *ResourceConsumer) sendConsumeCPURequest(millicores int) { func (rc *ResourceConsumer) sendConsumeMemRequest(megabytes int) { proxyRequest, err := framework.GetServicesProxyRequest(rc.framework.ClientSet, rc.framework.ClientSet.Core().RESTClient().Post()) framework.ExpectNoError(err) + + ctx, cancel := context.WithTimeout(context.Background(), framework.SingleCallTimeout) + defer cancel() + req := proxyRequest.Namespace(rc.framework.Namespace.Name). + Context(ctx). Name(rc.controllerName). Suffix("ConsumeMem"). Param("megabytes", strconv.Itoa(megabytes)). @@ -232,7 +243,12 @@ func (rc *ResourceConsumer) sendConsumeMemRequest(megabytes int) { func (rc *ResourceConsumer) sendConsumeCustomMetric(delta int) { proxyRequest, err := framework.GetServicesProxyRequest(rc.framework.ClientSet, rc.framework.ClientSet.Core().RESTClient().Post()) framework.ExpectNoError(err) + + ctx, cancel := context.WithTimeout(context.Background(), framework.SingleCallTimeout) + defer cancel() + req := proxyRequest.Namespace(rc.framework.Namespace.Name). + Context(ctx). Name(rc.controllerName). Suffix("BumpMetric"). Param("metric", customMetricName). diff --git a/test/e2e/cluster_logging_es.go b/test/e2e/cluster_logging_es.go index e37656e64fd..538a562f1df 100644 --- a/test/e2e/cluster_logging_es.go +++ b/test/e2e/cluster_logging_es.go @@ -17,6 +17,7 @@ limitations under the License. package e2e import ( + "context" "encoding/json" "fmt" "strconv" @@ -121,14 +122,23 @@ func checkElasticsearchReadiness(f *framework.Framework) error { framework.Logf("After %v failed to get services proxy request: %v", time.Since(start), errProxy) continue } + + ctx, cancel := context.WithTimeout(context.Background(), framework.SingleCallTimeout) + defer cancel() + // Query against the root URL for Elasticsearch. response := proxyRequest.Namespace(api.NamespaceSystem). + Context(ctx). Name("elasticsearch-logging"). Do() err = response.Error() response.StatusCode(&statusCode) if err != nil { + if ctx.Err() != nil { + framework.Failf("After %v proxy call to elasticsearch-loigging failed: %v", time.Since(start), err) + continue + } framework.Logf("After %v proxy call to elasticsearch-loigging failed: %v", time.Since(start), err) continue } @@ -153,12 +163,20 @@ func checkElasticsearchReadiness(f *framework.Framework) error { framework.Logf("After %v failed to get services proxy request: %v", time.Since(start), errProxy) continue } + + ctx, cancel := context.WithTimeout(context.Background(), framework.SingleCallTimeout) + defer cancel() + body, err = proxyRequest.Namespace(api.NamespaceSystem). + Context(ctx). Name("elasticsearch-logging"). Suffix("_cluster/health"). Param("level", "indices"). DoRaw() if err != nil { + if ctx.Err() != nil { + framework.Failf("Failed to get cluster health from elasticsearch: %v", err) + } continue } health := make(map[string]interface{}) @@ -195,9 +213,13 @@ func getMissingLinesCountElasticsearch(f *framework.Framework, expectedCount int return 0, fmt.Errorf("Failed to get services proxy request: %v", errProxy) } + ctx, cancel := context.WithTimeout(context.Background(), framework.SingleCallTimeout) + defer cancel() + // Ask Elasticsearch to return all the log lines that were tagged with the // pod name. Ask for ten times as many log lines because duplication is possible. body, err := proxyRequest.Namespace(api.NamespaceSystem). + Context(ctx). Name("elasticsearch-logging"). Suffix("_search"). // TODO: Change filter to only match records from current test run @@ -207,6 +229,9 @@ func getMissingLinesCountElasticsearch(f *framework.Framework, expectedCount int Param("size", strconv.Itoa(expectedCount*10)). DoRaw() if err != nil { + if ctx.Err() != nil { + framework.Failf("Failed to make proxy call to elasticsearch-logging: %v", err) + } return 0, fmt.Errorf("Failed to make proxy call to elasticsearch-logging: %v", err) } diff --git a/test/e2e/dashboard.go b/test/e2e/dashboard.go index d7125c4441a..0c325154b88 100644 --- a/test/e2e/dashboard.go +++ b/test/e2e/dashboard.go @@ -17,6 +17,7 @@ limitations under the License. package e2e import ( + "context" "net/http" "time" @@ -58,17 +59,26 @@ var _ = framework.KubeDescribe("Kubernetes Dashboard", func() { if errProxy != nil { framework.Logf("Get services proxy request failed: %v", errProxy) } + + ctx, cancel := context.WithTimeout(context.Background(), framework.SingleCallTimeout) + defer cancel() + // Query against the proxy URL for the kube-ui service. err := proxyRequest.Namespace(uiNamespace). + Context(ctx). Name(uiServiceName). Timeout(framework.SingleCallTimeout). Do(). StatusCode(&status). Error() - if status != http.StatusOK { - framework.Logf("Unexpected status from kubernetes-dashboard: %v", status) - } else if err != nil { + if err != nil { + if ctx.Err() != nil { + framework.Failf("Request to kube-ui failed: %v", err) + return true, err + } framework.Logf("Request to kube-ui failed: %v", err) + } else if status != http.StatusOK { + framework.Logf("Unexpected status from kubernetes-dashboard: %v", status) } // Don't return err here as it aborts polling. return status == http.StatusOK, nil diff --git a/test/e2e/dns.go b/test/e2e/dns.go index 50cf871707f..4e1dbabff0d 100644 --- a/test/e2e/dns.go +++ b/test/e2e/dns.go @@ -17,6 +17,7 @@ limitations under the License. package e2e import ( + "context" "fmt" "strings" "time" @@ -185,10 +186,15 @@ func assertFilesContain(fileNames []string, fileDir string, pod *v1.Pod, client if err != nil { return false, err } + + ctx, cancel := context.WithTimeout(context.Background(), framework.SingleCallTimeout) + defer cancel() + var contents []byte for _, fileName := range fileNames { if subResourceProxyAvailable { contents, err = client.Core().RESTClient().Get(). + Context(ctx). Namespace(pod.Namespace). Resource("pods"). SubResource("proxy"). @@ -197,6 +203,7 @@ func assertFilesContain(fileNames []string, fileDir string, pod *v1.Pod, client Do().Raw() } else { contents, err = client.Core().RESTClient().Get(). + Context(ctx). Prefix("proxy"). Resource("pods"). Namespace(pod.Namespace). @@ -205,7 +212,11 @@ func assertFilesContain(fileNames []string, fileDir string, pod *v1.Pod, client Do().Raw() } if err != nil { - framework.Logf("Unable to read %s from pod %s: %v", fileName, pod.Name, err) + if ctx.Err() != nil { + framework.Failf("Unable to read %s from pod %s: %v", fileName, pod.Name, err) + } else { + framework.Logf("Unable to read %s from pod %s: %v", fileName, pod.Name, err) + } failed = append(failed, fileName) } else if check && strings.TrimSpace(string(contents)) != expected { framework.Logf("File %s from pod %s contains '%s' instead of '%s'", fileName, pod.Name, string(contents), expected) diff --git a/test/e2e/example_k8petstore.go b/test/e2e/example_k8petstore.go index 6a03420ac29..550b441618e 100644 --- a/test/e2e/example_k8petstore.go +++ b/test/e2e/example_k8petstore.go @@ -17,6 +17,7 @@ limitations under the License. package e2e import ( + "context" "fmt" "os" "os/exec" @@ -52,16 +53,25 @@ func readTransactions(c clientset.Interface, ns string) (error, int) { if errProxy != nil { return errProxy, -1 } + + ctx, cancel := context.WithTimeout(context.Background(), framework.SingleCallTimeout) + defer cancel() + body, err := proxyRequest.Namespace(ns). + Context(ctx). Name("frontend"). Suffix("llen"). DoRaw() if err != nil { + if ctx.Err() != nil { + framework.Failf("Failed to read petstore transactions: %v", err) + } return err, -1 - } else { - totalTrans, err := strconv.Atoi(string(body)) - return err, totalTrans } + + totalTrans, err := strconv.Atoi(string(body)) + return err, totalTrans + } // runK8petstore runs the k8petstore application, bound to external nodeport, and @@ -150,7 +160,7 @@ T: // We should have exceeded the finalTransactionsExpected num of transactions. // If this fails, but there are transactions being created, we may need to recalibrate // the finalTransactionsExpected value - or else - your cluster is broken/slow ! - Ω(totalTransactions).Should(BeNumerically(">", finalTransactionsExpected)) + Expect(totalTransactions).To(BeNumerically(">", finalTransactionsExpected)) } var _ = framework.KubeDescribe("Pet Store [Feature:Example]", func() { diff --git a/test/e2e/examples.go b/test/e2e/examples.go index d6456d7fd6f..ba1d834055d 100644 --- a/test/e2e/examples.go +++ b/test/e2e/examples.go @@ -17,6 +17,7 @@ limitations under the License. package e2e import ( + "context" "fmt" "io/ioutil" "os" @@ -566,7 +567,12 @@ func makeHttpRequestToService(c clientset.Interface, ns, service, path string, t if errProxy != nil { break } + + ctx, cancel := context.WithTimeout(context.Background(), framework.SingleCallTimeout) + defer cancel() + result, err = proxyRequest.Namespace(ns). + Context(ctx). Name(service). Suffix(path). Do(). diff --git a/test/e2e/framework/kubelet_stats.go b/test/e2e/framework/kubelet_stats.go index d5be4aff6c9..c4691b13f69 100644 --- a/test/e2e/framework/kubelet_stats.go +++ b/test/e2e/framework/kubelet_stats.go @@ -18,6 +18,7 @@ package framework import ( "bytes" + "context" "encoding/json" "fmt" "sort" @@ -288,9 +289,13 @@ func getContainerInfo(c clientset.Interface, nodeName string, req *kubeletstats. return nil, err } + ctx, cancel := context.WithTimeout(context.Background(), SingleCallTimeout) + defer cancel() + var data []byte if subResourceProxyAvailable { data, err = c.Core().RESTClient().Post(). + Context(ctx). Resource("nodes"). SubResource("proxy"). Name(fmt.Sprintf("%v:%v", nodeName, ports.KubeletPort)). @@ -301,6 +306,7 @@ func getContainerInfo(c clientset.Interface, nodeName string, req *kubeletstats. } else { data, err = c.Core().RESTClient().Post(). + Context(ctx). Prefix("proxy"). Resource("nodes"). Name(fmt.Sprintf("%v:%v", nodeName, ports.KubeletPort)). diff --git a/test/e2e/framework/metrics_util.go b/test/e2e/framework/metrics_util.go index 8dcd6360dfa..79357c9fdab 100644 --- a/test/e2e/framework/metrics_util.go +++ b/test/e2e/framework/metrics_util.go @@ -18,6 +18,7 @@ package framework import ( "bytes" + "context" "encoding/json" "fmt" "io" @@ -335,7 +336,11 @@ func getSchedulingLatency(c clientset.Interface) (SchedulingLatency, error) { } } if masterRegistered { + ctx, cancel := context.WithTimeout(context.Background(), SingleCallTimeout) + defer cancel() + rawData, err := c.Core().RESTClient().Get(). + Context(ctx). Prefix("proxy"). Namespace(api.NamespaceSystem). Resource("pods"). diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index 07156a86cee..7acac5cf051 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -18,6 +18,7 @@ package framework import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -1568,9 +1569,14 @@ func (r podProxyResponseChecker) CheckAllResponses() (done bool, err error) { if err != nil { return false, err } + + ctx, cancel := context.WithTimeout(context.Background(), SingleCallTimeout) + defer cancel() + var body []byte if subResourceProxyAvailable { body, err = r.c.Core().RESTClient().Get(). + Context(ctx). Namespace(r.ns). Resource("pods"). SubResource("proxy"). @@ -1579,6 +1585,7 @@ func (r podProxyResponseChecker) CheckAllResponses() (done bool, err error) { Raw() } else { body, err = r.c.Core().RESTClient().Get(). + Context(ctx). Prefix("proxy"). Namespace(r.ns). Resource("pods"). @@ -1587,6 +1594,10 @@ func (r podProxyResponseChecker) CheckAllResponses() (done bool, err error) { Raw() } if err != nil { + if ctx.Err() != nil { + Failf("Controller %s: Failed to Get from replica %d [%s]: %v\n pod status: %#v", r.controllerName, i+1, pod.Name, err, pod.Status) + return false, err + } Logf("Controller %s: Failed to GET from replica %d [%s]: %v\npod status: %#v", r.controllerName, i+1, pod.Name, err, pod.Status) continue } @@ -1756,11 +1767,20 @@ func ServiceResponding(c clientset.Interface, ns, name string) error { Logf("Failed to get services proxy request: %v:", errProxy) return false, nil } + + ctx, cancel := context.WithTimeout(context.Background(), SingleCallTimeout) + defer cancel() + body, err := proxyRequest.Namespace(ns). + Context(ctx). Name(name). Do(). Raw() if err != nil { + if ctx.Err() != nil { + Failf("Failed to GET from service %s: %v", name, err) + return true, err + } Logf("Failed to GET from service %s: %v:", name, err) return false, nil } diff --git a/test/e2e/kibana_logging.go b/test/e2e/kibana_logging.go index 9bf466e1cb6..ab87aab3760 100644 --- a/test/e2e/kibana_logging.go +++ b/test/e2e/kibana_logging.go @@ -17,6 +17,7 @@ limitations under the License. package e2e import ( + "context" "time" "k8s.io/kubernetes/pkg/api" @@ -87,11 +88,20 @@ func ClusterLevelLoggingWithKibana(f *framework.Framework) { err = errProxy continue } + + ctx, cancel := context.WithTimeout(context.Background(), framework.SingleCallTimeout) + defer cancel() + // Query against the root URL for Kibana. _, err = proxyRequest.Namespace(api.NamespaceSystem). + Context(ctx). Name("kibana-logging"). DoRaw() if err != nil { + if ctx.Err() != nil { + framework.Failf("After %v proxy call to kibana-logging failed: %v", time.Since(start), err) + break + } framework.Logf("After %v proxy call to kibana-logging failed: %v", time.Since(start), err) continue } diff --git a/test/e2e/kubectl.go b/test/e2e/kubectl.go index c06c1a8e1db..3edc6959f1e 100644 --- a/test/e2e/kubectl.go +++ b/test/e2e/kubectl.go @@ -18,6 +18,7 @@ package e2e import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -1610,7 +1611,12 @@ func makeRequestToGuestbook(c clientset.Interface, cmd, value string, ns string) if errProxy != nil { return "", errProxy } + + ctx, cancel := context.WithTimeout(context.Background(), framework.SingleCallTimeout) + defer cancel() + result, err := proxyRequest.Namespace(ns). + Context(ctx). Name("frontend"). Suffix("/guestbook.php"). Param("cmd", cmd). @@ -1710,6 +1716,10 @@ func getUDData(jpgExpected string, ns string) func(clientset.Interface, string) if err != nil { return err } + + ctx, cancel := context.WithTimeout(context.Background(), framework.SingleCallTimeout) + defer cancel() + var body []byte if subResourceProxyAvailable { body, err = c.Core().RESTClient().Get(). @@ -1731,6 +1741,9 @@ func getUDData(jpgExpected string, ns string) func(clientset.Interface, string) Raw() } if err != nil { + if ctx.Err() != nil { + framework.Failf("Failed to retrieve data from container: %v", err) + } return err } framework.Logf("got data: %s", body) diff --git a/test/e2e/monitoring.go b/test/e2e/monitoring.go index f09640a5259..2dad612257f 100644 --- a/test/e2e/monitoring.go +++ b/test/e2e/monitoring.go @@ -18,6 +18,7 @@ package e2e import ( "bytes" + "context" "encoding/json" "fmt" "time" @@ -63,6 +64,10 @@ var ( // Query sends a command to the server and returns the Response func Query(c clientset.Interface, query string) (*influxdb.Response, error) { + + ctx, cancel := context.WithTimeout(context.Background(), framework.SingleCallTimeout) + defer cancel() + result, err := c.Core().RESTClient().Get(). Prefix("proxy"). Namespace("kube-system"). @@ -76,6 +81,9 @@ func Query(c clientset.Interface, query string) (*influxdb.Response, error) { Raw() if err != nil { + if ctx.Err() != nil { + framework.Failf("Failed to query influx db: %v", err) + } return nil, err } diff --git a/test/e2e/pre_stop.go b/test/e2e/pre_stop.go index 6c2eab0a30d..689f11c7a11 100644 --- a/test/e2e/pre_stop.go +++ b/test/e2e/pre_stop.go @@ -17,6 +17,7 @@ limitations under the License. package e2e import ( + "context" "encoding/json" "fmt" "time" @@ -122,9 +123,14 @@ func testPreStop(c clientset.Interface, ns string) { if err != nil { return false, err } + + ctx, cancel := context.WithTimeout(context.Background(), framework.SingleCallTimeout) + defer cancel() + var body []byte if subResourceProxyAvailable { body, err = c.Core().RESTClient().Get(). + Context(ctx). Namespace(ns). Resource("pods"). SubResource("proxy"). @@ -133,6 +139,7 @@ func testPreStop(c clientset.Interface, ns string) { DoRaw() } else { body, err = c.Core().RESTClient().Get(). + Context(ctx). Prefix("proxy"). Namespace(ns). Resource("pods"). @@ -141,6 +148,10 @@ func testPreStop(c clientset.Interface, ns string) { DoRaw() } if err != nil { + if ctx.Err() != nil { + framework.Failf("Error validating prestop: %v", err) + return true, err + } By(fmt.Sprintf("Error validating prestop: %v", err)) } else { framework.Logf("Saw: %s", string(body))