Refactor debug logging methods.

This commit is contained in:
Prashanth Balasubramanian 2015-11-08 10:58:21 -08:00
parent 8b35455664
commit eb4106fe29
5 changed files with 149 additions and 130 deletions

View File

@ -74,7 +74,7 @@ var _ = Describe("Examples e2e", func() {
expectedOnSentinel := "+monitor master"
By("starting redis bootstrap")
runKubectl("create", "-f", bootstrapYaml, nsFlag)
runKubectlOrDie("create", "-f", bootstrapYaml, nsFlag)
err := waitForPodRunningInNamespace(c, bootstrapPodName, ns)
Expect(err).NotTo(HaveOccurred())
@ -84,13 +84,13 @@ var _ = Describe("Examples e2e", func() {
Expect(err).NotTo(HaveOccurred())
By("setting up services and controllers")
runKubectl("create", "-f", sentinelServiceYaml, nsFlag)
runKubectl("create", "-f", sentinelControllerYaml, nsFlag)
runKubectl("create", "-f", controllerYaml, nsFlag)
runKubectlOrDie("create", "-f", sentinelServiceYaml, nsFlag)
runKubectlOrDie("create", "-f", sentinelControllerYaml, nsFlag)
runKubectlOrDie("create", "-f", controllerYaml, nsFlag)
By("scaling up the deployment")
runKubectl("scale", "rc", redisRC, "--replicas=3", nsFlag)
runKubectl("scale", "rc", sentinelRC, "--replicas=3", nsFlag)
runKubectlOrDie("scale", "rc", redisRC, "--replicas=3", nsFlag)
runKubectlOrDie("scale", "rc", sentinelRC, "--replicas=3", nsFlag)
By("checking up the services")
checkAllLogs := func() {
@ -110,7 +110,7 @@ var _ = Describe("Examples e2e", func() {
checkAllLogs()
By("turning down bootstrap")
runKubectl("delete", "-f", bootstrapYaml, nsFlag)
runKubectlOrDie("delete", "-f", bootstrapYaml, nsFlag)
err = waitForRCPodToDisappear(c, ns, redisRC, bootstrapPodName)
Expect(err).NotTo(HaveOccurred())
By("waiting for the new master election")
@ -131,8 +131,8 @@ var _ = Describe("Examples e2e", func() {
nsFlag := fmt.Sprintf("--namespace=%v", ns)
By("starting rabbitmq")
runKubectl("create", "-f", rabbitmqServiceYaml, nsFlag)
runKubectl("create", "-f", rabbitmqControllerYaml, nsFlag)
runKubectlOrDie("create", "-f", rabbitmqServiceYaml, nsFlag)
runKubectlOrDie("create", "-f", rabbitmqControllerYaml, nsFlag)
forEachPod(c, ns, "component", "rabbitmq", func(pod api.Pod) {
_, err := lookForStringInLog(ns, pod.Name, "rabbitmq", "Server startup complete", serverStartTimeout)
Expect(err).NotTo(HaveOccurred())
@ -141,15 +141,15 @@ var _ = Describe("Examples e2e", func() {
Expect(err).NotTo(HaveOccurred())
By("starting celery")
runKubectl("create", "-f", celeryControllerYaml, nsFlag)
runKubectlOrDie("create", "-f", celeryControllerYaml, nsFlag)
forEachPod(c, ns, "component", "celery", func(pod api.Pod) {
_, err := lookForStringInFile(ns, pod.Name, "celery", "/data/celery.log", " ready.", serverStartTimeout)
Expect(err).NotTo(HaveOccurred())
})
By("starting flower")
runKubectl("create", "-f", flowerServiceYaml, nsFlag)
runKubectl("create", "-f", flowerControllerYaml, nsFlag)
runKubectlOrDie("create", "-f", flowerServiceYaml, nsFlag)
runKubectlOrDie("create", "-f", flowerControllerYaml, nsFlag)
forEachPod(c, ns, "component", "flower", func(pod api.Pod) {
// Do nothing. just wait for it to be up and running.
})
@ -173,9 +173,9 @@ var _ = Describe("Examples e2e", func() {
nsFlag := fmt.Sprintf("--namespace=%v", ns)
By("starting master")
runKubectl("create", "-f", serviceJson, nsFlag)
runKubectl("create", "-f", masterJson, nsFlag)
runKubectl("create", "-f", driverJson, nsFlag)
runKubectlOrDie("create", "-f", serviceJson, nsFlag)
runKubectlOrDie("create", "-f", masterJson, nsFlag)
runKubectlOrDie("create", "-f", driverJson, nsFlag)
err := waitForPodRunningInNamespace(c, "spark-master", ns)
Expect(err).NotTo(HaveOccurred())
_, err = lookForStringInLog(ns, "spark-master", "spark-master", "Starting Spark master at", serverStartTimeout)
@ -188,7 +188,7 @@ var _ = Describe("Examples e2e", func() {
Expect(err).NotTo(HaveOccurred())
By("starting workers")
runKubectl("create", "-f", workerControllerJson, nsFlag)
runKubectlOrDie("create", "-f", workerControllerJson, nsFlag)
ScaleRC(c, ns, "spark-worker-controller", 2, true)
forEachPod(c, ns, "name", "spark-worker", func(pod api.Pod) {
_, err := lookForStringInLog(ns, pod.Name, "spark-worker", "Successfully registered with master", serverStartTimeout)
@ -208,8 +208,8 @@ var _ = Describe("Examples e2e", func() {
nsFlag := fmt.Sprintf("--namespace=%v", ns)
By("starting service and pod")
runKubectl("create", "-f", serviceYaml, nsFlag)
runKubectl("create", "-f", podYaml, nsFlag)
runKubectlOrDie("create", "-f", serviceYaml, nsFlag)
runKubectlOrDie("create", "-f", podYaml, nsFlag)
err := waitForPodRunningInNamespace(c, "cassandra", ns)
Expect(err).NotTo(HaveOccurred())
@ -220,7 +220,7 @@ var _ = Describe("Examples e2e", func() {
Expect(err).NotTo(HaveOccurred())
By("create and scale rc")
runKubectl("create", "-f", controllerYaml, nsFlag)
runKubectlOrDie("create", "-f", controllerYaml, nsFlag)
err = ScaleRC(c, ns, "cassandra", 2, true)
Expect(err).NotTo(HaveOccurred())
forEachPod(c, ns, "name", "cassandra", func(pod api.Pod) {
@ -230,7 +230,7 @@ var _ = Describe("Examples e2e", func() {
Expect(err).NotTo(HaveOccurred())
})
output := runKubectl("exec", "cassandra", nsFlag, "--", "nodetool", "status")
output := runKubectlOrDie("exec", "cassandra", nsFlag, "--", "nodetool", "status")
forEachPod(c, ns, "name", "cassandra", func(pod api.Pod) {
if !strings.Contains(output, pod.Status.PodIP) {
Failf("Pod ip %s not found in nodetool status", pod.Status.PodIP)
@ -253,8 +253,8 @@ var _ = Describe("Examples e2e", func() {
zookeeperPod := "zookeeper"
By("starting Zookeeper")
runKubectl("create", "-f", zookeeperPodJson, nsFlag)
runKubectl("create", "-f", zookeeperServiceJson, nsFlag)
runKubectlOrDie("create", "-f", zookeeperPodJson, nsFlag)
runKubectlOrDie("create", "-f", zookeeperServiceJson, nsFlag)
err := waitForPodRunningInNamespace(c, zookeeperPod, ns)
Expect(err).NotTo(HaveOccurred())
@ -265,8 +265,8 @@ var _ = Describe("Examples e2e", func() {
Expect(err).NotTo(HaveOccurred())
By("starting Nimbus")
runKubectl("create", "-f", nimbusPodJson, nsFlag)
runKubectl("create", "-f", nimbusServiceJson, nsFlag)
runKubectlOrDie("create", "-f", nimbusPodJson, nsFlag)
runKubectlOrDie("create", "-f", nimbusServiceJson, nsFlag)
err = waitForPodRunningInNamespace(c, "nimbus", ns)
Expect(err).NotTo(HaveOccurred())
@ -274,7 +274,7 @@ var _ = Describe("Examples e2e", func() {
Expect(err).NotTo(HaveOccurred())
By("starting workers")
runKubectl("create", "-f", workerControllerJson, nsFlag)
runKubectlOrDie("create", "-f", workerControllerJson, nsFlag)
forEachPod(c, ns, "name", "storm-worker", func(pod api.Pod) {
//do nothing, just wait for the pod to be running
})
@ -288,7 +288,7 @@ var _ = Describe("Examples e2e", func() {
By("checking if Nimbus responds to requests")
lookForString("No topologies running.", time.Minute, func() string {
return runKubectl("exec", "nimbus", nsFlag, "--", "bin/storm", "list")
return runKubectlOrDie("exec", "nimbus", nsFlag, "--", "bin/storm", "list")
})
})
})
@ -302,8 +302,8 @@ var _ = Describe("Examples e2e", func() {
httpYaml := mkpath("http-liveness.yaml")
nsFlag := fmt.Sprintf("--namespace=%v", ns)
runKubectl("create", "-f", execYaml, nsFlag)
runKubectl("create", "-f", httpYaml, nsFlag)
runKubectlOrDie("create", "-f", execYaml, nsFlag)
runKubectlOrDie("create", "-f", httpYaml, nsFlag)
checkRestart := func(podName string, timeout time.Duration) {
err := waitForPodRunningInNamespace(c, podName, ns)
Expect(err).NotTo(HaveOccurred())
@ -335,8 +335,8 @@ var _ = Describe("Examples e2e", func() {
nsFlag := fmt.Sprintf("--namespace=%v", ns)
By("creating secret and pod")
runKubectl("create", "-f", secretYaml, nsFlag)
runKubectl("create", "-f", podYaml, nsFlag)
runKubectlOrDie("create", "-f", secretYaml, nsFlag)
runKubectlOrDie("create", "-f", podYaml, nsFlag)
By("checking if secret was read correctly")
_, err := lookForStringInLog(ns, "secret-test-pod", "test-container", "value-1", serverStartTimeout)
@ -354,7 +354,7 @@ var _ = Describe("Examples e2e", func() {
podName := "dapi-test-pod"
By("creating the pod")
runKubectl("create", "-f", podYaml, nsFlag)
runKubectlOrDie("create", "-f", podYaml, nsFlag)
By("checking if name and namespace were passed correctly")
_, err := lookForStringInLog(ns, podName, "test-container", fmt.Sprintf("POD_NAMESPACE=%v", ns), serverStartTimeout)
@ -376,8 +376,8 @@ var _ = Describe("Examples e2e", func() {
nsFlag := fmt.Sprintf("--namespace=%v", ns)
By("starting rethinkdb")
runKubectl("create", "-f", driverServiceYaml, nsFlag)
runKubectl("create", "-f", rethinkDbControllerYaml, nsFlag)
runKubectlOrDie("create", "-f", driverServiceYaml, nsFlag)
runKubectlOrDie("create", "-f", rethinkDbControllerYaml, nsFlag)
checkDbInstances := func() {
forEachPod(c, ns, "db", "rethinkdb", func(pod api.Pod) {
_, err := lookForStringInLog(ns, pod.Name, "rethinkdb", "Server ready", serverStartTimeout)
@ -393,8 +393,8 @@ var _ = Describe("Examples e2e", func() {
checkDbInstances()
By("starting admin")
runKubectl("create", "-f", adminServiceYaml, nsFlag)
runKubectl("create", "-f", adminPodYaml, nsFlag)
runKubectlOrDie("create", "-f", adminServiceYaml, nsFlag)
runKubectlOrDie("create", "-f", adminPodYaml, nsFlag)
err = waitForPodRunningInNamespace(c, "rethinkdb-admin", ns)
Expect(err).NotTo(HaveOccurred())
checkDbInstances()
@ -416,8 +416,8 @@ var _ = Describe("Examples e2e", func() {
nsFlag := fmt.Sprintf("--namespace=%v", ns)
By("starting hazelcast")
runKubectl("create", "-f", serviceYaml, nsFlag)
runKubectl("create", "-f", controllerYaml, nsFlag)
runKubectlOrDie("create", "-f", serviceYaml, nsFlag)
runKubectlOrDie("create", "-f", controllerYaml, nsFlag)
forEachPod(c, ns, "name", "hazelcast", func(pod api.Pod) {
_, err := lookForStringInLog(ns, pod.Name, "hazelcast", "Members [1]", serverStartTimeout)
Expect(err).NotTo(HaveOccurred())
@ -476,11 +476,11 @@ var _ = Describe("Examples e2e", func() {
}
for _, ns := range namespaces {
runKubectl("create", "-f", backendRcYaml, getNsCmdFlag(ns))
runKubectlOrDie("create", "-f", backendRcYaml, getNsCmdFlag(ns))
}
for _, ns := range namespaces {
runKubectl("create", "-f", backendSvcYaml, getNsCmdFlag(ns))
runKubectlOrDie("create", "-f", backendSvcYaml, getNsCmdFlag(ns))
}
// wait for objects
@ -526,7 +526,7 @@ var _ = Describe("Examples e2e", func() {
// create a pod in each namespace
for _, ns := range namespaces {
newKubectlCommand("create", "-f", "-", getNsCmdFlag(ns)).withStdinData(updatedPodYaml).exec()
newKubectlCommand("create", "-f", "-", getNsCmdFlag(ns)).withStdinData(updatedPodYaml).execOrDie()
}
// wait until the pods have been scheduler, i.e. are not Pending anymore. Remember
@ -605,13 +605,13 @@ func forEachPod(c *client.Client, ns, selectorKey, selectorValue string, fn func
func lookForStringInLog(ns, podName, container, expectedString string, timeout time.Duration) (result string, err error) {
return lookForString(expectedString, timeout, func() string {
return runKubectl("log", podName, container, fmt.Sprintf("--namespace=%v", ns))
return runKubectlOrDie("log", podName, container, fmt.Sprintf("--namespace=%v", ns))
})
}
func lookForStringInFile(ns, podName, container, file, expectedString string, timeout time.Duration) (result string, err error) {
return lookForString(expectedString, timeout, func() string {
return runKubectl("exec", podName, "-c", container, fmt.Sprintf("--namespace=%v", ns), "--", "cat", file)
return runKubectlOrDie("exec", podName, "-c", container, fmt.Sprintf("--namespace=%v", ns), "--", "cat", file)
})
}
@ -620,7 +620,7 @@ func lookForStringInPodExec(ns, podName string, command []string, expectedString
// use the first container
args := []string{"exec", podName, fmt.Sprintf("--namespace=%v", ns), "--"}
args = append(args, command...)
return runKubectl(args...)
return runKubectlOrDie(args...)
})
}

View File

@ -65,6 +65,10 @@ var (
// Name of the loadbalancer controller within the cluster addon
lbContainerName = "l7-lb-controller"
// If set, the test tries to perform an HTTP GET on each url endpoint of
// the Ingress. Only set to false to short-circuit test runs in debugging.
verifyHTTPGET = true
// On average it takes ~6 minutes for a single backend to come online.
// We *don't* expect this poll to consistently take 15 minutes for every
// Ingress as GCE is creating/checking backends in parallel, but at the
@ -173,7 +177,9 @@ func createApp(c *client.Client, ns string, i int) {
// gcloudUnmarshal unmarshals json output of gcloud into given out interface.
func gcloudUnmarshal(resource, regex string, out interface{}) {
output, err := exec.Command("gcloud", "compute", resource, "list",
fmt.Sprintf("--regex=%v", regex), "-q", "--format=json").CombinedOutput()
fmt.Sprintf("--regex=%v", regex),
fmt.Sprintf("--project=%v", testContext.CloudConfig.ProjectID),
"-q", "--format=json").CombinedOutput()
if err != nil {
Failf("Error unmarshalling gcloud output: %v", err)
}
@ -204,7 +210,7 @@ func checkLeakedResources() error {
return nil
}
// run kubectl log on the L7 controller pod.
// kubectlLogLBController logs kubectl debug output for the L7 controller pod.
func kubectlLogLBController(c *client.Client) {
selector := labels.SelectorFromSet(labels.Set(map[string]string{"name": "glbc"}))
podList, err := c.Pods(api.NamespaceAll).List(selector, fields.Everything())
@ -218,21 +224,11 @@ func kubectlLogLBController(c *client.Client) {
}
for _, p := range podList.Items {
Logf("\nLast 100 log lines of %v\n", p.Name)
Logf(runKubectl("logs", p.Name, "--namespace=kube-system", "-c", lbContainerName, "--tail=100"))
l, _ := runKubectl("logs", p.Name, fmt.Sprintf("--namespace=%v", api.NamespaceSystem), "-c", lbContainerName, "--tail=100")
Logf(l)
}
}
// dumpDebugAndFail dumps verbose debug output before failing.
func dumpDebugAndFail(err string, ns string, c *client.Client) {
kubectlLogLBController(c)
Logf("\nOutput of kubectl describe ing:\n")
// TODO: runKubectl will hard fail if kubectl fails, swap it out for
// something more befitting for a debug dumper.
Logf(runKubectl("describe", "ing", fmt.Sprintf("--namespace=%v", ns)))
Failf(err)
}
var _ = Describe("GCE L7 LoadBalancer Controller", func() {
// These variables are initialized after framework's beforeEach.
var ns string
@ -254,6 +250,12 @@ var _ = Describe("GCE L7 LoadBalancer Controller", func() {
})
AfterEach(func() {
if CurrentGinkgoTestDescription().Failed {
kubectlLogLBController(client)
Logf("\nOutput of kubectl describe ing:\n")
desc, _ := runKubectl("describe", "ing", fmt.Sprintf("--namespace=%v", ns))
Logf(desc)
}
framework.afterEach()
err := wait.Poll(lbPollInterval, lbPollTimeout, func() (bool, error) {
if err := checkLeakedResources(); err != nil {
@ -301,13 +303,17 @@ var _ = Describe("GCE L7 LoadBalancer Controller", func() {
start := time.Now()
address, err := waitForIngressAddress(client, ing.Namespace, ing.Name, lbPollTimeout)
if err != nil {
dumpDebugAndFail(fmt.Sprintf("Ingress failed to acquire an IP address within %v", lbPollTimeout), ns, client)
Failf("Ingress failed to acquire an IP address within %v", lbPollTimeout)
}
Expect(err).NotTo(HaveOccurred())
By(fmt.Sprintf("Found address %v for ingress %v, took %v to come online",
address, ing.Name, time.Since(start)))
creationTimes = append(creationTimes, time.Since(start))
if !verifyHTTPGET {
continue
}
// Check that all rules respond to a simple GET.
for _, rules := range ing.Spec.Rules {
// As of Kubernetes 1.1 we only support HTTP Ingress.
@ -330,8 +336,8 @@ var _ = Describe("GCE L7 LoadBalancer Controller", func() {
return true, nil
})
if pollErr != nil {
dumpDebugAndFail(fmt.Sprintf("Failed to execute a successful GET within %v, Last response body for %v, host %v:\n%v\n\n%v",
lbPollTimeout, route, rules.Host, lastBody, pollErr), ns, client)
Failf("Failed to execute a successful GET within %v, Last response body for %v, host %v:\n%v\n\n%v",
lbPollTimeout, route, rules.Host, lastBody, pollErr)
}
rt := time.Since(GETStart)
By(fmt.Sprintf("Route %v host %v took %v to respond", route, rules.Host, rt))
@ -345,12 +351,15 @@ var _ = Describe("GCE L7 LoadBalancer Controller", func() {
sort.Sort(timeSlice(creationTimes))
perc50 := creationTimes[len(creationTimes)/2]
if perc50 > expectedLBCreationTime {
dumpDebugAndFail(fmt.Sprintf("Average creation time is too high: %+v", creationTimes), ns, client)
Failf("Average creation time is too high: %+v", creationTimes)
}
if !verifyHTTPGET {
return
}
sort.Sort(timeSlice(responseTimes))
perc50 = responseTimes[len(responseTimes)/2]
if perc50 > expectedLBHealthCheckTime {
dumpDebugAndFail(fmt.Sprintf("Average startup time is too high: %+v", responseTimes), ns, client)
Failf("Average startup time is too high: %+v", responseTimes)
}
})
})

View File

@ -95,7 +95,7 @@ var _ = Describe("Kubectl client", func() {
defer cleanup(nautilusPath, ns, updateDemoSelector)
By("creating a replication controller")
runKubectl("create", "-f", nautilusPath, fmt.Sprintf("--namespace=%v", ns))
runKubectlOrDie("create", "-f", nautilusPath, fmt.Sprintf("--namespace=%v", ns))
validateController(c, nautilusImage, 2, "update-demo", updateDemoSelector, getUDData("nautilus.jpg", ns), ns)
})
@ -103,22 +103,22 @@ var _ = Describe("Kubectl client", func() {
defer cleanup(nautilusPath, ns, updateDemoSelector)
By("creating a replication controller")
runKubectl("create", "-f", nautilusPath, fmt.Sprintf("--namespace=%v", ns))
runKubectlOrDie("create", "-f", nautilusPath, fmt.Sprintf("--namespace=%v", ns))
validateController(c, nautilusImage, 2, "update-demo", updateDemoSelector, getUDData("nautilus.jpg", ns), ns)
By("scaling down the replication controller")
runKubectl("scale", "rc", "update-demo-nautilus", "--replicas=1", "--timeout=5m", fmt.Sprintf("--namespace=%v", ns))
runKubectlOrDie("scale", "rc", "update-demo-nautilus", "--replicas=1", "--timeout=5m", fmt.Sprintf("--namespace=%v", ns))
validateController(c, nautilusImage, 1, "update-demo", updateDemoSelector, getUDData("nautilus.jpg", ns), ns)
By("scaling up the replication controller")
runKubectl("scale", "rc", "update-demo-nautilus", "--replicas=2", "--timeout=5m", fmt.Sprintf("--namespace=%v", ns))
runKubectlOrDie("scale", "rc", "update-demo-nautilus", "--replicas=2", "--timeout=5m", fmt.Sprintf("--namespace=%v", ns))
validateController(c, nautilusImage, 2, "update-demo", updateDemoSelector, getUDData("nautilus.jpg", ns), ns)
})
It("should do a rolling update of a replication controller [Conformance]", func() {
By("creating the initial replication controller")
runKubectl("create", "-f", nautilusPath, fmt.Sprintf("--namespace=%v", ns))
runKubectlOrDie("create", "-f", nautilusPath, fmt.Sprintf("--namespace=%v", ns))
validateController(c, nautilusImage, 2, "update-demo", updateDemoSelector, getUDData("nautilus.jpg", ns), ns)
By("rolling-update to new replication controller")
runKubectl("rolling-update", "update-demo-nautilus", "--update-period=1s", "-f", kittenPath, fmt.Sprintf("--namespace=%v", ns))
runKubectlOrDie("rolling-update", "update-demo-nautilus", "--update-period=1s", "-f", kittenPath, fmt.Sprintf("--namespace=%v", ns))
validateController(c, kittenImage, 2, "update-demo", updateDemoSelector, getUDData("kitten.jpg", ns), ns)
// Everything will hopefully be cleaned up when the namespace is deleted.
})
@ -135,7 +135,7 @@ var _ = Describe("Kubectl client", func() {
defer cleanup(guestbookPath, ns, frontendSelector, redisMasterSelector, redisSlaveSelector)
By("creating all guestbook components")
runKubectl("create", "-f", guestbookPath, fmt.Sprintf("--namespace=%v", ns))
runKubectlOrDie("create", "-f", guestbookPath, fmt.Sprintf("--namespace=%v", ns))
By("validating guestbook app")
validateGuestbookApp(c, ns)
@ -148,7 +148,7 @@ var _ = Describe("Kubectl client", func() {
BeforeEach(func() {
podPath = filepath.Join(testContext.RepoRoot, "docs/user-guide/pod.yaml")
By("creating the pod")
runKubectl("create", "-f", podPath, fmt.Sprintf("--namespace=%v", ns))
runKubectlOrDie("create", "-f", podPath, fmt.Sprintf("--namespace=%v", ns))
checkPodsRunningReady(c, ns, []string{simplePodName}, podStartTimeout)
})
AfterEach(func() {
@ -157,7 +157,7 @@ var _ = Describe("Kubectl client", func() {
It("should support exec", func() {
By("executing a command in the container")
execOutput := runKubectl("exec", fmt.Sprintf("--namespace=%v", ns), simplePodName, "echo", "running", "in", "container")
execOutput := runKubectlOrDie("exec", fmt.Sprintf("--namespace=%v", ns), simplePodName, "echo", "running", "in", "container")
if e, a := "running in container", execOutput; e != a {
Failf("Unexpected kubectl exec output. Wanted %q, got %q", e, a)
}
@ -165,7 +165,7 @@ var _ = Describe("Kubectl client", func() {
By("executing a command in the container with noninteractive stdin")
execOutput = newKubectlCommand("exec", fmt.Sprintf("--namespace=%v", ns), "-i", simplePodName, "cat").
withStdinData("abcd1234").
exec()
execOrDie()
if e, a := "abcd1234", execOutput; e != a {
Failf("Unexpected kubectl exec output. Wanted %q, got %q", e, a)
}
@ -181,7 +181,7 @@ var _ = Describe("Kubectl client", func() {
By("executing a command in the container with pseudo-interactive stdin")
execOutput = newKubectlCommand("exec", fmt.Sprintf("--namespace=%v", ns), "-i", simplePodName, "bash").
withStdinReader(r).
exec()
execOrDie()
if e, a := "hi", execOutput; e != a {
Failf("Unexpected kubectl exec output. Wanted %q, got %q", e, a)
}
@ -241,7 +241,7 @@ var _ = Describe("Kubectl client", func() {
}
// start exec-proxy-tester container
netexecPodPath := filepath.Join(testContext.RepoRoot, "test/images/netexec/pod.yaml")
runKubectl("create", "-f", netexecPodPath, fmt.Sprintf("--namespace=%v", ns))
runKubectlOrDie("create", "-f", netexecPodPath, fmt.Sprintf("--namespace=%v", ns))
checkPodsRunningReady(c, ns, []string{netexecContainer}, podStartTimeout)
// Clean up
defer cleanup(netexecPodPath, ns, netexecPodSelector)
@ -311,7 +311,7 @@ var _ = Describe("Kubectl client", func() {
By("Running kubectl in netexec via an HTTP proxy using " + proxyVar)
// start the proxy container
goproxyPodPath := filepath.Join(testContext.RepoRoot, "test/images/goproxy/pod.yaml")
runKubectl("create", "-f", goproxyPodPath, fmt.Sprintf("--namespace=%v", ns))
runKubectlOrDie("create", "-f", goproxyPodPath, fmt.Sprintf("--namespace=%v", ns))
checkPodsRunningReady(c, ns, []string{goproxyContainer}, podStartTimeout)
// get the proxy address
@ -348,7 +348,7 @@ var _ = Describe("Kubectl client", func() {
// Verify the proxy server logs saw the connection
expectedProxyLog := fmt.Sprintf("Accepting CONNECT to %s", strings.TrimRight(strings.TrimLeft(testContext.Host, "https://"), "/api"))
proxyLog := runKubectl("log", "goproxy", fmt.Sprintf("--namespace=%v", ns))
proxyLog := runKubectlOrDie("log", "goproxy", fmt.Sprintf("--namespace=%v", ns))
if !strings.Contains(proxyLog, expectedProxyLog) {
Failf("Missing expected log result on proxy server for %s. Expected: %q, got %q", proxyVar, expectedProxyLog, proxyLog)
@ -364,7 +364,7 @@ var _ = Describe("Kubectl client", func() {
By("executing a command with run and attach with stdin")
runOutput := newKubectlCommand(nsFlag, "run", "run-test", "--image=busybox", "--restart=Never", "--attach=true", "--stdin", "--", "sh", "-c", "cat && echo 'stdin closed'").
withStdinData("abcd1234").
exec()
execOrDie()
Expect(runOutput).To(ContainSubstring("abcd1234"))
Expect(runOutput).To(ContainSubstring("stdin closed"))
Expect(c.Pods(ns).Delete("run-test", api.NewDeleteOptions(0))).To(BeNil())
@ -372,7 +372,7 @@ var _ = Describe("Kubectl client", func() {
By("executing a command with run and attach without stdin")
runOutput = newKubectlCommand(fmt.Sprintf("--namespace=%v", ns), "run", "run-test-2", "--image=busybox", "--restart=Never", "--attach=true", "--leave-stdin-open=true", "--", "sh", "-c", "cat && echo 'stdin closed'").
withStdinData("abcd1234").
exec()
execOrDie()
Expect(runOutput).ToNot(ContainSubstring("abcd1234"))
Expect(runOutput).To(ContainSubstring("stdin closed"))
Expect(c.Pods(ns).Delete("run-test-2", api.NewDeleteOptions(0))).To(BeNil())
@ -380,7 +380,7 @@ var _ = Describe("Kubectl client", func() {
By("executing a command with run and attach with stdin with open stdin should remain running")
runOutput = newKubectlCommand(nsFlag, "run", "run-test-3", "--image=busybox", "--restart=Never", "--attach=true", "--leave-stdin-open=true", "--stdin", "--", "sh", "-c", "cat && echo 'stdin closed'").
withStdinData("abcd1234\n").
exec()
execOrDie()
Expect(runOutput).ToNot(ContainSubstring("stdin closed"))
if !checkPodsRunningReady(c, ns, []string{"run-test-3"}, time.Minute) {
Failf("Pod %q should still be running", "run-test-3")
@ -392,7 +392,7 @@ var _ = Describe("Kubectl client", func() {
if !checkPodsRunningReady(c, ns, []string{"run-test-3"}, 1*time.Second) {
Failf("Pod %q should still be running", "run-test-3")
}
logOutput := runKubectl(nsFlag, "logs", "run-test-3")
logOutput := runKubectlOrDie(nsFlag, "logs", "run-test-3")
Expect(logOutput).ToNot(ContainSubstring("stdin closed"))
return strings.Contains(logOutput, "abcd1234"), nil
})
@ -425,7 +425,7 @@ var _ = Describe("Kubectl client", func() {
Describe("Kubectl api-versions", func() {
It("should check if v1 is in available api versions [Conformance]", func() {
By("validating api verions")
output := runKubectl("api-versions")
output := runKubectlOrDie("api-versions")
if !strings.Contains(output, "v1") {
Failf("No v1 in kubectl api-versions")
}
@ -440,12 +440,12 @@ var _ = Describe("Kubectl client", func() {
controllerJson := mkpath("redis-master-controller.json")
nsFlag := fmt.Sprintf("--namespace=%v", ns)
By("creating Redis RC")
runKubectl("create", "-f", controllerJson, nsFlag)
runKubectlOrDie("create", "-f", controllerJson, nsFlag)
By("applying a modified configuration")
stdin := modifyReplicationControllerConfiguration(controllerJson)
newKubectlCommand("apply", "-f", "-", nsFlag).
withStdinReader(stdin).
exec()
execOrDie()
By("checking the result")
forEachReplicationController(c, ns, "app", "redis", validateReplicationControllerConfiguration)
})
@ -454,7 +454,7 @@ var _ = Describe("Kubectl client", func() {
Describe("Kubectl cluster-info", func() {
It("should check if Kubernetes master services is included in cluster-info [Conformance]", func() {
By("validating cluster-info")
output := runKubectl("cluster-info")
output := runKubectlOrDie("cluster-info")
// Can't check exact strings due to terminal control commands (colors)
requiredItems := []string{"Kubernetes master", "is running at"}
if providerIs("gce", "gke") {
@ -477,12 +477,12 @@ var _ = Describe("Kubectl client", func() {
serviceJson := mkpath("redis-master-service.json")
nsFlag := fmt.Sprintf("--namespace=%v", ns)
runKubectl("create", "-f", controllerJson, nsFlag)
runKubectl("create", "-f", serviceJson, nsFlag)
runKubectlOrDie("create", "-f", controllerJson, nsFlag)
runKubectlOrDie("create", "-f", serviceJson, nsFlag)
// Pod
forEachPod(c, ns, "app", "redis", func(pod api.Pod) {
output := runKubectl("describe", "pod", pod.Name, nsFlag)
output := runKubectlOrDie("describe", "pod", pod.Name, nsFlag)
requiredStrings := [][]string{
{"Name:", "redis-master-"},
{"Namespace:", ns},
@ -498,7 +498,7 @@ var _ = Describe("Kubectl client", func() {
})
// Rc
output := runKubectl("describe", "rc", "redis-master", nsFlag)
output := runKubectlOrDie("describe", "rc", "redis-master", nsFlag)
requiredStrings := [][]string{
{"Name:", "redis-master"},
{"Namespace:", ns},
@ -511,7 +511,7 @@ var _ = Describe("Kubectl client", func() {
checkOutput(output, requiredStrings)
// Service
output = runKubectl("describe", "service", "redis-master", nsFlag)
output = runKubectlOrDie("describe", "service", "redis-master", nsFlag)
requiredStrings = [][]string{
{"Name:", "redis-master"},
{"Namespace:", ns},
@ -528,7 +528,7 @@ var _ = Describe("Kubectl client", func() {
nodes, err := c.Nodes().List(labels.Everything(), fields.Everything())
Expect(err).NotTo(HaveOccurred())
node := nodes.Items[0]
output = runKubectl("describe", "node", node.Name)
output = runKubectlOrDie("describe", "node", node.Name)
requiredStrings = [][]string{
{"Name:", node.Name},
{"Labels:"},
@ -547,7 +547,7 @@ var _ = Describe("Kubectl client", func() {
checkOutput(output, requiredStrings)
// Namespace
output = runKubectl("describe", "namespace", ns)
output = runKubectlOrDie("describe", "namespace", ns)
requiredStrings = [][]string{
{"Name:", ns},
{"Labels:"},
@ -569,7 +569,7 @@ var _ = Describe("Kubectl client", func() {
redisPort := 6379
By("creating Redis RC")
runKubectl("create", "-f", controllerJson, nsFlag)
runKubectlOrDie("create", "-f", controllerJson, nsFlag)
forEachPod(c, ns, "app", "redis", func(pod api.Pod) {
lookForStringInLog(ns, pod.Name, "redis-master", "The server is now ready to accept connections", podStartTimeout)
})
@ -617,12 +617,12 @@ var _ = Describe("Kubectl client", func() {
}
By("exposing RC")
runKubectl("expose", "rc", "redis-master", "--name=rm2", "--port=1234", fmt.Sprintf("--target-port=%d", redisPort), nsFlag)
runKubectlOrDie("expose", "rc", "redis-master", "--name=rm2", "--port=1234", fmt.Sprintf("--target-port=%d", redisPort), nsFlag)
waitForService(c, ns, "rm2", true, poll, serviceStartTimeout)
validateService("rm2", 1234, serviceStartTimeout)
By("exposing service")
runKubectl("expose", "service", "rm2", "--name=rm3", "--port=2345", fmt.Sprintf("--target-port=%d", redisPort), nsFlag)
runKubectlOrDie("expose", "service", "rm2", "--name=rm3", "--port=2345", fmt.Sprintf("--target-port=%d", redisPort), nsFlag)
waitForService(c, ns, "rm3", true, poll, serviceStartTimeout)
validateService("rm3", 2345, serviceStartTimeout)
})
@ -635,7 +635,7 @@ var _ = Describe("Kubectl client", func() {
podPath = filepath.Join(testContext.RepoRoot, "docs/user-guide/pod.yaml")
By("creating the pod")
nsFlag = fmt.Sprintf("--namespace=%v", ns)
runKubectl("create", "-f", podPath, nsFlag)
runKubectlOrDie("create", "-f", podPath, nsFlag)
checkPodsRunningReady(c, ns, []string{simplePodName}, podStartTimeout)
})
AfterEach(func() {
@ -647,17 +647,17 @@ var _ = Describe("Kubectl client", func() {
labelValue := "testing-label-value"
By("adding the label " + labelName + " with value " + labelValue + " to a pod")
runKubectl("label", "pods", simplePodName, labelName+"="+labelValue, nsFlag)
runKubectlOrDie("label", "pods", simplePodName, labelName+"="+labelValue, nsFlag)
By("verifying the pod has the label " + labelName + " with the value " + labelValue)
output := runKubectl("get", "pod", simplePodName, "-L", labelName, nsFlag)
output := runKubectlOrDie("get", "pod", simplePodName, "-L", labelName, nsFlag)
if !strings.Contains(output, labelValue) {
Failf("Failed updating label " + labelName + " to the pod " + simplePodName)
}
By("removing the label " + labelName + " of a pod")
runKubectl("label", "pods", simplePodName, labelName+"-", nsFlag)
runKubectlOrDie("label", "pods", simplePodName, labelName+"-", nsFlag)
By("verifying the pod doesn't have the label " + labelName)
output = runKubectl("get", "pod", simplePodName, "-L", labelName, nsFlag)
output = runKubectlOrDie("get", "pod", simplePodName, "-L", labelName, nsFlag)
if strings.Contains(output, labelValue) {
Failf("Failed removing label " + labelName + " of the pod " + simplePodName)
}
@ -675,7 +675,7 @@ var _ = Describe("Kubectl client", func() {
rcPath = mkpath("redis-master-controller.json")
By("creating an rc")
nsFlag = fmt.Sprintf("--namespace=%v", ns)
runKubectl("create", "-f", rcPath, nsFlag)
runKubectlOrDie("create", "-f", rcPath, nsFlag)
})
AfterEach(func() {
cleanup(rcPath, ns, simplePodSelector)
@ -688,17 +688,17 @@ var _ = Describe("Kubectl client", func() {
Expect(err).NotTo(HaveOccurred())
By("limiting log lines")
out := runKubectl("log", pod.Name, containerName, nsFlag, "--tail=1")
out := runKubectlOrDie("log", pod.Name, containerName, nsFlag, "--tail=1")
Expect(len(out)).NotTo(BeZero())
Expect(len(strings.Split(out, "\n"))).To(Equal(1))
By("limiting log bytes")
out = runKubectl("log", pod.Name, containerName, nsFlag, "--limit-bytes=1")
out = runKubectlOrDie("log", pod.Name, containerName, nsFlag, "--limit-bytes=1")
Expect(len(strings.Split(out, "\n"))).To(Equal(1))
Expect(len(out)).To(Equal(1))
By("exposing timestamps")
out = runKubectl("log", pod.Name, containerName, nsFlag, "--tail=1", "--timestamps")
out = runKubectlOrDie("log", pod.Name, containerName, nsFlag, "--tail=1", "--timestamps")
lines := strings.Split(out, "\n")
Expect(len(lines)).To(Equal(1))
words := strings.Split(lines[0], " ")
@ -711,9 +711,9 @@ var _ = Describe("Kubectl client", func() {
By("restricting to a time range")
time.Sleep(1500 * time.Millisecond) // ensure that startup logs on the node are seen as older than 1s
out = runKubectl("log", pod.Name, containerName, nsFlag, "--since=1s")
out = runKubectlOrDie("log", pod.Name, containerName, nsFlag, "--since=1s")
recent := len(strings.Split(out, "\n"))
out = runKubectl("log", pod.Name, containerName, nsFlag, "--since=24h")
out = runKubectlOrDie("log", pod.Name, containerName, nsFlag, "--since=24h")
older := len(strings.Split(out, "\n"))
Expect(recent).To(BeNumerically("<", older))
})
@ -728,10 +728,10 @@ var _ = Describe("Kubectl client", func() {
controllerJson := mkpath("redis-master-controller.json")
nsFlag := fmt.Sprintf("--namespace=%v", ns)
By("creating Redis RC")
runKubectl("create", "-f", controllerJson, nsFlag)
runKubectlOrDie("create", "-f", controllerJson, nsFlag)
By("patching all pods")
forEachPod(c, ns, "app", "redis", func(pod api.Pod) {
runKubectl("patch", "pod", pod.Name, nsFlag, "-p", "{\"metadata\":{\"annotations\":{\"x\":\"y\"}}}")
runKubectlOrDie("patch", "pod", pod.Name, nsFlag, "-p", "{\"metadata\":{\"annotations\":{\"x\":\"y\"}}}")
})
By("checking annotations")
@ -751,7 +751,7 @@ var _ = Describe("Kubectl client", func() {
Describe("Kubectl version", func() {
It("should check is all data is printed [Conformance]", func() {
version := runKubectl("version")
version := runKubectlOrDie("version")
requiredItems := []string{"Client Version:", "Server Version:", "Major:", "Minor:", "GitCommit:"}
for _, item := range requiredItems {
if !strings.Contains(version, item) {
@ -771,14 +771,14 @@ var _ = Describe("Kubectl client", func() {
})
AfterEach(func() {
runKubectl("stop", "rc", rcName, nsFlag)
runKubectlOrDie("stop", "rc", rcName, nsFlag)
})
It("should create an rc from an image [Conformance]", func() {
image := "nginx"
By("running the image " + image)
runKubectl("run", rcName, "--image="+image, nsFlag)
runKubectlOrDie("run", rcName, "--image="+image, nsFlag)
By("verifying the rc " + rcName + " was created")
rc, err := c.ReplicationControllers(ns).Get(rcName)
if err != nil {
@ -797,7 +797,7 @@ var _ = Describe("Kubectl client", func() {
}
pods := podlist.Items
if pods == nil || len(pods) != 1 || len(pods[0].Spec.Containers) != 1 || pods[0].Spec.Containers[0].Image != image {
runKubectl("get", "pods", "-L", "run", nsFlag)
runKubectlOrDie("get", "pods", "-L", "run", nsFlag)
Failf("Failed creating 1 pod with expected image %s. Number of pods = %v", image, len(pods))
}
})
@ -814,14 +814,14 @@ var _ = Describe("Kubectl client", func() {
})
AfterEach(func() {
runKubectl("stop", "pods", podName, nsFlag)
runKubectlOrDie("stop", "pods", podName, nsFlag)
})
It("should create a pod from an image when restart is OnFailure [Conformance]", func() {
image := "nginx"
By("running the image " + image)
runKubectl("run", podName, "--restart=OnFailure", "--image="+image, nsFlag)
runKubectlOrDie("run", podName, "--restart=OnFailure", "--image="+image, nsFlag)
By("verifying the pod " + podName + " was created")
pod, err := c.Pods(ns).Get(podName)
if err != nil {
@ -840,7 +840,7 @@ var _ = Describe("Kubectl client", func() {
image := "nginx"
By("running the image " + image)
runKubectl("run", podName, "--restart=Never", "--image="+image, nsFlag)
runKubectlOrDie("run", podName, "--restart=Never", "--image="+image, nsFlag)
By("verifying the pod " + podName + " was created")
pod, err := c.Pods(ns).Get(podName)
if err != nil {

View File

@ -116,7 +116,7 @@ func runKubectlWithTimeout(timeout time.Duration, args ...string) string {
logOutput := make(chan string)
go func() {
defer GinkgoRecover()
logOutput <- runKubectl(args...)
logOutput <- runKubectlOrDie(args...)
}()
select {
case <-time.After(timeout):

View File

@ -926,14 +926,14 @@ func cleanup(filePath string, ns string, selectors ...string) {
if ns != "" {
nsArg = fmt.Sprintf("--namespace=%s", ns)
}
runKubectl("stop", "--grace-period=0", "-f", filePath, nsArg)
runKubectlOrDie("stop", "--grace-period=0", "-f", filePath, nsArg)
for _, selector := range selectors {
resources := runKubectl("get", "rc,svc", "-l", selector, "--no-headers", nsArg)
resources := runKubectlOrDie("get", "rc,svc", "-l", selector, "--no-headers", nsArg)
if resources != "" {
Failf("Resources left running after stop:\n%s", resources)
}
pods := runKubectl("get", "pods", "-l", selector, nsArg, "-o", "go-template={{ range .items }}{{ if not .metadata.deletionTimestamp }}{{ .metadata.name }}{{ \"\\n\" }}{{ end }}{{ end }}")
pods := runKubectlOrDie("get", "pods", "-l", selector, nsArg, "-o", "go-template={{ range .items }}{{ if not .metadata.deletionTimestamp }}{{ .metadata.name }}{{ \"\\n\" }}{{ end }}{{ end }}")
if pods != "" {
Failf("Pods left unterminated after stop:\n%s", pods)
}
@ -967,7 +967,7 @@ func validateController(c *client.Client, containerImage string, replicas int, c
By(fmt.Sprintf("waiting for all containers in %s pods to come up.", testname)) //testname should be selector
waitLoop:
for start := time.Now(); time.Since(start) < podStartTimeout; time.Sleep(5 * time.Second) {
getPodsOutput := runKubectl("get", "pods", "-o", "template", getPodsTemplate, "--api-version=v1", "-l", testname, fmt.Sprintf("--namespace=%v", ns))
getPodsOutput := runKubectlOrDie("get", "pods", "-o", "template", getPodsTemplate, "--api-version=v1", "-l", testname, fmt.Sprintf("--namespace=%v", ns))
pods := strings.Fields(getPodsOutput)
if numPods := len(pods); numPods != replicas {
By(fmt.Sprintf("Replicas for %s: expected=%d actual=%d", testname, replicas, numPods))
@ -975,13 +975,13 @@ waitLoop:
}
var runningPods []string
for _, podID := range pods {
running := runKubectl("get", "pods", podID, "-o", "template", getContainerStateTemplate, "--api-version=v1", fmt.Sprintf("--namespace=%v", ns))
running := runKubectlOrDie("get", "pods", podID, "-o", "template", getContainerStateTemplate, "--api-version=v1", fmt.Sprintf("--namespace=%v", ns))
if running != "true" {
Logf("%s is created but not running", podID)
continue waitLoop
}
currentImage := runKubectl("get", "pods", podID, "-o", "template", getImageTemplate, "--api-version=v1", fmt.Sprintf("--namespace=%v", ns))
currentImage := runKubectlOrDie("get", "pods", podID, "-o", "template", getImageTemplate, "--api-version=v1", fmt.Sprintf("--namespace=%v", ns))
if currentImage != containerImage {
Logf("%s is created but running wrong image; expected: %s, actual: %s", podID, containerImage, currentImage)
continue waitLoop
@ -1062,29 +1062,39 @@ func (b kubectlBuilder) withStdinReader(reader io.Reader) *kubectlBuilder {
return &b
}
func (b kubectlBuilder) exec() string {
func (b kubectlBuilder) execOrDie() string {
str, err := b.exec()
Expect(err).NotTo(HaveOccurred())
return str
}
func (b kubectlBuilder) exec() (string, error) {
var stdout, stderr bytes.Buffer
cmd := b.cmd
cmd.Stdout, cmd.Stderr = &stdout, &stderr
Logf("Running '%s %s'", cmd.Path, strings.Join(cmd.Args[1:], " ")) // skip arg[0] as it is printed separately
if err := cmd.Run(); err != nil {
Failf("Error running %v:\nCommand stdout:\n%v\nstderr:\n%v\n", cmd, cmd.Stdout, cmd.Stderr)
return ""
return "", fmt.Errorf("Error running %v:\nCommand stdout:\n%v\nstderr:\n%v\n", cmd, cmd.Stdout, cmd.Stderr)
}
Logf(stdout.String())
// TODO: trimspace should be unnecessary after switching to use kubectl binary directly
return strings.TrimSpace(stdout.String())
return strings.TrimSpace(stdout.String()), nil
}
// runKubectlOrDie is a convenience wrapper over kubectlBuilder
func runKubectlOrDie(args ...string) string {
return newKubectlCommand(args...).execOrDie()
}
// runKubectl is a convenience wrapper over kubectlBuilder
func runKubectl(args ...string) string {
func runKubectl(args ...string) (string, error) {
return newKubectlCommand(args...).exec()
}
// runKubectlInput is a convenience wrapper over kubectlBuilder that takes input to stdin
func runKubectlInput(data string, args ...string) string {
return newKubectlCommand(args...).withStdinData(data).exec()
// runKubectlOrDieInput is a convenience wrapper over kubectlBuilder that takes input to stdin
func runKubectlOrDieInput(data string, args ...string) string {
return newKubectlCommand(args...).withStdinData(data).execOrDie()
}
func startCmdAndStreamOutput(cmd *exec.Cmd) (stdout, stderr io.ReadCloser, err error) {
@ -1973,7 +1983,7 @@ func NewHostExecPodSpec(ns, name string) *api.Pod {
// RunHostCmd runs the given cmd in the context of the given pod using `kubectl exec`
// inside of a shell.
func RunHostCmd(ns, name, cmd string) string {
return runKubectl("exec", fmt.Sprintf("--namespace=%v", ns), name, "--", "/bin/sh", "-c", cmd)
return runKubectlOrDie("exec", fmt.Sprintf("--namespace=%v", ns), name, "--", "/bin/sh", "-c", cmd)
}
// LaunchHostExecPod launches a hostexec pod in the given namespace and waits