From eb43cbb5dc22ab0aa4403567815521f0a62244a6 Mon Sep 17 00:00:00 2001 From: Marek Biskup Date: Wed, 15 Jul 2015 09:52:15 +0200 Subject: [PATCH 1/2] e2e: reading stdin in kubectl --- test/e2e/util.go | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/test/e2e/util.go b/test/e2e/util.go index cb714030f85..3fe82d3f3a7 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -858,9 +858,26 @@ func kubectlCmd(args ...string) *exec.Cmd { return cmd } -func runKubectl(args ...string) string { +// kubectlBuilder is used to build, custimize and execute a kubectl Command. +// Add more functions to customize the builder as needed. +type kubectlBuilder struct { + cmd *exec.Cmd +} + +func newKubectlCommand(args ...string) *kubectlBuilder { + b := new(kubectlBuilder) + b.cmd = kubectlCmd(args...) + return b +} + +func (b kubectlBuilder) withStdinData(data string) *kubectlBuilder { + b.cmd.Stdin = strings.NewReader(data) + return &b +} + +func (b kubectlBuilder) exec() string { var stdout, stderr bytes.Buffer - cmd := kubectlCmd(args...) + cmd := b.cmd cmd.Stdout, cmd.Stderr = &stdout, &stderr Logf("Running '%s %s'", cmd.Path, strings.Join(cmd.Args, " ")) @@ -873,6 +890,11 @@ func runKubectl(args ...string) string { return strings.TrimSpace(stdout.String()) } +// runKubectl is a convenience wrapper over kubectlBuilder +func runKubectl(args ...string) string { + return newKubectlCommand(args...).exec() +} + func startCmdAndStreamOutput(cmd *exec.Cmd) (stdout, stderr io.ReadCloser, err error) { stdout, err = cmd.StdoutPipe() if err != nil { From 860822431bf3d0eb633ddd68fe7d8f2a47d19d20 Mon Sep 17 00:00:00 2001 From: Marek Biskup Date: Fri, 10 Jul 2015 11:54:54 +0200 Subject: [PATCH 2/2] e2e test for dns example --- test/e2e/examples.go | 134 ++++++++++++++++++++++++++++++++++++++- test/e2e/resize_nodes.go | 8 --- test/e2e/util.go | 36 ++++++++++- 3 files changed, 168 insertions(+), 10 deletions(-) diff --git a/test/e2e/examples.go b/test/e2e/examples.go index 902a6d891fb..1690edae4ee 100644 --- a/test/e2e/examples.go +++ b/test/e2e/examples.go @@ -18,6 +18,8 @@ package e2e import ( "fmt" + "io/ioutil" + "os" "path/filepath" "strings" "time" @@ -34,8 +36,17 @@ import ( const ( podListTimeout = time.Minute serverStartTimeout = podStartTimeout + 3*time.Minute + dnsReadyTimeout = time.Minute ) +const queryDnsPythonTemplate string = ` +import socket +try: + socket.gethostbyname('%s') + print 'ok' +except: + print 'err'` + var _ = Describe("Examples e2e", func() { var c *client.Client var ns string @@ -412,6 +423,103 @@ var _ = Describe("Examples e2e", func() { }) }) }) + + Describe("[Example]ClusterDns", func() { + It("should create pod that uses dns", func() { + mkpath := func(file string) string { + return filepath.Join(testContext.RepoRoot, "examples/cluster-dns", file) + } + + // contrary to the example, this test does not use contexts, for simplicity + // namespaces are passed directly. + // Also, for simplicity, we don't use yamls with namespaces, but we + // create testing namespaces instead. + + backendRcYaml := mkpath("dns-backend-rc.yaml") + backendRcName := "dns-backend" + backendSvcYaml := mkpath("dns-backend-service.yaml") + backendSvcName := "dns-backend" + backendPodName := "dns-backend" + frontendPodYaml := mkpath("dns-frontend-pod.yaml") + frontendPodName := "dns-frontend" + frontendPodContainerName := "dns-frontend" + + podOutput := "Hello World!" + + // we need two namespaces anyway, so let's forget about + // the one created in BeforeEach and create two new ones. + namespaces := []*api.Namespace{nil, nil} + for i := range namespaces { + var err error + namespaces[i], err = createTestingNS(fmt.Sprintf("dnsexample%d", i), c) + if namespaces[i] != nil { + defer c.Namespaces().Delete(namespaces[i].Name) + } + Expect(err).NotTo(HaveOccurred()) + } + + for _, ns := range namespaces { + runKubectl("create", "-f", backendRcYaml, getNsCmdFlag(ns)) + } + + for _, ns := range namespaces { + runKubectl("create", "-f", backendSvcYaml, getNsCmdFlag(ns)) + } + + // wait for objects + for _, ns := range namespaces { + waitForRCPodsRunning(c, ns.Name, backendRcName) + waitForService(c, ns.Name, backendSvcName, true, poll, serviceStartTimeout) + } + // it is not enough that pods are running because they may be set to running, but + // the application itself may have not been initialized. Just query the application. + for _, ns := range namespaces { + label := labels.SelectorFromSet(labels.Set(map[string]string{"name": backendRcName})) + pods, err := c.Pods(ns.Name).List(label, fields.Everything()) + Expect(err).NotTo(HaveOccurred()) + err = podsResponding(c, ns.Name, backendPodName, false, pods) + Expect(err).NotTo(HaveOccurred(), "waiting for all pods to respond") + Logf("found %d backend pods responding in namespace %s", len(pods.Items), ns.Name) + + err = serviceResponding(c, ns.Name, backendSvcName) + Expect(err).NotTo(HaveOccurred(), "waiting for the service to respond") + } + + // Now another tricky part: + // It may happen that the service name is not yet in DNS. + // So if we start our pod, it will fail. We must make sure + // the name is already resolvable. So let's try to query DNS from + // the pod we have, until we find our service name. + // This complicated code may be removed if the pod itself retried after + // dns error or timeout. + // This code is probably unnecessary, but let's stay on the safe side. + label := labels.SelectorFromSet(labels.Set(map[string]string{"name": backendPodName})) + pods, err := c.Pods(namespaces[0].Name).List(label, fields.Everything()) + + if err != nil || pods == nil || len(pods.Items) == 0 { + Failf("no running pods found") + } + podName := pods.Items[0].Name + + queryDns := fmt.Sprintf(queryDnsPythonTemplate, backendSvcName+"."+namespaces[0].Name) + _, err = lookForStringInPodExec(namespaces[0].Name, podName, []string{"python", "-c", queryDns}, "ok", dnsReadyTimeout) + Expect(err).NotTo(HaveOccurred(), "waiting for output from pod exec") + + updatedPodYaml := prepareResourceWithReplacedString(frontendPodYaml, "dns-backend.development.cluster.local", fmt.Sprintf("dns-backend.%s.cluster.local", namespaces[0].Name)) + + // create a pod in each namespace + for _, ns := range namespaces { + newKubectlCommand("create", "-f", "-", getNsCmdFlag(ns)).withStdinData(updatedPodYaml).exec() + } + // remember that we cannot wait for the pods to be running because our pods terminate by themselves. + + // wait for pods to print their result + for _, ns := range namespaces { + _, err := lookForStringInLog(ns.Name, frontendPodName, frontendPodContainerName, podOutput, podStartTimeout) + Expect(err).NotTo(HaveOccurred()) + } + }) + }) }) func makeHttpRequestToService(c *client.Client, ns, service, path string) (string, error) { @@ -426,6 +534,21 @@ func makeHttpRequestToService(c *client.Client, ns, service, path string) (strin return string(result), err } +func getNsCmdFlag(ns *api.Namespace) string { + return fmt.Sprintf("--namespace=%v", ns.Name) +} + +// pass enough context with the 'old' parameter so that it replaces what your really intended. +func prepareResourceWithReplacedString(inputFile, old, new string) string { + f, err := os.Open(inputFile) + Expect(err).NotTo(HaveOccurred()) + defer f.Close() + data, err := ioutil.ReadAll(f) + Expect(err).NotTo(HaveOccurred()) + podYaml := strings.Replace(string(data), old, new, 1) + return podYaml +} + func forEachPod(c *client.Client, ns, selectorKey, selectorValue string, fn func(api.Pod)) { var pods *api.PodList var err error @@ -458,6 +581,15 @@ func lookForStringInFile(ns, podName, container, file, expectedString string, ti }) } +func lookForStringInPodExec(ns, podName string, command []string, expectedString string, timeout time.Duration) (result string, err error) { + return lookForString(expectedString, timeout, func() string { + // use the first container + args := []string{"exec", podName, fmt.Sprintf("--namespace=%v", ns), "--"} + args = append(args, command...) + return runKubectl(args...) + }) +} + // Looks for the given string in the output of fn, repeatedly calling fn until // the timeout is reached or the string is found. Returns last log and possibly // error if the string was not found. @@ -468,6 +600,6 @@ func lookForString(expectedString string, timeout time.Duration, fn func() strin return } } - err = fmt.Errorf("Failed to find \"%s\"", expectedString) + err = fmt.Errorf("Failed to find \"%s\", last result: \"%s\"", expectedString, result) return } diff --git a/test/e2e/resize_nodes.go b/test/e2e/resize_nodes.go index f3488187cdf..4f91c51a649 100644 --- a/test/e2e/resize_nodes.go +++ b/test/e2e/resize_nodes.go @@ -285,14 +285,6 @@ func podsRunning(c *client.Client, pods *api.PodList) []error { return e } -func podsResponding(c *client.Client, ns, name string, wantName bool, pods *api.PodList) error { - By("trying to dial each unique pod") - retryTimeout := 2 * time.Minute - retryInterval := 5 * time.Second - label := labels.SelectorFromSet(labels.Set(map[string]string{"name": name})) - return wait.Poll(retryInterval, retryTimeout, podResponseChecker{c, ns, label, name, wantName, pods}.checkAllResponses) -} - func verifyPods(c *client.Client, ns, name string, wantName bool, replicas int) error { pods, err := podsCreated(c, ns, name, replicas) if err != nil { diff --git a/test/e2e/util.go b/test/e2e/util.go index 3fe82d3f3a7..836128dae16 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -85,6 +85,9 @@ const ( // be "ready" before the test starts, so this is small. podReadyBeforeTimeout = 20 * time.Second + podRespondingTimeout = 2 * time.Minute + serviceRespondingTimeout = 2 * time.Minute + // How wide to print pod names, by default. Useful for aligning printing to // quickly scan through output. podPrintWidth = 55 @@ -697,6 +700,37 @@ func (r podResponseChecker) checkAllResponses() (done bool, err error) { return true, nil } +func podsResponding(c *client.Client, ns, name string, wantName bool, pods *api.PodList) error { + By("trying to dial each unique pod") + label := labels.SelectorFromSet(labels.Set(map[string]string{"name": name})) + return wait.Poll(poll, podRespondingTimeout, podResponseChecker{c, ns, label, name, wantName, pods}.checkAllResponses) +} + +func serviceResponding(c *client.Client, ns, name string) error { + By(fmt.Sprintf("trying to dial the service %s.%s via the proxy", ns, name)) + + return wait.Poll(poll, serviceRespondingTimeout, func() (done bool, err error) { + body, err := c.Get(). + Prefix("proxy"). + Namespace(ns). + Resource("services"). + Name(name). + Do(). + Raw() + if err != nil { + Logf("Failed to GET from service %s: %v:", name, err) + return false, nil + } + got := string(body) + if len(got) == 0 { + Logf("Service %s: expected non-empty response", name) + return false, err // stop polling + } + Logf("Service %s: found nonempty answer: %s", name, got) + return true, nil + }) +} + func loadConfig() (*client.Config, error) { switch { case testContext.KubeConfig != "": @@ -880,7 +914,7 @@ func (b kubectlBuilder) exec() string { cmd := b.cmd cmd.Stdout, cmd.Stderr = &stdout, &stderr - Logf("Running '%s %s'", cmd.Path, strings.Join(cmd.Args, " ")) + Logf("Running '%s %s'", cmd.Path, strings.Join(cmd.Args[1:], " ")) // skip arg[0] as it is printed separately if err := cmd.Run(); err != nil { Failf("Error running %v:\nCommand stdout:\n%v\nstderr:\n%v\n", cmd, cmd.Stdout, cmd.Stderr) return ""