From 6db95790c5df5eade18a091b07aec9d1b474550d Mon Sep 17 00:00:00 2001 From: Robert Bailey Date: Thu, 26 Mar 2015 12:22:04 -0700 Subject: [PATCH] Revert "Refactor kubectl methods into utility class, so that we can easily integrate new tests and kube files written in json or other formats." --- test/e2e/kubectl.go | 142 +++++++++++++++++++++++++++++++++++++------- test/e2e/util.go | 117 ------------------------------------ 2 files changed, 122 insertions(+), 137 deletions(-) diff --git a/test/e2e/kubectl.go b/test/e2e/kubectl.go index ba8c0872c92..805abd43b83 100644 --- a/test/e2e/kubectl.go +++ b/test/e2e/kubectl.go @@ -17,13 +17,16 @@ limitations under the License. package e2e import ( + "bytes" "encoding/json" - "errors" + "fmt" + "os/exec" "path/filepath" "strings" "time" "github.com/GoogleCloudPlatform/kubernetes/pkg/client" + "github.com/GoogleCloudPlatform/kubernetes/pkg/client/clientcmd" . "github.com/onsi/ginkgo" ) @@ -62,7 +65,7 @@ var _ = Describe("kubectl", func() { By("creating a replication controller") runKubectl("create", "-f", nautilusPath) - validateController(c, nautilusImage, 2, "update-demo", updateDemoSelector, getUDData) + validateController(c, nautilusImage, 2) }) It("should scale a replication controller", func() { @@ -70,13 +73,13 @@ var _ = Describe("kubectl", func() { By("creating a replication controller") runKubectl("create", "-f", nautilusPath) - validateController(c, nautilusImage, 2, "update-demo", updateDemoSelector, getUDData) + validateController(c, nautilusImage, 2) By("scaling down the replication controller") runKubectl("resize", "rc", "update-demo-nautilus", "--replicas=1") - validateController(c, nautilusImage, 1, "update-demo", updateDemoSelector, getUDData) + validateController(c, nautilusImage, 1) By("scaling up the replication controller") runKubectl("resize", "rc", "update-demo-nautilus", "--replicas=2") - validateController(c, nautilusImage, 2, "update-demo", updateDemoSelector, getUDData) + validateController(c, nautilusImage, 2) }) It("should do a rolling update of a replication controller", func() { @@ -85,10 +88,10 @@ var _ = Describe("kubectl", func() { By("creating the initial replication controller") runKubectl("create", "-f", nautilusPath) - validateController(c, nautilusImage, 2, "update-demo", updateDemoSelector, getUDData) + validateController(c, nautilusImage, 2) By("rollingupdate to new replication controller") runKubectl("rollingupdate", "update-demo-nautilus", "--update-period=1s", "-f", kittenPath) - validateController(c, kittenImage, 2, "update-demo", updateDemoSelector, getUDData) + validateController(c, kittenImage, 2) }) }) @@ -150,12 +153,82 @@ func makeRequestToGuestbook(c *client.Client, cmd, value string) (string, error) return string(result), err } +func cleanup(filePath string, selectors ...string) { + By("using stop to clean up resources") + runKubectl("stop", "-f", filePath) + + for _, selector := range selectors { + resources := runKubectl("get", "pods,rc,se", "-l", selector, "--no-headers") + if resources != "" { + Failf("Resources left running after stop:\n%s", resources) + } + } +} + +func validateController(c *client.Client, image string, replicas int) { + + getPodsTemplate := "--template={{range.items}}{{.id}} {{end}}" + + // NB: kubectl adds the "exists" function to the standard template functions. + // This lets us check to see if the "running" entry exists for each of the containers + // we care about. Exists will never return an error and it's safe to check a chain of + // things, any one of which may not exist. In the below template, all of info, + // containername, and running might be nil, so the normal index function isn't very + // helpful. + // This template is unit-tested in kubectl, so if you change it, update the unit test. + // + // You can read about the syntax here: http://golang.org/pkg/text/template/ + getContainerStateTemplate := fmt.Sprintf(`--template={{and (exists . "currentState" "info" "%s" "state" "running")}}`, updateDemoContainer) + + getImageTemplate := fmt.Sprintf(`--template={{(index .currentState.info "%s").image}}`, updateDemoContainer) + + By(fmt.Sprintf("waiting for all containers in %s pods to come up.", updateDemoSelector)) + for start := time.Now(); time.Since(start) < podStartTimeout; time.Sleep(5 * time.Second) { + getPodsOutput := runKubectl("get", "pods", "-o", "template", getPodsTemplate, "-l", updateDemoSelector) + pods := strings.Fields(getPodsOutput) + if numPods := len(pods); numPods != replicas { + By(fmt.Sprintf("Replicas for %s: expected=%d actual=%d", updateDemoSelector, replicas, numPods)) + continue + } + var runningPods []string + for _, podID := range pods { + running := runKubectl("get", "pods", podID, "-o", "template", getContainerStateTemplate) + if running == "false" { + Logf("%s is created but not running", podID) + continue + } + + currentImage := runKubectl("get", "pods", podID, "-o", "template", getImageTemplate) + if currentImage != image { + Logf("%s is created but running wrong image; expected: %s, actual: %s", podID, image, currentImage) + continue + } + + data, err := getData(c, podID) + if err != nil { + Logf("%s is running right image but fetching data failed: %v", podID, err) + continue + } + if strings.Contains(data.image, image) { + Logf("%s is running right image but fetched data has the wrong info: %s", podID, data) + continue + } + + Logf("%s is verified up and running", podID) + runningPods = append(runningPods, podID) + } + if len(runningPods) == replicas { + return + } + } + Failf("Timed out after %v seconds waiting for %s pods to reach valid state", podStartTimeout.Seconds(), updateDemoSelector) +} + type updateDemoData struct { image string `json:"image"` } -// getUDData validates data.json in the update-demo (returns nil if data is ok). -func getUDData(c *client.Client, podID string) error { +func getData(c *client.Client, podID string) (*updateDemoData, error) { body, err := c.Get(). Prefix("proxy"). Resource("pods"). @@ -164,18 +237,47 @@ func getUDData(c *client.Client, podID string) error { Do(). Raw() if err != nil { - return err + return nil, err } Logf("got data: %s", body) var data updateDemoData - - if err := json.Unmarshal(body, &data); err != nil { - return err - } - - if strings.Contains(data.image, "update-demo") { - return nil - } else { - return errors.New("data served up in container is innaccurate") - } + err = json.Unmarshal(body, &data) + return &data, err +} + +func kubectlCmd(args ...string) *exec.Cmd { + defaultArgs := []string{} + if testContext.kubeConfig != "" { + defaultArgs = append(defaultArgs, "--"+clientcmd.RecommendedConfigPathFlag+"="+testContext.kubeConfig) + } else { + defaultArgs = append(defaultArgs, "--"+clientcmd.FlagAuthPath+"="+testContext.authConfig) + if testContext.certDir != "" { + defaultArgs = append(defaultArgs, + fmt.Sprintf("--certificate-authority=%s", filepath.Join(testContext.certDir, "ca.crt")), + fmt.Sprintf("--client-certificate=%s", filepath.Join(testContext.certDir, "kubecfg.crt")), + fmt.Sprintf("--client-key=%s", filepath.Join(testContext.certDir, "kubecfg.key"))) + } + } + kubectlArgs := append(defaultArgs, args...) + // TODO: Remove this once gcloud writes a proper entry in the kubeconfig file. + if testContext.provider == "gke" { + kubectlArgs = append(kubectlArgs, "--server="+testContext.host) + } + cmd := exec.Command("kubectl", kubectlArgs...) + Logf("Running '%s %s'", cmd.Path, strings.Join(cmd.Args, " ")) + return cmd +} + +func runKubectl(args ...string) string { + var stdout, stderr bytes.Buffer + cmd := kubectlCmd(args...) + cmd.Stdout, cmd.Stderr = &stdout, &stderr + + if err := cmd.Run(); err != nil { + Failf("Error running %v:\nCommand stdout:\n%v\nstderr:\n%v\n", cmd, cmd.Stdout, cmd.Stderr) + return "" + } + Logf(stdout.String()) + // TODO: trimspace should be unnecessary after switching to use kubectl binary directly + return strings.TrimSpace(stdout.String()) } diff --git a/test/e2e/util.go b/test/e2e/util.go index 5fa5acec780..93d846f7ed8 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -17,20 +17,16 @@ limitations under the License. package e2e import ( - "bytes" "fmt" "math/rand" - "os/exec" "path/filepath" "strconv" - "strings" "time" "github.com/GoogleCloudPlatform/kubernetes/pkg/api" "github.com/GoogleCloudPlatform/kubernetes/pkg/client" "github.com/GoogleCloudPlatform/kubernetes/pkg/client/clientcmd" "github.com/GoogleCloudPlatform/kubernetes/pkg/clientauth" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) @@ -186,116 +182,3 @@ func randomSuffix() string { func expectNoError(err error, explain ...interface{}) { ExpectWithOffset(1, err).NotTo(HaveOccurred(), explain...) } - -func cleanup(filePath string, selectors ...string) { - By("using stop to clean up resources") - runKubectl("stop", "-f", filePath) - - for _, selector := range selectors { - resources := runKubectl("get", "pods,rc,se", "-l", selector, "--no-headers") - if resources != "" { - Failf("Resources left running after stop:\n%s", resources) - } - } -} - -// validatorFn is the function which is individual tests will implement. -// we may want it to return more than just an error, at some point. -type validatorFn func(c *client.Client, podID string) error - -// validateController is a generic mechanism for testing RC's that are running. -// It takes a container name, a test name, and a validator function which is plugged in by a specific test. -// "containername": this is grepped for. -// "testname": which gets bubbled up to the logging/failure messages if errors happen. -// "validator" function: This function is given a podID and a client, and it can do some specific validations that way. -func validateController(c *client.Client, image string, replicas int, containername string, testname string, validator validatorFn) { - getPodsTemplate := "--template={{range.items}}{{.id}} {{end}}" - // NB: kubectl adds the "exists" function to the standard template functions. - // This lets us check to see if the "running" entry exists for each of the containers - // we care about. Exists will never return an error and it's safe to check a chain of - // things, any one of which may not exist. In the below template, all of info, - // containername, and running might be nil, so the normal index function isn't very - // helpful. - // This template is unit-tested in kubectl, so if you change it, update the unit test. - // You can read about the syntax here: http://golang.org/pkg/text/template/. - getContainerStateTemplate := fmt.Sprintf(`--template={{and (exists . "currentState" "info" "%s" "state" "running")}}`, containername) - - getImageTemplate := fmt.Sprintf(`--template={{(index .currentState.info "%s").image}}`, containername) - - By(fmt.Sprintf("waiting for all containers in %s pods to come up.", testname)) //testname should be selector - for start := time.Now(); time.Since(start) < podStartTimeout; time.Sleep(5 * time.Second) { - getPodsOutput := runKubectl("get", "pods", "-o", "template", getPodsTemplate, "-l", testname) - pods := strings.Fields(getPodsOutput) - if numPods := len(pods); numPods != replicas { - By(fmt.Sprintf("Replicas for %s: expected=%d actual=%d", testname, replicas, numPods)) - continue - } - var runningPods []string - for _, podID := range pods { - running := runKubectl("get", "pods", podID, "-o", "template", getContainerStateTemplate) - if running == "false" { - Logf("%s is created but not running", podID) - continue - } - - currentImage := runKubectl("get", "pods", podID, "-o", "template", getImageTemplate) - if currentImage != image { - Logf("%s is created but running wrong image; expected: %s, actual: %s", podID, image, currentImage) - continue - } - - // Call the generic validator function here. - // This might validate for example, that (1) getting a url works and (2) url is serving correct content. - if err := validator(c, podID); err != nil { - Logf("%s is running right image but validator function failed: %v", podID, err) - continue - } - - Logf("%s is verified up and running", podID) - runningPods = append(runningPods, podID) - } - // If we reach here, then all our checks passed. - if len(runningPods) == replicas { - return - } - } - // Reaching here means that one of more checks failed multiple times. Assuming its not a race condition, something is broken. - Failf("Timed out after %v seconds waiting for %s pods to reach valid state", podStartTimeout.Seconds(), testname) -} - -func kubectlCmd(args ...string) *exec.Cmd { - defaultArgs := []string{} - if testContext.kubeConfig != "" { - defaultArgs = append(defaultArgs, "--"+clientcmd.RecommendedConfigPathFlag+"="+testContext.kubeConfig) - } else { - defaultArgs = append(defaultArgs, "--"+clientcmd.FlagAuthPath+"="+testContext.authConfig) - if testContext.certDir != "" { - defaultArgs = append(defaultArgs, - fmt.Sprintf("--certificate-authority=%s", filepath.Join(testContext.certDir, "ca.crt")), - fmt.Sprintf("--client-certificate=%s", filepath.Join(testContext.certDir, "kubecfg.crt")), - fmt.Sprintf("--client-key=%s", filepath.Join(testContext.certDir, "kubecfg.key"))) - } - } - kubectlArgs := append(defaultArgs, args...) - // TODO: Remove this once gcloud writes a proper entry in the kubeconfig file. - if testContext.provider == "gke" { - kubectlArgs = append(kubectlArgs, "--server="+testContext.host) - } - cmd := exec.Command("kubectl", kubectlArgs...) - Logf("Running '%s %s'", cmd.Path, strings.Join(cmd.Args, " ")) - return cmd -} - -func runKubectl(args ...string) string { - var stdout, stderr bytes.Buffer - cmd := kubectlCmd(args...) - cmd.Stdout, cmd.Stderr = &stdout, &stderr - - if err := cmd.Run(); err != nil { - Failf("Error running %v:\nCommand stdout:\n%v\nstderr:\n%v\n", cmd, cmd.Stdout, cmd.Stderr) - return "" - } - Logf(stdout.String()) - // TODO: trimspace should be unnecessary after switching to use kubectl binary directly - return strings.TrimSpace(stdout.String()) -}