From 19fb97331d9d6eb473320675e3b435b4466ec6e4 Mon Sep 17 00:00:00 2001 From: Jacob Beacham Date: Thu, 17 Nov 2016 11:00:04 -0800 Subject: [PATCH] Add kubernetes-anywhere as a new e2e deployment option. The configuration in getConfig() comes mostly from the defaults in kubernetes-anywhere. --- hack/e2e.go | 195 ++++++++++++++++++++++++++---- hack/verify-flags/exceptions.txt | 3 + hack/verify-flags/known-flags.txt | 4 + 3 files changed, 180 insertions(+), 22 deletions(-) diff --git a/hack/e2e.go b/hack/e2e.go index 67da62c5979..26cb74fe8c2 100644 --- a/hack/e2e.go +++ b/hack/e2e.go @@ -18,6 +18,7 @@ limitations under the License. package main import ( + "bytes" "encoding/xml" "flag" "fmt" @@ -30,6 +31,7 @@ import ( "path/filepath" "strconv" "strings" + "text/template" "time" ) @@ -41,7 +43,7 @@ var ( "You can explicitly set to false if you're, e.g., testing client changes "+ "for which the server version doesn't make a difference.") checkLeakedResources = flag.Bool("check_leaked_resources", false, "Ensure project ends with the same resources") - deployment = flag.String("deployment", "bash", "up/down mechanism (defaults to cluster/kube-{up,down}.sh) (choices: bash/kops)") + deployment = flag.String("deployment", "bash", "up/down mechanism (defaults to cluster/kube-{up,down}.sh) (choices: bash/kops/kubernetes-anywhere)") down = flag.Bool("down", false, "If true, tear down the cluster before exiting.") dump = flag.String("dump", "", "If set, dump cluster logs to this location on test or cluster-up failure") kubemark = flag.Bool("kubemark", false, "If true, run kubemark tests.") @@ -62,12 +64,42 @@ var ( kopsNodes = flag.Int("kops-nodes", 2, "(kops only) Number of nodes to create.") kopsUpTimeout = flag.Duration("kops-up-timeout", 20*time.Minute, "(kops only) Time limit between 'kops config / kops update' and a response from the Kubernetes API.") + // kubernetes-anywhere specific flags. + kubernetesAnywherePath = flag.String("kubernetes-anywhere-path", "", "(kubernetes-anywhere only) Path to the kubernetes-anywhere directory. Must be set for kubernetes-anywhere.") + kubernetesAnywherePhase2Provider = flag.String("kubernetes-anywhere-phase2-provider", "ignition", "(kubernetes-anywhere only) Provider for phase2 bootstrapping. (Defaults to ignition).") + kubernetesAnywhereCluster = flag.String("kubernetes-anywhere-cluster", "", "(kubernetes-anywhere only) Cluster name. Must be set for kubernetes-anywhere.") + kubernetesAnywhereUpTimeout = flag.Duration("kubernetes-anywhere-up-timeout", 20*time.Minute, "(kubernetes-anywhere only) Time limit between starting a cluster and making a successful call to the Kubernetes API.") + // Deprecated flags. deprecatedPush = flag.Bool("push", false, "Deprecated. Does nothing.") deprecatedPushup = flag.Bool("pushup", false, "Deprecated. Does nothing.") deprecatedCtlCmd = flag.String("ctl", "", "Deprecated. Does nothing.") ) +const kubernetesAnywhereConfigTemplate = ` +.phase1.num_nodes=4 +.phase1.cluster_name="{{.Cluster}}" +.phase1.cloud_provider="gce" + +.phase1.gce.os_image="ubuntu-1604-xenial-v20160420c" +.phase1.gce.instance_type="n1-standard-2" +.phase1.gce.project="{{.Project}}" +.phase1.gce.region="us-central1" +.phase1.gce.zone="us-central1-b" +.phase1.gce.network="default" + +.phase2.installer_container="docker.io/colemickens/k8s-ignition:latest" +.phase2.docker_registry="gcr.io/google-containers" +.phase2.kubernetes_version="v1.4.1" +.phase2.provider="{{.Phase2Provider}}" + +.phase3.run_addons=y +.phase3.kube_proxy=y +.phase3.dashboard=y +.phase3.heapster=y +.phase3.kube_dns=y +` + func appendError(errs []error, err error) []error { if err != nil { return append(errs, err) @@ -421,6 +453,8 @@ func getDeployer() (deployer, error) { return bash{}, nil case "kops": return NewKops() + case "kubernetes-anywhere": + return NewKubernetesAnywhere() default: return nil, fmt.Errorf("Unknown deployment strategy %q", *deployment) } @@ -537,30 +571,11 @@ func (k kops) Up() error { // TODO(zmerlynn): More cluster validation. This should perhaps be // added to kops and not here, but this is a fine place to loop // for now. - for stop := time.Now().Add(*kopsUpTimeout); time.Now().Before(stop); time.Sleep(30 * time.Second) { - n, err := clusterSize(k) - if err != nil { - log.Printf("Can't get cluster size, sleeping: %v", err) - continue - } - if n < k.nodes+1 { - log.Printf("%d (current nodes) < %d (requested instances), sleeping", n, k.nodes+1) - continue - } - return nil - } - return fmt.Errorf("kops bringup timed out") + return waitForNodes(k, k.nodes+1, *kopsUpTimeout) } func (k kops) IsUp() error { - n, err := clusterSize(k) - if err != nil { - return err - } - if n <= 0 { - return fmt.Errorf("kops cluster found, but %d nodes reported", n) - } - return nil + return isUp(k) } func (k kops) SetupKubecfg() error { @@ -590,6 +605,115 @@ func (k kops) Down() error { return finishRunning("kops delete", exec.Command(k.path, "delete", "cluster", k.cluster, "--yes")) } +type kubernetesAnywhere struct { + path string + // These are exported only because their use in the config template requires it. + Phase2Provider string + Project string + Cluster string +} + +func NewKubernetesAnywhere() (*kubernetesAnywhere, error) { + if *kubernetesAnywherePath == "" { + return nil, fmt.Errorf("--kubernetes-anywhere-path is required") + } + + if *kubernetesAnywhereCluster == "" { + return nil, fmt.Errorf("--kubernetes-anywhere-cluster is required") + } + + project, ok := os.LookupEnv("PROJECT") + if !ok { + return nil, fmt.Errorf("The PROJECT environment variable is required to be set for kubernetes-anywhere") + } + + // Set KUBERNETES_CONFORMANCE_TEST so the auth info is picked up + // from kubectl instead of bash inference. + if err := os.Setenv("KUBERNETES_CONFORMANCE_TEST", "yes"); err != nil { + return nil, err + } + + k := &kubernetesAnywhere{ + path: *kubernetesAnywherePath, + Phase2Provider: *kubernetesAnywherePhase2Provider, + Project: project, + Cluster: *kubernetesAnywhereCluster, + } + + if err := k.writeConfig(); err != nil { + return nil, err + } + return k, nil +} + +func (k kubernetesAnywhere) getConfig() (string, error) { + // As needed, plumb through more CLI options to replace these defaults + tmpl, err := template.New("kubernetes-anywhere-config").Parse(kubernetesAnywhereConfigTemplate) + + if err != nil { + return "", fmt.Errorf("Error creating template for KubernetesAnywhere config: %v", err) + } + + var buf bytes.Buffer + if err = tmpl.Execute(&buf, k); err != nil { + return "", fmt.Errorf("Error executing template for KubernetesAnywhere config: %v", err) + } + + return buf.String(), nil +} + +func (k kubernetesAnywhere) writeConfig() error { + config, err := k.getConfig() + if err != nil { + return fmt.Errorf("Could not generate config: %v", err) + } + + f, err := os.Create(k.path + "/.config") + if err != nil { + return fmt.Errorf("Could not create file: %v", err) + } + defer f.Close() + + fmt.Fprint(f, config) + return nil +} + +func (k kubernetesAnywhere) Up() error { + cmd := exec.Command("make", "-C", k.path, "WAIT_FOR_KUBECONFIG=y", "deploy-cluster") + if err := finishRunning("deploy-cluster", cmd); err != nil { + return err + } + + nodes := 4 // For now, this is hardcoded in the config + return waitForNodes(k, nodes+1, *kubernetesAnywhereUpTimeout) +} + +func (k kubernetesAnywhere) IsUp() error { + return isUp(k) +} + +func (k kubernetesAnywhere) SetupKubecfg() error { + output, err := exec.Command("make", "--silent", "-C", k.path, "kubeconfig-path").Output() + if err != nil { + return fmt.Errorf("Could not get kubeconfig-path: %v", err) + } + kubecfg := strings.TrimSuffix(string(output), "\n") + + if err = os.Setenv("KUBECONFIG", kubecfg); err != nil { + return err + } + return nil +} + +func (k kubernetesAnywhere) Down() error { + err := finishRunning("get kubeconfig-path", exec.Command("make", "-C", k.path, "kubeconfig-path")) + if err != nil { + // This is expected if the cluster doesn't exist. + return nil + } + return finishRunning("destroy-cluster", exec.Command("make", "-C", k.path, "FORCE_DESTROY=y", "destroy-cluster")) +} + func clusterSize(deploy deployer) (int, error) { if err := deploy.SetupKubecfg(); err != nil { return -1, err @@ -633,6 +757,33 @@ func (e *CommandError) Error() string { return fmt.Sprintf("%q: %q", exitErr.Error(), stderr) } +func isUp(d deployer) error { + n, err := clusterSize(d) + if err != nil { + return err + } + if n <= 0 { + return fmt.Errorf("cluster found, but %d nodes reported", n) + } + return nil +} + +func waitForNodes(d deployer, nodes int, timeout time.Duration) error { + for stop := time.Now().Add(timeout); time.Now().Before(stop); time.Sleep(30 * time.Second) { + n, err := clusterSize(d) + if err != nil { + log.Printf("Can't get cluster size, sleeping: %v", err) + continue + } + if n < nodes { + log.Printf("%d (current nodes) < %d (requested instances), sleeping", n, nodes) + continue + } + return nil + } + return fmt.Errorf("waiting for nodes timed out") +} + func DumpClusterLogs(location string) error { log.Printf("Dumping cluster logs to: %v", location) return finishRunning("dump cluster logs", exec.Command("./cluster/log-dump.sh", location)) diff --git a/hack/verify-flags/exceptions.txt b/hack/verify-flags/exceptions.txt index 8299bdd22d9..85c7ccd4a73 100644 --- a/hack/verify-flags/exceptions.txt +++ b/hack/verify-flags/exceptions.txt @@ -85,6 +85,9 @@ federation/deploy/config.json.sample: "cluster_name": "cluster3-kubernetes" federation/deploy/config.json.sample: "num_nodes": 3, federation/deploy/config.json.sample: "num_nodes": 3, federation/deploy/config.json.sample: "num_nodes": 3, +hack/e2e.go:.phase1.cloud_provider="gce" +hack/e2e.go:.phase1.cluster_name="{{.Cluster}}" +hack/e2e.go:.phase1.num_nodes=4 hack/local-up-cluster.sh: advertise_address="--advertise_address=${API_HOST_IP}" hack/local-up-cluster.sh: runtime_config="--runtime-config=${RUNTIME_CONFIG}" hack/local-up-cluster.sh: advertise_address="" diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index b482d30356c..c7d24999e12 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -333,6 +333,10 @@ kubelet-read-only-port kubelet-root-dir kubelet-sync-frequency kubelet-timeout +kubernetes-anywhere-cluster +kubernetes-anywhere-path +kubernetes-anywhere-phase2-provider +kubernetes-anywhere-up-timeout kubernetes-service-node-port label-columns large-cluster-size-threshold