From 2ec06f8d4a632c83b775c2fad91c0e1ef8cb57ee Mon Sep 17 00:00:00 2001 From: Prashanth Balasubramanian Date: Mon, 4 Apr 2016 17:04:54 -0700 Subject: [PATCH 1/3] Generate a kube_uuid salt pillar. --- cluster/common.sh | 12 +++++++++++- cluster/gce/configure-vm.sh | 1 + 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/cluster/common.sh b/cluster/common.sh index de656d0dec3..9f91c7200d1 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -38,7 +38,6 @@ KUBE_RELEASE_VERSION_REGEX="^v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]* # kube::release::parse_and_validate_ci_version() KUBE_CI_VERSION_REGEX="^v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)-(beta|alpha)\\.(0|[1-9][0-9]*)(\\.(0|[1-9][0-9]*)\\+[-0-9a-z]*)?$" - # Generate kubeconfig data for the created cluster. # Assumed vars: # KUBE_USER @@ -202,6 +201,15 @@ function gen-kube-bearertoken() { KUBE_BEARER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) } +# Generate uuid +# This function only works on systems with /proc/sys/kernel/random/uuid. +# +# Vars set: +# KUBE_UUID +function gen-uuid { + KUBE_UUID=$(cat /proc/sys/kernel/random/uuid) +} + function load-or-gen-kube-basicauth() { if [[ ! -z "${KUBE_CONTEXT:-}" ]]; then @@ -456,6 +464,7 @@ function build-kube-env { local file=$2 build-runtime-config + gen-uuid rm -f ${file} cat >$file <>$file <>/srv/salt-overlay/pillar/cluster-params.sls From b066cb5357da330c887b4426acf26f10bfbdfd70 Mon Sep 17 00:00:00 2001 From: Prashanth Balasubramanian Date: Mon, 4 Apr 2016 17:28:52 -0700 Subject: [PATCH 2/3] Templatize l7 rc --- .../glbc/{glbc-controller.yaml => glbc.yaml} | 8 +++++++- cluster/common.sh | 16 +++++++++------- cluster/gce/configure-vm.sh | 2 +- cluster/saltbase/salt/kube-addons/init.sls | 1 + 4 files changed, 18 insertions(+), 9 deletions(-) rename cluster/addons/cluster-loadbalancing/glbc/{glbc-controller.yaml => glbc.yaml} (90%) diff --git a/cluster/addons/cluster-loadbalancing/glbc/glbc-controller.yaml b/cluster/addons/cluster-loadbalancing/glbc/glbc.yaml similarity index 90% rename from cluster/addons/cluster-loadbalancing/glbc/glbc-controller.yaml rename to cluster/addons/cluster-loadbalancing/glbc/glbc.yaml index 406458dfd6b..6a930141d61 100644 --- a/cluster/addons/cluster-loadbalancing/glbc/glbc-controller.yaml +++ b/cluster/addons/cluster-loadbalancing/glbc/glbc.yaml @@ -1,3 +1,8 @@ +{% set kube_uid = "" -%} +{% if pillar['kube_uid'] is defined -%} + {% set kube_uid = pillar['kube_uid'] %} +{% endif -%} + apiVersion: v1 kind: ReplicationController metadata: @@ -65,4 +70,5 @@ spec: memory: 50Mi args: - --default-backend-service=kube-system/default-http-backend - - --sync-period=300s + - --sync-period=60s + - --cluster-uid={{kube_uid}} diff --git a/cluster/common.sh b/cluster/common.sh index 9f91c7200d1..6d1b8a2908b 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -38,6 +38,7 @@ KUBE_RELEASE_VERSION_REGEX="^v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]* # kube::release::parse_and_validate_ci_version() KUBE_CI_VERSION_REGEX="^v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)-(beta|alpha)\\.(0|[1-9][0-9]*)(\\.(0|[1-9][0-9]*)\\+[-0-9a-z]*)?$" + # Generate kubeconfig data for the created cluster. # Assumed vars: # KUBE_USER @@ -201,13 +202,14 @@ function gen-kube-bearertoken() { KUBE_BEARER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) } -# Generate uuid -# This function only works on systems with /proc/sys/kernel/random/uuid. +# Generate uid +# This function only works on systems with python. It generates a time based +# UID instead of a UUID because GCE has a name length limit. # # Vars set: -# KUBE_UUID -function gen-uuid { - KUBE_UUID=$(cat /proc/sys/kernel/random/uuid) +# KUBE_UID +function gen-uid { + KUBE_UID=$(python -c 'import uuid; print uuid.uuid1().fields[0]') } @@ -464,7 +466,7 @@ function build-kube-env { local file=$2 build-runtime-config - gen-uuid + gen-uid rm -f ${file} cat >$file <>$file <>/srv/salt-overlay/pillar/cluster-params.sls diff --git a/cluster/saltbase/salt/kube-addons/init.sls b/cluster/saltbase/salt/kube-addons/init.sls index 3989edd838c..7d38b5b28bf 100644 --- a/cluster/saltbase/salt/kube-addons/init.sls +++ b/cluster/saltbase/salt/kube-addons/init.sls @@ -35,6 +35,7 @@ addon-dir-create: file.recurse: - source: salt://kube-addons/cluster-loadbalancing/glbc - include_pat: E@(^.+\.yaml$|^.+\.json$) + - template: jinja - user: root - group: root - dir_mode: 755 From 1f4e48ba59debb6ffb76e274c67567248b360c68 Mon Sep 17 00:00:00 2001 From: Prashanth Balasubramanian Date: Wed, 20 Apr 2016 16:14:53 -0700 Subject: [PATCH 3/3] E2E changes to support templated addon --- test/e2e/ingress.go | 114 ++++++++++++++------------------------------ 1 file changed, 36 insertions(+), 78 deletions(-) diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index 7e7bf2cda9d..08a73526f1f 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -253,76 +253,49 @@ func kubectlLogLBController(c *client.Client, ns string) { } type IngressController struct { - ns string - rcPath string - defaultSvcPath string - UID string - Project string - rc *api.ReplicationController - svc *api.Service - c *client.Client + ns string + rcPath string + UID string + Project string + rc *api.ReplicationController + svc *api.Service + c *client.Client } -func (cont *IngressController) create() { - - // TODO: This cop out is because it would be *more* brittle to duplicate all - // the name construction logic from the controller cross-repo. We will not - // need to be so paranoid about leaked resources once we figure out a solution - // for issues like #16337. Currently, all names should fall within 63 chars. - testName := fmt.Sprintf("k8s-fw-foo-app-X-%v--%v", cont.ns, cont.UID) - if len(testName) > nameLenLimit { - framework.Failf("Cannot reliably test the given namespace(%v)/uid(%v), too close to GCE limit of %v", - cont.ns, cont.UID, nameLenLimit) - } - - if cont.defaultSvcPath != "" { - svc := svcFromManifest(cont.defaultSvcPath) - svc.Namespace = cont.ns - svc.Labels = controllerLabels - svc.Spec.Selector = controllerLabels - cont.svc = svc - _, err := cont.c.Services(cont.ns).Create(cont.svc) - Expect(err).NotTo(HaveOccurred()) - } - rc := rcFromManifest(cont.rcPath) - +func (cont *IngressController) getL7AddonUID() (string, error) { listOpts := api.ListOptions{LabelSelector: labels.SelectorFromSet(labels.Set(clusterAddonLBLabels))} existingRCs, err := cont.c.ReplicationControllers(api.NamespaceSystem).List(listOpts) - Expect(err).NotTo(HaveOccurred()) + if err != nil { + return "", err + } if len(existingRCs.Items) != 1 { - framework.Failf("Unexpected number of lb cluster addons %v with label %v in kube-system namespace", len(existingRCs.Items), clusterAddonLBLabels) - } - - // Merge the existing spec and new spec. The modifications should not - // manifest as functional changes to the controller. Most importantly, the - // podTemplate shouldn't change (but for the additional test cmd line flags) - // to ensure we test actual cluster functionality across upgrades. - rc.Spec = existingRCs.Items[0].Spec - rc.Name = "glbc" - rc.Namespace = cont.ns - rc.Labels = controllerLabels - rc.Spec.Selector = controllerLabels - rc.Spec.Template.Labels = controllerLabels - rc.Spec.Replicas = 1 - - // These command line params are only recognized by v0.51 and above. - testArgs := []string{ - // Pass namespace uid so the controller will tag resources with it. - fmt.Sprintf("--cluster-uid=%v", cont.UID), - // Tell the controller to delete all resources as it quits. - fmt.Sprintf("--delete-all-on-quit=true"), - // Don't use the default Service from kube-system. - fmt.Sprintf("--default-backend-service=%v/%v", cont.svc.Namespace, cont.svc.Name), + return "", fmt.Errorf("Unexpected number of lb cluster addons %v with label %v in kube-system namespace", len(existingRCs.Items), clusterAddonLBLabels) } + rc := existingRCs.Items[0] + commandPrefix := "--cluster-uid=" for i, c := range rc.Spec.Template.Spec.Containers { if c.Name == lbContainerName { - rc.Spec.Template.Spec.Containers[i].Args = append(c.Args, testArgs...) + for _, arg := range rc.Spec.Template.Spec.Containers[i].Args { + if strings.HasPrefix(arg, commandPrefix) { + return strings.Replace(arg, commandPrefix, "", -1), nil + } + } } } - cont.rc = rc - _, err = cont.c.ReplicationControllers(cont.ns).Create(cont.rc) + return "", fmt.Errorf("Could not find cluster UID for L7 addon pod") +} + +func (cont *IngressController) init() { + uid, err := cont.getL7AddonUID() Expect(err).NotTo(HaveOccurred()) - Expect(framework.WaitForRCPodsRunning(cont.c, cont.ns, cont.rc.Name)).NotTo(HaveOccurred()) + cont.UID = uid + // There's a name limit imposed by GCE. The controller will truncate. + testName := fmt.Sprintf("k8s-fw-foo-app-X-%v--%v", cont.ns, cont.UID) + if len(testName) > nameLenLimit { + framework.Logf("WARNING: test name including cluster UID: %v is over the GCE limit of %v", testName, nameLenLimit) + } else { + framework.Logf("Deteced cluster UID %v", cont.UID) + } } func (cont *IngressController) Cleanup(del bool) error { @@ -457,24 +430,14 @@ var _ = framework.KubeDescribe("GCE L7 LoadBalancer Controller [Feature:Ingress] f.BeforeEach() client = f.Client ns = f.Namespace.Name - // Scaled down the existing Ingress controller so it doesn't interfere with the test. - Expect(framework.ScaleRCByLabels(client, api.NamespaceSystem, clusterAddonLBLabels, 0)).NotTo(HaveOccurred()) addonDir = filepath.Join( framework.TestContext.RepoRoot, "cluster", "addons", "cluster-loadbalancing", "glbc") - - nsParts := strings.Split(ns, "-") ingController = &IngressController{ - ns: ns, - // The UID in the namespace was generated by the master, so it's - // global to the cluster. - UID: nsParts[len(nsParts)-1], - Project: framework.TestContext.CloudConfig.ProjectID, - rcPath: filepath.Join(addonDir, "glbc-controller.yaml"), - defaultSvcPath: filepath.Join(addonDir, "default-svc.yaml"), - c: client, + ns: ns, + Project: framework.TestContext.CloudConfig.ProjectID, + c: client, } - ingController.create() - framework.Logf("Finished creating ingress controller") + ingController.init() // If we somehow get the same namespace uid as someone else in this // gce project, just back off. Expect(ingController.Cleanup(false)).NotTo(HaveOccurred()) @@ -509,9 +472,6 @@ var _ = framework.KubeDescribe("GCE L7 LoadBalancer Controller [Feature:Ingress] } return true, nil }) - // TODO: Remove this once issue #17802 is fixed - Expect(framework.ScaleRCByLabels(client, ingController.rc.Namespace, ingController.rc.Labels, 0)).NotTo(HaveOccurred()) - // If the controller failed to cleanup the test will fail, but we want to cleanup // resources before that. if pollErr != nil { @@ -520,8 +480,6 @@ var _ = framework.KubeDescribe("GCE L7 LoadBalancer Controller [Feature:Ingress] } framework.Failf("Failed to cleanup GCE L7 resources.") } - // Restore the cluster Addon. - Expect(framework.ScaleRCByLabels(client, api.NamespaceSystem, clusterAddonLBLabels, 1)).NotTo(HaveOccurred()) f.AfterEach() framework.Logf("Successfully verified GCE L7 loadbalancer via Ingress.") })