From f4d2334919bb9435d828cc142594bf02dc56d50c Mon Sep 17 00:00:00 2001 From: Prashanth Balasubramanian Date: Mon, 4 Apr 2016 17:28:52 -0700 Subject: [PATCH 1/3] Add a l7 static pod --- ...{glbc.yaml => default-svc-controller.yaml} | 38 ++----------- cluster/saltbase/salt/l7-gcp/glbc.manifest | 53 +++++++++++++++++++ cluster/saltbase/salt/l7-gcp/init.sls | 17 ++++++ cluster/saltbase/salt/top.sls | 3 ++ 4 files changed, 77 insertions(+), 34 deletions(-) rename cluster/addons/cluster-loadbalancing/glbc/{glbc.yaml => default-svc-controller.yaml} (50%) create mode 100644 cluster/saltbase/salt/l7-gcp/glbc.manifest create mode 100644 cluster/saltbase/salt/l7-gcp/init.sls diff --git a/cluster/addons/cluster-loadbalancing/glbc/glbc.yaml b/cluster/addons/cluster-loadbalancing/glbc/default-svc-controller.yaml similarity index 50% rename from cluster/addons/cluster-loadbalancing/glbc/glbc.yaml rename to cluster/addons/cluster-loadbalancing/glbc/default-svc-controller.yaml index e08a2abb015..4f3ce1e3b59 100644 --- a/cluster/addons/cluster-loadbalancing/glbc/glbc.yaml +++ b/cluster/addons/cluster-loadbalancing/glbc/default-svc-controller.yaml @@ -1,33 +1,26 @@ -{% set kube_uid = "" -%} -{% if pillar['kube_uid'] is defined -%} - {% set kube_uid = pillar['kube_uid'] %} -{% endif -%} - apiVersion: v1 kind: ReplicationController metadata: - name: l7-lb-controller-v0.6.2 + name: l7-default-backend-v1.0 namespace: kube-system labels: k8s-app: glbc - version: v0.6.2 + version: v1.0 kubernetes.io/cluster-service: "true" kubernetes.io/name: "GLBC" spec: - # There should never be more than 1 controller alive simultaneously. replicas: 1 selector: k8s-app: glbc - version: v0.6.2 + version: v1.0 template: metadata: labels: k8s-app: glbc - version: v0.6.2 + version: v1.0 name: glbc kubernetes.io/cluster-service: "true" spec: - terminationGracePeriodSeconds: 600 containers: - name: default-http-backend # Any image is permissible as long as: @@ -50,26 +43,3 @@ spec: requests: cpu: 10m memory: 20Mi - - image: gcr.io/google_containers/glbc:0.6.2 - livenessProbe: - httpGet: - path: /healthz - port: 8081 - scheme: HTTP - initialDelaySeconds: 30 - # healthz reaches out to GCE - periodSeconds: 30 - timeoutSeconds: 5 - name: l7-lb-controller - resources: - limits: - cpu: 100m - memory: 100Mi - requests: - cpu: 100m - memory: 50Mi - args: - - --default-backend-service=kube-system/default-http-backend - - --verbose=true - - --sync-period=60s - - --cluster-uid={{kube_uid}} diff --git a/cluster/saltbase/salt/l7-gcp/glbc.manifest b/cluster/saltbase/salt/l7-gcp/glbc.manifest new file mode 100644 index 00000000000..d386ded7cce --- /dev/null +++ b/cluster/saltbase/salt/l7-gcp/glbc.manifest @@ -0,0 +1,53 @@ +apiVersion: v1 +kind: Pod +metadata: + name: l7-lb-controller-v0.6.0 + namespace: kube-system + labels: + k8s-app: glbc + version: v0.6.0 + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "GLBC" +spec: + terminationGracePeriodSeconds: 600 + hostNetwork: true + containers: + # TODO: Push thing image once kubernetes/contrib/pull/680 is in. + - image: bprashanth/glbc:0.6.3 + imagePullPolicy: Always + livenessProbe: + httpGet: + path: /healthz + port: 8086 + scheme: HTTP + initialDelaySeconds: 30 + # healthz reaches out to GCE + periodSeconds: 30 + timeoutSeconds: 5 + name: l7-lb-controller + volumeMounts: + - mountPath: /etc/gce.conf + name: cloudconfig + readOnly: true + - mountPath: /var/log/glbc.log + name: logfile + readOnly: false + resources: + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 100m + memory: 50Mi + command: + # TODO: split this out into args when we no longer need to pipe stdout to a file #6428 + - sh + - -c + - '/glbc --default-backend-service=kube-system/default-http-backend --sync-period=60s --running-in-cluster=false --use-real-cloud=true --config-file-path=/etc/gce.conf --healthz-port=8086 1>>/var/log/glbc.log 2>&1' + volumes: + - hostPath: + path: /etc/gce.conf + name: cloudconfig + - hostPath: + path: /var/log/glbc.log + name: logfile diff --git a/cluster/saltbase/salt/l7-gcp/init.sls b/cluster/saltbase/salt/l7-gcp/init.sls new file mode 100644 index 00000000000..6617d6bf8d4 --- /dev/null +++ b/cluster/saltbase/salt/l7-gcp/init.sls @@ -0,0 +1,17 @@ +/etc/kubernetes/manifests/glbc.manifest: + file.managed: + - source: salt://l7-gcp/glbc.manifest + - template: jinja + - user: root + - group: root + - mode: 644 + - makedirs: true + - dir_mode: 755 + +/var/log/glbc.log: + file.managed: + - user: root + - group: root + - mode: 644 + + diff --git a/cluster/saltbase/salt/top.sls b/cluster/saltbase/salt/top.sls index cb2f9104185..000bbd9a9e9 100644 --- a/cluster/saltbase/salt/top.sls +++ b/cluster/saltbase/salt/top.sls @@ -56,6 +56,9 @@ base: - cni {% elif pillar.get('network_provider', '').lower() == 'cni' %} - cni +{% endif %} +{% if pillar.get('enable_l7_loadbalancing', '').lower() == 'glbc' %} + - l7-gcp {% endif %} - kube-apiserver - kube-controller-manager From e2b980b3e6b3a38d63ac004cd79d71b9637b9c78 Mon Sep 17 00:00:00 2001 From: Prashanth Balasubramanian Date: Sun, 22 May 2016 19:08:46 -0700 Subject: [PATCH 2/3] Fix e2es --- test/e2e/ingress.go | 44 +++++++------------------------------------- 1 file changed, 7 insertions(+), 37 deletions(-) diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index 08a73526f1f..a6bf921e60b 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -30,7 +30,6 @@ import ( "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/apis/extensions" client "k8s.io/kubernetes/pkg/client/unversioned" - "k8s.io/kubernetes/pkg/labels" utilexec "k8s.io/kubernetes/pkg/util/exec" "k8s.io/kubernetes/pkg/util/intstr" "k8s.io/kubernetes/pkg/util/wait" @@ -62,6 +61,10 @@ var ( testImage = "gcr.io/google_containers/n-way-http:1.0" httpContainerPort = 8080 + // Name of the config-map and key the ingress controller stores its uid in. + uidConfigMap = "ingress-uid" + uidKey = "uid" + expectedLBCreationTime = 7 * time.Minute expectedLBHealthCheckTime = 7 * time.Minute @@ -232,26 +235,6 @@ func gcloudDelete(resource, name, project string) { } } -// kubectlLogLBController logs kubectl debug output for the L7 controller pod. -func kubectlLogLBController(c *client.Client, ns string) { - selector := labels.SelectorFromSet(labels.Set(controllerLabels)) - options := api.ListOptions{LabelSelector: selector} - podList, err := c.Pods(api.NamespaceAll).List(options) - if err != nil { - framework.Logf("Cannot log L7 controller output, error listing pods %v", err) - return - } - if len(podList.Items) == 0 { - framework.Logf("Loadbalancer controller pod not found") - return - } - for _, p := range podList.Items { - framework.Logf("\nLast 100 log lines of %v\n", p.Name) - l, _ := framework.RunKubectl("logs", p.Name, fmt.Sprintf("--namespace=%v", ns), "-c", lbContainerName, "--tail=100") - framework.Logf(l) - } -} - type IngressController struct { ns string rcPath string @@ -263,24 +246,12 @@ type IngressController struct { } func (cont *IngressController) getL7AddonUID() (string, error) { - listOpts := api.ListOptions{LabelSelector: labels.SelectorFromSet(labels.Set(clusterAddonLBLabels))} - existingRCs, err := cont.c.ReplicationControllers(api.NamespaceSystem).List(listOpts) + cm, err := cont.c.ConfigMaps(api.NamespaceSystem).Get(uidConfigMap) if err != nil { return "", err } - if len(existingRCs.Items) != 1 { - return "", fmt.Errorf("Unexpected number of lb cluster addons %v with label %v in kube-system namespace", len(existingRCs.Items), clusterAddonLBLabels) - } - rc := existingRCs.Items[0] - commandPrefix := "--cluster-uid=" - for i, c := range rc.Spec.Template.Spec.Containers { - if c.Name == lbContainerName { - for _, arg := range rc.Spec.Template.Spec.Containers[i].Args { - if strings.HasPrefix(arg, commandPrefix) { - return strings.Replace(arg, commandPrefix, "", -1), nil - } - } - } + if uid, ok := cm.Data[uidKey]; ok { + return uid, nil } return "", fmt.Errorf("Could not find cluster UID for L7 addon pod") } @@ -448,7 +419,6 @@ var _ = framework.KubeDescribe("GCE L7 LoadBalancer Controller [Feature:Ingress] AfterEach(func() { framework.Logf("Average creation time %+v, health check time %+v", creationTimes, responseTimes) if CurrentGinkgoTestDescription().Failed { - kubectlLogLBController(client, ns) framework.Logf("\nOutput of kubectl describe ing:\n") desc, _ := framework.RunKubectl("describe", "ing", fmt.Sprintf("--namespace=%v", ns)) framework.Logf(desc) From c01a0583f75b22e7512558f64d4180a1b40beb95 Mon Sep 17 00:00:00 2001 From: Prashanth Balasubramanian Date: Sat, 28 May 2016 16:24:40 -0700 Subject: [PATCH 3/3] Trusty deployment --- build/common.sh | 1 + cluster/gce/gci/configure-helper.sh | 16 ++++++++++++---- cluster/saltbase/salt/l7-gcp/glbc.manifest | 14 +++++++------- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/build/common.sh b/build/common.sh index 2a40f21a964..41be8354b39 100755 --- a/build/common.sh +++ b/build/common.sh @@ -926,6 +926,7 @@ function kube::release::package_kube_manifests_tarball() { cp "${salt_dir}/kube-apiserver/abac-authz-policy.jsonl" "${dst_dir}" cp "${salt_dir}/kube-controller-manager/kube-controller-manager.manifest" "${dst_dir}" cp "${salt_dir}/kube-addons/kube-addon-manager.yaml" "${dst_dir}" + cp "${salt_dir}/l7-gcp/glbc.manifest" "${dst_dir}" cp "${KUBE_ROOT}/cluster/gce/trusty/configure-helper.sh" "${dst_dir}/trusty-configure-helper.sh" cp "${KUBE_ROOT}/cluster/gce/gci/configure-helper.sh" "${dst_dir}/gci-configure-helper.sh" cp "${KUBE_ROOT}/cluster/gce/gci/health-monitor.sh" "${dst_dir}/health-monitor.sh" diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 406b1c8d238..c4acd352ddb 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -558,7 +558,6 @@ function start-kube-apiserver { function start-kube-controller-manager { echo "Start kubernetes controller-manager" prepare-log-file /var/log/kube-controller-manager.log - # Calculate variables and assemble the command line. local params="${CONTROLLER_MANAGER_TEST_LOG_LEVEL:-"--v=2"} ${CONTROLLER_MANAGER_TEST_ARGS:-}" params+=" --cloud-provider=gce" @@ -706,9 +705,6 @@ function start-kube-addons { fi if [[ "${ENABLE_L7_LOADBALANCING:-}" == "glbc" ]]; then setup-addon-manifests "addons" "cluster-loadbalancing/glbc" - local -r glbc_yaml="${dst_dir}/cluster-loadbalancing/glbc/glbc.yaml" - remove-salt-config-comments "${glbc_yaml}" - sed -i -e "s@{{ *kube_uid *}}@${KUBE_UID:-}@g" "${glbc_yaml}" fi if [[ "${ENABLE_CLUSTER_DNS:-}" == "true" ]]; then setup-addon-manifests "addons" "dns" @@ -761,6 +757,17 @@ function start-fluentd { fi } +# Starts a l7 loadbalancing controller for ingress. +function start-lb-controller { + if [[ "${ENABLE_L7_LOADBALANCING:-}" == "glbc" ]]; then + echo "Starting GCE L7 pod" + prepare-log-file /var/log/glbc.log + local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/glbc.manifest" + cp "${src_file}" /etc/kubernetes/manifests/ + fi +} + + function reset-motd { # kubelet is installed both on the master and nodes, and the version is easy to parse (unlike kubectl) local -r version="$(/usr/bin/kubelet --version=true | cut -f2 -d " ")" @@ -829,6 +836,7 @@ if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then start-kube-scheduler start-kube-addons start-cluster-autoscaler + start-lb-controller else start-kube-proxy # Kube-registry-proxy. diff --git a/cluster/saltbase/salt/l7-gcp/glbc.manifest b/cluster/saltbase/salt/l7-gcp/glbc.manifest index d386ded7cce..1db2b4a6992 100644 --- a/cluster/saltbase/salt/l7-gcp/glbc.manifest +++ b/cluster/saltbase/salt/l7-gcp/glbc.manifest @@ -1,20 +1,18 @@ apiVersion: v1 kind: Pod metadata: - name: l7-lb-controller-v0.6.0 + name: l7-lb-controller-v0.6.3 namespace: kube-system labels: k8s-app: glbc - version: v0.6.0 + version: v0.6.3 kubernetes.io/cluster-service: "true" kubernetes.io/name: "GLBC" spec: terminationGracePeriodSeconds: 600 hostNetwork: true containers: - # TODO: Push thing image once kubernetes/contrib/pull/680 is in. - - image: bprashanth/glbc:0.6.3 - imagePullPolicy: Always + - image: gcr.io/google_containers/glbc:0.6.3 livenessProbe: httpGet: path: /healthz @@ -33,11 +31,13 @@ spec: name: logfile readOnly: false resources: + # Request and limits are set to accomodate this pod alongside the other + # master components on a single core master. limits: - cpu: 100m + cpu: 50m memory: 100Mi requests: - cpu: 100m + cpu: 10m memory: 50Mi command: # TODO: split this out into args when we no longer need to pipe stdout to a file #6428