Merge pull request #26048 from bprashanth/ing_master

Automatic merge from submit-queue

Run l7 controller on master 

Fixes https://github.com/kubernetes/kubernetes/issues/23663, needs https://github.com/kubernetes/contrib/pull/680
@roberthbailey @kubernetes/goog-cluster

build/common.sh

@@ -926,6 +926,7 @@ function kube::release::package_kube_manifests_tarball() {
   cp "${salt_dir}/kube-apiserver/abac-authz-policy.jsonl" "${dst_dir}"
   cp "${salt_dir}/kube-controller-manager/kube-controller-manager.manifest" "${dst_dir}"
   cp "${salt_dir}/kube-addons/kube-addon-manager.yaml" "${dst_dir}"
+  cp "${salt_dir}/l7-gcp/glbc.manifest" "${dst_dir}"
   cp "${KUBE_ROOT}/cluster/gce/trusty/configure-helper.sh" "${dst_dir}/trusty-configure-helper.sh"
   cp "${KUBE_ROOT}/cluster/gce/gci/configure-helper.sh" "${dst_dir}/gci-configure-helper.sh"
   cp "${KUBE_ROOT}/cluster/gce/gci/health-monitor.sh" "${dst_dir}/health-monitor.sh"

cluster/addons/cluster-loadbalancing/glbc/default-svc-controller.yaml

@@ -1,33 +1,26 @@
-{% set kube_uid = "" -%}
-{% if pillar['kube_uid'] is defined -%}
-  {% set kube_uid = pillar['kube_uid'] %}
-{% endif -%}
-
 apiVersion: v1
 kind: ReplicationController
 metadata:
-  name: l7-lb-controller-v0.6.2
+  name: l7-default-backend-v1.0
   namespace: kube-system
   labels:
     k8s-app: glbc
-    version: v0.6.2
+    version: v1.0
     kubernetes.io/cluster-service: "true"
     kubernetes.io/name: "GLBC"
 spec:
-  # There should never be more than 1 controller alive simultaneously.
   replicas: 1
   selector:
     k8s-app: glbc
-    version: v0.6.2
+    version: v1.0
   template:
     metadata:
       labels:
         k8s-app: glbc
-        version: v0.6.2
+        version: v1.0
         name: glbc
         kubernetes.io/cluster-service: "true"
     spec:
-      terminationGracePeriodSeconds: 600
       containers:
       - name: default-http-backend
         # Any image is permissible as long as:
@@ -50,26 +43,3 @@ spec:
           requests:
             cpu: 10m
             memory: 20Mi
-      - image: gcr.io/google_containers/glbc:0.6.2
-        livenessProbe:
-          httpGet:
-            path: /healthz
-            port: 8081
-            scheme: HTTP
-          initialDelaySeconds: 30
-          # healthz reaches out to GCE
-          periodSeconds: 30
-          timeoutSeconds: 5
-        name: l7-lb-controller
-        resources:
-          limits:
-            cpu: 100m
-            memory: 100Mi
-          requests:
-            cpu: 100m
-            memory: 50Mi
-        args:
-        - --default-backend-service=kube-system/default-http-backend
-        - --verbose=true
-        - --sync-period=60s
-        - --cluster-uid={{kube_uid}}

cluster/gce/gci/configure-helper.sh

@@ -558,7 +558,6 @@ function start-kube-apiserver {
 function start-kube-controller-manager {
   echo "Start kubernetes controller-manager"
   prepare-log-file /var/log/kube-controller-manager.log
- 
   # Calculate variables and assemble the command line.
   local params="${CONTROLLER_MANAGER_TEST_LOG_LEVEL:-"--v=2"} ${CONTROLLER_MANAGER_TEST_ARGS:-}"
   params+=" --cloud-provider=gce"
@@ -704,9 +703,6 @@ function start-kube-addons {
   fi
   if [[ "${ENABLE_L7_LOADBALANCING:-}" == "glbc" ]]; then
     setup-addon-manifests "addons" "cluster-loadbalancing/glbc"
-    local -r glbc_yaml="${dst_dir}/cluster-loadbalancing/glbc/glbc.yaml"
-    remove-salt-config-comments "${glbc_yaml}"
-    sed -i -e "s@{{ *kube_uid *}}@${KUBE_UID:-}@g" "${glbc_yaml}"
   fi
   if [[ "${ENABLE_CLUSTER_DNS:-}" == "true" ]]; then
     setup-addon-manifests "addons" "dns"
@@ -759,6 +755,17 @@ function start-fluentd {
   fi
 }
 
+# Starts a l7 loadbalancing controller for ingress.
+function start-lb-controller {
+  if [[ "${ENABLE_L7_LOADBALANCING:-}" == "glbc" ]]; then
+    echo "Starting GCE L7 pod"
+    prepare-log-file /var/log/glbc.log
+    local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/glbc.manifest"
+    cp "${src_file}" /etc/kubernetes/manifests/
+  fi
+}
+
+
 function reset-motd {
   # kubelet is installed both on the master and nodes, and the version is easy to parse (unlike kubectl)
   local -r version="$(/usr/bin/kubelet --version=true | cut -f2 -d " ")"
@@ -827,6 +834,7 @@ if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
   start-kube-scheduler
   start-kube-addons
   start-cluster-autoscaler
+  start-lb-controller
 else
   start-kube-proxy
   # Kube-registry-proxy.

cluster/saltbase/salt/l7-gcp/glbc.manifest

@@ -0,0 +1,53 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: l7-lb-controller-v0.6.3
+  namespace: kube-system
+  labels:
+    k8s-app: glbc
+    version: v0.6.3
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: "GLBC"
+spec:
+  terminationGracePeriodSeconds: 600
+  hostNetwork: true
+  containers:
+  - image: gcr.io/google_containers/glbc:0.6.3
+    livenessProbe:
+      httpGet:
+        path: /healthz
+        port: 8086
+        scheme: HTTP
+      initialDelaySeconds: 30
+      # healthz reaches out to GCE
+      periodSeconds: 30
+      timeoutSeconds: 5
+    name: l7-lb-controller
+    volumeMounts:
+    - mountPath: /etc/gce.conf
+      name: cloudconfig
+      readOnly: true
+    - mountPath: /var/log/glbc.log
+      name: logfile
+      readOnly: false
+    resources:
+      # Request and limits are set to accomodate this pod alongside the other
+      # master components on a single core master.
+      limits:
+        cpu: 50m
+        memory: 100Mi
+      requests:
+        cpu: 10m
+        memory: 50Mi
+    command:
+    # TODO: split this out into args when we no longer need to pipe stdout to a file #6428
+    - sh
+    - -c
+    - '/glbc --default-backend-service=kube-system/default-http-backend --sync-period=60s --running-in-cluster=false --use-real-cloud=true --config-file-path=/etc/gce.conf --healthz-port=8086 1>>/var/log/glbc.log 2>&1'
+  volumes:
+  - hostPath:
+      path: /etc/gce.conf
+    name: cloudconfig
+  - hostPath:
+      path: /var/log/glbc.log
+    name: logfile

cluster/saltbase/salt/l7-gcp/init.sls

@@ -0,0 +1,17 @@
+/etc/kubernetes/manifests/glbc.manifest:
+  file.managed:
+    - source: salt://l7-gcp/glbc.manifest
+    - template: jinja
+    - user: root
+    - group: root
+    - mode: 644
+    - makedirs: true
+    - dir_mode: 755
+
+/var/log/glbc.log:
+  file.managed:
+    - user: root
+    - group: root
+    - mode: 644
+
+

cluster/saltbase/salt/top.sls

@@ -56,6 +56,9 @@ base:
     - cni
 {% elif pillar.get('network_provider', '').lower() == 'cni' %}
     - cni
+{% endif %}
+{% if pillar.get('enable_l7_loadbalancing', '').lower() == 'glbc' %}
+    - l7-gcp
 {% endif %}
     - kube-apiserver
     - kube-controller-manager

test/e2e/ingress.go

@@ -30,7 +30,6 @@ import (
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/apis/extensions"
 	client "k8s.io/kubernetes/pkg/client/unversioned"
-	"k8s.io/kubernetes/pkg/labels"
 	utilexec "k8s.io/kubernetes/pkg/util/exec"
 	"k8s.io/kubernetes/pkg/util/intstr"
 	"k8s.io/kubernetes/pkg/util/wait"
@@ -62,6 +61,10 @@ var (
 	testImage         = "gcr.io/google_containers/n-way-http:1.0"
 	httpContainerPort = 8080
 
+	// Name of the config-map and key the ingress controller stores its uid in.
+	uidConfigMap = "ingress-uid"
+	uidKey       = "uid"
+
 	expectedLBCreationTime    = 7 * time.Minute
 	expectedLBHealthCheckTime = 7 * time.Minute
 
@@ -232,26 +235,6 @@ func gcloudDelete(resource, name, project string) {
 	}
 }
 
-// kubectlLogLBController logs kubectl debug output for the L7 controller pod.
-func kubectlLogLBController(c *client.Client, ns string) {
-	selector := labels.SelectorFromSet(labels.Set(controllerLabels))
-	options := api.ListOptions{LabelSelector: selector}
-	podList, err := c.Pods(api.NamespaceAll).List(options)
-	if err != nil {
-		framework.Logf("Cannot log L7 controller output, error listing pods %v", err)
-		return
-	}
-	if len(podList.Items) == 0 {
-		framework.Logf("Loadbalancer controller pod not found")
-		return
-	}
-	for _, p := range podList.Items {
-		framework.Logf("\nLast 100 log lines of %v\n", p.Name)
-		l, _ := framework.RunKubectl("logs", p.Name, fmt.Sprintf("--namespace=%v", ns), "-c", lbContainerName, "--tail=100")
-		framework.Logf(l)
-	}
-}
-
 type IngressController struct {
 	ns      string
 	rcPath  string
@@ -263,24 +246,12 @@ type IngressController struct {
 }
 
 func (cont *IngressController) getL7AddonUID() (string, error) {
-	listOpts := api.ListOptions{LabelSelector: labels.SelectorFromSet(labels.Set(clusterAddonLBLabels))}
+	cm, err := cont.c.ConfigMaps(api.NamespaceSystem).Get(uidConfigMap)
-	existingRCs, err := cont.c.ReplicationControllers(api.NamespaceSystem).List(listOpts)
 	if err != nil {
 		return "", err
 	}
-	if len(existingRCs.Items) != 1 {
+	if uid, ok := cm.Data[uidKey]; ok {
-		return "", fmt.Errorf("Unexpected number of lb cluster addons %v with label %v in kube-system namespace", len(existingRCs.Items), clusterAddonLBLabels)
+		return uid, nil
-	}
-	rc := existingRCs.Items[0]
-	commandPrefix := "--cluster-uid="
-	for i, c := range rc.Spec.Template.Spec.Containers {
-		if c.Name == lbContainerName {
-			for _, arg := range rc.Spec.Template.Spec.Containers[i].Args {
-				if strings.HasPrefix(arg, commandPrefix) {
-					return strings.Replace(arg, commandPrefix, "", -1), nil
-				}
-			}
-		}
 	}
 	return "", fmt.Errorf("Could not find cluster UID for L7 addon pod")
 }
@@ -448,7 +419,6 @@ var _ = framework.KubeDescribe("GCE L7 LoadBalancer Controller [Feature:Ingress]
 	AfterEach(func() {
 		framework.Logf("Average creation time %+v, health check time %+v", creationTimes, responseTimes)
 		if CurrentGinkgoTestDescription().Failed {
-			kubectlLogLBController(client, ns)
 			framework.Logf("\nOutput of kubectl describe ing:\n")
 			desc, _ := framework.RunKubectl("describe", "ing", fmt.Sprintf("--namespace=%v", ns))
 			framework.Logf(desc)